| blob | da267d2512a77364c7e2a56273c5fdd976fd2ff6 |
1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 [ -z "$CTDB_VARDIR" ] && {
6 if [ -d "/var/lib/ctdb" ] ; then
7 export CTDB_VARDIR="/var/lib/ctdb"
8 else
9 export CTDB_VARDIR="/var/ctdb"
10 fi
11 }
12 [ -z "$CTDB_ETCDIR" ] && {
13 export CTDB_ETCDIR="/etc"
14 }
16 #######################################
17 # pull in a system config file, if any
18 _loadconfig() {
20 if [ -z "$1" ] ; then
21 foo="${service_config:-${service_name}}"
22 if [ -n "$foo" ] ; then
23 loadconfig "$foo"
24 return
25 fi
26 fi
28 if [ "$1" != "ctdb" ] ; then
29 loadconfig "ctdb"
30 fi
32 if [ -z "$1" ] ; then
33 return
34 fi
36 if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
37 . $CTDB_ETCDIR/sysconfig/$1
38 elif [ -f $CTDB_ETCDIR/default/$1 ]; then
39 . $CTDB_ETCDIR/default/$1
40 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
41 . $CTDB_BASE/sysconfig/$1
42 fi
44 if [ "$1" = "ctdb" ] ; then
45 _config="${CTDB_BASE}/ctdbd.conf"
46 if [ -r "$_config" ] ; then
47 . "$_config"
48 fi
49 fi
50 }
52 loadconfig () {
53 _loadconfig "$@"
54 }
56 ##############################################################
58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
59 # configuration file.
60 debug ()
61 {
62 if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
63 # If there are arguments then echo them. Otherwise expect to
64 # use stdin, which allows us to pass lots of debug using a
65 # here document.
66 if [ -n "$1" ] ; then
67 echo "DEBUG: $*"
68 else
69 sed -e 's@^@DEBUG: @'
70 fi
71 else
72 if [ -z "$1" ] ; then
73 cat >/dev/null
74 fi
75 fi
76 }
78 die ()
79 {
80 _msg="$1"
81 _rc="${2:-1}"
83 echo "$_msg"
84 exit $_rc
85 }
87 # Log given message or stdin to either syslog or a CTDB log file
88 # $1 is the tag passed to logger if syslog is in use.
89 script_log ()
90 {
91 _tag="$1" ; shift
93 case "$CTDB_LOGGING" in
94 file:*|"")
95 if [ -n "$CTDB_LOGGING" ] ; then
96 _file="${CTDB_LOGGING#file:}"
97 else
98 _file="/var/log/log.ctdb"
99 fi
100 {
101 if [ -n "$*" ] ; then
102 echo "$*"
103 else
104 cat
105 fi
106 } >>"$_file"
107 ;;
108 *)
109 # Handle all syslog:* variants here too. There's no tool to do
110 # the lossy things, so just use logger.
111 logger -t "ctdbd: ${_tag}" $*
112 ;;
113 esac
114 }
116 # When things are run in the background in an eventscript then logging
117 # output might get lost. This is the "solution". :-)
118 background_with_logging ()
119 {
120 (
121 "$@" 2>&1 </dev/null |
122 script_log "${script_name}&"
123 )&
125 return 0
126 }
128 ##############################################################
129 # check number of args for different events
130 ctdb_check_args ()
131 {
132 case "$1" in
133 takeip|releaseip)
134 if [ $# != 4 ]; then
135 echo "ERROR: must supply interface, IP and maskbits"
136 exit 1
137 fi
138 ;;
139 updateip)
140 if [ $# != 5 ]; then
141 echo "ERROR: must supply old interface, new interface, IP and maskbits"
142 exit 1
143 fi
144 ;;
145 esac
146 }
148 ##############################################################
149 # determine on what type of system (init style) we are running
150 detect_init_style()
151 {
152 # only do detection if not already set:
153 [ -z "$CTDB_INIT_STYLE" ] || return
155 if [ -x /sbin/startproc ]; then
156 CTDB_INIT_STYLE="suse"
157 elif [ -x /sbin/start-stop-daemon ]; then
158 CTDB_INIT_STYLE="debian"
159 else
160 CTDB_INIT_STYLE="redhat"
161 fi
162 }
164 ######################################################
165 # simulate /sbin/service on platforms that don't have it
166 # _service() makes it easier to hook the service() function for
167 # testing.
168 _service ()
169 {
170 _service_name="$1"
171 _op="$2"
173 # do nothing, when no service was specified
174 [ -z "$_service_name" ] && return
176 if [ -x /sbin/service ]; then
177 $_nice /sbin/service "$_service_name" "$_op"
178 elif [ -x /usr/sbin/service ]; then
179 $_nice /usr/sbin/service "$_service_name" "$_op"
180 elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
181 $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
182 elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
183 $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
184 fi
185 }
187 service()
188 {
189 _nice=""
190 _service "$@"
191 }
193 ######################################################
194 # simulate /sbin/service (niced) on platforms that don't have it
195 nice_service()
196 {
197 _nice="nice"
198 _service "$@"
199 }
201 ######################################################
202 # wrapper around /proc/ settings to allow them to be hooked
203 # for testing
204 # 1st arg is relative path under /proc/, 2nd arg is value to set
205 set_proc ()
206 {
207 echo "$2" >"/proc/$1"
208 }
210 ######################################################
211 # wrapper around getting file contents from /proc/ to allow
212 # this to be hooked for testing
213 # 1st arg is relative path under /proc/
214 get_proc ()
215 {
216 cat "/proc/$1"
217 }
219 ######################################################
220 # Print up to $_max kernel stack traces for processes named $_program
221 program_stack_traces ()
222 {
223 _prog="$1"
224 _max="${2:-1}"
226 _count=1
227 for _pid in $(pidof "$_prog") ; do
228 [ $_count -le $_max ] || break
230 # Do this first to avoid racing with process exit
231 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
232 if [ -n "$_stack" ] ; then
233 echo "Stack trace for ${_prog}[${_pid}]:"
234 echo "$_stack"
235 _count=$(($_count + 1))
236 fi
237 done
238 }
240 ######################################################
241 # Check that an RPC service is healthy -
242 # this includes allowing a certain number of failures
243 # before marking the NFS service unhealthy.
244 #
245 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
246 #
247 # each triple is a set of 3 arguments: an operator, a
248 # fail count limit and an action string.
249 #
250 # For example:
251 #
252 # nfs_check_rpc_service "lockd" \
253 # -ge 15 "verbose restart unhealthy" \
254 # -eq 10 "restart:bs"
255 #
256 # says that if lockd is down for 15 iterations then do
257 # a verbose restart of lockd and mark the node unhealthy.
258 # Before this, after 10 iterations of failure, the
259 # service is restarted silently in the background.
260 # Order is important: the number of failures need to be
261 # specified in reverse order because processing stops
262 # after the first condition that is true.
263 ######################################################
264 nfs_check_rpc_service ()
265 {
266 _prog_name="$1" ; shift
268 if _nfs_check_rpc_common "$_prog_name" ; then
269 return
270 fi
272 while [ -n "$3" ] ; do
273 if _nfs_check_rpc_action "$1" "$2" "$3" ; then
274 break
275 fi
276 shift 3
277 done
278 }
280 # The new way of doing things...
281 nfs_check_rpc_services ()
282 {
283 # Files must end with .check - avoids editor backups, RPM fu, ...
284 for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
285 _t="${_f%.check}"
286 _prog_name="${_t##*/[0-9][0-9].}"
288 if _nfs_check_rpc_common "$_prog_name" ; then
289 # This RPC service is up, check next service...
290 continue
291 fi
293 # Check each line in the file in turn until one of the limit
294 # checks is hit...
295 while read _cmp _lim _rest ; do
296 # Skip comments
297 case "$_cmp" in
298 \#*) continue ;;
299 esac
301 if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
302 # Limit was hit on this line, no further checking...
303 break
304 fi
305 done <"$_f"
306 done
307 }
309 _nfs_check_rpc_common ()
310 {
311 _prog_name="$1"
313 # Some platforms don't have separate programs for all services.
314 case "$_prog_name" in
315 statd)
316 which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
317 esac
319 case "$_prog_name" in
320 nfsd)
321 _rpc_prog=nfs
322 _version=3
323 ;;
324 mountd)
325 _rpc_prog=mountd
326 _version=1
327 ;;
328 rquotad)
329 _rpc_prog=rquotad
330 _version=1
331 ;;
332 lockd)
333 _rpc_prog=nlockmgr
334 _version=4
335 ;;
336 statd)
337 _rpc_prog=status
338 _version=1
339 ;;
340 *)
341 echo "Internal error: unknown RPC program \"$_prog_name\"."
342 exit 1
343 esac
345 _service_name="nfs_${_prog_name}"
347 if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
348 ctdb_counter_init "$_service_name"
349 return 0
350 fi
352 ctdb_counter_incr "$_service_name"
354 return 1
355 }
357 _nfs_check_rpc_action ()
358 {
359 _cmp="$1"
360 _limit="$2"
361 _actions="$3"
363 if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
364 return 1
365 fi
367 for _action in $_actions ; do
368 case "$_action" in
369 verbose)
370 echo "$ctdb_check_rpc_out"
371 ;;
372 restart)
373 _nfs_restart_rpc_service "$_prog_name"
374 ;;
375 restart:b)
376 _nfs_restart_rpc_service "$_prog_name" true
377 ;;
378 unhealthy)
379 exit 1
380 ;;
381 *)
382 echo "Internal error: unknown action \"$_action\"."
383 exit 1
384 esac
385 done
387 return 0
388 }
390 _nfs_restart_rpc_service ()
391 {
392 _prog_name="$1"
393 _background="${2:-false}"
395 if $_background ; then
396 _maybe_background="background_with_logging"
397 else
398 _maybe_background=""
399 fi
401 _p="rpc.${_prog_name}"
403 case "$_prog_name" in
404 nfsd)
405 echo "Trying to restart NFS service"
406 $_maybe_background startstop_nfs restart
407 ;;
408 mountd)
409 echo "Trying to restart $_prog_name [${_p}]"
410 killall -q -9 "$_p"
411 nfs_dump_some_threads "$_p"
412 $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
413 ;;
414 rquotad)
415 echo "Trying to restart $_prog_name [${_p}]"
416 killall -q -9 "$_p"
417 nfs_dump_some_threads "$_p"
418 $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
419 ;;
420 lockd)
421 echo "Trying to restart lock manager service"
422 $_maybe_background startstop_nfslock restart
423 ;;
424 statd)
425 echo "Trying to restart $_prog_name [${_p}]"
426 killall -q -9 "$_p"
427 nfs_dump_some_threads "$_p"
428 $_maybe_background $_p \
429 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
430 ${STATD_PORT:+-p} $STATD_PORT \
431 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
432 ;;
433 *)
434 echo "Internal error: unknown RPC program \"$_prog_name\"."
435 exit 1
436 esac
437 }
439 ######################################################
440 # check that a rpc server is registered with portmap
441 # and responding to requests
442 # usage: ctdb_check_rpc SERVICE_NAME VERSION
443 ######################################################
444 ctdb_check_rpc ()
445 {
446 progname="$1"
447 version="$2"
449 _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
451 if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
452 ctdb_check_rpc_out="ERROR: $progname failed RPC check:
453 $ctdb_check_rpc_out"
454 echo "$ctdb_check_rpc_out"
455 return 1
456 fi
457 }
459 ######################################################
460 # Ensure $service_name is set
461 assert_service_name ()
462 {
463 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
464 }
466 ######################################################
467 # check a set of directories is available
468 # return 1 on a missing directory
469 # directories are read from stdin
470 ######################################################
471 ctdb_check_directories_probe()
472 {
473 while IFS="" read d ; do
474 case "$d" in
475 *%*)
476 continue
477 ;;
478 *)
479 [ -d "${d}/." ] || return 1
480 esac
481 done
482 }
484 ######################################################
485 # check a set of directories is available
486 # directories are read from stdin
487 ######################################################
488 ctdb_check_directories()
489 {
490 ctdb_check_directories_probe || {
491 echo "ERROR: $service_name directory \"$d\" not available"
492 exit 1
493 }
494 }
496 ######################################################
497 # check a set of tcp ports
498 # usage: ctdb_check_tcp_ports <ports...>
499 ######################################################
501 # This flag file is created when a service is initially started. It
502 # is deleted the first time TCP port checks for that service succeed.
503 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
504 # message if a port check fails.
505 _ctdb_check_tcp_common ()
506 {
507 assert_service_name
508 _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
509 }
511 ctdb_check_tcp_init ()
512 {
513 _ctdb_check_tcp_common
514 mkdir -p "${_ctdb_service_started_file%/*}" # dirname
515 touch "$_ctdb_service_started_file"
516 }
518 # Check whether something is listening on all of the given TCP ports
519 # using the "ctdb checktcpport" command.
520 ctdb_check_tcp_ports()
521 {
522 if [ -z "$1" ] ; then
523 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
524 exit 1
525 fi
527 for _p ; do # process each function argument (port)
528 _cmd="ctdb checktcpport $_p"
529 _out=$($_cmd 2>&1)
530 _ret=$?
531 case "$_ret" in
532 0)
533 _ctdb_check_tcp_common
534 if [ ! -f "$_ctdb_service_started_file" ] ; then
535 echo "ERROR: $service_name tcp port $_p is not responding"
536 debug "\"ctdb checktcpport $_p\" was able to bind to port"
537 else
538 echo "INFO: $service_name tcp port $_p is not responding"
539 fi
541 return 1
542 ;;
543 98)
544 # Couldn't bind, something already listening, next port...
545 continue
546 ;;
547 *)
548 echo "ERROR: unexpected error running \"ctdb checktcpport\""
549 debug <<EOF
550 ctdb checktcpport (exited with $_ret) with output:
551 $_out"
552 EOF
553 return $_ret
554 esac
555 done
557 # All ports listening
558 _ctdb_check_tcp_common
559 rm -f "$_ctdb_service_started_file"
560 return 0
561 }
563 ######################################################
564 # check a unix socket
565 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
566 ######################################################
567 ctdb_check_unix_socket() {
568 socket_path="$1"
569 [ -z "$socket_path" ] && return
571 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
572 echo "ERROR: $service_name socket $socket_path not found"
573 return 1
574 fi
575 }
577 ######################################################
578 # check a command returns zero status
579 # usage: ctdb_check_command <command>
580 ######################################################
581 ctdb_check_command ()
582 {
583 _out=$("$@" 2>&1) || {
584 echo "ERROR: $* returned error"
585 echo "$_out" | debug
586 exit 1
587 }
588 }
590 ################################################
591 # kill off any TCP connections with the given IP
592 ################################################
593 kill_tcp_connections ()
594 {
595 _ip="$1"
597 _oneway=false
598 if [ "$2" = "oneway" ] ; then
599 _oneway=true
600 fi
602 get_tcp_connections_for_ip "$_ip" | {
603 _killcount=0
604 _connections=""
605 _nl="
606 "
607 while read _dst _src; do
608 _destport="${_dst##*:}"
609 __oneway=$_oneway
610 case $_destport in
611 # we only do one-way killtcp for CIFS
612 139|445) __oneway=true ;;
613 esac
615 echo "Killing TCP connection $_src $_dst"
616 _connections="${_connections}${_nl}${_src} ${_dst}"
617 if ! $__oneway ; then
618 _connections="${_connections}${_nl}${_dst} ${_src}"
619 fi
621 _killcount=$(($_killcount + 1))
622 done
624 if [ $_killcount -eq 0 ] ; then
625 return
626 fi
628 echo "$_connections" | ctdb killtcp || {
629 echo "Failed to send killtcp control"
630 return
631 }
633 _count=0
634 while : ; do
635 _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
637 if [ $_remaining -eq 0 ] ; then
638 echo "Killed $_killcount TCP connections to released IP $_ip"
639 return
640 fi
642 _count=$(($_count + 1))
643 if [ $_count -gt 3 ] ; then
644 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
645 return
646 fi
648 echo "Waiting for $_remaining connections to be killed for IP $_ip"
649 sleep 1
650 done
651 }
652 }
654 ##################################################################
655 # kill off the local end for any TCP connections with the given IP
656 ##################################################################
657 kill_tcp_connections_local_only ()
658 {
659 kill_tcp_connections "$1" "oneway"
660 }
662 ##################################################################
663 # tickle any TCP connections with the given IP
664 ##################################################################
665 tickle_tcp_connections ()
666 {
667 _ip="$1"
669 get_tcp_connections_for_ip "$_ip" |
670 {
671 _failed=false
673 while read dest src; do
674 echo "Tickle TCP connection $src $dest"
675 ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
676 echo "Tickle TCP connection $dest $src"
677 ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
678 done
680 if $_failed ; then
681 echo "Failed to send tickle control"
682 fi
683 }
684 }
686 get_tcp_connections_for_ip ()
687 {
688 _ip="$1"
690 netstat -tn | awk -v ip=$_ip \
691 'index($1, "tcp") == 1 && \
692 (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
693 && $6 == "ESTABLISHED" \
694 {print $4" "$5}'
695 }
697 ##################################################################
698 # use statd-callout to update NFS lock info
699 ##################################################################
700 nfs_update_lock_info ()
701 {
702 if [ -x "$CTDB_BASE/statd-callout" ] ; then
703 "$CTDB_BASE/statd-callout" update
704 fi
705 }
707 ########################################################
708 # start/stop the Ganesha nfs service
709 ########################################################
710 startstop_ganesha()
711 {
712 _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
713 case "$1" in
714 start)
715 service "$_service_name" start
716 ;;
717 stop)
718 service "$_service_name" stop
719 ;;
720 restart)
721 service "$_service_name" stop
722 nfs_dump_some_threads "rpc.statd"
723 service "$_service_name" start
724 ;;
725 esac
726 }
728 ########################################################
729 # start/stop the nfs service on different platforms
730 ########################################################
731 startstop_nfs() {
732 PLATFORM="unknown"
733 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
734 PLATFORM="sles"
735 }
736 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
737 -r /usr/lib/systemd/system/nfs-lock.service ] && {
738 PLATFORM="rhel"
739 }
741 case $PLATFORM in
742 sles)
743 case $1 in
744 start)
745 service nfsserver start
746 ;;
747 stop)
748 service nfsserver stop > /dev/null 2>&1
749 ;;
750 restart)
751 set_proc "fs/nfsd/threads" 0
752 service nfsserver stop > /dev/null 2>&1
753 pkill -9 nfsd
754 nfs_dump_some_threads
755 service nfsserver start
756 ;;
757 esac
758 ;;
759 rhel)
760 case $1 in
761 start)
762 service nfslock start
763 service nfs start
764 ;;
765 stop)
766 service nfs stop
767 service nfslock stop
768 ;;
769 restart)
770 set_proc "fs/nfsd/threads" 0
771 service nfs stop > /dev/null 2>&1
772 service nfslock stop > /dev/null 2>&1
773 pkill -9 nfsd
774 nfs_dump_some_threads
775 service nfslock start
776 service nfs start
777 ;;
778 esac
779 ;;
780 *)
781 echo "Unknown platform. NFS is not supported with ctdb"
782 exit 1
783 ;;
784 esac
785 }
787 # Dump up to the configured number of nfsd thread backtraces.
788 nfs_dump_some_threads ()
789 {
790 _prog="${1:-nfsd}"
792 _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
793 [ $_num -gt 0 ] || return 0
795 program_stack_traces "$_prog" $_num
796 }
798 ########################################################
799 # start/stop the nfs lockmanager service on different platforms
800 ########################################################
801 startstop_nfslock() {
802 PLATFORM="unknown"
803 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
804 PLATFORM="sles"
805 }
806 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
807 -r /usr/lib/systemd/system/nfs-lock.service ] && {
808 PLATFORM="rhel"
809 }
811 case $PLATFORM in
812 sles)
813 # for sles there is no service for lockmanager
814 # so we instead just shutdown/restart nfs
815 case $1 in
816 start)
817 service nfsserver start
818 ;;
819 stop)
820 service nfsserver stop > /dev/null 2>&1
821 ;;
822 restart)
823 service nfsserver stop > /dev/null 2>&1
824 service nfsserver start
825 ;;
826 esac
827 ;;
828 rhel)
829 case $1 in
830 start)
831 service nfslock start
832 ;;
833 stop)
834 service nfslock stop > /dev/null 2>&1
835 ;;
836 restart)
837 service nfslock stop > /dev/null 2>&1
838 service nfslock start
839 ;;
840 esac
841 ;;
842 *)
843 echo "Unknown platform. NFS locking is not supported with ctdb"
844 exit 1
845 ;;
846 esac
847 }
849 ########################################################
851 add_ip_to_iface ()
852 {
853 _iface=$1
854 _ip=$2
855 _maskbits=$3
857 # Ensure interface is up
858 ip link set "$_iface" up || \
859 die "Failed to bringup interface $_iface"
861 # Only need to define broadcast for IPv4
862 case "$ip" in
863 *:*) _bcast="" ;;
864 *) _bcast="brd +" ;;
865 esac
867 ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
868 echo "Failed to add $_ip/$_maskbits on dev $_iface"
869 return 1
870 }
872 # Wait 5 seconds for IPv6 addresses to stop being tentative...
873 if [ -z "$_bcast" ] ; then
874 for _x in $(seq 1 10) ; do
875 ip addr show to "${_ip}/128" | grep -q "tentative" || break
876 sleep 0.5
877 done
879 # If the address was a duplicate then it won't be on the
880 # interface so flag an error.
881 _t=$(ip addr show to "${_ip}/128")
882 case "$_t" in
883 "")
884 echo "Failed to add $_ip/$_maskbits on dev $_iface"
885 return 1
886 ;;
887 *tentative*|*dadfailed*)
888 echo "Failed to add $_ip/$_maskbits on dev $_iface"
889 ip addr del "$_ip/$_maskbits" dev "$_iface"
890 return 1
891 ;;
892 esac
893 fi
894 }
896 delete_ip_from_iface()
897 {
898 _iface=$1
899 _ip=$2
900 _maskbits=$3
902 # This could be set globally for all interfaces but it is probably
903 # better to avoid surprises, so limit it the interfaces where CTDB
904 # has public IP addresses. There isn't anywhere else convenient
905 # to do this so just set it each time. This is much cheaper than
906 # remembering and re-adding secondaries.
907 set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
909 ip addr del "$_ip/$_maskbits" dev "$_iface" || {
910 echo "Failed to del $_ip on dev $_iface"
911 return 1
912 }
913 }
915 # If the given IP is hosted then print 2 items: maskbits and iface
916 ip_maskbits_iface ()
917 {
918 _addr="$1"
920 case "$_addr" in
921 *:*) _family="inet6" ; _bits=128 ;;
922 *) _family="inet" ; _bits=32 ;;
923 esac
925 ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
926 awk -v family="${_family}" \
927 'NR == 1 { iface = $2; sub(":$", "", iface) } \
928 $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
929 print mask, iface, family }'
930 }
932 drop_ip ()
933 {
934 _addr="${1%/*}" # Remove optional maskbits
936 set -- $(ip_maskbits_iface $_addr)
937 if [ -n "$1" ] ; then
938 _maskbits="$1"
939 _iface="$2"
940 echo "Removing public address $_addr/$_maskbits from device $_iface"
941 delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
942 fi
943 }
945 drop_all_public_ips ()
946 {
947 while read _ip _x ; do
948 drop_ip "$_ip"
949 done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
950 }
952 flush_route_cache ()
953 {
954 set_proc sys/net/ipv4/route/flush 1
955 set_proc sys/net/ipv6/route/flush 1
956 }
958 ########################################################
959 # Simple counters
960 _ctdb_counter_common () {
961 _service_name="${1:-${service_name:-${script_name}}}"
962 _counter_file="$ctdb_fail_dir/$_service_name"
963 mkdir -p "${_counter_file%/*}" # dirname
964 }
965 ctdb_counter_init () {
966 _ctdb_counter_common "$1"
968 >"$_counter_file"
969 }
970 ctdb_counter_incr () {
971 _ctdb_counter_common "$1"
973 # unary counting!
974 echo -n 1 >> "$_counter_file"
975 }
976 ctdb_check_counter () {
977 _msg="${1:-error}" # "error" - anything else is silent on fail
978 _op="${2:--ge}" # an integer operator supported by test
979 _limit="${3:-${service_fail_limit}}"
980 shift 3
981 _ctdb_counter_common "$1"
983 # unary counting!
984 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
985 _hit=false
986 if [ "$_op" != "%" ] ; then
987 if [ $_size $_op $_limit ] ; then
988 _hit=true
989 fi
990 else
991 if [ $(($_size $_op $_limit)) -eq 0 ] ; then
992 _hit=true
993 fi
994 fi
995 if $_hit ; then
996 if [ "$_msg" = "error" ] ; then
997 echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
998 exit 1
999 else
1000 return 1
1001 fi
1002 fi
1003 }
1005 ########################################################
1007 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1008 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1010 ctdb_setup_service_state_dir ()
1011 {
1012 service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1013 mkdir -p "$service_state_dir" || {
1014 echo "Error creating state dir \"$service_state_dir\""
1015 exit 1
1016 }
1017 }
1019 ########################################################
1020 # Managed status history, for auto-start/stop
1022 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1024 _ctdb_managed_common ()
1025 {
1026 _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1027 }
1029 ctdb_service_managed ()
1030 {
1031 _ctdb_managed_common
1032 mkdir -p "$ctdb_managed_dir"
1033 touch "$_ctdb_managed_file"
1034 }
1036 ctdb_service_unmanaged ()
1037 {
1038 _ctdb_managed_common
1039 rm -f "$_ctdb_managed_file"
1040 }
1042 is_ctdb_previously_managed_service ()
1043 {
1044 _ctdb_managed_common
1045 [ -f "$_ctdb_managed_file" ]
1046 }
1048 ########################################################
1049 # Check and set status
1051 log_status_cat ()
1052 {
1053 echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1054 }
1056 ctdb_checkstatus ()
1057 {
1058 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1059 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1060 return 1
1061 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1062 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1063 return 2
1064 else
1065 return 0
1066 fi
1067 }
1069 ctdb_setstatus ()
1070 {
1071 d="$ctdb_status_dir/$script_name"
1072 case "$1" in
1073 unhealthy|banned)
1074 mkdir -p "$d"
1075 cat "$2" >"$d/$1"
1076 ;;
1077 *)
1078 for i in "banned" "unhealthy" ; do
1079 rm -f "$d/$i"
1080 done
1081 ;;
1082 esac
1083 }
1085 ##################################################################
1086 # Reconfigure a service on demand
1088 _ctdb_service_reconfigure_common ()
1089 {
1090 _d="$ctdb_status_dir/${service_name}"
1091 mkdir -p "$_d"
1092 _ctdb_service_reconfigure_flag="$_d/reconfigure"
1093 }
1095 ctdb_service_needs_reconfigure ()
1096 {
1097 _ctdb_service_reconfigure_common
1098 [ -e "$_ctdb_service_reconfigure_flag" ]
1099 }
1101 ctdb_service_set_reconfigure ()
1102 {
1103 _ctdb_service_reconfigure_common
1104 >"$_ctdb_service_reconfigure_flag"
1105 }
1107 ctdb_service_unset_reconfigure ()
1108 {
1109 _ctdb_service_reconfigure_common
1110 rm -f "$_ctdb_service_reconfigure_flag"
1111 }
1113 ctdb_service_reconfigure ()
1114 {
1115 echo "Reconfiguring service \"${service_name}\"..."
1116 ctdb_service_unset_reconfigure
1117 service_reconfigure || return $?
1118 ctdb_counter_init
1119 }
1121 # Default service_reconfigure() function does nothing.
1122 service_reconfigure ()
1123 {
1124 :
1125 }
1127 ctdb_reconfigure_take_lock ()
1128 {
1129 _ctdb_service_reconfigure_common
1130 _lock="${_d}/reconfigure_lock"
1131 mkdir -p "${_lock%/*}" # dirname
1132 touch "$_lock"
1134 (
1135 flock 0
1136 # This is overkill but will work if we need to extend this to
1137 # allow certain events to run multiple times in parallel
1138 # (e.g. takeip) and write multiple PIDs to the file.
1139 read _locker_event
1140 if [ -n "$_locker_event" ] ; then
1141 while read _pid ; do
1142 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1143 kill -0 "$_pid" 2>/dev/null ; then
1144 exit 1
1145 fi
1146 done
1147 fi
1149 printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1150 exit 0
1151 ) <"$_lock"
1152 }
1154 ctdb_reconfigure_release_lock ()
1155 {
1156 _ctdb_service_reconfigure_common
1157 _lock="${_d}/reconfigure_lock"
1159 rm -f "$_lock"
1160 }
1162 ctdb_replay_monitor_status ()
1163 {
1164 echo "Replaying previous status for this script due to reconfigure..."
1165 # Leading separator ('|') is missing in some versions...
1166 _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1167 # Output looks like this:
1168 # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1169 # This is the cheapest way of getting fields in the middle.
1170 set -- $(IFS="|" ; echo $_out)
1171 _code="$3"
1172 _status="$4"
1173 # The error output field can include colons so we'll try to
1174 # preserve them. The weak checking at the beginning tries to make
1175 # this work for both broken (no leading '|') and fixed output.
1176 _out="${_out%|}"
1177 _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1178 case "$_status" in
1179 OK) : ;; # Do nothing special.
1180 TIMEDOUT)
1181 # Recast this as an error, since we can't exit with the
1182 # correct negative number.
1183 _code=1
1184 _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1185 ;;
1186 DISABLED)
1187 # Recast this as an OK, since we can't exit with the
1188 # correct negative number.
1189 _code=0
1190 _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1191 ;;
1192 *) : ;; # Must be ERROR, do nothing special.
1193 esac
1194 if [ -n "$_err_out" ] ; then
1195 echo "$_err_out"
1196 fi
1197 exit $_code
1198 }
1200 ctdb_service_check_reconfigure ()
1201 {
1202 assert_service_name
1204 # We only care about some events in this function. For others we
1205 # return now.
1206 case "$event_name" in
1207 monitor|ipreallocated|reconfigure) : ;;
1208 *) return 0 ;;
1209 esac
1211 if ctdb_reconfigure_take_lock ; then
1212 # No events covered by this function are running, so proceed
1213 # with gay abandon.
1214 case "$event_name" in
1215 reconfigure)
1216 (ctdb_service_reconfigure)
1217 exit $?
1218 ;;
1219 ipreallocated)
1220 if ctdb_service_needs_reconfigure ; then
1221 ctdb_service_reconfigure
1222 fi
1223 ;;
1224 esac
1226 ctdb_reconfigure_release_lock
1227 else
1228 # Somebody else is running an event we don't want to collide
1229 # with. We proceed with caution.
1230 case "$event_name" in
1231 reconfigure)
1232 # Tell whoever called us to retry.
1233 exit 2
1234 ;;
1235 ipreallocated)
1236 # Defer any scheduled reconfigure and just run the
1237 # rest of the ipreallocated event, as per the
1238 # eventscript. There's an assumption here that the
1239 # event doesn't depend on any scheduled reconfigure.
1240 # This is true in the current code.
1241 return 0
1242 ;;
1243 monitor)
1244 # There is most likely a reconfigure in progress so
1245 # the service is possibly unstable. As above, we
1246 # defer any scheduled reconfigured. We also replay
1247 # the previous monitor status since that's the best
1248 # information we have.
1249 ctdb_replay_monitor_status
1250 ;;
1251 esac
1252 fi
1253 }
1255 ##################################################################
1256 # Does CTDB manage this service? - and associated auto-start/stop
1258 ctdb_compat_managed_service ()
1259 {
1260 if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1261 CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1262 fi
1263 }
1265 is_ctdb_managed_service ()
1266 {
1267 assert_service_name
1269 # $t is used just for readability and to allow better accurate
1270 # matching via leading/trailing spaces
1271 t=" $CTDB_MANAGED_SERVICES "
1273 # Return 0 if "<space>$service_name<space>" appears in $t
1274 if [ "${t#* ${service_name} }" != "${t}" ] ; then
1275 return 0
1276 fi
1278 # If above didn't match then update $CTDB_MANAGED_SERVICES for
1279 # backward compatibility and try again.
1280 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
1281 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
1282 ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND" "winbind"
1283 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "apache2"
1284 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
1285 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
1286 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
1287 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
1288 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
1290 t=" $CTDB_MANAGED_SERVICES "
1292 # Return 0 if "<space>$service_name<space>" appears in $t
1293 [ "${t#* ${service_name} }" != "${t}" ]
1294 }
1296 ctdb_start_stop_service ()
1297 {
1298 assert_service_name
1300 # Allow service-start/service-stop pseudo-events to start/stop
1301 # services when we're not auto-starting/stopping and we're not
1302 # monitoring.
1303 case "$event_name" in
1304 service-start)
1305 if is_ctdb_managed_service ; then
1306 die 'service-start event not permitted when service is managed'
1307 fi
1308 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1309 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1310 fi
1311 ctdb_service_start
1312 exit $?
1313 ;;
1314 service-stop)
1315 if is_ctdb_managed_service ; then
1316 die 'service-stop event not permitted when service is managed'
1317 fi
1318 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1319 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1320 fi
1321 ctdb_service_stop
1322 exit $?
1323 ;;
1324 esac
1326 # Do nothing unless configured to...
1327 [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1329 [ "$event_name" = "monitor" ] || return 0
1331 if is_ctdb_managed_service ; then
1332 if ! is_ctdb_previously_managed_service ; then
1333 echo "Starting service \"$service_name\" - now managed"
1334 background_with_logging ctdb_service_start
1335 exit $?
1336 fi
1337 else
1338 if is_ctdb_previously_managed_service ; then
1339 echo "Stopping service \"$service_name\" - no longer managed"
1340 background_with_logging ctdb_service_stop
1341 exit $?
1342 fi
1343 fi
1344 }
1346 ctdb_service_start ()
1347 {
1348 # The service is marked managed if we've ever tried to start it.
1349 ctdb_service_managed
1351 service_start || return $?
1353 ctdb_counter_init
1354 ctdb_check_tcp_init
1355 }
1357 ctdb_service_stop ()
1358 {
1359 ctdb_service_unmanaged
1360 service_stop
1361 }
1363 # Default service_start() and service_stop() functions.
1365 # These may be overridden in an eventscript.
1366 service_start ()
1367 {
1368 service "$service_name" start
1369 }
1371 service_stop ()
1372 {
1373 service "$service_name" stop
1374 }
1376 ##################################################################
1378 ctdb_standard_event_handler ()
1379 {
1380 case "$1" in
1381 status)
1382 ctdb_checkstatus
1383 exit
1384 ;;
1385 setstatus)
1386 shift
1387 ctdb_setstatus "$@"
1388 exit
1389 ;;
1390 esac
1391 }
1393 iptables_wrapper ()
1394 {
1395 _family="$1" ; shift
1396 if [ "$_family" = "inet6" ] ; then
1397 _iptables_cmd="ip6tables"
1398 else
1399 _iptables_cmd="iptables"
1400 fi
1402 # iptables doesn't like being re-entered, so flock-wrap it.
1403 flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1404 }
1406 # AIX (and perhaps others?) doesn't have mktemp
1407 if ! which mktemp >/dev/null 2>&1 ; then
1408 mktemp ()
1409 {
1410 _dir=false
1411 if [ "$1" = "-d" ] ; then
1412 _dir=true
1413 shift
1414 fi
1415 _d="${TMPDIR:-/tmp}"
1416 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1417 md5sum | \
1418 sed -e 's@\(..........\).*@\1@')
1419 _t="${_d}/tmp.${_hex10}"
1420 (
1421 umask 077
1422 if $_dir ; then
1423 mkdir "$_t"
1424 else
1425 >"$_t"
1426 fi
1427 )
1428 echo "$_t"
1429 }
1430 fi
1432 ########################################################
1433 # tickle handling
1434 ########################################################
1436 update_tickles ()
1437 {
1438 _port="$1"
1440 tickledir="$CTDB_VARDIR/state/tickles"
1441 mkdir -p "$tickledir"
1443 # Who am I?
1444 _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1446 # What public IPs do I hold?
1447 _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1449 # IPs as a regexp choice
1450 _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1452 # Record connections to our public IPs in a temporary file
1453 _my_connections="${tickledir}/${_port}.connections"
1454 rm -f "$_my_connections"
1455 netstat -tn |
1456 awk -v destpat="^${_ipschoice}:${_port}\$" \
1457 '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1458 sort >"$_my_connections"
1460 # Record our current tickles in a temporary file
1461 _my_tickles="${tickledir}/${_port}.tickles"
1462 rm -f "$_my_tickles"
1463 for _i in $_ips ; do
1464 ctdb -X gettickles $_i $_port |
1465 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1466 done |
1467 sort >"$_my_tickles"
1469 # Add tickles for connections that we haven't already got tickles for
1470 comm -23 "$_my_connections" "$_my_tickles" |
1471 while read _src _dst ; do
1472 ctdb addtickle $_src $_dst
1473 done
1475 # Remove tickles for connections that are no longer there
1476 comm -13 "$_my_connections" "$_my_tickles" |
1477 while read _src _dst ; do
1478 ctdb deltickle $_src $_dst
1479 done
1481 rm -f "$_my_connections" "$_my_tickles"
1482 }
1484 ########################################################
1485 # load a site local config file
1486 ########################################################
1488 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1489 . "$CTDB_RC_LOCAL"
1490 }
1492 [ -x $CTDB_BASE/rc.local ] && {
1493 . $CTDB_BASE/rc.local
1494 }
1496 [ -d $CTDB_BASE/rc.local.d ] && {
1497 for i in $CTDB_BASE/rc.local.d/* ; do
1498 [ -x "$i" ] && . "$i"
1499 done
1500 }
1502 script_name="${0##*/}" # basename
1503 service_fail_limit=1
1504 event_name="$1"