| blob | f8f539ad53f76473bfe7d120038741521016b568 |
1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 if [ -z "$CTDB_BASE" ]; then
6 echo 'CTDB_BASE unset in CTDB functions file'
7 exit 1
8 fi
9 export CTDB_BASE
11 # CTDB_VARDIR is used elsewhere
12 # shellcheck disable=SC2034
13 CTDB_VARDIR="/usr/local/var/lib/ctdb"
15 CTDB="${CTDB:-/usr/local/bin/ctdb}"
17 # Only (and always) override these variables in test code
19 if [ -z "$CTDB_SCRIPT_VARDIR" ]; then
20 CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/scripts"
21 fi
23 if [ -z "$CTDB_SYS_ETCDIR" ]; then
24 CTDB_SYS_ETCDIR="/etc"
25 fi
27 if [ -z "$CTDB_HELPER_BINDIR" ]; then
28 CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
29 fi
31 #######################################
32 # pull in a system config file, if any
34 load_system_config()
35 {
36 for _i; do
38 if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/${_i}" ]; then
39 . "${CTDB_SYS_ETCDIR}/sysconfig/${_i}"
40 return
41 elif [ -f "${CTDB_SYS_ETCDIR}/default/${_i}" ]; then
42 . "${CTDB_SYS_ETCDIR}/default/${_i}"
43 return
44 fi
45 done
46 }
48 # load_script_options [ component script ]
49 # script is an event script name relative to a component
50 # component is currently ignored
51 load_script_options()
52 {
53 if [ $# -eq 2 ]; then
54 _script="$2"
55 elif [ $# -eq 0 ]; then
56 _script=""
57 else
58 die "usage: load_script_options [ component script ]"
59 fi
61 _options="${CTDB_BASE}/script.options"
63 if [ -r "$_options" ]; then
64 . "$_options"
65 fi
67 if [ -n "$_script" ]; then
68 _s="${CTDB_BASE}/events/legacy/${_script}"
69 else
70 _s="${0%.script}"
71 fi
72 _options="${_s}.options"
74 if [ -r "$_options" ]; then
75 . "$_options"
76 fi
77 }
79 ##############################################################
81 die()
82 {
83 _msg="$1"
84 _rc="${2:-1}"
86 echo "$_msg" >&2
87 exit "$_rc"
88 }
90 # Log given message or stdin to either syslog or a CTDB log file
91 # $1 is the tag passed to logger if syslog is in use.
92 script_log()
93 {
94 _tag="$1"
95 shift
97 case "$CTDB_LOGGING" in
98 file:)
99 if [ -n "$*" ] ; then
100 echo "$*"
101 else
102 cat
103 fi >&2
104 ;;
105 file:* | "")
106 if [ -n "$CTDB_LOGGING" ]; then
107 _file="${CTDB_LOGGING#file:}"
108 else
109 _file="/usr/local/var/log/log.ctdb"
110 fi
111 {
112 if [ -n "$*" ]; then
113 echo "$*"
114 else
115 cat
116 fi
117 } >>"$_file"
118 ;;
119 *)
120 # Handle all syslog:* variants here too. There's no tool to do
121 # the lossy things, so just use logger.
122 logger -t "ctdbd: ${_tag}" "$@"
123 ;;
124 esac
125 }
127 # When things are run in the background in an eventscript then logging
128 # output might get lost. This is the "solution". :-)
129 background_with_logging()
130 {
131 (
132 "$@" 2>&1 </dev/null |
133 script_log "${script_name}&"
134 ) &
136 return 0
137 }
139 ##############################################################
140 # check number of args for different events
141 ctdb_check_args()
142 {
143 case "$1" in
144 takeip | releaseip)
145 if [ $# != 4 ]; then
146 echo "ERROR: must supply interface, IP and maskbits"
147 exit 1
148 fi
149 ;;
150 updateip)
151 if [ $# != 5 ]; then
152 echo "ERROR: must supply old interface, new interface, IP and maskbits"
153 exit 1
154 fi
155 ;;
156 esac
157 }
159 ##############################################################
160 # determine on what type of system (init style) we are running
161 detect_init_style()
162 {
163 _init_style_file="${CTDB_SCRIPT_VARDIR}/init-style"
165 if [ ! -f "$_init_style_file" ]; then
166 if [ -n "$CTDB_INIT_STYLE" ]; then
167 echo "$CTDB_INIT_STYLE" >"$_init_style_file"
168 return
169 fi
171 # Subshell to contain variables in os-release file
172 (
173 _os_release="${CTDB_SYS_ETCDIR}/os-release"
174 if [ -f "$_os_release" ]; then
175 . "$_os_release"
176 case "$ID" in
177 centos | fedora | rhel)
178 echo "redhat"
179 ;;
180 debian | ubuntu)
181 echo "debian"
182 ;;
183 sles | suse)
184 echo "suse"
185 ;;
186 *)
187 case "$ID_LIKE" in
188 *centos* | *rhel*)
189 echo "redhat"
190 ;;
191 *)
192 echo "$ID"
193 ;;
194 esac
195 ;;
196 esac
197 else
198 echo "WARNING: unknown distribution ${ID}" >&2
199 echo "unknown"
200 fi
201 ) >"$_init_style_file"
202 fi
204 read -r CTDB_INIT_STYLE <"$_init_style_file"
205 }
207 ######################################################
208 # simulate /sbin/service on platforms that don't have it
209 # _service() makes it easier to hook the service() function for
210 # testing.
211 _service()
212 {
213 _service_name="$1"
214 _op="$2"
216 # do nothing, when no service was specified
217 [ -z "$_service_name" ] && return
219 if [ -x /sbin/service ]; then
220 $_nice /sbin/service "$_service_name" "$_op"
221 elif [ -x /usr/sbin/service ]; then
222 $_nice /usr/sbin/service "$_service_name" "$_op"
223 elif [ -x /bin/systemctl ]; then
224 $_nice /bin/systemctl "$_op" "$_service_name"
225 elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
226 $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
227 elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
228 $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
229 fi
230 }
232 service()
233 {
234 _nice=""
235 _service "$@"
236 }
238 ######################################################
239 # simulate /sbin/service (niced) on platforms that don't have it
240 nice_service()
241 {
242 _nice="nice"
243 _service "$@"
244 }
246 ######################################################
247 # Cached retrieval of PNN from local node. This never changes so why
248 # open a client connection to the server each time this is needed?
249 ctdb_get_pnn()
250 {
251 _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
252 if [ ! -f "$_pnn_file" ]; then
253 $CTDB pnn >"$_pnn_file"
254 fi
256 cat "$_pnn_file"
257 }
259 # Cached retrieval of private IP address from local node. This never
260 # changes.
261 ctdb_get_ip_address()
262 {
263 _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
264 if [ ! -f "$_ip_addr_file" ]; then
265 $CTDB -X nodestatus |
266 awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
267 fi
269 cat "$_ip_addr_file"
270 }
272 # Cache of public IP addresses assigned to this node. This function
273 # exists mainly so statd_callout does not need to talk to ctdbd, so
274 # can be run as non-root, but it may be used in other places. This
275 # must be updated/refreshed on failover. This is done in
276 # 10.interface, but doing it in "ipreallocated" isn't enough because
277 # clients may connect as soon as "takeip" completes. Also, the VNN in
278 # the daemon is only updated after the "releaseip" event completes, so
279 # "ctdb -X ip" can't be relied on there. Hence, complex updates
280 # involving locking for "takeip" & "releaseip". A future
281 # restructuring of the failover model will obsolete all of these
282 # moving parts.
283 CTDB_MY_PUBLIC_IPS_CACHE="${CTDB_SCRIPT_VARDIR}/my-public-ip-addresses"
284 update_my_public_ip_addresses()
285 {
286 _event="$1"
288 _f="$CTDB_MY_PUBLIC_IPS_CACHE"
289 _lock="${_f}.lock"
291 # In private CTDB state directory - no $$ security issue
292 _new="${_f}.new.$$"
293 {
294 flock --timeout 10 9 ||
295 die "ctdb_get_my_public_ip_addresses: timeout"
297 case "$_event" in
298 takeip)
299 _ip="$2"
300 # Redirect of stderr guards against initial
301 # missing file
302 cat "$_f" 2>/dev/null >"$_new"
303 echo "$_ip" >>"$_new"
304 ;;
305 releaseip)
306 _ip="$2"
307 # Redirect of stderr guards against initial
308 # missing file, which shouldn't happen in
309 # releaseip...
310 grep -Fvx "$_ip" "$_f" 2>/dev/null >"$_new"
311 ;;
312 ipreallocated)
313 _pnn=$(ctdb_get_pnn)
314 $CTDB -X ip |
315 awk -F'|' -v pnn="$_pnn" \
316 '$3 == pnn {print $2}' >"$_new"
317 ;;
318 esac
320 mv "$_new" "$_f"
322 } 9>"$_lock"
323 }
325 # Cached retrieval of database options for use by event scripts.
326 #
327 # If the variables are already set then they should not be overwritten
328 # - this should only happen during event script testing.
329 ctdb_get_db_options()
330 {
331 _db_opts_file="${CTDB_SCRIPT_VARDIR}/db_options.cache"
333 if [ ! -f "$_db_opts_file" ]; then
334 {
335 ctdb_translate_option "database" \
336 "volatile database directory" \
337 "CTDB_DBDIR"
338 ctdb_translate_option "database" \
339 "persistent database directory" \
340 "CTDB_DBDIR_PERSISTENT"
341 ctdb_translate_option "database" \
342 "state database directory" \
343 "CTDB_DBDIR_STATE"
344 } >"$_db_opts_file"
345 fi
347 . "$_db_opts_file"
348 }
350 ctdb_translate_option()
351 {
352 _section="$1"
353 _opt="$2"
354 _variable="$3"
356 # ctdb-config already prints an error if something goes wrong
357 _t=$("${CTDB_HELPER_BINDIR}/ctdb-config" get "$_section" "$_opt") ||
358 exit $?
359 echo "${_variable}=\"${_t}\""
360 }
362 ######################################################
363 # wrapper around /proc/ settings to allow them to be hooked
364 # for testing
365 # 1st arg is relative path under /proc/, 2nd arg is value to set
366 set_proc()
367 {
368 echo "$2" >"/proc/$1"
369 }
371 set_proc_maybe()
372 {
373 if [ -w "/proc/$1" ]; then
374 set_proc "$1" "$2"
375 fi
376 }
378 ######################################################
379 # wrapper around getting file contents from /proc/ to allow
380 # this to be hooked for testing
381 # 1st arg is relative path under /proc/
382 get_proc()
383 {
384 cat "/proc/$1"
385 }
387 ######################################################
388 # Print up to $_max kernel stack traces for processes named $_program
389 program_stack_traces()
390 {
391 _prog="$1"
392 _max="${2:-1}"
394 _count=1
395 for _pid in $(pidof "$_prog"); do
396 [ "$_count" -le "$_max" ] || break
398 # Do this first to avoid racing with process exit
399 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
400 if [ -n "$_stack" ]; then
401 echo "Stack trace for ${_prog}[${_pid}]:"
402 echo "$_stack"
403 _count=$((_count + 1))
404 fi
405 done
406 }
408 ######################################################
409 # Ensure $service_name is set
410 assert_service_name()
411 {
412 # service_name is set by the event script
413 # shellcheck disable=SC2154
414 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
415 }
417 ######################################################
418 # check a set of directories is available
419 # return 1 on a missing directory
420 # directories are read from stdin
421 ######################################################
422 ctdb_check_directories_probe()
423 {
424 while IFS="" read -r d; do
425 case "$d" in
426 *%*)
427 continue
428 ;;
429 *)
430 [ -d "${d}/." ] || return 1
431 ;;
432 esac
433 done
434 }
436 ######################################################
437 # check a set of directories is available
438 # directories are read from stdin
439 ######################################################
440 ctdb_check_directories()
441 {
442 ctdb_check_directories_probe || {
443 echo "ERROR: $service_name directory \"$d\" not available"
444 exit 1
445 }
446 }
448 ######################################################
449 # check a set of tcp ports
450 # usage: ctdb_check_tcp_ports <ports...>
451 ######################################################
453 # Check whether something is listening on all of the given TCP ports
454 # using the "ctdb checktcpport" command.
455 ctdb_check_tcp_ports()
456 {
457 if [ -z "$1" ]; then
458 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
459 exit 1
460 fi
462 for _p; do # process each function argument (port)
463 _cmd="$CTDB checktcpport $_p"
464 _out=$($_cmd 2>&1)
465 _ret=$?
466 case "$_ret" in
467 0)
468 echo "$service_name not listening on TCP port $_p"
469 return 1
470 ;;
471 98)
472 # Couldn't bind, something already listening, next port
473 continue
474 ;;
475 *)
476 echo "unexpected error (${_ret}) running \"${_cmd}\""
477 if [ -n "$_out" ]; then
478 echo "$_out"
479 fi
480 return $_ret
481 ;;
482 esac
483 done
485 # All ports listening
486 return 0
487 }
489 ######################################################
490 # check a unix socket
491 # usage: ctdb_check_unix_socket SOCKPATH
492 ######################################################
493 ctdb_check_unix_socket()
494 {
495 _sockpath="$1"
497 if [ -z "$_sockpath" ]; then
498 echo "ERROR: ctdb_check_unix_socket() requires socket path"
499 return 1
500 fi
502 _out=$(ss -l -x "src ${_sockpath}" | tail -n +2)
503 if [ -z "$_out" ]; then
504 echo "ERROR: ${service_name} not listening on ${_sockpath}"
505 return 1
506 fi
507 }
509 ################################################
510 # kill off any TCP connections with the given IP
511 ################################################
512 kill_tcp_connections()
513 {
514 _iface="$1"
515 _ip="$2"
517 _oneway=false
518 if [ "$3" = "oneway" ]; then
519 _oneway=true
520 fi
522 get_tcp_connections_for_ip "$_ip" | {
523 _killcount=0
524 _connections=""
525 _nl="
526 "
527 while read -r _dst _src; do
528 _destport="${_dst##*:}"
529 __oneway=$_oneway
530 case $_destport in
531 # we only do one-way killtcp for CIFS
532 139 | 445) __oneway=true ;;
533 esac
535 _connections="${_connections}${_nl}${_src} ${_dst}"
536 if ! $__oneway; then
537 _connections="${_connections}${_nl}${_dst} ${_src}"
538 fi
540 _killcount=$((_killcount + 1))
541 done
543 if [ $_killcount -eq 0 ]; then
544 return
545 fi
547 if [ -n "$CTDB_KILLTCP_DEBUGLEVEL" ]; then
548 _debuglevel="$CTDB_KILLTCP_DEBUGLEVEL"
549 else
550 _debuglevel="$CTDB_DEBUGLEVEL"
551 fi
552 echo "$_connections" |
553 CTDB_DEBUGLEVEL="$_debuglevel" \
554 "${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
555 echo "Failed to kill TCP connections"
556 return
557 }
559 _connections=$(get_tcp_connections_for_ip "$_ip")
560 if [ -z "$_connections" ]; then
561 _remaining=0
562 else
563 _remaining=$(echo "$_connections" | wc -l)
564 fi
566 _actually_killed=$((_killcount - _remaining))
568 _t="${_actually_killed}/${_killcount}"
569 echo "Killed ${_t} TCP connections to released IP $_ip"
571 if [ -n "$_connections" ]; then
572 echo "Remaining connections:"
573 echo "$_connections" | sed -e 's|^| |'
574 fi
575 }
576 }
578 ##################################################################
579 # kill off the local end for any TCP connections with the given IP
580 ##################################################################
581 kill_tcp_connections_local_only()
582 {
583 kill_tcp_connections "$@" "oneway"
584 }
586 ##################################################################
587 # tickle any TCP connections with the given IP
588 ##################################################################
589 tickle_tcp_connections()
590 {
591 _ip="$1"
593 # Get connections, both directions
594 _conns=$(get_tcp_connections_for_ip "$_ip" |
595 awk '{ print $1, $2 ; print $2, $1 }')
597 echo "$_conns" | awk '{ print "Tickle TCP connection", $1, $2 }'
598 echo "$_conns" | ctdb tickle
599 }
601 get_tcp_connections_for_ip()
602 {
603 _ip="$1"
605 ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
606 }
608 ########################################################
610 add_ip_to_iface()
611 {
612 _iface=$1
613 _ip=$2
614 _maskbits=$3
616 # Ensure interface is up
617 ip link set "$_iface" up ||
618 die "Failed to bringup interface $_iface"
620 # Only need to define broadcast for IPv4
621 case "$_ip" in
622 *:*) _bcast="" ;;
623 *) _bcast="brd +" ;;
624 esac
626 # Intentionally unquoted multi-word value here
627 # shellcheck disable=SC2086
628 ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
629 echo "Failed to add $_ip/$_maskbits on dev $_iface"
630 return 1
631 }
633 # Wait 5 seconds for IPv6 addresses to stop being tentative...
634 if [ -z "$_bcast" ]; then
635 for _x in $(seq 1 10); do
636 ip addr show to "${_ip}/128" | grep -q "tentative" || break
637 sleep 0.5
638 done
640 # If the address was a duplicate then it won't be on the
641 # interface so flag an error.
642 _t=$(ip addr show to "${_ip}/128")
643 case "$_t" in
644 "")
645 echo "Failed to add $_ip/$_maskbits on dev $_iface"
646 return 1
647 ;;
648 *tentative* | *dadfailed*)
649 echo "Failed to add $_ip/$_maskbits on dev $_iface"
650 ip addr del "$_ip/$_maskbits" dev "$_iface"
651 return 1
652 ;;
653 esac
654 fi
655 }
657 delete_ip_from_iface()
658 {
659 _iface=$1
660 _ip=$2
661 _maskbits=$3
663 # This could be set globally for all interfaces but it is probably
664 # better to avoid surprises, so limit it the interfaces where CTDB
665 # has public IP addresses. There isn't anywhere else convenient
666 # to do this so just set it each time. This is much cheaper than
667 # remembering and re-adding secondaries.
668 set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
670 ip addr del "$_ip/$_maskbits" dev "$_iface" || {
671 echo "Failed to del $_ip on dev $_iface"
672 return 1
673 }
674 }
676 # If the given IP is hosted then print 2 items: maskbits and iface
677 ip_maskbits_iface()
678 {
679 _addr="$1"
681 case "$_addr" in
682 *:*) _bits=128 ;;
683 *) _bits=32 ;;
684 esac
685 ip addr show to "${_addr}/${_bits}" 2>/dev/null |
686 awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
687 sub("@.*", "", iface) }
688 $1 ~ /inet/ { mask = $2; sub(".*/", "", mask);
689 print mask, iface }'
690 }
692 drop_ip()
693 {
694 _addr="${1%/*}" # Remove optional maskbits
696 # Intentional word splitting here
697 # shellcheck disable=SC2046
698 set -- $(ip_maskbits_iface "$_addr")
699 if [ -n "$1" ]; then
700 _maskbits="$1"
701 _iface="$2"
702 echo "Removing public address $_addr/$_maskbits from device $_iface"
703 delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
704 fi
705 }
707 have_public_addresses()
708 {
709 [ -f "${CTDB_BASE}/public_addresses" ]
710 }
712 # This sets $public_ifaces as a side-effect.
713 get_public_ifaces()
714 {
715 # Get all the interfaces listed in the public_addresses file
716 public_ifaces=$(sed -e '/^#.*/d' \
717 -e 's/^[^\t ]*[\t ]*//' \
718 -e 's/,/ /g' \
719 -e 's/[\t ]*$//' "${CTDB_BASE}/public_addresses")
721 # Get the interfaces for which CTDB has public IPs configured.
722 # That is, for all but the 1st line, get the 1st field.
723 ctdb_ifaces=$($CTDB -X ifaces | sed -e '1d' -e 's@^|@@' -e 's@|.*@@')
725 # Add $ctdb_ifaces and make $public_ifaces unique
726 # Use word splitting to squash whitespace
727 # shellcheck disable=SC2086
728 public_ifaces=$(echo $public_ifaces $ctdb_ifaces | tr ' ' '\n' | sort -u)
729 }
731 drop_all_public_ips()
732 {
733 # _x is intentionally ignored
734 # shellcheck disable=SC2034
735 while read -r _ip _x; do
736 case "$_ip" in
737 \#*) continue ;;
738 esac
739 drop_ip "$_ip"
740 done <"${CTDB_BASE}/public_addresses"
741 }
743 flush_route_cache()
744 {
745 set_proc_maybe sys/net/ipv4/route/flush 1
746 set_proc_maybe sys/net/ipv6/route/flush 1
747 }
749 ########################################################
750 # Interface monitoring
752 # If the interface is a virtual one (e.g. VLAN) then get the
753 # underlying interface
754 interface_get_real()
755 {
756 _iface="$1"
758 # If $_iface is a VLAN (i.e. contains an '@') then strip every
759 # before the '@', otherwise print the whole interface
760 echo "${_iface##*@}"
761 }
763 # Check whether an interface is operational
764 interface_monitor()
765 {
766 _iface="$1"
768 _iface_info=$(ip -br link show "$_iface" 2>&1) || {
769 echo "ERROR: Monitored interface ${_iface} does not exist"
770 return 1
771 }
773 # If the interface is a virtual one (e.g. VLAN) then get the
774 # underlying interface.
775 _realiface=$(interface_get_real "${_iface_info%% *}")
777 if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null); then
778 # This is a bond: various monitoring strategies
779 echo "$_bi" | grep -q 'Currently Active Slave: None' && {
780 echo "ERROR: No active slaves for bond device ${_realiface}"
781 return 1
782 }
783 echo "$_bi" | grep -q '^MII Status: up' || {
784 echo "ERROR: public network interface ${_realiface} is down"
785 return 1
786 }
787 echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
788 # This works around a bug in the driver where the
789 # overall bond status can be up but none of the actual
790 # physical interfaces have a link.
791 echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
792 echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
793 return 1
794 }
795 }
797 return 0
798 else
799 # Not a bond
800 case "$_iface" in
801 lo*)
802 # loopback is always working
803 return 0
804 ;;
805 ib*)
806 # we don't know how to test ib links
807 return 0
808 ;;
809 *)
810 ethtool "$_iface" | grep -q 'Link detected: yes' || {
811 # On some systems, this is not successful when a
812 # cable is plugged but the interface has not been
813 # brought up previously. Bring the interface up
814 # and try again...
815 ip link set "$_iface" up
816 ethtool "$_iface" | grep -q 'Link detected: yes' || {
817 echo "ERROR: No link on the public network interface ${_iface}"
818 return 1
819 }
820 }
821 return 0
822 ;;
823 esac
824 fi
825 }
827 ########################################################
828 # Simple counters
829 _ctdb_counter_common()
830 {
831 [ $# -le 1 ] || die "usage: _ctdb_counter_common [name]"
833 if [ $# -eq 1 ]; then
834 _counter_name="${1}.failcount"
835 else
836 _counter_name="failcount"
837 fi
839 if [ -z "$script_state_dir" ]; then
840 die "ctdb_counter_* functions need ctdb_setup_state_dir()"
841 fi
843 _counter_file="${script_state_dir}/${_counter_name}"
844 }
845 # Some code passes an argument
846 # shellcheck disable=SC2120
847 ctdb_counter_init()
848 {
849 _ctdb_counter_common "$1"
851 : >"$_counter_file"
852 }
853 ctdb_counter_incr()
854 {
855 _ctdb_counter_common "$1"
857 # unary counting using newlines!
858 echo >>"$_counter_file"
859 }
860 ctdb_counter_get()
861 {
862 _ctdb_counter_common "$1"
863 # unary counting!
864 _val=$(wc -c 2>/dev/null <"$_counter_file" || echo 0)
865 # Strip leading spaces from output of wc (on freebsd)
866 # shellcheck disable=SC2086
867 echo $_val
868 }
869 ctdb_counter_exists()
870 {
871 _ctdb_counter_common "$1"
872 [ -e "$_counter_file" ]
873 }
875 #
876 # Fail counter/threshold combination to control warnings and node unhealthy
877 #
879 _failcount_validate_threshold()
880 {
881 case "$1" in
882 "") return 1 ;; # A failure that doesn't need a warning
883 *)
884 if echo "$1" | grep -qx '[0-9]*'; then
885 return 0
886 fi
888 echo "WARNING: ${1} is an invalid threshold in \"${2}\" check"
889 return 1
890 ;;
891 esac
892 }
894 _failcount_common()
895 {
896 _thing="$1"
898 _counter=$(echo "$_thing" | sed -e 's@/@_SLASH_@g' -e 's@ @_@g')
899 }
901 failcount_init()
902 {
903 _thing="$1"
905 _failcount_common "$_thing"
907 ctdb_counter_init "$_counter"
908 }
910 failcount_reset()
911 {
912 _thing="$1"
914 _failcount_common "$_thing"
916 _failcount=$(ctdb_counter_get "$_counter")
917 if [ "$_failcount" -eq 0 ]; then
918 return
919 fi
921 printf 'NOTICE: %s: no longer failing\n' "$_thing"
922 ctdb_counter_init "$_counter"
923 }
925 failcount_incr()
926 {
927 _thing="$1"
928 _thresholds="$2"
929 _output="$3"
931 _failcount_common "$_thing"
933 ctdb_counter_incr "$_counter"
934 _failcount=$(ctdb_counter_get "$_counter")
936 case "$_thresholds" in
937 *:*)
938 _warn_threshold="${_thresholds%:*}"
939 _unhealthy_threshold="${_thresholds#*:}"
940 ;;
941 "")
942 _warn_threshold=1
943 _unhealthy_threshold=""
944 ;;
945 *)
946 _warn_threshold="$_thresholds"
947 _unhealthy_threshold=""
948 ;;
949 esac
951 if _failcount_validate_threshold "$_unhealthy_threshold" "$_thing"; then
952 if [ "$_failcount" -ge "$_unhealthy_threshold" ]; then
953 printf 'ERROR: %s: fail count %d >= threshold %d\n' \
954 "$_thing" \
955 "$_failcount" \
956 "$_unhealthy_threshold"
957 # Only print output when exceeding the
958 # unhealthy threshold
959 if [ "$_failcount" -eq "$_unhealthy_threshold" ] && \
960 [ -n "$_output" ]; then
961 echo "$_output"
962 fi
963 exit 1
964 fi
965 fi
967 if _failcount_validate_threshold "$_warn_threshold" "$_thing"; then
968 if [ "$_failcount" -lt "$_warn_threshold" ]; then
969 return 0
970 fi
971 fi
973 printf 'WARNING: %s: fail count %d >= threshold %d\n' \
974 "$_thing" \
975 "$_failcount" \
976 "$_warn_threshold"
977 if [ "$_failcount" -eq "$_warn_threshold" ] && [ -n "$_output" ]; then
978 # Only print output when exceeding the warning threshold
979 echo "$_output"
980 fi
981 }
983 ########################################################
985 # ctdb_setup_state_dir <type> <name>
986 # Sets/creates script_state_dir)
987 ctdb_setup_state_dir()
988 {
989 [ $# -eq 2 ] || die "usage: ctdb_setup_state_dir <type> <name>"
991 _type="$1"
992 _name="$2"
994 script_state_dir="${CTDB_SCRIPT_VARDIR}/${_type}/${_name}"
996 mkdir -p "$script_state_dir" ||
997 die "Error creating script state dir \"${script_state_dir}\""
998 }
1000 ##################################################################
1001 # Reconfigure a service on demand
1003 _ctdb_service_reconfigure_common()
1004 {
1005 if [ -z "$script_state_dir" ]; then
1006 die "ctdb_service_*_reconfigure() needs ctdb_setup_state_dir()"
1007 fi
1009 _ctdb_service_reconfigure_flag="${script_state_dir}/need_reconfigure"
1010 }
1012 ctdb_service_needs_reconfigure()
1013 {
1014 _ctdb_service_reconfigure_common
1015 [ -e "$_ctdb_service_reconfigure_flag" ]
1016 }
1018 ctdb_service_set_reconfigure()
1019 {
1020 _ctdb_service_reconfigure_common
1021 : >"$_ctdb_service_reconfigure_flag"
1022 }
1024 ctdb_service_unset_reconfigure()
1025 {
1026 _ctdb_service_reconfigure_common
1027 rm -f "$_ctdb_service_reconfigure_flag"
1028 }
1030 ctdb_service_reconfigure()
1031 {
1032 echo "Reconfiguring service \"${service_name}\"..."
1033 ctdb_service_unset_reconfigure
1034 service_reconfigure || return $?
1035 # Intentionally have this use $service_name as default
1036 # shellcheck disable=SC2119
1037 ctdb_counter_init
1038 }
1040 # Default service_reconfigure() function does nothing.
1041 service_reconfigure()
1042 {
1043 :
1044 }
1046 # Default service_start() and service_stop() functions.
1048 # These may be overridden in an eventscript.
1049 service_start()
1050 {
1051 service "$service_name" start
1052 }
1054 service_stop()
1055 {
1056 service "$service_name" stop
1057 }
1059 ##################################################################
1061 # This exists only for backward compatibility with 3rd party scripts
1062 # that call it
1063 ctdb_standard_event_handler()
1064 {
1065 :
1066 }
1068 iptables_wrapper()
1069 {
1070 _family="$1"
1071 shift
1072 if [ "$_family" = "inet6" ]; then
1073 _iptables_cmd="ip6tables"
1074 else
1075 _iptables_cmd="iptables"
1076 fi
1078 # iptables doesn't like being re-entered, so flock-wrap it.
1079 flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
1080 }
1082 # AIX (and perhaps others?) doesn't have mktemp
1083 # type is commonly supported and more portable than which(1)
1084 # shellcheck disable=SC2039
1085 if ! type mktemp >/dev/null 2>&1; then
1086 mktemp()
1087 {
1088 _dir=false
1089 if [ "$1" = "-d" ]; then
1090 _dir=true
1091 shift
1092 fi
1093 _d="${TMPDIR:-/tmp}"
1094 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null |
1095 cksum |
1096 awk '{print $1}')
1097 _t="${_d}/tmp.${_hex10}"
1098 (
1099 umask 077
1100 if $_dir; then
1101 mkdir "$_t"
1102 else
1103 : >"$_t"
1104 fi
1105 )
1106 echo "$_t"
1107 }
1108 fi
1110 ######################################################################
1111 # NFS callout handling
1113 nfs_callout_init()
1114 {
1115 _state_dir="$1"
1117 if [ -z "$CTDB_NFS_CALLOUT" ]; then
1118 CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
1119 fi
1120 # Always export, for statd callout
1121 export CTDB_NFS_CALLOUT
1123 # If the callout wants to use this then it must create it
1124 export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"
1126 # Export, if set, for use by clustered NFS callouts
1127 if [ -n "$CTDB_NFS_STATE_FS_TYPE" ]; then
1128 export CTDB_NFS_STATE_FS_TYPE
1129 fi
1130 if [ -n "$CTDB_NFS_STATE_MNT" ]; then
1131 export CTDB_NFS_STATE_MNT
1132 fi
1133 if [ -n "$CTDB_NFS_EXPORTS_FILE" ]; then
1134 export CTDB_NFS_EXPORTS_FILE
1135 fi
1137 nfs_callout_cache="${_state_dir}/nfs_callout_cache"
1138 nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
1139 nfs_callout_cache_ops="${nfs_callout_cache}/ops"
1140 }
1142 nfs_callout_register()
1143 {
1144 mkdir -p "$nfs_callout_cache_ops"
1145 rm -f "$nfs_callout_cache_ops"/*
1147 echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
1149 _t=$("$CTDB_NFS_CALLOUT" "register")
1150 if [ -n "$_t" ]; then
1151 echo "$_t" |
1152 while IFS="" read -r _op; do
1153 touch "${nfs_callout_cache_ops}/${_op}"
1154 done
1155 else
1156 touch "${nfs_callout_cache_ops}/ALL"
1157 fi
1158 }
1160 nfs_callout()
1161 {
1162 # Re-run registration if $CTDB_NFS_CALLOUT has changed
1163 _prev=""
1164 if [ -r "$nfs_callout_cache_callout" ]; then
1165 read -r _prev <"$nfs_callout_cache_callout"
1166 fi
1167 if [ "$CTDB_NFS_CALLOUT" != "$_prev" ]; then
1168 nfs_callout_register
1169 fi
1171 # Run the operation if it is registered...
1172 if [ -e "${nfs_callout_cache_ops}/${1}" ] ||
1173 [ -e "${nfs_callout_cache_ops}/ALL" ]; then
1174 "$CTDB_NFS_CALLOUT" "$@"
1175 fi
1176 }
1178 ########################################################
1179 # tickle handling
1180 ########################################################
1182 update_tickles()
1183 {
1184 _port="$1"
1186 tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
1187 mkdir -p "$tickledir"
1189 # What public IPs do I hold?
1190 _pnn=$(ctdb_get_pnn)
1191 _ips=$($CTDB -X ip | awk -F'|' -v pnn="$_pnn" '$3 == pnn {print $2}')
1193 # IPs and port as ss filters
1194 _ip_filter=""
1195 for _ip in $_ips; do
1196 _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
1197 done
1198 _port_filter="sport == :${_port}"
1200 # Record connections to our public IPs in a temporary file.
1201 # This temporary file is in CTDB's private state directory and
1202 # $$ is used to avoid a very rare race involving CTDB's script
1203 # debugging. No security issue, nothing to see here...
1204 _my_connections="${tickledir}/${_port}.connections.$$"
1205 # Parentheses are needed around the filters for precedence but
1206 # the parentheses can't be empty!
1207 #
1208 # Recent versions of ss print square brackets around IPv6
1209 # addresses. While it is desirable to update CTDB's address
1210 # parsing and printing code, something needs to be done here
1211 # for backward compatibility, so just delete the brackets.
1212 ss -tn state established \
1213 "${_ip_filter:+( ${_ip_filter} )}" \
1214 "${_port_filter:+( ${_port_filter} )}" |
1215 awk 'NR > 1 {print $4, $3}' |
1216 tr -d '][' |
1217 sort >"$_my_connections"
1219 # Record our current tickles in a temporary file
1220 _my_tickles="${tickledir}/${_port}.tickles.$$"
1221 for _i in $_ips; do
1222 $CTDB -X gettickles "$_i" "$_port" |
1223 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1224 done |
1225 sort >"$_my_tickles"
1227 # Add tickles for connections that we haven't already got tickles for
1228 comm -23 "$_my_connections" "$_my_tickles" |
1229 $CTDB addtickle
1231 # Remove tickles for connections that are no longer there
1232 comm -13 "$_my_connections" "$_my_tickles" |
1233 $CTDB deltickle
1235 rm -f "$_my_connections" "$_my_tickles"
1237 # Remove stale files from killed scripts
1238 # Files can't have spaces in name, more portable than -print0/-0
1239 # shellcheck disable=SC2038
1240 (cd "$tickledir" && find . -type f -mmin +10 | xargs -r rm)
1241 }
1243 ########################################################
1244 # load a site local config file
1245 ########################################################
1247 [ -x "${CTDB_BASE}/rc.local" ] && {
1248 . "${CTDB_BASE}/rc.local"
1249 }
1251 [ -d "${CTDB_BASE}/rc.local.d" ] && {
1252 for i in "${CTDB_BASE}/rc.local.d"/*; do
1253 [ -x "$i" ] && . "$i"
1254 done
1255 }
1257 script_name="${0##*/}" # basename