ctdb-scripts: Memory monitoring uses thresholds expressed as percentages
[Samba.git] / ctdb / tests / eventscripts / scripts / local.sh
blobce1c2510bd0b729ad5a9f46023e6e7459774dc7e
1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 # Augment PATH with relevant stubs/ directories. We do this by actually
4 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
5 # prepending that to $PATH in rc.local to avoid the PATH reset in
6 # functions.
8 EVENTSCRIPTS_PATH=""
10 if [ -d "${TEST_SUBDIR}/stubs" ] ; then
11 EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs"
12 case "$EVENTSCRIPTS_PATH" in
13 /*) : ;;
14 *) EVENTSCRIPTS_PATH="${PWD}/${EVENTSCRIPTS_PATH}" ;;
15 esac
18 export EVENTSCRIPTS_PATH
20 PATH="${EVENTSCRIPTS_PATH}:${PATH}"
22 export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts"
23 if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \
24 "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then
25 rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
27 mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR"
28 export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
30 export CTDB_LOGGING="file:${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb"
31 touch "${CTDB_LOGGING#file:}" || \
32 die "Unable to setup logging for \"$CTDB_LOGGING\""
34 if [ -d "${TEST_SUBDIR}/etc" ] ; then
35 cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR"
36 export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
37 else
38 die "Unable to setup \$CTDB_ETCDIR"
41 if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then
42 cp -prL "${TEST_SUBDIR}/etc-ctdb" "$EVENTSCRIPTS_TESTS_VAR_DIR"
43 export CTDB_BASE="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc-ctdb"
44 else
45 die "Unable to set \$CTDB_BASE"
47 export CTDB_BASE
49 if [ ! -d "${CTDB_BASE}/events.d" ] ; then
50 cat <<EOF
51 ERROR: Directory ${CTDB_BASE}/events.d does not exist.
53 That means that no eventscripts can be tested.
55 One possible explanation:
57 You have CTDB installed via RPMs (or similar), so the regular
58 CTDB_BASE directory is in /etc/ctdb/
60 BUT
62 You have done a regular "configure" and "make install" so the tests
63 are installed under /usr/local/.
65 If so, one possible hack to fix this is to create a symlink:
67 ln -s /etc/ctdb /usr/local/etc/ctdb
69 This is nasty but it works... :-)
70 EOF
71 exit 1
74 ######################################################################
76 if "$TEST_VERBOSE" ; then
77 debug () { echo "$@" ; }
78 else
79 debug () { : ; }
82 ######################################################################
84 # General setup fakery
86 setup_generic ()
88 debug "Setting up shares (3 existing shares)"
89 # Create 3 fake shares/exports.
90 export FAKE_SHARES=""
91 for i in $(seq 1 3) ; do
92 _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing"
93 mkdir -p "$_s"
94 FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}"
95 done
97 export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding"
98 mkdir -p "$FAKE_PROC_NET_BONDING"
99 rm -f "$FAKE_PROC_NET_BONDING"/*
101 export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down"
102 mkdir -p "$FAKE_ETHTOOL_LINK_DOWN"
103 rm -f "$FAKE_ETHTOOL_LINK_DOWN"/*
105 # This can only have 2 levels. We don't want to resort to usings
106 # something dangerous like "rm -r" setup time.
107 export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state"
108 mkdir -p "$FAKE_IP_STATE"
109 rm -f "$FAKE_IP_STATE"/*/*
110 rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true
111 rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true
114 export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
115 mkdir -p "${CTDB_DBDIR}/persistent"
117 export FAKE_TDBTOOL_SUPPORTS_CHECK="yes"
118 export FAKE_TDB_IS_OK
119 export FAKE_DATE_OUTPUT
121 export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
122 export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
125 tcp_port_down ()
127 for _i ; do
128 debug "Marking TCP port \"${_i}\" as not listening"
129 FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
130 done
133 shares_missing ()
135 _fmt="$1" ; shift
137 # Replace some shares with non-existent ones.
138 _t=""
139 _n=1
140 _nl="
142 export MISSING_SHARES_TEXT=""
143 for _i in $FAKE_SHARES ; do
144 if [ $_n = "$1" ] ; then
145 shift
146 _i="${_i%_existing}_missing"
147 debug "Replacing share $_n with missing share \"$_i\""
148 rmdir "$_i" 2>/dev/null || true
149 MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}")
151 _t="${_t}${_t:+ }${_i}"
152 _n=$(($_n + 1))
153 done
154 FAKE_SHARES="$_t"
157 # Setup some fake /proc/net/bonding files with just enough info for
158 # the eventscripts.
160 # arg1 is interface name, arg2 is currently active slave (use "None"
161 # if none), arg3 is MII status ("up" or "down").
162 setup_bond ()
164 _iface="$1"
165 _slave="${2:-${_iface}_sl_0}"
166 _mii_s="${3:-up}"
167 _mii_subs="${4:-${_mii_s:-up}}"
168 echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s"
169 cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF
170 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
171 Currently Active Slave: $_slave
172 # Status of the bond
173 MII Status: $_mii_s
174 # Status of 1st pretend adapter
175 MII Status: $_mii_subs
176 # Status of 2nd pretend adapter
177 MII Status: $_mii_subs
181 ethtool_interfaces_down ()
183 for _i ; do
184 echo "Marking interface $_i DOWN for ethtool"
185 touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
186 done
189 ethtool_interfaces_up ()
191 for _i ; do
192 echo "Marking interface $_i UP for ethtool"
193 rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
194 done
197 dump_routes ()
199 echo "# ip rule show"
200 ip rule show
202 ip rule show |
203 while read _p _x _i _x _t ; do
204 # Remove trailing colon after priority/preference.
205 _p="${_p%:}"
206 # Only remove rules that match our priority/preference.
207 [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
209 echo "# ip route show table $_t"
210 ip route show table "$_t"
211 done
214 # Copied from 13.per_ip_routing for now... so this is lazy testing :-(
215 ipv4_host_addr_to_net ()
217 _host="$1"
218 _maskbits="$2"
220 # Convert the host address to an unsigned long by splitting out
221 # the octets and doing the math.
222 _host_ul=0
223 for _o in $(export IFS="." ; echo $_host) ; do
224 _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
225 done
227 # Calculate the mask and apply it.
228 _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
229 _net_ul=$(( $_host_ul & $_mask_ul ))
231 # Now convert to a network address one byte at a time.
232 _net=""
233 for _o in $(seq 1 4) ; do
234 _net="$(($_net_ul & 255))${_net:+.}${_net}"
235 _net_ul=$(($_net_ul >> 8))
236 done
238 echo "${_net}/${_maskbits}"
241 ######################################################################
243 # CTDB fakery
245 # Evaluate an expression that probably calls functions or uses
246 # variables from the CTDB functions file. This is used for test
247 # initialisation.
248 eventscript_call ()
251 . "$CTDB_BASE/functions"
252 "$@"
256 # For now this creates the same public addresses each time. However,
257 # it could be made more flexible.
258 setup_public_addresses ()
260 if [ -f "$CTDB_PUBLIC_ADDRESSES" -a \
261 "${CTDB_PUBLIC_ADDRESSES%/*}" = "$EVENTSCRIPTS_TESTS_VAR_DIR" ] ; then
262 rm "$CTDB_PUBLIC_ADDRESSES"
265 export CTDB_PUBLIC_ADDRESSES=$(mktemp \
266 --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR" \
267 "public-addresses-XXXXXXXX")
269 echo "Setting up CTDB_PUBLIC_ADDRESSES=${CTDB_PUBLIC_ADDRESSES}"
270 cat >"$CTDB_PUBLIC_ADDRESSES" <<EOF
271 10.0.0.1/24 dev123
272 10.0.0.2/24 dev123
273 10.0.0.3/24 dev123
274 10.0.0.4/24 dev123
275 10.0.0.5/24 dev123
276 10.0.0.6/24 dev123
277 10.0.1.1/24 dev456
278 10.0.1.2/24 dev456
279 10.0.1.3/24 dev456
283 # Need to cope with ctdb_get_pnn(). If a test changes PNN then it
284 # needs to be using a different state directory, otherwise the wrong
285 # PNN can already be cached in the state directory.
286 ctdb_set_pnn ()
288 export FAKE_CTDB_PNN="$1"
289 echo "Setting up PNN ${FAKE_CTDB_PNN}"
291 export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb/${FAKE_CTDB_PNN}"
292 mkdir -p "$CTDB_VARDIR"
295 setup_ctdb ()
297 setup_generic
299 export FAKE_CTDB_NUMNODES="${1:-3}"
300 echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes"
302 ctdb_set_pnn "${2:-0}"
304 setup_public_addresses
306 export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb"
308 export FAKE_CTDB_EXTRA_CONFIG="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-config.sh"
309 rm -f "$FAKE_CTDB_EXTRA_CONFIG"
311 export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down"
312 mkdir -p "$FAKE_CTDB_IFACES_DOWN"
313 rm -f "$FAKE_CTDB_IFACES_DOWN"/*
315 export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus"
316 mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
317 rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
319 export CTDB_PARTIALLY_ONLINE_INTERFACES
321 export FAKE_CTDB_TUNABLES_OK="MonitorInterval TDBMutexEnabled DatabaseHashSize"
322 export FAKE_CTDB_TUNABLES_OBSOLETE="EventScriptUnhealthyOnTimeout"
325 setup_config ()
327 cat >"$FAKE_CTDB_EXTRA_CONFIG"
330 validate_percentage ()
332 case "$1" in
333 [0-9]|[0-9][0-9]|100) return 0 ;;
334 *) echo "WARNING: ${1} is an invalid percentage${2:+\" in }${2}${2:+\"}"
335 return 1
336 esac
339 setup_memcheck ()
341 _mem_usage="${1:-10}" # Default is 10%
342 _swap_usage="${2:-0}" # Default is 0%
344 setup_ctdb
346 _swap_total=5857276
347 _swap_free=$(( (100 - $_swap_usage) * $_swap_total / 100 ))
349 _mem_total=3940712
350 _mem_free=225268
351 _mem_buffers=146120
352 _mem_cached=$(( $_mem_total * (100 - $_mem_usage) / 100 - $_mem_free - $_mem_buffers ))
354 export FAKE_PROC_MEMINFO="\
355 MemTotal: ${_mem_total} kB
356 MemFree: ${_mem_free} kB
357 Buffers: ${_mem_buffers} kB
358 Cached: ${_mem_cached} kB
359 SwapCached: 56016 kB
360 Active: 2422104 kB
361 Inactive: 1019928 kB
362 Active(anon): 1917580 kB
363 Inactive(anon): 523080 kB
364 Active(file): 504524 kB
365 Inactive(file): 496848 kB
366 Unevictable: 4844 kB
367 Mlocked: 4844 kB
368 SwapTotal: ${_swap_total} kB
369 SwapFree: ${_swap_free} kB
370 ..."
372 export CTDB_MONITOR_FREE_MEMORY
373 export CTDB_MONITOR_FREE_MEMORY_WARN
374 export CTDB_CHECK_SWAP_IS_NOT_USED
377 setup_fscheck ()
379 export FAKE_FS_USE="${1:-10}" # Default is 10% usage
381 # Causes some variables to be exported
382 setup_ctdb
384 export CTDB_MONITOR_FILESYSTEM_USAGE
387 ctdb_get_interfaces ()
389 # The echo/subshell forces all the output onto 1 line.
390 echo $(ctdb ifaces -X | awk -F'|' 'FNR > 1 {print $2}')
393 ctdb_get_1_interface ()
395 _t=$(ctdb_get_interfaces)
396 echo ${_t%% *}
399 # Print all public addresses as: interface IP maskbits
400 # Each line is suitable for passing to takeip/releaseip
401 ctdb_get_all_public_addresses ()
403 _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
404 while IFS="/$IFS" read _ip _maskbits _ifaces ; do
405 echo "$_ifaces $_ip $_maskbits"
406 done <"$_f"
409 # Print public addresses on this node as: interface IP maskbits
410 # Each line is suitable for passing to takeip/releaseip
411 ctdb_get_my_public_addresses ()
413 ctdb ip -v -X | {
414 read _x # skip header line
416 while IFS="|" read _x _ip _x _iface _x ; do
417 [ -n "$_iface" ] || continue
418 while IFS="/$IFS" read _i _maskbits _x ; do
419 if [ "$_ip" = "$_i" ] ; then
420 echo $_iface $_ip $_maskbits
421 break
423 done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
424 done
428 # Prints the 1st public address as: interface IP maskbits
429 # This is suitable for passing to takeip/releaseip
430 ctdb_get_1_public_address ()
432 ctdb_get_my_public_addresses | { head -n 1 ; cat >/dev/null ; }
435 ctdb_not_implemented ()
437 export CTDB_NOT_IMPLEMENTED="$1"
438 ctdb_not_implemented="\
439 DEBUG: ctdb: command \"$1\" not implemented in stub"
442 ctdb_fake_scriptstatus ()
444 _code="$1"
445 _status="$2"
446 _err_out="$3"
448 _d1=$(date '+%s.%N')
449 _d2=$(date '+%s.%N')
451 echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
454 ######################################################################
456 setup_ctdb_policy_routing ()
458 service_name="per_ip_routing"
460 export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing"
461 export CTDB_PER_IP_ROUTING_RULE_PREF=100
462 export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000
463 export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000
465 # Tests need to create and populate this file
466 rm -f "$CTDB_PER_IP_ROUTING_CONF"
469 # Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF.
470 # $1 is the number of assigned IPs to use (<num>, all), defaulting to
471 # 1. If $2 is "default" then a default route is also added.
472 create_policy_routing_config ()
474 _num_ips="${1:-1}"
475 _should_add_default="$2"
477 ctdb_get_my_public_addresses |
478 if [ "$_num_ips" = "all" ] ; then
480 else
481 { head -n "$_num_ips" ; cat >/dev/null ; }
482 fi |
483 while read _dev _ip _bits ; do
484 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
485 _gw="${_net%.*}.254" # a dumb, calculated default
487 echo "$_ip $_net"
489 if [ "$_should_add_default" = "default" ] ; then
490 echo "$_ip 0.0.0.0/0 $_gw"
492 done >"$CTDB_PER_IP_ROUTING_CONF"
495 # Check the routes against those that are expected. $1 is the number
496 # of assigned IPs to use (<num>, all), defaulting to 1. If $2 is
497 # "default" then expect default routes to have been added.
498 check_routes ()
500 _num_ips="${1:-1}"
501 _should_add_default="$2"
503 _policy_rules=""
504 _policy_routes=""
506 ctdb_get_my_public_addresses |
507 if [ "$_num_ips" = "all" ] ; then
509 else
510 { head -n "$_num_ips" ; cat >/dev/null ; }
511 fi | {
512 while read _dev _ip _bits ; do
513 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
514 _gw="${_net%.*}.254" # a dumb, calculated default
516 _policy_rules="${_policy_rules}
517 ${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip "
518 _policy_routes="${_policy_routes}
519 # ip route show table ctdb.$_ip
520 $_net dev $_dev scope link "
522 if [ "$_should_add_default" = "default" ] ; then
523 _policy_routes="${_policy_routes}
524 default via $_gw dev $_dev "
526 done
528 ok <<EOF
529 # ip rule show
530 0: from all lookup local ${_policy_rules}
531 32766: from all lookup main
532 32767: from all lookup default ${_policy_routes}
535 simple_test_command dump_routes
536 } || test_fail
539 ######################################################################
541 ctdb_catdb_format_pairs ()
543 _count=0
545 while read _k _v ; do
546 _kn=$(echo -n "$_k" | wc -c)
547 _vn=$(echo -n "$_v" | wc -c)
548 cat <<EOF
549 key(${_kn}) = "${_k}"
550 dmaster: 0
551 rsn: 1
552 data(${_vn}) = "${_v}"
555 _count=$(($_count + 1))
556 done
558 echo "Dumped ${_count} records"
561 check_ctdb_tdb_statd_state ()
563 ctdb_get_my_public_addresses |
564 while read _x _sip _x ; do
565 for _cip ; do
566 echo "statd-state@${_sip}@${_cip}" "$FAKE_DATE_OUTPUT"
567 done
568 done |
569 ctdb_catdb_format_pairs | {
571 simple_test_command ctdb catdb ctdb.tdb
572 } || test_fail
575 check_statd_callout_smnotify ()
577 _state_even=$(( $(date '+%s') / 2 * 2))
578 _state_odd=$(($_state_even + 1))
580 nfs_load_config
582 ctdb_get_my_public_addresses |
583 while read _x _sip _x ; do
584 for _cip ; do
585 cat <<EOF
586 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_even}
587 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_even}
588 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_odd}
589 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_odd}
591 done
592 done | {
594 simple_test_event "notify"
595 } || test_fail
598 ######################################################################
600 setup_ctdb_natgw ()
602 debug "Setting up NAT gateway"
604 natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
605 mkdir -p "$natgw_config_dir"
607 # These will accumulate, 1 per test... but will be cleaned up at
608 # the end.
609 export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
611 # Read from stdin
612 while read _ip _master _dev ; do
613 echo "$_ip"
614 if [ "$_master" = "master" ] ; then
615 export FAKE_CTDB_NATGW_MASTER="$_ip"
617 done >"$CTDB_NATGW_NODES"
619 # Assume all of the nodes are on a /24 network and have IPv4
620 # addresses:
621 read _ip <"$CTDB_NATGW_NODES"
622 export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
624 # These are fixed. Probably don't use the same network for the
625 # private node IPs. To unset the default gateway just set it to
626 # "". :-)
627 export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
628 export CTDB_NATGW_PUBLIC_IFACE="eth1"
629 export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
630 export CTDB_NATGW_SLAVE_ONLY=""
633 ok_natgw_master_ip_addr_show ()
635 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
637 # This is based on CTDB_NATGW_PUBLIC_IP
638 _brd="10.1.1.255"
640 ok <<EOF
641 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
642 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
643 inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE}
644 valid_lft forever preferred_lft forever
648 ok_natgw_slave_ip_addr_show ()
650 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
651 ok <<EOF
652 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
653 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
657 ok_natgw_master_static_routes ()
659 _nl="
661 _t=""
662 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
663 # This is intentionally different to the code in 11.natgw ;-)
664 case "$_i" in
665 *@*)
666 _net=$(echo "$_i" | sed -e 's|@.*||')
667 _gw=$(echo "$_i" | sed -e 's|.*@||')
670 _net="$_i"
671 _gw="$CTDB_NATGW_DEFAULT_GATEWAY"
672 esac
674 [ -n "$_gw" ] || continue
675 _t="${_t}${_t:+${_nl}}"
676 _t="${_t}${_net} via ${_gw} dev ethXXX metric 10 "
677 done
678 _t=$(echo "$_t" | sort)
679 ok "$_t"
682 ok_natgw_slave_static_routes ()
684 _nl="
686 _t=""
687 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
688 # This is intentionally different to the code in 11.natgw ;-)
689 _net=$(echo "$_i" | sed -e 's|@.*||')
691 # The interface for the private network isn't specified as
692 # part of the NATGW configuration and isn't part of the
693 # command to add the route. It is implicitly added by "ip
694 # route" but our stub doesn't do this and adds "ethXXX".
695 _t="${_t}${_t:+${_nl}}"
696 _t="${_t}${_net} via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX metric 10 "
697 done
698 _t=$(echo "$_t" | sort)
699 ok "$_t"
702 ######################################################################
704 # Samba/winbind fakery
706 setup_samba ()
708 setup_ctdb
710 service_name="samba"
712 if [ "$1" != "down" ] ; then
714 debug "Marking Samba services as up, listening and managed by CTDB"
715 # Get into known state.
716 eventscript_call ctdb_service_managed
718 # All possible service names for all known distros.
719 for i in "smb" "nmb" "samba" ; do
720 service "$i" force-started
721 done
723 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
724 export CTDB_MANAGED_SERVICES="foo samba bar"
726 export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
727 export FAKE_WBINFO_FAIL="no"
729 # Some things in 50.samba are backgrounded and waited for. If
730 # we don't sleep at all then timeouts can happen. This avoids
731 # that... :-)
732 export FAKE_SLEEP_FORCE=0.1
733 else
734 debug "Marking Samba services as down, not listening and not managed by CTDB"
735 # Get into known state.
736 eventscript_call ctdb_service_unmanaged
738 # All possible service names for all known distros.
739 for i in "smb" "nmb" "samba" ; do
740 service "$i" force-stopped
741 done
743 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
744 export CTDB_MANAGED_SERVICES="foo bar"
745 unset CTDB_MANAGES_SAMBA
747 export FAKE_TCP_LISTEN=""
748 export FAKE_WBINFO_FAIL="yes"
752 setup_winbind ()
754 setup_ctdb
756 service_name="winbind"
758 if [ "$1" != "down" ] ; then
760 debug "Marking Winbind service as up and managed by CTDB"
761 # Get into known state.
762 eventscript_call ctdb_service_managed
764 service "winbind" force-started
766 export CTDB_MANAGED_SERVICES="foo winbind bar"
768 export FAKE_WBINFO_FAIL="no"
770 else
771 debug "Marking Winbind service as down and not managed by CTDB"
772 # Get into known state.
773 eventscript_call ctdb_service_unmanaged
775 service "winbind" force-stopped
777 export CTDB_MANAGED_SERVICES="foo bar"
778 unset CTDB_MANAGES_WINBIND
780 export FAKE_WBINFO_FAIL="yes"
784 wbinfo_down ()
786 debug "Making wbinfo commands fail"
787 FAKE_WBINFO_FAIL="yes"
790 ######################################################################
792 # NFS fakery
794 setup_nfs ()
796 setup_ctdb
798 service_name="nfs"
800 export FAKE_RPCINFO_SERVICES=""
802 export CTDB_NFS_SKIP_SHARE_CHECK="no"
804 export RPCNFSDCOUNT
806 # Reset the failcounts for nfs services.
807 eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
809 if [ "$1" != "down" ] ; then
810 debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB"
812 eventscript_call ctdb_service_managed
813 service "nfs" force-started
814 service "nfslock" force-started
816 export CTDB_MANAGED_SERVICES="foo nfs bar"
818 rpc_services_up \
819 "portmapper" "nfs" "mountd" "rquotad" "nlockmgr" "status"
821 nfs_setup_fake_threads "nfsd"
822 nfs_setup_fake_threads "rpc.foobar" # Just set the variable to empty
823 else
824 debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
826 eventscript_call ctdb_service_unmanaged
827 service "nfs" force-stopped
828 service "nfslock" force-stopped
830 export CTDB_MANAGED_SERVICES="foo bar"
831 unset CTDB_MANAGES_NFS
834 # This is really nasty. However, when we test NFS we don't
835 # actually test statd-callout. If we leave it there then left
836 # over, backgrounded instances of statd-callout will do horrible
837 # things with the "ctdb ip" stub and cause the actual
838 # statd-callout tests that follow to fail.
839 rm "${CTDB_BASE}/statd-callout"
842 setup_nfs_ganesha ()
844 setup_nfs "$@"
845 export CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-ganesha-callout"
846 if [ "$1" != "down" ] ; then
847 export CTDB_MANAGES_NFS="yes"
850 export CTDB_NFS_SKIP_SHARE_CHECK="yes"
853 rpc_services_down ()
855 for _i ; do
856 debug "Marking RPC service \"${_i}\" as unavailable"
857 FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
858 done
861 rpc_services_up ()
863 for _i ; do
864 debug "Marking RPC service \"${_i}\" as available"
865 case "$_i" in
866 portmapper) _t="2:4" ;;
867 nfs) _t="2:3" ;;
868 mountd) _t="1:3" ;;
869 rquotad) _t="1:2" ;;
870 nlockmgr) _t="3:4" ;;
871 status) _t="1:1" ;;
872 *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
873 esac
875 FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
876 done
880 nfs_load_config ()
882 _etc="$CTDB_ETCDIR" # shortcut for readability
883 for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
884 if [ -r "$_c" ] ; then
885 . "$_c"
886 break
888 done
891 nfs_setup_fake_threads ()
893 _prog="$1" ; shift
895 case "$_prog" in
896 nfsd)
897 export PROCFS_PATH=$(mktemp -d --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
898 _threads="${PROCFS_PATH}/fs/nfsd/threads"
899 mkdir -p $(dirname "$_threads")
900 echo $# >"$_threads"
901 export FAKE_NFSD_THREAD_PIDS="$*"
904 export FAKE_RPC_THREAD_PIDS="$*"
906 esac
909 program_stack_traces ()
911 _prog="$1"
912 _max="${2:-1}"
914 _count=1
915 for _pid in ${FAKE_NFSD_THREAD_PIDS:-$FAKE_RPC_THREAD_PIDS} ; do
916 [ $_count -le $_max ] || break
918 cat <<EOF
919 Stack trace for ${_prog}[${_pid}]:
920 [<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff
922 _count=$(($_count + 1))
923 done
926 guess_output ()
928 case "$1" in
929 $CTDB_NFS_CALLOUT\ start\ nlockmgr)
930 echo "&Starting nfslock: OK"
932 $CTDB_NFS_CALLOUT\ start\ nfs)
933 cat <<EOF
934 &Starting nfslock: OK
935 &Starting nfs: OK
939 : # Nothing
940 esac
943 # Set the required result for a particular RPC program having failed
944 # for a certain number of iterations. This is probably still a work
945 # in progress. Note that we could hook aggressively
946 # nfs_check_rpc_service() to try to implement this but we're better
947 # off testing nfs_check_rpc_service() using independent code... even
948 # if it is incomplete and hacky. So, if the 60.nfs eventscript
949 # changes and the tests start to fail then it may be due to this
950 # function being incomplete.
951 rpc_set_service_failure_response ()
953 _rpc_service="$1"
954 _numfails="${2:-1}" # default 1
956 # Default
957 ok_null
958 if [ $_numfails -eq 0 ] ; then
959 return
962 nfs_load_config
964 # A handy newline. :-)
965 _nl="
968 _dir="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}"
970 _file=$(ls "$_dir"/[0-9][0-9]."${_rpc_service}.check")
971 [ -r "$_file" ] || die "RPC check file \"$_file\" does not exist or is not unique"
973 _out=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
974 _rc_file=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
977 # Subshell to restrict scope variables...
979 # Defaults
980 family="tcp"
981 version=""
982 unhealthy_after=1
983 restart_every=0
984 service_stop_cmd=""
985 service_start_cmd=""
986 service_check_cmd=""
987 service_debug_cmd=""
989 # Don't bother syntax checking, eventscript does that...
990 . "$_file"
992 # Just use the first version, or use default. This is dumb but
993 # handles all the cases that we care about now...
994 if [ -n "$version" ] ; then
995 _ver="${version%% *}"
996 else
997 case "$_rpc_service" in
998 portmapper) _ver="" ;;
999 *) _ver=1 ;;
1000 esac
1002 _rpc_check_out="\
1003 $_rpc_service failed RPC check:
1004 rpcinfo: RPC: Program not registered
1005 program $_rpc_service${_ver:+ version }${_ver} is not available"
1007 if [ $unhealthy_after -gt 0 -a $_numfails -ge $unhealthy_after ] ; then
1008 _unhealthy=true
1009 echo 1 >"$_rc_file"
1010 echo "ERROR: ${_rpc_check_out}" >>"$_out"
1011 else
1012 _unhealthy=false
1013 echo 0 >"$_rc_file"
1016 if [ $restart_every -gt 0 -a $(($_numfails % $restart_every)) -eq 0 ] ; then
1017 if ! $_unhealthy ; then
1018 echo "WARNING: ${_rpc_check_out}" >>"$_out"
1021 echo "Trying to restart service \"${_rpc_service}\"..." >>"$_out"
1023 if [ -n "$service_debug_cmd" ] ; then
1024 $service_debug_cmd 2>&1 >>"$_out"
1027 guess_output "$service_start_cmd" >>"$_out"
1031 read _rc <"$_rc_file"
1032 required_result $_rc <"$_out"
1034 rm -f "$_out" "$_rc_file"
1037 ######################################################################
1039 # VSFTPD fakery
1041 setup_vsftpd ()
1043 service_name="vsftpd"
1045 if [ "$1" != "down" ] ; then
1046 die "setup_vsftpd up not implemented!!!"
1047 else
1048 debug "Setting up VSFTPD environment: service down, not managed by CTDB"
1050 eventscript_call ctdb_service_unmanaged
1051 service vsftpd force-stopped
1053 export CTDB_MANAGED_SERVICES="foo"
1054 unset CTDB_MANAGES_VSFTPD
1058 ######################################################################
1060 # HTTPD fakery
1062 setup_httpd ()
1064 if [ "$1" != "down" ] ; then
1065 die "setup_httpd up not implemented!!!"
1066 else
1067 debug "Setting up HTTPD environment: service down, not managed by CTDB"
1069 for service_name in "apache2" "httpd" ; do
1070 eventscript_call ctdb_service_unmanaged
1071 service "$service_name" force-stopped
1072 done
1074 export CTDB_MANAGED_SERVICES="foo"
1075 unset CTDB_MANAGES_HTTPD
1079 ######################################################################
1081 # multipathd fakery
1083 setup_multipathd ()
1085 for i ; do
1086 case "$i" in
1087 \!*)
1088 _t="${i#!}"
1089 echo "Marking ${_t} as having no active paths"
1090 FAKE_MULTIPATH_FAILURES="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}"
1093 _t="$i"
1094 esac
1095 CTDB_MONITOR_MPDEVICES="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}"
1096 done
1098 export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES
1099 export FAKE_SLEEP_FORCE=0.1
1102 ######################################################################
1104 # Result and test functions
1106 # Set some globals and print the summary.
1107 define_test ()
1109 desc="$1"
1111 _f=$(basename "$0" ".sh")
1113 # Remaining format should be NN.service.event.NNN or NN.service.NNN:
1114 _num="${_f##*.}"
1115 _f="${_f%.*}"
1117 case "$_f" in
1118 [0-9][0-9].*.*)
1119 script="${_f%.*}"
1120 event="${_f##*.}"
1121 script_dir="${CTDB_BASE}/events.d"
1123 [0-9][0-9].*)
1124 script="$_f"
1125 unset event
1126 script_dir="${CTDB_BASE}/events.d"
1128 *.*)
1129 script="${_f%.*}"
1130 event="${_f##*.}"
1131 script_dir="${CTDB_BASE}"
1134 script="${_f%.*}"
1135 unset event
1136 script_dir="${CTDB_BASE}"
1137 esac
1139 [ -r "${script_dir}/${script}" ] || \
1140 die "Internal error - unable to find script \"${script_dir}/${script}\""
1142 printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
1145 # Run an eventscript once. The test passes if the return code and
1146 # output match those required.
1148 # Any args are passed to the eventscript.
1150 simple_test ()
1152 [ -n "$event" ] || die 'simple_test: $event not set'
1154 args="$@"
1156 test_header ()
1158 echo "Running script \"$script $event${args:+ }$args\""
1161 extra_header ()
1163 cat <<EOF
1165 ##################################################
1166 CTDB_BASE="$CTDB_BASE"
1167 CTDB_ETCDIR="$CTDB_ETCDIR"
1168 ctdb client is "$(which ctdb)"
1169 ip command is "$(which ip)"
1173 script_test "${script_dir}/${script}" "$event" "$@"
1175 reset_test_header
1176 reset_extra_header
1179 simple_test_event ()
1181 # If something has previously failed then don't continue.
1182 : ${_passed:=true}
1183 $_passed || return 1
1185 event="$1" ; shift
1186 echo "=================================================="
1187 simple_test "$@"
1190 simple_test_command ()
1192 unit_test "$@"
1195 # Run an NFS eventscript iteratively.
1197 # - 1st argument is the number of iterations.
1199 # - 2nd argument is the NFS/RPC service being tested
1201 # rpcinfo (or $service_check_cmd) is used on each iteration to test
1202 # the availability of the service
1204 # If this is not set or null then no RPC service is checked and the
1205 # required output is not reset on each iteration. This is useful in
1206 # baseline tests to confirm that the eventscript and test
1207 # infrastructure is working correctly.
1209 # - Subsequent arguments come in pairs: an iteration number and
1210 # something to eval before that iteration. Each time an iteration
1211 # number is matched the associated argument is given to eval after
1212 # the default setup is done. The iteration numbers need to be given
1213 # in ascending order.
1215 # These arguments can allow a service to be started or stopped
1216 # before a particular iteration.
1218 nfs_iterate_test ()
1220 _repeats="$1"
1221 _rpc_service="$2"
1222 if [ -n "$2" ] ; then
1223 shift 2
1224 else
1225 shift
1228 echo "Running $_repeats iterations of \"$script $event\" $args"
1230 _iterate_failcount=0
1231 for _iteration in $(seq 1 $_repeats) ; do
1232 # This is not a numerical comparison because $1 will often not
1233 # be set.
1234 if [ "$_iteration" = "$1" ] ; then
1235 debug "##################################################"
1236 eval "$2"
1237 debug "##################################################"
1238 shift 2
1240 if [ -n "$_rpc_service" ] ; then
1241 _ok=false
1242 if [ -n "$service_check_cmd" ] ; then
1243 if eval "$service_check_cmd" ; then
1244 _ok=true
1246 else
1247 if rpcinfo -T tcp localhost "$_rpc_service" >/dev/null 2>&1 ; then
1248 _ok=true
1252 if $_ok ; then
1253 _iterate_failcount=0
1254 else
1255 _iterate_failcount=$(($_iterate_failcount + 1))
1257 rpc_set_service_failure_response "$_rpc_service" $_iterate_failcount
1259 _out=$(simple_test 2>&1)
1260 _ret=$?
1261 if "$TEST_VERBOSE" || [ $_ret -ne 0 ] ; then
1262 echo "##################################################"
1263 echo "Iteration ${_iteration}:"
1264 echo "$_out"
1266 if [ $_ret -ne 0 ] ; then
1267 exit $_ret
1269 done