1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 # Augment PATH with relevant stubs/ directories. We do this by actually
4 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
5 # prepending that to $PATH in rc.local to avoid the PATH reset in
10 if [ -d "${TEST_SUBDIR}/stubs" ] ; then
11 EVENTSCRIPTS_PATH
="${TEST_SUBDIR}/stubs"
12 case "$EVENTSCRIPTS_PATH" in
14 *) EVENTSCRIPTS_PATH
="${PWD}/${EVENTSCRIPTS_PATH}" ;;
18 export EVENTSCRIPTS_PATH
20 PATH
="${EVENTSCRIPTS_PATH}:${PATH}"
22 export EVENTSCRIPTS_TESTS_VAR_DIR
="${TEST_VAR_DIR}/unit_eventscripts"
23 if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \
24 "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then
25 rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
27 mkdir
-p "$EVENTSCRIPTS_TESTS_VAR_DIR"
28 export CTDB_VARDIR
="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
30 export CTDB_LOGGING
="file:${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb"
31 touch "${CTDB_LOGGING#file:}" || \
32 die
"Unable to setup logging for \"$CTDB_LOGGING\""
34 if [ -d "${TEST_SUBDIR}/etc" ] ; then
35 cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR"
36 export CTDB_ETCDIR
="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
38 die
"Unable to setup \$CTDB_ETCDIR"
41 if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then
42 cp -prL "${TEST_SUBDIR}/etc-ctdb" "$EVENTSCRIPTS_TESTS_VAR_DIR"
43 export CTDB_BASE
="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc-ctdb"
45 die
"Unable to set \$CTDB_BASE"
49 if [ ! -d "${CTDB_BASE}/events.d" ] ; then
51 ERROR: Directory ${CTDB_BASE}/events.d does not exist.
53 That means that no eventscripts can be tested.
55 One possible explanation:
57 You have CTDB installed via RPMs (or similar), so the regular
58 CTDB_BASE directory is in /etc/ctdb/
62 You have done a regular "configure" and "make install" so the tests
63 are installed under /usr/local/.
65 If so, one possible hack to fix this is to create a symlink:
67 ln -s /etc/ctdb /usr/local/etc/ctdb
69 This is nasty but it works... :-)
74 ######################################################################
76 if "$TEST_VERBOSE" ; then
77 debug
() { echo "$@" ; }
82 ######################################################################
84 # General setup fakery
88 debug
"Setting up shares (3 existing shares)"
89 # Create 3 fake shares/exports.
91 for i
in $
(seq 1 3) ; do
92 _s
="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing"
94 FAKE_SHARES
="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}"
97 export FAKE_PROC_NET_BONDING
="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding"
98 mkdir
-p "$FAKE_PROC_NET_BONDING"
99 rm -f "$FAKE_PROC_NET_BONDING"/*
101 export FAKE_ETHTOOL_LINK_DOWN
="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down"
102 mkdir
-p "$FAKE_ETHTOOL_LINK_DOWN"
103 rm -f "$FAKE_ETHTOOL_LINK_DOWN"/*
105 # This can only have 2 levels. We don't want to resort to usings
106 # something dangerous like "rm -r" setup time.
107 export FAKE_IP_STATE
="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state"
108 mkdir
-p "$FAKE_IP_STATE"
109 rm -f "$FAKE_IP_STATE"/*/*
110 rm -f "$FAKE_IP_STATE"/* 2>/dev
/null || true
111 rmdir "$FAKE_IP_STATE"/* 2>/dev
/null || true
114 export CTDB_DBDIR
="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
115 mkdir
-p "${CTDB_DBDIR}/persistent"
117 export FAKE_TDBTOOL_SUPPORTS_CHECK
="yes"
118 export FAKE_TDB_IS_OK
119 export FAKE_DATE_OUTPUT
121 export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
122 export FAKE_NETSTAT_TCP_ESTABLISHED_FILE
=$
(mktemp
--tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
128 debug
"Marking TCP port \"${_i}\" as not listening"
129 FAKE_TCP_LISTEN
=$
(echo "$FAKE_TCP_LISTEN" |
sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
137 # Replace some shares with non-existent ones.
142 export MISSING_SHARES_TEXT
=""
143 for _i
in $FAKE_SHARES ; do
144 if [ $_n = "$1" ] ; then
146 _i
="${_i%_existing}_missing"
147 debug
"Replacing share $_n with missing share \"$_i\""
148 rmdir "$_i" 2>/dev
/null || true
149 MISSING_SHARES_TEXT
="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}")
151 _t
="${_t}${_t:+ }${_i}"
157 # Setup some fake /proc/net/bonding files with just enough info for
160 # arg1 is interface name, arg2 is currently active slave (use "None"
161 # if none), arg3 is MII status ("up" or "down").
165 _slave
="${2:-${_iface}_sl_0}"
167 _mii_subs
="${4:-${_mii_s:-up}}"
168 echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s"
169 cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF
170 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
171 Currently Active Slave: $_slave
174 # Status of 1st pretend adapter
175 MII Status: $_mii_subs
176 # Status of 2nd pretend adapter
177 MII Status: $_mii_subs
181 ethtool_interfaces_down
()
184 echo "Marking interface $_i DOWN for ethtool"
185 touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
189 ethtool_interfaces_up
()
192 echo "Marking interface $_i UP for ethtool"
193 rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
199 echo "# ip rule show"
203 while read _p _x _i _x _t
; do
204 # Remove trailing colon after priority/preference.
206 # Only remove rules that match our priority/preference.
207 [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] ||
continue
209 echo "# ip route show table $_t"
210 ip route show table
"$_t"
214 # Copied from 13.per_ip_routing for now... so this is lazy testing :-(
215 ipv4_host_addr_to_net
()
220 # Convert the host address to an unsigned long by splitting out
221 # the octets and doing the math.
223 for _o
in $
(export IFS
="." ; echo $_host) ; do
224 _host_ul
=$
(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
227 # Calculate the mask and apply it.
228 _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
229 _net_ul=$(( $_host_ul & $_mask_ul ))
231 # Now convert to a network address one byte at a time.
233 for _o in $(seq 1 4) ; do
234 _net="$(($_net_ul & 255))${_net:+.}${_net}"
235 _net_ul=$(($_net_ul >> 8))
238 echo "${_net}/${_maskbits}"
241 ######################################################################
245 # Evaluate an expression that probably calls functions or uses
246 # variables from the CTDB functions file. This is used for test
251 .
"$CTDB_BASE/functions"
256 # For now this creates the same public addresses each time. However,
257 # it could be made more flexible.
258 setup_public_addresses
()
260 if [ -f "$CTDB_PUBLIC_ADDRESSES" -a \
261 "${CTDB_PUBLIC_ADDRESSES%/*}" = "$EVENTSCRIPTS_TESTS_VAR_DIR" ] ; then
262 rm "$CTDB_PUBLIC_ADDRESSES"
265 export CTDB_PUBLIC_ADDRESSES
=$
(mktemp \
266 --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR" \
267 "public-addresses-XXXXXXXX")
269 echo "Setting up CTDB_PUBLIC_ADDRESSES=${CTDB_PUBLIC_ADDRESSES}"
270 cat >"$CTDB_PUBLIC_ADDRESSES" <<EOF
283 # Need to cope with ctdb_get_pnn(). If a test changes PNN then it
284 # needs to be using a different state directory, otherwise the wrong
285 # PNN can already be cached in the state directory.
288 export FAKE_CTDB_PNN
="$1"
289 echo "Setting up PNN ${FAKE_CTDB_PNN}"
291 export CTDB_VARDIR
="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb/${FAKE_CTDB_PNN}"
292 mkdir
-p "$CTDB_VARDIR"
299 export FAKE_CTDB_NUMNODES
="${1:-3}"
300 echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes"
302 ctdb_set_pnn
"${2:-0}"
304 setup_public_addresses
306 export FAKE_CTDB_STATE
="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb"
308 export FAKE_CTDB_EXTRA_CONFIG
="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-config.sh"
309 rm -f "$FAKE_CTDB_EXTRA_CONFIG"
311 export FAKE_CTDB_IFACES_DOWN
="$FAKE_CTDB_STATE/ifaces-down"
312 mkdir
-p "$FAKE_CTDB_IFACES_DOWN"
313 rm -f "$FAKE_CTDB_IFACES_DOWN"/*
315 export FAKE_CTDB_SCRIPTSTATUS
="$FAKE_CTDB_STATE/scriptstatus"
316 mkdir
-p "$FAKE_CTDB_SCRIPTSTATUS"
317 rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
319 export CTDB_PARTIALLY_ONLINE_INTERFACES
321 export FAKE_CTDB_TUNABLES_OK
="MonitorInterval TDBMutexEnabled DatabaseHashSize"
322 export FAKE_CTDB_TUNABLES_OBSOLETE
="EventScriptUnhealthyOnTimeout"
327 cat >"$FAKE_CTDB_EXTRA_CONFIG"
334 _swap_total
="5857276"
336 if [ "$1" = "bad" ] ; then
341 _swap_free
="$_swap_total"
346 export FAKE_PROC_MEMINFO
="\
354 Active(anon): 1917580 kB
355 Inactive(anon): 523080 kB
356 Active(file): 504524 kB
357 Inactive(file): 496848 kB
360 SwapTotal: ${_swap_total} kB
361 SwapFree: ${_swap_free} kB
364 export FAKE_FREE_M
="\
365 total used free shared buffers cached
366 Mem: 3848 3634 213 0 142 ${_mem_cached}
367 -/+ buffers/cache: 2379 ${_mem_free}
370 export CTDB_MONITOR_FREE_MEMORY
371 export CTDB_MONITOR_FREE_MEMORY_WARN
372 export CTDB_CHECK_SWAP_IS_NOT_USED
375 ctdb_get_interfaces
()
377 # The echo/subshell forces all the output onto 1 line.
378 echo $
(ctdb ifaces
-X |
awk -F'|' 'FNR > 1 {print $2}')
381 ctdb_get_1_interface
()
383 _t
=$
(ctdb_get_interfaces
)
387 # Print all public addresses as: interface IP maskbits
388 # Each line is suitable for passing to takeip/releaseip
389 ctdb_get_all_public_addresses
()
391 _f
="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
392 while IFS
="/$IFS" read _ip _maskbits _ifaces
; do
393 echo "$_ifaces $_ip $_maskbits"
397 # Print public addresses on this node as: interface IP maskbits
398 # Each line is suitable for passing to takeip/releaseip
399 ctdb_get_my_public_addresses
()
402 read _x
# skip header line
404 while IFS
="|" read _x _ip _x _iface _x
; do
405 [ -n "$_iface" ] ||
continue
406 while IFS
="/$IFS" read _i _maskbits _x
; do
407 if [ "$_ip" = "$_i" ] ; then
408 echo $_iface $_ip $_maskbits
411 done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
416 # Prints the 1st public address as: interface IP maskbits
417 # This is suitable for passing to takeip/releaseip
418 ctdb_get_1_public_address
()
420 ctdb_get_my_public_addresses |
{ head -n 1 ; cat >/dev
/null
; }
423 ctdb_not_implemented
()
425 export CTDB_NOT_IMPLEMENTED
="$1"
426 ctdb_not_implemented
="\
427 DEBUG: ctdb: command \"$1\" not implemented in stub"
430 ctdb_fake_scriptstatus
()
439 echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
442 ######################################################################
444 setup_ctdb_policy_routing
()
446 service_name
="per_ip_routing"
448 export CTDB_PER_IP_ROUTING_CONF
="$CTDB_BASE/policy_routing"
449 export CTDB_PER_IP_ROUTING_RULE_PREF
=100
450 export CTDB_PER_IP_ROUTING_TABLE_ID_LOW
=1000
451 export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH
=2000
453 # Tests need to create and populate this file
454 rm -f "$CTDB_PER_IP_ROUTING_CONF"
457 # Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF.
458 # $1 is the number of assigned IPs to use (<num>, all), defaulting to
459 # 1. If $2 is "default" then a default route is also added.
460 create_policy_routing_config
()
463 _should_add_default
="$2"
465 ctdb_get_my_public_addresses |
466 if [ "$_num_ips" = "all" ] ; then
469 { head -n "$_num_ips" ; cat >/dev
/null
; }
471 while read _dev _ip _bits
; do
472 _net
=$
(ipv4_host_addr_to_net
"$_ip" "$_bits")
473 _gw
="${_net%.*}.1" # a dumb, calculated default
477 if [ "$_should_add_default" = "default" ] ; then
478 echo "$_ip 0.0.0.0/0 $_gw"
480 done >"$CTDB_PER_IP_ROUTING_CONF"
483 # Check the routes against those that are expected. $1 is the number
484 # of assigned IPs to use (<num>, all), defaulting to 1. If $2 is
485 # "default" then expect default routes to have been added.
489 _should_add_default
="$2"
494 ctdb_get_my_public_addresses |
495 if [ "$_num_ips" = "all" ] ; then
498 { head -n "$_num_ips" ; cat >/dev
/null
; }
500 while read _dev _ip _bits
; do
501 _net
=$
(ipv4_host_addr_to_net
"$_ip" "$_bits")
502 _gw
="${_net%.*}.1" # a dumb, calculated default
504 _policy_rules
="${_policy_rules}
505 ${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip "
506 _policy_routes
="${_policy_routes}
507 # ip route show table ctdb.$_ip
508 $_net dev $_dev scope link "
510 if [ "$_should_add_default" = "default" ] ; then
511 _policy_routes
="${_policy_routes}
512 default via $_gw dev $_dev "
518 0: from all lookup local ${_policy_rules}
519 32766: from all lookup main
520 32767: from all lookup default ${_policy_routes}
523 simple_test_command dump_routes
527 ######################################################################
529 ctdb_catdb_format_pairs
()
533 while read _k _v
; do
534 _kn
=$
(echo -n "$_k" |
wc -c)
535 _vn
=$
(echo -n "$_v" |
wc -c)
537 key(${_kn}) = "${_k}"
540 data(${_vn}) = "${_v}"
543 _count
=$
(($_count + 1))
546 echo "Dumped ${_count} records"
549 check_ctdb_tdb_statd_state
()
551 ctdb_get_my_public_addresses |
552 while read _x _sip _x
; do
554 echo "statd-state@${_sip}@${_cip}" "$FAKE_DATE_OUTPUT"
557 ctdb_catdb_format_pairs |
{
559 simple_test_command ctdb catdb ctdb.tdb
563 check_statd_callout_smnotify
()
565 _state_even
=$
(( $
(date '+%s') / 2 * 2))
566 _state_odd
=$
(($_state_even + 1))
570 ctdb_get_my_public_addresses |
571 while read _x _sip _x
; do
574 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_even}
575 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_even}
576 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_odd}
577 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_odd}
582 simple_test_event
"notify"
586 ######################################################################
590 debug
"Setting up NAT gateway"
592 natgw_config_dir
="${TEST_VAR_DIR}/natgw_config"
593 mkdir
-p "$natgw_config_dir"
595 # These will accumulate, 1 per test... but will be cleaned up at
597 export CTDB_NATGW_NODES
=$
(mktemp
--tmpdir="$natgw_config_dir")
600 while read _ip _master _dev
; do
602 if [ "$_master" = "master" ] ; then
603 export FAKE_CTDB_NATGW_MASTER
="$_ip"
605 done >"$CTDB_NATGW_NODES"
607 # Assume all of the nodes are on a /24 network and have IPv4
609 read _ip
<"$CTDB_NATGW_NODES"
610 export CTDB_NATGW_PRIVATE_NETWORK
="${_ip%.*}.0/24"
612 # These are fixed. Probably don't use the same network for the
613 # private node IPs. To unset the default gateway just set it to
615 export CTDB_NATGW_PUBLIC_IP
="10.1.1.121/24"
616 export CTDB_NATGW_PUBLIC_IFACE
="eth1"
617 export CTDB_NATGW_DEFAULT_GATEWAY
="10.1.1.254"
618 export CTDB_NATGW_SLAVE_ONLY
=""
621 ok_natgw_master_ip_addr_show
()
623 _mac
=$
(echo "$CTDB_NATGW_PUBLIC_IFACE" |
md5sum |
sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
625 # This is based on CTDB_NATGW_PUBLIC_IP
629 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
630 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
631 inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE}
632 valid_lft forever preferred_lft forever
636 ok_natgw_slave_ip_addr_show
()
638 _mac
=$
(echo "$CTDB_NATGW_PUBLIC_IFACE" |
md5sum |
sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
640 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
641 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
645 ok_natgw_master_static_routes
()
650 for _i
in $CTDB_NATGW_STATIC_ROUTES ; do
651 # This is intentionally different to the code in 11.natgw ;-)
654 _net
=$
(echo "$_i" |
sed -e 's|@.*||')
655 _gw
=$
(echo "$_i" |
sed -e 's|.*@||')
659 _gw
="$CTDB_NATGW_DEFAULT_GATEWAY"
662 [ -n "$_gw" ] ||
continue
663 _t
="${_t}${_t:+${_nl}}"
664 _t
="${_t}${_net} via ${_gw} dev ethXXX metric 10 "
666 _t
=$
(echo "$_t" |
sort)
670 ok_natgw_slave_static_routes
()
675 for _i
in $CTDB_NATGW_STATIC_ROUTES ; do
676 # This is intentionally different to the code in 11.natgw ;-)
677 _net
=$
(echo "$_i" |
sed -e 's|@.*||')
679 # The interface for the private network isn't specified as
680 # part of the NATGW configuration and isn't part of the
681 # command to add the route. It is implicitly added by "ip
682 # route" but our stub doesn't do this and adds "ethXXX".
683 _t
="${_t}${_t:+${_nl}}"
684 _t
="${_t}${_net} via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX metric 10 "
686 _t
=$
(echo "$_t" |
sort)
690 ######################################################################
692 # Samba/winbind fakery
700 if [ "$1" != "down" ] ; then
702 debug
"Marking Samba services as up, listening and managed by CTDB"
703 # Get into known state.
704 eventscript_call ctdb_service_managed
706 # All possible service names for all known distros.
707 for i
in "smb" "nmb" "samba" ; do
708 service
"$i" force-started
711 export CTDB_SAMBA_SKIP_SHARE_CHECK
="no"
712 export CTDB_MANAGED_SERVICES
="foo samba bar"
714 export FAKE_TCP_LISTEN
="0.0.0.0:445 0.0.0.0:139"
715 export FAKE_WBINFO_FAIL
="no"
717 # Some things in 50.samba are backgrounded and waited for. If
718 # we don't sleep at all then timeouts can happen. This avoids
720 export FAKE_SLEEP_FORCE
=0.1
722 debug
"Marking Samba services as down, not listening and not managed by CTDB"
723 # Get into known state.
724 eventscript_call ctdb_service_unmanaged
726 # All possible service names for all known distros.
727 for i
in "smb" "nmb" "samba" ; do
728 service
"$i" force-stopped
731 export CTDB_SAMBA_SKIP_SHARE_CHECK
="no"
732 export CTDB_MANAGED_SERVICES
="foo bar"
733 unset CTDB_MANAGES_SAMBA
735 export FAKE_TCP_LISTEN
=""
736 export FAKE_WBINFO_FAIL
="yes"
744 service_name
="winbind"
746 if [ "$1" != "down" ] ; then
748 debug
"Marking Winbind service as up and managed by CTDB"
749 # Get into known state.
750 eventscript_call ctdb_service_managed
752 service
"winbind" force-started
754 export CTDB_MANAGED_SERVICES
="foo winbind bar"
756 export FAKE_WBINFO_FAIL
="no"
759 debug
"Marking Winbind service as down and not managed by CTDB"
760 # Get into known state.
761 eventscript_call ctdb_service_unmanaged
763 service
"winbind" force-stopped
765 export CTDB_MANAGED_SERVICES
="foo bar"
766 unset CTDB_MANAGES_WINBIND
768 export FAKE_WBINFO_FAIL
="yes"
774 debug
"Making wbinfo commands fail"
775 FAKE_WBINFO_FAIL
="yes"
778 ######################################################################
788 export FAKE_RPCINFO_SERVICES
=""
790 export CTDB_NFS_SKIP_SHARE_CHECK
="no"
793 export CTDB_NFS_DUMP_STUCK_THREADS
795 # Reset the failcounts for nfs services.
796 eventscript_call
eval rm -f '$ctdb_fail_dir/nfs_*'
798 if [ "$1" != "down" ] ; then
799 debug
"Setting up NFS environment: all RPC services up, NFS managed by CTDB"
801 eventscript_call ctdb_service_managed
802 service
"nfs" force-started
803 service
"nfslock" force-started
805 export CTDB_MANAGED_SERVICES
="foo nfs bar"
807 rpc_services_up
"nfs" "mountd" "rquotad" "nlockmgr" "status"
809 nfs_setup_fake_threads
"nfsd"
810 nfs_setup_fake_threads
"rpc.foobar" # Just set the variable to empty
812 debug
"Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
814 eventscript_call ctdb_service_unmanaged
815 service
"nfs" force-stopped
816 service
"nfslock" force-stopped
818 export CTDB_MANAGED_SERVICES
="foo bar"
819 unset CTDB_MANAGES_NFS
822 # This is really nasty. However, when we test NFS we don't
823 # actually test statd-callout. If we leave it there then left
824 # over, backgrounded instances of statd-callout will do horrible
825 # things with the "ctdb ip" stub and cause the actual
826 # statd-callout tests that follow to fail.
827 rm "${CTDB_BASE}/statd-callout"
833 export CTDB_NFS_SERVER_MODE
="ganesha"
834 if [ "$1" != "down" ] ; then
835 export CTDB_MANAGES_NFS
="yes"
838 # We do not support testing the Ganesha-nfsd-specific part of the
840 export CTDB_SKIP_GANESHA_NFSD_CHECK
="yes"
841 export CTDB_NFS_SKIP_SHARE_CHECK
="yes"
847 debug
"Marking RPC service \"${_i}\" as unavailable"
848 FAKE_RPCINFO_SERVICES
=$
(echo "$FAKE_RPCINFO_SERVICES" |
sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
855 debug
"Marking RPC service \"${_i}\" as available"
860 nlockmgr
) _t
="3:4" ;;
862 *) die
"Internal error - unsupported RPC service \"${_i}\"" ;;
865 FAKE_RPCINFO_SERVICES
="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
872 _etc
="$CTDB_ETCDIR" # shortcut for readability
873 for _c
in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
874 if [ -r "$_c" ] ; then
881 nfs_setup_fake_threads
()
887 export PROCFS_PATH
=$
(mktemp
-d --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
888 _threads
="${PROCFS_PATH}/fs/nfsd/threads"
889 mkdir
-p $
(dirname "$_threads")
891 export FAKE_NFSD_THREAD_PIDS
="$*"
894 export FAKE_RPC_THREAD_PIDS
="$*"
899 program_stack_traces
()
905 for _pid
in ${FAKE_NFSD_THREAD_PIDS:-$FAKE_RPC_THREAD_PIDS} ; do
906 [ $_count -le $_max ] ||
break
909 Stack trace for ${_prog}[${_pid}]:
910 [<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff
912 _count
=$
(($_count + 1))
919 $CTDB_NFS_CALLOUT\ start\ nlockmgr
)
920 echo "&Starting nfslock: OK"
922 $CTDB_NFS_CALLOUT\ start\ nfs
)
924 &Starting nfslock: OK
933 # Set the required result for a particular RPC program having failed
934 # for a certain number of iterations. This is probably still a work
935 # in progress. Note that we could hook aggressively
936 # nfs_check_rpc_service() to try to implement this but we're better
937 # off testing nfs_check_rpc_service() using independent code... even
938 # if it is incomplete and hacky. So, if the 60.nfs eventscript
939 # changes and the tests start to fail then it may be due to this
940 # function being incomplete.
941 rpc_set_service_failure_response
()
944 _numfails
="${2:-1}" # default 1
948 if [ $_numfails -eq 0 ] ; then
954 # A handy newline. :-)
958 _dir
="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}"
960 _file
=$
(ls "$_dir"/[0-9][0-9].
"${_rpc_service}.check")
961 [ -r "$_file" ] || die
"RPC check file \"$_file\" does not exist or is not unique"
963 _out
=$
(mktemp
--tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
964 _rc_file
=$
(mktemp
--tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
967 # Subshell to restrict scope variables...
978 # Don't bother syntax checking, eventscript does that...
981 # Just use the first version, default to 1. This is dumb but
982 # handles all the cases that we care about now...
983 if [ -n "$version" ] ; then
984 _ver
="${version%% *}"
989 $_rpc_service failed RPC check:
990 rpcinfo: RPC: Program not registered
991 program $_rpc_service version $_ver is not available"
993 if [ $unhealthy_after -gt 0 -a $_numfails -ge $unhealthy_after ] ; then
996 echo "ERROR: ${_rpc_check_out}" >>"$_out"
1002 if [ $restart_every -gt 0 -a $
(($_numfails % $restart_every)) -eq 0 ] ; then
1003 if ! $_unhealthy ; then
1004 echo "WARNING: ${_rpc_check_out}" >>"$_out"
1007 echo "Trying to restart service \"${_rpc_service}\"..." >>"$_out"
1009 if [ -n "$service_debug_cmd" ] ; then
1010 $service_debug_cmd 2>&1 >>"$_out"
1013 guess_output
"$service_start_cmd" >>"$_out"
1017 read _rc
<"$_rc_file"
1018 required_result
$_rc <"$_out"
1020 rm -f "$_out" "$_rc_file"
1023 ######################################################################
1029 service_name
="vsftpd"
1031 if [ "$1" != "down" ] ; then
1032 die
"setup_vsftpd up not implemented!!!"
1034 debug
"Setting up VSFTPD environment: service down, not managed by CTDB"
1036 eventscript_call ctdb_service_unmanaged
1037 service vsftpd force-stopped
1039 export CTDB_MANAGED_SERVICES
="foo"
1040 unset CTDB_MANAGES_VSFTPD
1044 ######################################################################
1050 if [ "$1" != "down" ] ; then
1051 die
"setup_httpd up not implemented!!!"
1053 debug
"Setting up HTTPD environment: service down, not managed by CTDB"
1055 for service_name
in "apache2" "httpd" ; do
1056 eventscript_call ctdb_service_unmanaged
1057 service
"$service_name" force-stopped
1060 export CTDB_MANAGED_SERVICES
="foo"
1061 unset CTDB_MANAGES_HTTPD
1065 ######################################################################
1075 echo "Marking ${_t} as having no active paths"
1076 FAKE_MULTIPATH_FAILURES
="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}"
1081 CTDB_MONITOR_MPDEVICES
="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}"
1084 export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES
1085 export FAKE_SLEEP_FORCE
=0.1
1088 ######################################################################
1090 # Result and test functions
1092 # Set some globals and print the summary.
1097 _f
=$
(basename "$0" ".sh")
1099 # Remaining format should be NN.service.event.NNN or NN.service.NNN:
1107 script_dir
="${CTDB_BASE}/events.d"
1112 script_dir
="${CTDB_BASE}/events.d"
1117 script_dir
="${CTDB_BASE}"
1122 script_dir
="${CTDB_BASE}"
1125 [ -r "${script_dir}/${script}" ] || \
1126 die
"Internal error - unable to find script \"${script_dir}/${script}\""
1128 printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
1131 # Run an eventscript once. The test passes if the return code and
1132 # output match those required.
1134 # Any args are passed to the eventscript.
1138 [ -n "$event" ] || die
'simple_test: $event not set'
1144 echo "Running script \"$script $event${args:+ }$args\""
1151 ##################################################
1152 CTDB_BASE="$CTDB_BASE"
1153 CTDB_ETCDIR="$CTDB_ETCDIR"
1154 ctdb client is "$(which ctdb)"
1155 ip command is "$(which ip)"
1159 script_test
"${script_dir}/${script}" "$event" "$@"
1165 simple_test_event
()
1167 # If something has previously failed then don't continue.
1169 $_passed ||
return 1
1172 echo "=================================================="
1176 simple_test_command
()
1181 # Run an NFS eventscript iteratively.
1183 # - 1st argument is the number of iterations.
1185 # - 2nd argument is the NFS/RPC service being tested
1187 # rpcinfo is used on each iteration to test the availability of the
1190 # If this is not set or null then no RPC service is checked and the
1191 # required output is not reset on each iteration. This is useful in
1192 # baseline tests to confirm that the eventscript and test
1193 # infrastructure is working correctly.
1195 # - Subsequent arguments come in pairs: an iteration number and
1196 # something to eval before that iteration. Each time an iteration
1197 # number is matched the associated argument is given to eval after
1198 # the default setup is done. The iteration numbers need to be given
1199 # in ascending order.
1201 # These arguments can allow a service to be started or stopped
1202 # before a particular iteration.
1208 if [ -n "$2" ] ; then
1214 echo "Running $_repeats iterations of \"$script $event\" $args"
1216 _iterate_failcount
=0
1217 for _iteration
in $
(seq 1 $_repeats) ; do
1218 # This is not a numerical comparison because $1 will often not
1220 if [ "$_iteration" = "$1" ] ; then
1221 debug
"##################################################"
1223 debug
"##################################################"
1226 if [ -n "$_rpc_service" ] ; then
1227 if rpcinfo
-T tcp localhost
"$_rpc_service" >/dev
/null
2>&1 ; then
1228 _iterate_failcount
=0
1230 _iterate_failcount
=$
(($_iterate_failcount + 1))
1232 rpc_set_service_failure_response
"$_rpc_service" $_iterate_failcount
1234 _out
=$
(simple_test
2>&1)
1236 if "$TEST_VERBOSE" ||
[ $_ret -ne 0 ] ; then
1237 echo "##################################################"
1238 echo "Iteration ${_iteration}:"
1241 if [ $_ret -ne 0 ] ; then