1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 .
"${TEST_SCRIPTS_DIR}/common.sh"
5 ######################################################################
9 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then
10 CTDB_TEST_WRAPPER
="${CTDB_TEST_REMOTE_DIR}/test_wrap"
12 _d
=$
(cd ${TEST_SCRIPTS_DIR}; echo $PWD)
13 CTDB_TEST_WRAPPER
="$_d/test_wrap"
15 export CTDB_TEST_WRAPPER
17 # If $VALGRIND is set then use it whenever ctdb is called, but only if
18 # $CTDB is not already set.
19 [ -n "$CTDB" ] ||
export CTDB
="${VALGRIND}${VALGRIND:+ }ctdb"
22 PATH
="${TEST_SCRIPTS_DIR}:${PATH}"
24 ######################################################################
32 [ $
(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures
=$status
33 status
=$
(($testfailures+0))
35 # Avoid making a test fail from this point onwards. The test is
39 echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..."
41 eval "$ctdb_test_exit_hook" || true
42 unset ctdb_test_exit_hook
44 if $ctdb_test_restart_scheduled ||
! cluster_is_healthy
; then
48 # This could be made unconditional but then we might get
49 # duplication from the recovery in restart_ctdb. We want to
50 # leave the recovery in restart_ctdb so that future tests that
51 # might do a manual restart mid-test will benefit.
52 echo "Forcing a recovery..."
53 onnode
0 $CTDB recover
59 ctdb_test_exit_hook_add
()
61 ctdb_test_exit_hook
="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*"
66 scriptname
=$
(basename "$0")
68 ctdb_test_restart_scheduled
=false
70 trap "ctdb_test_exit" 0
73 ########################################
76 try_command_on_node
()
78 local nodespec
="$1" ; shift
83 while [ "${nodespec#-}" != "$nodespec" ] ; do
84 if [ "$nodespec" = "-v" ] ; then
87 onnode_opts
="${onnode_opts}${onnode_opts:+ }${nodespec}"
94 out
=$
(onnode
-q $onnode_opts "$nodespec" "$cmd" 2>&1) ||
{
96 echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
102 echo "Output of \"$cmd\":"
107 sanity_check_output
()
110 local regexp
="$2" # Should be anchored as necessary.
115 local num_lines
=$
(echo "$output" |
wc -l)
116 echo "There are $num_lines lines of output"
117 if [ $num_lines -lt $min_lines ] ; then
118 echo "BAD: that's less than the required number (${min_lines})"
123 local unexpected
# local doesn't pass through status of command on RHS.
124 unexpected
=$
(echo "$output" |
egrep -v "$regexp") || status
=$?
126 # Note that this is reversed.
127 if [ $status -eq 0 ] ; then
128 echo "BAD: unexpected lines in output:"
129 echo "$unexpected" |
cat -A
132 echo "Output lines look OK"
140 local ips
="$1" # list of "ip node" lines
142 echo "Sanity checking IPs..."
146 while read x ipp
; do
147 [ "$ipp" = "-1" ] && break
148 if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
155 echo "BAD: a node was -1 or IPs are only assigned to one node:"
157 echo "Are you running an old version of CTDB?"
161 # This returns a list of "ip node" lines in $out
165 try_command_on_node
$node \
166 "$CTDB ip -X | awk -F'|' 'NR > 1 { print \$2, \$3 }'"
169 _select_test_node_and_ips
()
171 try_command_on_node any \
172 "$CTDB ip -X -n all | awk -F'|' 'NR > 1 { print \$2, \$3 }'"
174 test_node
="" # this matches no PNN
177 while read ip pnn
; do
178 if [ -z "$test_node" -a "$pnn" != "-1" ] ; then
181 if [ "$pnn" = "$test_node" ] ; then
182 test_node_ips
="${test_node_ips}${test_node_ips:+ }${ip}"
184 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
186 echo "Selected node ${test_node} with IPs: ${test_node_ips}."
187 test_ip
="${test_node_ips%% *}"
190 *:*) test_prefix
="${test_ip}/128" ;;
191 *) test_prefix
="${test_ip}/32" ;;
194 [ -n "$test_node" ] ||
return 1
197 select_test_node_and_ips
()
200 while ! _select_test_node_and_ips
; do
201 echo "Unable to find a test node with IPs assigned"
202 if [ $timeout -le 0 ] ; then
203 echo "BAD: Too many attempts"
207 timeout
=$
(($timeout - 1))
214 get_test_ip_mask_and_iface
()
217 try_command_on_node
$test_node "$CTDB ip -v -X | awk -F'|' -v ip=$test_ip '\$2 == ip { print \$4 }'"
220 if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
222 try_command_on_node
$test_node ip addr show to
$test_ip
229 echo "$test_ip/$mask is on $iface"
232 #######################################
234 # Wait until either timeout expires or command succeeds. The command
235 # will be tried once per second, unless timeout has format T/I, where
236 # I is the recheck interval.
239 local timeout
="$1" ; shift # "$@" is the command...
244 interval
="${timeout#*/}"
245 timeout
="${timeout%/*}"
249 if [ "$1" = "!" ] ; then
254 echo -n "<${timeout}|"
256 while [ $t -gt 0 ] ; do
259 if { ! $negate && [ $rc -eq 0 ] ; } || \
260 { $negate && [ $rc -ne 0 ] ; } ; then
261 echo "|$(($timeout - $t))|"
266 for i
in $
(seq 1 $interval) ; do
269 t
=$
(($t - $interval))
281 for i
in $
(seq 1 $1) ; do
288 _cluster_is_healthy
()
290 $CTDB nodestatus all
>/dev
/null
293 _cluster_is_recovered
()
295 node_has_status all recovered
300 _cluster_is_healthy
&& _cluster_is_recovered
303 cluster_is_healthy
()
305 if onnode
0 $CTDB_TEST_WRAPPER _cluster_is_healthy
; then
306 echo "Cluster is HEALTHY"
307 if ! onnode
0 $CTDB_TEST_WRAPPER _cluster_is_recovered
; then
308 echo "WARNING: cluster in recovery mode!"
312 echo "Cluster is UNHEALTHY"
313 if ! ${ctdb_test_restart_scheduled:-false} ; then
314 echo "DEBUG AT $(date '+%F %T'):"
316 for i
in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do
327 local timeout
="${1:-120}"
329 echo "Waiting for cluster to become ready..."
331 wait_until
$timeout onnode
-q any
$CTDB_TEST_WRAPPER _cluster_is_ready
334 # This function is becoming nicely overloaded. Soon it will collapse! :-)
340 local bits fpat mpat rpat
342 (unhealthy
) bits
="?|?|?|1|*" ;;
343 (healthy
) bits
="?|?|?|0|*" ;;
344 (disconnected
) bits
="1|*" ;;
345 (connected
) bits
="0|*" ;;
346 (banned
) bits
="?|1|*" ;;
347 (unbanned
) bits
="?|0|*" ;;
348 (disabled
) bits
="?|?|1|*" ;;
349 (enabled
) bits
="?|?|0|*" ;;
350 (stopped
) bits
="?|?|?|?|1|*" ;;
351 (notstopped
) bits
="?|?|?|?|0|*" ;;
352 (frozen
) fpat
='^[[:space:]]+frozen[[:space:]]+1$' ;;
353 (unfrozen
) fpat
='^[[:space:]]+frozen[[:space:]]+0$' ;;
354 (monon
) mpat
='^Monitoring mode:ACTIVE \(0\)$' ;;
355 (monoff
) mpat
='^Monitoring mode:DISABLED \(1\)$' ;;
356 (recovered
) rpat
='^Recovery mode:RECOVERY \(1\)$' ;;
358 echo "node_has_status: unknown status \"$status\""
362 if [ -n "$bits" ] ; then
365 out
=$
($CTDB -X status
2>&1) ||
return 1
370 # This needs to be done in 2 steps to avoid false matches.
371 local line_bits
="${line#|${pnn}|*|}"
372 [ "$line_bits" = "$line" ] && continue
373 [ "${line_bits#${bits}}" != "$line_bits" ] && return 0
376 } <<<"$out" # Yay bash!
377 elif [ -n "$fpat" ] ; then
378 $CTDB statistics
-n "$pnn" |
egrep -q "$fpat"
379 elif [ -n "$mpat" ] ; then
380 $CTDB getmonmode
-n "$pnn" |
egrep -q "$mpat"
381 elif [ -n "$rpat" ] ; then
382 ! $CTDB status
-n "$pnn" |
egrep -q "$rpat"
384 echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
389 wait_until_node_has_status
()
393 local timeout
="${3:-30}"
394 local proxy_pnn
="${4:-any}"
396 echo "Waiting until node $pnn has status \"$status\"..."
398 if ! wait_until
$timeout onnode
$proxy_pnn $CTDB_TEST_WRAPPER node_has_status
"$pnn" "$status" ; then
399 for i
in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do
409 # Useful for superficially testing IP failover.
410 # IPs must be on the given node.
411 # If the first argument is '!' then the IPs must not be on the given node.
415 if [ "$1" = "!" ] ; then
416 negating
=true
; shift
418 local node
="$1" ; shift
423 all_ips_on_node
$node
426 for check
in $ips ; do
428 while read ip pnn
; do
429 if [ "$check" = "$ip" ] ; then
430 if [ "$pnn" = "$node" ] ; then
431 if $negating ; then return 1 ; fi
433 if ! $negating ; then return 1 ; fi
435 ips
="${ips/${ip}}" # Remove from list
438 # If we're negating and we didn't see the address then it
439 # isn't hosted by anyone!
441 ips
="${ips/${check}}"
443 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
446 ips
="${ips// }" # Remove any spaces.
450 wait_until_ips_are_on_node
()
452 # Go to some trouble to print a use description of what is happening
454 if [ "$1" == "!" ] ; then
461 [ "$i" != "!" ] ||
continue
462 if [ -z "$node" ] ; then
466 ips
="${ips}${ips:+, }${i}"
468 echo "Waiting for ${ips} to ${not}be assigned to node ${node}"
470 wait_until
60 ips_are_on_node
"$@"
479 all_ips_on_node
$node
481 while read ip pnn
; do
482 if [ "$node" = "$pnn" ] ; then
485 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
490 wait_until_node_has_some_ips
()
492 echo "Waiting for some IPs to be assigned to node ${test_node}"
494 wait_until
60 node_has_some_ips
"$@"
497 #######################################
499 _ctdb_hack_options
()
501 local ctdb_options
="$*"
503 case "$ctdb_options" in
504 *--start-as-stopped*)
505 export CTDB_START_AS_STOPPED
="yes"
511 _ctdb_hack_options
"$@"
513 if [ -e /etc
/redhat-release
] ; then
516 /etc
/init.d
/ctdb restart
520 # Restart CTDB on all nodes. Override for local daemons.
523 onnode
-p all
$CTDB_TEST_WRAPPER restart_ctdb_1
"$@"
526 # Nothing needed for a cluster. Override for local daemons.
534 # "$@" is passed to restart_ctdb_all.
536 echo -n "Restarting CTDB"
537 if $ctdb_test_restart_scheduled ; then
538 echo -n " (scheduled)"
543 for i
in $
(seq 1 5) ; do
544 _restart_ctdb_all
"$@" ||
{
545 echo "Restart failed. Trying again in a few seconds..."
550 wait_until_ready ||
{
551 echo "Cluster didn't become ready. Restarting..."
555 echo "Setting RerecoveryTimeout to 1"
556 onnode
-pq all
"$CTDB setvar RerecoveryTimeout 1"
558 # In recent versions of CTDB, forcing a recovery like this
559 # blocks until the recovery is complete. Hopefully this will
560 # help the cluster to stabilise before a subsequent test.
561 echo "Forcing a recovery..."
562 onnode
-q 0 $CTDB recover
565 if ! onnode
-q any
$CTDB_TEST_WRAPPER _cluster_is_recovered
; then
566 echo "Cluster has gone into recovery again, waiting..."
567 wait_until
30/2 onnode
-q any
$CTDB_TEST_WRAPPER _cluster_is_recovered
571 # Cluster is still healthy. Good, we're done!
572 if ! onnode
0 $CTDB_TEST_WRAPPER _cluster_is_healthy
; then
573 echo "Cluster became UNHEALTHY again [$(date)]"
574 onnode
-p all ctdb status
-X 2>&1
575 onnode
-p all ctdb scriptstatus
2>&1
580 echo "Doing a sync..."
581 onnode
-q 0 $CTDB sync
587 echo "Cluster UNHEALTHY... too many attempts..."
588 onnode
-p all ctdb status
-X 2>&1
589 onnode
-p all ctdb scriptstatus
2>&1
591 # Try to make the calling test fail
596 # Does nothing on cluster and should be overridden for local daemons
602 ctdb_restart_when_done
()
604 ctdb_test_restart_scheduled
=true
607 get_ctdbd_command_line_option
()
612 try_command_on_node
"$pnn" "$CTDB getpid" || \
613 die
"Unable to get PID of ctdbd on node $pnn"
615 local pid
="${out#*:}"
616 try_command_on_node
"$pnn" "ps -p $pid -o args hww" || \
617 die
"Unable to get command-line of PID $pid"
619 # Strip everything up to and including --option
620 local t
="${out#*--${option}}"
621 # Strip leading '=' or space if present
624 # Strip any following options and print
628 #######################################
630 wait_for_monitor_event
()
635 echo "Waiting for a monitor event on node ${pnn}..."
637 try_command_on_node
"$pnn" $CTDB scriptstatus ||
{
638 echo "Unable to get scriptstatus from node $pnn"
642 local ctdb_scriptstatus_original
="$out"
643 wait_until
120 _ctdb_scriptstatus_changed
646 _ctdb_scriptstatus_changed
()
648 try_command_on_node
"$pnn" $CTDB scriptstatus ||
{
649 echo "Unable to get scriptstatus from node $pnn"
653 [ "$out" != "$ctdb_scriptstatus_original" ]
656 #######################################
660 select_test_node_and_ips
662 nfs_first_export
=$
(showmount
-e $test_ip |
sed -n -e '2s/ .*//p')
664 echo "Creating test subdirectory..."
665 try_command_on_node
$test_node "mktemp -d --tmpdir=$nfs_first_export"
667 try_command_on_node
$test_node "chmod 777 $nfs_test_dir"
669 nfs_mnt_d
=$
(mktemp
-d)
670 nfs_local_file
="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE"
671 nfs_remote_file
="${nfs_test_dir}/TEST_FILE"
673 ctdb_test_exit_hook_add nfs_test_cleanup
675 echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..."
676 mount
-o timeo
=1,hard
,intr
,vers
=3 \
677 "[${test_ip}]:${nfs_first_export}" ${nfs_mnt_d}
682 rm -f "$nfs_local_file"
683 umount -f "$nfs_mnt_d"
685 onnode -q $test_node rmdir "$nfs_test_dir"
688 #######################################
690 # If the given IP is hosted then print 2 items: maskbits and iface
696 *:*) _family="inet6
" ; _bits=128 ;;
697 *) _family="inet
" ; _bits=32 ;;
700 ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
701 awk -v family="${_family}" \
702 'NR == 1 { iface = $2; sub(":$
", "", iface) } \
703 $1 ~ /inet/ { mask = $2; sub(".
*/", "", mask); \
704 print mask, iface, family }'
709 _addr="${1%/*}" # Remove optional maskbits
711 set -- $(ip_maskbits_iface $_addr)
712 if [ -n "$1" ] ; then
715 echo "Removing public address
$_addr/$_maskbits from device
$_iface"
716 ip addr del "$_ip/$_maskbits" dev "$_iface" >/dev/null 2>&1 || true
727 #######################################
729 # $1: pnn, $2: DB name
732 try_command_on_node -v $1 $CTDB getdbstatus "$2" |
733 sed -n -e "s@^path
: @@p
"
736 # $1: pnn, $2: DB name
737 db_ctdb_cattdb_count_records ()
739 try_command_on_node -v $1 $CTDB cattdb "$2" |
740 grep '^key' | grep -v '__db_sequence_number__' |
744 # $1: pnn, $2: DB name, $3: key string, $4: value string, $5: RSN (default 7)
747 _tdb=$(db_get_path $1 "$2")
749 try_command_on_node $1 $CTDB tstore "$_tdb" "$3" "$4" "$_rsn"
752 # $1: pnn, $2: DB name, $3: dbseqnum (must be < 255!!!!!)
753 db_ctdb_tstore_dbseqnum ()
755 # "__db_sequence_number__
" + trailing 0x00
756 _key='0x5f5f64625f73657175656e63655f6e756d6265725f5f00'
758 # Construct 8 byte (unit64_t) database sequence number. This
759 # probably breaks if $3 > 255
760 _value=$(printf "0x
%02x
%014x
" $3 0)
762 db_ctdb_tstore $1 "$2" "$_key" "$_value"
765 #######################################
767 # Make sure that $CTDB is set.
770 local="${TEST_SUBDIR}/scripts
/local.bash
"
771 if [ -r "$local" ] ; then