ctdb/tests/scripts/integration.bash

   1 # Hey Emacs, this is a -*- shell-script -*- !!!  :-)
   2
   3 . "${TEST_SCRIPTS_DIR}/common.sh"
   4
   5 ######################################################################
   6
   7 export CTDB_TIMEOUT=60
   8
   9 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then
  10     CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap"
  11 else
  12     _d=$(cd ${TEST_SCRIPTS_DIR}; echo $PWD)
  13     CTDB_TEST_WRAPPER="$_d/test_wrap"
  14 fi
  15 export CTDB_TEST_WRAPPER
  16
  17 # If $VALGRIND is set then use it whenever ctdb is called, but only if
  18 # $CTDB is not already set.
  19 [ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb"
  20
  21 # why???
  22 PATH="${TEST_SCRIPTS_DIR}:${PATH}"
  23
  24 ######################################################################
  25
  26 ctdb_test_exit ()
  27 {
  28     local status=$?
  29
  30     trap - 0
  31
  32     [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
  33     status=$(($testfailures+0))
  34
  35     # Avoid making a test fail from this point onwards.  The test is
  36     # now complete.
  37     set +e
  38
  39     echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..."
  40
  41     eval "$ctdb_test_exit_hook" || true
  42     unset ctdb_test_exit_hook
  43
  44     if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then
  45
  46         restart_ctdb
  47     else
  48         # This could be made unconditional but then we might get
  49         # duplication from the recovery in restart_ctdb.  We want to
  50         # leave the recovery in restart_ctdb so that future tests that
  51         # might do a manual restart mid-test will benefit.
  52         echo "Forcing a recovery..."
  53         onnode 0 $CTDB recover
  54     fi
  55
  56     exit $status
  57 }
  58
  59 ctdb_test_exit_hook_add ()
  60 {
  61     ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*"
  62 }
  63
  64 ctdb_test_init ()
  65 {
  66     scriptname=$(basename "$0")
  67     testfailures=0
  68     ctdb_test_restart_scheduled=false
  69
  70     trap "ctdb_test_exit" 0
  71 }
  72
  73 ########################################
  74
  75 # Sets: $out
  76 try_command_on_node ()
  77 {
  78     local nodespec="$1" ; shift
  79
  80     local verbose=false
  81     local onnode_opts=""
  82
  83     while [ "${nodespec#-}" != "$nodespec" ] ; do
  84         if [ "$nodespec" = "-v" ] ; then
  85             verbose=true
  86         else
  87             onnode_opts="${onnode_opts}${onnode_opts:+ }${nodespec}"
  88         fi
  89         nodespec="$1" ; shift
  90     done
  91
  92     local cmd="$*"
  93
  94     out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
  95
  96         echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
  97         echo "$out"
  98         return 1
  99     }
 100
 101     if $verbose ; then
 102         echo "Output of \"$cmd\":"
 103         echo "$out"
 104     fi
 105 }
 106
 107 sanity_check_output ()
 108 {
 109     local min_lines="$1"
 110     local regexp="$2" # Should be anchored as necessary.
 111     local output="$3"
 112
 113     local ret=0
 114
 115     local num_lines=$(echo "$output" | wc -l)
 116     echo "There are $num_lines lines of output"
 117     if [ $num_lines -lt $min_lines ] ; then
 118         echo "BAD: that's less than the required number (${min_lines})"
 119         ret=1
 120     fi
 121
 122     local status=0
 123     local unexpected # local doesn't pass through status of command on RHS.
 124     unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
 125
 126     # Note that this is reversed.
 127     if [ $status -eq 0 ] ; then
 128         echo "BAD: unexpected lines in output:"
 129         echo "$unexpected" | cat -A
 130         ret=1
 131     else
 132         echo "Output lines look OK"
 133     fi
 134
 135     return $ret
 136 }
 137
 138 sanity_check_ips ()
 139 {
 140     local ips="$1" # list of "ip node" lines
 141
 142     echo "Sanity checking IPs..."
 143
 144     local x ipp prev
 145     prev=""
 146     while read x ipp ; do
 147         [ "$ipp" = "-1" ] && break
 148         if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
 149             echo "OK"
 150             return 0
 151         fi
 152         prev="$ipp"
 153     done <<<"$ips"
 154
 155     echo "BAD: a node was -1 or IPs are only assigned to one node"
 156     echo "Are you running an old version of CTDB?"
 157     return 1
 158 }
 159
 160 # This returns a list of "ip node" lines in $out
 161 all_ips_on_node()
 162 {
 163     local node="$1"
 164     try_command_on_node $node \
 165         "$CTDB ip -Y | awk -F: 'NR > 1 { print \$2, \$3 }'"
 166 }
 167
 168 _select_test_node_and_ips ()
 169 {
 170     try_command_on_node any \
 171         "$CTDB ip -Y -n all | awk -F: 'NR > 1 { print \$2, \$3 }'"
 172
 173     test_node=""  # this matches no PNN
 174     test_node_ips=""
 175     local ip pnn
 176     while read ip pnn ; do
 177         if [ -z "$test_node" -a "$pnn" != "-1" ] ; then
 178             test_node="$pnn"
 179         fi
 180         if [ "$pnn" = "$test_node" ] ; then
 181             test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
 182         fi
 183     done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 184
 185     echo "Selected node ${test_node} with IPs: ${test_node_ips}."
 186     test_ip="${test_node_ips%% *}"
 187
 188     [ -n "$test_node" ] || return 1
 189 }
 190
 191 select_test_node_and_ips ()
 192 {
 193     local timeout=10
 194     while ! _select_test_node_and_ips ; do
 195         echo "Unable to find a test node with IPs assigned"
 196         if [ $timeout -le 0 ] ; then
 197             echo "BAD: Too many attempts"
 198             return 1
 199         fi
 200         sleep_for 1
 201         timeout=$(($timeout - 1))
 202     done
 203
 204     return 0
 205 }
 206
 207 # Sets: mask, iface
 208 get_test_ip_mask_and_iface ()
 209 {
 210     # Find the interface
 211     try_command_on_node $test_node "$CTDB ip -v -Y | awk -F: -v ip=$test_ip '\$2 == ip { print \$4 }'"
 212     iface="$out"
 213
 214     if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
 215         # Find the netmask
 216         try_command_on_node $test_node ip addr show to $test_ip
 217         mask="${out##*/}"
 218         mask="${mask%% *}"
 219     else
 220         mask="24"
 221     fi
 222
 223     echo "$test_ip/$mask is on $iface"
 224 }
 225
 226 #######################################
 227
 228 # Wait until either timeout expires or command succeeds.  The command
 229 # will be tried once per second, unless timeout has format T/I, where
 230 # I is the recheck interval.
 231 wait_until ()
 232 {
 233     local timeout="$1" ; shift # "$@" is the command...
 234
 235     local interval=1
 236     case "$timeout" in
 237         */*)
 238             interval="${timeout#*/}"
 239             timeout="${timeout%/*}"
 240     esac
 241
 242     local negate=false
 243     if [ "$1" = "!" ] ; then
 244         negate=true
 245         shift
 246     fi
 247
 248     echo -n "<${timeout}|"
 249     local t=$timeout
 250     while [ $t -gt 0 ] ; do
 251         local rc=0
 252         "$@" || rc=$?
 253         if { ! $negate && [ $rc -eq 0 ] ; } || \
 254             { $negate && [ $rc -ne 0 ] ; } ; then
 255             echo "|$(($timeout - $t))|"
 256             echo "OK"
 257             return 0
 258         fi
 259         local i
 260         for i in $(seq 1 $interval) ; do
 261             echo -n .
 262         done
 263         t=$(($t - $interval))
 264         sleep $interval
 265     done
 266
 267     echo "*TIMEOUT*"
 268
 269     return 1
 270 }
 271
 272 sleep_for ()
 273 {
 274     echo -n "=${1}|"
 275     for i in $(seq 1 $1) ; do
 276         echo -n '.'
 277         sleep 1
 278     done
 279     echo '|'
 280 }
 281
 282 _cluster_is_healthy ()
 283 {
 284     $CTDB nodestatus all >/dev/null
 285 }
 286
 287 _cluster_is_recovered ()
 288 {
 289     node_has_status all recovered
 290 }
 291
 292 _cluster_is_ready ()
 293 {
 294     _cluster_is_healthy && _cluster_is_recovered
 295 }
 296
 297 cluster_is_healthy ()
 298 {
 299     if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
 300         echo "Cluster is HEALTHY"
 301         if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_recovered ; then
 302           echo "WARNING: cluster in recovery mode!"
 303         fi
 304         return 0
 305     else
 306         echo "Cluster is UNHEALTHY"
 307         if ! ${ctdb_test_restart_scheduled:-false} ; then
 308             echo "DEBUG AT $(date '+%F %T'):"
 309             local i
 310             for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do
 311                 echo "$i"
 312                 $i || true
 313             done
 314         fi
 315         return 1
 316     fi
 317 }
 318
 319 wait_until_ready ()
 320 {
 321     local timeout="${1:-120}"
 322
 323     echo "Waiting for cluster to become ready..."
 324
 325     wait_until $timeout onnode -q any $CTDB_TEST_WRAPPER _cluster_is_ready
 326 }
 327
 328 # This function is becoming nicely overloaded.  Soon it will collapse!  :-)
 329 node_has_status ()
 330 {
 331     local pnn="$1"
 332     local status="$2"
 333
 334     local bits fpat mpat rpat
 335     case "$status" in
 336         (unhealthy)    bits="?:?:?:1:*" ;;
 337         (healthy)      bits="?:?:?:0:*" ;;
 338         (disconnected) bits="1:*" ;;
 339         (connected)    bits="0:*" ;;
 340         (banned)       bits="?:1:*" ;;
 341         (unbanned)     bits="?:0:*" ;;
 342         (disabled)     bits="?:?:1:*" ;;
 343         (enabled)      bits="?:?:0:*" ;;
 344         (stopped)      bits="?:?:?:?:1:*" ;;
 345         (notstopped)   bits="?:?:?:?:0:*" ;;
 346         (frozen)       fpat='^[[:space:]]+frozen[[:space:]]+1$' ;;
 347         (unfrozen)     fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
 348         (monon)        mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
 349         (monoff)       mpat='^Monitoring mode:DISABLED \(1\)$' ;;
 350         (recovered)    rpat='^Recovery mode:RECOVERY \(1\)$' ;;
 351         *)
 352             echo "node_has_status: unknown status \"$status\""
 353             return 1
 354     esac
 355
 356     if [ -n "$bits" ] ; then
 357         local out x line
 358
 359         out=$($CTDB -Y status 2>&1) || return 1
 360
 361         {
 362             read x
 363             while read line ; do
 364                 # This needs to be done in 2 steps to avoid false matches.
 365                 local line_bits="${line#:${pnn}:*:}"
 366                 [ "$line_bits" = "$line" ] && continue
 367                 [ "${line_bits#${bits}}" != "$line_bits" ] && return 0
 368             done
 369             return 1
 370         } <<<"$out" # Yay bash!
 371     elif [ -n "$fpat" ] ; then
 372         $CTDB statistics -n "$pnn" | egrep -q "$fpat"
 373     elif [ -n "$mpat" ] ; then
 374         $CTDB getmonmode -n "$pnn" | egrep -q "$mpat"
 375     elif [ -n "$rpat" ] ; then
 376         ! $CTDB status -n "$pnn" | egrep -q "$rpat"
 377     else
 378         echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
 379         return 1
 380     fi
 381 }
 382
 383 wait_until_node_has_status ()
 384 {
 385     local pnn="$1"
 386     local status="$2"
 387     local timeout="${3:-30}"
 388     local proxy_pnn="${4:-any}"
 389
 390     echo "Waiting until node $pnn has status \"$status\"..."
 391
 392     if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then
 393         for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do
 394             echo "$i"
 395             $i || true
 396         done
 397
 398         return 1
 399     fi
 400
 401 }
 402
 403 # Useful for superficially testing IP failover.
 404 # IPs must be on the given node.
 405 # If the first argument is '!' then the IPs must not be on the given node.
 406 ips_are_on_node ()
 407 {
 408     local negating=false
 409     if [ "$1" = "!" ] ; then
 410         negating=true ; shift
 411     fi
 412     local node="$1" ; shift
 413     local ips="$*"
 414
 415     local out
 416
 417     all_ips_on_node $node
 418
 419     local check
 420     for check in $ips ; do
 421         local ip pnn
 422         while read ip pnn ; do
 423             if [ "$check" = "$ip" ] ; then
 424                 if [ "$pnn" = "$node" ] ; then
 425                     if $negating ; then return 1 ; fi
 426                 else
 427                     if ! $negating ; then return 1 ; fi
 428                 fi
 429                 ips="${ips/${ip}}" # Remove from list
 430                 break
 431             fi
 432             # If we're negating and we didn't see the address then it
 433             # isn't hosted by anyone!
 434             if $negating ; then
 435                 ips="${ips/${check}}"
 436             fi
 437         done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 438     done
 439
 440     ips="${ips// }" # Remove any spaces.
 441     [ -z "$ips" ]
 442 }
 443
 444 wait_until_ips_are_on_node ()
 445 {
 446     # Go to some trouble to print a use description of what is happening
 447     local not=""
 448     if [ "$1" == "!" ] ; then
 449         not="no longer "
 450     fi
 451     local node=""
 452     local ips=""
 453     local i
 454     for i ; do
 455         [ "$i" != "!" ] || continue
 456         if [ -z "$node" ] ; then
 457             node="$i"
 458             continue
 459         fi
 460         ips="${ips}${ips:+, }${i}"
 461     done
 462     echo "Waiting for ${ips} to ${not}be assigned to node ${node}"
 463
 464     wait_until 60 ips_are_on_node "$@"
 465 }
 466
 467 node_has_some_ips ()
 468 {
 469     local node="$1"
 470
 471     local out
 472
 473     all_ips_on_node $node
 474
 475     while read ip pnn ; do
 476         if [ "$node" = "$pnn" ] ; then
 477             return 0
 478         fi
 479     done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 480
 481     return 1
 482 }
 483
 484 wait_until_node_has_some_ips ()
 485 {
 486     echo "Waiting for node to have some IPs..."
 487
 488     wait_until 60 node_has_some_ips "$@"
 489 }
 490
 491 #######################################
 492
 493 _ctdb_hack_options ()
 494 {
 495     local ctdb_options="$*"
 496
 497     case "$ctdb_options" in
 498         *--start-as-stopped*)
 499             export CTDB_START_AS_STOPPED="yes"
 500     esac
 501 }
 502
 503 restart_ctdb_1 ()
 504 {
 505     _ctdb_hack_options "$@"
 506
 507     if [ -e /etc/redhat-release ] ; then
 508         service ctdb restart
 509     else
 510         /etc/init.d/ctdb restart
 511     fi
 512 }
 513
 514 # Restart CTDB on all nodes.  Override for local daemons.
 515 _restart_ctdb_all ()
 516 {
 517     onnode -p all $CTDB_TEST_WRAPPER restart_ctdb_1 "$@"
 518 }
 519
 520 # Nothing needed for a cluster.  Override for local daemons.
 521 setup_ctdb ()
 522 {
 523     :
 524 }
 525
 526 restart_ctdb ()
 527 {
 528     # "$@" is passed to restart_ctdb_all.
 529
 530     echo -n "Restarting CTDB"
 531     if $ctdb_test_restart_scheduled ; then
 532         echo -n " (scheduled)"
 533     fi
 534     echo "..."
 535
 536     local i
 537     for i in $(seq 1 5) ; do
 538         _restart_ctdb_all "$@" || {
 539             echo "Restart failed.  Trying again in a few seconds..."
 540             sleep_for 5
 541             continue
 542         }
 543
 544         wait_until_ready || {
 545             echo "Cluster didn't become ready.  Restarting..."
 546             continue
 547         }
 548
 549         echo "Setting RerecoveryTimeout to 1"
 550         onnode -pq all "$CTDB setvar RerecoveryTimeout 1"
 551
 552         # In recent versions of CTDB, forcing a recovery like this
 553         # blocks until the recovery is complete.  Hopefully this will
 554         # help the cluster to stabilise before a subsequent test.
 555         echo "Forcing a recovery..."
 556         onnode -q 0 $CTDB recover
 557         sleep_for 2
 558
 559         if ! onnode -q any $CTDB_TEST_WRAPPER _cluster_is_recovered ; then
 560             echo "Cluster has gone into recovery again, waiting..."
 561             wait_until 30/2 onnode -q any $CTDB_TEST_WRAPPER _cluster_is_recovered
 562         fi
 563
 564
 565         # Cluster is still healthy.  Good, we're done!
 566         if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
 567             echo "Cluster became UNHEALTHY again [$(date)]"
 568             onnode -p all ctdb status -Y 2>&1
 569             onnode -p all ctdb scriptstatus 2>&1
 570             echo "Restarting..."
 571             continue
 572         fi
 573
 574         echo "Doing a sync..."
 575         onnode -q 0 $CTDB sync
 576
 577         echo "ctdb is ready"
 578         return 0
 579     done
 580
 581     echo "Cluster UNHEALTHY...  too many attempts..."
 582     onnode -p all ctdb status -Y 2>&1
 583     onnode -p all ctdb scriptstatus 2>&1
 584
 585     # Try to make the calling test fail
 586     status=1
 587     return 1
 588 }
 589
 590 # Does nothing on cluster and should be overridden for local daemons
 591 maybe_stop_ctdb ()
 592 {
 593     :
 594 }
 595
 596 ctdb_restart_when_done ()
 597 {
 598     ctdb_test_restart_scheduled=true
 599 }
 600
 601 get_ctdbd_command_line_option ()
 602 {
 603     local pnn="$1"
 604     local option="$2"
 605
 606     try_command_on_node "$pnn" "$CTDB getpid" || \
 607         die "Unable to get PID of ctdbd on node $pnn"
 608
 609     local pid="${out#*:}"
 610     try_command_on_node "$pnn" "ps -p $pid -o args hww" || \
 611         die "Unable to get command-line of PID $pid"
 612
 613     # Strip everything up to and including --option
 614     local t="${out#*--${option}}"
 615     # Strip leading '=' or space if present
 616     t="${t#=}"
 617     t="${t# }"
 618     # Strip any following options and print
 619     echo "${t%% -*}"
 620 }
 621
 622 #######################################
 623
 624 wait_for_monitor_event ()
 625 {
 626     local pnn="$1"
 627     local timeout=120
 628
 629     echo "Waiting for a monitor event on node ${pnn}..."
 630
 631     try_command_on_node "$pnn" $CTDB scriptstatus || {
 632         echo "Unable to get scriptstatus from node $pnn"
 633         return 1
 634     }
 635
 636     local ctdb_scriptstatus_original="$out"
 637     wait_until 120 _ctdb_scriptstatus_changed
 638 }
 639
 640 _ctdb_scriptstatus_changed ()
 641 {
 642     try_command_on_node "$pnn" $CTDB scriptstatus || {
 643         echo "Unable to get scriptstatus from node $pnn"
 644         return 1
 645     }
 646
 647     [ "$out" != "$ctdb_scriptstatus_original" ]
 648 }
 649
 650 #######################################
 651
 652 nfs_test_setup ()
 653 {
 654     select_test_node_and_ips
 655
 656     nfs_first_export=$(showmount -e $test_ip | sed -n -e '2s/ .*//p')
 657
 658     echo "Creating test subdirectory..."
 659     try_command_on_node $test_node "mktemp -d --tmpdir=$nfs_first_export"
 660     nfs_test_dir="$out"
 661     try_command_on_node $test_node "chmod 777 $nfs_test_dir"
 662
 663     nfs_mnt_d=$(mktemp -d)
 664     nfs_local_file="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE"
 665     nfs_remote_file="${nfs_test_dir}/TEST_FILE"
 666
 667     ctdb_test_exit_hook_add nfs_test_cleanup
 668
 669     echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..."
 670     mount -o timeo=1,hard,intr,vers=3 \
 671         ${test_ip}:${nfs_first_export} ${nfs_mnt_d}
 672 }
 673
 674 nfs_test_cleanup ()
 675 {
 676     rm -f "$nfs_local_file"
 677     umount -f "$nfs_mnt_d"
 678     rmdir "$nfs_mnt_d"
 679     onnode -q $test_node rmdir "$nfs_test_dir"
 680 }
 681
 682 #######################################
 683
 684 # $1: pnn, $2: DB name
 685 db_get_path ()
 686 {
 687     try_command_on_node -v $1 $CTDB getdbstatus "$2" |
 688     sed -n -e "s@^path: @@p"
 689 }
 690
 691 # $1: pnn, $2: DB name
 692 db_ctdb_cattdb_count_records ()
 693 {
 694     try_command_on_node -v $1 $CTDB cattdb "$2" |
 695     grep '^key' | grep -v '__db_sequence_number__' |
 696     wc -l
 697 }
 698
 699 # $1: pnn, $2: DB name, $3: key string, $4: value string, $5: RSN (default 7)
 700 db_ctdb_tstore ()
 701 {
 702     _tdb=$(db_get_path $1 "$2")
 703     _rsn="${5:-7}"
 704     try_command_on_node $1 $CTDB tstore "$_tdb" "$3" "$4" "$_rsn"
 705 }
 706
 707 # $1: pnn, $2: DB name, $3: dbseqnum (must be < 255!!!!!)
 708 db_ctdb_tstore_dbseqnum ()
 709 {
 710     # "__db_sequence_number__" + trailing 0x00
 711     _key='0x5f5f64625f73657175656e63655f6e756d6265725f5f00'
 712
 713     # Construct 8 byte (unit64_t) database sequence number.  This
 714     # probably breaks if $3 > 255
 715     _value=$(printf "0x%02x%014x" $3 0)
 716
 717     db_ctdb_tstore $1 "$2" "$_key" "$_value"
 718 }
 719
 720 #######################################
 721
 722 # Make sure that $CTDB is set.
 723 : ${CTDB:=ctdb}
 724
 725 local="${TEST_SUBDIR}/scripts/local.bash"
 726 if [ -r "$local" ] ; then
 727     . "$local"
 728 fi