ctdb/tests/scripts/integration.bash

   1 # Hey Emacs, this is a -*- shell-script -*- !!!  :-)
   2
   3 . "${TEST_SCRIPTS_DIR}/common.sh"
   4
   5 ######################################################################
   6
   7 export CTDB_TIMEOUT=60
   8
   9 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then
  10     CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap"
  11 else
  12     _d=$(cd ${TEST_SCRIPTS_DIR}; echo $PWD)
  13     CTDB_TEST_WRAPPER="$_d/test_wrap"
  14 fi
  15 export CTDB_TEST_WRAPPER
  16
  17 # If $VALGRIND is set then use it whenever ctdb is called, but only if
  18 # $CTDB is not already set.
  19 [ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb"
  20
  21 # why???
  22 PATH="${TEST_SCRIPTS_DIR}:${PATH}"
  23
  24 ######################################################################
  25
  26 ctdb_test_exit ()
  27 {
  28     local status=$?
  29
  30     trap - 0
  31
  32     [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
  33     status=$(($testfailures+0))
  34
  35     # Avoid making a test fail from this point onwards.  The test is
  36     # now complete.
  37     set +e
  38
  39     echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..."
  40
  41     eval "$ctdb_test_exit_hook" || true
  42     unset ctdb_test_exit_hook
  43
  44     if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then
  45
  46         restart_ctdb
  47     else
  48         # This could be made unconditional but then we might get
  49         # duplication from the recovery in restart_ctdb.  We want to
  50         # leave the recovery in restart_ctdb so that future tests that
  51         # might do a manual restart mid-test will benefit.
  52         echo "Forcing a recovery..."
  53         onnode 0 $CTDB recover
  54     fi
  55
  56     exit $status
  57 }
  58
  59 ctdb_test_exit_hook_add ()
  60 {
  61     ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*"
  62 }
  63
  64 ctdb_test_init ()
  65 {
  66     scriptname=$(basename "$0")
  67     testfailures=0
  68     ctdb_test_restart_scheduled=false
  69
  70     trap "ctdb_test_exit" 0
  71 }
  72
  73 ########################################
  74
  75 # Sets: $out
  76 try_command_on_node ()
  77 {
  78     local nodespec="$1" ; shift
  79
  80     local verbose=false
  81     local onnode_opts=""
  82
  83     while [ "${nodespec#-}" != "$nodespec" ] ; do
  84         if [ "$nodespec" = "-v" ] ; then
  85             verbose=true
  86         else
  87             onnode_opts="${onnode_opts}${onnode_opts:+ }${nodespec}"
  88         fi
  89         nodespec="$1" ; shift
  90     done
  91
  92     local cmd="$*"
  93
  94     out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
  95
  96         echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
  97         echo "$out"
  98         return 1
  99     }
 100
 101     if $verbose ; then
 102         echo "Output of \"$cmd\":"
 103         echo "$out"
 104     fi
 105 }
 106
 107 sanity_check_output ()
 108 {
 109     local min_lines="$1"
 110     local regexp="$2" # Should be anchored as necessary.
 111     local output="$3"
 112
 113     local ret=0
 114
 115     local num_lines=$(echo "$output" | wc -l)
 116     echo "There are $num_lines lines of output"
 117     if [ $num_lines -lt $min_lines ] ; then
 118         echo "BAD: that's less than the required number (${min_lines})"
 119         ret=1
 120     fi
 121
 122     local status=0
 123     local unexpected # local doesn't pass through status of command on RHS.
 124     unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
 125
 126     # Note that this is reversed.
 127     if [ $status -eq 0 ] ; then
 128         echo "BAD: unexpected lines in output:"
 129         echo "$unexpected" | cat -A
 130         ret=1
 131     else
 132         echo "Output lines look OK"
 133     fi
 134
 135     return $ret
 136 }
 137
 138 sanity_check_ips ()
 139 {
 140     local ips="$1" # list of "ip node" lines
 141
 142     echo "Sanity checking IPs..."
 143
 144     local x ipp prev
 145     prev=""
 146     while read x ipp ; do
 147         [ "$ipp" = "-1" ] && break
 148         if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
 149             echo "OK"
 150             return 0
 151         fi
 152         prev="$ipp"
 153     done <<<"$ips"
 154
 155     echo "BAD: a node was -1 or IPs are only assigned to one node"
 156     echo "Are you running an old version of CTDB?"
 157     return 1
 158 }
 159
 160 # This returns a list of "ip node" lines in $out
 161 all_ips_on_node()
 162 {
 163     local node=$@
 164     try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'"
 165 }
 166
 167 _select_test_node_and_ips ()
 168 {
 169     all_ips_on_node 0
 170
 171     test_node=""  # this matches no PNN
 172     test_node_ips=""
 173     local ip pnn
 174     while read ip pnn ; do
 175         if [ -z "$test_node" -a "$pnn" != "-1" ] ; then
 176             test_node="$pnn"
 177         fi
 178         if [ "$pnn" = "$test_node" ] ; then
 179             test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
 180         fi
 181     done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 182
 183     echo "Selected node ${test_node} with IPs: ${test_node_ips}."
 184     test_ip="${test_node_ips%% *}"
 185
 186     [ -n "$test_node" ] || return 1
 187 }
 188
 189 select_test_node_and_ips ()
 190 {
 191     local timeout=10
 192     while ! _select_test_node_and_ips ; do
 193         echo "Unable to find a test node with IPs assigned"
 194         if [ $timeout -le 0 ] ; then
 195             echo "BAD: Too many attempts"
 196             return 1
 197         fi
 198         sleep_for 1
 199         timeout=$(($timeout - 1))
 200     done
 201
 202     return 0
 203 }
 204
 205 #######################################
 206
 207 # Wait until either timeout expires or command succeeds.  The command
 208 # will be tried once per second.
 209 wait_until ()
 210 {
 211     local timeout="$1" ; shift # "$@" is the command...
 212
 213     local negate=false
 214     if [ "$1" = "!" ] ; then
 215         negate=true
 216         shift
 217     fi
 218
 219     echo -n "<${timeout}|"
 220     local t=$timeout
 221     while [ $t -gt 0 ] ; do
 222         local rc=0
 223         "$@" || rc=$?
 224         if { ! $negate && [ $rc -eq 0 ] ; } || \
 225             { $negate && [ $rc -ne 0 ] ; } ; then
 226             echo "|$(($timeout - $t))|"
 227             echo "OK"
 228             return 0
 229         fi
 230         echo -n .
 231         t=$(($t - 1))
 232         sleep 1
 233     done
 234
 235     echo "*TIMEOUT*"
 236
 237     return 1
 238 }
 239
 240 sleep_for ()
 241 {
 242     echo -n "=${1}|"
 243     for i in $(seq 1 $1) ; do
 244         echo -n '.'
 245         sleep 1
 246     done
 247     echo '|'
 248 }
 249
 250 _cluster_is_healthy ()
 251 {
 252     $CTDB nodestatus all >/dev/null && \
 253         node_has_status 0 recovered
 254 }
 255
 256 cluster_is_healthy ()
 257 {
 258     if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
 259         echo "Cluster is HEALTHY"
 260         return 0
 261     else
 262         echo "Cluster is UNHEALTHY"
 263         if ! ${ctdb_test_restart_scheduled:-false} ; then
 264             echo "DEBUG AT $(date '+%F %T'):"
 265             local i
 266             for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do
 267                 echo "$i"
 268                 $i || true
 269             done
 270         fi
 271         return 1
 272     fi
 273 }
 274
 275 wait_until_healthy ()
 276 {
 277     local timeout="${1:-120}"
 278
 279     echo "Waiting for cluster to become healthy..."
 280
 281     wait_until 120 _cluster_is_healthy
 282 }
 283
 284 # This function is becoming nicely overloaded.  Soon it will collapse!  :-)
 285 node_has_status ()
 286 {
 287     local pnn="$1"
 288     local status="$2"
 289
 290     local bits fpat mpat rpat
 291     case "$status" in
 292         (unhealthy)    bits="?:?:?:1:*" ;;
 293         (healthy)      bits="?:?:?:0:*" ;;
 294         (disconnected) bits="1:*" ;;
 295         (connected)    bits="0:*" ;;
 296         (banned)       bits="?:1:*" ;;
 297         (unbanned)     bits="?:0:*" ;;
 298         (disabled)     bits="?:?:1:*" ;;
 299         (enabled)      bits="?:?:0:*" ;;
 300         (stopped)      bits="?:?:?:?:1:*" ;;
 301         (notstopped)   bits="?:?:?:?:0:*" ;;
 302         (frozen)       fpat='^[[:space:]]+frozen[[:space:]]+1$' ;;
 303         (unfrozen)     fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
 304         (monon)        mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
 305         (monoff)       mpat='^Monitoring mode:DISABLED \(1\)$' ;;
 306         (recovered)    rpat='^Recovery mode:NORMAL \(0\)$' ;;
 307         *)
 308             echo "node_has_status: unknown status \"$status\""
 309             return 1
 310     esac
 311
 312     if [ -n "$bits" ] ; then
 313         local out x line
 314
 315         out=$($CTDB -Y status 2>&1) || return 1
 316
 317         {
 318             read x
 319             while read line ; do
 320                 # This needs to be done in 2 steps to avoid false matches.
 321                 local line_bits="${line#:${pnn}:*:}"
 322                 [ "$line_bits" = "$line" ] && continue
 323                 [ "${line_bits#${bits}}" != "$line_bits" ] && return 0
 324             done
 325             return 1
 326         } <<<"$out" # Yay bash!
 327     elif [ -n "$fpat" ] ; then
 328         $CTDB statistics -n "$pnn" | egrep -q "$fpat"
 329     elif [ -n "$mpat" ] ; then
 330         $CTDB getmonmode -n "$pnn" | egrep -q "$mpat"
 331     elif [ -n "$rpat" ] ; then
 332         $CTDB status -n "$pnn" | egrep -q "$rpat"
 333     else
 334         echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
 335         return 1
 336     fi
 337 }
 338
 339 wait_until_node_has_status ()
 340 {
 341     local pnn="$1"
 342     local status="$2"
 343     local timeout="${3:-30}"
 344     local proxy_pnn="${4:-any}"
 345
 346     echo "Waiting until node $pnn has status \"$status\"..."
 347
 348     if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then
 349         for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do
 350             echo "$i"
 351             $i || true
 352         done
 353
 354         return 1
 355     fi
 356
 357 }
 358
 359 # Useful for superficially testing IP failover.
 360 # IPs must be on nodes matching nodeglob.
 361 # If the first argument is '!' then the IPs must not be on nodes
 362 # matching nodeglob.
 363 ips_are_on_nodeglob ()
 364 {
 365     local negating=false
 366     if [ "$1" = "!" ] ; then
 367         negating=true ; shift
 368     fi
 369     local nodeglob="$1" ; shift
 370     local ips="$*"
 371
 372     local out
 373
 374     all_ips_on_node 1
 375
 376     for check in $ips ; do
 377         while read ip pnn ; do
 378             if [ "$check" = "$ip" ] ; then
 379                 case "$pnn" in
 380                     ($nodeglob) if $negating ; then return 1 ; fi ;;
 381                     (*) if ! $negating ; then return 1 ; fi  ;;
 382                 esac
 383                 ips="${ips/${ip}}" # Remove from list
 384                 break
 385             fi
 386             # If we're negating and we didn't see the address then it
 387             # isn't hosted by anyone!
 388             if $negating ; then
 389                 ips="${ips/${check}}"
 390             fi
 391         done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 392     done
 393
 394     ips="${ips// }" # Remove any spaces.
 395     [ -z "$ips" ]
 396 }
 397
 398 wait_until_ips_are_on_nodeglob ()
 399 {
 400     echo "Waiting for IPs to fail over..."
 401
 402     wait_until 60 ips_are_on_nodeglob "$@"
 403 }
 404
 405 node_has_some_ips ()
 406 {
 407     local node="$1"
 408
 409     local out
 410
 411     all_ips_on_node 1
 412
 413     while read ip pnn ; do
 414         if [ "$node" = "$pnn" ] ; then
 415             return 0
 416         fi
 417     done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 418
 419     return 1
 420 }
 421
 422 wait_until_node_has_some_ips ()
 423 {
 424     echo "Waiting for node to have some IPs..."
 425
 426     wait_until 60 node_has_some_ips "$@"
 427 }
 428
 429 #######################################
 430
 431 _ctdb_hack_options ()
 432 {
 433     local ctdb_options="$*"
 434
 435     case "$ctdb_options" in
 436         *--start-as-stopped*)
 437             export CTDB_START_AS_STOPPED="yes"
 438     esac
 439 }
 440
 441 _restart_ctdb ()
 442 {
 443     _ctdb_hack_options "$@"
 444
 445     if [ -e /etc/redhat-release ] ; then
 446         service ctdb restart
 447     else
 448         /etc/init.d/ctdb restart
 449     fi
 450 }
 451
 452 # Nothing needed for a cluster.  Override for local daemons.
 453 setup_ctdb ()
 454 {
 455     :
 456 }
 457
 458 restart_ctdb ()
 459 {
 460     # "$@" is passed to ctdbd start.
 461
 462     echo -n "Restarting CTDB"
 463     if $ctdb_test_restart_scheduled ; then
 464         echo -n " (scheduled)"
 465     fi
 466     echo "..."
 467
 468     local i
 469     for i in $(seq 1 5) ; do
 470         if [ -n "$CTDB_NODES_SOCKETS" ] ; then
 471             daemons_stop
 472             daemons_start "$@"
 473         else
 474             onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@"
 475         fi || {
 476             echo "Restart failed.  Trying again in a few seconds..."
 477             sleep_for 5
 478             continue
 479         }
 480
 481         onnode -q 1  $CTDB_TEST_WRAPPER wait_until_healthy || {
 482             echo "Cluster didn't become healthy.  Restarting..."
 483             continue
 484         }
 485
 486         echo "Setting RerecoveryTimeout to 1"
 487         onnode -pq all "$CTDB setvar RerecoveryTimeout 1"
 488
 489         # In recent versions of CTDB, forcing a recovery like this
 490         # blocks until the recovery is complete.  Hopefully this will
 491         # help the cluster to stabilise before a subsequent test.
 492         echo "Forcing a recovery..."
 493         onnode -q 0 $CTDB recover
 494         sleep_for 1
 495
 496         # Cluster is still healthy.  Good, we're done!
 497         if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
 498             echo "Cluster became UNHEALTHY again [$(date)]"
 499             onnode -p all ctdb status -Y 2>&1
 500             onnode -p all ctdb scriptstatus 2>&1
 501             echo "Restarting..."
 502             continue
 503         fi
 504
 505         echo "Doing a sync..."
 506         onnode -q 0 $CTDB sync
 507
 508         echo "ctdb is ready"
 509         return 0
 510     done
 511
 512     echo "Cluster UNHEALTHY...  too many attempts..."
 513     onnode -p all ctdb status -Y 2>&1
 514     onnode -p all ctdb scriptstatus 2>&1
 515
 516     # Try to make the calling test fail
 517     status=1
 518     return 1
 519 }
 520
 521 ctdb_restart_when_done ()
 522 {
 523     ctdb_test_restart_scheduled=true
 524 }
 525
 526 get_ctdbd_command_line_option ()
 527 {
 528     local pnn="$1"
 529     local option="$2"
 530
 531     try_command_on_node "$pnn" "$CTDB getpid" || \
 532         die "Unable to get PID of ctdbd on node $pnn"
 533
 534     local pid="${out#*:}"
 535     try_command_on_node "$pnn" "ps -p $pid -o args hww" || \
 536         die "Unable to get command-line of PID $pid"
 537
 538     # Strip everything up to and including --option
 539     local t="${out#*--${option}}"
 540     # Strip leading '=' or space if present
 541     t="${t#=}"
 542     t="${t# }"
 543     # Strip any following options and print
 544     echo "${t%% -*}"
 545 }
 546
 547 #######################################
 548
 549 wait_for_monitor_event ()
 550 {
 551     local pnn="$1"
 552     local timeout=120
 553
 554     echo "Waiting for a monitor event on node ${pnn}..."
 555
 556     try_command_on_node "$pnn" $CTDB scriptstatus || {
 557         echo "Unable to get scriptstatus from node $pnn"
 558         return 1
 559     }
 560
 561     local ctdb_scriptstatus_original="$out"
 562     wait_until 120 _ctdb_scriptstatus_changed
 563 }
 564
 565 _ctdb_scriptstatus_changed ()
 566 {
 567     try_command_on_node "$pnn" $CTDB scriptstatus || {
 568         echo "Unable to get scriptstatus from node $pnn"
 569         return 1
 570     }
 571
 572     [ "$out" != "$ctdb_scriptstatus_original" ]
 573 }
 574
 575 #######################################
 576
 577 nfs_test_setup ()
 578 {
 579     select_test_node_and_ips
 580
 581     nfs_first_export=$(showmount -e $test_ip | sed -n -e '2s/ .*//p')
 582
 583     echo "Creating test subdirectory..."
 584     try_command_on_node $test_node "mktemp -d --tmpdir=$nfs_first_export"
 585     nfs_test_dir="$out"
 586     try_command_on_node $test_node "chmod 777 $nfs_test_dir"
 587
 588     nfs_mnt_d=$(mktemp -d)
 589     nfs_local_file="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE"
 590     nfs_remote_file="${nfs_test_dir}/TEST_FILE"
 591
 592     ctdb_test_exit_hook_add nfs_test_cleanup
 593
 594     echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..."
 595     mount -o timeo=1,hard,intr,vers=3 \
 596         ${test_ip}:${nfs_first_export} ${nfs_mnt_d}
 597 }
 598
 599 nfs_test_cleanup ()
 600 {
 601     rm -f "$nfs_local_file"
 602     umount -f "$nfs_mnt_d"
 603     rmdir "$nfs_mnt_d"
 604     onnode -q $test_node rmdir "$nfs_test_dir"
 605 }
 606
 607 #######################################
 608
 609 # $1: pnn, $2: DB name
 610 db_get_path ()
 611 {
 612     try_command_on_node -v $1 $CTDB getdbstatus "$2" |
 613     sed -n -e "s@^path: @@p"
 614 }
 615
 616 # $1: pnn, $2: DB name
 617 db_ctdb_cattdb_count_records ()
 618 {
 619     try_command_on_node -v $1 $CTDB cattdb "$2" |
 620     grep '^key' | grep -v '__db_sequence_number__' |
 621     wc -l
 622 }
 623
 624 # $1: pnn, $2: DB name, $3: key string, $4: value string, $5: RSN (default 7)
 625 db_ctdb_tstore ()
 626 {
 627     _tdb=$(db_get_path $1 "$2")
 628     _rsn="${5:-7}"
 629     try_command_on_node $1 $CTDB tstore "$_tdb" "$3" "$4" "$_rsn"
 630 }
 631
 632 # $1: pnn, $2: DB name, $3: dbseqnum (must be < 255!!!!!)
 633 db_ctdb_tstore_dbseqnum ()
 634 {
 635     # "__db_sequence_number__" + trailing 0x00
 636     _key='0x5f5f64625f73657175656e63655f6e756d6265725f5f00'
 637
 638     # Construct 8 byte (unit64_t) database sequence number.  This
 639     # probably breaks if $3 > 255
 640     _value=$(printf "0x%02x%014x" $3 0)
 641
 642     db_ctdb_tstore $1 "$2" "$_key" "$_value"
 643 }
 644
 645 #######################################
 646
 647 # Make sure that $CTDB is set.
 648 : ${CTDB:=ctdb}
 649
 650 local="${TEST_SUBDIR}/scripts/local.bash"
 651 if [ -r "$local" ] ; then
 652     . "$local"
 653 fi