ctdb/tests/complex/34_nfs_tickle_restart.sh

   1 #!/bin/bash
   2
   3 test_info()
   4 {
   5     cat <<EOF
   6 Verify that a newly started CTDB node gets updated tickle details
   7
   8 Prerequisites:
   9
  10 * An active CTDB cluster with at least 2 nodes with public addresses.
  11
  12 * Test must be run on a real or virtual cluster rather than against
  13   local daemons.
  14
  15 * Cluster nodes must be listening on the NFS TCP port (2049).
  16
  17 Steps:
  18
  19 As with 31_nfs_tickle.sh but restart a node after the tickle is
  20 registered.
  21
  22 Expected results:
  23
  24 * CTDB should correctly communicated tickles to new CTDB instances as
  25   they join the cluster.
  26 EOF
  27 }
  28
  29 . "${TEST_SCRIPTS_DIR}/integration.bash"
  30
  31 set -e
  32
  33 ctdb_test_init "$@"
  34
  35 ctdb_test_check_real_cluster
  36
  37 cluster_is_healthy
  38
  39 # Reset configuration
  40 ctdb_restart_when_done
  41
  42 # We need this for later, so we know how long to run nc for.
  43 try_command_on_node any $CTDB getvar MonitorInterval
  44 monitor_interval="${out#*= }"
  45 #echo "Monitor interval on node $test_node is $monitor_interval seconds."
  46
  47 select_test_node_and_ips
  48 try_command_on_node $test_node "$CTDB listnodes -X"
  49 listnodes_output="$out"
  50 numnodes=$(wc -l <<<"$listnodes_output")
  51
  52 test_port=2049
  53
  54 echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with netcat..."
  55
  56 nc -d -w 600 $test_ip $test_port &
  57 nc_pid=$!
  58 ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1"
  59
  60 wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc"
  61 src_socket="$out"
  62 echo "Source socket is $src_socket"
  63
  64 wait_for_monitor_event $test_node
  65
  66 echo "Wait until NFS connection is tracked by CTDB on test node ..."
  67 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
  68
  69 echo "Select a node to restart ctdbd"
  70 rn=$(awk -F'|' -v test_node=$test_node \
  71     '$2 != test_node { print $2 ; exit }' <<<"$listnodes_output")
  72
  73 echo "Restarting CTDB on node ${rn}"
  74 try_command_on_node $rn $CTDB_TEST_WRAPPER restart_ctdb_1
  75
  76 # In some theoretical world this is racy.  In practice, the node will
  77 # take quite a while to become healthy, so this will beat any
  78 # assignment of IPs to the node.
  79 echo "Setting NoIPTakeover on node ${rn}"
  80 try_command_on_node $rn $CTDB setvar NoIPTakeover 1
  81
  82 wait_until_ready
  83
  84 echo "Getting TickleUpdateInterval..."
  85 try_command_on_node $test_node $CTDB getvar TickleUpdateInterval
  86 update_interval="$out"
  87
  88 echo "Wait until NFS connection is tracked by CTDB on all nodes..."
  89 if ! wait_until $(($update_interval * 2)) \
  90     check_tickles_all $numnodes $test_ip $test_port $src_socket ; then
  91     echo "BAD: connection not tracked on all nodes:"
  92     echo "$out"
  93     exit 1
  94 fi
  95
  96 # We could go on to test whether the tickle ACK gets sent.  However,
  97 # this is tested in previous tests and the use of NoIPTakeover
  98 # complicates things on a 2 node cluster.