ctdb/tests/complex/31_nfs_tickle.sh

   1 #!/bin/bash
   2
   3 test_info()
   4 {
   5     cat <<EOF
   6 Verify that NFS connections are monitored and that NFS tickles are sent.
   7
   8 We create a connection to the NFS server on a node and confirm that
   9 this connection is registered in the nfs-tickles/ subdirectory in
  10 shared storage.  Then kill ctdbd on the relevant NFS server node and
  11 ensure that the takeover node sends an appropriate reset packet.
  12
  13 Prerequisites:
  14
  15 * An active CTDB cluster with at least 2 nodes with public addresses.
  16
  17 * Test must be run on a real or virtual cluster rather than against
  18   local daemons.
  19
  20 * Test must not be run from a cluster node.
  21
  22 * Cluster nodes must be listening on the NFS TCP port (2049).
  23
  24 Expected results:
  25
  26 * CTDB should correctly record the socket and on failover the takeover
  27   node should send a reset packet.
  28 EOF
  29 }
  30
  31 . "${TEST_SCRIPTS_DIR}/integration.bash"
  32
  33 set -e
  34
  35 ctdb_test_init "$@"
  36
  37 ctdb_test_check_real_cluster
  38
  39 cluster_is_healthy
  40
  41 # Reset configuration
  42 ctdb_restart_when_done
  43
  44 # We need this for later, so we know how long to run nc for.
  45 try_command_on_node any $CTDB getvar MonitorInterval
  46 monitor_interval="${out#*= }"
  47 #echo "Monitor interval on node $test_node is $monitor_interval seconds."
  48
  49 select_test_node_and_ips
  50 try_command_on_node $test_node "$CTDB listnodes | wc -l"
  51 numnodes="$out"
  52
  53 test_port=2049
  54
  55 echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with netcat..."
  56
  57 nc -d -w $(($monitor_interval * 4)) $test_ip $test_port &
  58 nc_pid=$!
  59 ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1"
  60
  61 wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc"
  62 src_socket="$out"
  63 echo "Source socket is $src_socket"
  64
  65 wait_for_monitor_event $test_node
  66
  67 echo "Wait until NFS connection is tracked by CTDB on test node ..."
  68 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
  69
  70 echo "Getting TicklesUpdateInterval..."
  71 try_command_on_node $test_node $CTDB getvar TickleUpdateInterval
  72 update_interval="$out"
  73
  74 echo "Wait until NFS connection is tracked by CTDB on all nodes..."
  75 wait_until $(($update_interval * 2)) \
  76     check_tickles_all $numnodes  $test_ip $test_port $src_socket
  77
  78 tcptickle_sniff_start $src_socket "${test_ip}:${test_port}"
  79
  80 # We need to be nasty to make that the node being failed out doesn't
  81 # get a chance to send any tickles and confuse our sniff.  IPs also
  82 # need to be dropped because we're simulating a dead node rather than
  83 # a CTDB failure.  To properly handle a CTDB failure we would need a
  84 # watchdog to drop the IPs when CTDB disappears.
  85 echo "Killing ctdbd on ${test_node}..."
  86 try_command_on_node -v $test_node "killall -9 ctdbd ; $CTDB_TEST_WRAPPER drop_ips ${test_node_ips}"
  87
  88 wait_until_node_has_status $test_node disconnected
  89
  90 tcptickle_sniff_wait_show