ctdb/tests/complex/31_nfs_tickle.sh

   1 #!/bin/bash
   2
   3 test_info()
   4 {
   5     cat <<EOF
   6 Verify that NFS connections are monitored and that NFS tickles are sent.
   7
   8 We create a connection to the NFS server on a node and confirm that
   9 this connection is registered in the nfs-tickles/ subdirectory in
  10 shared storage.  Then disable the relevant NFS server node and ensure
  11 that it send an appropriate reset packet.
  12
  13 Prerequisites:
  14
  15 * An active CTDB cluster with at least 2 nodes with public addresses.
  16
  17 * Test must be run on a real or virtual cluster rather than against
  18   local daemons.
  19
  20 * Test must not be run from a cluster node.
  21
  22 * Cluster nodes must be listening on the NFS TCP port (2049).
  23
  24 Steps:
  25
  26 1. Verify that the cluster is healthy.
  27 2. Connect from the current host (test client) to TCP port 2049 using
  28    the public address of a cluster node.
  29 3. Determine the source socket used for the connection.
  30 4. Ensure that CTDB records the source socket details in the nfs-tickles
  31    directory on shared storage.
  32 5. Disable the node that the connection has been made to.
  33 6. Verify that a TCP tickle (a reset packet) is sent to the test client.
  34
  35 Expected results:
  36
  37 * CTDB should correctly record the socket and should send a reset
  38   packet when the node is disabled.
  39 EOF
  40 }
  41
  42 . "${TEST_SCRIPTS_DIR}/integration.bash"
  43
  44 set -e
  45
  46 ctdb_test_init "$@"
  47
  48 ctdb_test_check_real_cluster
  49
  50 cluster_is_healthy
  51
  52 # Reset configuration
  53 ctdb_restart_when_done
  54
  55 # We need this for later, so we know how long to run nc for.
  56 try_command_on_node any $CTDB getvar MonitorInterval
  57 monitor_interval="${out#*= }"
  58 #echo "Monitor interval on node $test_node is $monitor_interval seconds."
  59
  60 select_test_node_and_ips
  61 try_command_on_node $test_node "$CTDB listnodes | wc -l"
  62 numnodes="$out"
  63
  64 test_port=2049
  65
  66 echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with netcat..."
  67
  68 nc -d -w $(($monitor_interval * 4)) $test_ip $test_port &
  69 nc_pid=$!
  70 ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1"
  71
  72 wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc"
  73 src_socket="$out"
  74 echo "Source socket is $src_socket"
  75
  76 wait_for_monitor_event $test_node
  77
  78 echo "Wait until NFS connection is tracked by CTDB on test node ..."
  79 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
  80
  81 echo "Getting TicklesUpdateInterval..."
  82 try_command_on_node $test_node $CTDB getvar TickleUpdateInterval
  83 update_interval="$out"
  84
  85 echo "Wait until NFS connection is tracked by CTDB on all nodes..."
  86 wait_until $(($update_interval * 2)) \
  87     check_tickles_all $numnodes  $test_ip $test_port $src_socket
  88
  89 tcptickle_sniff_start $src_socket "${test_ip}:${test_port}"
  90
  91 # We need to be nasty to make that the node being failed out doesn't
  92 # get a chance to send any tickles and confuse our sniff.
  93 echo "Killing ctdbd on ${test_node}..."
  94 try_command_on_node $test_node killall -9 ctdbd
  95
  96 wait_until_node_has_status $test_node disconnected
  97
  98 tcptickle_sniff_wait_show