ctdb/tests/complex/90_debug_hung_script.sh

   1 #!/bin/bash
   2
   3 test_info()
   4 {
   5     cat <<EOF
   6 Verify CTDB's debugging of timed out eventscripts
   7
   8 Prerequisites:
   9
  10 * An active CTDB cluster with monitoring enabled
  11
  12 Expected results:
  13
  14 * When an eventscript times out the correct debugging is executed.
  15 EOF
  16 }
  17
  18 . "${TEST_SCRIPTS_DIR}/integration.bash"
  19
  20 set -e
  21
  22 ctdb_test_init "$@"
  23
  24 ctdb_test_check_real_cluster
  25
  26 cluster_is_healthy
  27
  28 # No need for restart when done
  29
  30 # This is overkill but it at least provides a valid test node
  31 select_test_node_and_ips
  32
  33 ####################
  34
  35 # Set this if CTDB is installed in a non-standard location on cluster
  36 # nodes
  37 [ -n "$CTDB_BASE" ] || CTDB_BASE="/etc/ctdb"
  38
  39 ####################
  40
  41 echo "Enable eventscript for testing timeouts..."
  42 ctdb_test_exit_hook_add "onnode -q $test_node $CTDB disablescript 99.timeout"
  43 try_command_on_node $test_node $CTDB enablescript "99.timeout"
  44
  45 ####################
  46
  47 echo "Setting monitor events to time out..."
  48 rc_local_d="${CTDB_BASE}/rc.local.d"
  49 try_command_on_node $test_node mkdir -p "$rc_local_d"
  50
  51 rc_local_f="${rc_local_d}/timeout_config.$$"
  52 ctdb_test_exit_hook_add "onnode $test_node rm -f $rc_local_f"
  53
  54 try_command_on_node $test_node mktemp
  55 debug_output="$out"
  56 ctdb_test_exit_hook_add "onnode $test_node rm -f $debug_output"
  57
  58 try_command_on_node -i $test_node tee "$rc_local_f" <<<"\
  59 CTDB_RUN_TIMEOUT_MONITOR=yes
  60 CTDB_DEBUG_HUNG_SCRIPT_LOGFILE=\"$debug_output\"
  61 CTDB_DEBUG_HUNG_SCRIPT_STACKPAT='exportfs\|rpcinfo\|sleep'"
  62
  63 try_command_on_node $test_node chmod +x "$rc_local_f"
  64
  65 ####################
  66
  67 wait_for_monitor_event $test_node
  68
  69 echo "Waiting for debugging output to appear..."
  70 # Use test -s because the file is created above using mktemp
  71 wait_until 60 onnode $test_node test -s "$debug_output"
  72
  73 echo "Checking output of hung script debugging..."
  74 try_command_on_node -v $test_node cat "$debug_output"
  75
  76 while IFS="" read pattern ; do
  77     if grep -- "^${pattern}\$" <<<"$out" >/dev/null ; then
  78         echo "GOOD: output contains \"$pattern\""
  79     else
  80         echo "BAD: output does not contain \"$pattern\""
  81         exit 1
  82     fi
  83 done <<'EOF'
  84 ===== Start of hung script debug for PID=".*", event="monitor" =====
  85 ===== End of hung script debug for PID=".*", event="monitor" =====
  86 pstree -p -a .*:
  87  *\`-99\\.timeout,.* /etc/ctdb/events.d/99.timeout monitor
  88  *\`-sleep,.*
  89 ---- Stack trace of interesting process [0-9]*\\[sleep\\] ----
  90 [<[0-9a-f]*>] .*sleep+.*
  91 ---- ctdb scriptstatus monitor: ----
  92 [0-9]* scripts were executed last monitor cycle
  93 99\\.timeout *Status:TIMEDOUT.*
  94  *OUTPUT:sleeping for [0-9]* seconds\\.\\.\\.
  95 EOF