ctdb/tests/local_daemons.sh

   1 #!/bin/sh
   2
   3 set -u
   4
   5 export CTDB_TEST_MODE="yes"
   6
   7 # Following 2 lines may be modified by installation script
   8 CTDB_TESTS_ARE_INSTALLED=false
   9 CTDB_TEST_DIR=$(dirname "$0")
  10 export CTDB_TESTS_ARE_INSTALLED CTDB_TEST_DIR
  11
  12 export TEST_SCRIPTS_DIR="${CTDB_TEST_DIR}/scripts"
  13
  14 . "${TEST_SCRIPTS_DIR}/common.sh"
  15
  16 if ! $CTDB_TESTS_ARE_INSTALLED ; then
  17         hdir="$CTDB_SCRIPTS_HELPER_BINDIR"
  18         export CTDB_EVENTD="${hdir}/ctdb-eventd"
  19         export CTDB_EVENT_HELPER="${hdir}/ctdb-event"
  20         export CTDB_LOCK_HELPER="${hdir}/ctdb_lock_helper"
  21         export CTDB_RECOVERY_HELPER="${hdir}/ctdb_recovery_helper"
  22         export CTDB_TAKEOVER_HELPER="${hdir}/ctdb_takeover_helper"
  23         export CTDB_CLUSTER_MUTEX_HELPER="${hdir}/ctdb_mutex_fcntl_helper"
  24 fi
  25
  26 ########################################
  27
  28 # If the given IP is hosted then print 2 items: maskbits and iface
  29 have_ip ()
  30 {
  31         _addr="$1"
  32
  33         case "$_addr" in
  34         *:*) _bits=128 ;;
  35         *)   _bits=32  ;;
  36         esac
  37
  38         _t=$(ip addr show to "${_addr}/${_bits}")
  39         [ -n "$_t" ]
  40 }
  41
  42 setup_nodes ()
  43 {
  44         _num_nodes="$1"
  45         _use_ipv6="$2"
  46
  47         _have_all_ips=true
  48         for _i in $(seq 0 $((_num_nodes - 1)) ) ; do
  49                 if $_use_ipv6 ; then
  50                         _j=$(printf "%04x" $((0x5f00 + 1 + _i)) )
  51                         _node_ip="fd00::5357:${_j}"
  52                         if have_ip "$_node_ip" ; then
  53                                 echo "$_node_ip"
  54                         else
  55                                 cat >&2 <<EOF
  56 ERROR: ${_node_ip} not on an interface, please add it
  57 EOF
  58                                 _have_all_ips=false
  59                         fi
  60                 else
  61                         _c=$(( _i / 100 ))
  62                         _d=$(( 1 + (_i % 100) ))
  63                         echo "127.0.${_c}.${_d}"
  64                 fi
  65         done
  66
  67         # Fail if we don't have all of the IPv6 addresses assigned
  68         $_have_all_ips
  69 }
  70
  71 setup_public_addresses ()
  72 {
  73         _num_nodes="$1"
  74         _node_no_ips="$2"
  75         _use_ipv6="$3"
  76
  77         for _i in $(seq 0 $((_num_nodes - 1)) ) ; do
  78                 if  [ "$_i" -eq "$_node_no_ips" ] ; then
  79                         continue
  80                 fi
  81
  82                 # 2 public addresses on most nodes, just to make
  83                 # things interesting
  84                 if $_use_ipv6 ; then
  85                         printf 'fc00:10::1:%x/64 lo\n' $((1 + _i))
  86                         printf 'fc00:10::2:%x/64 lo\n' $((1 + _i))
  87                 else
  88                         _c1=$(( 100 + (_i / 100) ))
  89                         _c2=$(( 200 + (_i / 100) ))
  90                         _d=$(( 1 + (_i % 100) ))
  91                         printf '192.168.%d.%d/24 lo\n' "$_c1" "$_d"
  92                         printf '192.168.%d.%d/24 lo\n' "$_c2" "$_d"
  93                 fi
  94         done
  95 }
  96
  97 setup_socket_wrapper ()
  98 {
  99         _socket_wrapper_so="$1"
 100
 101         _so="${directory}/libsocket-wrapper.so"
 102         if [ ! -f "$_socket_wrapper_so" ] ; then
 103                 die "$0 setup: Unable to find ${_socket_wrapper_so}"
 104         fi
 105
 106         # Find absolute path if only relative is given
 107         case "$_socket_wrapper_so" in
 108         /*) : ;;
 109         *) _socket_wrapper_so="${PWD}/${_socket_wrapper_so}" ;;
 110         esac
 111
 112         rm -f "$_so"
 113         ln -s "$_socket_wrapper_so" "$_so"
 114
 115         _d="${directory}/sw"
 116         rm -rf "$_d"
 117         mkdir -p "$_d"
 118 }
 119
 120 local_daemons_setup_usage ()
 121 {
 122         cat >&2 <<EOF
 123 $0 <directory> setup [ <options>... ]
 124
 125 Options:
 126   -C            Comment out given config item (default: item uncommented)
 127   -F            Disable failover (default: failover enabled)
 128   -N <file>     Nodes file (default: automatically generated)
 129   -n <num>      Number of nodes (default: 3)
 130   -P <file>     Public addresses file (default: automatically generated)
 131   -R            Use a command for the cluster lock (default: use a file)
 132   -r <time>     Like -R and set recheck interval to <time> (default: use a file)
 133   -S <library>  Socket wrapper shared library to preload (default: none)
 134   -6            Generate IPv6 IPs for nodes, public addresses (default: IPv4)
 135 EOF
 136
 137         exit 1
 138 }
 139
 140 local_daemons_setup ()
 141 {
 142         _commented_config=""
 143         _disable_failover=false
 144         _nodes_file=""
 145         _num_nodes=3
 146         _public_addresses_file=""
 147         _cluster_lock_use_command=false
 148         _cluster_lock_recheck_interval=""
 149         _socket_wrapper=""
 150         _use_ipv6=false
 151
 152         set -e
 153
 154         while getopts "C:FN:n:P:Rr:S:6h?" _opt ; do
 155                 case "$_opt" in
 156                 C) _t="${_commented_config}${_commented_config:+|}"
 157                    _commented_config="${_t}${OPTARG}"
 158                    ;;
 159                 F) _disable_failover=true ;;
 160                 N) _nodes_file="$OPTARG" ;;
 161                 n) _num_nodes="$OPTARG" ;;
 162                 P) _public_addresses_file="$OPTARG" ;;
 163                 R) _cluster_lock_use_command=true ;;
 164                 r) _cluster_lock_use_command=true
 165                    _cluster_lock_recheck_interval="$OPTARG"
 166                    ;;
 167                 S) _socket_wrapper="$OPTARG" ;;
 168                 6) _use_ipv6=true ;;
 169                 \?|h) local_daemons_setup_usage ;;
 170                 esac
 171         done
 172         shift $((OPTIND - 1))
 173
 174         mkdir -p "$directory"
 175
 176         _nodes_all="${directory}/nodes"
 177         if [ -n "$_nodes_file" ] ; then
 178                 cp "$_nodes_file" "$_nodes_all"
 179         else
 180                 setup_nodes "$_num_nodes" $_use_ipv6 >"$_nodes_all"
 181         fi
 182
 183         # If there are (strictly) greater than 2 nodes then we'll
 184         # "randomly" choose a node to have no public addresses
 185         _node_no_ips=-1
 186         if [ "$_num_nodes" -gt 2 ] ; then
 187                 _node_no_ips=$(($$ % _num_nodes))
 188         fi
 189
 190         _public_addresses_all="${directory}/public_addresses"
 191         if [ -n "$_public_addresses_file" ] ; then
 192                 cp "$_public_addresses_file" "$_public_addresses_all"
 193         else
 194                 setup_public_addresses "$_num_nodes" \
 195                                        $_node_no_ips \
 196                                        "$_use_ipv6" >"$_public_addresses_all"
 197         fi
 198
 199         _cluster_lock_dir="${directory}/shared/.ctdb"
 200         mkdir -p "$_cluster_lock_dir"
 201         _cluster_lock="${_cluster_lock_dir}/cluster.lock"
 202         if $_cluster_lock_use_command ; then
 203                 _helper="${CTDB_SCRIPTS_HELPER_BINDIR}/ctdb_mutex_fcntl_helper"
 204                 _t="! ${_helper} ${_cluster_lock}"
 205                 if [ -n "$_cluster_lock_recheck_interval" ] ; then
 206                         _t="${_t} ${_cluster_lock_recheck_interval}"
 207                 fi
 208                 _cluster_lock="$_t"
 209         fi
 210
 211         if [ -n "$_socket_wrapper" ] ; then
 212                 setup_socket_wrapper "$_socket_wrapper"
 213         fi
 214
 215         for _n in $(seq 0 $((_num_nodes - 1))) ; do
 216                 # CTDB_TEST_SUITE_DIR needs to be correctly set so
 217                 # setup_ctdb_base() finds the etc-ctdb/ subdirectory
 218                 # and the test event script is correctly installed
 219                 # shellcheck disable=SC2034
 220                 CTDB_TEST_SUITE_DIR="$CTDB_TEST_DIR" \
 221                            setup_ctdb_base "$directory" "node.${_n}" \
 222                                 functions notify.sh debug-hung-script.sh
 223
 224                 cp "$_nodes_all" "${CTDB_BASE}/nodes"
 225
 226                 _public_addresses="${CTDB_BASE}/public_addresses"
 227
 228                 if  [ -z "$_public_addresses_file" ] && \
 229                             [ "$_node_no_ips" -eq "$_n" ] ; then
 230                         echo "Node ${_n} will have no public IPs."
 231                         : >"$_public_addresses"
 232                 else
 233                         cp "$_public_addresses_all" "$_public_addresses"
 234                 fi
 235
 236                 _node_ip=$(sed -n -e "$((_n + 1))p" "$_nodes_all")
 237
 238                 _db_dir="${CTDB_BASE}/db"
 239                 for _d in "volatile" "persistent" "state" ; do
 240                         mkdir -p "${_db_dir}/${_d}"
 241                 done
 242
 243                 cat >"${CTDB_BASE}/ctdb.conf" <<EOF
 244 [logging]
 245         location = file:${CTDB_BASE}/log.ctdb
 246         log level = INFO
 247
 248 [cluster]
 249         cluster lock = ${_cluster_lock}
 250         node address = ${_node_ip}
 251
 252 [database]
 253         volatile database directory = ${_db_dir}/volatile
 254         persistent database directory = ${_db_dir}/persistent
 255         state database directory = ${_db_dir}/state
 256
 257 [failover]
 258         disabled = ${_disable_failover}
 259
 260 [event]
 261         debug script = debug-hung-script.sh
 262 EOF
 263
 264                 (
 265                         IFS='|'
 266                         for _c in $_commented_config ; do
 267                                 # Quote all backslashes due to double-quotes
 268                                 sed -i -e "s|^\\t\\(${_c}\\) = |\\t# \\1 = |" \
 269                                     "${CTDB_BASE}/ctdb.conf"
 270                         done
 271                 )
 272         done
 273 }
 274
 275 local_daemons_ssh_usage ()
 276 {
 277         cat >&2 <<EOF
 278 usage: $0 <directory> ssh [ -n ] <ip> <command>
 279 EOF
 280
 281         exit 1
 282 }
 283
 284 local_daemons_ssh ()
 285 {
 286         if [ $# -lt 2 ] ; then
 287                 local_daemons_ssh_usage
 288         fi
 289
 290         # Only try to respect ssh -n option, others can't be used so discard them
 291         _close_stdin=false
 292         while getopts "nh?" _opt ; do
 293                 case "$_opt" in
 294                 n) _close_stdin=true ;;
 295                 \?|h) local_daemons_ssh_usage ;;
 296                 *) : ;;
 297                 esac
 298         done
 299         shift $((OPTIND - 1))
 300
 301         if [ $# -lt 2 ] ; then
 302                 local_daemons_ssh_usage
 303         fi
 304
 305         _nodes="${directory}/nodes"
 306
 307         # IP address of node. onnode can pass hostnames but not in these tests
 308         _ip="$1" ; shift
 309         # "$*" is command
 310
 311
 312         # Determine the correct CTDB base directory
 313         _num=$(awk -v ip="$_ip" '$1 == ip { print NR }' "$_nodes")
 314         _node=$((_num - 1))
 315         export CTDB_BASE="${directory}/node.${_node}"
 316
 317         if [ ! -d "$CTDB_BASE" ] ; then
 318                 die "$0 ssh: Unable to find base for node ${_ip}"
 319         fi
 320
 321         if $_close_stdin ; then
 322                 exec sh -c "$*" </dev/null
 323         else
 324                 exec sh -c "$*"
 325         fi
 326 }
 327
 328 onnode_common ()
 329 {
 330         # onnode will execute this, which fakes ssh against local daemons
 331         export ONNODE_SSH="${0} ${directory} ssh"
 332
 333         # onnode just needs the nodes file, so use the common one
 334         export CTDB_BASE="$directory"
 335 }
 336
 337 local_daemons_generic_usage ()
 338 {
 339         cat >&2 <<EOF
 340 usage: $0 <directory> ${1} <nodes>
 341
 342 <nodes> can be  "all", a node number or any specification supported by onnode
 343 EOF
 344
 345         exit 1
 346 }
 347
 348 local_daemons_start_socket_wrapper ()
 349 {
 350         _so="${directory}/libsocket-wrapper.so"
 351         _d="${directory}/sw"
 352
 353         if [ -d "$_d" ] && [ -f "$_so" ] ; then
 354                 export SOCKET_WRAPPER_DIR="$_d"
 355                 export LD_PRELOAD="$_so"
 356                 export SOCKET_WRAPPER_DIR_ALLOW_ORIG="1"
 357         fi
 358 }
 359
 360 local_daemons_start ()
 361 {
 362         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 363                 local_daemons_generic_usage "start"
 364         fi
 365
 366         local_daemons_start_socket_wrapper
 367
 368         _nodes="$1"
 369
 370         onnode_common
 371
 372         onnode -i "$_nodes" "${VALGRIND:-} ctdbd"
 373 }
 374
 375 local_daemons_stop ()
 376 {
 377         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 378                 local_daemons_generic_usage "stop"
 379         fi
 380
 381         _nodes="$1"
 382
 383         onnode_common
 384
 385         onnode -p "$_nodes" \
 386                 "if [ -e \"\${CTDB_BASE}/run/ctdbd.pid\" ] ; then \
 387                         ${CTDB:-${VALGRIND:-} ctdb} shutdown ; \
 388                  fi"
 389 }
 390
 391 local_daemons_onnode_usage ()
 392 {
 393         cat >&2 <<EOF
 394 usage: $0 <directory> onnode <nodes> <command>...
 395
 396 <nodes> can be  "all", a node number or any specification supported by onnode
 397 EOF
 398
 399         exit 1
 400 }
 401
 402 local_daemons_onnode ()
 403 {
 404         if [ $# -lt 2 ] || [ "$1" = "-h" ] ; then
 405                 local_daemons_onnode_usage
 406         fi
 407
 408         _nodes="$1"
 409         shift
 410
 411         onnode_common
 412
 413         onnode "$_nodes" "$@"
 414 }
 415
 416 local_daemons_print_socket ()
 417 {
 418         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 419                 local_daemons_generic_usage "print-socket"
 420         fi
 421
 422         _nodes="$1"
 423         shift
 424
 425         onnode_common
 426
 427         _path="${CTDB_SCRIPTS_HELPER_BINDIR}/ctdb-path"
 428         onnode -q "$_nodes" "${VALGRIND:-} ${_path} socket ctdbd"
 429 }
 430
 431 local_daemons_print_log ()
 432 {
 433         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 434                 local_daemons_generic_usage "print-log"
 435         fi
 436
 437         _nodes="$1"
 438         shift
 439
 440         onnode_common
 441
 442         # shellcheck disable=SC2016
 443         # $CTDB_BASE must only be expanded under onnode, not in top-level shell
 444         onnode -q "$_nodes" 'cat ${CTDB_BASE}/log.ctdb' |
 445         sort
 446
 447 }
 448
 449 local_daemons_tail_log ()
 450 {
 451         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 452                 local_daemons_generic_usage "tail-log"
 453         fi
 454
 455         _nodes="$1"
 456         shift
 457
 458         onnode_common
 459
 460         # shellcheck disable=SC2016,SC2046
 461         # $CTDB_BASE must only be expanded under onnode, not in top-level shell
 462         # Intentional word splitting to separate log filenames
 463         tail -f $(onnode -q "$_nodes" 'echo ${CTDB_BASE}/log.ctdb')
 464 }
 465
 466 usage ()
 467 {
 468         cat <<EOF
 469 usage: $0 <directory> <command> [ <options>... ]
 470
 471 Commands:
 472   setup          Set up daemon configuration according to given options
 473   start          Start specified daemon(s)
 474   stop           Stop specified daemon(s)
 475   onnode         Run a command in the environment of specified daemon(s)
 476   print-socket   Print the Unix domain socket used by specified daemon(s)
 477   print-log      Print logs for specified daemon(s) to stdout
 478   tail-log       Follow logs for specified daemon(s) to stdout
 479
 480 All commands use <directory> for daemon configuration
 481
 482 Run command with -h option to see per-command usage
 483 EOF
 484
 485         exit 1
 486 }
 487
 488 if [ $# -lt 2 ] ; then
 489         usage
 490 fi
 491
 492 directory="$1"
 493 command="$2"
 494 shift 2
 495
 496 case "$command" in
 497 setup) local_daemons_setup "$@" ;;
 498 ssh) local_daemons_ssh "$@" ;; # Internal, not shown by usage()
 499 start) local_daemons_start "$@" ;;
 500 stop) local_daemons_stop "$@" ;;
 501 onnode) local_daemons_onnode "$@" ;;
 502 print-socket) local_daemons_print_socket "$@" ;;
 503 print-log) local_daemons_print_log "$@" ;;
 504 tail-log) local_daemons_tail_log "$@" ;;
 505 *) usage ;;
 506 esac