ctdb/tests/eventscripts/scripts/local.sh

   1 # Hey Emacs, this is a -*- shell-script -*- !!!  :-)
   2
   3 # Augment PATH with relevant stubs/ directories.  We do this by actually
   4 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
   5 # prepending that to $PATH in rc.local to avoid the PATH reset in
   6 # functions.
   7
   8 EVENTSCRIPTS_PATH=""
   9
  10 if [ -d "${TEST_SUBDIR}/stubs" ] ; then
  11     EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs"
  12 fi
  13
  14 export EVENTSCRIPTS_PATH
  15
  16 PATH="${EVENTSCRIPTS_PATH}:${PATH}"
  17
  18 export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts"
  19 if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \
  20     "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then
  21     rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
  22 fi
  23 mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR"
  24 export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
  25
  26 export CTDB_LOGFILE="${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb"
  27 touch "$CTDB_LOGFILE" || die "Unable to create CTDB_LOGFILE=$CTDB_LOGFILE"
  28
  29 if [ -d "${TEST_SUBDIR}/etc" ] ; then
  30     cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR"
  31     export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
  32 else
  33     die "Unable to setup \$CTDB_ETCDIR"
  34 fi
  35
  36 if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then
  37     cp -prL "${TEST_SUBDIR}/etc-ctdb" "$EVENTSCRIPTS_TESTS_VAR_DIR"
  38     export CTDB_BASE="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc-ctdb"
  39 else
  40     die "Unable to set \$CTDB_BASE"
  41 fi
  42 export CTDB_BASE
  43
  44 if [ ! -d "${CTDB_BASE}/events.d" ] ; then
  45     cat <<EOF
  46 ERROR: Directory ${CTDB_BASE}/events.d does not exist.
  47
  48 That means that no eventscripts can be tested.
  49
  50 One possible explanation:
  51
  52   You have CTDB installed via RPMs (or similar), so the regular
  53   CTDB_BASE directory is in /etc/ctdb/
  54
  55   BUT
  56
  57   You have done a regular "configure" and "make install" so the tests
  58   are installed under /usr/local/.
  59
  60 If so, one possible hack to fix this is to create a symlink:
  61
  62   ln -s /etc/ctdb /usr/local/etc/ctdb
  63
  64 This is nasty but it works...  :-)
  65 EOF
  66     exit 1
  67 fi
  68
  69 ######################################################################
  70
  71 if "$TEST_VERBOSE" ; then
  72     debug () { echo "$@" ; }
  73 else
  74     debug () { : ; }
  75 fi
  76
  77 eventscripts_tests_cleanup_hooks=""
  78
  79 # This loses quoting!
  80 eventscripts_test_add_cleanup ()
  81 {
  82     eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*"
  83 }
  84
  85 trap 'eval $eventscripts_tests_cleanup_hooks' 0
  86
  87
  88 ######################################################################
  89
  90 # General setup fakery
  91
  92 setup_generic ()
  93 {
  94     debug "Setting up shares (3 existing shares)"
  95     # Create 3 fake shares/exports.
  96     export FAKE_SHARES=""
  97     for i in $(seq 1 3) ; do
  98         _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing"
  99         mkdir -p "$_s"
 100         FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}"
 101     done
 102
 103     export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding"
 104     mkdir -p "$FAKE_PROC_NET_BONDING"
 105     rm -f "$FAKE_PROC_NET_BONDING"/*
 106
 107     export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down"
 108     mkdir -p "$FAKE_ETHTOOL_LINK_DOWN"
 109     rm -f "$FAKE_ETHTOOL_LINK_DOWN"/*
 110
 111     # This can only have 2 levels.  We don't want to resort to usings
 112     # something dangerous like "rm -r" setup time.
 113     export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state"
 114     mkdir -p "$FAKE_IP_STATE"
 115     rm -f "$FAKE_IP_STATE"/*/*
 116     rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true
 117     rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true
 118
 119
 120     export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
 121     mkdir -p "${CTDB_DBDIR}/persistent"
 122
 123     export FAKE_TDBTOOL_SUPPORTS_CHECK="yes"
 124     export FAKE_TDB_IS_OK
 125     export FAKE_DATE_OUTPUT
 126
 127     export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
 128     export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
 129 }
 130
 131 tcp_port_down ()
 132 {
 133     for _i ; do
 134         debug "Marking TCP port \"${_i}\" as not listening"
 135         FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
 136     done
 137 }
 138
 139 shares_missing ()
 140 {
 141     _fmt="$1" ; shift
 142
 143     # Replace some shares with non-existent ones.
 144     _t=""
 145     _n=1
 146     _nl="
 147 "
 148     export MISSING_SHARES_TEXT=""
 149     for _i in $FAKE_SHARES ; do
 150         if [ $_n = "$1" ] ; then
 151             shift
 152             _i="${_i%_existing}_missing"
 153             debug "Replacing share $_n with missing share \"$_i\""
 154             rmdir "$_i" 2>/dev/null || true
 155             MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}")
 156         fi
 157         _t="${_t}${_t:+ }${_i}"
 158         _n=$(($_n + 1))
 159     done
 160     FAKE_SHARES="$_t"
 161 }
 162
 163 # Setup some fake /proc/net/bonding files with just enough info for
 164 # the eventscripts.
 165
 166 # arg1 is interface name, arg2 is currently active slave (use "None"
 167 # if none), arg3 is MII status ("up" or "down").
 168 setup_bond ()
 169 {
 170     _iface="$1"
 171     _slave="${2:-${_iface}_sl_0}"
 172     _mii_s="${3:-up}"
 173     _mii_subs="${4:-${_mii_s:-up}}"
 174     echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s"
 175     cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF
 176 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
 177 Currently Active Slave: $_slave
 178 # Status of the bond
 179 MII Status: $_mii_s
 180 # Status of 1st pretend adapter
 181 MII Status: $_mii_subs
 182 # Status of 2nd pretend adapter
 183 MII Status: $_mii_subs
 184 EOF
 185 }
 186
 187 ethtool_interfaces_down ()
 188 {
 189     for _i ; do
 190         echo "Marking interface $_i DOWN for ethtool"
 191         touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
 192     done
 193 }
 194
 195 ethtool_interfaces_up ()
 196 {
 197     for _i ; do
 198         echo "Marking interface $_i UP for ethtool"
 199         rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
 200     done
 201 }
 202
 203 dump_routes ()
 204 {
 205     echo "# ip rule show"
 206     ip rule show
 207
 208     ip rule show |
 209     while read _p _x _i _x _t ; do
 210         # Remove trailing colon after priority/preference.
 211         _p="${_p%:}"
 212         # Only remove rules that match our priority/preference.
 213         [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
 214
 215         echo "# ip route show table $_t"
 216         ip route show table "$_t"
 217     done
 218 }
 219
 220 # Copied from 13.per_ip_routing for now... so this is lazy testing  :-(
 221 ipv4_host_addr_to_net ()
 222 {
 223     _host="$1"
 224     _maskbits="$2"
 225
 226     # Convert the host address to an unsigned long by splitting out
 227     # the octets and doing the math.
 228     _host_ul=0
 229     for _o in $(export IFS="." ; echo $_host) ; do
 230         _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
 231     done
 232
 233     # Calculate the mask and apply it.
 234     _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
 235     _net_ul=$(( $_host_ul & $_mask_ul ))
 236
 237     # Now convert to a network address one byte at a time.
 238     _net=""
 239     for _o in $(seq 1 4) ; do
 240         _net="$(($_net_ul & 255))${_net:+.}${_net}"
 241         _net_ul=$(($_net_ul >> 8))
 242     done
 243
 244     echo "${_net}/${_maskbits}"
 245 }
 246
 247 ######################################################################
 248
 249 # CTDB fakery
 250
 251 # Evaluate an expression that probably calls functions or uses
 252 # variables from the CTDB functions file.  This is used for test
 253 # initialisation.
 254 eventscript_call ()
 255 {
 256     (
 257         . "$CTDB_BASE/functions"
 258         "$@"
 259     )
 260 }
 261
 262 # Set output for ctdb command.  Option 1st argument is return code.
 263 ctdb_set_output ()
 264 {
 265     _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out"
 266     cat >"$_out"
 267
 268     _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc"
 269     echo "${1:-0}" >"$_rc"
 270
 271     eventscripts_test_add_cleanup "rm -f $_out $_rc"
 272 }
 273
 274 setup_ctdb ()
 275 {
 276     setup_generic
 277
 278     export FAKE_CTDB_NUMNODES="${1:-3}"
 279     echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes"
 280
 281     export FAKE_CTDB_PNN="${2:-0}"
 282     echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}"
 283
 284     export CTDB_PUBLIC_ADDRESSES="${CTDB_BASE}/public_addresses"
 285     if [ -n "$3" ] ; then
 286         echo "Setting up CTDB_PUBLIC_ADDRESSES: $3"
 287         CTDB_PUBLIC_ADDRESSES=$(mktemp)
 288         for _i in $3 ; do
 289             _ip="${_i%@*}"
 290             _ifaces="${_i#*@}"
 291             echo "${_ip} ${_ifaces}" >>"$CTDB_PUBLIC_ADDRESSES"
 292         done
 293         eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES"
 294     fi
 295
 296     export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb"
 297
 298     export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down"
 299     mkdir -p "$FAKE_CTDB_IFACES_DOWN"
 300     rm -f "$FAKE_CTDB_IFACES_DOWN"/*
 301
 302     export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus"
 303     mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
 304     rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
 305
 306     export CTDB_PARTIALLY_ONLINE_INTERFACES
 307 }
 308
 309 setup_memcheck ()
 310 {
 311     setup_ctdb
 312
 313     _swap_total="5857276"
 314
 315     if [ "$1" = "bad" ] ; then
 316         _swap_free="   4352"
 317         _mem_cached=" 112"
 318         _mem_free=" 468"
 319     else
 320         _swap_free="$_swap_total"
 321         _mem_cached="1112"
 322         _mem_free="1468"
 323     fi
 324
 325     export FAKE_PROC_MEMINFO="\
 326 MemTotal:        3940712 kB
 327 MemFree:          225268 kB
 328 Buffers:          146120 kB
 329 Cached:          1139348 kB
 330 SwapCached:        56016 kB
 331 Active:          2422104 kB
 332 Inactive:        1019928 kB
 333 Active(anon):    1917580 kB
 334 Inactive(anon):   523080 kB
 335 Active(file):     504524 kB
 336 Inactive(file):   496848 kB
 337 Unevictable:        4844 kB
 338 Mlocked:            4844 kB
 339 SwapTotal:       ${_swap_total} kB
 340 SwapFree:        ${_swap_free} kB
 341 ..."
 342
 343     export FAKE_FREE_M="\
 344              total       used       free     shared    buffers     cached
 345 Mem:          3848       3634        213          0        142       ${_mem_cached}
 346 -/+ buffers/cache:       2379       ${_mem_free}
 347 Swap:         5719        246       5473"
 348
 349     export CTDB_MONITOR_FREE_MEMORY
 350     export CTDB_MONITOR_FREE_MEMORY_WARN
 351     export CTDB_CHECK_SWAP_IS_NOT_USED
 352 }
 353
 354 ctdb_get_interfaces ()
 355 {
 356     # The echo/subshell forces all the output onto 1 line.
 357     echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}')
 358 }
 359
 360 ctdb_get_1_interface ()
 361 {
 362     _t=$(ctdb_get_interfaces)
 363     echo ${_t%% *}
 364 }
 365
 366 # Print all public addresses as: interface IP maskbits
 367 # Each line is suitable for passing to takeip/releaseip
 368 ctdb_get_all_public_addresses ()
 369 {
 370     _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
 371     while IFS="/$IFS" read _ip _maskbits _ifaces ; do
 372         echo "$_ifaces $_ip $_maskbits"
 373     done <"$_f"
 374 }
 375
 376 # Print public addresses on this node as: interface IP maskbits
 377 # Each line is suitable for passing to takeip/releaseip
 378 ctdb_get_my_public_addresses ()
 379 {
 380     ctdb ip -v -Y | {
 381         read _x # skip header line
 382
 383         while IFS=":" read _x _ip _x _iface _x ; do
 384             [ -n "$_iface" ] || continue
 385             while IFS="/$IFS" read _i _maskbits _x ; do
 386                 if [ "$_ip" = "$_i" ] ; then
 387                     echo $_iface $_ip $_maskbits
 388                     break
 389                 fi
 390             done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
 391         done
 392     }
 393 }
 394
 395 # Prints the 1st public address as: interface IP maskbits
 396 # This is suitable for passing to takeip/releaseip
 397 ctdb_get_1_public_address ()
 398 {
 399     ctdb_get_my_public_addresses | { head -n 1 ; cat >/dev/null ; }
 400 }
 401
 402 ctdb_not_implemented ()
 403 {
 404     export CTDB_NOT_IMPLEMENTED="$1"
 405     ctdb_not_implemented="\
 406 DEBUG: ctdb: command \"$1\" not implemented in stub"
 407 }
 408
 409 ctdb_fake_scriptstatus ()
 410 {
 411     _code="$1"
 412     _status="$2"
 413     _err_out="$3"
 414
 415     _d1=$(date '+%s.%N')
 416     _d2=$(date '+%s.%N')
 417
 418     echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
 419 }
 420
 421 ######################################################################
 422
 423 setup_ctdb_policy_routing ()
 424 {
 425     service_name="per_ip_routing"
 426
 427     export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing"
 428     export CTDB_PER_IP_ROUTING_RULE_PREF=100
 429     export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000
 430     export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000
 431
 432     # Tests need to create and populate this file
 433     rm -f "$CTDB_PER_IP_ROUTING_CONF"
 434 }
 435
 436 # Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF.
 437 # $1 is the number of assigned IPs to use (<num>, all), defaulting to
 438 # 1.  If $2 is "default" then a default route is also added.
 439 create_policy_routing_config ()
 440 {
 441     _num_ips="${1:-1}"
 442     _should_add_default="$2"
 443
 444     ctdb_get_my_public_addresses |
 445     if [ "$_num_ips" = "all" ] ; then
 446         cat
 447     else
 448         { head -n "$_num_ips" ; cat >/dev/null ; }
 449     fi |
 450     while read _dev _ip _bits ; do
 451         _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
 452         _gw="${_net%.*}.1" # a dumb, calculated default
 453
 454         echo "$_ip $_net"
 455
 456         if [ "$_should_add_default" = "default" ] ; then
 457             echo "$_ip 0.0.0.0/0 $_gw"
 458         fi
 459     done >"$CTDB_PER_IP_ROUTING_CONF"
 460 }
 461
 462 # Check the routes against those that are expected.  $1 is the number
 463 # of assigned IPs to use (<num>, all), defaulting to 1.  If $2 is
 464 # "default" then expect default routes to have been added.
 465 check_routes ()
 466 {
 467     _num_ips="${1:-1}"
 468     _should_add_default="$2"
 469
 470     _policy_rules=""
 471     _policy_routes=""
 472
 473     ctdb_get_my_public_addresses |
 474     if [ "$_num_ips" = "all" ] ; then
 475         cat
 476     else
 477         { head -n "$_num_ips" ; cat >/dev/null ; }
 478     fi | {
 479         while read _dev _ip _bits ; do
 480             _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
 481             _gw="${_net%.*}.1" # a dumb, calculated default
 482
 483             _policy_rules="${_policy_rules}
 484 ${CTDB_PER_IP_ROUTING_RULE_PREF}:       from $_ip lookup ctdb.$_ip "
 485             _policy_routes="${_policy_routes}
 486 # ip route show table ctdb.$_ip
 487 $_net dev $_dev  scope link "
 488
 489             if [ "$_should_add_default" = "default" ] ; then
 490                 _policy_routes="${_policy_routes}
 491 default via $_gw dev $_dev "
 492             fi
 493         done
 494
 495         ok <<EOF
 496 # ip rule show
 497 0:      from all lookup local ${_policy_rules}
 498 32766:  from all lookup main
 499 32767:  from all lookup default ${_policy_routes}
 500 EOF
 501
 502         simple_test_command dump_routes
 503     }
 504 }
 505
 506 ######################################################################
 507
 508 setup_ctdb_natgw ()
 509 {
 510     debug "Setting up NAT gateway"
 511
 512     natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
 513     mkdir -p "$natgw_config_dir"
 514
 515     # These will accumulate, 1 per test... but will be cleaned up at
 516     # the end.
 517     export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
 518
 519     # Read from stdin
 520     while read _ip _master _dev ; do
 521         echo "$_ip"
 522         if [ "$_master" = "master" ] ; then
 523             export FAKE_CTDB_NATGW_MASTER="$_ip"
 524         fi
 525     done >"$CTDB_NATGW_NODES"
 526
 527     # Assume all of the nodes are on a /24 network and have IPv4
 528     # addresses:
 529     read _ip <"$CTDB_NATGW_NODES"
 530     export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
 531
 532     # These are fixed.  Probably don't use the same network for the
 533     # private node IPs.  To unset the default gateway just set it to
 534     # "".  :-)
 535     export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
 536     export CTDB_NATGW_PUBLIC_IFACE="eth1"
 537     export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
 538     export CTDB_NATGW_SLAVE_ONLY=""
 539 }
 540
 541 ok_natgw_master_ip_addr_show ()
 542 {
 543     _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
 544
 545     # This is based on CTDB_NATGW_PUBLIC_IP
 546     _brd="10.1.1.255"
 547
 548 ok <<EOF
 549 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
 550     link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
 551     inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE}
 552        valid_lft forever preferred_lft forever
 553 EOF
 554 }
 555
 556 ok_natgw_slave_ip_addr_show ()
 557 {
 558     _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
 559 ok <<EOF
 560 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
 561     link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
 562 EOF
 563 }
 564
 565 ok_natgw_master_static_routes ()
 566 {
 567     _nl="
 568 "
 569     _t=""
 570     for _i in $CTDB_NATGW_STATIC_ROUTES ; do
 571         # This is intentionally different to the code in 11.natgw ;-)
 572         case "$_i" in
 573             *@*)
 574                 _net=$(echo "$_i" | sed -e 's|@.*||')
 575                 _gw=$(echo "$_i" | sed -e 's|.*@||')
 576                 ;;
 577             *)
 578                 _net="$_i"
 579                 _gw="$CTDB_NATGW_DEFAULT_GATEWAY"
 580         esac
 581
 582         [ -n "$_gw" ] || continue
 583         _t="${_t}${_t:+${_nl}}"
 584         _t="${_t}${_net} via ${_gw} dev ethXXX  metric 10 "
 585     done
 586     ok "$_t"
 587 }
 588
 589 ok_natgw_slave_static_routes ()
 590 {
 591     _nl="
 592 "
 593     _t=""
 594     for _i in $CTDB_NATGW_STATIC_ROUTES ; do
 595         # This is intentionally different to the code in 11.natgw ;-)
 596         _net=$(echo "$_i" | sed -e 's|@.*||')
 597
 598         # The interface for the private network isn't specified as
 599         # part of the NATGW configuration and isn't part of the
 600         # command to add the route.  It is implicitly added by "ip
 601         # route" but our stub doesn't do this and adds "ethXXX".
 602         _t="${_t}${_t:+${_nl}}"
 603         _t="${_t}${_net} via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX  metric 10 "
 604     done
 605     ok "$_t"
 606 }
 607
 608 ######################################################################
 609
 610 # Samba/winbind fakery
 611
 612 setup_samba ()
 613 {
 614     setup_ctdb
 615
 616     service_name="samba"
 617
 618     if [ "$1" != "down" ] ; then
 619
 620         debug "Marking Samba services as up, listening and managed by CTDB"
 621         # Get into known state.
 622         eventscript_call ctdb_service_managed
 623
 624         # All possible service names for all known distros.
 625         for i in "smb" "nmb" "samba" ; do
 626             service "$i" force-started
 627         done
 628
 629         export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
 630         export CTDB_MANAGED_SERVICES="foo samba bar"
 631
 632         export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
 633         export FAKE_WBINFO_FAIL="no"
 634
 635         # Some things in 50.samba are backgrounded and waited for.  If
 636         # we don't sleep at all then timeouts can happen.  This avoids
 637         # that...  :-)
 638         export FAKE_SLEEP_FORCE=0.1
 639     else
 640         debug "Marking Samba services as down, not listening and not managed by CTDB"
 641         # Get into known state.
 642         eventscript_call ctdb_service_unmanaged
 643
 644         # All possible service names for all known distros.
 645         for i in "smb" "nmb" "samba" ; do
 646             service "$i" force-stopped
 647         done
 648
 649         export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
 650         export CTDB_MANAGED_SERVICES="foo bar"
 651         unset CTDB_MANAGES_SAMBA
 652
 653         export FAKE_TCP_LISTEN=""
 654         export FAKE_WBINFO_FAIL="yes"
 655     fi
 656
 657     # This is ugly but if this file isn't removed before each test
 658     # then configuration changes between tests don't stick.
 659     rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache"
 660 }
 661
 662 setup_winbind ()
 663 {
 664     setup_ctdb
 665
 666     service_name="winbind"
 667
 668     if [ "$1" != "down" ] ; then
 669
 670         debug "Marking Winbind service as up and managed by CTDB"
 671         # Get into known state.
 672         eventscript_call ctdb_service_managed
 673
 674         service "winbind" force-started
 675
 676         export CTDB_MANAGED_SERVICES="foo winbind bar"
 677
 678         export FAKE_WBINFO_FAIL="no"
 679
 680     else
 681         debug "Marking Winbind service as down and not managed by CTDB"
 682         # Get into known state.
 683         eventscript_call ctdb_service_unmanaged
 684
 685         service "winbind" force-stopped
 686
 687         export CTDB_MANAGED_SERVICES="foo bar"
 688         unset CTDB_MANAGES_WINBIND
 689
 690         export FAKE_WBINFO_FAIL="yes"
 691     fi
 692 }
 693
 694 wbinfo_down ()
 695 {
 696     debug "Making wbinfo commands fail"
 697     FAKE_WBINFO_FAIL="yes"
 698 }
 699
 700 ######################################################################
 701
 702 # NFS fakery
 703
 704 setup_nfs ()
 705 {
 706     setup_ctdb
 707
 708     service_name="nfs"
 709
 710     export FAKE_RPCINFO_SERVICES=""
 711
 712     export CTDB_NFS_SKIP_SHARE_CHECK="no"
 713
 714     export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
 715     export CTDB_NFS_DUMP_STUCK_THREADS
 716
 717     # Reset the failcounts for nfs services.
 718     eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
 719
 720     if [ "$1" != "down" ] ; then
 721         debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB"
 722
 723         eventscript_call ctdb_service_managed
 724         service "nfs" force-started  # might not be enough
 725
 726         export CTDB_MANAGED_SERVICES="foo nfs bar"
 727
 728         rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status"
 729     else
 730         debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
 731
 732         eventscript_call ctdb_service_unmanaged
 733         service "nfs" force-stopped  # might not be enough
 734         eventscript_call startstop_nfs stop
 735
 736         export CTDB_MANAGED_SERVICES="foo bar"
 737         unset CTDB_MANAGES_NFS
 738     fi
 739 }
 740
 741 setup_nfs_ganesha ()
 742 {
 743     setup_nfs "$@"
 744     export CTDB_NFS_SERVER_MODE="ganesha"
 745     if [ "$1" != "down" ] ; then
 746         export CTDB_MANAGES_NFS="yes"
 747     fi
 748
 749     # We do not support testing the Ganesha-nfsd-specific part of the
 750     # eventscript.
 751     export CTDB_SKIP_GANESHA_NFSD_CHECK="yes"
 752     export CTDB_NFS_SKIP_SHARE_CHECK="yes"
 753 }
 754
 755 rpc_services_down ()
 756 {
 757     for _i ; do
 758         debug "Marking RPC service \"${_i}\" as unavailable"
 759         FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
 760     done
 761 }
 762
 763 rpc_services_up ()
 764 {
 765     for _i ; do
 766         debug "Marking RPC service \"${_i}\" as available"
 767         case "$_i" in
 768             nfs)      _t="2:3" ;;
 769             mountd)   _t="1:3" ;;
 770             rquotad)  _t="1:2" ;;
 771             nlockmgr) _t="3:4" ;;
 772             status)   _t="1:1" ;;
 773             *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
 774         esac
 775
 776         FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
 777     done
 778 }
 779
 780 # Set the required result for a particular RPC program having failed
 781 # for a certain number of iterations.  This is probably still a work
 782 # in progress.  Note that we could hook aggressively
 783 # nfs_check_rpc_service() to try to implement this but we're better
 784 # off testing nfs_check_rpc_service() using independent code...  even
 785 # if it is incomplete and hacky.  So, if the 60.nfs eventscript
 786 # changes and the tests start to fail then it may be due to this
 787 # function being incomplete.
 788 rpc_set_service_failure_response ()
 789 {
 790     _progname="$1"
 791     # The number of failures defaults to the iteration number.  This
 792     # will be true when we fail from the 1st iteration... but we need
 793     # the flexibility to set the number of failures.
 794     _numfails="${2:-${iteration}}"
 795
 796     _etc="$CTDB_ETCDIR" # shortcut for readability
 797     for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
 798         if [ -r "$_c" ] ; then
 799             . "$_c"
 800             break
 801         fi
 802     done
 803
 804     # A handy newline.  :-)
 805     _nl="
 806 "
 807
 808     # Default
 809     ok_null
 810
 811     _file=$(ls "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9]."${_progname}.check")
 812     [ -r "$_file" ] || die "RPC check file \"$_file\" does not exist or is not unique"
 813
 814     while read _op _li _actions ; do
 815         # Skip comments
 816         case "$_op" in
 817             \#*) continue ;;
 818         esac
 819
 820         _hit=false
 821         if [ "$_op" != "%" ] ; then
 822             if [ $_numfails $_op $_li ] ; then
 823                 _hit=true
 824             fi
 825         else
 826             if [ $(($_numfails $_op $_li)) -eq 0 ] ; then
 827                 _hit=true
 828             fi
 829         fi
 830         if $_hit ; then
 831             _out=""
 832             _rc=0
 833             for _action in $_actions ; do
 834                 case "$_action" in
 835                     verbose)
 836                         _ver=1
 837                         _pn="$_progname"
 838                         case "$_progname" in
 839                             nfsd) _ver=3 ; _pn="nfs" ;;
 840                             lockd) _ver=4 ; _pn="nlockmgr" ;;
 841                             statd) _pn="status" ;;
 842                         esac
 843                         _out="\
 844 ERROR: $_pn failed RPC check:
 845 rpcinfo: RPC: Program not registered
 846 program $_pn version $_ver is not available"
 847                         ;;
 848                     restart*)
 849                         _p="rpc.${_progname}"
 850                         case "$_action" in
 851                             *:b) _bg="&" ;;
 852                             *)   _bg=""  ;;
 853                         esac
 854                         case "$_progname" in
 855                             nfsd)
 856                                 _t="\
 857 Trying to restart NFS service"
 858
 859                                 if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then
 860                                     for _pid in $FAKE_NFSD_THREAD_PIDS ; do
 861                                         _t="\
 862 $_t
 863 ${_bg}Stack trace for stuck nfsd thread [${_pid}]:
 864 ${_bg}[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff"
 865                                     done
 866                                 fi
 867
 868                                 _t="\
 869 ${_t}
 870 ${_bg}Starting nfslock: OK
 871 ${_bg}Starting nfs: OK"
 872                                 ;;
 873                             lockd)
 874                                 _t="\
 875 Trying to restart lock manager service
 876 ${_bg}Starting nfslock: OK"
 877                                 ;;
 878                             *)
 879                                 _t="Trying to restart $_progname [${_p}]"
 880                         esac
 881                         _out="${_out}${_out:+${_nl}}${_t}"
 882                         ;;
 883                     unhealthy)
 884                         _rc=1
 885                 esac
 886             done
 887             required_result $_rc "$_out"
 888             return
 889         fi
 890     done <"$_file"
 891 }
 892
 893 ######################################################################
 894
 895 # VSFTPD fakery
 896
 897 setup_vsftpd ()
 898 {
 899     service_name="vsftpd"
 900
 901     if [ "$1" != "down" ] ; then
 902         die "setup_vsftpd up not implemented!!!"
 903     else
 904         debug "Setting up VSFTPD environment: service down, not managed by CTDB"
 905
 906         eventscript_call ctdb_service_unmanaged
 907         service vsftpd force-stopped
 908
 909         export CTDB_MANAGED_SERVICES="foo"
 910         unset CTDB_MANAGES_VSFTPD
 911     fi
 912 }
 913
 914 ######################################################################
 915
 916 # HTTPD fakery
 917
 918 setup_httpd ()
 919 {
 920     if [ "$1" != "down" ] ; then
 921         die "setup_httpd up not implemented!!!"
 922     else
 923         debug "Setting up HTTPD environment: service down, not managed by CTDB"
 924
 925         for service_name in "apache2" "httpd" ; do
 926             eventscript_call ctdb_service_unmanaged
 927             service "$service_name" force-stopped
 928         done
 929
 930         export CTDB_MANAGED_SERVICES="foo"
 931         unset CTDB_MANAGES_HTTPD
 932     fi
 933 }
 934
 935 ######################################################################
 936
 937 # multipathd fakery
 938
 939 setup_multipathd ()
 940 {
 941     for i ; do
 942         case "$i" in
 943             \!*)
 944                 _t="${i#!}"
 945                 echo "Marking ${_t} as having no active paths"
 946                 FAKE_MULTIPATH_FAILURES="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}"
 947                 ;;
 948             *)
 949                 _t="$i"
 950         esac
 951         CTDB_MONITOR_MPDEVICES="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}"
 952     done
 953
 954     export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES
 955     export FAKE_SLEEP_FORCE=0.1
 956 }
 957
 958 ######################################################################
 959
 960 # Result and test functions
 961
 962 # Set some globals and print the summary.
 963 define_test ()
 964 {
 965     desc="$1"
 966
 967     _f=$(basename "$0" ".sh")
 968
 969     # Remaining format should be NN.service.event.NNN or NN.service.NNN:
 970     _num="${_f##*.}"
 971     _f="${_f%.*}"
 972     case "$_f" in
 973         *.*.*)
 974             script="${_f%.*}"
 975             event="${_f##*.}"
 976             ;;
 977         *.*)
 978             script="$_f"
 979             unset event
 980             ;;
 981         *)
 982             die "Internal error - unknown testcase filename format"
 983     esac
 984
 985     printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
 986 }
 987
 988 _extra_header ()
 989 {
 990     cat <<EOF
 991 CTDB_BASE="$CTDB_BASE"
 992 CTDB_ETCDIR="$CTDB_ETCDIR"
 993 ctdb client is "$(which ctdb)"
 994 ip command is "$(which ip)"
 995 EOF
 996 }
 997
 998 # Run an eventscript once.  The test passes if the return code and
 999 # output match those required.
1000
1001 # Any args are passed to the eventscript.
1002
1003 simple_test ()
1004 {
1005     [ -n "$event" ] || die 'simple_test: $event not set'
1006
1007     _extra_header=$(_extra_header)
1008
1009     echo "Running eventscript \"$script $event${1:+ }$*\""
1010     _shell=""
1011     if $TEST_COMMAND_TRACE ; then
1012         _shell="sh -x"
1013     else
1014         _shell="sh"
1015     fi
1016     _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
1017
1018     result_check "$_extra_header"
1019 }
1020
1021 simple_test_event ()
1022 {
1023     # If something has previously failed then don't continue.
1024     : ${_passed:=true}
1025     $_passed || return 1
1026
1027     event="$1" ; shift
1028     echo "=================================================="
1029     simple_test "$@"
1030 }
1031
1032 simple_test_command ()
1033 {
1034     # If something has previously failed then don't continue.
1035     : ${_passed:=true}
1036     $_passed || return 1
1037
1038     echo "=================================================="
1039     echo "Running command \"$*\""
1040     _out=$("$@" 2>&1)
1041
1042     result_check
1043 }
1044
1045 check_ctdb_logfile ()
1046 {
1047     # If something has previously failed then don't continue.
1048     : ${_passed:=true}
1049     $_passed || return 1
1050
1051     echo "=================================================="
1052     echo "Checking CTDB_LOGFILE=\"${CTDB_LOGFILE}\""
1053     _out=$(cat "$CTDB_LOGFILE" 2>&1)
1054
1055     result_check
1056 }
1057
1058 # Run an eventscript iteratively.
1059 # - 1st argument is the number of iterations.
1060 # - 2nd argument is something to eval to do setup for every iteration.
1061 #   The easiest thing to do here is to define a function and pass it
1062 #   here.
1063 # - Subsequent arguments come in pairs: an iteration number and
1064 #   something to eval for that iteration.  Each time an iteration
1065 #   number is matched the associated argument is given to eval after
1066 #   the default setup is done.  The iteration numbers need to be given
1067 #   in ascending order.
1068 #
1069 # Some optional args can be given *before* these, surrounded by extra
1070 # "--" args.  These args are passed to the eventscript.  Quoting is
1071 # lost.
1072 #
1073 # One use of the 2nd and further arguments is to call
1074 # required_result() to change what is expected of a particular
1075 # iteration.
1076 iterate_test ()
1077 {
1078     [ -n "$event" ] || die 'simple_test: $event not set'
1079
1080     args=""
1081     if [ "$1" = "--" ] ; then
1082         shift
1083         while [ "$1" != "--" ] ; do
1084             args="${args}${args:+ }$1"
1085             shift
1086         done
1087         shift
1088     fi
1089
1090     _repeats="$1"
1091     _setup_default="$2"
1092     shift 2
1093
1094     echo "Running $_repeats iterations of \"$script $event\" $args"
1095
1096     _result=true
1097
1098     for iteration in $(seq 1 $_repeats) ; do
1099         # This is inefficient because the iteration-specific setup
1100         # might completely replace the default one.  However, running
1101         # the default is good because it allows you to revert to a
1102         # particular result without needing to specify it explicitly.
1103         eval $_setup_default
1104         if [ $iteration = "$1" ] ; then
1105             eval $2
1106             shift 2
1107         fi
1108
1109         _shell=""
1110         if $TEST_COMMAND_TRACE ; then
1111             _shell="sh -x"
1112         else
1113             _shell="sh"
1114         fi
1115         _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1)
1116         _rc=$?
1117
1118         _fout=$(echo "$_out" | result_filter)
1119
1120         if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
1121             _passed=true
1122         else
1123             _passed=false
1124             _result=false
1125         fi
1126
1127         result_print "$_passed" "$_out" "$_rc" "Iteration $iteration"
1128     done
1129
1130     result_footer "$_result" "$(_extra_header)"
1131 }