ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh

   1 setup()
   2 {
   3         setup_public_addresses
   4         setup_shares
   5
   6         # shellcheck disable=SC2034
   7         # Used in expected output
   8         service_name="nfs"
   9
  10         if [ -z "$CTDB_NFS_DISTRO_STYLE" ]; then
  11                 # Currently supported: sysvinit-redhat, systemd-redhat
  12                 CTDB_NFS_DISTRO_STYLE="systemd-redhat"
  13         fi
  14
  15         export FAKE_RPCINFO_SERVICES=""
  16
  17         setup_script_options <<EOF
  18 CTDB_NFS_SKIP_SHARE_CHECK="no"
  19 # This doesn't even need to exist
  20 CTDB_NFS_EXPORTS_FILE="${CTDB_TEST_TMP_DIR}/etc-exports"
  21 EOF
  22
  23         export RPCNFSDCOUNT
  24
  25         if [ "$1" != "down" ]; then
  26                 debug <<EOF
  27 Setting up NFS environment: all RPC services up, NFS managed by CTDB
  28 EOF
  29
  30                 case "$CTDB_NFS_DISTRO_STYLE" in
  31                 sysvinit-*)
  32                         service "nfs" force-started
  33                         service "nfslock" force-started
  34                         ;;
  35                 systemd-*)
  36                         service "nfs-service" force-started
  37                         service "nfs-mountd" force-started
  38                         service "rpc-rquotad" force-started
  39                         service "rpc-statd" force-started
  40                         ;;
  41                 esac
  42
  43                 rpc_services_up \
  44                         "portmapper" "nfs" "mountd" "rquotad" \
  45                         "nlockmgr" "status"
  46
  47                 nfs_setup_fake_threads "nfsd"
  48                 nfs_setup_fake_threads "rpc.foobar" # Set the variable to empty
  49         else
  50                 debug <<EOF
  51 Setting up NFS environment: all RPC services down, NFS not managed by CTDB
  52 EOF
  53
  54                 case "$CTDB_NFS_DISTRO_STYLE" in
  55                 sysvinit-*)
  56                         service "nfs" force-stopped
  57                         service "nfslock" force-stopped
  58                         service "nfs-kernel-server" force-stopped
  59                         ;;
  60                 systemd-*)
  61                         service "nfs-server" force-stopped
  62                         service "nfs-mountd" force-stopped
  63                         service "rpc-quotad" force-stopped
  64                         service "rpc-statd" force-stopped
  65                         ;;
  66                 esac
  67         fi
  68
  69         # This is really nasty.  However, when we test NFS we don't
  70         # actually test statd-callout. If we leave it there then left
  71         # over, backgrounded instances of statd-callout will do
  72         # horrible things with the "ctdb ip" stub and cause the actual
  73         # statd-callout tests that follow to fail.
  74         rm "${CTDB_BASE}/statd-callout"
  75 }
  76
  77 rpc_services_down()
  78 {
  79         _out=""
  80         for _s in $FAKE_RPCINFO_SERVICES; do
  81                 for _i; do
  82                         if [ "$_i" = "${_s%%:*}" ]; then
  83                                 debug "Marking RPC service \"${_i}\" as UNAVAILABLE"
  84                                 continue 2
  85                         fi
  86                 done
  87                 _out="${_out}${_out:+ }${_s}"
  88         done
  89         FAKE_RPCINFO_SERVICES="$_out"
  90 }
  91
  92 rpc_services_up()
  93 {
  94         _out="$FAKE_RPCINFO_SERVICES"
  95         for _i; do
  96                 debug "Marking RPC service \"${_i}\" as available"
  97                 case "$_i" in
  98                 portmapper) _t="2:4" ;;
  99                 nfs) _t="2:3" ;;
 100                 mountd) _t="1:3" ;;
 101                 rquotad) _t="1:2" ;;
 102                 nlockmgr) _t="3:4" ;;
 103                 status) _t="1:1" ;;
 104                 *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
 105                 esac
 106
 107                 _out="${_out}${_out:+ }${_i}:${_t}"
 108         done
 109         export FAKE_RPCINFO_SERVICES="$_out"
 110 }
 111
 112 nfs_setup_fake_threads()
 113 {
 114         _prog="$1"
 115         shift
 116
 117         case "$_prog" in
 118         nfsd)
 119                 export PROCFS_PATH="${CTDB_TEST_TMP_DIR}/proc"
 120                 _threads="${PROCFS_PATH}/fs/nfsd/threads"
 121                 mkdir -p "$(dirname "$_threads")"
 122                 echo $# >"$_threads"
 123                 export FAKE_NFSD_THREAD_PIDS="$*"
 124                 ;;
 125         *)
 126                 export FAKE_RPC_THREAD_PIDS="$*"
 127                 ;;
 128         esac
 129 }
 130
 131 guess_output()
 132 {
 133         case "$1" in
 134         "${CTDB_NFS_CALLOUT} start nlockmgr")
 135                 case "$CTDB_NFS_DISTRO_STYLE" in
 136                 sysvinit-redhat)
 137                         echo "&Starting nfslock: OK"
 138                         ;;
 139                 sysvinit-debian)
 140                         cat <<EOF
 141 &Starting nfs-kernel-server: OK
 142 EOF
 143                         ;;
 144                 systemd-*)
 145                         echo "&Starting rpc-statd: OK"
 146                         ;;
 147                 esac
 148                 ;;
 149         "${CTDB_NFS_CALLOUT} start nfs")
 150                 case "$CTDB_NFS_DISTRO_STYLE" in
 151                 sysvinit-redhat)
 152                         cat <<EOF
 153 &Starting nfslock: OK
 154 &Starting nfs: OK
 155 EOF
 156                         ;;
 157                 sysvinit-debian)
 158                         cat <<EOF
 159 &Starting nfs-kernel-server: OK
 160 EOF
 161                         ;;
 162                 systemd-redhat)
 163                         cat <<EOF
 164 &Starting rpc-statd: OK
 165 &Starting nfs-server: OK
 166 &Starting rpc-rquotad: OK
 167 EOF
 168                         ;;
 169                 systemd-debian)
 170                         cat <<EOF
 171 &Starting rpc-statd: OK
 172 &Starting nfs-server: OK
 173 &Starting quotarpc: OK
 174 EOF
 175                         ;;
 176                 esac
 177                 ;;
 178         "${CTDB_NFS_CALLOUT} stop mountd")
 179                 case "$CTDB_NFS_DISTRO_STYLE" in
 180                 systemd-*)
 181                         echo "Stopping nfs-mountd: OK"
 182                         ;;
 183                 esac
 184                 ;;
 185         "${CTDB_NFS_CALLOUT} stop rquotad")
 186                 case "$CTDB_NFS_DISTRO_STYLE" in
 187                 systemd-redhat)
 188                         echo "Stopping rpc-rquotad: OK"
 189                         ;;
 190                 systemd-debian)
 191                         if service "quotarpc" status >/dev/null; then
 192                                 echo "Stopping quotarpc: OK"
 193                         else
 194                                 echo "service: can't stop quotarpc - not running"
 195                         fi
 196                         ;;
 197                 esac
 198                 ;;
 199         "${CTDB_NFS_CALLOUT} stop status")
 200                 case "$CTDB_NFS_DISTRO_STYLE" in
 201                 systemd-*)
 202                         echo "Stopping rpc-statd: OK"
 203                         ;;
 204                 esac
 205                 ;;
 206         "${CTDB_NFS_CALLOUT} start mountd")
 207                 case "$CTDB_NFS_DISTRO_STYLE" in
 208                 systemd-*)
 209                         echo "&Starting nfs-mountd: OK"
 210                         ;;
 211                 esac
 212                 ;;
 213         "${CTDB_NFS_CALLOUT} start rquotad")
 214                 case "$CTDB_NFS_DISTRO_STYLE" in
 215                 systemd-redhat)
 216                         echo "&Starting rpc-rquotad: OK"
 217                         ;;
 218                 systemd-debian)
 219                         echo "&Starting quotarpc: OK"
 220                         ;;
 221                 esac
 222                 ;;
 223         "${CTDB_NFS_CALLOUT} start status")
 224                 case "$CTDB_NFS_DISTRO_STYLE" in
 225                 systemd-*)
 226                         echo "&Starting rpc-statd: OK"
 227                         ;;
 228                 esac
 229                 ;;
 230         *)
 231                 : # Nothing
 232                 ;;
 233         esac
 234 }
 235
 236 # Set the required result for a particular RPC program having failed
 237 # for a certain number of iterations.  This is probably still a work
 238 # in progress.  Note that we could hook aggressively
 239 # nfs_check_rpc_service() to try to implement this but we're better
 240 # off testing nfs_check_rpc_service() using independent code...  even
 241 # if it is incomplete and hacky.  So, if the 60.nfs eventscript
 242 # changes and the tests start to fail then it may be due to this
 243 # function being incomplete.
 244 rpc_set_service_failure_response()
 245 {
 246         _rpc_service="$1"
 247         _numfails="${2:-1}" # default 1
 248
 249         # Default
 250         ok_null
 251         if [ "$_numfails" -eq 0 ]; then
 252                 return
 253         fi
 254
 255         nfs_load_config
 256
 257         # A handy newline.  :-)
 258         _nl="
 259 "
 260
 261         _dir="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}"
 262
 263         _file=$(ls "$_dir"/[0-9][0-9]."${_rpc_service}.check")
 264         [ -r "$_file" ] ||
 265                 die "RPC check file \"$_file\" does not exist or is not unique"
 266
 267         _out="${CTDB_TEST_TMP_DIR}/rpc_failure_output"
 268         : >"$_out"
 269         _rc_file="${CTDB_TEST_TMP_DIR}/rpc_result"
 270
 271         (
 272                 # Subshell to restrict scope variables...
 273
 274                 # Defaults
 275                 # shellcheck disable=SC2034
 276                 # Unused, but for completeness, possible future use
 277                 family="tcp"
 278                 version=""
 279                 unhealthy_after=1
 280                 restart_every=0
 281                 service_stop_cmd=""
 282                 service_start_cmd=""
 283                 # shellcheck disable=SC2034
 284                 # Unused, but for completeness, possible future use
 285                 service_check_cmd=""
 286                 service_debug_cmd=""
 287
 288                 # Don't bother syntax checking, eventscript does that...
 289                 . "$_file"
 290
 291                 # Just use the first version, or use default.  This is
 292                 # dumb but handles all the cases that we care about
 293                 # now...
 294                 if [ -n "$version" ]; then
 295                         _ver="${version%% *}"
 296                 else
 297                         case "$_rpc_service" in
 298                         portmapper) _ver="" ;;
 299                         *) _ver=1 ;;
 300                         esac
 301                 fi
 302                 _rpc_check_out="\
 303 $_rpc_service failed RPC check:
 304 rpcinfo: RPC: Program not registered
 305 program $_rpc_service${_ver:+ version }${_ver} is not available"
 306
 307                 if [ $unhealthy_after -gt 0 ] &&
 308                         [ "$_numfails" -ge $unhealthy_after ]; then
 309                         _unhealthy=true
 310                         echo 1 >"$_rc_file"
 311                         echo "ERROR: ${_rpc_check_out}" >>"$_out"
 312                 else
 313                         _unhealthy=false
 314                         echo 0 >"$_rc_file"
 315                 fi
 316
 317                 if [ $restart_every -gt 0 ] &&
 318                         [ $((_numfails % restart_every)) -eq 0 ]; then
 319                         if ! $_unhealthy; then
 320                                 echo "WARNING: ${_rpc_check_out}" >>"$_out"
 321                         fi
 322
 323                         echo "Trying to restart service \"${_rpc_service}\"..." \
 324                                 >>"$_out"
 325
 326                         guess_output "$service_stop_cmd" >>"$_out"
 327
 328                         if [ -n "$service_debug_cmd" ]; then
 329                                 $service_debug_cmd >>"$_out" 2>&1
 330                         fi
 331
 332                         guess_output "$service_start_cmd" >>"$_out"
 333                 fi
 334         )
 335
 336         read -r _rc <"$_rc_file"
 337         required_result "$_rc" <"$_out"
 338
 339         rm -f "$_out" "$_rc_file"
 340 }
 341
 342 program_stack_traces()
 343 {
 344         _prog="$1"
 345         _max="${2:-1}"
 346
 347         _count=1
 348         if [ "$_prog" = "nfsd" ]; then
 349                 _pids="$FAKE_NFSD_THREAD_PIDS"
 350         else
 351                 _pids="$FAKE_RPC_THREAD_PIDS"
 352         fi
 353         for _pid in $_pids; do
 354                 [ $_count -le "$_max" ] || break
 355
 356                 program_stack_trace "$_prog" "$_pid"
 357                 _count=$((_count + 1))
 358         done
 359 }
 360
 361 # Run an NFS eventscript iteratively.
 362 #
 363 # - 1st argument is the number of iterations.
 364 #
 365 # - 2nd argument is the NFS/RPC service being tested
 366 #
 367 #   rpcinfo is used on each iteration to test the availability of the
 368 #   service
 369 #
 370 #   If this is not set or null then no RPC service is checked and the
 371 #   required output is not reset on each iteration.  This is useful in
 372 #   baseline tests to confirm that the eventscript and test
 373 #   infrastructure is working correctly.
 374 #
 375 # - Subsequent arguments come in pairs: an iteration number and
 376 #   something to eval before that iteration.  Each time an iteration
 377 #   number is matched the associated argument is given to eval after
 378 #   the default setup is done.  The iteration numbers need to be given
 379 #   in ascending order.
 380 #
 381 #   These arguments can allow a service to be started or stopped
 382 #   before a particular iteration.
 383 #
 384 nfs_iterate_test()
 385 {
 386         _repeats="$1"
 387         _rpc_service="$2"
 388         if [ -n "$2" ]; then
 389                 shift 2
 390         else
 391                 shift
 392         fi
 393
 394         # shellcheck disable=SC2154
 395         # Variables defined in define_test()
 396         echo "Running $_repeats iterations of \"$script $event\" $args"
 397
 398         _iterate_failcount=0
 399         for _iteration in $(seq 1 "$_repeats"); do
 400                 # This is not a numerical comparison because $1 will
 401                 # often not be set.
 402                 if [ "$_iteration" = "$1" ]; then
 403                         debug <<EOF
 404 ##################################################
 405 EOF
 406                         eval "$2"
 407                         debug <<EOF
 408 ##################################################
 409 EOF
 410                         shift 2
 411                 fi
 412                 if [ -n "$_rpc_service" ]; then
 413                         if rpcinfo -T tcp localhost "$_rpc_service" \
 414                                    >/dev/null 2>&1 ; then
 415                                 _iterate_failcount=0
 416                         else
 417                                 _iterate_failcount=$((_iterate_failcount + 1))
 418                         fi
 419                         rpc_set_service_failure_response \
 420                                 "$_rpc_service" $_iterate_failcount
 421                 fi
 422                 _out=$(simple_test 2>&1)
 423                 _ret=$?
 424                 if "$CTDB_TEST_VERBOSE" || [ $_ret -ne 0 ]; then
 425                         cat <<EOF
 426 ##################################################
 427 Iteration ${_iteration}:
 428 $_out
 429 EOF
 430                 fi
 431                 if [ $_ret -ne 0 ]; then
 432                         exit $_ret
 433                 fi
 434         done
 435 }