ctdb/utils/ceph/test_ceph_rados_reclock.sh

   1 #!/bin/bash
   2 # standalone test for ctdb_mutex_ceph_rados_helper
   3 #
   4 # Copyright (C) David Disseldorp 2016-2020
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, see <http://www.gnu.org/licenses/>.
  18
  19 # XXX The following parameters may require configuration:
  20 CLUSTER="ceph"                          # Name of the Ceph cluster under test
  21 USER="client.admin"                     # Ceph user - a keyring must exist
  22 POOL="rbd"                              # RADOS pool - must exist
  23 OBJECT="ctdb_reclock"                   # RADOS object: target for lock requests
  24
  25 # test procedure:
  26 # - using ctdb_mutex_ceph_rados_helper, take a lock on the Ceph RADOS object at
  27 #   CLUSTER/$POOL/$OBJECT using the Ceph keyring for $USER
  28 #   + confirm that lock is obtained, via ctdb_mutex_ceph_rados_helper "0" output
  29 # - check for ceph-mgr service registration
  30 # - check RADOS object lock state, using the "rados lock info" command
  31 # - attempt to obtain the lock again, using ctdb_mutex_ceph_rados_helper
  32 #   + confirm that the lock is not successfully taken ("1" output=contention)
  33 # - tell the first locker to drop the lock and exit, via SIGTERM
  34 # - once the first locker has exited, attempt to get the lock again
  35 #   + confirm that this attempt succeeds
  36
  37 function _fail() {
  38         echo "FAILED: $*"
  39         exit 1
  40 }
  41
  42 # this test requires the Ceph "rados" binary, and "jq" json parser
  43 which jq > /dev/null || exit 1
  44 which rados > /dev/null || exit 1
  45 which ceph > /dev/null || exit 1
  46 which ctdb_mutex_ceph_rados_helper || exit 1
  47
  48 TMP_DIR="$(mktemp --directory)" || exit 1
  49 rados -p "$POOL" rm "$OBJECT"
  50
  51 # explicitly disable lock expiry (duration=0), to ensure that we don't get
  52 # intermittent failures (due to renewal) from the lock state diff further down
  53 (ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" 0 \
  54                                                         > ${TMP_DIR}/first) &
  55 locker_pid=$!
  56
  57 # TODO wait for ctdb_mutex_ceph_rados_helper to write one byte to stdout,
  58 # indicating lock acquisition success/failure
  59 sleep 1
  60
  61 first_out=$(cat ${TMP_DIR}/first)
  62 [ "$first_out" == "0" ] \
  63         || _fail "expected lock acquisition (0), but got $first_out"
  64
  65 ceph service dump > ${TMP_DIR}/service_dump
  66 SERVICE_COUNT=$(jq -r '.services.ctdb.daemons | length' ${TMP_DIR}/service_dump)
  67 [ $SERVICE_COUNT -gt 0 ] || _fail "lock holder missing from ceph service dump"
  68
  69 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
  70                                                 > ${TMP_DIR}/lock_state_first
  71
  72 # echo "with lock: `cat ${TMP_DIR}/lock_state_first`"
  73
  74 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_first)"
  75 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
  76         || _fail "unexpected lock name: $LOCK_NAME"
  77 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_first)"
  78 [ "$LOCK_TYPE" == "exclusive" ] \
  79         || _fail "unexpected lock type: $LOCK_TYPE"
  80
  81 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_first)"
  82 [ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT"
  83 LOCKER_COOKIE="$(jq -r '.lockers[0].cookie' ${TMP_DIR}/lock_state_first)"
  84 [ "$LOCKER_COOKIE" == "ctdb_reclock_mutex" ] \
  85         || _fail "unexpected locker cookie: $LOCKER_COOKIE"
  86 LOCKER_DESC="$(jq -r '.lockers[0].description' ${TMP_DIR}/lock_state_first)"
  87 [ "$LOCKER_DESC" == "CTDB cluster lock" ] \
  88         || _fail "unexpected locker description: $LOCKER_DESC"
  89 LOCKER_EXP="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_first)"
  90 [ "$LOCKER_EXP" == "0.000000" ] \
  91         || _fail "unexpected locker expiration: $LOCKER_EXP"
  92
  93 # second attempt while first is still holding the lock - expect failure
  94 ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" \
  95                                                         > ${TMP_DIR}/second
  96 second_out=$(cat ${TMP_DIR}/second)
  97 [ "$second_out" == "1" ] \
  98         || _fail "expected lock contention (1), but got $second_out"
  99
 100 # confirm lock state didn't change
 101 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
 102                                                 > ${TMP_DIR}/lock_state_second
 103
 104 diff ${TMP_DIR}/lock_state_first ${TMP_DIR}/lock_state_second \
 105                                         || _fail "unexpected lock state change"
 106
 107 # tell first locker to drop the lock and terminate
 108 kill $locker_pid || exit 1
 109
 110 wait $locker_pid &> /dev/null
 111
 112 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
 113                                                 > ${TMP_DIR}/lock_state_third
 114 # echo "without lock: `cat ${TMP_DIR}/lock_state_third`"
 115
 116 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_third)"
 117 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
 118         || _fail "unexpected lock name: $LOCK_NAME"
 119 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_third)"
 120 [ "$LOCK_TYPE" == "exclusive" ] \
 121         || _fail "unexpected lock type: $LOCK_TYPE"
 122
 123 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_third)"
 124 [ $LOCK_COUNT -eq 0 ] \
 125         || _fail "didn\'t expect any locks in rados state, got $LOCK_COUNT"
 126
 127 exec >${TMP_DIR}/third -- ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" &
 128 locker_pid=$!
 129
 130 sleep 1
 131
 132 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
 133                                                 > ${TMP_DIR}/lock_state_fourth
 134 # echo "with lock again: `cat ${TMP_DIR}/lock_state_fourth`"
 135
 136 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_fourth)"
 137 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
 138         || _fail "unexpected lock name: $LOCK_NAME"
 139 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_fourth)"
 140 [ "$LOCK_TYPE" == "exclusive" ] \
 141         || _fail "unexpected lock type: $LOCK_TYPE"
 142
 143 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_fourth)"
 144 [ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT"
 145 LOCKER_COOKIE="$(jq -r '.lockers[0].cookie' ${TMP_DIR}/lock_state_fourth)"
 146 [ "$LOCKER_COOKIE" == "ctdb_reclock_mutex" ] \
 147         || _fail "unexpected locker cookie: $LOCKER_COOKIE"
 148 LOCKER_DESC="$(jq -r '.lockers[0].description' ${TMP_DIR}/lock_state_fourth)"
 149 [ "$LOCKER_DESC" == "CTDB cluster lock" ] \
 150         || _fail "unexpected locker description: $LOCKER_DESC"
 151
 152 kill $locker_pid || exit 1
 153 wait $locker_pid &> /dev/null
 154
 155 third_out=$(cat ${TMP_DIR}/third)
 156 [ "$third_out" == "0" ] \
 157         || _fail "expected lock acquisition (0), but got $third_out"
 158
 159 # test renew / expire behaviour using a 1s expiry (update period = 500ms)
 160 exec >${TMP_DIR}/forth -- ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" \
 161                                                         "$POOL" "$OBJECT" 1 &
 162 locker_pid=$!
 163
 164 sleep 1
 165
 166 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
 167                                                 > ${TMP_DIR}/lock_state_fifth_a
 168 #echo "with lock fifth: `cat ${TMP_DIR}/lock_state_fifth_a`"
 169
 170 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_fifth_a)"
 171 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
 172         || _fail "unexpected lock name: $LOCK_NAME"
 173 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_fifth_a)"
 174 [ "$LOCK_TYPE" == "exclusive" ] \
 175         || _fail "unexpected lock type: $LOCK_TYPE"
 176 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_fifth_a)"
 177 [ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT"
 178 LOCKER_EXP_A="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_a)"
 179 [ "$LOCKER_EXP_A" != "0.000000" ] \
 180         || _fail "unexpected locker expiration: $LOCKER_EXP_A"
 181 sleep 1 # sleep until renewal
 182 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
 183                                                 > ${TMP_DIR}/lock_state_fifth_b
 184 LOCKER_EXP_B="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_b)"
 185 [ "$LOCKER_EXP_B" != "0.000000" ] \
 186         || _fail "unexpected locker expiration: $LOCKER_EXP_B"
 187 #echo "lock expiration before renewal $LOCKER_EXP_A, after renewal $LOCKER_EXP_B"
 188 [ "$LOCKER_EXP_B" != "$LOCKER_EXP_A" ] \
 189         || _fail "locker expiration matches: $LOCKER_EXP_B"
 190
 191 # no chance to drop the lock, rely on expiry
 192 kill -KILL $locker_pid || exit 1
 193 wait $locker_pid &> /dev/null
 194 sleep 1 # sleep until lock expiry
 195
 196 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
 197                                                 > ${TMP_DIR}/lock_state_sixth
 198 #echo "lock expiry sixth: `cat ${TMP_DIR}/lock_state_sixth`"
 199
 200 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_sixth)"
 201 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
 202         || _fail "unexpected lock name: $LOCK_NAME"
 203 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_sixth)"
 204 [ "$LOCK_TYPE" == "exclusive" ] \
 205         || _fail "unexpected lock type: $LOCK_TYPE"
 206 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_sixth)"
 207 [ $LOCK_COUNT -eq 0 ] || _fail "expected 0 locks in rados state, got $LOCK_COUNT"
 208
 209 rm ${TMP_DIR}/*
 210 rmdir $TMP_DIR
 211
 212 echo "$0: all tests passed"