ctdb/config/statd-callout

   1 #!/bin/sh
   2
   3 # This must run as root as CTDB tool commands need to access CTDB socket
   4 [ "$(id -u)" -eq 0 ] || exec sudo "$0" "$@"
   5
   6 # this script needs to be installed so that statd points to it with the -H
   7 # command line argument. The easiest way to do that is to put something like this in
   8 # /etc/sysconfig/nfs:
   9 #   STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
  10
  11 [ -n "$CTDB_BASE" ] || \
  12     CTDB_BASE=$(d=$(dirname "$0") ; cd -P "$d" ; echo "$PWD")
  13
  14 . "${CTDB_BASE}/functions"
  15
  16 # Overwrite this so we get some logging
  17 die ()
  18 {
  19     script_log "statd-callout" "$@"
  20     exit 1
  21 }
  22
  23 loadconfig ctdb
  24 loadconfig nfs
  25
  26 [ -n "$NFS_HOSTNAME" ] || \
  27     die "NFS_HOSTNAME is not configured. statd-callout failed"
  28
  29 # A handy newline
  30 nl="
  31 "
  32
  33 service_state_dir=$(ctdb_setup_service_state_dir "statd-callout") || exit $?
  34
  35 cd "$service_state_dir" || \
  36     die "Failed to change directory to \"${service_state_dir}\""
  37
  38 pnn=$(ctdb_get_pnn)
  39
  40 case "$1" in
  41     # Keep a single file to keep track of the last "add-client" or
  42     # "del-client'.  These get pushed to ctdb.tdb during "update",
  43     # which will generally be run once each "monitor" cycle.  In this
  44     # way we avoid scalability problems with flood of persistent
  45     # transactions after a "notify" when all the clients re-take their
  46     # locks.
  47
  48     add-client)
  49         # statd does not tell us to which IP the client connected so
  50         # we must add it to all the IPs that we serve
  51         cip="$2"
  52         date=$(date '+%s')
  53         # x is intentionally ignored
  54         # shellcheck disable=SC2034
  55         $CTDB ip -X |
  56         tail -n +2 |
  57         while IFS="|" read x sip node x ; do
  58             [ "$node" = "$pnn" ] || continue # not us
  59             key="statd-state@${sip}@${cip}"
  60             echo "\"${key}\" \"${date}\"" >"$key"
  61         done
  62         ;;
  63
  64     del-client)
  65         # statd does not tell us from which IP the client disconnected
  66         # so we must add it to all the IPs that we serve
  67         cip="$2"
  68         # x is intentionally ignored
  69         # shellcheck disable=SC2034
  70         $CTDB ip -X |
  71         tail -n +2 |
  72         while IFS="|" read x sip node x ; do
  73             [ "$node" = "$pnn" ] || continue # not us
  74             key="statd-state@${sip}@${cip}"
  75             echo "\"${key}\" \"\"" >"$key"
  76         done
  77         ;;
  78
  79     update)
  80         files=$(echo statd-state@*)
  81         if [ "$files" = "statd-state@*" ] ; then
  82             # No files!
  83             exit 0
  84         fi
  85         # Filter out lines for any IP addresses that are not currently
  86         # hosted public IP addresses.
  87         ctdb_ips=$($CTDB ip | tail -n +2)
  88         sed_expr=$(echo "$ctdb_ips" |
  89             awk -v pnn="$pnn" 'pnn == $2 {
  90                 ip = $1; gsub(/\./, "\\.", ip);
  91                 printf "/statd-state@%s@/p\n", ip }')
  92         # Intentional multi-word expansion for multiple files
  93         # shellcheck disable=SC2086
  94         if sed -n "$sed_expr" $files | $CTDB ptrans "ctdb.tdb" ; then
  95             rm $files
  96         fi
  97         ;;
  98
  99     notify)
 100         # we must restart the lockmanager (on all nodes) so that we get
 101         # a clusterwide grace period (so other clients don't take out
 102         # conflicting locks through other nodes before all locks have been
 103         # reclaimed)
 104
 105         # we need these settings to make sure that no tcp connections survive
 106         # across a very fast failover/failback
 107         #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
 108         #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
 109         #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
 110
 111         # Delete the notification list for statd, we don't want it to
 112         # ping any clients
 113         rm -f /var/lib/nfs/statd/sm/*
 114         rm -f /var/lib/nfs/statd/sm.bak/*
 115
 116         # we must keep a monotonically increasing state variable for the entire
 117         # cluster  so state always increases when ip addresses fail from one
 118         # node to another
 119         # We use epoch and hope the nodes are close enough in clock.
 120         # Even numbers mean service is shut down, odd numbers mean
 121         # service is started.
 122         # Intentionally round to an even number
 123         # shellcheck disable=SC2017
 124         state_even=$(( $(date '+%s') / 2 * 2))
 125
 126         # We must also let some time pass between stopping and
 127         # restarting the lock manager.  Otherwise there is a window
 128         # where the lock manager will respond "strangely" immediately
 129         # after restarting it, which causes clients to fail to reclaim
 130         # their locks.
 131         nfs_callout_init
 132         "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
 133         sleep 2
 134         "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
 135
 136         # we now need to send out additional statd notifications to ensure
 137         # that clients understand that the lockmanager has restarted.
 138         # we have three cases:
 139         # 1, clients that ignore the ip address the stat notification came from
 140         #    and ONLY care about the 'name' in the notify packet.
 141         #    these clients ONLY work with lock failover IFF that name
 142         #    can be resolved into an ipaddress that matches the one used
 143         #    to mount the share.  (==linux clients)
 144         #    This is handled when starting lockmanager above,  but those
 145         #    packets are sent from the "wrong" ip address, something linux
 146         #    clients are ok with, buth other clients will barf at.
 147         # 2, Some clients only accept statd packets IFF they come from the
 148         #    'correct' ip address.
 149         # 2a,Send out the notification using the 'correct' ip address and also
 150         #    specify the 'correct' hostname in the statd packet.
 151         #    Some clients require both the correct source address and also the
 152         #    correct name. (these clients also ONLY work if the ip addresses
 153         #    used to map the share can be resolved into the name returned in
 154         #    the notify packet.)
 155         # 2b,Other clients require that the source ip address of the notify
 156         #    packet matches the ip address used to take out the lock.
 157         #    I.e. that the correct source address is used.
 158         #    These clients also require that the statd notify packet contains
 159         #    the name as the ip address used when the lock was taken out.
 160         #
 161         # Both 2a and 2b are commonly used in lockmanagers since they maximize
 162         # probability that the client will accept the statd notify packet and
 163         # not just ignore it.
 164         # For all IPs we serve, collect info and push to the config database
 165
 166         # Construct a sed expression to take catdb output and produce pairs of:
 167         #   server-IP client-IP
 168         # but only for the server-IPs that are hosted on this node.
 169         ctdb_all_ips=$($CTDB ip all | tail -n +2)
 170         sed_expr=$(echo "$ctdb_all_ips" |
 171             awk -v pnn="$pnn" 'pnn == $2 {
 172                 ip = $1; gsub(/\./, "\\.", ip);
 173                 printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
 174
 175         statd_state=$($CTDB catdb ctdb.tdb | sed -n "$sed_expr" | sort)
 176         [ -n "$statd_state" ] || exit 0
 177
 178         smnotify="${CTDB_HELPER_BINDIR}/smnotify"
 179         prev=""
 180         echo "$statd_state" | {
 181             # This all needs to be in the same command group at the
 182             # end of the pipe so it doesn't get lost when the loop
 183             # completes.
 184             items=""
 185             while read sip cip ; do
 186                 # Collect item to delete from the DB
 187                 key="statd-state@${sip}@${cip}"
 188                 item="\"${key}\" \"\""
 189                 items="${items}${items:+${nl}}${item}"
 190
 191                 # NOTE: Consider optimising smnotify to read all the
 192                 # data from stdin and then run it in the background.
 193
 194                 # Reset stateval for each serverip
 195                 [ "$sip" = "$prev" ] || stateval="$state_even"
 196                 # Send notifies for server shutdown
 197                 "$smnotify" --client="$cip" --ip="$sip" \
 198                             --server="$sip" --stateval="$stateval"
 199                 "$smnotify" --client="$cip" --ip="$sip" \
 200                             --server="$NFS_HOSTNAME" --stateval="$stateval"
 201                 # Send notifies for server startup
 202                 stateval=$((stateval + 1))
 203                 "$smnotify" --client="$cip" --ip="$sip" \
 204                             --server="$sip" --stateval="$stateval"
 205                 "$smnotify" --client="$cip" --ip="$sip" \
 206                             --server="$NFS_HOSTNAME" --stateval="$stateval"
 207             done
 208
 209             echo "$items" | $CTDB ptrans "ctdb.tdb"
 210         }
 211
 212         # Remove any stale touch files (i.e. for IPs not currently
 213         # hosted on this node and created since the last "update").
 214         # There's nothing else we can do with them at this stage.
 215         echo "$ctdb_all_ips" |
 216             awk -v pnn="$pnn" 'pnn != $2 { print $1 }' |
 217             while read sip ; do
 218                 rm -f "statd-state@${sip}@"*
 219             done
 220         ;;
 221 esac