ctdb/config/statd-callout

   1 #!/bin/sh
   2
   3 # This must run as root as CTDB tool commands need to access CTDB socket
   4 [ "$(id -u)" -eq 0 ] || exec sudo "$0" "$@"
   5
   6 # statd must be configured to use this script as its high availability call-out.
   7 #
   8 # In most Linux versions this can be done using something like the following...
   9 #
  10 # /etc/sysconfig/nfs (Red Hat) or /etc/default/nfs-common (Debian):
  11 #   NFS_HOSTNAME=myhostname
  12 #   STATD_HOSTNAME="${NFS_HOSTNAME} -H /etc/ctdb/statd-callout"
  13 #
  14 # Newer Red Hat Linux variants instead use /etc/nfs.conf:
  15 #   [statd]
  16 #     name = myhostname
  17 #     ha-callout = /etc/ctdb/statd-callout
  18
  19 [ -n "$CTDB_BASE" ] || \
  20         CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && echo "$PWD")
  21
  22 . "${CTDB_BASE}/functions"
  23
  24 # Overwrite this so we get some logging
  25 die ()
  26 {
  27     script_log "statd-callout" "$@"
  28     exit 1
  29 }
  30
  31 # Try different variables to find config file for NFS_HOSTNAME
  32 load_system_config "nfs" "nfs-common"
  33
  34 # If NFS_HOSTNAME not set then try to pull it out of /etc/nfs.conf
  35 if [ -z "$NFS_HOSTNAME" ]; then
  36         if type nfsconf >/dev/null 2>&1; then
  37                 NFS_HOSTNAME=$(nfsconf --get statd name)
  38         elif type git >/dev/null 2>&1; then
  39                 # git to the rescue!
  40                 NFS_HOSTNAME=$(git config --file=/etc/nfs.conf statd.name)
  41         fi
  42 fi
  43
  44 [ -n "$NFS_HOSTNAME" ] || \
  45     die "NFS_HOSTNAME is not configured. statd-callout failed"
  46
  47 ############################################################
  48
  49 ctdb_setup_state_dir "service" "nfs"
  50
  51 # script_state_dir set by ctdb_setup_state_dir()
  52 # shellcheck disable=SC2154
  53 d="${script_state_dir}/statd-callout"
  54
  55 mkdir -p "$d" || die "Failed to create directory \"${d}\""
  56 cd "$d" || die "Failed to change directory to \"${d}\""
  57
  58 pnn=$(ctdb_get_pnn)
  59
  60 ############################################################
  61
  62 send_notifies ()
  63 {
  64         _smnotify="${CTDB_HELPER_BINDIR}/smnotify"
  65
  66         # State must monotonically increase, across the entire
  67         # cluster.  Use seconds since epoch and hope the time is in
  68         # sync across nodes.  Even numbers mean service is shut down,
  69         # odd numbers mean service is started.
  70
  71         # Intentionally round to an even number
  72         # shellcheck disable=SC2017
  73         _state_even=$(( $(date '+%s') / 2 * 2))
  74
  75         _prev=""
  76         while read _sip _cip ; do
  77                 # NOTE: Consider optimising smnotify to read all the
  78                 # data from stdin and then run it in the background.
  79
  80                 # Reset stateval for each serverip
  81                 if [ "$_sip" != "$_prev" ] ; then
  82                         _stateval="$_state_even"
  83                 fi
  84
  85                 # Send notifies for server shutdown
  86                 "$_smnotify" --client="$_cip" --ip="$_sip" \
  87                              --server="$_sip" --stateval="$_stateval"
  88                 "$_smnotify" --client="$_cip" --ip="$_sip" \
  89                              --server="$NFS_HOSTNAME" --stateval="$_stateval"
  90
  91                 # Send notifies for server startup
  92                 _stateval=$((_stateval + 1))
  93                 "$_smnotify" --client="$_cip" --ip="$_sip" \
  94                              --server="$_sip" --stateval="$_stateval"
  95                 "$_smnotify" --client="$_cip" --ip="$_sip" \
  96                              --server="$NFS_HOSTNAME" --stateval="$_stateval"
  97         done
  98 }
  99
 100 delete_records ()
 101 {
 102         while read _sip _cip ; do
 103                 _key="statd-state@${_sip}@${_cip}"
 104                 echo "\"${_key}\" \"\""
 105         done | $CTDB ptrans "ctdb.tdb"
 106 }
 107
 108 ############################################################
 109
 110 case "$1" in
 111     # Keep a single file to keep track of the last "add-client" or
 112     # "del-client'.  These get pushed to ctdb.tdb during "update",
 113     # which will generally be run once each "monitor" cycle.  In this
 114     # way we avoid scalability problems with flood of persistent
 115     # transactions after a "notify" when all the clients re-take their
 116     # locks.
 117
 118     add-client)
 119         # statd does not tell us to which IP the client connected so
 120         # we must add it to all the IPs that we serve
 121         cip="$2"
 122         date=$(date '+%s')
 123         # x is intentionally ignored
 124         # shellcheck disable=SC2034
 125         $CTDB ip -X |
 126         tail -n +2 |
 127         while IFS="|" read x sip node x ; do
 128             [ "$node" = "$pnn" ] || continue # not us
 129             key="statd-state@${sip}@${cip}"
 130             echo "\"${key}\" \"${date}\"" >"$key"
 131         done
 132         ;;
 133
 134     del-client)
 135         # statd does not tell us from which IP the client disconnected
 136         # so we must add it to all the IPs that we serve
 137         cip="$2"
 138         # x is intentionally ignored
 139         # shellcheck disable=SC2034
 140         $CTDB ip -X |
 141         tail -n +2 |
 142         while IFS="|" read x sip node x ; do
 143             [ "$node" = "$pnn" ] || continue # not us
 144             key="statd-state@${sip}@${cip}"
 145             echo "\"${key}\" \"\"" >"$key"
 146         done
 147         ;;
 148
 149     update)
 150         files=$(echo statd-state@*)
 151         if [ "$files" = "statd-state@*" ] ; then
 152             # No files!
 153             exit 0
 154         fi
 155         # Filter out lines for any IP addresses that are not currently
 156         # hosted public IP addresses.
 157         ctdb_ips=$($CTDB ip | tail -n +2)
 158         sed_expr=$(echo "$ctdb_ips" |
 159             awk -v pnn="$pnn" 'pnn == $2 {
 160                 ip = $1; gsub(/\./, "\\.", ip);
 161                 printf "/statd-state@%s@/p\n", ip }')
 162         # Intentional multi-word expansion for multiple files
 163         # shellcheck disable=SC2086
 164         items=$(sed -n "$sed_expr" $files)
 165         if [ -n "$items" ] ; then
 166                 if echo "$items"  | $CTDB ptrans "ctdb.tdb" ; then
 167                         # shellcheck disable=SC2086
 168                         rm $files
 169                 fi
 170         fi
 171         ;;
 172
 173     notify)
 174         # we must restart the lockmanager (on all nodes) so that we get
 175         # a clusterwide grace period (so other clients don't take out
 176         # conflicting locks through other nodes before all locks have been
 177         # reclaimed)
 178
 179         # we need these settings to make sure that no tcp connections survive
 180         # across a very fast failover/failback
 181         #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
 182         #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
 183         #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
 184
 185         # Delete the notification list for statd, we don't want it to
 186         # ping any clients
 187         rm -f /var/lib/nfs/statd/sm/*
 188         rm -f /var/lib/nfs/statd/sm.bak/*
 189
 190         # We must also let some time pass between stopping and
 191         # restarting the lock manager.  Otherwise there is a window
 192         # where the lock manager will respond "strangely" immediately
 193         # after restarting it, which causes clients to fail to reclaim
 194         # their locks.
 195         nfs_callout_init
 196         "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
 197         sleep 2
 198         "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
 199
 200         # we now need to send out additional statd notifications to ensure
 201         # that clients understand that the lockmanager has restarted.
 202         # we have three cases:
 203         # 1, clients that ignore the ip address the stat notification came from
 204         #    and ONLY care about the 'name' in the notify packet.
 205         #    these clients ONLY work with lock failover IFF that name
 206         #    can be resolved into an ipaddress that matches the one used
 207         #    to mount the share.  (==linux clients)
 208         #    This is handled when starting lockmanager above,  but those
 209         #    packets are sent from the "wrong" ip address, something linux
 210         #    clients are ok with, buth other clients will barf at.
 211         # 2, Some clients only accept statd packets IFF they come from the
 212         #    'correct' ip address.
 213         # 2a,Send out the notification using the 'correct' ip address and also
 214         #    specify the 'correct' hostname in the statd packet.
 215         #    Some clients require both the correct source address and also the
 216         #    correct name. (these clients also ONLY work if the ip addresses
 217         #    used to map the share can be resolved into the name returned in
 218         #    the notify packet.)
 219         # 2b,Other clients require that the source ip address of the notify
 220         #    packet matches the ip address used to take out the lock.
 221         #    I.e. that the correct source address is used.
 222         #    These clients also require that the statd notify packet contains
 223         #    the name as the ip address used when the lock was taken out.
 224         #
 225         # Both 2a and 2b are commonly used in lockmanagers since they maximize
 226         # probability that the client will accept the statd notify packet and
 227         # not just ignore it.
 228         # For all IPs we serve, collect info and push to the config database
 229
 230         # Construct a sed expression to take catdb output and produce pairs of:
 231         #   server-IP client-IP
 232         # but only for the server-IPs that are hosted on this node.
 233         ctdb_all_ips=$($CTDB ip all | tail -n +2)
 234         sed_expr=$(echo "$ctdb_all_ips" |
 235             awk -v pnn="$pnn" 'pnn == $2 {
 236                 ip = $1; gsub(/\./, "\\.", ip);
 237                 printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
 238
 239         statd_state=$($CTDB catdb ctdb.tdb | sed -n "$sed_expr" | sort)
 240         [ -n "$statd_state" ] || exit 0
 241
 242         echo "$statd_state" | send_notifies
 243         echo "$statd_state" | delete_records
 244
 245         # Remove any stale touch files (i.e. for IPs not currently
 246         # hosted on this node and created since the last "update").
 247         # There's nothing else we can do with them at this stage.
 248         echo "$ctdb_all_ips" |
 249             awk -v pnn="$pnn" 'pnn != $2 { print $1 }' |
 250             while read sip ; do
 251                 rm -f "statd-state@${sip}@"*
 252             done
 253         ;;
 254 esac