ctdb/config/statd-callout

   1 #!/bin/sh
   2
   3 # This must run as root as CTDB tool commands need to access CTDB socket
   4 [ $(id -u) -eq 0 ] || exec sudo "$0" "$@"
   5
   6 # this script needs to be installed so that statd points to it with the -H
   7 # command line argument. The easiest way to do that is to put something like this in
   8 # /etc/sysconfig/nfs:
   9 #   STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
  10
  11 [ -n "$CTDB_BASE" ] || \
  12     export CTDB_BASE=$(cd -P $(dirname "$0") ; echo "$PWD")
  13
  14 . $CTDB_BASE/functions
  15
  16 # Overwrite this so we get some logging
  17 die ()
  18 {
  19     script_log "statd-callout" "$@"
  20     exit 1
  21 }
  22
  23 loadconfig ctdb
  24 loadconfig nfs
  25
  26 [ -n "$NFS_HOSTNAME" ] || \
  27     die "NFS_HOSTNAME is not configured. statd-callout failed"
  28
  29 # A handy newline
  30 nl="
  31 "
  32
  33 case "$1" in
  34     add-client)
  35         # statd does not tell us to which IP the client connected so
  36         # we must add it to all the IPs that we serve
  37         cip="$2"
  38         pnn=$(ctdb xpnn | sed -e 's/.*://')
  39         date=$(date '+%s')
  40         ctdb ip -Y |
  41         tail -n +2 | {
  42             # This all needs to be in the end of the pipe so it
  43             # doesn't get lost
  44             items=""
  45             while IFS=":" read x sip node x ; do
  46                 [ "$node" = "$pnn" ] || continue # not us
  47                 key="statd-state@${sip}@${cip}"
  48                 item="\"${key}\" \"${date}\""
  49                 items="${items}${items:+${nl}}${item}"
  50             done
  51             if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
  52                 die "Failed to add clients"
  53             fi
  54         }
  55         ;;
  56   del-client)
  57         # statd does not tell us from which IP the client disconnected
  58         # so we must add it to all the IPs that we serve
  59         cip="$2"
  60         pnn=$(ctdb xpnn | sed -e 's/.*://')
  61         ctdb ip -Y |
  62         tail -n +2 | {
  63             # This all needs to be in the end of the pipe so it
  64             # doesn't get lost
  65             items=""
  66             while IFS=":" read x sip node x ; do
  67                 [ "$node" = "$pnn" ] || continue # not us
  68                 key="statd-state@${sip}@${cip}"
  69                 item="\"${key}\" \"\""
  70                 items="${items}${items:+${nl}}${item}"
  71             done
  72             if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
  73                 die "Failed to delete clients"
  74             fi
  75         }
  76         ;;
  77     notify)
  78         # we must restart the lockmanager (on all nodes) so that we get
  79         # a clusterwide grace period (so other clients dont take out
  80         # conflicting locks through other nodes before all locks have been
  81         # reclaimed)
  82
  83         # we need these settings to make sure that no tcp connections survive
  84         # across a very fast failover/failback
  85         #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
  86         #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
  87         #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
  88
  89         # Delete the notification list for statd, we dont want it to
  90         # ping any clients
  91         rm -f /var/lib/nfs/statd/sm/*
  92         rm -f /var/lib/nfs/statd/sm.bak/*
  93
  94         # we must keep a monotonically increasing state variable for the entire
  95         # cluster  so state always increases when ip addresses fail from one
  96         # node to another
  97         # We use epoch and hope the nodes are close enough in clock.
  98         # Even numbers mean service is shut down, odd numbers mean
  99         # service is started.
 100         state_even=$(( $(date '+%s') / 2 * 2))
 101
 102         # we must also let some time pass between stopping and restarting the
 103         # lockmanager since othervise there is a window where the lockmanager
 104         # will respond "strangely" immediately after restarting it, which
 105         # causes clients to fail to reclaim the locks.
 106         #
 107         if [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] ; then
 108             startstop_nfslock stop >/dev/null 2>&1
 109             sleep 2
 110             startstop_nfslock start >/dev/null 2>&1
 111         fi
 112
 113         # we now need to send out additional statd notifications to ensure
 114         # that clients understand that the lockmanager has restarted.
 115         # we have three cases:
 116         # 1, clients that ignore the ip address the stat notification came from
 117         #    and ONLY care about the 'name' in the notify packet.
 118         #    these clients ONLY work with lock failover IFF that name
 119         #    can be resolved into an ipaddress that matches the one used
 120         #    to mount the share.  (==linux clients)
 121         #    This is handled when starting lockmanager above,  but those
 122         #    packets are sent from the "wrong" ip address, something linux
 123         #    clients are ok with, buth other clients will barf at.
 124         # 2, Some clients only accept statd packets IFF they come from the
 125         #    'correct' ip address.
 126         # 2a,Send out the notification using the 'correct' ip address and also
 127         #    specify the 'correct' hostname in the statd packet.
 128         #    Some clients require both the correct source address and also the
 129         #    correct name. (these clients also ONLY work if the ip addresses
 130         #    used to map the share can be resolved into the name returned in
 131         #    the notify packet.)
 132         # 2b,Other clients require that the source ip address of the notify
 133         #    packet matches the ip address used to take out the lock.
 134         #    I.e. that the correct source address is used.
 135         #    These clients also require that the statd notify packet contains
 136         #    the name as the ip address used when the lock was taken out.
 137         #
 138         # Both 2a and 2b are commonly used in lockmanagers since they maximize
 139         # probability that the client will accept the statd notify packet and
 140         # not just ignore it.
 141         # For all IPs we serve, collect info and push to the config database
 142         pnn=$(ctdb xpnn | sed -e 's/.*://')
 143
 144         # Construct a sed expression to take catdb output and produce pairs of:
 145         #   server-IP client-IP
 146         # but only for the server-IPs that are hosted on this node.
 147         sed_expr=$(ctdb ip | tail -n +2 |
 148             awk -v pnn=$pnn 'pnn == $2 { printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", gensub(/\./, "\\\\.", "g", $1) }')
 149
 150         statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort)
 151         [ -n "$statd_state" ] || exit 0
 152
 153         # The following is dangerous if this script times out before
 154         # all of the smnotify commands are run.  Revert to individual
 155         # pdelete commands for now and consider optimising smnotify to
 156         # read all the data from stdin and then run it in the
 157         # background.
 158         #
 159         # Delete all the items from the TDB
 160         #if ! echo "$statd_state" | \
 161         #    awk '{ printf "\"statd-state@%s@%s\" \"\"\n", $1, $2 }') | \
 162         #    ctdb ptrans ctdb.tdb ; then
 163
 164         #    die "Yikes!"
 165         #fi
 166
 167         prev=""
 168         echo "$statd_state" |
 169         while read sip cip ; do
 170             # Delete the entry from the DB
 171             ctdb pdelete ctdb.tdb "statd-state@${sip}@${cip}"
 172             # Reset stateval for each serverip
 173             [ "$sip" = "$prev" ] || stateval="$state_even"
 174             # Send notifies for server shutdown
 175             smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
 176             smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
 177             # Send notifies for server startup
 178             stateval=$(($stateval + 1))
 179             smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
 180             smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
 181         done
 182         ;;
 183 esac