3 # This must run as root as CTDB tool commands need to access CTDB socket
4 [ "$(id -u)" -eq 0 ] ||
exec sudo
"$0" "$@"
6 # statd must be configured to use this script as its high availability call-out.
8 # In most Linux versions this can be done using something like the following...
10 # /etc/sysconfig/nfs (Red Hat) or /etc/default/nfs-common (Debian):
11 # NFS_HOSTNAME=myhostname
12 # STATD_HOSTNAME="${NFS_HOSTNAME} -H /etc/ctdb/statd-callout"
14 # Newer Red Hat Linux variants instead use /etc/nfs.conf:
17 # ha-callout = /etc/ctdb/statd-callout
19 [ -n "$CTDB_BASE" ] || \
20 CTDB_BASE
=$
(d
=$
(dirname "$0") && cd -P "$d" && echo "$PWD")
22 .
"${CTDB_BASE}/functions"
24 # Overwrite this so we get some logging
27 script_log
"statd-callout" "$@"
31 # Try different variables to find config file for NFS_HOSTNAME
32 load_system_config
"nfs" "nfs-common"
34 # If NFS_HOSTNAME not set then try to pull it out of /etc/nfs.conf
35 if [ -z "$NFS_HOSTNAME" ]; then
36 if type nfsconf
>/dev
/null
2>&1; then
37 NFS_HOSTNAME
=$
(nfsconf
--get statd name
)
38 elif type git
>/dev
/null
2>&1; then
40 NFS_HOSTNAME
=$
(git config
--file=/etc
/nfs.conf statd.name
)
44 [ -n "$NFS_HOSTNAME" ] || \
45 die
"NFS_HOSTNAME is not configured. statd-callout failed"
47 ############################################################
49 ctdb_setup_state_dir
"service" "nfs"
51 # script_state_dir set by ctdb_setup_state_dir()
52 # shellcheck disable=SC2154
53 d
="${script_state_dir}/statd-callout"
55 mkdir
-p "$d" || die
"Failed to create directory \"${d}\""
56 cd "$d" || die
"Failed to change directory to \"${d}\""
60 ############################################################
64 _smnotify
="${CTDB_HELPER_BINDIR}/smnotify"
66 # State must monotonically increase, across the entire
67 # cluster. Use seconds since epoch and hope the time is in
68 # sync across nodes. Even numbers mean service is shut down,
69 # odd numbers mean service is started.
71 # Intentionally round to an even number
72 # shellcheck disable=SC2017
73 _state_even
=$
(( $
(date '+%s') / 2 * 2))
76 while read _sip _cip
; do
77 # NOTE: Consider optimising smnotify to read all the
78 # data from stdin and then run it in the background.
80 # Reset stateval for each serverip
81 if [ "$_sip" != "$_prev" ] ; then
82 _stateval
="$_state_even"
85 # Send notifies for server shutdown
86 "$_smnotify" --client="$_cip" --ip="$_sip" \
87 --server="$_sip" --stateval="$_stateval"
88 "$_smnotify" --client="$_cip" --ip="$_sip" \
89 --server="$NFS_HOSTNAME" --stateval="$_stateval"
91 # Send notifies for server startup
92 _stateval
=$
((_stateval
+ 1))
93 "$_smnotify" --client="$_cip" --ip="$_sip" \
94 --server="$_sip" --stateval="$_stateval"
95 "$_smnotify" --client="$_cip" --ip="$_sip" \
96 --server="$NFS_HOSTNAME" --stateval="$_stateval"
102 while read _sip _cip
; do
103 _key
="statd-state@${_sip}@${_cip}"
104 echo "\"${_key}\" \"\""
105 done |
$CTDB ptrans
"ctdb.tdb"
108 ############################################################
111 # Keep a single file to keep track of the last "add-client" or
112 # "del-client'. These get pushed to ctdb.tdb during "update",
113 # which will generally be run once each "monitor" cycle. In this
114 # way we avoid scalability problems with flood of persistent
115 # transactions after a "notify" when all the clients re-take their
119 # statd does not tell us to which IP the client connected so
120 # we must add it to all the IPs that we serve
123 # x is intentionally ignored
124 # shellcheck disable=SC2034
127 while IFS
="|" read x sip node x
; do
128 [ "$node" = "$pnn" ] ||
continue # not us
129 key
="statd-state@${sip}@${cip}"
130 echo "\"${key}\" \"${date}\"" >"$key"
135 # statd does not tell us from which IP the client disconnected
136 # so we must add it to all the IPs that we serve
138 # x is intentionally ignored
139 # shellcheck disable=SC2034
142 while IFS
="|" read x sip node x
; do
143 [ "$node" = "$pnn" ] ||
continue # not us
144 key
="statd-state@${sip}@${cip}"
145 echo "\"${key}\" \"\"" >"$key"
150 files
=$
(echo statd-state@
*)
151 if [ "$files" = "statd-state@*" ] ; then
155 # Filter out lines for any IP addresses that are not currently
156 # hosted public IP addresses.
157 ctdb_ips
=$
($CTDB ip |
tail -n +2)
158 sed_expr
=$
(echo "$ctdb_ips" |
159 awk -v pnn
="$pnn" 'pnn == $2 {
160 ip = $1; gsub(/\./, "\\.", ip);
161 printf "/statd-state@%s@/p\n", ip }')
162 # Intentional multi-word expansion for multiple files
163 # shellcheck disable=SC2086
164 items
=$
(sed -n "$sed_expr" $files)
165 if [ -n "$items" ] ; then
166 if echo "$items" |
$CTDB ptrans
"ctdb.tdb" ; then
167 # shellcheck disable=SC2086
174 # we must restart the lockmanager (on all nodes) so that we get
175 # a clusterwide grace period (so other clients don't take out
176 # conflicting locks through other nodes before all locks have been
179 # we need these settings to make sure that no tcp connections survive
180 # across a very fast failover/failback
181 #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
182 #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
183 #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
185 # Delete the notification list for statd, we don't want it to
187 rm -f /var
/lib
/nfs
/statd
/sm
/*
188 rm -f /var
/lib
/nfs
/statd
/sm.bak
/*
190 # We must also let some time pass between stopping and
191 # restarting the lock manager. Otherwise there is a window
192 # where the lock manager will respond "strangely" immediately
193 # after restarting it, which causes clients to fail to reclaim
196 "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev
/null
2>&1
198 "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev
/null
2>&1
200 # we now need to send out additional statd notifications to ensure
201 # that clients understand that the lockmanager has restarted.
202 # we have three cases:
203 # 1, clients that ignore the ip address the stat notification came from
204 # and ONLY care about the 'name' in the notify packet.
205 # these clients ONLY work with lock failover IFF that name
206 # can be resolved into an ipaddress that matches the one used
207 # to mount the share. (==linux clients)
208 # This is handled when starting lockmanager above, but those
209 # packets are sent from the "wrong" ip address, something linux
210 # clients are ok with, buth other clients will barf at.
211 # 2, Some clients only accept statd packets IFF they come from the
212 # 'correct' ip address.
213 # 2a,Send out the notification using the 'correct' ip address and also
214 # specify the 'correct' hostname in the statd packet.
215 # Some clients require both the correct source address and also the
216 # correct name. (these clients also ONLY work if the ip addresses
217 # used to map the share can be resolved into the name returned in
218 # the notify packet.)
219 # 2b,Other clients require that the source ip address of the notify
220 # packet matches the ip address used to take out the lock.
221 # I.e. that the correct source address is used.
222 # These clients also require that the statd notify packet contains
223 # the name as the ip address used when the lock was taken out.
225 # Both 2a and 2b are commonly used in lockmanagers since they maximize
226 # probability that the client will accept the statd notify packet and
227 # not just ignore it.
228 # For all IPs we serve, collect info and push to the config database
230 # Construct a sed expression to take catdb output and produce pairs of:
231 # server-IP client-IP
232 # but only for the server-IPs that are hosted on this node.
233 ctdb_all_ips
=$
($CTDB ip all |
tail -n +2)
234 sed_expr
=$
(echo "$ctdb_all_ips" |
235 awk -v pnn
="$pnn" 'pnn == $2 {
236 ip = $1; gsub(/\./, "\\.", ip);
237 printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
239 statd_state
=$
($CTDB catdb ctdb.tdb |
sed -n "$sed_expr" |
sort)
240 [ -n "$statd_state" ] ||
exit 0
242 echo "$statd_state" | send_notifies
243 echo "$statd_state" | delete_records
245 # Remove any stale touch files (i.e. for IPs not currently
246 # hosted on this node and created since the last "update").
247 # There's nothing else we can do with them at this stage.
248 echo "$ctdb_all_ips" |
249 awk -v pnn
="$pnn" 'pnn != $2 { print $1 }' |
251 rm -f "statd-state@${sip}@"*