3 # This must run as root as CTDB tool commands need to access CTDB socket
4 [ "$(id -u)" -eq 0 ] ||
exec sudo
"$0" "$@"
6 # this script needs to be installed so that statd points to it with the -H
7 # command line argument. The easiest way to do that is to put something like this in
9 # STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
11 [ -n "$CTDB_BASE" ] || \
12 CTDB_BASE
=$
(d
=$
(dirname "$0") ; cd -P "$d" ; echo "$PWD")
14 .
"${CTDB_BASE}/functions"
16 # Overwrite this so we get some logging
19 script_log
"statd-callout" "$@"
23 load_system_config
"nfs"
25 [ -n "$NFS_HOSTNAME" ] || \
26 die
"NFS_HOSTNAME is not configured. statd-callout failed"
28 ############################################################
30 ctdb_setup_state_dir
"service" "nfs"
32 # script_state_dir set by ctdb_setup_state_dir()
33 # shellcheck disable=SC2154
34 d
="${script_state_dir}/statd-callout"
36 mkdir
-p "$d" || die
"Failed to create directory \"${d}\""
37 cd "$d" || die
"Failed to change directory to \"${d}\""
41 ############################################################
45 _smnotify
="${CTDB_HELPER_BINDIR}/smnotify"
47 # State must monotonically increase, across the entire
48 # cluster. Use seconds since epoch and hope the time is in
49 # sync across nodes. Even numbers mean service is shut down,
50 # odd numbers mean service is started.
52 # Intentionally round to an even number
53 # shellcheck disable=SC2017
54 _state_even
=$
(( $
(date '+%s') / 2 * 2))
57 while read _sip _cip
; do
58 # NOTE: Consider optimising smnotify to read all the
59 # data from stdin and then run it in the background.
61 # Reset stateval for each serverip
62 if [ "$_sip" != "$_prev" ] ; then
63 _stateval
="$_state_even"
66 # Send notifies for server shutdown
67 "$_smnotify" --client="$_cip" --ip="$_sip" \
68 --server="$_sip" --stateval="$_stateval"
69 "$_smnotify" --client="$_cip" --ip="$_sip" \
70 --server="$NFS_HOSTNAME" --stateval="$_stateval"
72 # Send notifies for server startup
73 _stateval
=$
((_stateval
+ 1))
74 "$_smnotify" --client="$_cip" --ip="$_sip" \
75 --server="$_sip" --stateval="$_stateval"
76 "$_smnotify" --client="$_cip" --ip="$_sip" \
77 --server="$NFS_HOSTNAME" --stateval="$_stateval"
83 while read _sip _cip
; do
84 _key
="statd-state@${_sip}@${_cip}"
85 echo "\"${_key}\" \"\""
86 done |
$CTDB ptrans
"ctdb.tdb"
89 ############################################################
92 # Keep a single file to keep track of the last "add-client" or
93 # "del-client'. These get pushed to ctdb.tdb during "update",
94 # which will generally be run once each "monitor" cycle. In this
95 # way we avoid scalability problems with flood of persistent
96 # transactions after a "notify" when all the clients re-take their
100 # statd does not tell us to which IP the client connected so
101 # we must add it to all the IPs that we serve
104 # x is intentionally ignored
105 # shellcheck disable=SC2034
108 while IFS
="|" read x sip node x
; do
109 [ "$node" = "$pnn" ] ||
continue # not us
110 key
="statd-state@${sip}@${cip}"
111 echo "\"${key}\" \"${date}\"" >"$key"
116 # statd does not tell us from which IP the client disconnected
117 # so we must add it to all the IPs that we serve
119 # x is intentionally ignored
120 # shellcheck disable=SC2034
123 while IFS
="|" read x sip node x
; do
124 [ "$node" = "$pnn" ] ||
continue # not us
125 key
="statd-state@${sip}@${cip}"
126 echo "\"${key}\" \"\"" >"$key"
131 files
=$
(echo statd-state@
*)
132 if [ "$files" = "statd-state@*" ] ; then
136 # Filter out lines for any IP addresses that are not currently
137 # hosted public IP addresses.
138 ctdb_ips
=$
($CTDB ip |
tail -n +2)
139 sed_expr
=$
(echo "$ctdb_ips" |
140 awk -v pnn
="$pnn" 'pnn == $2 {
141 ip = $1; gsub(/\./, "\\.", ip);
142 printf "/statd-state@%s@/p\n", ip }')
143 # Intentional multi-word expansion for multiple files
144 # shellcheck disable=SC2086
145 items
=$
(sed -n "$sed_expr" $files)
146 if [ -n "$items" ] ; then
147 if echo "$items" |
$CTDB ptrans
"ctdb.tdb" ; then
148 # shellcheck disable=SC2086
155 # we must restart the lockmanager (on all nodes) so that we get
156 # a clusterwide grace period (so other clients don't take out
157 # conflicting locks through other nodes before all locks have been
160 # we need these settings to make sure that no tcp connections survive
161 # across a very fast failover/failback
162 #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
163 #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
164 #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
166 # Delete the notification list for statd, we don't want it to
168 rm -f /var
/lib
/nfs
/statd
/sm
/*
169 rm -f /var
/lib
/nfs
/statd
/sm.bak
/*
171 # We must also let some time pass between stopping and
172 # restarting the lock manager. Otherwise there is a window
173 # where the lock manager will respond "strangely" immediately
174 # after restarting it, which causes clients to fail to reclaim
177 "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev
/null
2>&1
179 "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev
/null
2>&1
181 # we now need to send out additional statd notifications to ensure
182 # that clients understand that the lockmanager has restarted.
183 # we have three cases:
184 # 1, clients that ignore the ip address the stat notification came from
185 # and ONLY care about the 'name' in the notify packet.
186 # these clients ONLY work with lock failover IFF that name
187 # can be resolved into an ipaddress that matches the one used
188 # to mount the share. (==linux clients)
189 # This is handled when starting lockmanager above, but those
190 # packets are sent from the "wrong" ip address, something linux
191 # clients are ok with, buth other clients will barf at.
192 # 2, Some clients only accept statd packets IFF they come from the
193 # 'correct' ip address.
194 # 2a,Send out the notification using the 'correct' ip address and also
195 # specify the 'correct' hostname in the statd packet.
196 # Some clients require both the correct source address and also the
197 # correct name. (these clients also ONLY work if the ip addresses
198 # used to map the share can be resolved into the name returned in
199 # the notify packet.)
200 # 2b,Other clients require that the source ip address of the notify
201 # packet matches the ip address used to take out the lock.
202 # I.e. that the correct source address is used.
203 # These clients also require that the statd notify packet contains
204 # the name as the ip address used when the lock was taken out.
206 # Both 2a and 2b are commonly used in lockmanagers since they maximize
207 # probability that the client will accept the statd notify packet and
208 # not just ignore it.
209 # For all IPs we serve, collect info and push to the config database
211 # Construct a sed expression to take catdb output and produce pairs of:
212 # server-IP client-IP
213 # but only for the server-IPs that are hosted on this node.
214 ctdb_all_ips
=$
($CTDB ip all |
tail -n +2)
215 sed_expr
=$
(echo "$ctdb_all_ips" |
216 awk -v pnn
="$pnn" 'pnn == $2 {
217 ip = $1; gsub(/\./, "\\.", ip);
218 printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
220 statd_state
=$
($CTDB catdb ctdb.tdb |
sed -n "$sed_expr" |
sort)
221 [ -n "$statd_state" ] ||
exit 0
223 echo "$statd_state" | send_notifies
224 echo "$statd_state" | delete_records
226 # Remove any stale touch files (i.e. for IPs not currently
227 # hosted on this node and created since the last "update").
228 # There's nothing else we can do with them at this stage.
229 echo "$ctdb_all_ips" |
230 awk -v pnn
="$pnn" 'pnn != $2 { print $1 }' |
232 rm -f "statd-state@${sip}@"*