3 # This must run as root as CTDB tool commands need to access CTDB socket
4 [ $(id -u) -eq 0 ] || exec sudo "$0" "$@"
6 # this script needs to be installed so that statd points to it with the -H
7 # command line argument. The easiest way to do that is to put something like this in
9 # STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
11 [ -n "$CTDB_BASE" ] || \
12 export CTDB_BASE=$(cd -P $(dirname "$0") ; echo "$PWD")
14 . $CTDB_BASE/functions
16 # Overwrite this so we get some logging
19 script_log "statd-callout" "$@"
26 [ -n "$NFS_HOSTNAME" ] || \
27 die "NFS_HOSTNAME is not configured. statd-callout failed"
35 # statd does not tell us to which IP the client connected so
36 # we must add it to all the IPs that we serve
38 pnn=$(ctdb xpnn | sed -e 's/.*://')
42 # This all needs to be in the end of the pipe so it
45 while IFS=":" read x sip node x ; do
46 [ "$node" = "$pnn" ] || continue # not us
47 key="statd-state@${sip}@${cip}"
48 item="\"${key}\" \"${date}\""
49 items="${items}${items:+${nl}}${item}"
51 if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
52 die "Failed to add clients"
57 # statd does not tell us from which IP the client disconnected
58 # so we must add it to all the IPs that we serve
60 pnn=$(ctdb xpnn | sed -e 's/.*://')
63 # This all needs to be in the end of the pipe so it
66 while IFS=":" read x sip node x ; do
67 [ "$node" = "$pnn" ] || continue # not us
68 key="statd-state@${sip}@${cip}"
69 item="\"${key}\" \"\""
70 items="${items}${items:+${nl}}${item}"
72 if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
73 die "Failed to delete clients"
78 # we must restart the lockmanager (on all nodes) so that we get
79 # a clusterwide grace period (so other clients dont take out
80 # conflicting locks through other nodes before all locks have been
83 # we need these settings to make sure that no tcp connections survive
84 # across a very fast failover/failback
85 #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
86 #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
87 #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
89 # Delete the notification list for statd, we dont want it to
91 rm -f /var/lib/nfs/statd/sm/*
92 rm -f /var/lib/nfs/statd/sm.bak/*
94 # we must keep a monotonically increasing state variable for the entire
95 # cluster so state always increases when ip addresses fail from one
97 # We use epoch and hope the nodes are close enough in clock.
98 # Even numbers mean service is shut down, odd numbers mean
100 state_even=$(( $(date '+%s') / 2 * 2))
102 # we must also let some time pass between stopping and restarting the
103 # lockmanager since othervise there is a window where the lockmanager
104 # will respond "strangely" immediately after restarting it, which
105 # causes clients to fail to reclaim the locks.
107 if [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] ; then
108 startstop_nfslock stop >/dev/null 2>&1
110 startstop_nfslock start >/dev/null 2>&1
113 # we now need to send out additional statd notifications to ensure
114 # that clients understand that the lockmanager has restarted.
115 # we have three cases:
116 # 1, clients that ignore the ip address the stat notification came from
117 # and ONLY care about the 'name' in the notify packet.
118 # these clients ONLY work with lock failover IFF that name
119 # can be resolved into an ipaddress that matches the one used
120 # to mount the share. (==linux clients)
121 # This is handled when starting lockmanager above, but those
122 # packets are sent from the "wrong" ip address, something linux
123 # clients are ok with, buth other clients will barf at.
124 # 2, Some clients only accept statd packets IFF they come from the
125 # 'correct' ip address.
126 # 2a,Send out the notification using the 'correct' ip address and also
127 # specify the 'correct' hostname in the statd packet.
128 # Some clients require both the correct source address and also the
129 # correct name. (these clients also ONLY work if the ip addresses
130 # used to map the share can be resolved into the name returned in
131 # the notify packet.)
132 # 2b,Other clients require that the source ip address of the notify
133 # packet matches the ip address used to take out the lock.
134 # I.e. that the correct source address is used.
135 # These clients also require that the statd notify packet contains
136 # the name as the ip address used when the lock was taken out.
138 # Both 2a and 2b are commonly used in lockmanagers since they maximize
139 # probability that the client will accept the statd notify packet and
140 # not just ignore it.
141 # For all IPs we serve, collect info and push to the config database
142 pnn=$(ctdb xpnn | sed -e 's/.*://')
144 # Construct a sed expression to take catdb output and produce pairs of:
145 # server-IP client-IP
146 # but only for the server-IPs that are hosted on this node.
147 sed_expr=$(ctdb ip | tail -n +2 |
148 awk -v pnn=$pnn 'pnn == $2 { printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", gensub(/\./, "\\\\.", "g", $1) }')
150 statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort)
152 # The following is dangerous if this script times out before
153 # all of the smnotify commands are run. Revert to individual
154 # pdelete commands for now and consider optimising smnotify to
155 # read all the data from stdin and then run it in the
158 # Delete all the items from the TDB
159 #if ! echo "$statd_state" | \
160 # awk '{ printf "\"statd-state@%s@%s\" \"\"\n", $1, $2 }') | \
161 # ctdb ptrans ctdb.tdb ; then
167 echo "$statd_state" |
168 while read sip cip ; do
169 # Delete the entry from the DB
170 ctdb pdelete ctdb.tdb "statd-state@${sip}@${cip}"
171 # Reset stateval for each serverip
172 [ "$sip" = "$prev" ] || stateval="$state_even"
173 # Send notifies for server shutdown
174 smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
175 smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
176 # Send notifies for server startup
177 stateval=$(($stateval + 1))
178 smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
179 smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval