# /etc/sysconfig/nfs:
# STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
-. /etc/ctdb/functions
+[ -z "$CTDB_BASE" ] && {
+ export CTDB_BASE="/etc/ctdb"
+}
+
+. $CTDB_BASE/functions
+loadconfig ctdb
loadconfig nfs
-[ -z "$STATD_SHARED_DIRECTORY" ] && exit 0
+[ -z "$STATD_SHARED_DIRECTORY" ] && {
+ echo STATD_SHARED_DIRECTORY not configured. statd-callout failed.
+ exit 0
+}
[ -d $STATD_SHARED_DIRECTORY ] || exit 0
+[ -z $NFS_HOSTNAME ] && {
+ echo NFS_HOSTNAME is not configured. statd-callout failed.
+ exit 0
+}
+
case "$1" in
add-client)
- for f in `/bin/ls /etc/ctdb/state/statd/ip/*`; do
- ip=`/bin/basename $f`
+ # the callout does not tell us to which ip the client connected
+ # so we must add it to all the ips that we serve
+ for f in $CTDB_BASE/state/statd/ip/*; do
+ ip=`basename $f`
[ -d $STATD_SHARED_DIRECTORY/$ip ] || /bin/mkdir $STATD_SHARED_DIRECTORY/$ip
- /bin/touch $STATD_SHARED_DIRECTORY/$ip/sm/$2
+ touch $STATD_SHARED_DIRECTORY/$ip/$2
done
;;
del-client)
- for f in `/bin/ls /etc/ctdb/state/statd/ip/*`; do
- ip=`/bin/basename $f`
- /bin/rm -f $STATD_SHARED_DIRECTORY/$ip/sm/$2
+ # the callout does not tell us to which ip the client connected
+ # so we must add it to all the ips that we serve
+ for f in $CTDB_BASE/state/statd/ip/*; do
+ ip=`basename $f`
+ /bin/rm -f $STATD_SHARED_DIRECTORY/$ip/$2
done
;;
notify)
- # restart the local lock manager and statd
- /etc/init.d/nfslock stop > /dev/null 2>&1
- /etc/init.d/nfslock start > /dev/null 2>&1
- # send out notifications to any additional ips we now serve
- for f in `/bin/ls /etc/ctdb/state/statd/ip/*`; do
- ip=`/bin/basename $f`
- [ -d $STATD_SHARED_DIRECTORY/$ip ] && {
- # we must copy to a different directory since rpc.statd gets
- # "upset" if sm-notify touches the files.
- /bin/rm -rf /tmp/statd/$ip
- /bin/mkdir -p /tmp/statd/$ip
- /bin/cp -apr $STATD_SHARED_DIRECTORY/$ip/* /tmp/statd/$ip
- [ -x /usr/sbin/sm-notify ] && /usr/sbin/sm-notify -P /tmp/statd/$ip -v $ip -n
+ # we must restart the lockmanager (on all nodes) so that we get
+ # a clusterwide grace period (so other clients dont take out
+ # conflicting locks through other nodes before all locks have been
+ # reclaimed)
+
+ # we need these settings to make sure that no tcp connections survive
+ # across a very fast failover/failback
+ #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
+ #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
+ #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
+
+ # rebuild the state directory for the local statd to use the correct
+ # state value and to initally send notifications to all clients
+ rm -f /var/lib/nfs/statd/sm/*
+ rm -f /var/lib/nfs/statd/sm.bak/*
+ cat $STATD_SHARED_DIRECTORY/state >/var/lib/nfs/statd/state
+
+
+ # we must keep a monotonically increasing state variable for the entire
+ # cluster so state always increases when ip addresses fail from one
+ # node to another
+ [ ! -f $STATD_SHARED_DIRECTORY/state ] && {
+ echo 1 | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
+ }
+ # read current state
+ STATE=`od -t d4 $STATD_SHARED_DIRECTORY/state | head -1 | sed -e "s/^[0-9]*[^0-9]*//"`
+ # write current state+2 back to the state file
+ # the /2 *2 are to ensure that state is odd. state must be odd.
+ STATE=`expr $STATE "/" 2 "*" 2 "+" 3`
+ echo $STATE | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
+
+
+
+ # we must also let some time pass between stopping and restarting the
+ # lockmanager since othervise there is a window where the lockmanager
+ # will respond "strangely" immediately after restarting it, which
+ # causes clients to fail to reclaim the locks.
+ #
+ startstop_nfslock stop > /dev/null 2>&1
+ sleep 2
+
+ # copy all monitored clients on this node to the local lockmanager
+ for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
+ ip=`basename $f`
+ [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
+ for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
+ client=`basename $g`
+ touch /var/lib/nfs/statd/sm/$client
+ done
+ }
+ done
+
+ # now start lockmanager again with the new state directory.
+ startstop_nfslock start > /dev/null 2>&1
+
+ # we now need to send out additional statd notifications to ensure
+ # that clients understand that the lockmanager has restarted.
+ # we have three cases:
+ # 1, clients that ignore the ip address the stat notification came from
+ # and ONLY care about the 'name' in the notify packet.
+ # these clients ONLY work with lock failover IFF that name
+ # can be resolved into an ipaddress that matches the one used
+ # to mount the share. (==linux clients)
+ # This is handled when starting lockmanager above, but those
+ # packets are sent from the "wrong" ip address, something linux
+ # clients are ok with, buth other clients will barf at.
+ # 2, Some clients only accept statd packets IFF they come from the
+ # 'correct' ip address.
+ # 2a,Send out the notification using the 'correct' ip address and also
+ # specify the 'correct' hostname in the statd packet.
+ # Some clients require both the correct source address and also the
+ # correct name. (these clients also ONLY work if the ip addresses
+ # used to map the share can be resolved into the name returned in
+ # the notify packet.)
+ # 2b,Other clients require that the source ip address of the notify
+ # packet matches the ip address used to take out the lock.
+ # I.e. that the correct source address is used.
+ # These clients also require that the statd notify packet contains
+ # the name as the ip address used when the lock was taken out.
+ #
+ # Both 2a and 2b are commonly used in lockmanagers since they maximize
+ # probability that the client will accept the statd notify packet and
+ # not just ignore it.
+ for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
+ ip=`basename $f`
+ [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
+ for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
+ client=`basename $g`
+# /bin/rm -f $g
+ # send out notifications from the "correct" address
+ # (the same addresse as where the lock was taken out
+ # on) some clients require that the source address
+ # matches where the lock was taken out.
+ # also send it both as a name that the client
+ # hopefully can resolve into the server ip and
+ # and also by specifying the raw ip address as name.
+ /usr/bin/smnotify --client=$client --ip=$ip --server=$ip --stateval=$STATE
+ /usr/bin/smnotify --client=$client --ip=$ip --server=$NFS_HOSTNAME --stateval=$STATE
+ done
}
done
;;
git.samba.org / metze / ctdb / wip.git / blobdiff