X-Git-Url: http://git.samba.org/?a=blobdiff_plain;f=config%2Fstatd-callout;h=168975c51e1c36b5669906eeeaeea85677ee328d;hb=d41b802250ddc0a89581eb6285edfd66bdc7a78a;hp=3f645ec732ef1f2b348af0614a44223ffc864c3a;hpb=65f3e2bc722e314b2c51c3bfdc544b408a8a64cf;p=metze%2Fctdb%2Fwip.git diff --git a/config/statd-callout b/config/statd-callout index 3f645ec7..168975c5 100755 --- a/config/statd-callout +++ b/config/statd-callout @@ -5,41 +5,144 @@ # /etc/sysconfig/nfs: # STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout" -. /etc/ctdb/functions +[ -z "$CTDB_BASE" ] && { + export CTDB_BASE="/etc/ctdb" +} + +. $CTDB_BASE/functions +loadconfig ctdb loadconfig nfs -[ -z "$STATD_SHARED_DIRECTORY" ] && exit 0 +[ -z "$STATD_SHARED_DIRECTORY" ] && { + echo STATD_SHARED_DIRECTORY not configured. statd-callout failed. + exit 0 +} [ -d $STATD_SHARED_DIRECTORY ] || exit 0 +[ -z $NFS_HOSTNAME ] && { + echo NFS_HOSTNAME is not configured. statd-callout failed. + exit 0 +} + case "$1" in add-client) - for f in `/bin/ls /etc/ctdb/state/statd/ip/*`; do - ip=`/bin/basename $f` + # the callout does not tell us to which ip the client connected + # so we must add it to all the ips that we serve + for f in $CTDB_BASE/state/statd/ip/*; do + ip=`basename $f` [ -d $STATD_SHARED_DIRECTORY/$ip ] || /bin/mkdir $STATD_SHARED_DIRECTORY/$ip - /bin/touch $STATD_SHARED_DIRECTORY/$ip/sm/$2 + touch $STATD_SHARED_DIRECTORY/$ip/$2 done ;; del-client) - for f in `/bin/ls /etc/ctdb/state/statd/ip/*`; do - ip=`/bin/basename $f` - /bin/rm -f $STATD_SHARED_DIRECTORY/$ip/sm/$2 + # the callout does not tell us to which ip the client connected + # so we must add it to all the ips that we serve + for f in $CTDB_BASE/state/statd/ip/*; do + ip=`basename $f` + /bin/rm -f $STATD_SHARED_DIRECTORY/$ip/$2 done ;; notify) - # restart the local lock manager and statd - /etc/init.d/nfslock stop > /dev/null 2>&1 - /etc/init.d/nfslock start > /dev/null 2>&1 - # send out notifications to any additional ips we now serve - for f in `/bin/ls /etc/ctdb/state/statd/ip/*`; do - ip=`/bin/basename $f` - [ -d $STATD_SHARED_DIRECTORY/$ip ] && { - # we must copy to a different directory since rpc.statd gets - # "upset" if sm-notify touches the files. - /bin/rm -rf /tmp/statd/$ip - /bin/mkdir -p /tmp/statd/$ip - /bin/cp -apr $STATD_SHARED_DIRECTORY/$ip/* /tmp/statd/$ip - [ -x /usr/sbin/sm-notify ] && /usr/sbin/sm-notify -P /tmp/statd/$ip -v $ip -n + # we must restart the lockmanager (on all nodes) so that we get + # a clusterwide grace period (so other clients dont take out + # conflicting locks through other nodes before all locks have been + # reclaimed) + + # we need these settings to make sure that no tcp connections survive + # across a very fast failover/failback + #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout + #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets + #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans + + # rebuild the state directory for the local statd to use the correct + # state value and to initally send notifications to all clients + rm -f /var/lib/nfs/statd/sm/* + rm -f /var/lib/nfs/statd/sm.bak/* + cat $STATD_SHARED_DIRECTORY/state >/var/lib/nfs/statd/state + + + # we must keep a monotonically increasing state variable for the entire + # cluster so state always increases when ip addresses fail from one + # node to another + [ ! -f $STATD_SHARED_DIRECTORY/state ] && { + echo 1 | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state + } + # read current state + STATE=`od -t d4 $STATD_SHARED_DIRECTORY/state | head -1 | sed -e "s/^[0-9]*[^0-9]*//"` + # write current state+2 back to the state file + # the /2 *2 are to ensure that state is odd. state must be odd. + STATE=`expr $STATE "/" 2 "*" 2 "+" 3` + echo $STATE | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state + + + + # we must also let some time pass between stopping and restarting the + # lockmanager since othervise there is a window where the lockmanager + # will respond "strangely" immediately after restarting it, which + # causes clients to fail to reclaim the locks. + # + startstop_nfslock stop > /dev/null 2>&1 + sleep 2 + + # copy all monitored clients on this node to the local lockmanager + for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do + ip=`basename $f` + [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && { + for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do + client=`basename $g` + touch /var/lib/nfs/statd/sm/$client + done + } + done + + # now start lockmanager again with the new state directory. + startstop_nfslock start > /dev/null 2>&1 + + # we now need to send out additional statd notifications to ensure + # that clients understand that the lockmanager has restarted. + # we have three cases: + # 1, clients that ignore the ip address the stat notification came from + # and ONLY care about the 'name' in the notify packet. + # these clients ONLY work with lock failover IFF that name + # can be resolved into an ipaddress that matches the one used + # to mount the share. (==linux clients) + # This is handled when starting lockmanager above, but those + # packets are sent from the "wrong" ip address, something linux + # clients are ok with, buth other clients will barf at. + # 2, Some clients only accept statd packets IFF they come from the + # 'correct' ip address. + # 2a,Send out the notification using the 'correct' ip address and also + # specify the 'correct' hostname in the statd packet. + # Some clients require both the correct source address and also the + # correct name. (these clients also ONLY work if the ip addresses + # used to map the share can be resolved into the name returned in + # the notify packet.) + # 2b,Other clients require that the source ip address of the notify + # packet matches the ip address used to take out the lock. + # I.e. that the correct source address is used. + # These clients also require that the statd notify packet contains + # the name as the ip address used when the lock was taken out. + # + # Both 2a and 2b are commonly used in lockmanagers since they maximize + # probability that the client will accept the statd notify packet and + # not just ignore it. + for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do + ip=`basename $f` + [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && { + for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do + client=`basename $g` +# /bin/rm -f $g + # send out notifications from the "correct" address + # (the same addresse as where the lock was taken out + # on) some clients require that the source address + # matches where the lock was taken out. + # also send it both as a name that the client + # hopefully can resolve into the server ip and + # and also by specifying the raw ip address as name. + /usr/bin/smnotify --client=$client --ip=$ip --server=$ip --stateval=$STATE + /usr/bin/smnotify --client=$client --ip=$ip --server=$NFS_HOSTNAME --stateval=$STATE + done } done ;;