#!/bin/sh
+# This must run as root as CTDB tool commands need to access CTDB socket
+[ $(id -u) -eq 0 ] || exec sudo "$0" "$@"
+
# this script needs to be installed so that statd points to it with the -H
# command line argument. The easiest way to do that is to put something like this in
# /etc/sysconfig/nfs:
# STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
-[ -z "$CTDB_BASE" ] && {
- export CTDB_BASE="/etc/ctdb"
-}
+[ -n "$CTDB_BASE" ] || \
+ export CTDB_BASE=$(cd -P $(dirname "$0") ; echo "$PWD")
. $CTDB_BASE/functions
-loadconfig nfs
-[ -z "$STATD_SHARED_DIRECTORY" ] && {
- echo STATD_SHARED_DIRECTORY not configured. statd-callout failed.
- exit 0
+# Overwrite this so we get some logging
+die ()
+{
+ script_log "statd-callout" "$@"
+ exit 1
}
-[ -d $STATD_SHARED_DIRECTORY ] || exit 0
+loadconfig ctdb
+loadconfig nfs
+
+[ -n "$NFS_HOSTNAME" ] || \
+ die "NFS_HOSTNAME is not configured. statd-callout failed"
-[ -z $NFS_HOSTNAME ] && {
- echo NFS_HOSTNAME is not configured. statd-callout failed.
- exit 0
-}
+# A handy newline
+nl="
+"
case "$1" in
- add-client)
- # the callout does not tell us to which ip the client connected
- # so we must add it to all the ips that we serve
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/*`; do
- ip=`basename $f`
- [ -d $STATD_SHARED_DIRECTORY/$ip ] || /bin/mkdir $STATD_SHARED_DIRECTORY/$ip
- touch $STATD_SHARED_DIRECTORY/$ip/$2
- done
+ add-client)
+ # statd does not tell us to which IP the client connected so
+ # we must add it to all the IPs that we serve
+ cip="$2"
+ pnn=$(ctdb xpnn | sed -e 's/.*://')
+ date=$(date '+%s')
+ ctdb ip -X |
+ tail -n +2 | {
+ # This all needs to be in the end of the pipe so it
+ # doesn't get lost
+ items=""
+ while IFS="|" read x sip node x ; do
+ [ "$node" = "$pnn" ] || continue # not us
+ key="statd-state@${sip}@${cip}"
+ item="\"${key}\" \"${date}\""
+ items="${items}${items:+${nl}}${item}"
+ done
+ if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
+ die "Failed to add clients"
+ fi
+ }
;;
del-client)
- # the callout does not tell us to which ip the client connected
- # so we must add it to all the ips that we serve
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/*`; do
- ip=`basename $f`
- /bin/rm -f $STATD_SHARED_DIRECTORY/$ip/$2
- done
+ # statd does not tell us from which IP the client disconnected
+ # so we must add it to all the IPs that we serve
+ cip="$2"
+ pnn=$(ctdb xpnn | sed -e 's/.*://')
+ ctdb ip -X |
+ tail -n +2 | {
+ # This all needs to be in the end of the pipe so it
+ # doesn't get lost
+ items=""
+ while IFS="|" read x sip node x ; do
+ [ "$node" = "$pnn" ] || continue # not us
+ key="statd-state@${sip}@${cip}"
+ item="\"${key}\" \"\""
+ items="${items}${items:+${nl}}${item}"
+ done
+ if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
+ die "Failed to delete clients"
+ fi
+ }
;;
- notify)
+ notify)
# we must restart the lockmanager (on all nodes) so that we get
# a clusterwide grace period (so other clients dont take out
# conflicting locks through other nodes before all locks have been
#echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
#echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
- # rebuild the state directory for the local statd to use the correct
- # state value and to initally send notifications to all clients
+ # Delete the notification list for statd, we dont want it to
+ # ping any clients
rm -f /var/lib/nfs/statd/sm/*
rm -f /var/lib/nfs/statd/sm.bak/*
- cat $STATD_SHARED_DIRECTORY/state >/var/lib/nfs/statd/state
-
# we must keep a monotonically increasing state variable for the entire
# cluster so state always increases when ip addresses fail from one
# node to another
- [ ! -f $STATD_SHARED_DIRECTORY/state ] && {
- echo 1 | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
- }
- # read current state
- STATE=`od -t d4 $STATD_SHARED_DIRECTORY/state | head -1 | sed -e "s/^[0-9]*[^0-9]*//"`
- # write current state+2 back to the state file
- # the /2 *2 are to ensure that state is odd. state must be odd.
- STATE=`expr $STATE "/" 2 "*" 2 "+" 3`
- echo $STATE | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
-
-
+ # We use epoch and hope the nodes are close enough in clock.
+ # Even numbers mean service is shut down, odd numbers mean
+ # service is started.
+ state_even=$(( $(date '+%s') / 2 * 2))
# we must also let some time pass between stopping and restarting the
# lockmanager since othervise there is a window where the lockmanager
# will respond "strangely" immediately after restarting it, which
# causes clients to fail to reclaim the locks.
#
- service nfslock stop > /dev/null 2>&1
- sleep 2
-
- # copy all monitored clients on this node to the local lockmanager
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
- ip=`basename $f`
- [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
- for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
- client=`basename $g`
- touch /var/lib/nfs/statd/sm/$client
- done
- }
- done
-
- # now start lockmanager again with the new state directory.
- service nfslock start > /dev/null 2>&1
+ if [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] ; then
+ startstop_nfslock stop >/dev/null 2>&1
+ sleep 2
+ startstop_nfslock start >/dev/null 2>&1
+ fi
# we now need to send out additional statd notifications to ensure
# that clients understand that the lockmanager has restarted.
# Both 2a and 2b are commonly used in lockmanagers since they maximize
# probability that the client will accept the statd notify packet and
# not just ignore it.
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
- ip=`basename $f`
- [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
- for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
- client=`basename $g`
-# /bin/rm -f $g
- # send out notifications from the "correct" address
- # (the same addresse as where the lock was taken out
- # on) some clients require that the source address
- # matches where the lock was taken out.
- # also send it both as a name that the client
- # hopefully can resolve into the server ip and
- # and also by specifying the raw ip address as name.
- /usr/bin/smnotify --client=$client --ip=$ip --server=$ip --stateval=$STATE
- /usr/bin/smnotify --client=$client --ip=$ip --server=$NFS_HOSTNAME --stateval=$STATE
- done
- }
+ # For all IPs we serve, collect info and push to the config database
+ pnn=$(ctdb xpnn | sed -e 's/.*://')
+
+ # Construct a sed expression to take catdb output and produce pairs of:
+ # server-IP client-IP
+ # but only for the server-IPs that are hosted on this node.
+ sed_expr=$(ctdb ip | tail -n +2 |
+ awk -v pnn=$pnn 'pnn == $2 { \
+ ip = $1; gsub(/\./, "\\\\.", ip); \
+ printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
+
+ statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort)
+ [ -n "$statd_state" ] || exit 0
+
+ # The following is dangerous if this script times out before
+ # all of the smnotify commands are run. Revert to individual
+ # pdelete commands for now and consider optimising smnotify to
+ # read all the data from stdin and then run it in the
+ # background.
+ #
+ # Delete all the items from the TDB
+ #if ! echo "$statd_state" | \
+ # awk '{ printf "\"statd-state@%s@%s\" \"\"\n", $1, $2 }') | \
+ # ctdb ptrans ctdb.tdb ; then
+
+ # die "Yikes!"
+ #fi
+
+ prev=""
+ echo "$statd_state" |
+ while read sip cip ; do
+ # Delete the entry from the DB
+ ctdb pdelete ctdb.tdb "statd-state@${sip}@${cip}"
+ # Reset stateval for each serverip
+ [ "$sip" = "$prev" ] || stateval="$state_even"
+ # Send notifies for server shutdown
+ smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
+ smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
+ # Send notifies for server startup
+ stateval=$(($stateval + 1))
+ smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
+ smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
done
;;
esac
git.samba.org / vlendec / samba-autobuild / .git / blobdiff