#!/bin/sh
+# This must run as root as CTDB tool commands need to access CTDB socket
+[ $(id -u) -eq 0 ] || exec sudo "$0" "$@"
+
# this script needs to be installed so that statd points to it with the -H
# command line argument. The easiest way to do that is to put something like this in
# /etc/sysconfig/nfs:
# STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
-[ -z "$CTDB_BASE" ] && {
- export CTDB_BASE="/etc/ctdb"
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") ; cd -P "$d" ; dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+# Overwrite this so we get some logging
+die ()
+{
+ script_log "statd-callout" "$@"
+ exit 1
}
-. $CTDB_BASE/functions
+loadconfig ctdb
loadconfig nfs
-[ -z "$STATD_SHARED_DIRECTORY" ] && {
- echo STATD_SHARED_DIRECTORY not configured. statd-callout failed.
- exit 0
-}
+[ -n "$NFS_HOSTNAME" ] || \
+ die "NFS_HOSTNAME is not configured. statd-callout failed"
-[ -d $STATD_SHARED_DIRECTORY ] || exit 0
+# A handy newline
+nl="
+"
-[ -z $NFS_HOSTNAME ] && {
- echo NFS_HOSTNAME is not configured. statd-callout failed.
- exit 0
-}
+ctdb_setup_service_state_dir "statd-callout"
+
+cd "$service_state_dir" || \
+ die "Failed to change directory to \"${service_state_dir}\""
case "$1" in
- add-client)
- # the callout does not tell us to which ip the client connected
- # so we must add it to all the ips that we serve
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/*`; do
- ip=`/bin/basename $f`
- [ -d $STATD_SHARED_DIRECTORY/$ip ] || /bin/mkdir $STATD_SHARED_DIRECTORY/$ip
- /bin/touch $STATD_SHARED_DIRECTORY/$ip/$2
+ # Keep a single file to keep track of the last "add-client" or
+ # "del-client'. These get pushed to ctdb.tdb during "update",
+ # which will generally be run once each "monitor" cycle. In this
+ # way we avoid scalability problems with flood of persistent
+ # transactions after a "notify" when all the clients re-take their
+ # locks.
+
+ add-client)
+ # statd does not tell us to which IP the client connected so
+ # we must add it to all the IPs that we serve
+ cip="$2"
+ ctdb_get_pnn
+ date=$(date '+%s')
+ $CTDB ip -X |
+ tail -n +2 |
+ while IFS="|" read x sip node x ; do
+ [ "$node" = "$pnn" ] || continue # not us
+ key="statd-state@${sip}@${cip}"
+ echo "\"${key}\" \"${date}\"" >"$key"
done
;;
- del-client)
- # the callout does not tell us to which ip the client connected
- # so we must add it to all the ips that we serve
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/*`; do
- ip=`/bin/basename $f`
- /bin/rm -f $STATD_SHARED_DIRECTORY/$ip/$2
+
+ del-client)
+ # statd does not tell us from which IP the client disconnected
+ # so we must add it to all the IPs that we serve
+ cip="$2"
+ ctdb_get_pnn
+ $CTDB ip -X |
+ tail -n +2 |
+ while IFS="|" read x sip node x ; do
+ [ "$node" = "$pnn" ] || continue # not us
+ key="statd-state@${sip}@${cip}"
+ echo "\"${key}\" \"\"" >"$key"
done
;;
- notify)
+
+ update)
+ files=$(echo statd-state@*)
+ if [ "$files" = "statd-state@*" ] ; then
+ # No files!
+ exit 0
+ fi
+ # Filter out lines for any IP addresses that are not currently
+ # hosted public IP addresses.
+ ctdb_get_pnn
+ ctdb_ips=$($CTDB ip | tail -n +2)
+ sed_expr=$(echo "$ctdb_ips" |
+ awk -v pnn="$pnn" 'pnn == $2 { \
+ ip = $1; gsub(/\./, "\\.", ip); \
+ printf "/statd-state@%s@/p\n", ip }')
+ if cat $files | sed -n "$sed_expr" | $CTDB ptrans "ctdb.tdb" ; then
+ rm $files
+ fi
+ ;;
+
+ notify)
# we must restart the lockmanager (on all nodes) so that we get
- # a clusterwide grace period (so other clients dont take out
+ # a clusterwide grace period (so other clients don't take out
# conflicting locks through other nodes before all locks have been
# reclaimed)
# we need these settings to make sure that no tcp connections survive
# across a very fast failover/failback
- echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
- echo 0 > /proc/sys/net/ipv4/tcp_fin_timeout
- echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
+ #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
+ #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
+ #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
- # rebuild the state directory for the local statd to use the correct
- # state value and to initally send notifications to all clients
+ # Delete the notification list for statd, we don't want it to
+ # ping any clients
rm -f /var/lib/nfs/statd/sm/*
rm -f /var/lib/nfs/statd/sm.bak/*
- cat $STATD_SHARED_DIRECTORY/state >/var/lib/nfs/statd/state
-
# we must keep a monotonically increasing state variable for the entire
# cluster so state always increases when ip addresses fail from one
# node to another
- [ ! -f $STATD_SHARED_DIRECTORY/state ] && {
- echo 1 | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
- }
- # read current state
- STATE=`od -t d4 $STATD_SHARED_DIRECTORY/state | head -1 | sed -e "s/^[0-9]*[^0-9]*//"`
- # write current state+2 back to the state file
- # the /2 *2 are to ensure that state is odd. state must be odd.
- STATE=`expr $STATE "/" 2 "*" 2 "+" 3`
- echo $STATE | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
-
-
-
- # we must also let some time pass between stopping and restarting the
- # lockmanager since othervise there is a window where the lockmanager
- # will respond "strangely" immediately after restarting it, which
- # causes clients to fail to reclaim the locks.
- #
- service nfslock stop > /dev/null 2>&1
- sleep 2
-
- # copy all monitored clients on this node to the local lockmanager
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
- ip=`/bin/basename $f`
- [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
- for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
- client=`/bin/basename $g`
- touch /var/lib/nfs/statd/sm/$client
- done
- }
- done
-
- # now start lockmanager again with the new state directory.
- service nfslock start > /dev/null 2>&1
+ # We use epoch and hope the nodes are close enough in clock.
+ # Even numbers mean service is shut down, odd numbers mean
+ # service is started.
+ state_even=$(( $(date '+%s') / 2 * 2))
+
+ # We must also let some time pass between stopping and
+ # restarting the lock manager. Otherwise there is a window
+ # where the lock manager will respond "strangely" immediately
+ # after restarting it, which causes clients to fail to reclaim
+ # their locks.
+ "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
+ sleep 2
+ "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
# we now need to send out additional statd notifications to ensure
# that clients understand that the lockmanager has restarted.
# Both 2a and 2b are commonly used in lockmanagers since they maximize
# probability that the client will accept the statd notify packet and
# not just ignore it.
- for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
- ip=`/bin/basename $f`
- [ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
- for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
- client=`/bin/basename $g`
-# /bin/rm -f $g
- # send out notifications from the "correct" address
- # (the same addresse as where the lock was taken out
- # on) some clients require that the source address
- # matches where the lock was taken out.
- # also send it both as a name that the client
- # hopefully can resolve into the server ip and
- # and also by specifying the raw ip address as name.
- /usr/bin/smnotify --client=$client --ip=$ip --server=$ip --stateval=$STATE
- /usr/bin/smnotify --client=$client --ip=$ip --server=$NFS_HOSTNAME --stateval=$STATE
- done
- }
- done
+ # For all IPs we serve, collect info and push to the config database
+ ctdb_get_pnn
+
+ # Construct a sed expression to take catdb output and produce pairs of:
+ # server-IP client-IP
+ # but only for the server-IPs that are hosted on this node.
+ ctdb_all_ips=$($CTDB ip all | tail -n +2)
+ sed_expr=$(echo "$ctdb_all_ips" |
+ awk -v pnn="$pnn" 'pnn == $2 { \
+ ip = $1; gsub(/\./, "\\.", ip); \
+ printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
+
+ statd_state=$($CTDB catdb ctdb.tdb | sed -n "$sed_expr" | sort)
+ [ -n "$statd_state" ] || exit 0
+
+ smnotify="${CTDB_HELPER_BINDIR}/smnotify"
+ prev=""
+ echo "$statd_state" | {
+ # This all needs to be in the same command group at the
+ # end of the pipe so it doesn't get lost when the loop
+ # completes.
+ items=""
+ while read sip cip ; do
+ # Collect item to delete from the DB
+ key="statd-state@${sip}@${cip}"
+ item="\"${key}\" \"\""
+ items="${items}${items:+${nl}}${item}"
+
+ # NOTE: Consider optimising smnotify to read all the
+ # data from stdin and then run it in the background.
+
+ # Reset stateval for each serverip
+ [ "$sip" = "$prev" ] || stateval="$state_even"
+ # Send notifies for server shutdown
+ "$smnotify" --client="$cip" --ip="$sip" \
+ --server="$sip" --stateval="$stateval"
+ "$smnotify" --client="$cip" --ip="$sip" \
+ --server="$NFS_HOSTNAME" --stateval="$stateval"
+ # Send notifies for server startup
+ stateval=$(($stateval + 1))
+ "$smnotify" --client="$cip" --ip="$sip" \
+ --server="$sip" --stateval="$stateval"
+ "$smnotify" --client="$cip" --ip="$sip" \
+ --server="$NFS_HOSTNAME" --stateval="$stateval"
+ done
+
+ echo "$items" | $CTDB ptrans "ctdb.tdb"
+ }
+
+ # Remove any stale touch files (i.e. for IPs not currently
+ # hosted on this node and created since the last "update").
+ # There's nothing else we can do with them at this stage.
+ echo "$ctdb_all_ips" |
+ awk -v pnn="$pnn" 'pnn != $2 { print $1 }' |
+ while read sip ; do
+ rm -f "statd-state@${sip}@"*
+ done
;;
esac
git.samba.org / vlendec / samba-autobuild / .git / blobdiff