ctdb-scripts: Quote some variable expansions
[vlendec/samba-autobuild/.git] / ctdb / config / functions
index bcef4c78b0b8cce1b38414aeb23ca877132b2edd..64b7c012d90adca8b7a4371f1518a48501e125bc 100755 (executable)
@@ -2,19 +2,56 @@
 
 # utility functions for ctdb event scripts
 
-[ -z "$CTDB_VARDIR" ] && {
-    if [ -d "/var/lib/ctdb" ] ; then
-       export CTDB_VARDIR="/var/lib/ctdb"
-    else
-       export CTDB_VARDIR="/var/ctdb"
-    fi
-}
-[ -z "$CTDB_ETCDIR" ] && {
-    export CTDB_ETCDIR="/etc"
-}
+if [ -z "$CTDB_BASE" ] ; then
+    echo 'CTDB_BASE unset in CTDB functions file'
+    exit 1
+fi
+export CTDB_BASE
+
+CTDB_VARDIR="/usr/local/var/lib/ctdb"
+ctdb_rundir="/usr/local/var/run/ctdb"
+
+CTDB="${CTDB:-/usr/local/bin/ctdb}"
+
+# Only (and always) override these variables in test code
+
+if [ -z "$CTDB_SCRIPT_VARDIR" ] ; then
+    CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/state"
+fi
+
+if [ -z "$CTDB_SYS_ETCDIR" ] ; then
+    CTDB_SYS_ETCDIR="/etc"
+fi
+
+if [ -z "$CTDB_HELPER_BINDIR" ] ; then
+    CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
+fi
 
 #######################################
 # pull in a system config file, if any
+
+rewrite_ctdb_options ()
+{
+    case "$CTDB_DBDIR" in
+       tmpfs|tmpfs:*)
+           _opts_defaults="mode=700"
+           # Get any extra options specified after colon
+           if [ "$CTDB_DBDIR" = "tmpfs" ] ; then
+               _opts=""
+           else
+               _opts="${CTDB_DBDIR#tmpfs:}"
+           fi
+           # This is an internal variable, only used by ctdbd_wrapper.
+           # It is OK to repeat mount options - last value wins
+           CTDB_DBDIR_TMPFS_OPTIONS="${_opts_defaults}${_opts:+,}${_opts}"
+
+           CTDB_DBDIR="${ctdb_rundir}/CTDB_DBDIR"
+           ;;
+       *)
+           CTDB_DBDIR_TMPFS_OPTIONS=""
+    esac
+}
+
 _loadconfig() {
 
     if [ -z "$1" ] ; then
@@ -33,19 +70,20 @@ _loadconfig() {
        return
     fi
 
-    if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
-       . $CTDB_ETCDIR/sysconfig/$1
-    elif [ -f $CTDB_ETCDIR/default/$1 ]; then
-       . $CTDB_ETCDIR/default/$1
-    elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
-       . $CTDB_BASE/sysconfig/$1
+    if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/$1" ]; then
+       . "${CTDB_SYS_ETCDIR}/sysconfig/$1"
+    elif [ -f "${CTDB_SYS_ETCDIR}/default/$1" ]; then
+       . "${CTDB_SYS_ETCDIR}/default/$1"
+    elif [ -f "${CTDB_BASE}/sysconfig/$1" ]; then
+       . "${CTDB_BASE}/sysconfig/$1"
     fi
 
     if [ "$1" = "ctdb" ] ; then
-       _config="${CTDB_BASE}/ctdbd.conf"
+       _config="${CTDBD_CONF:-${CTDB_BASE}/ctdbd.conf}"
        if [ -r "$_config" ] ; then
            . "$_config"
        fi
+       rewrite_ctdb_options
     fi
 }
 
@@ -59,7 +97,7 @@ loadconfig () {
 # configuration file.
 debug ()
 {
-    if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
+    if [ "${CTDB_SCRIPT_DEBUGLEVEL:-2}" -ge 4 ] ; then
        # If there are arguments then echo them.  Otherwise expect to
        # use stdin, which allows us to pass lots of debug using a
        # here document.
@@ -80,7 +118,7 @@ die ()
     _msg="$1"
     _rc="${2:-1}"
 
-    echo "$_msg"
+    echo "$_msg" >&2
     exit $_rc
 }
 
@@ -95,7 +133,7 @@ script_log ()
            if [ -n "$CTDB_LOGGING" ] ; then
                _file="${CTDB_LOGGING#file:}"
            else
-               _file="/var/log/log.ctdb"
+               _file="/usr/local/var/log/log.ctdb"
            fi
            {
                if [ -n "$*" ] ; then
@@ -108,7 +146,7 @@ script_log ()
        *)
            # Handle all syslog:* variants here too.  There's no tool to do
            # the lossy things, so just use logger.
-           logger -t "ctdbd: ${_tag}" $*
+           logger -t "ctdbd: ${_tag}" "$*"
            ;;
     esac
 }
@@ -177,10 +215,12 @@ _service ()
       $_nice /sbin/service "$_service_name" "$_op"
   elif [ -x /usr/sbin/service ]; then
       $_nice /usr/sbin/service "$_service_name" "$_op"
-  elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
-      $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
-  elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
-      $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
+  elif [ -x /bin/systemctl ]; then
+      $_nice /bin/systemctl "$_op" "$_service_name"
+  elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
+      $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
+  elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
+      $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
   fi
 }
 
@@ -204,14 +244,27 @@ nice_service()
 # This sets $pnn - this avoid an unnecessary subprocess.
 ctdb_get_pnn ()
 {
-    _pnn_file="$CTDB_VARDIR/state/my-pnn"
+    _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
     if [ ! -f "$_pnn_file" ] ; then
-       ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
+       $CTDB pnn | sed -e 's@.*:@@' >"$_pnn_file"
     fi
 
     read pnn <"$_pnn_file"
 }
 
+# Cached retrieval of private IP address from local node.  This never
+# changes.  Sets $ip_address to avoid an unnecessary subprocess.
+ctdb_get_ip_address ()
+{
+    _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
+    if [ ! -f "$_ip_addr_file" ] ; then
+       $CTDB -X nodestatus |
+           awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
+    fi
+
+    read ip_address <"$_ip_addr_file"
+}
+
 ######################################################
 # wrapper around /proc/ settings to allow them to be hooked
 # for testing
@@ -258,237 +311,6 @@ program_stack_traces ()
     done
 }
 
-######################################################
-# Check that an RPC service is healthy -
-# this includes allowing a certain number of failures
-# before marking the NFS service unhealthy.
-#
-# usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
-#
-# each triple is a set of 3 arguments: an operator, a 
-# fail count limit and an action string.
-#
-# For example:
-#
-#      nfs_check_rpc_service "lockd" \
-#          -ge 15 "verbose restart unhealthy" \
-#          -eq 10 "restart:bs"
-#
-# says that if lockd is down for 15 iterations then do
-# a verbose restart of lockd and mark the node unhealthy.
-# Before this, after 10 iterations of failure, the
-# service is restarted silently in the background.
-# Order is important: the number of failures need to be
-# specified in reverse order because processing stops
-# after the first condition that is true.
-######################################################
-nfs_check_rpc_service ()
-{
-    _prog_name="$1" ; shift
-
-    if _nfs_check_rpc_common "$_prog_name" ; then
-       return
-    fi
-
-    while [ -n "$3" ] ; do
-       if _nfs_check_rpc_action "$1" "$2" "$3" ; then
-           break
-       fi
-       shift 3
-    done
-}
-
-# The new way of doing things...
-nfs_check_rpc_services ()
-{
-    # Files must end with .check - avoids editor backups, RPM fu, ...
-    for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
-       _t="${_f%.check}"
-       _prog_name="${_t##*/[0-9][0-9].}"
-
-       # If $_prog_name contains '@' then the bit after it is the
-       # address family.
-       _family="${_prog_name#*@}"
-       if [ "$_family" = "$_prog_name" ] ; then
-           _family=""
-       else
-           _prog_name="${_prog_name%@*}"
-       fi
-
-       if _nfs_check_rpc_common "$_prog_name" "$_family" ; then
-           # This RPC service is up, check next service...
-           continue
-       fi
-
-       # Check each line in the file in turn until one of the limit
-       # checks is hit...
-       while read _cmp _lim _rest ; do
-           # Skip comments
-           case "$_cmp" in
-               \#*) continue ;;
-           esac
-
-           if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
-               # Limit was hit on this line, no further checking...
-               break
-           fi
-       done <"$_f"
-    done
-}
-
-_nfs_check_rpc_common ()
-{
-    _prog_name="$1"
-    _family="$2"
-
-    # Some platforms don't have separate programs for all services.
-    case "$_prog_name" in
-       statd)
-           type "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
-    esac
-
-    case "$_prog_name" in
-       nfsd)
-           _rpc_prog=nfs
-           _version=3
-           ;;
-       mountd)
-           _rpc_prog=mountd
-           _version=1
-           ;;
-       rquotad)
-           _rpc_prog=rquotad
-           _version=1
-           ;;
-       lockd)
-           _rpc_prog=nlockmgr
-           _version=4
-           ;;
-       statd)
-           _rpc_prog=status
-           _version=1
-           ;;
-       *)
-           echo "Internal error: unknown RPC program \"$_prog_name\"."
-           exit 1
-    esac
-
-    _service_name="nfs_${_prog_name}${_family:+_}${_family}"
-
-    if ctdb_check_rpc "$_rpc_prog" "$_version" "$_family" >/dev/null ; then
-       ctdb_counter_init "$_service_name"
-       return 0
-    fi
-
-    ctdb_counter_incr "$_service_name"
-
-    return 1
-}
-
-_nfs_check_rpc_action ()
-{
-    _cmp="$1"
-    _limit="$2"
-    _actions="$3"
-
-    if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
-       return 1
-    fi
-
-    for _action in $_actions ; do
-       case "$_action" in
-           verbose)
-               echo "$ctdb_check_rpc_out"
-               ;;
-           restart)
-               _nfs_restart_rpc_service "$_prog_name"
-               ;;
-           restart:b)
-               _nfs_restart_rpc_service "$_prog_name" true
-               ;;
-           unhealthy)
-               exit 1
-               ;;
-           *)
-               echo "Internal error: unknown action \"$_action\"."
-               exit 1
-       esac
-    done
-
-    return 0
-}
-
-_nfs_restart_rpc_service ()
-{
-    _prog_name="$1"
-    _background="${2:-false}"
-
-    if $_background ; then
-       _maybe_background="background_with_logging"
-    else
-       _maybe_background=""
-    fi
-
-    _p="rpc.${_prog_name}"
-
-    case "$_prog_name" in
-       nfsd)
-           echo "Trying to restart NFS service"
-           $_maybe_background startstop_nfs restart
-           ;;
-       mountd)
-           echo "Trying to restart $_prog_name [${_p}]"
-           killall -q -9 "$_p"
-           nfs_dump_some_threads "$_p"
-           $_maybe_background $_p $RPCMOUNTDOPTS \
-                              ${MOUNTD_PORT:+-p} $MOUNTD_PORT
-           ;;
-       rquotad)
-           echo "Trying to restart $_prog_name [${_p}]"
-           killall -q -9 "$_p"
-           nfs_dump_some_threads "$_p"
-           $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
-           ;;
-       lockd)
-           echo "Trying to restart lock manager service"
-           $_maybe_background startstop_nfslock restart
-           ;;
-       statd)
-           echo "Trying to restart $_prog_name [${_p}]"
-           killall -q -9 "$_p"
-           nfs_dump_some_threads "$_p"
-           $_maybe_background $_p \
-               ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
-               ${STATD_PORT:+-p} $STATD_PORT \
-               ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
-           ;;
-       *)
-           echo "Internal error: unknown RPC program \"$_prog_name\"."
-           exit 1
-    esac
-}
-
-######################################################
-# check that a rpc server is registered with portmap
-# and responding to requests
-# usage: ctdb_check_rpc SERVICE_NAME VERSION
-######################################################
-ctdb_check_rpc ()
-{
-    progname="$1"
-    version="$2"
-    _family="${3:-tcp}"
-
-    _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
-
-    if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost $progname $version 2>&1) ; then
-       ctdb_check_rpc_out="ERROR: $progname failed RPC check:
-$ctdb_check_rpc_out"
-       echo "$ctdb_check_rpc_out"
-       return 1
-    fi
-}
-
 ######################################################
 # Ensure $service_name is set
 assert_service_name ()
@@ -538,7 +360,8 @@ ctdb_check_directories()
 _ctdb_check_tcp_common ()
 {
     assert_service_name
-    _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
+    _d="${CTDB_SCRIPT_VARDIR}/failcount"
+    _ctdb_service_started_file="${_d}/${service_name}.started"
 }
 
 ctdb_check_tcp_init ()
@@ -558,7 +381,7 @@ ctdb_check_tcp_ports()
     fi
 
     for _p ; do  # process each function argument (port)
-       _cmd="ctdb checktcpport $_p"
+       _cmd="$CTDB checktcpport $_p"
        _out=$($_cmd 2>&1)
        _ret=$?
        case "$_ret" in
@@ -580,7 +403,7 @@ ctdb_check_tcp_ports()
            *)
                echo "ERROR: unexpected error running \"ctdb checktcpport\""
                debug <<EOF
-ctdb checktcpport (exited with $_ret) with output:
+$CTDB checktcpport (exited with $_ret) with output:
 $_out"
 EOF
                return $_ret
@@ -625,10 +448,11 @@ ctdb_check_command ()
 ################################################
 kill_tcp_connections ()
 {
-    _ip="$1"
+    _iface="$1"
+    _ip="$2"
 
     _oneway=false
-    if [ "$2" = "oneway" ] ; then
+    if [ "$3" = "oneway" ] ; then
        _oneway=true
     fi
 
@@ -658,29 +482,21 @@ kill_tcp_connections ()
            return
        fi
 
-       echo "$_connections" | ctdb killtcp || {
-           echo "Failed to send killtcp control"
-           return
+       echo "$_connections" | \
+               "${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
+               echo "Failed to kill TCP connections"
+               return
        }
 
-       _count=0
-       while : ; do
-           _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
+       _remaining=$(get_tcp_connections_for_ip "$_ip" | wc -l)
 
-           if [ $_remaining -eq 0 ] ; then
+       if [ $_remaining -eq 0 ] ; then
                echo "Killed $_killcount TCP connections to released IP $_ip"
                return
-           fi
-
-           _count=$(($_count + 1))
-           if [ $_count -gt 3 ] ; then
-               echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
-               return
-           fi
+       fi
 
-           echo "Waiting for $_remaining connections to be killed for IP $_ip"
-           sleep 1
-       done
+       _t="${_remaining}/${_killcount}"
+       echo "Failed to kill TCP connections for IP $_ip (${_t} remaining)"
     }
 }
 
@@ -689,7 +505,7 @@ kill_tcp_connections ()
 ##################################################################
 kill_tcp_connections_local_only ()
 {
-    kill_tcp_connections "$1" "oneway"
+    kill_tcp_connections "$@" "oneway"
 }
 
 ##################################################################
@@ -705,9 +521,9 @@ tickle_tcp_connections ()
 
        while read dest src; do
            echo "Tickle TCP connection $src $dest"
-           ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
+           $CTDB tickle "$src" "$dest" >/dev/null 2>&1 || _failed=true
            echo "Tickle TCP connection $dest $src"
-           ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
+           $CTDB tickle "$dest" "$src" >/dev/null 2>&1 || _failed=true
        done
 
        if $_failed ; then
@@ -720,163 +536,7 @@ get_tcp_connections_for_ip ()
 {
     _ip="$1"
 
-    netstat -tn | awk -v ip=$_ip \
-       'index($1, "tcp") == 1 && \
-        (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
-        && $6 == "ESTABLISHED" \
-        {print $4" "$5}'
-}
-
-##################################################################
-# use statd-callout to update NFS lock info
-##################################################################
-nfs_update_lock_info ()
-{
-    if [ -x "$CTDB_BASE/statd-callout" ] ; then
-       "$CTDB_BASE/statd-callout" update
-    fi
-}
-
-########################################################
-# start/stop the Ganesha nfs service
-########################################################
-startstop_ganesha()
-{
-    _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
-    case "$1" in
-       start)
-           service "$_service_name" start
-           ;;
-       stop)
-           service "$_service_name" stop
-           ;;
-       restart)
-           service "$_service_name" stop
-           nfs_dump_some_threads "rpc.statd"
-           service "$_service_name" start
-           ;;
-    esac
-}
-
-########################################################
-# start/stop the nfs service on different platforms
-########################################################
-startstop_nfs() {
-       PLATFORM="unknown"
-       [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
-               PLATFORM="sles"
-       }
-       [ -x $CTDB_ETCDIR/init.d/nfslock -o \
-           -r /usr/lib/systemd/system/nfs-lock.service ] && {
-               PLATFORM="rhel"
-       }
-
-       case $PLATFORM in
-       sles)
-               case $1 in
-               start)
-                       service nfsserver start
-                       ;;
-               stop)
-                       service nfsserver stop > /dev/null 2>&1
-                       ;;
-               restart)
-                       set_proc "fs/nfsd/threads" 0
-                       service nfsserver stop > /dev/null 2>&1
-                       pkill -9 nfsd
-                       nfs_dump_some_threads
-                       service nfsserver start
-                       ;;
-               esac
-               ;;
-       rhel)
-               case $1 in
-               start)
-                       service nfslock start
-                       service nfs start
-                       ;;
-               stop)
-                       service nfs stop
-                       service nfslock stop
-                       ;;
-               restart)
-                       set_proc "fs/nfsd/threads" 0
-                       service nfs stop > /dev/null 2>&1
-                       service nfslock stop > /dev/null 2>&1
-                       pkill -9 nfsd
-                       nfs_dump_some_threads
-                       service nfslock start
-                       service nfs start
-                       ;;
-               esac
-               ;;
-       *)
-               echo "Unknown platform. NFS is not supported with ctdb"
-               exit 1
-               ;;
-       esac
-}
-
-# Dump up to the configured number of nfsd thread backtraces.
-nfs_dump_some_threads ()
-{
-    _prog="${1:-nfsd}"
-
-    _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
-    [ $_num -gt 0 ] || return 0
-
-    program_stack_traces "$_prog" $_num
-}
-
-########################################################
-# start/stop the nfs lockmanager service on different platforms
-########################################################
-startstop_nfslock() {
-       PLATFORM="unknown"
-       [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
-               PLATFORM="sles"
-       }
-       [ -x $CTDB_ETCDIR/init.d/nfslock -o \
-           -r /usr/lib/systemd/system/nfs-lock.service ] && {
-               PLATFORM="rhel"
-       }
-
-       case $PLATFORM in
-       sles)
-               # for sles there is no service for lockmanager
-               # so we instead just shutdown/restart nfs
-               case $1 in
-               start)
-                       service nfsserver start
-                       ;;
-               stop)
-                       service nfsserver stop > /dev/null 2>&1
-                       ;;
-               restart)
-                       service nfsserver stop > /dev/null 2>&1
-                       service nfsserver start
-                       ;;
-               esac
-               ;;
-       rhel)
-               case $1 in
-               start)
-                       service nfslock start
-                       ;;
-               stop)
-                       service nfslock stop > /dev/null 2>&1
-                       ;;
-               restart)
-                       service nfslock stop > /dev/null 2>&1
-                       service nfslock start
-                       ;;
-               esac
-               ;;
-       *)
-               echo "Unknown platform. NFS locking is not supported with ctdb"
-               exit 1
-               ;;
-       esac
+    ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
 }
 
 ########################################################
@@ -892,7 +552,7 @@ add_ip_to_iface ()
        die "Failed to bringup interface $_iface"
 
     # Only need to define broadcast for IPv4
-    case "$ip" in
+    case "$_ip" in
         *:*) _bcast=""      ;;
        *)   _bcast="brd +" ;;
     esac
@@ -951,27 +611,27 @@ ip_maskbits_iface ()
     _addr="$1"
 
     case "$_addr" in
-       *:*) _family="inet6" ; _bits=128 ;;
-       *)   _family="inet"  ; _bits=32  ;;
+       *:*) _bits=128 ;;
+       *)   _bits=32  ;;
     esac
 
     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
-       awk -v family="${_family}" \
-           'NR == 1 { iface = $2; sub(":$", "", iface) } \
+       awk 'NR == 1 { iface = $2; sub(":$", "", iface) ; \
+                      sub("@.*", "", iface) } \
              $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
-                           print mask, iface, family }'
+                           print mask, iface }'
 }
 
 drop_ip ()
 {
     _addr="${1%/*}"  # Remove optional maskbits
 
-    set -- $(ip_maskbits_iface $_addr)
+    set -- $(ip_maskbits_iface "$_addr")
     if [ -n "$1" ] ; then
        _maskbits="$1"
        _iface="$2"
        echo "Removing public address $_addr/$_maskbits from device $_iface"
-       delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
+       delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
     fi
 }
 
@@ -988,11 +648,105 @@ flush_route_cache ()
     set_proc_maybe sys/net/ipv6/route/flush 1
 }
 
+########################################################
+# Interface monitoring
+
+# If the interface is a virtual one (e.g. VLAN) then get the
+# underlying interface
+interface_get_real ()
+{
+    # Output of "ip link show <iface>"
+    _iface_info="$1"
+
+    # Extract the full interface description to see if it is a VLAN
+    _t=$(echo "$_iface_info" |
+               awk 'NR == 1 { iface = $2; sub(":$", "", iface) ; \
+                              print iface }')
+    case "$_t" in
+       *@*)
+           # VLAN: use the underlying interface, after the '@'
+           echo "${_t##*@}"
+           ;;
+       *)
+           # Not a regular VLAN.  For backward compatibility, assume
+           # there is some other sort of VLAN that doesn't have the
+           # '@' in the output and only use what is before a '.'.  If
+           # there is no '.' then this will be the whole interface
+           # name.
+           echo "${_t%%.*}"
+    esac
+}
+
+# Check whether an interface is operational
+interface_monitor ()
+{
+    _iface="$1"
+
+    _iface_info=$(ip link show "$_iface" 2>&1) || {
+       echo "ERROR: Monitored interface ${_iface} does not exist"
+       return 1
+    }
+
+
+    # If the interface is a virtual one (e.g. VLAN) then get the
+    # underlying interface.
+    _realiface=$(interface_get_real "$_iface_info")
+
+    if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null) ; then
+       # This is a bond: various monitoring strategies
+       echo "$_bi" | grep -q 'Currently Active Slave: None' && {
+           echo "ERROR: No active slaves for bond device ${_realiface}"
+           return 1
+       }
+       echo "$_bi" | grep -q '^MII Status: up' || {
+           echo "ERROR: public network interface ${_realiface} is down"
+           return 1
+       }
+       echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
+           # This works around a bug in the driver where the
+           # overall bond status can be up but none of the actual
+           # physical interfaces have a link.
+           echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
+               echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
+               return 1
+           }
+       }
+
+       return 0
+    else
+       # Not a bond
+       case "$_iface" in
+           lo*)
+               # loopback is always working
+               return 0
+               ;;
+           ib*)
+               # we don't know how to test ib links
+               return 0
+               ;;
+           *)
+               ethtool "$_iface" | grep -q 'Link detected: yes' || {
+                   # On some systems, this is not successful when a
+                   # cable is plugged but the interface has not been
+                   # brought up previously. Bring the interface up
+                   # and try again...
+                   ip link set "$_iface" up
+                   ethtool "$_iface" | grep -q 'Link detected: yes' || {
+                       echo "ERROR: No link on the public network interface ${_iface}"
+                       return 1
+                   }
+               }
+               return 0
+               ;;
+       esac
+    fi
+}
+
 ########################################################
 # Simple counters
 _ctdb_counter_common () {
     _service_name="${1:-${service_name:-${script_name}}}"
-    _counter_file="$ctdb_fail_dir/$_service_name"
+    _counter_file="${CTDB_SCRIPT_VARDIR}/failcount/${_service_name}"
     mkdir -p "${_counter_file%/*}" # dirname
 }
 ctdb_counter_init () {
@@ -1006,15 +760,19 @@ ctdb_counter_incr () {
     # unary counting!
     echo -n 1 >> "$_counter_file"
 }
+ctdb_counter_get () {
+    _ctdb_counter_common "$1"
+    # unary counting!
+    stat -c "%s" "$_counter_file" 2>/dev/null || echo 0
+}
 ctdb_check_counter () {
     _msg="${1:-error}"  # "error"  - anything else is silent on fail
     _op="${2:--ge}"  # an integer operator supported by test
     _limit="${3:-${service_fail_limit}}"
     shift 3
-    _ctdb_counter_common "$1"
 
-    # unary counting!
-    _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
+    _size=$(ctdb_counter_get "$1")
+
     _hit=false
     if [ "$_op" != "%" ] ; then
        if [ $_size $_op $_limit ] ; then
@@ -1037,12 +795,9 @@ ctdb_check_counter () {
 
 ########################################################
 
-ctdb_status_dir="$CTDB_VARDIR/state/service_status"
-ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
-
 ctdb_setup_service_state_dir ()
 {
-    service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
+    service_state_dir="${CTDB_SCRIPT_VARDIR}/service_state/${1:-${service_name}}"
     mkdir -p "$service_state_dir" || {
        echo "Error creating state dir \"$service_state_dir\""
        exit 1
@@ -1052,17 +807,15 @@ ctdb_setup_service_state_dir ()
 ########################################################
 # Managed status history, for auto-start/stop
 
-ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
-
 _ctdb_managed_common ()
 {
-    _ctdb_managed_file="$ctdb_managed_dir/$service_name"
+    _ctdb_managed_file="${CTDB_SCRIPT_VARDIR}/managed_history/${service_name}"
 }
 
 ctdb_service_managed ()
 {
     _ctdb_managed_common
-    mkdir -p "$ctdb_managed_dir"
+    mkdir -p "${_ctdb_managed_file%/*}" # dirname
     touch "$_ctdb_managed_file"
 }
 
@@ -1078,49 +831,12 @@ is_ctdb_previously_managed_service ()
     [ -f "$_ctdb_managed_file" ]
 }
 
-########################################################
-# Check and set status
-
-log_status_cat ()
-{
-    echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
-}
-
-ctdb_checkstatus ()
-{
-    if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
-       log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
-       return 1
-    elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
-       log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
-       return 2
-    else
-       return 0
-    fi
-}
-
-ctdb_setstatus ()
-{
-    d="$ctdb_status_dir/$script_name"
-    case "$1" in
-       unhealthy|banned)
-           mkdir -p "$d"
-           cat "$2" >"$d/$1"
-           ;;
-       *)
-           for i in "banned" "unhealthy" ; do
-               rm -f "$d/$i"
-           done
-           ;;
-    esac
-}
-
 ##################################################################
 # Reconfigure a service on demand
 
 _ctdb_service_reconfigure_common ()
 {
-    _d="$ctdb_status_dir/${service_name}"
+    _d="${CTDB_SCRIPT_VARDIR}/service_status/${service_name}"
     mkdir -p "$_d"
     _ctdb_service_reconfigure_flag="$_d/reconfigure"
 }
@@ -1196,7 +912,7 @@ ctdb_replay_monitor_status ()
 {
     echo "Replaying previous status for this script due to reconfigure..."
     # Leading separator ('|') is missing in some versions...
-    _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
+    _out=$($CTDB scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
     # Output looks like this:
     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
     # This is the cheapest way of getting fields in the middle.
@@ -1318,7 +1034,6 @@ is_ctdb_managed_service ()
     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
-    ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
 
     t=" $CTDB_MANAGED_SERVICES "
 
@@ -1410,17 +1125,7 @@ service_stop ()
 
 ctdb_standard_event_handler ()
 {
-    case "$1" in
-       status)
-           ctdb_checkstatus
-           exit
-           ;;
-       setstatus)
-            shift
-           ctdb_setstatus "$@"
-           exit
-           ;;
-    esac
+    :
 }
 
 iptables_wrapper ()
@@ -1433,7 +1138,7 @@ iptables_wrapper ()
     fi
 
     # iptables doesn't like being re-entered, so flock-wrap it.
-    flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
+    flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
 }
 
 # AIX (and perhaps others?) doesn't have mktemp
@@ -1462,6 +1167,69 @@ if ! type mktemp >/dev/null 2>&1 ; then
     }
 fi
 
+######################################################################
+# NFS callout handling
+
+nfs_callout_init ()
+{
+       if [ -z "$CTDB_NFS_CALLOUT" ] ; then
+               CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
+       fi
+       # Always export, for statd callout
+       export CTDB_NFS_CALLOUT
+
+       # If the callout wants to use this then it must create it
+       export CTDB_NFS_CALLOUT_STATE_DIR="${service_state_dir}/callout-state"
+
+       # Export, if set, for use by clustered NFS callouts
+       if [ -n "$CTDB_NFS_STATE_FS_TYPE" ] ; then
+               export CTDB_NFS_STATE_FS_TYPE
+       fi
+       if [ -n "$CTDB_NFS_STATE_MNT" ] ; then
+               export CTDB_NFS_STATE_MNT
+       fi
+
+       nfs_callout_cache="${service_state_dir}/nfs_callout_cache"
+       nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
+       nfs_callout_cache_ops="${nfs_callout_cache}/ops"
+}
+
+nfs_callout_register ()
+{
+    mkdir -p "$nfs_callout_cache_ops"
+    rm -f "$nfs_callout_cache_ops"/*
+
+    echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
+
+    _t=$(eval "$CTDB_NFS_CALLOUT" "register")
+    if [ -n "$_t" ] ; then
+       echo "$_t" |
+           while IFS="" read _op ; do
+               touch "${nfs_callout_cache_ops}/${_op}"
+           done
+    else
+       touch "${nfs_callout_cache_ops}/ALL"
+    fi
+}
+
+nfs_callout ()
+{
+    # Re-run registration if $CTDB_NFS_CALLOUT has changed
+    _prev=""
+    if [ -r "$nfs_callout_cache_callout" ] ; then
+       read _prev <"$nfs_callout_cache_callout"
+    fi
+    if [ "$CTDB_NFS_CALLOUT" != "$_prev" ] ; then
+       nfs_callout_register
+    fi
+
+    # Run the operation if it is registered...
+    if [ -e "${nfs_callout_cache_ops}/${1}" ] || \
+          [ -e "${nfs_callout_cache_ops}/ALL" ]; then
+       eval "$CTDB_NFS_CALLOUT" "$@"
+    fi
+}
+
 ########################################################
 # tickle handling
 ########################################################
@@ -1470,30 +1238,38 @@ update_tickles ()
 {
        _port="$1"
 
-       tickledir="$CTDB_VARDIR/state/tickles"
+       tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
        mkdir -p "$tickledir"
 
        ctdb_get_pnn
 
        # What public IPs do I hold?
-       _ips=$(ctdb -X ip | awk -F'|' -v pnn=$pnn '$3 == pnn {print $2}')
-
-       # IPs as a regexp choice
-       _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
+       _ips=$($CTDB -X ip | awk -F'|' -v pnn="$pnn" '$3 == pnn {print $2}')
 
-       # Record connections to our public IPs in a temporary file
-       _my_connections="${tickledir}/${_port}.connections"
-       rm -f "$_my_connections"
-       netstat -tn |
-       awk -v destpat="^${_ipschoice}:${_port}\$" \
-         '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
+       # IPs and port as ss filters
+       _ip_filter=""
+       for _ip in $_ips ; do
+           _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
+       done
+       _port_filter="sport == :${_port}"
+
+       # Record connections to our public IPs in a temporary file.
+       # This temporary file is in CTDB's private state directory and
+       # $$ is used to avoid a very rare race involving CTDB's script
+       # debugging.  No security issue, nothing to see here...
+       _my_connections="${tickledir}/${_port}.connections.$$"
+       # Parentheses are needed around the filters for precedence but
+       # the parentheses can't be empty!
+       ss -tn state established \
+          "${_ip_filter:+( ${_ip_filter} )}" \
+          "${_port_filter:+( ${_port_filter} )}" |
+       awk 'NR > 1 {print $4, $3}' |
        sort >"$_my_connections"
 
        # Record our current tickles in a temporary file
-       _my_tickles="${tickledir}/${_port}.tickles"
-       rm -f "$_my_tickles"
+       _my_tickles="${tickledir}/${_port}.tickles.$$"
        for _i in $_ips ; do
-               ctdb -X gettickles $_i $_port |
+               $CTDB -X gettickles "$_i" "$_port" |
                awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
        done |
        sort >"$_my_tickles"
@@ -1501,16 +1277,19 @@ update_tickles ()
        # Add tickles for connections that we haven't already got tickles for
        comm -23 "$_my_connections" "$_my_tickles" |
        while read _src _dst ; do
-               ctdb addtickle $_src $_dst
+               $CTDB addtickle "$_src" "$_dst"
        done
 
        # Remove tickles for connections that are no longer there
        comm -13 "$_my_connections" "$_my_tickles" |
        while read _src _dst ; do
-               ctdb deltickle $_src $_dst
+               $CTDB deltickle "$_src" "$_dst"
        done
 
-       rm -f "$_my_connections" "$_my_tickles" 
+       rm -f "$_my_connections" "$_my_tickles"
+
+       # Remove stale files from killed scripts
+       find "$tickledir" -type f -mmin +10 | xargs -r rm
 }
 
 ########################################################
@@ -1521,12 +1300,12 @@ update_tickles ()
        . "$CTDB_RC_LOCAL"
 }
 
-[ -x $CTDB_BASE/rc.local ] && {
-       . $CTDB_BASE/rc.local
+[ -x "${CTDB_BASE}/rc.local" ] && {
+       . "${CTDB_BASE}/rc.local"
 }
 
-[ -d $CTDB_BASE/rc.local.d ] && {
-       for i in $CTDB_BASE/rc.local.d/* ; do
+[ -d "${CTDB_BASE}/rc.local.d" ] && {
+       for i in "${CTDB_BASE}/rc.local.d"/* ; do
                [ -x "$i" ] && . "$i"
        done
 }