ctdb-scripts: Quote some variable expansions
[vlendec/samba-autobuild/.git] / ctdb / config / functions
index 6f972711d2547409796d05330a000c19e5debea0..64b7c012d90adca8b7a4371f1518a48501e125bc 100755 (executable)
@@ -2,19 +2,56 @@
 
 # utility functions for ctdb event scripts
 
-[ -z "$CTDB_VARDIR" ] && {
-    if [ -d "/var/lib/ctdb" ] ; then
-       export CTDB_VARDIR="/var/lib/ctdb"
-    else
-       export CTDB_VARDIR="/var/ctdb"
-    fi
-}
-[ -z "$CTDB_ETCDIR" ] && {
-    export CTDB_ETCDIR="/etc"
-}
+if [ -z "$CTDB_BASE" ] ; then
+    echo 'CTDB_BASE unset in CTDB functions file'
+    exit 1
+fi
+export CTDB_BASE
+
+CTDB_VARDIR="/usr/local/var/lib/ctdb"
+ctdb_rundir="/usr/local/var/run/ctdb"
+
+CTDB="${CTDB:-/usr/local/bin/ctdb}"
+
+# Only (and always) override these variables in test code
+
+if [ -z "$CTDB_SCRIPT_VARDIR" ] ; then
+    CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/state"
+fi
+
+if [ -z "$CTDB_SYS_ETCDIR" ] ; then
+    CTDB_SYS_ETCDIR="/etc"
+fi
+
+if [ -z "$CTDB_HELPER_BINDIR" ] ; then
+    CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
+fi
 
 #######################################
 # pull in a system config file, if any
+
+rewrite_ctdb_options ()
+{
+    case "$CTDB_DBDIR" in
+       tmpfs|tmpfs:*)
+           _opts_defaults="mode=700"
+           # Get any extra options specified after colon
+           if [ "$CTDB_DBDIR" = "tmpfs" ] ; then
+               _opts=""
+           else
+               _opts="${CTDB_DBDIR#tmpfs:}"
+           fi
+           # This is an internal variable, only used by ctdbd_wrapper.
+           # It is OK to repeat mount options - last value wins
+           CTDB_DBDIR_TMPFS_OPTIONS="${_opts_defaults}${_opts:+,}${_opts}"
+
+           CTDB_DBDIR="${ctdb_rundir}/CTDB_DBDIR"
+           ;;
+       *)
+           CTDB_DBDIR_TMPFS_OPTIONS=""
+    esac
+}
+
 _loadconfig() {
 
     if [ -z "$1" ] ; then
@@ -33,19 +70,20 @@ _loadconfig() {
        return
     fi
 
-    if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
-       . $CTDB_ETCDIR/sysconfig/$1
-    elif [ -f $CTDB_ETCDIR/default/$1 ]; then
-       . $CTDB_ETCDIR/default/$1
-    elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
-       . $CTDB_BASE/sysconfig/$1
+    if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/$1" ]; then
+       . "${CTDB_SYS_ETCDIR}/sysconfig/$1"
+    elif [ -f "${CTDB_SYS_ETCDIR}/default/$1" ]; then
+       . "${CTDB_SYS_ETCDIR}/default/$1"
+    elif [ -f "${CTDB_BASE}/sysconfig/$1" ]; then
+       . "${CTDB_BASE}/sysconfig/$1"
     fi
 
     if [ "$1" = "ctdb" ] ; then
-       _config="${CTDB_BASE}/ctdbd.conf"
+       _config="${CTDBD_CONF:-${CTDB_BASE}/ctdbd.conf}"
        if [ -r "$_config" ] ; then
            . "$_config"
        fi
+       rewrite_ctdb_options
     fi
 }
 
@@ -59,7 +97,7 @@ loadconfig () {
 # configuration file.
 debug ()
 {
-    if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
+    if [ "${CTDB_SCRIPT_DEBUGLEVEL:-2}" -ge 4 ] ; then
        # If there are arguments then echo them.  Otherwise expect to
        # use stdin, which allows us to pass lots of debug using a
        # here document.
@@ -80,7 +118,7 @@ die ()
     _msg="$1"
     _rc="${2:-1}"
 
-    echo "$_msg"
+    echo "$_msg" >&2
     exit $_rc
 }
 
@@ -95,7 +133,7 @@ script_log ()
            if [ -n "$CTDB_LOGGING" ] ; then
                _file="${CTDB_LOGGING#file:}"
            else
-               _file="/var/log/log.ctdb"
+               _file="/usr/local/var/log/log.ctdb"
            fi
            {
                if [ -n "$*" ] ; then
@@ -108,7 +146,7 @@ script_log ()
        *)
            # Handle all syslog:* variants here too.  There's no tool to do
            # the lossy things, so just use logger.
-           logger -t "ctdbd: ${_tag}" $*
+           logger -t "ctdbd: ${_tag}" "$*"
            ;;
     esac
 }
@@ -177,10 +215,12 @@ _service ()
       $_nice /sbin/service "$_service_name" "$_op"
   elif [ -x /usr/sbin/service ]; then
       $_nice /usr/sbin/service "$_service_name" "$_op"
-  elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
-      $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
-  elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
-      $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
+  elif [ -x /bin/systemctl ]; then
+      $_nice /bin/systemctl "$_op" "$_service_name"
+  elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
+      $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
+  elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
+      $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
   fi
 }
 
@@ -204,14 +244,27 @@ nice_service()
 # This sets $pnn - this avoid an unnecessary subprocess.
 ctdb_get_pnn ()
 {
-    _pnn_file="$CTDB_VARDIR/state/my-pnn"
+    _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
     if [ ! -f "$_pnn_file" ] ; then
-       ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
+       $CTDB pnn | sed -e 's@.*:@@' >"$_pnn_file"
     fi
 
     read pnn <"$_pnn_file"
 }
 
+# Cached retrieval of private IP address from local node.  This never
+# changes.  Sets $ip_address to avoid an unnecessary subprocess.
+ctdb_get_ip_address ()
+{
+    _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
+    if [ ! -f "$_ip_addr_file" ] ; then
+       $CTDB -X nodestatus |
+           awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
+    fi
+
+    read ip_address <"$_ip_addr_file"
+}
+
 ######################################################
 # wrapper around /proc/ settings to allow them to be hooked
 # for testing
@@ -258,199 +311,6 @@ program_stack_traces ()
     done
 }
 
-######################################################
-# Check the health of NFS services
-#
-# Use .check files in given directory.
-# Default is "${CTDB_BASE}/nfs-checks.d/"
-######################################################
-nfs_check_services ()
-{
-    _dir="${1:-${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}}"
-
-    # Files must end with .check - avoids editor backups, RPM fu, ...
-    for _f in "$_dir"/[0-9][0-9].*.check ; do
-       _t="${_f%.check}"
-       _progname="${_t##*/[0-9][0-9].}"
-
-       nfs_check_service "$_progname" <"$_f"
-    done
-}
-
-######################################################
-# Check the health of an NFS service
-#
-# $1 - progname, passed to rpcinfo (looked up in /etc/rpc)
-#
-# Reads variables from stdin
-#
-# Variables are:
-#
-# * family             - "tcp" or "udp" or space separated list
-#                        default: tcp, not used with "service_check_cmd"
-# * version            - optional, RPC service version number
-#                        default is to omit to check for any version,
-#                        not used with "service_check_cmd"
-# * unhealthy_after    - number of check fails before unhealthy
-#                        default: 1
-# * restart_every      - number of check fails before restart
-#                        default: 0, meaning no restart
-# * service_stop_cmd   - command to stop service
-#                        default: no default, must be provided if
-#                                 restart_every > 0
-# * service_start_cmd  - command to start service
-#                        default: no default, must be provided if
-#                                 restart_every > 0
-# * service_check_cmd  - command to check health of service
-#                        default is to check RPC service using rpcinfo
-# * service_debug_cmd  - command to debug a service after trying to stop it;
-#                        for example, it can be useful to print stack
-#                        traces of threads that have not exited, since
-#                        they may be stuck doing I/O;
-#                        no default, see also function program_stack_traces()
-#
-# Quoting in values is not preserved
-#
-######################################################
-nfs_check_service ()
-{
-    _progname="$1"
-
-    (
-       # Subshell to restrict scope variables...
-
-       # Defaults
-       family="tcp"
-       version=""
-       unhealthy_after=1
-       restart_every=0
-       service_stop_cmd=""
-       service_start_cmd=""
-       service_check_cmd=""
-       service_debug_cmd=""
-
-       # Eval line-by-line.  Expands variable references in values.
-       # Also allows variable name checking, which seems useful.
-       while read _line ; do
-           case "$_line" in
-               \#*|"") : ;; # Ignore comments, blank lines
-
-               family=*|version=*|\
-               unhealthy_after=*|restart_every=*|\
-               service_stop_cmd=*|service_start_cmd=*|\
-               service_check_cmd=*|service_debug_cmd=*)
-
-                   eval "$_line"
-                   ;;
-               *)
-                   echo "ERROR: Unknown variable for ${_progname}: ${_line}"
-                   exit 1
-           esac
-       done
-
-       _service_name="nfs_${_progname}"
-
-       _ok=false
-       if [ -n "$service_check_cmd" ] ; then
-           # Using eval means variables can contain semicolon separated commands
-           if eval "$service_check_cmd" ; then
-               _ok=true
-           fi
-       else
-           if nfs_check_rpcinfo \
-                  "$_progname" "$version" "$family" >/dev/null ; then
-               _ok=true
-           fi
-       fi
-
-       if $_ok ; then
-           if [ $unhealthy_after -ne 1 -o $restart_every -ne 0 ] ; then
-               ctdb_counter_init "$_service_name"
-           fi
-           exit 0
-       fi
-
-       ctdb_counter_incr "$_service_name"
-       _failcount=$(ctdb_counter_get "$_service_name")
-
-       _unhealthy=false
-       if [ $unhealthy_after -gt 0 ] ; then
-           if [ $_failcount -ge $unhealthy_after ] ; then
-               _unhealthy=true
-               echo "ERROR: $ctdb_check_rpc_out"
-           fi
-       fi
-
-       if [ $restart_every -gt 0 ] ; then
-           if [ $(($_failcount % $restart_every)) -eq 0 ] ; then
-               if ! $_unhealthy ; then
-                   echo "WARNING: $ctdb_check_rpc_out"
-               fi
-               nfs_restart_service
-           fi
-       fi
-
-       if $_unhealthy ; then
-           exit 1
-       fi
-
-       return 0
-    ) || exit 1
-}
-
-# Uses: stop_service, start_service, debug_stuck_threads
-nfs_restart_service ()
-{
-    if [ -z "$service_stop_cmd" -o -z "$service_start_cmd" ] ; then
-       die "ERROR: Can not restart service \"${_progname}\" without corresponding service_start_cmd/service_stop_cmd settings"
-    fi
-
-    echo "Trying to restart service \"${_progname}\"..."
-    # Using eval means variables can contain semicolon separated commands
-    eval "$service_stop_cmd"
-    if [ -n "$service_debug_cmd" ] ; then
-       eval "$service_debug_cmd"
-    fi
-    background_with_logging eval "$service_start_cmd"
-}
-
-######################################################
-# Check an RPC service with rpcinfo
-######################################################
-ctdb_check_rpc ()
-{
-    _progname="$1"        # passed to rpcinfo (looked up in /etc/rpc)
-    _version="$2"         # optional, not passed if empty/unset
-    _family="${3:-tcp}"   # optional, default is "tcp"
-
-    _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
-
-    if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost \
-                                     $_progname $_version 2>&1) ; then
-       ctdb_check_rpc_out="$_progname failed RPC check:
-$ctdb_check_rpc_out"
-       echo "$ctdb_check_rpc_out"
-       return 1
-    fi
-}
-
-nfs_check_rpcinfo ()
-{
-    _progname="$1"        # passed to rpcinfo (looked up in /etc/rpc)
-    _versions="$2"        # optional, space separated, not passed if empty/unset
-    _families="${3:-tcp}" # optional, space separated, default is "tcp"
-
-    for _family in $_families ; do
-       if [ -n "$_versions" ] ; then
-           for _version in $_versions ; do
-               ctdb_check_rpc $_progname $_version $_family || return $?
-           done
-       else
-           ctdb_check_rpc $_progname "" $_family || return $?
-       fi
-    done
-}
-
 ######################################################
 # Ensure $service_name is set
 assert_service_name ()
@@ -500,7 +360,8 @@ ctdb_check_directories()
 _ctdb_check_tcp_common ()
 {
     assert_service_name
-    _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
+    _d="${CTDB_SCRIPT_VARDIR}/failcount"
+    _ctdb_service_started_file="${_d}/${service_name}.started"
 }
 
 ctdb_check_tcp_init ()
@@ -520,7 +381,7 @@ ctdb_check_tcp_ports()
     fi
 
     for _p ; do  # process each function argument (port)
-       _cmd="ctdb checktcpport $_p"
+       _cmd="$CTDB checktcpport $_p"
        _out=$($_cmd 2>&1)
        _ret=$?
        case "$_ret" in
@@ -542,7 +403,7 @@ ctdb_check_tcp_ports()
            *)
                echo "ERROR: unexpected error running \"ctdb checktcpport\""
                debug <<EOF
-ctdb checktcpport (exited with $_ret) with output:
+$CTDB checktcpport (exited with $_ret) with output:
 $_out"
 EOF
                return $_ret
@@ -587,10 +448,11 @@ ctdb_check_command ()
 ################################################
 kill_tcp_connections ()
 {
-    _ip="$1"
+    _iface="$1"
+    _ip="$2"
 
     _oneway=false
-    if [ "$2" = "oneway" ] ; then
+    if [ "$3" = "oneway" ] ; then
        _oneway=true
     fi
 
@@ -620,29 +482,21 @@ kill_tcp_connections ()
            return
        fi
 
-       echo "$_connections" | ctdb killtcp || {
-           echo "Failed to send killtcp control"
-           return
+       echo "$_connections" | \
+               "${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
+               echo "Failed to kill TCP connections"
+               return
        }
 
-       _count=0
-       while : ; do
-           _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
+       _remaining=$(get_tcp_connections_for_ip "$_ip" | wc -l)
 
-           if [ $_remaining -eq 0 ] ; then
+       if [ $_remaining -eq 0 ] ; then
                echo "Killed $_killcount TCP connections to released IP $_ip"
                return
-           fi
-
-           _count=$(($_count + 1))
-           if [ $_count -gt 3 ] ; then
-               echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
-               return
-           fi
+       fi
 
-           echo "Waiting for $_remaining connections to be killed for IP $_ip"
-           sleep 1
-       done
+       _t="${_remaining}/${_killcount}"
+       echo "Failed to kill TCP connections for IP $_ip (${_t} remaining)"
     }
 }
 
@@ -651,7 +505,7 @@ kill_tcp_connections ()
 ##################################################################
 kill_tcp_connections_local_only ()
 {
-    kill_tcp_connections "$1" "oneway"
+    kill_tcp_connections "$@" "oneway"
 }
 
 ##################################################################
@@ -667,9 +521,9 @@ tickle_tcp_connections ()
 
        while read dest src; do
            echo "Tickle TCP connection $src $dest"
-           ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
+           $CTDB tickle "$src" "$dest" >/dev/null 2>&1 || _failed=true
            echo "Tickle TCP connection $dest $src"
-           ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
+           $CTDB tickle "$dest" "$src" >/dev/null 2>&1 || _failed=true
        done
 
        if $_failed ; then
@@ -682,21 +536,7 @@ get_tcp_connections_for_ip ()
 {
     _ip="$1"
 
-    netstat -tn | awk -v ip=$_ip \
-       'index($1, "tcp") == 1 && \
-        (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
-        && $6 == "ESTABLISHED" \
-        {print $4" "$5}'
-}
-
-##################################################################
-# use statd-callout to update NFS lock info
-##################################################################
-nfs_update_lock_info ()
-{
-    if [ -x "$CTDB_BASE/statd-callout" ] ; then
-       "$CTDB_BASE/statd-callout" update
-    fi
+    ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
 }
 
 ########################################################
@@ -712,7 +552,7 @@ add_ip_to_iface ()
        die "Failed to bringup interface $_iface"
 
     # Only need to define broadcast for IPv4
-    case "$ip" in
+    case "$_ip" in
         *:*) _bcast=""      ;;
        *)   _bcast="brd +" ;;
     esac
@@ -771,28 +611,27 @@ ip_maskbits_iface ()
     _addr="$1"
 
     case "$_addr" in
-       *:*) _family="inet6" ; _bits=128 ;;
-       *)   _family="inet"  ; _bits=32  ;;
+       *:*) _bits=128 ;;
+       *)   _bits=32  ;;
     esac
 
     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
-       awk -v family="${_family}" \
-           'NR == 1 { iface = $2; sub(":$", "", iface) ; \
+       awk 'NR == 1 { iface = $2; sub(":$", "", iface) ; \
                       sub("@.*", "", iface) } \
              $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
-                           print mask, iface, family }'
+                           print mask, iface }'
 }
 
 drop_ip ()
 {
     _addr="${1%/*}"  # Remove optional maskbits
 
-    set -- $(ip_maskbits_iface $_addr)
+    set -- $(ip_maskbits_iface "$_addr")
     if [ -n "$1" ] ; then
        _maskbits="$1"
        _iface="$2"
        echo "Removing public address $_addr/$_maskbits from device $_iface"
-       delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
+       delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
     fi
 }
 
@@ -809,11 +648,105 @@ flush_route_cache ()
     set_proc_maybe sys/net/ipv6/route/flush 1
 }
 
+########################################################
+# Interface monitoring
+
+# If the interface is a virtual one (e.g. VLAN) then get the
+# underlying interface
+interface_get_real ()
+{
+    # Output of "ip link show <iface>"
+    _iface_info="$1"
+
+    # Extract the full interface description to see if it is a VLAN
+    _t=$(echo "$_iface_info" |
+               awk 'NR == 1 { iface = $2; sub(":$", "", iface) ; \
+                              print iface }')
+    case "$_t" in
+       *@*)
+           # VLAN: use the underlying interface, after the '@'
+           echo "${_t##*@}"
+           ;;
+       *)
+           # Not a regular VLAN.  For backward compatibility, assume
+           # there is some other sort of VLAN that doesn't have the
+           # '@' in the output and only use what is before a '.'.  If
+           # there is no '.' then this will be the whole interface
+           # name.
+           echo "${_t%%.*}"
+    esac
+}
+
+# Check whether an interface is operational
+interface_monitor ()
+{
+    _iface="$1"
+
+    _iface_info=$(ip link show "$_iface" 2>&1) || {
+       echo "ERROR: Monitored interface ${_iface} does not exist"
+       return 1
+    }
+
+
+    # If the interface is a virtual one (e.g. VLAN) then get the
+    # underlying interface.
+    _realiface=$(interface_get_real "$_iface_info")
+
+    if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null) ; then
+       # This is a bond: various monitoring strategies
+       echo "$_bi" | grep -q 'Currently Active Slave: None' && {
+           echo "ERROR: No active slaves for bond device ${_realiface}"
+           return 1
+       }
+       echo "$_bi" | grep -q '^MII Status: up' || {
+           echo "ERROR: public network interface ${_realiface} is down"
+           return 1
+       }
+       echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
+           # This works around a bug in the driver where the
+           # overall bond status can be up but none of the actual
+           # physical interfaces have a link.
+           echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
+               echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
+               return 1
+           }
+       }
+
+       return 0
+    else
+       # Not a bond
+       case "$_iface" in
+           lo*)
+               # loopback is always working
+               return 0
+               ;;
+           ib*)
+               # we don't know how to test ib links
+               return 0
+               ;;
+           *)
+               ethtool "$_iface" | grep -q 'Link detected: yes' || {
+                   # On some systems, this is not successful when a
+                   # cable is plugged but the interface has not been
+                   # brought up previously. Bring the interface up
+                   # and try again...
+                   ip link set "$_iface" up
+                   ethtool "$_iface" | grep -q 'Link detected: yes' || {
+                       echo "ERROR: No link on the public network interface ${_iface}"
+                       return 1
+                   }
+               }
+               return 0
+               ;;
+       esac
+    fi
+}
+
 ########################################################
 # Simple counters
 _ctdb_counter_common () {
     _service_name="${1:-${service_name:-${script_name}}}"
-    _counter_file="$ctdb_fail_dir/$_service_name"
+    _counter_file="${CTDB_SCRIPT_VARDIR}/failcount/${_service_name}"
     mkdir -p "${_counter_file%/*}" # dirname
 }
 ctdb_counter_init () {
@@ -862,12 +795,9 @@ ctdb_check_counter () {
 
 ########################################################
 
-ctdb_status_dir="$CTDB_VARDIR/state/service_status"
-ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
-
 ctdb_setup_service_state_dir ()
 {
-    service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
+    service_state_dir="${CTDB_SCRIPT_VARDIR}/service_state/${1:-${service_name}}"
     mkdir -p "$service_state_dir" || {
        echo "Error creating state dir \"$service_state_dir\""
        exit 1
@@ -877,17 +807,15 @@ ctdb_setup_service_state_dir ()
 ########################################################
 # Managed status history, for auto-start/stop
 
-ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
-
 _ctdb_managed_common ()
 {
-    _ctdb_managed_file="$ctdb_managed_dir/$service_name"
+    _ctdb_managed_file="${CTDB_SCRIPT_VARDIR}/managed_history/${service_name}"
 }
 
 ctdb_service_managed ()
 {
     _ctdb_managed_common
-    mkdir -p "$ctdb_managed_dir"
+    mkdir -p "${_ctdb_managed_file%/*}" # dirname
     touch "$_ctdb_managed_file"
 }
 
@@ -903,49 +831,12 @@ is_ctdb_previously_managed_service ()
     [ -f "$_ctdb_managed_file" ]
 }
 
-########################################################
-# Check and set status
-
-log_status_cat ()
-{
-    echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
-}
-
-ctdb_checkstatus ()
-{
-    if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
-       log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
-       return 1
-    elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
-       log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
-       return 2
-    else
-       return 0
-    fi
-}
-
-ctdb_setstatus ()
-{
-    d="$ctdb_status_dir/$script_name"
-    case "$1" in
-       unhealthy|banned)
-           mkdir -p "$d"
-           cat "$2" >"$d/$1"
-           ;;
-       *)
-           for i in "banned" "unhealthy" ; do
-               rm -f "$d/$i"
-           done
-           ;;
-    esac
-}
-
 ##################################################################
 # Reconfigure a service on demand
 
 _ctdb_service_reconfigure_common ()
 {
-    _d="$ctdb_status_dir/${service_name}"
+    _d="${CTDB_SCRIPT_VARDIR}/service_status/${service_name}"
     mkdir -p "$_d"
     _ctdb_service_reconfigure_flag="$_d/reconfigure"
 }
@@ -1021,7 +912,7 @@ ctdb_replay_monitor_status ()
 {
     echo "Replaying previous status for this script due to reconfigure..."
     # Leading separator ('|') is missing in some versions...
-    _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
+    _out=$($CTDB scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
     # Output looks like this:
     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
     # This is the cheapest way of getting fields in the middle.
@@ -1234,17 +1125,7 @@ service_stop ()
 
 ctdb_standard_event_handler ()
 {
-    case "$1" in
-       status)
-           ctdb_checkstatus
-           exit
-           ;;
-       setstatus)
-            shift
-           ctdb_setstatus "$@"
-           exit
-           ;;
-    esac
+    :
 }
 
 iptables_wrapper ()
@@ -1257,7 +1138,7 @@ iptables_wrapper ()
     fi
 
     # iptables doesn't like being re-entered, so flock-wrap it.
-    flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
+    flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
 }
 
 # AIX (and perhaps others?) doesn't have mktemp
@@ -1286,6 +1167,69 @@ if ! type mktemp >/dev/null 2>&1 ; then
     }
 fi
 
+######################################################################
+# NFS callout handling
+
+nfs_callout_init ()
+{
+       if [ -z "$CTDB_NFS_CALLOUT" ] ; then
+               CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
+       fi
+       # Always export, for statd callout
+       export CTDB_NFS_CALLOUT
+
+       # If the callout wants to use this then it must create it
+       export CTDB_NFS_CALLOUT_STATE_DIR="${service_state_dir}/callout-state"
+
+       # Export, if set, for use by clustered NFS callouts
+       if [ -n "$CTDB_NFS_STATE_FS_TYPE" ] ; then
+               export CTDB_NFS_STATE_FS_TYPE
+       fi
+       if [ -n "$CTDB_NFS_STATE_MNT" ] ; then
+               export CTDB_NFS_STATE_MNT
+       fi
+
+       nfs_callout_cache="${service_state_dir}/nfs_callout_cache"
+       nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
+       nfs_callout_cache_ops="${nfs_callout_cache}/ops"
+}
+
+nfs_callout_register ()
+{
+    mkdir -p "$nfs_callout_cache_ops"
+    rm -f "$nfs_callout_cache_ops"/*
+
+    echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
+
+    _t=$(eval "$CTDB_NFS_CALLOUT" "register")
+    if [ -n "$_t" ] ; then
+       echo "$_t" |
+           while IFS="" read _op ; do
+               touch "${nfs_callout_cache_ops}/${_op}"
+           done
+    else
+       touch "${nfs_callout_cache_ops}/ALL"
+    fi
+}
+
+nfs_callout ()
+{
+    # Re-run registration if $CTDB_NFS_CALLOUT has changed
+    _prev=""
+    if [ -r "$nfs_callout_cache_callout" ] ; then
+       read _prev <"$nfs_callout_cache_callout"
+    fi
+    if [ "$CTDB_NFS_CALLOUT" != "$_prev" ] ; then
+       nfs_callout_register
+    fi
+
+    # Run the operation if it is registered...
+    if [ -e "${nfs_callout_cache_ops}/${1}" ] || \
+          [ -e "${nfs_callout_cache_ops}/ALL" ]; then
+       eval "$CTDB_NFS_CALLOUT" "$@"
+    fi
+}
+
 ########################################################
 # tickle handling
 ########################################################
@@ -1294,30 +1238,38 @@ update_tickles ()
 {
        _port="$1"
 
-       tickledir="$CTDB_VARDIR/state/tickles"
+       tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
        mkdir -p "$tickledir"
 
        ctdb_get_pnn
 
        # What public IPs do I hold?
-       _ips=$(ctdb -X ip | awk -F'|' -v pnn=$pnn '$3 == pnn {print $2}')
+       _ips=$($CTDB -X ip | awk -F'|' -v pnn="$pnn" '$3 == pnn {print $2}')
 
-       # IPs as a regexp choice
-       _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
-
-       # Record connections to our public IPs in a temporary file
-       _my_connections="${tickledir}/${_port}.connections"
-       rm -f "$_my_connections"
-       netstat -tn |
-       awk -v destpat="^${_ipschoice}:${_port}\$" \
-         '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
+       # IPs and port as ss filters
+       _ip_filter=""
+       for _ip in $_ips ; do
+           _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
+       done
+       _port_filter="sport == :${_port}"
+
+       # Record connections to our public IPs in a temporary file.
+       # This temporary file is in CTDB's private state directory and
+       # $$ is used to avoid a very rare race involving CTDB's script
+       # debugging.  No security issue, nothing to see here...
+       _my_connections="${tickledir}/${_port}.connections.$$"
+       # Parentheses are needed around the filters for precedence but
+       # the parentheses can't be empty!
+       ss -tn state established \
+          "${_ip_filter:+( ${_ip_filter} )}" \
+          "${_port_filter:+( ${_port_filter} )}" |
+       awk 'NR > 1 {print $4, $3}' |
        sort >"$_my_connections"
 
        # Record our current tickles in a temporary file
-       _my_tickles="${tickledir}/${_port}.tickles"
-       rm -f "$_my_tickles"
+       _my_tickles="${tickledir}/${_port}.tickles.$$"
        for _i in $_ips ; do
-               ctdb -X gettickles $_i $_port |
+               $CTDB -X gettickles "$_i" "$_port" |
                awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
        done |
        sort >"$_my_tickles"
@@ -1325,16 +1277,19 @@ update_tickles ()
        # Add tickles for connections that we haven't already got tickles for
        comm -23 "$_my_connections" "$_my_tickles" |
        while read _src _dst ; do
-               ctdb addtickle $_src $_dst
+               $CTDB addtickle "$_src" "$_dst"
        done
 
        # Remove tickles for connections that are no longer there
        comm -13 "$_my_connections" "$_my_tickles" |
        while read _src _dst ; do
-               ctdb deltickle $_src $_dst
+               $CTDB deltickle "$_src" "$_dst"
        done
 
-       rm -f "$_my_connections" "$_my_tickles" 
+       rm -f "$_my_connections" "$_my_tickles"
+
+       # Remove stale files from killed scripts
+       find "$tickledir" -type f -mmin +10 | xargs -r rm
 }
 
 ########################################################
@@ -1345,12 +1300,12 @@ update_tickles ()
        . "$CTDB_RC_LOCAL"
 }
 
-[ -x $CTDB_BASE/rc.local ] && {
-       . $CTDB_BASE/rc.local
+[ -x "${CTDB_BASE}/rc.local" ] && {
+       . "${CTDB_BASE}/rc.local"
 }
 
-[ -d $CTDB_BASE/rc.local.d ] && {
-       for i in $CTDB_BASE/rc.local.d/* ; do
+[ -d "${CTDB_BASE}/rc.local.d" ] && {
+       for i in "${CTDB_BASE}/rc.local.d"/* ; do
                [ -x "$i" ] && . "$i"
        done
 }