Tests: initial eventscript unit tests.
authorMartin Schwenke <martin@meltin.net>
Tue, 28 Jun 2011 07:29:13 +0000 (17:29 +1000)
committerMartin Schwenke <martin@meltin.net>
Mon, 8 Aug 2011 03:38:07 +0000 (13:38 +1000)
Signed-off-by: Martin Schwenke <martin@meltin.net>
60 files changed:
tests/eventscripts/README [new file with mode: 0644]
tests/eventscripts/common.sh [new file with mode: 0644]
tests/eventscripts/etc-ctdb/events.d [new symlink]
tests/eventscripts/etc-ctdb/functions [new symlink]
tests/eventscripts/etc-ctdb/interface_modify.sh [new symlink]
tests/eventscripts/etc-ctdb/public_addresses [new file with mode: 0644]
tests/eventscripts/etc-ctdb/rc.local [new file with mode: 0755]
tests/eventscripts/etc-ctdb/rc.local.nfs.monitor.get-limits [new file with mode: 0755]
tests/eventscripts/etc-ctdb/statd-callout [new file with mode: 0755]
tests/eventscripts/etc/init.d/nfs [new file with mode: 0755]
tests/eventscripts/etc/init.d/nfslock [new file with mode: 0755]
tests/eventscripts/etc/samba/smb.conf [new file with mode: 0644]
tests/eventscripts/etc/sysconfig/nfs [new file with mode: 0644]
tests/eventscripts/run_tests.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.init.001.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.init.002.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.001.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.002.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.003.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.004.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.005.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.006.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.007.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.008.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.009.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.010.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.011.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.monitor.012.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.releaseip.001.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.releaseip.002.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.startup.001.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.startup.002.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.takeip.001.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.takeip.002.sh [new file with mode: 0755]
tests/eventscripts/simple/10.interface.takeip.003.sh [new file with mode: 0755]
tests/eventscripts/simple/40.vsftpd.monitor.001.sh [new file with mode: 0755]
tests/eventscripts/simple/41.httpd.monitor.001.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.001.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.050.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.051.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.101.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.102.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.103.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.104.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.105.sh [new file with mode: 0755]
tests/eventscripts/simple/50.samba.monitor.106.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.001.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.100.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.101.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.111.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.112.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.121.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.122.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.131.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.132.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.141.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.142.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.151.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.152.sh [new file with mode: 0755]
tests/eventscripts/simple/60.nfs.monitor.153.sh [new file with mode: 0755]

diff --git a/tests/eventscripts/README b/tests/eventscripts/README
new file mode 100644 (file)
index 0000000..33bea9e
--- /dev/null
@@ -0,0 +1,43 @@
+eventscript unit tests
+======================
+
+This directory contains some eventscript unit tests for CTDB.  These
+tests can be run as a non-privileged user.  There are a lot of stub
+implementations of commands (located in bin/) used to make the
+eventscripts think they're running against a real system.
+
+Examples:
+
+* ./run_tests.sh
+
+  Run all tests, displaying minimal output.
+
+* ./run_tests.sh -s
+
+  Run all tests, displaying minimal output and a summary.
+
+* ./run_tests.sh  -s simple/*.sh
+
+  Run all the tests in the simple/ subdirectory.
+
+* ./run_tests.sh -v -s
+
+  Run all tests, displaying extra output and a summary.
+
+* ./run_tests.sh -sq
+
+  Run all tests, displaying only a summary.
+
+* EVENTSCRIPTS_TESTS_TRACE="sh -x" \
+    ./run_tests.sh simple/10.interface.startup.002.sh
+
+  Run a test and have the eventscript itself run with "sh -x".  This
+  will usually make a test fail because the (undesirable) trace output
+  will be included with the output of the eventscript.  However, this
+  is useful for finding out why a test might be failing.  You can just
+  drop the "-x" (minimal command-line editing) to see if changes have
+  made a test pass.
+
+The simple/ subdirectory contains tests that exercise only a single
+eventscript.  Another directory containing tests that exercise
+interactions between eventscripts is coming soon...  :-)
diff --git a/tests/eventscripts/common.sh b/tests/eventscripts/common.sh
new file mode 100644 (file)
index 0000000..ec5454c
--- /dev/null
@@ -0,0 +1,772 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!  :-)
+
+# Augment PATH with relevant bin/ directories.  We do this by actually
+# setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
+# prepending that to $PATH in rc.local to avoid the PATH reset in
+# functions.
+
+EVENTSCRIPTS_PATH=""
+
+if [ -d "${EVENTSCRIPTS_TESTS_DIR}/bin" ] ; then
+    EVENTSCRIPTS_PATH="${EVENTSCRIPTS_TESTS_DIR}/bin"
+fi
+
+export EVENTSCRIPTS_TESTCASE_DIR=$(dirname "$0")
+if [ $(basename "$EVENTSCRIPTS_TESTCASE_DIR") = "eventscripts" ] ; then
+    # Just a test script, no testcase subdirectory.
+    EVENTSCRIPTS_TESTCASE_DIR="$EVENTSCRIPTS_TESTS_DIR"
+else
+    if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/bin" ] ; then
+       EVENTSCRIPTS_PATH="${EVENTSCRIPTS_TESTCASE_DIR}/bin:${EVENTSCRIPTS_PATH}"
+    fi
+fi
+
+export EVENTSCRIPTS_PATH
+
+PATH="${EVENTSCRIPTS_PATH}:${PATH}"
+
+if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc" ] ; then
+    CTDB_ETCDIR="${EVENTSCRIPTS_TESTCASE_DIR}/etc"
+elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc" ] ; then
+    CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_DIR}/etc"
+else
+    echo "Unable to set \$CTDB_ETCDIR" >&2
+    exit 1
+fi
+export CTDB_ETCDIR
+
+if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc-ctdb" ] ; then
+    CTDB_BASE="${EVENTSCRIPTS_TESTCASE_DIR}/etc-ctdb"
+elif [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb" ] ; then
+    CTDB_BASE="${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb"
+elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb" ] ; then
+    CTDB_BASE="${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb"
+else
+    echo "Unable to set \$CTDB_BASE" >&2
+    exit 1
+fi
+export CTDB_BASE
+
+export EVENTSCRIPTS_TESTS_VAR_DIR="${EVENTSCRIPTS_TESTS_DIR}/var"
+if [ "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/var" ] ; then
+    rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
+fi
+mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR"
+export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
+
+######################################################################
+
+if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] ; then
+    debug () { echo "$@" ; }
+else
+    debug () { : ; }
+fi
+
+eventscripts_tests_cleanup_hooks=""
+
+# This loses quoting!
+eventscripts_test_add_cleanup ()
+{
+    eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*"
+}
+
+trap 'eval $eventscripts_tests_cleanup_hooks' 0
+
+
+######################################################################
+
+# General setup fakery
+
+setup_generic ()
+{
+    debug "Setting up shares (3 existing shares)"
+    # Create 3 fake shares/exports.
+    export FAKE_SHARES=""
+    for i in $(seq 1 3) ; do
+       _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing"
+       mkdir -p "$_s"
+       FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}"
+    done
+
+    export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding"
+    mkdir -p "$FAKE_PROC_NET_BONDING"
+    rm -f "$FAKE_PROC_NET_BONDING"/*
+
+    export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down"
+    mkdir -p "$FAKE_ETHTOOL_LINK_DOWN"
+    rm -f "$FAKE_ETHTOOL_LINK_DOWN"/*
+
+    # This can only have 2 levels.  We don't want to resort to usings
+    # something dangerous like "rm -r" setup time.
+    export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state"
+    mkdir -p "$FAKE_IP_STATE"
+    rm -f "$FAKE_IP_STATE"/*/*
+    rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true
+    rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true
+}
+
+tcp_port_down ()
+{
+    for _i ; do
+       debug "Marking TCP port \"${_i}\" as not listening"
+       FAKE_NETSTAT_TCP_LISTEN=$(echo "$FAKE_NETSTAT_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
+    done
+}
+
+shares_missing ()
+{
+    _fmt="$1" ; shift
+
+    # Replace some shares with non-existent ones.
+    _t=""
+    _n=1
+    _nl="
+"
+    export MISSING_SHARES_TEXT=""
+    for _i in $FAKE_SHARES ; do
+       if [ $_n = "$1" ] ; then
+           shift
+           _i="${_i%_existing}_missing"
+           debug "Replacing share $_n with missing share \"$_i\""
+           rmdir "$_i" 2>/dev/null || true
+           MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}")
+       fi
+       _t="${_t}${_t:+ }${_i}"
+       _n=$(($_n + 1))
+    done
+    FAKE_SHARES="$_t"
+}
+
+# Setup some fake /proc/net/bonding files with just enough info for
+# the eventscripts.
+
+# arg1 is interface name, arg2 is currently active slave (use "None"
+# if none), arg3 is MII status ("up" or "down").
+setup_bond ()
+{
+    _iface="$1"
+    _slave="${2:-${_iface}_sl_0}"
+    _mii_s="${3:-up}"
+    echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s"
+    cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF
+Currently Active Slave: $_slave
+MII Status: $_mii_s
+EOF
+}
+
+ethtool_interfaces_down ()
+{
+    for _i ; do
+       echo "Marking interface $_i DOWN for ethtool"
+       touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
+    done
+}
+
+ethtool_interfaces_up ()
+{
+    for _i ; do
+       echo "Marking interface $_i UP for ethtool"
+       rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
+    done
+}
+
+######################################################################
+
+# CTDB fakery
+
+# Evaluate an expression that probably calls functions or uses
+# variables from the CTDB functions file.  This is used for test
+# initialisation.
+eventscript_call ()
+{
+    (
+       . "$CTDB_BASE/functions"
+       "$@"
+    )
+}
+
+# Set output for ctdb command.  Option 1st argument is return code.
+ctdb_set_output ()
+{
+    _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out"
+    cat >"$_out"
+
+    _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc"
+    echo "${1:-0}" >"$_rc"
+
+    eventscripts_test_add_cleanup "rm -f $_out $_rc"
+}
+
+setup_ctdb ()
+{
+    setup_generic
+
+    export FAKE_CTDB_NUMNODES="${1:-3}"
+    echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes"
+
+    export FAKE_CTDB_PNN="${2:-0}"
+    echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}"
+
+    if [ -n "$3" ] ; then
+       echo "Setting up CTDB_PUBLIC_ADDRESSES: $3"
+       export CTDB_PUBLIC_ADDRESSES=$(mktemp)
+       for _i in $3 ; do
+           _ip="${_i%@*}"
+           _ifaces="${_i#*@}"
+           echo "${_ip} ${_ifaces}" >>"$CTDB_PUBLIC_ADDRESSES"
+       done
+       eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES"
+    fi
+
+    export FAKE_CTDB_IFACES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb/ifaces-down"
+    mkdir -p "$FAKE_CTDB_IFACES_DOWN"
+    rm -f "$FAKE_CTDB_IFACES_DOWN"/*
+
+    export FAKE_CTDB_NODES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/nodes-down"
+    mkdir -p "$FAKE_CTDB_NODES_DOWN"
+    rm -f "$FAKE_CTDB_NODES_DOWN"/*
+}
+
+
+
+ctdb_nodes_up ()
+{
+    for _i ; do
+       rm -f "${FAKE_CTDB_NODES_DOWN}/${_i}"
+    done
+}
+
+ctdb_nodes_down ()
+{
+    for _i ; do
+       touch "${FAKE_CTDB_NODES_DOWN}/${_i}"
+    done
+}
+
+ctdb_get_interfaces ()
+{
+    # The echo/subshell forces all the output onto 1 line.
+    echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}')
+}
+
+ctdb_get_1_interface ()
+{
+    _t=$(ctdb_get_interfaces)
+    echo ${_t%% *}
+}
+
+ctdb_get_public_addresses ()
+{
+    _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
+    echo $(if [ -r "$_f" ] ; then 
+       while read _ip _iface ; do
+           echo "${_ip}@${_iface}"
+       done <"$_f"
+       fi)
+}
+
+# Prints the 1st public address as: interface IP maskbits
+# This is suitable for passing to 10.interfaces takeip/releaseip
+ctdb_get_1_public_address ()
+{
+    _addrs=$(ctdb_get_public_addresses)
+    echo "${_addrs%% *}" | sed -r -e 's#(.*)/(.*)@(.*)#\3 \1 \2#g'
+}
+
+######################################################################
+
+# Samba fakery
+
+setup_samba ()
+{
+    setup_ctdb
+
+    if [ "$1" != "down" ] ; then
+
+       debug "Marking Samba services as up, listening and managed by CTDB"
+        # Get into known state.
+       for i in "samba" "winbind" ; do
+           eventscript_call ctdb_service_managed "$i"
+       done
+        # All possible service names for all known distros.
+       for i in "smb" "nmb" "winbind" "samba" ; do
+           service "$i" force-started
+       done
+
+       export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
+       export CTDB_MANAGED_SERVICES="foo samba winbind bar"
+
+       export FAKE_NETSTAT_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
+       export FAKE_WBINFO_FAIL="no"
+    else
+       debug "Marking Samba services as down, not listening and not managed by CTDB"
+        # Get into known state.
+       for i in "samba" "winbind" ; do
+           eventscript_call ctdb_service_unmanaged "$i"
+       done
+        # All possible service names for all known distros.
+       for i in "smb" "nmb" "winbind" "samba" ; do
+           service "$i" force-stopped
+       done
+
+       export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
+       export CTDB_MANAGED_SERVICES="foo bar"
+       unset CTDB_MANAGES_SAMBA
+       unset CTDB_MANAGES_WINBIND
+
+       export FAKE_NETSTAT_TCP_LISTEN=""
+       export FAKE_WBINFO_FAIL="yes"
+    fi
+
+    # This is ugly but if this file isn't removed before each test
+    # then configuration changes between tests don't stick.
+    rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache"
+}
+
+wbinfo_down ()
+{
+    debug "Making wbinfo commands fail"
+    FAKE_WBINFO_FAIL="yes"
+}
+
+######################################################################
+
+# NFS fakery
+
+setup_nfs ()
+{
+    setup_ctdb
+
+    export FAKE_RPCINFO_SERVICES=""
+
+    export CTDB_NFS_SKIP_SHARE_CHECK="no"
+    export CTDB_NFS_SKIP_KNFSD_ALIVE_CHECK="no"
+
+    # Reset the failcounts for nfs services.
+    eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
+
+    rpc_fail_limits_file="${EVENTSCRIPTS_TESTS_VAR_DIR}/rpc_fail_limits"
+
+    # Force this file to exist so tests can be individually run.
+    if [ ! -f "$rpc_fail_limits_file" ] ; then
+       # This is gross... but is needed to fake through the nfs monitor event.
+       eventscript_call ctdb_service_managed "nfs"
+       service "nfs" force-started  # might not be enough
+       CTDB_RC_LOCAL="$CTDB_BASE/rc.local.nfs.monitor.get-limits" \
+           CTDB_MANAGES_NFS="yes" \
+           "${CTDB_BASE}/events.d/60.nfs" "monitor" >"$rpc_fail_limits_file"
+    fi
+    
+    if [ "$1" != "down" ] ; then
+       debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB"
+
+       eventscript_call ctdb_service_managed "nfs"
+       service "nfs" force-started  # might not be enough
+
+       export CTDB_MANAGED_SERVICES="foo nfs bar"
+
+       rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status"
+    else
+       debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
+
+       eventscript_call ctdb_service_unmanaged "nfs"
+       service "nfs" force-stopped  # might not be enough
+       eventscript_call startstop_nfs stop
+
+       export CTDB_MANAGED_SERVICES="foo bar"
+       unset CTDB_MANAGES_NFS
+    fi
+}
+
+rpc_services_down ()
+{
+    for _i ; do
+       debug "Marking RPC service \"${_i}\" as unavailable"
+       FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
+    done
+}
+
+rpc_services_up ()
+{
+    for _i ; do
+       debug "Marking RPC service \"${_i}\" as available"
+       case "$_i" in
+           nfs)      _t="2:3" ;;
+           mountd)   _t="1:3" ;;
+           rquotad)  _t="1:2" ;;
+           nlockmgr) _t="3:4" ;;
+           status)   _t="1:1" ;;
+           *)
+               echo "Internal error - unsupported RPC service \"${_i}\""
+               exit 1
+       esac
+
+       FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
+    done
+}
+
+# Set the required result for a particular RPC program having failed
+# for a certain number of iterations.  This is probably still a work
+# in progress.  Note that we could hook aggressively
+# nfs_check_rpc_service() to try to implement this but we're better
+# off testing nfs_check_rpc_service() using independent code...  even
+# if it is incomplete and hacky.  So, if the 60.nfs eventscript
+# changes and the tests start to fail then it may be due to this
+# function being incomplete.
+rpc_set_service_failure_response ()
+{
+    _progname="$1"
+    # The number of failures defaults to the iteration number.  This
+    # will be true when we fail from the 1st iteration... but we need
+    # the flexibility to set the number of failures.
+    _numfails="${2:-${iteration}}"
+
+    _c="${CTDB_ETCDIR}/sysconfig/nfs"
+    if [ -r "$_c" ] ; then
+       . "$_c"
+    fi
+
+    # A handy newline.  :-)
+    _nl="
+"
+
+    # Default
+    ok_null
+
+    _ts=$(sed -n -e "s@^${_progname} @@p" "$rpc_fail_limits_file")
+
+    while [ -n "$_ts" ] ; do
+       # Get the triple: operator, fail limit and actions.
+       _op="${_ts%% *}" ; _ts="${_ts#* }"
+       _li="${_ts%% *}" ; _ts="${_ts#* }"
+       # We've lost some of the quoting but we can simulate
+       # because we know an operator is always the first in a
+       # triple.
+       _actions=""
+       while [ -n "$_ts" ] ; do
+           # If this is an operator then we've got all of the
+           # actions.
+           case "$_ts" in
+               -*) break ;;
+           esac
+
+           _actions="${_actions}${_actions:+ }${_ts%% *}"
+           # Special case for end of list.
+           if [ "$_ts" != "${_ts#* }" ] ; then
+               _ts="${_ts#* }"
+           else
+               _ts=""
+           fi
+       done
+
+       if [ "$_numfails" "$_op" "$_li" ] ; then
+           _out=""
+           _rc=0
+           for _action in $_actions ; do
+               case "$_action" in
+                   verbose)
+                       _ver=1
+                       _pn="$_progname"
+                       case "$_progname" in
+                           knfsd) _ver=3 ; _pn="nfs" ;;
+                           lockd) _ver=4 ; _pn="nlockmgr" ;;
+                           statd) _pn="status" ;;
+                       esac
+                       _out="\
+ERROR: $_pn failed RPC check:
+rpcinfo: RPC: Program not registered
+program $_pn version $_ver is not available"
+                       ;;
+                   restart|restart:*)
+                       _opts=""
+                       _p="rpc.${_progname}"
+                       case "$_progname" in
+                           statd)
+                               _opts="${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}"
+                               ;;
+                       esac
+                       case "${_progname}${_action#restart}" in
+                           knfsd)
+                               _t="\
+Trying to restart NFS service
+Starting nfslock: OK
+Starting nfs: OK"
+                               ;;
+                           knfsd:bs)
+                               _t="Trying to restart NFS service"
+                               ;;
+                           lockd)
+                               _t="\
+Trying to restart lock manager service
+Stopping nfslock: OK
+Starting nfslock: OK"
+                               ;;
+                           lockd:*)
+                               _t="Trying to restart lock manager service"
+                               ;;
+                           *)
+                               _t="Trying to restart $_progname [${_p}${_opts}]"
+                       esac
+                       _out="${_out}${_out:+${_nl}}${_t}"
+                       ;;
+                   unhealthy)
+                       _rc=1
+               esac
+           done
+           required_result $_rc "$_out"
+           return
+       fi
+    done
+}
+
+######################################################################
+
+# VSFTPD fakery
+
+setup_vsftpd ()
+{
+    if [ "$1" != "down" ] ; then
+       echo "setup_vsftpd up not implemented!!!"
+       exit 1
+    else
+       debug "Setting up VSFTPD environment: service down, not managed by CTDB"
+
+       eventscript_call ctdb_service_unmanaged vsftpd
+       service vsftpd force-stopped
+
+       export CTDB_MANAGED_SERVICES="foo"
+       unset CTDB_MANAGES_VSFTPD
+    fi
+}
+
+######################################################################
+
+# HTTPD fakery
+
+setup_httpd ()
+{
+    if [ "$1" != "down" ] ; then
+       echo "setup_httpd up not implemented!!!"
+       exit 1
+    else
+       debug "Setting up HTTPD environment: service down, not managed by CTDB"
+
+       for i in "apache2" "httpd" ; do
+           eventscript_call ctdb_service_unmanaged "$i"
+           service "$i" force-stopped
+       done
+
+       export CTDB_MANAGED_SERVICES="foo"
+       unset CTDB_MANAGES_HTTPD
+    fi
+}
+
+######################################################################
+
+# Result and test functions
+
+# Set some globals and print the summary.
+define_test ()
+{
+    desc="$1"
+
+    _f="$0"
+    _f="${_f#./}"      # strip leading ./
+    _f="${_f#simple/}" # strip leading simple/
+    _f="${_f%%/*}"     # if subdir, strip off file
+    _f="${_f%.sh}"     # strip off .sh suffix if any
+
+    # Remaining format should be NN.service.event.NNN:
+    _num="${_f##*.}"
+    _f="${_f%.*}"
+    event="${_f##*.}"
+    script="${_f%.*}"
+
+    printf "%-14s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
+}
+
+# Set the required result for a test.
+# - Argument 1 is exit code.
+# - Argument 2, if present is the required test output but "--"
+#   indicates empty output.
+# If argument 2 is not present or null then read required test output
+# from stdin.
+required_result ()
+{
+    required_rc="${1:-0}"
+    if [ -n "$2" ] ; then
+       if [ "$2" = "--" ] ; then
+           required_output=""
+       else
+           required_output="$2"
+       fi
+    else
+       if ! tty -s ; then
+           required_output=$(cat)
+       else
+           required_output=""
+       fi
+    fi
+}
+
+ok ()
+{
+    required_result 0 "$@"
+}
+
+ok_null ()
+{
+    ok --
+}
+
+result_print ()
+{
+    _passed="$1"
+    _out="$2"
+    _rc="$3"
+    _iteration="$4"
+
+    if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then
+       if [ -n "$_iteration" ] ; then
+           cat <<EOF
+
+##################################################
+##################################################
+Iteration $_iteration
+EOF
+       fi
+
+cat <<EOF
+##################################################
+Output (Exit status: ${_rc}):
+##################################################
+$_out
+EOF
+    fi
+
+    if ! $_passed ; then
+       cat <<EOF
+##################################################
+Required output (Exit status: ${required_rc}):
+##################################################
+$required_output
+EOF
+    fi
+}
+
+result_footer ()
+{
+    _passed="$1"
+
+    if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then
+
+       cat <<EOF
+##################################################
+CTDB_BASE="$CTDB_BASE"
+CTDB_ETCDIR="$CTDB_ETCDIR"
+ctdb client is "$(which ctdb)"
+##################################################
+EOF
+    fi
+
+    echo
+
+    if $_passed ; then
+       echo "PASSED"
+       return 0
+    else
+       echo "FAILED"
+       return 1
+    fi
+}
+
+# Run an eventscript once.  The test passes if the return code and
+# output match those required.
+
+# Any args are passed to the eventscript.
+
+# Eventscript tracing can be done by setting:
+#   EVENTSCRIPTS_TESTS_TRACE="sh -x"
+
+# or similar.  This will almost certainly make a test fail but is
+# useful for debugging.
+simple_test ()
+{
+    echo "Running \"${CTDB_BASE}/events.d/$script $event\""
+    _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
+    _rc=$?
+
+    if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then
+       _passed=true
+    else
+       _passed=false
+    fi
+
+    result_print "$_passed" "$_out" "$_rc"
+    result_footer "$_passed"
+}
+
+# Run an eventscript iteratively.
+# - 1st argument is the number of iterations.
+# - 2nd argument is something to eval to do setup for every iteration.
+#   The easiest thing to do here is to define a function and pass it
+#   here.
+# - Subsequent arguments come in pairs: an iteration number and
+#   something to eval for that iteration.  Each time an iteration
+#   number is matched the associated argument is given to eval after
+#   the default setup is done.  The iteration numbers need to be given
+#   in ascending order.
+#
+# Some optional args can be given *before* these, surrounded by extra
+# "--" args.  These args are passed to the eventscript.  Quoting is
+# lost.
+#
+# One use of the 2nd and further arguments is to call
+# required_result() to change what is expected of a particular
+# iteration.
+iterate_test ()
+{
+    args=""
+    if [ "$1" = "--" ] ; then
+       shift
+       while [ "$1" != "--" ] ; do
+           args="${args}${args:+ }$1"
+           shift
+       done
+       shift
+    fi
+
+    _repeats="$1"
+    _setup_default="$2"
+    shift 2
+
+    echo "Running $_repeats iterations of \"${CTDB_BASE}/events.d/$script $event\" $args"
+
+    _result=true
+
+    for iteration in $(seq 1 $_repeats) ; do
+       # This is inefficient because the iteration-specific setup
+       # might completely replace the default one.  However, running
+       # the default is good because it allows you to revert to a
+       # particular result without needing to specify it explicitly.
+       eval $_setup_default
+       if [ $iteration = "$1" ] ; then
+           eval $2
+           shift 2
+       fi
+
+       _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1)
+       _rc=$?
+
+       if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then
+           _passed=true
+       else
+           _passed=false
+           _result=false
+       fi
+
+       result_print "$_passed" "$_out" "$_rc" "$iteration"
+    done
+
+    result_footer "$_result"
+}
diff --git a/tests/eventscripts/etc-ctdb/events.d b/tests/eventscripts/etc-ctdb/events.d
new file mode 120000 (symlink)
index 0000000..69d2396
--- /dev/null
@@ -0,0 +1 @@
+../../../config/events.d
\ No newline at end of file
diff --git a/tests/eventscripts/etc-ctdb/functions b/tests/eventscripts/etc-ctdb/functions
new file mode 120000 (symlink)
index 0000000..86ba904
--- /dev/null
@@ -0,0 +1 @@
+../../../config/functions
\ No newline at end of file
diff --git a/tests/eventscripts/etc-ctdb/interface_modify.sh b/tests/eventscripts/etc-ctdb/interface_modify.sh
new file mode 120000 (symlink)
index 0000000..94f555c
--- /dev/null
@@ -0,0 +1 @@
+../../../config/interface_modify.sh
\ No newline at end of file
diff --git a/tests/eventscripts/etc-ctdb/public_addresses b/tests/eventscripts/etc-ctdb/public_addresses
new file mode 100644 (file)
index 0000000..5421cd7
--- /dev/null
@@ -0,0 +1,4 @@
+10.0.0.1/24 dev123
+10.0.0.2/24 dev123
+10.0.1.1/24 dev456
+10.0.1.2/24 dev456
diff --git a/tests/eventscripts/etc-ctdb/rc.local b/tests/eventscripts/etc-ctdb/rc.local
new file mode 100755 (executable)
index 0000000..ae93ae5
--- /dev/null
@@ -0,0 +1,46 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!  :-)
+
+# Use a "service" command in $PATH if one exists.
+service ()
+{
+    if _t=$(which "service" 2>/dev/null) ; then
+       "$_t" "$@"
+    else
+       _nice=""
+       _service "$@"
+    fi
+}
+
+nice_service ()
+{
+    if _t=$(which "service" 2>/dev/null) ; then
+       nice "$_t" "$@"
+    else
+       _nice="nice"
+       _service "$@"
+    fi
+}
+
+# Always succeeds
+set_proc () { : ; }
+
+get_proc ()
+{
+    case "$1" in
+       net/bonding/*)
+           cat "$FAKE_PROC_NET_BONDING/${1##*/}"
+           ;;
+       sys/net/ipv4/conf/all/arp_filter)
+           echo 1
+           ;;
+       *)
+           echo "get_proc: \"$1\" not implemented"
+           exit 1
+    esac
+}
+
+# Always succeeds
+iptables () { : ; }
+
+CTDB_INIT_STYLE="redhat"
+PATH="${EVENTSCRIPTS_PATH}:$PATH"
diff --git a/tests/eventscripts/etc-ctdb/rc.local.nfs.monitor.get-limits b/tests/eventscripts/etc-ctdb/rc.local.nfs.monitor.get-limits
new file mode 100755 (executable)
index 0000000..96e4cff
--- /dev/null
@@ -0,0 +1,25 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!  :-)
+
+# This scripts nobble the 60.nfs monitor event so that it prints out
+# the service fail limits for each RPC service.
+
+CTDB_INIT_STYLE="redhat"
+PATH="${EVENTSCRIPTS_PATH}:$PATH"
+
+service () { : ; }
+
+update_tickles () { : ; }
+
+ctdb_setup_service_state_dir "nfs"
+
+CTDB_NFS_SKIP_KNFSD_ALIVE_CHECK="no"
+CTDB_NFS_SKIP_SHARE_CHECK="yes"
+
+# Ugly but necessary - if this file was touched less then 60 seconds
+# ago then this skips some code.
+touch "$service_state_dir/update-trigger"
+
+nfs_check_rpc_service ()
+{
+    echo "$*"
+}
diff --git a/tests/eventscripts/etc-ctdb/statd-callout b/tests/eventscripts/etc-ctdb/statd-callout
new file mode 100755 (executable)
index 0000000..51779bd
--- /dev/null
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# For now, always succeed.
+
+exit 0
diff --git a/tests/eventscripts/etc/init.d/nfs b/tests/eventscripts/etc/init.d/nfs
new file mode 100755 (executable)
index 0000000..43eb308
--- /dev/null
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# This is not used.  The fake "service" script is used instead.  This
+# is only needed to shut up functions like startstop_nfs(), which look
+# for this script.
+
+exit 0
diff --git a/tests/eventscripts/etc/init.d/nfslock b/tests/eventscripts/etc/init.d/nfslock
new file mode 100755 (executable)
index 0000000..43eb308
--- /dev/null
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# This is not used.  The fake "service" script is used instead.  This
+# is only needed to shut up functions like startstop_nfs(), which look
+# for this script.
+
+exit 0
diff --git a/tests/eventscripts/etc/samba/smb.conf b/tests/eventscripts/etc/samba/smb.conf
new file mode 100644 (file)
index 0000000..da89db2
--- /dev/null
@@ -0,0 +1,42 @@
+[global]
+       # enable clustering
+       clustering=yes
+       ctdb:registry.tdb=yes
+
+       security = ADS
+       auth methods = guest sam winbind
+
+       netbios name = cluster1
+       workgroup = CLUSTER1
+       realm = CLUSTER1.COM
+       server string = "Clustered Samba"
+       disable netbios = yes
+       disable spoolss = yes
+       fileid:mapping = fsname
+       use mmap = yes
+       gpfs:sharemodes = yes
+       gpfs:leases = yes
+       passdb backend = tdbsam
+       preferred master = no
+       kernel oplocks = yes
+       syslog = 1
+       host msdfs = no
+       notify:inotify = no
+       vfs objects = shadow_copy2 syncops gpfs fileid
+       shadow:snapdir = .snapshots
+       shadow:fixinodes = yes
+       wide links = no
+       smbd:backgroundqueue = False
+       read only = no
+       use sendfile = yes
+       strict locking = yes
+       posix locking = yes
+       large readwrite = yes
+       force unknown acl user = yes
+       nfs4:mode = special
+       nfs4:chown = yes
+       nfs4:acedup = merge
+       nfs4:sidmap = /etc/samba/sidmap.tdb
+       map readonly = no
+       ea support = yes
+       dmapi support = no
diff --git a/tests/eventscripts/etc/sysconfig/nfs b/tests/eventscripts/etc/sysconfig/nfs
new file mode 100644 (file)
index 0000000..090d786
--- /dev/null
@@ -0,0 +1,2 @@
+NFS_HOSTNAME="cluster1"
+STATD_HOSTNAME="$NFS_HOSTNAME -H /etc/ctdb/statd-callout "
diff --git a/tests/eventscripts/run_tests.sh b/tests/eventscripts/run_tests.sh
new file mode 100755 (executable)
index 0000000..cb2c8f2
--- /dev/null
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+# Eventscript unit test harness.
+
+cd $(dirname "$0")
+export EVENTSCRIPTS_TESTS_DIR=$(pwd)
+
+test_dir=$(dirname "$EVENTSCRIPTS_TESTS_DIR")
+
+opts="-d"
+
+for i ; do
+    case "$i" in
+       -v)
+           export EVENTSCRIPT_TESTS_VERBOSE="yes"
+           shift
+           ;;
+       -*)
+           opts="$opts $i"
+           shift
+           ;;
+       *)
+           break
+    esac
+done
+
+tests=""
+if [ -z "$*" ] ; then
+    tests=$(ls simple/[0-9][0-9].*.*.[0-9][0-9][0-9].sh simple/[0-9][0-9].*.*.[0-9][0-9][0-9]/run_test.sh 2>/dev/null)
+fi
+
+"$test_dir/scripts/run_tests" $opts "$@" $tests || exit 1
+
+exit 0
diff --git a/tests/eventscripts/simple/10.interface.init.001.sh b/tests/eventscripts/simple/10.interface.init.001.sh
new file mode 100755 (executable)
index 0000000..5a61ab3
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "no public addresses"
+
+setup_ctdb
+
+export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist"
+
+ok "No public addresses file found. Nothing to do for 10.interfaces"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.init.002.sh b/tests/eventscripts/simple/10.interface.init.002.sh
new file mode 100755 (executable)
index 0000000..43bacd5
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "all interfaces up"
+
+setup_ctdb
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.001.sh b/tests/eventscripts/simple/10.interface.monitor.001.sh
new file mode 100755 (executable)
index 0000000..5a61ab3
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "no public addresses"
+
+setup_ctdb
+
+export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist"
+
+ok "No public addresses file found. Nothing to do for 10.interfaces"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.002.sh b/tests/eventscripts/simple/10.interface.monitor.002.sh
new file mode 100755 (executable)
index 0000000..43bacd5
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "all interfaces up"
+
+setup_ctdb
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.003.sh b/tests/eventscripts/simple/10.interface.monitor.003.sh
new file mode 100755 (executable)
index 0000000..5dd4328
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 interface down"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+ethtool_interfaces_down $iface
+
+required_result 1 "ERROR: No link on the public network interface $iface"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.004.sh b/tests/eventscripts/simple/10.interface.monitor.004.sh
new file mode 100755 (executable)
index 0000000..d215c40
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "all interfaces up, 1 is a bond"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+setup_bond $iface
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.005.sh b/tests/eventscripts/simple/10.interface.monitor.005.sh
new file mode 100755 (executable)
index 0000000..00db01f
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 bond, no active slaves"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+setup_bond $iface "None"
+
+required_result 1 "ERROR: No active slaves for bond device $iface"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.006.sh b/tests/eventscripts/simple/10.interface.monitor.006.sh
new file mode 100755 (executable)
index 0000000..862a972
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 bond, active slaves, link down"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+setup_bond $iface "" "down"
+
+required_result 1 "ERROR: public network interface $iface is down"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.007.sh b/tests/eventscripts/simple/10.interface.monitor.007.sh
new file mode 100755 (executable)
index 0000000..004247e
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "unknown interface, up"
+
+setup_ctdb
+
+export CTDB_PUBLIC_INTERFACE="dev999"
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.008.sh b/tests/eventscripts/simple/10.interface.monitor.008.sh
new file mode 100755 (executable)
index 0000000..aa59151
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "unknown interface, down, up"
+
+setup_ctdb
+
+iface="dev999"
+export CTDB_PUBLIC_INTERFACE="$iface"
+
+#EVENTSCRIPTS_TESTS_TRACE="sh -x"
+iterate_test 3 "ok_null" \
+    1 'ethtool_interfaces_down "$iface" ; required_result 1 "ERROR: No link on the public network interface $iface"' \
+    2 'ethtool_interfaces_up "$iface"'
diff --git a/tests/eventscripts/simple/10.interface.monitor.009.sh b/tests/eventscripts/simple/10.interface.monitor.009.sh
new file mode 100755 (executable)
index 0000000..ab890f3
--- /dev/null
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 down"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+export CTDB_PARTIALLY_ONLINE_INTERFACES="yes"
+
+ethtool_interfaces_down "$iface"
+
+ok "ERROR: No link on the public network interface $iface"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.010.sh b/tests/eventscripts/simple/10.interface.monitor.010.sh
new file mode 100755 (executable)
index 0000000..476d2c9
--- /dev/null
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, all down"
+
+setup_ctdb
+
+ifaces=$(ctdb_get_interfaces)
+
+export CTDB_PARTIALLY_ONLINE_INTERFACES="yes"
+
+ethtool_interfaces_down $ifaces
+
+msg=$(for i in $ifaces ; do echo "ERROR: No link on the public network interface $i" ; done)
+
+required_result 1 "$msg"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.011.sh b/tests/eventscripts/simple/10.interface.monitor.011.sh
new file mode 100755 (executable)
index 0000000..3e5ba34
--- /dev/null
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+setup_bond $iface "None"
+
+export CTDB_PARTIALLY_ONLINE_INTERFACES="yes"
+
+ethtool_interfaces_down "$iface"
+
+ok "ERROR: No active slaves for bond device $iface"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.monitor.012.sh b/tests/eventscripts/simple/10.interface.monitor.012.sh
new file mode 100755 (executable)
index 0000000..d27c423
--- /dev/null
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down"
+
+setup_ctdb
+
+ifaces=$(ctdb_get_interfaces)
+
+for i in $ifaces ; do
+    setup_bond $i "None"
+done
+
+export CTDB_PARTIALLY_ONLINE_INTERFACES="yes"
+
+ethtool_interfaces_down $ifaces
+
+msg=$(for i in $ifaces ; do echo "ERROR: No active slaves for bond device $i" ; done)
+
+required_result 1 "$msg"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.releaseip.001.sh b/tests/eventscripts/simple/10.interface.releaseip.001.sh
new file mode 100755 (executable)
index 0000000..56f01d6
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "error - no args given"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+required_result 1 "must supply interface, IP and maskbits"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.releaseip.002.sh b/tests/eventscripts/simple/10.interface.releaseip.002.sh
new file mode 100755 (executable)
index 0000000..d08d27c
--- /dev/null
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "error - remove a non-existent ip"
+
+setup_ctdb
+
+public_address=$(ctdb_get_1_public_address)
+ip="${public_address% *}" ; ip="${ip#* }"
+
+required_result 1 <<EOF
+RTNETLINK answers: Cannot assign requested address
+Failed to del ${ip} on dev ${public_address%% *}
+EOF
+
+simple_test $public_address
diff --git a/tests/eventscripts/simple/10.interface.startup.001.sh b/tests/eventscripts/simple/10.interface.startup.001.sh
new file mode 100755 (executable)
index 0000000..5a61ab3
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "no public addresses"
+
+setup_ctdb
+
+export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist"
+
+ok "No public addresses file found. Nothing to do for 10.interfaces"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.startup.002.sh b/tests/eventscripts/simple/10.interface.startup.002.sh
new file mode 100755 (executable)
index 0000000..43bacd5
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "all interfaces up"
+
+setup_ctdb
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.takeip.001.sh b/tests/eventscripts/simple/10.interface.takeip.001.sh
new file mode 100755 (executable)
index 0000000..56f01d6
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "error - no args given"
+
+setup_ctdb
+
+iface=$(ctdb_get_1_interface)
+
+required_result 1 "must supply interface, IP and maskbits"
+
+simple_test
diff --git a/tests/eventscripts/simple/10.interface.takeip.002.sh b/tests/eventscripts/simple/10.interface.takeip.002.sh
new file mode 100755 (executable)
index 0000000..9ee361a
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "add an ip"
+
+setup_ctdb
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test $public_address
diff --git a/tests/eventscripts/simple/10.interface.takeip.003.sh b/tests/eventscripts/simple/10.interface.takeip.003.sh
new file mode 100755 (executable)
index 0000000..8997d71
--- /dev/null
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "error - add same IP twice"
+
+setup_ctdb
+
+public_address=$(ctdb_get_1_public_address)
+
+# This is a bit gross and contrived.  The method of quoting the error
+# message so it makes it to required_result() is horrible.  Hopefully
+# improvements will come.
+
+err2="\
+RTNETLINK answers: File exists
+Failed to add 10.0.0.1/24 on dev dev123"
+
+#EVENTSCRIPTS_TESTS_TRACE="sh -x"
+iterate_test -- $public_address -- 2 "ok_null" \
+    2 'required_result 1 "$err2"'
diff --git a/tests/eventscripts/simple/40.vsftpd.monitor.001.sh b/tests/eventscripts/simple/40.vsftpd.monitor.001.sh
new file mode 100755 (executable)
index 0000000..fe2afd1
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "not managed, check no-op"
+
+setup_vsftpd "down"
+
+ok_null
+
+simple_test $cmd
diff --git a/tests/eventscripts/simple/41.httpd.monitor.001.sh b/tests/eventscripts/simple/41.httpd.monitor.001.sh
new file mode 100755 (executable)
index 0000000..c46a45c
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "not managed, check no-op"
+
+setup_httpd "down"
+
+ok_null
+
+simple_test $cmd
diff --git a/tests/eventscripts/simple/50.samba.monitor.001.sh b/tests/eventscripts/simple/50.samba.monitor.001.sh
new file mode 100755 (executable)
index 0000000..77fc2eb
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "not managed, check no-op"
+
+setup_samba "down"
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.050.sh b/tests/eventscripts/simple/50.samba.monitor.050.sh
new file mode 100755 (executable)
index 0000000..7a1b422
--- /dev/null
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "auto-start, simple"
+
+setup_samba "down"
+
+export CTDB_MANAGED_SERVICES="foo samba winbind bar"
+
+ok <<EOF
+Starting service "samba" - now managed
+Starting winbind: OK
+Starting smb: OK
+EOF
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.051.sh b/tests/eventscripts/simple/50.samba.monitor.051.sh
new file mode 100755 (executable)
index 0000000..3a53ac8
--- /dev/null
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "auto-stop, simple"
+
+setup_samba
+
+export CTDB_MANAGED_SERVICES="foo"
+unset CTDB_MANAGES_SAMBA
+unset CTDB_MANAGES_WINBIND
+
+ok <<EOF
+Stopping service "samba" - no longer managed
+Stopping smb: OK
+Stopping winbind: OK
+EOF
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.101.sh b/tests/eventscripts/simple/50.samba.monitor.101.sh
new file mode 100755 (executable)
index 0000000..d4b80bb
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "all OK"
+
+setup_samba
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.102.sh b/tests/eventscripts/simple/50.samba.monitor.102.sh
new file mode 100755 (executable)
index 0000000..2f1f446
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "winbind down"
+
+setup_samba
+wbinfo_down
+
+required_result 1 "ERROR: winbind - wbinfo -p returned error"
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.103.sh b/tests/eventscripts/simple/50.samba.monitor.103.sh
new file mode 100755 (executable)
index 0000000..8854780
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 445 down"
+
+setup_samba
+tcp_port_down 445
+
+required_result 1 "ERROR: samba tcp port 445 is not responding"
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.104.sh b/tests/eventscripts/simple/50.samba.monitor.104.sh
new file mode 100755 (executable)
index 0000000..a29e783
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down"
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 "ERROR: samba tcp port 139 is not responding"
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.105.sh b/tests/eventscripts/simple/50.samba.monitor.105.sh
new file mode 100755 (executable)
index 0000000..624c4a5
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "non-existent share path"
+
+setup_samba
+shares_missing "ERROR: samba directory \"%s\" not available" 2
+
+required_result 1 "$MISSING_SHARES_TEXT"
+
+simple_test
diff --git a/tests/eventscripts/simple/50.samba.monitor.106.sh b/tests/eventscripts/simple/50.samba.monitor.106.sh
new file mode 100755 (executable)
index 0000000..f54cc06
--- /dev/null
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "non-existent share - not checked"
+
+setup_samba
+shares_missing "ERROR: samba directory \"%s\" not available" 2
+
+export CTDB_SAMBA_SKIP_SHARE_CHECK="yes"
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/60.nfs.monitor.001.sh b/tests/eventscripts/simple/60.nfs.monitor.001.sh
new file mode 100755 (executable)
index 0000000..5b260ac
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "not managed, check no-op"
+
+setup_nfs "down"
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/60.nfs.monitor.100.sh b/tests/eventscripts/simple/60.nfs.monitor.100.sh
new file mode 100755 (executable)
index 0000000..95f4dc6
--- /dev/null
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "get RPC service fail limits/actions"
+
+setup_nfs
+
+set -e
+
+rm -f "$rpc_fail_limits_file"
+CTDB_RC_LOCAL="$CTDB_BASE/rc.local.nfs.monitor.get-limits" \
+    "${CTDB_BASE}/events.d/60.nfs" "monitor" >"$rpc_fail_limits_file"
+
+services="knfsd|mountd|rquotad|lockd|statd"
+
+echo "Doing rough check of file format..."
+
+! grep -v -E "^(${services}) " "$rpc_fail_limits_file"
diff --git a/tests/eventscripts/simple/60.nfs.monitor.101.sh b/tests/eventscripts/simple/60.nfs.monitor.101.sh
new file mode 100755 (executable)
index 0000000..657db97
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "all services available"
+
+setup_nfs
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/60.nfs.monitor.111.sh b/tests/eventscripts/simple/60.nfs.monitor.111.sh
new file mode 100755 (executable)
index 0000000..8dcde02
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "knfsd down, 1 iteration"
+
+setup_nfs
+rpc_services_down "nfs"
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/60.nfs.monitor.112.sh b/tests/eventscripts/simple/60.nfs.monitor.112.sh
new file mode 100755 (executable)
index 0000000..de796eb
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "knfsd down, 6 iterations"
+
+# knfsd fails and attempts to restart it fail.
+
+setup_nfs
+rpc_services_down "nfs"
+
+iterate_test 6 'ok_null' \
+    2 'rpc_set_service_failure_response "knfsd"' \
+    4 'rpc_set_service_failure_response "knfsd"' \
+    6 'rpc_set_service_failure_response "knfsd"'
diff --git a/tests/eventscripts/simple/60.nfs.monitor.121.sh b/tests/eventscripts/simple/60.nfs.monitor.121.sh
new file mode 100755 (executable)
index 0000000..062c3f6
--- /dev/null
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "lockd down, 15 iterations"
+
+# This simulates an ongoing failure in the eventscript's automated
+# attempts to restart the service.  That is, the eventscript is unable
+# to restart the service.
+
+setup_nfs
+rpc_services_down "nlockmgr"
+
+#EVENTSCRIPTS_TESTS_TRACE="sh -x"
+iterate_test 15 "ok_null" \
+    10 "rpc_set_service_failure_response 'lockd'" \
+    15 "rpc_set_service_failure_response 'lockd'"
diff --git a/tests/eventscripts/simple/60.nfs.monitor.122.sh b/tests/eventscripts/simple/60.nfs.monitor.122.sh
new file mode 100755 (executable)
index 0000000..b93dba7
--- /dev/null
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "lockd down, 15 iterations, back up after 10"
+
+# This simulates a success the eventscript's automated attempts to
+# restart the service.
+
+setup_nfs
+rpc_services_down "nlockmgr"
+
+# Iteration 10 should try to restart rpc.lockd.  However, our test
+# stub rpc.lockd does nothing, so we have to explicitly flag it as up.
+
+iterate_test 15 "ok_null" \
+    10 "rpc_set_service_failure_response 'lockd'" \
+    11 "rpc_services_up nlockmgr"
+
diff --git a/tests/eventscripts/simple/60.nfs.monitor.131.sh b/tests/eventscripts/simple/60.nfs.monitor.131.sh
new file mode 100755 (executable)
index 0000000..84d20b7
--- /dev/null
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "rquotad down, 5 iterations"
+
+setup_nfs
+rpc_services_down "rquotad"
+
+iterate_test 5 'rpc_set_service_failure_response "rquotad"'
diff --git a/tests/eventscripts/simple/60.nfs.monitor.132.sh b/tests/eventscripts/simple/60.nfs.monitor.132.sh
new file mode 100755 (executable)
index 0000000..fad9a4c
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "rquotad down, 5 iterations, back up after 1"
+
+# rquotad fails once but then comes back of its own accord after 1
+# failure.
+
+setup_nfs
+rpc_services_down "rquotad"
+
+iterate_test 5 'ok_null' \
+    1 'rpc_set_service_failure_response "rquotad"' \
+    2 'rpc_services_up "rquotad"'
diff --git a/tests/eventscripts/simple/60.nfs.monitor.141.sh b/tests/eventscripts/simple/60.nfs.monitor.141.sh
new file mode 100755 (executable)
index 0000000..377de6e
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "statd down, 6 iterations"
+
+# statd fails and attempts to restart it fail.
+
+setup_nfs
+rpc_services_down "status"
+
+iterate_test 6 'ok_null' \
+    2 'rpc_set_service_failure_response "statd"' \
+    4 'rpc_set_service_failure_response "statd"' \
+    6 'rpc_set_service_failure_response "statd"'
diff --git a/tests/eventscripts/simple/60.nfs.monitor.142.sh b/tests/eventscripts/simple/60.nfs.monitor.142.sh
new file mode 100755 (executable)
index 0000000..3ca3cf6
--- /dev/null
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "statd down, 8 iterations, back up after 2"
+
+# statd fails and the first attempt to restart it succeeds.
+
+setup_nfs
+rpc_services_down "status"
+
+iterate_test 8 'ok_null' \
+    2 'rpc_set_service_failure_response "statd"' \
+    3 'rpc_services_up "status"'
diff --git a/tests/eventscripts/simple/60.nfs.monitor.151.sh b/tests/eventscripts/simple/60.nfs.monitor.151.sh
new file mode 100755 (executable)
index 0000000..af2dd26
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "mountd down, 1 iteration"
+
+setup_nfs
+rpc_services_down "mountd"
+
+ok_null
+
+simple_test
diff --git a/tests/eventscripts/simple/60.nfs.monitor.152.sh b/tests/eventscripts/simple/60.nfs.monitor.152.sh
new file mode 100755 (executable)
index 0000000..9aad819
--- /dev/null
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "mountd down, 10 iterations"
+
+# This simulates an ongoing failure in the eventscript's automated
+# attempts to restart the service.  That is, the eventscript is unable
+# to restart the service.
+
+setup_nfs
+rpc_services_down "mountd"
+
+iterate_test 10 "ok_null" \
+    5 "rpc_set_service_failure_response 'mountd'" \
+    10 "rpc_set_service_failure_response 'mountd'"
+
+#export FAKE_NETSTAT_TCP_ESTABLISHED="10.0.0.1:2049|10.254.254.1:12301 10.0.0.1:2049|10.254.254.1:12302 10.0.0.1:2049|10.254.254.1:12303 10.0.0.1:2049|10.254.254.2:12304 10.0.0.1:2049|10.254.254.2:12305"
diff --git a/tests/eventscripts/simple/60.nfs.monitor.153.sh b/tests/eventscripts/simple/60.nfs.monitor.153.sh
new file mode 100755 (executable)
index 0000000..6b2750c
--- /dev/null
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "mountd down, 10 iterations, back up after 5"
+
+setup_nfs
+rpc_services_down "mountd"
+
+# Iteration 5 should try to restart rpc.mountd.  However, our test
+# stub rpc.mountd does nothing, so we have to explicitly flag it as
+# up.
+iterate_test 10 "ok_null" \
+    5 "rpc_set_service_failure_response 'mountd'" \
+    6 "rpc_services_up mountd"