utils/smnotify/gen_xdr.c
utils/smnotify/smnotify.h
nodes.txt
+public_addresses.txt
rec.lock
test.db
+tests/bin
+tests/events.d/00.ctdb_test_trigger
+tests/var
server/ctdb_keepalive.o server/ctdb_logging.o server/ctdb_uptime.c \
$(CTDB_CLIENT_OBJ) $(CTDB_TCP_OBJ) @INFINIBAND_WRAPPER_OBJ@
-TEST_BINS=bin/ctdb_bench bin/ctdb_fetch bin/ctdb_store bin/ctdb_randrec bin/ctdb_persistent \
- bin/ctdb_traverse bin/rb_test bin/ctdb_transaction \
+TEST_BINS=tests/bin/ctdb_bench tests/bin/ctdb_fetch tests/bin/ctdb_store \
+ tests/bin/ctdb_randrec tests/bin/ctdb_persistent \
+ tests/bin/ctdb_traverse tests/bin/rb_test tests/bin/ctdb_transaction \
@INFINIBAND_BINS@
BINS = bin/ctdb @CTDB_SCSI_IO@ bin/ctdb_ipmux bin/smnotify
SBINS = bin/ctdbd
-DIRS = lib bin
+DIRS = lib bin tests/bin
.SUFFIXES: .c .o .h .1 .1.xml .1.html
@echo Generating $@
rpcgen -l utils/smnotify/smnotify.x > utils/smnotify/gen_smnotify.c
-bin/rb_test: $(CTDB_CLIENT_OBJ) tests/rb_test.o
+tests/bin/rb_test: $(CTDB_CLIENT_OBJ) tests/src/rb_test.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/rb_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/rb_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ctdb_bench: $(CTDB_CLIENT_OBJ) tests/ctdb_bench.o
+tests/bin/ctdb_bench: $(CTDB_CLIENT_OBJ) tests/src/ctdb_bench.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/ctdb_bench.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/ctdb_bench.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ctdb_fetch: $(CTDB_CLIENT_OBJ) tests/ctdb_fetch.o
+tests/bin/ctdb_fetch: $(CTDB_CLIENT_OBJ) tests/src/ctdb_fetch.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/ctdb_fetch.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/ctdb_fetch.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ctdb_store: $(CTDB_CLIENT_OBJ) tests/ctdb_store.o
+tests/bin/ctdb_store: $(CTDB_CLIENT_OBJ) tests/src/ctdb_store.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/ctdb_store.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/ctdb_store.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ctdb_traverse: $(CTDB_CLIENT_OBJ) tests/ctdb_traverse.o
+tests/bin/ctdb_traverse: $(CTDB_CLIENT_OBJ) tests/src/ctdb_traverse.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/ctdb_traverse.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/ctdb_traverse.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ctdb_randrec: $(CTDB_CLIENT_OBJ) tests/ctdb_randrec.o
+tests/bin/ctdb_randrec: $(CTDB_CLIENT_OBJ) tests/src/ctdb_randrec.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/ctdb_randrec.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/ctdb_randrec.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ctdb_persistent: $(CTDB_CLIENT_OBJ) tests/ctdb_persistent.o
+tests/bin/ctdb_persistent: $(CTDB_CLIENT_OBJ) tests/src/ctdb_persistent.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/ctdb_persistent.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/ctdb_persistent.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ctdb_transaction: $(CTDB_CLIENT_OBJ) tests/ctdb_transaction.o
+tests/bin/ctdb_transaction: $(CTDB_CLIENT_OBJ) tests/src/ctdb_transaction.o
@echo Linking $@
- @$(CC) $(CFLAGS) -o $@ tests/ctdb_transaction.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+ @$(CC) $(CFLAGS) -o $@ tests/src/ctdb_transaction.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
-bin/ibwrapper_test: $(CTDB_CLIENT_OBJ) ib/ibwrapper_test.o
+tests/bin/ibwrapper_test: $(CTDB_CLIENT_OBJ) ib/ibwrapper_test.o
@echo Linking $@
@$(CC) $(CFLAGS) -o $@ ib/ibwrapper_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o ib/ibw_ctdb.o ib/ibw_ctdb_init.o"
INFINIBAND_LIBS="-lrdmacm -libverbs"
- INFINIBAND_BINS="bin/ibwrapper_test"
+ INFINIBAND_BINS="tests/bin/ibwrapper_test"
AC_CHECK_HEADERS(infiniband/verbs.h, [], [
echo "ERROR: you need infiniband/verbs.h when ib enabled!"
--- /dev/null
+Introduction
+------------
+
+The run_tests script can be run as follows:
+
+ scripts/run_tests simple/*.sh
+
+It can also be run from other places (e.g. the top level ctdb source
+directory), as it works out where the tree is.
+
+The pseudo-test simple/00_ctdb_init.sh causes ctdb to be (re)started
+on all nodes to attempt to force the cluster into a healthy state. By
+default (i.e. if CTDB_TEST_REAL_CLUSTER is unset - see below) this
+causes some local daemons to be started on the local machine. Tests
+can also be run against a real or virtual cluster. All tests
+communicate with cluster nodes using onnode - when using local daemons
+this is accomplished via some test hooks in onnode and the ctdb
+client.
+
+Command-line options
+--------------------
+
+The most useful option is "-s", which causes a summary of test results
+to be printed at the end of testing.
+
+Environment variables
+---------------------
+
+run_tests supports several environment variables, mostly implemented
+in scripts/ctdb_test_env. These are:
+
+* CTDB_TEST_REAL_CLUSTER
+
+ If set, testing will occur on a real or virtual cluster.
+
+ Assumptions:
+
+ - The ctdb client command can be found via $PATH on the nodes.
+
+ - Password-less ssh access to the cluster nodes is permitted from
+ the test host.
+
+ - $CTDB_NODES_FILE is set to the location of a file similar to
+ /etc/ctdb/nodes. The file can be obtained by scping it from one
+ of the cluster nodes.
+
+ - See CTDB_TEST_REMOTE_DIR.
+
+ If not set, testing will proceed against local daemons.
+
+* CTDB_TEST_REMOTE_DIR
+
+ This may be required when running against a real or virtual cluster
+ (as opposed to local daemons).
+
+ If set, this points to a directory containing the contents of the
+ tests/scripts/ directory, as well as all of the test binaries. This
+ can be accomplished in a couple of ways:
+
+ * By copying the relevant scripts/binaries to some directory.
+
+ * Building an RPM containing all of the test code that is required
+ on the cluster nodes and installing it on each node. Hopefully
+ this will be supported in a future version of the CTDB packaging
+ process.
+
+ If not set, the test system assumes that the CTDB tree is available
+ in the same location on the cluster nodes as on the test host. This
+ could be accomplished by copying or by sharing with NFS (or
+ similar).
+
+* VALGRIND
+
+ This can be used to cause all invocations of the ctdb client (and,
+ with local daemons, the ctdbd daemons themselves) to occur under
+ valgrind.
+
+ The easiest way of doing this is something like:
+
+ VALGRIND="valgrind -q" scripts/run_tests ...
+
+ NOTE: Some libc calls seem to do weird things and perhaps cause
+ spurious output from ctdbd at start time. Please read valgrind
+ output carefully before reporting bugs. :-)
+
+* CTDB
+
+ How to invoke the ctdb client. If not already set and if $VALGRIND
+ is set, this is set to "$VALGRIND ctdb". If this is not already set
+ but $VALGRIND is not set, this is simply set to "ctdb"
+
+Look, no run_test!
+------------------
+
+If you want to integrate individual tests into some other test
+environment you can use scripts/ctdb_test_env to wrap individual
+tests. They will return 0 on success, non-zero otherwise, and will
+print the output omitting the test header/footer that surrounds test
+output when tests are run using run_tests. So, you can do something
+like:
+
+ for i in simple/*.sh ; do
+ my_test_framework_wrapper scripts/ctdb_test_env $i
+ done
+
+to have your own framework process the test results and output.
--- /dev/null
+* Make tests know about IPv6.
+* Tests that write to database.
+* Tests that check actual network connectivity on failover.
+* Handle interrupting tests better.
+++ /dev/null
-#!/bin/sh
-
-killall -q ctdb_bench ctdbd
-
-NUMNODES=2
-if [ $# -gt 0 ]; then
- NUMNODES=$1
-fi
-
-rm -f nodes.txt
-for i in `seq 1 $NUMNODES`; do
- echo 127.0.0.$i >> nodes.txt
-done
-
-tests/start_daemons.sh $NUMNODES nodes.txt || exit 1
-
-killall -9 ctdb_bench
-echo "Trying $NUMNODES nodes"
-for i in `seq 1 $NUMNODES`; do
- $VALGRIND bin/ctdb_bench --socket sock.$i -n $NUMNODES $* &
-done
-
-wait
-bin/ctdb shutdown --socket sock.1 -n all
+++ /dev/null
-#!/bin/sh
-
-killall -q ctdbd
-
-tests/start_daemons.sh 2 || exit 1
-
-echo "Testing ping"
-$VALGRIND bin/ctdb ping || exit 1
-
-echo "Testing status"
-$VALGRIND bin/ctdb status || exit 1
-
-echo "Testing statistics"
-$VALGRIND bin/ctdb -n all statistics || exit 1
-
-echo "Testing statisticsreset"
-$VALGRIND bin/ctdb -n all statisticsreset || exit 1
-
-echo "Testing debug"
-$VALGRIND bin/ctdb -n all setdebug 3 || exit 1
-$VALGRIND bin/ctdb -n all getdebug || exit 1
-$VALGRIND bin/ctdb -n all setdebug 0 || exit 1
-$VALGRIND bin/ctdb -n all getdebug || exit 1
-
-echo "Attaching to some databases"
-$VALGRIND bin/ctdb attach test1.tdb || exit 1
-$VALGRIND bin/ctdb attach test2.tdb || exit 1
-
-echo "Testing getdbmap"
-$VALGRIND bin/ctdb getdbmap || exit 1
-
-echo "Testing status"
-$VALGRIND bin/ctdb status || exit 1
-
-echo "Testing variables"
-$VALGRIND bin/ctdb listvars || exit 1
-$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
-$VALGRIND bin/ctdb setvar TraverseTimeout 10 || exit 1
-$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
-
-sleep 1
-
-echo "Testing shutdown"
-$VALGRIND bin/ctdb shutdown -n all || exit 1
-
-sleep 1
-
-echo "All done"
-killall -q ctdbd
-exit 0
+++ /dev/null
-#!/bin/sh
-
-NUMNODES=2
-if [ $# -gt 0 ]; then
- NUMNODES=$1
-fi
-
-trap 'echo "Killing test"; killall -9 -q ctdbd ctdb_fetch; exit 1' INT TERM
-
-tests/start_daemons.sh $NUMNODES || exit 1
-
-
-killall -9 -q ctdb_fetch
-for i in `seq 1 $NUMNODES`; do
- $VALGRIND bin/ctdb_fetch --socket sock.$i -n $NUMNODES $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-exit 0
+++ /dev/null
-127.0.0.1
-127.0.0.2
-127.0.0.3
-127.0.0.4
+++ /dev/null
-#!/bin/sh
-
-NUMNODES=4
-if [ $# -gt 0 ]; then
- NUMNODES=$1
-fi
-
-killall -9 -q ctdb_persistent ctdbd
-
-rm -rf test.db/persistent
-
-echo "Starting $NUMNODES daemons for SAFE persistent writes"
-tests/start_daemons.sh $NUMNODES || exit 1
-
-trap 'echo "Killing test"; killall -9 -q ctdbd ctdb_persistent; exit 1' INT TERM
-
-
-for i in `seq 1 $NUMNODES`; do
- $VALGRIND bin/ctdb_persistent --timelimit 30 --socket sock.$i $* &
- $VALGRIND bin/ctdb_persistent --timelimit 30 --socket sock.$i $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-killall -9 ctdbd
-
-
-
-echo "Starting $NUMNODES daemons for UNSAFE persistent writes"
-tests/start_daemons.sh $NUMNODES || exit 1
-
-killall -9 -q ctdb_persistent
-
-for i in `seq 1 $NUMNODES`; do
- $VALGRIND bin/ctdb_persistent --unsafe-writes --timelimit 30 --socket sock.$i $* &
- $VALGRIND bin/ctdb_persistent --unsafe-writes --timelimit 30 --socket sock.$i $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-killall -9 ctdbd
-
-
-
-exit 0
+++ /dev/null
-10.99.99.0/24
-10.99.99.1/24
-10.99.99.2/24
-10.99.99.3/24
#!/bin/sh
-trap 'echo "Killing test"; killall -9 -q ctdbd; exit 1' INT TERM
-
-tests/fetch.sh 4 || exit 1
-tests/bench.sh 4 || exit 1
-tests/ctdbd.sh || exit 1
+tests/scripts/run_tests -s tests/simple/*.sh || exit 1
echo "All OK"
exit 0
--- /dev/null
+#!/bin/bash
+
+ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd)
+export CTDB_DIR=$(dirname $(dirname $ctdb_test_scripts_dir))
+var_dir=$CTDB_DIR/tests/var
+
+######################################################################
+
+ctdb_tools_dir=$CTDB_DIR/tools
+
+PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}"
+
+export CTDB_TIMEOUT=60
+
+######################################################################
+
+if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then
+ CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap"
+else
+ CTDB_TEST_WRAPPER="${ctdb_test_scripts_dir}/test_wrap"
+fi
+export CTDB_TEST_WRAPPER
+
+# If we're not running on a real cluster then we need a local copy of
+# ctdb (and other stuff) in $PATH and we will use local daemons.
+if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+ export CTDB_TEST_NUM_DAEMONS=3
+
+ export CTDB_NODES_SOCKETS=""
+ for i in $(seq 0 $(($CTDB_TEST_NUM_DAEMONS -1))) ; do
+ CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}"
+ done
+
+ PATH="${CTDB_DIR}/bin:${CTDB_DIR}/tests/bin:${PATH}"
+fi
+
+# If $VALGRIND is set then use it whenever ctdb is called, but only if
+# $CTDB is not already set.
+[ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb"
+
+######################################################################
+
+"$@"
--- /dev/null
+# Hey Emacs, this is a -*- shell-script -*- !!! :-)
+
+fail ()
+{
+ echo "$*"
+ exit 1
+}
+
+######################################################################
+
+ctdb_test_begin ()
+{
+ local name="$1"
+
+ teststarttime=$(date '+%s')
+ testduration=0
+
+ echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--"
+ echo "Running test $name ($(date '+%T'))"
+ echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--"
+}
+
+ctdb_test_end ()
+{
+ local name="$1" ; shift
+ local status="$1" ; shift
+ # "$@" is command-line
+
+ local interp="SKIPPED"
+ local statstr=" (reason $*)"
+ if [ -n "$status" ] ; then
+ if [ $status -eq 0 ] ; then
+ interp="PASSED"
+ statstr=""
+ echo "ALL OK: $*"
+ else
+ interp="FAILED"
+ statstr=" (status $status)"
+ testfailures=$(($testfailures+1))
+ fi
+ fi
+
+ testduration=$(($(date +%s)-$teststarttime))
+
+ echo "=========================================================================="
+ echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)"
+ echo "=========================================================================="
+
+}
+
+test_exit ()
+{
+ exit $(($testfailures+0))
+}
+
+ctdb_test_exit ()
+{
+ local status=$?
+
+ trap - 0
+
+ [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
+
+ eval "$ctdb_test_exit_hook"
+ unset ctdb_test_exit_hook
+
+ if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then
+ echo "Restarting ctdb on all nodes to get back into known state..."
+ restart_ctdb
+ else
+ # This could be made unconditional but then we might get
+ # duplication from the recovery in restart_ctdb. We want to
+ # leave the recovery in restart_ctdb so that future tests that
+ # might do a manual restart mid-test will benefit.
+ echo "Forcing a recovery..."
+ onnode 0 ctdb recover
+ fi
+
+ test_exit
+}
+
+ctdb_test_run ()
+{
+ local name="$1" ; shift
+
+ [ -n "$1" ] || set -- "$name"
+
+ ctdb_test_begin "$name"
+
+ local status=0
+ "$@" || status=$?
+
+ ctdb_test_end "$name" "$status" "$*"
+
+ return $status
+}
+
+ctdb_test_usage()
+{
+ local status=${1:-2}
+
+ cat <<EOF
+Usage: $0 [option]
+
+Options:
+ -h, --help show this screen.
+ -v, --version show test case version.
+ --category show the test category (ACL, CTDB, Samba ...).
+ -d, --description show test case description.
+ --summary show short test case summary.
+EOF
+
+ exit $status
+}
+
+ctdb_test_version ()
+{
+ [ -n "$CTDB_DIR" ] || fail "Can not determine version."
+
+ (cd "$CTDB_DIR" && git describe)
+}
+
+ctdb_test_cmd_options()
+{
+ [ -n "$1" ] || return 0
+
+ case "$1" in
+ -h|--help) ctdb_test_usage 0 ;;
+ -v|--version) ctdb_test_version ;;
+ --category) echo "CTDB" ;;
+ -d|--description) test_info ;;
+ --summary) test_info | head -1 ;;
+ *)
+ echo "Error: Unknown parameter = $1"
+ echo
+ ctdb_test_usage 2
+ ;;
+ esac
+
+ exit 0
+}
+
+ctdb_test_init ()
+{
+ scriptname=$(basename "$0")
+ testfailures=0
+
+ ctdb_test_cmd_options $@
+
+ trap "ctdb_test_exit" 0
+}
+
+########################################
+
+# Sets: $out
+try_command_on_node ()
+{
+ local nodespec="$1" ; shift
+
+ local verbose=false
+ local onnode_opts=""
+
+ while [ "${nodespec#-}" != "$nodespec" ] ; do
+ if [ "$nodespec" = "-v" ] ; then
+ verbose=true
+ else
+ onnode_opts="$nodespec"
+ fi
+ nodespec="$1" ; shift
+ done
+
+ local cmd="$*"
+
+ out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
+
+ echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
+ echo "$out"
+ return 1
+ }
+
+ if $verbose ; then
+ echo "Output of \"$cmd\":"
+ echo "$out"
+ fi
+}
+
+sanity_check_output ()
+{
+ local min_lines="$1"
+ local regexp="$2" # Should be anchored as necessary.
+ local output="$3"
+
+ local ret=0
+
+ local num_lines=$(echo "$output" | wc -l)
+ echo "There are $num_lines lines of output"
+ if [ $num_lines -lt $min_lines ] ; then
+ echo "BAD: that's less than the required number (${min_lines})"
+ ret=1
+ fi
+
+ local status=0
+ local unexpected # local doesn't pass through status of command on RHS.
+ unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
+
+ # Note that this is reversed.
+ if [ $status -eq 0 ] ; then
+ echo "BAD: unexpected lines in output:"
+ echo "$unexpected" | cat -A
+ ret=1
+ else
+ echo "Output lines look OK"
+ fi
+
+ return $ret
+}
+
+sanity_check_ips ()
+{
+ local ips="$1" # Output of "ctdb ip -n all"
+
+ echo "Sanity checking IPs..."
+
+ local x ipp prev
+ prev=""
+ while read x ipp ; do
+ [ "$ipp" = "-1" ] && break
+ if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+ echo "OK"
+ return 0
+ fi
+ prev="$ipp"
+ done <<<"$ips"
+
+ echo "BAD: a node was -1 or IPs are only assigned to one node"
+ echo "Are you running an old version of CTDB?"
+ return 1
+}
+
+#######################################
+
+# Wait until either timeout expires or command succeeds. The command
+# will be tried once per second.
+wait_until ()
+{
+ local timeout="$1" ; shift # "$@" is the command...
+
+ echo -n "<${timeout}|"
+ while [ $timeout -gt 0 ] ; do
+ if "$@" ; then
+ echo '|'
+ echo "OK"
+ return 0
+ fi
+ echo -n .
+ timeout=$(($timeout - 1))
+ sleep 1
+ done
+
+ echo "*TIMEOUT*"
+
+ return 1
+}
+
+sleep_for ()
+{
+ echo -n "=${1}|"
+ for i in $(seq 1 $1) ; do
+ echo -n '.'
+ sleep 1
+ done
+ echo '|'
+}
+
+_cluster_is_healthy ()
+{
+ local out x count line
+
+ out=$(ctdb -Y status 2>&1) || return 1
+
+ {
+ read x
+ count=0
+ while read line ; do
+ count=$(($count + 1))
+ [ "${line#:*:*:}" != "0:0:0:0:" ] && return 1
+ done
+ [ $count -gt 0 ] && return $?
+ } <<<"$out" # Yay bash!
+}
+
+cluster_is_healthy ()
+{
+ if _cluster_is_healthy ; then
+ echo "Cluster is HEALTHY"
+ exit 0
+ else
+ echo "Cluster is UNHEALTHY"
+ exit 1
+ fi
+}
+
+wait_until_healthy ()
+{
+ local timeout="${1:-120}"
+
+ echo "Waiting for cluster to become healthy..."
+
+ wait_until 120 _cluster_is_healthy
+}
+
+# This function is becoming nicely overloaded. Soon it will collapse! :-)
+node_has_status ()
+{
+ local pnn="$1"
+ local status="$2"
+
+ local bits fpat mpat
+ case "$status" in
+ (unhealthy) bits="?:?:?:1" ;;
+ (healthy) bits="?:?:?:0" ;;
+ (disconnected) bits="1:?:?:?" ;;
+ (connected) bits="0:?:?:?" ;;
+ (banned) bits="?:1:?:?" ;;
+ (unbanned) bits="?:0:?:?" ;;
+ (disabled) bits="?:?:1:?" ;;
+ (enabled) bits="?:?:0:?" ;;
+ (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;;
+ (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
+ (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
+ (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;;
+ *)
+ echo "node_has_status: unknown status \"$status\""
+ return 1
+ esac
+
+ if [ -n "$bits" ] ; then
+ local out x line
+
+ out=$(ctdb -Y status 2>&1) || return 1
+
+ {
+ read x
+ while read line ; do
+ [ "${line#:${pnn}:*:${bits}:}" = "" ] && return 0
+ done
+ return 1
+ } <<<"$out" # Yay bash!
+ elif [ -n "$fpat" ] ; then
+ ctdb statistics -n "$pnn" | egrep -q "$fpat"
+ elif [ -n "$mpat" ] ; then
+ ctdb getmonmode -n "$pnn" | egrep -q "$mpat"
+ else
+ echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
+ return 1
+ fi
+}
+
+wait_until_node_has_status ()
+{
+ local pnn="$1"
+ local status="$2"
+ local timeout="${3:-30}"
+
+ echo "Waiting until node $pnn has status \"$status\"..."
+
+ wait_until $timeout node_has_status "$pnn" "$status"
+}
+
+# Useful for superficially testing IP failover.
+# IPs must be on nodes matching nodeglob.
+ips_are_on_nodeglob ()
+{
+ local nodeglob="$1" ; shift
+ local ips="$*"
+
+ local out
+
+ try_command_on_node 1 ctdb ip -n all
+
+ while read ip pnn ; do
+ for check in $ips ; do
+ if [ "$check" = "$ip" ] ; then
+ case "$pnn" in
+ ($nodeglob) : ;;
+ (*) return 1 ;;
+ esac
+ ips="${ips/${ip}}" # Remove from list
+ fi
+ done
+ done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+ ips="${ips// }" # Remove any spaces.
+ [ -z "$ips" ]
+}
+
+wait_until_ips_are_on_nodeglob ()
+{
+ echo "Waiting for IPs to fail over..."
+
+ wait_until 60 ips_are_on_nodeglob "$@"
+}
+
+#######################################
+
+daemons_stop ()
+{
+ echo "Attempting to politely shutdown daemons..."
+ onnode 1 ctdb shutdown -n all || true
+
+ echo "Sleeping for a while..."
+ sleep_for 1
+
+ if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then
+ echo "Killing remaining daemons..."
+ pkill -f $CTDB_DIR/bin/ctdbd
+
+ if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then
+ echo "Once more with feeling.."
+ pkill -9 $CTDB_DIR/bin/ctdbd
+ fi
+ fi
+
+ local var_dir=$CTDB_DIR/tests/var
+ rm -rf $var_dir/test.db
+}
+
+daemons_setup ()
+{
+ local num_nodes="${1:-2}" # default is 2 nodes
+
+ local var_dir=$CTDB_DIR/tests/var
+
+ mkdir -p $var_dir/test.db/persistent
+
+ local nodes=$var_dir/nodes.txt
+ local public_addresses=$var_dir/public_addresses.txt
+ local no_public_addresses=$var_dir/no_public_addresses.txt
+ rm -f $nodes $public_addresses $no_public_addresses
+
+ # If there are (strictly) greater than 2 nodes then we'll randomly
+ # choose a node to have no public addresses.
+ local no_public_ips=-1
+ [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes))
+ echo "$no_public_ips" >$no_public_addresses
+
+ local i
+ for i in $(seq 1 $num_nodes) ; do
+ if [ "${CTDB_USE_IPV6}x" != "x" ]; then
+ echo ::$i >> $nodes
+ ip addr add ::$i/128 dev lo
+ else
+ echo 127.0.0.$i >> $nodes
+ # 2 public addresses on most nodes, just to make things interesting.
+ if [ $(($i - 1)) -ne $no_public_ips ] ; then
+ echo "192.0.2.$i/24 lo" >> $public_addresses
+ echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses
+ fi
+ fi
+ done
+}
+
+daemons_start ()
+{
+ local num_nodes="${1:-2}" # default is 2 nodes
+ shift # "$@" gets passed to ctdbd
+
+ local var_dir=$CTDB_DIR/tests/var
+
+ local nodes=$var_dir/nodes.txt
+ local public_addresses=$var_dir/public_addresses.txt
+ local no_public_addresses=$var_dir/no_public_addresses.txt
+
+ local no_public_ips=-1
+ [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses
+
+ local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent"
+
+ echo "Starting $num_nodes ctdb daemons..."
+ if [ "$no_public_ips" != -1 ] ; then
+ echo "Node $no_public_ips will have no public IPs."
+ fi
+
+ for i in $(seq 0 $(($num_nodes - 1))) ; do
+ if [ $(id -u) -eq 0 ]; then
+ ctdb_options="$ctdb_options --public-interface=lo"
+ fi
+
+ if [ $i -eq $no_public_ips ] ; then
+ ctdb_options="$ctdb_options --public-addresses=/dev/null"
+ else
+ ctdb_options="$ctdb_options --public-addresses=$public_addresses"
+ fi
+
+ # Need full path so we can use "pkill -f" to kill the daemons.
+ $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" ||return 1
+ done
+
+ if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then
+ ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1
+ fi
+}
+
+#######################################
+
+_restart_ctdb ()
+{
+ if [ -e /etc/redhat-release ] ; then
+ service ctdb restart
+ else
+ /etc/init.d/ctdb restart
+ fi
+}
+
+setup_ctdb ()
+{
+ if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+ daemons_setup $CTDB_TEST_NUM_DAEMONS
+ fi
+}
+
+restart_ctdb ()
+{
+ if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+ daemons_stop
+ daemons_start $CTDB_TEST_NUM_DAEMONS
+ else
+ onnode -pq all $CTDB_TEST_WRAPPER _restart_ctdb
+ fi || return 1
+
+ onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1
+
+ echo "Setting RerecoveryTimeout to 1"
+ onnode -pq all "ctdb setvar RerecoveryTimeout 1"
+
+ # In recent versions of CTDB, forcing a recovery like this blocks
+ # until the recovery is complete. Hopefully this will help the
+ # cluster to stabilise before a subsequent test.
+ echo "Forcing a recovery..."
+ onnode -q 0 ctdb recover
+
+ #echo "Sleeping to allow ctdb to settle..."
+ #sleep_for 10
+
+ echo "ctdb is ready"
+}
+
+#######################################
+
+install_eventscript ()
+{
+ local script_name="$1"
+ local script_contents="$2"
+
+ if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+ # The quoting here is *very* fragile. However, we do
+ # experience the joy of installing a short script using
+ # onnode, and without needing to know the IP addresses of the
+ # nodes.
+ onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\""
+ else
+ f="${CTDB_DIR}/tests/events.d/${script_name}"
+ echo "$script_contents" >"$f"
+ chmod 755 "$f"
+ fi
+}
+
+uninstall_eventscript ()
+{
+ local script_name="$1"
+
+ if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+ onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\""
+ else
+ rm -vf "${CTDB_DIR}/tests/events.d/${script_name}"
+ fi
+}
--- /dev/null
+#!/bin/bash
+
+# The ability of ctdb_test_env to take tests on the command-line is
+# nice, but here we need to hack around it with that colon to reset
+# the arguments that it sees.
+. $(dirname $0)/ctdb_test_env :
+
+. ctdb_test_functions.bash
+
+usage() {
+ cat <<EOF
+Usage: run_tests [OPTIONS] [TESTS]
+
+EOF
+ exit 1
+}
+
+######################################################################
+
+with_summary=false
+
+temp=$(getopt -n "$prog" -o "xhs" -l help -- "$@")
+
+[ $? != 0 ] && usage
+
+eval set -- "$temp"
+
+while true ; do
+ case "$1" in
+ -x) set -x; shift ;;
+ -s) with_summary=true ; shift ;;
+ --) shift ; break ;;
+ *) usage ;;
+ esac
+done
+
+######################################################################
+
+tests_total=0
+tests_passed=0
+summary=""
+
+rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@')
+ww=$((rows - 7))
+
+for f; do
+ [ -x $f ] || fail "test \"$f\" is not executable"
+ tests_total=$(($tests_total + 1))
+ if ctdb_test_run "$f" ; then
+ tests_passed=$(($tests_passed + 1))
+ t="PASSED"
+ else
+ t="FAILED"
+ fi
+ summary=$(printf "%s\n%-${ww}s%s" "$summary" "$f" "$t")
+done
+
+if $with_summary ; then
+ echo "$summary"
+ echo
+ echo "${tests_passed}/${tests_total} tests passed"
+fi
+
+test_exit
--- /dev/null
+#!/bin/bash
+
+# Execute the given command. The intention is that it is a function
+# from ctdb_test_functions.bash.
+
+PATH="$(dirname $0):${PATH}"
+
+. ctdb_test_functions.bash
+
+"$@"
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Restart the ctdbd daemons of a CTDB cluster.
+
+No error if ctdbd is not already running on the cluster.
+
+Prerequisites:
+
+* Nodes must be accessible via 'onnode'.
+
+Steps:
+
+1. Restart the ctdb daemons on all nodes using a method according to
+ the test environment and platform.
+
+Expected results:
+
+* The cluster is healthy within a reasonable timeframe.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+echo "Restarting ctdb on all nodes..."
+setup_ctdb
+restart_ctdb
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Install an event script on all nodes that helps detect ctdb events.
+
+We could install separate scripts for particular events later as
+needed. However, the script installed here will allow detection of
+all events. It also allows a file to be created to indicate that a
+node should be marked as unhealthy.
+
+Prerequisites:
+
+* Nodes must be accessible via 'onnode'.
+
+Steps:
+
+1. Use the install_eventscript to install the eventscript.
+
+Expected results:
+
+* The script is successfully installed on all nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+script='#!/bin/sh
+out=$(ctdb pnn)
+pnn="${out#PNN:}"
+
+# Allow creation of flag files that are removed to confirm that events
+# are taking place.
+rm -f "/tmp/ctdb-test-flag.${1}.${pnn}"
+
+# Allow creation of a trigger file to make a monitor event fail and
+# force a node to be marked as unhealthy. This avoids having to look
+# at log files to confirm that monitoring is working. Note that
+# ${pnn} is needed in the filename if we are testing using local
+# daemons so we put in there regardless.
+trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}"
+detected="/tmp/ctdb-test-unhealthy-detected.${pnn}"
+if [ "$1" = "monitor" ] ; then
+ if [ -e "$trigger" ] ; then
+ echo "${0}: Unhealthy because \"$trigger\" detected"
+ touch "$detected"
+ exit 1
+ elif [ -e "$detected" -a ! -e "$trigger" ] ; then
+ echo "${0}: Healthy again, \"$trigger\" no longer detected"
+ rm "$detected"
+ fi
+fi
+
+exit 0
+'
+
+install_eventscript "00.ctdb_test_trigger" "$script"
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Use 'onnode' to confirm connectivity between all cluster nodes.
+
+Steps:
+
+1. Do a recursive "onnode all" to make sure all the nodes can connect
+ to each other. On a cluster this ensures that SSH keys are known
+ between all hosts, which will stop output being corrupted with
+ messages about nodes being added to the list of known hosts.
+
+Expected results:
+
+* 'onnode' works between all nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+
+#
+
+echo "Checking connectivity between nodes..."
+onnode all onnode all true
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify the output of the 'ctdb version' command.
+
+This test assumes an RPM-based installation and needs to be skipped on
+non-RPM systems.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run the 'ctdb version' command on one of the cluster nodes.
+3. Compare the version displayed with that listed by the rpm command
+ for the ctdb package.
+
+Expected results:
+
+* The 'ctdb version' command displays the ctdb version number.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+if ! try_command_on_node -v 0 "rpm -q ctdb" ; then
+ echo "No useful output from rpm, SKIPPING rest of test".
+ exit 0
+fi
+rpm_ver="${out#ctdb-}"
+
+try_command_on_node -v 0 "$CTDB version"
+ctdb_ver="${out#CTDB version: }"
+
+if [ "$ctdb_ver" = "$rpm_ver" ] ; then
+ echo "OK: CTDB version = RPM version"
+else
+ echo "BAD: CTDB version != RPM version"
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb listvars' shows a list of all tunable variables.
+
+This test simply checks that at least 5 sane looking lines are
+printed. It does not check that the list is complete or that the
+values are sane.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb listvars' and verify that it shows a list of tunable
+ variables and their current values.
+
+Expected results:
+
+* 'ctdb listvars' works as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB listvars"
+
+sanity_check_output \
+ 5 \
+ '^[[:alpha:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \
+ "$out"
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb getvar' works correctly.
+
+Expands on the steps below as it actually checks the values of all
+variables listed by 'ctdb listvars'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb getvars <varname>' with a valid variable name (possibly
+ obtained via 'ctdb listvars'.
+3. Verify that the command displays the correct value of the variable
+ (corroborate with the value shown by 'ctdb listvars'.
+
+Expected results:
+
+* 'ctdb getvar' shows the correct value of the variable.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB listvars"
+
+echo "Veryifying all variable values using \"ctdb getvar\"..."
+
+echo "$out" |
+while read var x val ; do
+ try_command_on_node 0 "$CTDB getvar $var"
+
+ val2=$(echo $out | sed -e 's@.*[[:space:]]@@')
+
+ if [ "$val" != "$val2" ] ; then
+ echo "MISMATCH on $var: $val != $val2"
+ exit 1
+ fi
+done
+
+testfailures=$?
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb setvar' works correctly.
+
+Doesn't strictly follow the procedure outlines below, since it doesn't
+pick a variable from the output of 'ctdb listvars'. However, it
+verifies the value with 'ctdb getvar' in addition to 'ctdb listvars'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get a list of all the ctdb tunable variables, using the 'ctdb
+ listvars' command.
+3. Set the value of one of the variables using the 'setvar' control on
+ one of the nodes. E.g. 'ctdb setvar DeterministicIPs 0'.
+4. Verify that the 'listvars' control now shows the new value for the
+ variable.
+
+Expected results:
+
+* After setting a value using 'ctdb setvar', 'ctdb listvars' shows the
+ modified value of the variable.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+var="RecoverTimeout"
+
+try_command_on_node -v 0 $CTDB getvar $var
+
+val="${out#*= }"
+
+echo "Going to try incrementing it..."
+
+incr=$(($val + 1))
+
+try_command_on_node 0 $CTDB setvar $var $incr
+
+echo "That seemed to work, let's check the value..."
+
+try_command_on_node -v 0 $CTDB getvar $var
+
+newval="${out#*= }"
+
+if [ "$incr" != "$newval" ] ; then
+ echo "Nope, that didn't work..."
+ exit 1
+fi
+
+echo "Look's good! Now verifying with \"ctdb listvars\""
+try_command_on_node -v 0 "$CTDB listvars | grep '^$var'"
+
+check="${out#*= }"
+
+if [ "$incr" != "$check" ] ; then
+ echo "Nope, that didn't work. Restarting ctdb to get back into known state..."
+ restart_ctdb
+ exit 1
+fi
+
+echo "Look's good! Putting the old value back..."
+cmd="$CTDB setvar $var $val"
+try_command_on_node 0 $cmd
+
+echo "All done..."
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb listnodes' shows the list of nodes in a ctdb cluster.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb listnodes' on all the nodes of the cluster.
+3. Verify that one all the nodes the command displays a list of
+ current cluster nodes.
+
+Expected results:
+
+* 'ctdb listnodes' displays the correct information.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB listnodes"
+
+num_nodes=$(echo "$out" | wc -l)
+
+# Each line should look like an IP address.
+sanity_check_output \
+ 2 \
+ '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' \
+ "$out"
+
+out_0="$out"
+
+echo "Checking other nodes..."
+
+n=1
+while [ $n -lt $num_nodes ] ; do
+ echo -n "Node ${n}: "
+ try_command_on_node $n "$CTDB listnodes"
+ if [ "$out_0" = "$out" ] ; then
+ echo "OK"
+ else
+ echo "DIFFERs from node 0:"
+ echo "$out"
+ testfailures=1
+ fi
+ n=$(($n + 1))
+done
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb getpid' works as expected.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb getpid -n <number>' on the nodes to check the PID of the
+ ctdbd process.
+3. Verify that the output is valid.
+4. Verify that with the '-n all' option the command shows the PIDs on
+ all the nodes
+
+Expected results:
+
+* 'ctdb getpid' shows valid output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# This is an attempt at being independent of the number of nodes
+# reported by "ctdb getpid -n all".
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+echo "There are $num_nodes nodes..."
+
+# Call getpid a few different ways and make sure the answer is always the same.
+
+try_command_on_node -v 0 "onnode -q all $CTDB getpid"
+pids_onnode="$out"
+
+try_command_on_node -v 0 "$CTDB getpid -n all"
+pids_getpid_all="$out"
+
+cmd=""
+n=0
+while [ $n -lt $num_nodes ] ; do
+ cmd="${cmd}${cmd:+; }$CTDB getpid -n $n"
+ n=$(($n + 1))
+done
+try_command_on_node -v 0 "( $cmd )"
+pids_getpid_n="$out"
+
+if [ "$pids_onnode" = "$pids_getpid_all" -a \
+ "$pids_getpid_all" = "$pids_getpid_n" ] ; then
+ echo "They're the same... cool!"
+else
+ echo "Error: they differ."
+ testfailures=1
+fi
+
+echo "Checking each PID for validity"
+
+n=0
+while [ $n -lt $num_nodes ] ; do
+ read line
+ pid=${line#Pid:}
+ try_command_on_node $n "ls -l /proc/${pid}/exe | sed -e 's@.*/@@'"
+ echo -n "Node ${n}, PID ${pid} looks to be running \"$out\" - "
+ if [ "$out" = "ctdbd" ] ; then
+ echo "GOOD!"
+ elif [ -n "$VALGRIND" -a "$out" = "memcheck" ] ; then
+ # We could check cmdline too if this isn't good enough.
+ echo "GOOD enough!"
+ else
+ echo "BAD!"
+ testfailures=1
+ fi
+ n=$(($n + 1))
+done <<<"$pids_onnode"
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb process-exists' shows correct information.
+
+The implementation is creative about how it gets PIDs for existing and
+non-existing processes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. On one of the cluster nodes, get the PID of an existing process
+ (using ps wax).
+3. Run 'ctdb process-exists <pid>' on the node and verify that the
+ correct output is shown.
+4. Run 'ctdb process-exists <pid>' with a pid of a non-existent
+ process and verify that the correct output is shown.
+
+Expected results:
+
+* 'ctdb process-exists' shows the correct output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+# Create a background process on $test_node that will last for 60 seconds.
+# It should still be there when we check.
+try_command_on_node $test_node 'sleep 60 >/dev/null 2>&1 & echo $!'
+pid="$out"
+
+echo "Checking for PID $pid on node $test_node"
+# set -e is good, but avoid it here
+status=0
+onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$?
+echo "$out"
+
+if [ $status -eq 0 ] ; then
+ echo "OK"
+else
+ echo "BAD"
+ testfailures=1
+fi
+
+# Now just echo the PID of the shell from the onnode process on node
+# 2. This PID will disappear and PIDs shouldn't roll around fast
+# enough to trick the test... but there is a chance that will happen!
+try_command_on_node $test_node 'echo $$'
+pid="$out"
+
+echo "Checking for PID $pid on node $test_node"
+# set -e is good, but avoid it here
+status=0
+onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$?
+echo "$out"
+
+if [ $status -ne 0 ] ; then
+ echo "OK"
+else
+ echo "BAD"
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify the operation of 'ctdb isnotrecmaster'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb isnotrecmaster' on each node.
+
+3. Verify that only 1 node shows the output 'This node is the
+ recmaster' and all the other nodes show the output 'This node is
+ not the recmaster'.
+
+Expected results:
+
+* 'ctdb isnotrecmaster' shows the correct output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+cmd="$CTDB isnotrecmaster || true"
+try_command_on_node all "$cmd"
+echo "Output of \"$cmd\":"
+echo "$out"
+
+num_all_lines=$(echo "$out" | wc -l)
+num_rm_lines=$(echo "$out" | fgrep -c 'this node is the recmaster') || true
+num_not_rm_lines=$(echo "$out" | fgrep -c 'this node is not the recmaster') || true
+
+if [ $num_rm_lines -eq 1 ] ; then
+ echo "OK, there is only 1 recmaster"
+else
+ echo "BAD, there are ${num_rm_lines} nodes claiming to be the recmaster"
+ testfailures=1
+fi
+
+if [ $(($num_all_lines - $num_not_rm_lines)) -eq 1 ] ; then
+ echo "OK, all the other nodes claim not to be the recmaster"
+else
+ echo "BAD, there are only ${num_not_rm_lines} nodes claiming not to be the recmaster"
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify the operation of the 'ctdb ping' command.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run the 'ctdb ping' command on one of the nodes and verify that it
+ shows valid and expected output.
+3. Shutdown one of the cluster nodes, using the 'ctdb shutdown'
+ command.
+4. Run the 'ctdb ping -n <node>' command from another node to this
+ node.
+5. Verify that the command is not successful since th ctdb daemon is
+ not running on the node.
+
+Expected results:
+
+* The 'ctdb ping' command shows valid and expected output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB ping -n 1"
+
+sanity_check_output \
+ 1 \
+ '^response from 1 time=[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \
+ "$out"
+
+try_command_on_node 0 "$CTDB shutdown -n 1"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status 1 disconnected
+
+try_command_on_node -v 0 "! $CTDB ping -n 1"
+
+sanity_check_output \
+ 1 \
+ "(: ctdb_control error: 'ctdb_control to disconnected node'|Unable to get ping response from node 1|Node 1 is DISCONNECTED)" \
+ "$out"
+
+echo "Expect a restart..."
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb ip' shows the correct output.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb ip' on one of the nodes and verify the list of IP
+ addresses displayed (cross check the result with the output of
+ 'ip addr show' on the node).
+3. Verify that colon-separated output is generated with the -Y option.
+
+Expected results:
+
+* 'ctdb ip' shows the list of public IPs being served by a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 1 $CTDB ip -n all
+ips=$(echo "$out" | sed -e '1d')
+colons=$(echo "$ips" | sed -e 's@^@:@' -e 's@$@:@' -e 's@ @:@')
+
+while read ip pnn ; do
+ try_command_on_node $pnn "ip addr show"
+ if [ "${out/inet ${ip}\/}" != "$out" ] ; then
+ echo "GOOD: node $pnn appears to have $ip assigned"
+ else
+ echo "BAD: node $pnn does not appear to have $ip assigned"
+ testfailures=1
+ fi
+done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
+
+[ "$testfailures" != 1 ] && echo "Looks good!"
+
+cmd="$CTDB -Y ip -n all | sed -e '1d'"
+echo "Checking that \"$cmd\" produces expected output..."
+
+try_command_on_node 1 "$cmd"
+if [ "$out" = "$colons" ] ; then
+ echo "Yep, looks good!"
+else
+ echo "Nope, it looks like this:"
+ echo "$out"
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb getdebug' works as expected.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get the current debug level on a node, using 'ctdb getdebug -n <node>'.
+3. Verify that colon-separated output is generated with the -Y option.
+4. Verify that the '-n all' option shows the debug level on all nodes.
+
+Expected results:
+
+* 'ctdb getdebug' shows the debug level on all the nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+
+try_command_on_node -v 1 "onnode -q all $CTDB getdebug"
+getdebug_onnode="$out"
+
+sanity_check_output \
+ $num_nodes \
+ '^Node [[:digit:]] is at debug level [[:alpha:]]+ \([[:digit:]]\)$' \
+ "$out"
+
+try_command_on_node -v 1 "$CTDB getdebug -n all"
+getdebug_all="$out"
+
+cmd=""
+n=0
+while [ $n -lt $num_nodes ] ; do
+ cmd="${cmd}${cmd:+; }$CTDB getdebug -n $n"
+ n=$(($n + 1))
+done
+try_command_on_node -v 1 "$cmd"
+getdebug_n="$out"
+
+if [ "$getdebug_onnode" = "$getdebug_all" -a \
+ "$getdebug_all" = "$getdebug_n" ] ; then
+ echo "They're the same... cool!"
+else
+ echo "Error: they differ."
+ testfailures=1
+fi
+
+colons=""
+nl="
+"
+while read line ; do
+ t=$(echo "$line" | sed -r -e 's@Node [[:digit:]] is at debug level ([[:alpha:]]+) \((-?[[:digit:]])\)$@:\1:\2:@')
+ colons="${colons}${colons:+${nl}}:Name:Level:${nl}${t}"
+done <<<"$getdebug_onnode"
+
+cmd="$CTDB -Y getdebug -n all"
+echo "Checking that \"$cmd\" produces expected output..."
+
+try_command_on_node 1 "$cmd"
+if [ "$out" = "$colons" ] ; then
+ echo "Yep, looks good!"
+else
+ echo "Nope, it looks like this:"
+ echo "$out"
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb setdebug' works as expected.
+
+This is a little superficial. It checks that CTDB thinks the debug
+level has been changed but doesn't actually check that logging occurs
+at the new level.
+
+A test should also be added to see if setting the debug value via a
+numerical value works too.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get the current debug level on a node, using 'ctdb getdebug'.
+3. Change the debug level to some other value (e.g. EMERG) using
+ 'ctdb setdebug'.
+4. Verify that the new debug level is correctly set using 'ctdb getdebug'.
+
+Expected results:
+
+* 'ctdb setdebug' correctly sets the debug level on a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+get_debug ()
+{
+ # Sets; check_debug
+ local node="$1"
+
+ local out
+
+ try_command_on_node -v $node "$CTDB getdebug"
+ check_debug=$(echo "$out" |
+ sed -r -e 's@Node [[:digit:]] is at debug level ([[:alpha:]]+) \(-?[[:digit:]]\)$@\1@')
+}
+
+set_and_check_debug ()
+{
+ local node="$1"
+ local level="$2"
+
+ echo "Setting debug level on node ${node} to ${level}."
+ try_command_on_node $node "$CTDB setdebug ${level}"
+
+ local check_debug
+ get_debug $node
+
+ if [ "$level" = "$check_debug" ] ; then
+ echo "That seemed to work... cool!"
+ else
+ echo "BAD: Debug level should have changed to \"$level\" but it is \"$check_debug\"."
+ testfailures=1
+ fi
+}
+
+get_debug 1
+initial_debug="$check_debug"
+
+new_debug="EMERG"
+[ "$initial_debug" = "$new_debug" ] && new_debug="ALERT"
+
+set_and_check_debug 1 "$new_debug"
+
+if [ "$testfailures" != 1 ] ; then
+ echo "Returning the debug level to its initial value..."
+ set_and_check_debug 1 "$initial_debug"
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb statistics' works as expected.
+
+This is pretty superficial and could do more validation.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb statistics' on a node, and verify that the output is
+ valid.
+3. Repeat the command with the '-n all' option and verify that the
+ output is valid.
+
+Expected results:
+
+* 'ctdb statistics' shows valid output on all the nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+pattern='^(CTDB version 1|Gathered statistics for [[:digit:]]+ nodes|[[:space:]]+[[:alpha:]_]+[[:space:]]+[[:digit:]]+|[[:space:]]+(node|client|timeouts)|[[:space:]]+[[:alpha:]_]+_latency[[:space:]]+[[:digit:]]+\.[[:digit:]]+[[:space:]]sec)$'
+
+try_command_on_node -v 1 "$CTDB statistics"
+
+sanity_check_output 38 "$pattern" "$out"
+
+try_command_on_node -v 1 "$CTDB statistics -n all"
+
+sanity_check_output 38 "$pattern" "$out"
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb statisticsreset' works as expected.
+
+This is pretty superficial. It just checks that a few particular
+items reduce.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb statisticsreset' on all nodes and verify that it executes
+ successfully.
+
+Expected results:
+
+* 'ctdb statisticsreset' executes successfully.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+
+get_stat ()
+{
+ local label="$1"
+ local out="$2"
+
+ echo "$out" | sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" | head -1
+}
+
+check_reduced ()
+{
+ local label="$1"
+ local before="$2"
+ local after="$3"
+
+ if [ $after -lt $before ] ; then
+ echo "GOOD: ${label} reduced from ${before} to ${after}"
+ else
+ echo "BAD: ${label} did not reduce from ${before} to ${after}"
+ testfailures=1
+ fi
+}
+
+n=0
+while [ $n -lt $num_nodes ] ; do
+ echo "Getting initial statistics for node ${n}..."
+
+ try_command_on_node -v $n $CTDB statistics
+
+ before_req_control=$(get_stat "req_control" "$out")
+ before_reply_control=$(get_stat "reply_control" "$out")
+ before_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+
+ try_command_on_node $n $CTDB statisticsreset
+
+ try_command_on_node -v $n $CTDB statistics
+
+ after_req_control=$(get_stat "req_control" "$out")
+ after_reply_control=$(get_stat "reply_control" "$out")
+ after_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+
+ check_reduced "req_control" "$before_req_control" "$after_req_control"
+ check_reduced "reply_control" "$before_reply_control" "$after_reply_control"
+ check_reduced "node_packets_recv" "$before_node_packets_recv" "$after_node_packets_recv"
+
+ n=$(($n + 1))
+done
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that an IP address can be added to a node using 'ctdb addip'.
+
+This test goes to some trouble to figure out which IP address to add
+but assumes a 24-bit subnet mask. It does not handle IPv6. It does
+not do any network level checks that the new IP address is reachable
+but simply trusts 'ctdb ip' that the address has been added. There is
+also an extra prerequisite that the node being added to already has
+public addresses - this is difficult to avoid if the extra address is
+to be sensibly chosen.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+ served.
+3. Add an additional public address to be served by the node, using
+ 'ctdb addip'.
+4. Verify that this IP address has been added to the list of IP
+ addresses being served by the node, using the 'ctdb ip' command.
+
+Expected results:
+
+* 'ctdb ip' adds an IP address to the list of public IP addresses
+ being served by a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we just want a node that has public IPs.
+# This will work and is economically semi-randomly. :-)
+read x test_node <<<"$out"
+
+test_node_ips=""
+all_ips=""
+while read ip pnn ; do
+ all_ips="${all_ips}${all_ips:+ }${ip}"
+ [ "$pnn" = "$test_node" ] && \
+ test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+done <<<"$out"
+
+echo "Selected node ${test_node} with IPs: $test_node_ips"
+
+# Try to find a free IP adddress. This is inefficient but should
+# succeed quickly.
+try_command_on_node $test_node "ip addr show"
+all_test_node_ips=$(echo "$out" | sed -rn -e 's@^[[:space:]]+inet[[:space:]]+([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+/[[:digit:]]+).*[[:space:]]([^[:space:]]+)+$@\1:\2@p')
+
+add_ip=""
+
+# Use an IP already on one of the nodes, remove the last octet and
+# loop through the possible IP addreses.
+for i in $test_node_ips ; do
+ prefix="${i%.*}"
+ for j in $(seq 1 254) ; do
+ try="${prefix}.${j}"
+ # Try to make sure it isn't used anywhere!
+
+ # First, make sure it isn't an existing public address on the
+ # cluster.
+ for k in $all_ips ; do
+ [ "$try" = "$k" ] && continue 2
+ done
+
+ # Also make sure it isn't some other address in use on the
+ # node.
+ for k in $all_test_node_ips ; do
+ [ "$try" = "${k%/*}" ] && continue 2
+ done
+
+ # Get the interface details for $i, which our address is a
+ # close relative of. This should never fail but it can't hurt
+ # to be careful...
+ for k in $all_test_node_ips ; do
+ if [ "$i" = "${k%/*}" ] ; then
+ # Found one!
+ add_ip="${try}/${k#*/}"
+ break 3
+ fi
+ done
+ done
+done
+
+if [ -n "$add_ip" ] ; then
+ echo "Adding IP: ${add_ip/:/ on interface }"
+ try_command_on_node $test_node $CTDB addip ${add_ip/:/ }
+
+ echo "Waiting for IP to be added..."
+ wait_until 60 ips_are_on_nodeglob $test_node $test_node_ips ${add_ip%/*}
+
+ echo "That worked!"
+else
+ echo "BAD: Unable to find IP address to add."
+ testfailures=1
+fi
+
+echo "Restarting cluster to restore configuration..."
+restart_ctdb
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that a node's public IP address can be deleted using 'ctdb deleteip'.
+
+This test does not do any network level checks that the IP address is
+no longer reachable but simply trusts 'ctdb ip' that the address has
+been deleted.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+ served.
+3. Delete one public IP address being be served by the node, using
+ 'ctdb delip'.
+4. Verify that the delete IP address is no longer listed using the
+ 'ctdb ip' command.
+
+Expected results:
+
+* 'ctdb delip' removes an IP address from the list of public IP
+ addresses being served by a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
+
+# Select an IP/node to remove.
+num_ips=$(echo "$out" | wc -l)
+num_to_remove=$(($RANDOM % $num_ips))
+
+# Find the details in the list.
+i=0
+while [ $i -le $num_to_remove ] ; do
+ read ip_to_remove test_node
+ i=$(($i + 1))
+done <<<"$out"
+
+echo "Attempting to remove ${ip_to_remove} from node ${test_node}."
+try_command_on_node $test_node $CTDB delip $ip_to_remove
+
+echo "Sleeping..."
+sleep_for 1
+
+test_node_ips=""
+while read ip pnn ; do
+ [ "$pnn" = "$test_node" ] && \
+ test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then
+ echo "That worked! Restarting cluster to restore configuration..."
+ restart_ctdb
+else
+ echo "BAD: The remove IP address is still there!"
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify 'ctdb freeze' works correctly.
+
+This is a superficial test that simply checks that 'ctdb statistics'
+reports the node becomes frozen. No checks are done to ensure that
+client access to databases is blocked.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb freeze -n <node>' to freeze the databases on one of the
+ nodes.
+3. Run 'ctdb statistics' to verify that 'frozen' has the value '1' on
+ the node.
+
+Expected results:
+
+* When the database is frozen, the 'frozen' variable in the
+ 'ctdb statistics' output is set to 1 on the node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+echo "Freezing node $test_node"
+
+try_command_on_node 0 $CTDB freeze -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
+
+echo "That worked! Restarting cluster to restore configuration..."
+
+restart_ctdb
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify 'ctdb thaw' works correctly.
+
+This is a superficial test that simply checks that 'ctdb statistics'
+reports the node becomes unfrozen. No checks are done to ensure that
+client access to databases is unblocked.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb freeze -n <node>' to freeze the databases on one of the
+ nodes.
+3. Run 'ctdb statistics' to verify that 'frozen' has the value '1' on
+ the node.
+4, Now run 'ctdb thaw -n <node>' on the same node.
+5. Run 'ctdb statistics' to verify that 'frozen' once again has the
+ value '0' on the node.
+
+
+Expected results:
+
+* 'ctdb thaw' causes a node to 'thaw' and the status change can be
+ seem via 'ctdb statistics'.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+echo "Freezing node $test_node"
+
+try_command_on_node 0 $CTDB freeze -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
+
+echo "That worked! Now thawing node $test_node"
+
+try_command_on_node 0 $CTDB thaw -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unfrozen
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify 'ctdb getmonmode' works correctly.
+
+This test doesn't actually verify that enabling and disabling
+monitoring mode actually does that. It trusts ctdb that the
+monitoring mode is modified as requested. 21_ctdb_disablemonitor.sh
+does some more useful checking.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb getmodmode -n <node>' to get the current monitoring mode.
+3. Verify that it shows monitoring as 'active'.
+4. Verify that the command prints the output in colon-separated format
+ when run with the '-Y' option.
+5. Disable monitoring on the node using 'ctdb disablemonitor'.
+6. Verify that it shows monitoring as 'disabled'.
+
+Expected results:
+
+* 'ctdb getmonmode' works as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+try_command_on_node -v 0 $CTDB getmonmode -n $test_node
+
+sanity_check_output \
+ 1 \
+ '^Monitoring mode:ACTIVE \(0\)$' \
+ "$out"
+
+colons=$(printf ':mode:\n:0:')
+
+try_command_on_node -v 0 $CTDB -Y getmonmode -n $test_node
+
+if [ "$out" = "$colons" ] ; then
+ echo "Looks OK"
+else
+ echo "BAD: -Y output isn't what was expected"
+ testfailures=1
+fi
+
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
+
+echo "That worked! Restarting cluster to restore configuration..."
+
+restart_ctdb
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb disablemonitor' works correctly.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+* 00_ctdb_install_eventscript.sh successfully installed its event
+ script.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Create a file called /tmp/ctdb-test-unhealthy-trigger.<node> on a
+ node and verify that the status of the node changes to unhealthy
+ within the interval indicated by the MonitorInterval variable.
+3. Check that the file /tmp/ctdb-test-unhealthy-detected.<node> is
+ created, indicating that the event script is the reason the node
+ has been marked as unhealthy.
+4. Now disable monitoring on the node using 'ctdb disablemonitor -n <node>.
+5. Verify that the message 'Monitoring mode:DISABLED' is printed.
+6. Remove /tmp/ctdb-test-unhealthy-detected.<node> and ensure that it
+ is not recreated within the interval indicated by the
+ MonitorInterval variable.
+7. Remove /tmp/ctdb-test-unhealthy-trigger.<node>.
+8. Verify that the status of the node continues to be 'UNHEALTHY',
+ since monitoring has been disabled.
+
+Expected results:
+
+* When monitoring is disabled, event scripts are not executed and the
+ state of nodes is not monitored.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+# We need this for later, so we know how long to sleep.
+try_command_on_node -v 0 $CTDB getvar MonitorInterval -n $test_node
+monitor_interval="${out#*= }"
+echo "Monitor interval on node $test_node is $monitor_interval seconds."
+
+trigger="/tmp/ctdb-test-unhealthy-trigger.${test_node}"
+detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
+
+recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
+try_command_on_node $test_node touch "$recovered_flag"
+
+ctdb_test_exit_hook="onnode $test_node rm -vf $trigger"
+
+echo "Creating trigger file on node $test_node to make it unhealthy..."
+try_command_on_node $test_node touch "$trigger"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $monitor_interval
+
+try_command_on_node -v $test_node ls -l "$detected"
+
+# Wait until recovery is complete before disabling monitoring,
+# otherwise completion of the recover can turn monitoring back on!
+echo "Waiting until recovery is complete..."
+wait_until 30 onnode $test_node ! test -e "$recovered_flag"
+
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
+
+sanity_check_output \
+ 1 \
+ '^Monitoring mode:DISABLED$' \
+ "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
+
+try_command_on_node -v $test_node rm -v "$detected"
+
+sleep_for $monitor_interval
+
+try_command_on_node $test_node test ! -e "$detected"
+
+echo "OK: flag file was not recreated so monitoring must be disabled."
+
+echo "Removing trigger file. Monitoring is disabled so node will stay unhealthy."
+
+try_command_on_node -v $test_node rm -v "$trigger"
+
+sleep_for $monitor_interval
+
+onnode 0 $CTDB_TEST_WRAPPER node_has_status $test_node unhealthy
+
+echo "OK, that all worked. Expect a restart..."
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb enablemonitor' works correctly.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+* 00_ctdb_install_eventscript.sh successfully installed its event
+ script.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable monitoring on a node using 'ctdb disablemonitor -n <node>.
+3. Create a file called /tmp/ctdb-test-unhealthy-trigger.<node> on the
+ node.
+4. Verify that the status of the node does not change to unhealthy
+ within the interval indicated by the MonitorInterval variable,
+ since monitoring is disabled.
+5. Now enable monitoring on the node using 'ctdb enablemonitor -n <node>.
+6. Verify that the status of the node changes to unhealthy within the
+ interval indicated by the MonitorInterval variable.
+7. Check that the file /tmp/ctdb-test-unhealthy-detected.<node> is
+ created, indicating that the event script is the reason the node
+ has been marked as unhealthy.
+
+Expected results:
+
+* When monitoring is enabled on a node, event scripts are executed and
+ status changes are monitored.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+# We need this for later, so we know how long to sleep.
+try_command_on_node -v 0 $CTDB getvar MonitorInterval -n $test_node
+monitor_interval="${out#*= }"
+echo "Monitor interval on node $test_node is $monitor_interval seconds."
+
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
+
+sanity_check_output \
+ 1 \
+ '^Monitoring mode:DISABLED$' \
+ "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
+
+trigger="/tmp/ctdb-test-unhealthy-trigger.${test_node}"
+detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
+
+recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
+try_command_on_node $test_node touch "$recovered_flag"
+
+ctdb_test_exit_hook="onnode $test_node rm -vf $trigger; restart_ctdb"
+
+echo "Creating trigger file on node $test_node to see if it goes unhealthy..."
+try_command_on_node $test_node touch "$trigger"
+
+sleep_for $monitor_interval
+
+try_command_on_node 0 $CTDB_TEST_WRAPPER node_has_status $test_node healthy
+
+try_command_on_node $test_node test ! -e "$detected"
+
+echo "OK: flag file was not created so monitoring must be disabled."
+
+try_command_on_node -v 0 $CTDB enablemonitor -n $test_node
+
+sanity_check_output \
+ 1 \
+ '^Monitoring mode:ACTIVE$' \
+ "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monon
+
+sleep_for $monitor_interval
+
+try_command_on_node $test_node test -e "$detected"
+
+echo "OK: flag file was created so monitoring must be enabled."
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $monitor_interval
+
+try_command_on_node -v $test_node ls -l "$detected"
+
+echo "OK, that all worked. Expect a restart..."
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb moveip' allows movement of public IPs between cluster nodes.
+
+To work, this test unsets DeterministicIPs and sets NoIPFailback.
+
+This test does not do any network level checks that the IP address is
+no longer reachable but simply trusts 'ctdb ip' that the address has
+been deleted.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+ served.
+3. Use 'ctdb moveip' to move an address from one node to another.
+4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node.
+
+Expected results:
+
+* 'ctdb moveip' allows an IP address to be moved between cluster nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# Restart when done since things are likely to be broken.
+ctdb_test_exit_hook="restart_ctdb"
+
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+echo "There are $num_nodes nodes..."
+
+if [ $num_nodes -lt 2 ] ; then
+ echo "Less than 2 nodes!"
+ exit 1
+fi
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
+
+sanity_check_ips "$out"
+
+# Select an IP/node to move.
+num_ips=$(echo "$out" | wc -l)
+num_to_move=$(($RANDOM % $num_ips))
+
+# Find the details in the list.
+i=0
+while [ $i -le $num_to_move ] ; do
+ read ip_to_move test_node
+ i=$(($i + 1))
+done <<<"$out"
+
+# Can only move address to a node that is willing to host $ip_to_move.
+# This inefficient but shouldn't take long or get stuck.
+to_node=$test_node
+while [ $test_node -eq $to_node ] ; do
+ n=$(($RANDOM % $num_ips))
+ i=0
+ while [ $i -le $n ] ; do
+ read x to_node
+ i=$(($i + 1))
+ done <<<"$out"
+done
+
+echo "Turning off DeterministicIPs..."
+try_command_on_node 0 $CTDB setvar DeterministicIPs 0 -n all
+
+echo "Turning on NoIPFailback..."
+try_command_on_node 0 $CTDB setvar NoIPFailback 1 -n all
+
+echo "Attempting to move ${ip_to_move} from node ${test_node} to node ${to_node}."
+try_command_on_node $test_node $CTDB moveip $ip_to_move $to_node
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ip_to_move ; then
+ echo "IP moved from ${test_node}."
+else
+ echo "BAD: IP didn't move from ${test_node}."
+ exit 1
+fi
+
+if wait_until_ips_are_on_nodeglob "$to_node" $ip_to_move ; then
+ echo "IP moved to ${to_node}."
+else
+ echo "BAD: IP didn't move to ${to_node}."
+ exit 1
+fi
+
+echo "OK, that worked... expect a restart..."
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb getdbmap' operates as expected.
+
+This test creates some test databases using 'ctdb attach'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get the database on using 'ctdb getdbmap'.
+3. Verify that the output is valid.
+
+Expected results:
+
+* 'ctdb getdbmap' shows a valid listing of databases.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# Restart when done since things are likely to be broken.
+ctdb_test_exit_hook="restart_ctdb"
+
+make_temp_db_filename ()
+{
+ dd if=/dev/urandom count=1 bs=512 2>/dev/null |
+ md5sum |
+ awk '{printf "%s.tdb\n", $1}'
+}
+
+try_command_on_node -v 0 "$CTDB getdbmap"
+
+db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[:space:]]+ path:[^[:space:]]+)$'
+
+sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+
+num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
+
+for i in $(seq 1 5) ; do
+ f=$(make_temp_db_filename)
+ echo "Creating test database: $f"
+ try_command_on_node 0 $CTDB attach "$f"
+ try_command_on_node 0 $CTDB getdbmap
+ sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+ num=$(echo "$out" | sed -n -e '1s/^.*://p')
+ if [ $num = $(($num_db_init + $i)) ] ; then
+ echo "OK: correct number of additional databases"
+ else
+ echo "BAD: no additional database"
+ exit 1
+ fi
+ if [ "${out/name:${f} /}" != "$out" ] ; then
+ echo "OK: getdbmap knows about \"$f\""
+ else
+ echo "BAD: getdbmap does not know about \"$f\""
+ exit 1
+ fi
+done
+
+echo "OK, that worked... expect a restart..."
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that 'ctdb dumpmemory' shows expected output.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb dumpmemory' and verify that it shows expected output
+3. Verify that the command takes the '-n all' option and that it
+ causes output for all nodes to be displayed.
+
+Expected results:
+
+* 'ctdb dumpmemory' sows valid output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB dumpmemory"
+
+pat='^([[:space:]].+[[:space:]]+contains[[:space:]]+[[:digit:]]+ bytes in[[:space:]]+[[:digit:]]+ blocks \(ref [[:digit:]]+\)[[:space:]]+0x[[:xdigit:]]+|[[:space:]]+reference to: .+|full talloc report on .+ \(total[[:space:]]+[[:digit:]]+ bytes in [[:digit:]]+ blocks\))$'
+
+sanity_check_output 10 "$pat" "$out"
+
+echo "Checking output using '-n all'..."
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+try_command_on_node 0 "$CTDB dumpmemory" -n all
+sanity_check_output 10 "$pat" "$out"
+
+if [ $(fgrep -c 'full talloc report on' <<<"$out") -eq $num_nodes ] ; then
+ echo "OK: there looks to be output for all $num_nodes nodes"
+else
+ echo "BAD: there not look to be output for all $num_nodes nodes"
+ exit 1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify an error occurs if a ctdb command is run against a node without a ctdbd.
+
+That is, check that an error message is printed if an attempt is made
+to execute a ctdb command against a node that is not running ctdbd.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Shutdown ctdb on a node using 'ctdb shutdown -n <node>'.
+3. Verify that the status of the node changes to 'DISCONNECTED'.
+4. Now run 'ctdb ip -n <node>' from another node.
+5. Verify that an error message is printed stating that the node is
+ disconnected.
+6. Execute some other commands against the shutdown node. For example,
+ disable, enable, ban, unban, listvars.
+7. For each command, verify that an error message is printed stating
+ that the node is disconnected.
+
+Expected results:
+
+* For a node on which ctdb is not running, all commands display an
+ error message stating that the node is disconnected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+echo "There are $num_nodes nodes."
+
+echo "Shutting down node ${test_node}..."
+try_command_on_node $test_node $CTDB shutdown
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disconnected
+
+pat="ctdb_control error: 'ctdb_control to disconnected node'|Node $test_node is DISCONNECTED"
+
+for i in ip disable enable "ban 0" unban listvars ; do
+ try_command_on_node -v 0 ! $CTDB $i -n $test_node
+
+ if egrep -q "$pat" <<<"$out" ; then
+ echo "OK: \"ctdb ${i}\" fails with \"disconnected node\""
+ else
+ echo "BAD: \"ctdb ${i}\" does not fail with \"disconnected node\""
+ exit 1
+ fi
+done
+
+echo "That all looks OK. Restarting cluster..."
+
+restart_ctdb
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify the operation of 'ctdb disable'.
+
+This is a superficial test of the 'ctdb disable' command. It trusts
+information from CTDB that indicates that the IP failover has happened
+correctly. Another test should check that the failover has actually
+happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable one of the nodes using 'ctdb disable -n <node>'.
+3. Verify that the status of the node changes to 'disabled'.
+4. Verify that the IP addreses served by the disabled node are failed
+ over to other nodes.
+
+Expected results:
+
+* The status of the disabled node changes as expected and IP addresses
+ failover as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we just want a node that has public IPs.
+# This will work and is economically semi-randomly. :-)
+read x test_node <<<"$out"
+
+ips=""
+while read ip pnn ; do
+ if [ "$pnn" = "$test_node" ] ; then
+ ips="${ips}${ips:+ }${ip}"
+ fi
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+echo "Disabling node $test_node"
+
+try_command_on_node 1 $CTDB disable -n $test_node
+
+# Avoid a potential race condition...
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+ echo "All IPs moved."
+else
+ echo "Some IPs didn't move."
+ testfailures=1
+fi
+
+echo "Expect a restart..."
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify the operation of 'ctdb enable'.
+
+This is a superficial test of the 'ctdb enable' command. It trusts
+information from CTDB that indicates that the IP failover has happened
+correctly. Another test should check that the failover has actually
+happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable one of the nodes using 'ctdb disable -n <node>'.
+3. Verify that the status of the node changes to 'disabled'.
+4. Verify that the public IP addreses served by the disabled node are
+ failed over to other nodes.
+5. Enable the disabled node using 'ctdb enable -n '<node>'.
+6. Verify that the status changes back to 'OK'.
+7. Verify that the public IP addreses served by the disabled node are
+ failed back to the node.
+
+
+Expected results:
+
+* The status of a re-enabled node changes as expected and IP addresses
+ fail back as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+########################################
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we just want a node that has public IPs.
+# This will work and is economically semi-randomly. :-)
+read x test_node <<<"$out"
+
+ips=""
+while read ip pnn ; do
+ if [ "$pnn" = "$test_node" ] ; then
+ ips="${ips}${ips:+ }${ip}"
+ fi
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+echo "Disabling node $test_node"
+try_command_on_node 1 $CTDB disable -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+ echo "All IPs moved."
+else
+ echo "Some IPs didn't move."
+ testfailures=1
+fi
+
+echo "Reenabling node $test_node"
+try_command_on_node 1 $CTDB enable -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node enabled
+
+# BUG: this is only guaranteed if DeterministicIPs is 1 and
+# NoIPFailback is 0.
+if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
+ echo "All IPs moved."
+else
+ echo "Some IPs didn't move."
+ testfailures=1
+fi
+
+echo "All done!"
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify the operation of the 'ctdb ban' command.
+
+This is a superficial test of the 'ctdb ban' command. It trusts
+information from CTDB that indicates that the IP failover has
+happened correctly. Another test should check that the failover
+has actually happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Ban one of the nodes using the 'ctdb ban <timeout>' command.
+3. Before the ban timeout expires, verify that the status of the
+ node changes to 'banned'.
+4. Verify that the public IP addresses that were being served by
+ the node are failed over to one of the other nodes.
+5. When the ban expires ensure that the status of the node changes
+ back to 'OK' and that the public IP addresses move back to the
+ node.
+
+Expected results:
+
+* The status of the banned nodes changes as expected and IP addresses
+ failover as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Finding out which node is the recovery master..."
+try_command_on_node -v 0 "$CTDB recmaster"
+recmaster=$out
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we want a node that has public IPs and that
+# is not the recmaster. We pick the first one that satisfies both
+# conditions. We avoid the recmaster because banning the recmaster
+# (obviously) causes the recmaster to change... and changing the
+# recmaster causes all nodes to become unbanned!
+test_node=""
+
+ips=""
+while read ip pnn ; do
+ [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
+ [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+if [ -z "$test_node" ] ; then
+ echo "BAD: unable to select a suitable node for banning."
+ exit 1
+fi
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+ban_time=15
+
+echo "Banning node $test_node for $ban_time seconds"
+try_command_on_node 1 $CTDB ban $ban_time -n $test_node
+
+# Avoid a potential race condition...
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+ echo "All IPs moved."
+else
+ echo "Some IPs didn't move."
+ testfailures=1
+fi
+
+echo "Sleeping until ban expires..."
+sleep_for $ban_time
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unbanned
+
+# BUG: this is only guaranteed if DeterministicIPs is 1 and
+# NoIPFailback is 0.
+if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
+ echo "All IPs moved."
+else
+ echo "Some IPs didn't move."
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify the operation of the 'ctdb unban' command.
+
+This is a superficial test of the 'ctdb uban' command. It trusts
+information from CTDB that indicates that the IP failover and failback
+has happened correctly. Another test should check that the failover
+and failback has actually happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Ban one of the nodes using the 'ctdb ban <timeout>' command.
+3. Before the ban timeout expires, verify that the status of the
+ node changes to 'banned'.
+4. Verify that the public IP addresses that were being served by
+ the node are failed over to one of the other nodes.
+5. Before the ban timeout expires, use 'ctdb unban' to unban the
+ node.
+6. Verify that the status of the node changes back to 'OK' and that
+ the public IP addresses move back to the node.
+
+Expected results:
+
+* The 'ctdb unban' command successfully unbans a banned node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Finding out which node is the recovery master..."
+try_command_on_node -v 0 "$CTDB recmaster"
+recmaster=$out
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# See 41_ctdb_ban.sh for an explanation of why test_node is chosen
+# like this.
+test_node=""
+
+ips=""
+while read ip pnn ; do
+ [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
+ [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+if [ -z "$test_node" ] ; then
+ echo "BAD: unable to select a suitable node for banning."
+ exit 1
+fi
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+ban_time=60
+
+echo "Banning node $test_node for $ban_time seconds"
+try_command_on_node 1 $CTDB ban $ban_time -n $test_node
+
+# Avoid a potential race condition...
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+ echo "All IPs moved."
+else
+ echo "Some IPs didn't move."
+ testfailures=1
+fi
+
+echo "Unbanning node $test_node"
+try_command_on_node 1 $CTDB unban -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unbanned
+
+# BUG: this is only guaranteed if DeterministicIPs is 1 and
+# NoIPFailback is 0.
+if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
+ echo "All IPs moved."
+else
+ echo "Some IPs didn't move."
+ testfailures=1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Run the ctdb_bench test and sanity check the output.
+
+This doesn't test for performance regressions or similarly anything
+useful. Only vague sanity checking of results is done.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run ctdb_bench on all nodes with default options.
+3. Ensure that the number of +ve and -ive messages are within 1% of
+ each other.
+4. Ensure that the number of messages per second is greater than 10.
+
+Expected results:
+
+* ctdb_bench runs without error and prints reasonable results.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+echo "Running ctdb_bench on all $num_nodes nodes."
+try_command_on_node -v -pq all $CTDB_TEST_WRAPPER $VALGRIND ctdb_bench -n $num_nodes
+
+# Get the last line of output.
+while read line ; do
+ prev=$line
+done <<<"$out"
+
+pat='^(Ring: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\)[[:space:]]?|Waiting for cluster[[:space:]]?)+$'
+sanity_check_output 1 "$pat" "$out"
+
+# $prev should look like this:
+# Ring: 10670.93 msgs/sec (+ve=53391 -ve=53373)
+stuff="${prev##*Ring: }"
+mps="${stuff% msgs/sec*}"
+
+if [ ${mps%.*} -ge 10 ] ; then
+ echo "OK: $mps msgs/sec >= 10 msgs/sec"
+else
+ echo "BAD: $mps msgs/sec < 10 msgs/sec"
+ exit 1
+fi
+
+stuff="${stuff#*msgs/sec (+ve=}"
+positive="${stuff%% *}"
+
+if [ $positive -gt 0 ] ; then
+ echo "OK: +ive ($positive) > 0"
+else
+ echo "BAD: +ive ($positive) = 0"
+ exit 1
+fi
+
+stuff="${stuff#*-ve=}"
+negative="${stuff%)}"
+
+if [ $negative -gt 0 ] ; then
+ echo "OK: -ive ($negative) > 0"
+else
+ echo "BAD: -ive ($negative) = 0"
+ exit 1
+fi
+
+perc_diff=$(( ($positive - $negative) * 100 / $positive ))
+perc_diff=${perc_diff#-}
+
+if [ $perc_diff -le 1 ] ; then
+ echo "OK: percentage difference between +ive and -ive ($perc_diff%) <= 1%"
+else
+ echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > 1%"
+ exit 1
+fi
+
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Run the ctdb_fetch test and sanity check the output.
+
+This doesn't test for performance regressions or similarly anything
+useful. Only vague sanity checking of results is done.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run ctdb_fetch on all nodes with default options.
+3. Ensure that the number of +ve and -ive messages are within 1% of
+ each other.
+4. Ensure that the number of messages per second is greater than 10.
+
+Expected results:
+
+* ctdb_fetch runs without error and prints reasonable results.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+echo "Running ctdb_fetch on all $num_nodes nodes."
+try_command_on_node -v -pq all $CTDB_TEST_WRAPPER $VALGRIND ctdb_fetch -n $num_nodes
+
+# Get the last line of output.
+while read line ; do
+ prev=$line
+done <<<"$out"
+
+pat='^(Ring: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\)[[:space:]]?|Waiting for cluster[[:space:]]?)+$'
+sanity_check_output 1 "$pat" "$out"
+
+# $prev should look like this:
+# Ring: 10670.93 msgs/sec (+ve=53391 -ve=53373)
+stuff="${prev##*Ring: }"
+mps="${stuff% msgs/sec*}"
+
+if [ ${mps%.*} -ge 10 ] ; then
+ echo "OK: $mps msgs/sec >= 10 msgs/sec"
+else
+ echo "BAD: $mps msgs/sec < 10 msgs/sec"
+ exit 1
+fi
+
+stuff="${stuff#*msgs/sec (+ve=}"
+positive="${stuff%% *}"
+
+if [ $positive -gt 0 ] ; then
+ echo "OK: +ive ($positive) > 0"
+else
+ echo "BAD: +ive ($positive) = 0"
+ exit 1
+fi
+
+stuff="${stuff#*-ve=}"
+negative="${stuff%)}"
+
+if [ $negative -gt 0 ] ; then
+ echo "OK: -ive ($negative) > 0"
+else
+ echo "BAD: -ive ($negative) = 0"
+ exit 1
+fi
+
+perc_diff=$(( ($positive - $negative) * 100 / $positive ))
+perc_diff=${perc_diff#-}
+
+if [ $perc_diff -le 1 ] ; then
+ echo "OK: percentage difference between +ive and -ive ($perc_diff%) <= 1%"
+else
+ echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > 1%"
+ exit 1
+fi
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that the ctdb_transaction test succeeds.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_transaction on each node with a 30 second
+ timeout.
+3. Ensure that all ctdb_transaction processes complete successfully.
+
+Expected results:
+
+* ctdb_transaction runs without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=30"
+
+echo "Running ctdb_transaction on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that the ctdb_persistent test succeeds for safe persistent writes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_persistent on each node with a 30 second
+ timeout.
+3. Ensure that all ctdb_persistent processes complete successfully.
+
+Expected results:
+
+* ctdb_persistent tests safe persistent writes without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --timelimit=30"
+
+echo "Running ctdb_persistent on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Verify that the ctdb_persistent test succeeds for unsafe persistent writes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_persistent on each node with a 30 second
+ timeout and with the --unsafe-writes option.
+3. Ensure that all ctdb_persistent processes complete successfully.
+
+Expected results:
+
+* ctdb_persistent tests unsafe persistent writes without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --unsafe-writes --timelimit=30"
+
+echo "Running ctdb_persistent --unsafe-writes on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
--- /dev/null
+#!/bin/bash
+
+test_info()
+{
+ cat <<EOF
+Uninstall the event script used for testing..
+
+Prerequisites:
+
+* Nodes must be accessible via 'onnode'.
+
+Steps:
+
+1.
+
+Expected results:
+
+* The script is successfully uninstalled from all nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+uninstall_eventscript "00.ctdb_test_trigger"
#include "system/filesys.h"
#include "popt.h"
#include "cmdline.h"
+#include "ctdb.h"
+#include "ctdb_private.h"
#include <sys/time.h>
#include <time.h>
int incr = *(int *)data.dptr;
int *count = (int *)private_data;
int dest;
+
(*count)++;
- dest = (ctdb_get_pnn(ctdb) + incr) % num_nodes;
+ dest = (ctdb_get_pnn(ctdb) + num_nodes + incr) % num_nodes;
ctdb_send_message(ctdb, dest, srvid, data);
if (incr == 1) {
msg_plus++;
}
}
+
+void send_start_messages(struct ctdb_context *ctdb, int incr)
+{
+ /* two messages are injected into the ring, moving
+ in opposite directions */
+ int dest;
+ TDB_DATA data;
+
+ data.dptr = (uint8_t *)&incr;
+ data.dsize = sizeof(incr);
+
+ dest = (ctdb_get_pnn(ctdb) + num_nodes + incr) % num_nodes;
+ ctdb_send_message(ctdb, dest, 0, data);
+}
+
+static void each_second(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+
+ /* we kickstart the ring into action by inserting messages from node
+ with pnn 0.
+ it may happen that some other node does not yet have ctdb_bench
+ running in which case the ring is broken and the messages are lost.
+ if so, once every second try again to restart the ring
+ */
+ if (msg_plus == 0) {
+// printf("no messages recevied, try again to kickstart the ring in forward direction...\n");
+ send_start_messages(ctdb, 1);
+ }
+ if (msg_minus == 0) {
+// printf("no messages recevied, try again to kickstart the ring in reverse direction...\n");
+ send_start_messages(ctdb, -1);
+ }
+ event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb);
+}
+
+static void dummy_event(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+ event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), dummy_event, ctdb);
+}
+
/*
benchmark sending messages in a ring around the nodes
*/
int pnn=ctdb_get_pnn(ctdb);
if (pnn == 0) {
- /* two messages are injected into the ring, moving
- in opposite directions */
- int dest, incr;
- TDB_DATA data;
-
- data.dptr = (uint8_t *)&incr;
- data.dsize = sizeof(incr);
-
- incr = 1;
- dest = (ctdb_get_pnn(ctdb) + incr) % num_nodes;
- ctdb_send_message(ctdb, dest, 0, data);
-
- incr = -1;
- dest = (ctdb_get_pnn(ctdb) + incr) % num_nodes;
- ctdb_send_message(ctdb, dest, 0, data);
+ event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb);
+ } else {
+ event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), dummy_event, ctdb);
}
-
- start_timer();
+ start_timer();
while (end_timer() < timelimit) {
if (pnn == 0 && msg_count % 10000 == 0) {
printf("Ring: %.2f msgs/sec (+ve=%d -ve=%d)\r",
msg_count/end_timer(), msg_plus, msg_minus);
}
-/*
- handler for reconfigure message
-*/
-static void reconfigure_handler(struct ctdb_context *ctdb, uint64_t srvid,
- TDB_DATA data, void *private_data)
-{
- int *ready = (int *)private_data;
- *ready = 1;
-}
-
-
/*
main program
*/
int ret;
poptContext pc;
struct event_context *ev;
- int cluster_ready=0;
pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
/* initialise ctdb */
ctdb = ctdb_cmdline_client(ev);
- ctdb_set_message_handler(ctdb, CTDB_SRVID_RECONFIGURE, reconfigure_handler,
- &cluster_ready);
-
/* attach to a specific database */
ctdb_db = ctdb_attach(ctdb, "test.tdb", false, 0);
if (!ctdb_db) {
+++ /dev/null
-#!/bin/sh
-
-NUMNODES=2
-if [ $# -gt 0 ]; then
- NUMNODES=$1
-fi
-shift
-
-NODES="./tests/nodes.txt"
-rm -f $NODES
-for i in `seq 1 $NUMNODES`; do
- if [ "${CTDB_USE_IPV6}x" != "x" ]; then
- echo ::$i >> $NODES
- ip addr add ::$i/128 dev lo
- else
- echo 127.0.0.$i >> $NODES
- fi
-done
-
-killall -q ctdbd
-rm -rf test.db/persistent/*
-
-CTDB_OPTIONS="--reclock=rec.lock --nlist $NODES --event-script-dir=tests/events.d --logfile=- -d 0 --dbdir=test.db --dbdir-persistent=test.db/persistent $*"
-
-echo "Starting $NUMNODES ctdb daemons"
-for i in `seq 1 $NUMNODES`; do
- if [ `id -u` -eq 0 ]; then
- CTDB_OPTIONS="$CTDB_OPTIONS --public-interface=lo"
- fi
-
- $VALGRIND bin/ctdbd --socket=sock.$i $CTDB_OPTIONS || exit 1
-done
-ln -sf $PWD/sock.1 /tmp/ctdb.socket || exit 1
-
-while bin/ctdb status | egrep "DISCONNECTED|UNHEALTHY" > /dev/null; do
- echo "`date` Waiting for recovery"
- sleep 1;
-done
-
-echo "$NUMNODES daemons started"
-
-exit 0
+++ /dev/null
-#!/bin/sh
-
-NUMNODES=4
-if [ $# -gt 0 ]; then
- NUMNODES=$1
-fi
-
-killall -9 -q ctdb_transaction ctdbd
-
-rm -rf test.db/transaction
-
-echo "Starting $NUMNODES daemons for transaction writes"
-tests/start_daemons.sh $NUMNODES || exit 1
-
-trap 'echo "Killing test"; killall -9 -q ctdbd ctdb_transaction; exit 1' INT TERM
-
-VALGRIND="valgrind -q"
-for i in `seq 1 $NUMNODES`; do
- $VALGRIND bin/ctdb_transaction --timelimit 30 --socket sock.$i $* &
- $VALGRIND bin/ctdb_transaction --timelimit 30 --socket sock.$i $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-killall -9 ctdbd
-
-exit 0
esac
if [ -z "$ctdb_status_output" ] ; then
+ # FIXME: need to do something if $CTDB_NODES_SOCKETS is set.
ctdb_status_output=$(ctdb -Y status 2>/dev/null)
if [ $? -ne 0 ] ; then
echo "${prog}: unable to get status of CTDB nodes" >&2
ctdb_recmaster=""
get_nodes ()
{
- [ -f "$CTDB_NODES_FILE" ] || CTDB_NODES_FILE=/etc/ctdb/nodes
- local all_nodes=$(egrep '^[[:alnum:]]' $CTDB_NODES_FILE)
+ local all_nodes
+
+ if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+ all_nodes="$CTDB_NODES_SOCKETS"
+ else
+ [ -f "$CTDB_NODES_FILE" ] || CTDB_NODES_FILE=/etc/ctdb/nodes
+ all_nodes=$(egrep '^[[:alnum:]]' $CTDB_NODES_FILE)
+ fi
local nodes=""
local n
done
}
+fakessh ()
+{
+ CTDB_SOCKET="$1" sh -c "$2"
+}
+
######################################################################
parse_options "$@"
$current && command="cd $PWD && $command"
-SSH_OPTS=
-# Could "2>/dev/null || true" but want to see errors from typos in file.
-[ -r /etc/ctdb/onnode.conf ] && . /etc/ctdb/onnode.conf
-[ -n "$SSH" ] || SSH=ssh
-if [ "$SSH" = "ssh" ] ; then
- ssh_opts="-n"
-else
- : # rsh? All bets are off!
+ssh_opts=
+if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+ SSH=fakessh
+else
+ # Could "2>/dev/null || true" but want to see errors from typos in file.
+ [ -r /etc/ctdb/onnode.conf ] && . /etc/ctdb/onnode.conf
+ [ -n "$SSH" ] || SSH=ssh
+ if [ "$SSH" = "ssh" ] ; then
+ ssh_opts="-n"
+ else
+ : # rsh? All bets are off!
+ fi
fi
######################################################################
git.samba.org / martins / samba.git / commitdiff