######################################################################
-#. /root/SOFS/autosofs/scripts/tester.bash
-
ctdb_test_begin ()
{
local name="$1"
trap - 0
[ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
+ status=$(($testfailures+0))
+
+ # Avoid making a test fail from this point onwards. The test is
+ # now complete.
+ set +e
+
+ echo "*** TEST COMPLETE (RC=$status), CLEANING UP..."
+
+ eval "$ctdb_test_exit_hook" || true
+ unset ctdb_test_exit_hook
- eval "$ctdb_test_exit_hook"
+ if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then
- if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then
- echo "Restarting ctdb on all nodes to get back into known state..."
restart_ctdb
+ else
+ # This could be made unconditional but then we might get
+ # duplication from the recovery in restart_ctdb. We want to
+ # leave the recovery in restart_ctdb so that future tests that
+ # might do a manual restart mid-test will benefit.
+ echo "Forcing a recovery..."
+ onnode 0 ctdb recover
fi
- test_exit
+ exit $status
+}
+
+ctdb_test_exit_hook_add ()
+{
+ ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*"
}
ctdb_test_run ()
{
scriptname=$(basename "$0")
testfailures=0
+ ctdb_test_restart_scheduled=false
ctdb_test_cmd_options $@
trap "ctdb_test_exit" 0
}
+ctdb_test_check_real_cluster ()
+{
+ [ -n "$CTDB_TEST_REAL_CLUSTER" ] && return 0
+
+ echo "ERROR: This test must be run on a real/virtual cluster, not local daemons."
+ return 1
+}
+
########################################
# Sets: $out
local nodespec="$1" ; shift
local verbose=false
- if [ "$nodespec" = "-v" ] ; then
- verbose=true
+ local onnode_opts=""
+
+ while [ "${nodespec#-}" != "$nodespec" ] ; do
+ if [ "$nodespec" = "-v" ] ; then
+ verbose=true
+ else
+ onnode_opts="$nodespec"
+ fi
nodespec="$1" ; shift
- fi
+ done
+
local cmd="$*"
- out=$(onnode -q "$nodespec" "$cmd" 2>&1) || {
+ out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
echo "$out"
return $ret
}
+sanity_check_ips ()
+{
+ local ips="$1" # Output of "ctdb ip -n all"
+
+ echo "Sanity checking IPs..."
+
+ local x ipp prev
+ prev=""
+ while read x ipp ; do
+ [ "$ipp" = "-1" ] && break
+ if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+ echo "OK"
+ return 0
+ fi
+ prev="$ipp"
+ done <<<"$ips"
+
+ echo "BAD: a node was -1 or IPs are only assigned to one node"
+ echo "Are you running an old version of CTDB?"
+ return 1
+}
+
+select_test_node_and_ips ()
+{
+ try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+ # When selecting test_node we just want a node that has public
+ # IPs. This will work and is economically semi-random. :-)
+ local x
+ read x test_node <<<"$out"
+
+ test_node_ips=""
+ local ip pnn
+ while read ip pnn ; do
+ if [ "$pnn" = "$test_node" ] ; then
+ test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+ fi
+ done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+ echo "Selected node ${test_node} with IPs: ${test_node_ips}."
+ test_ip="${test_node_ips%% *}"
+}
+
#######################################
# Wait until either timeout expires or command succeeds. The command
local timeout="$1" ; shift # "$@" is the command...
echo -n "<${timeout}|"
- while [ $timeout -gt 0 ] ; do
+ local t=$timeout
+ while [ $t -gt 0 ] ; do
if "$@" ; then
- echo '|'
+ echo "|$(($timeout - $t))|"
echo "OK"
return 0
fi
echo -n .
- timeout=$(($timeout - 1))
+ t=$(($t - 1))
sleep 1
done
cluster_is_healthy ()
{
- if _cluster_is_healthy ; then
+ if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
echo "Cluster is HEALTHY"
- exit 0
+ return 0
else
echo "Cluster is UNHEALTHY"
- exit 1
+ if ! ${ctdb_test_restart_scheduled:-false} ; then
+ echo "DEBUG:"
+ local i
+ for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do
+ echo "$i"
+ $i || true
+ done
+ fi
+ return 1
fi
}
wait_until 60 ips_are_on_nodeglob "$@"
}
+get_src_socket ()
+{
+ local proto="$1"
+ local dst_socket="$2"
+ local pid="$3"
+ local prog="$4"
+
+ local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$"
+ out=$(netstat -tanp |
+ egrep "$pat" |
+ awk '{ print $4 }')
+
+ [ -n "$out" ]
+}
+
+wait_until_get_src_socket ()
+{
+ local proto="$1"
+ local dst_socket="$2"
+ local pid="$3"
+ local prog="$4"
+
+ echo "Waiting for ${prog} to establish connection to ${dst_socket}..."
+
+ wait_until 5 get_src_socket "$@"
+}
+
+#######################################
+
+# filename will be in $tcpdump_filename, pid in $tcpdump_pid
+tcpdump_start ()
+{
+ tcpdump_filter="$1" # global
+
+ echo "Running tcpdump..."
+ tcpdump_filename=$(mktemp)
+ ctdb_test_exit_hook_add "rm -f $tcpdump_filename"
+
+ # The only way of being sure that tcpdump is listening is to send
+ # some packets that it will see. So we use dummy pings - the -U
+ # option to tcpdump ensures that packets are flushed to the file
+ # as they are captured.
+ local dummy_addr="127.3.2.1"
+ local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo"
+ tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" &
+ ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1"
+
+ echo "Waiting for tcpdump output file to be ready..."
+ ping -q "$dummy_addr" >/dev/null 2>&1 &
+ ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1"
+
+ tcpdump_listen_for_dummy ()
+ {
+ tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1
+ }
+
+ wait_until 10 tcpdump_listen_for_dummy
+}
+
+# By default, wait for 1 matching packet.
+tcpdump_wait ()
+{
+ local count="${1:-1}"
+ local filter="${2:-${tcpdump_filter}}"
+
+ tcpdump_check ()
+ {
+ local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l)
+ [ $found -ge $count ]
+ }
+
+ echo "Waiting for tcpdump to capture some packets..."
+ if ! wait_until 30 tcpdump_check ; then
+ echo "DEBUG:"
+ local i
+ for i in "ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do
+ echo "$i"
+ $i || true
+ done
+ return 1
+ fi
+}
+
+tcpdump_show ()
+{
+ local filter="${1:-${tcpdump_filter}}"
+
+ tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null
+}
+
+tcptickle_sniff_start ()
+{
+ local src="$1"
+ local dst="$2"
+
+ local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}"
+ local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}"
+ local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234
+ local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)"
+ tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0"
+ local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})"
+
+ tcpdump_start "$filter"
+}
+
+tcptickle_sniff_wait_show ()
+{
+ tcpdump_wait 1 "$tcptickle_reset"
+
+ echo "GOOD: here are some TCP tickle packets:"
+ tcpdump_show
+}
+
+gratarp_sniff_start ()
+{
+ tcpdump_start "arp host ${test_ip}"
+}
+
+gratarp_sniff_wait_show ()
+{
+ tcpdump_wait 2
+
+ echo "GOOD: this should be the some gratuitous ARPs:"
+ tcpdump_show
+}
+
+
#######################################
daemons_stop ()
echo "Sleeping for a while..."
sleep_for 1
- if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then
+ if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then
echo "Killing remaining daemons..."
- killall $CTDB_DIR/bin/ctdbd
+ pkill -f $CTDB_DIR/bin/ctdbd
- if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then
+ if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then
echo "Once more with feeling.."
- killall -9 $CTDB_DIR/bin/ctdbd
+ pkill -9 $CTDB_DIR/bin/ctdbd
fi
fi
- var_dir=$CTDB_DIR/tests/var
+ local var_dir=$CTDB_DIR/tests/var
rm -rf $var_dir/test.db
}
local no_public_ips=-1
[ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses
- local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent"
+ local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent"
echo "Starting $num_nodes ctdb daemons..."
if [ "$no_public_ips" != -1 ] ; then
ctdb_options="$ctdb_options --public-addresses=$public_addresses"
fi
- $VALGRIND bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" || return 1
+ # Need full path so we can use "pkill -f" to kill the daemons.
+ $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" ||return 1
done
if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then
restart_ctdb ()
{
- if [ -n "$CTDB_NODES_SOCKETS" ] ; then
- daemons_stop
- daemons_start $CTDB_TEST_NUM_DAEMONS
- else
- onnode -pq all $CTDB_TEST_WRAPPER _restart_ctdb
- fi || return 1
+ echo -n "Restarting CTDB"
+ if $ctdb_test_restart_scheduled ; then
+ echo -n " (scheduled)"
+ fi
+ echo "..."
+
+ local i
+ for i in $(seq 1 5) ; do
+ if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+ daemons_stop
+ daemons_start $CTDB_TEST_NUM_DAEMONS
+ else
+ onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb
+ fi && break
+
+ echo "That didn't seem to work - sleeping a while and trying again..."
+ sleep_for 5
+ done
onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1
# cluster to stabilise before a subsequent test.
echo "Forcing a recovery..."
onnode -q 0 ctdb recover
-
- #echo "Sleeping to allow ctdb to settle..."
- #sleep_for 10
+ sleep_for 1
+ echo "Forcing a recovery..."
+ onnode -q 0 ctdb recover
echo "ctdb is ready"
}
+ctdb_restart_when_done ()
+{
+ ctdb_test_restart_scheduled=true
+}
+
#######################################
install_eventscript ()
git.samba.org / metze / ctdb / wip.git / blobdiff