Steps:
1. Verify that the cluster is healthy.
-2. Determine the CTDB MonitorInterval setting and remember it.
-3. Create a temporary directory using mktemp, remember the name in
- $mydir.
-4. Create an executable file /etc/ctdb/rc.local.d/fake-exportfs that
- contains a definiton for the function exportfs, which prints a
- share definition for a directory $mydir/foo (which does not
- currently exist).
-5. Create an executable file
+2. Determine a timeout for state changes by adding MonitorInterval
+ and EventScriptTimeout.
+3. Create a temporary directory on the test node using mktemp,
+ remember the name in $mydir.
+4. On the test node create an executable file
+ /etc/ctdb/rc.local.d/fake-exportfs that contains a definiton for
+ the function exportfs, which prints a share definition for a
+ directory $mydir/foo (which does not currently exist).
+5. On the test node create an executable file
/etc/ctdb/rc.local.d/nfs-skip-share-check that replaces the
loadconfig() function by one with equivalent functionality, but
which also sets CTDB_NFS_SKIP_SHARE_CHECK="no" if loading
"ctdb" configuration.
-6. Wait for a maximum of MonitorInterval seconds for the node to
- become unhealthy.
+6. Wait for the test node to become unhealthy.
7. Create the directory $mydir/foo.
-8. Wait for a maximum of MonitorInterval seconds for the node to
- become healthy.
+8. Wait for the test node to become healthy.
9. Modify /etc/ctdb/rc.local.d/nfs-skip-share-check so that it sets
CTDB_NFS_SKIP_SHARE_CHECK to "yes".
10. Remove the directory $mydir/foo.
-11. Wait for MonitorInterval and confirm that the the node is still
+11. Wait for a monitor event and confirm that the the node is still
healthy.
Expected results:
# We need this for later, so we know how long to sleep.
try_command_on_node $test_node $CTDB getvar MonitorInterval
-monitor_interval=$((${out#*= } + 1))
+monitor_interval=${out#*= }
+try_command_on_node $test_node $CTDB getvar EventScriptTimeout
+event_script_timeout=${out#*= }
+
+monitor_timeout=$(($monitor_interval + $event_script_timeout))
+
+echo "Using timeout of ${monitor_timeout}s (MonitorInterval + EventScriptTimeout)..."
+
mydir=$(onnode -q $test_node mktemp -d)
rc_local_d="${CTDB_BASE:-/etc/ctdb}/rc.local.d"
-mkdir -p "$rc_local_d"
my_exit_hook ()
{
+ ctdb_test_eventscript_uninstall
onnode -q $test_node "rm -f $mydir/*"
onnode -q $test_node "rmdir --ignore-fail-on-non-empty $mydir"
onnode -q $test_node "rm -f \"$rc_local_d/\"*"
ctdb_test_exit_hook_add my_exit_hook
+ctdb_test_eventscript_install
+
foo_dir=$mydir/foo
try_command_on_node -v $test_node "mkdir -p \"$rc_local_d\""
echo "Installing \"$n\" with CTDB_NSF_SKIP_SHARE_CHECK=no..."
try_command_on_node $test_node "echo '$n_contents' >\"$n\" ; chmod +x \"$n\""
-wait_until_node_has_status $test_node unhealthy $monitor_interval
+wait_until_node_has_status $test_node unhealthy $monitor_timeout
try_command_on_node -v $test_node "mkdir $foo_dir"
-wait_until_node_has_status $test_node healthy $monitor_interval
+wait_until_node_has_status $test_node healthy $monitor_timeout
echo "Re-installing \"$n\" with CTDB_NFS_SKIP_SHARE_CHECK=yes..."
try_command_on_node $test_node "echo '${n_contents/=no/=yes}' >\"$n\" ; chmod +x \"$n\""
try_command_on_node -v $test_node "rmdir $foo_dir"
-echo "Waiting for MonitorInterval to ensure that node $test_node stays healthy..."
-sleep_for $monitor_interval
+wait_for_monitor_event $test_node
wait_until_node_has_status $test_node healthy 1
Steps:
1. Verify that the cluster is healthy.
-2. Determine the CTDB MonitorInterval setting and remember it.
+2. Determine a timeout for state changes by adding MonitorInterval
+ and EventScriptTimeout.
3. Create a temporary directory using mktemp, remember the name in
$mydir.
4. Create an executable file /etc/ctdb/rc.local.d/fake-testparm that
9. Modify /etc/ctdb/rc.local.d/samba-skip-share-check so that it sets
CTDB_SAMBA_SKIP_SHARE_CHECK="yes".
10. Remove the directory $mydir/foo.
-11. Wait for MonitorInterval and confirm that the the node is still
+11. Wait for a monitor event and confirm that the the node is still
healthy.
Expected results:
select_test_node_and_ips
+# We need this for later, so we know how long to sleep.
# We need this for later, so we know how long to sleep.
try_command_on_node $test_node $CTDB getvar MonitorInterval
-monitor_interval=$((${out#*= } + 1))
+monitor_interval=${out#*= }
+try_command_on_node $test_node $CTDB getvar EventScriptTimeout
+event_script_timeout=${out#*= }
+
+monitor_timeout=$(($monitor_interval + $event_script_timeout))
+
+echo "Using timeout of ${monitor_timeout}s (MonitorInterval + EventScriptTimeout)..."
mydir=$(onnode -q $test_node mktemp -d)
rc_local_d="${CTDB_BASE:-/etc/ctdb}/rc.local.d"
-mkdir -p "$rc_local_d"
my_exit_hook ()
{
+ ctdb_test_eventscript_uninstall
onnode -q $test_node "rm -f $mydir/*"
onnode -q $test_node "rmdir --ignore-fail-on-non-empty $mydir"
onnode -q $test_node "rm -f \"$rc_local_d/\"*"
ctdb_test_exit_hook_add my_exit_hook
+ctdb_test_eventscript_install
+
foo_dir=$mydir/foo
try_command_on_node -v $test_node "mkdir -p \"$rc_local_d\""
echo "Installing \"$n\" with CTDB_SAMBA_SKIP_SHARE_CHECK=no..."
try_command_on_node $test_node "echo '$n_contents' >\"$n\" ; chmod +x \"$n\""
-wait_until_node_has_status $test_node unhealthy $monitor_interval
+wait_until_node_has_status $test_node unhealthy $monitor_timeout
try_command_on_node -v $test_node "mkdir $foo_dir"
-wait_until_node_has_status $test_node healthy $monitor_interval
+wait_until_node_has_status $test_node healthy $monitor_timeout
echo "Re-installing \"$n\" with CTDB_SAMBA_SKIP_SHARE_CHECK=yes..."
try_command_on_node $test_node "echo '${n_contents/=no/=yes}' >\"$n\" ; chmod +x \"$n\""
try_command_on_node -v $test_node "rmdir $foo_dir"
-echo "Waiting for MonitorInterval to ensure that node $test_node stays healthy..."
-sleep_for $monitor_interval
+wait_for_monitor_event $test_node
wait_until_node_has_status $test_node healthy 1
{
local timeout="$1" ; shift # "$@" is the command...
+ local negate=false
+ if [ "$1" = "!" ] ; then
+ negate=true
+ shift
+ fi
+
echo -n "<${timeout}|"
local t=$timeout
while [ $t -gt 0 ] ; do
- if "$@" ; then
+ ("$@")
+ local rc=$?
+ if { ! $negate && [ $rc -eq 0 ] ; } || \
+ { $negate && [ $rc -ne 0 ] ; } ; then
echo "|$(($timeout - $t))|"
echo "OK"
return 0
rm -vf "${CTDB_DIR}/tests/events.d/${script_name}"
fi
}
+
+#######################################
+
+# This section deals with the 99.ctdb_test eventscript.
+
+# Metafunctions: Handle a ctdb-test file on a node.
+# given event.
+ctdb_test_eventscript_file_create ()
+{
+ local pnn="$1"
+ local type="$2"
+
+ try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}"
+}
+
+ctdb_test_eventscript_file_remove ()
+{
+ local pnn="$1"
+ local type="$2"
+
+ try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}"
+}
+
+ctdb_test_eventscript_file_exists ()
+{
+ local pnn="$1"
+ local type="$2"
+
+ try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1
+}
+
+
+# Handle a flag file on a node that is removed by 99.ctdb_test on the
+# given event.
+ctdb_test_eventscript_flag ()
+{
+ local cmd="$1"
+ local pnn="$2"
+ local event="$3"
+
+ ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}"
+}
+
+
+# Handle a trigger that causes 99.ctdb_test to fail it's monitor
+# event.
+ctdb_test_eventscript_unhealthy_trigger ()
+{
+ local cmd="$1"
+ local pnn="$2"
+
+ ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger"
+}
+
+# Handle the file that 99.ctdb_test created to show that it has marked
+# a node unhealthy because it detected the above trigger.
+ctdb_test_eventscript_unhealthy_detected ()
+{
+ local cmd="$1"
+ local pnn="$2"
+
+ ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected"
+}
+
+# Note that the eventscript can't use the above functions!
+ctdb_test_eventscript_install ()
+{
+
+ local script='#!/bin/sh
+out=$(ctdb pnn)
+pnn="${out#PNN:}"
+
+rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}"
+
+trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}"
+detected="/tmp/ctdb-test-unhealthy-detected.${pnn}"
+if [ "$1" = "monitor" ] ; then
+ if [ -e "$trigger" ] ; then
+ echo "${0}: Unhealthy because \"$trigger\" detected"
+ touch "$detected"
+ exit 1
+ elif [ -e "$detected" -a ! -e "$trigger" ] ; then
+ echo "${0}: Healthy again, \"$trigger\" no longer detected"
+ rm "$detected"
+ fi
+fi
+
+exit 0
+'
+ install_eventscript "99.ctdb_test" "$script"
+}
+
+ctdb_test_eventscript_uninstall ()
+{
+ uninstall_eventscript "99.ctdb_test"
+}
+
+# Note that this only works if you know all other monitor events will
+# succeed. You also need to install the eventscript before using it.
+wait_for_monitor_event ()
+{
+ local pnn="$1"
+
+ echo "Waiting for a monitor event on node $pnn to complete..."
+ ctdb_test_eventscript_flag create $pnn "monitor"
+
+ wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor"
+
+}
+++ /dev/null
-#!/bin/bash
-
-test_info()
-{
- cat <<EOF
-Install an event script on all nodes that helps detect ctdb events.
-
-We could install separate scripts for particular events later as
-needed. However, the script installed here will allow detection of
-all events. It also allows a file to be created to indicate that a
-node should be marked as unhealthy.
-
-Prerequisites:
-
-* Nodes must be accessible via 'onnode'.
-
-Steps:
-
-1. Use the install_eventscript to install the eventscript.
-
-Expected results:
-
-* The script is successfully installed on all nodes.
-EOF
-}
-
-. ctdb_test_functions.bash
-
-script='#!/bin/sh
-out=$(ctdb pnn)
-pnn="${out#PNN:}"
-
-# Allow creation of flag files that are removed to confirm that events
-# are taking place.
-rm -f "/tmp/ctdb-test-flag.${1}.${pnn}"
-
-# Allow creation of a trigger file to make a monitor event fail and
-# force a node to be marked as unhealthy. This avoids having to look
-# at log files to confirm that monitoring is working. Note that
-# ${pnn} is needed in the filename if we are testing using local
-# daemons so we put in there regardless.
-trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}"
-detected="/tmp/ctdb-test-unhealthy-detected.${pnn}"
-if [ "$1" = "monitor" ] ; then
- if [ -e "$trigger" ] ; then
- echo "${0}: Unhealthy because \"$trigger\" detected"
- touch "$detected"
- exit 1
- elif [ -e "$detected" -a ! -e "$trigger" ] ; then
- echo "${0}: Healthy again, \"$trigger\" no longer detected"
- rm "$detected"
- fi
-fi
-
-exit 0
-'
-
-install_eventscript "00.ctdb_test_trigger" "$script"
+++ /dev/null
-#!/bin/bash
-
-test_info()
-{
- cat <<EOF
-Uninstall the event script used for testing..
-
-Prerequisites:
-
-* Nodes must be accessible via 'onnode'.
-
-Steps:
-
-1.
-
-Expected results:
-
-* The script is successfully uninstalled from all nodes.
-EOF
-}
-
-. ctdb_test_functions.bash
-
-ctdb_test_init "$@"
-
-uninstall_eventscript "00.ctdb_test_trigger"
git.samba.org / ctdb.git / commitdiff