Clean up handling the of CTDB restarts in testcases.

Glitches during restarts of the CTDB cluster have been causing some
tests to fail.  This is because restarts are initiated in the body of
many tests.  This adds a simple function ctdb_restart_when_done, which
schedules a restart using an existing hook in the test exit code.
This function is now used in tests that need to restart CTDB.

Signed-off-by: Martin Schwenke <martin@meltin.net>

diff --git a/tests/scripts/ctdb_test_functions.bash b/tests/scripts/ctdb_test_functions.bash
index 1930ae163a037e69f77d24012fedeb9ea70029ec..210e6c4853c194e5b6662b5652d6d5d81dd3ea12 100644 (file)
--- a/tests/scripts/ctdb_test_functions.bash
+++ b/tests/scripts/ctdb_test_functions.bash
@@ -591,6 +591,10 @@ setup_ctdb ()
 
 restart_ctdb ()
 {
+    if [ "$1" = "-v" ] ; then
+       echo "Restarting CTDB (scheduled)..."
+    fi
+    
     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
        daemons_stop
        daemons_start $CTDB_TEST_NUM_DAEMONS
@@ -615,6 +619,13 @@ restart_ctdb ()
     echo "ctdb is ready"
 }
 
+ctdb_restart_when_done ()
+{
+    ctdb_test_exit_hook_add restart_ctdb -v
+}
+
+
+
 #######################################
 
 install_eventscript ()

diff --git a/tests/simple/04_ctdb_setvar.sh b/tests/simple/04_ctdb_setvar.sh
index 9348308f2d4f341c798184ce029e5042fdca0efb..ceb6c2361d56c90821c62353d9f08d0bf97e24da 100755 (executable)
--- a/tests/simple/04_ctdb_setvar.sh
+++ b/tests/simple/04_ctdb_setvar.sh
@@ -38,6 +38,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 var="RecoverTimeout"
 
 try_command_on_node -v 0 $CTDB getvar $var
@@ -67,8 +70,7 @@ try_command_on_node -v 0 "$CTDB listvars | grep '^$var'"
 check="${out#*= }"
 
 if [ "$incr" != "$check" ] ; then
-    echo "Nope, that didn't work.  Restarting ctdb to get back into known state..."
-    restart_ctdb
+    echo "Nope, that didn't work..."
     exit 1
 fi
 

diff --git a/tests/simple/16_ctdb_config_add_ip.sh b/tests/simple/16_ctdb_config_add_ip.sh
index 50fc9c5cb947481b547d8b69f93aaa67da7c8430..634a39aec51ddf165b888e07670291d954aab41a 100755 (executable)
--- a/tests/simple/16_ctdb_config_add_ip.sh
+++ b/tests/simple/16_ctdb_config_add_ip.sh
@@ -42,6 +42,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 echo "Getting list of public IPs..."
 try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
 
@@ -112,7 +115,4 @@ else
     testfailures=1
 fi
 
-echo "Restarting cluster to restore configuration..."
-restart_ctdb
-
 ctdb_test_exit

diff --git a/tests/simple/17_ctdb_config_delete_ip.sh b/tests/simple/17_ctdb_config_delete_ip.sh
index 822c7023c261fa729511c17456a9de33f1f11eb0..bab45ddd50c40f6b5c491ee4e930cbff4abb2a87 100755 (executable)
--- a/tests/simple/17_ctdb_config_delete_ip.sh
+++ b/tests/simple/17_ctdb_config_delete_ip.sh
@@ -38,6 +38,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 echo "Getting list of public IPs..."
 try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
 
@@ -65,8 +68,7 @@ while read ip pnn ; do
 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 
 if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then
-    echo "That worked!  Restarting cluster to restore configuration..."
-    restart_ctdb
+    echo "GOOD: That worked!"
 else
     echo "BAD: The remove IP address is still there!"
     testfailures=1

diff --git a/tests/simple/18_ctdb_freeze.sh b/tests/simple/18_ctdb_freeze.sh
index 5f2ccb0a1376b2bb769fa0963c05e5429eae07ed..009e2a745c6eb3853873d56e591239ebded9c3ef 100755 (executable)
--- a/tests/simple/18_ctdb_freeze.sh
+++ b/tests/simple/18_ctdb_freeze.sh
@@ -36,6 +36,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 test_node=1
 
 echo "Freezing node $test_node"
@@ -44,8 +47,6 @@ try_command_on_node 0 $CTDB freeze -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
 
-echo "That worked!  Restarting cluster to restore configuration..."
-
-restart_ctdb
+echo "GOOD: That worked!"
 
 ctdb_test_exit

diff --git a/tests/simple/20_ctdb_getmonmode.sh b/tests/simple/20_ctdb_getmonmode.sh
index 802f302945e703d2fe34e0e14f2c877a56b5dfe6..6a55341297d3f188a1b7061df504cad6b71180e5 100755 (executable)
--- a/tests/simple/20_ctdb_getmonmode.sh
+++ b/tests/simple/20_ctdb_getmonmode.sh
@@ -38,6 +38,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 test_node=1
 
 try_command_on_node -v 0 $CTDB getmonmode -n $test_node
@@ -62,8 +65,6 @@ try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
 
-echo "That worked!  Restarting cluster to restore configuration..."
-
-restart_ctdb
+echo "GOOD: That worked!"
 
 ctdb_test_exit

diff --git a/tests/simple/21_ctdb_disablemonitor.sh b/tests/simple/21_ctdb_disablemonitor.sh
index 85855e4ca4dd7294f207c57c3e2d45204a29fc49..23ae052f5cbe370c81f9c865e03485830038b59f 100755 (executable)
--- a/tests/simple/21_ctdb_disablemonitor.sh
+++ b/tests/simple/21_ctdb_disablemonitor.sh
@@ -57,7 +57,7 @@ detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
 recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
 try_command_on_node $test_node touch "$recovered_flag"
 
-ctdb_test_exit_hook="onnode $test_node rm -vf $trigger"
+ctdb_test_exit_hook_add "onnode $test_node rm -vf $trigger"
 
 echo "Creating trigger file on node $test_node to make it unhealthy..."
 try_command_on_node $test_node touch "$trigger"

diff --git a/tests/simple/22_ctdb_enablemonitor.sh b/tests/simple/22_ctdb_enablemonitor.sh
index f7d68091c85f19978deae9f03d1575b129bf9da4..98fc7e2700e1835cb104cdf2c16a6134777aeecc 100755 (executable)
--- a/tests/simple/22_ctdb_enablemonitor.sh
+++ b/tests/simple/22_ctdb_enablemonitor.sh
@@ -42,6 +42,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 test_node=1
 
 # We need this for later, so we know how long to sleep.
@@ -64,7 +67,7 @@ detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
 recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
 try_command_on_node $test_node touch "$recovered_flag"
 
-ctdb_test_exit_hook="onnode $test_node rm -vf $trigger; restart_ctdb"
+ctdb_test_exit_hook_add "onnode $test_node rm -vf $trigger"
 
 echo "Creating trigger file on node $test_node to see if it goes unhealthy..."
 try_command_on_node $test_node touch "$trigger"
@@ -96,6 +99,6 @@ onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $mon
 
 try_command_on_node -v $test_node ls -l "$detected"
 
-echo "OK, that all worked.  Expect a restart..."
+echo "GOOD: That all worked..."
 
 ctdb_test_exit

diff --git a/tests/simple/23_ctdb_moveip.sh b/tests/simple/23_ctdb_moveip.sh
index c91bbbee9e473fd4e0f9e389f9b6da1ccd8d5fd7..ca7fe11fb4c2d0a0f4e6b42656a9f8daedf182ff 100755 (executable)
--- a/tests/simple/23_ctdb_moveip.sh
+++ b/tests/simple/23_ctdb_moveip.sh
@@ -37,8 +37,8 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-# Restart when done since things are likely to be broken.
-ctdb_test_exit_hook="restart_ctdb"
+# Reset configuration
+ctdb_restart_when_done
 
 try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
@@ -100,6 +100,6 @@ else
     exit 1
 fi
 
-echo "OK, that worked... expect a restart..."
+echo "GOOD: That worked..."
 
 ctdb_test_exit

diff --git a/tests/simple/24_ctdb_getdbmap.sh b/tests/simple/24_ctdb_getdbmap.sh
index 49ebda07429109d9c8133a4926e0ab37613c8fbf..c621a304ad624dbb41680d7f6afad6f38da4ff36 100755 (executable)
--- a/tests/simple/24_ctdb_getdbmap.sh
+++ b/tests/simple/24_ctdb_getdbmap.sh
@@ -31,8 +31,8 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-# Restart when done since things are likely to be broken.
-ctdb_test_exit_hook="restart_ctdb"
+# Reset configuration
+ctdb_restart_when_done
 
 make_temp_db_filename ()
 {
@@ -70,6 +70,6 @@ for i in $(seq 1 5) ; do
     fi
 done
 
-echo "OK, that worked... expect a restart..."
+echo "GOOD: That worked..."
 
 ctdb_test_exit

diff --git a/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
index 72e0acd7a651d2c388187554b5c67a6a4216de89..4287cba1fbe4885fa723b651f2d797bf451d2172 100755 (executable)
--- a/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@@ -64,8 +64,6 @@ for i in ip disable enable "ban 0" unban listvars ; do
     fi
 done
 
-echo "That all looks OK.  Restarting cluster..."
-
-restart_ctdb
+echo "OK, that all worked.  Expect a restart..."
 
 ctdb_test_exit