ctdb-tests: Actually wait for record to migrate to lmaster node

This test has been failing with:

  Wait until record is migrated to lmaster node 0
  <30|BAD: node 0 is not dmaster
  dmaster: 1
  rsn: 8
  flags: 0x00010000 MIGRATED_WITH_DATA
  data(6) = "value1"
  *** TEST COMPLETED (RC=1) AT 2021-02-02 06:18:48, CLEANING UP...

This should never happen.  If this really fails then the wait should
time out.

The problem is that wait_until() does:

  "$@" || _rc=$?

and vacuum_test_key_dmaster() currently calls ctdb_test_fail() on
failure, which causes the shell to exit.  Instead, pass a variant to
wait_until() that simply returns the correct status instead of
exiting.

An alternative would be to change the statement in wait_until() to do:

  ("$@") || _rc=$?

so it captures the exit.  However, this is a global change and
requires more thought.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Jeremy Allison <jra@samba.org>

diff --git a/ctdb/tests/INTEGRATION/database/scripts/local.bash b/ctdb/tests/INTEGRATION/database/scripts/local.bash
index 5334b1cb7d8a3aa8eba850a0c11a764179711d99..ae2e0d58de5f34bc6b7e1ca520f7957123d3a329 100644 (file)
--- a/ctdb/tests/INTEGRATION/database/scripts/local.bash
+++ b/ctdb/tests/INTEGRATION/database/scripts/local.bash
@@ -30,7 +30,7 @@ check_cattdb_num_records ()
        return $ret
 }
 
-vacuum_test_key_dmaster ()
+_key_dmaster_check ()
 {
        local node="$1"
        local db="$2"
@@ -41,10 +41,42 @@ vacuum_test_key_dmaster ()
 
        # shellcheck disable=SC2154
        # $outfile is set above by try_command_on_node()
-       if ! grep -Fqx "dmaster: ${dmaster}" "$outfile" ; then
-               echo "BAD: node ${dmaster} is not dmaster"
-               cat "$outfile"
-               ctdb_test_fail
+       grep -Fqx "dmaster: ${dmaster}" "$outfile"
+}
+
+_key_dmaster_fail ()
+{
+       local dmaster="$1"
+
+       echo "BAD: node ${dmaster} is not dmaster"
+       # shellcheck disable=SC2154
+       # $outfile is set by the caller via _key_dmaster_check()
+       cat "$outfile"
+       ctdb_test_fail
+}
+
+vacuum_test_key_dmaster ()
+{
+       local node="$1"
+       local db="$2"
+       local key="$3"
+       local dmaster="${4:-${node}}"
+
+       if ! _key_dmaster_check "$node" "$db" "$key" "$dmaster" ; then
+               _key_dmaster_fail "$dmaster"
+       fi
+}
+
+vacuum_test_wait_key_dmaster ()
+{
+       local node="$1"
+       local db="$2"
+       local key="$3"
+       local dmaster="${4:-${node}}"
+
+       if ! wait_until 30 \
+            _key_dmaster_check "$node" "$db" "$key" "$dmaster" ; then
+               _key_dmaster_fail "$dmaster"
        fi
 }
 

diff --git a/ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh b/ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh
index b6adbf73c87056fc2834f7a4ced2590ff72a6de0..d16482e2e63e3aad087b2cd30bab644a799718eb 100755 (executable)
--- a/ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh
+++ b/ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh
@@ -100,7 +100,7 @@ try_command_on_node "$non_lmaster" "kill ${pid}"
 ctdb_test_cleanup_pid_clear
 
 echo "Wait until record is migrated to lmaster node ${lmaster}"
-wait_until 30 vacuum_test_key_dmaster "$lmaster" "$db" "$key"
+vacuum_test_wait_key_dmaster "$lmaster" "$db" "$key"
 
 echo
 echo "Confirm that all nodes still have the record"