ctdb-tests: Need to drop public IPs in kill-failover tests

These tests simulate a dead node rather than a CTDB failure, so drop
IP addresses when killing a "node" to avoid problems with duplicates.

To cope with a CTDB failure a watchdog would be needed to ensure that
the public IPs are dropped when CTDB dies.  Let's not do that now.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
Autobuild-User(master): Martin Schwenke <martins@samba.org>
Autobuild-Date(master): Fri Dec  5 23:29:39 CET 2014 on sn-devel-104

diff --git a/ctdb/tests/complex/31_nfs_tickle.sh b/ctdb/tests/complex/31_nfs_tickle.sh
index 5aeb870d3c65a1015300a292899bf05bb8a31124..fbb30c14cc7f7c3aaec3a52a184ca252982be4b7 100755 (executable)
--- a/ctdb/tests/complex/31_nfs_tickle.sh
+++ b/ctdb/tests/complex/31_nfs_tickle.sh
@@ -89,9 +89,12 @@ wait_until $(($update_interval * 2)) \
 tcptickle_sniff_start $src_socket "${test_ip}:${test_port}"
 
 # We need to be nasty to make that the node being failed out doesn't
-# get a chance to send any tickles and confuse our sniff.
+# get a chance to send any tickles and confuse our sniff.  IPs also
+# need to be dropped because we're simulating a dead node rather than
+# a CTDB failure.  To properly handle a CTDB failure we would need a
+# watchdog to drop the IPs when CTDB disappears.
 echo "Killing ctdbd on ${test_node}..."
-try_command_on_node $test_node killall -9 ctdbd
+try_command_on_node -v $test_node "killall -9 ctdbd ; $CTDB_TEST_WRAPPER drop_ips ${test_node_ips}"
 
 wait_until_node_has_status $test_node disconnected
 

diff --git a/ctdb/tests/complex/45_failover_nfs_kill.sh b/ctdb/tests/complex/45_failover_nfs_kill.sh
index 52b423fb12b29456b4bef1310e1fa7fffef56bec..aada3711d11a8aabd096a13572432b398c098a53 100755 (executable)
--- a/ctdb/tests/complex/45_failover_nfs_kill.sh
+++ b/ctdb/tests/complex/45_failover_nfs_kill.sh
@@ -61,7 +61,12 @@ gratarp_sniff_start
 echo "Killing node $test_node"
 try_command_on_node $test_node $CTDB getpid
 pid=${out#*:}
-try_command_on_node $test_node kill -9 $pid
+# We need to be nasty to make that the node being failed out doesn't
+# get a chance to send any tickles or doing anything else clever.  IPs
+# also need to be dropped because we're simulating a dead node rather
+# than a CTDB failure.  To properly handle a CTDB failure we would
+# need a watchdog to drop the IPs when CTDB disappears.
+try_command_on_node -v $test_node "kill -9 $pid ; $CTDB_TEST_WRAPPER drop_ips ${test_node_ips}"
 wait_until_node_has_status $test_node disconnected
 
 gratarp_sniff_wait_show

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 6bec78ef987b5a48be3d9307917fb413cff4aa56..0d27c93643fabfe460ad0e4288d7bf3495401c74 100644 (file)
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -687,6 +687,44 @@ nfs_test_cleanup ()
 
 #######################################
 
+# If the given IP is hosted then print 2 items: maskbits and iface
+ip_maskbits_iface ()
+{
+    _addr="$1"
+
+    case "$_addr" in
+       *:*) _family="inet6" ; _bits=128 ;;
+       *)   _family="inet"  ; _bits=32  ;;
+    esac
+
+    ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
+       awk -v family="${_family}" \
+           'NR == 1 { iface = gensub(":$", "", 1, $2) } \
+             $1 ~ /inet/ { print gensub(".*/", "", 1, $2), iface, family }'
+}
+
+drop_ip ()
+{
+    _addr="${1%/*}"  # Remove optional maskbits
+
+    set -- $(ip_maskbits_iface $_addr)
+    if [ -n "$1" ] ; then
+       _maskbits="$1"
+       _iface="$2"
+       echo "Removing public address $_addr/$_maskbits from device $_iface"
+       ip addr del "$_ip/$_maskbits" dev "$_iface" >/dev/null 2>&1 || true
+    fi
+}
+
+drop_ips ()
+{
+    for _ip ; do
+       drop_ip "$_ip"
+    done
+}
+
+#######################################
+
 # $1: pnn, $2: DB name
 db_get_path ()
 {