ctdb-tests: Add tests for takeover helper

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

diff --git a/ctdb/tests/run_tests.sh b/ctdb/tests/run_tests.sh
index c39b92dcd85dccc2ace64a282fe55dc6bf1d2247..f128d959252bb402e8582c94363fd4e488b09c7e 100755 (executable)
--- a/ctdb/tests/run_tests.sh
+++ b/ctdb/tests/run_tests.sh
@@ -257,7 +257,8 @@ export TEST_SCRIPTS_DIR="${test_dir}/scripts"
 # If no tests specified then run some defaults
 if [ -z "$1" ] ; then
     if [ -n "$TEST_LOCAL_DAEMONS" ] ; then
-       set -- onnode takeover tool eventscripts cunit eventd shellcheck simple
+       set -- onnode takeover takeover_helper tool eventscripts \
+           cunit eventd shellcheck simple
     else
        set -- simple complex
     fi

diff --git a/ctdb/tests/takeover_helper/000.sh b/ctdb/tests/takeover_helper/000.sh
new file mode 100755 (executable)
index 0000000..3cb9635
--- /dev/null
+++ b/ctdb/tests/takeover_helper/000.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, no IPs"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+EOF
+
+required_result 0 <<EOF
+No nodes available to host public IPs yet
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/010.sh b/ctdb/tests/takeover_helper/010.sh
new file mode 100755 (executable)
index 0000000..1275156
--- /dev/null
+++ b/ctdb/tests/takeover_helper/010.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/011.sh b/ctdb/tests/takeover_helper/011.sh
new file mode 100755 (executable)
index 0000000..12a2a1a
--- /dev/null
+++ b/ctdb/tests/takeover_helper/011.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, 1 ok, IPs all unassigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 2
+10.0.0.33 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/012.sh b/ctdb/tests/takeover_helper/012.sh
new file mode 100755 (executable)
index 0000000..04e4508
--- /dev/null
+++ b/ctdb/tests/takeover_helper/012.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, 1 IP unassigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 2
+10.0.0.33 1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/013.sh b/ctdb/tests/takeover_helper/013.sh
new file mode 100755 (executable)
index 0000000..ad55564
--- /dev/null
+++ b/ctdb/tests/takeover_helper/013.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, 1 unhealthy, IPs all assigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 0
+10.0.0.33 1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/014.sh b/ctdb/tests/takeover_helper/014.sh
new file mode 100755 (executable)
index 0000000..bb36893
--- /dev/null
+++ b/ctdb/tests/takeover_helper/014.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all unhealthy, all IPs assigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 2
+10.0.0.33 1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/015.sh b/ctdb/tests/takeover_helper/015.sh
new file mode 100755 (executable)
index 0000000..2841412
--- /dev/null
+++ b/ctdb/tests/takeover_helper/015.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all unhealthy, all IPs assigned, NoIPHostOnAllDisabled"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ctdb_cmd setvar NoIPHostOnAllDisabled 1
+
+ok <<EOF
+Failed to find node to cover ip 10.0.0.33
+Failed to find node to cover ip 10.0.0.32
+Failed to find node to cover ip 10.0.0.31
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/016.sh b/ctdb/tests/takeover_helper/016.sh
new file mode 100755 (executable)
index 0000000..4d2e4fe
--- /dev/null
+++ b/ctdb/tests/takeover_helper/016.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs all unassigned, DisableIPFailover"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+EOF
+
+ctdb_cmd setvar DisableIPFailover 1
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/017.sh b/ctdb/tests/takeover_helper/017.sh
new file mode 100755 (executable)
index 0000000..e5bcd20
--- /dev/null
+++ b/ctdb/tests/takeover_helper/017.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs unbalanced, NoIPFailback"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  1
+EOF
+
+ctdb_cmd setvar NoIPFailback 1
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/018.sh b/ctdb/tests/takeover_helper/018.sh
new file mode 100755 (executable)
index 0000000..61a26dd
--- /dev/null
+++ b/ctdb/tests/takeover_helper/018.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs unbalanced"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  1
+EOF
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/019.sh b/ctdb/tests/takeover_helper/019.sh
new file mode 100755 (executable)
index 0000000..0802611
--- /dev/null
+++ b/ctdb/tests/takeover_helper/019.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, 1 node unhealthy, IPs all assigned, NoIPTakeover"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+EOF
+
+ctdb_cmd setvar NoIPTakeover 1
+
+ok <<EOF
+Failed to find node to cover ip 10.0.0.32
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 -1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/020.sh b/ctdb/tests/takeover_helper/020.sh
new file mode 100755 (executable)
index 0000000..111b5a4
--- /dev/null
+++ b/ctdb/tests/takeover_helper/020.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all unhealthy, IPs all assigned, NoIPTakeover"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+EOF
+
+ctdb_cmd setvar NoIPTakeover 1
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/021.sh b/ctdb/tests/takeover_helper/021.sh
new file mode 100755 (executable)
index 0000000..ad8e59f
--- /dev/null
+++ b/ctdb/tests/takeover_helper/021.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs all assigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  -1
+EOF
+
+ctdb_cmd setvar NoIPTakeover 1
+
+ok <<EOF
+Failed to find node to cover ip 10.0.0.34
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 -1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/022.sh b/ctdb/tests/takeover_helper/022.sh
new file mode 100755 (executable)
index 0000000..e8c5a96
--- /dev/null
+++ b/ctdb/tests/takeover_helper/022.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs very unbalanced, no force rebalance"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+10.0.0.36  2
+EOF
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 2
+10.0.0.36 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/023.sh b/ctdb/tests/takeover_helper/023.sh
new file mode 100755 (executable)
index 0000000..a76afef
--- /dev/null
+++ b/ctdb/tests/takeover_helper/023.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs very unbalanced, force rebalance 1"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+10.0.0.36  2
+EOF
+
+ok <<EOF
+Forcing rebalancing of IPs to node 1
+EOF
+test_takeover_helper 1
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 1
+10.0.0.36 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/024.sh b/ctdb/tests/takeover_helper/024.sh
new file mode 100755 (executable)
index 0000000..af7480c
--- /dev/null
+++ b/ctdb/tests/takeover_helper/024.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs very unbalanced, force rebalance all"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+10.0.0.36  2
+EOF
+
+ok <<EOF
+Forcing rebalancing of IPs to node 1
+Forcing rebalancing of IPs to node 0
+Forcing rebalancing of IPs to node 2
+EOF
+test_takeover_helper 1,0,2
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 0
+10.0.0.36 1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/025.sh b/ctdb/tests/takeover_helper/025.sh
new file mode 100755 (executable)
index 0000000..28db486
--- /dev/null
+++ b/ctdb/tests/takeover_helper/025.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all assigned randomly, deterministic IPs"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  0
+10.0.0.33  2
+EOF
+
+ctdb_cmd setvar IPAllocAlgorithm 0
+
+ok <<EOF
+Deterministic IPs enabled. Resetting all ip allocations
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/026.sh b/ctdb/tests/takeover_helper/026.sh
new file mode 100755 (executable)
index 0000000..08a7b6d
--- /dev/null
+++ b/ctdb/tests/takeover_helper/026.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs assigned, unbalanced, non-deterministic IPs"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+EOF
+
+ctdb_cmd setvar IPAllocAlgorithm 1
+
+ok_null
+test_takeover_helper
+
+# This is non-deterministic - LCP2 would not rebalance without
+# force-rebalance-nodes
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/110.sh b/ctdb/tests/takeover_helper/110.sh
new file mode 100755 (executable)
index 0000000..56dc16c
--- /dev/null
+++ b/ctdb/tests/takeover_helper/110.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, no IPs, IPREALLOCATED error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+CONTROLFAILS
+137    1       ERROR   CTDB_CONTROL_IPREALLOCATED fake failure
+
+EOF
+
+required_result 255 <<EOF
+No nodes available to host public IPs yet
+IPREALLOCATED failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/111.sh b/ctdb/tests/takeover_helper/111.sh
new file mode 100755 (executable)
index 0000000..d14868b
--- /dev/null
+++ b/ctdb/tests/takeover_helper/111.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, IPREALLOCATED error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+137    1       ERROR   CTDB_CONTROL_IPREALLOCATED fake failure
+EOF
+
+required_result 255 <<EOF
+IPREALLOCATED failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/120.sh b/ctdb/tests/takeover_helper/120.sh
new file mode 100755 (executable)
index 0000000..af780d6
--- /dev/null
+++ b/ctdb/tests/takeover_helper/120.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, TAKEOVER_IP error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+89     1       ERROR   CTDB_CONTROL_TAKEOVER_IP fake failure
+EOF
+
+required_result 255 <<EOF
+TAKEOVER_IP 10.0.0.32 failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 -1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/121.sh b/ctdb/tests/takeover_helper/121.sh
new file mode 100755 (executable)
index 0000000..cc113da
--- /dev/null
+++ b/ctdb/tests/takeover_helper/121.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, 2/3 IPs assigned, TAKEOVER_IP error (redundant)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  2
+10.0.0.32  1
+10.0.0.33  -1
+
+CONTROLFAILS
+89     1       ERROR   CTDB_CONTROL_TAKEOVER_IP fake failure
+EOF
+
+required_result 255 <<EOF
+TAKEOVER_IP 10.0.0.32 failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/122.sh b/ctdb/tests/takeover_helper/122.sh
new file mode 100755 (executable)
index 0000000..d823b09
--- /dev/null
+++ b/ctdb/tests/takeover_helper/122.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, 2/3 IPs assigned, TAKEOVER_IP error (target)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  2
+10.0.0.32  1
+10.0.0.33  -1
+
+CONTROLFAILS
+89     0       ERROR   CTDB_CONTROL_TAKEOVER_IP fake failure
+EOF
+
+required_result 255 <<EOF
+TAKEOVER_IP 10.0.0.33 failed on node 0, ret=-1
+Assigning banning credits to node 0
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/130.sh b/ctdb/tests/takeover_helper/130.sh
new file mode 100755 (executable)
index 0000000..83735d4
--- /dev/null
+++ b/ctdb/tests/takeover_helper/130.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, RELEASE_IP error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+88     2       ERROR   CTDB_CONTROL_RELEASE_IP fake failure
+EOF
+
+required_result 255 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 2, ret=-1
+RELEASE_IP 10.0.0.32 failed on node 2, ret=-1
+Assigning banning credits to node 2
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/131.sh b/ctdb/tests/takeover_helper/131.sh
new file mode 100755 (executable)
index 0000000..4e0cd46
--- /dev/null
+++ b/ctdb/tests/takeover_helper/131.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, all IPs assigned, RELEASE_IP error (redundant)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+
+CONTROLFAILS
+88     0       ERROR   CTDB_CONTROL_RELEASE_IP fake failure
+EOF
+
+required_result 255 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 0, ret=-1
+Assigning banning credits to node 0
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 -1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/132.sh b/ctdb/tests/takeover_helper/132.sh
new file mode 100755 (executable)
index 0000000..a1a4ce5
--- /dev/null
+++ b/ctdb/tests/takeover_helper/132.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, all IPs assigned, RELEASE_IP error (target)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+
+CONTROLFAILS
+88     1       ERROR   CTDB_CONTROL_RELEASE_IP fake failure
+EOF
+
+required_result 255 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 1, ret=-1
+RELEASE_IP 10.0.0.32 failed on node 1, ret=-1
+RELEASE_IP 10.0.0.31 failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/140.sh b/ctdb/tests/takeover_helper/140.sh
new file mode 100755 (executable)
index 0000000..60a03f2
--- /dev/null
+++ b/ctdb/tests/takeover_helper/140.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_PUBLIC_IPS error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+90     2       ERROR   CTDB_CONTROL_GET_PUBLIC_IPS fake failure
+EOF
+
+required_result 255 <<EOF
+control GET_PUBLIC_IPS failed on node 2, ret=-1
+Failed to fetch known public IPs
+takeover run failed, ret=-1
+EOF
+test_takeover_helper

diff --git a/ctdb/tests/takeover_helper/150.sh b/ctdb/tests/takeover_helper/150.sh
new file mode 100755 (executable)
index 0000000..56042b4
--- /dev/null
+++ b/ctdb/tests/takeover_helper/150.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_NODEMAP error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+91     0       ERROR   CTDB_CONTROL_GET_NODEMAP fake failure
+EOF
+
+required_result 255 <<EOF
+control GET_NODEMAP failed, ret=-1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper

diff --git a/ctdb/tests/takeover_helper/160.sh b/ctdb/tests/takeover_helper/160.sh
new file mode 100755 (executable)
index 0000000..c09f649
--- /dev/null
+++ b/ctdb/tests/takeover_helper/160.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_ALL_TUNABLES error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+53     0       ERROR   CTDB_CONTROL_GET_ALL_TUNABLES fake failure
+EOF
+
+required_result 255 <<EOF
+control GET_ALL_TUNABLES failed, ret=-1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper

diff --git a/ctdb/tests/takeover_helper/210.sh b/ctdb/tests/takeover_helper/210.sh
new file mode 100755 (executable)
index 0000000..207949e
--- /dev/null
+++ b/ctdb/tests/takeover_helper/210.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, no IPs, IPREALLOCATED timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+CONTROLFAILS
+137    1       TIMEOUT CTDB_CONTROL_IPREALLOCATED fake timeout
+
+EOF
+
+required_result 110 <<EOF
+No nodes available to host public IPs yet
+IPREALLOCATED failed on node 1, ret=110
+Assigning banning credits to node 1
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/211.sh b/ctdb/tests/takeover_helper/211.sh
new file mode 100755 (executable)
index 0000000..f4eca68
--- /dev/null
+++ b/ctdb/tests/takeover_helper/211.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, IPREALLOCATED timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+137    1       TIMEOUT CTDB_CONTROL_IPREALLOCATED fake timeout
+EOF
+
+required_result 110 <<EOF
+IPREALLOCATED failed on node 1, ret=110
+Assigning banning credits to node 1
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/220.sh b/ctdb/tests/takeover_helper/220.sh
new file mode 100755 (executable)
index 0000000..d8c7823
--- /dev/null
+++ b/ctdb/tests/takeover_helper/220.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, TAKEOVER_IP timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+89     1       TIMEOUT CTDB_CONTROL_TAKEOVER_IP fake timeout
+EOF
+
+required_result 110 <<EOF
+TAKEOVER_IP 10.0.0.32 failed to node 1, ret=110
+Assigning banning credits to node 1
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 -1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/230.sh b/ctdb/tests/takeover_helper/230.sh
new file mode 100755 (executable)
index 0000000..9b77c2b
--- /dev/null
+++ b/ctdb/tests/takeover_helper/230.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, RELEASE_IP timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+88     2       TIMEOUT CTDB_CONTROL_RELEASE_IP fake timeout
+EOF
+
+required_result 110 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 2, ret=110
+RELEASE_IP 10.0.0.32 failed on node 2, ret=110
+Assigning banning credits to node 2
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all

diff --git a/ctdb/tests/takeover_helper/240.sh b/ctdb/tests/takeover_helper/240.sh
new file mode 100755 (executable)
index 0000000..00004a5
--- /dev/null
+++ b/ctdb/tests/takeover_helper/240.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_PUBLIC_IPS timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+90     2       TIMEOUT CTDB_CONTROL_GET_PUBLIC_IPS fake timeout
+EOF
+
+required_result 110 <<EOF
+control GET_PUBLIC_IPS failed on node 2, ret=110
+Failed to fetch known public IPs
+takeover run failed, ret=110
+EOF
+test_takeover_helper

diff --git a/ctdb/tests/takeover_helper/250.sh b/ctdb/tests/takeover_helper/250.sh
new file mode 100755 (executable)
index 0000000..1422474
--- /dev/null
+++ b/ctdb/tests/takeover_helper/250.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_NODEMAP timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+91     0       TIMEOUT CTDB_CONTROL_GET_NODEMAP fake timeout
+EOF
+
+required_result 110 <<EOF
+control GET_NODEMAP failed to node 0, ret=110
+takeover run failed, ret=110
+EOF
+test_takeover_helper

diff --git a/ctdb/tests/takeover_helper/260.sh b/ctdb/tests/takeover_helper/260.sh
new file mode 100755 (executable)
index 0000000..645faa2
--- /dev/null
+++ b/ctdb/tests/takeover_helper/260.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_ALL_TUNABLES timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+53     0       TIMEOUT CTDB_CONTROL_GET_ALL_TUNABLES fake timeout
+EOF
+
+required_result 110 <<EOF
+control GET_ALL_TUNABLES failed, ret=110
+takeover run failed, ret=110
+EOF
+test_takeover_helper

diff --git a/ctdb/tests/takeover_helper/scripts/local.sh b/ctdb/tests/takeover_helper/scripts/local.sh
new file mode 100644 (file)
index 0000000..4dd20d7
--- /dev/null
+++ b/ctdb/tests/takeover_helper/scripts/local.sh
@@ -0,0 +1,109 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!  :-)
+
+# Augment PATH with stubs/ directory.
+
+if "$TEST_VERBOSE" ; then
+       debug () { echo "$@" ; }
+else
+       debug () { : ; }
+fi
+
+. "${TEST_SCRIPTS_DIR}/script_install_paths.sh"
+
+PATH="$PATH:$CTDB_SCRIPTS_TOOLS_HELPER_DIR"
+
+ctdbd_socket="${TEST_VAR_DIR}/ctdbd.socket.$$"
+ctdbd_pidfile="${TEST_VAR_DIR}/ctdbd.pid.$$"
+
+define_test ()
+{
+       _f=$(basename "$0" ".sh")
+
+       printf "%-28s - %s\n" "$_f" "$1"
+
+       if [ -z "$FAKE_CTDBD_DEBUGLEVEL" ] ; then
+               FAKE_CTDBD_DEBUGLEVEL="ERR"
+       fi
+       if [ -z "$HELPER_DEBUGLEVEL" ] ; then
+               HELPER_DEBUGLEVEL="NOTICE"
+       fi
+       if [ -z "$CTDB_DEBUGLEVEL" ] ; then
+               CTDB_DEBUGLEVEL="ERR"
+       fi
+}
+
+cleanup_ctdbd ()
+{
+       debug "Cleaning up fake ctdbd"
+
+       pid=$(cat "$ctdbd_pidfile" 2>/dev/null || echo)
+       if [ -n "$pid" ] ; then
+               kill $pid || true
+               rm -f "$ctdbd_pidfile"
+       fi
+       rm -f "$ctdbd_socket"
+}
+
+setup_ctdbd ()
+{
+       debug "Setting up fake ctdbd"
+
+       $VALGRIND fake_ctdbd -d "$FAKE_CTDBD_DEBUGLEVEL" \
+                 -s "$ctdbd_socket" -p "$ctdbd_pidfile"
+       # This current translates to a 6 second timeout for the
+       # important controls
+       ctdb --socket $ctdbd_socket setvar TakeoverTimeout 2
+       test_cleanup cleanup_ctdbd
+}
+
+ctdbd_getpid ()
+{
+       cat "$ctdbd_pidfile"
+}
+
+# Render non-printable characters.  The helper prints the status as
+# binary, so render it for easy comparison.
+result_filter ()
+{
+       sed -e 's|ctdb-takeover\[[0-9]*\]: ||'
+}
+
+ctdb_cmd ()
+{
+       echo Running: ctdb -d "$CTDB_DEBUGLEVEL" --socket $ctdbd_socket "$@"
+       ctdb -d "$CTDB_DEBUGLEVEL" --socket $ctdbd_socket "$@"
+}
+
+test_ctdb_ip_all ()
+{
+       unit_test ctdb -d "$CTDB_DEBUGLEVEL" \
+                 --socket $ctdbd_socket ip all || exit $?
+}
+
+takeover_helper_out="${TEST_VAR_DIR}/takover_helper.out"
+
+takeover_helper_format_outfd ()
+{
+       od -A n -t d4 "$takeover_helper_out" | sed -e 's|^[[:space:]]*||'
+}
+
+test_takeover_helper ()
+{
+       (
+               export CTDB_DEBUGLEVEL="$HELPER_DEBUGLEVEL"
+               export CTDB_LOGGING="file:"
+               unit_test ctdb_takeover_helper 3 "$ctdbd_socket" "$@" \
+                         3>"$takeover_helper_out"
+       ) || exit $?
+
+       case "$required_rc" in
+       255) _t="-1" ;;
+       *) _t="$required_rc" ;;
+       esac
+       ok "$_t"
+
+       unit_test_notrace takeover_helper_format_outfd
+       _ret=$?
+       rm "$takeover_helper_out"
+       [ $? -eq 0 ] || exit $?
+}

diff --git a/ctdb/wscript b/ctdb/wscript
index 66959cde0915062bbeebd7442ecc27c27081a6c8..b951dd69f6c490f9bcad21d95fa358b736842805 100644 (file)
--- a/ctdb/wscript
+++ b/ctdb/wscript
@@ -838,6 +838,7 @@ def build(bld):
         'shellcheck',
         'simple',
         'takeover',
+        'takeover_helper',
         'tool'
     ]