ctdb-tests: Add tests to ensure reset of server end of connections

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

diff --git a/ctdb/tests/complex/36_smb_reset_server.sh b/ctdb/tests/complex/36_smb_reset_server.sh
new file mode 100755 (executable)
index 0000000..5723ac7
--- /dev/null
+++ b/ctdb/tests/complex/36_smb_reset_server.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the server end of an SMB connection is correctly reset
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 nodes with public addresses.
+
+* Test must be run on a real or virtual cluster rather than against
+  local daemons.
+
+* Test must not be run from a cluster node.
+
+* Clustered Samba must be listening on TCP port 445.
+
+Expected results:
+
+* CTDB should correctly record the connection and the releasing node
+  should reset the server end of the connection.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init "$@"
+
+ctdb_test_check_real_cluster
+
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
+
+# We need this for later, so we know how long to sleep.
+try_command_on_node 0 $CTDB getvar MonitorInterval
+monitor_interval="${out#*= }"
+
+select_test_node_and_ips
+
+test_port=445
+
+echo "Set NoIPTakeover=1 on all nodes"
+try_command_on_node all $CTDB setvar NoIPTakeover 1
+
+echo "Give the recovery daemon some time to reload tunables"
+sleep_for 5
+
+echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with nc..."
+
+nc -d -w $(($monitor_interval * 4)) $test_ip $test_port &
+nc_pid=$!
+ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1"
+
+wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc"
+src_socket="$out"
+echo "Source socket is $src_socket"
+
+# This should happen as soon as connection is up... but unless we wait
+# we sometimes beat the registration.
+echo "Checking if CIFS connection is tracked by CTDB on test node..."
+wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
+echo "$out"
+
+if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
+    echo "GOOD: CIFS connection tracked OK by CTDB."
+else
+    echo "BAD: Socket not tracked by CTDB."
+    exit 1
+fi
+
+# It would be nice if ss consistently used local/peer instead of src/dst
+ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
+
+try_command_on_node $test_node \
+                   "ss -tn state established '${ss_filter}' | tail -n +2"
+if [ -z "$out" ] ; then
+       echo "BAD: ss did not list the socket"
+       exit 1
+fi
+echo "GOOD: ss lists the socket:"
+echo "$out"
+
+echo "Disabling node $test_node"
+try_command_on_node 1 $CTDB disable -n $test_node
+wait_until_node_has_status $test_node disabled
+
+try_command_on_node $test_node \
+                   "ss -tn state established '${ss_filter}' | tail -n +2"
+if [ -n "$out" ] ; then
+       echo "BAD: ss listed the socket after failover"
+       exit 1
+fi
+echo "GOOD: ss no longer lists the socket"

diff --git a/ctdb/tests/complex/37_nfs_reset_server.sh b/ctdb/tests/complex/37_nfs_reset_server.sh
new file mode 100755 (executable)
index 0000000..d6dd0f0
--- /dev/null
+++ b/ctdb/tests/complex/37_nfs_reset_server.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the server end of an NFS connection is correctly reset
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 nodes with public addresses.
+
+* Test must be run on a real or virtual cluster rather than against
+  local daemons.
+
+* Test must not be run from a cluster node.
+
+* Cluster nodes must be listening on the NFS TCP port (2049).
+
+Expected results:
+
+* CTDB should correctly record the connection and the releasing node
+  should reset the server end of the connection.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init "$@"
+
+ctdb_test_check_real_cluster
+
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
+
+# We need this for later, so we know how long to sleep.
+try_command_on_node 0 $CTDB getvar MonitorInterval
+monitor_interval="${out#*= }"
+
+select_test_node_and_ips
+
+test_port=2049
+
+echo "Set NoIPTakeover=1 on all nodes"
+try_command_on_node all $CTDB setvar NoIPTakeover 1
+
+echo "Give the recovery daemon some time to reload tunables"
+sleep_for 5
+
+echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with nc..."
+
+nc -d -w $(($monitor_interval * 4)) $test_ip $test_port &
+nc_pid=$!
+ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1"
+
+wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc"
+src_socket="$out"
+echo "Source socket is $src_socket"
+
+echo "Wait until NFS connection is tracked by CTDB on test node ..."
+wait_until $((monitor_interval * 2)) \
+          check_tickles $test_node $test_ip $test_port $src_socket
+echo "$out"
+
+# It would be nice if ss consistently used local/peer instead of src/dst
+ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
+
+try_command_on_node $test_node \
+                   "ss -tn state established '${ss_filter}' | tail -n +2"
+if [ -z "$out" ] ; then
+       echo "BAD: ss did not list the socket"
+       exit 1
+fi
+echo "GOOD: ss lists the socket:"
+echo "$out"
+
+echo "Disabling node $test_node"
+try_command_on_node 1 $CTDB disable -n $test_node
+wait_until_node_has_status $test_node disabled
+
+try_command_on_node $test_node \
+                   "ss -tn state established '${ss_filter}' | tail -n +2"
+if [ -n "$out" ] ; then
+       echo "BAD: ss listed the socket after failover"
+       exit 1
+fi
+echo "GOOD: ss no longer lists the socket"