tests/eventscripts: Unit tests for $CTDB_NFS_DUMP_STUCK_THREADS

Includes minor test infrastructure updates.

Signed-off-by: Martin Schwenke <martin@meltin.net>
(This used to be ctdb commit cd4358b01c6c3d413b431f5760029d2b163b9c03)

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.113.sh b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh
new file mode 100755 (executable)
index 0000000..caa4989
--- /dev/null
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "knfsd down, 6 iterations, dump 5 threads, none hung"
+
+# knfsd fails and attempts to restart it fail.
+setup_nfs
+rpc_services_down "nfs"
+
+# Additionally, any hung threads should have stack traces dumped.
+CTDB_NFS_DUMP_STUCK_THREADS=5
+FAKE_NFSD_THREAD_PIDS=""
+
+iterate_test 6 'ok_null' \
+    2 'rpc_set_service_failure_response "nfsd"' \
+    4 'rpc_set_service_failure_response "nfsd"' \
+    6 'rpc_set_service_failure_response "nfsd"'

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.114.sh b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh
new file mode 100755 (executable)
index 0000000..8279395
--- /dev/null
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "knfsd down, 6 iterations, dump 5 threads, 3 hung"
+
+# knfsd fails and attempts to restart it fail.
+setup_nfs
+rpc_services_down "nfs"
+
+# Additionally, any hung threads should have stack traces dumped.
+CTDB_NFS_DUMP_STUCK_THREADS=5
+FAKE_NFSD_THREAD_PIDS="1001 1002 1003"
+
+iterate_test 6 'ok_null' \
+    2 'rpc_set_service_failure_response "nfsd"' \
+    4 'rpc_set_service_failure_response "nfsd"' \
+    6 'rpc_set_service_failure_response "nfsd"'

diff --git a/ctdb/tests/eventscripts/etc-ctdb/rc.local b/ctdb/tests/eventscripts/etc-ctdb/rc.local
index bc98f72c2b9d580b17344ab2ba811c4250869ac2..9cd4d5513615ff1028391775054872c1b9a5e56c 100755 (executable)
--- a/ctdb/tests/eventscripts/etc-ctdb/rc.local
+++ b/ctdb/tests/eventscripts/etc-ctdb/rc.local
@@ -36,6 +36,9 @@ get_proc ()
        fs/nfsd/threads)
            echo "$FAKE_NFSD_THREAD_PIDS" | wc -w
            ;;
+       */stack)
+           echo "[<ffffffff87654321>] fake_stack_trace_for_pid_${1}+0x0/0xff"
+           ;;
        *)
            echo "get_proc: \"$1\" not implemented"
            exit 1

diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh
index cb08b30dce5c5f4cb60b834def3263bbc295454d..6e2f15ce2e087027287a109b14832b00fe81d872 100644 (file)
--- a/ctdb/tests/eventscripts/scripts/local.sh
+++ b/ctdb/tests/eventscripts/scripts/local.sh
@@ -556,6 +556,7 @@ setup_nfs ()
     export CTDB_NFS_SKIP_SHARE_CHECK="no"
 
     export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
+    export CTDB_NFS_DUMP_STUCK_THREADS
 
     # Reset the failcounts for nfs services.
     eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
@@ -675,7 +676,19 @@ program $_pn version $_ver is not available"
                        case "${_progname}${_action#restart}" in
                            nfsd)
                                _t="\
-Trying to restart NFS service
+Trying to restart NFS service"
+
+                               if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then
+                                   for _pid in $FAKE_NFSD_THREAD_PIDS ; do
+                                       _t="\
+$_t
+Stack trace for stuck nfsd thread [${_pid}]:
+[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff"
+                                   done
+                               fi
+
+                               _t="\
+${_t}
 Starting nfslock: OK
 Starting nfs: OK"
                                ;;

diff --git a/ctdb/tests/eventscripts/stubs/pidof b/ctdb/tests/eventscripts/stubs/pidof
new file mode 100755 (executable)
index 0000000..b6ad6d8
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/pidof
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+case "$1" in
+    nfsd)
+       echo "$FAKE_NFSD_THREAD_PIDS"
+       ;;
+    *)
+       echo "pidof: \"$1\" not implemented"
+       exit 1
+esac