tests/eventscripts: Tests for memory checking in 00.ctdb

... plus updates to test infrastructure to support.

Signed-off-by: Martin Schwenke <martin@meltin.net>

diff --git a/tests/eventscripts/00.ctdb.monitor.001.sh b/tests/eventscripts/00.ctdb.monitor.001.sh
new file mode 100755 (executable)
index 0000000..4290d13
--- /dev/null
+++ b/tests/eventscripts/00.ctdb.monitor.001.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, no checks enabled"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY=""
+CTDB_MONITOR_FREE_MEMORY_WARN=""
+CTDB_CHECK_SWAP_IS_NOT_USED="no"
+
+ok_null
+
+simple_test

diff --git a/tests/eventscripts/00.ctdb.monitor.002.sh b/tests/eventscripts/00.ctdb.monitor.002.sh
new file mode 100755 (executable)
index 0000000..6e94012
--- /dev/null
+++ b/tests/eventscripts/00.ctdb.monitor.002.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, good situation, all enabled"
+
+setup_memcheck
+
+CTDB_MONITOR_FREE_MEMORY="500"
+CTDB_MONITOR_FREE_MEMORY_WARN="1000"
+CTDB_CHECK_SWAP_IS_NOT_USED="yes"
+
+ok_null
+
+simple_test

diff --git a/tests/eventscripts/00.ctdb.monitor.003.sh b/tests/eventscripts/00.ctdb.monitor.003.sh
new file mode 100755 (executable)
index 0000000..9e63ab5
--- /dev/null
+++ b/tests/eventscripts/00.ctdb.monitor.003.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, only swap check"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY=""
+CTDB_MONITOR_FREE_MEMORY_WARN=""
+CTDB_CHECK_SWAP_IS_NOT_USED="yes"
+
+ok <<EOF
+We are swapping:
+$FAKE_PROC_MEMINFO
+$(ps foobar)
+EOF
+
+simple_test

diff --git a/tests/eventscripts/00.ctdb.monitor.004.sh b/tests/eventscripts/00.ctdb.monitor.004.sh
new file mode 100755 (executable)
index 0000000..fdf2032
--- /dev/null
+++ b/tests/eventscripts/00.ctdb.monitor.004.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, only memory warning"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY=""
+CTDB_MONITOR_FREE_MEMORY_WARN="500"
+CTDB_CHECK_SWAP_IS_NOT_USED="no"
+
+ok <<EOF
+WARNING: free memory is low - 468MB free <=  ${CTDB_MONITOR_FREE_MEMORY_WARN}MB (CTDB threshold)
+EOF
+
+simple_test

diff --git a/tests/eventscripts/00.ctdb.monitor.005.sh b/tests/eventscripts/00.ctdb.monitor.005.sh
new file mode 100755 (executable)
index 0000000..a46851a
--- /dev/null
+++ b/tests/eventscripts/00.ctdb.monitor.005.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, only memory critical"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY="500"
+CTDB_MONITOR_FREE_MEMORY_WARN=""
+CTDB_CHECK_SWAP_IS_NOT_USED="no"
+
+ok <<EOF
+CRITICAL: OOM - 468MB free <= ${CTDB_MONITOR_FREE_MEMORY}MB (CTDB threshold)
+CRITICAL: Shutting down CTDB!!!
+$FAKE_PROC_MEMINFO
+$(ps foobar)
+CTDB says BYE!
+EOF
+
+simple_test

diff --git a/tests/eventscripts/etc-ctdb/rc.local b/tests/eventscripts/etc-ctdb/rc.local
index b11c7ecbf7e74bfc8055cfd7dfeca07ddb2a3ba1..6052d87f130335febd62d26f45e6cd994cc1431f 100755 (executable)
--- a/tests/eventscripts/etc-ctdb/rc.local
+++ b/tests/eventscripts/etc-ctdb/rc.local
@@ -39,6 +39,9 @@ get_proc ()
        */stack)
            echo "[<ffffffff87654321>] fake_stack_trace_for_pid_${1}+0x0/0xff"
            ;;
+       meminfo)
+           echo "$FAKE_PROC_MEMINFO"
+           ;;
        *)
            echo "get_proc: \"$1\" not implemented"
            exit 1

diff --git a/tests/eventscripts/scripts/local.sh b/tests/eventscripts/scripts/local.sh
index 00da7732dd2322ee56c43089e5357ca84a6f0957..a522e788979bc05ec86703e8428fc47917fc2cbb 100644 (file)
--- a/tests/eventscripts/scripts/local.sh
+++ b/tests/eventscripts/scripts/local.sh
@@ -311,6 +311,51 @@ setup_ctdb ()
     export CTDB_PARTIALLY_ONLINE_INTERFACES
 }
 
+setup_memcheck ()
+{
+    setup_ctdb
+
+    _swap_total="5857276"
+
+    if [ "$1" = "bad" ] ; then
+       _swap_free="   4352"
+       _mem_cached=" 112"
+       _mem_free=" 468"
+    else
+       _swap_free="$_swap_total"
+       _mem_cached="1112"
+       _mem_free="1468"
+    fi
+
+    export FAKE_PROC_MEMINFO="\
+MemTotal:        3940712 kB
+MemFree:          225268 kB
+Buffers:          146120 kB
+Cached:          1139348 kB
+SwapCached:        56016 kB
+Active:          2422104 kB
+Inactive:        1019928 kB
+Active(anon):    1917580 kB
+Inactive(anon):   523080 kB
+Active(file):     504524 kB
+Inactive(file):   496848 kB
+Unevictable:        4844 kB
+Mlocked:            4844 kB
+SwapTotal:       ${_swap_total} kB
+SwapFree:        ${_swap_free} kB
+..."
+
+    export FAKE_FREE_M="\
+             total       used       free     shared    buffers     cached
+Mem:          3848       3634        213          0        142       ${_mem_cached}
+-/+ buffers/cache:       2379       ${_mem_free}
+Swap:         5719        246       5473"
+
+    export CTDB_MONITOR_FREE_MEMORY
+    export CTDB_MONITOR_FREE_MEMORY_WARN
+    export CTDB_CHECK_SWAP_IS_NOT_USED
+}
+
 ctdb_get_interfaces ()
 {
     # The echo/subshell forces all the output onto 1 line.

diff --git a/tests/eventscripts/stubs/ctdb b/tests/eventscripts/stubs/ctdb
index 58007f2e2b0a7aa3464c7e8fe5ddac8fdfe79826..da84ed7cdfd7dcb542acd35233fb8f22f75f860a 100755 (executable)
--- a/tests/eventscripts/stubs/ctdb
+++ b/tests/eventscripts/stubs/ctdb
@@ -124,8 +124,8 @@ ip_reallocate ()
                # Have non-zero flags
                _this=0
                for _j in "$FAKE_CTDB_STATE/node-state/"*"/$_i" ; do
-                   _t="${_j%/*}" # dirname
-                   _f="${_t%/*}" # basename
+                   _tf="${_j%/*}" # dirname
+                   _f="${_tf##*/}" # basename
                    _this=$(( $_this | $_f ))
                done
            else
@@ -224,6 +224,13 @@ ctdb_disable ()
 
 ######################################################################
 
+ctdb_shutdown ()
+{
+    echo "CTDB says BYE!"
+}
+
+######################################################################
+
 case "$1" in
     gettickles)
        setup_tickles
@@ -322,5 +329,6 @@ case "$1" in
     enable)      ctdb_enable "$@";;
     disable)     ctdb_disable "$@";;
     moveip)      ctdb_moveip "$@";;
+    shutdown)    ctdb_shutdown "$@";;
     *) not_implemented "$1" ;;
 esac

diff --git a/tests/eventscripts/stubs/free b/tests/eventscripts/stubs/free
new file mode 100755 (executable)
index 0000000..6453509
--- /dev/null
+++ b/tests/eventscripts/stubs/free
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ "$1" = "-m" ] ; then
+    echo "$FAKE_FREE_M"
+    exit 0
+else
+    echo "free: not implemented - $*"
+    exit 1
+fi

diff --git a/tests/eventscripts/stubs/ps b/tests/eventscripts/stubs/ps
new file mode 100755 (executable)
index 0000000..5abeaf9
--- /dev/null
+++ b/tests/eventscripts/stubs/ps
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+cat <<EOF
+USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+root         2  0.0  0.0      0     0 ?        S    Aug28   0:00 [kthreadd]
+root         3  0.0  0.0      0     0 ?        S    Aug28   0:43  \_ [ksoftirqd/0]
+...
+root         1  0.0  0.0   2976   624 ?        Ss   Aug28   0:07 init [2]  
+root       495  0.0  0.0   3888  1640 ?        Ss   Aug28   0:00 udevd --daemon
+...
+[MORE FAKE ps OUTPUT]
+EOF