. "${TEST_SCRIPTS_DIR}/common.sh"
-# If we're not running on a real cluster then we need a local copy of
-# ctdb (and other stuff) in $PATH and we will use local daemons.
-if [ -n "$TEST_LOCAL_DAEMONS" ] ; then
- export CTDB_NODES_SOCKETS=""
- for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
- CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${TEST_VAR_DIR}/sock.${i}"
- done
-
- # Use in-tree binaries if running against local daemons.
- # Otherwise CTDB need to be installed on all nodes.
- if [ -n "$ctdb_dir" -a -d "${ctdb_dir}/bin" ] ; then
- PATH="${ctdb_dir}/bin:${PATH}"
- fi
-
- export CTDB_NODES="${TEST_VAR_DIR}/nodes.txt"
-fi
-
######################################################################
export CTDB_TIMEOUT=60
if [ "$nodespec" = "-v" ] ; then
verbose=true
else
- onnode_opts="$nodespec"
+ onnode_opts="${onnode_opts}${onnode_opts:+ }${nodespec}"
fi
nodespec="$1" ; shift
done
_cluster_is_healthy ()
{
- local out x count line
-
- out=$($CTDB -Y status 2>/dev/null) || return 1
-
- {
- read x
- count=0
- while read line ; do
- # We need to see valid lines if we're going to be healthy.
- [ "${line#:[0-9]}" != "$line" ] && count=$(($count + 1))
- # A line indicating a node is unhealthy causes failure.
- [ "${line##:*:*:*1:}" != "$line" ] && return 1
- done
- [ $count -gt 0 ] && return $?
- } <<<"$out" # Yay bash!
+ $CTDB nodestatus all >/dev/null && \
+ node_has_status 0 recovered
}
cluster_is_healthy ()
local pnn="$1"
local status="$2"
- local bits fpat mpat
+ local bits fpat mpat rpat
case "$status" in
(unhealthy) bits="?:?:?:1:*" ;;
(healthy) bits="?:?:?:0:*" ;;
(unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
(monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
(monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;;
+ (recovered) rpat='^Recovery mode:NORMAL \(0\)$' ;;
*)
echo "node_has_status: unknown status \"$status\""
return 1
$CTDB statistics -n "$pnn" | egrep -q "$fpat"
elif [ -n "$mpat" ] ; then
$CTDB getmonmode -n "$pnn" | egrep -q "$mpat"
+ elif [ -n "$rpat" ] ; then
+ $CTDB status -n "$pnn" | egrep -q "$rpat"
else
echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
return 1
# Useful for superficially testing IP failover.
# IPs must be on nodes matching nodeglob.
+# If the first argument is '!' then the IPs must not be on nodes
+# matching nodeglob.
ips_are_on_nodeglob ()
{
+ local negating=false
+ if [ "$1" = "!" ] ; then
+ negating=true ; shift
+ fi
local nodeglob="$1" ; shift
local ips="$*"
all_ips_on_node 1
- while read ip pnn ; do
- for check in $ips ; do
+ for check in $ips ; do
+ while read ip pnn ; do
if [ "$check" = "$ip" ] ; then
case "$pnn" in
- ($nodeglob) : ;;
- (*) return 1 ;;
+ ($nodeglob) if $negating ; then return 1 ; fi ;;
+ (*) if ! $negating ; then return 1 ; fi ;;
esac
ips="${ips/${ip}}" # Remove from list
+ break
fi
- done
- done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+ # If we're negating and we didn't see the address then it
+ # isn't hosted by anyone!
+ if $negating ; then
+ ips="${ips/${check}}"
+ fi
+ done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+ done
ips="${ips// }" # Remove any spaces.
[ -z "$ips" ]
#######################################
-daemons_stop ()
-{
- echo "Attempting to politely shutdown daemons..."
- onnode 1 $CTDB shutdown -n all || true
-
- echo "Sleeping for a while..."
- sleep_for 1
-
- local pat="ctdbd --socket=.* --nlist .* --nopublicipcheck"
- if pgrep -f "$pat" >/dev/null ; then
- echo "Killing remaining daemons..."
- pkill -f "$pat"
-
- if pgrep -f "$pat" >/dev/null ; then
- echo "Once more with feeling.."
- pkill -9 -f "$pat"
- fi
- fi
-
- rm -rf "${TEST_VAR_DIR}/test.db"
-}
-
-daemons_setup ()
-{
- mkdir -p "${TEST_VAR_DIR}/test.db/persistent"
-
- local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all"
- local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt"
- rm -f $CTDB_NODES $public_addresses_all $no_public_addresses
-
- # If there are (strictly) greater than 2 nodes then we'll randomly
- # choose a node to have no public addresses.
- local no_public_ips=-1
- [ $TEST_LOCAL_DAEMONS -gt 2 ] && no_public_ips=$(($RANDOM % $TEST_LOCAL_DAEMONS))
- echo "$no_public_ips" >$no_public_addresses
-
- # When running certain tests we add and remove eventscripts, so we
- # need to be able to modify the events.d/ directory. Therefore,
- # we use a temporary events.d/ directory under $TEST_VAR_DIR. We
- # copy the actual test eventscript(s) in there from the original
- # events.d/ directory that sits alongside $TEST_SCRIPT_DIR.
- local top=$(dirname "$TEST_SCRIPTS_DIR")
- local events_d="${top}/events.d"
- mkdir -p "${TEST_VAR_DIR}/events.d"
- cp -p "${events_d}/"* "${TEST_VAR_DIR}/events.d/"
-
- local i
- for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do
- if [ "${CTDB_USE_IPV6}x" != "x" ]; then
- echo ::$i >>"$CTDB_NODES"
- ip addr add ::$i/128 dev lo
- else
- echo 127.0.0.$i >>"$CTDB_NODES"
- # 2 public addresses on most nodes, just to make things interesting.
- if [ $(($i - 1)) -ne $no_public_ips ] ; then
- echo "192.0.2.$i/24 lo" >>"$public_addresses_all"
- echo "192.0.2.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >>"$public_addresses_all"
- fi
- fi
- done
-}
-
-daemons_start_1 ()
-{
- local pnn="$1"
- shift # "$@" gets passed to ctdbd
-
- local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all"
- local public_addresses_mine="${TEST_VAR_DIR}/public_addresses.${pnn}"
- local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt"
-
- local no_public_ips=-1
- [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses
-
- if [ "$no_public_ips" = $pnn ] ; then
- echo "Node $no_public_ips will have no public IPs."
- fi
-
- local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES")
- local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --listen=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state"
-
- if [ -n "$TEST_LOCAL_DAEMONS" ] ; then
- ctdb_options="$ctdb_options --public-interface=lo"
- fi
-
- if [ $pnn -eq $no_public_ips ] ; then
- ctdb_options="$ctdb_options --public-addresses=/dev/null"
- else
- cp "$public_addresses_all" "$public_addresses_mine"
- ctdb_options="$ctdb_options --public-addresses=$public_addresses_mine"
- fi
-
- # We'll use "pkill -f" to kill the daemons with
- # "--socket=.* --nlist .* --nopublicipcheck" as context.
- $VALGRIND ctdbd --socket="${TEST_VAR_DIR}/sock.$pnn" $ctdb_options "$@" ||return 1
-}
-
-daemons_start ()
-{
- # "$@" gets passed to ctdbd
-
- echo "Starting $TEST_LOCAL_DAEMONS ctdb daemons..."
-
- for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
- daemons_start_1 $i "$@"
- done
-
- if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then
- ln -sf "${TEST_VAR_DIR}/sock.0" /tmp/ctdb.socket || return 1
- fi
-}
-
-#######################################
-
_ctdb_hack_options ()
{
local ctdb_options="$*"
- # We really just want to pass CTDB_OPTIONS but on RH
- # /etc/sysconfig/ctdb can, and frequently does, set that variable.
- # So instead, we hack badly. We'll add these as we use them.
- # Note that these may still be overridden by the above file... but
- # we tend to use the exotic options here... so that is unlikely.
-
case "$ctdb_options" in
*--start-as-stopped*)
export CTDB_START_AS_STOPPED="yes"
echo "Forcing a recovery..."
onnode -q 0 $CTDB recover
sleep_for 1
- echo "Forcing a recovery..."
- onnode -q 0 $CTDB recover
echo "ctdb is ready"
}
continue
}
- local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1)
-
echo "Setting RerecoveryTimeout to 1"
onnode -pq all "$CTDB setvar RerecoveryTimeout 1"
echo "Forcing a recovery..."
onnode -q 0 $CTDB recover
sleep_for 1
- echo "Forcing a recovery..."
- onnode -q 0 $CTDB recover
# Cluster is still healthy. Good, we're done!
if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
- echo "Cluster become UNHEALTHY again. Restarting..."
+ echo "Cluster became UNHEALTHY again [$(date)]"
+ onnode -p all ctdb status -Y 2>&1
+ onnode -p all ctdb scriptstatus 2>&1
+ echo "Restarting..."
continue
fi
done
echo "Cluster UNHEALTHY... too many attempts..."
- echo "$debug_out"
+ onnode -p all ctdb status -Y 2>&1
+ onnode -p all ctdb scriptstatus 2>&1
+
# Try to make the calling test fail
status=1
return 1
#######################################
-install_eventscript ()
+wait_for_monitor_event ()
{
- local script_name="$1"
- local script_contents="$2"
+ local pnn="$1"
+ local timeout=120
- if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
- # The quoting here is *very* fragile. However, we do
- # experience the joy of installing a short script using
- # onnode, and without needing to know the IP addresses of the
- # nodes.
- onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\""
- else
- f="${TEST_VAR_DIR}/events.d/${script_name}"
- echo "$script_contents" >"$f"
- chmod 755 "$f"
- fi
+ echo "Waiting for a monitor event on node ${pnn}..."
+
+ try_command_on_node "$pnn" $CTDB scriptstatus || {
+ echo "Unable to get scriptstatus from node $pnn"
+ return 1
+ }
+
+ local ctdb_scriptstatus_original="$out"
+ wait_until 120 _ctdb_scriptstatus_changed
}
-uninstall_eventscript ()
+_ctdb_scriptstatus_changed ()
{
- local script_name="$1"
+ try_command_on_node "$pnn" $CTDB scriptstatus || {
+ echo "Unable to get scriptstatus from node $pnn"
+ return 1
+ }
- if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
- onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\""
- else
- rm -vf "${TEST_VAR_DIR}/events.d/${script_name}"
- fi
+ [ "$out" != "$ctdb_scriptstatus_original" ]
}
#######################################
-# This section deals with the 99.ctdb_test eventscript.
-
-# Metafunctions: Handle a ctdb-test file on a node.
-# given event.
-ctdb_test_eventscript_file_create ()
+nfs_test_setup ()
{
- local pnn="$1"
- local type="$2"
+ select_test_node_and_ips
- try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}"
-}
+ nfs_first_export=$(showmount -e $test_ip | sed -n -e '2s/ .*//p')
-ctdb_test_eventscript_file_remove ()
-{
- local pnn="$1"
- local type="$2"
+ echo "Creating test subdirectory..."
+ try_command_on_node $test_node "mktemp -d --tmpdir=$nfs_first_export"
+ nfs_test_dir="$out"
+ try_command_on_node $test_node "chmod 777 $nfs_test_dir"
- try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}"
-}
+ nfs_mnt_d=$(mktemp -d)
+ nfs_local_file="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE"
+ nfs_remote_file="${nfs_test_dir}/TEST_FILE"
-ctdb_test_eventscript_file_exists ()
-{
- local pnn="$1"
- local type="$2"
+ ctdb_test_exit_hook_add nfs_test_cleanup
- try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1
+ echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..."
+ mount -o timeo=1,hard,intr,vers=3 \
+ ${test_ip}:${nfs_first_export} ${nfs_mnt_d}
}
-
-# Handle a flag file on a node that is removed by 99.ctdb_test on the
-# given event.
-ctdb_test_eventscript_flag ()
+nfs_test_cleanup ()
{
- local cmd="$1"
- local pnn="$2"
- local event="$3"
-
- ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}"
+ rm -f "$nfs_local_file"
+ umount -f "$nfs_mnt_d"
+ rmdir "$nfs_mnt_d"
+ onnode -q $test_node rmdir "$nfs_test_dir"
}
+#######################################
-# Handle a trigger that causes 99.ctdb_test to fail it's monitor
-# event.
-ctdb_test_eventscript_unhealthy_trigger ()
+# $1: pnn, $2: DB name
+db_get_path ()
{
- local cmd="$1"
- local pnn="$2"
-
- ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger"
+ try_command_on_node -v $1 $CTDB getdbstatus "$2" |
+ sed -n -e "s@^path: @@p"
}
-# Handle the file that 99.ctdb_test created to show that it has marked
-# a node unhealthy because it detected the above trigger.
-ctdb_test_eventscript_unhealthy_detected ()
+# $1: pnn, $2: DB name
+db_ctdb_cattdb_count_records ()
{
- local cmd="$1"
- local pnn="$2"
-
- ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected"
+ try_command_on_node -v $1 $CTDB cattdb "$2" |
+ grep '^key' | grep -v '__db_sequence_number__' |
+ wc -l
}
-# Handle a trigger that causes 99.ctdb_test to timeout it's monitor
-# event. This should cause the node to be banned.
-ctdb_test_eventscript_timeout_trigger ()
+# $1: pnn, $2: DB name, $3: key string, $4: value string, $5: RSN (default 7)
+db_ctdb_tstore ()
{
- local cmd="$1"
- local pnn="$2"
- local event="$3"
-
- ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout"
+ _tdb=$(db_get_path $1 "$2")
+ _rsn="${5:-7}"
+ try_command_on_node $1 $CTDB tstore "$_tdb" "$3" "$4" "$_rsn"
}
-# Note that the eventscript can't use the above functions!
-ctdb_test_eventscript_install ()
+# $1: pnn, $2: DB name, $3: dbseqnum (must be < 255!!!!!)
+db_ctdb_tstore_dbseqnum ()
{
+ # "__db_sequence_number__" + trailing 0x00
+ _key='0x5f5f64625f73657175656e63655f6e756d6265725f5f00'
- local script='#!/bin/sh
-out=$(ctdb pnn)
-pnn="${out#PNN:}"
-
-rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}"
-
-trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}"
-detected="/tmp/ctdb-test-unhealthy-detected.${pnn}"
-timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}"
-case "$1" in
- monitor)
- if [ -e "$trigger" ] ; then
- echo "${0}: Unhealthy because \"$trigger\" detected"
- touch "$detected"
- exit 1
- elif [ -e "$detected" -a ! -e "$trigger" ] ; then
- echo "${0}: Healthy again, \"$trigger\" no longer detected"
- rm "$detected"
- fi
-
- ;;
- *)
- if [ -e "$timeout_trigger" ] ; then
- echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected"
- sleep 9999
- fi
- ;;
- *)
-
-esac
+ # Construct 8 byte (unit64_t) database sequence number. This
+ # probably breaks if $3 > 255
+ _value=$(printf "0x%02x%014x" $3 0)
-exit 0
-'
- install_eventscript "99.ctdb_test" "$script"
+ db_ctdb_tstore $1 "$2" "$_key" "$_value"
}
-ctdb_test_eventscript_uninstall ()
-{
- uninstall_eventscript "99.ctdb_test"
-}
-
-# Note that this only works if you know all other monitor events will
-# succeed. You also need to install the eventscript before using it.
-wait_for_monitor_event ()
-{
- local pnn="$1"
-
- echo "Waiting for a monitor event on node ${pnn}..."
- ctdb_test_eventscript_flag create $pnn "monitor"
-
- wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor"
-
-}
+#######################################
# Make sure that $CTDB is set.
: ${CTDB:=ctdb}
git.samba.org / bbaumbach / samba-autobuild / .git / blobdiff