1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 . "${TEST_SCRIPTS_DIR}/common.sh"
5 # If we're not running on a real cluster then we need a local copy of
6 # ctdb (and other stuff) in $PATH and we will use local daemons.
7 if [ -n "$TEST_LOCAL_DAEMONS" ] ; then
8 export CTDB_NODES_SOCKETS=""
9 for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
10 CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${TEST_VAR_DIR}/sock.${i}"
13 # Use in-tree binaries if running against local daemons.
14 # Otherwise CTDB need to be installed on all nodes.
15 if [ -n "$ctdb_dir" -a -d "${ctdb_dir}/bin" ] ; then
16 PATH="${ctdb_dir}/bin:${PATH}"
17 export CTDB_LOCK_HELPER="${ctdb_dir}/bin/ctdb_lock_helper"
20 export CTDB_NODES="${TEST_VAR_DIR}/nodes.txt"
23 ######################################################################
25 export CTDB_TIMEOUT=60
27 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then
28 CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap"
30 _d=$(cd ${TEST_SCRIPTS_DIR}; echo $PWD)
31 CTDB_TEST_WRAPPER="$_d/test_wrap"
33 export CTDB_TEST_WRAPPER
35 # If $VALGRIND is set then use it whenever ctdb is called, but only if
36 # $CTDB is not already set.
37 [ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb"
40 PATH="${TEST_SCRIPTS_DIR}:${PATH}"
42 ######################################################################
44 ctdb_check_time_logs ()
53 out=$(onnode all tail -n 20 "${TEST_VAR_DIR}/ctdb.test.time.log" 2>&1)
55 if [ $? -eq 0 ] ; then
60 node="${line#>> NODE: }"
67 if [ -n "$ds_prev" ] && \
68 [ $(($ds_curr - $ds_prev)) -ge $threshold ] ; then
69 echo "Node $node had time jump of $(($ds_curr - $ds_prev))ds between $(date +'%T' -d @${ds_prev%?}) and $(date +'%T' -d @${ds_curr%?})"
78 echo Error getting time logs
81 echo "Check time sync (test client first):"
84 echo "Information from test client:"
87 echo "Information from cluster nodes:"
88 onnode all "top -b -n 1 ; echo '/proc/slabinfo' ; cat /proc/slabinfo"
98 [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
99 status=$(($testfailures+0))
101 # Avoid making a test fail from this point onwards. The test is
105 echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..."
107 if [ -z "$TEST_LOCAL_DAEMONS" -a -n "$CTDB_TEST_TIME_LOGGING" -a \
108 $status -ne 0 ] ; then
112 eval "$ctdb_test_exit_hook" || true
113 unset ctdb_test_exit_hook
115 if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then
119 # This could be made unconditional but then we might get
120 # duplication from the recovery in restart_ctdb. We want to
121 # leave the recovery in restart_ctdb so that future tests that
122 # might do a manual restart mid-test will benefit.
123 echo "Forcing a recovery..."
124 onnode 0 $CTDB recover
130 ctdb_test_exit_hook_add ()
132 ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*"
137 scriptname=$(basename "$0")
139 ctdb_test_restart_scheduled=false
141 trap "ctdb_test_exit" 0
144 ########################################
147 try_command_on_node ()
149 local nodespec="$1" ; shift
154 while [ "${nodespec#-}" != "$nodespec" ] ; do
155 if [ "$nodespec" = "-v" ] ; then
158 onnode_opts="$nodespec"
160 nodespec="$1" ; shift
165 out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
167 echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
173 echo "Output of \"$cmd\":"
178 sanity_check_output ()
181 local regexp="$2" # Should be anchored as necessary.
186 local num_lines=$(echo "$output" | wc -l)
187 echo "There are $num_lines lines of output"
188 if [ $num_lines -lt $min_lines ] ; then
189 echo "BAD: that's less than the required number (${min_lines})"
194 local unexpected # local doesn't pass through status of command on RHS.
195 unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
197 # Note that this is reversed.
198 if [ $status -eq 0 ] ; then
199 echo "BAD: unexpected lines in output:"
200 echo "$unexpected" | cat -A
203 echo "Output lines look OK"
211 local ips="$1" # list of "ip node" lines
213 echo "Sanity checking IPs..."
217 while read x ipp ; do
218 [ "$ipp" = "-1" ] && break
219 if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
226 echo "BAD: a node was -1 or IPs are only assigned to one node"
227 echo "Are you running an old version of CTDB?"
231 # This returns a list of "ip node" lines in $out
235 try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'"
238 _select_test_node_and_ips ()
242 test_node="" # this matches no PNN
245 while read ip pnn ; do
246 if [ -z "$test_node" -a "$pnn" != "-1" ] ; then
249 if [ "$pnn" = "$test_node" ] ; then
250 test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
252 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
254 echo "Selected node ${test_node} with IPs: ${test_node_ips}."
255 test_ip="${test_node_ips%% *}"
257 [ -n "$test_node" ] || return 1
260 select_test_node_and_ips ()
263 while ! _select_test_node_and_ips ; do
264 echo "Unable to find a test node with IPs assigned"
265 if [ $timeout -le 0 ] ; then
266 echo "BAD: Too many attempts"
270 timeout=$(($timeout - 1))
276 #######################################
278 # Wait until either timeout expires or command succeeds. The command
279 # will be tried once per second.
282 local timeout="$1" ; shift # "$@" is the command...
285 if [ "$1" = "!" ] ; then
290 echo -n "<${timeout}|"
292 while [ $t -gt 0 ] ; do
295 if { ! $negate && [ $rc -eq 0 ] ; } || \
296 { $negate && [ $rc -ne 0 ] ; } ; then
297 echo "|$(($timeout - $t))|"
314 for i in $(seq 1 $1) ; do
321 _cluster_is_healthy ()
323 $CTDB nodestatus all >/dev/null && \
324 node_has_status 0 recovered
327 cluster_is_healthy ()
329 if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
330 echo "Cluster is HEALTHY"
333 echo "Cluster is UNHEALTHY"
334 if ! ${ctdb_test_restart_scheduled:-false} ; then
335 echo "DEBUG AT $(date '+%F %T'):"
337 for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do
346 wait_until_healthy ()
348 local timeout="${1:-120}"
350 echo "Waiting for cluster to become healthy..."
352 wait_until 120 _cluster_is_healthy
355 # This function is becoming nicely overloaded. Soon it will collapse! :-)
361 local bits fpat mpat rpat
363 (unhealthy) bits="?:?:?:1:*" ;;
364 (healthy) bits="?:?:?:0:*" ;;
365 (disconnected) bits="1:*" ;;
366 (connected) bits="0:*" ;;
367 (banned) bits="?:1:*" ;;
368 (unbanned) bits="?:0:*" ;;
369 (disabled) bits="?:?:1:*" ;;
370 (enabled) bits="?:?:0:*" ;;
371 (stopped) bits="?:?:?:?:1:*" ;;
372 (notstopped) bits="?:?:?:?:0:*" ;;
373 (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;;
374 (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
375 (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
376 (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;;
377 (recovered) rpat='^Recovery mode:NORMAL \(0\)$' ;;
379 echo "node_has_status: unknown status \"$status\""
383 if [ -n "$bits" ] ; then
386 out=$($CTDB -Y status 2>&1) || return 1
391 # This needs to be done in 2 steps to avoid false matches.
392 local line_bits="${line#:${pnn}:*:}"
393 [ "$line_bits" = "$line" ] && continue
394 [ "${line_bits#${bits}}" != "$line_bits" ] && return 0
397 } <<<"$out" # Yay bash!
398 elif [ -n "$fpat" ] ; then
399 $CTDB statistics -n "$pnn" | egrep -q "$fpat"
400 elif [ -n "$mpat" ] ; then
401 $CTDB getmonmode -n "$pnn" | egrep -q "$mpat"
402 elif [ -n "$rpat" ] ; then
403 $CTDB status -n "$pnn" | egrep -q "$rpat"
405 echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
410 wait_until_node_has_status ()
414 local timeout="${3:-30}"
415 local proxy_pnn="${4:-any}"
417 echo "Waiting until node $pnn has status \"$status\"..."
419 if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then
420 for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do
430 # Useful for superficially testing IP failover.
431 # IPs must be on nodes matching nodeglob.
432 # If the first argument is '!' then the IPs must not be on nodes
434 ips_are_on_nodeglob ()
437 if [ "$1" = "!" ] ; then
438 negating=true ; shift
440 local nodeglob="$1" ; shift
447 for check in $ips ; do
448 while read ip pnn ; do
449 if [ "$check" = "$ip" ] ; then
451 ($nodeglob) if $negating ; then return 1 ; fi ;;
452 (*) if ! $negating ; then return 1 ; fi ;;
454 ips="${ips/${ip}}" # Remove from list
457 # If we're negating and we didn't see the address then it
458 # isn't hosted by anyone!
460 ips="${ips/${check}}"
462 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
465 ips="${ips// }" # Remove any spaces.
469 wait_until_ips_are_on_nodeglob ()
471 echo "Waiting for IPs to fail over..."
473 wait_until 60 ips_are_on_nodeglob "$@"
484 while read ip pnn ; do
485 if [ "$node" = "$pnn" ] ; then
488 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
493 wait_until_node_has_some_ips ()
495 echo "Waiting for node to have some IPs..."
497 wait_until 60 node_has_some_ips "$@"
504 ip addr show to "$_ip" | awk '$1 == "inet" { print $2 }'
507 #######################################
511 echo "Attempting to politely shutdown daemons..."
512 onnode 1 $CTDB shutdown -n all || true
514 echo "Sleeping for a while..."
517 local pat="ctdbd --socket=.* --nlist .* --nopublicipcheck"
518 if pgrep -f "$pat" >/dev/null ; then
519 echo "Killing remaining daemons..."
522 if pgrep -f "$pat" >/dev/null ; then
523 echo "Once more with feeling.."
528 rm -rf "${TEST_VAR_DIR}/test.db"
533 mkdir -p "${TEST_VAR_DIR}/test.db/persistent"
535 local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all"
536 local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt"
537 rm -f $CTDB_NODES $public_addresses_all $no_public_addresses
539 # If there are (strictly) greater than 2 nodes then we'll randomly
540 # choose a node to have no public addresses.
541 local no_public_ips=-1
542 [ $TEST_LOCAL_DAEMONS -gt 2 ] && no_public_ips=$(($RANDOM % $TEST_LOCAL_DAEMONS))
543 echo "$no_public_ips" >$no_public_addresses
545 # When running certain tests we add and remove eventscripts, so we
546 # need to be able to modify the events.d/ directory. Therefore,
547 # we use a temporary events.d/ directory under $TEST_VAR_DIR. We
548 # copy the actual test eventscript(s) in there from the original
549 # events.d/ directory that sits alongside $TEST_SCRIPT_DIR.
550 local top=$(dirname "$TEST_SCRIPTS_DIR")
551 local events_d="${top}/events.d"
552 mkdir -p "${TEST_VAR_DIR}/events.d"
553 cp -p "${events_d}/"* "${TEST_VAR_DIR}/events.d/"
556 for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do
557 if [ "${CTDB_USE_IPV6}x" != "x" ]; then
558 echo ::$i >>"$CTDB_NODES"
559 ip addr add ::$i/128 dev lo
561 echo 127.0.0.$i >>"$CTDB_NODES"
562 # 2 public addresses on most nodes, just to make things interesting.
563 if [ $(($i - 1)) -ne $no_public_ips ] ; then
564 echo "192.168.234.$i/24 lo" >>"$public_addresses_all"
565 echo "192.168.234.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >>"$public_addresses_all"
574 shift # "$@" gets passed to ctdbd
576 local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all"
577 local public_addresses_mine="${TEST_VAR_DIR}/public_addresses.${pnn}"
578 local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt"
580 local no_public_ips=-1
581 [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses
583 if [ "$no_public_ips" = $pnn ] ; then
584 echo "Node $no_public_ips will have no public IPs."
587 local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES")
588 local ctdb_options="--sloppy-start --reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --listen=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state"
590 if [ $pnn -eq $no_public_ips ] ; then
591 ctdb_options="$ctdb_options --public-addresses=/dev/null"
593 cp "$public_addresses_all" "$public_addresses_mine"
594 ctdb_options="$ctdb_options --public-addresses=$public_addresses_mine"
597 # We'll use "pkill -f" to kill the daemons with
598 # "--socket=.* --nlist .* --nopublicipcheck" as context.
599 $VALGRIND ctdbd --socket="${TEST_VAR_DIR}/sock.$pnn" $ctdb_options "$@" ||return 1
604 # "$@" gets passed to ctdbd
606 echo "Starting $TEST_LOCAL_DAEMONS ctdb daemons..."
608 for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
609 daemons_start_1 $i "$@"
612 if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then
613 ln -sf "${TEST_VAR_DIR}/sock.0" /tmp/ctdb.socket || return 1
617 #######################################
619 _ctdb_hack_options ()
621 local ctdb_options="$*"
623 case "$ctdb_options" in
624 *--start-as-stopped*)
625 export CTDB_START_AS_STOPPED="yes"
631 _ctdb_hack_options "$@"
633 if [ -e /etc/redhat-release ] ; then
636 /etc/init.d/ctdb restart
642 _ctdb_hack_options "$@"
644 /etc/init.d/ctdb start
649 if [ -n "$CTDB_NODES_SOCKETS" ] ; then
654 # Common things to do after starting one or more nodes.
657 onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1
659 echo "Setting RerecoveryTimeout to 1"
660 onnode -pq all "$CTDB setvar RerecoveryTimeout 1"
662 # In recent versions of CTDB, forcing a recovery like this blocks
663 # until the recovery is complete. Hopefully this will help the
664 # cluster to stabilise before a subsequent test.
665 echo "Forcing a recovery..."
666 onnode -q 0 $CTDB recover
672 # This assumes that ctdbd is not running on the given node.
676 shift # "$@" is passed to ctdbd start.
678 echo -n "Starting CTDB on node ${pnn}..."
680 if [ -n "$CTDB_NODES_SOCKETS" ] ; then
681 daemons_start_1 $pnn "$@"
683 onnode $pnn $CTDB_TEST_WRAPPER _ctdb_start "$@"
686 # If we're starting only 1 node then we're doing something weird.
687 ctdb_restart_when_done
692 # "$@" is passed to ctdbd start.
694 echo -n "Restarting CTDB"
695 if $ctdb_test_restart_scheduled ; then
696 echo -n " (scheduled)"
701 for i in $(seq 1 5) ; do
702 if [ -n "$CTDB_NODES_SOCKETS" ] ; then
706 onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@"
708 echo "Restart failed. Trying again in a few seconds..."
713 onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || {
714 echo "Cluster didn't become healthy. Restarting..."
718 echo "Setting RerecoveryTimeout to 1"
719 onnode -pq all "$CTDB setvar RerecoveryTimeout 1"
721 # In recent versions of CTDB, forcing a recovery like this
722 # blocks until the recovery is complete. Hopefully this will
723 # help the cluster to stabilise before a subsequent test.
724 echo "Forcing a recovery..."
725 onnode -q 0 $CTDB recover
728 # Cluster is still healthy. Good, we're done!
729 if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
730 echo "Cluster became UNHEALTHY again [$(date)]"
731 onnode -p all ctdb status -Y 2>&1
732 onnode -p all ctdb scriptstatus 2>&1
737 echo "Doing a sync..."
738 onnode -q 0 $CTDB sync
744 echo "Cluster UNHEALTHY... too many attempts..."
745 onnode -p all ctdb status -Y 2>&1
746 onnode -p all ctdb scriptstatus 2>&1
748 # Try to make the calling test fail
753 ctdb_restart_when_done ()
755 ctdb_test_restart_scheduled=true
758 get_ctdbd_command_line_option ()
763 try_command_on_node "$pnn" "$CTDB getpid" || \
764 die "Unable to get PID of ctdbd on node $pnn"
766 local pid="${out#*:}"
767 try_command_on_node "$pnn" "ps -p $pid -o args hww" || \
768 die "Unable to get command-line of PID $pid"
770 # Strip everything up to and including --option
771 local t="${out#*--${option}}"
772 # Strip leading '=' or space if present
775 # Strip any following options and print
779 #######################################
781 install_eventscript ()
783 local script_name="$1"
784 local script_contents="$2"
786 if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
787 # The quoting here is *very* fragile. However, we do
788 # experience the joy of installing a short script using
789 # onnode, and without needing to know the IP addresses of the
791 onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\""
793 f="${TEST_VAR_DIR}/events.d/${script_name}"
794 echo "$script_contents" >"$f"
799 uninstall_eventscript ()
801 local script_name="$1"
803 if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
804 onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\""
806 rm -vf "${TEST_VAR_DIR}/events.d/${script_name}"
810 #######################################
812 # This section deals with the 99.ctdb_test eventscript.
814 # Metafunctions: Handle a ctdb-test file on a node.
816 ctdb_test_eventscript_file_create ()
821 try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}"
824 ctdb_test_eventscript_file_remove ()
829 try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}"
832 ctdb_test_eventscript_file_exists ()
837 try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1
841 # Handle a flag file on a node that is removed by 99.ctdb_test on the
843 ctdb_test_eventscript_flag ()
849 ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}"
853 # Handle a trigger that causes 99.ctdb_test to fail it's monitor
855 ctdb_test_eventscript_unhealthy_trigger ()
860 ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger"
863 # Handle the file that 99.ctdb_test created to show that it has marked
864 # a node unhealthy because it detected the above trigger.
865 ctdb_test_eventscript_unhealthy_detected ()
870 ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected"
873 # Handle a trigger that causes 99.ctdb_test to timeout it's monitor
874 # event. This should cause the node to be banned.
875 ctdb_test_eventscript_timeout_trigger ()
881 ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout"
884 # Note that the eventscript can't use the above functions!
885 ctdb_test_eventscript_install ()
888 local script='#!/bin/sh
892 rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}"
894 trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}"
895 detected="/tmp/ctdb-test-unhealthy-detected.${pnn}"
896 timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}"
899 if [ -e "$trigger" ] ; then
900 echo "${0}: Unhealthy because \"$trigger\" detected"
903 elif [ -e "$detected" -a ! -e "$trigger" ] ; then
904 echo "${0}: Healthy again, \"$trigger\" no longer detected"
910 if [ -e "$timeout_trigger" ] ; then
911 echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected"
921 install_eventscript "99.ctdb_test" "$script"
924 ctdb_test_eventscript_uninstall ()
926 uninstall_eventscript "99.ctdb_test"
929 # Note that this only works if you know all other monitor events will
930 # succeed. You also need to install the eventscript before using it.
931 wait_for_monitor_event ()
935 echo "Waiting for a monitor event on node ${pnn}..."
936 ctdb_test_eventscript_flag create $pnn "monitor"
938 wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor"
942 #######################################
946 select_test_node_and_ips
948 nfs_first_export=$(showmount -e $test_ip | sed -n -e '2s/ .*//p')
950 echo "Creating test subdirectory..."
951 try_command_on_node $test_node "mktemp -d --tmpdir=$nfs_first_export"
953 try_command_on_node $test_node "chmod 777 $nfs_test_dir"
955 nfs_mnt_d=$(mktemp -d)
956 nfs_local_file="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE"
957 nfs_remote_file="${nfs_test_dir}/TEST_FILE"
959 ctdb_test_exit_hook_add nfs_test_cleanup
961 echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..."
962 mount -o timeo=1,hard,intr,vers=3 \
963 ${test_ip}:${nfs_first_export} ${nfs_mnt_d}
968 rm -f "$nfs_local_file"
969 umount -f "$nfs_mnt_d"
971 onnode -q $test_node rmdir "$nfs_test_dir"
976 #######################################
978 # Make sure that $CTDB is set.
981 local="${TEST_SUBDIR}/scripts/local.bash"
982 if [ -r "$local" ] ; then