1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 . "${TEST_SCRIPTS_DIR}/common.sh"
5 ######################################################################
9 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then
10 CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap"
12 _d=$(cd "$TEST_SCRIPTS_DIR" && echo "$PWD")
13 CTDB_TEST_WRAPPER="$_d/test_wrap"
15 export CTDB_TEST_WRAPPER
17 # If $VALGRIND is set then use it whenever ctdb is called, but only if
18 # $CTDB is not already set.
19 [ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb"
22 PATH="${TEST_SCRIPTS_DIR}:${PATH}"
24 ######################################################################
26 ctdb_test_on_cluster ()
28 [ -z "$CTDB_TEST_LOCAL_DAEMONS" ]
37 # run_tests.sh pipes stdout into tee. If the tee process is
38 # killed then any attempt to write to stdout (e.g. echo) will
39 # result in SIGPIPE, terminating the caller. Ignore SIGPIPE to
40 # ensure that all clean-up is run.
43 # Avoid making a test fail from this point onwards. The test is
47 echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..."
49 eval "$ctdb_test_exit_hook" || true
50 unset ctdb_test_exit_hook
52 echo "Stopping cluster..."
53 ctdb_nodes_stop || ctdb_test_error "Cluster shutdown failed"
58 ctdb_test_exit_hook_add ()
60 ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*"
63 # Setting cleanup_pid to <pid>@<node> will cause <pid> to be killed on
64 # <node> when the test completes. To cancel, just unset cleanup_pid.
65 ctdb_test_cleanup_pid=""
66 ctdb_test_cleanup_pid_exit_hook ()
68 if [ -n "$ctdb_test_cleanup_pid" ] ; then
69 local pid="${ctdb_test_cleanup_pid%@*}"
70 local node="${ctdb_test_cleanup_pid#*@}"
72 try_command_on_node "$node" "kill ${pid}"
76 ctdb_test_exit_hook_add ctdb_test_cleanup_pid_exit_hook
78 ctdb_test_cleanup_pid_set ()
83 ctdb_test_cleanup_pid="${pid}@${node}"
86 ctdb_test_cleanup_pid_clear ()
88 ctdb_test_cleanup_pid=""
91 # -n option means do not configure/start cluster
94 trap "ctdb_test_exit" 0
96 ctdb_nodes_stop >/dev/null 2>&1 || true
98 if [ "$1" != "-n" ] ; then
99 echo "Configuring cluster..."
100 setup_ctdb || ctdb_test_error "Cluster configuration failed"
102 echo "Starting cluster..."
103 ctdb_init || ctdb_test_error "Cluster startup failed"
106 echo "*** SETUP COMPLETE AT $(date '+%F %T'), RUNNING TEST..."
109 ctdb_nodes_start_custom ()
111 if ctdb_test_on_cluster ; then
112 ctdb_test_error "ctdb_nodes_start_custom() on real cluster"
115 ctdb_nodes_stop >/dev/null 2>&1 || true
117 echo "Configuring cluster..."
118 setup_ctdb "$@" || ctdb_test_error "Cluster configuration failed"
120 echo "Starting cluster..."
121 ctdb_init || ctdb_test_fail "Cluster startup failed"
124 ctdb_test_skip_on_cluster ()
126 if ctdb_test_on_cluster ; then
128 "SKIPPING this test - only runs against local daemons"
133 ctdb_nodes_restart ()
136 ctdb_nodes_start "$@"
139 ########################################
141 # Sets: $out, $outfile
142 # * The first 1KB of output is put into $out
143 # * Tests should use $outfile for handling large output
144 # * $outfile is removed after each test
146 outfile="${CTDB_TEST_TMP_DIR}/try_command_on_node.out"
153 ctdb_test_exit_hook_add outfile_cleanup
155 try_command_on_node ()
157 local nodespec="$1" ; shift
162 while [ "${nodespec#-}" != "$nodespec" ] ; do
163 if [ "$nodespec" = "-v" ] ; then
166 onnode_opts="${onnode_opts}${onnode_opts:+ }${nodespec}"
168 nodespec="$1" ; shift
174 # Intentionally unquoted - might be empty
175 # shellcheck disable=SC2086
176 onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 || status=$?
177 out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
179 if [ $status -ne 0 ] ; then
180 echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
186 echo "Output of \"$cmd\":"
187 cat "$outfile" || true
196 local options nodespec
201 options="${options}${options:+ }${1}"
211 # shellcheck disable=SC2086
212 # $options can be multi-word
213 try_command_on_node $options "$nodespec" "${thing} $*"
218 _run_onnode "$CTDB" "$@"
223 _run_onnode "${CTDB_TEST_WRAPPER} ${VALGRIND}" "$@"
228 _run_onnode "${CTDB_TEST_WRAPPER}" "$@"
231 sanity_check_output ()
234 local regexp="$2" # Should be anchored as necessary.
239 num_lines=$(wc -l <"$outfile" | tr -d '[:space:]')
240 echo "There are $num_lines lines of output"
241 if [ "$num_lines" -lt "$min_lines" ] ; then
242 ctdb_test_fail "BAD: that's less than the required number (${min_lines})"
246 local unexpected # local doesn't pass through status of command on RHS.
247 unexpected=$(grep -Ev "$regexp" "$outfile") || status=$?
249 # Note that this is reversed.
250 if [ $status -eq 0 ] ; then
251 echo "BAD: unexpected lines in output:"
252 echo "$unexpected" | cat -A
255 echo "Output lines look OK"
263 try_command_on_node any ctdb pnn || return 1
266 echo "Selected node ${test_node}"
269 # This returns a list of "ip node" lines in $outfile
273 try_command_on_node "$node" \
274 "$CTDB ip -X | awk -F'|' 'NR > 1 { print \$2, \$3 }'"
277 _select_test_node_and_ips ()
279 try_command_on_node any \
280 "$CTDB ip -X all | awk -F'|' 'NR > 1 { print \$2, \$3 }'"
282 test_node="" # this matches no PNN
285 while read -r ip pnn ; do
286 if [ -z "$test_node" ] && [ "$pnn" != "-1" ] ; then
289 if [ "$pnn" = "$test_node" ] ; then
290 test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
294 echo "Selected node ${test_node} with IPs: ${test_node_ips}."
295 test_ip="${test_node_ips%% *}"
297 # test_prefix used by caller
298 # shellcheck disable=SC2034
300 *:*) test_prefix="${test_ip}/128" ;;
301 *) test_prefix="${test_ip}/32" ;;
304 [ -n "$test_node" ] || return 1
307 select_test_node_and_ips ()
310 while ! _select_test_node_and_ips ; do
311 echo "Unable to find a test node with IPs assigned"
312 if [ $timeout -le 0 ] ; then
313 ctdb_test_error "BAD: Too many attempts"
317 timeout=$((timeout - 1))
324 get_test_ip_mask_and_iface ()
327 ctdb_onnode "$test_node" "ip -v -X"
328 iface=$(awk -F'|' -v ip="$test_ip" '$2 == ip { print $4 }' "$outfile")
330 if ctdb_test_on_cluster ; then
332 try_command_on_node "$test_node" ip addr show to "$test_ip"
339 echo "$test_ip/$mask is on $iface"
344 try_command_on_node -q all "$CTDB pnn"
348 # The subtlety is that "ctdb delip" will fail if the IP address isn't
349 # configured on a node...
350 delete_ip_from_all_nodes ()
358 for _pnn in $all_pnns ; do
359 all_ips_on_node "$_pnn"
360 while read -r _i _ ; do
361 if [ "$_ip" = "$_i" ] ; then
362 _nodes="${_nodes}${_nodes:+,}${_pnn}"
367 try_command_on_node -pq "$_nodes" "$CTDB delip $_ip"
370 #######################################
375 for i in $(seq 1 "$1") ; do
382 _cluster_is_healthy ()
384 $CTDB nodestatus all >/dev/null
387 _cluster_is_recovered ()
389 node_has_status 0 recovered
394 _cluster_is_healthy && _cluster_is_recovered
397 cluster_is_healthy ()
399 if onnode 0 "$CTDB_TEST_WRAPPER" _cluster_is_healthy ; then
400 echo "Cluster is HEALTHY"
401 if ! onnode 0 "$CTDB_TEST_WRAPPER" _cluster_is_recovered ; then
402 echo "WARNING: cluster in recovery mode!"
407 echo "Cluster is UNHEALTHY"
409 echo "DEBUG AT $(date '+%F %T'):"
411 for i in "onnode -q 0 $CTDB status" \
412 "onnode -q 0 onnode all $CTDB scriptstatus" ; do
422 local timeout="${1:-120}"
424 echo "Waiting for cluster to become ready..."
426 wait_until "$timeout" onnode -q any "$CTDB_TEST_WRAPPER" _cluster_is_ready
429 # This function is becoming nicely overloaded. Soon it will collapse! :-)
437 ! $CTDB status -n "$pnn" | \
438 grep -Eq '^Recovery mode:RECOVERY \(1\)$'
442 ! $CTDB status | grep -Eq "^hash:.* lmaster:${pnn}\$"
449 unhealthy) bits="?|?|?|1|*" ;;
450 healthy) bits="?|?|?|0|*" ;;
451 disconnected) bits="1|*" ;;
452 connected) bits="0|*" ;;
453 banned) bits="?|1|*" ;;
454 unbanned) bits="?|0|*" ;;
455 disabled) bits="?|?|1|*" ;;
456 enabled) bits="?|?|0|*" ;;
457 stopped) bits="?|?|?|?|1|*" ;;
458 notstopped) bits="?|?|?|?|0|*" ;;
460 echo "node_has_status: unknown status \"$status\""
465 out=$($CTDB -X status 2>&1) || return 1
469 while read -r line ; do
470 # This needs to be done in 2 steps to
471 # avoid false matches.
472 local line_bits="${line#|${pnn}|*|}"
473 [ "$line_bits" = "$line" ] && continue
474 [ "${line_bits#${bits}}" != "$line_bits" ] && \
478 } <<<"$out" # Yay bash!
481 wait_until_node_has_status ()
485 local timeout="${3:-30}"
486 local proxy_pnn="${4:-any}"
488 echo "Waiting until node $pnn has status \"$status\"..."
490 if ! wait_until "$timeout" onnode "$proxy_pnn" \
491 "$CTDB_TEST_WRAPPER" node_has_status "$pnn" "$status" ; then
493 for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do
503 # Useful for superficially testing IP failover.
504 # IPs must be on the given node.
505 # If the first argument is '!' then the IPs must not be on the given node.
509 if [ "$1" = "!" ] ; then
510 negating=true ; shift
512 local node="$1" ; shift
517 all_ips_on_node "$node"
520 for check in $ips ; do
522 while read -r ip pnn ; do
523 if [ "$check" = "$ip" ] ; then
524 if [ "$pnn" = "$node" ] ; then
525 if $negating ; then return 1 ; fi
527 if ! $negating ; then return 1 ; fi
529 ips="${ips/${ip}}" # Remove from list
532 # If we're negating and we didn't see the address then it
533 # isn't hosted by anyone!
535 ips="${ips/${check}}"
540 ips="${ips// }" # Remove any spaces.
544 wait_until_ips_are_on_node ()
546 # Go to some trouble to print a use description of what is happening
548 if [ "$1" == "!" ] ; then
555 [ "$i" != "!" ] || continue
556 if [ -z "$node" ] ; then
560 ips="${ips}${ips:+, }${i}"
562 echo "Waiting for ${ips} to ${not}be assigned to node ${node}"
564 wait_until 60 ips_are_on_node "$@"
573 all_ips_on_node "$node"
575 while read -r ip pnn ; do
576 if [ "$node" = "$pnn" ] ; then
584 wait_until_node_has_some_ips ()
586 echo "Waiting for some IPs to be assigned to node ${test_node}"
588 wait_until 60 node_has_some_ips "$@"
591 wait_until_node_has_no_ips ()
593 echo "Waiting until no IPs are assigned to node ${test_node}"
595 wait_until 60 ! node_has_some_ips "$@"
598 #######################################
602 if ! ctdb_nodes_start ; then
603 echo "Cluster start failed"
607 if ! wait_until_ready 120 ; then
608 echo "Cluster didn't become ready"
612 echo "Setting RerecoveryTimeout to 1"
613 onnode -pq all "$CTDB setvar RerecoveryTimeout 1"
615 echo "Forcing a recovery..."
616 onnode -q 0 "$CTDB recover"
619 if ! onnode -q all "$CTDB_TEST_WRAPPER _cluster_is_recovered" ; then
620 echo "Cluster has gone into recovery again, waiting..."
621 if ! wait_until 30/2 onnode -q all \
622 "$CTDB_TEST_WRAPPER _cluster_is_recovered" ; then
623 echo "Cluster did not come out of recovery"
628 if ! onnode 0 "$CTDB_TEST_WRAPPER _cluster_is_healthy" ; then
629 echo "Cluster became UNHEALTHY again [$(date)]"
633 echo "Doing a sync..."
634 onnode -q 0 "$CTDB sync"
642 echo "${CTDB_BASE:-${CTDB_SCRIPTS_BASE}}"
645 #######################################
647 wait_for_monitor_event ()
652 echo "Waiting for a monitor event on node ${pnn}..."
654 ctdb_onnode "$pnn" scriptstatus || {
655 echo "Unable to get scriptstatus from node $pnn"
659 mv "$outfile" "${outfile}.orig"
661 wait_until 120 _ctdb_scriptstatus_changed
664 _ctdb_scriptstatus_changed ()
666 ctdb_onnode "$pnn" scriptstatus || {
667 echo "Unable to get scriptstatus from node $pnn"
671 ! diff "$outfile" "${outfile}.orig" >/dev/null
674 #######################################
676 # If the given IP is hosted then print 2 items: maskbits and iface
682 *:*) _family="inet6" ; _bits=128 ;;
683 *) _family="inet" ; _bits=32 ;;
686 # Literal backslashes in awk script
687 # shellcheck disable=SC1004
688 ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
689 awk -v family="${_family}" \
690 'NR == 1 { iface = $2; sub(":$", "", iface) } \
691 $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
692 print mask, iface, family }'
697 _addr="${1%/*}" # Remove optional maskbits
699 # Intentional word splitting
700 # shellcheck disable=SC2046,SC2086
701 set -- $(ip_maskbits_iface $_addr)
702 if [ -n "$1" ] ; then
705 echo "Removing public address $_addr/$_maskbits from device $_iface"
706 ip addr del "$_ip/$_maskbits" dev "$_iface" >/dev/null 2>&1 || true
717 #######################################
719 # $1: pnn, $2: DB name
722 ctdb_onnode -v "$1" "getdbstatus $2" | sed -n -e "s@^path: @@p"
725 # $1: pnn, $2: DB name
726 db_ctdb_cattdb_count_records ()
728 # Count the number of keys, excluding any that begin with '_'.
729 # This excludes at least the sequence number record in
730 # persistent/replicated databases. The trailing "|| :" forces
731 # the command to succeed when no records are matched.
732 ctdb_onnode "$1" "cattdb $2 | grep -c '^key([0-9][0-9]*) = \"[^_]' || :"
736 # $1: pnn, $2: DB name, $3: key string, $4: value string, $5: RSN (default 7)
739 _tdb=$(db_get_path "$1" "$2")
741 ctdb_onnode "$1" tstore "$_tdb" "$3" "$4" "$_rsn"
744 # $1: pnn, $2: DB name, $3: dbseqnum (must be < 255!!!!!)
745 db_ctdb_tstore_dbseqnum ()
747 # "__db_sequence_number__" + trailing 0x00
748 _key='0x5f5f64625f73657175656e63655f6e756d6265725f5f00'
750 # Construct 8 byte (unit64_t) database sequence number. This
751 # probably breaks if $3 > 255
752 _value=$(printf "0x%02x%014x" "$3" 0)
754 db_ctdb_tstore "$1" "$2" "$_key" "$_value"
757 ########################################
759 # Make sure that $CTDB is set.
760 if [ -z "$CTDB" ] ; then
764 if ctdb_test_on_cluster ; then
765 . "${TEST_SCRIPTS_DIR}/integration_real_cluster.bash"
767 . "${TEST_SCRIPTS_DIR}/integration_local_daemons.bash"
771 local="${CTDB_TEST_SUITE_DIR}/scripts/local.bash"
772 if [ -r "$local" ] ; then