ctdb/tests/simple/scripts/local_daemons.bash

   1 # If we're not running on a real cluster then we need a local copy of
   2 # ctdb (and other stuff) in $PATH and we will use local daemons.
   3
   4 # Use in-tree binaries if running against local daemons.
   5 # Otherwise CTDB need to be installed on all nodes.
   6 if [ -n "$ctdb_dir" -a -d "${ctdb_dir}/bin" ] ; then
   7         # ctdbd_wrapper is in config/ directory
   8         PATH="${ctdb_dir}/bin:${ctdb_dir}/config:${PATH}"
   9         hdir="${ctdb_dir}/bin"
  10         export CTDB_EVENTD="${hdir}/ctdb_eventd"
  11         export CTDB_EVENT_HELPER="${hdir}/ctdb_event"
  12         export CTDB_LOCK_HELPER="${hdir}/ctdb_lock_helper"
  13         export CTDB_RECOVERY_HELPER="${hdir}/ctdb_recovery_helper"
  14         export CTDB_TAKEOVER_HELPER="${hdir}/ctdb_takeover_helper"
  15         export CTDB_CLUSTER_MUTEX_HELPER="${hdir}/ctdb_mutex_fcntl_helper"
  16 fi
  17
  18 if [ -n "$TEST_SOCKET_WRAPPER_SO_PATH" ] ; then
  19         export LD_PRELOAD="$TEST_SOCKET_WRAPPER_SO_PATH"
  20         export SOCKET_WRAPPER_DIR="${SIMPLE_TESTS_VAR_DIR}/sw"
  21         mkdir -p "$SOCKET_WRAPPER_DIR"
  22 fi
  23
  24 # onnode will execute this, which fakes ssh against local daemons
  25 export ONNODE_SSH="${TEST_SUBDIR}/scripts/ssh_local_daemons.sh"
  26
  27 #######################################
  28
  29 # If the given IP is hosted then print 2 items: maskbits and iface
  30 have_ip ()
  31 {
  32         local addr="$1"
  33         local bits t
  34
  35         case "$addr" in
  36         *:*) bits=128 ;;
  37         *)   bits=32  ;;
  38         esac
  39
  40         t=$(ip addr show to "${addr}/${bits}")
  41         [ -n "$t" ]
  42 }
  43
  44 node_dir ()
  45 {
  46         local pnn="$1"
  47
  48         echo "${SIMPLE_TESTS_VAR_DIR}/node.${pnn}"
  49 }
  50
  51 node_conf ()
  52 {
  53         local pnn="$1"
  54
  55         local node_dir=$(node_dir "$pnn")
  56         echo "${node_dir}/ctdbd.conf"
  57 }
  58
  59 node_pidfile ()
  60 {
  61         local pnn="$1"
  62
  63         local node_dir=$(node_dir "$pnn")
  64         echo "${node_dir}/ctdbd.pid"
  65 }
  66
  67 node_socket ()
  68 {
  69         local pnn="$1"
  70
  71         local node_dir=$(node_dir "$pnn")
  72         echo "${node_dir}/ctdbd.socket"
  73 }
  74
  75 setup_nodes ()
  76 {
  77         local have_all_ips=true
  78         local i
  79         for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do
  80                 if [ -n "$CTDB_USE_IPV6" ]; then
  81                         local j=$(printf "%02x" $i)
  82                         local node_ip="fd00::5357:5f${j}"
  83                         if have_ip "$node_ip" ; then
  84                                 echo "$node_ip"
  85                         else
  86                                 cat >&2 <<EOF
  87 ERROR: ${node_ip} not on an interface, please add it
  88 EOF
  89                                 have_all_ips=false
  90                         fi
  91                 else
  92                         local j=$((i + 10))
  93                         echo "127.0.0.${j}"
  94                 fi
  95         done
  96
  97         # Fail if we don't have all of the IPv6 addresses assigned
  98         $have_all_ips
  99 }
 100
 101 setup_public_addresses ()
 102 {
 103         local pnn_no_ips="$1"
 104
 105         local i
 106         for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do
 107                 if  [ $((i - 1)) -eq $pnn_no_ips ] ; then
 108                         continue
 109                 fi
 110
 111                 # 2 public addresses on most nodes, just to make
 112                 # things interesting
 113                 local j=$((i + TEST_LOCAL_DAEMONS))
 114                 if [ -n "$CTDB_USE_IPV6" ]; then
 115                         printf "fc00:10::1:%x/64 lo\n" "$i"
 116                         printf "fc00:10::1:%x/64 lo\n" "$j"
 117                 else
 118                         printf "192.168.234.%x/24 lo\n" "$i"
 119                         printf "192.168.234.%x/24 lo\n" "$j"
 120                 fi
 121         done
 122 }
 123
 124 setup_ctdb ()
 125 {
 126         local no_public_addresses=false
 127         local no_event_scripts=false
 128         case "$1" in
 129         --no-public-addresses) no_public_addresses=true ;;
 130         --no-event-scripts)    no_event_scripts=true    ;;
 131         esac
 132
 133         nodes_file="${SIMPLE_TESTS_VAR_DIR}/nodes"
 134         setup_nodes >"$nodes_file" || return 1
 135
 136         # If there are (strictly) greater than 2 nodes then we'll
 137         # randomly choose a node to have no public addresses
 138         local pnn_no_ips=-1
 139         if [ $TEST_LOCAL_DAEMONS -gt 2 ] ; then
 140                 pnn_no_ips=$((RANDOM % TEST_LOCAL_DAEMONS))
 141         fi
 142
 143         local public_addresses_all="${SIMPLE_TESTS_VAR_DIR}/public_addresses"
 144         setup_public_addresses $pnn_no_ips >"$public_addresses_all"
 145
 146         local pnn
 147         for pnn in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 148                 local node_dir=$(node_dir "$pnn")
 149
 150                 setup_ctdb_base "$SIMPLE_TESTS_VAR_DIR" "node.${pnn}" \
 151                                 functions
 152
 153                 if [ "$node_dir" != "$CTDB_BASE" ] ; then
 154                         die "Inconsistent CTDB_BASE"
 155                 fi
 156
 157                 cp "$nodes_file" "${CTDB_BASE}/nodes"
 158
 159                 local public_addresses="${node_dir}/public_addresses"
 160
 161                 if  $no_public_addresses || [ $pnn_no_ips -eq $pnn ] ; then
 162                         echo "Node ${pnn} will have no public IPs."
 163                         : >"$public_addresses"
 164                 else
 165                         cp "$public_addresses_all" "$public_addresses"
 166                 fi
 167
 168                 local node_ip=$(sed -n -e "$(($pnn + 1))p" "$nodes_file")
 169
 170                 local conf=$(node_conf "$pnn")
 171
 172                 local db_dir="${node_dir}/db"
 173                 mkdir -p "${db_dir}/persistent"
 174
 175                 if $no_event_scripts ; then
 176                         rm -vf "${CTDB_BASE}/events.d/"*
 177                 fi
 178
 179                 cat >"$conf" <<EOF
 180 CTDB_RECOVERY_LOCK="${SIMPLE_TESTS_VAR_DIR}/rec.lock"
 181 CTDB_NODE_ADDRESS="${node_ip}"
 182 CTDB_LOGGING="file:${node_dir}/log.ctdb"
 183 CTDB_DEBUGLEVEL=INFO
 184 CTDB_DBDIR="${db_dir}"
 185 CTDB_DBDIR_PERSISTENT="${db_dir}/persistent"
 186 CTDB_DBDIR_STATE="${db_dir}/state"
 187 CTDB_NOSETSCHED=yes
 188 EOF
 189         done
 190 }
 191
 192 start_ctdb_1 ()
 193 {
 194         local pnn="$1"
 195         local node_dir=$(node_dir "$pnn")
 196         local pidfile=$(node_pidfile "$pnn")
 197         local conf=$(node_conf "$pnn")
 198         local socket=$(node_socket "$pnn")
 199
 200         CTDBD="${VALGRIND} ctdbd --sloppy-start --nopublicipcheck" \
 201              CTDB_BASE="$node_dir" \
 202              CTDBD_CONF="$conf" \
 203              CTDB_PIDFILE="$pidfile" \
 204              CTDB_SOCKET="$socket" \
 205              ctdbd_wrapper start
 206 }
 207
 208 daemons_start ()
 209 {
 210     echo "Starting $TEST_LOCAL_DAEMONS ctdb daemons..."
 211
 212     local pnn
 213     for pnn in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 214         start_ctdb_1 "$pnn"
 215     done
 216 }
 217
 218 stop_ctdb_1 ()
 219 {
 220         local pnn="$1"
 221         local node_dir=$(node_dir "$pnn")
 222         local pidfile=$(node_pidfile "$pnn")
 223         local conf=$(node_conf "$pnn")
 224         local socket=$(node_socket "$pnn")
 225
 226         CTDB_BASE="$node_dir" \
 227                  CTDBD_CONF="$conf" \
 228                  CTDB_PIDFILE="$pidfile" \
 229                  CTDB_SOCKET="$socket" \
 230                  ctdbd_wrapper stop
 231 }
 232
 233 daemons_stop ()
 234 {
 235     echo "Stopping $TEST_LOCAL_DAEMONS ctdb daemons..."
 236
 237     local pnn
 238     for pnn in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 239         stop_ctdb_1 "$pnn"
 240     done
 241
 242     rm -rf "${SIMPLE_TESTS_VAR_DIR}/test.db"
 243 }
 244
 245 restart_ctdb_1 ()
 246 {
 247         stop_ctdb_1 "$1"
 248         start_ctdb_1 "$1"
 249 }
 250
 251 maybe_stop_ctdb ()
 252 {
 253     daemons_stop
 254 }
 255
 256 ctdb_stop_all ()
 257 {
 258         daemons_stop
 259 }
 260
 261 _ctdb_start_all ()
 262 {
 263         daemons_start
 264 }
 265
 266 ps_ctdbd ()
 267 {
 268         # If this fails to find processes then the tests fails, so
 269         # look at full command-line so this will work with valgrind.
 270         # Note that the output could be generated with pgrep's -a
 271         # option but it doesn't exist in older versions.
 272         ps -p $(pgrep -f '\<ctdbd\>' | xargs | sed -e 's| |,|g') -o args ww
 273         echo
 274 }
 275
 276 # onnode will use CTDB_BASES to help the ctdb tool connection to each
 277 # daemon
 278 export CTDB_BASES=""
 279 for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 280         node_dir=$(node_dir "$i")
 281         CTDB_BASES="${CTDB_BASES}${CTDB_BASES:+ }${node_dir}"
 282 done
 283
 284 # Need a default CTDB_BASE for onnode (to find the functions file).
 285 # Any node will do, so pick the 1st...
 286 export CTDB_BASE="${CTDB_BASES%% *}"