ctdb/tests/simple/scripts/local_daemons.bash

   1 # If we're not running on a real cluster then we need a local copy of
   2 # ctdb (and other stuff) in $PATH and we will use local daemons.
   3
   4 # Use in-tree binaries if running against local daemons.
   5 # Otherwise CTDB need to be installed on all nodes.
   6 if [ -n "$ctdb_dir" -a -d "${ctdb_dir}/bin" ] ; then
   7         # ctdbd_wrapper is in config/ directory
   8         PATH="${ctdb_dir}/bin:${ctdb_dir}/config:${PATH}"
   9         hdir="${ctdb_dir}/bin"
  10         export CTDB_EVENTD="${hdir}/ctdb_eventd"
  11         export CTDB_EVENT_HELPER="${hdir}/ctdb_event"
  12         export CTDB_LOCK_HELPER="${hdir}/ctdb_lock_helper"
  13         export CTDB_RECOVERY_HELPER="${hdir}/ctdb_recovery_helper"
  14         export CTDB_TAKEOVER_HELPER="${hdir}/ctdb_takeover_helper"
  15         export CTDB_CLUSTER_MUTEX_HELPER="${hdir}/ctdb_mutex_fcntl_helper"
  16 fi
  17
  18 export CTDB_NODES="${SIMPLE_TESTS_VAR_DIR}/nodes.txt"
  19
  20 if [ -n "$TEST_SOCKET_WRAPPER_SO_PATH" ] ; then
  21         export LD_PRELOAD="$TEST_SOCKET_WRAPPER_SO_PATH"
  22         export SOCKET_WRAPPER_DIR="${SIMPLE_TESTS_VAR_DIR}/sw"
  23         mkdir -p "$SOCKET_WRAPPER_DIR"
  24 fi
  25
  26 # onnode will execute this, which fakes ssh against local daemons
  27 export ONNODE_SSH="${TEST_SUBDIR}/scripts/ssh_local_daemons.sh"
  28
  29 #######################################
  30
  31 config_from_environment ()
  32 {
  33         # Override from the environment.  This would be easier if env was
  34         # guaranteed to quote its output so it could be reused.
  35         env |
  36         grep '^CTDB_' |
  37         sed -e 's@=\([^"]\)@="\1@' -e 's@[^"]$@&"@' -e 's@="$@&"@'
  38 }
  39
  40 # If the given IP is hosted then print 2 items: maskbits and iface
  41 have_ip ()
  42 {
  43         local addr="$1"
  44         local bits t
  45
  46         case "$addr" in
  47         *:*) bits=128 ;;
  48         *)   bits=32  ;;
  49         esac
  50
  51         t=$(ip addr show to "${addr}/${bits}")
  52         [ -n "$t" ]
  53 }
  54
  55 node_dir ()
  56 {
  57         local pnn="$1"
  58
  59         echo "${SIMPLE_TESTS_VAR_DIR}/node.${pnn}"
  60 }
  61
  62 node_conf ()
  63 {
  64         local pnn="$1"
  65
  66         local node_dir=$(node_dir "$pnn")
  67         echo "${node_dir}/ctdbd.conf"
  68 }
  69
  70 node_pidfile ()
  71 {
  72         local pnn="$1"
  73
  74         local node_dir=$(node_dir "$pnn")
  75         echo "${node_dir}/ctdbd.pid"
  76 }
  77
  78 node_socket ()
  79 {
  80         local pnn="$1"
  81
  82         local node_dir=$(node_dir "$pnn")
  83         echo "${node_dir}/ctdbd.socket"
  84 }
  85
  86 setup_nodes ()
  87 {
  88         local have_all_ips=true
  89         local i
  90         for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do
  91                 if [ -n "$CTDB_USE_IPV6" ]; then
  92                         local j=$(printf "%02x" $i)
  93                         local node_ip="fd00::5357:5f${j}"
  94                         if have_ip "$node_ip" ; then
  95                                 echo "$node_ip"
  96                         else
  97                                 cat >&2 <<EOF
  98 ERROR: ${node_ip} not on an interface, please add it
  99 EOF
 100                                 have_all_ips=false
 101                         fi
 102                 else
 103                         local j=$((i + 10))
 104                         echo "127.0.0.${j}"
 105                 fi
 106         done
 107
 108         # Fail if we don't have all of the IPv6 addresses assigned
 109         $have_all_ips
 110 }
 111
 112 setup_public_addresses ()
 113 {
 114         local pnn_no_ips="$1"
 115
 116         local i
 117         for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do
 118                 if  [ $((i - 1)) -eq $pnn_no_ips ] ; then
 119                         continue
 120                 fi
 121
 122                 # 2 public addresses on most nodes, just to make
 123                 # things interesting
 124                 local j=$((i + TEST_LOCAL_DAEMONS))
 125                 if [ -n "$CTDB_USE_IPV6" ]; then
 126                         printf "fc00:10::1:%x/64 lo\n" "$i"
 127                         printf "fc00:10::1:%x/64 lo\n" "$j"
 128                 else
 129                         printf "192.168.234.%x/24 lo\n" "$i"
 130                         printf "192.168.234.%x/24 lo\n" "$j"
 131                 fi
 132         done
 133 }
 134
 135 setup_ctdb ()
 136 {
 137         # When running certain tests we add and remove eventscripts, so we
 138         # need to be able to modify the events.d/ directory.  Therefore,
 139         # we use a temporary events.d/ directory under $TEST_VAR_DIR.  We
 140         # copy the actual test eventscript(s) in there from the original
 141         # events.d/ directory that sits alongside $TEST_SCRIPT_DIR.
 142         local top=$(dirname "$TEST_SCRIPTS_DIR")
 143         local events_d="${top}/events.d"
 144         mkdir -p "${TEST_VAR_DIR}/events.d"
 145         cp -p "${events_d}/"* "${TEST_VAR_DIR}/events.d/"
 146
 147         setup_nodes >"$CTDB_NODES" || return 1
 148
 149         # If there are (strictly) greater than 2 nodes then we'll
 150         # randomly choose a node to have no public addresses
 151         local pnn_no_ips=-1
 152         if [ $TEST_LOCAL_DAEMONS -gt 2 ] ; then
 153                 pnn_no_ips=$((RANDOM % TEST_LOCAL_DAEMONS))
 154         fi
 155
 156         local public_addresses_all="${SIMPLE_TESTS_VAR_DIR}/public_addresses"
 157         setup_public_addresses $pnn_no_ips >"$public_addresses_all"
 158
 159         local pnn
 160         for pnn in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 161                 local node_dir=$(node_dir "$pnn")
 162
 163                 setup_ctdb_base "$SIMPLE_TESTS_VAR_DIR" "node.${pnn}" \
 164                                 functions
 165
 166                 if [ "$node_dir" != "$CTDB_BASE" ] ; then
 167                         die "Inconsistent CTDB_BASE"
 168                 fi
 169
 170                 local public_addresses="${node_dir}/public_addresses"
 171
 172                 if  [ $pnn_no_ips -eq $pnn ] ; then
 173                         echo "Node ${pnn} will have no public IPs."
 174                         : >"$public_addresses"
 175                 else
 176                         cp "$public_addresses_all" "$public_addresses"
 177                 fi
 178
 179                 local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES")
 180
 181                 local conf=$(node_conf "$pnn")
 182                 local socket=$(node_socket "$pnn")
 183
 184                 local db_dir="${node_dir}/db"
 185                 mkdir -p "${db_dir}/persistent"
 186
 187                 cat >"$conf" <<EOF
 188 CTDB_RECOVERY_LOCK="${SIMPLE_TESTS_VAR_DIR}/rec.lock"
 189 CTDB_NODES="$CTDB_NODES"
 190 CTDB_NODE_ADDRESS="${node_ip}"
 191 CTDB_EVENT_SCRIPT_DIR="${TEST_VAR_DIR}/events.d"
 192 CTDB_LOGGING="file:${node_dir}/log.ctdb"
 193 CTDB_DEBUGLEVEL=INFO
 194 CTDB_DBDIR="${db_dir}"
 195 CTDB_DBDIR_PERSISTENT="${db_dir}/persistent"
 196 CTDB_DBDIR_STATE="${db_dir}/state"
 197 CTDB_PUBLIC_ADDRESSES="${public_addresses}"
 198 CTDB_SOCKET="${socket}"
 199 CTDB_NOSETSCHED=yes
 200 EOF
 201
 202                 # Append any configuration variables set in environment to
 203                 # configuration file so they affect CTDB after each restart.
 204                 config_from_environment >>"$conf"
 205         done
 206 }
 207
 208 start_ctdb_1 ()
 209 {
 210         local pnn="$1"
 211         local node_dir=$(node_dir "$pnn")
 212         local pidfile=$(node_pidfile "$pnn")
 213         local conf=$(node_conf "$pnn")
 214
 215         # If there is any CTDB configuration in the environment then
 216         # append it to the regular configuration in a temporary
 217         # configuration file and use it just this once.
 218         local tmp_conf=""
 219         local env_conf=$(config_from_environment)
 220         if [ -n "$env_conf" ] ; then
 221                 tmp_conf=$(mktemp --tmpdir="$SIMPLE_TESTS_VAR_DIR")
 222                 cat "$conf" >"$tmp_conf"
 223                 echo "$env_conf" >>"$tmp_conf"
 224                 conf="$tmp_conf"
 225         fi
 226
 227         CTDBD="${VALGRIND} ctdbd --sloppy-start --nopublicipcheck" \
 228              CTDB_BASE="$node_dir" \
 229              CTDBD_CONF="$conf" \
 230              CTDB_PIDFILE="$pidfile" \
 231              ctdbd_wrapper start
 232
 233         if [ -n "$tmp_conf" ] ; then
 234                 rm -f "$tmp_conf"
 235         fi
 236 }
 237
 238 daemons_start ()
 239 {
 240     echo "Starting $TEST_LOCAL_DAEMONS ctdb daemons..."
 241
 242     local pnn
 243     for pnn in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 244         start_ctdb_1 "$pnn"
 245     done
 246 }
 247
 248 stop_ctdb_1 ()
 249 {
 250         local pnn="$1"
 251         local node_dir=$(node_dir "$pnn")
 252         local pidfile=$(node_pidfile "$pnn")
 253         local conf=$(node_conf "$pnn")
 254
 255         CTDB_BASE="$node_dir" \
 256                  CTDBD_CONF="$conf" \
 257                  CTDB_PIDFILE="$pidfile" \
 258                  ctdbd_wrapper stop
 259 }
 260
 261 daemons_stop ()
 262 {
 263     echo "Stopping $TEST_LOCAL_DAEMONS ctdb daemons..."
 264
 265     local pnn
 266     for pnn in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 267         stop_ctdb_1 "$pnn"
 268     done
 269
 270     rm -rf "${SIMPLE_TESTS_VAR_DIR}/test.db"
 271 }
 272
 273 restart_ctdb_1 ()
 274 {
 275         stop_ctdb_1 "$1"
 276         start_ctdb_1 "$1"
 277 }
 278
 279 maybe_stop_ctdb ()
 280 {
 281     daemons_stop
 282 }
 283
 284 ctdb_stop_all ()
 285 {
 286         daemons_stop
 287 }
 288
 289 _ctdb_start_all ()
 290 {
 291         daemons_start
 292 }
 293
 294 ps_ctdbd ()
 295 {
 296         # If this fails to find processes then the tests fails, so
 297         # look at full command-line so this will work with valgrind.
 298         # Note that the output could be generated with pgrep's -a
 299         # option but it doesn't exist in older versions.
 300         ps -p $(pgrep -f '\<ctdbd\>' | xargs | sed -e 's| |,|g') -o args ww
 301         echo
 302 }
 303
 304 # onnode will use CTDB_NODES_SOCKETS to help the ctdb tool connection
 305 # to each daemon
 306 export CTDB_NODES_SOCKETS=""
 307 for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do
 308     socket=$(node_socket "$i")
 309     CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${socket}"
 310 done
 311
 312 # Need a default CTDB_BASE for onnode (to find the functions file).
 313 # Any node will do, so pick the 1st...
 314 export CTDB_BASE=$(node_dir 0)