1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 # Augment PATH with relevant stubs/ directories. We do this by actually
4 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
5 # prepending that to $PATH in rc.local to avoid the PATH reset in
10 if [ -d "${TEST_SUBDIR}/stubs" ] ; then
11 EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs"
12 case "$EVENTSCRIPTS_PATH" in
14 *) EVENTSCRIPTS_PATH="${PWD}/${EVENTSCRIPTS_PATH}" ;;
18 export EVENTSCRIPTS_PATH
20 PATH="${EVENTSCRIPTS_PATH}:${PATH}"
22 export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts"
23 if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \
24 "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then
25 rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
27 mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR"
28 export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
30 export CTDB_LOGGING="file:${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb"
31 touch "${CTDB_LOGGING#file:}" || \
32 die "Unable to setup logging for \"$CTDB_LOGGING\""
34 if [ -d "${TEST_SUBDIR}/etc" ] ; then
35 cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR"
36 export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
38 die "Unable to setup \$CTDB_ETCDIR"
41 if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then
42 cp -prL "${TEST_SUBDIR}/etc-ctdb" "$EVENTSCRIPTS_TESTS_VAR_DIR"
43 export CTDB_BASE="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc-ctdb"
45 die "Unable to set \$CTDB_BASE"
49 if [ ! -d "${CTDB_BASE}/events.d" ] ; then
51 ERROR: Directory ${CTDB_BASE}/events.d does not exist.
53 That means that no eventscripts can be tested.
55 One possible explanation:
57 You have CTDB installed via RPMs (or similar), so the regular
58 CTDB_BASE directory is in /etc/ctdb/
62 You have done a regular "configure" and "make install" so the tests
63 are installed under /usr/local/.
65 If so, one possible hack to fix this is to create a symlink:
67 ln -s /etc/ctdb /usr/local/etc/ctdb
69 This is nasty but it works... :-)
74 ######################################################################
76 if "$TEST_VERBOSE" ; then
77 debug () { echo "$@" ; }
82 eventscripts_tests_cleanup_hooks=""
85 eventscripts_test_add_cleanup ()
87 eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*"
90 trap 'eval $eventscripts_tests_cleanup_hooks' 0
93 ######################################################################
95 # General setup fakery
99 debug "Setting up shares (3 existing shares)"
100 # Create 3 fake shares/exports.
101 export FAKE_SHARES=""
102 for i in $(seq 1 3) ; do
103 _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing"
105 FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}"
108 export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding"
109 mkdir -p "$FAKE_PROC_NET_BONDING"
110 rm -f "$FAKE_PROC_NET_BONDING"/*
112 export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down"
113 mkdir -p "$FAKE_ETHTOOL_LINK_DOWN"
114 rm -f "$FAKE_ETHTOOL_LINK_DOWN"/*
116 # This can only have 2 levels. We don't want to resort to usings
117 # something dangerous like "rm -r" setup time.
118 export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state"
119 mkdir -p "$FAKE_IP_STATE"
120 rm -f "$FAKE_IP_STATE"/*/*
121 rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true
122 rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true
125 export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
126 mkdir -p "${CTDB_DBDIR}/persistent"
128 export FAKE_TDBTOOL_SUPPORTS_CHECK="yes"
129 export FAKE_TDB_IS_OK
130 export FAKE_DATE_OUTPUT
132 export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
133 export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
139 debug "Marking TCP port \"${_i}\" as not listening"
140 FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
148 # Replace some shares with non-existent ones.
153 export MISSING_SHARES_TEXT=""
154 for _i in $FAKE_SHARES ; do
155 if [ $_n = "$1" ] ; then
157 _i="${_i%_existing}_missing"
158 debug "Replacing share $_n with missing share \"$_i\""
159 rmdir "$_i" 2>/dev/null || true
160 MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}")
162 _t="${_t}${_t:+ }${_i}"
168 # Setup some fake /proc/net/bonding files with just enough info for
171 # arg1 is interface name, arg2 is currently active slave (use "None"
172 # if none), arg3 is MII status ("up" or "down").
176 _slave="${2:-${_iface}_sl_0}"
178 _mii_subs="${4:-${_mii_s:-up}}"
179 echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s"
180 cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF
181 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
182 Currently Active Slave: $_slave
185 # Status of 1st pretend adapter
186 MII Status: $_mii_subs
187 # Status of 2nd pretend adapter
188 MII Status: $_mii_subs
192 ethtool_interfaces_down ()
195 echo "Marking interface $_i DOWN for ethtool"
196 touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
200 ethtool_interfaces_up ()
203 echo "Marking interface $_i UP for ethtool"
204 rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
210 echo "# ip rule show"
214 while read _p _x _i _x _t ; do
215 # Remove trailing colon after priority/preference.
217 # Only remove rules that match our priority/preference.
218 [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
220 echo "# ip route show table $_t"
221 ip route show table "$_t"
225 # Copied from 13.per_ip_routing for now... so this is lazy testing :-(
226 ipv4_host_addr_to_net ()
231 # Convert the host address to an unsigned long by splitting out
232 # the octets and doing the math.
234 for _o in $(export IFS="." ; echo $_host) ; do
235 _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
238 # Calculate the mask and apply it.
239 _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
240 _net_ul=$(( $_host_ul & $_mask_ul ))
242 # Now convert to a network address one byte at a time.
244 for _o in $(seq 1 4) ; do
245 _net="$(($_net_ul & 255))${_net:+.}${_net}"
246 _net_ul=$(($_net_ul >> 8))
249 echo "${_net}/${_maskbits}"
252 ######################################################################
256 # Evaluate an expression that probably calls functions or uses
257 # variables from the CTDB functions file. This is used for test
262 . "$CTDB_BASE/functions"
267 # Set output for ctdb command. Option 1st argument is return code.
270 _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out"
273 _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc"
274 echo "${1:-0}" >"$_rc"
276 eventscripts_test_add_cleanup "rm -f $_out $_rc"
279 # For now this creates the same public addresses each time. However,
280 # it could be made more flexible.
281 setup_public_addresses ()
283 if [ -f "$CTDB_PUBLIC_ADDRESSES" -a \
284 "${CTDB_PUBLIC_ADDRESSES%/*}" = "$EVENTSCRIPTS_TESTS_VAR_DIR" ] ; then
285 rm "$CTDB_PUBLIC_ADDRESSES"
288 export CTDB_PUBLIC_ADDRESSES=$(mktemp \
289 --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR" \
290 "public-addresses-XXXXXXXX")
292 echo "Setting up CTDB_PUBLIC_ADDRESSES=${CTDB_PUBLIC_ADDRESSES}"
293 cat >"$CTDB_PUBLIC_ADDRESSES" <<EOF
310 export FAKE_CTDB_NUMNODES="${1:-3}"
311 echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes"
313 export FAKE_CTDB_PNN="${2:-0}"
314 echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}"
316 setup_public_addresses
318 export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb"
320 export FAKE_CTDB_EXTRA_CONFIG="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-config.sh"
321 rm -f "$FAKE_CTDB_EXTRA_CONFIG"
323 export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down"
324 mkdir -p "$FAKE_CTDB_IFACES_DOWN"
325 rm -f "$FAKE_CTDB_IFACES_DOWN"/*
327 export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus"
328 mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
329 rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
331 export CTDB_PARTIALLY_ONLINE_INTERFACES
333 export FAKE_CTDB_TUNABLES_OK="MonitorInterval TDBMutexEnabled DatabaseHashSize"
334 export FAKE_CTDB_TUNABLES_OBSOLETE="EventScriptUnhealthyOnTimeout"
339 cat >"$FAKE_CTDB_EXTRA_CONFIG"
346 _swap_total="5857276"
348 if [ "$1" = "bad" ] ; then
353 _swap_free="$_swap_total"
358 export FAKE_PROC_MEMINFO="\
366 Active(anon): 1917580 kB
367 Inactive(anon): 523080 kB
368 Active(file): 504524 kB
369 Inactive(file): 496848 kB
372 SwapTotal: ${_swap_total} kB
373 SwapFree: ${_swap_free} kB
376 export FAKE_FREE_M="\
377 total used free shared buffers cached
378 Mem: 3848 3634 213 0 142 ${_mem_cached}
379 -/+ buffers/cache: 2379 ${_mem_free}
382 export CTDB_MONITOR_FREE_MEMORY
383 export CTDB_MONITOR_FREE_MEMORY_WARN
384 export CTDB_CHECK_SWAP_IS_NOT_USED
387 ctdb_get_interfaces ()
389 # The echo/subshell forces all the output onto 1 line.
390 echo $(ctdb ifaces -X | awk -F'|' 'FNR > 1 {print $2}')
393 ctdb_get_1_interface ()
395 _t=$(ctdb_get_interfaces)
399 # Print all public addresses as: interface IP maskbits
400 # Each line is suitable for passing to takeip/releaseip
401 ctdb_get_all_public_addresses ()
403 _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
404 while IFS="/$IFS" read _ip _maskbits _ifaces ; do
405 echo "$_ifaces $_ip $_maskbits"
409 # Print public addresses on this node as: interface IP maskbits
410 # Each line is suitable for passing to takeip/releaseip
411 ctdb_get_my_public_addresses ()
414 read _x # skip header line
416 while IFS="|" read _x _ip _x _iface _x ; do
417 [ -n "$_iface" ] || continue
418 while IFS="/$IFS" read _i _maskbits _x ; do
419 if [ "$_ip" = "$_i" ] ; then
420 echo $_iface $_ip $_maskbits
423 done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
428 # Prints the 1st public address as: interface IP maskbits
429 # This is suitable for passing to takeip/releaseip
430 ctdb_get_1_public_address ()
432 ctdb_get_my_public_addresses | { head -n 1 ; cat >/dev/null ; }
435 ctdb_not_implemented ()
437 export CTDB_NOT_IMPLEMENTED="$1"
438 ctdb_not_implemented="\
439 DEBUG: ctdb: command \"$1\" not implemented in stub"
442 ctdb_fake_scriptstatus ()
451 echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
454 ######################################################################
456 setup_ctdb_policy_routing ()
458 service_name="per_ip_routing"
460 export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing"
461 export CTDB_PER_IP_ROUTING_RULE_PREF=100
462 export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000
463 export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000
465 # Tests need to create and populate this file
466 rm -f "$CTDB_PER_IP_ROUTING_CONF"
469 # Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF.
470 # $1 is the number of assigned IPs to use (<num>, all), defaulting to
471 # 1. If $2 is "default" then a default route is also added.
472 create_policy_routing_config ()
475 _should_add_default="$2"
477 ctdb_get_my_public_addresses |
478 if [ "$_num_ips" = "all" ] ; then
481 { head -n "$_num_ips" ; cat >/dev/null ; }
483 while read _dev _ip _bits ; do
484 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
485 _gw="${_net%.*}.1" # a dumb, calculated default
489 if [ "$_should_add_default" = "default" ] ; then
490 echo "$_ip 0.0.0.0/0 $_gw"
492 done >"$CTDB_PER_IP_ROUTING_CONF"
495 # Check the routes against those that are expected. $1 is the number
496 # of assigned IPs to use (<num>, all), defaulting to 1. If $2 is
497 # "default" then expect default routes to have been added.
501 _should_add_default="$2"
506 ctdb_get_my_public_addresses |
507 if [ "$_num_ips" = "all" ] ; then
510 { head -n "$_num_ips" ; cat >/dev/null ; }
512 while read _dev _ip _bits ; do
513 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
514 _gw="${_net%.*}.1" # a dumb, calculated default
516 _policy_rules="${_policy_rules}
517 ${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip "
518 _policy_routes="${_policy_routes}
519 # ip route show table ctdb.$_ip
520 $_net dev $_dev scope link "
522 if [ "$_should_add_default" = "default" ] ; then
523 _policy_routes="${_policy_routes}
524 default via $_gw dev $_dev "
530 0: from all lookup local ${_policy_rules}
531 32766: from all lookup main
532 32767: from all lookup default ${_policy_routes}
535 simple_test_command dump_routes
539 ######################################################################
541 ctdb_catdb_format_pairs ()
545 while read _k _v ; do
546 _kn=$(echo -n "$_k" | wc -c)
547 _vn=$(echo -n "$_v" | wc -c)
549 key(${_kn}) = "${_k}"
552 data(${_vn}) = "${_v}"
555 _count=$(($_count + 1))
558 echo "Dumped ${_count} records"
561 check_ctdb_tdb_statd_state ()
563 ctdb_get_my_public_addresses |
564 while read _x _sip _x ; do
566 echo "statd-state@${_sip}@${_cip}" "$FAKE_DATE_OUTPUT"
569 ctdb_catdb_format_pairs | {
571 simple_test_command ctdb catdb ctdb.tdb
575 check_statd_callout_smnotify ()
577 _state_even=$(( $(date '+%s') / 2 * 2))
578 _state_odd=$(($_state_even + 1))
582 ctdb_get_my_public_addresses |
583 while read _x _sip _x ; do
586 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_even}
587 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_even}
588 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_odd}
589 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_odd}
594 simple_test_event "notify"
598 ######################################################################
602 debug "Setting up NAT gateway"
604 natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
605 mkdir -p "$natgw_config_dir"
607 # These will accumulate, 1 per test... but will be cleaned up at
609 export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
612 while read _ip _master _dev ; do
614 if [ "$_master" = "master" ] ; then
615 export FAKE_CTDB_NATGW_MASTER="$_ip"
617 done >"$CTDB_NATGW_NODES"
619 # Assume all of the nodes are on a /24 network and have IPv4
621 read _ip <"$CTDB_NATGW_NODES"
622 export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
624 # These are fixed. Probably don't use the same network for the
625 # private node IPs. To unset the default gateway just set it to
627 export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
628 export CTDB_NATGW_PUBLIC_IFACE="eth1"
629 export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
630 export CTDB_NATGW_SLAVE_ONLY=""
633 ok_natgw_master_ip_addr_show ()
635 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
637 # This is based on CTDB_NATGW_PUBLIC_IP
641 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
642 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
643 inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE}
644 valid_lft forever preferred_lft forever
648 ok_natgw_slave_ip_addr_show ()
650 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
652 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
653 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
657 ok_natgw_master_static_routes ()
662 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
663 # This is intentionally different to the code in 11.natgw ;-)
666 _net=$(echo "$_i" | sed -e 's|@.*||')
667 _gw=$(echo "$_i" | sed -e 's|.*@||')
671 _gw="$CTDB_NATGW_DEFAULT_GATEWAY"
674 [ -n "$_gw" ] || continue
675 _t="${_t}${_t:+${_nl}}"
676 _t="${_t}${_net} via ${_gw} dev ethXXX metric 10 "
681 ok_natgw_slave_static_routes ()
686 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
687 # This is intentionally different to the code in 11.natgw ;-)
688 _net=$(echo "$_i" | sed -e 's|@.*||')
690 # The interface for the private network isn't specified as
691 # part of the NATGW configuration and isn't part of the
692 # command to add the route. It is implicitly added by "ip
693 # route" but our stub doesn't do this and adds "ethXXX".
694 _t="${_t}${_t:+${_nl}}"
695 _t="${_t}${_net} via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX metric 10 "
700 ######################################################################
702 # Samba/winbind fakery
710 if [ "$1" != "down" ] ; then
712 debug "Marking Samba services as up, listening and managed by CTDB"
713 # Get into known state.
714 eventscript_call ctdb_service_managed
716 # All possible service names for all known distros.
717 for i in "smb" "nmb" "samba" ; do
718 service "$i" force-started
721 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
722 export CTDB_MANAGED_SERVICES="foo samba bar"
724 export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
725 export FAKE_WBINFO_FAIL="no"
727 # Some things in 50.samba are backgrounded and waited for. If
728 # we don't sleep at all then timeouts can happen. This avoids
730 export FAKE_SLEEP_FORCE=0.1
732 debug "Marking Samba services as down, not listening and not managed by CTDB"
733 # Get into known state.
734 eventscript_call ctdb_service_unmanaged
736 # All possible service names for all known distros.
737 for i in "smb" "nmb" "samba" ; do
738 service "$i" force-stopped
741 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
742 export CTDB_MANAGED_SERVICES="foo bar"
743 unset CTDB_MANAGES_SAMBA
745 export FAKE_TCP_LISTEN=""
746 export FAKE_WBINFO_FAIL="yes"
749 # This is ugly but if this file isn't removed before each test
750 # then configuration changes between tests don't stick.
751 rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache"
758 service_name="winbind"
760 if [ "$1" != "down" ] ; then
762 debug "Marking Winbind service as up and managed by CTDB"
763 # Get into known state.
764 eventscript_call ctdb_service_managed
766 service "winbind" force-started
768 export CTDB_MANAGED_SERVICES="foo winbind bar"
770 export FAKE_WBINFO_FAIL="no"
773 debug "Marking Winbind service as down and not managed by CTDB"
774 # Get into known state.
775 eventscript_call ctdb_service_unmanaged
777 service "winbind" force-stopped
779 export CTDB_MANAGED_SERVICES="foo bar"
780 unset CTDB_MANAGES_WINBIND
782 export FAKE_WBINFO_FAIL="yes"
788 debug "Making wbinfo commands fail"
789 FAKE_WBINFO_FAIL="yes"
792 ######################################################################
802 export FAKE_RPCINFO_SERVICES=""
804 export CTDB_NFS_SKIP_SHARE_CHECK="no"
806 export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
807 export CTDB_NFS_DUMP_STUCK_THREADS FAKE_RPC_THREAD_PIDS
809 # Reset the failcounts for nfs services.
810 eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
812 if [ "$1" != "down" ] ; then
813 debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB"
815 eventscript_call ctdb_service_managed
816 service "nfs" force-started # might not be enough
818 export CTDB_MANAGED_SERVICES="foo nfs bar"
820 rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status"
822 debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
824 eventscript_call ctdb_service_unmanaged
825 service "nfs" force-stopped # might not be enough
826 eventscript_call startstop_nfs stop
828 export CTDB_MANAGED_SERVICES="foo bar"
829 unset CTDB_MANAGES_NFS
836 export CTDB_NFS_SERVER_MODE="ganesha"
837 if [ "$1" != "down" ] ; then
838 export CTDB_MANAGES_NFS="yes"
841 # We do not support testing the Ganesha-nfsd-specific part of the
843 export CTDB_SKIP_GANESHA_NFSD_CHECK="yes"
844 export CTDB_NFS_SKIP_SHARE_CHECK="yes"
850 debug "Marking RPC service \"${_i}\" as unavailable"
851 FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
858 debug "Marking RPC service \"${_i}\" as available"
863 nlockmgr) _t="3:4" ;;
865 *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
868 FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
875 _etc="$CTDB_ETCDIR" # shortcut for readability
876 for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
877 if [ -r "$_c" ] ; then
885 # Set the required result for a particular RPC program having failed
886 # for a certain number of iterations. This is probably still a work
887 # in progress. Note that we could hook aggressively
888 # nfs_check_rpc_service() to try to implement this but we're better
889 # off testing nfs_check_rpc_service() using independent code... even
890 # if it is incomplete and hacky. So, if the 60.nfs eventscript
891 # changes and the tests start to fail then it may be due to this
892 # function being incomplete.
893 rpc_set_service_failure_response ()
896 # The number of failures defaults to the iteration number. This
897 # will be true when we fail from the 1st iteration... but we need
898 # the flexibility to set the number of failures.
899 _numfails="${2:-${iteration}}"
903 # A handy newline. :-)
910 _file=$(ls "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9]."${_progname}.check")
911 [ -r "$_file" ] || die "RPC check file \"$_file\" does not exist or is not unique"
913 while read _op _li _actions ; do
920 if [ "$_op" != "%" ] ; then
921 if [ $_numfails $_op $_li ] ; then
925 if [ $(($_numfails $_op $_li)) -eq 0 ] ; then
932 for _action in $_actions ; do
938 nfsd) _ver=3 ; _pn="nfs" ;;
939 lockd) _ver=4 ; _pn="nlockmgr" ;;
940 statd) _pn="status" ;;
943 ERROR: $_pn failed RPC check:
944 rpcinfo: RPC: Program not registered
945 program $_pn version $_ver is not available"
948 _p="rpc.${_progname}"
956 Trying to restart NFS service"
958 if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then
959 for _pid in $FAKE_NFSD_THREAD_PIDS ; do
962 ${_bg}Stack trace for nfsd[${_pid}]:
963 ${_bg}[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff"
969 ${_bg}Starting nfslock: OK
970 ${_bg}Starting nfs: OK"
974 Trying to restart lock manager service
975 ${_bg}Starting nfslock: OK"
978 _t="Trying to restart $_progname [${_p}]"
979 if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then
980 for _pid in $FAKE_RPC_THREAD_PIDS ; do
983 Stack trace for ${_p}[${_pid}]:
984 [<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff"
988 _out="${_out}${_out:+${_nl}}${_t}"
994 required_result $_rc "$_out"
1000 ######################################################################
1006 service_name="vsftpd"
1008 if [ "$1" != "down" ] ; then
1009 die "setup_vsftpd up not implemented!!!"
1011 debug "Setting up VSFTPD environment: service down, not managed by CTDB"
1013 eventscript_call ctdb_service_unmanaged
1014 service vsftpd force-stopped
1016 export CTDB_MANAGED_SERVICES="foo"
1017 unset CTDB_MANAGES_VSFTPD
1021 ######################################################################
1027 if [ "$1" != "down" ] ; then
1028 die "setup_httpd up not implemented!!!"
1030 debug "Setting up HTTPD environment: service down, not managed by CTDB"
1032 for service_name in "apache2" "httpd" ; do
1033 eventscript_call ctdb_service_unmanaged
1034 service "$service_name" force-stopped
1037 export CTDB_MANAGED_SERVICES="foo"
1038 unset CTDB_MANAGES_HTTPD
1042 ######################################################################
1052 echo "Marking ${_t} as having no active paths"
1053 FAKE_MULTIPATH_FAILURES="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}"
1058 CTDB_MONITOR_MPDEVICES="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}"
1061 export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES
1062 export FAKE_SLEEP_FORCE=0.1
1065 ######################################################################
1067 # Result and test functions
1069 # Set some globals and print the summary.
1074 _f=$(basename "$0" ".sh")
1076 # Remaining format should be NN.service.event.NNN or NN.service.NNN:
1084 script_dir="${CTDB_BASE}/events.d"
1089 script_dir="${CTDB_BASE}/events.d"
1094 script_dir="${CTDB_BASE}"
1099 script_dir="${CTDB_BASE}"
1102 [ -x "${script_dir}/${script}" ] || \
1103 die "Internal error - unable to find script \"${script_dir}/${script}\""
1105 printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
1111 CTDB_BASE="$CTDB_BASE"
1112 CTDB_ETCDIR="$CTDB_ETCDIR"
1113 ctdb client is "$(which ctdb)"
1114 ip command is "$(which ip)"
1118 # Run an eventscript once. The test passes if the return code and
1119 # output match those required.
1121 # Any args are passed to the eventscript.
1125 [ -n "$event" ] || die 'simple_test: $event not set'
1127 _extra_header=$(_extra_header)
1129 echo "Running script \"$script $event${1:+ }$*\""
1131 if $TEST_COMMAND_TRACE ; then
1136 _out=$($_shell "${script_dir}/${script}" "$event" "$@" 2>&1)
1138 result_check "$_extra_header"
1141 simple_test_event ()
1143 # If something has previously failed then don't continue.
1145 $_passed || return 1
1148 echo "=================================================="
1152 simple_test_command ()
1154 # If something has previously failed then don't continue.
1156 $_passed || return 1
1158 echo "=================================================="
1159 echo "Running command \"$*\""
1165 # Run an eventscript iteratively.
1166 # - 1st argument is the number of iterations.
1167 # - 2nd argument is something to eval to do setup for every iteration.
1168 # The easiest thing to do here is to define a function and pass it
1170 # - Subsequent arguments come in pairs: an iteration number and
1171 # something to eval for that iteration. Each time an iteration
1172 # number is matched the associated argument is given to eval after
1173 # the default setup is done. The iteration numbers need to be given
1174 # in ascending order.
1176 # Some optional args can be given *before* these, surrounded by extra
1177 # "--" args. These args are passed to the eventscript. Quoting is
1180 # One use of the 2nd and further arguments is to call
1181 # required_result() to change what is expected of a particular
1185 [ -n "$event" ] || die 'simple_test: $event not set'
1188 if [ "$1" = "--" ] ; then
1190 while [ "$1" != "--" ] ; do
1191 args="${args}${args:+ }$1"
1201 echo "Running $_repeats iterations of \"$script $event\" $args"
1205 for iteration in $(seq 1 $_repeats) ; do
1206 # This is inefficient because the iteration-specific setup
1207 # might completely replace the default one. However, running
1208 # the default is good because it allows you to revert to a
1209 # particular result without needing to specify it explicitly.
1210 eval $_setup_default
1211 if [ $iteration = "$1" ] ; then
1217 if $TEST_COMMAND_TRACE ; then
1222 _out=$($_shell "${script_dir}/${script}" "$event" $args 2>&1)
1225 _fout=$(echo "$_out" | result_filter)
1227 if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
1234 result_print "$_passed" "$_out" "$_rc" "Iteration $iteration"
1237 result_footer "$_result" "$(_extra_header)"