# utility functions for ctdb event scripts
+PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
+
#######################################
# pull in a system config file, if any
loadconfig() {
- name="$1"
- if [ -f /etc/sysconfig/$name ]; then
- . /etc/sysconfig/$name
- elif [ -f /etc/default/$name ]; then
- . /etc/default/$name
- elif [ -f $CTDB_BASE/sysconfig/$name ]; then
- . $CTDB_BASE/sysconfig/$name
+
+ if [ "$1" != "ctdb" ] ; then
+ loadconfig "ctdb"
+ fi
+
+ if [ -z "$1" ] ; then
+ foo="${service_config:-${service_name}}"
+ if [ -n "$foo" ] ; then
+ loadconfig "$foo"
+ fi
+ fi
+
+ if [ -f /etc/sysconfig/$1 ]; then
+ . /etc/sysconfig/$1
+ elif [ -f /etc/default/$1 ]; then
+ . /etc/default/$1
+ elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
+ . $CTDB_BASE/sysconfig/$1
fi
}
+##############################################################
+# determine on what type of system (init style) we are running
+detect_init_style() {
+ # only do detection if not already set:
+ test "x$CTDB_INIT_STYLE" != "x" && return
+
+ if [ -x /sbin/startproc ]; then
+ CTDB_INIT_STYLE="suse"
+ elif [ -x /sbin/start-stop-daemon ]; then
+ CTDB_INIT_STYLE="debian"
+ else
+ CTDB_INIT_STYLE="redhat"
+ fi
+}
######################################################
# simulate /sbin/service on platforms that don't have it
service() {
- service_name="$1"
- op="$2"
+ _service_name="$1"
+ _op="$2"
+
+ # do nothing, when no service was specified
+ [ -z "$_service_name" ] && return
+
if [ -x /sbin/service ]; then
- /sbin/service "$service_name" "$op"
- elif [ -x /etc/init.d/$service_name ]; then
- /etc/init.d/$service_name "$op"
- elif [ -x /etc/rc.d/init.d/$service_name ]; then
- /etc/rc.d/init.d/$service_name "$op"
+ /sbin/service "$_service_name" "$_op"
+ elif [ -x /etc/init.d/$_service_name ]; then
+ /etc/init.d/$_service_name "$_op"
+ elif [ -x /etc/rc.d/init.d/$_service_name ]; then
+ /etc/rc.d/init.d/$_service_name "$_op"
fi
}
######################################################
# simulate /sbin/service (niced) on platforms that don't have it
nice_service() {
- service_name="$1"
- op="$2"
- if [ -x /sbin/service ]; then
- nice /sbin/service "$service_name" "$op"
- elif [ -x /etc/init.d/$service_name ]; then
- nice /etc/init.d/$service_name "$op"
- elif [ -x /etc/rc.d/init.d/$service_name ]; then
- nice /etc/rc.d/init.d/$service_name "$op"
- fi
+ # do nothing, when no service was specified
+ [ -z "$1" ] && return
+
+ nice service "$@"
}
######################################################
(netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
else
echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
- return
+ return 127
fi
done
[ $all_ok -eq 1 ] || sleep 1
ctdb status > /dev/null 2>&1 || {
echo "ctdb daemon has died. Exiting tcp wait $service_name"
- exit 1
+ return 1
}
done
echo "Local tcp services for $service_name are up"
}
-
-######################################################
-# wait for a set of directories
-# usage: ctdb_wait_directories SERVICE_NAME <directories...>
-######################################################
-ctdb_wait_directories() {
- service_name="$1"
- shift
- wait_dirs="$*"
- [ -z "$wait_dirs" ] && return;
- all_ok=0
- echo "Waiting for local directories for $service_name"
- while [ $all_ok -eq 0 ]; do
- all_ok=1
- for d in $wait_dirs; do
- [ -d $d ] || all_ok=0
- done
- [ $all_ok -eq 1 ] || sleep 1
- ctdb status > /dev/null 2>&1 || {
- echo "ctdb daemon has died. Exiting directory wait for $service_name"
- exit 1
- }
- done
- echo "Local directories for $service_name are available"
-}
-
-
######################################################
# check that a rpc server is registered with portmap
# and responding to requests
# usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
######################################################
ctdb_check_rpc() {
- service_name="$1"
+ progname="$1"
prognum="$2"
version="$3"
rpcinfo -u localhost $prognum $version > /dev/null || {
- echo "ERROR: $service_name not responding to rpc requests"
+ echo "ERROR: $progname not responding to rpc requests"
exit 1
}
}
######################################################
# check a set of directories is available
-# return 0 on a missing directory
+# return 1 on a missing directory
# usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
######################################################
ctdb_check_directories_probe() {
- service_name="$1"
- shift
- wait_dirs="$*"
- [ -z "$wait_dirs" ] && return;
- for d in $wait_dirs; do
- [ -d $d ] || return 1
- done
- return 0
+ while IFS="" read d ; do
+ case "$d" in
+ *%*)
+ continue
+ ;;
+ *)
+ [ -d "$d" ] || return 1
+ esac
+ done
}
######################################################
# usage: ctdb_check_directories SERVICE_NAME <directories...>
######################################################
ctdb_check_directories() {
- service_name="$1"
- shift
- wait_dirs="$*"
- ctdb_check_directories_probe "$service_name" $wait_dirs || {
- echo "ERROR: $service_name directory $d not available"
- exit 1
- }
+ n="${1:-${service_name}}"
+ ctdb_check_directories_probe || {
+ echo "ERROR: $n directory \"$d\" not available"
+ exit 1
+ }
}
######################################################
# check a set of tcp ports
-# usage: ctdb_check_tcp_ports SERVICE_NAME <ports...>
+# usage: ctdb_check_tcp_ports <ports...>
######################################################
ctdb_check_tcp_ports() {
- service_name="$1"
- shift
- wait_ports="$*"
- [ -z "$wait_ports" ] && return;
- for p in $wait_ports; do
- all_ok=1
- if [ -x /usr/bin/netcat ]; then
- /usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
- elif [ -x /usr/bin/nc ]; then
- /usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
- elif [ -x /usr/bin/netstat ]; then
- (netstat -a -n | egrep "0.0.0.0:$p .*LISTEN" > /dev/null ) || all_ok=0
- elif [ -x /bin/netstat ]; then
- (netstat -a -n | egrep "0.0.0.0:$p .*LISTEN" > /dev/null ) || all_ok=0
- fi
- [ $all_ok -eq 1 ] || {
- echo "ERROR: $service_name tcp port $p is not responding"
- exit 1
- }
- done
+
+ for p ; do
+ if ! netstat -a -t -n | grep -q "0\.0\.0\.0:$p .*LISTEN" ; then
+ if ! netstat -a -t -n | grep -q ":::$p .*LISTEN" ; then
+ echo "ERROR: $service_name tcp port $p is not responding"
+ return 1
+ fi
+ fi
+ done
+}
+
+######################################################
+# check a unix socket
+# usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
+######################################################
+ctdb_check_unix_socket() {
+ socket_path="$1"
+ [ -z "$socket_path" ] && return
+
+ if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
+ echo "ERROR: $service_name socket $socket_path not found"
+ return 1
+ fi
}
######################################################
echo "Killing TCP connection $srcip:$srcport $destip:$destport"
ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
case $destport in
- # we only do one-way killtcp for NFS and CIFS
- 139|445|2049) : ;;
+ # we only do one-way killtcp for CIFS
+ 139|445) : ;;
# for all others we do 2-way
*)
ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
echo "killed $_killcount TCP connections to released IP $_IP"
}
+##################################################################
+# kill off the local end for any TCP connections with the given IP
+##################################################################
+kill_tcp_connections_local_only() {
+ _IP="$1"
+ _failed=0
+
+ _killcount=0
+ connfile="$CTDB_BASE/state/connections.$_IP"
+ netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
+ netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
+
+ while read dest src; do
+ srcip=`echo $src | sed -e "s/:[^:]*$//"`
+ srcport=`echo $src | sed -e "s/^.*://"`
+ destip=`echo $dest | sed -e "s/:[^:]*$//"`
+ destport=`echo $dest | sed -e "s/^.*://"`
+ echo "Killing TCP connection $srcip:$srcport $destip:$destport"
+ ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
+ _killcount=`expr $_killcount + 1`
+ done < $connfile
+ /bin/rm -f $connfile
+
+ [ $_failed = 0 ] || {
+ echo "Failed to send killtcp control"
+ return;
+ }
+ [ $_killcount -gt 0 ] || {
+ return;
+ }
+ _count=0
+ while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
+ sleep 1
+ _count=`expr $_count + 1`
+ [ $_count -gt 3 ] && {
+ echo "Timed out killing tcp connections for IP $_IP"
+ return;
+ }
+ done
+ echo "killed $_killcount TCP connections to released IP $_IP"
+}
+
########################################################
# start/stop the nfs service on different platforms
########################################################
esac
}
+########################################################
+# remove an ip address from an interface
+########################################################
+remove_ip() {
+ # the ip tool will delete all secondary IPs if this is the primary.
+ # To work around this _very_ annoying behaviour we have to keep a
+ # record of the secondaries and re-add them afterwards. yuck
+ secondaries=""
+ if ip addr list dev $2 primary | grep -q "inet $1 " ; then
+ secondaries=`ip addr list dev $2 secondary | grep " inet " | awk '{print $2}'`
+ fi
+ ip addr del $1 dev $2 >/dev/null 2>/dev/null || failed=1
+ [ -z "$secondaries" ] || {
+ for i in $secondaries; do
+ if ip addr list dev $2 | grep -q "inet $i" ; then
+ echo "kept secondary $i on dev $2"
+ else
+ echo "re-adding secondary address $i to dev $2"
+ ip addr add $i dev $2 || failed=1
+ fi
+ done
+ }
+}
+
+########################################################
+# some simple logic for counting events - per eventscript
+# usage: ctdb_counter_init
+# ctdb_counter_incr
+# ctdb_check_counter_limit <limit>
+# ctdb_check_counter_limit succeeds when count >= <limit>
+########################################################
+_ctdb_counter_common () {
+ _counter_file="$ctdb_fail_dir/$service_name"
+ mkdir -p "${_counter_file%/*}" # dirname
+}
+ctdb_counter_init () {
+ _ctdb_counter_common
+
+ >"$_counter_file"
+}
+ctdb_counter_incr () {
+ _ctdb_counter_common
+
+ # unary counting!
+ echo -n 1 >> "$_counter_file"
+}
+ctdb_check_counter_limit () {
+ _ctdb_counter_common
+
+ _limit="${1:-${service_fail_limit}}"
+ _quiet="$2"
+
+ # unary counting!
+ _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
+ if [ $_size -ge $_limit ] ; then
+ echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
+ exit 1
+ elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
+ echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
+ fi
+}
+########################################################
+
+ctdb_spool_dir="/var/spool/ctdb"
+ctdb_status_dir="$ctdb_spool_dir/status"
+ctdb_fail_dir="$ctdb_spool_dir/failcount"
+ctdb_active_dir="$ctdb_spool_dir/active"
+
+log_status_cat ()
+{
+ echo "node is \"$1\", problem with \"${script_name}\": $(cat $2)"
+}
+
+ctdb_checkstatus ()
+{
+ if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
+ log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
+ return 1
+ elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
+ log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
+ return 2
+ else
+ return 0
+ fi
+}
+
+ctdb_setstatus ()
+{
+ d="$ctdb_status_dir/$script_name"
+ case "$1" in
+ unhealthy|banned)
+ mkdir -p "$d"
+ cat "$2" >"$d/$1"
+ ;;
+ *)
+ for i in "banned" "unhealthy" ; do
+ rm -f "$d/$i"
+ done
+ ;;
+ esac
+}
+
+ctdb_service_needs_reconfigure ()
+{
+ [ -e "$ctdb_status_dir/$service_name/reconfigure" ]
+}
+
+ctdb_service_set_reconfigure ()
+{
+ d="$ctdb_status_dir/$service_name"
+ mkdir -p "$d"
+ >"$d/reconfigure"
+}
+
+ctdb_service_unset_reconfigure ()
+{
+ rm -f "$ctdb_status_dir/$service_name/reconfigure"
+}
+
+ctdb_service_reconfigure ()
+{
+ if [ -n "$service_reconfigure" ] ; then
+ eval $service_reconfigure
+ else
+ service "$service_name" restart
+ fi
+ ctdb_service_unset_reconfigure
+ ctdb_counter_init
+}
+
+ctdb_compat_managed_service ()
+{
+ if [ "$1" = "yes" ] ; then
+ t="$t $2 "
+ fi
+}
+
+is_ctdb_managed_service ()
+{
+ t=" $CTDB_MANAGED_SERVICES "
+
+ ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
+ ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
+ ctdb_compat_managed_service "$CTDB_MANAGES_SCP" "scp"
+ ctdb_compat_managed_service "$CTDB_MANAGES_WINDBIND" "windbind"
+ ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
+ ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
+ ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
+ ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
+
+ # Returns 0 if "<space>$service_name<space>" appears in $t
+ [ "${t#* ${service_name} }" != "${t}" ]
+}
+
+ctdb_start_stop_service ()
+{
+ _active="$ctdb_active_dir/$service_name"
+
+ if is_ctdb_managed_service ; then
+ if ! [ -e "$_active" ] ; then
+ echo "Starting service $service_name"
+ ctdb_service_start || exit $?
+ mkdir -p "$ctdb_active_dir"
+ touch "$_active"
+ exit 0
+ fi
+ elif ! is_ctdb_managed_service ; then
+ if [ -e "$_active" ] ; then
+ echo "Stopping service $service_name"
+ ctdb_service_stop || exit $?
+ rm -f "$_active"
+ fi
+ exit 0
+ fi
+}
+
+ctdb_service_start ()
+{
+ if [ -n "$service_start" ] ; then
+ eval $service_start
+ else
+ service "$service_name" start
+ fi
+ ctdb_counter_init
+}
+
+ctdb_service_stop ()
+{
+ if [ -n "$service_stop" ] ; then
+ eval $service_stop
+ else
+ service "$service_name" stop
+ fi
+}
+
+ctdb_standard_event_handler ()
+{
+ case "$1" in
+ status)
+ ctdb_checkstatus
+ exit
+ ;;
+ setstatus)
+ ctdb_setstatus "$@"
+ exit
+ ;;
+ esac
+}
+
########################################################
# load a site local config file
########################################################
. $CTDB_BASE/rc.local
}
+[ -d $CTDB_BASE/rc.local.d ] && {
+ for i in $CTDB_BASE/rc.local.d/* ; do
+ [ -x "$i" ] && . "$i"
+ done
+}
+
+script_name="${0##*/}" # basename
+service_name="$script_name" # default is just the script name
+service_fail_limit=1