1 # utility functions for ctdb event scripts
3 PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
5 #######################################
6 # pull in a system config file, if any
10 foo="${service_config:-${service_name}}"
11 if [ -n "$foo" ] ; then
14 elif [ "$1" != "ctdb" ] ; then
19 if [ -f /etc/sysconfig/$1 ]; then
21 elif [ -f /etc/default/$1 ]; then
23 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
24 . $CTDB_BASE/sysconfig/$1
28 ##############################################################
29 # determine on what type of system (init style) we are running
31 # only do detection if not already set:
32 test "x$CTDB_INIT_STYLE" != "x" && return
34 if [ -x /sbin/startproc ]; then
35 CTDB_INIT_STYLE="suse"
36 elif [ -x /sbin/start-stop-daemon ]; then
37 CTDB_INIT_STYLE="debian"
39 CTDB_INIT_STYLE="redhat"
43 ######################################################
44 # simulate /sbin/service on platforms that don't have it
49 # do nothing, when no service was specified
50 [ -z "$_service_name" ] && return
52 if [ -x /sbin/service ]; then
53 /sbin/service "$_service_name" "$_op"
54 elif [ -x /etc/init.d/$_service_name ]; then
55 /etc/init.d/$_service_name "$_op"
56 elif [ -x /etc/rc.d/init.d/$_service_name ]; then
57 /etc/rc.d/init.d/$_service_name "$_op"
61 ######################################################
62 # simulate /sbin/service (niced) on platforms that don't have it
64 # do nothing, when no service was specified
70 ######################################################
71 # wait for a command to return a zero exit status
72 # usage: ctdb_wait_command SERVICE_NAME <command>
73 ######################################################
77 [ -z "$wait_cmd" ] && return;
79 echo "Waiting for service $service_name to start"
80 while [ $all_ok -eq 0 ]; do
81 $wait_cmd > /dev/null 2>&1 && all_ok=1
82 ctdb status > /dev/null 2>&1 || {
83 echo "ctdb daemon has died. Exiting wait for $service_name"
86 [ $all_ok -eq 1 ] || sleep 1
88 echo "Local service $service_name is up"
92 ######################################################
93 # wait for a set of tcp ports
94 # usage: ctdb_wait_tcp_ports SERVICE_NAME <ports...>
95 ######################################################
96 ctdb_wait_tcp_ports() {
100 [ -z "$wait_ports" ] && return;
102 echo "Waiting for tcp service $service_name to start"
103 while [ $all_ok -eq 0 ]; do
105 for p in $wait_ports; do
106 if [ -x /usr/bin/netcat ]; then
107 /usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
108 elif [ -x /usr/bin/nc ]; then
109 /usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
110 elif [ -x /usr/bin/netstat ]; then
111 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
112 elif [ -x /bin/netstat ]; then
113 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
115 echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
119 [ $all_ok -eq 1 ] || sleep 1
120 ctdb status > /dev/null 2>&1 || {
121 echo "ctdb daemon has died. Exiting tcp wait $service_name"
125 echo "Local tcp services for $service_name are up"
129 ######################################################
130 # check that a rpc server is registered with portmap
131 # and responding to requests
132 # usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
133 ######################################################
138 rpcinfo -u localhost $prognum $version > /dev/null || {
139 echo "ERROR: $progname not responding to rpc requests"
144 ######################################################
145 # check a set of directories is available
146 # return 1 on a missing directory
147 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
148 ######################################################
149 ctdb_check_directories_probe() {
150 while IFS="" read d ; do
156 [ -d "$d" ] || return 1
161 ######################################################
162 # check a set of directories is available
163 # usage: ctdb_check_directories SERVICE_NAME <directories...>
164 ######################################################
165 ctdb_check_directories() {
166 n="${1:-${service_name}}"
167 ctdb_check_directories_probe || {
168 echo "ERROR: $n directory \"$d\" not available"
173 ######################################################
174 # check a set of tcp ports
175 # usage: ctdb_check_tcp_ports <ports...>
176 ######################################################
177 ctdb_check_tcp_ports() {
180 if ! netstat -a -t -n | grep -q "0\.0\.0\.0:$p .*LISTEN" ; then
181 if ! netstat -a -t -n | grep -q ":::$p .*LISTEN" ; then
182 echo "ERROR: $service_name tcp port $p is not responding"
189 ######################################################
190 # check a unix socket
191 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
192 ######################################################
193 ctdb_check_unix_socket() {
195 [ -z "$socket_path" ] && return
197 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
198 echo "ERROR: $service_name socket $socket_path not found"
203 ######################################################
204 # check a command returns zero status
205 # usage: ctdb_check_command SERVICE_NAME <command>
206 ######################################################
207 ctdb_check_command() {
210 [ -z "$wait_cmd" ] && return;
211 $wait_cmd > /dev/null 2>&1 || {
212 echo "ERROR: $service_name - $wait_cmd returned error"
217 ################################################
218 # kill off any TCP connections with the given IP
219 ################################################
220 kill_tcp_connections() {
225 connfile="$CTDB_BASE/state/connections.$_IP"
226 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
227 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
229 while read dest src; do
230 srcip=`echo $src | sed -e "s/:[^:]*$//"`
231 srcport=`echo $src | sed -e "s/^.*://"`
232 destip=`echo $dest | sed -e "s/:[^:]*$//"`
233 destport=`echo $dest | sed -e "s/^.*://"`
234 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
235 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
237 # we only do one-way killtcp for CIFS
239 # for all others we do 2-way
241 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
244 _killcount=`expr $_killcount + 1`
248 [ $_failed = 0 ] || {
249 echo "Failed to send killtcp control"
252 [ $_killcount -gt 0 ] || {
256 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
258 _count=`expr $_count + 1`
259 [ $_count -gt 3 ] && {
260 echo "Timed out killing tcp connections for IP $_IP"
264 echo "killed $_killcount TCP connections to released IP $_IP"
267 ##################################################################
268 # kill off the local end for any TCP connections with the given IP
269 ##################################################################
270 kill_tcp_connections_local_only() {
275 connfile="$CTDB_BASE/state/connections.$_IP"
276 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
277 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
279 while read dest src; do
280 srcip=`echo $src | sed -e "s/:[^:]*$//"`
281 srcport=`echo $src | sed -e "s/^.*://"`
282 destip=`echo $dest | sed -e "s/:[^:]*$//"`
283 destport=`echo $dest | sed -e "s/^.*://"`
284 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
285 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
286 _killcount=`expr $_killcount + 1`
290 [ $_failed = 0 ] || {
291 echo "Failed to send killtcp control"
294 [ $_killcount -gt 0 ] || {
298 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
300 _count=`expr $_count + 1`
301 [ $_count -gt 3 ] && {
302 echo "Timed out killing tcp connections for IP $_IP"
306 echo "killed $_killcount TCP connections to released IP $_IP"
309 ##################################################################
310 # tickle any TCP connections with the given IP
311 ##################################################################
312 tickle_tcp_connections() {
317 connfile="$CTDB_BASE/state/connections.$_IP"
318 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
319 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
321 while read dest src; do
322 srcip=`echo $src | sed -e "s/:[^:]*$//"`
323 srcport=`echo $src | sed -e "s/^.*://"`
324 destip=`echo $dest | sed -e "s/:[^:]*$//"`
325 destport=`echo $dest | sed -e "s/^.*://"`
326 echo "Tickle TCP connection $srcip:$srcport $destip:$destport"
327 ctdb tickle $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
328 echo "Tickle TCP connection $destip:$destport $srcip:$srcport"
329 ctdb tickle $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
333 [ $_failed = 0 ] || {
334 echo "Failed to send tickle control"
339 ########################################################
340 # start/stop the nfs service on different platforms
341 ########################################################
344 [ -x /etc/init.d/nfsserver ] && {
347 [ -x /etc/init.d/nfslock ] && {
355 service nfsserver start
358 service nfsserver stop > /dev/null 2>&1
365 service nfslock start
369 service nfs stop > /dev/null 2>&1
370 service nfslock stop > /dev/null 2>&1
375 echo "Unknown platform. NFS is not supported with ctdb"
381 ########################################################
382 # start/stop the nfs lockmanager service on different platforms
383 ########################################################
384 startstop_nfslock() {
386 [ -x /etc/init.d/nfsserver ] && {
389 [ -x /etc/init.d/nfslock ] && {
395 # for sles there is no service for lockmanager
396 # so we instead just shutdown/restart nfs
399 service nfsserver start
402 service nfsserver stop > /dev/null 2>&1
409 service nfslock start
412 service nfslock stop > /dev/null 2>&1
417 echo "Unknown platform. NFS locking is not supported with ctdb"
423 # better use delete_ip_from_iface() together with add_ip_to_iface
425 # the ip tool will delete all secondary IPs if this is the primary.
426 # To work around this _very_ annoying behaviour we have to keep a
427 # record of the secondaries and re-add them afterwards. yuck
429 if ip addr list dev $2 primary | grep -q "inet $1 " ; then
430 secondaries=`ip addr list dev $2 secondary | grep " inet " | awk '{print $2}'`
432 ip addr del $1 dev $2 >/dev/null 2>/dev/null || failed=1
433 [ -z "$secondaries" ] || {
434 for i in $secondaries; do
435 if ip addr list dev $2 | grep -q "inet $i" ; then
436 echo "kept secondary $i on dev $2"
438 echo "re-adding secondary address $i to dev $2"
439 ip addr add $i dev $2 || failed=1
450 local _lockfile="$CTDB_BASE/state/interface_modify.$_iface.flock"
452 test -f $_lockfile || {
456 flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh add "$_iface" "$_ip" "$_maskbits"
460 delete_ip_from_iface()
465 local _lockfile="$CTDB_BASE/state/interface_modify.$_iface.flock"
467 test -f $_lockfile || {
471 flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh delete "$_iface" "$_ip" "$_maskbits"
475 ########################################################
476 # some simple logic for counting events - per eventscript
477 # usage: ctdb_counter_init
479 # ctdb_check_counter_limit <limit>
480 # ctdb_check_counter_limit succeeds when count >= <limit>
481 ########################################################
482 _ctdb_counter_common () {
483 _counter_file="$ctdb_fail_dir/$service_name"
484 mkdir -p "${_counter_file%/*}" # dirname
486 ctdb_counter_init () {
491 ctdb_counter_incr () {
495 echo -n 1 >> "$_counter_file"
497 ctdb_check_counter_limit () {
500 _limit="${1:-${service_fail_limit}}"
504 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
505 if [ $_size -ge $_limit ] ; then
506 echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
508 elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
509 echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
512 ########################################################
514 ctdb_spool_dir="/var/spool/ctdb"
515 ctdb_status_dir="$ctdb_spool_dir/status"
516 ctdb_fail_dir="$ctdb_spool_dir/failcount"
517 ctdb_active_dir="$ctdb_spool_dir/active"
521 echo "node is \"$1\", problem with \"${script_name}\": $(cat $2)"
526 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
527 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
529 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
530 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
539 d="$ctdb_status_dir/$script_name"
546 for i in "banned" "unhealthy" ; do
553 ctdb_service_needs_reconfigure ()
555 [ -e "$ctdb_status_dir/$service_name/reconfigure" ]
558 ctdb_service_set_reconfigure ()
560 d="$ctdb_status_dir/$service_name"
565 ctdb_service_unset_reconfigure ()
567 rm -f "$ctdb_status_dir/$service_name/reconfigure"
570 ctdb_service_reconfigure ()
572 if [ -n "$service_reconfigure" ] ; then
573 eval $service_reconfigure
575 service "$service_name" restart
577 ctdb_service_unset_reconfigure
581 ctdb_compat_managed_service ()
583 if [ "$1" = "yes" ] ; then
588 is_ctdb_managed_service ()
590 t=" $CTDB_MANAGED_SERVICES "
592 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
593 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
594 ctdb_compat_managed_service "$CTDB_MANAGES_SCP" "scp"
595 ctdb_compat_managed_service "$CTDB_MANAGES_WINDBIND" "windbind"
596 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
597 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
598 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
599 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
601 # Returns 0 if "<space>$service_name<space>" appears in $t
602 [ "${t#* ${service_name} }" != "${t}" ]
605 ctdb_start_stop_service ()
607 _active="$ctdb_active_dir/$service_name"
609 if is_ctdb_managed_service ; then
610 if ! [ -e "$_active" ] ; then
611 echo "Starting service $service_name"
612 ctdb_service_start || exit $?
613 mkdir -p "$ctdb_active_dir"
617 elif ! is_ctdb_managed_service ; then
618 if [ -e "$_active" ] ; then
619 echo "Stopping service $service_name"
620 ctdb_service_stop || exit $?
627 ctdb_service_start ()
629 if [ -n "$service_start" ] ; then
632 service "$service_name" start
639 if [ -n "$service_stop" ] ; then
642 service "$service_name" stop
646 ctdb_standard_event_handler ()
660 ipv4_host_addr_to_net_addr()
665 local HOST0=$(echo $HOST | awk -F . '{print $4}')
666 local HOST1=$(echo $HOST | awk -F . '{print $3}')
667 local HOST2=$(echo $HOST | awk -F . '{print $2}')
668 local HOST3=$(echo $HOST | awk -F . '{print $1}')
670 local HOST_NUM=$(( $HOST0 + $HOST1 * 256 + $HOST2 * (256 ** 2) + $HOST3 * (256 ** 3) ))
672 local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
674 local NET_NUM=$(( $HOST_NUM & $MASK_NUM))
676 local NET0=$(( $NET_NUM & 255 ))
677 local NET1=$(( ($NET_NUM & (255 * 256)) / 256 ))
678 local NET2=$(( ($NET_NUM & (255 * 256**2)) / 256**2 ))
679 local NET3=$(( ($NET_NUM & (255 * 256**3)) / 256**3 ))
681 echo "$NET3.$NET2.$NET1.$NET0"
684 ipv4_maskbits_to_net_mask()
688 local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
690 local MASK0=$(( $MASK_NUM & 255 ))
691 local MASK1=$(( ($MASK_NUM & (255 * 256)) / 256 ))
692 local MASK2=$(( ($MASK_NUM & (255 * 256**2)) / 256**2 ))
693 local MASK3=$(( ($MASK_NUM & (255 * 256**3)) / 256**3 ))
695 echo "$MASK3.$MASK2.$MASK1.$MASK0"
703 local N=`echo $ADDR | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*//'`
704 test -n "$N" && fail=1
706 local ADDR0=$(echo $ADDR | awk -F . '{print $4}')
707 local ADDR1=$(echo $ADDR | awk -F . '{print $3}')
708 local ADDR2=$(echo $ADDR | awk -F . '{print $2}')
709 local ADDR3=$(echo $ADDR | awk -F . '{print $1}')
711 test "$ADDR0" -gt 255 && fail=1
712 test "$ADDR1" -gt 255 && fail=1
713 test "$ADDR2" -gt 255 && fail=1
714 test "$ADDR3" -gt 255 && fail=1
716 test x"$fail" != x"0" && {
717 #echo "IPv4: '$ADDR' is not a valid address"
724 ########################################################
725 # load a site local config file
726 ########################################################
728 [ -x $CTDB_BASE/rc.local ] && {
729 . $CTDB_BASE/rc.local
732 [ -d $CTDB_BASE/rc.local.d ] && {
733 for i in $CTDB_BASE/rc.local.d/* ; do
734 [ -x "$i" ] && . "$i"
738 script_name="${0##*/}" # basename
739 service_name="$script_name" # default is just the script name