ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
   6
   7 [ -z "$CTDB_VARDIR" ] && {
   8     if [ -d "/var/lib/ctdb" ] ; then
   9         export CTDB_VARDIR="/var/lib/ctdb"
  10     else
  11         export CTDB_VARDIR="/var/ctdb"
  12     fi
  13 }
  14 [ -z "$CTDB_ETCDIR" ] && {
  15     export CTDB_ETCDIR="/etc"
  16 }
  17
  18 #######################################
  19 # pull in a system config file, if any
  20 _loadconfig() {
  21
  22     if [ -z "$1" ] ; then
  23         foo="${service_config:-${service_name}}"
  24         if [ -n "$foo" ] ; then
  25             loadconfig "$foo"
  26         fi
  27     elif [ "$1" != "ctdb" ] ; then
  28         loadconfig "ctdb"
  29     fi
  30
  31     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  32         . $CTDB_ETCDIR/sysconfig/$1
  33     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  34         . $CTDB_ETCDIR/default/$1
  35     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  36         . $CTDB_BASE/sysconfig/$1
  37     fi
  38 }
  39
  40 loadconfig () {
  41     _loadconfig "$@"
  42 }
  43
  44 ##############################################################
  45
  46 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  47 # configuration file.
  48 debug ()
  49 {
  50     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  51         # If there are arguments then echo them.  Otherwise expect to
  52         # use stdin, which allows us to pass lots of debug using a
  53         # here document.
  54         if [ -n "$1" ] ; then
  55             echo "DEBUG: $*"
  56         elif ! tty -s ; then
  57             sed -e 's@^@DEBUG: @'
  58         fi
  59     fi
  60 }
  61
  62 die ()
  63 {
  64     _msg="$1"
  65     _rc="${2:-1}"
  66
  67     echo "$_msg"
  68     exit $_rc
  69 }
  70
  71 # Log given message or stdin to either syslog or a CTDB log file
  72 # $1 is the tag passed to logger if syslog is in use.
  73 script_log ()
  74 {
  75     _tag="$1" ; shift
  76
  77     _using_syslog=false
  78     if [ "$CTDB_SYSLOG" = "yes" -o -z "$CTDB_LOGFILE" ] ; then
  79         _using_syslog=true
  80     fi
  81     case "$CTDB_OPTIONS" in
  82         *--syslog*) _using_syslog=true ;;
  83     esac
  84
  85     if $_using_syslog ; then
  86         logger -t "ctdbd: ${_tag}" $*
  87     else
  88         {
  89             if [ -n "$*" ] ; then
  90                 echo "$*"
  91             else
  92                 cat
  93             fi
  94         } >>"${CTDB_LOGFILE:-/var/log/log.ctdb}"
  95     fi
  96 }
  97
  98 # When things are run in the background in an eventscript then logging
  99 # output might get lost.  This is the "solution".  :-)
 100 background_with_logging ()
 101 {
 102     (
 103         "$@" 2>&1 </dev/null |
 104         script_log "${script_name}&"
 105     )&
 106
 107     return 0
 108 }
 109
 110 ##############################################################
 111 # check number of args for different events
 112 ctdb_check_args ()
 113 {
 114     case "$1" in
 115         takeip|releaseip)
 116             if [ $# != 4 ]; then
 117                 echo "ERROR: must supply interface, IP and maskbits"
 118                 exit 1
 119             fi
 120             ;;
 121         updateip)
 122             if [ $# != 5 ]; then
 123                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 124                 exit 1
 125             fi
 126             ;;
 127     esac
 128 }
 129
 130 ##############################################################
 131 # determine on what type of system (init style) we are running
 132 detect_init_style() {
 133     # only do detection if not already set:
 134     test "x$CTDB_INIT_STYLE" != "x" && return
 135
 136     if [ -x /sbin/startproc ]; then
 137         CTDB_INIT_STYLE="suse"
 138     elif [ -x /sbin/start-stop-daemon ]; then
 139         CTDB_INIT_STYLE="debian"
 140     else
 141         CTDB_INIT_STYLE="redhat"
 142     fi
 143 }
 144
 145 ######################################################
 146 # simulate /sbin/service on platforms that don't have it
 147 # _service() makes it easier to hook the service() function for
 148 # testing.
 149 _service ()
 150 {
 151   _service_name="$1"
 152   _op="$2"
 153
 154   # do nothing, when no service was specified
 155   [ -z "$_service_name" ] && return
 156
 157   if [ -x /sbin/service ]; then
 158       $_nice /sbin/service "$_service_name" "$_op"
 159   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 160       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 161   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 162       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 163   fi
 164 }
 165
 166 service()
 167 {
 168     _nice=""
 169     _service "$@"
 170 }
 171
 172 ######################################################
 173 # simulate /sbin/service (niced) on platforms that don't have it
 174 nice_service()
 175 {
 176     _nice="nice"
 177     _service "$@"
 178 }
 179
 180 ######################################################
 181 # wrapper around /proc/ settings to allow them to be hooked
 182 # for testing
 183 # 1st arg is relative path under /proc/, 2nd arg is value to set
 184 set_proc ()
 185 {
 186     echo "$2" >"/proc/$1"
 187 }
 188
 189 ######################################################
 190 # wrapper around getting file contents from /proc/ to allow
 191 # this to be hooked for testing
 192 # 1st arg is relative path under /proc/
 193 get_proc ()
 194 {
 195     cat "/proc/$1"
 196 }
 197
 198 ######################################################
 199 # Check that an RPC service is healthy -
 200 # this includes allowing a certain number of failures
 201 # before marking the NFS service unhealthy.
 202 #
 203 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 204 #
 205 # each triple is a set of 3 arguments: an operator, a
 206 # fail count limit and an action string.
 207 #
 208 # For example:
 209 #
 210 #       nfs_check_rpc_service "lockd" \
 211 #           -ge 15 "verbose restart unhealthy" \
 212 #           -eq 10 "restart:bs"
 213 #
 214 # says that if lockd is down for 15 iterations then do
 215 # a verbose restart of lockd and mark the node unhealthy.
 216 # Before this, after 10 iterations of failure, the
 217 # service is restarted silently in the background.
 218 # Order is important: the number of failures need to be
 219 # specified in reverse order because processing stops
 220 # after the first condition that is true.
 221 ######################################################
 222 nfs_check_rpc_service ()
 223 {
 224     _prog_name="$1" ; shift
 225
 226     _v=""
 227     case "$1" in
 228         -*) : ;;
 229         *) _v="$1" ; shift ;;
 230     esac
 231
 232     _version=${_v:-1}
 233     _rpc_prog="$_prog_name"
 234     _restart=""
 235     _opts=""
 236     case "$_prog_name" in
 237         knfsd)
 238             _rpc_prog=nfs
 239             _version=${_v:-3}
 240             _restart="echo 'Trying to restart NFS service'"
 241             _restart="${_restart}; startstop_nfs restart"
 242             ;;
 243         ganesha)
 244             _rpc_prog=nfs
 245             _version=${_v:-3}
 246             _restart="echo 'Trying to restart Ganesha NFS service'"
 247             _restart="${_restart}; startstop_ganesha restart"
 248             ;;
 249         mountd)
 250             _opts="${MOUNTD_PORT:+ -p }${MOUNTD_PORT}"
 251             ;;
 252         rquotad)
 253             _opts="${RQUOTAD_PORT:+ -p }${RQUOTAD_PORT}"
 254             ;;
 255         lockd)
 256             _rpc_prog=nlockmgr
 257             _version=${_v:-4}
 258             _restart="echo 'Trying to restart lock manager service'"
 259             _restart="${_restart}; startstop_nfslock restart"
 260             ;;
 261         statd)
 262             _rpc_prog=status
 263             _opts="${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}"
 264             _opts="${_opts}${STATD_PORT:+ -p }${STATD_PORT}"
 265             _opts="${_opts}${STATD_OUTGOING_PORT:+ -o }${STATD_OUTGOING_PORT}"
 266             ;;
 267         *)
 268             echo "Internal error: unknown RPC program \"$_prog_name\"."
 269             exit 1
 270     esac
 271
 272     _service_name="nfs_${_prog_name}"
 273
 274     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 275         ctdb_counter_init "$_service_name"
 276         return 0
 277     fi
 278
 279     ctdb_counter_incr "$_service_name"
 280
 281     while [ -n "$3" ] ; do
 282         ctdb_check_counter "quiet" "$1" "$2" "$_service_name" || {
 283             for _action in $3 ; do
 284                 case "$_action" in
 285                     verbose)
 286                         echo "$ctdb_check_rpc_out"
 287                         ;;
 288                     restart|restart:*)
 289                         # No explicit command specified, construct rpc command.
 290                         if [ -z "$_restart" ] ; then
 291                             _p="rpc.${_prog_name}"
 292                             _restart="echo 'Trying to restart $_prog_name [${_p}${_opts}]'"
 293                             _restart="${_restart}; killall -q -9 $_p"
 294                             _restart="${_restart}; $_p $_opts"
 295                         fi
 296
 297                         # Process restart flags...
 298                         _flags="${_action#restart:}"
 299                         # There may not have been a colon...
 300                         [ "$_flags" != "$_action" ] || _flags=""
 301                         # q=quiet - everything to /dev/null
 302                         if [ "${_flags#*q}" != "$_flags" ] ; then
 303                             _restart="{ ${_restart} ; } >/dev/null 2>&1"
 304                         fi
 305                         # s=stealthy - last command to /dev/null
 306                         if [ "${_flags#*s}" != "$_flags" ] ; then
 307                             _restart="${_restart} >/dev/null 2>&1"
 308                         fi
 309                         # b=background - the whole thing, easy and reliable
 310                         if [ "${_flags#*b}" != "$_flags" ] ; then
 311                             _restart="{ ${_restart} ; } &"
 312                         fi
 313
 314                         # Do it!
 315                         eval "${_restart}"
 316                         ;;
 317                     unhealthy)
 318                         exit 1
 319                         ;;
 320                     *)
 321                         echo "Internal error: unknown action \"$_action\"."
 322                         exit 1
 323                 esac
 324             done
 325
 326             # Only process the first action group.
 327             break
 328         }
 329         shift 3
 330     done
 331 }
 332
 333 ######################################################
 334 # check that a rpc server is registered with portmap
 335 # and responding to requests
 336 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 337 ######################################################
 338 ctdb_check_rpc ()
 339 {
 340     progname="$1"
 341     version="$2"
 342
 343     if ! ctdb_check_rpc_out=$(rpcinfo -u localhost $progname $version 2>&1) ; then
 344         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 345 $ctdb_check_rpc_out"
 346         echo "$ctdb_check_rpc_out"
 347         return 1
 348     fi
 349 }
 350
 351 ######################################################
 352 # check a set of directories is available
 353 # return 1 on a missing directory
 354 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
 355 ######################################################
 356 ctdb_check_directories_probe() {
 357     while IFS="" read d ; do
 358         case "$d" in
 359             *%*)
 360                 continue
 361                 ;;
 362             *)
 363                 [ -d "${d}/." ] || return 1
 364         esac
 365     done
 366 }
 367
 368 ######################################################
 369 # check a set of directories is available
 370 # usage: ctdb_check_directories SERVICE_NAME <directories...>
 371 ######################################################
 372 ctdb_check_directories() {
 373     n="${1:-${service_name}}"
 374     ctdb_check_directories_probe || {
 375         echo "ERROR: $n directory \"$d\" not available"
 376         exit 1
 377     }
 378 }
 379
 380 ######################################################
 381 # check a set of tcp ports
 382 # usage: ctdb_check_tcp_ports <ports...>
 383 ######################################################
 384
 385 # This flag file is created when a service is initially started.  It
 386 # is deleted the first time TCP port checks for that service succeed.
 387 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 388 # message if a port check fails.
 389 _ctdb_check_tcp_common ()
 390 {
 391     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 392 }
 393
 394 ctdb_check_tcp_init ()
 395 {
 396     _ctdb_check_tcp_common
 397     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 398     touch "$_ctdb_service_started_file"
 399 }
 400
 401 ctdb_check_tcp_ports()
 402 {
 403     if [ -z "$1" ] ; then
 404         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 405         exit 1
 406     fi
 407
 408     # Set default value for CTDB_TCP_PORT_CHECKS if unset.
 409     # If any of these defaults are unsupported then this variable can
 410     # be overridden in /etc/sysconfig/ctdb or via a file in
 411     # /etc/ctdb/rc.local.d/.
 412     : ${CTDB_TCP_PORT_CHECKERS:=ctdb nmap netstat}
 413
 414     for _c in $CTDB_TCP_PORT_CHECKERS ; do
 415         ctdb_check_tcp_ports_$_c "$@"
 416         case "$?" in
 417             0)
 418                 _ctdb_check_tcp_common
 419                 rm -f "$_ctdb_service_started_file"
 420                 return 0
 421                 ;;
 422             1)
 423                 _ctdb_check_tcp_common
 424                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 425                     echo "ERROR: $service_name tcp port $_p is not responding"
 426                     debug <<EOF
 427 $ctdb_check_tcp_ports_debug
 428 EOF
 429                 else
 430                     echo "INFO: $service_name tcp port $_p is not responding"
 431                 fi
 432
 433                 return 1
 434                 ;;
 435             127)
 436                 debug <<EOF
 437 ctdb_check_ports - checker $_c not implemented
 438 output from checker was:
 439 $ctdb_check_tcp_ports_debug
 440 EOF
 441                 ;;
 442             *)
 443
 444         esac
 445     done
 446
 447     echo "INTERNAL ERROR: ctdb_check_ports - no working checkers in CTDB_TCP_PORT_CHECKERS=\"$CTDB_TCP_PORT_CHECKERS\""
 448
 449     return 127
 450 }
 451
 452 ctdb_check_tcp_ports_netstat ()
 453 {
 454     _cmd='netstat -l -t -n'
 455     _ns=$($_cmd 2>&1)
 456     if [ $? -eq 127 ] ; then
 457         # netstat probably not installed - unlikely?
 458         ctdb_check_tcp_ports_debug="$_ns"
 459         return 127
 460     fi
 461
 462     for _p ; do  # process each function argument (port)
 463         for _a in '0\.0\.0\.0' '::' ; do
 464             _pat="[[:space:]]${_a}:${_p}[[:space:]]+[^[:space:]]+[[:space:]]+LISTEN"
 465             if echo "$_ns" | grep -E -q "$_pat" ; then
 466                 # We matched the port, so process next port
 467                 continue 2
 468             fi
 469         done
 470
 471         # We didn't match the port, so flag an error.
 472         ctdb_check_tcp_ports_debug="$_cmd shows this output:
 473 $_ns"
 474         return 1
 475     done
 476
 477     return 0
 478 }
 479
 480 ctdb_check_tcp_ports_nmap ()
 481 {
 482     # nmap wants a comma-separated list of ports
 483     _ports=""
 484     for _p ; do
 485         _ports="${_ports}${_ports:+,}${_p}"
 486     done
 487
 488     _cmd="nmap -n -oG - -PS 127.0.0.1 -p $_ports"
 489
 490     _nmap_out=$($_cmd 2>&1)
 491     if [ $? -eq 127 ] ; then
 492         # nmap probably not installed
 493         ctdb_check_tcp_ports_debug="$_nmap_out"
 494         return 127
 495     fi
 496
 497     # get the port-related output
 498     _port_info=$(echo "$_nmap_out" | sed -n -r -e 's@^.*Ports:[[:space:]]@@p')
 499
 500     for _p ; do
 501         # looking for something like this:
 502         #  445/open/tcp//microsoft-ds///
 503         # possibly followed by a comma
 504         _t="$_p/open/tcp//"
 505         case "$_port_info" in
 506             # The info we're after must be either at the beginning of
 507             # the string or it must follow a space.
 508             $_t*|*\ $_t*) : ;;
 509             *)
 510                 # Nope, flag an error...
 511                 ctdb_check_tcp_ports_debug="$_cmd shows this output:
 512 $_nmap_out"
 513                 return 1
 514         esac
 515     done
 516
 517     return 0
 518 }
 519
 520 # Use the new "ctdb checktcpport" command to check the port.
 521 # This is very cheap.
 522 ctdb_check_tcp_ports_ctdb ()
 523 {
 524     for _p ; do  # process each function argument (port)
 525         _cmd="ctdb checktcpport $_p"
 526         _out=$($_cmd 2>&1)
 527         _ret=$?
 528         case "$_ret" in
 529             0)
 530                 ctdb_check_tcp_ports_debug="\"$_cmd\" was able to bind to port"
 531                 return 1
 532                 ;;
 533             98)
 534                 # Couldn't bind, something already listening, next port...
 535                 continue
 536                 ;;
 537             *)
 538                 ctdb_check_tcp_ports_debug="$_cmd (exited with $_ret) with output:
 539 $_out"
 540                 # assume not implemented
 541                 return 127
 542         esac
 543     done
 544
 545     return 0
 546 }
 547
 548 ######################################################
 549 # check a unix socket
 550 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 551 ######################################################
 552 ctdb_check_unix_socket() {
 553     socket_path="$1"
 554     [ -z "$socket_path" ] && return
 555
 556     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 557         echo "ERROR: $service_name socket $socket_path not found"
 558         return 1
 559     fi
 560 }
 561
 562 ######################################################
 563 # check a command returns zero status
 564 # usage: ctdb_check_command SERVICE_NAME <command>
 565 ######################################################
 566 ctdb_check_command() {
 567   service_name="$1"
 568   wait_cmd="$2"
 569   [ -z "$wait_cmd" ] && return;
 570   $wait_cmd > /dev/null 2>&1 || {
 571       echo "ERROR: $service_name - $wait_cmd returned error"
 572       exit 1
 573   }
 574 }
 575
 576 ################################################
 577 # kill off any TCP connections with the given IP
 578 ################################################
 579 kill_tcp_connections() {
 580     _IP="$1"
 581     _failed=0
 582
 583     _killcount=0
 584     connfile="$CTDB_VARDIR/state/connections.$_IP"
 585     mkdir -p "${connfile%/*}" # dirname
 586     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 587     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 588
 589     while read dest src; do
 590         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 591         srcport=`echo $src | sed -e "s/^.*://"`
 592         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 593         destport=`echo $dest | sed -e "s/^.*://"`
 594         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 595         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 596         case $destport in
 597           # we only do one-way killtcp for CIFS
 598           139|445) : ;;
 599           # for all others we do 2-way
 600           *)
 601                 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
 602                 ;;
 603         esac
 604         _killcount=`expr $_killcount + 1`
 605      done < $connfile
 606     rm -f $connfile
 607
 608     [ $_failed = 0 ] || {
 609         echo "Failed to send killtcp control"
 610         return;
 611     }
 612     [ $_killcount -gt 0 ] || {
 613         return;
 614     }
 615     _count=0
 616     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 617         sleep 1
 618         _count=`expr $_count + 1`
 619         [ $_count -gt 3 ] && {
 620             echo "Timed out killing tcp connections for IP $_IP"
 621             return;
 622         }
 623     done
 624     echo "killed $_killcount TCP connections to released IP $_IP"
 625 }
 626
 627 ##################################################################
 628 # kill off the local end for any TCP connections with the given IP
 629 ##################################################################
 630 kill_tcp_connections_local_only() {
 631     _IP="$1"
 632     _failed=0
 633
 634     _killcount=0
 635     connfile="$CTDB_VARDIR/state/connections.$_IP"
 636     mkdir -p "${connfile%/*}" # dirname
 637     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 638     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 639
 640     while read dest src; do
 641         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 642         srcport=`echo $src | sed -e "s/^.*://"`
 643         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 644         destport=`echo $dest | sed -e "s/^.*://"`
 645         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 646         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 647         _killcount=`expr $_killcount + 1`
 648      done < $connfile
 649     rm -f $connfile
 650
 651     [ $_failed = 0 ] || {
 652         echo "Failed to send killtcp control"
 653         return;
 654     }
 655     [ $_killcount -gt 0 ] || {
 656         return;
 657     }
 658     _count=0
 659     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 660         sleep 1
 661         _count=`expr $_count + 1`
 662         [ $_count -gt 3 ] && {
 663             echo "Timed out killing tcp connections for IP $_IP"
 664             return;
 665         }
 666     done
 667     echo "killed $_killcount TCP connections to released IP $_IP"
 668 }
 669
 670 ##################################################################
 671 # tickle any TCP connections with the given IP
 672 ##################################################################
 673 tickle_tcp_connections() {
 674     _IP="$1"
 675     _failed=0
 676
 677     _killcount=0
 678     connfile="$CTDB_VARDIR/state/connections.$_IP"
 679     mkdir -p "${connfile%/*}" # dirname
 680     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 681     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 682
 683     while read dest src; do
 684         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 685         srcport=`echo $src | sed -e "s/^.*://"`
 686         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 687         destport=`echo $dest | sed -e "s/^.*://"`
 688         echo "Tickle TCP connection $srcip:$srcport $destip:$destport"
 689         ctdb tickle $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 690         echo "Tickle TCP connection $destip:$destport $srcip:$srcport"
 691         ctdb tickle $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
 692      done < $connfile
 693     rm -f $connfile
 694
 695     [ $_failed = 0 ] || {
 696         echo "Failed to send tickle control"
 697         return;
 698     }
 699 }
 700
 701 ########################################################
 702 # start/stop the Ganesha nfs service
 703 ########################################################
 704 startstop_ganesha()
 705 {
 706     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 707     case "$1" in
 708         start)
 709             service "$_service_name" start
 710             ;;
 711         stop)
 712             service "$_service_name" stop
 713             ;;
 714         restart)
 715             service "$_service_name" restart
 716             ;;
 717     esac
 718 }
 719
 720 ########################################################
 721 # start/stop the nfs service on different platforms
 722 ########################################################
 723 startstop_nfs() {
 724         PLATFORM="unknown"
 725         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 726                 PLATFORM="sles"
 727         }
 728         [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
 729                 PLATFORM="rhel"
 730         }
 731
 732         case $PLATFORM in
 733         sles)
 734                 case $1 in
 735                 start)
 736                         service nfsserver start
 737                         ;;
 738                 stop)
 739                         service nfsserver stop > /dev/null 2>&1
 740                         ;;
 741                 restart)
 742                         set_proc "fs/nfsd/threads" 0
 743                         service nfsserver stop > /dev/null 2>&1
 744                         pkill -9 nfsd
 745                         service nfsserver start
 746                         ;;
 747                 esac
 748                 ;;
 749         rhel)
 750                 case $1 in
 751                 start)
 752                         service nfslock start
 753                         service nfs start
 754                         ;;
 755                 stop)
 756                         service nfs stop
 757                         service nfslock stop
 758                         ;;
 759                 restart)
 760                         set_proc "fs/nfsd/threads" 0
 761                         service nfs stop > /dev/null 2>&1
 762                         service nfslock stop > /dev/null 2>&1
 763                         pkill -9 nfsd
 764                         service nfslock start
 765                         service nfs start
 766                         ;;
 767                 esac
 768                 ;;
 769         *)
 770                 echo "Unknown platform. NFS is not supported with ctdb"
 771                 exit 1
 772                 ;;
 773         esac
 774 }
 775
 776 ########################################################
 777 # start/stop the nfs lockmanager service on different platforms
 778 ########################################################
 779 startstop_nfslock() {
 780         PLATFORM="unknown"
 781         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 782                 PLATFORM="sles"
 783         }
 784         [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
 785                 PLATFORM="rhel"
 786         }
 787
 788         case $PLATFORM in
 789         sles)
 790                 # for sles there is no service for lockmanager
 791                 # so we instead just shutdown/restart nfs
 792                 case $1 in
 793                 start)
 794                         service nfsserver start
 795                         ;;
 796                 stop)
 797                         service nfsserver stop > /dev/null 2>&1
 798                         ;;
 799                 restart)
 800                         service nfsserver stop
 801                         service nfsserver start
 802                         ;;
 803                 esac
 804                 ;;
 805         rhel)
 806                 case $1 in
 807                 start)
 808                         service nfslock start
 809                         ;;
 810                 stop)
 811                         service nfslock stop > /dev/null 2>&1
 812                         ;;
 813                 restart)
 814                         service nfslock stop
 815                         service nfslock start
 816                         ;;
 817                 esac
 818                 ;;
 819         *)
 820                 echo "Unknown platform. NFS locking is not supported with ctdb"
 821                 exit 1
 822                 ;;
 823         esac
 824 }
 825
 826 add_ip_to_iface()
 827 {
 828     _iface=$1
 829     _ip=$2
 830     _maskbits=$3
 831
 832     _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
 833     mkdir -p "${_lockfile%/*}" # dirname
 834     [ -f "$_lockfile" ] || touch "$_lockfile"
 835
 836     (
 837         # Note: use of return/exit/die() below only gets us out of the
 838         # sub-shell, which is actually what we want.  That is, the
 839         # function should just return non-zero.
 840
 841         flock --timeout 30 0 || \
 842             die "add_ip_to_iface: unable to get lock for ${_iface}"
 843
 844         # Ensure interface is up
 845         ip link set "$_iface" up || \
 846             die "Failed to bringup interface $_iface"
 847
 848         ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || \
 849             die "Failed to add $_ip/$_maskbits on dev $_iface"
 850     ) <"$_lockfile"
 851
 852     # Do nothing here - return above only gets us out of the subshell
 853     # and doing anything here will affect the return code.
 854 }
 855
 856 delete_ip_from_iface()
 857 {
 858     _iface=$1
 859     _ip=$2
 860     _maskbits=$3
 861
 862     _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
 863     mkdir -p "${_lockfile%/*}" # dirname
 864     [ -f "$_lockfile" ] || touch "$_lockfile"
 865
 866     (
 867         # Note: use of return/exit/die() below only gets us out of the
 868         # sub-shell, which is actually what we want.  That is, the
 869         # function should just return non-zero.
 870
 871         flock --timeout 30 0 || \
 872             die "delete_ip_from_iface: unable to get lock for ${_iface}"
 873
 874         _im="$_ip/$_maskbits"  # shorthand for readability
 875
 876         # "ip addr del" will delete all secondary IPs if this is the
 877         # primary.  To work around this _very_ annoying behaviour we
 878         # have to keep a record of the secondaries and re-add them
 879         # afterwards.  Yuck!
 880
 881         _secondaries=""
 882         if ip addr list dev "$_iface" primary | grep -Fq "inet $_im " ; then
 883             _secondaries=$(ip addr list dev "$_iface" secondary | \
 884                 awk '$1 == "inet" { print $2 }')
 885         fi
 886
 887         local _rc=0
 888         ip addr del "$_im" dev "$_iface" || {
 889             echo "Failed to del $_ip on dev $_iface"
 890             _rc=1
 891         }
 892
 893         if [ -n "$_secondaries" ] ; then
 894             for _i in $_secondaries; do
 895                 if ip addr list dev "$_iface" | grep -Fq "inet $_i" ; then
 896                     echo "Kept secondary $_i on dev $_iface"
 897                 else
 898                     echo "Re-adding secondary address $_i to dev $_iface"
 899                     ip addr add $_i brd + dev $_iface || {
 900                         echo "Failed to re-add address $_i to dev $_iface"
 901                         _rc=1
 902                     }
 903                 fi
 904             done
 905         fi
 906
 907         return $_rc
 908     ) <"$_lockfile"
 909
 910     # Do nothing here - return above only gets us out of the subshell
 911     # and doing anything here will affect the return code.
 912 }
 913
 914 # If the given IP is hosted then print 2 items: maskbits and iface
 915 ip_maskbits_iface ()
 916 {
 917     _addr="$1"
 918
 919     ip addr show to "${_addr}/32" 2>/dev/null | \
 920         awk '$1 == "inet" { print gensub(".*/", "", 1, $2), $NF }'
 921 }
 922
 923 drop_ip ()
 924 {
 925     _addr="${1%/*}"  # Remove optional maskbits
 926     _log_tag="$2"
 927
 928     set -- $(ip_maskbits_iface $_addr)
 929     if [ -n "$1" ] ; then
 930         _maskbits="$1"
 931         _iface="$2"
 932         if [ -n "$_log_tag" ] ; then
 933             script_log "$_log_tag" \
 934                 "Removing public address $_addr/$_maskbits from device $_iface"
 935         fi
 936         ip addr del $_addr/$_maskbits dev $_iface >/dev/null 2>&1
 937     fi
 938 }
 939
 940 drop_all_public_ips ()
 941 {
 942     _log_tag="$1"
 943
 944     while read _ip _x ; do
 945         drop_ip "$_ip" "$_log_tag"
 946     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 947 }
 948
 949 ########################################################
 950 # some simple logic for counting events - per eventscript
 951 # usage: ctdb_counter_init
 952 #        ctdb_counter_incr
 953 #        ctdb_check_counter_limit <limit>
 954 # ctdb_check_counter_limit fails when count >= <limit>
 955 ########################################################
 956 _ctdb_counter_common () {
 957     _service_name="${1:-${service_name}}"
 958     _counter_file="$ctdb_fail_dir/$_service_name"
 959     mkdir -p "${_counter_file%/*}" # dirname
 960 }
 961 ctdb_counter_init () {
 962     _ctdb_counter_common "$1"
 963
 964     >"$_counter_file"
 965 }
 966 ctdb_counter_incr () {
 967     _ctdb_counter_common "$1"
 968
 969     # unary counting!
 970     echo -n 1 >> "$_counter_file"
 971 }
 972 ctdb_check_counter_limit () {
 973     _ctdb_counter_common
 974
 975     _limit="${1:-${service_fail_limit}}"
 976     _quiet="$2"
 977
 978     # unary counting!
 979     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 980     if [ $_size -ge $_limit ] ; then
 981         echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
 982         exit 1
 983     elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
 984         echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
 985     fi
 986 }
 987 ctdb_check_counter () {
 988     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 989     _op="${2:--ge}"  # an integer operator supported by test
 990     _limit="${3:-${service_fail_limit}}"
 991     shift 3
 992     _ctdb_counter_common "$1"
 993
 994     # unary counting!
 995     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 996     if [ $_size $_op $_limit ] ; then
 997         if [ "$_msg" = "error" ] ; then
 998             echo "ERROR: $_limit consecutive failures for $_service_name, marking node unhealthy"
 999             exit 1
1000         else
1001             return 1
1002         fi
1003     fi
1004 }
1005
1006 ########################################################
1007
1008 ctdb_status_dir="$CTDB_VARDIR/status"
1009 ctdb_fail_dir="$CTDB_VARDIR/failcount"
1010
1011 ctdb_setup_service_state_dir ()
1012 {
1013     service_state_dir="$CTDB_VARDIR/state/${1:-${service_name}}"
1014     mkdir -p "$service_state_dir" || {
1015         echo "Error creating state dir \"$service_state_dir\""
1016         exit 1
1017     }
1018 }
1019
1020 ########################################################
1021 # Managed status history, for auto-start/stop
1022
1023 ctdb_managed_dir="$CTDB_VARDIR/managed_history"
1024
1025 _ctdb_managed_common ()
1026 {
1027     _service_name="${1:-${service_name}}"
1028     _ctdb_managed_file="$ctdb_managed_dir/$_service_name"
1029 }
1030
1031 ctdb_service_managed ()
1032 {
1033     _ctdb_managed_common "$@"
1034     mkdir -p "$ctdb_managed_dir"
1035     touch "$_ctdb_managed_file"
1036 }
1037
1038 ctdb_service_unmanaged ()
1039 {
1040     _ctdb_managed_common "$@"
1041     rm -f "$_ctdb_managed_file"
1042 }
1043
1044 is_ctdb_previously_managed_service ()
1045 {
1046     _ctdb_managed_common "$@"
1047     [ -f "$_ctdb_managed_file" ]
1048 }
1049
1050 ########################################################
1051 # Check and set status
1052
1053 log_status_cat ()
1054 {
1055     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1056 }
1057
1058 ctdb_checkstatus ()
1059 {
1060     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1061         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1062         return 1
1063     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1064         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1065         return 2
1066     else
1067         return 0
1068     fi
1069 }
1070
1071 ctdb_setstatus ()
1072 {
1073     d="$ctdb_status_dir/$script_name"
1074     case "$1" in
1075         unhealthy|banned)
1076             mkdir -p "$d"
1077             cat "$2" >"$d/$1"
1078             ;;
1079         *)
1080             for i in "banned" "unhealthy" ; do
1081                 rm -f "$d/$i"
1082             done
1083             ;;
1084     esac
1085 }
1086
1087 ##################################################################
1088 # Reconfigure a service on demand
1089
1090 _ctdb_service_reconfigure_common ()
1091 {
1092     _d="$ctdb_status_dir/${service_name}"
1093     mkdir -p "$_d"
1094     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1095 }
1096
1097 ctdb_service_needs_reconfigure ()
1098 {
1099     _ctdb_service_reconfigure_common
1100     [ -e "$_ctdb_service_reconfigure_flag" ]
1101 }
1102
1103 ctdb_service_set_reconfigure ()
1104 {
1105     _ctdb_service_reconfigure_common
1106     >"$_ctdb_service_reconfigure_flag"
1107 }
1108
1109 ctdb_service_unset_reconfigure ()
1110 {
1111     _ctdb_service_reconfigure_common
1112     rm -f "$_ctdb_service_reconfigure_flag"
1113 }
1114
1115 ctdb_service_reconfigure ()
1116 {
1117     echo "Reconfiguring service \"${service_name}\"..."
1118     ctdb_service_unset_reconfigure
1119     service_reconfigure || return $?
1120     ctdb_counter_init
1121 }
1122
1123 # Default service_reconfigure() function does nothing.
1124 service_reconfigure ()
1125 {
1126     :
1127 }
1128
1129 ctdb_reconfigure_try_lock ()
1130 {
1131     _ctdb_service_reconfigure_common
1132     _lock="${_d}/reconfigure_lock"
1133     mkdir -p "${_lock%/*}" # dirname
1134     touch "$_lock"
1135
1136     (
1137         flock 0
1138         # This is overkill but will work if we need to extend this to
1139         # allow certain events to run multiple times in parallel
1140         # (e.g. takeip) and write multiple PIDs to the file.
1141         read _locker_event
1142         if [ -n "$_locker_event" ] ; then
1143             while read _pid ; do
1144                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1145                     kill -0 "$_pid" 2>/dev/null ; then
1146                     exit 1
1147                 fi
1148             done
1149         fi
1150
1151         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1152         exit 0
1153     ) <"$_lock"
1154 }
1155
1156 ctdb_replay_monitor_status ()
1157 {
1158     echo "Replaying previous status for this script due to reconfigure..."
1159     # Leading colon (':') is missing in some versions...
1160     _out=$(ctdb scriptstatus -Y | grep -E "^:?monitor:${script_name}:")
1161     # Output looks like this:
1162     # :monitor:60.nfs:1:ERROR:1314764004.030861:1314764004.035514:foo bar:
1163     # This is the cheapest way of getting fields in the middle.
1164     set -- $(IFS=":" ; echo $_out)
1165     _code="$3"
1166     _status="$4"
1167     # The error output field can include colons so we'll try to
1168     # preserve them.  The weak checking at the beginning tries to make
1169     # this work for both broken (no leading ':') and fixed output.
1170     _out="${_out%:}"
1171     _err_out="${_out#*monitor:${script_name}:*:*:*:*:}"
1172     case "$_status" in
1173         OK) : ;;  # Do nothing special.
1174         TIMEDOUT)
1175             # Recast this as an error, since we can't exit with the
1176             # correct negative number.
1177             _code=1
1178             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1179             ;;
1180         DISABLED)
1181             # Recast this as an OK, since we can't exit with the
1182             # correct negative number.
1183             _code=0
1184             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1185             ;;
1186         *) : ;;  # Must be ERROR, do nothing special.
1187     esac
1188     echo "$_err_out"
1189     exit $_code
1190 }
1191
1192 ctdb_service_check_reconfigure ()
1193 {
1194     # We only care about some events in this function.  For others we
1195     # return now.
1196     case "$event_name" in
1197         monitor|ipreallocated|reconfigure) : ;;
1198         *) return 0 ;;
1199     esac
1200
1201     if ctdb_reconfigure_try_lock ; then
1202         # No events covered by this function are running, so proceed
1203         # with gay abandon.
1204         case "$event_name" in
1205             reconfigure)
1206                 (ctdb_service_reconfigure)
1207                 exit $?
1208                 ;;
1209             ipreallocated)
1210                 if ctdb_service_needs_reconfigure ; then
1211                     ctdb_service_reconfigure
1212                 fi
1213                 ;;
1214             monitor)
1215                 if ctdb_service_needs_reconfigure ; then
1216                     ctdb_service_reconfigure
1217                     # Given that the reconfigure might not have
1218                     # resulted in the service being stable yet, we
1219                     # replay the previous status since that's the best
1220                     # information we have.
1221                     ctdb_replay_monitor_status
1222                 fi
1223                 ;;
1224         esac
1225     else
1226         # Somebody else is running an event we don't want to collide
1227         # with.  We proceed with caution.
1228         case "$event_name" in
1229             reconfigure)
1230                 # Tell whoever called us to retry.
1231                 exit 2
1232                 ;;
1233             ipreallocated)
1234                 # Defer any scheduled reconfigure and just run the
1235                 # rest of the ipreallocated event, as per the
1236                 # eventscript.  There's an assumption here that the
1237                 # event doesn't depend on any scheduled reconfigure.
1238                 # This is true in the current code.
1239                 return 0
1240                 ;;
1241             monitor)
1242                 # There is most likely a reconfigure in progress so
1243                 # the service is possibly unstable.  As above, we
1244                 # defer any scheduled reconfigured.  We also replay
1245                 # the previous monitor status since that's the best
1246                 # information we have.
1247                 ctdb_replay_monitor_status
1248                 ;;
1249         esac
1250     fi
1251 }
1252
1253 ##################################################################
1254 # Does CTDB manage this service? - and associated auto-start/stop
1255
1256 ctdb_compat_managed_service ()
1257 {
1258     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1259         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1260     fi
1261 }
1262
1263 is_ctdb_managed_service ()
1264 {
1265     # $t is used just for readability and to allow better accurate
1266     # matching via leading/trailing spaces
1267     t=" $CTDB_MANAGED_SERVICES "
1268
1269     # Return 0 if "<space>$service_name<space>" appears in $t
1270     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1271         return 0
1272     fi
1273
1274     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1275     # backward compatibility and try again.
1276     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1277     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1278     ctdb_compat_managed_service "$CTDB_MANAGES_SCP"      "scp"
1279     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1280     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1281     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1282     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1283     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1284     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1285     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1286
1287     t=" $CTDB_MANAGED_SERVICES "
1288
1289     # Return 0 if "<space>$service_name<space>" appears in $t
1290     [ "${t#* ${service_name} }" != "${t}" ]
1291 }
1292
1293 ctdb_start_stop_service ()
1294 {
1295     # Allow service-start/service-stop pseudo-events to start/stop
1296     # services when we're not auto-starting/stopping and we're not
1297     # monitoring.
1298     case "$event_name" in
1299         service-start)
1300             if is_ctdb_managed_service ; then
1301                 die 'service-start event not permitted when service is managed'
1302             fi
1303             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1304                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1305             fi
1306             ctdb_service_start
1307             exit $?
1308             ;;
1309         service-stop)
1310             if is_ctdb_managed_service ; then
1311                 die 'service-stop event not permitted when service is managed'
1312             fi
1313             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1314                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1315             fi
1316             ctdb_service_stop
1317             exit $?
1318             ;;
1319     esac
1320
1321     # Do nothing unless configured to...
1322     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1323
1324     [ "$event_name" = "monitor" ] || return 0
1325
1326     if is_ctdb_managed_service ; then
1327         if ! is_ctdb_previously_managed_service ; then
1328             echo "Starting service \"$service_name\" - now managed"
1329             background_with_logging ctdb_service_start
1330             exit $?
1331         fi
1332     else
1333         if is_ctdb_previously_managed_service ; then
1334             echo "Stopping service \"$service_name\" - no longer managed"
1335             background_with_logging ctdb_service_stop
1336             exit $?
1337         fi
1338     fi
1339 }
1340
1341 ctdb_service_start ()
1342 {
1343     # The service is marked managed if we've ever tried to start it.
1344     ctdb_service_managed "$@"
1345
1346     # Here we only want $1.  If no argument is passed then
1347     # service_start needs to know.
1348     service_start "$@" || return $?
1349
1350     ctdb_counter_init "$@"
1351     ctdb_check_tcp_init
1352 }
1353
1354 ctdb_service_stop ()
1355 {
1356     ctdb_service_unmanaged "$@"
1357     service_stop "$@"
1358 }
1359
1360 # Default service_start() and service_stop() functions.
1361
1362 # These may be overridden in an eventscript.  When overriding, the
1363 # following convention must be followed.  If these functions are
1364 # called with no arguments then they may use internal logic to
1365 # determine whether the service is managed and, therefore, whether
1366 # they should take any action.  However, if the service name is
1367 # specified as an argument then an attempt must be made to start or
1368 # stop the service.  This is because the auto-start/stop code calls
1369 # them with the service name as an argument.
1370 service_start ()
1371 {
1372     service "${1:-${service_name}}" start
1373 }
1374
1375 service_stop ()
1376 {
1377     service "${1:-${service_name}}" stop
1378 }
1379
1380 ##################################################################
1381
1382 ctdb_standard_event_handler ()
1383 {
1384     case "$1" in
1385         status)
1386             ctdb_checkstatus
1387             exit
1388             ;;
1389         setstatus)
1390             shift
1391             ctdb_setstatus "$@"
1392             exit
1393             ;;
1394     esac
1395 }
1396
1397 # iptables doesn't like being re-entered, so flock-wrap it.
1398 iptables()
1399 {
1400         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1401 }
1402
1403 ########################################################
1404 # tickle handling
1405 ########################################################
1406
1407 update_tickles ()
1408 {
1409         _port="$1"
1410
1411         tickledir="$CTDB_VARDIR/state/tickles"
1412         mkdir -p "$tickledir"
1413
1414         # Who am I?
1415         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1416
1417         # What public IPs do I hold?
1418         _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}')
1419
1420         # IPs as a regexp choice
1421         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1422
1423         # Record connections to our public IPs in a temporary file
1424         _my_connections="${tickledir}/${_port}.connections"
1425         rm -f "$_my_connections"
1426         netstat -tn |
1427         awk -v destpat="^${_ipschoice}:${_port}\$" \
1428           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1429         sort >"$_my_connections"
1430
1431         # Record our current tickles in a temporary file
1432         _my_tickles="${tickledir}/${_port}.tickles"
1433         rm -f "$_my_tickles"
1434         for _i in $_ips ; do
1435                 ctdb -Y gettickles $_i $_port |
1436                 awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1437         done |
1438         sort >"$_my_tickles"
1439
1440         # Add tickles for connections that we haven't already got tickles for
1441         comm -23 "$_my_connections" "$_my_tickles" |
1442         while read _src _dst ; do
1443                 ctdb addtickle $_src $_dst
1444         done
1445
1446         # Remove tickles for connections that are no longer there
1447         comm -13 "$_my_connections" "$_my_tickles" |
1448         while read _src _dst ; do
1449                 ctdb deltickle $_src $_dst
1450         done
1451
1452         rm -f "$_my_connections" "$_my_tickles"
1453 }
1454
1455 ########################################################
1456 # load a site local config file
1457 ########################################################
1458
1459 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1460         . "$CTDB_RC_LOCAL"
1461 }
1462
1463 [ -x $CTDB_BASE/rc.local ] && {
1464         . $CTDB_BASE/rc.local
1465 }
1466
1467 [ -d $CTDB_BASE/rc.local.d ] && {
1468         for i in $CTDB_BASE/rc.local.d/* ; do
1469                 [ -x "$i" ] && . "$i"
1470         done
1471 }
1472
1473 script_name="${0##*/}"       # basename
1474 service_name="$script_name"  # default is just the script name
1475 service_fail_limit=1
1476 event_name="$1"