ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         else
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     else
  72         if [ -z "$1" ] ; then
  73             cat >/dev/null
  74         fi
  75     fi
  76 }
  77
  78 die ()
  79 {
  80     _msg="$1"
  81     _rc="${2:-1}"
  82
  83     echo "$_msg"
  84     exit $_rc
  85 }
  86
  87 # Log given message or stdin to either syslog or a CTDB log file
  88 # $1 is the tag passed to logger if syslog is in use.
  89 script_log ()
  90 {
  91     _tag="$1" ; shift
  92
  93     case "$CTDB_LOGGING" in
  94         file:*|"")
  95             if [ -n "$CTDB_LOGGING" ] ; then
  96                 _file="${CTDB_LOGGING#file:}"
  97             else
  98                 _file="/var/log/log.ctdb"
  99             fi
 100             {
 101                 if [ -n "$*" ] ; then
 102                     echo "$*"
 103                 else
 104                     cat
 105                 fi
 106             } >>"$_file"
 107             ;;
 108         *)
 109             # Handle all syslog:* variants here too.  There's no tool to do
 110             # the lossy things, so just use logger.
 111             logger -t "ctdbd: ${_tag}" $*
 112             ;;
 113     esac
 114 }
 115
 116 # When things are run in the background in an eventscript then logging
 117 # output might get lost.  This is the "solution".  :-)
 118 background_with_logging ()
 119 {
 120     (
 121         "$@" 2>&1 </dev/null |
 122         script_log "${script_name}&"
 123     )&
 124
 125     return 0
 126 }
 127
 128 ##############################################################
 129 # check number of args for different events
 130 ctdb_check_args ()
 131 {
 132     case "$1" in
 133         takeip|releaseip)
 134             if [ $# != 4 ]; then
 135                 echo "ERROR: must supply interface, IP and maskbits"
 136                 exit 1
 137             fi
 138             ;;
 139         updateip)
 140             if [ $# != 5 ]; then
 141                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 142                 exit 1
 143             fi
 144             ;;
 145     esac
 146 }
 147
 148 ##############################################################
 149 # determine on what type of system (init style) we are running
 150 detect_init_style()
 151 {
 152     # only do detection if not already set:
 153     [ -z "$CTDB_INIT_STYLE" ] || return
 154
 155     if [ -x /sbin/startproc ]; then
 156         CTDB_INIT_STYLE="suse"
 157     elif [ -x /sbin/start-stop-daemon ]; then
 158         CTDB_INIT_STYLE="debian"
 159     else
 160         CTDB_INIT_STYLE="redhat"
 161     fi
 162 }
 163
 164 ######################################################
 165 # simulate /sbin/service on platforms that don't have it
 166 # _service() makes it easier to hook the service() function for
 167 # testing.
 168 _service ()
 169 {
 170   _service_name="$1"
 171   _op="$2"
 172
 173   # do nothing, when no service was specified
 174   [ -z "$_service_name" ] && return
 175
 176   if [ -x /sbin/service ]; then
 177       $_nice /sbin/service "$_service_name" "$_op"
 178   elif [ -x /usr/sbin/service ]; then
 179       $_nice /usr/sbin/service "$_service_name" "$_op"
 180   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 181       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 182   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 183       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 184   fi
 185 }
 186
 187 service()
 188 {
 189     _nice=""
 190     _service "$@"
 191 }
 192
 193 ######################################################
 194 # simulate /sbin/service (niced) on platforms that don't have it
 195 nice_service()
 196 {
 197     _nice="nice"
 198     _service "$@"
 199 }
 200
 201 ######################################################
 202 # Cached retrieval of PNN from local node.  This never changes so why
 203 # open a client connection to the server each time this is needed?
 204 # This sets $pnn - this avoid an unnecessary subprocess.
 205 ctdb_get_pnn ()
 206 {
 207     _pnn_file="$CTDB_VARDIR/state/my-pnn"
 208     if [ ! -f "$_pnn_file" ] ; then
 209         ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
 210     fi
 211
 212     read pnn <"$_pnn_file"
 213 }
 214
 215 ######################################################
 216 # wrapper around /proc/ settings to allow them to be hooked
 217 # for testing
 218 # 1st arg is relative path under /proc/, 2nd arg is value to set
 219 set_proc ()
 220 {
 221     echo "$2" >"/proc/$1"
 222 }
 223
 224 set_proc_maybe ()
 225 {
 226     if [ -w "/proc/$1" ] ; then
 227         set_proc "$1" "$2"
 228     fi
 229 }
 230
 231 ######################################################
 232 # wrapper around getting file contents from /proc/ to allow
 233 # this to be hooked for testing
 234 # 1st arg is relative path under /proc/
 235 get_proc ()
 236 {
 237     cat "/proc/$1"
 238 }
 239
 240 ######################################################
 241 # Print up to $_max kernel stack traces for processes named $_program
 242 program_stack_traces ()
 243 {
 244     _prog="$1"
 245     _max="${2:-1}"
 246
 247     _count=1
 248     for _pid in $(pidof "$_prog") ; do
 249         [ $_count -le $_max ] || break
 250
 251         # Do this first to avoid racing with process exit
 252         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 253         if [ -n "$_stack" ] ; then
 254             echo "Stack trace for ${_prog}[${_pid}]:"
 255             echo "$_stack"
 256             _count=$(($_count + 1))
 257         fi
 258     done
 259 }
 260
 261 ######################################################
 262 # Check that an RPC service is healthy -
 263 # this includes allowing a certain number of failures
 264 # before marking the NFS service unhealthy.
 265 #
 266 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 267 #
 268 # each triple is a set of 3 arguments: an operator, a
 269 # fail count limit and an action string.
 270 #
 271 # For example:
 272 #
 273 #       nfs_check_rpc_service "lockd" \
 274 #           -ge 15 "verbose restart unhealthy" \
 275 #           -eq 10 "restart:bs"
 276 #
 277 # says that if lockd is down for 15 iterations then do
 278 # a verbose restart of lockd and mark the node unhealthy.
 279 # Before this, after 10 iterations of failure, the
 280 # service is restarted silently in the background.
 281 # Order is important: the number of failures need to be
 282 # specified in reverse order because processing stops
 283 # after the first condition that is true.
 284 ######################################################
 285 nfs_check_rpc_service ()
 286 {
 287     _prog_name="$1" ; shift
 288
 289     if _nfs_check_rpc_common "$_prog_name" ; then
 290         return
 291     fi
 292
 293     while [ -n "$3" ] ; do
 294         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 295             break
 296         fi
 297         shift 3
 298     done
 299 }
 300
 301 # The new way of doing things...
 302 nfs_check_rpc_services ()
 303 {
 304     # Files must end with .check - avoids editor backups, RPM fu, ...
 305     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 306         _t="${_f%.check}"
 307         _prog_name="${_t##*/[0-9][0-9].}"
 308
 309         # If $_prog_name contains '@' then the bit after it is the
 310         # address family.
 311         _family="${_prog_name#*@}"
 312         if [ "$_family" = "$_prog_name" ] ; then
 313             _family=""
 314         else
 315             _prog_name="${_prog_name%@*}"
 316         fi
 317
 318         if _nfs_check_rpc_common "$_prog_name" "$_family" ; then
 319             # This RPC service is up, check next service...
 320             continue
 321         fi
 322
 323         # Check each line in the file in turn until one of the limit
 324         # checks is hit...
 325         while read _cmp _lim _rest ; do
 326             # Skip comments
 327             case "$_cmp" in
 328                 \#*) continue ;;
 329             esac
 330
 331             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 332                 # Limit was hit on this line, no further checking...
 333                 break
 334             fi
 335         done <"$_f"
 336     done
 337 }
 338
 339 _nfs_check_rpc_common ()
 340 {
 341     _prog_name="$1"
 342     _family="$2"
 343
 344     # Some platforms don't have separate programs for all services.
 345     case "$_prog_name" in
 346         statd)
 347             type "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 348     esac
 349
 350     case "$_prog_name" in
 351         nfsd)
 352             _rpc_prog=nfs
 353             _version=3
 354             ;;
 355         mountd)
 356             _rpc_prog=mountd
 357             _version=1
 358             ;;
 359         rquotad)
 360             _rpc_prog=rquotad
 361             _version=1
 362             ;;
 363         lockd)
 364             _rpc_prog=nlockmgr
 365             _version=4
 366             ;;
 367         statd)
 368             _rpc_prog=status
 369             _version=1
 370             ;;
 371         *)
 372             echo "Internal error: unknown RPC program \"$_prog_name\"."
 373             exit 1
 374     esac
 375
 376     _service_name="nfs_${_prog_name}${_family:+_}${_family}"
 377
 378     if ctdb_check_rpc "$_rpc_prog" "$_version" "$_family" >/dev/null ; then
 379         ctdb_counter_init "$_service_name"
 380         return 0
 381     fi
 382
 383     ctdb_counter_incr "$_service_name"
 384
 385     return 1
 386 }
 387
 388 _nfs_check_rpc_action ()
 389 {
 390     _cmp="$1"
 391     _limit="$2"
 392     _actions="$3"
 393
 394     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 395         return 1
 396     fi
 397
 398     for _action in $_actions ; do
 399         case "$_action" in
 400             verbose)
 401                 echo "$ctdb_check_rpc_out"
 402                 ;;
 403             restart)
 404                 _nfs_restart_rpc_service "$_prog_name"
 405                 ;;
 406             restart:b)
 407                 _nfs_restart_rpc_service "$_prog_name" true
 408                 ;;
 409             unhealthy)
 410                 exit 1
 411                 ;;
 412             *)
 413                 echo "Internal error: unknown action \"$_action\"."
 414                 exit 1
 415         esac
 416     done
 417
 418     return 0
 419 }
 420
 421 _nfs_restart_rpc_service ()
 422 {
 423     _prog_name="$1"
 424     _background="${2:-false}"
 425
 426     if $_background ; then
 427         _maybe_background="background_with_logging"
 428     else
 429         _maybe_background=""
 430     fi
 431
 432     _p="rpc.${_prog_name}"
 433
 434     case "$_prog_name" in
 435         nfsd)
 436             echo "Trying to restart NFS service"
 437             $_maybe_background startstop_nfs restart
 438             ;;
 439         mountd)
 440             echo "Trying to restart $_prog_name [${_p}]"
 441             killall -q -9 "$_p"
 442             nfs_dump_some_threads "$_p"
 443             $_maybe_background $_p $RPCMOUNTDOPTS \
 444                                ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 445             ;;
 446         rquotad)
 447             echo "Trying to restart $_prog_name [${_p}]"
 448             killall -q -9 "$_p"
 449             nfs_dump_some_threads "$_p"
 450             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 451             ;;
 452         lockd)
 453             echo "Trying to restart lock manager service"
 454             $_maybe_background startstop_nfslock restart
 455             ;;
 456         statd)
 457             echo "Trying to restart $_prog_name [${_p}]"
 458             killall -q -9 "$_p"
 459             nfs_dump_some_threads "$_p"
 460             $_maybe_background $_p \
 461                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 462                 ${STATD_PORT:+-p} $STATD_PORT \
 463                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 464             ;;
 465         *)
 466             echo "Internal error: unknown RPC program \"$_prog_name\"."
 467             exit 1
 468     esac
 469 }
 470
 471 ######################################################
 472 # check that a rpc server is registered with portmap
 473 # and responding to requests
 474 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 475 ######################################################
 476 ctdb_check_rpc ()
 477 {
 478     progname="$1"
 479     version="$2"
 480     _family="${3:-tcp}"
 481
 482     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 483
 484     if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost $progname $version 2>&1) ; then
 485         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 486 $ctdb_check_rpc_out"
 487         echo "$ctdb_check_rpc_out"
 488         return 1
 489     fi
 490 }
 491
 492 ######################################################
 493 # Ensure $service_name is set
 494 assert_service_name ()
 495 {
 496     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 497 }
 498
 499 ######################################################
 500 # check a set of directories is available
 501 # return 1 on a missing directory
 502 # directories are read from stdin
 503 ######################################################
 504 ctdb_check_directories_probe()
 505 {
 506     while IFS="" read d ; do
 507         case "$d" in
 508             *%*)
 509                 continue
 510                 ;;
 511             *)
 512                 [ -d "${d}/." ] || return 1
 513         esac
 514     done
 515 }
 516
 517 ######################################################
 518 # check a set of directories is available
 519 # directories are read from stdin
 520 ######################################################
 521 ctdb_check_directories()
 522 {
 523     ctdb_check_directories_probe || {
 524         echo "ERROR: $service_name directory \"$d\" not available"
 525         exit 1
 526     }
 527 }
 528
 529 ######################################################
 530 # check a set of tcp ports
 531 # usage: ctdb_check_tcp_ports <ports...>
 532 ######################################################
 533
 534 # This flag file is created when a service is initially started.  It
 535 # is deleted the first time TCP port checks for that service succeed.
 536 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 537 # message if a port check fails.
 538 _ctdb_check_tcp_common ()
 539 {
 540     assert_service_name
 541     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 542 }
 543
 544 ctdb_check_tcp_init ()
 545 {
 546     _ctdb_check_tcp_common
 547     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 548     touch "$_ctdb_service_started_file"
 549 }
 550
 551 # Check whether something is listening on all of the given TCP ports
 552 # using the "ctdb checktcpport" command.
 553 ctdb_check_tcp_ports()
 554 {
 555     if [ -z "$1" ] ; then
 556         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 557         exit 1
 558     fi
 559
 560     for _p ; do  # process each function argument (port)
 561         _cmd="ctdb checktcpport $_p"
 562         _out=$($_cmd 2>&1)
 563         _ret=$?
 564         case "$_ret" in
 565             0)
 566                 _ctdb_check_tcp_common
 567                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 568                     echo "ERROR: $service_name tcp port $_p is not responding"
 569                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 570                 else
 571                     echo "INFO: $service_name tcp port $_p is not responding"
 572                 fi
 573
 574                 return 1
 575                 ;;
 576             98)
 577                 # Couldn't bind, something already listening, next port...
 578                 continue
 579                 ;;
 580             *)
 581                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 582                 debug <<EOF
 583 ctdb checktcpport (exited with $_ret) with output:
 584 $_out"
 585 EOF
 586                 return $_ret
 587         esac
 588     done
 589
 590     # All ports listening
 591     _ctdb_check_tcp_common
 592     rm -f "$_ctdb_service_started_file"
 593     return 0
 594 }
 595
 596 ######################################################
 597 # check a unix socket
 598 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 599 ######################################################
 600 ctdb_check_unix_socket() {
 601     socket_path="$1"
 602     [ -z "$socket_path" ] && return
 603
 604     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 605         echo "ERROR: $service_name socket $socket_path not found"
 606         return 1
 607     fi
 608 }
 609
 610 ######################################################
 611 # check a command returns zero status
 612 # usage: ctdb_check_command <command>
 613 ######################################################
 614 ctdb_check_command ()
 615 {
 616     _out=$("$@" 2>&1) || {
 617         echo "ERROR: $* returned error"
 618         echo "$_out" | debug
 619         exit 1
 620     }
 621 }
 622
 623 ################################################
 624 # kill off any TCP connections with the given IP
 625 ################################################
 626 kill_tcp_connections ()
 627 {
 628     _ip="$1"
 629
 630     _oneway=false
 631     if [ "$2" = "oneway" ] ; then
 632         _oneway=true
 633     fi
 634
 635     get_tcp_connections_for_ip "$_ip" | {
 636         _killcount=0
 637         _connections=""
 638         _nl="
 639 "
 640         while read _dst _src; do
 641             _destport="${_dst##*:}"
 642             __oneway=$_oneway
 643             case $_destport in
 644                 # we only do one-way killtcp for CIFS
 645                 139|445) __oneway=true ;;
 646             esac
 647
 648             echo "Killing TCP connection $_src $_dst"
 649             _connections="${_connections}${_nl}${_src} ${_dst}"
 650             if ! $__oneway ; then
 651                 _connections="${_connections}${_nl}${_dst} ${_src}"
 652             fi
 653
 654             _killcount=$(($_killcount + 1))
 655         done
 656
 657         if [ $_killcount -eq 0 ] ; then
 658             return
 659         fi
 660
 661         echo "$_connections" | ctdb killtcp || {
 662             echo "Failed to send killtcp control"
 663             return
 664         }
 665
 666         _count=0
 667         while : ; do
 668             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 669
 670             if [ $_remaining -eq 0 ] ; then
 671                 echo "Killed $_killcount TCP connections to released IP $_ip"
 672                 return
 673             fi
 674
 675             _count=$(($_count + 1))
 676             if [ $_count -gt 3 ] ; then
 677                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 678                 return
 679             fi
 680
 681             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 682             sleep 1
 683         done
 684     }
 685 }
 686
 687 ##################################################################
 688 # kill off the local end for any TCP connections with the given IP
 689 ##################################################################
 690 kill_tcp_connections_local_only ()
 691 {
 692     kill_tcp_connections "$1" "oneway"
 693 }
 694
 695 ##################################################################
 696 # tickle any TCP connections with the given IP
 697 ##################################################################
 698 tickle_tcp_connections ()
 699 {
 700     _ip="$1"
 701
 702     get_tcp_connections_for_ip "$_ip" |
 703     {
 704         _failed=false
 705
 706         while read dest src; do
 707             echo "Tickle TCP connection $src $dest"
 708             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 709             echo "Tickle TCP connection $dest $src"
 710             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 711         done
 712
 713         if $_failed ; then
 714             echo "Failed to send tickle control"
 715         fi
 716     }
 717 }
 718
 719 get_tcp_connections_for_ip ()
 720 {
 721     _ip="$1"
 722
 723     netstat -tn | awk -v ip=$_ip \
 724         'index($1, "tcp") == 1 && \
 725          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 726          && $6 == "ESTABLISHED" \
 727          {print $4" "$5}'
 728 }
 729
 730 ##################################################################
 731 # use statd-callout to update NFS lock info
 732 ##################################################################
 733 nfs_update_lock_info ()
 734 {
 735     if [ -x "$CTDB_BASE/statd-callout" ] ; then
 736         "$CTDB_BASE/statd-callout" update
 737     fi
 738 }
 739
 740 ########################################################
 741 # start/stop the Ganesha nfs service
 742 ########################################################
 743 startstop_ganesha()
 744 {
 745     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 746     case "$1" in
 747         start)
 748             service "$_service_name" start
 749             ;;
 750         stop)
 751             service "$_service_name" stop
 752             ;;
 753         restart)
 754             service "$_service_name" stop
 755             nfs_dump_some_threads "rpc.statd"
 756             service "$_service_name" start
 757             ;;
 758     esac
 759 }
 760
 761 ########################################################
 762 # start/stop the nfs service on different platforms
 763 ########################################################
 764 startstop_nfs() {
 765         PLATFORM="unknown"
 766         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 767                 PLATFORM="sles"
 768         }
 769         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 770             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 771                 PLATFORM="rhel"
 772         }
 773
 774         case $PLATFORM in
 775         sles)
 776                 case $1 in
 777                 start)
 778                         service nfsserver start
 779                         ;;
 780                 stop)
 781                         service nfsserver stop > /dev/null 2>&1
 782                         ;;
 783                 restart)
 784                         set_proc "fs/nfsd/threads" 0
 785                         service nfsserver stop > /dev/null 2>&1
 786                         pkill -9 nfsd
 787                         nfs_dump_some_threads
 788                         service nfsserver start
 789                         ;;
 790                 esac
 791                 ;;
 792         rhel)
 793                 case $1 in
 794                 start)
 795                         service nfslock start
 796                         service nfs start
 797                         ;;
 798                 stop)
 799                         service nfs stop
 800                         service nfslock stop
 801                         ;;
 802                 restart)
 803                         set_proc "fs/nfsd/threads" 0
 804                         service nfs stop > /dev/null 2>&1
 805                         service nfslock stop > /dev/null 2>&1
 806                         pkill -9 nfsd
 807                         nfs_dump_some_threads
 808                         service nfslock start
 809                         service nfs start
 810                         ;;
 811                 esac
 812                 ;;
 813         *)
 814                 echo "Unknown platform. NFS is not supported with ctdb"
 815                 exit 1
 816                 ;;
 817         esac
 818 }
 819
 820 # Dump up to the configured number of nfsd thread backtraces.
 821 nfs_dump_some_threads ()
 822 {
 823     _prog="${1:-nfsd}"
 824
 825     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 826     [ $_num -gt 0 ] || return 0
 827
 828     program_stack_traces "$_prog" $_num
 829 }
 830
 831 ########################################################
 832 # start/stop the nfs lockmanager service on different platforms
 833 ########################################################
 834 startstop_nfslock() {
 835         PLATFORM="unknown"
 836         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 837                 PLATFORM="sles"
 838         }
 839         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 840             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 841                 PLATFORM="rhel"
 842         }
 843
 844         case $PLATFORM in
 845         sles)
 846                 # for sles there is no service for lockmanager
 847                 # so we instead just shutdown/restart nfs
 848                 case $1 in
 849                 start)
 850                         service nfsserver start
 851                         ;;
 852                 stop)
 853                         service nfsserver stop > /dev/null 2>&1
 854                         ;;
 855                 restart)
 856                         service nfsserver stop > /dev/null 2>&1
 857                         service nfsserver start
 858                         ;;
 859                 esac
 860                 ;;
 861         rhel)
 862                 case $1 in
 863                 start)
 864                         service nfslock start
 865                         ;;
 866                 stop)
 867                         service nfslock stop > /dev/null 2>&1
 868                         ;;
 869                 restart)
 870                         service nfslock stop > /dev/null 2>&1
 871                         service nfslock start
 872                         ;;
 873                 esac
 874                 ;;
 875         *)
 876                 echo "Unknown platform. NFS locking is not supported with ctdb"
 877                 exit 1
 878                 ;;
 879         esac
 880 }
 881
 882 ########################################################
 883
 884 add_ip_to_iface ()
 885 {
 886     _iface=$1
 887     _ip=$2
 888     _maskbits=$3
 889
 890     # Ensure interface is up
 891     ip link set "$_iface" up || \
 892         die "Failed to bringup interface $_iface"
 893
 894     # Only need to define broadcast for IPv4
 895     case "$ip" in
 896         *:*) _bcast=""      ;;
 897         *)   _bcast="brd +" ;;
 898     esac
 899
 900     ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
 901         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 902         return 1
 903     }
 904
 905     # Wait 5 seconds for IPv6 addresses to stop being tentative...
 906     if [ -z "$_bcast" ] ; then
 907         for _x in $(seq 1 10) ; do
 908             ip addr show to "${_ip}/128" | grep -q "tentative" || break
 909             sleep 0.5
 910         done
 911
 912         # If the address was a duplicate then it won't be on the
 913         # interface so flag an error.
 914         _t=$(ip addr show to "${_ip}/128")
 915         case "$_t" in
 916             "")
 917                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 918                 return 1
 919                 ;;
 920             *tentative*|*dadfailed*)
 921                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 922                 ip addr del "$_ip/$_maskbits" dev "$_iface"
 923                 return 1
 924                 ;;
 925         esac
 926     fi
 927 }
 928
 929 delete_ip_from_iface()
 930 {
 931     _iface=$1
 932     _ip=$2
 933     _maskbits=$3
 934
 935     # This could be set globally for all interfaces but it is probably
 936     # better to avoid surprises, so limit it the interfaces where CTDB
 937     # has public IP addresses.  There isn't anywhere else convenient
 938     # to do this so just set it each time.  This is much cheaper than
 939     # remembering and re-adding secondaries.
 940     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 941
 942     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 943         echo "Failed to del $_ip on dev $_iface"
 944         return 1
 945     }
 946 }
 947
 948 # If the given IP is hosted then print 2 items: maskbits and iface
 949 ip_maskbits_iface ()
 950 {
 951     _addr="$1"
 952
 953     case "$_addr" in
 954         *:*) _family="inet6" ; _bits=128 ;;
 955         *)   _family="inet"  ; _bits=32  ;;
 956     esac
 957
 958     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 959         awk -v family="${_family}" \
 960             'NR == 1 { iface = $2; sub(":$", "", iface) } \
 961              $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
 962                            print mask, iface, family }'
 963 }
 964
 965 drop_ip ()
 966 {
 967     _addr="${1%/*}"  # Remove optional maskbits
 968
 969     set -- $(ip_maskbits_iface $_addr)
 970     if [ -n "$1" ] ; then
 971         _maskbits="$1"
 972         _iface="$2"
 973         echo "Removing public address $_addr/$_maskbits from device $_iface"
 974         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 975     fi
 976 }
 977
 978 drop_all_public_ips ()
 979 {
 980     while read _ip _x ; do
 981         drop_ip "$_ip"
 982     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 983 }
 984
 985 flush_route_cache ()
 986 {
 987     set_proc_maybe sys/net/ipv4/route/flush 1
 988     set_proc_maybe sys/net/ipv6/route/flush 1
 989 }
 990
 991 ########################################################
 992 # Simple counters
 993 _ctdb_counter_common () {
 994     _service_name="${1:-${service_name:-${script_name}}}"
 995     _counter_file="$ctdb_fail_dir/$_service_name"
 996     mkdir -p "${_counter_file%/*}" # dirname
 997 }
 998 ctdb_counter_init () {
 999     _ctdb_counter_common "$1"
1000
1001     >"$_counter_file"
1002 }
1003 ctdb_counter_incr () {
1004     _ctdb_counter_common "$1"
1005
1006     # unary counting!
1007     echo -n 1 >> "$_counter_file"
1008 }
1009 ctdb_check_counter () {
1010     _msg="${1:-error}"  # "error"  - anything else is silent on fail
1011     _op="${2:--ge}"  # an integer operator supported by test
1012     _limit="${3:-${service_fail_limit}}"
1013     shift 3
1014     _ctdb_counter_common "$1"
1015
1016     # unary counting!
1017     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
1018     _hit=false
1019     if [ "$_op" != "%" ] ; then
1020         if [ $_size $_op $_limit ] ; then
1021             _hit=true
1022         fi
1023     else
1024         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
1025             _hit=true
1026         fi
1027     fi
1028     if $_hit ; then
1029         if [ "$_msg" = "error" ] ; then
1030             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1031             exit 1
1032         else
1033             return 1
1034         fi
1035     fi
1036 }
1037
1038 ########################################################
1039
1040 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1041 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1042
1043 ctdb_setup_service_state_dir ()
1044 {
1045     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1046     mkdir -p "$service_state_dir" || {
1047         echo "Error creating state dir \"$service_state_dir\""
1048         exit 1
1049     }
1050 }
1051
1052 ########################################################
1053 # Managed status history, for auto-start/stop
1054
1055 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1056
1057 _ctdb_managed_common ()
1058 {
1059     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1060 }
1061
1062 ctdb_service_managed ()
1063 {
1064     _ctdb_managed_common
1065     mkdir -p "$ctdb_managed_dir"
1066     touch "$_ctdb_managed_file"
1067 }
1068
1069 ctdb_service_unmanaged ()
1070 {
1071     _ctdb_managed_common
1072     rm -f "$_ctdb_managed_file"
1073 }
1074
1075 is_ctdb_previously_managed_service ()
1076 {
1077     _ctdb_managed_common
1078     [ -f "$_ctdb_managed_file" ]
1079 }
1080
1081 ########################################################
1082 # Check and set status
1083
1084 log_status_cat ()
1085 {
1086     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1087 }
1088
1089 ctdb_checkstatus ()
1090 {
1091     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1092         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1093         return 1
1094     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1095         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1096         return 2
1097     else
1098         return 0
1099     fi
1100 }
1101
1102 ctdb_setstatus ()
1103 {
1104     d="$ctdb_status_dir/$script_name"
1105     case "$1" in
1106         unhealthy|banned)
1107             mkdir -p "$d"
1108             cat "$2" >"$d/$1"
1109             ;;
1110         *)
1111             for i in "banned" "unhealthy" ; do
1112                 rm -f "$d/$i"
1113             done
1114             ;;
1115     esac
1116 }
1117
1118 ##################################################################
1119 # Reconfigure a service on demand
1120
1121 _ctdb_service_reconfigure_common ()
1122 {
1123     _d="$ctdb_status_dir/${service_name}"
1124     mkdir -p "$_d"
1125     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1126 }
1127
1128 ctdb_service_needs_reconfigure ()
1129 {
1130     _ctdb_service_reconfigure_common
1131     [ -e "$_ctdb_service_reconfigure_flag" ]
1132 }
1133
1134 ctdb_service_set_reconfigure ()
1135 {
1136     _ctdb_service_reconfigure_common
1137     >"$_ctdb_service_reconfigure_flag"
1138 }
1139
1140 ctdb_service_unset_reconfigure ()
1141 {
1142     _ctdb_service_reconfigure_common
1143     rm -f "$_ctdb_service_reconfigure_flag"
1144 }
1145
1146 ctdb_service_reconfigure ()
1147 {
1148     echo "Reconfiguring service \"${service_name}\"..."
1149     ctdb_service_unset_reconfigure
1150     service_reconfigure || return $?
1151     ctdb_counter_init
1152 }
1153
1154 # Default service_reconfigure() function does nothing.
1155 service_reconfigure ()
1156 {
1157     :
1158 }
1159
1160 ctdb_reconfigure_take_lock ()
1161 {
1162     _ctdb_service_reconfigure_common
1163     _lock="${_d}/reconfigure_lock"
1164     mkdir -p "${_lock%/*}" # dirname
1165     touch "$_lock"
1166
1167     (
1168         flock 0
1169         # This is overkill but will work if we need to extend this to
1170         # allow certain events to run multiple times in parallel
1171         # (e.g. takeip) and write multiple PIDs to the file.
1172         read _locker_event
1173         if [ -n "$_locker_event" ] ; then
1174             while read _pid ; do
1175                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1176                     kill -0 "$_pid" 2>/dev/null ; then
1177                     exit 1
1178                 fi
1179             done
1180         fi
1181
1182         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1183         exit 0
1184     ) <"$_lock"
1185 }
1186
1187 ctdb_reconfigure_release_lock ()
1188 {
1189     _ctdb_service_reconfigure_common
1190     _lock="${_d}/reconfigure_lock"
1191
1192     rm -f "$_lock"
1193 }
1194
1195 ctdb_replay_monitor_status ()
1196 {
1197     echo "Replaying previous status for this script due to reconfigure..."
1198     # Leading separator ('|') is missing in some versions...
1199     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1200     # Output looks like this:
1201     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1202     # This is the cheapest way of getting fields in the middle.
1203     set -- $(IFS="|" ; echo $_out)
1204     _code="$3"
1205     _status="$4"
1206     # The error output field can include colons so we'll try to
1207     # preserve them.  The weak checking at the beginning tries to make
1208     # this work for both broken (no leading '|') and fixed output.
1209     _out="${_out%|}"
1210     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1211     case "$_status" in
1212         OK) : ;;  # Do nothing special.
1213         TIMEDOUT)
1214             # Recast this as an error, since we can't exit with the
1215             # correct negative number.
1216             _code=1
1217             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1218             ;;
1219         DISABLED)
1220             # Recast this as an OK, since we can't exit with the
1221             # correct negative number.
1222             _code=0
1223             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1224             ;;
1225         *) : ;;  # Must be ERROR, do nothing special.
1226     esac
1227     if [ -n "$_err_out" ] ; then
1228         echo "$_err_out"
1229     fi
1230     exit $_code
1231 }
1232
1233 ctdb_service_check_reconfigure ()
1234 {
1235     assert_service_name
1236
1237     # We only care about some events in this function.  For others we
1238     # return now.
1239     case "$event_name" in
1240         monitor|ipreallocated|reconfigure) : ;;
1241         *) return 0 ;;
1242     esac
1243
1244     if ctdb_reconfigure_take_lock ; then
1245         # No events covered by this function are running, so proceed
1246         # with gay abandon.
1247         case "$event_name" in
1248             reconfigure)
1249                 (ctdb_service_reconfigure)
1250                 exit $?
1251                 ;;
1252             ipreallocated)
1253                 if ctdb_service_needs_reconfigure ; then
1254                     ctdb_service_reconfigure
1255                 fi
1256                 ;;
1257         esac
1258
1259         ctdb_reconfigure_release_lock
1260     else
1261         # Somebody else is running an event we don't want to collide
1262         # with.  We proceed with caution.
1263         case "$event_name" in
1264             reconfigure)
1265                 # Tell whoever called us to retry.
1266                 exit 2
1267                 ;;
1268             ipreallocated)
1269                 # Defer any scheduled reconfigure and just run the
1270                 # rest of the ipreallocated event, as per the
1271                 # eventscript.  There's an assumption here that the
1272                 # event doesn't depend on any scheduled reconfigure.
1273                 # This is true in the current code.
1274                 return 0
1275                 ;;
1276             monitor)
1277                 # There is most likely a reconfigure in progress so
1278                 # the service is possibly unstable.  As above, we
1279                 # defer any scheduled reconfigured.  We also replay
1280                 # the previous monitor status since that's the best
1281                 # information we have.
1282                 ctdb_replay_monitor_status
1283                 ;;
1284         esac
1285     fi
1286 }
1287
1288 ##################################################################
1289 # Does CTDB manage this service? - and associated auto-start/stop
1290
1291 ctdb_compat_managed_service ()
1292 {
1293     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1294         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1295     fi
1296 }
1297
1298 is_ctdb_managed_service ()
1299 {
1300     assert_service_name
1301
1302     # $t is used just for readability and to allow better accurate
1303     # matching via leading/trailing spaces
1304     t=" $CTDB_MANAGED_SERVICES "
1305
1306     # Return 0 if "<space>$service_name<space>" appears in $t
1307     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1308         return 0
1309     fi
1310
1311     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1312     # backward compatibility and try again.
1313     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1314     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1315     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1316     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1317     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1318     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1319     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1320     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1321     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1322
1323     t=" $CTDB_MANAGED_SERVICES "
1324
1325     # Return 0 if "<space>$service_name<space>" appears in $t
1326     [ "${t#* ${service_name} }" != "${t}" ]
1327 }
1328
1329 ctdb_start_stop_service ()
1330 {
1331     assert_service_name
1332
1333     # Allow service-start/service-stop pseudo-events to start/stop
1334     # services when we're not auto-starting/stopping and we're not
1335     # monitoring.
1336     case "$event_name" in
1337         service-start)
1338             if is_ctdb_managed_service ; then
1339                 die 'service-start event not permitted when service is managed'
1340             fi
1341             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1342                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1343             fi
1344             ctdb_service_start
1345             exit $?
1346             ;;
1347         service-stop)
1348             if is_ctdb_managed_service ; then
1349                 die 'service-stop event not permitted when service is managed'
1350             fi
1351             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1352                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1353             fi
1354             ctdb_service_stop
1355             exit $?
1356             ;;
1357     esac
1358
1359     # Do nothing unless configured to...
1360     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1361
1362     [ "$event_name" = "monitor" ] || return 0
1363
1364     if is_ctdb_managed_service ; then
1365         if ! is_ctdb_previously_managed_service ; then
1366             echo "Starting service \"$service_name\" - now managed"
1367             background_with_logging ctdb_service_start
1368             exit $?
1369         fi
1370     else
1371         if is_ctdb_previously_managed_service ; then
1372             echo "Stopping service \"$service_name\" - no longer managed"
1373             background_with_logging ctdb_service_stop
1374             exit $?
1375         fi
1376     fi
1377 }
1378
1379 ctdb_service_start ()
1380 {
1381     # The service is marked managed if we've ever tried to start it.
1382     ctdb_service_managed
1383
1384     service_start || return $?
1385
1386     ctdb_counter_init
1387     ctdb_check_tcp_init
1388 }
1389
1390 ctdb_service_stop ()
1391 {
1392     ctdb_service_unmanaged
1393     service_stop
1394 }
1395
1396 # Default service_start() and service_stop() functions.
1397
1398 # These may be overridden in an eventscript.
1399 service_start ()
1400 {
1401     service "$service_name" start
1402 }
1403
1404 service_stop ()
1405 {
1406     service "$service_name" stop
1407 }
1408
1409 ##################################################################
1410
1411 ctdb_standard_event_handler ()
1412 {
1413     case "$1" in
1414         status)
1415             ctdb_checkstatus
1416             exit
1417             ;;
1418         setstatus)
1419             shift
1420             ctdb_setstatus "$@"
1421             exit
1422             ;;
1423     esac
1424 }
1425
1426 iptables_wrapper ()
1427 {
1428     _family="$1" ; shift
1429     if [ "$_family" = "inet6" ] ; then
1430         _iptables_cmd="ip6tables"
1431     else
1432         _iptables_cmd="iptables"
1433     fi
1434
1435     # iptables doesn't like being re-entered, so flock-wrap it.
1436     flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1437 }
1438
1439 # AIX (and perhaps others?) doesn't have mktemp
1440 if ! type mktemp >/dev/null 2>&1 ; then
1441     mktemp ()
1442     {
1443         _dir=false
1444         if [ "$1" = "-d" ] ; then
1445             _dir=true
1446             shift
1447         fi
1448         _d="${TMPDIR:-/tmp}"
1449         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1450             md5sum | \
1451             sed -e 's@\(..........\).*@\1@')
1452         _t="${_d}/tmp.${_hex10}"
1453         (
1454             umask 077
1455             if $_dir ; then
1456                 mkdir "$_t"
1457             else
1458                 >"$_t"
1459             fi
1460         )
1461         echo "$_t"
1462     }
1463 fi
1464
1465 ########################################################
1466 # tickle handling
1467 ########################################################
1468
1469 update_tickles ()
1470 {
1471         _port="$1"
1472
1473         tickledir="$CTDB_VARDIR/state/tickles"
1474         mkdir -p "$tickledir"
1475
1476         ctdb_get_pnn
1477
1478         # What public IPs do I hold?
1479         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$pnn '$3 == pnn {print $2}')
1480
1481         # IPs as a regexp choice
1482         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1483
1484         # Record connections to our public IPs in a temporary file
1485         _my_connections="${tickledir}/${_port}.connections"
1486         rm -f "$_my_connections"
1487         netstat -tn |
1488         awk -v destpat="^${_ipschoice}:${_port}\$" \
1489           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1490         sort >"$_my_connections"
1491
1492         # Record our current tickles in a temporary file
1493         _my_tickles="${tickledir}/${_port}.tickles"
1494         rm -f "$_my_tickles"
1495         for _i in $_ips ; do
1496                 ctdb -X gettickles $_i $_port |
1497                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1498         done |
1499         sort >"$_my_tickles"
1500
1501         # Add tickles for connections that we haven't already got tickles for
1502         comm -23 "$_my_connections" "$_my_tickles" |
1503         while read _src _dst ; do
1504                 ctdb addtickle $_src $_dst
1505         done
1506
1507         # Remove tickles for connections that are no longer there
1508         comm -13 "$_my_connections" "$_my_tickles" |
1509         while read _src _dst ; do
1510                 ctdb deltickle $_src $_dst
1511         done
1512
1513         rm -f "$_my_connections" "$_my_tickles"
1514 }
1515
1516 ########################################################
1517 # load a site local config file
1518 ########################################################
1519
1520 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1521         . "$CTDB_RC_LOCAL"
1522 }
1523
1524 [ -x $CTDB_BASE/rc.local ] && {
1525         . $CTDB_BASE/rc.local
1526 }
1527
1528 [ -d $CTDB_BASE/rc.local.d ] && {
1529         for i in $CTDB_BASE/rc.local.d/* ; do
1530                 [ -x "$i" ] && . "$i"
1531         done
1532 }
1533
1534 script_name="${0##*/}"       # basename
1535 service_fail_limit=1
1536 event_name="$1"