ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         elif ! tty -s ; then
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     fi
  72 }
  73
  74 die ()
  75 {
  76     _msg="$1"
  77     _rc="${2:-1}"
  78
  79     echo "$_msg"
  80     exit $_rc
  81 }
  82
  83 # Log given message or stdin to either syslog or a CTDB log file
  84 # $1 is the tag passed to logger if syslog is in use.
  85 script_log ()
  86 {
  87     _tag="$1" ; shift
  88
  89     case "$CTDB_LOGGING" in
  90         file:*|"")
  91             if [ -n "$CTDB_LOGGING" ] ; then
  92                 _file="${CTDB_LOGGING#file:}"
  93             else
  94                 _file="/var/log/log.ctdb"
  95             fi
  96             {
  97                 if [ -n "$*" ] ; then
  98                     echo "$*"
  99                 else
 100                     cat
 101                 fi
 102             } >>"$_file"
 103             ;;
 104         *)
 105             # Handle all syslog:* variants here too.  There's no tool to do
 106             # the lossy things, so just use logger.
 107             logger -t "ctdbd: ${_tag}" $*
 108             ;;
 109     esac
 110 }
 111
 112 # When things are run in the background in an eventscript then logging
 113 # output might get lost.  This is the "solution".  :-)
 114 background_with_logging ()
 115 {
 116     (
 117         "$@" 2>&1 </dev/null |
 118         script_log "${script_name}&"
 119     )&
 120
 121     return 0
 122 }
 123
 124 ##############################################################
 125 # check number of args for different events
 126 ctdb_check_args ()
 127 {
 128     case "$1" in
 129         takeip|releaseip)
 130             if [ $# != 4 ]; then
 131                 echo "ERROR: must supply interface, IP and maskbits"
 132                 exit 1
 133             fi
 134             ;;
 135         updateip)
 136             if [ $# != 5 ]; then
 137                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 138                 exit 1
 139             fi
 140             ;;
 141     esac
 142 }
 143
 144 ##############################################################
 145 # determine on what type of system (init style) we are running
 146 detect_init_style()
 147 {
 148     # only do detection if not already set:
 149     [ -z "$CTDB_INIT_STYLE" ] || return
 150
 151     if [ -x /sbin/startproc ]; then
 152         CTDB_INIT_STYLE="suse"
 153     elif [ -x /sbin/start-stop-daemon ]; then
 154         CTDB_INIT_STYLE="debian"
 155     else
 156         CTDB_INIT_STYLE="redhat"
 157     fi
 158 }
 159
 160 ######################################################
 161 # simulate /sbin/service on platforms that don't have it
 162 # _service() makes it easier to hook the service() function for
 163 # testing.
 164 _service ()
 165 {
 166   _service_name="$1"
 167   _op="$2"
 168
 169   # do nothing, when no service was specified
 170   [ -z "$_service_name" ] && return
 171
 172   if [ -x /sbin/service ]; then
 173       $_nice /sbin/service "$_service_name" "$_op"
 174   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 175       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 176   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 177       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 178   fi
 179 }
 180
 181 service()
 182 {
 183     _nice=""
 184     _service "$@"
 185 }
 186
 187 ######################################################
 188 # simulate /sbin/service (niced) on platforms that don't have it
 189 nice_service()
 190 {
 191     _nice="nice"
 192     _service "$@"
 193 }
 194
 195 ######################################################
 196 # wrapper around /proc/ settings to allow them to be hooked
 197 # for testing
 198 # 1st arg is relative path under /proc/, 2nd arg is value to set
 199 set_proc ()
 200 {
 201     echo "$2" >"/proc/$1"
 202 }
 203
 204 ######################################################
 205 # wrapper around getting file contents from /proc/ to allow
 206 # this to be hooked for testing
 207 # 1st arg is relative path under /proc/
 208 get_proc ()
 209 {
 210     cat "/proc/$1"
 211 }
 212
 213 ######################################################
 214 # Print up to $_max kernel stack traces for processes named $_program
 215 program_stack_traces ()
 216 {
 217     _prog="$1"
 218     _max="${2:-1}"
 219
 220     _count=1
 221     for _pid in $(pidof "$_prog") ; do
 222         [ $_count -le $_max ] || break
 223
 224         # Do this first to avoid racing with process exit
 225         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 226         if [ -n "$_stack" ] ; then
 227             echo "Stack trace for ${_prog}[${_pid}]:"
 228             echo "$_stack"
 229             _count=$(($_count + 1))
 230         fi
 231     done
 232 }
 233
 234 ######################################################
 235 # Check that an RPC service is healthy -
 236 # this includes allowing a certain number of failures
 237 # before marking the NFS service unhealthy.
 238 #
 239 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 240 #
 241 # each triple is a set of 3 arguments: an operator, a
 242 # fail count limit and an action string.
 243 #
 244 # For example:
 245 #
 246 #       nfs_check_rpc_service "lockd" \
 247 #           -ge 15 "verbose restart unhealthy" \
 248 #           -eq 10 "restart:bs"
 249 #
 250 # says that if lockd is down for 15 iterations then do
 251 # a verbose restart of lockd and mark the node unhealthy.
 252 # Before this, after 10 iterations of failure, the
 253 # service is restarted silently in the background.
 254 # Order is important: the number of failures need to be
 255 # specified in reverse order because processing stops
 256 # after the first condition that is true.
 257 ######################################################
 258 nfs_check_rpc_service ()
 259 {
 260     _prog_name="$1" ; shift
 261
 262     if _nfs_check_rpc_common "$_prog_name" ; then
 263         return
 264     fi
 265
 266     while [ -n "$3" ] ; do
 267         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 268             break
 269         fi
 270         shift 3
 271     done
 272 }
 273
 274 # The new way of doing things...
 275 nfs_check_rpc_services ()
 276 {
 277     # Files must end with .check - avoids editor backups, RPM fu, ...
 278     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 279         _t="${_f%.check}"
 280         _prog_name="${_t##*/[0-9][0-9].}"
 281
 282         if _nfs_check_rpc_common "$_prog_name" ; then
 283             # This RPC service is up, check next service...
 284             continue
 285         fi
 286
 287         # Check each line in the file in turn until one of the limit
 288         # checks is hit...
 289         while read _cmp _lim _rest ; do
 290             # Skip comments
 291             case "$_cmp" in
 292                 \#*) continue ;;
 293             esac
 294
 295             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 296                 # Limit was hit on this line, no further checking...
 297                 break
 298             fi
 299         done <"$_f"
 300     done
 301 }
 302
 303 _nfs_check_rpc_common ()
 304 {
 305     _prog_name="$1"
 306
 307     # Some platforms don't have separate programs for all services.
 308     case "$_prog_name" in
 309         statd)
 310             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 311     esac
 312
 313     case "$_prog_name" in
 314         nfsd)
 315             _rpc_prog=nfs
 316             _version=3
 317             ;;
 318         mountd)
 319             _rpc_prog=mountd
 320             _version=1
 321             ;;
 322         rquotad)
 323             _rpc_prog=rquotad
 324             _version=1
 325             ;;
 326         lockd)
 327             _rpc_prog=nlockmgr
 328             _version=4
 329             ;;
 330         statd)
 331             _rpc_prog=status
 332             _version=1
 333             ;;
 334         *)
 335             echo "Internal error: unknown RPC program \"$_prog_name\"."
 336             exit 1
 337     esac
 338
 339     _service_name="nfs_${_prog_name}"
 340
 341     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 342         ctdb_counter_init "$_service_name"
 343         return 0
 344     fi
 345
 346     ctdb_counter_incr "$_service_name"
 347
 348     return 1
 349 }
 350
 351 _nfs_check_rpc_action ()
 352 {
 353     _cmp="$1"
 354     _limit="$2"
 355     _actions="$3"
 356
 357     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 358         return 1
 359     fi
 360
 361     for _action in $_actions ; do
 362         case "$_action" in
 363             verbose)
 364                 echo "$ctdb_check_rpc_out"
 365                 ;;
 366             restart)
 367                 _nfs_restart_rpc_service "$_prog_name"
 368                 ;;
 369             restart:b)
 370                 _nfs_restart_rpc_service "$_prog_name" true
 371                 ;;
 372             unhealthy)
 373                 exit 1
 374                 ;;
 375             *)
 376                 echo "Internal error: unknown action \"$_action\"."
 377                 exit 1
 378         esac
 379     done
 380
 381     return 0
 382 }
 383
 384 _nfs_restart_rpc_service ()
 385 {
 386     _prog_name="$1"
 387     _background="${2:-false}"
 388
 389     if $_background ; then
 390         _maybe_background="background_with_logging"
 391     else
 392         _maybe_background=""
 393     fi
 394
 395     _p="rpc.${_prog_name}"
 396
 397     case "$_prog_name" in
 398         nfsd)
 399             echo "Trying to restart NFS service"
 400             $_maybe_background startstop_nfs restart
 401             ;;
 402         mountd)
 403             echo "Trying to restart $_prog_name [${_p}]"
 404             killall -q -9 "$_p"
 405             nfs_dump_some_threads "$_p"
 406             $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 407             ;;
 408         rquotad)
 409             echo "Trying to restart $_prog_name [${_p}]"
 410             killall -q -9 "$_p"
 411             nfs_dump_some_threads "$_p"
 412             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 413             ;;
 414         lockd)
 415             echo "Trying to restart lock manager service"
 416             $_maybe_background startstop_nfslock restart
 417             ;;
 418         statd)
 419             echo "Trying to restart $_prog_name [${_p}]"
 420             killall -q -9 "$_p"
 421             nfs_dump_some_threads "$_p"
 422             $_maybe_background $_p \
 423                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 424                 ${STATD_PORT:+-p} $STATD_PORT \
 425                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 426             ;;
 427         *)
 428             echo "Internal error: unknown RPC program \"$_prog_name\"."
 429             exit 1
 430     esac
 431 }
 432
 433 ######################################################
 434 # check that a rpc server is registered with portmap
 435 # and responding to requests
 436 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 437 ######################################################
 438 ctdb_check_rpc ()
 439 {
 440     progname="$1"
 441     version="$2"
 442
 443     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 444
 445     if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
 446         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 447 $ctdb_check_rpc_out"
 448         echo "$ctdb_check_rpc_out"
 449         return 1
 450     fi
 451 }
 452
 453 ######################################################
 454 # Ensure $service_name is set
 455 assert_service_name ()
 456 {
 457     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 458 }
 459
 460 ######################################################
 461 # check a set of directories is available
 462 # return 1 on a missing directory
 463 # directories are read from stdin
 464 ######################################################
 465 ctdb_check_directories_probe()
 466 {
 467     while IFS="" read d ; do
 468         case "$d" in
 469             *%*)
 470                 continue
 471                 ;;
 472             *)
 473                 [ -d "${d}/." ] || return 1
 474         esac
 475     done
 476 }
 477
 478 ######################################################
 479 # check a set of directories is available
 480 # directories are read from stdin
 481 ######################################################
 482 ctdb_check_directories()
 483 {
 484     ctdb_check_directories_probe || {
 485         echo "ERROR: $service_name directory \"$d\" not available"
 486         exit 1
 487     }
 488 }
 489
 490 ######################################################
 491 # check a set of tcp ports
 492 # usage: ctdb_check_tcp_ports <ports...>
 493 ######################################################
 494
 495 # This flag file is created when a service is initially started.  It
 496 # is deleted the first time TCP port checks for that service succeed.
 497 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 498 # message if a port check fails.
 499 _ctdb_check_tcp_common ()
 500 {
 501     assert_service_name
 502     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 503 }
 504
 505 ctdb_check_tcp_init ()
 506 {
 507     _ctdb_check_tcp_common
 508     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 509     touch "$_ctdb_service_started_file"
 510 }
 511
 512 # Check whether something is listening on all of the given TCP ports
 513 # using the "ctdb checktcpport" command.
 514 ctdb_check_tcp_ports()
 515 {
 516     if [ -z "$1" ] ; then
 517         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 518         exit 1
 519     fi
 520
 521     for _p ; do  # process each function argument (port)
 522         _cmd="ctdb checktcpport $_p"
 523         _out=$($_cmd 2>&1)
 524         _ret=$?
 525         case "$_ret" in
 526             0)
 527                 _ctdb_check_tcp_common
 528                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 529                     echo "ERROR: $service_name tcp port $_p is not responding"
 530                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 531                 else
 532                     echo "INFO: $service_name tcp port $_p is not responding"
 533                 fi
 534
 535                 return 1
 536                 ;;
 537             98)
 538                 # Couldn't bind, something already listening, next port...
 539                 continue
 540                 ;;
 541             *)
 542                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 543                 debug <<EOF
 544 ctdb checktcpport (exited with $_ret) with output:
 545 $_out"
 546 EOF
 547                 return $_ret
 548         esac
 549     done
 550
 551     # All ports listening
 552     _ctdb_check_tcp_common
 553     rm -f "$_ctdb_service_started_file"
 554     return 0
 555 }
 556
 557 ######################################################
 558 # check a unix socket
 559 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 560 ######################################################
 561 ctdb_check_unix_socket() {
 562     socket_path="$1"
 563     [ -z "$socket_path" ] && return
 564
 565     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 566         echo "ERROR: $service_name socket $socket_path not found"
 567         return 1
 568     fi
 569 }
 570
 571 ######################################################
 572 # check a command returns zero status
 573 # usage: ctdb_check_command <command>
 574 ######################################################
 575 ctdb_check_command ()
 576 {
 577     _out=$("$@" 2>&1) || {
 578         echo "ERROR: $* returned error"
 579         echo "$_out" | debug
 580         exit 1
 581     }
 582 }
 583
 584 ################################################
 585 # kill off any TCP connections with the given IP
 586 ################################################
 587 kill_tcp_connections ()
 588 {
 589     _ip="$1"
 590
 591     _oneway=false
 592     if [ "$2" = "oneway" ] ; then
 593         _oneway=true
 594     fi
 595
 596     get_tcp_connections_for_ip "$_ip" | {
 597         _killcount=0
 598         _connections=""
 599         _nl="
 600 "
 601         while read _dst _src; do
 602             _destport="${_dst##*:}"
 603             __oneway=$_oneway
 604             case $_destport in
 605                 # we only do one-way killtcp for CIFS
 606                 139|445) __oneway=true ;;
 607             esac
 608
 609             echo "Killing TCP connection $_src $_dst"
 610             _connections="${_connections}${_nl}${_src} ${_dst}"
 611             if ! $__oneway ; then
 612                 _connections="${_connections}${_nl}${_dst} ${_src}"
 613             fi
 614
 615             _killcount=$(($_killcount + 1))
 616         done
 617
 618         if [ $_killcount -eq 0 ] ; then
 619             return
 620         fi
 621
 622         echo "$_connections" | ctdb killtcp || {
 623             echo "Failed to send killtcp control"
 624             return
 625         }
 626
 627         _count=0
 628         while : ; do
 629             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 630
 631             if [ $_remaining -eq 0 ] ; then
 632                 echo "Killed $_killcount TCP connections to released IP $_ip"
 633                 return
 634             fi
 635
 636             _count=$(($_count + 1))
 637             if [ $_count -gt 3 ] ; then
 638                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 639                 return
 640             fi
 641
 642             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 643             sleep 1
 644         done
 645     }
 646 }
 647
 648 ##################################################################
 649 # kill off the local end for any TCP connections with the given IP
 650 ##################################################################
 651 kill_tcp_connections_local_only ()
 652 {
 653     kill_tcp_connections "$1" "oneway"
 654 }
 655
 656 ##################################################################
 657 # tickle any TCP connections with the given IP
 658 ##################################################################
 659 tickle_tcp_connections ()
 660 {
 661     _ip="$1"
 662
 663     get_tcp_connections_for_ip "$_ip" |
 664     {
 665         _failed=false
 666
 667         while read dest src; do
 668             echo "Tickle TCP connection $src $dest"
 669             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 670             echo "Tickle TCP connection $dest $src"
 671             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 672         done
 673
 674         if $_failed ; then
 675             echo "Failed to send tickle control"
 676         fi
 677     }
 678 }
 679
 680 get_tcp_connections_for_ip ()
 681 {
 682     _ip="$1"
 683
 684     netstat -tn | awk -v ip=$_ip \
 685         'index($1, "tcp") == 1 && \
 686          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 687          && $6 == "ESTABLISHED" \
 688          {print $4" "$5}'
 689 }
 690
 691 ########################################################
 692 # start/stop the Ganesha nfs service
 693 ########################################################
 694 startstop_ganesha()
 695 {
 696     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 697     case "$1" in
 698         start)
 699             service "$_service_name" start
 700             ;;
 701         stop)
 702             service "$_service_name" stop
 703             ;;
 704         restart)
 705             service "$_service_name" stop
 706             nfs_dump_some_threads "rpc.statd"
 707             service "$_service_name" start
 708             ;;
 709     esac
 710 }
 711
 712 ########################################################
 713 # start/stop the nfs service on different platforms
 714 ########################################################
 715 startstop_nfs() {
 716         PLATFORM="unknown"
 717         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 718                 PLATFORM="sles"
 719         }
 720         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 721             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 722                 PLATFORM="rhel"
 723         }
 724
 725         case $PLATFORM in
 726         sles)
 727                 case $1 in
 728                 start)
 729                         service nfsserver start
 730                         ;;
 731                 stop)
 732                         service nfsserver stop > /dev/null 2>&1
 733                         ;;
 734                 restart)
 735                         set_proc "fs/nfsd/threads" 0
 736                         service nfsserver stop > /dev/null 2>&1
 737                         pkill -9 nfsd
 738                         nfs_dump_some_threads
 739                         service nfsserver start
 740                         ;;
 741                 esac
 742                 ;;
 743         rhel)
 744                 case $1 in
 745                 start)
 746                         service nfslock start
 747                         service nfs start
 748                         ;;
 749                 stop)
 750                         service nfs stop
 751                         service nfslock stop
 752                         ;;
 753                 restart)
 754                         set_proc "fs/nfsd/threads" 0
 755                         service nfs stop > /dev/null 2>&1
 756                         service nfslock stop > /dev/null 2>&1
 757                         pkill -9 nfsd
 758                         nfs_dump_some_threads
 759                         service nfslock start
 760                         service nfs start
 761                         ;;
 762                 esac
 763                 ;;
 764         *)
 765                 echo "Unknown platform. NFS is not supported with ctdb"
 766                 exit 1
 767                 ;;
 768         esac
 769 }
 770
 771 # Dump up to the configured number of nfsd thread backtraces.
 772 nfs_dump_some_threads ()
 773 {
 774     _prog="${1:-nfsd}"
 775
 776     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 777     [ $_num -gt 0 ] || return 0
 778
 779     program_stack_traces "$_prog" $_num
 780 }
 781
 782 ########################################################
 783 # start/stop the nfs lockmanager service on different platforms
 784 ########################################################
 785 startstop_nfslock() {
 786         PLATFORM="unknown"
 787         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 788                 PLATFORM="sles"
 789         }
 790         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 791             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 792                 PLATFORM="rhel"
 793         }
 794
 795         case $PLATFORM in
 796         sles)
 797                 # for sles there is no service for lockmanager
 798                 # so we instead just shutdown/restart nfs
 799                 case $1 in
 800                 start)
 801                         service nfsserver start
 802                         ;;
 803                 stop)
 804                         service nfsserver stop > /dev/null 2>&1
 805                         ;;
 806                 restart)
 807                         service nfsserver stop > /dev/null 2>&1
 808                         service nfsserver start
 809                         ;;
 810                 esac
 811                 ;;
 812         rhel)
 813                 case $1 in
 814                 start)
 815                         service nfslock start
 816                         ;;
 817                 stop)
 818                         service nfslock stop > /dev/null 2>&1
 819                         ;;
 820                 restart)
 821                         service nfslock stop > /dev/null 2>&1
 822                         service nfslock start
 823                         ;;
 824                 esac
 825                 ;;
 826         *)
 827                 echo "Unknown platform. NFS locking is not supported with ctdb"
 828                 exit 1
 829                 ;;
 830         esac
 831 }
 832
 833 # Periodically update the statd database
 834 nfs_statd_update ()
 835 {
 836     _update_period="$1"
 837
 838     _statd_update_trigger="$service_state_dir/update-trigger"
 839     [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
 840
 841     _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
 842     _current_time=$(date +"%s")
 843     if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
 844         touch "$_statd_update_trigger"
 845         $CTDB_BASE/statd-callout updatelocal &
 846         $CTDB_BASE/statd-callout updateremote &
 847     fi
 848 }
 849
 850 ########################################################
 851
 852 add_ip_to_iface ()
 853 {
 854     _iface=$1
 855     _ip=$2
 856     _maskbits=$3
 857
 858     # Ensure interface is up
 859     ip link set "$_iface" up || \
 860         die "Failed to bringup interface $_iface"
 861
 862     ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || {
 863         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 864         return 1
 865     }
 866 }
 867
 868 delete_ip_from_iface()
 869 {
 870     _iface=$1
 871     _ip=$2
 872     _maskbits=$3
 873
 874     # This could be set globally for all interfaces but it is probably
 875     # better to avoid surprises, so limit it the interfaces where CTDB
 876     # has public IP addresses.  There isn't anywhere else convenient
 877     # to do this so just set it each time.  This is much cheaper than
 878     # remembering and re-adding secondaries.
 879     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 880
 881     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 882         echo "Failed to del $_ip on dev $_iface"
 883         return 1
 884     }
 885 }
 886
 887 # If the given IP is hosted then print 2 items: maskbits and iface
 888 ip_maskbits_iface ()
 889 {
 890     _addr="$1"
 891
 892     ip addr show to "${_addr}/32" 2>/dev/null | \
 893         awk '$1 == "inet" { print gensub(".*/", "", 1, $2), $NF }'
 894 }
 895
 896 drop_ip ()
 897 {
 898     _addr="${1%/*}"  # Remove optional maskbits
 899
 900     set -- $(ip_maskbits_iface $_addr)
 901     if [ -n "$1" ] ; then
 902         _maskbits="$1"
 903         _iface="$2"
 904         echo "Removing public address $_addr/$_maskbits from device $_iface"
 905         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 906     fi
 907 }
 908
 909 drop_all_public_ips ()
 910 {
 911     while read _ip _x ; do
 912         drop_ip "$_ip"
 913     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 914 }
 915
 916 ########################################################
 917 # Simple counters
 918 _ctdb_counter_common () {
 919     _service_name="${1:-${service_name:-${script_name}}}"
 920     _counter_file="$ctdb_fail_dir/$_service_name"
 921     mkdir -p "${_counter_file%/*}" # dirname
 922 }
 923 ctdb_counter_init () {
 924     _ctdb_counter_common "$1"
 925
 926     >"$_counter_file"
 927 }
 928 ctdb_counter_incr () {
 929     _ctdb_counter_common "$1"
 930
 931     # unary counting!
 932     echo -n 1 >> "$_counter_file"
 933 }
 934 ctdb_check_counter () {
 935     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 936     _op="${2:--ge}"  # an integer operator supported by test
 937     _limit="${3:-${service_fail_limit}}"
 938     shift 3
 939     _ctdb_counter_common "$1"
 940
 941     # unary counting!
 942     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 943     _hit=false
 944     if [ "$_op" != "%" ] ; then
 945         if [ $_size $_op $_limit ] ; then
 946             _hit=true
 947         fi
 948     else
 949         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
 950             _hit=true
 951         fi
 952     fi
 953     if $_hit ; then
 954         if [ "$_msg" = "error" ] ; then
 955             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
 956             exit 1
 957         else
 958             return 1
 959         fi
 960     fi
 961 }
 962
 963 ########################################################
 964
 965 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
 966 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
 967
 968 ctdb_setup_service_state_dir ()
 969 {
 970     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
 971     mkdir -p "$service_state_dir" || {
 972         echo "Error creating state dir \"$service_state_dir\""
 973         exit 1
 974     }
 975 }
 976
 977 ########################################################
 978 # Managed status history, for auto-start/stop
 979
 980 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
 981
 982 _ctdb_managed_common ()
 983 {
 984     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
 985 }
 986
 987 ctdb_service_managed ()
 988 {
 989     _ctdb_managed_common
 990     mkdir -p "$ctdb_managed_dir"
 991     touch "$_ctdb_managed_file"
 992 }
 993
 994 ctdb_service_unmanaged ()
 995 {
 996     _ctdb_managed_common
 997     rm -f "$_ctdb_managed_file"
 998 }
 999
1000 is_ctdb_previously_managed_service ()
1001 {
1002     _ctdb_managed_common
1003     [ -f "$_ctdb_managed_file" ]
1004 }
1005
1006 ########################################################
1007 # Check and set status
1008
1009 log_status_cat ()
1010 {
1011     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1012 }
1013
1014 ctdb_checkstatus ()
1015 {
1016     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1017         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1018         return 1
1019     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1020         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1021         return 2
1022     else
1023         return 0
1024     fi
1025 }
1026
1027 ctdb_setstatus ()
1028 {
1029     d="$ctdb_status_dir/$script_name"
1030     case "$1" in
1031         unhealthy|banned)
1032             mkdir -p "$d"
1033             cat "$2" >"$d/$1"
1034             ;;
1035         *)
1036             for i in "banned" "unhealthy" ; do
1037                 rm -f "$d/$i"
1038             done
1039             ;;
1040     esac
1041 }
1042
1043 ##################################################################
1044 # Reconfigure a service on demand
1045
1046 _ctdb_service_reconfigure_common ()
1047 {
1048     _d="$ctdb_status_dir/${service_name}"
1049     mkdir -p "$_d"
1050     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1051 }
1052
1053 ctdb_service_needs_reconfigure ()
1054 {
1055     _ctdb_service_reconfigure_common
1056     [ -e "$_ctdb_service_reconfigure_flag" ]
1057 }
1058
1059 ctdb_service_set_reconfigure ()
1060 {
1061     _ctdb_service_reconfigure_common
1062     >"$_ctdb_service_reconfigure_flag"
1063 }
1064
1065 ctdb_service_unset_reconfigure ()
1066 {
1067     _ctdb_service_reconfigure_common
1068     rm -f "$_ctdb_service_reconfigure_flag"
1069 }
1070
1071 ctdb_service_reconfigure ()
1072 {
1073     echo "Reconfiguring service \"${service_name}\"..."
1074     ctdb_service_unset_reconfigure
1075     service_reconfigure || return $?
1076     ctdb_counter_init
1077 }
1078
1079 # Default service_reconfigure() function does nothing.
1080 service_reconfigure ()
1081 {
1082     :
1083 }
1084
1085 ctdb_reconfigure_take_lock ()
1086 {
1087     _ctdb_service_reconfigure_common
1088     _lock="${_d}/reconfigure_lock"
1089     mkdir -p "${_lock%/*}" # dirname
1090     touch "$_lock"
1091
1092     (
1093         flock 0
1094         # This is overkill but will work if we need to extend this to
1095         # allow certain events to run multiple times in parallel
1096         # (e.g. takeip) and write multiple PIDs to the file.
1097         read _locker_event
1098         if [ -n "$_locker_event" ] ; then
1099             while read _pid ; do
1100                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1101                     kill -0 "$_pid" 2>/dev/null ; then
1102                     exit 1
1103                 fi
1104             done
1105         fi
1106
1107         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1108         exit 0
1109     ) <"$_lock"
1110 }
1111
1112 ctdb_reconfigure_release_lock ()
1113 {
1114     _ctdb_service_reconfigure_common
1115     _lock="${_d}/reconfigure_lock"
1116
1117     rm -f "$_lock"
1118 }
1119
1120 ctdb_replay_monitor_status ()
1121 {
1122     echo "Replaying previous status for this script due to reconfigure..."
1123     # Leading separator ('|') is missing in some versions...
1124     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1125     # Output looks like this:
1126     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1127     # This is the cheapest way of getting fields in the middle.
1128     set -- $(IFS="|" ; echo $_out)
1129     _code="$3"
1130     _status="$4"
1131     # The error output field can include colons so we'll try to
1132     # preserve them.  The weak checking at the beginning tries to make
1133     # this work for both broken (no leading '|') and fixed output.
1134     _out="${_out%|}"
1135     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1136     case "$_status" in
1137         OK) : ;;  # Do nothing special.
1138         TIMEDOUT)
1139             # Recast this as an error, since we can't exit with the
1140             # correct negative number.
1141             _code=1
1142             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1143             ;;
1144         DISABLED)
1145             # Recast this as an OK, since we can't exit with the
1146             # correct negative number.
1147             _code=0
1148             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1149             ;;
1150         *) : ;;  # Must be ERROR, do nothing special.
1151     esac
1152     if [ -n "$_err_out" ] ; then
1153         echo "$_err_out"
1154     fi
1155     exit $_code
1156 }
1157
1158 ctdb_service_check_reconfigure ()
1159 {
1160     assert_service_name
1161
1162     # We only care about some events in this function.  For others we
1163     # return now.
1164     case "$event_name" in
1165         monitor|ipreallocated|reconfigure) : ;;
1166         *) return 0 ;;
1167     esac
1168
1169     if ctdb_reconfigure_take_lock ; then
1170         # No events covered by this function are running, so proceed
1171         # with gay abandon.
1172         case "$event_name" in
1173             reconfigure)
1174                 (ctdb_service_reconfigure)
1175                 exit $?
1176                 ;;
1177             ipreallocated)
1178                 if ctdb_service_needs_reconfigure ; then
1179                     ctdb_service_reconfigure
1180                 fi
1181                 ;;
1182         esac
1183
1184         ctdb_reconfigure_release_lock
1185     else
1186         # Somebody else is running an event we don't want to collide
1187         # with.  We proceed with caution.
1188         case "$event_name" in
1189             reconfigure)
1190                 # Tell whoever called us to retry.
1191                 exit 2
1192                 ;;
1193             ipreallocated)
1194                 # Defer any scheduled reconfigure and just run the
1195                 # rest of the ipreallocated event, as per the
1196                 # eventscript.  There's an assumption here that the
1197                 # event doesn't depend on any scheduled reconfigure.
1198                 # This is true in the current code.
1199                 return 0
1200                 ;;
1201             monitor)
1202                 # There is most likely a reconfigure in progress so
1203                 # the service is possibly unstable.  As above, we
1204                 # defer any scheduled reconfigured.  We also replay
1205                 # the previous monitor status since that's the best
1206                 # information we have.
1207                 ctdb_replay_monitor_status
1208                 ;;
1209         esac
1210     fi
1211 }
1212
1213 ##################################################################
1214 # Does CTDB manage this service? - and associated auto-start/stop
1215
1216 ctdb_compat_managed_service ()
1217 {
1218     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1219         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1220     fi
1221 }
1222
1223 is_ctdb_managed_service ()
1224 {
1225     assert_service_name
1226
1227     # $t is used just for readability and to allow better accurate
1228     # matching via leading/trailing spaces
1229     t=" $CTDB_MANAGED_SERVICES "
1230
1231     # Return 0 if "<space>$service_name<space>" appears in $t
1232     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1233         return 0
1234     fi
1235
1236     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1237     # backward compatibility and try again.
1238     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1239     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1240     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1241     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1242     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1243     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1244     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1245     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1246     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1247
1248     t=" $CTDB_MANAGED_SERVICES "
1249
1250     # Return 0 if "<space>$service_name<space>" appears in $t
1251     [ "${t#* ${service_name} }" != "${t}" ]
1252 }
1253
1254 ctdb_start_stop_service ()
1255 {
1256     assert_service_name
1257
1258     # Allow service-start/service-stop pseudo-events to start/stop
1259     # services when we're not auto-starting/stopping and we're not
1260     # monitoring.
1261     case "$event_name" in
1262         service-start)
1263             if is_ctdb_managed_service ; then
1264                 die 'service-start event not permitted when service is managed'
1265             fi
1266             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1267                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1268             fi
1269             ctdb_service_start
1270             exit $?
1271             ;;
1272         service-stop)
1273             if is_ctdb_managed_service ; then
1274                 die 'service-stop event not permitted when service is managed'
1275             fi
1276             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1277                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1278             fi
1279             ctdb_service_stop
1280             exit $?
1281             ;;
1282     esac
1283
1284     # Do nothing unless configured to...
1285     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1286
1287     [ "$event_name" = "monitor" ] || return 0
1288
1289     if is_ctdb_managed_service ; then
1290         if ! is_ctdb_previously_managed_service ; then
1291             echo "Starting service \"$service_name\" - now managed"
1292             background_with_logging ctdb_service_start
1293             exit $?
1294         fi
1295     else
1296         if is_ctdb_previously_managed_service ; then
1297             echo "Stopping service \"$service_name\" - no longer managed"
1298             background_with_logging ctdb_service_stop
1299             exit $?
1300         fi
1301     fi
1302 }
1303
1304 ctdb_service_start ()
1305 {
1306     # The service is marked managed if we've ever tried to start it.
1307     ctdb_service_managed
1308
1309     service_start || return $?
1310
1311     ctdb_counter_init
1312     ctdb_check_tcp_init
1313 }
1314
1315 ctdb_service_stop ()
1316 {
1317     ctdb_service_unmanaged
1318     service_stop
1319 }
1320
1321 # Default service_start() and service_stop() functions.
1322
1323 # These may be overridden in an eventscript.
1324 service_start ()
1325 {
1326     service "$service_name" start
1327 }
1328
1329 service_stop ()
1330 {
1331     service "$service_name" stop
1332 }
1333
1334 ##################################################################
1335
1336 ctdb_standard_event_handler ()
1337 {
1338     case "$1" in
1339         status)
1340             ctdb_checkstatus
1341             exit
1342             ;;
1343         setstatus)
1344             shift
1345             ctdb_setstatus "$@"
1346             exit
1347             ;;
1348     esac
1349 }
1350
1351 # iptables doesn't like being re-entered, so flock-wrap it.
1352 iptables()
1353 {
1354         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1355 }
1356
1357 # AIX (and perhaps others?) doesn't have mktemp
1358 if ! which mktemp >/dev/null 2>&1 ; then
1359     mktemp ()
1360     {
1361         _dir=false
1362         if [ "$1" = "-d" ] ; then
1363             _dir=true
1364             shift
1365         fi
1366         _d="${TMPDIR:-/tmp}"
1367         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1368             md5sum | \
1369             sed -e 's@\(..........\).*@\1@')
1370         _t="${_d}/tmp.${_hex10}"
1371         (
1372             umask 077
1373             if $_dir ; then
1374                 mkdir "$_t"
1375             else
1376                 >"$_t"
1377             fi
1378         )
1379         echo "$_t"
1380     }
1381 fi
1382
1383 ########################################################
1384 # tickle handling
1385 ########################################################
1386
1387 update_tickles ()
1388 {
1389         _port="$1"
1390
1391         tickledir="$CTDB_VARDIR/state/tickles"
1392         mkdir -p "$tickledir"
1393
1394         # Who am I?
1395         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1396
1397         # What public IPs do I hold?
1398         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1399
1400         # IPs as a regexp choice
1401         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1402
1403         # Record connections to our public IPs in a temporary file
1404         _my_connections="${tickledir}/${_port}.connections"
1405         rm -f "$_my_connections"
1406         netstat -tn |
1407         awk -v destpat="^${_ipschoice}:${_port}\$" \
1408           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1409         sort >"$_my_connections"
1410
1411         # Record our current tickles in a temporary file
1412         _my_tickles="${tickledir}/${_port}.tickles"
1413         rm -f "$_my_tickles"
1414         for _i in $_ips ; do
1415                 ctdb -X gettickles $_i $_port |
1416                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1417         done |
1418         sort >"$_my_tickles"
1419
1420         # Add tickles for connections that we haven't already got tickles for
1421         comm -23 "$_my_connections" "$_my_tickles" |
1422         while read _src _dst ; do
1423                 ctdb addtickle $_src $_dst
1424         done
1425
1426         # Remove tickles for connections that are no longer there
1427         comm -13 "$_my_connections" "$_my_tickles" |
1428         while read _src _dst ; do
1429                 ctdb deltickle $_src $_dst
1430         done
1431
1432         rm -f "$_my_connections" "$_my_tickles"
1433 }
1434
1435 ########################################################
1436 # load a site local config file
1437 ########################################################
1438
1439 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1440         . "$CTDB_RC_LOCAL"
1441 }
1442
1443 [ -x $CTDB_BASE/rc.local ] && {
1444         . $CTDB_BASE/rc.local
1445 }
1446
1447 [ -d $CTDB_BASE/rc.local.d ] && {
1448         for i in $CTDB_BASE/rc.local.d/* ; do
1449                 [ -x "$i" ] && . "$i"
1450         done
1451 }
1452
1453 script_name="${0##*/}"       # basename
1454 service_fail_limit=1
1455 event_name="$1"