ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         elif ! tty -s ; then
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     fi
  72 }
  73
  74 die ()
  75 {
  76     _msg="$1"
  77     _rc="${2:-1}"
  78
  79     echo "$_msg"
  80     exit $_rc
  81 }
  82
  83 # Log given message or stdin to either syslog or a CTDB log file
  84 # $1 is the tag passed to logger if syslog is in use.
  85 script_log ()
  86 {
  87     _tag="$1" ; shift
  88
  89     case "$CTDB_LOGGING" in
  90         file:*|"")
  91             if [ -n "$CTDB_LOGGING" ] ; then
  92                 _file="${CTDB_LOGGING#file:}"
  93             else
  94                 _file="/var/log/log.ctdb"
  95             fi
  96             {
  97                 if [ -n "$*" ] ; then
  98                     echo "$*"
  99                 else
 100                     cat
 101                 fi
 102             } >>"$_file"
 103             ;;
 104         *)
 105             # Handle all syslog:* variants here too.  There's no tool to do
 106             # the lossy things, so just use logger.
 107             logger -t "ctdbd: ${_tag}" $*
 108             ;;
 109     esac
 110 }
 111
 112 # When things are run in the background in an eventscript then logging
 113 # output might get lost.  This is the "solution".  :-)
 114 background_with_logging ()
 115 {
 116     (
 117         "$@" 2>&1 </dev/null |
 118         script_log "${script_name}&"
 119     )&
 120
 121     return 0
 122 }
 123
 124 ##############################################################
 125 # check number of args for different events
 126 ctdb_check_args ()
 127 {
 128     case "$1" in
 129         takeip|releaseip)
 130             if [ $# != 4 ]; then
 131                 echo "ERROR: must supply interface, IP and maskbits"
 132                 exit 1
 133             fi
 134             ;;
 135         updateip)
 136             if [ $# != 5 ]; then
 137                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 138                 exit 1
 139             fi
 140             ;;
 141     esac
 142 }
 143
 144 ##############################################################
 145 # determine on what type of system (init style) we are running
 146 detect_init_style()
 147 {
 148     # only do detection if not already set:
 149     [ -z "$CTDB_INIT_STYLE" ] || return
 150
 151     if [ -x /sbin/startproc ]; then
 152         CTDB_INIT_STYLE="suse"
 153     elif [ -x /sbin/start-stop-daemon ]; then
 154         CTDB_INIT_STYLE="debian"
 155     else
 156         CTDB_INIT_STYLE="redhat"
 157     fi
 158 }
 159
 160 ######################################################
 161 # simulate /sbin/service on platforms that don't have it
 162 # _service() makes it easier to hook the service() function for
 163 # testing.
 164 _service ()
 165 {
 166   _service_name="$1"
 167   _op="$2"
 168
 169   # do nothing, when no service was specified
 170   [ -z "$_service_name" ] && return
 171
 172   if [ -x /sbin/service ]; then
 173       $_nice /sbin/service "$_service_name" "$_op"
 174   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 175       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 176   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 177       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 178   fi
 179 }
 180
 181 service()
 182 {
 183     _nice=""
 184     _service "$@"
 185 }
 186
 187 ######################################################
 188 # simulate /sbin/service (niced) on platforms that don't have it
 189 nice_service()
 190 {
 191     _nice="nice"
 192     _service "$@"
 193 }
 194
 195 ######################################################
 196 # wrapper around /proc/ settings to allow them to be hooked
 197 # for testing
 198 # 1st arg is relative path under /proc/, 2nd arg is value to set
 199 set_proc ()
 200 {
 201     echo "$2" >"/proc/$1"
 202 }
 203
 204 ######################################################
 205 # wrapper around getting file contents from /proc/ to allow
 206 # this to be hooked for testing
 207 # 1st arg is relative path under /proc/
 208 get_proc ()
 209 {
 210     cat "/proc/$1"
 211 }
 212
 213 ######################################################
 214 # Print up to $_max kernel stack traces for processes named $_program
 215 program_stack_traces ()
 216 {
 217     _prog="$1"
 218     _max="${2:-1}"
 219
 220     _count=1
 221     for _pid in $(pidof "$_prog") ; do
 222         [ $_count -le $_max ] || break
 223
 224         # Do this first to avoid racing with process exit
 225         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 226         if [ -n "$_stack" ] ; then
 227             echo "Stack trace for ${_prog}[${_pid}]:"
 228             echo "$_stack"
 229             _count=$(($_count + 1))
 230         fi
 231     done
 232 }
 233
 234 ######################################################
 235 # Check that an RPC service is healthy -
 236 # this includes allowing a certain number of failures
 237 # before marking the NFS service unhealthy.
 238 #
 239 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 240 #
 241 # each triple is a set of 3 arguments: an operator, a
 242 # fail count limit and an action string.
 243 #
 244 # For example:
 245 #
 246 #       nfs_check_rpc_service "lockd" \
 247 #           -ge 15 "verbose restart unhealthy" \
 248 #           -eq 10 "restart:bs"
 249 #
 250 # says that if lockd is down for 15 iterations then do
 251 # a verbose restart of lockd and mark the node unhealthy.
 252 # Before this, after 10 iterations of failure, the
 253 # service is restarted silently in the background.
 254 # Order is important: the number of failures need to be
 255 # specified in reverse order because processing stops
 256 # after the first condition that is true.
 257 ######################################################
 258 nfs_check_rpc_service ()
 259 {
 260     _prog_name="$1" ; shift
 261
 262     if _nfs_check_rpc_common "$_prog_name" ; then
 263         return
 264     fi
 265
 266     while [ -n "$3" ] ; do
 267         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 268             break
 269         fi
 270         shift 3
 271     done
 272 }
 273
 274 # The new way of doing things...
 275 nfs_check_rpc_services ()
 276 {
 277     # Files must end with .check - avoids editor backups, RPM fu, ...
 278     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 279         _t="${_f%.check}"
 280         _prog_name="${_t##*/[0-9][0-9].}"
 281
 282         if _nfs_check_rpc_common "$_prog_name" ; then
 283             # This RPC service is up, check next service...
 284             continue
 285         fi
 286
 287         # Check each line in the file in turn until one of the limit
 288         # checks is hit...
 289         while read _cmp _lim _rest ; do
 290             # Skip comments
 291             case "$_cmp" in
 292                 \#*) continue ;;
 293             esac
 294
 295             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 296                 # Limit was hit on this line, no further checking...
 297                 break
 298             fi
 299         done <"$_f"
 300     done
 301 }
 302
 303 _nfs_check_rpc_common ()
 304 {
 305     _prog_name="$1"
 306
 307     # Some platforms don't have separate programs for all services.
 308     case "$_prog_name" in
 309         statd)
 310             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 311     esac
 312
 313     case "$_prog_name" in
 314         nfsd)
 315             _rpc_prog=nfs
 316             _version=3
 317             ;;
 318         mountd)
 319             _rpc_prog=mountd
 320             _version=1
 321             ;;
 322         rquotad)
 323             _rpc_prog=rquotad
 324             _version=1
 325             ;;
 326         lockd)
 327             _rpc_prog=nlockmgr
 328             _version=4
 329             ;;
 330         statd)
 331             _rpc_prog=status
 332             _version=1
 333             ;;
 334         *)
 335             echo "Internal error: unknown RPC program \"$_prog_name\"."
 336             exit 1
 337     esac
 338
 339     _service_name="nfs_${_prog_name}"
 340
 341     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 342         ctdb_counter_init "$_service_name"
 343         return 0
 344     fi
 345
 346     ctdb_counter_incr "$_service_name"
 347
 348     return 1
 349 }
 350
 351 _nfs_check_rpc_action ()
 352 {
 353     _cmp="$1"
 354     _limit="$2"
 355     _actions="$3"
 356
 357     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 358         return 1
 359     fi
 360
 361     for _action in $_actions ; do
 362         case "$_action" in
 363             verbose)
 364                 echo "$ctdb_check_rpc_out"
 365                 ;;
 366             restart)
 367                 _nfs_restart_rpc_service "$_prog_name"
 368                 ;;
 369             restart:b)
 370                 _nfs_restart_rpc_service "$_prog_name" true
 371                 ;;
 372             unhealthy)
 373                 exit 1
 374                 ;;
 375             *)
 376                 echo "Internal error: unknown action \"$_action\"."
 377                 exit 1
 378         esac
 379     done
 380
 381     return 0
 382 }
 383
 384 _nfs_restart_rpc_service ()
 385 {
 386     _prog_name="$1"
 387     _background="${2:-false}"
 388
 389     if $_background ; then
 390         _maybe_background="background_with_logging"
 391     else
 392         _maybe_background=""
 393     fi
 394
 395     _p="rpc.${_prog_name}"
 396
 397     case "$_prog_name" in
 398         nfsd)
 399             echo "Trying to restart NFS service"
 400             $_maybe_background startstop_nfs restart
 401             ;;
 402         mountd)
 403             echo "Trying to restart $_prog_name [${_p}]"
 404             killall -q -9 "$_p"
 405             nfs_dump_some_threads "$_p"
 406             $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 407             ;;
 408         rquotad)
 409             echo "Trying to restart $_prog_name [${_p}]"
 410             killall -q -9 "$_p"
 411             nfs_dump_some_threads "$_p"
 412             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 413             ;;
 414         lockd)
 415             echo "Trying to restart lock manager service"
 416             $_maybe_background startstop_nfslock restart
 417             ;;
 418         statd)
 419             echo "Trying to restart $_prog_name [${_p}]"
 420             killall -q -9 "$_p"
 421             nfs_dump_some_threads "$_p"
 422             $_maybe_background $_p \
 423                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 424                 ${STATD_PORT:+-p} $STATD_PORT \
 425                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 426             ;;
 427         *)
 428             echo "Internal error: unknown RPC program \"$_prog_name\"."
 429             exit 1
 430     esac
 431 }
 432
 433 ######################################################
 434 # check that a rpc server is registered with portmap
 435 # and responding to requests
 436 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 437 ######################################################
 438 ctdb_check_rpc ()
 439 {
 440     progname="$1"
 441     version="$2"
 442
 443     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 444
 445     if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
 446         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 447 $ctdb_check_rpc_out"
 448         echo "$ctdb_check_rpc_out"
 449         return 1
 450     fi
 451 }
 452
 453 ######################################################
 454 # Ensure $service_name is set
 455 assert_service_name ()
 456 {
 457     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 458 }
 459
 460 ######################################################
 461 # check a set of directories is available
 462 # return 1 on a missing directory
 463 # directories are read from stdin
 464 ######################################################
 465 ctdb_check_directories_probe()
 466 {
 467     while IFS="" read d ; do
 468         case "$d" in
 469             *%*)
 470                 continue
 471                 ;;
 472             *)
 473                 [ -d "${d}/." ] || return 1
 474         esac
 475     done
 476 }
 477
 478 ######################################################
 479 # check a set of directories is available
 480 # directories are read from stdin
 481 ######################################################
 482 ctdb_check_directories()
 483 {
 484     ctdb_check_directories_probe || {
 485         echo "ERROR: $service_name directory \"$d\" not available"
 486         exit 1
 487     }
 488 }
 489
 490 ######################################################
 491 # check a set of tcp ports
 492 # usage: ctdb_check_tcp_ports <ports...>
 493 ######################################################
 494
 495 # This flag file is created when a service is initially started.  It
 496 # is deleted the first time TCP port checks for that service succeed.
 497 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 498 # message if a port check fails.
 499 _ctdb_check_tcp_common ()
 500 {
 501     assert_service_name
 502     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 503 }
 504
 505 ctdb_check_tcp_init ()
 506 {
 507     _ctdb_check_tcp_common
 508     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 509     touch "$_ctdb_service_started_file"
 510 }
 511
 512 # Check whether something is listening on all of the given TCP ports
 513 # using the "ctdb checktcpport" command.
 514 ctdb_check_tcp_ports()
 515 {
 516     if [ -z "$1" ] ; then
 517         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 518         exit 1
 519     fi
 520
 521     for _p ; do  # process each function argument (port)
 522         _cmd="ctdb checktcpport $_p"
 523         _out=$($_cmd 2>&1)
 524         _ret=$?
 525         case "$_ret" in
 526             0)
 527                 _ctdb_check_tcp_common
 528                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 529                     echo "ERROR: $service_name tcp port $_p is not responding"
 530                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 531                 else
 532                     echo "INFO: $service_name tcp port $_p is not responding"
 533                 fi
 534
 535                 return 1
 536                 ;;
 537             98)
 538                 # Couldn't bind, something already listening, next port...
 539                 continue
 540                 ;;
 541             *)
 542                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 543                 debug <<EOF
 544 ctdb checktcpport (exited with $_ret) with output:
 545 $_out"
 546 EOF
 547                 return $_ret
 548         esac
 549     done
 550
 551     # All ports listening
 552     _ctdb_check_tcp_common
 553     rm -f "$_ctdb_service_started_file"
 554     return 0
 555 }
 556
 557 ######################################################
 558 # check a unix socket
 559 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 560 ######################################################
 561 ctdb_check_unix_socket() {
 562     socket_path="$1"
 563     [ -z "$socket_path" ] && return
 564
 565     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 566         echo "ERROR: $service_name socket $socket_path not found"
 567         return 1
 568     fi
 569 }
 570
 571 ######################################################
 572 # check a command returns zero status
 573 # usage: ctdb_check_command <command>
 574 ######################################################
 575 ctdb_check_command ()
 576 {
 577     _out=$("$@" 2>&1) || {
 578         echo "ERROR: $* returned error"
 579         echo "$_out" | debug
 580         exit 1
 581     }
 582 }
 583
 584 ################################################
 585 # kill off any TCP connections with the given IP
 586 ################################################
 587 kill_tcp_connections ()
 588 {
 589     _ip="$1"
 590
 591     _oneway=false
 592     if [ "$2" = "oneway" ] ; then
 593         _oneway=true
 594     fi
 595
 596     get_tcp_connections_for_ip "$_ip" | {
 597         _killcount=0
 598         _connections=""
 599         _nl="
 600 "
 601         while read _dst _src; do
 602             _destport="${_dst##*:}"
 603             __oneway=$_oneway
 604             case $_destport in
 605                 # we only do one-way killtcp for CIFS
 606                 139|445) __oneway=true ;;
 607             esac
 608
 609             echo "Killing TCP connection $_src $_dst"
 610             _connections="${_connections}${_nl}${_src} ${_dst}"
 611             if ! $__oneway ; then
 612                 _connections="${_connections}${_nl}${_dst} ${_src}"
 613             fi
 614
 615             _killcount=$(($_killcount + 1))
 616         done
 617
 618         if [ $_killcount -eq 0 ] ; then
 619             return
 620         fi
 621
 622         echo "$_connections" | ctdb killtcp || {
 623             echo "Failed to send killtcp control"
 624             return
 625         }
 626
 627         _count=0
 628         while : ; do
 629             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 630
 631             if [ $_remaining -eq 0 ] ; then
 632                 echo "Killed $_killcount TCP connections to released IP $_ip"
 633                 return
 634             fi
 635
 636             _count=$(($_count + 1))
 637             if [ $_count -gt 3 ] ; then
 638                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 639                 return
 640             fi
 641
 642             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 643             sleep 1
 644         done
 645     }
 646 }
 647
 648 ##################################################################
 649 # kill off the local end for any TCP connections with the given IP
 650 ##################################################################
 651 kill_tcp_connections_local_only ()
 652 {
 653     kill_tcp_connections "$1" "oneway"
 654 }
 655
 656 ##################################################################
 657 # tickle any TCP connections with the given IP
 658 ##################################################################
 659 tickle_tcp_connections ()
 660 {
 661     _ip="$1"
 662
 663     get_tcp_connections_for_ip "$_ip" |
 664     {
 665         _failed=false
 666
 667         while read dest src; do
 668             echo "Tickle TCP connection $src $dest"
 669             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 670             echo "Tickle TCP connection $dest $src"
 671             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 672         done
 673
 674         if $_failed ; then
 675             echo "Failed to send tickle control"
 676         fi
 677     }
 678 }
 679
 680 get_tcp_connections_for_ip ()
 681 {
 682     _ip="$1"
 683
 684     netstat -tn | awk -v ip=$_ip \
 685         'index($1, "tcp") == 1 && \
 686          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 687          && $6 == "ESTABLISHED" \
 688          {print $4" "$5}'
 689 }
 690
 691 ########################################################
 692 # start/stop the Ganesha nfs service
 693 ########################################################
 694 startstop_ganesha()
 695 {
 696     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 697     case "$1" in
 698         start)
 699             service "$_service_name" start
 700             ;;
 701         stop)
 702             service "$_service_name" stop
 703             ;;
 704         restart)
 705             service "$_service_name" stop
 706             nfs_dump_some_threads "rpc.statd"
 707             service "$_service_name" start
 708             ;;
 709     esac
 710 }
 711
 712 ########################################################
 713 # start/stop the nfs service on different platforms
 714 ########################################################
 715 startstop_nfs() {
 716         PLATFORM="unknown"
 717         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 718                 PLATFORM="sles"
 719         }
 720         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 721             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 722                 PLATFORM="rhel"
 723         }
 724
 725         case $PLATFORM in
 726         sles)
 727                 case $1 in
 728                 start)
 729                         service nfsserver start
 730                         ;;
 731                 stop)
 732                         service nfsserver stop > /dev/null 2>&1
 733                         ;;
 734                 restart)
 735                         set_proc "fs/nfsd/threads" 0
 736                         service nfsserver stop > /dev/null 2>&1
 737                         pkill -9 nfsd
 738                         nfs_dump_some_threads
 739                         service nfsserver start
 740                         ;;
 741                 esac
 742                 ;;
 743         rhel)
 744                 case $1 in
 745                 start)
 746                         service nfslock start
 747                         service nfs start
 748                         ;;
 749                 stop)
 750                         service nfs stop
 751                         service nfslock stop
 752                         ;;
 753                 restart)
 754                         set_proc "fs/nfsd/threads" 0
 755                         service nfs stop > /dev/null 2>&1
 756                         service nfslock stop > /dev/null 2>&1
 757                         pkill -9 nfsd
 758                         nfs_dump_some_threads
 759                         service nfslock start
 760                         service nfs start
 761                         ;;
 762                 esac
 763                 ;;
 764         *)
 765                 echo "Unknown platform. NFS is not supported with ctdb"
 766                 exit 1
 767                 ;;
 768         esac
 769 }
 770
 771 # Dump up to the configured number of nfsd thread backtraces.
 772 nfs_dump_some_threads ()
 773 {
 774     _prog="${1:-nfsd}"
 775
 776     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 777     [ $_num -gt 0 ] || return 0
 778
 779     program_stack_traces "$_prog" $_num
 780 }
 781
 782 ########################################################
 783 # start/stop the nfs lockmanager service on different platforms
 784 ########################################################
 785 startstop_nfslock() {
 786         PLATFORM="unknown"
 787         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 788                 PLATFORM="sles"
 789         }
 790         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 791             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 792                 PLATFORM="rhel"
 793         }
 794
 795         case $PLATFORM in
 796         sles)
 797                 # for sles there is no service for lockmanager
 798                 # so we instead just shutdown/restart nfs
 799                 case $1 in
 800                 start)
 801                         service nfsserver start
 802                         ;;
 803                 stop)
 804                         service nfsserver stop > /dev/null 2>&1
 805                         ;;
 806                 restart)
 807                         service nfsserver stop > /dev/null 2>&1
 808                         service nfsserver start
 809                         ;;
 810                 esac
 811                 ;;
 812         rhel)
 813                 case $1 in
 814                 start)
 815                         service nfslock start
 816                         ;;
 817                 stop)
 818                         service nfslock stop > /dev/null 2>&1
 819                         ;;
 820                 restart)
 821                         service nfslock stop > /dev/null 2>&1
 822                         service nfslock start
 823                         ;;
 824                 esac
 825                 ;;
 826         *)
 827                 echo "Unknown platform. NFS locking is not supported with ctdb"
 828                 exit 1
 829                 ;;
 830         esac
 831 }
 832
 833 # Periodically update the statd database
 834 nfs_statd_update ()
 835 {
 836     _update_period="$1"
 837
 838     _statd_update_trigger="$service_state_dir/update-trigger"
 839     [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
 840
 841     _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
 842     _current_time=$(date +"%s")
 843     if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
 844         touch "$_statd_update_trigger"
 845         $CTDB_BASE/statd-callout updatelocal &
 846         $CTDB_BASE/statd-callout updateremote &
 847     fi
 848 }
 849
 850 ########################################################
 851
 852 add_ip_to_iface ()
 853 {
 854     _iface=$1
 855     _ip=$2
 856     _maskbits=$3
 857
 858     # Ensure interface is up
 859     ip link set "$_iface" up || \
 860         die "Failed to bringup interface $_iface"
 861
 862     ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || {
 863         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 864         return 1
 865     }
 866 }
 867
 868 delete_ip_from_iface()
 869 {
 870     _iface=$1
 871     _ip=$2
 872     _maskbits=$3
 873
 874     # This could be set globally for all interfaces but it is probably
 875     # better to avoid surprises, so limit it the interfaces where CTDB
 876     # has public IP addresses.  There isn't anywhere else convenient
 877     # to do this so just set it each time.  This is much cheaper than
 878     # remembering and re-adding secondaries.
 879     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 880
 881     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 882         echo "Failed to del $_ip on dev $_iface"
 883         return 1
 884     }
 885 }
 886
 887 # If the given IP is hosted then print 2 items: maskbits and iface
 888 ip_maskbits_iface ()
 889 {
 890     _addr="$1"
 891
 892     case "$_addr" in
 893         *:*) _family="inet6" ; _bits=128 ;;
 894         *)   _family="inet"  ; _bits=32  ;;
 895     esac
 896
 897     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 898         awk -v family="${_family}" \
 899             'NR == 1 { iface = gensub(":$", "", 1, $2) } \
 900              $1 ~ /inet/ { print gensub(".*/", "", 1, $2), iface, family }'
 901 }
 902
 903 drop_ip ()
 904 {
 905     _addr="${1%/*}"  # Remove optional maskbits
 906
 907     set -- $(ip_maskbits_iface $_addr)
 908     if [ -n "$1" ] ; then
 909         _maskbits="$1"
 910         _iface="$2"
 911         echo "Removing public address $_addr/$_maskbits from device $_iface"
 912         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 913     fi
 914 }
 915
 916 drop_all_public_ips ()
 917 {
 918     while read _ip _x ; do
 919         drop_ip "$_ip"
 920     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 921 }
 922
 923 ########################################################
 924 # Simple counters
 925 _ctdb_counter_common () {
 926     _service_name="${1:-${service_name:-${script_name}}}"
 927     _counter_file="$ctdb_fail_dir/$_service_name"
 928     mkdir -p "${_counter_file%/*}" # dirname
 929 }
 930 ctdb_counter_init () {
 931     _ctdb_counter_common "$1"
 932
 933     >"$_counter_file"
 934 }
 935 ctdb_counter_incr () {
 936     _ctdb_counter_common "$1"
 937
 938     # unary counting!
 939     echo -n 1 >> "$_counter_file"
 940 }
 941 ctdb_check_counter () {
 942     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 943     _op="${2:--ge}"  # an integer operator supported by test
 944     _limit="${3:-${service_fail_limit}}"
 945     shift 3
 946     _ctdb_counter_common "$1"
 947
 948     # unary counting!
 949     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 950     _hit=false
 951     if [ "$_op" != "%" ] ; then
 952         if [ $_size $_op $_limit ] ; then
 953             _hit=true
 954         fi
 955     else
 956         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
 957             _hit=true
 958         fi
 959     fi
 960     if $_hit ; then
 961         if [ "$_msg" = "error" ] ; then
 962             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
 963             exit 1
 964         else
 965             return 1
 966         fi
 967     fi
 968 }
 969
 970 ########################################################
 971
 972 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
 973 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
 974
 975 ctdb_setup_service_state_dir ()
 976 {
 977     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
 978     mkdir -p "$service_state_dir" || {
 979         echo "Error creating state dir \"$service_state_dir\""
 980         exit 1
 981     }
 982 }
 983
 984 ########################################################
 985 # Managed status history, for auto-start/stop
 986
 987 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
 988
 989 _ctdb_managed_common ()
 990 {
 991     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
 992 }
 993
 994 ctdb_service_managed ()
 995 {
 996     _ctdb_managed_common
 997     mkdir -p "$ctdb_managed_dir"
 998     touch "$_ctdb_managed_file"
 999 }
1000
1001 ctdb_service_unmanaged ()
1002 {
1003     _ctdb_managed_common
1004     rm -f "$_ctdb_managed_file"
1005 }
1006
1007 is_ctdb_previously_managed_service ()
1008 {
1009     _ctdb_managed_common
1010     [ -f "$_ctdb_managed_file" ]
1011 }
1012
1013 ########################################################
1014 # Check and set status
1015
1016 log_status_cat ()
1017 {
1018     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1019 }
1020
1021 ctdb_checkstatus ()
1022 {
1023     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1024         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1025         return 1
1026     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1027         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1028         return 2
1029     else
1030         return 0
1031     fi
1032 }
1033
1034 ctdb_setstatus ()
1035 {
1036     d="$ctdb_status_dir/$script_name"
1037     case "$1" in
1038         unhealthy|banned)
1039             mkdir -p "$d"
1040             cat "$2" >"$d/$1"
1041             ;;
1042         *)
1043             for i in "banned" "unhealthy" ; do
1044                 rm -f "$d/$i"
1045             done
1046             ;;
1047     esac
1048 }
1049
1050 ##################################################################
1051 # Reconfigure a service on demand
1052
1053 _ctdb_service_reconfigure_common ()
1054 {
1055     _d="$ctdb_status_dir/${service_name}"
1056     mkdir -p "$_d"
1057     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1058 }
1059
1060 ctdb_service_needs_reconfigure ()
1061 {
1062     _ctdb_service_reconfigure_common
1063     [ -e "$_ctdb_service_reconfigure_flag" ]
1064 }
1065
1066 ctdb_service_set_reconfigure ()
1067 {
1068     _ctdb_service_reconfigure_common
1069     >"$_ctdb_service_reconfigure_flag"
1070 }
1071
1072 ctdb_service_unset_reconfigure ()
1073 {
1074     _ctdb_service_reconfigure_common
1075     rm -f "$_ctdb_service_reconfigure_flag"
1076 }
1077
1078 ctdb_service_reconfigure ()
1079 {
1080     echo "Reconfiguring service \"${service_name}\"..."
1081     ctdb_service_unset_reconfigure
1082     service_reconfigure || return $?
1083     ctdb_counter_init
1084 }
1085
1086 # Default service_reconfigure() function does nothing.
1087 service_reconfigure ()
1088 {
1089     :
1090 }
1091
1092 ctdb_reconfigure_take_lock ()
1093 {
1094     _ctdb_service_reconfigure_common
1095     _lock="${_d}/reconfigure_lock"
1096     mkdir -p "${_lock%/*}" # dirname
1097     touch "$_lock"
1098
1099     (
1100         flock 0
1101         # This is overkill but will work if we need to extend this to
1102         # allow certain events to run multiple times in parallel
1103         # (e.g. takeip) and write multiple PIDs to the file.
1104         read _locker_event
1105         if [ -n "$_locker_event" ] ; then
1106             while read _pid ; do
1107                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1108                     kill -0 "$_pid" 2>/dev/null ; then
1109                     exit 1
1110                 fi
1111             done
1112         fi
1113
1114         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1115         exit 0
1116     ) <"$_lock"
1117 }
1118
1119 ctdb_reconfigure_release_lock ()
1120 {
1121     _ctdb_service_reconfigure_common
1122     _lock="${_d}/reconfigure_lock"
1123
1124     rm -f "$_lock"
1125 }
1126
1127 ctdb_replay_monitor_status ()
1128 {
1129     echo "Replaying previous status for this script due to reconfigure..."
1130     # Leading separator ('|') is missing in some versions...
1131     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1132     # Output looks like this:
1133     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1134     # This is the cheapest way of getting fields in the middle.
1135     set -- $(IFS="|" ; echo $_out)
1136     _code="$3"
1137     _status="$4"
1138     # The error output field can include colons so we'll try to
1139     # preserve them.  The weak checking at the beginning tries to make
1140     # this work for both broken (no leading '|') and fixed output.
1141     _out="${_out%|}"
1142     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1143     case "$_status" in
1144         OK) : ;;  # Do nothing special.
1145         TIMEDOUT)
1146             # Recast this as an error, since we can't exit with the
1147             # correct negative number.
1148             _code=1
1149             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1150             ;;
1151         DISABLED)
1152             # Recast this as an OK, since we can't exit with the
1153             # correct negative number.
1154             _code=0
1155             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1156             ;;
1157         *) : ;;  # Must be ERROR, do nothing special.
1158     esac
1159     if [ -n "$_err_out" ] ; then
1160         echo "$_err_out"
1161     fi
1162     exit $_code
1163 }
1164
1165 ctdb_service_check_reconfigure ()
1166 {
1167     assert_service_name
1168
1169     # We only care about some events in this function.  For others we
1170     # return now.
1171     case "$event_name" in
1172         monitor|ipreallocated|reconfigure) : ;;
1173         *) return 0 ;;
1174     esac
1175
1176     if ctdb_reconfigure_take_lock ; then
1177         # No events covered by this function are running, so proceed
1178         # with gay abandon.
1179         case "$event_name" in
1180             reconfigure)
1181                 (ctdb_service_reconfigure)
1182                 exit $?
1183                 ;;
1184             ipreallocated)
1185                 if ctdb_service_needs_reconfigure ; then
1186                     ctdb_service_reconfigure
1187                 fi
1188                 ;;
1189         esac
1190
1191         ctdb_reconfigure_release_lock
1192     else
1193         # Somebody else is running an event we don't want to collide
1194         # with.  We proceed with caution.
1195         case "$event_name" in
1196             reconfigure)
1197                 # Tell whoever called us to retry.
1198                 exit 2
1199                 ;;
1200             ipreallocated)
1201                 # Defer any scheduled reconfigure and just run the
1202                 # rest of the ipreallocated event, as per the
1203                 # eventscript.  There's an assumption here that the
1204                 # event doesn't depend on any scheduled reconfigure.
1205                 # This is true in the current code.
1206                 return 0
1207                 ;;
1208             monitor)
1209                 # There is most likely a reconfigure in progress so
1210                 # the service is possibly unstable.  As above, we
1211                 # defer any scheduled reconfigured.  We also replay
1212                 # the previous monitor status since that's the best
1213                 # information we have.
1214                 ctdb_replay_monitor_status
1215                 ;;
1216         esac
1217     fi
1218 }
1219
1220 ##################################################################
1221 # Does CTDB manage this service? - and associated auto-start/stop
1222
1223 ctdb_compat_managed_service ()
1224 {
1225     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1226         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1227     fi
1228 }
1229
1230 is_ctdb_managed_service ()
1231 {
1232     assert_service_name
1233
1234     # $t is used just for readability and to allow better accurate
1235     # matching via leading/trailing spaces
1236     t=" $CTDB_MANAGED_SERVICES "
1237
1238     # Return 0 if "<space>$service_name<space>" appears in $t
1239     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1240         return 0
1241     fi
1242
1243     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1244     # backward compatibility and try again.
1245     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1246     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1247     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1248     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1249     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1250     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1251     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1252     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1253     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1254
1255     t=" $CTDB_MANAGED_SERVICES "
1256
1257     # Return 0 if "<space>$service_name<space>" appears in $t
1258     [ "${t#* ${service_name} }" != "${t}" ]
1259 }
1260
1261 ctdb_start_stop_service ()
1262 {
1263     assert_service_name
1264
1265     # Allow service-start/service-stop pseudo-events to start/stop
1266     # services when we're not auto-starting/stopping and we're not
1267     # monitoring.
1268     case "$event_name" in
1269         service-start)
1270             if is_ctdb_managed_service ; then
1271                 die 'service-start event not permitted when service is managed'
1272             fi
1273             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1274                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1275             fi
1276             ctdb_service_start
1277             exit $?
1278             ;;
1279         service-stop)
1280             if is_ctdb_managed_service ; then
1281                 die 'service-stop event not permitted when service is managed'
1282             fi
1283             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1284                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1285             fi
1286             ctdb_service_stop
1287             exit $?
1288             ;;
1289     esac
1290
1291     # Do nothing unless configured to...
1292     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1293
1294     [ "$event_name" = "monitor" ] || return 0
1295
1296     if is_ctdb_managed_service ; then
1297         if ! is_ctdb_previously_managed_service ; then
1298             echo "Starting service \"$service_name\" - now managed"
1299             background_with_logging ctdb_service_start
1300             exit $?
1301         fi
1302     else
1303         if is_ctdb_previously_managed_service ; then
1304             echo "Stopping service \"$service_name\" - no longer managed"
1305             background_with_logging ctdb_service_stop
1306             exit $?
1307         fi
1308     fi
1309 }
1310
1311 ctdb_service_start ()
1312 {
1313     # The service is marked managed if we've ever tried to start it.
1314     ctdb_service_managed
1315
1316     service_start || return $?
1317
1318     ctdb_counter_init
1319     ctdb_check_tcp_init
1320 }
1321
1322 ctdb_service_stop ()
1323 {
1324     ctdb_service_unmanaged
1325     service_stop
1326 }
1327
1328 # Default service_start() and service_stop() functions.
1329
1330 # These may be overridden in an eventscript.
1331 service_start ()
1332 {
1333     service "$service_name" start
1334 }
1335
1336 service_stop ()
1337 {
1338     service "$service_name" stop
1339 }
1340
1341 ##################################################################
1342
1343 ctdb_standard_event_handler ()
1344 {
1345     case "$1" in
1346         status)
1347             ctdb_checkstatus
1348             exit
1349             ;;
1350         setstatus)
1351             shift
1352             ctdb_setstatus "$@"
1353             exit
1354             ;;
1355     esac
1356 }
1357
1358 # iptables doesn't like being re-entered, so flock-wrap it.
1359 iptables()
1360 {
1361         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1362 }
1363
1364 # AIX (and perhaps others?) doesn't have mktemp
1365 if ! which mktemp >/dev/null 2>&1 ; then
1366     mktemp ()
1367     {
1368         _dir=false
1369         if [ "$1" = "-d" ] ; then
1370             _dir=true
1371             shift
1372         fi
1373         _d="${TMPDIR:-/tmp}"
1374         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1375             md5sum | \
1376             sed -e 's@\(..........\).*@\1@')
1377         _t="${_d}/tmp.${_hex10}"
1378         (
1379             umask 077
1380             if $_dir ; then
1381                 mkdir "$_t"
1382             else
1383                 >"$_t"
1384             fi
1385         )
1386         echo "$_t"
1387     }
1388 fi
1389
1390 ########################################################
1391 # tickle handling
1392 ########################################################
1393
1394 update_tickles ()
1395 {
1396         _port="$1"
1397
1398         tickledir="$CTDB_VARDIR/state/tickles"
1399         mkdir -p "$tickledir"
1400
1401         # Who am I?
1402         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1403
1404         # What public IPs do I hold?
1405         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1406
1407         # IPs as a regexp choice
1408         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1409
1410         # Record connections to our public IPs in a temporary file
1411         _my_connections="${tickledir}/${_port}.connections"
1412         rm -f "$_my_connections"
1413         netstat -tn |
1414         awk -v destpat="^${_ipschoice}:${_port}\$" \
1415           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1416         sort >"$_my_connections"
1417
1418         # Record our current tickles in a temporary file
1419         _my_tickles="${tickledir}/${_port}.tickles"
1420         rm -f "$_my_tickles"
1421         for _i in $_ips ; do
1422                 ctdb -X gettickles $_i $_port |
1423                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1424         done |
1425         sort >"$_my_tickles"
1426
1427         # Add tickles for connections that we haven't already got tickles for
1428         comm -23 "$_my_connections" "$_my_tickles" |
1429         while read _src _dst ; do
1430                 ctdb addtickle $_src $_dst
1431         done
1432
1433         # Remove tickles for connections that are no longer there
1434         comm -13 "$_my_connections" "$_my_tickles" |
1435         while read _src _dst ; do
1436                 ctdb deltickle $_src $_dst
1437         done
1438
1439         rm -f "$_my_connections" "$_my_tickles"
1440 }
1441
1442 ########################################################
1443 # load a site local config file
1444 ########################################################
1445
1446 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1447         . "$CTDB_RC_LOCAL"
1448 }
1449
1450 [ -x $CTDB_BASE/rc.local ] && {
1451         . $CTDB_BASE/rc.local
1452 }
1453
1454 [ -d $CTDB_BASE/rc.local.d ] && {
1455         for i in $CTDB_BASE/rc.local.d/* ; do
1456                 [ -x "$i" ] && . "$i"
1457         done
1458 }
1459
1460 script_name="${0##*/}"       # basename
1461 service_fail_limit=1
1462 event_name="$1"