ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         elif ! tty -s ; then
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     fi
  72 }
  73
  74 die ()
  75 {
  76     _msg="$1"
  77     _rc="${2:-1}"
  78
  79     echo "$_msg"
  80     exit $_rc
  81 }
  82
  83 # Log given message or stdin to either syslog or a CTDB log file
  84 # $1 is the tag passed to logger if syslog is in use.
  85 script_log ()
  86 {
  87     _tag="$1" ; shift
  88
  89     case "$CTDB_LOGGING" in
  90         file:*|"")
  91             if [ -n "$CTDB_LOGGING" ] ; then
  92                 _file="${CTDB_LOGGING#file:}"
  93             else
  94                 _file="/var/log/log.ctdb"
  95             fi
  96             {
  97                 if [ -n "$*" ] ; then
  98                     echo "$*"
  99                 else
 100                     cat
 101                 fi
 102             } >>"$_file"
 103             ;;
 104         *)
 105             # Handle all syslog:* variants here too.  There's no tool to do
 106             # the lossy things, so just use logger.
 107             logger -t "ctdbd: ${_tag}" $*
 108             ;;
 109     esac
 110 }
 111
 112 # When things are run in the background in an eventscript then logging
 113 # output might get lost.  This is the "solution".  :-)
 114 background_with_logging ()
 115 {
 116     (
 117         "$@" 2>&1 </dev/null |
 118         script_log "${script_name}&"
 119     )&
 120
 121     return 0
 122 }
 123
 124 ##############################################################
 125 # check number of args for different events
 126 ctdb_check_args ()
 127 {
 128     case "$1" in
 129         takeip|releaseip)
 130             if [ $# != 4 ]; then
 131                 echo "ERROR: must supply interface, IP and maskbits"
 132                 exit 1
 133             fi
 134             ;;
 135         updateip)
 136             if [ $# != 5 ]; then
 137                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 138                 exit 1
 139             fi
 140             ;;
 141     esac
 142 }
 143
 144 ##############################################################
 145 # determine on what type of system (init style) we are running
 146 detect_init_style()
 147 {
 148     # only do detection if not already set:
 149     [ -z "$CTDB_INIT_STYLE" ] || return
 150
 151     if [ -x /sbin/startproc ]; then
 152         CTDB_INIT_STYLE="suse"
 153     elif [ -x /sbin/start-stop-daemon ]; then
 154         CTDB_INIT_STYLE="debian"
 155     else
 156         CTDB_INIT_STYLE="redhat"
 157     fi
 158 }
 159
 160 ######################################################
 161 # simulate /sbin/service on platforms that don't have it
 162 # _service() makes it easier to hook the service() function for
 163 # testing.
 164 _service ()
 165 {
 166   _service_name="$1"
 167   _op="$2"
 168
 169   # do nothing, when no service was specified
 170   [ -z "$_service_name" ] && return
 171
 172   if [ -x /sbin/service ]; then
 173       $_nice /sbin/service "$_service_name" "$_op"
 174   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 175       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 176   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 177       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 178   fi
 179 }
 180
 181 service()
 182 {
 183     _nice=""
 184     _service "$@"
 185 }
 186
 187 ######################################################
 188 # simulate /sbin/service (niced) on platforms that don't have it
 189 nice_service()
 190 {
 191     _nice="nice"
 192     _service "$@"
 193 }
 194
 195 ######################################################
 196 # wrapper around /proc/ settings to allow them to be hooked
 197 # for testing
 198 # 1st arg is relative path under /proc/, 2nd arg is value to set
 199 set_proc ()
 200 {
 201     echo "$2" >"/proc/$1"
 202 }
 203
 204 ######################################################
 205 # wrapper around getting file contents from /proc/ to allow
 206 # this to be hooked for testing
 207 # 1st arg is relative path under /proc/
 208 get_proc ()
 209 {
 210     cat "/proc/$1"
 211 }
 212
 213 ######################################################
 214 # Print up to $_max kernel stack traces for processes named $_program
 215 program_stack_traces ()
 216 {
 217     _prog="$1"
 218     _max="${2:-1}"
 219
 220     _count=1
 221     for _pid in $(pidof "$_prog") ; do
 222         [ $_count -le $_max ] || break
 223
 224         # Do this first to avoid racing with process exit
 225         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 226         if [ -n "$_stack" ] ; then
 227             echo "Stack trace for ${_prog}[${_pid}]:"
 228             echo "$_stack"
 229             _count=$(($_count + 1))
 230         fi
 231     done
 232 }
 233
 234 ######################################################
 235 # Check that an RPC service is healthy -
 236 # this includes allowing a certain number of failures
 237 # before marking the NFS service unhealthy.
 238 #
 239 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 240 #
 241 # each triple is a set of 3 arguments: an operator, a
 242 # fail count limit and an action string.
 243 #
 244 # For example:
 245 #
 246 #       nfs_check_rpc_service "lockd" \
 247 #           -ge 15 "verbose restart unhealthy" \
 248 #           -eq 10 "restart:bs"
 249 #
 250 # says that if lockd is down for 15 iterations then do
 251 # a verbose restart of lockd and mark the node unhealthy.
 252 # Before this, after 10 iterations of failure, the
 253 # service is restarted silently in the background.
 254 # Order is important: the number of failures need to be
 255 # specified in reverse order because processing stops
 256 # after the first condition that is true.
 257 ######################################################
 258 nfs_check_rpc_service ()
 259 {
 260     _prog_name="$1" ; shift
 261
 262     if _nfs_check_rpc_common "$_prog_name" ; then
 263         return
 264     fi
 265
 266     while [ -n "$3" ] ; do
 267         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 268             break
 269         fi
 270         shift 3
 271     done
 272 }
 273
 274 # The new way of doing things...
 275 nfs_check_rpc_services ()
 276 {
 277     # Files must end with .check - avoids editor backups, RPM fu, ...
 278     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 279         _t="${_f%.check}"
 280         _prog_name="${_t##*/[0-9][0-9].}"
 281
 282         if _nfs_check_rpc_common "$_prog_name" ; then
 283             # This RPC service is up, check next service...
 284             continue
 285         fi
 286
 287         # Check each line in the file in turn until one of the limit
 288         # checks is hit...
 289         while read _cmp _lim _rest ; do
 290             # Skip comments
 291             case "$_cmp" in
 292                 \#*) continue ;;
 293             esac
 294
 295             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 296                 # Limit was hit on this line, no further checking...
 297                 break
 298             fi
 299         done <"$_f"
 300     done
 301 }
 302
 303 _nfs_check_rpc_common ()
 304 {
 305     _prog_name="$1"
 306
 307     # Some platforms don't have separate programs for all services.
 308     case "$_prog_name" in
 309         statd)
 310             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 311     esac
 312
 313     case "$_prog_name" in
 314         nfsd)
 315             _rpc_prog=nfs
 316             _version=3
 317             ;;
 318         mountd)
 319             _rpc_prog=mountd
 320             _version=1
 321             ;;
 322         rquotad)
 323             _rpc_prog=rquotad
 324             _version=1
 325             ;;
 326         lockd)
 327             _rpc_prog=nlockmgr
 328             _version=4
 329             ;;
 330         statd)
 331             _rpc_prog=status
 332             _version=1
 333             ;;
 334         *)
 335             echo "Internal error: unknown RPC program \"$_prog_name\"."
 336             exit 1
 337     esac
 338
 339     _service_name="nfs_${_prog_name}"
 340
 341     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 342         ctdb_counter_init "$_service_name"
 343         return 0
 344     fi
 345
 346     ctdb_counter_incr "$_service_name"
 347
 348     return 1
 349 }
 350
 351 _nfs_check_rpc_action ()
 352 {
 353     _cmp="$1"
 354     _limit="$2"
 355     _actions="$3"
 356
 357     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 358         return 1
 359     fi
 360
 361     for _action in $_actions ; do
 362         case "$_action" in
 363             verbose)
 364                 echo "$ctdb_check_rpc_out"
 365                 ;;
 366             restart)
 367                 _nfs_restart_rpc_service "$_prog_name"
 368                 ;;
 369             restart:b)
 370                 _nfs_restart_rpc_service "$_prog_name" true
 371                 ;;
 372             unhealthy)
 373                 exit 1
 374                 ;;
 375             *)
 376                 echo "Internal error: unknown action \"$_action\"."
 377                 exit 1
 378         esac
 379     done
 380
 381     return 0
 382 }
 383
 384 _nfs_restart_rpc_service ()
 385 {
 386     _prog_name="$1"
 387     _background="${2:-false}"
 388
 389     if $_background ; then
 390         _maybe_background="background_with_logging"
 391     else
 392         _maybe_background=""
 393     fi
 394
 395     _p="rpc.${_prog_name}"
 396
 397     case "$_prog_name" in
 398         nfsd)
 399             echo "Trying to restart NFS service"
 400             $_maybe_background startstop_nfs restart
 401             ;;
 402         mountd)
 403             echo "Trying to restart $_prog_name [${_p}]"
 404             killall -q -9 "$_p"
 405             nfs_dump_some_threads "$_p"
 406             $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 407             ;;
 408         rquotad)
 409             echo "Trying to restart $_prog_name [${_p}]"
 410             killall -q -9 "$_p"
 411             nfs_dump_some_threads "$_p"
 412             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 413             ;;
 414         lockd)
 415             echo "Trying to restart lock manager service"
 416             $_maybe_background startstop_nfslock restart
 417             ;;
 418         statd)
 419             echo "Trying to restart $_prog_name [${_p}]"
 420             killall -q -9 "$_p"
 421             nfs_dump_some_threads "$_p"
 422             $_maybe_background $_p \
 423                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 424                 ${STATD_PORT:+-p} $STATD_PORT \
 425                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 426             ;;
 427         *)
 428             echo "Internal error: unknown RPC program \"$_prog_name\"."
 429             exit 1
 430     esac
 431 }
 432
 433 ######################################################
 434 # check that a rpc server is registered with portmap
 435 # and responding to requests
 436 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 437 ######################################################
 438 ctdb_check_rpc ()
 439 {
 440     progname="$1"
 441     version="$2"
 442
 443     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 444
 445     if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
 446         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 447 $ctdb_check_rpc_out"
 448         echo "$ctdb_check_rpc_out"
 449         return 1
 450     fi
 451 }
 452
 453 ######################################################
 454 # Ensure $service_name is set
 455 assert_service_name ()
 456 {
 457     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 458 }
 459
 460 ######################################################
 461 # check a set of directories is available
 462 # return 1 on a missing directory
 463 # directories are read from stdin
 464 ######################################################
 465 ctdb_check_directories_probe()
 466 {
 467     while IFS="" read d ; do
 468         case "$d" in
 469             *%*)
 470                 continue
 471                 ;;
 472             *)
 473                 [ -d "${d}/." ] || return 1
 474         esac
 475     done
 476 }
 477
 478 ######################################################
 479 # check a set of directories is available
 480 # directories are read from stdin
 481 ######################################################
 482 ctdb_check_directories()
 483 {
 484     ctdb_check_directories_probe || {
 485         echo "ERROR: $service_name directory \"$d\" not available"
 486         exit 1
 487     }
 488 }
 489
 490 ######################################################
 491 # check a set of tcp ports
 492 # usage: ctdb_check_tcp_ports <ports...>
 493 ######################################################
 494
 495 # This flag file is created when a service is initially started.  It
 496 # is deleted the first time TCP port checks for that service succeed.
 497 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 498 # message if a port check fails.
 499 _ctdb_check_tcp_common ()
 500 {
 501     assert_service_name
 502     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 503 }
 504
 505 ctdb_check_tcp_init ()
 506 {
 507     _ctdb_check_tcp_common
 508     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 509     touch "$_ctdb_service_started_file"
 510 }
 511
 512 # Check whether something is listening on all of the given TCP ports
 513 # using the "ctdb checktcpport" command.
 514 ctdb_check_tcp_ports()
 515 {
 516     if [ -z "$1" ] ; then
 517         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 518         exit 1
 519     fi
 520
 521     for _p ; do  # process each function argument (port)
 522         _cmd="ctdb checktcpport $_p"
 523         _out=$($_cmd 2>&1)
 524         _ret=$?
 525         case "$_ret" in
 526             0)
 527                 _ctdb_check_tcp_common
 528                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 529                     echo "ERROR: $service_name tcp port $_p is not responding"
 530                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 531                 else
 532                     echo "INFO: $service_name tcp port $_p is not responding"
 533                 fi
 534
 535                 return 1
 536                 ;;
 537             98)
 538                 # Couldn't bind, something already listening, next port...
 539                 continue
 540                 ;;
 541             *)
 542                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 543                 debug <<EOF
 544 ctdb checktcpport (exited with $_ret) with output:
 545 $_out"
 546 EOF
 547                 return $_ret
 548         esac
 549     done
 550
 551     # All ports listening
 552     _ctdb_check_tcp_common
 553     rm -f "$_ctdb_service_started_file"
 554     return 0
 555 }
 556
 557 ######################################################
 558 # check a unix socket
 559 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 560 ######################################################
 561 ctdb_check_unix_socket() {
 562     socket_path="$1"
 563     [ -z "$socket_path" ] && return
 564
 565     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 566         echo "ERROR: $service_name socket $socket_path not found"
 567         return 1
 568     fi
 569 }
 570
 571 ######################################################
 572 # check a command returns zero status
 573 # usage: ctdb_check_command <command>
 574 ######################################################
 575 ctdb_check_command ()
 576 {
 577     _out=$("$@" 2>&1) || {
 578         echo "ERROR: $* returned error"
 579         echo "$_out" | debug
 580         exit 1
 581     }
 582 }
 583
 584 ################################################
 585 # kill off any TCP connections with the given IP
 586 ################################################
 587 kill_tcp_connections ()
 588 {
 589     _ip="$1"
 590
 591     _oneway=false
 592     if [ "$2" = "oneway" ] ; then
 593         _oneway=true
 594     fi
 595
 596     get_tcp_connections_for_ip "$_ip" | {
 597         _killcount=0
 598         _connections=""
 599         _nl="
 600 "
 601         while read _dst _src; do
 602             _destport="${_dst##*:}"
 603             __oneway=$_oneway
 604             case $_destport in
 605                 # we only do one-way killtcp for CIFS
 606                 139|445) __oneway=true ;;
 607             esac
 608
 609             echo "Killing TCP connection $_src $_dst"
 610             _connections="${_connections}${_nl}${_src} ${_dst}"
 611             if ! $__oneway ; then
 612                 _connections="${_connections}${_nl}${_dst} ${_src}"
 613             fi
 614
 615             _killcount=$(($_killcount + 1))
 616         done
 617
 618         if [ $_killcount -eq 0 ] ; then
 619             return
 620         fi
 621
 622         echo "$_connections" | ctdb killtcp || {
 623             echo "Failed to send killtcp control"
 624             return
 625         }
 626
 627         _count=0
 628         while : ; do
 629             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 630
 631             if [ $_remaining -eq 0 ] ; then
 632                 echo "Killed $_killcount TCP connections to released IP $_ip"
 633                 return
 634             fi
 635
 636             _count=$(($_count + 1))
 637             if [ $_count -gt 3 ] ; then
 638                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 639                 return
 640             fi
 641
 642             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 643             sleep 1
 644         done
 645     }
 646 }
 647
 648 ##################################################################
 649 # kill off the local end for any TCP connections with the given IP
 650 ##################################################################
 651 kill_tcp_connections_local_only ()
 652 {
 653     kill_tcp_connections "$1" "oneway"
 654 }
 655
 656 ##################################################################
 657 # tickle any TCP connections with the given IP
 658 ##################################################################
 659 tickle_tcp_connections ()
 660 {
 661     _ip="$1"
 662
 663     get_tcp_connections_for_ip "$_ip" |
 664     {
 665         _failed=false
 666
 667         while read dest src; do
 668             echo "Tickle TCP connection $src $dest"
 669             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 670             echo "Tickle TCP connection $dest $src"
 671             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 672         done
 673
 674         if $_failed ; then
 675             echo "Failed to send tickle control"
 676         fi
 677     }
 678 }
 679
 680 get_tcp_connections_for_ip ()
 681 {
 682     _ip="$1"
 683
 684     netstat -tn | awk -v ip=$_ip \
 685         'index($1, "tcp") == 1 && \
 686          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 687          && $6 == "ESTABLISHED" \
 688          {print $4" "$5}'
 689 }
 690
 691 ########################################################
 692 # start/stop the Ganesha nfs service
 693 ########################################################
 694 startstop_ganesha()
 695 {
 696     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 697     case "$1" in
 698         start)
 699             service "$_service_name" start
 700             ;;
 701         stop)
 702             service "$_service_name" stop
 703             ;;
 704         restart)
 705             service "$_service_name" stop
 706             nfs_dump_some_threads "rpc.statd"
 707             service "$_service_name" start
 708             ;;
 709     esac
 710 }
 711
 712 ########################################################
 713 # start/stop the nfs service on different platforms
 714 ########################################################
 715 startstop_nfs() {
 716         PLATFORM="unknown"
 717         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 718                 PLATFORM="sles"
 719         }
 720         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 721             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 722                 PLATFORM="rhel"
 723         }
 724
 725         case $PLATFORM in
 726         sles)
 727                 case $1 in
 728                 start)
 729                         service nfsserver start
 730                         ;;
 731                 stop)
 732                         service nfsserver stop > /dev/null 2>&1
 733                         ;;
 734                 restart)
 735                         set_proc "fs/nfsd/threads" 0
 736                         service nfsserver stop > /dev/null 2>&1
 737                         pkill -9 nfsd
 738                         nfs_dump_some_threads
 739                         service nfsserver start
 740                         ;;
 741                 esac
 742                 ;;
 743         rhel)
 744                 case $1 in
 745                 start)
 746                         service nfslock start
 747                         service nfs start
 748                         ;;
 749                 stop)
 750                         service nfs stop
 751                         service nfslock stop
 752                         ;;
 753                 restart)
 754                         set_proc "fs/nfsd/threads" 0
 755                         service nfs stop > /dev/null 2>&1
 756                         service nfslock stop > /dev/null 2>&1
 757                         pkill -9 nfsd
 758                         nfs_dump_some_threads
 759                         service nfslock start
 760                         service nfs start
 761                         ;;
 762                 esac
 763                 ;;
 764         *)
 765                 echo "Unknown platform. NFS is not supported with ctdb"
 766                 exit 1
 767                 ;;
 768         esac
 769 }
 770
 771 # Dump up to the configured number of nfsd thread backtraces.
 772 nfs_dump_some_threads ()
 773 {
 774     _prog="${1:-nfsd}"
 775
 776     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 777     [ $_num -gt 0 ] || return 0
 778
 779     program_stack_traces "$_prog" $_num
 780 }
 781
 782 ########################################################
 783 # start/stop the nfs lockmanager service on different platforms
 784 ########################################################
 785 startstop_nfslock() {
 786         PLATFORM="unknown"
 787         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 788                 PLATFORM="sles"
 789         }
 790         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 791             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 792                 PLATFORM="rhel"
 793         }
 794
 795         case $PLATFORM in
 796         sles)
 797                 # for sles there is no service for lockmanager
 798                 # so we instead just shutdown/restart nfs
 799                 case $1 in
 800                 start)
 801                         service nfsserver start
 802                         ;;
 803                 stop)
 804                         service nfsserver stop > /dev/null 2>&1
 805                         ;;
 806                 restart)
 807                         service nfsserver stop > /dev/null 2>&1
 808                         service nfsserver start
 809                         ;;
 810                 esac
 811                 ;;
 812         rhel)
 813                 case $1 in
 814                 start)
 815                         service nfslock start
 816                         ;;
 817                 stop)
 818                         service nfslock stop > /dev/null 2>&1
 819                         ;;
 820                 restart)
 821                         service nfslock stop > /dev/null 2>&1
 822                         service nfslock start
 823                         ;;
 824                 esac
 825                 ;;
 826         *)
 827                 echo "Unknown platform. NFS locking is not supported with ctdb"
 828                 exit 1
 829                 ;;
 830         esac
 831 }
 832
 833 # Periodically update the statd database
 834 nfs_statd_update ()
 835 {
 836     _update_period="$1"
 837
 838     _statd_update_trigger="$service_state_dir/update-trigger"
 839     [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
 840
 841     _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
 842     _current_time=$(date +"%s")
 843     if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
 844         touch "$_statd_update_trigger"
 845         $CTDB_BASE/statd-callout updatelocal &
 846         $CTDB_BASE/statd-callout updateremote &
 847     fi
 848 }
 849
 850 ########################################################
 851
 852 add_ip_to_iface ()
 853 {
 854     _iface=$1
 855     _ip=$2
 856     _maskbits=$3
 857
 858     # Ensure interface is up
 859     ip link set "$_iface" up || \
 860         die "Failed to bringup interface $_iface"
 861
 862     # Only need to define broadcast for IPv4
 863     case "$ip" in
 864         *:*) _bcast=""      ;;
 865         *)   _bcast="brd +" ;;
 866     esac
 867
 868     ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
 869         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 870         return 1
 871     }
 872
 873     # Wait 5 seconds for IPv6 addresses to stop being tentative...
 874     if [ -z "$_bcast" ] ; then
 875         for _x in $(seq 1 10) ; do
 876             ip addr show to "${_ip}/128" | grep -q "tentative" || break
 877             sleep 0.5
 878         done
 879
 880         # If the address was a duplicate then it won't be on the
 881         # interface so flag an error.
 882         _t=$(ip addr show to "${_ip}/128")
 883         case "$_t" in
 884             "")
 885                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 886                 return 1
 887                 ;;
 888             *tentative*|*dadfailed*)
 889                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 890                 ip addr del "$_ip/$_maskbits" dev "$_iface"
 891                 return 1
 892                 ;;
 893         esac
 894     fi
 895 }
 896
 897 delete_ip_from_iface()
 898 {
 899     _iface=$1
 900     _ip=$2
 901     _maskbits=$3
 902
 903     # This could be set globally for all interfaces but it is probably
 904     # better to avoid surprises, so limit it the interfaces where CTDB
 905     # has public IP addresses.  There isn't anywhere else convenient
 906     # to do this so just set it each time.  This is much cheaper than
 907     # remembering and re-adding secondaries.
 908     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 909
 910     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 911         echo "Failed to del $_ip on dev $_iface"
 912         return 1
 913     }
 914 }
 915
 916 # If the given IP is hosted then print 2 items: maskbits and iface
 917 ip_maskbits_iface ()
 918 {
 919     _addr="$1"
 920
 921     case "$_addr" in
 922         *:*) _family="inet6" ; _bits=128 ;;
 923         *)   _family="inet"  ; _bits=32  ;;
 924     esac
 925
 926     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 927         awk -v family="${_family}" \
 928             'NR == 1 { iface = gensub(":$", "", 1, $2) } \
 929              $1 ~ /inet/ { print gensub(".*/", "", 1, $2), iface, family }'
 930 }
 931
 932 drop_ip ()
 933 {
 934     _addr="${1%/*}"  # Remove optional maskbits
 935
 936     set -- $(ip_maskbits_iface $_addr)
 937     if [ -n "$1" ] ; then
 938         _maskbits="$1"
 939         _iface="$2"
 940         echo "Removing public address $_addr/$_maskbits from device $_iface"
 941         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 942     fi
 943 }
 944
 945 drop_all_public_ips ()
 946 {
 947     while read _ip _x ; do
 948         drop_ip "$_ip"
 949     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 950 }
 951
 952 flush_route_cache ()
 953 {
 954     set_proc sys/net/ipv4/route/flush 1
 955     set_proc sys/net/ipv6/route/flush 1
 956 }
 957
 958 ########################################################
 959 # Simple counters
 960 _ctdb_counter_common () {
 961     _service_name="${1:-${service_name:-${script_name}}}"
 962     _counter_file="$ctdb_fail_dir/$_service_name"
 963     mkdir -p "${_counter_file%/*}" # dirname
 964 }
 965 ctdb_counter_init () {
 966     _ctdb_counter_common "$1"
 967
 968     >"$_counter_file"
 969 }
 970 ctdb_counter_incr () {
 971     _ctdb_counter_common "$1"
 972
 973     # unary counting!
 974     echo -n 1 >> "$_counter_file"
 975 }
 976 ctdb_check_counter () {
 977     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 978     _op="${2:--ge}"  # an integer operator supported by test
 979     _limit="${3:-${service_fail_limit}}"
 980     shift 3
 981     _ctdb_counter_common "$1"
 982
 983     # unary counting!
 984     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 985     _hit=false
 986     if [ "$_op" != "%" ] ; then
 987         if [ $_size $_op $_limit ] ; then
 988             _hit=true
 989         fi
 990     else
 991         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
 992             _hit=true
 993         fi
 994     fi
 995     if $_hit ; then
 996         if [ "$_msg" = "error" ] ; then
 997             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
 998             exit 1
 999         else
1000             return 1
1001         fi
1002     fi
1003 }
1004
1005 ########################################################
1006
1007 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1008 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1009
1010 ctdb_setup_service_state_dir ()
1011 {
1012     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1013     mkdir -p "$service_state_dir" || {
1014         echo "Error creating state dir \"$service_state_dir\""
1015         exit 1
1016     }
1017 }
1018
1019 ########################################################
1020 # Managed status history, for auto-start/stop
1021
1022 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1023
1024 _ctdb_managed_common ()
1025 {
1026     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1027 }
1028
1029 ctdb_service_managed ()
1030 {
1031     _ctdb_managed_common
1032     mkdir -p "$ctdb_managed_dir"
1033     touch "$_ctdb_managed_file"
1034 }
1035
1036 ctdb_service_unmanaged ()
1037 {
1038     _ctdb_managed_common
1039     rm -f "$_ctdb_managed_file"
1040 }
1041
1042 is_ctdb_previously_managed_service ()
1043 {
1044     _ctdb_managed_common
1045     [ -f "$_ctdb_managed_file" ]
1046 }
1047
1048 ########################################################
1049 # Check and set status
1050
1051 log_status_cat ()
1052 {
1053     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1054 }
1055
1056 ctdb_checkstatus ()
1057 {
1058     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1059         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1060         return 1
1061     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1062         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1063         return 2
1064     else
1065         return 0
1066     fi
1067 }
1068
1069 ctdb_setstatus ()
1070 {
1071     d="$ctdb_status_dir/$script_name"
1072     case "$1" in
1073         unhealthy|banned)
1074             mkdir -p "$d"
1075             cat "$2" >"$d/$1"
1076             ;;
1077         *)
1078             for i in "banned" "unhealthy" ; do
1079                 rm -f "$d/$i"
1080             done
1081             ;;
1082     esac
1083 }
1084
1085 ##################################################################
1086 # Reconfigure a service on demand
1087
1088 _ctdb_service_reconfigure_common ()
1089 {
1090     _d="$ctdb_status_dir/${service_name}"
1091     mkdir -p "$_d"
1092     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1093 }
1094
1095 ctdb_service_needs_reconfigure ()
1096 {
1097     _ctdb_service_reconfigure_common
1098     [ -e "$_ctdb_service_reconfigure_flag" ]
1099 }
1100
1101 ctdb_service_set_reconfigure ()
1102 {
1103     _ctdb_service_reconfigure_common
1104     >"$_ctdb_service_reconfigure_flag"
1105 }
1106
1107 ctdb_service_unset_reconfigure ()
1108 {
1109     _ctdb_service_reconfigure_common
1110     rm -f "$_ctdb_service_reconfigure_flag"
1111 }
1112
1113 ctdb_service_reconfigure ()
1114 {
1115     echo "Reconfiguring service \"${service_name}\"..."
1116     ctdb_service_unset_reconfigure
1117     service_reconfigure || return $?
1118     ctdb_counter_init
1119 }
1120
1121 # Default service_reconfigure() function does nothing.
1122 service_reconfigure ()
1123 {
1124     :
1125 }
1126
1127 ctdb_reconfigure_take_lock ()
1128 {
1129     _ctdb_service_reconfigure_common
1130     _lock="${_d}/reconfigure_lock"
1131     mkdir -p "${_lock%/*}" # dirname
1132     touch "$_lock"
1133
1134     (
1135         flock 0
1136         # This is overkill but will work if we need to extend this to
1137         # allow certain events to run multiple times in parallel
1138         # (e.g. takeip) and write multiple PIDs to the file.
1139         read _locker_event
1140         if [ -n "$_locker_event" ] ; then
1141             while read _pid ; do
1142                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1143                     kill -0 "$_pid" 2>/dev/null ; then
1144                     exit 1
1145                 fi
1146             done
1147         fi
1148
1149         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1150         exit 0
1151     ) <"$_lock"
1152 }
1153
1154 ctdb_reconfigure_release_lock ()
1155 {
1156     _ctdb_service_reconfigure_common
1157     _lock="${_d}/reconfigure_lock"
1158
1159     rm -f "$_lock"
1160 }
1161
1162 ctdb_replay_monitor_status ()
1163 {
1164     echo "Replaying previous status for this script due to reconfigure..."
1165     # Leading separator ('|') is missing in some versions...
1166     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1167     # Output looks like this:
1168     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1169     # This is the cheapest way of getting fields in the middle.
1170     set -- $(IFS="|" ; echo $_out)
1171     _code="$3"
1172     _status="$4"
1173     # The error output field can include colons so we'll try to
1174     # preserve them.  The weak checking at the beginning tries to make
1175     # this work for both broken (no leading '|') and fixed output.
1176     _out="${_out%|}"
1177     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1178     case "$_status" in
1179         OK) : ;;  # Do nothing special.
1180         TIMEDOUT)
1181             # Recast this as an error, since we can't exit with the
1182             # correct negative number.
1183             _code=1
1184             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1185             ;;
1186         DISABLED)
1187             # Recast this as an OK, since we can't exit with the
1188             # correct negative number.
1189             _code=0
1190             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1191             ;;
1192         *) : ;;  # Must be ERROR, do nothing special.
1193     esac
1194     if [ -n "$_err_out" ] ; then
1195         echo "$_err_out"
1196     fi
1197     exit $_code
1198 }
1199
1200 ctdb_service_check_reconfigure ()
1201 {
1202     assert_service_name
1203
1204     # We only care about some events in this function.  For others we
1205     # return now.
1206     case "$event_name" in
1207         monitor|ipreallocated|reconfigure) : ;;
1208         *) return 0 ;;
1209     esac
1210
1211     if ctdb_reconfigure_take_lock ; then
1212         # No events covered by this function are running, so proceed
1213         # with gay abandon.
1214         case "$event_name" in
1215             reconfigure)
1216                 (ctdb_service_reconfigure)
1217                 exit $?
1218                 ;;
1219             ipreallocated)
1220                 if ctdb_service_needs_reconfigure ; then
1221                     ctdb_service_reconfigure
1222                 fi
1223                 ;;
1224         esac
1225
1226         ctdb_reconfigure_release_lock
1227     else
1228         # Somebody else is running an event we don't want to collide
1229         # with.  We proceed with caution.
1230         case "$event_name" in
1231             reconfigure)
1232                 # Tell whoever called us to retry.
1233                 exit 2
1234                 ;;
1235             ipreallocated)
1236                 # Defer any scheduled reconfigure and just run the
1237                 # rest of the ipreallocated event, as per the
1238                 # eventscript.  There's an assumption here that the
1239                 # event doesn't depend on any scheduled reconfigure.
1240                 # This is true in the current code.
1241                 return 0
1242                 ;;
1243             monitor)
1244                 # There is most likely a reconfigure in progress so
1245                 # the service is possibly unstable.  As above, we
1246                 # defer any scheduled reconfigured.  We also replay
1247                 # the previous monitor status since that's the best
1248                 # information we have.
1249                 ctdb_replay_monitor_status
1250                 ;;
1251         esac
1252     fi
1253 }
1254
1255 ##################################################################
1256 # Does CTDB manage this service? - and associated auto-start/stop
1257
1258 ctdb_compat_managed_service ()
1259 {
1260     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1261         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1262     fi
1263 }
1264
1265 is_ctdb_managed_service ()
1266 {
1267     assert_service_name
1268
1269     # $t is used just for readability and to allow better accurate
1270     # matching via leading/trailing spaces
1271     t=" $CTDB_MANAGED_SERVICES "
1272
1273     # Return 0 if "<space>$service_name<space>" appears in $t
1274     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1275         return 0
1276     fi
1277
1278     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1279     # backward compatibility and try again.
1280     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1281     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1282     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1283     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1284     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1285     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1286     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1287     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1288     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1289
1290     t=" $CTDB_MANAGED_SERVICES "
1291
1292     # Return 0 if "<space>$service_name<space>" appears in $t
1293     [ "${t#* ${service_name} }" != "${t}" ]
1294 }
1295
1296 ctdb_start_stop_service ()
1297 {
1298     assert_service_name
1299
1300     # Allow service-start/service-stop pseudo-events to start/stop
1301     # services when we're not auto-starting/stopping and we're not
1302     # monitoring.
1303     case "$event_name" in
1304         service-start)
1305             if is_ctdb_managed_service ; then
1306                 die 'service-start event not permitted when service is managed'
1307             fi
1308             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1309                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1310             fi
1311             ctdb_service_start
1312             exit $?
1313             ;;
1314         service-stop)
1315             if is_ctdb_managed_service ; then
1316                 die 'service-stop event not permitted when service is managed'
1317             fi
1318             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1319                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1320             fi
1321             ctdb_service_stop
1322             exit $?
1323             ;;
1324     esac
1325
1326     # Do nothing unless configured to...
1327     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1328
1329     [ "$event_name" = "monitor" ] || return 0
1330
1331     if is_ctdb_managed_service ; then
1332         if ! is_ctdb_previously_managed_service ; then
1333             echo "Starting service \"$service_name\" - now managed"
1334             background_with_logging ctdb_service_start
1335             exit $?
1336         fi
1337     else
1338         if is_ctdb_previously_managed_service ; then
1339             echo "Stopping service \"$service_name\" - no longer managed"
1340             background_with_logging ctdb_service_stop
1341             exit $?
1342         fi
1343     fi
1344 }
1345
1346 ctdb_service_start ()
1347 {
1348     # The service is marked managed if we've ever tried to start it.
1349     ctdb_service_managed
1350
1351     service_start || return $?
1352
1353     ctdb_counter_init
1354     ctdb_check_tcp_init
1355 }
1356
1357 ctdb_service_stop ()
1358 {
1359     ctdb_service_unmanaged
1360     service_stop
1361 }
1362
1363 # Default service_start() and service_stop() functions.
1364
1365 # These may be overridden in an eventscript.
1366 service_start ()
1367 {
1368     service "$service_name" start
1369 }
1370
1371 service_stop ()
1372 {
1373     service "$service_name" stop
1374 }
1375
1376 ##################################################################
1377
1378 ctdb_standard_event_handler ()
1379 {
1380     case "$1" in
1381         status)
1382             ctdb_checkstatus
1383             exit
1384             ;;
1385         setstatus)
1386             shift
1387             ctdb_setstatus "$@"
1388             exit
1389             ;;
1390     esac
1391 }
1392
1393 # iptables doesn't like being re-entered, so flock-wrap it.
1394 iptables ()
1395 {
1396         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1397 }
1398 ip6tables ()
1399 {
1400         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/ip6tables "$@"
1401 }
1402 iptables_wrapper ()
1403 {
1404     _family="$1" ; shift
1405     if [ "$_family" = "inet6" ] ; then
1406         ip6tables "$@"
1407     else
1408         iptables "$@"
1409     fi
1410 }
1411
1412 # AIX (and perhaps others?) doesn't have mktemp
1413 if ! which mktemp >/dev/null 2>&1 ; then
1414     mktemp ()
1415     {
1416         _dir=false
1417         if [ "$1" = "-d" ] ; then
1418             _dir=true
1419             shift
1420         fi
1421         _d="${TMPDIR:-/tmp}"
1422         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1423             md5sum | \
1424             sed -e 's@\(..........\).*@\1@')
1425         _t="${_d}/tmp.${_hex10}"
1426         (
1427             umask 077
1428             if $_dir ; then
1429                 mkdir "$_t"
1430             else
1431                 >"$_t"
1432             fi
1433         )
1434         echo "$_t"
1435     }
1436 fi
1437
1438 ########################################################
1439 # tickle handling
1440 ########################################################
1441
1442 update_tickles ()
1443 {
1444         _port="$1"
1445
1446         tickledir="$CTDB_VARDIR/state/tickles"
1447         mkdir -p "$tickledir"
1448
1449         # Who am I?
1450         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1451
1452         # What public IPs do I hold?
1453         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1454
1455         # IPs as a regexp choice
1456         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1457
1458         # Record connections to our public IPs in a temporary file
1459         _my_connections="${tickledir}/${_port}.connections"
1460         rm -f "$_my_connections"
1461         netstat -tn |
1462         awk -v destpat="^${_ipschoice}:${_port}\$" \
1463           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1464         sort >"$_my_connections"
1465
1466         # Record our current tickles in a temporary file
1467         _my_tickles="${tickledir}/${_port}.tickles"
1468         rm -f "$_my_tickles"
1469         for _i in $_ips ; do
1470                 ctdb -X gettickles $_i $_port |
1471                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1472         done |
1473         sort >"$_my_tickles"
1474
1475         # Add tickles for connections that we haven't already got tickles for
1476         comm -23 "$_my_connections" "$_my_tickles" |
1477         while read _src _dst ; do
1478                 ctdb addtickle $_src $_dst
1479         done
1480
1481         # Remove tickles for connections that are no longer there
1482         comm -13 "$_my_connections" "$_my_tickles" |
1483         while read _src _dst ; do
1484                 ctdb deltickle $_src $_dst
1485         done
1486
1487         rm -f "$_my_connections" "$_my_tickles"
1488 }
1489
1490 ########################################################
1491 # load a site local config file
1492 ########################################################
1493
1494 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1495         . "$CTDB_RC_LOCAL"
1496 }
1497
1498 [ -x $CTDB_BASE/rc.local ] && {
1499         . $CTDB_BASE/rc.local
1500 }
1501
1502 [ -d $CTDB_BASE/rc.local.d ] && {
1503         for i in $CTDB_BASE/rc.local.d/* ; do
1504                 [ -x "$i" ] && . "$i"
1505         done
1506 }
1507
1508 script_name="${0##*/}"       # basename
1509 service_fail_limit=1
1510 event_name="$1"