config/functions

   1 # utility functions for ctdb event scripts
   2
   3 PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
   4
   5 #######################################
   6 # pull in a system config file, if any
   7 loadconfig() {
   8
   9     if [ "$1" != "ctdb" ] ; then
  10         loadconfig "ctdb"
  11     fi
  12
  13     if [ -z "$1" ] ; then
  14         foo="${service_config:-${service_name}}"
  15         if [ -n "$foo" ] ; then
  16             loadconfig "$foo"
  17         fi
  18     fi
  19
  20     if [ -f /etc/sysconfig/$1 ]; then
  21         . /etc/sysconfig/$1
  22     elif [ -f /etc/default/$1 ]; then
  23         . /etc/default/$1
  24     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  25         . $CTDB_BASE/sysconfig/$1
  26     fi
  27 }
  28
  29 ##############################################################
  30 # determine on what type of system (init style) we are running
  31 detect_init_style() {
  32     # only do detection if not already set:
  33     test "x$CTDB_INIT_STYLE" != "x" && return
  34
  35     if [ -x /sbin/startproc ]; then
  36         CTDB_INIT_STYLE="suse"
  37     elif [ -x /sbin/start-stop-daemon ]; then
  38         CTDB_INIT_STYLE="debian"
  39     else
  40         CTDB_INIT_STYLE="redhat"
  41     fi
  42 }
  43
  44 ######################################################
  45 # simulate /sbin/service on platforms that don't have it
  46 service() {
  47   _service_name="$1"
  48   _op="$2"
  49
  50   # do nothing, when no service was specified
  51   [ -z "$_service_name" ] && return
  52
  53   if [ -x /sbin/service ]; then
  54       /sbin/service "$_service_name" "$_op"
  55   elif [ -x /etc/init.d/$_service_name ]; then
  56       /etc/init.d/$_service_name "$_op"
  57   elif [ -x /etc/rc.d/init.d/$_service_name ]; then
  58       /etc/rc.d/init.d/$_service_name "$_op"
  59   fi
  60 }
  61
  62 ######################################################
  63 # simulate /sbin/service (niced) on platforms that don't have it
  64 nice_service() {
  65   # do nothing, when no service was specified
  66   [ -z "$1" ] && return
  67
  68     nice service "$@"
  69 }
  70
  71 ######################################################
  72 # wait for a command to return a zero exit status
  73 # usage: ctdb_wait_command SERVICE_NAME <command>
  74 ######################################################
  75 ctdb_wait_command() {
  76   service_name="$1"
  77   wait_cmd="$2"
  78   [ -z "$wait_cmd" ] && return;
  79   all_ok=0
  80   echo "Waiting for service $service_name to start"
  81   while [ $all_ok -eq 0 ]; do
  82           $wait_cmd > /dev/null 2>&1 && all_ok=1
  83           ctdb status > /dev/null 2>&1 || {
  84                 echo "ctdb daemon has died. Exiting wait for $service_name"
  85                 exit 1
  86           }
  87           [ $all_ok -eq 1 ] || sleep 1
  88   done
  89   echo "Local service $service_name is up"
  90 }
  91
  92
  93 ######################################################
  94 # wait for a set of tcp ports
  95 # usage: ctdb_wait_tcp_ports SERVICE_NAME <ports...>
  96 ######################################################
  97 ctdb_wait_tcp_ports() {
  98   service_name="$1"
  99   shift
 100   wait_ports="$*"
 101   [ -z "$wait_ports" ] && return;
 102   all_ok=0
 103   echo "Waiting for tcp service $service_name to start"
 104   while [ $all_ok -eq 0 ]; do
 105           all_ok=1
 106           for p in $wait_ports; do
 107               if [ -x /usr/bin/netcat ]; then
 108                   /usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
 109               elif [ -x /usr/bin/nc ]; then
 110                   /usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
 111               elif [ -x /usr/bin/netstat ]; then
 112                   (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
 113               elif [ -x /bin/netstat ]; then
 114                   (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
 115               else
 116                   echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
 117                   return 127
 118               fi
 119           done
 120           [ $all_ok -eq 1 ] || sleep 1
 121           ctdb status > /dev/null 2>&1 || {
 122                 echo "ctdb daemon has died. Exiting tcp wait $service_name"
 123                 return 1
 124           }
 125   done
 126   echo "Local tcp services for $service_name are up"
 127 }
 128
 129
 130 ######################################################
 131 # check that a rpc server is registered with portmap
 132 # and responding to requests
 133 # usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
 134 ######################################################
 135 ctdb_check_rpc() {
 136     progname="$1"
 137     prognum="$2"
 138     version="$3"
 139     rpcinfo -u localhost $prognum $version > /dev/null || {
 140             echo "ERROR: $progname not responding to rpc requests"
 141             exit 1
 142     }
 143 }
 144
 145 ######################################################
 146 # check a set of directories is available
 147 # return 1 on a missing directory
 148 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
 149 ######################################################
 150 ctdb_check_directories_probe() {
 151     while IFS="" read d ; do
 152         case "$d" in
 153             *%*)
 154                 continue
 155                 ;;
 156             *)
 157                 [ -d "$d" ] || return 1
 158         esac
 159     done
 160 }
 161
 162 ######################################################
 163 # check a set of directories is available
 164 # usage: ctdb_check_directories SERVICE_NAME <directories...>
 165 ######################################################
 166 ctdb_check_directories() {
 167     n="${1:-${service_name}}"
 168     ctdb_check_directories_probe || {
 169         echo "ERROR: $n directory \"$d\" not available"
 170         exit 1
 171     }
 172 }
 173
 174 ######################################################
 175 # check a set of tcp ports
 176 # usage: ctdb_check_tcp_ports <ports...>
 177 ######################################################
 178 ctdb_check_tcp_ports() {
 179
 180     for p ; do
 181         if ! netstat -a -t -n | grep -q "0\.0\.0\.0:$p .*LISTEN" ; then
 182             if ! netstat -a -t -n | grep -q ":::$p .*LISTEN" ; then
 183                 echo "ERROR: $service_name tcp port $p is not responding"
 184                 return 1
 185             fi
 186         fi
 187     done
 188 }
 189
 190 ######################################################
 191 # check a unix socket
 192 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 193 ######################################################
 194 ctdb_check_unix_socket() {
 195     socket_path="$1"
 196     [ -z "$socket_path" ] && return
 197
 198     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 199         echo "ERROR: $service_name socket $socket_path not found"
 200         return 1
 201     fi
 202 }
 203
 204 ######################################################
 205 # check a command returns zero status
 206 # usage: ctdb_check_command SERVICE_NAME <command>
 207 ######################################################
 208 ctdb_check_command() {
 209   service_name="$1"
 210   wait_cmd="$2"
 211   [ -z "$wait_cmd" ] && return;
 212   $wait_cmd > /dev/null 2>&1 || {
 213       echo "ERROR: $service_name - $wait_cmd returned error"
 214       exit 1
 215   }
 216 }
 217
 218 ################################################
 219 # kill off any TCP connections with the given IP
 220 ################################################
 221 kill_tcp_connections() {
 222     _IP="$1"
 223     _failed=0
 224
 225     _killcount=0
 226     connfile="$CTDB_BASE/state/connections.$_IP"
 227     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 228     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 229
 230     while read dest src; do
 231         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 232         srcport=`echo $src | sed -e "s/^.*://"`
 233         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 234         destport=`echo $dest | sed -e "s/^.*://"`
 235         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 236         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 237         case $destport in
 238           # we only do one-way killtcp for CIFS
 239           139|445) : ;;
 240           # for all others we do 2-way
 241           *)
 242                 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
 243                 ;;
 244         esac
 245         _killcount=`expr $_killcount + 1`
 246      done < $connfile
 247     /bin/rm -f $connfile
 248
 249     [ $_failed = 0 ] || {
 250         echo "Failed to send killtcp control"
 251         return;
 252     }
 253     [ $_killcount -gt 0 ] || {
 254         return;
 255     }
 256     _count=0
 257     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 258         sleep 1
 259         _count=`expr $_count + 1`
 260         [ $_count -gt 3 ] && {
 261             echo "Timed out killing tcp connections for IP $_IP"
 262             return;
 263         }
 264     done
 265     echo "killed $_killcount TCP connections to released IP $_IP"
 266 }
 267
 268 ##################################################################
 269 # kill off the local end for any TCP connections with the given IP
 270 ##################################################################
 271 kill_tcp_connections_local_only() {
 272     _IP="$1"
 273     _failed=0
 274
 275     _killcount=0
 276     connfile="$CTDB_BASE/state/connections.$_IP"
 277     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 278     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 279
 280     while read dest src; do
 281         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 282         srcport=`echo $src | sed -e "s/^.*://"`
 283         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 284         destport=`echo $dest | sed -e "s/^.*://"`
 285         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 286         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 287         _killcount=`expr $_killcount + 1`
 288      done < $connfile
 289     /bin/rm -f $connfile
 290
 291     [ $_failed = 0 ] || {
 292         echo "Failed to send killtcp control"
 293         return;
 294     }
 295     [ $_killcount -gt 0 ] || {
 296         return;
 297     }
 298     _count=0
 299     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 300         sleep 1
 301         _count=`expr $_count + 1`
 302         [ $_count -gt 3 ] && {
 303             echo "Timed out killing tcp connections for IP $_IP"
 304             return;
 305         }
 306     done
 307     echo "killed $_killcount TCP connections to released IP $_IP"
 308 }
 309
 310 ########################################################
 311 # start/stop the nfs service on different platforms
 312 ########################################################
 313 startstop_nfs() {
 314         PLATFORM="unknown"
 315         [ -x /etc/init.d/nfsserver ] && {
 316                 PLATFORM="sles"
 317         }
 318         [ -x /etc/init.d/nfslock ] && {
 319                 PLATFORM="rhel"
 320         }
 321
 322         case $PLATFORM in
 323         sles)
 324                 case $1 in
 325                 start)
 326                         service nfsserver start
 327                         ;;
 328                 stop)
 329                         service nfsserver stop > /dev/null 2>&1
 330                         ;;
 331                 esac
 332                 ;;
 333         rhel)
 334                 case $1 in
 335                 start)
 336                         service nfslock start
 337                         service nfs start
 338                         ;;
 339                 stop)
 340                         service nfs stop > /dev/null 2>&1
 341                         service nfslock stop > /dev/null 2>&1
 342                         ;;
 343                 esac
 344                 ;;
 345         *)
 346                 echo "Unknown platform. NFS is not supported with ctdb"
 347                 exit 1
 348                 ;;
 349         esac
 350 }
 351
 352 ########################################################
 353 # start/stop the nfs lockmanager service on different platforms
 354 ########################################################
 355 startstop_nfslock() {
 356         PLATFORM="unknown"
 357         [ -x /etc/init.d/nfsserver ] && {
 358                 PLATFORM="sles"
 359         }
 360         [ -x /etc/init.d/nfslock ] && {
 361                 PLATFORM="rhel"
 362         }
 363
 364         case $PLATFORM in
 365         sles)
 366                 # for sles there is no service for lockmanager
 367                 # so we instead just shutdown/restart nfs
 368                 case $1 in
 369                 start)
 370                         service nfsserver start
 371                         ;;
 372                 stop)
 373                         service nfsserver stop > /dev/null 2>&1
 374                         ;;
 375                 esac
 376                 ;;
 377         rhel)
 378                 case $1 in
 379                 start)
 380                         service nfslock start
 381                         ;;
 382                 stop)
 383                         service nfslock stop > /dev/null 2>&1
 384                         ;;
 385                 esac
 386                 ;;
 387         *)
 388                 echo "Unknown platform. NFS locking is not supported with ctdb"
 389                 exit 1
 390                 ;;
 391         esac
 392 }
 393
 394 ########################################################
 395 # remove an ip address from an interface
 396 ########################################################
 397 remove_ip() {
 398         # the ip tool will delete all secondary IPs if this is the primary.
 399         # To work around this _very_ annoying behaviour we have to keep a
 400         # record of the secondaries and re-add them afterwards. yuck
 401         secondaries=""
 402         if ip addr list dev $2 primary | grep -q "inet $1 " ; then
 403             secondaries=`ip addr list dev $2 secondary | grep " inet " | awk '{print $2}'`
 404         fi
 405         ip addr del $1 dev $2 >/dev/null 2>/dev/null || failed=1
 406         [ -z "$secondaries" ] || {
 407             for i in $secondaries; do
 408                 if ip addr list dev $2 | grep -q "inet $i" ; then
 409                     echo "kept secondary $i on dev $2"
 410                 else
 411                     echo "re-adding secondary address $i to dev $2"
 412                     ip addr add $i dev $2 || failed=1
 413                 fi
 414             done
 415         }
 416 }
 417
 418 ########################################################
 419 # some simple logic for counting events - per eventscript
 420 # usage: ctdb_counter_init
 421 #        ctdb_counter_incr
 422 #        ctdb_check_counter_limit <limit>
 423 # ctdb_check_counter_limit succeeds when count >= <limit>
 424 ########################################################
 425 _ctdb_counter_common () {
 426     _counter_file="$ctdb_fail_dir/$service_name"
 427     mkdir -p "${_counter_file%/*}" # dirname
 428 }
 429 ctdb_counter_init () {
 430     _ctdb_counter_common
 431
 432     >"$_counter_file"
 433 }
 434 ctdb_counter_incr () {
 435     _ctdb_counter_common
 436
 437     # unary counting!
 438     echo -n 1 >> "$_counter_file"
 439 }
 440 ctdb_check_counter_limit () {
 441     _ctdb_counter_common
 442
 443     _limit="${1:-${service_fail_limit}}"
 444     _quiet="$2"
 445
 446     # unary counting!
 447     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 448     if [ $_size -ge $_limit ] ; then
 449         echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
 450         exit 1
 451     elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
 452         echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
 453     fi
 454 }
 455 ########################################################
 456
 457 ctdb_spool_dir="/var/spool/ctdb"
 458 ctdb_status_dir="$ctdb_spool_dir/status"
 459 ctdb_fail_dir="$ctdb_spool_dir/failcount"
 460 ctdb_active_dir="$ctdb_spool_dir/active"
 461
 462 log_status_cat ()
 463 {
 464     echo "node is \"$1\", problem with \"${script_name}\": $(cat $2)"
 465 }
 466
 467 ctdb_checkstatus ()
 468 {
 469     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
 470         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
 471         return 1
 472     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
 473         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
 474         return 2
 475     else
 476         return 0
 477     fi
 478 }
 479
 480 ctdb_setstatus ()
 481 {
 482     d="$ctdb_status_dir/$script_name"
 483     case "$1" in
 484         unhealthy|banned)
 485             mkdir -p "$d"
 486             cat "$2" >"$d/$1"
 487             ;;
 488         *)
 489             for i in "banned" "unhealthy" ; do
 490                 rm -f "$d/$i"
 491             done
 492             ;;
 493     esac
 494 }
 495
 496 ctdb_service_needs_reconfigure ()
 497 {
 498     [ -e "$ctdb_status_dir/$service_name/reconfigure" ]
 499 }
 500
 501 ctdb_service_set_reconfigure ()
 502 {
 503     d="$ctdb_status_dir/$service_name"
 504     mkdir -p "$d"
 505     >"$d/reconfigure"
 506 }
 507
 508 ctdb_service_unset_reconfigure ()
 509 {
 510     rm -f "$ctdb_status_dir/$service_name/reconfigure"
 511 }
 512
 513 ctdb_service_reconfigure ()
 514 {
 515     if [ -n "$service_reconfigure" ] ; then
 516         eval $service_reconfigure
 517     else
 518         service "$service_name" restart
 519     fi
 520     ctdb_service_unset_reconfigure
 521     ctdb_counter_init
 522 }
 523
 524 ctdb_compat_managed_service ()
 525 {
 526     if [ "$1" = "yes" ] ; then
 527         t="$t $2 "
 528     fi
 529 }
 530
 531 is_ctdb_managed_service ()
 532 {
 533     t=" $CTDB_MANAGED_SERVICES "
 534
 535     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
 536     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
 537     ctdb_compat_managed_service "$CTDB_MANAGES_SCP"      "scp"
 538     ctdb_compat_managed_service "$CTDB_MANAGES_WINDBIND" "windbind"
 539     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
 540     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
 541     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
 542     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
 543
 544     # Returns 0 if "<space>$service_name<space>" appears in $t
 545     [ "${t#* ${service_name} }" != "${t}" ]
 546 }
 547
 548 ctdb_start_stop_service ()
 549 {
 550     _active="$ctdb_active_dir/$service_name"
 551
 552     if is_ctdb_managed_service ; then
 553         if ! [ -e "$_active" ] ; then
 554             echo "Starting service $service_name"
 555             ctdb_service_start || exit $?
 556             mkdir -p "$ctdb_active_dir"
 557             touch "$_active"
 558             exit 0
 559         fi
 560     elif ! is_ctdb_managed_service ; then
 561         if [ -e "$_active" ] ; then
 562             echo "Stopping service $service_name"
 563             ctdb_service_stop || exit $?
 564             rm -f "$_active"
 565         fi
 566         exit 0
 567     fi
 568 }
 569
 570 ctdb_service_start ()
 571 {
 572     if [ -n "$service_start" ] ; then
 573         eval $service_start
 574     else
 575         service "$service_name" start
 576     fi
 577     ctdb_counter_init
 578 }
 579
 580 ctdb_service_stop ()
 581 {
 582     if [ -n "$service_stop" ] ; then
 583         eval $service_stop
 584     else
 585         service "$service_name" stop
 586     fi
 587 }
 588
 589 ########################################################
 590 # load a site local config file
 591 ########################################################
 592
 593 [ -x $CTDB_BASE/rc.local ] && {
 594         . $CTDB_BASE/rc.local
 595 }
 596
 597 [ -d $CTDB_BASE/rc.local.d ] && {
 598         for i in $CTDB_BASE/rc.local.d/* ; do
 599                 [ -x "$i" ] && . "$i"
 600         done
 601 }
 602
 603 # A reasonable default is the basename of the eventscript.
 604 script_name="${0##*/}" # basename
 605 service_name="$script_name"
 606 service_fail_limit=1
 607
 608 ctdb_event="$1" ; shift
 609 cmd="$ctdb_event"
 610
 611 case "$ctdb_event" in
 612     status)
 613         ctdb_checkstatus
 614         exit
 615         ;;
 616     setstatus)
 617         ctdb_setstatus "$@"
 618         exit
 619         ;;
 620 esac