+++ /dev/null
-#!/usr/bin/env python
-
-# ctdb ip takeover code
-
-# Copyright (C) Martin Schwenke, Ronnie Sahlberg 2010, 2011
-
-# Based on original CTDB C code:
-#
-# Copyright (C) Ronnie Sahlberg 2007
-# Copyright (C) Andrew Tridgell 2007
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-
-import os
-import sys
-# Use optparse since newer argparse not available in RHEL5/EPEL.
-from optparse import OptionParser
-import copy
-import random
-import itertools
-
-# For parsing IP addresses
-import socket
-import struct
-
-# For external algorithm
-import subprocess
-import re
-
-options = None
-
-def process_args(extra_options=[]):
- global options
-
- parser = OptionParser(option_list=extra_options)
-
- parser.add_option("--nd",
- action="store_false", dest="deterministic_public_ips",
- default=True,
- help="turn off deterministic_public_ips")
- parser.add_option("--ni",
- action="store_true", dest="no_ip_failback", default=False,
- help="turn on no_ip_failback")
- parser.add_option("-L", "--lcp2",
- action="store_true", dest="lcp2", default=False,
- help="use LCP2 IP rebalancing algorithm [default: %default]")
- parser.add_option("-e", "--external",
- action="store_true", dest="external", default=False,
- help="use external test program to implement IP allocation algorithm [default: %default]")
- parser.add_option("-b", "--balance",
- action="store_true", dest="balance", default=False,
- help="show (im)balance information after each event")
- parser.add_option("-d", "--diff",
- action="store_true", dest="diff", default=False,
- help="show IP address movements for each event")
- parser.add_option("-n", "--no-print",
- action="store_false", dest="show", default=True,
- help="don't show IP address layout after each event")
- parser.add_option("-v", "--verbose",
- action="count", dest="verbose", default=0,
- help="print information and actions taken to stdout")
- parser.add_option("-r", "--retries",
- action="store", type="int", dest="retries", default=5,
- help="number of retry loops for rebalancing non-deterministic failback [default: %default]")
- parser.add_option("-i", "--iterations",
- action="store", type="int", dest="iterations",
- default=1000,
- help="number of iterations to run in test [default: %default]")
- parser.add_option("-o", "--odds",
- action="store", type="int", dest="odds", default=4,
- help="make the chances of a failover 1 in ODDS [default: %default]")
- parser.add_option("-A", "--aggressive",
- action="store_true", dest="aggressive", default=False,
- help="apply ODDS to try to flip each node [default: %default]")
-
- def seed_callback(option, opt, value, parser):
- random.seed(value)
- parser.add_option("-s", "--seed",
- action="callback", type="int", callback=seed_callback,
- help="initial random number seed for random events")
-
- parser.add_option("-x", "--exit",
- action="store_true", dest="exit", default=False,
- help="exit on the 1st gratuitous IP move or IP imbalance")
- parser.add_option("-H", "--hard-imbalance-limit",
- action="store", type="int", dest="hard_limit", default=1,
- help="exceeding this limit causes termination [default: %default]")
- parser.add_option("-S", "--soft-imbalance-limit",
- action="store", type="int", dest="soft_limit", default=1,
- help="exceeding this limit increments a counter [default: %default]")
-
- (options, args) = parser.parse_args()
-
- if len(args) != 0:
- parser.error("too many arguments")
-
- # Could use a callback for this or change the default, but
- # laziness is sometimes a virtue. ;-)
- if options.lcp2:
- options.deterministic_public_ips = False
-
-def print_begin(t, delim='='):
- print delim * 40
- print "%s:" % (t)
-
-def print_end():
- print "-" * 40
-
-def verbose_begin(t):
- if options.verbose > 0:
- print_begin(t)
-
-def verbose_end():
- if options.verbose > 0:
- print_end()
-
-def verbose_print(t):
- if options.verbose > 0:
- if not type(t) == list:
- t = [t]
- if t != []:
- print "\n".join([str(i) for i in t])
-
-# more than this and we switch to the logging module... :-)
-def debug_begin(t):
- if options.verbose > 1:
- print_begin(t, '-')
-
-def debug_end():
- if options.verbose > 1:
- print_end()
-
-def debug_print(t):
- if options.verbose > 1:
- if not type(t) == list:
- t = [t]
- if t != []:
- print "\n".join([str(i) for i in t])
-
-def ip_to_list_of_ints(ip):
- # Be lazy... but only expose errors in IPv4 addresses, since
- # they'll be more commonly used. :-)
- try:
- l = socket.inet_pton(socket.AF_INET6, ip)
- except:
- # Pad with leading 0s. This makes IPv4 addresses comparable
- # with IPv6 but reduces the overall effectiveness of the
- # algorithm. The alternative would be to treat these
- # addresses separately while trying to keep all the IPs in
- # overall balance.
- l = "".join(itertools.repeat("\0", 12)) + \
- socket.inet_pton(socket.AF_INET, ip)
-
- return map(lambda x: struct.unpack('B', x)[0], l)
-
-def ip_distance(ip1, ip2):
- """Calculate the distance between 2 IPs.
-
- This is the length of the longtest common prefix between the IPs.
- It is calculated by XOR-ing the 2 IPs together and counting the
- number of leading zeroes."""
-
- distance = 0
- for (o1, o2) in zip(ip_to_list_of_ints(ip1), ip_to_list_of_ints(ip2)):
- # XOR this pair of octets
- x = o1 ^ o2
- # count number leading zeroes
- if x == 0:
- distance += 8
- else:
- # bin() gives minimal length '0bNNN' string
- distance += (8 - (len(bin(x)) - 2))
- break
-
- return distance
-
-def ip_distance_2_sum(ip, ips):
- """Calculate the IP distance for the given IP relative to IPs.
-
- This could be made more efficient by insering ip_distance_2 into
- the loop in this function. However, that would result in some
- loss of clarity and also will not be necessary in a C
- implemntation."""
-
- sum = 0
- for i in ips:
- sum += ip_distance(ip, i) ** 2
-
- return sum
-
-def imbalance_metric(ips):
- """Return the imbalance metric for a group of IPs.
-
- This is the sum of squares of the IP distances between each pair of IPs."""
- if len(ips) > 1:
- (h, t) = (ips[0], ips[1:])
- return ip_distance_2_sum(h, t) + imbalance_metric(t)
- else:
- return 0
-
-def mean(l):
- return float(sum(l))/len(l)
-
-class Node(object):
- def __init__(self, public_addresses):
- # List of list allows groups of IPs to be passed in. They're
- # not actually used in the algorithm but are just used by
- # calculate_imbalance() for checking the simulation. Note
- # that people can pass in garbage and make this code
- # fail... but we're all friends here in simulation world...
- # :-)
- if type(public_addresses[0]) is str:
- self.public_addresses = set(public_addresses)
- self.ip_groups = []
- else:
- # flatten
- self.public_addresses = set([i for s in public_addresses for i in s])
- self.ip_groups = public_addresses
-
- self.current_addresses = set()
- self.healthy = True
- self.imbalance = -1
-
- def __str__(self):
- return "%s %s%s" % \
- ("*" if len(self.public_addresses) == 0 else \
- (" " if self.healthy else "#"),
- sorted(list(self.current_addresses)),
- " %d" % self.imbalance if options.lcp2 else "")
-
- def can_node_serve_ip(self, ip):
- return ip in self.public_addresses
-
- def node_ip_coverage(self, ips=None):
- return len([a for a in self.current_addresses if ips == None or a in ips])
-
- def set_imbalance(self, imbalance=-1):
- """Set the imbalance metric to the given value. If none given
- then calculate it."""
-
- if imbalance != -1:
- self.imbalance = imbalance
- else:
- self.imbalance = imbalance_metric(list(self.current_addresses))
-
- def get_imbalance(self):
- return self.imbalance
-
-class Cluster(object):
- def __init__(self):
- self.nodes = []
- self.deterministic_public_ips = options.deterministic_public_ips
- self.no_ip_failback = options.no_ip_failback
- self.all_public_ips = set()
-
- # Statistics
- self.ip_moves = []
- self.grat_ip_moves = []
- self.imbalance = []
- self.imbalance_groups = []
- self.imbalance_count = 0
- self.imbalance_groups_count = itertools.repeat(0)
- self.imbalance_metric = []
- self.events = -1
- self.num_unhealthy = []
-
- self.prev = None
-
- def __str__(self):
- return "\n".join(["%2d %s" % (i, n) \
- for (i, n) in enumerate(self.nodes)])
-
- # This is naive. It assumes that IP groups are indicated by the
- # 1st node having IP groups.
- def have_ip_groups(self):
- return (len(self.nodes[0].ip_groups) > 0)
-
- def print_statistics(self):
- print_begin("STATISTICS")
- print "Events: %6d" % self.events
- print "Total IP moves: %6d" % sum(self.ip_moves)
- print "Gratuitous IP moves: %6d" % sum(self.grat_ip_moves)
- print "Max imbalance: %6d" % max(self.imbalance)
- if self.have_ip_groups():
- print "Max group imbalance counts: ", map(max, zip(*self.imbalance_groups))
- print "Mean imbalance: %f" % mean(self.imbalance)
- if self.have_ip_groups():
- print "Mean group imbalances counts: ", map(mean, zip(*self.imbalance_groups))
- print "Final imbalance: %6d" % self.imbalance[-1]
- if self.have_ip_groups():
- print "Final group imbalances: ", self.imbalance_groups[-1]
- if options.lcp2:
- print "Max LCP2 imbalance : %6d" % max(self.imbalance_metric)
- print "Soft imbalance count: %6d" % self.imbalance_count
- if self.have_ip_groups():
- print "Soft imbalance group counts: ", self.imbalance_groups_count
- if options.lcp2:
- print "Final LCP2 imbalance : %6d" % self.imbalance_metric[-1]
- print "Maximum unhealthy: %6d" % max(self.num_unhealthy)
- print_end()
-
- def find_pnn_with_ip(self, ip):
- for (i, n) in enumerate(self.nodes):
- if ip in n.current_addresses:
- return i
- return -1
-
- def quietly_remove_ip(self, ip):
- # Remove address from old node.
- old = self.find_pnn_with_ip(ip)
- if old != -1:
- self.nodes[old].current_addresses.remove(ip)
-
- def add_node(self, node):
- self.nodes.append(node)
- self.all_public_ips |= node.public_addresses
-
- def healthy(self, *pnns):
- verbose_begin("HEALTHY")
-
- for pnn in pnns:
- self.nodes[pnn].healthy = True
- verbose_print(pnn)
-
- verbose_end()
-
- def unhealthy(self, *pnns):
-
- verbose_begin("UNHEALTHY")
-
- for pnn in pnns:
- self.nodes[pnn].healthy = False
- verbose_print(pnn)
-
- verbose_end()
-
- def do_something_random(self):
-
- """Make random node(s) healthy or unhealthy.
-
- If options.aggressive is False then: If all nodes are healthy
- or unhealthy, then invert one of them; otherwise, there's a 1
- in options.odds chance of making another node unhealthy.
-
- If options.aggressive is True then: For each node there is a 1
- in options.odds chance of flipping the state of that node
- between healthy and unhealthy."""
-
- if not options.aggressive:
- num_nodes = len(self.nodes)
- healthy_pnns = [i for (i,n) in enumerate(self.nodes) if n.healthy]
- num_healthy = len(healthy_pnns)
-
- if num_nodes == num_healthy:
- self.unhealthy(random.randint(0, num_nodes-1))
- elif num_healthy == 0:
- self.healthy(random.randint(0, num_nodes-1))
- elif random.randint(1, options.odds) == 1:
- self.unhealthy(random.choice(healthy_pnns))
- else:
- all_pnns = range(num_nodes)
- unhealthy_pnns = sorted(list(set(all_pnns) - set(healthy_pnns)))
- self.healthy(random.choice(unhealthy_pnns))
- else:
- # We need to make at least one change or we retry...x
- changed = False
- while not changed:
- for (pnn, n) in enumerate(self.nodes):
- if random.randint(1, options.odds) == 1:
- changed = True
- if n.healthy:
- self.unhealthy(pnn)
- else:
- self.healthy(pnn)
-
- def random_iterations(self):
- i = 1
- while i <= options.iterations:
- verbose_begin("EVENT %d" % i)
- verbose_end()
- self.do_something_random()
- if self.recover() and options.exit:
- break
- i += 1
-
- self.print_statistics()
-
- def imbalance_for_ips(self, ips):
-
- imbalance = 0
-
- maxnode = -1
- minnode = -1
-
- for ip in ips:
- for (i, n) in enumerate(self.nodes):
-
- if not n.healthy or not n.can_node_serve_ip(ip):
- continue
-
- num = n.node_ip_coverage(ips)
-
- if maxnode == -1 or num > maxnum:
- maxnode = i
- maxnum = num
-
- if minnode == -1 or num < minnum:
- minnode = i
- minnum = num
-
- if maxnode == -1 or minnode == -1:
- continue
-
- i = maxnum - minnum
- #if i < 2:
- # i = 0
- imbalance = max([imbalance, i])
-
- return imbalance
-
-
- def calculate_imbalance(self):
-
- # First, do all the assigned IPs.
- assigned = sorted([ip
- for n in self.nodes
- for ip in n.current_addresses])
-
- i = self.imbalance_for_ips(assigned)
-
- ig = []
- # FIXME? If dealing with IP groups, assume the nodes are all
- # the same.
- for ips in self.nodes[0].ip_groups:
- gi = self.imbalance_for_ips(ips)
- ig.append(gi)
-
- return (i, ig)
-
-
- def diff(self):
- """Calculate differences in IP assignments between self and prev.
-
- Gratuitous IP moves (from a healthy node to a healthy node)
- are prefixed by !!."""
-
- ip_moves = 0
- grat_ip_moves = 0
- details = []
-
- for (new, n) in enumerate(self.nodes):
- for ip in n.current_addresses:
- old = self.prev.find_pnn_with_ip(ip)
- if old != new:
- ip_moves += 1
- if old != -1 and \
- self.prev.nodes[new].healthy and \
- self.nodes[new].healthy and \
- self.nodes[old].healthy and \
- self.prev.nodes[old].healthy:
- prefix = "!!"
- grat_ip_moves += 1
- else:
- prefix = " "
- details.append("%s %s: %d -> %d" %
- (prefix, ip, old, new))
-
- return (ip_moves, grat_ip_moves, details)
-
- def find_takeover_node(self, ip):
-
- pnn = -1
- min = 0
- for (i, n) in enumerate(self.nodes):
- if not n.healthy:
- continue
-
- if not n.can_node_serve_ip(ip):
- continue
-
- num = n.node_ip_coverage()
-
- if (pnn == -1):
- pnn = i
- min = num
- else:
- if num < min:
- pnn = i
- min = num
-
- if pnn == -1:
- verbose_print("Could not find node to take over public address %s" % ip)
- return False
-
- self.nodes[pnn].current_addresses.add(ip)
-
- verbose_print("%s -> %d" % (ip, pnn))
- return True
-
- def basic_allocate_unassigned(self):
-
- assigned = set([ip for n in self.nodes for ip in n.current_addresses])
- unassigned = sorted(list(self.all_public_ips - assigned))
-
- for ip in unassigned:
- self.find_takeover_node(ip)
-
- def basic_failback(self, retries_l):
-
- assigned = sorted([ip
- for n in self.nodes
- for ip in n.current_addresses])
- for ip in assigned:
-
- maxnode = -1
- minnode = -1
- for (i, n) in enumerate(self.nodes):
- if not n.healthy:
- continue
-
- if not n.can_node_serve_ip(ip):
- continue
-
- num = n.node_ip_coverage()
-
- if maxnode == -1:
- maxnode = i
- maxnum = num
- else:
- if num > maxnum:
- maxnode = i
- maxnum = num
- if minnode == -1:
- minnode = i
- minnum = num
- else:
- if num < minnum:
- minnode = i
- minnum = num
-
- if maxnode == -1:
- print "Could not find maxnode. May not be able to serve ip", ip
- continue
-
- #if self.deterministic_public_ips:
- # continue
-
- if maxnum > minnum + 1 and retries_l[0] < options.retries:
- # Remove the 1st ip from maxnode
- t = sorted(list(self.nodes[maxnode].current_addresses))
- realloc = t[0]
- verbose_print("%s <- %d" % (realloc, maxnode))
- self.nodes[maxnode].current_addresses.remove(realloc)
- # Redo the outer loop.
- retries_l[0] += 1
- return True
-
- return False
-
-
- def lcp2_allocate_unassigned(self):
-
- # Assign as many unassigned addresses as possible. Keep
- # selecting the optimal assignment until we don't manage to
- # assign anything.
- assigned = set([ip for n in self.nodes for ip in n.current_addresses])
- unassigned = sorted(list(self.all_public_ips - assigned))
-
- should_loop = True
- while len(unassigned) > 0 and should_loop:
- should_loop = False
-
- debug_begin(" CONSIDERING MOVES (UNASSIGNED)")
-
- minnode = -1
- mindsum = 0
- minip = None
-
- for ip in unassigned:
- for dstnode in range(len(self.nodes)):
- if self.nodes[dstnode].can_node_serve_ip(ip) and \
- self.nodes[dstnode].healthy:
- dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses)
- dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum
- debug_print(" %s -> %d [+%d]" % \
- (ip,
- dstnode,
- dstimbl - self.nodes[dstnode].get_imbalance()))
-
- if (minnode == -1) or (dstdsum < mindsum):
- minnode = dstnode
- minimbl = dstimbl
- mindsum = dstdsum
- minip = ip
- should_loop = True
- debug_end()
-
- if minnode != -1:
- self.nodes[minnode].current_addresses.add(minip)
- self.nodes[minnode].set_imbalance(self.nodes[minnode].get_imbalance() + mindsum)
- verbose_print("%s -> %d [+%d]" % (minip, minnode, mindsum))
- unassigned.remove(minip)
-
- for ip in unassigned:
- verbose_print("Could not find node to take over public address %s" % ip)
-
- def lcp2_failback(self, targets):
-
- # Get the node with the highest imbalance metric.
- srcnode = -1
- maximbl = 0
- for (pnn, n) in enumerate(self.nodes):
- b = n.get_imbalance()
- if (srcnode == -1) or (b > maximbl):
- srcnode = pnn
- maximbl = b
-
- # This means that all nodes had 0 or 1 addresses, so can't
- # be imbalanced.
- if maximbl == 0:
- return False
-
- # We'll need this a few times...
- ips = self.nodes[srcnode].current_addresses
-
- # Find an IP and destination node that best reduces imbalance.
- optimum = None
- debug_begin(" CONSIDERING MOVES FROM %d [%d]" % (srcnode, maximbl))
- for ip in ips:
- # What is this IP address costing the source node?
- srcdsum = ip_distance_2_sum(ip, ips - set([ip]))
- srcimbl = maximbl - srcdsum
-
- # Consider this IP address would cost each potential
- # destination node. Destination nodes are limited to
- # those that are newly healthy, since we don't want to
- # do gratuitous failover of IPs just to make minor
- # balance improvements.
- for dstnode in targets:
- if self.nodes[dstnode].can_node_serve_ip(ip) and \
- self.nodes[dstnode].healthy:
- dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses)
- dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum
- debug_print(" %d [%d] -> %s -> %d [+%d]" % \
- (srcnode,
- srcimbl - self.nodes[srcnode].get_imbalance(),
- ip,
- dstnode,
- dstimbl - self.nodes[dstnode].get_imbalance()))
-
- if (dstimbl < maximbl) and (dstdsum < srcdsum):
- if optimum is None:
- optimum = (ip, srcnode, srcimbl, dstnode, dstimbl)
- else:
- (x, sn, si, dn, di) = optimum
- if (srcimbl + dstimbl) < (si + di):
- optimum = (ip, srcnode, srcimbl, dstnode, dstimbl)
- debug_end()
-
- if optimum is not None:
- # We found a move that makes things better...
- (ip, srcnode, srcimbl, dstnode, dstimbl) = optimum
- ini_srcimbl = self.nodes[srcnode].get_imbalance()
- ini_dstimbl = self.nodes[dstnode].get_imbalance()
-
- self.nodes[srcnode].current_addresses.remove(ip)
- self.nodes[srcnode].set_imbalance(srcimbl)
-
- self.nodes[dstnode].current_addresses.add(ip)
- self.nodes[dstnode].set_imbalance(dstimbl)
-
- verbose_print("%d [%d] -> %s -> %d [+%d]" % \
- (srcnode,
- srcimbl - ini_srcimbl,
- ip,
- dstnode,
- dstimbl - ini_dstimbl))
-
- return True
-
- return False
-
- def ctdb_takeover_run_python(self):
-
- # Don't bother with the num_healthy stuff. It is an
- # irrelevant detail.
-
- # We just keep the allocate IPs in the current_addresses field
- # of the node. This needs to readable, not efficient!
-
- if self.deterministic_public_ips:
- # Remap everything.
- addr_list = sorted(list(self.all_public_ips))
- for (i, ip) in enumerate(addr_list):
- self.quietly_remove_ip(ip)
- # Add addresses to new node.
- pnn = i % len(self.nodes)
- self.nodes[pnn].current_addresses.add(ip)
- verbose_print("%s -> %d" % (ip, pnn))
-
- # Remove public addresses from unhealthy nodes.
- for (pnn, n) in enumerate(self.nodes):
- if not n.healthy:
- verbose_print(["%s <- %d" % (ip, pnn)
- for ip in n.current_addresses])
- n.current_addresses = set()
-
- # If a node can't serve an assigned address then remove it.
- for n in self.nodes:
- verbose_print(["%s <- %d" % (ip, pnn)
- for ip in n.current_addresses - n.public_addresses])
- n.current_addresses &= n.public_addresses
-
- if options.lcp2:
- newly_healthy = [pnn for (pnn, n) in enumerate(self.nodes)
- if len(n.current_addresses) == 0 and n.healthy]
- for n in self.nodes:
- n.set_imbalance()
-
- # We'll only retry the balancing act up to options.retries
- # times (for the basic non-deterministic algorithm). This
- # nonsense gives us a reference on the retries count in
- # Python. It will be easier in C. :-)
- # For LCP2 we reassignas many IPs from heavily "loaded" nodes
- # to nodes that are newly healthy, looping until we fail to
- # reassign an IP.
- retries_l = [0]
- should_loop = True
- while should_loop:
- should_loop = False
-
- if options.lcp2:
- self.lcp2_allocate_unassigned()
- else:
- self.basic_allocate_unassigned()
-
- if self.no_ip_failback or self.deterministic_public_ips:
- break
-
- if options.lcp2:
- if len(newly_healthy) == 0:
- break
- should_loop = self.lcp2_failback(newly_healthy)
- else:
- should_loop = self.basic_failback(retries_l)
-
- def ctdb_takeover_run_external(self):
-
- # Written while asleep...
-
- # Convert the cluster state to something that be fed to
- # ctdb_takeover_tests ipalloc ...
-
- in_lines = []
- for ip in sorted(list(self.all_public_ips)):
- allowed = []
- assigned = -1
- for (i, n) in enumerate(self.nodes):
- if n.can_node_serve_ip(ip):
- allowed.append("%s" % i)
- if ip in n.current_addresses:
- assigned = i
- line = "%s\t%d\t%s" % (ip, assigned, ",".join(allowed))
- in_lines.append(line)
-
- nodestates = ",".join(["0" if n.healthy else "1" for n in self.nodes])
-
- if options.lcp2:
- os.environ["CTDB_LCP2"] = "yes"
- if options.verbose > 1:
- os.environ["CTDB_TEST_LOGLEVEL"] = "4"
- elif options.verbose == 1:
- os.environ["CTDB_TEST_LOGLEVEL"] = "3"
- else:
- os.environ["CTDB_TEST_LOGLEVEL"] = "0"
-
- p = subprocess.Popen("../../bin/ctdb_takeover_tests ipalloc %s 2>&1" % nodestates,
- shell=True,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
- p.stdin.write("\n".join(in_lines))
- p.stdin.close()
-
- # Flush all of the assigned IPs.
- for n in self.nodes:
- n.current_addresses = set()
-
- # Uses the results to populate the current_addresses for each
- # node.
- for line in p.stdout.read().split("\n"):
- # Some lines are debug, some are the final IP
- # configuration. Let's use a gross hack that assumes any
- # line with 2 words is IP configuration. That will do for
- # now.
- words = re.split("\s+", line)
- if len(words) == 2:
- # Add the IP as current for the specified node.
- self.nodes[int(words[1])].current_addresses.add(words[0])
- else:
- # First 3 words are log date/time, remove them...
- print " ".join(words[3:])
-
- # Now fake up the LCP calculations.
- for n in self.nodes:
- n.set_imbalance()
-
- def ctdb_takeover_run(self):
-
- self.events += 1
-
- if options.external:
- return self.ctdb_takeover_run_external()
- else:
- return self.ctdb_takeover_run_python()
-
- def recover(self):
- verbose_begin("TAKEOVER")
-
- self.ctdb_takeover_run()
-
- verbose_end()
-
- grat_ip_moves = 0
-
- if self.prev is not None:
- (ip_moves, grat_ip_moves, details) = self.diff()
- self.ip_moves.append(ip_moves)
- self.grat_ip_moves.append(grat_ip_moves)
-
- if options.diff:
- print_begin("DIFF")
- print "\n".join(details)
- print_end()
-
- (imbalance, imbalance_groups) = self.calculate_imbalance()
- self.imbalance.append(imbalance)
- self.imbalance_groups.append(imbalance_groups)
-
- if imbalance > options.soft_limit:
- self.imbalance_count += 1
-
- # There must be a cleaner way...
- t = []
- for (c, i) in zip(self.imbalance_groups_count, imbalance_groups):
- if i > options.soft_limit:
- t.append(c + i)
- else:
- t.append(c)
- self.imbalance_groups_count = t
-
- imbalance_metric = max([n.get_imbalance() for n in self.nodes])
- self.imbalance_metric.append(imbalance_metric)
- if options.balance:
- print_begin("IMBALANCE")
- print "ALL IPS:", imbalance
- if self.have_ip_groups():
- print "IP GROUPS:", imbalance_groups
- if options.lcp2:
- print "LCP2 IMBALANCE:", imbalance_metric
- print_end()
-
- num_unhealthy = len(self.nodes) - \
- len([n for n in self.nodes if n.healthy])
- self.num_unhealthy.append(num_unhealthy)
-
- if options.show:
- print_begin("STATE")
- print self
- print_end()
-
- self.prev = None
- self.prev = copy.deepcopy(self)
-
- # True is bad!
- return (grat_ip_moves > 0) or \
- (not self.have_ip_groups() and imbalance > options.hard_limit) or \
- (self.have_ip_groups() and (max(imbalance_groups) > options.hard_limit))
git.samba.org / kamenim / samba-autobuild / .git / commitdiff