along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include "includes.h"
-#include "lib/tevent/tevent.h"
-#include "lib/tdb/include/tdb.h"
+#include "tdb.h"
#include "lib/util/dlinklist.h"
#include "system/network.h"
#include "system/filesys.h"
#define CTDB_ARP_INTERVAL 1
#define CTDB_ARP_REPEAT 3
+/* Flags used in IP allocation algorithms. */
+struct ctdb_ipflags {
+ bool noiptakeover;
+ bool noiphost;
+};
+
struct ctdb_iface {
struct ctdb_iface *prev, *next;
const char *name;
CTDB_NO_MEMORY_FATAL(ctdb, i);
i->name = talloc_strdup(i, iface);
CTDB_NO_MEMORY(ctdb, i->name);
- i->link_up = false;
+ /*
+ * If link_up defaults to true then IPs can be allocated to a
+ * node during the first recovery. However, then an interface
+ * could have its link marked down during the startup event,
+ * causing the IP to move almost immediately. If link_up
+ * defaults to false then, during normal operation, IPs added
+ * to a new interface can't be assigned until a monitor cycle
+ * has occurred and marked the new interfaces up. This makes
+ * IP allocation unpredictable. The following is a neat
+ * compromise: early in startup link_up defaults to false, so
+ * IPs can't be assigned, and after startup IPs can be
+ * assigned immediately.
+ */
+ i->link_up = (ctdb->runstate == CTDB_RUNSTATE_RUNNING);
DLIST_ADD(ctdb->ifaces, i);
return 0;
}
+static bool vnn_has_interface_with_name(struct ctdb_vnn *vnn,
+ const char *name)
+{
+ int n;
+
+ for (n = 0; vnn->ifaces[n] != NULL; n++) {
+ if (strcmp(name, vnn->ifaces[n]) == 0) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/* If any interfaces now have no possible IPs then delete them. This
+ * implementation is naive (i.e. simple) rather than clever
+ * (i.e. complex). Given that this is run on delip and that operation
+ * is rare, this doesn't need to be efficient - it needs to be
+ * foolproof. One alternative is reference counting, where the logic
+ * is distributed and can, therefore, be broken in multiple places.
+ * Another alternative is to build a red-black tree of interfaces that
+ * can have addresses (by walking ctdb->vnn and ctdb->single_ip_vnn
+ * once) and then walking ctdb->ifaces once and deleting those not in
+ * the tree. Let's go to one of those if the naive implementation
+ * causes problems... :-)
+ */
+static void ctdb_remove_orphaned_ifaces(struct ctdb_context *ctdb,
+ struct ctdb_vnn *vnn,
+ TALLOC_CTX *mem_ctx)
+{
+ struct ctdb_iface *i;
+
+ /* For each interface, check if there's an IP using it. */
+ for(i=ctdb->ifaces; i; i=i->next) {
+ struct ctdb_vnn *tv;
+ bool found;
+
+ /* Only consider interfaces named in the given VNN. */
+ if (!vnn_has_interface_with_name(vnn, i->name)) {
+ continue;
+ }
+
+ /* Is the "single IP" on this interface? */
+ if ((ctdb->single_ip_vnn != NULL) &&
+ (ctdb->single_ip_vnn->ifaces[0] != NULL) &&
+ (strcmp(i->name, ctdb->single_ip_vnn->ifaces[0]) == 0)) {
+ /* Found, next interface please... */
+ continue;
+ }
+ /* Search for a vnn with this interface. */
+ found = false;
+ for (tv=ctdb->vnn; tv; tv=tv->next) {
+ if (vnn_has_interface_with_name(tv, i->name)) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ /* None of the VNNs are using this interface. */
+ DLIST_REMOVE(ctdb->ifaces, i);
+ /* Caller will free mem_ctx when convenient. */
+ talloc_steal(mem_ctx, i);
+ }
+ }
+}
+
+
static struct ctdb_iface *ctdb_find_iface(struct ctdb_context *ctdb,
const char *iface)
{
struct ctdb_iface *i;
- /* Verify that we dont have an entry for this ip yet */
for (i=ctdb->ifaces;i;i=i->next) {
if (strcmp(i->name, iface) == 0) {
return i;
{
int i;
+ if (vnn->delete_pending) {
+ return false;
+ }
+
if (vnn->iface && vnn->iface->link_up) {
return true;
}
return;
}
+static int ctdb_takeip_destructor(struct ctdb_do_takeip_state *state)
+{
+ state->vnn->update_in_flight = false;
+ return 0;
+}
+
/*
take over an ip address
*/
int ret;
struct ctdb_do_takeip_state *state;
+ if (vnn->update_in_flight) {
+ DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u rejected "
+ "update for this IP already in flight\n",
+ ctdb_addr_to_str(&vnn->public_address),
+ vnn->public_netmask_bits));
+ return -1;
+ }
+
ret = ctdb_vnn_assign_iface(ctdb, vnn);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Takeover of IP %s/%u failed to "
state->c = talloc_steal(ctdb, c);
state->vnn = vnn;
+ vnn->update_in_flight = true;
+ talloc_set_destructor(state, ctdb_takeip_destructor);
+
DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u on interface %s\n",
ctdb_addr_to_str(&vnn->public_address),
vnn->public_netmask_bits,
state,
ctdb_do_takeip_callback,
state,
- false,
CTDB_EVENT_TAKE_IP,
"%s %s %u",
ctdb_vnn_iface_string(vnn),
return;
}
+static int ctdb_updateip_destructor(struct ctdb_do_updateip_state *state)
+{
+ state->vnn->update_in_flight = false;
+ return 0;
+}
+
/*
update (move) an ip address
*/
struct ctdb_iface *old = vnn->iface;
const char *new_name;
+ if (vnn->update_in_flight) {
+ DEBUG(DEBUG_NOTICE,("Update of IP %s/%u rejected "
+ "update for this IP already in flight\n",
+ ctdb_addr_to_str(&vnn->public_address),
+ vnn->public_netmask_bits));
+ return -1;
+ }
+
ctdb_vnn_unassign_iface(ctdb, vnn);
ret = ctdb_vnn_assign_iface(ctdb, vnn);
if (ret != 0) {
state->old = old;
state->vnn = vnn;
+ vnn->update_in_flight = true;
+ talloc_set_destructor(state, ctdb_updateip_destructor);
+
DEBUG(DEBUG_NOTICE,("Update of IP %s/%u from "
"interface %s to %s\n",
ctdb_addr_to_str(&vnn->public_address),
state,
ctdb_do_updateip_callback,
state,
- false,
CTDB_EVENT_UPDATE_IP,
"%s %s %s %u",
state->old->name,
}
if (vnn->iface) {
- if (vnn->iface->link_up) {
- /* only move when the rebalance gains something */
- if (vnn->iface->references > (best_iface->references + 1)) {
+ if (vnn->iface != best_iface) {
+ if (!vnn->iface->link_up) {
do_updateip = true;
+ } else if (vnn->iface->references > (best_iface->references + 1)) {
+ /* only move when the rebalance gains something */
+ do_updateip = true;
}
- } else if (vnn->iface != best_iface) {
- do_updateip = true;
}
}
}
}
+static void do_delete_ip(struct ctdb_context *ctdb, struct ctdb_vnn *vnn)
+{
+ TALLOC_CTX *mem_ctx = talloc_new(ctdb);
+
+ DLIST_REMOVE(ctdb->vnn, vnn);
+ ctdb_remove_orphaned_ifaces(ctdb, vnn, mem_ctx);
+ ctdb_vnn_unassign_iface(ctdb, vnn);
+ talloc_free(vnn);
+ talloc_free(mem_ctx);
+}
+
/*
called when releaseip event finishes
*/
ctdb_ban_self(ctdb);
}
+ if (ctdb->do_checkpublicip && ctdb_sys_have_ip(state->addr)) {
+ DEBUG(DEBUG_ERR, ("IP %s still hosted during release IP callback, failing\n",
+ ctdb_addr_to_str(state->addr)));
+ ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
+ talloc_free(state);
+ return;
+ }
+
/* send a message to all clients of this node telling them
that the cluster has been reconfigured and they should
release any sockets on this IP */
ctdb_vnn_unassign_iface(ctdb, state->vnn);
+ /* Process the IP if it has been marked for deletion */
+ if (state->vnn->delete_pending) {
+ do_delete_ip(ctdb, state->vnn);
+ state->vnn = NULL;
+ }
+
/* the control succeeded */
ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
talloc_free(state);
}
+static int ctdb_releaseip_destructor(struct takeover_callback_state *state)
+{
+ if (state->vnn != NULL) {
+ state->vnn->update_in_flight = false;
+ }
+ return 0;
+}
+
/*
release an ip address
*/
struct takeover_callback_state *state;
struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
struct ctdb_vnn *vnn;
+ char *iface;
/* update our vnn list */
vnn = find_public_ip_vnn(ctdb, &pip->addr);
talloc_free(vnn->takeover_ctx);
vnn->takeover_ctx = NULL;
+ /* Some ctdb tool commands (e.g. moveip, rebalanceip) send
+ * lazy multicast to drop an IP from any node that isn't the
+ * intended new node. The following causes makes ctdbd ignore
+ * a release for any address it doesn't host.
+ */
if (ctdb->do_checkpublicip) {
-
if (!ctdb_sys_have_ip(&pip->addr)) {
DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u on interface %s (ip not held)\n",
ctdb_addr_to_str(&pip->addr),
ctdb_vnn_unassign_iface(ctdb, vnn);
return 0;
}
-
+ } else {
if (vnn->iface == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " release_ip of IP %s is known to the kernel, "
- "but we have no interface assigned, has someone manually configured it? Ignore for now.\n",
- ctdb_addr_to_str(&vnn->public_address)));
+ DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u (ip not held)\n",
+ ctdb_addr_to_str(&pip->addr),
+ vnn->public_netmask_bits));
return 0;
}
+ }
- } else if (vnn->iface == NULL) {
- DEBUG(DEBUG_ERR, ("No interface found for IP %s.\n",
- ctdb_addr_to_str(&vnn->public_address)));
- return 0;
+ /* There is a potential race between take_ip and us because we
+ * update the VNN via a callback that run when the
+ * eventscripts have been run. Avoid the race by allowing one
+ * update to be in flight at a time.
+ */
+ if (vnn->update_in_flight) {
+ DEBUG(DEBUG_NOTICE,("Release of IP %s/%u rejected "
+ "update for this IP already in flight\n",
+ ctdb_addr_to_str(&vnn->public_address),
+ vnn->public_netmask_bits));
+ return -1;
+ }
+
+ if (ctdb->do_checkpublicip) {
+ iface = ctdb_sys_find_ifname(&pip->addr);
+ if (iface == NULL) {
+ DEBUG(DEBUG_ERR, ("Could not find which interface the ip address is hosted on. can not release it\n"));
+ return 0;
+ }
+ if (vnn->iface == NULL) {
+ DEBUG(DEBUG_WARNING,
+ ("Public IP %s is hosted on interface %s but we have no VNN\n",
+ ctdb_addr_to_str(&pip->addr),
+ iface));
+ } else if (strcmp(iface, ctdb_vnn_iface_string(vnn)) != 0) {
+ DEBUG(DEBUG_WARNING,
+ ("Public IP %s is hosted on inteterface %s but VNN says %s\n",
+ ctdb_addr_to_str(&pip->addr),
+ iface,
+ ctdb_vnn_iface_string(vnn)));
+ /* Should we fix vnn->iface? If we do, what
+ * happens to reference counts?
+ */
+ }
+ } else {
+ iface = strdup(ctdb_vnn_iface_string(vnn));
}
DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s node:%d\n",
ctdb_addr_to_str(&pip->addr),
- vnn->public_netmask_bits,
- ctdb_vnn_iface_string(vnn),
+ vnn->public_netmask_bits,
+ iface,
pip->pnn));
state = talloc(ctdb, struct takeover_callback_state);
*state->addr = pip->addr;
state->vnn = vnn;
+ vnn->update_in_flight = true;
+ talloc_set_destructor(state, ctdb_releaseip_destructor);
+
ret = ctdb_event_script_callback(ctdb,
state, release_ip_callback, state,
- false,
CTDB_EVENT_RELEASE_IP,
"%s %s %u",
- ctdb_vnn_iface_string(vnn),
+ iface,
ctdb_addr_to_str(&pip->addr),
vnn->public_netmask_bits);
+ free(iface);
if (ret != 0) {
DEBUG(DEBUG_ERR,(__location__ " Failed to release IP %s on interface %s\n",
ctdb_addr_to_str(&pip->addr),
static int ctdb_add_public_address(struct ctdb_context *ctdb,
ctdb_sock_addr *addr,
- unsigned mask, const char *ifaces)
+ unsigned mask, const char *ifaces,
+ bool check_address)
{
struct ctdb_vnn *vnn;
uint32_t num = 0;
vnn->public_address = *addr;
vnn->public_netmask_bits = mask;
vnn->pnn = -1;
- if (ctdb_sys_have_ip(addr)) {
- DEBUG(DEBUG_ERR,("We are already hosting public address '%s'. setting PNN to ourself:%d\n", ctdb_addr_to_str(addr), ctdb->pnn));
- vnn->pnn = ctdb->pnn;
+ if (check_address) {
+ if (ctdb_sys_have_ip(addr)) {
+ DEBUG(DEBUG_ERR,("We are already hosting public address '%s'. setting PNN to ourself:%d\n", ctdb_addr_to_str(addr), ctdb->pnn));
+ vnn->pnn = ctdb->pnn;
+ }
}
for (i=0; vnn->ifaces[i]; i++) {
talloc_free(vnn);
return -1;
}
- if (i == 0) {
- vnn->iface = ctdb_find_iface(ctdb, vnn->ifaces[i]);
- }
}
DLIST_ADD(ctdb->vnn, vnn);
return 0;
}
-/*
- setup the event script directory
-*/
-int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char *script_dir)
-{
- ctdb->event_script_dir = talloc_strdup(ctdb, script_dir);
- CTDB_NO_MEMORY(ctdb, ctdb->event_script_dir);
- return 0;
-}
-
static void ctdb_check_interfaces_event(struct event_context *ev, struct timed_event *te,
struct timeval t, void *private_data)
{
}
-static int ctdb_start_monitoring_interfaces(struct ctdb_context *ctdb)
+int ctdb_start_monitoring_interfaces(struct ctdb_context *ctdb)
{
if (ctdb->check_public_ifaces_ctx != NULL) {
talloc_free(ctdb->check_public_ifaces_ctx);
/*
setup the public address lists from a file
*/
-int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist)
+int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses)
{
char **lines;
int nlines;
int i;
- lines = file_lines_load(alist, &nlines, ctdb);
+ lines = file_lines_load(ctdb->public_addresses_file, &nlines, ctdb);
if (lines == NULL) {
- ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", alist);
+ ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", ctdb->public_addresses_file);
return -1;
}
while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
talloc_free(lines);
return -1;
}
- if (ctdb_add_public_address(ctdb, &addr, mask, ifaces)) {
+ if (ctdb_add_public_address(ctdb, &addr, mask, ifaces, check_addresses)) {
DEBUG(DEBUG_CRIT,("Failed to add line %u to the public address list\n", i+1));
talloc_free(lines);
return -1;
}
- if (ctdb->do_checkpublicip) {
- ctdb_start_monitoring_interfaces(ctdb);
- }
-
talloc_free(lines);
return 0;
}
return 0;
}
+struct ctdb_public_ip_list {
+ struct ctdb_public_ip_list *next;
+ uint32_t pnn;
+ ctdb_sock_addr addr;
+};
+
/* Given a physical node, return the number of
public addresses that is currently assigned to this node.
*/
}
-/* Check if this is a public ip known to the node, i.e. can that
- node takeover this ip ?
+/* Can the given node host the given IP: is the public IP known to the
+ * node and is NOIPHOST unset?
*/
-static int can_node_serve_ip(struct ctdb_context *ctdb, int32_t pnn,
- struct ctdb_public_ip_list *ip)
+static bool can_node_host_ip(struct ctdb_context *ctdb, int32_t pnn,
+ struct ctdb_ipflags ipflags,
+ struct ctdb_public_ip_list *ip)
{
struct ctdb_all_public_ips *public_ips;
int i;
+ if (ipflags.noiphost) {
+ return false;
+ }
+
public_ips = ctdb->nodes[pnn]->available_public_ips;
if (public_ips == NULL) {
- return -1;
+ return false;
}
- for (i=0;i<public_ips->num;i++) {
+ for (i=0; i<public_ips->num; i++) {
if (ctdb_same_ip(&ip->addr, &public_ips->ips[i].addr)) {
/* yes, this node can serve this public ip */
- return 0;
+ return true;
}
}
- return -1;
+ return false;
}
+static bool can_node_takeover_ip(struct ctdb_context *ctdb, int32_t pnn,
+ struct ctdb_ipflags ipflags,
+ struct ctdb_public_ip_list *ip)
+{
+ if (ipflags.noiptakeover) {
+ return false;
+ }
+
+ return can_node_host_ip(ctdb, pnn, ipflags, ip);
+}
/* search the node lists list for a node to takeover this ip.
pick the node that currently are serving the least number of ips
so that the ips get spread out evenly.
*/
static int find_takeover_node(struct ctdb_context *ctdb,
- struct ctdb_node_map *nodemap, uint32_t mask,
+ struct ctdb_ipflags *ipflags,
struct ctdb_public_ip_list *ip,
struct ctdb_public_ip_list *all_ips)
{
int pnn, min=0, num;
- int i;
+ int i, numnodes;
+ numnodes = talloc_array_length(ipflags);
pnn = -1;
- for (i=0;i<nodemap->num;i++) {
- if (nodemap->nodes[i].flags & NODE_FLAGS_NOIPTAKEOVER) {
- /* This node is not allowed to takeover any addresses
- */
- continue;
- }
-
- if (nodemap->nodes[i].flags & mask) {
- /* This node is not healty and can not be used to serve
- a public address
- */
- continue;
- }
-
+ for (i=0; i<numnodes; i++) {
/* verify that this node can serve this ip */
- if (can_node_serve_ip(ctdb, i, ip)) {
+ if (!can_node_takeover_ip(ctdb, i, ipflags[i], ip)) {
/* no it couldnt so skip to the next node */
continue;
}
tmp_ip = talloc_zero(ctdb->ip_tree, struct ctdb_public_ip_list);
CTDB_NO_MEMORY_NULL(ctdb, tmp_ip);
- tmp_ip->pnn = public_ips->ips[j].pnn;
+ /* Do not use information about IP addresses hosted
+ * on other nodes, it may not be accurate */
+ if (public_ips->ips[j].pnn == ctdb->nodes[i]->pnn) {
+ tmp_ip->pnn = public_ips->ips[j].pnn;
+ } else {
+ tmp_ip->pnn = -1;
+ }
tmp_ip->addr = public_ips->ips[j].addr;
tmp_ip->next = NULL;
* finding the best node for each.
*/
static void basic_allocate_unassigned(struct ctdb_context *ctdb,
- struct ctdb_node_map *nodemap,
- uint32_t mask,
+ struct ctdb_ipflags *ipflags,
struct ctdb_public_ip_list *all_ips)
{
struct ctdb_public_ip_list *tmp_ip;
*/
for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
if (tmp_ip->pnn == -1) {
- if (find_takeover_node(ctdb, nodemap, mask, tmp_ip, all_ips)) {
+ if (find_takeover_node(ctdb, ipflags, tmp_ip, all_ips)) {
DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
ctdb_addr_to_str(&tmp_ip->addr)));
}
/* Basic non-deterministic rebalancing algorithm.
*/
-static bool basic_failback(struct ctdb_context *ctdb,
- struct ctdb_node_map *nodemap,
- uint32_t mask,
+static void basic_failback(struct ctdb_context *ctdb,
+ struct ctdb_ipflags *ipflags,
struct ctdb_public_ip_list *all_ips,
- int num_ips,
- int *retries)
+ int num_ips)
{
- int i;
- int maxnode, maxnum=0, minnode, minnum=0, num;
+ int i, numnodes;
+ int maxnode, maxnum, minnode, minnum, num, retries;
struct ctdb_public_ip_list *tmp_ip;
+ numnodes = talloc_array_length(ipflags);
+ retries = 0;
+
+try_again:
+ maxnum=0;
+ minnum=0;
+
/* for each ip address, loop over all nodes that can serve
this ip and make sure that the difference between the node
serving the most and the node serving the least ip's are
*/
maxnode = -1;
minnode = -1;
- for (i=0;i<nodemap->num;i++) {
- if (nodemap->nodes[i].flags & mask) {
- continue;
- }
-
- /* Only check nodes that are allowed to takeover an ip */
- if (nodemap->nodes[i].flags & NODE_FLAGS_NOIPTAKEOVER) {
- continue;
- }
-
+ for (i=0; i<numnodes; i++) {
/* only check nodes that can actually serve this ip */
- if (can_node_serve_ip(ctdb, i, tmp_ip)) {
+ if (!can_node_takeover_ip(ctdb, i, ipflags[i], tmp_ip)) {
/* no it couldnt so skip to the next node */
continue;
}
continue;
}
- /* If we want deterministic IPs then dont try to reallocate
- them to spread out the load.
- */
- if (1 == ctdb->tunable.deterministic_public_ips) {
- continue;
- }
-
/* if the spread between the smallest and largest coverage by
a node is >=2 we steal one of the ips from the node with
most coverage to even things out a bit.
want to spend too much time balancing the ip coverage.
*/
if ( (maxnum > minnum+1)
- && (*retries < (num_ips + 5)) ){
+ && (retries < (num_ips + 5)) ){
struct ctdb_public_ip_list *tmp;
- /* mark one of maxnode's vnn's as unassigned and try
- again
- */
+ /* Reassign one of maxnode's VNNs */
for (tmp=all_ips;tmp;tmp=tmp->next) {
if (tmp->pnn == maxnode) {
- tmp->pnn = -1;
- (*retries)++;
- return true;
+ (void)find_takeover_node(ctdb, ipflags, tmp, all_ips);
+ retries++;
+ goto try_again;;
}
}
}
}
-
- return false;
-}
-
-struct ctdb_rebalancenodes {
- struct ctdb_rebalancenodes *next;
- uint32_t pnn;
-};
-static struct ctdb_rebalancenodes *force_rebalance_list = NULL;
-
-
-/* set this flag to force the node to be rebalanced even if it just didnt
- become healthy again.
-*/
-void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn)
-{
- struct ctdb_rebalancenodes *rebalance;
-
- for (rebalance = force_rebalance_list; rebalance; rebalance = rebalance->next) {
- if (rebalance->pnn == pnn) {
- return;
- }
- }
-
- rebalance = talloc(ctdb, struct ctdb_rebalancenodes);
- rebalance->pnn = pnn;
- rebalance->next = force_rebalance_list;
- force_rebalance_list = rebalance;
}
-/* Do necessary LCP2 initialisation. Bury it in a function here so
- * that we can unit test it.
- */
-static void lcp2_init(struct ctdb_context * tmp_ctx,
- struct ctdb_node_map * nodemap,
- uint32_t mask,
- struct ctdb_public_ip_list *all_ips,
- uint32_t **lcp2_imbalances,
- bool **newly_healthy)
+static void lcp2_init(struct ctdb_context *tmp_ctx,
+ struct ctdb_ipflags *ipflags,
+ struct ctdb_public_ip_list *all_ips,
+ uint32_t *force_rebalance_nodes,
+ uint32_t **lcp2_imbalances,
+ bool **rebalance_candidates)
{
- int i;
+ int i, numnodes;
struct ctdb_public_ip_list *tmp_ip;
- *newly_healthy = talloc_array(tmp_ctx, bool, nodemap->num);
- CTDB_NO_MEMORY_FATAL(tmp_ctx, *newly_healthy);
- *lcp2_imbalances = talloc_array(tmp_ctx, uint32_t, nodemap->num);
+ numnodes = talloc_array_length(ipflags);
+
+ *rebalance_candidates = talloc_array(tmp_ctx, bool, numnodes);
+ CTDB_NO_MEMORY_FATAL(tmp_ctx, *rebalance_candidates);
+ *lcp2_imbalances = talloc_array(tmp_ctx, uint32_t, numnodes);
CTDB_NO_MEMORY_FATAL(tmp_ctx, *lcp2_imbalances);
- for (i=0;i<nodemap->num;i++) {
+ for (i=0; i<numnodes; i++) {
(*lcp2_imbalances)[i] = lcp2_imbalance(all_ips, i);
- /* First step: is the node "healthy"? */
- (*newly_healthy)[i] = ! (bool)(nodemap->nodes[i].flags & mask);
+ /* First step: assume all nodes are candidates */
+ (*rebalance_candidates)[i] = true;
}
- /* 2nd step: if a ndoe has IPs assigned then it must have been
- * healthy before, so we remove it from consideration... */
+ /* 2nd step: if a node has IPs assigned then it must have been
+ * healthy before, so we remove it from consideration. This
+ * is overkill but is all we have because we don't maintain
+ * state between takeover runs. An alternative would be to
+ * keep state and invalidate it every time the recovery master
+ * changes.
+ */
for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
if (tmp_ip->pnn != -1) {
- (*newly_healthy)[tmp_ip->pnn] = false;
+ (*rebalance_candidates)[tmp_ip->pnn] = false;
}
}
/* 3rd step: if a node is forced to re-balance then
we allow failback onto the node */
- while (force_rebalance_list != NULL) {
- struct ctdb_rebalancenodes *next = force_rebalance_list->next;
-
- if (force_rebalance_list->pnn <= nodemap->num) {
- (*newly_healthy)[force_rebalance_list->pnn] = true;
+ if (force_rebalance_nodes == NULL) {
+ return;
+ }
+ for (i = 0; i < talloc_array_length(force_rebalance_nodes); i++) {
+ uint32_t pnn = force_rebalance_nodes[i];
+ if (pnn >= numnodes) {
+ DEBUG(DEBUG_ERR,
+ (__location__ "unknown node %u\n", pnn));
+ continue;
}
- DEBUG(DEBUG_ERR,("During ipreallocation, forced rebalance of node %d\n", force_rebalance_list->pnn));
- talloc_free(force_rebalance_list);
- force_rebalance_list = next;
+ DEBUG(DEBUG_NOTICE,
+ ("Forcing rebalancing of IPs to node %u\n", pnn));
+ (*rebalance_candidates)[pnn] = true;
}
}
* the IP/node combination that will cost the least.
*/
static void lcp2_allocate_unassigned(struct ctdb_context *ctdb,
- struct ctdb_node_map *nodemap,
- uint32_t mask,
- struct ctdb_public_ip_list *all_ips,
- uint32_t *lcp2_imbalances)
+ struct ctdb_ipflags *ipflags,
+ struct ctdb_public_ip_list *all_ips,
+ uint32_t *lcp2_imbalances)
{
struct ctdb_public_ip_list *tmp_ip;
- int dstnode;
+ int dstnode, numnodes;
int minnode;
uint32_t mindsum, dstdsum, dstimbl, minimbl;
bool should_loop = true;
bool have_unassigned = true;
+ numnodes = talloc_array_length(ipflags);
+
while (have_unassigned && should_loop) {
should_loop = false;
continue;
}
- for (dstnode=0; dstnode < nodemap->num; dstnode++) {
- /* Only check nodes that are allowed to takeover an ip */
- if (nodemap->nodes[dstnode].flags & NODE_FLAGS_NOIPTAKEOVER) {
- continue;
- }
-
- /* only check nodes that can actually serve this ip */
- if (can_node_serve_ip(ctdb, dstnode, tmp_ip)) {
+ for (dstnode=0; dstnode<numnodes; dstnode++) {
+ /* only check nodes that can actually takeover this ip */
+ if (!can_node_takeover_ip(ctdb, dstnode,
+ ipflags[dstnode],
+ tmp_ip)) {
/* no it couldnt so skip to the next node */
continue;
}
- if (nodemap->nodes[dstnode].flags & mask) {
- continue;
- }
dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
dstimbl = lcp2_imbalances[dstnode] + dstdsum;
* combination to move from the source node.
*/
static bool lcp2_failback_candidate(struct ctdb_context *ctdb,
- struct ctdb_node_map *nodemap,
+ struct ctdb_ipflags *ipflags,
struct ctdb_public_ip_list *all_ips,
int srcnode,
- uint32_t candimbl,
uint32_t *lcp2_imbalances,
- bool *newly_healthy)
+ bool *rebalance_candidates)
{
- int dstnode, mindstnode;
+ int dstnode, mindstnode, numnodes;
uint32_t srcimbl, srcdsum, dstimbl, dstdsum;
uint32_t minsrcimbl, mindstimbl;
struct ctdb_public_ip_list *minip;
struct ctdb_public_ip_list *tmp_ip;
/* Find an IP and destination node that best reduces imbalance. */
+ srcimbl = 0;
minip = NULL;
minsrcimbl = 0;
mindstnode = -1;
mindstimbl = 0;
+ numnodes = talloc_array_length(ipflags);
+
DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
- DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES FROM %d [%d]\n", srcnode, candimbl));
+ DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES FROM %d [%d]\n",
+ srcnode, lcp2_imbalances[srcnode]));
for (tmp_ip=all_ips; tmp_ip; tmp_ip=tmp_ip->next) {
/* Only consider addresses on srcnode. */
/* What is this IP address costing the source node? */
srcdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, srcnode);
- srcimbl = candimbl - srcdsum;
+ srcimbl = lcp2_imbalances[srcnode] - srcdsum;
/* Consider this IP address would cost each potential
* destination node. Destination nodes are limited to
* to do gratuitous failover of IPs just to make minor
* balance improvements.
*/
- for (dstnode=0; dstnode < nodemap->num; dstnode++) {
- if (! newly_healthy[dstnode]) {
- continue;
- }
-
- /* Only check nodes that are allowed to takeover an ip */
- if (nodemap->nodes[dstnode].flags & NODE_FLAGS_NOIPTAKEOVER) {
+ for (dstnode=0; dstnode<numnodes; dstnode++) {
+ if (!rebalance_candidates[dstnode]) {
continue;
}
- /* only check nodes that can actually serve this ip */
- if (can_node_serve_ip(ctdb, dstnode, tmp_ip)) {
+ /* only check nodes that can actually takeover this ip */
+ if (!can_node_takeover_ip(ctdb, dstnode,
+ ipflags[dstnode], tmp_ip)) {
/* no it couldnt so skip to the next node */
continue;
}
dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
dstimbl = lcp2_imbalances[dstnode] + dstdsum;
DEBUG(DEBUG_DEBUG,(" %d [%d] -> %s -> %d [+%d]\n",
- srcnode, srcimbl - lcp2_imbalances[srcnode],
+ srcnode, -srcdsum,
ctdb_addr_to_str(&(tmp_ip->addr)),
- dstnode, dstimbl - lcp2_imbalances[dstnode]));
+ dstnode, dstdsum));
- if ((dstimbl < candimbl) && (dstdsum < srcdsum) && \
+ if ((dstimbl < lcp2_imbalances[srcnode]) &&
+ (dstdsum < srcdsum) && \
((mindstnode == -1) || \
((srcimbl + dstimbl) < (minsrcimbl + mindstimbl)))) {
mindstnode, mindstimbl - lcp2_imbalances[mindstnode]));
- lcp2_imbalances[srcnode] = srcimbl;
+ lcp2_imbalances[srcnode] = minsrcimbl;
lcp2_imbalances[mindstnode] = mindstimbl;
minip->pnn = mindstnode;
* node with the highest LCP2 imbalance, and then determines the best
* IP/destination node combination to move from the source node.
*/
-static bool lcp2_failback(struct ctdb_context *ctdb,
- struct ctdb_node_map *nodemap,
- uint32_t mask,
+static void lcp2_failback(struct ctdb_context *ctdb,
+ struct ctdb_ipflags *ipflags,
struct ctdb_public_ip_list *all_ips,
uint32_t *lcp2_imbalances,
- bool *newly_healthy)
+ bool *rebalance_candidates)
{
- int i, num_newly_healthy;
+ int i, numnodes;
struct lcp2_imbalance_pnn * lips;
- bool ret;
+ bool again;
- /* It is only worth continuing if we have suitable target
- * nodes to transfer IPs to. This check is much cheaper than
- * continuing on...
- */
- num_newly_healthy = 0;
- for (i = 0; i < nodemap->num; i++) {
- if (newly_healthy[i]) {
- num_newly_healthy++;
- }
- }
- if (num_newly_healthy == 0) {
- return false;
- }
+ numnodes = talloc_array_length(ipflags);
+try_again:
/* Put the imbalances and nodes into an array, sort them and
* iterate through candidates. Usually the 1st one will be
* used, so this doesn't cost much...
*/
- lips = talloc_array(ctdb, struct lcp2_imbalance_pnn, nodemap->num);
- for (i = 0; i < nodemap->num; i++) {
+ DEBUG(DEBUG_DEBUG,("+++++++++++++++++++++++++++++++++++++++++\n"));
+ DEBUG(DEBUG_DEBUG,("Selecting most imbalanced node from:\n"));
+ lips = talloc_array(ctdb, struct lcp2_imbalance_pnn, numnodes);
+ for (i=0; i<numnodes; i++) {
lips[i].imbalance = lcp2_imbalances[i];
lips[i].pnn = i;
+ DEBUG(DEBUG_DEBUG,(" %d [%d]\n", i, lcp2_imbalances[i]));
}
- qsort(lips, nodemap->num, sizeof(struct lcp2_imbalance_pnn),
+ qsort(lips, numnodes, sizeof(struct lcp2_imbalance_pnn),
lcp2_cmp_imbalance_pnn);
- ret = false;
- for (i = 0; i < nodemap->num; i++) {
+ again = false;
+ for (i=0; i<numnodes; i++) {
/* This means that all nodes had 0 or 1 addresses, so
* can't be imbalanced.
*/
}
if (lcp2_failback_candidate(ctdb,
- nodemap,
+ ipflags,
all_ips,
lips[i].pnn,
- lips[i].imbalance,
lcp2_imbalances,
- newly_healthy)) {
- ret = true;
+ rebalance_candidates)) {
+ again = true;
break;
}
}
talloc_free(lips);
- return ret;
+ if (again) {
+ goto try_again;
+ }
}
-/* The calculation part of the IP allocation algorithm. */
-static void ctdb_takeover_run_core(struct ctdb_context *ctdb,
- struct ctdb_node_map *nodemap,
- struct ctdb_public_ip_list **all_ips_p)
+static void unassign_unsuitable_ips(struct ctdb_context *ctdb,
+ struct ctdb_ipflags *ipflags,
+ struct ctdb_public_ip_list *all_ips)
{
- int i, num_healthy, retries, num_ips;
- uint32_t mask;
- struct ctdb_public_ip_list *all_ips, *tmp_ip;
- uint32_t *lcp2_imbalances;
- bool *newly_healthy;
-
- TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
-
- /* Count how many completely healthy nodes we have */
- num_healthy = 0;
- for (i=0;i<nodemap->num;i++) {
- if (!(nodemap->nodes[i].flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED))) {
- num_healthy++;
- }
- }
-
- if (num_healthy > 0) {
- /* We have healthy nodes, so only consider them for
- serving public addresses
- */
- mask = NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED;
- } else {
- /* We didnt have any completely healthy nodes so
- use "disabled" nodes as a fallback
- */
- mask = NODE_FLAGS_INACTIVE;
- }
-
- /* since nodes only know about those public addresses that
- can be served by that particular node, no single node has
- a full list of all public addresses that exist in the cluster.
- Walk over all node structures and create a merged list of
- all public addresses that exist in the cluster.
-
- keep the tree of ips around as ctdb->ip_tree
- */
- all_ips = create_merged_ip_list(ctdb);
- *all_ips_p = all_ips; /* minimal code changes */
-
- /* Count how many ips we have */
- num_ips = 0;
- for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
- num_ips++;
- }
-
- /* If we want deterministic ip allocations, i.e. that the ip addresses
- will always be allocated the same way for a specific set of
- available/unavailable nodes.
- */
- if (1 == ctdb->tunable.deterministic_public_ips) {
- DEBUG(DEBUG_NOTICE,("Deterministic IPs enabled. Resetting all ip allocations\n"));
- for (i=0,tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next,i++) {
- tmp_ip->pnn = i%nodemap->num;
- }
- }
-
-
- /* mark all public addresses with a masked node as being served by
- node -1
- */
- for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
- if (tmp_ip->pnn == -1) {
- continue;
- }
- if (nodemap->nodes[tmp_ip->pnn].flags & mask) {
- tmp_ip->pnn = -1;
- }
- }
+ struct ctdb_public_ip_list *tmp_ip;
/* verify that the assigned nodes can serve that public ip
and set it to -1 if not
if (tmp_ip->pnn == -1) {
continue;
}
- if (can_node_serve_ip(ctdb, tmp_ip->pnn, tmp_ip) != 0) {
+ if (!can_node_host_ip(ctdb, tmp_ip->pnn,
+ ipflags[tmp_ip->pnn], tmp_ip) != 0) {
/* this node can not serve this ip. */
+ DEBUG(DEBUG_DEBUG,("Unassign IP: %s from %d\n",
+ ctdb_addr_to_str(&(tmp_ip->addr)),
+ tmp_ip->pnn));
tmp_ip->pnn = -1;
}
}
+}
- if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
- lcp2_init(tmp_ctx, nodemap, mask, all_ips, &lcp2_imbalances, &newly_healthy);
- }
+static void ip_alloc_deterministic_ips(struct ctdb_context *ctdb,
+ struct ctdb_ipflags *ipflags,
+ struct ctdb_public_ip_list *all_ips)
+{
+ struct ctdb_public_ip_list *tmp_ip;
+ int i, numnodes;
+
+ numnodes = talloc_array_length(ipflags);
- /* now we must redistribute all public addresses with takeover node
- -1 among the nodes available
+ DEBUG(DEBUG_NOTICE,("Deterministic IPs enabled. Resetting all ip allocations\n"));
+ /* Allocate IPs to nodes in a modulo fashion so that IPs will
+ * always be allocated the same way for a specific set of
+ * available/unavailable nodes.
*/
- retries = 0;
-try_again:
- if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
- lcp2_allocate_unassigned(ctdb, nodemap, mask, all_ips, lcp2_imbalances);
- } else {
- basic_allocate_unassigned(ctdb, nodemap, mask, all_ips);
+
+ for (i=0,tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next,i++) {
+ tmp_ip->pnn = i % numnodes;
}
- /* If we dont want ips to fail back after a node becomes healthy
- again, we wont even try to reallocat the ip addresses so that
- they are evenly spread out.
- This can NOT be used at the same time as DeterministicIPs !
- */
+ /* IP failback doesn't make sense with deterministic
+ * IPs, since the modulo step above implicitly fails
+ * back IPs to their "home" node.
+ */
if (1 == ctdb->tunable.no_ip_failback) {
- if (1 == ctdb->tunable.deterministic_public_ips) {
- DEBUG(DEBUG_ERR, ("ERROR: You can not use 'DeterministicIPs' and 'NoIPFailback' at the same time\n"));
- }
- goto finished;
+ DEBUG(DEBUG_WARNING, ("WARNING: 'NoIPFailback' set but ignored - incompatible with 'DeterministicIPs\n"));
}
+ unassign_unsuitable_ips(ctdb, ipflags, all_ips);
- /* now, try to make sure the ip adresses are evenly distributed
- across the node.
- */
- if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
- if (lcp2_failback(ctdb, nodemap, mask, all_ips, lcp2_imbalances, newly_healthy)) {
- goto try_again;
- }
- } else {
- if (basic_failback(ctdb, nodemap, mask, all_ips, num_ips, &retries)) {
- goto try_again;
- }
- }
+ basic_allocate_unassigned(ctdb, ipflags, all_ips);
+
+ /* No failback here! */
+}
+
+static void ip_alloc_nondeterministic_ips(struct ctdb_context *ctdb,
+ struct ctdb_ipflags *ipflags,
+ struct ctdb_public_ip_list *all_ips)
+{
+ /* This should be pushed down into basic_failback. */
+ struct ctdb_public_ip_list *tmp_ip;
+ int num_ips = 0;
+ for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
+ num_ips++;
+ }
+
+ unassign_unsuitable_ips(ctdb, ipflags, all_ips);
+
+ basic_allocate_unassigned(ctdb, ipflags, all_ips);
+
+ /* If we don't want IPs to fail back then don't rebalance IPs. */
+ if (1 == ctdb->tunable.no_ip_failback) {
+ return;
+ }
+
+ /* Now, try to make sure the ip adresses are evenly distributed
+ across the nodes.
+ */
+ basic_failback(ctdb, ipflags, all_ips, num_ips);
+}
+
+static void ip_alloc_lcp2(struct ctdb_context *ctdb,
+ struct ctdb_ipflags *ipflags,
+ struct ctdb_public_ip_list *all_ips,
+ uint32_t *force_rebalance_nodes)
+{
+ uint32_t *lcp2_imbalances;
+ bool *rebalance_candidates;
+ int numnodes, num_rebalance_candidates, i;
+
+ TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+
+ unassign_unsuitable_ips(ctdb, ipflags, all_ips);
+
+ lcp2_init(tmp_ctx, ipflags, all_ips,force_rebalance_nodes,
+ &lcp2_imbalances, &rebalance_candidates);
+
+ lcp2_allocate_unassigned(ctdb, ipflags, all_ips, lcp2_imbalances);
+
+ /* If we don't want IPs to fail back then don't rebalance IPs. */
+ if (1 == ctdb->tunable.no_ip_failback) {
+ goto finished;
+ }
+
+ /* It is only worth continuing if we have suitable target
+ * nodes to transfer IPs to. This check is much cheaper than
+ * continuing on...
+ */
+ numnodes = talloc_array_length(ipflags);
+ num_rebalance_candidates = 0;
+ for (i=0; i<numnodes; i++) {
+ if (rebalance_candidates[i]) {
+ num_rebalance_candidates++;
+ }
+ }
+ if (num_rebalance_candidates == 0) {
+ goto finished;
+ }
+
+ /* Now, try to make sure the ip adresses are evenly distributed
+ across the nodes.
+ */
+ lcp2_failback(ctdb, ipflags, all_ips,
+ lcp2_imbalances, rebalance_candidates);
+
+finished:
+ talloc_free(tmp_ctx);
+}
+
+static bool all_nodes_are_disabled(struct ctdb_node_map *nodemap)
+{
+ int i, num_healthy;
+
+ /* Count how many completely healthy nodes we have */
+ num_healthy = 0;
+ for (i=0;i<nodemap->num;i++) {
+ if (!(nodemap->nodes[i].flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED))) {
+ num_healthy++;
+ }
+ }
+
+ return num_healthy == 0;
+}
+
+/* The calculation part of the IP allocation algorithm. */
+static void ctdb_takeover_run_core(struct ctdb_context *ctdb,
+ struct ctdb_ipflags *ipflags,
+ struct ctdb_public_ip_list **all_ips_p,
+ uint32_t *force_rebalance_nodes)
+{
+ /* since nodes only know about those public addresses that
+ can be served by that particular node, no single node has
+ a full list of all public addresses that exist in the cluster.
+ Walk over all node structures and create a merged list of
+ all public addresses that exist in the cluster.
+
+ keep the tree of ips around as ctdb->ip_tree
+ */
+ *all_ips_p = create_merged_ip_list(ctdb);
+
+ if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
+ ip_alloc_lcp2(ctdb, ipflags, *all_ips_p, force_rebalance_nodes);
+ } else if (1 == ctdb->tunable.deterministic_public_ips) {
+ ip_alloc_deterministic_ips(ctdb, ipflags, *all_ips_p);
+ } else {
+ ip_alloc_nondeterministic_ips(ctdb, ipflags, *all_ips_p);
+ }
- /* finished distributing the public addresses, now just send the
- info out to the nodes */
-finished:
/* at this point ->pnn is the node which will own each IP
or -1 if there is no node that can cover this ip
*/
return;
}
-static void noiptakeover_cb(struct ctdb_context *ctdb, uint32_t pnn, int32_t res, TDB_DATA outdata, void *callback)
+struct get_tunable_callback_data {
+ const char *tunable;
+ uint32_t *out;
+ bool fatal;
+};
+
+static void get_tunable_callback(struct ctdb_context *ctdb, uint32_t pnn,
+ int32_t res, TDB_DATA outdata,
+ void *callback)
+{
+ struct get_tunable_callback_data *cd =
+ (struct get_tunable_callback_data *)callback;
+ int size;
+
+ if (res != 0) {
+ /* Already handled in fail callback */
+ return;
+ }
+
+ if (outdata.dsize != sizeof(uint32_t)) {
+ DEBUG(DEBUG_ERR,("Wrong size of returned data when reading \"%s\" tunable from node %d. Expected %d bytes but received %d bytes\n",
+ cd->tunable, pnn, (int)sizeof(uint32_t),
+ (int)outdata.dsize));
+ cd->fatal = true;
+ return;
+ }
+
+ size = talloc_array_length(cd->out);
+ if (pnn >= size) {
+ DEBUG(DEBUG_ERR,("Got %s reply from node %d but nodemap only has %d entries\n",
+ cd->tunable, pnn, size));
+ return;
+ }
+
+
+ cd->out[pnn] = *(uint32_t *)outdata.dptr;
+}
+
+static void get_tunable_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
+ int32_t res, TDB_DATA outdata,
+ void *callback)
+{
+ struct get_tunable_callback_data *cd =
+ (struct get_tunable_callback_data *)callback;
+
+ switch (res) {
+ case -ETIME:
+ DEBUG(DEBUG_ERR,
+ ("Timed out getting tunable \"%s\" from node %d\n",
+ cd->tunable, pnn));
+ cd->fatal = true;
+ break;
+ case -EINVAL:
+ case -1:
+ DEBUG(DEBUG_WARNING,
+ ("Tunable \"%s\" not implemented on node %d\n",
+ cd->tunable, pnn));
+ break;
+ default:
+ DEBUG(DEBUG_ERR,
+ ("Unexpected error getting tunable \"%s\" from node %d\n",
+ cd->tunable, pnn));
+ cd->fatal = true;
+ }
+}
+
+static uint32_t *get_tunable_from_nodes(struct ctdb_context *ctdb,
+ TALLOC_CTX *tmp_ctx,
+ struct ctdb_node_map *nodemap,
+ const char *tunable,
+ uint32_t default_value)
+{
+ TDB_DATA data;
+ struct ctdb_control_get_tunable *t;
+ uint32_t *nodes;
+ uint32_t *tvals;
+ struct get_tunable_callback_data callback_data;
+ int i;
+
+ tvals = talloc_array(tmp_ctx, uint32_t, nodemap->num);
+ CTDB_NO_MEMORY_NULL(ctdb, tvals);
+ for (i=0; i<nodemap->num; i++) {
+ tvals[i] = default_value;
+ }
+
+ callback_data.out = tvals;
+ callback_data.tunable = tunable;
+ callback_data.fatal = false;
+
+ data.dsize = offsetof(struct ctdb_control_get_tunable, name) + strlen(tunable) + 1;
+ data.dptr = talloc_size(tmp_ctx, data.dsize);
+ t = (struct ctdb_control_get_tunable *)data.dptr;
+ t->length = strlen(tunable)+1;
+ memcpy(t->name, tunable, t->length);
+ nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_TUNABLE,
+ nodes, 0, TAKEOVER_TIMEOUT(),
+ false, data,
+ get_tunable_callback,
+ get_tunable_fail_callback,
+ &callback_data) != 0) {
+ if (callback_data.fatal) {
+ talloc_free(tvals);
+ tvals = NULL;
+ }
+ }
+ talloc_free(nodes);
+ talloc_free(data.dptr);
+
+ return tvals;
+}
+
+struct get_runstate_callback_data {
+ enum ctdb_runstate *out;
+ bool fatal;
+};
+
+static void get_runstate_callback(struct ctdb_context *ctdb, uint32_t pnn,
+ int32_t res, TDB_DATA outdata,
+ void *callback_data)
{
- struct ctdb_node_map *nodemap = (struct ctdb_node_map *)callback;
+ struct get_runstate_callback_data *cd =
+ (struct get_runstate_callback_data *)callback_data;
+ int size;
if (res != 0) {
- DEBUG(DEBUG_ERR,("Failure to read NoIPTakeover tunable from remote node %d\n", pnn));
+ /* Already handled in fail callback */
return;
}
if (outdata.dsize != sizeof(uint32_t)) {
- DEBUG(DEBUG_ERR,("Wrong size of returned data when reading NoIPTakeover tunable from node %d. Expected %d bytes but received %d bytes\n", pnn, (int)sizeof(uint32_t), (int)outdata.dsize));
+ DEBUG(DEBUG_ERR,("Wrong size of returned data when getting runstate from node %d. Expected %d bytes but received %d bytes\n",
+ pnn, (int)sizeof(uint32_t),
+ (int)outdata.dsize));
+ cd->fatal = true;
+ return;
+ }
+
+ size = talloc_array_length(cd->out);
+ if (pnn >= size) {
+ DEBUG(DEBUG_ERR,("Got reply from node %d but nodemap only has %d entries\n",
+ pnn, size));
+ return;
+ }
+
+ cd->out[pnn] = (enum ctdb_runstate)*(uint32_t *)outdata.dptr;
+}
+
+static void get_runstate_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
+ int32_t res, TDB_DATA outdata,
+ void *callback)
+{
+ struct get_runstate_callback_data *cd =
+ (struct get_runstate_callback_data *)callback;
+
+ switch (res) {
+ case -ETIME:
+ DEBUG(DEBUG_ERR,
+ ("Timed out getting runstate from node %d\n", pnn));
+ cd->fatal = true;
+ break;
+ default:
+ DEBUG(DEBUG_WARNING,
+ ("Error getting runstate from node %d - assuming runstates not supported\n",
+ pnn));
+ }
+}
+
+static enum ctdb_runstate * get_runstate_from_nodes(struct ctdb_context *ctdb,
+ TALLOC_CTX *tmp_ctx,
+ struct ctdb_node_map *nodemap,
+ enum ctdb_runstate default_value)
+{
+ uint32_t *nodes;
+ enum ctdb_runstate *rs;
+ struct get_runstate_callback_data callback_data;
+ int i;
+
+ rs = talloc_array(tmp_ctx, enum ctdb_runstate, nodemap->num);
+ CTDB_NO_MEMORY_NULL(ctdb, rs);
+ for (i=0; i<nodemap->num; i++) {
+ rs[i] = default_value;
+ }
+
+ callback_data.out = rs;
+ callback_data.fatal = false;
+
+ nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_RUNSTATE,
+ nodes, 0, TAKEOVER_TIMEOUT(),
+ true, tdb_null,
+ get_runstate_callback,
+ get_runstate_fail_callback,
+ &callback_data) != 0) {
+ if (callback_data.fatal) {
+ free(rs);
+ rs = NULL;
+ }
+ }
+ talloc_free(nodes);
+
+ return rs;
+}
+
+/* Set internal flags for IP allocation:
+ * Clear ip flags
+ * Set NOIPTAKOVER ip flags from per-node NoIPTakeover tunable
+ * Set NOIPHOST ip flag for each INACTIVE node
+ * if all nodes are disabled:
+ * Set NOIPHOST ip flags from per-node NoIPHostOnAllDisabled tunable
+ * else
+ * Set NOIPHOST ip flags for disabled nodes
+ */
+static struct ctdb_ipflags *
+set_ipflags_internal(struct ctdb_context *ctdb,
+ TALLOC_CTX *tmp_ctx,
+ struct ctdb_node_map *nodemap,
+ uint32_t *tval_noiptakeover,
+ uint32_t *tval_noiphostonalldisabled,
+ enum ctdb_runstate *runstate)
+{
+ int i;
+ struct ctdb_ipflags *ipflags;
+
+ /* Clear IP flags - implicit due to talloc_zero */
+ ipflags = talloc_zero_array(tmp_ctx, struct ctdb_ipflags, nodemap->num);
+ CTDB_NO_MEMORY_NULL(ctdb, ipflags);
+
+ for (i=0;i<nodemap->num;i++) {
+ /* Can not take IPs on node with NoIPTakeover set */
+ if (tval_noiptakeover[i] != 0) {
+ ipflags[i].noiptakeover = true;
+ }
+
+ /* Can not host IPs on node not in RUNNING state */
+ if (runstate[i] != CTDB_RUNSTATE_RUNNING) {
+ ipflags[i].noiphost = true;
+ continue;
+ }
+ /* Can not host IPs on INACTIVE node */
+ if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
+ ipflags[i].noiphost = true;
+ }
+ }
+
+ if (all_nodes_are_disabled(nodemap)) {
+ /* If all nodes are disabled, can not host IPs on node
+ * with NoIPHostOnAllDisabled set
+ */
+ for (i=0;i<nodemap->num;i++) {
+ if (tval_noiphostonalldisabled[i] != 0) {
+ ipflags[i].noiphost = true;
+ }
+ }
+ } else {
+ /* If some nodes are not disabled, then can not host
+ * IPs on DISABLED node
+ */
+ for (i=0;i<nodemap->num;i++) {
+ if (nodemap->nodes[i].flags & NODE_FLAGS_DISABLED) {
+ ipflags[i].noiphost = true;
+ }
+ }
+ }
+
+ return ipflags;
+}
+
+static struct ctdb_ipflags *set_ipflags(struct ctdb_context *ctdb,
+ TALLOC_CTX *tmp_ctx,
+ struct ctdb_node_map *nodemap)
+{
+ uint32_t *tval_noiptakeover;
+ uint32_t *tval_noiphostonalldisabled;
+ struct ctdb_ipflags *ipflags;
+ enum ctdb_runstate *runstate;
+
+
+ tval_noiptakeover = get_tunable_from_nodes(ctdb, tmp_ctx, nodemap,
+ "NoIPTakeover", 0);
+ if (tval_noiptakeover == NULL) {
+ return NULL;
+ }
+
+ tval_noiphostonalldisabled =
+ get_tunable_from_nodes(ctdb, tmp_ctx, nodemap,
+ "NoIPHostOnAllDisabled", 0);
+ if (tval_noiphostonalldisabled == NULL) {
+ /* Caller frees tmp_ctx */
+ return NULL;
+ }
+
+ /* Any nodes where CTDB_CONTROL_GET_RUNSTATE is not supported
+ * will default to CTDB_RUNSTATE_RUNNING. This ensures
+ * reasonable behaviour on a mixed cluster during upgrade.
+ */
+ runstate = get_runstate_from_nodes(ctdb, tmp_ctx, nodemap,
+ CTDB_RUNSTATE_RUNNING);
+ if (runstate == NULL) {
+ /* Caller frees tmp_ctx */
+ return NULL;
+ }
+
+ ipflags = set_ipflags_internal(ctdb, tmp_ctx, nodemap,
+ tval_noiptakeover,
+ tval_noiphostonalldisabled,
+ runstate);
+
+ talloc_free(tval_noiptakeover);
+ talloc_free(tval_noiphostonalldisabled);
+ talloc_free(runstate);
+
+ return ipflags;
+}
+
+struct iprealloc_callback_data {
+ bool *retry_nodes;
+ int retry_count;
+ client_async_callback fail_callback;
+ void *fail_callback_data;
+ struct ctdb_node_map *nodemap;
+};
+
+static void iprealloc_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
+ int32_t res, TDB_DATA outdata,
+ void *callback)
+{
+ int numnodes;
+ struct iprealloc_callback_data *cd =
+ (struct iprealloc_callback_data *)callback;
+
+ numnodes = talloc_array_length(cd->retry_nodes);
+ if (pnn > numnodes) {
+ DEBUG(DEBUG_ERR,
+ ("ipreallocated failure from node %d, "
+ "but only %d nodes in nodemap\n",
+ pnn, numnodes));
return;
}
- if (pnn >= nodemap->num) {
- DEBUG(DEBUG_ERR,("Got NoIPTakeover reply from node %d but nodemap only has %d entries\n", pnn, nodemap->num));
+ /* Can't run the "ipreallocated" event on a INACTIVE node */
+ if (cd->nodemap->nodes[pnn].flags & NODE_FLAGS_INACTIVE) {
+ DEBUG(DEBUG_WARNING,
+ ("ipreallocated failed on inactive node %d, ignoring\n",
+ pnn));
return;
}
- if (*(uint32_t *)outdata.dptr != 0) {
- nodemap->nodes[pnn].flags |= NODE_FLAGS_NOIPTAKEOVER;
+ switch (res) {
+ case -ETIME:
+ /* If the control timed out then that's a real error,
+ * so call the real fail callback
+ */
+ if (cd->fail_callback) {
+ cd->fail_callback(ctdb, pnn, res, outdata,
+ cd->fail_callback_data);
+ } else {
+ DEBUG(DEBUG_WARNING,
+ ("iprealloc timed out but no callback registered\n"));
+ }
+ break;
+ default:
+ /* If not a timeout then either the ipreallocated
+ * eventscript (or some setup) failed. This might
+ * have failed because the IPREALLOCATED control isn't
+ * implemented - right now there is no way of knowing
+ * because the error codes are all folded down to -1.
+ * Consider retrying using EVENTSCRIPT control...
+ */
+ DEBUG(DEBUG_WARNING,
+ ("ipreallocated failure from node %d, flagging retry\n",
+ pnn));
+ cd->retry_nodes[pnn] = true;
+ cd->retry_count++;
+ }
+}
+
+struct takeover_callback_data {
+ bool *node_failed;
+ client_async_callback fail_callback;
+ void *fail_callback_data;
+ struct ctdb_node_map *nodemap;
+};
+
+static void takeover_run_fail_callback(struct ctdb_context *ctdb,
+ uint32_t node_pnn, int32_t res,
+ TDB_DATA outdata, void *callback_data)
+{
+ struct takeover_callback_data *cd =
+ talloc_get_type_abort(callback_data,
+ struct takeover_callback_data);
+ int i;
+
+ for (i = 0; i < cd->nodemap->num; i++) {
+ if (node_pnn == cd->nodemap->nodes[i].pnn) {
+ break;
+ }
+ }
+
+ if (i == cd->nodemap->num) {
+ DEBUG(DEBUG_ERR, (__location__ " invalid PNN %u\n", node_pnn));
+ return;
+ }
+
+ if (!cd->node_failed[i]) {
+ cd->node_failed[i] = true;
+ cd->fail_callback(ctdb, node_pnn, res, outdata,
+ cd->fail_callback_data);
}
}
/*
make any IP alias changes for public addresses that are necessary
*/
-int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
+ uint32_t *force_rebalance_nodes,
+ client_async_callback fail_callback, void *callback_data)
{
- int i;
+ int i, j, ret;
struct ctdb_public_ip ip;
struct ctdb_public_ipv4 ipv4;
- struct ctdb_control_get_tunable *t;
uint32_t *nodes;
struct ctdb_public_ip_list *all_ips, *tmp_ip;
TDB_DATA data;
struct client_async_data *async_data;
struct ctdb_client_control_state *state;
TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+ struct ctdb_ipflags *ipflags;
+ struct takeover_callback_data *takeover_data;
+ struct iprealloc_callback_data iprealloc_data;
+ bool *retry_data;
/*
* ip failover is completely disabled, just send out the
goto ipreallocated;
}
-
- /* assume all nodes do support failback */
- for (i=0;i<nodemap->num;i++) {
- nodemap->nodes[i].flags &= ~NODE_FLAGS_NOIPTAKEOVER;
- }
- data.dsize = offsetof(struct ctdb_control_get_tunable, name) + strlen("NoIPTakeover") + 1;
- data.dptr = talloc_size(tmp_ctx, data.dsize);
- t = (struct ctdb_control_get_tunable *)data.dptr;
- t->length = strlen("NoIPTakeover")+1;
- memcpy(t->name, "NoIPTakeover", t->length);
- nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
- if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_TUNABLE,
- nodes, 0, TAKEOVER_TIMEOUT(),
- false, data,
- noiptakeover_cb, NULL,
- nodemap) != 0) {
- DEBUG(DEBUG_ERR, (__location__ " ctdb_control to get noiptakeover tunable failed\n"));
+ ipflags = set_ipflags(ctdb, tmp_ctx, nodemap);
+ if (ipflags == NULL) {
+ DEBUG(DEBUG_ERR,("Failed to set IP flags - aborting takeover run\n"));
+ talloc_free(tmp_ctx);
+ return -1;
}
- talloc_free(nodes);
- talloc_free(data.dptr);
-
ZERO_STRUCT(ip);
/* Do the IP reassignment calculations */
- ctdb_takeover_run_core(ctdb, nodemap, &all_ips);
+ ctdb_takeover_run_core(ctdb, ipflags, &all_ips, force_rebalance_nodes);
+
+ /* Now tell all nodes to release any public IPs should not
+ * host. This will be a NOOP on nodes that don't currently
+ * hold the given IP.
+ */
+ takeover_data = talloc_zero(tmp_ctx, struct takeover_callback_data);
+ CTDB_NO_MEMORY_FATAL(ctdb, takeover_data);
+
+ takeover_data->node_failed = talloc_zero_array(tmp_ctx,
+ bool, nodemap->num);
+ CTDB_NO_MEMORY_FATAL(ctdb, takeover_data->node_failed);
+ takeover_data->fail_callback = fail_callback;
+ takeover_data->fail_callback_data = callback_data;
+ takeover_data->nodemap = nodemap;
- /* now tell all nodes to delete any alias that they should not
- have. This will be a NOOP on nodes that don't currently
- hold the given alias */
async_data = talloc_zero(tmp_ctx, struct client_async_data);
CTDB_NO_MEMORY_FATAL(ctdb, async_data);
+ async_data->fail_callback = takeover_run_fail_callback;
+ async_data->callback_data = takeover_data;
+
for (i=0;i<nodemap->num;i++) {
/* don't talk to unconnected nodes, but do talk to banned nodes */
if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
/* tell all nodes to get their own IPs */
async_data = talloc_zero(tmp_ctx, struct client_async_data);
CTDB_NO_MEMORY_FATAL(ctdb, async_data);
+
+ async_data->fail_callback = fail_callback;
+ async_data->callback_data = callback_data;
+
for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
if (tmp_ip->pnn == -1) {
/* this IP won't be taken over */
}
ipreallocated:
- /* tell all nodes to update natwg */
- /* send the flags update natgw on all connected nodes */
- data.dptr = discard_const("ipreallocated");
- data.dsize = strlen((char *)data.dptr) + 1;
+ /*
+ * Tell all nodes to run eventscripts to process the
+ * "ipreallocated" event. This can do a lot of things,
+ * including restarting services to reconfigure them if public
+ * IPs have moved. Once upon a time this event only used to
+ * update natwg.
+ */
+ retry_data = talloc_zero_array(tmp_ctx, bool, nodemap->num);
+ CTDB_NO_MEMORY_FATAL(ctdb, retry_data);
+ iprealloc_data.retry_nodes = retry_data;
+ iprealloc_data.retry_count = 0;
+ iprealloc_data.fail_callback = fail_callback;
+ iprealloc_data.fail_callback_data = callback_data;
+ iprealloc_data.nodemap = nodemap;
+
nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
- if (ctdb_client_async_control(ctdb, CTDB_CONTROL_RUN_EVENTSCRIPTS,
- nodes, 0, TAKEOVER_TIMEOUT(),
- false, data,
- NULL, NULL,
- NULL) != 0) {
- DEBUG(DEBUG_ERR, (__location__ " ctdb_control to updatenatgw failed\n"));
+ ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_IPREALLOCATED,
+ nodes, 0, TAKEOVER_TIMEOUT(),
+ false, tdb_null,
+ NULL, iprealloc_fail_callback,
+ &iprealloc_data);
+ if (ret != 0) {
+ /* If the control failed then we should retry to any
+ * nodes flagged by iprealloc_fail_callback using the
+ * EVENTSCRIPT control. This is a best-effort at
+ * backward compatiblity when running a mixed cluster
+ * where some nodes have not yet been upgraded to
+ * support the IPREALLOCATED control.
+ */
+ DEBUG(DEBUG_WARNING,
+ ("Retry ipreallocated to some nodes using eventscript control\n"));
+
+ nodes = talloc_array(tmp_ctx, uint32_t,
+ iprealloc_data.retry_count);
+ CTDB_NO_MEMORY_FATAL(ctdb, nodes);
+
+ j = 0;
+ for (i=0; i<nodemap->num; i++) {
+ if (iprealloc_data.retry_nodes[i]) {
+ nodes[j] = i;
+ j++;
+ }
+ }
+
+ data.dptr = discard_const("ipreallocated");
+ data.dsize = strlen((char *)data.dptr) + 1;
+ ret = ctdb_client_async_control(ctdb,
+ CTDB_CONTROL_RUN_EVENTSCRIPTS,
+ nodes, 0, TAKEOVER_TIMEOUT(),
+ false, data,
+ NULL, fail_callback,
+ callback_data);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " failed to send control to run eventscripts with \"ipreallocated\"\n"));
+ }
}
talloc_free(tmp_ctx);
- return 0;
+ return ret;
}
struct ctdb_vnn *vnn;
ctdb_sock_addr addr;
+ /* If we don't have public IPs, tickles are useless */
+ if (ctdb->vnn == NULL) {
+ return 0;
+ }
+
switch (indata.dsize) {
case sizeof(struct ctdb_control_tcp):
old_addr = (struct ctdb_control_tcp *)indata.dptr;
struct ctdb_tcp_connection tcp;
struct ctdb_vnn *vnn;
+ /* If we don't have public IPs, tickles are useless */
+ if (ctdb->vnn == NULL) {
+ return 0;
+ }
+
vnn = find_public_ip_vnn(ctdb, &p->dst_addr);
if (vnn == NULL) {
DEBUG(DEBUG_INFO,(__location__ " got TCP_ADD control for an address which is not a public address '%s'\n",
/* If this is the first tickle */
if (tcparray == NULL) {
- tcparray = talloc_size(ctdb->nodes,
- offsetof(struct ctdb_tcp_array, connections) +
- sizeof(struct ctdb_tcp_connection) * 1);
+ tcparray = talloc(vnn, struct ctdb_tcp_array);
CTDB_NO_MEMORY(ctdb, tcparray);
vnn->tcp_array = tcparray;
/* Do we already have this tickle ?*/
tcp.src_addr = p->src_addr;
tcp.dst_addr = p->dst_addr;
- if (ctdb_tcp_find(vnn->tcp_array, &tcp) != NULL) {
+ if (ctdb_tcp_find(tcparray, &tcp) != NULL) {
DEBUG(DEBUG_DEBUG,("Already had tickle info for %s:%u for vnn:%u\n",
ctdb_addr_to_str(&tcp.dst_addr),
ntohs(tcp.dst_addr.ip.sin_port),
tcparray->num+1);
CTDB_NO_MEMORY(ctdb, tcparray->connections);
- vnn->tcp_array = tcparray;
tcparray->connections[tcparray->num].src_addr = p->src_addr;
tcparray->connections[tcparray->num].dst_addr = p->dst_addr;
tcparray->num++;
-
+
DEBUG(DEBUG_INFO,("Added tickle info for %s:%u from vnn %u\n",
ctdb_addr_to_str(&tcp.dst_addr),
ntohs(tcp.dst_addr.ip.sin_port),
{
struct ctdb_tcp_connection *conn = (struct ctdb_tcp_connection *)indata.dptr;
+ /* If we don't have public IPs, tickles are useless */
+ if (ctdb->vnn == NULL) {
+ return 0;
+ }
+
ctdb_remove_tcp_connection(ctdb, conn);
return 0;
/*
- called when a daemon restarts - send all tickes for all public addresses
- we are serving immediately to the new node.
+ Called when another daemon starts - caises all tickles for all
+ public addresses we are serving to be sent to the new node on the
+ next check. This actually causes the next scheduled call to
+ tdb_update_tcp_tickles() to update all nodes. This is simple and
+ doesn't require careful error handling.
*/
-int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t vnn)
+int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t pnn)
{
-/*XXX here we should send all tickes we are serving to the new node */
+ struct ctdb_vnn *vnn;
+
+ for (vnn = ctdb->vnn; vnn != NULL; vnn = vnn->next) {
+ vnn->tcp_update_needed = true;
+ }
+
return 0;
}
void ctdb_release_all_ips(struct ctdb_context *ctdb)
{
struct ctdb_vnn *vnn;
+ int count = 0;
for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
if (!ctdb_sys_have_ip(&vnn->public_address)) {
if (!vnn->iface) {
continue;
}
+
+ DEBUG(DEBUG_INFO,("Release of IP %s/%u on interface %s node:-1\n",
+ ctdb_addr_to_str(&vnn->public_address),
+ vnn->public_netmask_bits,
+ ctdb_vnn_iface_string(vnn)));
+
ctdb_event_script_args(ctdb, CTDB_EVENT_RELEASE_IP, "%s %s %u",
ctdb_vnn_iface_string(vnn),
ctdb_addr_to_str(&vnn->public_address),
vnn->public_netmask_bits);
release_kill_clients(ctdb, &vnn->public_address);
ctdb_vnn_unassign_iface(ctdb, vnn);
+ count++;
}
+
+ DEBUG(DEBUG_NOTICE,(__location__ " Released %d public IPs\n", count));
}
if (vnn->iface == cur) {
info->active_idx = i;
}
- strcpy(info->ifaces[i].name, cur->name);
+ strncpy(info->ifaces[i].name, cur->name, sizeof(info->ifaces[i].name)-1);
info->ifaces[i].link_state = cur->link_up;
info->ifaces[i].references = cur->references;
}
* list->tickles.num) {
DEBUG(DEBUG_ERR,("Bad indata in ctdb_control_set_tcp_tickle_list\n"));
return -1;
- }
+ }
vnn = find_public_ip_vnn(ctdb, &list->addr);
if (vnn == NULL) {
- DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
+ DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
ctdb_addr_to_str(&list->addr)));
return 1;
talloc_free(vnn->tcp_array);
vnn->tcp_array = NULL;
- tcparray = talloc(ctdb->nodes, struct ctdb_tcp_array);
+ tcparray = talloc(vnn, struct ctdb_tcp_array);
CTDB_NO_MEMORY(ctdb, tcparray);
tcparray->num = list->tickles.num;
tcparray->connections = talloc_array(tcparray, struct ctdb_tcp_connection, tcparray->num);
CTDB_NO_MEMORY(ctdb, tcparray->connections);
- memcpy(tcparray->connections, &list->tickles.connections[0],
+ memcpy(tcparray->connections, &list->tickles.connections[0],
sizeof(struct ctdb_tcp_connection)*tcparray->num);
/* We now have a new fresh tickle list array for this vnn */
- vnn->tcp_array = talloc_steal(vnn, tcparray);
-
+ vnn->tcp_array = tcparray;
+
return 0;
}
/*
set the list of all tcp tickles for a public address
*/
-static int ctdb_ctrl_set_tcp_tickles(struct ctdb_context *ctdb,
- struct timeval timeout, uint32_t destnode,
- ctdb_sock_addr *addr,
- struct ctdb_tcp_array *tcparray)
+static int ctdb_send_set_tcp_tickles_for_ip(struct ctdb_context *ctdb,
+ ctdb_sock_addr *addr,
+ struct ctdb_tcp_array *tcparray)
{
int ret, num;
TDB_DATA data;
memcpy(&list->tickles.connections[0], tcparray->connections, sizeof(struct ctdb_tcp_connection) * num);
}
- ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
+ ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, 0,
CTDB_CONTROL_SET_TCP_TICKLE_LIST,
0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
if (ret != 0) {
if (!vnn->tcp_update_needed) {
continue;
}
- ret = ctdb_ctrl_set_tcp_tickles(ctdb,
- TAKEOVER_TIMEOUT(),
- CTDB_BROADCAST_CONNECTED,
- &vnn->public_address,
- vnn->tcp_array);
+ ret = ctdb_send_set_tcp_tickles_for_ip(ctdb,
+ &vnn->public_address,
+ vnn->tcp_array);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to send the tickle update for public address %s\n",
ctdb_addr_to_str(&vnn->public_address)));
+ } else {
+ vnn->tcp_update_needed = false;
}
}
return -1;
}
- ret = ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0]);
+ DEBUG(DEBUG_NOTICE,("Add IP %s\n", ctdb_addr_to_str(&pub->addr)));
+
+ ret = ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0], true);
if (ret != 0) {
DEBUG(DEBUG_ERR,(__location__ " Failed to add public address\n"));
return 0;
}
+struct delete_ip_callback_state {
+ struct ctdb_req_control *c;
+};
+
/*
called when releaseip event finishes for del_public_address
*/
-static void delete_ip_callback(struct ctdb_context *ctdb, int status,
- void *private_data)
+static void delete_ip_callback(struct ctdb_context *ctdb,
+ int32_t status, TDB_DATA data,
+ const char *errormsg,
+ void *private_data)
{
+ struct delete_ip_callback_state *state =
+ talloc_get_type(private_data, struct delete_ip_callback_state);
+
+ /* If release failed then fail. */
+ ctdb_request_control_reply(ctdb, state->c, NULL, status, errormsg);
talloc_free(private_data);
}
-int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
+int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb,
+ struct ctdb_req_control *c,
+ TDB_DATA indata, bool *async_reply)
{
struct ctdb_control_ip_iface *pub = (struct ctdb_control_ip_iface *)indata.dptr;
struct ctdb_vnn *vnn;
- int ret;
/* verify the size of indata */
if (indata.dsize < offsetof(struct ctdb_control_ip_iface, iface)) {
return -1;
}
+ DEBUG(DEBUG_NOTICE,("Delete IP %s\n", ctdb_addr_to_str(&pub->addr)));
+
/* walk over all public addresses until we find a match */
for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
if (ctdb_same_ip(&vnn->public_address, &pub->addr)) {
- TALLOC_CTX *mem_ctx;
-
- DLIST_REMOVE(ctdb->vnn, vnn);
- if (vnn->pnn != ctdb->pnn) {
- if (vnn->iface != NULL) {
- ctdb_vnn_unassign_iface(ctdb, vnn);
+ if (vnn->pnn == ctdb->pnn) {
+ struct delete_ip_callback_state *state;
+ struct ctdb_public_ip *ip;
+ TDB_DATA data;
+ int ret;
+
+ vnn->delete_pending = true;
+
+ state = talloc(ctdb,
+ struct delete_ip_callback_state);
+ CTDB_NO_MEMORY(ctdb, state);
+ state->c = c;
+
+ ip = talloc(state, struct ctdb_public_ip);
+ if (ip == NULL) {
+ DEBUG(DEBUG_ERR,
+ (__location__ " Out of memory\n"));
+ talloc_free(state);
+ return -1;
+ }
+ ip->pnn = -1;
+ ip->addr = pub->addr;
+
+ data.dsize = sizeof(struct ctdb_public_ip);
+ data.dptr = (unsigned char *)ip;
+
+ ret = ctdb_daemon_send_control(ctdb,
+ ctdb_get_pnn(ctdb),
+ 0,
+ CTDB_CONTROL_RELEASE_IP,
+ 0, 0,
+ data,
+ delete_ip_callback,
+ state);
+ if (ret == -1) {
+ DEBUG(DEBUG_ERR,
+ (__location__ "Unable to send "
+ "CTDB_CONTROL_RELEASE_IP\n"));
+ talloc_free(state);
+ return -1;
}
- talloc_free(vnn);
- return 0;
- }
- vnn->pnn = -1;
- mem_ctx = talloc_new(ctdb);
- talloc_steal(mem_ctx, vnn);
- ret = ctdb_event_script_callback(ctdb,
- mem_ctx, delete_ip_callback, mem_ctx,
- false,
- CTDB_EVENT_RELEASE_IP,
- "%s %s %u",
- ctdb_vnn_iface_string(vnn),
- ctdb_addr_to_str(&vnn->public_address),
- vnn->public_netmask_bits);
- if (vnn->iface != NULL) {
- ctdb_vnn_unassign_iface(ctdb, vnn);
- }
- if (ret != 0) {
- return -1;
+ state->c = talloc_steal(state, c);
+ *async_reply = true;
+ } else {
+ /* This IP is not hosted on the
+ * current node so just delete it
+ * now. */
+ do_delete_ip(ctdb, vnn);
}
+
return 0;
}
}
+ DEBUG(DEBUG_ERR,("Delete IP of unknown public IP address %s\n",
+ ctdb_addr_to_str(&pub->addr)));
return -1;
}
+
+struct ipreallocated_callback_state {
+ struct ctdb_req_control *c;
+};
+
+static void ctdb_ipreallocated_callback(struct ctdb_context *ctdb,
+ int status, void *p)
+{
+ struct ipreallocated_callback_state *state =
+ talloc_get_type(p, struct ipreallocated_callback_state);
+
+ if (status != 0) {
+ DEBUG(DEBUG_ERR,
+ (" \"ipreallocated\" event script failed (status %d)\n",
+ status));
+ if (status == -ETIME) {
+ ctdb_ban_self(ctdb);
+ }
+ }
+
+ ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
+ talloc_free(state);
+}
+
+/* A control to run the ipreallocated event */
+int32_t ctdb_control_ipreallocated(struct ctdb_context *ctdb,
+ struct ctdb_req_control *c,
+ bool *async_reply)
+{
+ int ret;
+ struct ipreallocated_callback_state *state;
+
+ state = talloc(ctdb, struct ipreallocated_callback_state);
+ CTDB_NO_MEMORY(ctdb, state);
+
+ DEBUG(DEBUG_INFO,(__location__ " Running \"ipreallocated\" event\n"));
+
+ ret = ctdb_event_script_callback(ctdb, state,
+ ctdb_ipreallocated_callback, state,
+ CTDB_EVENT_IPREALLOCATED,
+ "%s", "");
+
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to run \"ipreallocated\" event \n"));
+ talloc_free(state);
+ return -1;
+ }
+
+ /* tell the control that we will be reply asynchronously */
+ state->c = talloc_steal(state, c);
+ *async_reply = true;
+
+ return 0;
+}
+
+
/* This function is called from the recovery daemon to verify that a remote
node has the expected ip allocation.
This is verified against ctdb->ip_tree
*/
-int verify_remote_ip_allocation(struct ctdb_context *ctdb, struct ctdb_all_public_ips *ips)
+int verify_remote_ip_allocation(struct ctdb_context *ctdb,
+ struct ctdb_all_public_ips *ips,
+ uint32_t pnn)
{
struct ctdb_public_ip_list *tmp_ip;
int i;
for (i=0; i<ips->num; i++) {
tmp_ip = trbt_lookuparray32(ctdb->ip_tree, IP_KEYLEN, ip_key(&ips->ips[i].addr));
if (tmp_ip == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Could not find host for address %s, reassign ips\n", ctdb_addr_to_str(&ips->ips[i].addr)));
+ DEBUG(DEBUG_ERR,("Node %u has new or unknown public IP %s\n", pnn, ctdb_addr_to_str(&ips->ips[i].addr)));
return -1;
}
}
if (tmp_ip->pnn != ips->ips[i].pnn) {
- DEBUG(DEBUG_ERR,("Inconsistent ip allocation. Trigger reallocation. Thinks %s is held by node %u while it is held by node %u\n", ctdb_addr_to_str(&ips->ips[i].addr), ips->ips[i].pnn, tmp_ip->pnn));
+ DEBUG(DEBUG_ERR,
+ ("Inconsistent IP allocation - node %u thinks %s is held by node %u while it is assigned to node %u\n",
+ pnn,
+ ctdb_addr_to_str(&ips->ips[i].addr),
+ ips->ips[i].pnn, tmp_ip->pnn));
return -1;
}
}
return 0;
}
+
+
+struct ctdb_reloadips_handle {
+ struct ctdb_context *ctdb;
+ struct ctdb_req_control *c;
+ int status;
+ int fd[2];
+ pid_t child;
+ struct fd_event *fde;
+};
+
+static int ctdb_reloadips_destructor(struct ctdb_reloadips_handle *h)
+{
+ if (h == h->ctdb->reload_ips) {
+ h->ctdb->reload_ips = NULL;
+ }
+ if (h->c != NULL) {
+ ctdb_request_control_reply(h->ctdb, h->c, NULL, h->status, NULL);
+ h->c = NULL;
+ }
+ ctdb_kill(h->ctdb, h->child, SIGKILL);
+ return 0;
+}
+
+static void ctdb_reloadips_timeout_event(struct event_context *ev,
+ struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
+
+ talloc_free(h);
+}
+
+static void ctdb_reloadips_child_handler(struct event_context *ev, struct fd_event *fde,
+ uint16_t flags, void *private_data)
+{
+ struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
+
+ char res;
+ int ret;
+
+ ret = read(h->fd[0], &res, 1);
+ if (ret < 1 || res != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Reloadips child process returned error\n"));
+ res = 1;
+ }
+ h->status = res;
+
+ talloc_free(h);
+}
+
+static int ctdb_reloadips_child(struct ctdb_context *ctdb)
+{
+ TALLOC_CTX *mem_ctx = talloc_new(NULL);
+ struct ctdb_all_public_ips *ips;
+ struct ctdb_vnn *vnn;
+ struct client_async_data *async_data;
+ struct timeval timeout;
+ TDB_DATA data;
+ struct ctdb_client_control_state *state;
+ bool first_add;
+ int i, ret;
+
+ CTDB_NO_MEMORY(ctdb, mem_ctx);
+
+ /* Read IPs from local node */
+ ret = ctdb_ctrl_get_public_ips(ctdb, TAKEOVER_TIMEOUT(),
+ CTDB_CURRENT_NODE, mem_ctx, &ips);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,
+ ("Unable to fetch public IPs from local node\n"));
+ talloc_free(mem_ctx);
+ return -1;
+ }
+
+ /* Read IPs file - this is safe since this is a child process */
+ ctdb->vnn = NULL;
+ if (ctdb_set_public_addresses(ctdb, false) != 0) {
+ DEBUG(DEBUG_ERR,("Failed to re-read public addresses file\n"));
+ talloc_free(mem_ctx);
+ return -1;
+ }
+
+ async_data = talloc_zero(mem_ctx, struct client_async_data);
+ CTDB_NO_MEMORY(ctdb, async_data);
+
+ /* Compare IPs between node and file for IPs to be deleted */
+ for (i = 0; i < ips->num; i++) {
+ /* */
+ for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
+ if (ctdb_same_ip(&vnn->public_address,
+ &ips->ips[i].addr)) {
+ /* IP is still in file */
+ break;
+ }
+ }
+
+ if (vnn == NULL) {
+ /* Delete IP ips->ips[i] */
+ struct ctdb_control_ip_iface *pub;
+
+ DEBUG(DEBUG_NOTICE,
+ ("IP %s no longer configured, deleting it\n",
+ ctdb_addr_to_str(&ips->ips[i].addr)));
+
+ pub = talloc_zero(mem_ctx,
+ struct ctdb_control_ip_iface);
+ CTDB_NO_MEMORY(ctdb, pub);
+
+ pub->addr = ips->ips[i].addr;
+ pub->mask = 0;
+ pub->len = 0;
+
+ timeout = TAKEOVER_TIMEOUT();
+
+ data.dsize = offsetof(struct ctdb_control_ip_iface,
+ iface) + pub->len;
+ data.dptr = (uint8_t *)pub;
+
+ state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
+ CTDB_CONTROL_DEL_PUBLIC_IP,
+ 0, data, async_data,
+ &timeout, NULL);
+ if (state == NULL) {
+ DEBUG(DEBUG_ERR,
+ (__location__
+ " failed sending CTDB_CONTROL_DEL_PUBLIC_IP\n"));
+ goto failed;
+ }
+
+ ctdb_client_async_add(async_data, state);
+ }
+ }
+
+ /* Compare IPs between node and file for IPs to be added */
+ first_add = true;
+ for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
+ for (i = 0; i < ips->num; i++) {
+ if (ctdb_same_ip(&vnn->public_address,
+ &ips->ips[i].addr)) {
+ /* IP already on node */
+ break;
+ }
+ }
+ if (i == ips->num) {
+ /* Add IP ips->ips[i] */
+ struct ctdb_control_ip_iface *pub;
+ const char *ifaces = NULL;
+ uint32_t len;
+ int iface = 0;
+
+ DEBUG(DEBUG_NOTICE,
+ ("New IP %s configured, adding it\n",
+ ctdb_addr_to_str(&vnn->public_address)));
+ if (first_add) {
+ uint32_t pnn = ctdb_get_pnn(ctdb);
+
+ data.dsize = sizeof(pnn);
+ data.dptr = (uint8_t *)&pnn;
+
+ ret = ctdb_client_send_message(
+ ctdb,
+ CTDB_BROADCAST_CONNECTED,
+ CTDB_SRVID_REBALANCE_NODE,
+ data);
+ if (ret != 0) {
+ DEBUG(DEBUG_WARNING,
+ ("Failed to send message to force node reallocation - IPs may be unbalanced\n"));
+ }
+
+ first_add = false;
+ }
+
+ ifaces = vnn->ifaces[0];
+ iface = 1;
+ while (vnn->ifaces[iface] != NULL) {
+ ifaces = talloc_asprintf(vnn, "%s,%s", ifaces,
+ vnn->ifaces[iface]);
+ iface++;
+ }
+
+ len = strlen(ifaces) + 1;
+ pub = talloc_zero_size(mem_ctx,
+ offsetof(struct ctdb_control_ip_iface, iface) + len);
+ CTDB_NO_MEMORY(ctdb, pub);
+
+ pub->addr = vnn->public_address;
+ pub->mask = vnn->public_netmask_bits;
+ pub->len = len;
+ memcpy(&pub->iface[0], ifaces, pub->len);
+
+ timeout = TAKEOVER_TIMEOUT();
+
+ data.dsize = offsetof(struct ctdb_control_ip_iface,
+ iface) + pub->len;
+ data.dptr = (uint8_t *)pub;
+
+ state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
+ CTDB_CONTROL_ADD_PUBLIC_IP,
+ 0, data, async_data,
+ &timeout, NULL);
+ if (state == NULL) {
+ DEBUG(DEBUG_ERR,
+ (__location__
+ " failed sending CTDB_CONTROL_ADD_PUBLIC_IP\n"));
+ goto failed;
+ }
+
+ ctdb_client_async_add(async_data, state);
+ }
+ }
+
+ if (ctdb_client_async_wait(ctdb, async_data) != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Add/delete IPs failed\n"));
+ goto failed;
+ }
+
+ talloc_free(mem_ctx);
+ return 0;
+
+failed:
+ talloc_free(mem_ctx);
+ return -1;
+}
+
+/* This control is sent to force the node to re-read the public addresses file
+ and drop any addresses we should nnot longer host, and add new addresses
+ that we are now able to host
+*/
+int32_t ctdb_control_reload_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
+{
+ struct ctdb_reloadips_handle *h;
+ pid_t parent = getpid();
+
+ if (ctdb->reload_ips != NULL) {
+ talloc_free(ctdb->reload_ips);
+ ctdb->reload_ips = NULL;
+ }
+
+ h = talloc(ctdb, struct ctdb_reloadips_handle);
+ CTDB_NO_MEMORY(ctdb, h);
+ h->ctdb = ctdb;
+ h->c = NULL;
+ h->status = -1;
+
+ if (pipe(h->fd) == -1) {
+ DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
+ talloc_free(h);
+ return -1;
+ }
+
+ h->child = ctdb_fork(ctdb);
+ if (h->child == (pid_t)-1) {
+ DEBUG(DEBUG_ERR, ("Failed to fork a child for reloadips\n"));
+ close(h->fd[0]);
+ close(h->fd[1]);
+ talloc_free(h);
+ return -1;
+ }
+
+ /* child process */
+ if (h->child == 0) {
+ signed char res = 0;
+
+ close(h->fd[0]);
+ debug_extra = talloc_asprintf(NULL, "reloadips:");
+
+ ctdb_set_process_name("ctdb_reloadips");
+ if (switch_from_server_to_client(ctdb, "reloadips-child") != 0) {
+ DEBUG(DEBUG_CRIT,("ERROR: Failed to switch reloadips child into client mode\n"));
+ res = -1;
+ } else {
+ res = ctdb_reloadips_child(ctdb);
+ if (res != 0) {
+ DEBUG(DEBUG_ERR,("Failed to reload ips on local node\n"));
+ }
+ }
+
+ write(h->fd[1], &res, 1);
+ /* make sure we die when our parent dies */
+ while (ctdb_kill(ctdb, parent, 0) == 0 || errno != ESRCH) {
+ sleep(5);
+ }
+ _exit(0);
+ }
+
+ h->c = talloc_steal(h, c);
+
+ close(h->fd[1]);
+ set_close_on_exec(h->fd[0]);
+
+ talloc_set_destructor(h, ctdb_reloadips_destructor);
+
+
+ h->fde = event_add_fd(ctdb->ev, h, h->fd[0],
+ EVENT_FD_READ, ctdb_reloadips_child_handler,
+ (void *)h);
+ tevent_fd_set_auto_close(h->fde);
+
+ event_add_timed(ctdb->ev, h,
+ timeval_current_ofs(120, 0),
+ ctdb_reloadips_timeout_event, h);
+
+ /* we reply later */
+ *async_reply = true;
+ return 0;
+}
git.samba.org / garming / samba-autobuild / .git / blobdiff