int ctdb_set_event_script(struct ctdb_context *ctdb, const char *script);
int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char *script_dir);
int ctdb_set_notification_script(struct ctdb_context *ctdb, const char *script);
-void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn);
-int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, client_async_callback fail_callback, void *callback_data);
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
+ uint32_t *force_rebalance_nodes,
+ client_async_callback fail_callback, void *callback_data);
int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
TDB_DATA indata);
bool takeover_run_in_progress;
TALLOC_CTX *takeover_runs_disable_ctx;
struct ctdb_control_get_ifaces *ifaces;
- TALLOC_CTX *deferred_rebalance_ctx;
+ uint32_t *force_rebalance_nodes;
};
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
}
}
- ret = ctdb_takeover_run(rec->ctdb, nodemap, takeover_fail_callback,
+ ret = ctdb_takeover_run(rec->ctdb, nodemap,
+ rec->force_rebalance_nodes,
+ takeover_fail_callback,
banning_credits_on_fail ? rec : NULL);
/* Reenable takeover runs and IP checks on other nodes */
}
ok = true;
+ /* Takeover run was successful so clear force rebalance targets */
+ TALLOC_FREE(rec->force_rebalance_nodes);
done:
rec->need_takeover_run = !ok;
talloc_free(nodes);
}
-static void ctdb_rebalance_timeout(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *p)
+static void ctdb_rebalance_timeout(struct event_context *ev,
+ struct timed_event *te,
+ struct timeval t, void *p)
{
struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
- DEBUG(DEBUG_NOTICE,
- ("Rebalance all nodes that have had ip assignment changes.\n"));
+ if (rec->force_rebalance_nodes == NULL) {
+ DEBUG(DEBUG_ERR,
+ ("Rebalance timeout occurred - no nodes to rebalance\n"));
+ return;
+ }
+ DEBUG(DEBUG_NOTICE,
+ ("Rebalance timeout occurred - do takeover run\n"));
do_takeover_run(rec, rec->nodemap, false);
-
- talloc_free(rec->deferred_rebalance_ctx);
- rec->deferred_rebalance_ctx = NULL;
}
-static void recd_node_rebalance_handler(struct ctdb_context *ctdb, uint64_t srvid,
- TDB_DATA data, void *private_data)
+static void recd_node_rebalance_handler(struct ctdb_context *ctdb,
+ uint64_t srvid,
+ TDB_DATA data, void *private_data)
{
uint32_t pnn;
+ uint32_t *t;
+ int len;
struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+ if (rec->recmaster != ctdb_get_pnn(ctdb)) {
+ return;
+ }
+
if (data.dsize != sizeof(uint32_t)) {
DEBUG(DEBUG_ERR,(__location__ " Incorrect size of node rebalance message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(uint32_t)));
return;
pnn = *(uint32_t *)&data.dptr[0];
- lcp2_forcerebalance(ctdb, pnn);
- DEBUG(DEBUG_NOTICE,("Received message to perform node rebalancing for node %d\n", pnn));
+ DEBUG(DEBUG_NOTICE,("Setting up rebalance of IPs to node %u\n", pnn));
- if (rec->deferred_rebalance_ctx != NULL) {
- talloc_free(rec->deferred_rebalance_ctx);
+ /* Copy any existing list of nodes. There's probably some
+ * sort of realloc variant that will do this but we need to
+ * make sure that freeing the old array also cancels the timer
+ * event for the timeout... not sure if realloc will do that.
+ */
+ len = (rec->force_rebalance_nodes != NULL) ?
+ talloc_array_length(rec->force_rebalance_nodes) :
+ 0;
+
+ /* This allows duplicates to be added but they don't cause
+ * harm. A call to add a duplicate PNN arguably means that
+ * the timeout should be reset, so this is the simplest
+ * solution.
+ */
+ t = talloc_zero_array(rec, uint32_t, len+1);
+ CTDB_NO_MEMORY_VOID(ctdb, t);
+ if (len > 0) {
+ memcpy(t, rec->force_rebalance_nodes, sizeof(uint32_t) * len);
}
- rec->deferred_rebalance_ctx = talloc_new(rec);
- event_add_timed(ctdb->ev, rec->deferred_rebalance_ctx,
+ t[len] = pnn;
+
+ talloc_free(rec->force_rebalance_nodes);
+
+ rec->force_rebalance_nodes = t;
+ event_add_timed(ctdb->ev, rec->force_rebalance_nodes,
timeval_current_ofs(ctdb->tunable.deferred_rebalance_on_node_add, 0),
ctdb_rebalance_timeout, rec);
}
return;
}
- /* if we are not the recmaster we can safely ignore any ip reallocate requests */
+ /* If we are not the recmaster then do some housekeeping */
if (rec->recmaster != pnn) {
+ /* Ignore any IP reallocate requests - only recmaster
+ * processes them
+ */
TALLOC_FREE(rec->reallocate_requests);
+ /* Clear any nodes that should be force rebalanced in
+ * the next takeover run. If the recovery master role
+ * has moved then we don't want to process these some
+ * time in the future.
+ */
+ TALLOC_FREE(rec->force_rebalance_nodes);
}
/* This is a special case. When recovery daemon is started, recmaster
}
}
-struct ctdb_rebalancenodes {
- struct ctdb_rebalancenodes *next;
- uint32_t pnn;
-};
-static struct ctdb_rebalancenodes *force_rebalance_list = NULL;
-
-
-/* set this flag to force the node to be rebalanced even if it just didnt
- become healthy again.
-*/
-void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn)
-{
- struct ctdb_rebalancenodes *rebalance;
-
- for (rebalance = force_rebalance_list; rebalance; rebalance = rebalance->next) {
- if (rebalance->pnn == pnn) {
- return;
- }
- }
-
- rebalance = talloc(ctdb, struct ctdb_rebalancenodes);
- rebalance->pnn = pnn;
- rebalance->next = force_rebalance_list;
- force_rebalance_list = rebalance;
-}
-
-/* Do necessary LCP2 initialisation. Bury it in a function here so
- * that we can unit test it.
- */
static void lcp2_init(struct ctdb_context *tmp_ctx,
struct ctdb_ipflags *ipflags,
struct ctdb_public_ip_list *all_ips,
+ uint32_t *force_rebalance_nodes,
uint32_t **lcp2_imbalances,
bool **rebalance_candidates)
{
/* 3rd step: if a node is forced to re-balance then
we allow failback onto the node */
- while (force_rebalance_list != NULL) {
- struct ctdb_rebalancenodes *next = force_rebalance_list->next;
-
- if (force_rebalance_list->pnn <= numnodes) {
- (*rebalance_candidates)[force_rebalance_list->pnn] = true;
+ if (force_rebalance_nodes == NULL) {
+ return;
+ }
+ for (i = 0; i < talloc_array_length(force_rebalance_nodes); i++) {
+ uint32_t pnn = force_rebalance_nodes[i];
+ if (pnn >= numnodes) {
+ DEBUG(DEBUG_ERR,
+ (__location__ "unknown node %u\n", pnn));
+ continue;
}
- DEBUG(DEBUG_ERR,("During ipreallocation, forced rebalance of node %d\n", force_rebalance_list->pnn));
- talloc_free(force_rebalance_list);
- force_rebalance_list = next;
+ DEBUG(DEBUG_NOTICE,
+ ("Forcing rebalancing of IPs to node %u\n", pnn));
+ (*rebalance_candidates)[pnn] = true;
}
}
static void ip_alloc_lcp2(struct ctdb_context *ctdb,
struct ctdb_ipflags *ipflags,
- struct ctdb_public_ip_list *all_ips)
+ struct ctdb_public_ip_list *all_ips,
+ uint32_t *force_rebalance_nodes)
{
uint32_t *lcp2_imbalances;
bool *rebalance_candidates;
unassign_unsuitable_ips(ctdb, ipflags, all_ips);
- lcp2_init(tmp_ctx, ipflags, all_ips,
+ lcp2_init(tmp_ctx, ipflags, all_ips,force_rebalance_nodes,
&lcp2_imbalances, &rebalance_candidates);
lcp2_allocate_unassigned(ctdb, ipflags, all_ips, lcp2_imbalances);
/* The calculation part of the IP allocation algorithm. */
static void ctdb_takeover_run_core(struct ctdb_context *ctdb,
struct ctdb_ipflags *ipflags,
- struct ctdb_public_ip_list **all_ips_p)
+ struct ctdb_public_ip_list **all_ips_p,
+ uint32_t *force_rebalance_nodes)
{
/* since nodes only know about those public addresses that
can be served by that particular node, no single node has
*all_ips_p = create_merged_ip_list(ctdb);
if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
- ip_alloc_lcp2(ctdb, ipflags, *all_ips_p);
+ ip_alloc_lcp2(ctdb, ipflags, *all_ips_p, force_rebalance_nodes);
} else if (1 == ctdb->tunable.deterministic_public_ips) {
ip_alloc_deterministic_ips(ctdb, ipflags, *all_ips_p);
} else {
make any IP alias changes for public addresses that are necessary
*/
int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
+ uint32_t *force_rebalance_nodes,
client_async_callback fail_callback, void *callback_data)
{
int i, j, ret;
ZERO_STRUCT(ip);
/* Do the IP reassignment calculations */
- ctdb_takeover_run_core(ctdb, ipflags, &all_ips);
+ ctdb_takeover_run_core(ctdb, ipflags, &all_ips, force_rebalance_nodes);
/* Now tell all nodes to release any public IPs should not
* host. This will be a NOOP on nodes that don't currently
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
- lcp2_init(ctdb, ipflags, all_ips, &lcp2_imbalances, &newly_healthy);
+ lcp2_init(ctdb, ipflags, all_ips, NULL,
+ &lcp2_imbalances, &newly_healthy);
lcp2_allocate_unassigned(ctdb, ipflags,
all_ips, lcp2_imbalances);
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
- lcp2_init(ctdb, ipflags, all_ips, &lcp2_imbalances, &newly_healthy);
+ lcp2_init(ctdb, ipflags, all_ips, NULL,
+ &lcp2_imbalances, &newly_healthy);
lcp2_failback(ctdb, ipflags,
all_ips, lcp2_imbalances, newly_healthy);
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
- lcp2_init(ctdb, ipflags, all_ips, &lcp2_imbalances, &newly_healthy);
+ lcp2_init(ctdb, ipflags, all_ips, NULL,
+ &lcp2_imbalances, &newly_healthy);
lcp2_failback(ctdb, ipflags,
all_ips, lcp2_imbalances, newly_healthy);
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags,
read_ips_for_multiple_nodes);
- ctdb_takeover_run_core(ctdb, ipflags, &all_ips);
+ ctdb_takeover_run_core(ctdb, ipflags, &all_ips, NULL);
print_ctdb_public_ip_list(all_ips);