The current implementation has a few flaws:
* A takeover run is called unconditionally when the timer goes even if
the recovery master role has moved. This means a node other than
the recovery master can incorrectly do a takeover run.
* The rebalancing target nodes are cleared in the setup for a takeover
run, regardless of whether the takeover run succeeds.
* The timer to force a rebalance isn't cleared if another takeover run
occurs before the deadline. Any forced rebalancing will happen in
the first takeover run and when the timer expires some time later
then an unnecessary takeover run will occur.
* If the recovery master role moves then the rebalancing data will
stay on the original node and affect the next takeover run to occur
if the recovery master role should come back to the original node.
Instead, store an array of rebalance target nodes in the recovery
master context. This is passed as an extra argument to
ctdb_takeover_run() each time it is called and is cleared when a
takeover run succeeds. The timer hangs off the array of rebalance
target nodes, which is cleared if the node isn't the recovery master.
This means that it is possible to lose rebalance data if the recovery
master role moves. However, that's a difficult problem to solve. The
best way of approaching it is probably to try to stop the recovery
master role from jumping around unnecesarily when inactive nodes join
the cluster.
The long term solution is to avoid this nonsense completely. The IP
allocation algorithm needs to cache state between runs so that it
knows which nodes have just become healthy. This also needs recovery
master stability.
Signed-off-by: Martin Schwenke <martin@meltin.net>
(This used to be ctdb commit
c51c1efe5fc7fa668597f2acd435dee16e410fc9)
int ctdb_set_event_script(struct ctdb_context *ctdb, const char *script);
int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char *script_dir);
int ctdb_set_notification_script(struct ctdb_context *ctdb, const char *script);
-void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn);
-int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, client_async_callback fail_callback, void *callback_data);
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
+ uint32_t *force_rebalance_nodes,
+ client_async_callback fail_callback, void *callback_data);
int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
TDB_DATA indata);
bool takeover_run_in_progress;
TALLOC_CTX *takeover_runs_disable_ctx;
struct ctdb_control_get_ifaces *ifaces;
- TALLOC_CTX *deferred_rebalance_ctx;
+ uint32_t *force_rebalance_nodes;
};
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
}
}
- ret = ctdb_takeover_run(rec->ctdb, nodemap, takeover_fail_callback,
+ ret = ctdb_takeover_run(rec->ctdb, nodemap,
+ rec->force_rebalance_nodes,
+ takeover_fail_callback,
banning_credits_on_fail ? rec : NULL);
/* Reenable takeover runs and IP checks on other nodes */
}
ok = true;
+ /* Takeover run was successful so clear force rebalance targets */
+ TALLOC_FREE(rec->force_rebalance_nodes);
done:
rec->need_takeover_run = !ok;
talloc_free(nodes);
}
-static void ctdb_rebalance_timeout(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *p)
+static void ctdb_rebalance_timeout(struct event_context *ev,
+ struct timed_event *te,
+ struct timeval t, void *p)
{
struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
- DEBUG(DEBUG_NOTICE,
- ("Rebalance all nodes that have had ip assignment changes.\n"));
+ if (rec->force_rebalance_nodes == NULL) {
+ DEBUG(DEBUG_ERR,
+ ("Rebalance timeout occurred - no nodes to rebalance\n"));
+ return;
+ }
+ DEBUG(DEBUG_NOTICE,
+ ("Rebalance timeout occurred - do takeover run\n"));
do_takeover_run(rec, rec->nodemap, false);
-
- talloc_free(rec->deferred_rebalance_ctx);
- rec->deferred_rebalance_ctx = NULL;
}
-static void recd_node_rebalance_handler(struct ctdb_context *ctdb, uint64_t srvid,
- TDB_DATA data, void *private_data)
+static void recd_node_rebalance_handler(struct ctdb_context *ctdb,
+ uint64_t srvid,
+ TDB_DATA data, void *private_data)
{
uint32_t pnn;
+ uint32_t *t;
+ int len;
struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+ if (rec->recmaster != ctdb_get_pnn(ctdb)) {
+ return;
+ }
+
if (data.dsize != sizeof(uint32_t)) {
DEBUG(DEBUG_ERR,(__location__ " Incorrect size of node rebalance message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(uint32_t)));
return;
pnn = *(uint32_t *)&data.dptr[0];
- lcp2_forcerebalance(ctdb, pnn);
- DEBUG(DEBUG_NOTICE,("Received message to perform node rebalancing for node %d\n", pnn));
+ DEBUG(DEBUG_NOTICE,("Setting up rebalance of IPs to node %u\n", pnn));
- if (rec->deferred_rebalance_ctx != NULL) {
- talloc_free(rec->deferred_rebalance_ctx);
+ /* Copy any existing list of nodes. There's probably some
+ * sort of realloc variant that will do this but we need to
+ * make sure that freeing the old array also cancels the timer
+ * event for the timeout... not sure if realloc will do that.
+ */
+ len = (rec->force_rebalance_nodes != NULL) ?
+ talloc_array_length(rec->force_rebalance_nodes) :
+ 0;
+
+ /* This allows duplicates to be added but they don't cause
+ * harm. A call to add a duplicate PNN arguably means that
+ * the timeout should be reset, so this is the simplest
+ * solution.
+ */
+ t = talloc_zero_array(rec, uint32_t, len+1);
+ CTDB_NO_MEMORY_VOID(ctdb, t);
+ if (len > 0) {
+ memcpy(t, rec->force_rebalance_nodes, sizeof(uint32_t) * len);
}
- rec->deferred_rebalance_ctx = talloc_new(rec);
- event_add_timed(ctdb->ev, rec->deferred_rebalance_ctx,
+ t[len] = pnn;
+
+ talloc_free(rec->force_rebalance_nodes);
+
+ rec->force_rebalance_nodes = t;
+ event_add_timed(ctdb->ev, rec->force_rebalance_nodes,
timeval_current_ofs(ctdb->tunable.deferred_rebalance_on_node_add, 0),
ctdb_rebalance_timeout, rec);
}
return;
}
- /* if we are not the recmaster we can safely ignore any ip reallocate requests */
+ /* If we are not the recmaster then do some housekeeping */
if (rec->recmaster != pnn) {
+ /* Ignore any IP reallocate requests - only recmaster
+ * processes them
+ */
TALLOC_FREE(rec->reallocate_requests);
+ /* Clear any nodes that should be force rebalanced in
+ * the next takeover run. If the recovery master role
+ * has moved then we don't want to process these some
+ * time in the future.
+ */
+ TALLOC_FREE(rec->force_rebalance_nodes);
}
/* This is a special case. When recovery daemon is started, recmaster
}
}
-struct ctdb_rebalancenodes {
- struct ctdb_rebalancenodes *next;
- uint32_t pnn;
-};
-static struct ctdb_rebalancenodes *force_rebalance_list = NULL;
-
-
-/* set this flag to force the node to be rebalanced even if it just didnt
- become healthy again.
-*/
-void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn)
-{
- struct ctdb_rebalancenodes *rebalance;
-
- for (rebalance = force_rebalance_list; rebalance; rebalance = rebalance->next) {
- if (rebalance->pnn == pnn) {
- return;
- }
- }
-
- rebalance = talloc(ctdb, struct ctdb_rebalancenodes);
- rebalance->pnn = pnn;
- rebalance->next = force_rebalance_list;
- force_rebalance_list = rebalance;
-}
-
-/* Do necessary LCP2 initialisation. Bury it in a function here so
- * that we can unit test it.
- */
static void lcp2_init(struct ctdb_context *tmp_ctx,
struct ctdb_ipflags *ipflags,
struct ctdb_public_ip_list *all_ips,
+ uint32_t *force_rebalance_nodes,
uint32_t **lcp2_imbalances,
bool **rebalance_candidates)
{
/* 3rd step: if a node is forced to re-balance then
we allow failback onto the node */
- while (force_rebalance_list != NULL) {
- struct ctdb_rebalancenodes *next = force_rebalance_list->next;
-
- if (force_rebalance_list->pnn <= numnodes) {
- (*rebalance_candidates)[force_rebalance_list->pnn] = true;
+ if (force_rebalance_nodes == NULL) {
+ return;
+ }
+ for (i = 0; i < talloc_array_length(force_rebalance_nodes); i++) {
+ uint32_t pnn = force_rebalance_nodes[i];
+ if (pnn >= numnodes) {
+ DEBUG(DEBUG_ERR,
+ (__location__ "unknown node %u\n", pnn));
+ continue;
}
- DEBUG(DEBUG_ERR,("During ipreallocation, forced rebalance of node %d\n", force_rebalance_list->pnn));
- talloc_free(force_rebalance_list);
- force_rebalance_list = next;
+ DEBUG(DEBUG_NOTICE,
+ ("Forcing rebalancing of IPs to node %u\n", pnn));
+ (*rebalance_candidates)[pnn] = true;
}
}
static void ip_alloc_lcp2(struct ctdb_context *ctdb,
struct ctdb_ipflags *ipflags,
- struct ctdb_public_ip_list *all_ips)
+ struct ctdb_public_ip_list *all_ips,
+ uint32_t *force_rebalance_nodes)
{
uint32_t *lcp2_imbalances;
bool *rebalance_candidates;
unassign_unsuitable_ips(ctdb, ipflags, all_ips);
- lcp2_init(tmp_ctx, ipflags, all_ips,
+ lcp2_init(tmp_ctx, ipflags, all_ips,force_rebalance_nodes,
&lcp2_imbalances, &rebalance_candidates);
lcp2_allocate_unassigned(ctdb, ipflags, all_ips, lcp2_imbalances);
/* The calculation part of the IP allocation algorithm. */
static void ctdb_takeover_run_core(struct ctdb_context *ctdb,
struct ctdb_ipflags *ipflags,
- struct ctdb_public_ip_list **all_ips_p)
+ struct ctdb_public_ip_list **all_ips_p,
+ uint32_t *force_rebalance_nodes)
{
/* since nodes only know about those public addresses that
can be served by that particular node, no single node has
*all_ips_p = create_merged_ip_list(ctdb);
if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
- ip_alloc_lcp2(ctdb, ipflags, *all_ips_p);
+ ip_alloc_lcp2(ctdb, ipflags, *all_ips_p, force_rebalance_nodes);
} else if (1 == ctdb->tunable.deterministic_public_ips) {
ip_alloc_deterministic_ips(ctdb, ipflags, *all_ips_p);
} else {
make any IP alias changes for public addresses that are necessary
*/
int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
+ uint32_t *force_rebalance_nodes,
client_async_callback fail_callback, void *callback_data)
{
int i, j, ret;
ZERO_STRUCT(ip);
/* Do the IP reassignment calculations */
- ctdb_takeover_run_core(ctdb, ipflags, &all_ips);
+ ctdb_takeover_run_core(ctdb, ipflags, &all_ips, force_rebalance_nodes);
/* Now tell all nodes to release any public IPs should not
* host. This will be a NOOP on nodes that don't currently
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
- lcp2_init(ctdb, ipflags, all_ips, &lcp2_imbalances, &newly_healthy);
+ lcp2_init(ctdb, ipflags, all_ips, NULL,
+ &lcp2_imbalances, &newly_healthy);
lcp2_allocate_unassigned(ctdb, ipflags,
all_ips, lcp2_imbalances);
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
- lcp2_init(ctdb, ipflags, all_ips, &lcp2_imbalances, &newly_healthy);
+ lcp2_init(ctdb, ipflags, all_ips, NULL,
+ &lcp2_imbalances, &newly_healthy);
lcp2_failback(ctdb, ipflags,
all_ips, lcp2_imbalances, newly_healthy);
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
- lcp2_init(ctdb, ipflags, all_ips, &lcp2_imbalances, &newly_healthy);
+ lcp2_init(ctdb, ipflags, all_ips, NULL,
+ &lcp2_imbalances, &newly_healthy);
lcp2_failback(ctdb, ipflags,
all_ips, lcp2_imbalances, newly_healthy);
ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags,
read_ips_for_multiple_nodes);
- ctdb_takeover_run_core(ctdb, ipflags, &all_ips);
+ ctdb_takeover_run_core(ctdb, ipflags, &all_ips, NULL);
print_ctdb_public_ip_list(all_ips);
git.samba.org / kai / samba-autobuild / .git / commitdiff