protocol design and packet details
*/
#include "includes.h"
-#include "lib/tevent/tevent.h"
-#include "lib/tdb/include/tdb.h"
+#include "tdb.h"
#include "lib/util/dlinklist.h"
#include "system/network.h"
#include "system/filesys.h"
#include "../include/ctdb_private.h"
+#include "../common/rb_tree.h"
+
+struct ctdb_sticky_record {
+ struct ctdb_context *ctdb;
+ struct ctdb_db_context *ctdb_db;
+ TDB_CONTEXT *pindown;
+};
/*
find the ctdb_db from a db index
* to its local ctdb (ctdb_request_call). If the node is not itself
* the record's DMASTER, it first redirects the packet to the
* record's LMASTER. The LMASTER then redirects the call packet to
- * the current DMASTER. But there is a race: The record may have
- * been migrated off the DMASTER while the redirected packet is
- * on the wire (or in the local queue). So in case the record has
- * migrated off the new destinaton of the call packet, instead of
- * going back to the LMASTER to get the new DMASTER, we try to
- * reduce rountrips by first chasing the record a couple of times
- * before giving up the direct chase and finally going back to the
- * LMASTER (again). Note that this works because auf this: When
+ * the current DMASTER. Note that this works because of this: When
* a record is migrated off a node, then the new DMASTER is stored
* in the record's copy on the former DMASTER.
- *
- * The maximum number of attempts for direct chase to make before
- * going back to the LMASTER is configurable by the tunable
- * "MaxRedirectCount".
*/
-static void ctdb_call_send_redirect(struct ctdb_context *ctdb,
+static void ctdb_call_send_redirect(struct ctdb_context *ctdb,
+ struct ctdb_db_context *ctdb_db,
TDB_DATA key,
struct ctdb_req_call *c,
struct ctdb_ltdb_header *header)
{
-
uint32_t lmaster = ctdb_lmaster(ctdb, &key);
+
+ c->hdr.destnode = lmaster;
if (ctdb->pnn == lmaster) {
c->hdr.destnode = header->dmaster;
- } else if ((c->hopcount % ctdb->tunable.max_redirect_count) == 0) {
- c->hdr.destnode = lmaster;
- } else {
- c->hdr.destnode = header->dmaster;
}
c->hopcount++;
+
+ if (c->hopcount%100 > 95) {
+ DEBUG(DEBUG_WARNING,("High hopcount %d dbid:%s "
+ "key:0x%08x reqid=%08x pnn:%d src:%d lmaster:%d "
+ "header->dmaster:%d dst:%d\n",
+ c->hopcount, ctdb_db->db_name, ctdb_hash(&key),
+ c->hdr.reqid, ctdb->pnn, c->hdr.srcnode, lmaster,
+ header->dmaster, c->hdr.destnode));
+ }
+
ctdb_queue_packet(ctdb, &c->hdr);
}
talloc_free(r);
}
+static void ctdb_sticky_pindown_timeout(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_sticky_record *sr = talloc_get_type(private_data,
+ struct ctdb_sticky_record);
+
+ DEBUG(DEBUG_ERR,("Pindown timeout db:%s unstick record\n", sr->ctdb_db->db_name));
+ if (sr->pindown != NULL) {
+ talloc_free(sr->pindown);
+ sr->pindown = NULL;
+ }
+}
+
+static int
+ctdb_set_sticky_pindown(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key)
+{
+ TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+ uint32_t *k;
+ struct ctdb_sticky_record *sr;
+
+ k = talloc_zero_size(tmp_ctx, ((key.dsize + 3) & 0xfffffffc) + 4);
+ if (k == NULL) {
+ DEBUG(DEBUG_ERR,("Failed to allocate key for sticky record\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ k[0] = (key.dsize + 3) / 4 + 1;
+ memcpy(&k[1], key.dptr, key.dsize);
+
+ sr = trbt_lookuparray32(ctdb_db->sticky_records, k[0], &k[0]);
+ if (sr == NULL) {
+ talloc_free(tmp_ctx);
+ return 0;
+ }
+
+ talloc_free(tmp_ctx);
+
+ if (sr->pindown == NULL) {
+ DEBUG(DEBUG_ERR,("Pinning down record in %s for %d ms\n", ctdb_db->db_name, ctdb->tunable.sticky_pindown));
+ sr->pindown = talloc_new(sr);
+ if (sr->pindown == NULL) {
+ DEBUG(DEBUG_ERR,("Failed to allocate pindown context for sticky record\n"));
+ return -1;
+ }
+ event_add_timed(ctdb->ev, sr->pindown, timeval_current_ofs(ctdb->tunable.sticky_pindown / 1000, (ctdb->tunable.sticky_pindown * 1000) % 1000000), ctdb_sticky_pindown_timeout, sr);
+ }
+
+ return 0;
+}
+
/*
called when a CTDB_REPLY_DMASTER packet comes in, or when the lmaster
gets a CTDB_REQUEST_DMASTER for itself. We become the dmaster.
DEBUG(DEBUG_DEBUG,("pnn %u dmaster response %08x\n", ctdb->pnn, ctdb_hash(&key)));
ZERO_STRUCT(header);
- header.rsn = rsn + 1;
+ header.rsn = rsn;
header.dmaster = ctdb->pnn;
header.flags = record_flags;
return;
}
+ /* we just became DMASTER and this database is "sticky",
+ see if the record is flagged as "hot" and set up a pin-down
+ context to stop migrations for a little while if so
+ */
+ if (ctdb_db->sticky) {
+ ctdb_set_sticky_pindown(ctdb, ctdb_db, key);
+ }
if (state == NULL) {
DEBUG(DEBUG_ERR,("pnn %u Invalid reqid %u in ctdb_become_dmaster from node %u\n",
/* fetch the current record */
ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, key, &header, hdr, &data2,
- ctdb_call_input_pkt, ctdb, False);
+ ctdb_call_input_pkt, ctdb, false);
if (ret == -1) {
ctdb_fatal(ctdb, "ctdb_req_dmaster failed to fetch record");
return;
}
}
+static void ctdb_sticky_record_timeout(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_sticky_record *sr = talloc_get_type(private_data,
+ struct ctdb_sticky_record);
+ talloc_free(sr);
+}
+
+static void *ctdb_make_sticky_record_callback(void *parm, void *data)
+{
+ if (data) {
+ DEBUG(DEBUG_ERR,("Already have sticky record registered. Free old %p and create new %p\n", data, parm));
+ talloc_free(data);
+ }
+ return parm;
+}
+
+static int
+ctdb_make_record_sticky(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key)
+{
+ TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+ uint32_t *k;
+ struct ctdb_sticky_record *sr;
+
+ k = talloc_zero_size(tmp_ctx, ((key.dsize + 3) & 0xfffffffc) + 4);
+ if (k == NULL) {
+ DEBUG(DEBUG_ERR,("Failed to allocate key for sticky record\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ k[0] = (key.dsize + 3) / 4 + 1;
+ memcpy(&k[1], key.dptr, key.dsize);
+
+ sr = trbt_lookuparray32(ctdb_db->sticky_records, k[0], &k[0]);
+ if (sr != NULL) {
+ talloc_free(tmp_ctx);
+ return 0;
+ }
+
+ sr = talloc(ctdb_db->sticky_records, struct ctdb_sticky_record);
+ if (sr == NULL) {
+ talloc_free(tmp_ctx);
+ DEBUG(DEBUG_ERR,("Failed to allocate sticky record structure\n"));
+ return -1;
+ }
+
+ sr->ctdb = ctdb;
+ sr->ctdb_db = ctdb_db;
+ sr->pindown = NULL;
+
+ DEBUG(DEBUG_ERR,("Make record sticky for %d seconds in db %s key:0x%08x.\n",
+ ctdb->tunable.sticky_duration,
+ ctdb_db->db_name, ctdb_hash(&key)));
+
+ trbt_insertarray32_callback(ctdb_db->sticky_records, k[0], &k[0], ctdb_make_sticky_record_callback, sr);
+
+ event_add_timed(ctdb->ev, sr, timeval_current_ofs(ctdb->tunable.sticky_duration, 0), ctdb_sticky_record_timeout, sr);
+
+ talloc_free(tmp_ctx);
+ return 0;
+}
+
+struct pinned_down_requeue_handle {
+ struct ctdb_context *ctdb;
+ struct ctdb_req_header *hdr;
+};
+
+struct pinned_down_deferred_call {
+ struct ctdb_context *ctdb;
+ struct ctdb_req_header *hdr;
+};
+
+static void pinned_down_requeue(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct pinned_down_requeue_handle *handle = talloc_get_type(private_data, struct pinned_down_requeue_handle);
+ struct ctdb_context *ctdb = handle->ctdb;
+
+ talloc_steal(ctdb, handle->hdr);
+ ctdb_call_input_pkt(ctdb, handle->hdr);
+
+ talloc_free(handle);
+}
+
+static int pinned_down_destructor(struct pinned_down_deferred_call *pinned_down)
+{
+ struct ctdb_context *ctdb = pinned_down->ctdb;
+ struct pinned_down_requeue_handle *handle = talloc(ctdb, struct pinned_down_requeue_handle);
+
+ handle->ctdb = pinned_down->ctdb;
+ handle->hdr = pinned_down->hdr;
+ talloc_steal(handle, handle->hdr);
+
+ event_add_timed(ctdb->ev, handle, timeval_zero(), pinned_down_requeue, handle);
+
+ return 0;
+}
+
+static int
+ctdb_defer_pinned_down_request(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_req_header *hdr)
+{
+ TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+ uint32_t *k;
+ struct ctdb_sticky_record *sr;
+ struct pinned_down_deferred_call *pinned_down;
+
+ k = talloc_zero_size(tmp_ctx, ((key.dsize + 3) & 0xfffffffc) + 4);
+ if (k == NULL) {
+ DEBUG(DEBUG_ERR,("Failed to allocate key for sticky record\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ k[0] = (key.dsize + 3) / 4 + 1;
+ memcpy(&k[1], key.dptr, key.dsize);
+
+ sr = trbt_lookuparray32(ctdb_db->sticky_records, k[0], &k[0]);
+ if (sr == NULL) {
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ talloc_free(tmp_ctx);
+
+ if (sr->pindown == NULL) {
+ return -1;
+ }
+
+ pinned_down = talloc(sr->pindown, struct pinned_down_deferred_call);
+ if (pinned_down == NULL) {
+ DEBUG(DEBUG_ERR,("Failed to allocate structure for deferred pinned down request\n"));
+ return -1;
+ }
+
+ pinned_down->ctdb = ctdb;
+ pinned_down->hdr = hdr;
+
+ talloc_set_destructor(pinned_down, pinned_down_destructor);
+ talloc_steal(pinned_down, hdr);
+
+ return 0;
+}
+
+static void
+ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key, int hopcount)
+{
+ int i, id;
+
+ /* smallest value is always at index 0 */
+ if (hopcount <= ctdb_db->statistics.hot_keys[0].count) {
+ return;
+ }
+
+ /* see if we already know this key */
+ for (i = 0; i < MAX_HOT_KEYS; i++) {
+ if (key.dsize != ctdb_db->statistics.hot_keys[i].key.dsize) {
+ continue;
+ }
+ if (memcmp(key.dptr, ctdb_db->statistics.hot_keys[i].key.dptr, key.dsize)) {
+ continue;
+ }
+ /* found an entry for this key */
+ if (hopcount <= ctdb_db->statistics.hot_keys[i].count) {
+ return;
+ }
+ ctdb_db->statistics.hot_keys[i].count = hopcount;
+ goto sort_keys;
+ }
+
+ if (ctdb_db->statistics.num_hot_keys < MAX_HOT_KEYS) {
+ id = ctdb_db->statistics.num_hot_keys;
+ ctdb_db->statistics.num_hot_keys++;
+ } else {
+ id = 0;
+ }
+
+ if (ctdb_db->statistics.hot_keys[id].key.dptr != NULL) {
+ talloc_free(ctdb_db->statistics.hot_keys[id].key.dptr);
+ }
+ ctdb_db->statistics.hot_keys[id].key.dsize = key.dsize;
+ ctdb_db->statistics.hot_keys[id].key.dptr = talloc_memdup(ctdb_db, key.dptr, key.dsize);
+ ctdb_db->statistics.hot_keys[id].count = hopcount;
+ DEBUG(DEBUG_NOTICE,("Updated hot key database=%s key=0x%08x id=%d hop_count=%d\n",
+ ctdb_db->db_name, ctdb_hash(&key), id, hopcount));
+
+sort_keys:
+ for (i = 1; i < MAX_HOT_KEYS; i++) {
+ if (ctdb_db->statistics.hot_keys[i].count == 0) {
+ continue;
+ }
+ if (ctdb_db->statistics.hot_keys[i].count < ctdb_db->statistics.hot_keys[0].count) {
+ hopcount = ctdb_db->statistics.hot_keys[i].count;
+ ctdb_db->statistics.hot_keys[i].count = ctdb_db->statistics.hot_keys[0].count;
+ ctdb_db->statistics.hot_keys[0].count = hopcount;
+
+ key = ctdb_db->statistics.hot_keys[i].key;
+ ctdb_db->statistics.hot_keys[i].key = ctdb_db->statistics.hot_keys[0].key;
+ ctdb_db->statistics.hot_keys[0].key = key;
+ }
+ }
+}
/*
called when a CTDB_REQ_CALL packet comes in
struct ctdb_ltdb_header header;
struct ctdb_call *call;
struct ctdb_db_context *ctdb_db;
+ int tmp_count, bucket;
if (ctdb->methods == NULL) {
DEBUG(DEBUG_INFO,(__location__ " Failed ctdb_request_call. Transport is DOWN\n"));
call->reply_data.dptr = NULL;
call->reply_data.dsize = 0;
+
+ /* If this record is pinned down we should defer the
+ request until the pindown times out
+ */
+ if (ctdb_db->sticky) {
+ if (ctdb_defer_pinned_down_request(ctdb, ctdb_db, call->key, hdr) == 0) {
+ DEBUG(DEBUG_WARNING,
+ ("Defer request for pinned down record in %s\n", ctdb_db->db_name));
+ talloc_free(call);
+ return;
+ }
+ }
+
+
/* determine if we are the dmaster for this key. This also
fetches the record data (if any), thus avoiding a 2nd fetch of the data
if the call will be answered locally */
ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, call->key, &header, hdr, &data,
- ctdb_call_input_pkt, ctdb, False);
+ ctdb_call_input_pkt, ctdb, false);
if (ret == -1) {
ctdb_send_error(ctdb, hdr, ret, "ltdb fetch failed in ctdb_request_call");
+ talloc_free(call);
return;
}
if (ret == -2) {
DEBUG(DEBUG_INFO,(__location__ " deferred ctdb_request_call\n"));
+ talloc_free(call);
return;
}
}
if (header.flags & CTDB_REC_RO_REVOKE_COMPLETE) {
- header.flags &= ~(CTDB_REC_RO_HAVE_DELEGATIONS|CTDB_REC_RO_HAVE_READONLY|CTDB_REC_RO_REVOKING_READONLY|CTDB_REC_RO_REVOKE_COMPLETE);
+ header.flags &= ~CTDB_REC_RO_FLAGS;
+ CTDB_INCREMENT_STAT(ctdb, total_ro_revokes);
+ CTDB_INCREMENT_DB_STAT(ctdb_db, db_ro_revokes);
if (ctdb_ltdb_store(ctdb_db, call->key, &header, data) != 0) {
ctdb_fatal(ctdb, "Failed to write header with cleared REVOKE flag");
}
return;
}
- /* if we are not the dmaster and are not hosting any delegations,
- then send a redirect to the requesting node */
+ /*
+ * If we are not the dmaster and are not hosting any delegations,
+ * then we redirect the request to the node than can answer it
+ * (the lmaster or the dmaster).
+ */
if ((header.dmaster != ctdb->pnn)
&& (!(header.flags & CTDB_REC_RO_HAVE_DELEGATIONS)) ) {
talloc_free(data.dptr);
- ctdb_call_send_redirect(ctdb, call->key, c, &header);
+ ctdb_call_send_redirect(ctdb, ctdb_db, call->key, c, &header);
ret = ctdb_ltdb_unlock(ctdb_db, call->key);
if (ret != 0) {
DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
}
+ talloc_free(call);
return;
}
}
ctdb_queue_packet(ctdb, &r->hdr);
+ CTDB_INCREMENT_STAT(ctdb, total_ro_delegations);
+ CTDB_INCREMENT_DB_STAT(ctdb_db, db_ro_delegations);
talloc_free(r);
+ talloc_free(call);
return;
}
CTDB_UPDATE_STAT(ctdb, max_hop_count, c->hopcount);
+ tmp_count = c->hopcount;
+ bucket = 0;
+ while (tmp_count) {
+ tmp_count >>= 2;
+ bucket++;
+ }
+ if (bucket >= MAX_COUNT_BUCKETS) {
+ bucket = MAX_COUNT_BUCKETS - 1;
+ }
+ CTDB_INCREMENT_STAT(ctdb, hop_count_bucket[bucket]);
+ CTDB_INCREMENT_DB_STAT(ctdb_db, hop_count_bucket[bucket]);
+ ctdb_update_db_stat_hot_keys(ctdb_db, call->key, c->hopcount);
+
+ /* If this database supports sticky records, then check if the
+ hopcount is big. If it is it means the record is hot and we
+ should make it sticky.
+ */
+ if (ctdb_db->sticky && c->hopcount >= ctdb->tunable.hopcount_make_sticky) {
+ ctdb_make_record_sticky(ctdb, ctdb_db, call->key);
+ }
+
/* Try if possible to migrate the record off to the caller node.
* From the clients perspective a fetch of the data is just as
* expensive as a migration.
*/
if (c->hdr.srcnode != ctdb->pnn) {
- if (ctdb_db->transaction_active) {
+ if (ctdb_db->persistent_state) {
DEBUG(DEBUG_INFO, (__location__ " refusing migration"
" of key %s while transaction is active\n",
(char *)call->key.dptr));
if (ret != 0) {
DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
}
- return;
}
+ talloc_free(call);
+ return;
}
ret = ctdb_call_local(ctdb_db, call, &header, hdr, &data, true);
ctdb_queue_packet(ctdb, &r->hdr);
talloc_free(r);
+ talloc_free(call);
}
-/*
- called when a CTDB_REPLY_CALL packet comes in
-
- This packet comes in response to a CTDB_REQ_CALL request packet. It
- contains any reply data from the call
-*/
+/**
+ * called when a CTDB_REPLY_CALL packet comes in
+ *
+ * This packet comes in response to a CTDB_REQ_CALL request packet. It
+ * contains any reply data from the call
+ */
void ctdb_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
{
struct ctdb_reply_call *c = (struct ctdb_reply_call *)hdr;
key.dsize = state->c->keylen;
key.dptr = state->c->data;
ret = ctdb_ltdb_lock_requeue(ctdb_db, key, hdr,
- ctdb_call_input_pkt, ctdb, False);
+ ctdb_call_input_pkt, ctdb, false);
if (ret == -2) {
return;
}
}
-/*
- called when a CTDB_REPLY_DMASTER packet comes in
-
- This packet comes in from the lmaster response to a CTDB_REQ_CALL
- request packet. It means that the current dmaster wants to give us
- the dmaster role
-*/
+/**
+ * called when a CTDB_REPLY_DMASTER packet comes in
+ *
+ * This packet comes in from the lmaster in response to a CTDB_REQ_CALL
+ * request packet. It means that the current dmaster wants to give us
+ * the dmaster role.
+ */
void ctdb_reply_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
{
struct ctdb_reply_dmaster *c = (struct ctdb_reply_dmaster *)hdr;
}
ret = ctdb_ltdb_lock_requeue(ctdb_db, key, hdr,
- ctdb_call_input_pkt, ctdb, False);
+ ctdb_call_input_pkt, ctdb, false);
if (ret == -2) {
return;
}
if (rc->fd[1] != -1) {
close(rc->fd[1]);
}
- kill(rc->child, SIGKILL);
+ ctdb_kill(rc->ctdb, rc->child, SIGKILL);
DLIST_REMOVE(rc->ctdb_db->revokechild_active, rc);
return 0;
struct ctdb_revoke_state *revoke_state = private_data;
struct ctdb_client_control_state *state;
- state = ctdb_ctrl_updaterecord_send(ctdb, revoke_state, timeval_current_ofs(5,0), pnn, revoke_state->ctdb_db, revoke_state->key, revoke_state->header, revoke_state->data);
+ state = ctdb_ctrl_updaterecord_send(ctdb, revoke_state, timeval_current_ofs(ctdb->tunable.control_timeout,0), pnn, revoke_state->ctdb_db, revoke_state->key, revoke_state->header, revoke_state->data);
if (state == NULL) {
DEBUG(DEBUG_ERR,("Failure to send update record to revoke readonly delegation\n"));
revoke_state->status = -1;
ctdb_trackingdb_traverse(ctdb, tdata, revoke_send_cb, state);
- event_add_timed(ctdb->ev, state, timeval_current_ofs(5, 0), ctdb_revoke_timeout_handler, state);
+ event_add_timed(ctdb->ev, state, timeval_current_ofs(ctdb->tunable.control_timeout, 0), ctdb_revoke_timeout_handler, state);
while (state->finished == 0) {
event_loop_once(ctdb->ev);
int ret;
header->flags &= ~(CTDB_REC_RO_REVOKING_READONLY|CTDB_REC_RO_HAVE_DELEGATIONS|CTDB_REC_RO_HAVE_READONLY);
+ header->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA;
header->rsn -= 1;
if ((rc = talloc_zero(ctdb_db, struct revokechild_handle)) == NULL) {
close(rc->fd[0]);
debug_extra = talloc_asprintf(NULL, "revokechild-%s:", ctdb_db->db_name);
+ ctdb_set_process_name("ctdb_revokechild");
if (switch_from_server_to_client(ctdb, "revokechild-%s", ctdb_db->db_name) != 0) {
DEBUG(DEBUG_ERR,("Failed to switch from server to client for revokechild process\n"));
c = 1;
child_finished:
write(rc->fd[1], &c, 1);
/* make sure we die when our parent dies */
- while (kill(parent, 0) == 0 || errno != ESRCH) {
+ while (ctdb_kill(ctdb, parent, 0) == 0 || errno != ESRCH) {
sleep(5);
}
_exit(0);
git.samba.org / samba.git / blobdiff