2 ctdb_call protocol code
4 Copyright (C) Andrew Tridgell 2006
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>.
20 see http://wiki.samba.org/index.php/Samba_%26_Clustering for
21 protocol design and packet details
24 #include "system/network.h"
25 #include "system/filesys.h"
30 #include "lib/util/dlinklist.h"
31 #include "lib/util/debug.h"
32 #include "lib/util/samba_util.h"
33 #include "lib/util/sys_rw.h"
34 #include "lib/util/util_process.h"
36 #include "ctdb_private.h"
37 #include "ctdb_client.h"
39 #include "common/rb_tree.h"
40 #include "common/reqid.h"
41 #include "common/system.h"
42 #include "common/common.h"
43 #include "common/logging.h"
44 #include "common/hash_count.h"
46 struct ctdb_sticky_record {
47 struct ctdb_context *ctdb;
48 struct ctdb_db_context *ctdb_db;
53 find the ctdb_db from a db index
55 struct ctdb_db_context *find_ctdb_db(struct ctdb_context *ctdb, uint32_t id)
57 struct ctdb_db_context *ctdb_db;
59 for (ctdb_db=ctdb->db_list; ctdb_db; ctdb_db=ctdb_db->next) {
60 if (ctdb_db->db_id == id) {
68 a varient of input packet that can be used in lock requeue
70 static void ctdb_call_input_pkt(void *p, struct ctdb_req_header *hdr)
72 struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
73 ctdb_input_pkt(ctdb, hdr);
80 static void ctdb_send_error(struct ctdb_context *ctdb,
81 struct ctdb_req_header *hdr, uint32_t status,
82 const char *fmt, ...) PRINTF_ATTRIBUTE(4,5);
83 static void ctdb_send_error(struct ctdb_context *ctdb,
84 struct ctdb_req_header *hdr, uint32_t status,
88 struct ctdb_reply_error_old *r;
92 if (ctdb->methods == NULL) {
93 DEBUG(DEBUG_INFO,(__location__ " Failed to send error. Transport is DOWN\n"));
98 msg = talloc_vasprintf(ctdb, fmt, ap);
100 ctdb_fatal(ctdb, "Unable to allocate error in ctdb_send_error\n");
104 msglen = strlen(msg)+1;
105 len = offsetof(struct ctdb_reply_error_old, msg);
106 r = ctdb_transport_allocate(ctdb, msg, CTDB_REPLY_ERROR, len + msglen,
107 struct ctdb_reply_error_old);
108 CTDB_NO_MEMORY_FATAL(ctdb, r);
110 r->hdr.destnode = hdr->srcnode;
111 r->hdr.reqid = hdr->reqid;
114 memcpy(&r->msg[0], msg, msglen);
116 ctdb_queue_packet(ctdb, &r->hdr);
123 * send a redirect reply
125 * The logic behind this function is this:
127 * A client wants to grab a record and sends a CTDB_REQ_CALL packet
128 * to its local ctdb (ctdb_request_call). If the node is not itself
129 * the record's DMASTER, it first redirects the packet to the
130 * record's LMASTER. The LMASTER then redirects the call packet to
131 * the current DMASTER. Note that this works because of this: When
132 * a record is migrated off a node, then the new DMASTER is stored
133 * in the record's copy on the former DMASTER.
135 static void ctdb_call_send_redirect(struct ctdb_context *ctdb,
136 struct ctdb_db_context *ctdb_db,
138 struct ctdb_req_call_old *c,
139 struct ctdb_ltdb_header *header)
141 uint32_t lmaster = ctdb_lmaster(ctdb, &key);
143 c->hdr.destnode = lmaster;
144 if (ctdb->pnn == lmaster) {
145 c->hdr.destnode = header->dmaster;
149 if (c->hopcount%100 > 95) {
150 DEBUG(DEBUG_WARNING,("High hopcount %d dbid:%s "
151 "key:0x%08x reqid=%08x pnn:%d src:%d lmaster:%d "
152 "header->dmaster:%d dst:%d\n",
153 c->hopcount, ctdb_db->db_name, ctdb_hash(&key),
154 c->hdr.reqid, ctdb->pnn, c->hdr.srcnode, lmaster,
155 header->dmaster, c->hdr.destnode));
158 ctdb_queue_packet(ctdb, &c->hdr);
165 caller must have the chainlock before calling this routine. Caller must be
168 static void ctdb_send_dmaster_reply(struct ctdb_db_context *ctdb_db,
169 struct ctdb_ltdb_header *header,
170 TDB_DATA key, TDB_DATA data,
171 uint32_t new_dmaster,
174 struct ctdb_context *ctdb = ctdb_db->ctdb;
175 struct ctdb_reply_dmaster_old *r;
179 if (ctdb->pnn != ctdb_lmaster(ctdb, &key)) {
180 DEBUG(DEBUG_ALERT,(__location__ " Caller is not lmaster!\n"));
184 header->dmaster = new_dmaster;
185 ret = ctdb_ltdb_store(ctdb_db, key, header, data);
187 ctdb_fatal(ctdb, "ctdb_send_dmaster_reply unable to update dmaster");
191 if (ctdb->methods == NULL) {
192 ctdb_fatal(ctdb, "ctdb_send_dmaster_reply cant update dmaster since transport is down");
196 /* put the packet on a temporary context, allowing us to safely free
197 it below even if ctdb_reply_dmaster() has freed it already */
198 tmp_ctx = talloc_new(ctdb);
200 /* send the CTDB_REPLY_DMASTER */
201 len = offsetof(struct ctdb_reply_dmaster_old, data) + key.dsize + data.dsize + sizeof(uint32_t);
202 r = ctdb_transport_allocate(ctdb, tmp_ctx, CTDB_REPLY_DMASTER, len,
203 struct ctdb_reply_dmaster_old);
204 CTDB_NO_MEMORY_FATAL(ctdb, r);
206 r->hdr.destnode = new_dmaster;
207 r->hdr.reqid = reqid;
208 r->hdr.generation = ctdb_db->generation;
209 r->rsn = header->rsn;
210 r->keylen = key.dsize;
211 r->datalen = data.dsize;
212 r->db_id = ctdb_db->db_id;
213 memcpy(&r->data[0], key.dptr, key.dsize);
214 memcpy(&r->data[key.dsize], data.dptr, data.dsize);
215 memcpy(&r->data[key.dsize+data.dsize], &header->flags, sizeof(uint32_t));
217 ctdb_queue_packet(ctdb, &r->hdr);
219 talloc_free(tmp_ctx);
223 send a dmaster request (give another node the dmaster for a record)
225 This is always sent to the lmaster, which ensures that the lmaster
226 always knows who the dmaster is. The lmaster will then send a
227 CTDB_REPLY_DMASTER to the new dmaster
229 static void ctdb_call_send_dmaster(struct ctdb_db_context *ctdb_db,
230 struct ctdb_req_call_old *c,
231 struct ctdb_ltdb_header *header,
232 TDB_DATA *key, TDB_DATA *data)
234 struct ctdb_req_dmaster_old *r;
235 struct ctdb_context *ctdb = ctdb_db->ctdb;
237 uint32_t lmaster = ctdb_lmaster(ctdb, key);
239 if (ctdb->methods == NULL) {
240 ctdb_fatal(ctdb, "Failed ctdb_call_send_dmaster since transport is down");
244 if (data->dsize != 0) {
245 header->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA;
248 if (lmaster == ctdb->pnn) {
249 ctdb_send_dmaster_reply(ctdb_db, header, *key, *data,
250 c->hdr.srcnode, c->hdr.reqid);
254 len = offsetof(struct ctdb_req_dmaster_old, data) + key->dsize + data->dsize
256 r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REQ_DMASTER, len,
257 struct ctdb_req_dmaster_old);
258 CTDB_NO_MEMORY_FATAL(ctdb, r);
259 r->hdr.destnode = lmaster;
260 r->hdr.reqid = c->hdr.reqid;
261 r->hdr.generation = ctdb_db->generation;
263 r->rsn = header->rsn;
264 r->dmaster = c->hdr.srcnode;
265 r->keylen = key->dsize;
266 r->datalen = data->dsize;
267 memcpy(&r->data[0], key->dptr, key->dsize);
268 memcpy(&r->data[key->dsize], data->dptr, data->dsize);
269 memcpy(&r->data[key->dsize + data->dsize], &header->flags, sizeof(uint32_t));
271 header->dmaster = c->hdr.srcnode;
272 if (ctdb_ltdb_store(ctdb_db, *key, header, *data) != 0) {
273 ctdb_fatal(ctdb, "Failed to store record in ctdb_call_send_dmaster");
276 ctdb_queue_packet(ctdb, &r->hdr);
281 static void ctdb_sticky_pindown_timeout(struct tevent_context *ev,
282 struct tevent_timer *te,
283 struct timeval t, void *private_data)
285 struct ctdb_sticky_record *sr = talloc_get_type(private_data,
286 struct ctdb_sticky_record);
288 DEBUG(DEBUG_ERR,("Pindown timeout db:%s unstick record\n", sr->ctdb_db->db_name));
289 if (sr->pindown != NULL) {
290 talloc_free(sr->pindown);
296 ctdb_set_sticky_pindown(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key)
298 TALLOC_CTX *tmp_ctx = talloc_new(NULL);
300 struct ctdb_sticky_record *sr;
302 k = ctdb_key_to_idkey(tmp_ctx, key);
304 DEBUG(DEBUG_ERR,("Failed to allocate key for sticky record\n"));
305 talloc_free(tmp_ctx);
309 sr = trbt_lookuparray32(ctdb_db->sticky_records, k[0], &k[0]);
311 talloc_free(tmp_ctx);
315 talloc_free(tmp_ctx);
317 if (sr->pindown == NULL) {
318 DEBUG(DEBUG_ERR,("Pinning down record in %s for %d ms\n", ctdb_db->db_name, ctdb->tunable.sticky_pindown));
319 sr->pindown = talloc_new(sr);
320 if (sr->pindown == NULL) {
321 DEBUG(DEBUG_ERR,("Failed to allocate pindown context for sticky record\n"));
324 tevent_add_timer(ctdb->ev, sr->pindown,
325 timeval_current_ofs(ctdb->tunable.sticky_pindown / 1000,
326 (ctdb->tunable.sticky_pindown * 1000) % 1000000),
327 ctdb_sticky_pindown_timeout, sr);
334 called when a CTDB_REPLY_DMASTER packet comes in, or when the lmaster
335 gets a CTDB_REQUEST_DMASTER for itself. We become the dmaster.
337 must be called with the chainlock held. This function releases the chainlock
339 static void ctdb_become_dmaster(struct ctdb_db_context *ctdb_db,
340 struct ctdb_req_header *hdr,
341 TDB_DATA key, TDB_DATA data,
342 uint64_t rsn, uint32_t record_flags)
344 struct ctdb_call_state *state;
345 struct ctdb_context *ctdb = ctdb_db->ctdb;
346 struct ctdb_ltdb_header header;
349 DEBUG(DEBUG_DEBUG,("pnn %u dmaster response %08x\n", ctdb->pnn, ctdb_hash(&key)));
353 header.dmaster = ctdb->pnn;
354 header.flags = record_flags;
356 state = reqid_find(ctdb->idr, hdr->reqid, struct ctdb_call_state);
359 if (state->call->flags & CTDB_CALL_FLAG_VACUUM_MIGRATION) {
361 * We temporarily add the VACUUM_MIGRATED flag to
362 * the record flags, so that ctdb_ltdb_store can
363 * decide whether the record should be stored or
366 header.flags |= CTDB_REC_FLAG_VACUUM_MIGRATED;
370 if (ctdb_ltdb_store(ctdb_db, key, &header, data) != 0) {
371 ctdb_fatal(ctdb, "ctdb_reply_dmaster store failed\n");
373 ret = ctdb_ltdb_unlock(ctdb_db, key);
375 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
380 /* we just became DMASTER and this database is "sticky",
381 see if the record is flagged as "hot" and set up a pin-down
382 context to stop migrations for a little while if so
384 if (ctdb_db->sticky) {
385 ctdb_set_sticky_pindown(ctdb, ctdb_db, key);
389 DEBUG(DEBUG_ERR,("pnn %u Invalid reqid %u in ctdb_become_dmaster from node %u\n",
390 ctdb->pnn, hdr->reqid, hdr->srcnode));
392 ret = ctdb_ltdb_unlock(ctdb_db, key);
394 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
399 if (key.dsize != state->call->key.dsize || memcmp(key.dptr, state->call->key.dptr, key.dsize)) {
400 DEBUG(DEBUG_ERR, ("Got bogus DMASTER packet reqid:%u from node %u. Key does not match key held in matching idr.\n", hdr->reqid, hdr->srcnode));
402 ret = ctdb_ltdb_unlock(ctdb_db, key);
404 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
409 if (hdr->reqid != state->reqid) {
410 /* we found a record but it was the wrong one */
411 DEBUG(DEBUG_ERR, ("Dropped orphan in ctdb_become_dmaster with reqid:%u\n from node %u", hdr->reqid, hdr->srcnode));
413 ret = ctdb_ltdb_unlock(ctdb_db, key);
415 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
420 (void) hash_count_increment(ctdb_db->migratedb, key);
422 ctdb_call_local(ctdb_db, state->call, &header, state, &data, true);
424 ret = ctdb_ltdb_unlock(ctdb_db, state->call->key);
426 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
429 state->state = CTDB_CALL_DONE;
430 if (state->async.fn) {
431 state->async.fn(state);
435 struct dmaster_defer_call {
436 struct dmaster_defer_call *next, *prev;
437 struct ctdb_context *ctdb;
438 struct ctdb_req_header *hdr;
441 struct dmaster_defer_queue {
442 struct ctdb_db_context *ctdb_db;
444 struct dmaster_defer_call *deferred_calls;
447 static void dmaster_defer_reprocess(struct tevent_context *ev,
448 struct tevent_timer *te,
452 struct dmaster_defer_call *call = talloc_get_type(
453 private_data, struct dmaster_defer_call);
455 ctdb_input_pkt(call->ctdb, call->hdr);
459 static int dmaster_defer_queue_destructor(struct dmaster_defer_queue *ddq)
461 /* Ignore requests, if database recovery happens in-between. */
462 if (ddq->generation != ddq->ctdb_db->generation) {
466 while (ddq->deferred_calls != NULL) {
467 struct dmaster_defer_call *call = ddq->deferred_calls;
469 DLIST_REMOVE(ddq->deferred_calls, call);
471 talloc_steal(call->ctdb, call);
472 tevent_add_timer(call->ctdb->ev, call, timeval_zero(),
473 dmaster_defer_reprocess, call);
478 static void *insert_ddq_callback(void *parm, void *data)
487 * This function is used to reigster a key in database that needs to be updated.
488 * Any requests for that key should get deferred till this is completed.
490 static int dmaster_defer_setup(struct ctdb_db_context *ctdb_db,
491 struct ctdb_req_header *hdr,
495 struct dmaster_defer_queue *ddq;
497 k = ctdb_key_to_idkey(hdr, key);
499 DEBUG(DEBUG_ERR, ("Failed to allocate key for dmaster defer setup\n"));
504 ddq = trbt_lookuparray32(ctdb_db->defer_dmaster, k[0], k);
506 if (ddq->generation == ctdb_db->generation) {
511 /* Recovery ocurred - get rid of old queue. All the deferred
512 * requests will be resent anyway from ctdb_call_resend_db.
517 ddq = talloc(hdr, struct dmaster_defer_queue);
519 DEBUG(DEBUG_ERR, ("Failed to allocate dmaster defer queue\n"));
523 ddq->ctdb_db = ctdb_db;
524 ddq->generation = hdr->generation;
525 ddq->deferred_calls = NULL;
527 trbt_insertarray32_callback(ctdb_db->defer_dmaster, k[0], k,
528 insert_ddq_callback, ddq);
529 talloc_set_destructor(ddq, dmaster_defer_queue_destructor);
535 static int dmaster_defer_add(struct ctdb_db_context *ctdb_db,
536 struct ctdb_req_header *hdr,
539 struct dmaster_defer_queue *ddq;
540 struct dmaster_defer_call *call;
543 k = ctdb_key_to_idkey(hdr, key);
545 DEBUG(DEBUG_ERR, ("Failed to allocate key for dmaster defer add\n"));
549 ddq = trbt_lookuparray32(ctdb_db->defer_dmaster, k[0], k);
557 if (ddq->generation != hdr->generation) {
558 talloc_set_destructor(ddq, NULL);
563 call = talloc(ddq, struct dmaster_defer_call);
565 DEBUG(DEBUG_ERR, ("Failed to allocate dmaster defer call\n"));
569 call->ctdb = ctdb_db->ctdb;
570 call->hdr = talloc_steal(call, hdr);
572 DLIST_ADD_END(ddq->deferred_calls, call);
578 called when a CTDB_REQ_DMASTER packet comes in
580 this comes into the lmaster for a record when the current dmaster
581 wants to give up the dmaster role and give it to someone else
583 void ctdb_request_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
585 struct ctdb_req_dmaster_old *c = (struct ctdb_req_dmaster_old *)hdr;
586 TDB_DATA key, data, data2;
587 struct ctdb_ltdb_header header;
588 struct ctdb_db_context *ctdb_db;
589 uint32_t record_flags = 0;
594 key.dsize = c->keylen;
595 data.dptr = c->data + c->keylen;
596 data.dsize = c->datalen;
597 len = offsetof(struct ctdb_req_dmaster_old, data) + key.dsize + data.dsize
599 if (len <= c->hdr.length) {
600 memcpy(&record_flags, &c->data[c->keylen + c->datalen],
601 sizeof(record_flags));
604 ctdb_db = find_ctdb_db(ctdb, c->db_id);
606 ctdb_send_error(ctdb, hdr, -1,
607 "Unknown database in request. db_id==0x%08x",
612 dmaster_defer_setup(ctdb_db, hdr, key);
614 /* fetch the current record */
615 ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, key, &header, hdr, &data2,
616 ctdb_call_input_pkt, ctdb, false);
618 ctdb_fatal(ctdb, "ctdb_req_dmaster failed to fetch record");
622 DEBUG(DEBUG_INFO,(__location__ " deferring ctdb_request_dmaster\n"));
626 if (ctdb_lmaster(ctdb, &key) != ctdb->pnn) {
627 DEBUG(DEBUG_ERR, ("dmaster request to non-lmaster "
628 "db=%s lmaster=%u gen=%u curgen=%u\n",
629 ctdb_db->db_name, ctdb_lmaster(ctdb, &key),
630 hdr->generation, ctdb_db->generation));
631 ctdb_fatal(ctdb, "ctdb_req_dmaster to non-lmaster");
634 DEBUG(DEBUG_DEBUG,("pnn %u dmaster request on %08x for %u from %u\n",
635 ctdb->pnn, ctdb_hash(&key), c->dmaster, c->hdr.srcnode));
637 /* its a protocol error if the sending node is not the current dmaster */
638 if (header.dmaster != hdr->srcnode) {
639 DEBUG(DEBUG_ALERT,("pnn %u dmaster request for new-dmaster %u from non-master %u real-dmaster=%u key %08x dbid 0x%08x gen=%u curgen=%u c->rsn=%llu header.rsn=%llu reqid=%u keyval=0x%08x\n",
640 ctdb->pnn, c->dmaster, hdr->srcnode, header.dmaster, ctdb_hash(&key),
641 ctdb_db->db_id, hdr->generation, ctdb->vnn_map->generation,
642 (unsigned long long)c->rsn, (unsigned long long)header.rsn, c->hdr.reqid,
643 (key.dsize >= 4)?(*(uint32_t *)key.dptr):0));
644 if (header.rsn != 0 || header.dmaster != ctdb->pnn) {
645 DEBUG(DEBUG_ERR,("ctdb_req_dmaster from non-master. Force a recovery.\n"));
647 ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
648 ctdb_ltdb_unlock(ctdb_db, key);
653 if (header.rsn > c->rsn) {
654 DEBUG(DEBUG_ALERT,("pnn %u dmaster request with older RSN new-dmaster %u from %u real-dmaster=%u key %08x dbid 0x%08x gen=%u curgen=%u c->rsn=%llu header.rsn=%llu reqid=%u\n",
655 ctdb->pnn, c->dmaster, hdr->srcnode, header.dmaster, ctdb_hash(&key),
656 ctdb_db->db_id, hdr->generation, ctdb->vnn_map->generation,
657 (unsigned long long)c->rsn, (unsigned long long)header.rsn, c->hdr.reqid));
660 /* use the rsn from the sending node */
663 /* store the record flags from the sending node */
664 header.flags = record_flags;
666 /* check if the new dmaster is the lmaster, in which case we
667 skip the dmaster reply */
668 if (c->dmaster == ctdb->pnn) {
669 ctdb_become_dmaster(ctdb_db, hdr, key, data, c->rsn, record_flags);
671 ctdb_send_dmaster_reply(ctdb_db, &header, key, data, c->dmaster, hdr->reqid);
673 ret = ctdb_ltdb_unlock(ctdb_db, key);
675 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
680 static void ctdb_sticky_record_timeout(struct tevent_context *ev,
681 struct tevent_timer *te,
682 struct timeval t, void *private_data)
684 struct ctdb_sticky_record *sr = talloc_get_type(private_data,
685 struct ctdb_sticky_record);
689 static void *ctdb_make_sticky_record_callback(void *parm, void *data)
692 DEBUG(DEBUG_ERR,("Already have sticky record registered. Free old %p and create new %p\n", data, parm));
699 ctdb_make_record_sticky(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key)
701 TALLOC_CTX *tmp_ctx = talloc_new(NULL);
703 struct ctdb_sticky_record *sr;
705 k = ctdb_key_to_idkey(tmp_ctx, key);
707 DEBUG(DEBUG_ERR,("Failed to allocate key for sticky record\n"));
708 talloc_free(tmp_ctx);
712 sr = trbt_lookuparray32(ctdb_db->sticky_records, k[0], &k[0]);
714 talloc_free(tmp_ctx);
718 sr = talloc(ctdb_db->sticky_records, struct ctdb_sticky_record);
720 talloc_free(tmp_ctx);
721 DEBUG(DEBUG_ERR,("Failed to allocate sticky record structure\n"));
726 sr->ctdb_db = ctdb_db;
729 DEBUG(DEBUG_ERR,("Make record sticky for %d seconds in db %s key:0x%08x.\n",
730 ctdb->tunable.sticky_duration,
731 ctdb_db->db_name, ctdb_hash(&key)));
733 trbt_insertarray32_callback(ctdb_db->sticky_records, k[0], &k[0], ctdb_make_sticky_record_callback, sr);
735 tevent_add_timer(ctdb->ev, sr,
736 timeval_current_ofs(ctdb->tunable.sticky_duration, 0),
737 ctdb_sticky_record_timeout, sr);
739 talloc_free(tmp_ctx);
743 struct pinned_down_requeue_handle {
744 struct ctdb_context *ctdb;
745 struct ctdb_req_header *hdr;
748 struct pinned_down_deferred_call {
749 struct ctdb_context *ctdb;
750 struct ctdb_req_header *hdr;
753 static void pinned_down_requeue(struct tevent_context *ev,
754 struct tevent_timer *te,
755 struct timeval t, void *private_data)
757 struct pinned_down_requeue_handle *handle = talloc_get_type(private_data, struct pinned_down_requeue_handle);
758 struct ctdb_context *ctdb = handle->ctdb;
760 talloc_steal(ctdb, handle->hdr);
761 ctdb_call_input_pkt(ctdb, handle->hdr);
766 static int pinned_down_destructor(struct pinned_down_deferred_call *pinned_down)
768 struct ctdb_context *ctdb = pinned_down->ctdb;
769 struct pinned_down_requeue_handle *handle = talloc(ctdb, struct pinned_down_requeue_handle);
771 handle->ctdb = pinned_down->ctdb;
772 handle->hdr = pinned_down->hdr;
773 talloc_steal(handle, handle->hdr);
775 tevent_add_timer(ctdb->ev, handle, timeval_zero(),
776 pinned_down_requeue, handle);
782 ctdb_defer_pinned_down_request(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_req_header *hdr)
784 TALLOC_CTX *tmp_ctx = talloc_new(NULL);
786 struct ctdb_sticky_record *sr;
787 struct pinned_down_deferred_call *pinned_down;
789 k = ctdb_key_to_idkey(tmp_ctx, key);
791 DEBUG(DEBUG_ERR,("Failed to allocate key for sticky record\n"));
792 talloc_free(tmp_ctx);
796 sr = trbt_lookuparray32(ctdb_db->sticky_records, k[0], &k[0]);
798 talloc_free(tmp_ctx);
802 talloc_free(tmp_ctx);
804 if (sr->pindown == NULL) {
808 pinned_down = talloc(sr->pindown, struct pinned_down_deferred_call);
809 if (pinned_down == NULL) {
810 DEBUG(DEBUG_ERR,("Failed to allocate structure for deferred pinned down request\n"));
814 pinned_down->ctdb = ctdb;
815 pinned_down->hdr = hdr;
817 talloc_set_destructor(pinned_down, pinned_down_destructor);
818 talloc_steal(pinned_down, hdr);
824 ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key,
830 /* smallest value is always at index 0 */
831 if (count <= ctdb_db->statistics.hot_keys[0].count) {
835 /* see if we already know this key */
836 for (i = 0; i < MAX_HOT_KEYS; i++) {
837 if (key.dsize != ctdb_db->statistics.hot_keys[i].key.dsize) {
840 if (memcmp(key.dptr, ctdb_db->statistics.hot_keys[i].key.dptr, key.dsize)) {
843 /* found an entry for this key */
844 if (count <= ctdb_db->statistics.hot_keys[i].count) {
847 ctdb_db->statistics.hot_keys[i].count = count;
851 if (ctdb_db->statistics.num_hot_keys < MAX_HOT_KEYS) {
852 id = ctdb_db->statistics.num_hot_keys;
853 ctdb_db->statistics.num_hot_keys++;
858 if (ctdb_db->statistics.hot_keys[id].key.dptr != NULL) {
859 talloc_free(ctdb_db->statistics.hot_keys[id].key.dptr);
861 ctdb_db->statistics.hot_keys[id].key.dsize = key.dsize;
862 ctdb_db->statistics.hot_keys[id].key.dptr = talloc_memdup(ctdb_db, key.dptr, key.dsize);
863 ctdb_db->statistics.hot_keys[id].count = count;
865 keystr = hex_encode_talloc(ctdb_db,
866 (unsigned char *)key.dptr, key.dsize);
867 DEBUG(DEBUG_NOTICE,("Updated hot key database=%s key=%s id=%d "
868 "count=%d\n", ctdb_db->db_name,
869 keystr ? keystr : "" , id, count));
873 for (i = 1; i < MAX_HOT_KEYS; i++) {
874 if (ctdb_db->statistics.hot_keys[i].count == 0) {
877 if (ctdb_db->statistics.hot_keys[i].count < ctdb_db->statistics.hot_keys[0].count) {
878 count = ctdb_db->statistics.hot_keys[i].count;
879 ctdb_db->statistics.hot_keys[i].count = ctdb_db->statistics.hot_keys[0].count;
880 ctdb_db->statistics.hot_keys[0].count = count;
882 key = ctdb_db->statistics.hot_keys[i].key;
883 ctdb_db->statistics.hot_keys[i].key = ctdb_db->statistics.hot_keys[0].key;
884 ctdb_db->statistics.hot_keys[0].key = key;
890 called when a CTDB_REQ_CALL packet comes in
892 void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
894 struct ctdb_req_call_old *c = (struct ctdb_req_call_old *)hdr;
896 struct ctdb_reply_call_old *r;
898 struct ctdb_ltdb_header header;
899 struct ctdb_call *call;
900 struct ctdb_db_context *ctdb_db;
901 int tmp_count, bucket;
903 if (ctdb->methods == NULL) {
904 DEBUG(DEBUG_INFO,(__location__ " Failed ctdb_request_call. Transport is DOWN\n"));
909 ctdb_db = find_ctdb_db(ctdb, c->db_id);
911 ctdb_send_error(ctdb, hdr, -1,
912 "Unknown database in request. db_id==0x%08x",
917 call = talloc(hdr, struct ctdb_call);
918 CTDB_NO_MEMORY_FATAL(ctdb, call);
920 call->call_id = c->callid;
921 call->key.dptr = c->data;
922 call->key.dsize = c->keylen;
923 call->call_data.dptr = c->data + c->keylen;
924 call->call_data.dsize = c->calldatalen;
925 call->reply_data.dptr = NULL;
926 call->reply_data.dsize = 0;
929 /* If this record is pinned down we should defer the
930 request until the pindown times out
932 if (ctdb_db->sticky) {
933 if (ctdb_defer_pinned_down_request(ctdb, ctdb_db, call->key, hdr) == 0) {
935 ("Defer request for pinned down record in %s\n", ctdb_db->db_name));
941 if (dmaster_defer_add(ctdb_db, hdr, call->key) == 0) {
946 /* determine if we are the dmaster for this key. This also
947 fetches the record data (if any), thus avoiding a 2nd fetch of the data
948 if the call will be answered locally */
950 ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, call->key, &header, hdr, &data,
951 ctdb_call_input_pkt, ctdb, false);
953 ctdb_send_error(ctdb, hdr, ret, "ltdb fetch failed in ctdb_request_call");
958 DEBUG(DEBUG_INFO,(__location__ " deferred ctdb_request_call\n"));
963 /* Dont do READONLY if we don't have a tracking database */
964 if ((c->flags & CTDB_WANT_READONLY) && !ctdb_db->readonly) {
965 c->flags &= ~CTDB_WANT_READONLY;
968 if (header.flags & CTDB_REC_RO_REVOKE_COMPLETE) {
969 header.flags &= ~CTDB_REC_RO_FLAGS;
970 CTDB_INCREMENT_STAT(ctdb, total_ro_revokes);
971 CTDB_INCREMENT_DB_STAT(ctdb_db, db_ro_revokes);
972 if (ctdb_ltdb_store(ctdb_db, call->key, &header, data) != 0) {
973 ctdb_fatal(ctdb, "Failed to write header with cleared REVOKE flag");
975 /* and clear out the tracking data */
976 if (tdb_delete(ctdb_db->rottdb, call->key) != 0) {
977 DEBUG(DEBUG_ERR,(__location__ " Failed to clear out trackingdb record\n"));
981 /* if we are revoking, we must defer all other calls until the revoke
984 if (header.flags & CTDB_REC_RO_REVOKING_READONLY) {
985 talloc_free(data.dptr);
986 ret = ctdb_ltdb_unlock(ctdb_db, call->key);
988 if (ctdb_add_revoke_deferred_call(ctdb, ctdb_db, call->key, hdr, ctdb_call_input_pkt, ctdb) != 0) {
989 ctdb_fatal(ctdb, "Failed to add deferred call for revoke child");
996 * If we are not the dmaster and are not hosting any delegations,
997 * then we redirect the request to the node than can answer it
998 * (the lmaster or the dmaster).
1000 if ((header.dmaster != ctdb->pnn)
1001 && (!(header.flags & CTDB_REC_RO_HAVE_DELEGATIONS)) ) {
1002 talloc_free(data.dptr);
1003 ctdb_call_send_redirect(ctdb, ctdb_db, call->key, c, &header);
1005 ret = ctdb_ltdb_unlock(ctdb_db, call->key);
1007 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
1013 if ( (!(c->flags & CTDB_WANT_READONLY))
1014 && (header.flags & (CTDB_REC_RO_HAVE_DELEGATIONS|CTDB_REC_RO_HAVE_READONLY)) ) {
1015 header.flags |= CTDB_REC_RO_REVOKING_READONLY;
1016 if (ctdb_ltdb_store(ctdb_db, call->key, &header, data) != 0) {
1017 ctdb_fatal(ctdb, "Failed to store record with HAVE_DELEGATIONS set");
1019 ret = ctdb_ltdb_unlock(ctdb_db, call->key);
1021 if (ctdb_start_revoke_ro_record(ctdb, ctdb_db, call->key, &header, data) != 0) {
1022 ctdb_fatal(ctdb, "Failed to start record revoke");
1024 talloc_free(data.dptr);
1026 if (ctdb_add_revoke_deferred_call(ctdb, ctdb_db, call->key, hdr, ctdb_call_input_pkt, ctdb) != 0) {
1027 ctdb_fatal(ctdb, "Failed to add deferred call for revoke child");
1034 /* If this is the first request for delegation. bump rsn and set
1035 * the delegations flag
1037 if ((c->flags & CTDB_WANT_READONLY)
1038 && (c->callid == CTDB_FETCH_WITH_HEADER_FUNC)
1039 && (!(header.flags & CTDB_REC_RO_HAVE_DELEGATIONS))) {
1041 header.flags |= CTDB_REC_RO_HAVE_DELEGATIONS;
1042 if (ctdb_ltdb_store(ctdb_db, call->key, &header, data) != 0) {
1043 ctdb_fatal(ctdb, "Failed to store record with HAVE_DELEGATIONS set");
1046 if ((c->flags & CTDB_WANT_READONLY)
1047 && (call->call_id == CTDB_FETCH_WITH_HEADER_FUNC)) {
1050 tdata = tdb_fetch(ctdb_db->rottdb, call->key);
1051 if (ctdb_trackingdb_add_pnn(ctdb, &tdata, c->hdr.srcnode) != 0) {
1052 ctdb_fatal(ctdb, "Failed to add node to trackingdb");
1054 if (tdb_store(ctdb_db->rottdb, call->key, tdata, TDB_REPLACE) != 0) {
1055 ctdb_fatal(ctdb, "Failed to store trackingdb data");
1059 ret = ctdb_ltdb_unlock(ctdb_db, call->key);
1061 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
1064 len = offsetof(struct ctdb_reply_call_old, data) + data.dsize + sizeof(struct ctdb_ltdb_header);
1065 r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REPLY_CALL, len,
1066 struct ctdb_reply_call_old);
1067 CTDB_NO_MEMORY_FATAL(ctdb, r);
1068 r->hdr.destnode = c->hdr.srcnode;
1069 r->hdr.reqid = c->hdr.reqid;
1070 r->hdr.generation = ctdb_db->generation;
1072 r->datalen = data.dsize + sizeof(struct ctdb_ltdb_header);
1074 header.flags |= CTDB_REC_RO_HAVE_READONLY;
1075 header.flags &= ~CTDB_REC_RO_HAVE_DELEGATIONS;
1076 memcpy(&r->data[0], &header, sizeof(struct ctdb_ltdb_header));
1079 memcpy(&r->data[sizeof(struct ctdb_ltdb_header)], data.dptr, data.dsize);
1082 ctdb_queue_packet(ctdb, &r->hdr);
1083 CTDB_INCREMENT_STAT(ctdb, total_ro_delegations);
1084 CTDB_INCREMENT_DB_STAT(ctdb_db, db_ro_delegations);
1091 CTDB_UPDATE_STAT(ctdb, max_hop_count, c->hopcount);
1092 tmp_count = c->hopcount;
1098 if (bucket >= MAX_COUNT_BUCKETS) {
1099 bucket = MAX_COUNT_BUCKETS - 1;
1101 CTDB_INCREMENT_STAT(ctdb, hop_count_bucket[bucket]);
1102 CTDB_INCREMENT_DB_STAT(ctdb_db, hop_count_bucket[bucket]);
1104 /* If this database supports sticky records, then check if the
1105 hopcount is big. If it is it means the record is hot and we
1106 should make it sticky.
1108 if (ctdb_db->sticky && c->hopcount >= ctdb->tunable.hopcount_make_sticky) {
1109 ctdb_make_record_sticky(ctdb, ctdb_db, call->key);
1113 /* Try if possible to migrate the record off to the caller node.
1114 * From the clients perspective a fetch of the data is just as
1115 * expensive as a migration.
1117 if (c->hdr.srcnode != ctdb->pnn) {
1118 if (ctdb_db->persistent_state) {
1119 DEBUG(DEBUG_INFO, (__location__ " refusing migration"
1120 " of key %s while transaction is active\n",
1121 (char *)call->key.dptr));
1123 DEBUG(DEBUG_DEBUG,("pnn %u starting migration of %08x to %u\n",
1124 ctdb->pnn, ctdb_hash(&(call->key)), c->hdr.srcnode));
1125 ctdb_call_send_dmaster(ctdb_db, c, &header, &(call->key), &data);
1126 talloc_free(data.dptr);
1128 ret = ctdb_ltdb_unlock(ctdb_db, call->key);
1130 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
1137 ret = ctdb_call_local(ctdb_db, call, &header, hdr, &data, true);
1139 DEBUG(DEBUG_ERR,(__location__ " ctdb_call_local failed\n"));
1143 ret = ctdb_ltdb_unlock(ctdb_db, call->key);
1145 DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
1148 len = offsetof(struct ctdb_reply_call_old, data) + call->reply_data.dsize;
1149 r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REPLY_CALL, len,
1150 struct ctdb_reply_call_old);
1151 CTDB_NO_MEMORY_FATAL(ctdb, r);
1152 r->hdr.destnode = hdr->srcnode;
1153 r->hdr.reqid = hdr->reqid;
1154 r->hdr.generation = ctdb_db->generation;
1155 r->status = call->status;
1156 r->datalen = call->reply_data.dsize;
1157 if (call->reply_data.dsize) {
1158 memcpy(&r->data[0], call->reply_data.dptr, call->reply_data.dsize);
1161 ctdb_queue_packet(ctdb, &r->hdr);
1168 * called when a CTDB_REPLY_CALL packet comes in
1170 * This packet comes in response to a CTDB_REQ_CALL request packet. It
1171 * contains any reply data from the call
1173 void ctdb_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
1175 struct ctdb_reply_call_old *c = (struct ctdb_reply_call_old *)hdr;
1176 struct ctdb_call_state *state;
1178 state = reqid_find(ctdb->idr, hdr->reqid, struct ctdb_call_state);
1179 if (state == NULL) {
1180 DEBUG(DEBUG_ERR, (__location__ " reqid %u not found\n", hdr->reqid));
1184 if (hdr->reqid != state->reqid) {
1185 /* we found a record but it was the wrong one */
1186 DEBUG(DEBUG_ERR, ("Dropped orphaned call reply with reqid:%u\n",hdr->reqid));
1191 /* read only delegation processing */
1192 /* If we got a FETCH_WITH_HEADER we should check if this is a ro
1193 * delegation since we may need to update the record header
1195 if (state->c->callid == CTDB_FETCH_WITH_HEADER_FUNC) {
1196 struct ctdb_db_context *ctdb_db = state->ctdb_db;
1197 struct ctdb_ltdb_header *header = (struct ctdb_ltdb_header *)&c->data[0];
1198 struct ctdb_ltdb_header oldheader;
1199 TDB_DATA key, data, olddata;
1202 if (!(header->flags & CTDB_REC_RO_HAVE_READONLY)) {
1207 key.dsize = state->c->keylen;
1208 key.dptr = state->c->data;
1209 ret = ctdb_ltdb_lock_requeue(ctdb_db, key, hdr,
1210 ctdb_call_input_pkt, ctdb, false);
1215 DEBUG(DEBUG_ERR,(__location__ " Failed to get lock in ctdb_reply_call\n"));
1219 ret = ctdb_ltdb_fetch(ctdb_db, key, &oldheader, state, &olddata);
1221 DEBUG(DEBUG_ERR, ("Failed to fetch old record in ctdb_reply_call\n"));
1222 ctdb_ltdb_unlock(ctdb_db, key);
1226 if (header->rsn <= oldheader.rsn) {
1227 ctdb_ltdb_unlock(ctdb_db, key);
1231 if (c->datalen < sizeof(struct ctdb_ltdb_header)) {
1232 DEBUG(DEBUG_ERR,(__location__ " Got FETCH_WITH_HEADER reply with too little data: %d bytes\n", c->datalen));
1233 ctdb_ltdb_unlock(ctdb_db, key);
1237 data.dsize = c->datalen - sizeof(struct ctdb_ltdb_header);
1238 data.dptr = &c->data[sizeof(struct ctdb_ltdb_header)];
1239 ret = ctdb_ltdb_store(ctdb_db, key, header, data);
1241 DEBUG(DEBUG_ERR, ("Failed to store new record in ctdb_reply_call\n"));
1242 ctdb_ltdb_unlock(ctdb_db, key);
1246 ctdb_ltdb_unlock(ctdb_db, key);
1250 state->call->reply_data.dptr = c->data;
1251 state->call->reply_data.dsize = c->datalen;
1252 state->call->status = c->status;
1254 talloc_steal(state, c);
1256 state->state = CTDB_CALL_DONE;
1257 if (state->async.fn) {
1258 state->async.fn(state);
1264 * called when a CTDB_REPLY_DMASTER packet comes in
1266 * This packet comes in from the lmaster in response to a CTDB_REQ_CALL
1267 * request packet. It means that the current dmaster wants to give us
1270 void ctdb_reply_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
1272 struct ctdb_reply_dmaster_old *c = (struct ctdb_reply_dmaster_old *)hdr;
1273 struct ctdb_db_context *ctdb_db;
1275 uint32_t record_flags = 0;
1279 ctdb_db = find_ctdb_db(ctdb, c->db_id);
1280 if (ctdb_db == NULL) {
1281 DEBUG(DEBUG_ERR,("Unknown db_id 0x%x in ctdb_reply_dmaster\n", c->db_id));
1286 key.dsize = c->keylen;
1287 data.dptr = &c->data[key.dsize];
1288 data.dsize = c->datalen;
1289 len = offsetof(struct ctdb_reply_dmaster_old, data) + key.dsize + data.dsize
1291 if (len <= c->hdr.length) {
1292 memcpy(&record_flags, &c->data[c->keylen + c->datalen],
1293 sizeof(record_flags));
1296 dmaster_defer_setup(ctdb_db, hdr, key);
1298 ret = ctdb_ltdb_lock_requeue(ctdb_db, key, hdr,
1299 ctdb_call_input_pkt, ctdb, false);
1304 DEBUG(DEBUG_ERR,(__location__ " Failed to get lock in ctdb_reply_dmaster\n"));
1308 ctdb_become_dmaster(ctdb_db, hdr, key, data, c->rsn, record_flags);
1313 called when a CTDB_REPLY_ERROR packet comes in
1315 void ctdb_reply_error(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
1317 struct ctdb_reply_error_old *c = (struct ctdb_reply_error_old *)hdr;
1318 struct ctdb_call_state *state;
1320 state = reqid_find(ctdb->idr, hdr->reqid, struct ctdb_call_state);
1321 if (state == NULL) {
1322 DEBUG(DEBUG_ERR,("pnn %u Invalid reqid %u in ctdb_reply_error\n",
1323 ctdb->pnn, hdr->reqid));
1327 if (hdr->reqid != state->reqid) {
1328 /* we found a record but it was the wrong one */
1329 DEBUG(DEBUG_ERR, ("Dropped orphaned error reply with reqid:%u\n",hdr->reqid));
1333 talloc_steal(state, c);
1335 state->state = CTDB_CALL_ERROR;
1336 state->errmsg = (char *)c->msg;
1337 if (state->async.fn) {
1338 state->async.fn(state);
1346 static int ctdb_call_destructor(struct ctdb_call_state *state)
1348 DLIST_REMOVE(state->ctdb_db->pending_calls, state);
1349 reqid_remove(state->ctdb_db->ctdb->idr, state->reqid);
1355 called when a ctdb_call needs to be resent after a reconfigure event
1357 static void ctdb_call_resend(struct ctdb_call_state *state)
1359 struct ctdb_context *ctdb = state->ctdb_db->ctdb;
1361 state->generation = state->ctdb_db->generation;
1363 /* use a new reqid, in case the old reply does eventually come in */
1364 reqid_remove(ctdb->idr, state->reqid);
1365 state->reqid = reqid_new(ctdb->idr, state);
1366 state->c->hdr.reqid = state->reqid;
1368 /* update the generation count for this request, so its valid with the new vnn_map */
1369 state->c->hdr.generation = state->generation;
1371 /* send the packet to ourselves, it will be redirected appropriately */
1372 state->c->hdr.destnode = ctdb->pnn;
1374 ctdb_queue_packet(ctdb, &state->c->hdr);
1375 DEBUG(DEBUG_NOTICE,("resent ctdb_call for db %s reqid %u generation %u\n",
1376 state->ctdb_db->db_name, state->reqid, state->generation));
1380 resend all pending calls on recovery
1382 void ctdb_call_resend_db(struct ctdb_db_context *ctdb_db)
1384 struct ctdb_call_state *state, *next;
1386 for (state = ctdb_db->pending_calls; state; state = next) {
1388 ctdb_call_resend(state);
1392 void ctdb_call_resend_all(struct ctdb_context *ctdb)
1394 struct ctdb_db_context *ctdb_db;
1396 for (ctdb_db = ctdb->db_list; ctdb_db; ctdb_db = ctdb_db->next) {
1397 ctdb_call_resend_db(ctdb_db);
1402 this allows the caller to setup a async.fn
1404 static void call_local_trigger(struct tevent_context *ev,
1405 struct tevent_timer *te,
1406 struct timeval t, void *private_data)
1408 struct ctdb_call_state *state = talloc_get_type(private_data, struct ctdb_call_state);
1409 if (state->async.fn) {
1410 state->async.fn(state);
1416 construct an event driven local ctdb_call
1418 this is used so that locally processed ctdb_call requests are processed
1419 in an event driven manner
1421 struct ctdb_call_state *ctdb_call_local_send(struct ctdb_db_context *ctdb_db,
1422 struct ctdb_call *call,
1423 struct ctdb_ltdb_header *header,
1426 struct ctdb_call_state *state;
1427 struct ctdb_context *ctdb = ctdb_db->ctdb;
1430 state = talloc_zero(ctdb_db, struct ctdb_call_state);
1431 CTDB_NO_MEMORY_NULL(ctdb, state);
1433 talloc_steal(state, data->dptr);
1435 state->state = CTDB_CALL_DONE;
1436 state->call = talloc(state, struct ctdb_call);
1437 CTDB_NO_MEMORY_NULL(ctdb, state->call);
1438 *(state->call) = *call;
1439 state->ctdb_db = ctdb_db;
1441 ret = ctdb_call_local(ctdb_db, state->call, header, state, data, true);
1443 DEBUG(DEBUG_DEBUG,("ctdb_call_local() failed, ignoring return code %d\n", ret));
1446 tevent_add_timer(ctdb->ev, state, timeval_zero(),
1447 call_local_trigger, state);
1454 make a remote ctdb call - async send. Called in daemon context.
1456 This constructs a ctdb_call request and queues it for processing.
1457 This call never blocks.
1459 struct ctdb_call_state *ctdb_daemon_call_send_remote(struct ctdb_db_context *ctdb_db,
1460 struct ctdb_call *call,
1461 struct ctdb_ltdb_header *header)
1464 struct ctdb_call_state *state;
1465 struct ctdb_context *ctdb = ctdb_db->ctdb;
1467 if (ctdb->methods == NULL) {
1468 DEBUG(DEBUG_INFO,(__location__ " Failed send packet. Transport is down\n"));
1472 state = talloc_zero(ctdb_db, struct ctdb_call_state);
1473 CTDB_NO_MEMORY_NULL(ctdb, state);
1474 state->call = talloc(state, struct ctdb_call);
1475 CTDB_NO_MEMORY_NULL(ctdb, state->call);
1477 state->reqid = reqid_new(ctdb->idr, state);
1478 state->ctdb_db = ctdb_db;
1479 talloc_set_destructor(state, ctdb_call_destructor);
1481 len = offsetof(struct ctdb_req_call_old, data) + call->key.dsize + call->call_data.dsize;
1482 state->c = ctdb_transport_allocate(ctdb, state, CTDB_REQ_CALL, len,
1483 struct ctdb_req_call_old);
1484 CTDB_NO_MEMORY_NULL(ctdb, state->c);
1485 state->c->hdr.destnode = header->dmaster;
1487 /* this limits us to 16k outstanding messages - not unreasonable */
1488 state->c->hdr.reqid = state->reqid;
1489 state->c->hdr.generation = ctdb_db->generation;
1490 state->c->flags = call->flags;
1491 state->c->db_id = ctdb_db->db_id;
1492 state->c->callid = call->call_id;
1493 state->c->hopcount = 0;
1494 state->c->keylen = call->key.dsize;
1495 state->c->calldatalen = call->call_data.dsize;
1496 memcpy(&state->c->data[0], call->key.dptr, call->key.dsize);
1497 memcpy(&state->c->data[call->key.dsize],
1498 call->call_data.dptr, call->call_data.dsize);
1499 *(state->call) = *call;
1500 state->call->call_data.dptr = &state->c->data[call->key.dsize];
1501 state->call->key.dptr = &state->c->data[0];
1503 state->state = CTDB_CALL_WAIT;
1504 state->generation = ctdb_db->generation;
1506 DLIST_ADD(ctdb_db->pending_calls, state);
1508 ctdb_queue_packet(ctdb, &state->c->hdr);
1514 make a remote ctdb call - async recv - called in daemon context
1516 This is called when the program wants to wait for a ctdb_call to complete and get the
1517 results. This call will block unless the call has already completed.
1519 int ctdb_daemon_call_recv(struct ctdb_call_state *state, struct ctdb_call *call)
1521 while (state->state < CTDB_CALL_DONE) {
1522 tevent_loop_once(state->ctdb_db->ctdb->ev);
1524 if (state->state != CTDB_CALL_DONE) {
1525 ctdb_set_error(state->ctdb_db->ctdb, "%s", state->errmsg);
1530 if (state->call->reply_data.dsize) {
1531 call->reply_data.dptr = talloc_memdup(call,
1532 state->call->reply_data.dptr,
1533 state->call->reply_data.dsize);
1534 call->reply_data.dsize = state->call->reply_data.dsize;
1536 call->reply_data.dptr = NULL;
1537 call->reply_data.dsize = 0;
1539 call->status = state->call->status;
1546 send a keepalive packet to the other node
1548 void ctdb_send_keepalive(struct ctdb_context *ctdb, uint32_t destnode)
1550 struct ctdb_req_keepalive_old *r;
1552 if (ctdb->methods == NULL) {
1553 DEBUG(DEBUG_INFO,(__location__ " Failed to send keepalive. Transport is DOWN\n"));
1557 r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REQ_KEEPALIVE,
1558 sizeof(struct ctdb_req_keepalive_old),
1559 struct ctdb_req_keepalive_old);
1560 CTDB_NO_MEMORY_FATAL(ctdb, r);
1561 r->hdr.destnode = destnode;
1564 CTDB_INCREMENT_STAT(ctdb, keepalive_packets_sent);
1566 ctdb_queue_packet(ctdb, &r->hdr);
1573 struct revokechild_deferred_call {
1574 struct ctdb_context *ctdb;
1575 struct ctdb_req_header *hdr;
1576 deferred_requeue_fn fn;
1580 struct revokechild_handle {
1581 struct revokechild_handle *next, *prev;
1582 struct ctdb_context *ctdb;
1583 struct ctdb_db_context *ctdb_db;
1584 struct tevent_fd *fde;
1591 struct revokechild_requeue_handle {
1592 struct ctdb_context *ctdb;
1593 struct ctdb_req_header *hdr;
1594 deferred_requeue_fn fn;
1598 static void deferred_call_requeue(struct tevent_context *ev,
1599 struct tevent_timer *te,
1600 struct timeval t, void *private_data)
1602 struct revokechild_requeue_handle *requeue_handle = talloc_get_type(private_data, struct revokechild_requeue_handle);
1604 requeue_handle->fn(requeue_handle->ctx, requeue_handle->hdr);
1605 talloc_free(requeue_handle);
1608 static int deferred_call_destructor(struct revokechild_deferred_call *deferred_call)
1610 struct ctdb_context *ctdb = deferred_call->ctdb;
1611 struct revokechild_requeue_handle *requeue_handle = talloc(ctdb, struct revokechild_requeue_handle);
1613 requeue_handle->ctdb = ctdb;
1614 requeue_handle->hdr = deferred_call->hdr;
1615 requeue_handle->fn = deferred_call->fn;
1616 requeue_handle->ctx = deferred_call->ctx;
1617 talloc_steal(requeue_handle, requeue_handle->hdr);
1619 /* Always delay revoke requests. Either wait for the read/write
1620 * operation to complete, or if revoking failed wait for recovery to
1623 tevent_add_timer(ctdb->ev, requeue_handle,
1624 timeval_current_ofs(1, 0),
1625 deferred_call_requeue, requeue_handle);
1631 static int revokechild_destructor(struct revokechild_handle *rc)
1633 if (rc->fde != NULL) {
1634 talloc_free(rc->fde);
1637 if (rc->fd[0] != -1) {
1640 if (rc->fd[1] != -1) {
1643 ctdb_kill(rc->ctdb, rc->child, SIGKILL);
1645 DLIST_REMOVE(rc->ctdb_db->revokechild_active, rc);
1649 static void revokechild_handler(struct tevent_context *ev,
1650 struct tevent_fd *fde,
1651 uint16_t flags, void *private_data)
1653 struct revokechild_handle *rc = talloc_get_type(private_data,
1654 struct revokechild_handle);
1658 ret = sys_read(rc->fd[0], &c, 1);
1660 DEBUG(DEBUG_ERR,("Failed to read status from revokechild. errno:%d\n", errno));
1666 DEBUG(DEBUG_ERR,("revokechild returned failure. status:%d\n", c));
1675 struct ctdb_revoke_state {
1676 struct ctdb_db_context *ctdb_db;
1678 struct ctdb_ltdb_header *header;
1685 static void update_record_cb(struct ctdb_client_control_state *state)
1687 struct ctdb_revoke_state *revoke_state;
1691 if (state == NULL) {
1694 revoke_state = state->async.private_data;
1696 state->async.fn = NULL;
1697 ret = ctdb_control_recv(state->ctdb, state, state, NULL, &res, NULL);
1698 if ((ret != 0) || (res != 0)) {
1699 DEBUG(DEBUG_ERR,("Recv for revoke update record failed ret:%d res:%d\n", ret, res));
1700 revoke_state->status = -1;
1703 revoke_state->count--;
1704 if (revoke_state->count <= 0) {
1705 revoke_state->finished = 1;
1709 static void revoke_send_cb(struct ctdb_context *ctdb, uint32_t pnn, void *private_data)
1711 struct ctdb_revoke_state *revoke_state = private_data;
1712 struct ctdb_client_control_state *state;
1714 state = ctdb_ctrl_updaterecord_send(ctdb, revoke_state, timeval_current_ofs(ctdb->tunable.control_timeout,0), pnn, revoke_state->ctdb_db, revoke_state->key, revoke_state->header, revoke_state->data);
1715 if (state == NULL) {
1716 DEBUG(DEBUG_ERR,("Failure to send update record to revoke readonly delegation\n"));
1717 revoke_state->status = -1;
1720 state->async.fn = update_record_cb;
1721 state->async.private_data = revoke_state;
1723 revoke_state->count++;
1727 static void ctdb_revoke_timeout_handler(struct tevent_context *ev,
1728 struct tevent_timer *te,
1729 struct timeval yt, void *private_data)
1731 struct ctdb_revoke_state *state = private_data;
1733 DEBUG(DEBUG_ERR,("Timed out waiting for revoke to finish\n"));
1734 state->finished = 1;
1738 static int ctdb_revoke_all_delegations(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA tdata, TDB_DATA key, struct ctdb_ltdb_header *header, TDB_DATA data)
1740 struct ctdb_revoke_state *state = talloc_zero(ctdb, struct ctdb_revoke_state);
1741 struct ctdb_ltdb_header new_header;
1744 state->ctdb_db = ctdb_db;
1746 state->header = header;
1749 ctdb_trackingdb_traverse(ctdb, tdata, revoke_send_cb, state);
1751 tevent_add_timer(ctdb->ev, state,
1752 timeval_current_ofs(ctdb->tunable.control_timeout, 0),
1753 ctdb_revoke_timeout_handler, state);
1755 while (state->finished == 0) {
1756 tevent_loop_once(ctdb->ev);
1759 if (ctdb_ltdb_lock(ctdb_db, key) != 0) {
1760 DEBUG(DEBUG_ERR,("Failed to chainlock the database in revokechild\n"));
1764 if (ctdb_ltdb_fetch(ctdb_db, key, &new_header, state, &new_data) != 0) {
1765 ctdb_ltdb_unlock(ctdb_db, key);
1766 DEBUG(DEBUG_ERR,("Failed for fetch tdb record in revokechild\n"));
1771 if (new_header.rsn > header->rsn) {
1772 ctdb_ltdb_unlock(ctdb_db, key);
1773 DEBUG(DEBUG_ERR,("RSN too high in tdb record in revokechild\n"));
1777 if ( (new_header.flags & (CTDB_REC_RO_REVOKING_READONLY|CTDB_REC_RO_HAVE_DELEGATIONS)) != (CTDB_REC_RO_REVOKING_READONLY|CTDB_REC_RO_HAVE_DELEGATIONS) ) {
1778 ctdb_ltdb_unlock(ctdb_db, key);
1779 DEBUG(DEBUG_ERR,("Flags are wrong in tdb record in revokechild\n"));
1785 * If revoke on all nodes succeed, revoke is complete. Otherwise,
1786 * remove CTDB_REC_RO_REVOKING_READONLY flag and retry.
1788 if (state->status == 0) {
1790 new_header.flags |= CTDB_REC_RO_REVOKE_COMPLETE;
1792 DEBUG(DEBUG_NOTICE, ("Revoke all delegations failed, retrying.\n"));
1793 new_header.flags &= ~CTDB_REC_RO_REVOKING_READONLY;
1795 if (ctdb_ltdb_store(ctdb_db, key, &new_header, new_data) != 0) {
1796 ctdb_ltdb_unlock(ctdb_db, key);
1797 DEBUG(DEBUG_ERR,("Failed to write new record in revokechild\n"));
1801 ctdb_ltdb_unlock(ctdb_db, key);
1808 int ctdb_start_revoke_ro_record(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_ltdb_header *header, TDB_DATA data)
1811 struct revokechild_handle *rc;
1812 pid_t parent = getpid();
1815 header->flags &= ~(CTDB_REC_RO_REVOKING_READONLY|CTDB_REC_RO_HAVE_DELEGATIONS|CTDB_REC_RO_HAVE_READONLY);
1816 header->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA;
1819 if ((rc = talloc_zero(ctdb_db, struct revokechild_handle)) == NULL) {
1820 DEBUG(DEBUG_ERR,("Failed to allocate revokechild_handle\n"));
1824 tdata = tdb_fetch(ctdb_db->rottdb, key);
1825 if (tdata.dsize > 0) {
1829 tdata.dptr = talloc_memdup(rc, tdata.dptr, tdata.dsize);
1835 rc->ctdb_db = ctdb_db;
1839 talloc_set_destructor(rc, revokechild_destructor);
1841 rc->key.dsize = key.dsize;
1842 rc->key.dptr = talloc_memdup(rc, key.dptr, key.dsize);
1843 if (rc->key.dptr == NULL) {
1844 DEBUG(DEBUG_ERR,("Failed to allocate key for revokechild_handle\n"));
1851 DEBUG(DEBUG_ERR,("Failed to allocate key for revokechild_handle\n"));
1857 rc->child = ctdb_fork(ctdb);
1858 if (rc->child == (pid_t)-1) {
1859 DEBUG(DEBUG_ERR,("Failed to fork child for revokechild\n"));
1864 if (rc->child == 0) {
1868 prctl_set_comment("ctdb_revokechild");
1869 if (switch_from_server_to_client(ctdb) != 0) {
1870 DEBUG(DEBUG_ERR,("Failed to switch from server to client for revokechild process\n"));
1872 goto child_finished;
1875 c = ctdb_revoke_all_delegations(ctdb, ctdb_db, tdata, key, header, data);
1878 sys_write(rc->fd[1], &c, 1);
1879 ctdb_wait_for_process_to_exit(parent);
1885 set_close_on_exec(rc->fd[0]);
1887 /* This is an active revokechild child process */
1888 DLIST_ADD_END(ctdb_db->revokechild_active, rc);
1890 rc->fde = tevent_add_fd(ctdb->ev, rc, rc->fd[0], TEVENT_FD_READ,
1891 revokechild_handler, (void *)rc);
1892 if (rc->fde == NULL) {
1893 DEBUG(DEBUG_ERR,("Failed to set up fd event for revokechild process\n"));
1896 tevent_fd_set_auto_close(rc->fde);
1901 int ctdb_add_revoke_deferred_call(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_req_header *hdr, deferred_requeue_fn fn, void *call_context)
1903 struct revokechild_handle *rc;
1904 struct revokechild_deferred_call *deferred_call;
1906 for (rc = ctdb_db->revokechild_active; rc; rc = rc->next) {
1907 if (rc->key.dsize == 0) {
1910 if (rc->key.dsize != key.dsize) {
1913 if (!memcmp(rc->key.dptr, key.dptr, key.dsize)) {
1919 DEBUG(DEBUG_ERR,("Failed to add deferred call to revoke list. revoke structure not found\n"));
1923 deferred_call = talloc(rc, struct revokechild_deferred_call);
1924 if (deferred_call == NULL) {
1925 DEBUG(DEBUG_ERR,("Failed to allocate deferred call structure for revoking record\n"));
1929 deferred_call->ctdb = ctdb;
1930 deferred_call->hdr = hdr;
1931 deferred_call->fn = fn;
1932 deferred_call->ctx = call_context;
1934 talloc_set_destructor(deferred_call, deferred_call_destructor);
1935 talloc_steal(deferred_call, hdr);
1940 static void ctdb_migration_count_handler(TDB_DATA key, uint64_t counter,
1943 struct ctdb_db_context *ctdb_db = talloc_get_type_abort(
1944 private_data, struct ctdb_db_context);
1947 value = (counter < INT_MAX ? counter : INT_MAX);
1948 ctdb_update_db_stat_hot_keys(ctdb_db, key, value);
1951 static void ctdb_migration_cleandb_event(struct tevent_context *ev,
1952 struct tevent_timer *te,
1953 struct timeval current_time,
1956 struct ctdb_db_context *ctdb_db = talloc_get_type_abort(
1957 private_data, struct ctdb_db_context);
1959 if (ctdb_db->migratedb == NULL) {
1963 hash_count_expire(ctdb_db->migratedb, NULL);
1965 te = tevent_add_timer(ctdb_db->ctdb->ev, ctdb_db->migratedb,
1966 tevent_timeval_current_ofs(10, 0),
1967 ctdb_migration_cleandb_event, ctdb_db);
1970 ("Memory error in migration cleandb event for %s\n",
1972 TALLOC_FREE(ctdb_db->migratedb);
1976 int ctdb_migration_init(struct ctdb_db_context *ctdb_db)
1978 struct timeval one_second = { 1, 0 };
1979 struct tevent_timer *te;
1982 if (ctdb_db->persistent) {
1986 ret = hash_count_init(ctdb_db, one_second,
1987 ctdb_migration_count_handler, ctdb_db,
1988 &ctdb_db->migratedb);
1991 ("Memory error in migration init for %s\n",
1996 te = tevent_add_timer(ctdb_db->ctdb->ev, ctdb_db->migratedb,
1997 tevent_timeval_current_ofs(10, 0),
1998 ctdb_migration_cleandb_event, ctdb_db);
2001 ("Memory error in migration init for %s\n",
2003 TALLOC_FREE(ctdb_db->migratedb);