3 provide API to do non-blocking locks for single or all databases
5 Copyright (C) Amitay Isaacs 2012
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #include "system/filesys.h"
22 #include "system/network.h"
27 #include "lib/tdb_wrap/tdb_wrap.h"
28 #include "lib/util/dlinklist.h"
29 #include "lib/util/debug.h"
30 #include "lib/util/samba_util.h"
31 #include "lib/util/sys_rw.h"
33 #include "ctdb_private.h"
35 #include "common/common.h"
36 #include "common/logging.h"
39 * Non-blocking Locking API
41 * 1. Create a child process to do blocking locks.
42 * 2. Once the locks are obtained, signal parent process via fd.
43 * 3. Invoke registered callback routine with locking status.
44 * 4. If the child process cannot get locks within certain time,
45 * execute an external script to debug.
47 * ctdb_lock_record() - get a lock on a record
48 * ctdb_lock_db() - get a lock on a DB
50 * auto_mark - whether to mark/unmark DBs in before/after callback
51 * = false is used for freezing databases for
52 * recovery since the recovery cannot start till
53 * databases are locked on all the nodes.
54 * = true is used for record locks.
62 static const char * const lock_type_str[] = {
69 /* lock_context is the common part for a lock request */
71 struct lock_context *next, *prev;
73 struct ctdb_context *ctdb;
74 struct ctdb_db_context *ctdb_db;
78 struct lock_request *request;
81 struct tevent_fd *tfd;
82 struct tevent_timer *ttimer;
83 struct timeval start_time;
88 /* lock_request is the client specific part for a lock request */
90 struct lock_context *lctx;
91 void (*callback)(void *, bool);
96 int ctdb_db_iterator(struct ctdb_context *ctdb, ctdb_db_handler_t handler,
99 struct ctdb_db_context *ctdb_db;
102 for (ctdb_db = ctdb->db_list; ctdb_db; ctdb_db = ctdb_db->next) {
103 ret = handler(ctdb_db, private_data);
113 * lock all databases - mark only
115 static int db_lock_mark_handler(struct ctdb_db_context *ctdb_db,
118 int tdb_transaction_write_lock_mark(struct tdb_context *);
120 DEBUG(DEBUG_INFO, ("marking locked database %s\n", ctdb_db->db_name));
122 if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
123 DEBUG(DEBUG_ERR, ("Failed to mark (transaction lock) database %s\n",
128 if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
129 DEBUG(DEBUG_ERR, ("Failed to mark (all lock) database %s\n",
137 int ctdb_lockdb_mark(struct ctdb_db_context *ctdb_db)
139 if (!ctdb_db_frozen(ctdb_db)) {
141 ("Attempt to mark database locked when not frozen\n"));
145 return db_lock_mark_handler(ctdb_db, NULL);
149 * lock all databases - unmark only
151 static int db_lock_unmark_handler(struct ctdb_db_context *ctdb_db,
154 int tdb_transaction_write_lock_unmark(struct tdb_context *);
156 DEBUG(DEBUG_INFO, ("unmarking locked database %s\n", ctdb_db->db_name));
158 if (tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb) != 0) {
159 DEBUG(DEBUG_ERR, ("Failed to unmark (transaction lock) database %s\n",
164 if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
165 DEBUG(DEBUG_ERR, ("Failed to unmark (all lock) database %s\n",
173 int ctdb_lockdb_unmark(struct ctdb_db_context *ctdb_db)
175 if (!ctdb_db_frozen(ctdb_db)) {
177 ("Attempt to unmark database locked when not frozen\n"));
181 return db_lock_unmark_handler(ctdb_db, NULL);
184 static void ctdb_lock_schedule(struct ctdb_context *ctdb);
187 * Destructor to kill the child locking process
189 static int ctdb_lock_context_destructor(struct lock_context *lock_ctx)
191 if (lock_ctx->request) {
192 lock_ctx->request->lctx = NULL;
194 if (lock_ctx->child > 0) {
195 ctdb_kill(lock_ctx->ctdb, lock_ctx->child, SIGTERM);
196 if (lock_ctx->type == LOCK_RECORD) {
197 DLIST_REMOVE(lock_ctx->ctdb_db->lock_current, lock_ctx);
199 DLIST_REMOVE(lock_ctx->ctdb->lock_current, lock_ctx);
201 if (lock_ctx->ctdb_db) {
202 lock_ctx->ctdb_db->lock_num_current--;
204 CTDB_DECREMENT_STAT(lock_ctx->ctdb, locks.num_current);
205 if (lock_ctx->ctdb_db) {
206 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_current);
209 if (lock_ctx->type == LOCK_RECORD) {
210 DLIST_REMOVE(lock_ctx->ctdb_db->lock_pending, lock_ctx);
212 DLIST_REMOVE(lock_ctx->ctdb->lock_pending, lock_ctx);
214 CTDB_DECREMENT_STAT(lock_ctx->ctdb, locks.num_pending);
215 if (lock_ctx->ctdb_db) {
216 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_pending);
220 ctdb_lock_schedule(lock_ctx->ctdb);
227 * Destructor to remove lock request
229 static int ctdb_lock_request_destructor(struct lock_request *lock_request)
231 if (lock_request->lctx == NULL) {
235 lock_request->lctx->request = NULL;
236 TALLOC_FREE(lock_request->lctx);
242 * Process all the callbacks waiting for lock
244 * If lock has failed, callback is executed with locked=false
246 static void process_callbacks(struct lock_context *lock_ctx, bool locked)
248 struct lock_request *request;
249 bool auto_mark = lock_ctx->auto_mark;
251 if (auto_mark && locked) {
252 switch (lock_ctx->type) {
254 tdb_chainlock_mark(lock_ctx->ctdb_db->ltdb->tdb, lock_ctx->key);
258 ctdb_lockdb_mark(lock_ctx->ctdb_db);
263 request = lock_ctx->request;
265 /* Since request may be freed in the callback, unset the lock
266 * context, so request destructor will not free lock context.
268 request->lctx = NULL;
271 /* Since request may be freed in the callback, unset the request */
272 lock_ctx->request = NULL;
274 request->callback(request->private_data, locked);
281 switch (lock_ctx->type) {
283 tdb_chainlock_unmark(lock_ctx->ctdb_db->ltdb->tdb, lock_ctx->key);
287 ctdb_lockdb_unmark(lock_ctx->ctdb_db);
292 talloc_free(lock_ctx);
296 static int lock_bucket_id(double t)
298 double ms = 1.e-3, s = 1;
303 } else if (t < 10*ms) {
305 } else if (t < 100*ms) {
307 } else if (t < 1*s) {
309 } else if (t < 2*s) {
311 } else if (t < 4*s) {
313 } else if (t < 8*s) {
315 } else if (t < 16*s) {
317 } else if (t < 32*s) {
319 } else if (t < 64*s) {
329 * Callback routine when the required locks are obtained.
330 * Called from parent context
332 static void ctdb_lock_handler(struct tevent_context *ev,
333 struct tevent_fd *tfd,
337 struct lock_context *lock_ctx;
343 lock_ctx = talloc_get_type_abort(private_data, struct lock_context);
345 /* cancel the timeout event */
346 TALLOC_FREE(lock_ctx->ttimer);
348 t = timeval_elapsed(&lock_ctx->start_time);
349 id = lock_bucket_id(t);
351 /* Read the status from the child process */
352 if (sys_read(lock_ctx->fd[0], &c, 1) != 1) {
355 locked = (c == 0 ? true : false);
358 /* Update statistics */
359 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.num_calls);
360 if (lock_ctx->ctdb_db) {
361 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_calls);
365 if (lock_ctx->ctdb_db) {
366 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.buckets[id]);
367 CTDB_UPDATE_LATENCY(lock_ctx->ctdb, lock_ctx->ctdb_db,
368 lock_type_str[lock_ctx->type], locks.latency,
369 lock_ctx->start_time);
371 CTDB_UPDATE_DB_LATENCY(lock_ctx->ctdb_db, lock_type_str[lock_ctx->type], locks.latency, t);
372 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.buckets[id]);
375 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.num_failed);
376 if (lock_ctx->ctdb_db) {
377 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_failed);
381 process_callbacks(lock_ctx, locked);
386 * Callback routine when required locks are not obtained within timeout
387 * Called from parent context
389 static void ctdb_lock_timeout_handler(struct tevent_context *ev,
390 struct tevent_timer *ttimer,
391 struct timeval current_time,
394 static char debug_locks[PATH_MAX+1] = "";
395 static struct timeval last_debug_time;
396 struct lock_context *lock_ctx;
397 struct ctdb_context *ctdb;
403 lock_ctx = talloc_get_type_abort(private_data, struct lock_context);
404 ctdb = lock_ctx->ctdb;
406 elapsed_time = timeval_elapsed(&lock_ctx->start_time);
408 ("Unable to get %s lock on database %s for %.0lf seconds\n",
409 (lock_ctx->type == LOCK_RECORD ? "RECORD" : "DB"),
410 lock_ctx->ctdb_db->db_name, elapsed_time));
412 /* If a node stopped/banned, don't spam the logs */
413 if (ctdb->nodes[ctdb->pnn]->flags & NODE_FLAGS_INACTIVE) {
414 goto skip_lock_debug;
417 /* Restrict log debugging to once per second */
418 now = timeval_current();
419 if (last_debug_time.tv_sec == now.tv_sec) {
420 goto skip_lock_debug;
423 last_debug_time.tv_sec = now.tv_sec;
425 if (ctdb_set_helper("lock debugging helper",
426 debug_locks, sizeof(debug_locks),
428 getenv("CTDB_BASE"), "debug_locks.sh")) {
431 execl(debug_locks, debug_locks, NULL);
434 ctdb_track_child(ctdb, pid);
438 " Unable to setup lock debugging\n"));
443 /* Back-off logging if lock is not obtained for a long time */
444 if (elapsed_time < 100.0) {
446 } else if (elapsed_time < 1000.0) {
452 /* reset the timeout timer */
453 // talloc_free(lock_ctx->ttimer);
454 lock_ctx->ttimer = tevent_add_timer(ctdb->ev,
456 timeval_current_ofs(new_timer, 0),
457 ctdb_lock_timeout_handler,
461 static int db_flags(struct ctdb_db_context *ctdb_db)
463 int tdb_flags = TDB_DEFAULT;
465 #ifdef TDB_MUTEX_LOCKING
466 if (!ctdb_db->persistent && ctdb_db->ctdb->tunable.mutex_enabled) {
467 tdb_flags = (TDB_MUTEX_LOCKING | TDB_CLEAR_IF_FIRST);
473 static bool lock_helper_args(TALLOC_CTX *mem_ctx,
474 struct lock_context *lock_ctx, int fd,
475 int *argc, const char ***argv)
477 const char **args = NULL;
480 switch (lock_ctx->type) {
490 /* Add extra argument for null termination */
493 args = talloc_array(mem_ctx, const char *, nargs);
498 args[0] = talloc_asprintf(args, "%d", getpid());
499 args[1] = talloc_asprintf(args, "%d", fd);
501 switch (lock_ctx->type) {
503 args[2] = talloc_strdup(args, "RECORD");
504 args[3] = talloc_strdup(args, lock_ctx->ctdb_db->db_path);
505 args[4] = talloc_asprintf(args, "0x%x",
506 db_flags(lock_ctx->ctdb_db));
507 if (lock_ctx->key.dsize == 0) {
508 args[5] = talloc_strdup(args, "NULL");
510 args[5] = hex_encode_talloc(args, lock_ctx->key.dptr, lock_ctx->key.dsize);
515 args[2] = talloc_strdup(args, "DB");
516 args[3] = talloc_strdup(args, lock_ctx->ctdb_db->db_path);
517 args[4] = talloc_asprintf(args, "0x%x",
518 db_flags(lock_ctx->ctdb_db));
522 /* Make sure last argument is NULL */
523 args[nargs-1] = NULL;
525 for (i=0; i<nargs-1; i++) {
526 if (args[i] == NULL) {
538 * Find a lock request that can be scheduled
540 static struct lock_context *ctdb_find_lock_context(struct ctdb_context *ctdb)
542 struct lock_context *lock_ctx, *next_ctx;
543 struct ctdb_db_context *ctdb_db;
545 /* First check if there are database lock requests */
547 for (lock_ctx = ctdb->lock_pending; lock_ctx != NULL;
548 lock_ctx = next_ctx) {
550 if (lock_ctx->request != NULL) {
551 /* Found a lock context with a request */
555 next_ctx = lock_ctx->next;
557 DEBUG(DEBUG_INFO, ("Removing lock context without lock "
559 DLIST_REMOVE(ctdb->lock_pending, lock_ctx);
560 CTDB_DECREMENT_STAT(ctdb, locks.num_pending);
561 if (lock_ctx->ctdb_db) {
562 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db,
565 talloc_free(lock_ctx);
568 /* Next check database queues */
569 for (ctdb_db = ctdb->db_list; ctdb_db; ctdb_db = ctdb_db->next) {
570 if (ctdb_db->lock_num_current ==
571 ctdb->tunable.lock_processes_per_db) {
575 for (lock_ctx = ctdb_db->lock_pending; lock_ctx != NULL;
576 lock_ctx = next_ctx) {
578 next_ctx = lock_ctx->next;
580 if (lock_ctx->request != NULL) {
584 DEBUG(DEBUG_INFO, ("Removing lock context without "
586 DLIST_REMOVE(ctdb_db->lock_pending, lock_ctx);
587 CTDB_DECREMENT_STAT(ctdb, locks.num_pending);
588 CTDB_DECREMENT_DB_STAT(ctdb_db, locks.num_pending);
589 talloc_free(lock_ctx);
597 * Schedule a new lock child process
598 * Set up callback handler and timeout handler
600 static void ctdb_lock_schedule(struct ctdb_context *ctdb)
602 struct lock_context *lock_ctx;
605 static char prog[PATH_MAX+1] = "";
608 if (!ctdb_set_helper("lock helper",
611 CTDB_HELPER_BINDIR, "ctdb_lock_helper")) {
612 ctdb_die(ctdb, __location__
613 " Unable to set lock helper\n");
616 /* Find a lock context with requests */
617 lock_ctx = ctdb_find_lock_context(ctdb);
618 if (lock_ctx == NULL) {
622 lock_ctx->child = -1;
623 ret = pipe(lock_ctx->fd);
625 DEBUG(DEBUG_ERR, ("Failed to create pipe in ctdb_lock_schedule\n"));
629 set_close_on_exec(lock_ctx->fd[0]);
631 /* Create data for child process */
632 tmp_ctx = talloc_new(lock_ctx);
633 if (tmp_ctx == NULL) {
634 DEBUG(DEBUG_ERR, ("Failed to allocate memory for helper args\n"));
635 close(lock_ctx->fd[0]);
636 close(lock_ctx->fd[1]);
640 if (! ctdb->do_setsched) {
641 ret = setenv("CTDB_NOSETSCHED", "1", 1);
644 ("Failed to set CTDB_NOSETSCHED variable\n"));
648 /* Create arguments for lock helper */
649 if (!lock_helper_args(tmp_ctx, lock_ctx, lock_ctx->fd[1],
651 DEBUG(DEBUG_ERR, ("Failed to create lock helper args\n"));
652 close(lock_ctx->fd[0]);
653 close(lock_ctx->fd[1]);
654 talloc_free(tmp_ctx);
658 lock_ctx->child = ctdb_vfork_exec(lock_ctx, ctdb, prog, argc,
659 (const char **)args);
660 if (lock_ctx->child == -1) {
661 DEBUG(DEBUG_ERR, ("Failed to create a child in ctdb_lock_schedule\n"));
662 close(lock_ctx->fd[0]);
663 close(lock_ctx->fd[1]);
664 talloc_free(tmp_ctx);
669 close(lock_ctx->fd[1]);
671 talloc_free(tmp_ctx);
673 /* Set up timeout handler */
674 lock_ctx->ttimer = tevent_add_timer(ctdb->ev,
676 timeval_current_ofs(10, 0),
677 ctdb_lock_timeout_handler,
679 if (lock_ctx->ttimer == NULL) {
680 ctdb_kill(ctdb, lock_ctx->child, SIGTERM);
681 lock_ctx->child = -1;
682 close(lock_ctx->fd[0]);
686 /* Set up callback */
687 lock_ctx->tfd = tevent_add_fd(ctdb->ev,
693 if (lock_ctx->tfd == NULL) {
694 TALLOC_FREE(lock_ctx->ttimer);
695 ctdb_kill(ctdb, lock_ctx->child, SIGTERM);
696 lock_ctx->child = -1;
697 close(lock_ctx->fd[0]);
700 tevent_fd_set_auto_close(lock_ctx->tfd);
702 /* Move the context from pending to current */
703 if (lock_ctx->type == LOCK_RECORD) {
704 DLIST_REMOVE(lock_ctx->ctdb_db->lock_pending, lock_ctx);
705 DLIST_ADD_END(lock_ctx->ctdb_db->lock_current, lock_ctx);
707 DLIST_REMOVE(ctdb->lock_pending, lock_ctx);
708 DLIST_ADD_END(ctdb->lock_current, lock_ctx);
710 CTDB_DECREMENT_STAT(lock_ctx->ctdb, locks.num_pending);
711 CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.num_current);
712 if (lock_ctx->ctdb_db) {
713 lock_ctx->ctdb_db->lock_num_current++;
714 CTDB_DECREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_pending);
715 CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db, locks.num_current);
721 * Lock record / db depending on type
723 static struct lock_request *ctdb_lock_internal(TALLOC_CTX *mem_ctx,
724 struct ctdb_context *ctdb,
725 struct ctdb_db_context *ctdb_db,
728 void (*callback)(void *, bool),
733 struct lock_context *lock_ctx = NULL;
734 struct lock_request *request;
736 if (callback == NULL) {
737 DEBUG(DEBUG_WARNING, ("No callback function specified, not locking\n"));
741 lock_ctx = talloc_zero(ctdb, struct lock_context);
742 if (lock_ctx == NULL) {
743 DEBUG(DEBUG_ERR, ("Failed to create a new lock context\n"));
747 if ((request = talloc_zero(mem_ctx, struct lock_request)) == NULL) {
748 talloc_free(lock_ctx);
752 lock_ctx->type = type;
753 lock_ctx->ctdb = ctdb;
754 lock_ctx->ctdb_db = ctdb_db;
755 lock_ctx->key.dsize = key.dsize;
757 lock_ctx->key.dptr = talloc_memdup(lock_ctx, key.dptr, key.dsize);
758 if (lock_ctx->key.dptr == NULL) {
759 DEBUG(DEBUG_ERR, (__location__ "Memory allocation error\n"));
760 talloc_free(lock_ctx);
761 talloc_free(request);
764 lock_ctx->key_hash = ctdb_hash(&key);
766 lock_ctx->key.dptr = NULL;
768 lock_ctx->priority = priority;
769 lock_ctx->auto_mark = auto_mark;
771 lock_ctx->request = request;
772 lock_ctx->child = -1;
774 /* Non-record locks are required by recovery and should be scheduled
775 * immediately, so keep them at the head of the pending queue.
777 if (lock_ctx->type == LOCK_RECORD) {
778 DLIST_ADD_END(ctdb_db->lock_pending, lock_ctx);
780 DLIST_ADD_END(ctdb->lock_pending, lock_ctx);
782 CTDB_INCREMENT_STAT(ctdb, locks.num_pending);
784 CTDB_INCREMENT_DB_STAT(ctdb_db, locks.num_pending);
787 /* Start the timer when we activate the context */
788 lock_ctx->start_time = timeval_current();
790 request->lctx = lock_ctx;
791 request->callback = callback;
792 request->private_data = private_data;
794 talloc_set_destructor(request, ctdb_lock_request_destructor);
795 talloc_set_destructor(lock_ctx, ctdb_lock_context_destructor);
797 ctdb_lock_schedule(ctdb);
804 * obtain a lock on a record in a database
806 struct lock_request *ctdb_lock_record(TALLOC_CTX *mem_ctx,
807 struct ctdb_db_context *ctdb_db,
810 void (*callback)(void *, bool),
813 return ctdb_lock_internal(mem_ctx,
826 * obtain a lock on a database
828 struct lock_request *ctdb_lock_db(TALLOC_CTX *mem_ctx,
829 struct ctdb_db_context *ctdb_db,
831 void (*callback)(void *, bool),
834 return ctdb_lock_internal(mem_ctx,