ctdb-cluster-mutex: Factor out cluster mutex code
authorMartin Schwenke <martin@meltin.net>
Wed, 17 Feb 2016 03:32:03 +0000 (14:32 +1100)
committerAmitay Isaacs <amitay@samba.org>
Thu, 28 Apr 2016 07:39:17 +0000 (09:39 +0200)
Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
ctdb/server/ctdb_cluster_mutex.c [new file with mode: 0644]
ctdb/server/ctdb_cluster_mutex.h [new file with mode: 0644]
ctdb/server/ctdb_recover.c
ctdb/tests/src/ctdbd_test.c
ctdb/wscript

diff --git a/ctdb/server/ctdb_cluster_mutex.c b/ctdb/server/ctdb_cluster_mutex.c
new file mode 100644 (file)
index 0000000..12950c4
--- /dev/null
@@ -0,0 +1,266 @@
+/*
+   CTDB cluster mutex handling
+
+   Copyright (C) Andrew Tridgell  2007
+   Copyright (C) Ronnie Sahlberg  2007
+   Copyright (C) Martin Schwenke  2016
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <tevent.h>
+
+#include "replace.h"
+#include "system/network.h"
+
+#include "lib/util/debug.h"
+#include "lib/util/time.h"
+#include "lib/util/strv.h"
+#include "lib/util/strv_util.h"
+
+#include "ctdb_private.h"
+#include "common/common.h"
+#include "common/logging.h"
+#include "common/system.h"
+
+#include "ctdb_cluster_mutex.h"
+
+struct ctdb_cluster_mutex_handle {
+       struct ctdb_context *ctdb;
+       cluster_mutex_handler_t handler;
+       void *private_data;
+       int fd[2];
+       struct tevent_timer *te;
+       struct tevent_fd *fde;
+       pid_t child;
+       struct timeval start_time;
+};
+
+void ctdb_cluster_mutex_set_handler(struct ctdb_cluster_mutex_handle *h,
+                                   cluster_mutex_handler_t handler,
+                                   void *private_data)
+{
+       h->handler = handler;
+       h->private_data = private_data;
+}
+
+static void cluster_mutex_timeout(struct tevent_context *ev,
+                                 struct tevent_timer *te,
+                                 struct timeval t, void *private_data)
+{
+       struct ctdb_cluster_mutex_handle *h =
+               talloc_get_type(private_data, struct ctdb_cluster_mutex_handle);
+       double latency = timeval_elapsed(&h->start_time);
+
+       if (h->handler != NULL) {
+               h->handler(h->ctdb, '2', latency, h, h->private_data);
+       }
+}
+
+
+/* When the handle is freed it causes any child holding the mutex to
+ * be killed, thus freeing the mutex */
+static int cluster_mutex_destructor(struct ctdb_cluster_mutex_handle *h)
+{
+       if (h->fd[0] != -1) {
+               h->fd[0] = -1;
+       }
+       ctdb_kill(h->ctdb, h->child, SIGTERM);
+       return 0;
+}
+
+/* this is called when the client process has completed ctdb_recovery_lock()
+   and has written data back to us through the pipe.
+*/
+static void cluster_mutex_handler(struct tevent_context *ev,
+                                 struct tevent_fd *fde,
+                                 uint16_t flags, void *private_data)
+{
+       struct ctdb_cluster_mutex_handle *h=
+               talloc_get_type(private_data, struct ctdb_cluster_mutex_handle);
+       double latency = timeval_elapsed(&h->start_time);
+       char c = '0';
+       int ret;
+
+       /* Got response from child process so abort timeout */
+       TALLOC_FREE(h->te);
+
+       ret = sys_read(h->fd[0], &c, 1);
+
+       /* If the child wrote status then just pass it to the handler.
+        * If no status was written then this is an unexpected error
+        * so pass generic error code to handler. */
+       if (h->handler != NULL) {
+               h->handler(h->ctdb, ret == 1 ? c : '3', latency,
+                          h, h->private_data);
+       }
+}
+
+static char cluster_mutex_helper[PATH_MAX+1] = "";
+
+static bool cluster_mutex_helper_args(TALLOC_CTX *mem_ctx,
+                                     const char *argstring, char ***argv)
+{
+       int nargs, i, ret, n;
+       bool is_command = false;
+       char **args = NULL;
+       char *strv = NULL;
+       char *t = NULL;
+
+       if (argstring != NULL && argstring[0] == '!') {
+               /* This is actually a full command */
+               is_command = true;
+               t = discard_const(&argstring[1]);
+       } else {
+               is_command = false;
+               t = discard_const(argstring);
+       }
+
+       ret = strv_split(mem_ctx, &strv, t, " \t");
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,
+                     ("Unable to parse mutex helper string \"%s\" (%s)\n",
+                      argstring, strerror(ret)));
+               return false;
+       }
+       n = strv_count(strv);
+
+       args = talloc_array(mem_ctx, char *, n + (is_command ? 1 : 2));
+
+       if (args == NULL) {
+               DEBUG(DEBUG_ERR,(__location__ " out of memory\n"));
+               return false;
+       }
+
+       nargs = 0;
+
+       if (! is_command) {
+               if (!ctdb_set_helper("cluster mutex helper",
+                                    cluster_mutex_helper,
+                                    sizeof(cluster_mutex_helper),
+                                    "CTDB_CLUSTER_MUTEX_HELPER",
+                                    CTDB_HELPER_BINDIR,
+                                    "ctdb_mutex_fcntl_helper")) {
+                       DEBUG(DEBUG_ERR,("ctdb exiting with error: %s\n",
+                                        __location__
+                                        " Unable to set cluster mutex helper\n"));
+                       exit(1);
+               }
+
+               args[nargs++] = cluster_mutex_helper;
+       }
+
+       t = NULL;
+       for (i = 0; i < n; i++) {
+               /* Don't copy, just keep cmd_args around */
+               t = strv_next(strv, t);
+               args[nargs++] = t;
+       }
+
+       /* Make sure last argument is NULL */
+       args[nargs] = NULL;
+
+       *argv = args;
+       return true;
+}
+
+struct ctdb_cluster_mutex_handle *
+ctdb_cluster_mutex(struct ctdb_context *ctdb,
+                  const char *argstring,
+                  int timeout)
+{
+       struct ctdb_cluster_mutex_handle *h;
+       char **args;
+       int ret;
+
+       h = talloc(ctdb, struct ctdb_cluster_mutex_handle);
+       if (h == NULL) {
+               DEBUG(DEBUG_ERR, (__location__ " out of memory\n"));
+               return NULL;
+       }
+
+       h->start_time = timeval_current();
+       h->fd[0] = -1;
+       h->fd[1] = -1;
+
+       ret = pipe(h->fd);
+       if (ret != 0) {
+               talloc_free(h);
+               DEBUG(DEBUG_ERR, (__location__ " Failed to open pipe\n"));
+               return NULL;
+       }
+       set_close_on_exec(h->fd[0]);
+
+       /* Create arguments for lock helper */
+       if (!cluster_mutex_helper_args(h, argstring, &args)) {
+               close(h->fd[0]);
+               close(h->fd[1]);
+               talloc_free(h);
+               return NULL;
+       }
+
+       h->child = ctdb_fork(ctdb);
+       if (h->child == (pid_t)-1) {
+               close(h->fd[0]);
+               close(h->fd[1]);
+               talloc_free(h);
+               return NULL;
+       }
+
+       if (h->child == 0) {
+               /* Make stdout point to the pipe */
+               close(STDOUT_FILENO);
+               dup2(h->fd[1], STDOUT_FILENO);
+               close(h->fd[1]);
+
+               execv(args[0], args);
+
+               /* Only happens on error */
+               DEBUG(DEBUG_ERR, (__location__ "execv() failed\n"));
+               _exit(1);
+       }
+
+       /* Parent */
+
+       DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d\n", h->fd[0]));
+       set_close_on_exec(h->fd[0]);
+
+       close(h->fd[1]);
+       h->fd[1] = -1;
+
+       talloc_set_destructor(h, cluster_mutex_destructor);
+
+       if (timeout != 0) {
+               h->te = tevent_add_timer(ctdb->ev, h,
+                                        timeval_current_ofs(timeout, 0),
+                                        cluster_mutex_timeout, h);
+       } else {
+               h->te = NULL;
+       }
+
+       h->fde = tevent_add_fd(ctdb->ev, h, h->fd[0], TEVENT_FD_READ,
+                              cluster_mutex_handler, (void *)h);
+
+       if (h->fde == NULL) {
+               talloc_free(h);
+               return NULL;
+       }
+       tevent_fd_set_auto_close(h->fde);
+
+       h->ctdb = ctdb;
+       h->handler = NULL;
+       h->private_data = NULL;
+
+       return h;
+}
diff --git a/ctdb/server/ctdb_cluster_mutex.h b/ctdb/server/ctdb_cluster_mutex.h
new file mode 100644 (file)
index 0000000..9131eb3
--- /dev/null
@@ -0,0 +1,48 @@
+/*
+   CTDB cluster mutex handling
+
+   Copyright (C) Andrew Tridgell  2007
+   Copyright (C) Ronnie Sahlberg  2007
+   Copyright (C) Martin Schwenke  2016
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __CTDB_CLUSTER_MUTEX_H__
+#define __CTDB_CLUSTER_MUTEX_H__
+
+#include "replace.h"
+#include "system/network.h"
+
+#include "ctdb_private.h"
+
+struct ctdb_cluster_mutex_handle;
+
+typedef void (*cluster_mutex_handler_t) (
+       struct ctdb_context *ctdb,
+       char status,
+       double latency,
+       struct ctdb_cluster_mutex_handle *h,
+       void *private_data);
+
+void ctdb_cluster_mutex_set_handler(struct ctdb_cluster_mutex_handle *h,
+                                   cluster_mutex_handler_t handler,
+                                   void *private_data);
+
+struct ctdb_cluster_mutex_handle *
+ctdb_cluster_mutex(struct ctdb_context *ctdb,
+                  const char *argstring,
+                  int timeout);
+
+#endif /* __CTDB_IPALLOC_H__ */
index b8f71ff537769d0f00a31049103cb80d1cdda89d..fe7e6e9cb35f86faa67dafd3a7e14f70a420fe72 100644 (file)
@@ -32,8 +32,6 @@
 #include "lib/util/debug.h"
 #include "lib/util/time.h"
 #include "lib/util/util_process.h"
-#include "lib/util/strv.h"
-#include "lib/util/strv_util.h"
 
 #include "ctdb_private.h"
 #include "ctdb_client.h"
@@ -42,6 +40,8 @@
 #include "common/common.h"
 #include "common/logging.h"
 
+#include "ctdb_cluster_mutex.h"
+
 int 
 ctdb_control_getvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
 {
@@ -749,33 +749,6 @@ int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb,
        return 0;
 }
 
-struct ctdb_cluster_mutex_handle;
-typedef void (*cluster_mutex_handler_t) (
-       struct ctdb_context *ctdb,
-       char status,
-       double latency,
-       struct ctdb_cluster_mutex_handle *h,
-       void *private_data);
-
-struct ctdb_cluster_mutex_handle {
-       struct ctdb_context *ctdb;
-       cluster_mutex_handler_t handler;
-       void *private_data;
-       int fd[2];
-       struct tevent_timer *te;
-       struct tevent_fd *fde;
-       pid_t child;
-       struct timeval start_time;
-};
-
-static void ctdb_cluster_mutex_set_handler(struct ctdb_cluster_mutex_handle *h,
-                                          cluster_mutex_handler_t handler,
-                                          void *private_data)
-{
-       h->handler = handler;
-       h->private_data = private_data;
-}
-
 static void set_recmode_handler(struct ctdb_context *ctdb,
                                char status,
                                double latency,
@@ -839,62 +812,6 @@ static void set_recmode_handler(struct ctdb_context *ctdb,
        talloc_free(h);
 }
 
-/*
-  called if our set_recmode child times out. this would happen if
-  ctdb_recovery_lock() would block.
- */
-static void cluster_mutex_timeout(struct tevent_context *ev,
-                                 struct tevent_timer *te,
-                                 struct timeval t, void *private_data)
-{
-       struct ctdb_cluster_mutex_handle *h =
-               talloc_get_type(private_data, struct ctdb_cluster_mutex_handle);
-       double latency = timeval_elapsed(&h->start_time);
-
-       if (h->handler != NULL) {
-               h->handler(h->ctdb, '2', latency, h, h->private_data);
-       }
-}
-
-
-/* When the handle is freed it causes any child holding the mutex to
- * be killed, thus freeing the mutex */
-static int cluster_mutex_destructor(struct ctdb_cluster_mutex_handle *h)
-{
-       if (h->fd[0] != -1) {
-               h->fd[0] = -1;
-       }
-       ctdb_kill(h->ctdb, h->child, SIGTERM);
-       return 0;
-}
-
-/* this is called when the client process has completed ctdb_recovery_lock()
-   and has written data back to us through the pipe.
-*/
-static void cluster_mutex_handler(struct tevent_context *ev,
-                                 struct tevent_fd *fde,
-                                 uint16_t flags, void *private_data)
-{
-       struct ctdb_cluster_mutex_handle *h=
-               talloc_get_type(private_data, struct ctdb_cluster_mutex_handle);
-       double latency = timeval_elapsed(&h->start_time);
-       char c = '0';
-       int ret;
-
-       /* Got response from child process so abort timeout */
-       TALLOC_FREE(h->te);
-
-       ret = sys_read(h->fd[0], &c, 1);
-
-       /* If the child wrote status then just pass it to the handler.
-        * If no status was written then this is an unexpected error
-        * so pass generic error code to handler. */
-       if (h->handler != NULL) {
-               h->handler(h->ctdb, ret == 1 ? c : '3', latency,
-                          h, h->private_data);
-       }
-}
-
 static void
 ctdb_drop_all_ips_event(struct tevent_context *ev, struct tevent_timer *te,
                        struct timeval t, void *private_data)
@@ -908,165 +825,6 @@ ctdb_drop_all_ips_event(struct tevent_context *ev, struct tevent_timer *te,
        ctdb_release_all_ips(ctdb);
 }
 
-static char cluster_mutex_helper[PATH_MAX+1] = "";
-
-static bool cluster_mutex_helper_args(TALLOC_CTX *mem_ctx,
-                                     const char *argstring, char ***argv)
-{
-       int nargs, i, ret, n;
-       bool is_command = false;
-       char **args = NULL;
-       char *strv = NULL;
-       char *t = NULL;
-
-       if (argstring != NULL && argstring[0] == '!') {
-               /* This is actually a full command */
-               is_command = true;
-               t = discard_const(&argstring[1]);
-       } else {
-               is_command = false;
-               t = discard_const(argstring);
-       }
-
-       ret = strv_split(mem_ctx, &strv, t, " \t");
-       if (ret != 0) {
-               DEBUG(DEBUG_ERR,
-                     ("Unable to parse mutex helper string \"%s\" (%s)\n",
-                      argstring, strerror(ret)));
-               return false;
-       }
-       n = strv_count(strv);
-
-       args = talloc_array(mem_ctx, char *, n + (is_command ? 1 : 2));
-
-       if (args == NULL) {
-               DEBUG(DEBUG_ERR,(__location__ " out of memory\n"));
-               return false;
-       }
-
-       nargs = 0;
-
-       if (! is_command) {
-               if (!ctdb_set_helper("cluster mutex helper",
-                                    cluster_mutex_helper,
-                                    sizeof(cluster_mutex_helper),
-                                    "CTDB_CLUSTER_MUTEX_HELPER",
-                                    CTDB_HELPER_BINDIR,
-                                    "ctdb_mutex_fcntl_helper")) {
-                       DEBUG(DEBUG_ERR,("ctdb exiting with error: %s\n",
-                                        __location__
-                                        " Unable to set cluster mutex helper\n"));
-                       exit(1);
-               }
-
-               args[nargs++] = cluster_mutex_helper;
-       }
-
-       t = NULL;
-       for (i = 0; i < n; i++) {
-               /* Don't copy, just keep cmd_args around */
-               t = strv_next(strv, t);
-               args[nargs++] = t;
-       }
-
-       /* Make sure last argument is NULL */
-       args[nargs] = NULL;
-
-       *argv = args;
-       return true;
-}
-
-static struct ctdb_cluster_mutex_handle *
-ctdb_cluster_mutex(struct ctdb_context *ctdb,
-                  const char *argstring,
-                  int timeout)
-{
-       struct ctdb_cluster_mutex_handle *h;
-       char **args;
-       int ret;
-
-       h = talloc(ctdb, struct ctdb_cluster_mutex_handle);
-       if (h == NULL) {
-               DEBUG(DEBUG_ERR, (__location__ " out of memory\n"));
-               return NULL;
-       }
-
-       h->start_time = timeval_current();
-       h->fd[0] = -1;
-       h->fd[1] = -1;
-
-       ret = pipe(h->fd);
-       if (ret != 0) {
-               talloc_free(h);
-               DEBUG(DEBUG_ERR, (__location__ " Failed to open pipe\n"));
-               return NULL;
-       }
-       set_close_on_exec(h->fd[0]);
-
-       /* Create arguments for lock helper */
-       if (!cluster_mutex_helper_args(h, argstring, &args)) {
-               close(h->fd[0]);
-               close(h->fd[1]);
-               talloc_free(h);
-               return NULL;
-       }
-
-       h->child = ctdb_fork(ctdb);
-       if (h->child == (pid_t)-1) {
-               close(h->fd[0]);
-               close(h->fd[1]);
-               talloc_free(h);
-               return NULL;
-       }
-
-       if (h->child == 0) {
-               /* Make stdout point to the pipe */
-               close(STDOUT_FILENO);
-               dup2(h->fd[1], STDOUT_FILENO);
-               close(h->fd[1]);
-
-               execv(args[0], args);
-
-               /* Only happens on error */
-               DEBUG(DEBUG_ERR, (__location__ "execv() failed\n"));
-               _exit(1);
-       }
-
-       /* Parent */
-
-       DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d\n", h->fd[0]));
-       set_close_on_exec(h->fd[0]);
-
-       close(h->fd[1]);
-       h->fd[1] = -1;
-
-       talloc_set_destructor(h, cluster_mutex_destructor);
-
-       if (timeout != 0) {
-               h->te = tevent_add_timer(ctdb->ev, h,
-                                        timeval_current_ofs(timeout, 0),
-                                        cluster_mutex_timeout, h);
-       } else {
-               h->te = NULL;
-       }
-
-       h->fde = tevent_add_fd(ctdb->ev, h, h->fd[0], TEVENT_FD_READ,
-                              cluster_mutex_handler, (void *)h);
-
-       if (h->fde == NULL) {
-               talloc_free(h);
-               return NULL;
-       }
-       tevent_fd_set_auto_close(h->fde);
-
-       h->ctdb = ctdb;
-       h->handler = NULL;
-       h->private_data = NULL;
-
-       return h;
-}
-
-
 /*
  * Set up an event to drop all public ips if we remain in recovery for too
  * long
index 77fac08a6ad70cbd09a59134e7cf1d53c3e4d135..7d66712200ca6c3040b9c3ca445b9f5a07678023 100644 (file)
@@ -64,6 +64,7 @@ bool fast_start;
 #include "server/ctdb_takeover.c"
 #include "server/ctdb_persistent.c"
 #include "server/ctdb_keepalive.c"
+#include "server/ctdb_cluster_mutex.c"
 #include "server/ctdb_logging.c"
 #include "server/ctdb_logging_syslog.c"
 #include "server/ctdb_logging_file.c"
index 6a50a3966fa67583d2815bbf7f73cf09d9b350cc..c51d963e7c672cf8a743a051df8e79482863710c 100755 (executable)
@@ -394,6 +394,7 @@ def build(bld):
                                              ctdb_traverse.c eventscript.c
                                              ctdb_takeover.c
                                              ctdb_persistent.c ctdb_keepalive.c
+                                             ctdb_cluster_mutex.c
                                              ctdb_logging.c
                                              ctdb_logging_syslog.c
                                              ctdb_logging_file.c