messaging: Don't do self-sends in messaging_send_all
[nivanova/samba-autobuild/.git] / source3 / lib / messages.c
index aaaee52e3a6f53398c22f7ea4b537b9425f5b54e..561616df6e44da9c989f90253cf2fc534a22422c 100644 (file)
 */
 
 #include "includes.h"
+#include "lib/util/server_id.h"
 #include "dbwrap/dbwrap.h"
 #include "serverid.h"
 #include "messages.h"
 #include "lib/util/tevent_unix.h"
 #include "lib/background.h"
 #include "lib/messages_dgm.h"
+#include "lib/util/iov_buf.h"
+#include "lib/util/server_id_db.h"
+#include "lib/messages_dgm_ref.h"
+#include "lib/messages_ctdb.h"
+#include "lib/messages_ctdb_ref.h"
+#include "lib/messages_util.h"
+#include "cluster_support.h"
+#include "ctdbd_conn.h"
+#include "ctdb_srvids.h"
+
+#ifdef CLUSTER_SUPPORT
+#include "ctdb_protocol.h"
+#endif
 
 struct messaging_callback {
        struct messaging_callback *prev, *next;
-       uint32 msg_type;
+       uint32_t msg_type;
        void (*fn)(struct messaging_context *msg, void *private_data, 
                   uint32_t msg_type, 
                   struct server_id server_id, DATA_BLOB *data);
        void *private_data;
 };
 
+struct messaging_registered_ev {
+       struct tevent_context *ev;
+       struct tevent_immediate *im;
+       size_t refcount;
+};
+
 struct messaging_context {
        struct server_id id;
        struct tevent_context *event_ctx;
        struct messaging_callback *callbacks;
 
+       struct messaging_rec *posted_msgs;
+
+       struct messaging_registered_ev *event_contexts;
+
        struct tevent_req **new_waiters;
-       unsigned num_new_waiters;
+       size_t num_new_waiters;
 
        struct tevent_req **waiters;
-       unsigned num_waiters;
+       size_t num_waiters;
 
-       struct messaging_backend *remote;
-};
+       void *msg_dgm_ref;
+       void *msg_ctdb_ref;
 
-struct messaging_hdr {
-       uint32_t msg_type;
-       struct server_id dst;
-       struct server_id src;
+       struct server_id_db *names_db;
 };
 
+static struct messaging_rec *messaging_rec_dup(TALLOC_CTX *mem_ctx,
+                                              struct messaging_rec *rec);
+static bool messaging_dispatch_classic(struct messaging_context *msg_ctx,
+                                      struct messaging_rec *rec);
+static bool messaging_dispatch_waiters(struct messaging_context *msg_ctx,
+                                      struct tevent_context *ev,
+                                      struct messaging_rec *rec);
+static void messaging_dispatch_rec(struct messaging_context *msg_ctx,
+                                  struct tevent_context *ev,
+                                  struct messaging_rec *rec);
+
 /****************************************************************************
  A useful function for testing the message system.
 ****************************************************************************/
@@ -101,134 +133,247 @@ static void ping_message(struct messaging_context *msg_ctx,
        messaging_send(msg_ctx, src, MSG_PONG, data);
 }
 
-/****************************************************************************
- Register/replace a dispatch function for a particular message type.
- JRA changed Dec 13 2006. Only one message handler now permitted per type.
- *NOTE*: Dispatch functions must be able to cope with incoming
- messages on an *odd* byte boundary.
-****************************************************************************/
+struct messaging_rec *messaging_rec_create(
+       TALLOC_CTX *mem_ctx, struct server_id src, struct server_id dst,
+       uint32_t msg_type, const struct iovec *iov, int iovlen,
+       const int *fds, size_t num_fds)
+{
+       ssize_t buflen;
+       uint8_t *buf;
+       struct messaging_rec *result;
 
-struct msg_all {
-       struct messaging_context *msg_ctx;
-       int msg_type;
-       uint32 msg_flag;
-       const void *buf;
-       size_t len;
-       int n_sent;
-};
+       if (num_fds > INT8_MAX) {
+               return NULL;
+       }
 
-/****************************************************************************
- Send one of the messages for the broadcast.
-****************************************************************************/
+       buflen = iov_buflen(iov, iovlen);
+       if (buflen == -1) {
+               return NULL;
+       }
+       buf = talloc_array(mem_ctx, uint8_t, buflen);
+       if (buf == NULL) {
+               return NULL;
+       }
+       iov_buf(iov, iovlen, buf, buflen);
+
+       {
+               struct messaging_rec rec;
+               int64_t fds64[num_fds];
+               size_t i;
 
-static int traverse_fn(struct db_record *rec, const struct server_id *id,
-                      uint32_t msg_flags, void *state)
+               for (i=0; i<num_fds; i++) {
+                       fds64[i] = fds[i];
+               }
+
+               rec = (struct messaging_rec) {
+                       .msg_version = MESSAGE_VERSION, .msg_type = msg_type,
+                       .src = src, .dest = dst,
+                       .buf.data = buf, .buf.length = buflen,
+                       .num_fds = num_fds, .fds = fds64,
+               };
+
+               result = messaging_rec_dup(mem_ctx, &rec);
+       }
+
+       TALLOC_FREE(buf);
+
+       return result;
+}
+
+static bool messaging_register_event_context(struct messaging_context *ctx,
+                                            struct tevent_context *ev)
 {
-       struct msg_all *msg_all = (struct msg_all *)state;
-       NTSTATUS status;
+       size_t i, num_event_contexts;
+       struct messaging_registered_ev *free_reg = NULL;
+       struct messaging_registered_ev *tmp;
 
-       /* Don't send if the receiver hasn't registered an interest. */
+       num_event_contexts = talloc_array_length(ctx->event_contexts);
 
-       if((msg_flags & msg_all->msg_flag) == 0) {
-               return 0;
+       for (i=0; i<num_event_contexts; i++) {
+               struct messaging_registered_ev *reg = &ctx->event_contexts[i];
+
+               if (reg->ev == ev) {
+                       reg->refcount += 1;
+                       return true;
+               }
+               if (reg->refcount == 0) {
+                       if (reg->ev != NULL) {
+                               abort();
+                       }
+                       free_reg = reg;
+               }
        }
 
-       /* If the msg send fails because the pid was not found (i.e. smbd died), 
-        * the msg has already been deleted from the messages.tdb.*/
+       if (free_reg == NULL) {
+               tmp = talloc_realloc(ctx, ctx->event_contexts,
+                                    struct messaging_registered_ev,
+                                    num_event_contexts+1);
+               if (tmp == NULL) {
+                       return false;
+               }
+               ctx->event_contexts = tmp;
 
-       status = messaging_send_buf(msg_all->msg_ctx, *id, msg_all->msg_type,
-                                   (const uint8_t *)msg_all->buf, msg_all->len);
+               free_reg = &ctx->event_contexts[num_event_contexts];
+       }
 
-       if (NT_STATUS_EQUAL(status, NT_STATUS_INVALID_HANDLE)) {
-               struct server_id_buf idbuf;
+       *free_reg = (struct messaging_registered_ev) { .ev = ev, .refcount = 1 };
 
-               /*
-                * If the pid was not found delete the entry from
-                * serverid.tdb
-                */
+       return true;
+}
 
-               DEBUG(2, ("pid %s doesn't exist\n",
-                         server_id_str_buf(*id, &idbuf)));
+static bool messaging_deregister_event_context(struct messaging_context *ctx,
+                                              struct tevent_context *ev)
+{
+       size_t i, num_event_contexts;
 
-               dbwrap_record_delete(rec);
+       num_event_contexts = talloc_array_length(ctx->event_contexts);
+
+       for (i=0; i<num_event_contexts; i++) {
+               struct messaging_registered_ev *reg = &ctx->event_contexts[i];
+
+               if (reg->ev == ev) {
+                       if (reg->refcount == 0) {
+                               return false;
+                       }
+                       reg->refcount -= 1;
+
+                       if (reg->refcount == 0) {
+                               /*
+                                * Not strictly necessary, just
+                                * paranoia
+                                */
+                               reg->ev = NULL;
+
+                               /*
+                                * Do not talloc_free(reg->im),
+                                * recycle immediates events.
+                                */
+                       }
+                       return true;
+               }
        }
-       msg_all->n_sent++;
-       return 0;
+       return false;
 }
 
-/**
- * Send a message to all smbd processes.
- *
- * It isn't very efficient, but should be OK for the sorts of
- * applications that use it. When we need efficient broadcast we can add
- * it.
- *
- * @param n_sent Set to the number of messages sent.  This should be
- * equal to the number of processes, but be careful for races.
- *
- * @retval True for success.
- **/
-bool message_send_all(struct messaging_context *msg_ctx,
-                     int msg_type,
-                     const void *buf, size_t len,
-                     int *n_sent)
+static void messaging_post_main_event_context(struct tevent_context *ev,
+                                             struct tevent_immediate *im,
+                                             void *private_data)
 {
-       struct msg_all msg_all;
-
-       msg_all.msg_type = msg_type;
-       if (msg_type < 0x100) {
-               msg_all.msg_flag = FLAG_MSG_GENERAL;
-       } else if (msg_type > 0x100 && msg_type < 0x200) {
-               msg_all.msg_flag = FLAG_MSG_NMBD;
-       } else if (msg_type > 0x200 && msg_type < 0x300) {
-               msg_all.msg_flag = FLAG_MSG_PRINT_GENERAL;
-       } else if (msg_type > 0x300 && msg_type < 0x400) {
-               msg_all.msg_flag = FLAG_MSG_SMBD;
-       } else if (msg_type > 0x400 && msg_type < 0x600) {
-               msg_all.msg_flag = FLAG_MSG_WINBIND;
-       } else if (msg_type > 4000 && msg_type < 5000) {
-               msg_all.msg_flag = FLAG_MSG_DBWRAP;
-       } else {
-               return false;
+       struct messaging_context *ctx = talloc_get_type_abort(
+               private_data, struct messaging_context);
+
+       while (ctx->posted_msgs != NULL) {
+               struct messaging_rec *rec = ctx->posted_msgs;
+               bool consumed;
+
+               DLIST_REMOVE(ctx->posted_msgs, rec);
+
+               consumed = messaging_dispatch_classic(ctx, rec);
+               if (!consumed) {
+                       consumed = messaging_dispatch_waiters(
+                               ctx, ctx->event_ctx, rec);
+               }
+
+               if (!consumed) {
+                       uint8_t i;
+
+                       for (i=0; i<rec->num_fds; i++) {
+                               close(rec->fds[i]);
+                       }
+               }
+
+               TALLOC_FREE(rec);
        }
+}
+
+static void messaging_post_sub_event_context(struct tevent_context *ev,
+                                            struct tevent_immediate *im,
+                                            void *private_data)
+{
+       struct messaging_context *ctx = talloc_get_type_abort(
+               private_data, struct messaging_context);
+       struct messaging_rec *rec, *next;
+
+       for (rec = ctx->posted_msgs; rec != NULL; rec = next) {
+               bool consumed;
+
+               next = rec->next;
+
+               consumed = messaging_dispatch_waiters(ctx, ev, rec);
+               if (consumed) {
+                       DLIST_REMOVE(ctx->posted_msgs, rec);
+                       TALLOC_FREE(rec);
+               }
+       }
+}
+
+static bool messaging_alert_event_contexts(struct messaging_context *ctx)
+{
+       size_t i, num_event_contexts;
+
+       num_event_contexts = talloc_array_length(ctx->event_contexts);
 
-       msg_all.buf = buf;
-       msg_all.len = len;
-       msg_all.n_sent = 0;
-       msg_all.msg_ctx = msg_ctx;
+       for (i=0; i<num_event_contexts; i++) {
+               struct messaging_registered_ev *reg = &ctx->event_contexts[i];
 
-       serverid_traverse(traverse_fn, &msg_all);
-       if (n_sent)
-               *n_sent = msg_all.n_sent;
+               if (reg->refcount == 0) {
+                       continue;
+               }
+
+               if (reg->im == NULL) {
+                       reg->im = tevent_create_immediate(
+                               ctx->event_contexts);
+               }
+               if (reg->im == NULL) {
+                       DBG_WARNING("Could not create immediate\n");
+                       continue;
+               }
+
+               /*
+                * We depend on schedule_immediate to work
+                * multiple times. Might be a bit inefficient,
+                * but this needs to be proven in tests. The
+                * alternatively would be to track whether the
+                * immediate has already been scheduled. For
+                * now, avoid that complexity here.
+                */
+
+               if (reg->ev == ctx->event_ctx) {
+                       tevent_schedule_immediate(
+                               reg->im, reg->ev,
+                               messaging_post_main_event_context,
+                               ctx);
+               } else {
+                       tevent_schedule_immediate(
+                               reg->im, reg->ev,
+                               messaging_post_sub_event_context,
+                               ctx);
+               }
+
+       }
        return true;
 }
 
-static void messaging_recv_cb(const uint8_t *msg, size_t msg_len,
+static void messaging_recv_cb(struct tevent_context *ev,
+                             const uint8_t *msg, size_t msg_len,
                              int *fds, size_t num_fds,
                              void *private_data)
 {
        struct messaging_context *msg_ctx = talloc_get_type_abort(
                private_data, struct messaging_context);
-       const struct messaging_hdr *hdr;
        struct server_id_buf idbuf;
        struct messaging_rec rec;
        int64_t fds64[MIN(num_fds, INT8_MAX)];
        size_t i;
 
-       if (msg_len < sizeof(*hdr)) {
-               for (i=0; i < num_fds; i++) {
-                       close(fds[i]);
-               }
-               DEBUG(1, ("message too short: %u\n", (unsigned)msg_len));
-               return;
+       if (msg_len < MESSAGE_HDR_LENGTH) {
+               DBG_WARNING("message too short: %zu\n", msg_len);
+               goto close_fail;
        }
 
        if (num_fds > INT8_MAX) {
-               for (i=0; i < num_fds; i++) {
-                       close(fds[i]);
-               }
-               DEBUG(1, ("too many fds: %u\n", (unsigned)num_fds));
-               return;
+               DBG_WARNING("too many fds: %zu\n", num_fds);
+               goto close_fail;
        }
 
        /*
@@ -240,77 +385,168 @@ static void messaging_recv_cb(const uint8_t *msg, size_t msg_len,
                fds[i] = -1;
        }
 
-       /*
-        * messages_dgm guarantees alignment, so we can cast here
-        */
-       hdr = (const struct messaging_hdr *)msg;
-
-       DEBUG(10, ("%s: Received message 0x%x len %u (num_fds:%u) from %s\n",
-                  __func__, (unsigned)hdr->msg_type,
-                  (unsigned)(msg_len - sizeof(*hdr)),
-                  (unsigned)num_fds,
-                  server_id_str_buf(hdr->src, &idbuf)));
-
        rec = (struct messaging_rec) {
                .msg_version = MESSAGE_VERSION,
-               .msg_type = hdr->msg_type,
-               .src = hdr->src,
-               .dest = hdr->dst,
-               .buf.data = discard_const_p(uint8, msg) + sizeof(*hdr),
-               .buf.length = msg_len - sizeof(*hdr),
+               .buf.data = discard_const_p(uint8_t, msg) + MESSAGE_HDR_LENGTH,
+               .buf.length = msg_len - MESSAGE_HDR_LENGTH,
                .num_fds = num_fds,
                .fds = fds64,
        };
 
-       messaging_dispatch_rec(msg_ctx, &rec);
+       message_hdr_get(&rec.msg_type, &rec.src, &rec.dest, msg);
+
+       DBG_DEBUG("Received message 0x%x len %zu (num_fds:%zu) from %s\n",
+                 (unsigned)rec.msg_type, rec.buf.length, num_fds,
+                 server_id_str_buf(rec.src, &idbuf));
+
+       messaging_dispatch_rec(msg_ctx, ev, &rec);
+       return;
+
+close_fail:
+       for (i=0; i < num_fds; i++) {
+               close(fds[i]);
+       }
 }
 
 static int messaging_context_destructor(struct messaging_context *ctx)
 {
-       messaging_dgm_destroy();
+       size_t i;
+
+       for (i=0; i<ctx->num_new_waiters; i++) {
+               if (ctx->new_waiters[i] != NULL) {
+                       tevent_req_set_cleanup_fn(ctx->new_waiters[i], NULL);
+                       ctx->new_waiters[i] = NULL;
+               }
+       }
+       for (i=0; i<ctx->num_waiters; i++) {
+               if (ctx->waiters[i] != NULL) {
+                       tevent_req_set_cleanup_fn(ctx->waiters[i], NULL);
+                       ctx->waiters[i] = NULL;
+               }
+       }
+
+       /*
+        * The immediates from messaging_alert_event_contexts
+        * reference "ctx". Don't let them outlive the
+        * messaging_context we're destroying here.
+        */
+       TALLOC_FREE(ctx->event_contexts);
+
        return 0;
 }
 
-struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx, 
-                                        struct tevent_context *ev)
+static const char *private_path(const char *name)
+{
+       return talloc_asprintf(talloc_tos(), "%s/%s", lp_private_dir(), name);
+}
+
+static NTSTATUS messaging_init_internal(TALLOC_CTX *mem_ctx,
+                                       struct tevent_context *ev,
+                                       struct messaging_context **pmsg_ctx)
 {
+       TALLOC_CTX *frame;
        struct messaging_context *ctx;
-       NTSTATUS status;
+       NTSTATUS status = NT_STATUS_UNSUCCESSFUL;
        int ret;
+       const char *lck_path;
+       const char *priv_path;
+       bool ok;
 
-       if (!(ctx = talloc_zero(mem_ctx, struct messaging_context))) {
-               return NULL;
+       lck_path = lock_path("msg.lock");
+       if (lck_path == NULL) {
+               return NT_STATUS_NO_MEMORY;
        }
 
-       ctx->id = procid_self();
-       ctx->event_ctx = ev;
+       ok = directory_create_or_exist_strict(lck_path,
+                                             sec_initial_uid(),
+                                             0755);
+       if (!ok) {
+               DBG_DEBUG("Could not create lock directory: %s\n",
+                         strerror(errno));
+               return NT_STATUS_ACCESS_DENIED;
+       }
 
-       sec_init();
+       priv_path = private_path("msg.sock");
+       if (priv_path == NULL) {
+               return NT_STATUS_NO_MEMORY;
+       }
 
-       ret = messaging_dgm_init(ctx->event_ctx, ctx->id,
-                                lp_cache_directory(), sec_initial_uid(),
-                                messaging_recv_cb, ctx);
+       ok = directory_create_or_exist_strict(priv_path, sec_initial_uid(),
+                                             0700);
+       if (!ok) {
+               DBG_DEBUG("Could not create msg directory: %s\n",
+                         strerror(errno));
+               return NT_STATUS_ACCESS_DENIED;
+       }
 
-       if (ret != 0) {
-               DEBUG(2, ("messaging_dgm_init failed: %s\n", strerror(ret)));
-               TALLOC_FREE(ctx);
-               return NULL;
+       frame = talloc_stackframe();
+       if (frame == NULL) {
+               return NT_STATUS_NO_MEMORY;
+       }
+
+       ctx = talloc_zero(frame, struct messaging_context);
+       if (ctx == NULL) {
+               status = NT_STATUS_NO_MEMORY;
+               goto done;
        }
 
+       ctx->id = (struct server_id) {
+               .pid = getpid(), .vnn = NONCLUSTER_VNN
+       };
+
+       ctx->event_ctx = ev;
+
+       ok = messaging_register_event_context(ctx, ev);
+       if (!ok) {
+               status = NT_STATUS_NO_MEMORY;
+               goto done;
+       }
+
+       sec_init();
+
+       ctx->msg_dgm_ref = messaging_dgm_ref(ctx,
+                                            ctx->event_ctx,
+                                            &ctx->id.unique_id,
+                                            priv_path,
+                                            lck_path,
+                                            messaging_recv_cb,
+                                            ctx,
+                                            &ret);
+       if (ctx->msg_dgm_ref == NULL) {
+               DEBUG(2, ("messaging_dgm_ref failed: %s\n", strerror(ret)));
+               status = map_nt_error_from_unix(ret);
+               goto done;
+       }
        talloc_set_destructor(ctx, messaging_context_destructor);
 
+#ifdef CLUSTER_SUPPORT
        if (lp_clustering()) {
-               status = messaging_ctdbd_init(ctx, ctx, &ctx->remote);
-
-               if (!NT_STATUS_IS_OK(status)) {
-                       DEBUG(2, ("messaging_ctdbd_init failed: %s\n",
-                                 nt_errstr(status)));
-                       TALLOC_FREE(ctx);
-                       return NULL;
+               ctx->msg_ctdb_ref = messaging_ctdb_ref(
+                       ctx, ctx->event_ctx,
+                       lp_ctdbd_socket(), lp_ctdb_timeout(),
+                       ctx->id.unique_id, messaging_recv_cb, ctx, &ret);
+               if (ctx->msg_ctdb_ref == NULL) {
+                       DBG_NOTICE("messaging_ctdb_ref failed: %s\n",
+                                  strerror(ret));
+                       status = map_nt_error_from_unix(ret);
+                       goto done;
                }
        }
+#endif
+
        ctx->id.vnn = get_my_vnn();
 
+       ctx->names_db = server_id_db_init(ctx,
+                                         ctx->id,
+                                         lp_lock_directory(),
+                                         0,
+                                         TDB_INCOMPATIBLE_HASH|TDB_CLEAR_IF_FIRST);
+       if (ctx->names_db == NULL) {
+               DBG_DEBUG("server_id_db_init failed\n");
+               status = NT_STATUS_NO_MEMORY;
+               goto done;
+       }
+
        messaging_register(ctx, NULL, MSG_PING, ping_message);
 
        /* Register some debugging related messages */
@@ -319,9 +555,45 @@ struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
        register_dmalloc_msgs(ctx);
        debug_register_msgs(ctx);
 
+       {
+               struct server_id_buf tmp;
+               DBG_DEBUG("my id: %s\n", server_id_str_buf(ctx->id, &tmp));
+       }
+
+       *pmsg_ctx = talloc_steal(mem_ctx, ctx);
+
+       status = NT_STATUS_OK;
+done:
+       TALLOC_FREE(frame);
+
+       return status;
+}
+
+struct messaging_context *messaging_init(TALLOC_CTX *mem_ctx,
+                                        struct tevent_context *ev)
+{
+       struct messaging_context *ctx = NULL;
+       NTSTATUS status;
+
+       status = messaging_init_internal(mem_ctx,
+                                        ev,
+                                        &ctx);
+       if (!NT_STATUS_IS_OK(status)) {
+               return NULL;
+       }
+
        return ctx;
 }
 
+NTSTATUS messaging_init_client(TALLOC_CTX *mem_ctx,
+                              struct tevent_context *ev,
+                              struct messaging_context **pmsg_ctx)
+{
+       return messaging_init_internal(mem_ctx,
+                                       ev,
+                                       pmsg_ctx);
+}
+
 struct server_id messaging_server_id(const struct messaging_context *msg_ctx)
 {
        return msg_ctx->id;
@@ -332,34 +604,46 @@ struct server_id messaging_server_id(const struct messaging_context *msg_ctx)
  */
 NTSTATUS messaging_reinit(struct messaging_context *msg_ctx)
 {
-       NTSTATUS status;
        int ret;
+       char *lck_path;
 
-       messaging_dgm_destroy();
+       TALLOC_FREE(msg_ctx->msg_dgm_ref);
+       TALLOC_FREE(msg_ctx->msg_ctdb_ref);
 
-       msg_ctx->id = procid_self();
+       msg_ctx->id = (struct server_id) {
+               .pid = getpid(), .vnn = msg_ctx->id.vnn
+       };
 
-       ret = messaging_dgm_init(msg_ctx->event_ctx, msg_ctx->id,
-                                lp_cache_directory(), sec_initial_uid(),
-                                messaging_recv_cb, msg_ctx);
-       if (ret != 0) {
-               DEBUG(0, ("messaging_dgm_init failed: %s\n", strerror(errno)));
-               return map_nt_error_from_unix(ret);
+       lck_path = lock_path("msg.lock");
+       if (lck_path == NULL) {
+               return NT_STATUS_NO_MEMORY;
        }
 
-       TALLOC_FREE(msg_ctx->remote);
+       msg_ctx->msg_dgm_ref = messaging_dgm_ref(
+               msg_ctx, msg_ctx->event_ctx, &msg_ctx->id.unique_id,
+               private_path("msg.sock"), lck_path,
+               messaging_recv_cb, msg_ctx, &ret);
 
-       if (lp_clustering()) {
-               status = messaging_ctdbd_init(msg_ctx, msg_ctx,
-                                             &msg_ctx->remote);
+       if (msg_ctx->msg_dgm_ref == NULL) {
+               DEBUG(2, ("messaging_dgm_ref failed: %s\n", strerror(ret)));
+               return map_nt_error_from_unix(ret);
+       }
 
-               if (!NT_STATUS_IS_OK(status)) {
-                       DEBUG(1, ("messaging_ctdbd_init failed: %s\n",
-                                 nt_errstr(status)));
-                       return status;
+       if (lp_clustering()) {
+               msg_ctx->msg_ctdb_ref = messaging_ctdb_ref(
+                       msg_ctx, msg_ctx->event_ctx,
+                       lp_ctdbd_socket(), lp_ctdb_timeout(),
+                       msg_ctx->id.unique_id, messaging_recv_cb, msg_ctx,
+                       &ret);
+               if (msg_ctx->msg_ctdb_ref == NULL) {
+                       DBG_NOTICE("messaging_ctdb_ref failed: %s\n",
+                                  strerror(ret));
+                       return map_nt_error_from_unix(ret);
                }
        }
 
+       server_id_db_reinit(msg_ctx->names_db, msg_ctx->id);
+
        return NT_STATUS_OK;
 }
 
@@ -442,10 +726,12 @@ NTSTATUS messaging_send(struct messaging_context *msg_ctx,
                        struct server_id server, uint32_t msg_type,
                        const DATA_BLOB *data)
 {
-       struct iovec iov;
+       struct iovec iov = {0};
 
-       iov.iov_base = data->data;
-       iov.iov_len = data->length;
+       if (data != NULL) {
+               iov.iov_base = data->data;
+               iov.iov_len = data->length;
+       };
 
        return messaging_send_iov(msg_ctx, server, msg_type, &iov, 1, NULL, 0);
 }
@@ -458,85 +744,175 @@ NTSTATUS messaging_send_buf(struct messaging_context *msg_ctx,
        return messaging_send(msg_ctx, server, msg_type, &blob);
 }
 
-NTSTATUS messaging_send_iov(struct messaging_context *msg_ctx,
-                           struct server_id server, uint32_t msg_type,
+static int messaging_post_self(struct messaging_context *msg_ctx,
+                              struct server_id src, struct server_id dst,
+                              uint32_t msg_type,
+                              const struct iovec *iov, int iovlen,
+                              const int *fds, size_t num_fds)
+{
+       struct messaging_rec *rec;
+       bool ok;
+
+       rec = messaging_rec_create(
+               msg_ctx, src, dst, msg_type, iov, iovlen, fds, num_fds);
+       if (rec == NULL) {
+               return ENOMEM;
+       }
+
+       ok = messaging_alert_event_contexts(msg_ctx);
+       if (!ok) {
+               TALLOC_FREE(rec);
+               return ENOMEM;
+       }
+
+       DLIST_ADD_END(msg_ctx->posted_msgs, rec);
+
+       return 0;
+}
+
+int messaging_send_iov_from(struct messaging_context *msg_ctx,
+                           struct server_id src, struct server_id dst,
+                           uint32_t msg_type,
                            const struct iovec *iov, int iovlen,
                            const int *fds, size_t num_fds)
 {
        int ret;
-       struct messaging_hdr hdr;
+       uint8_t hdr[MESSAGE_HDR_LENGTH];
        struct iovec iov2[iovlen+1];
 
-       if (server_id_is_disconnected(&server)) {
-               return NT_STATUS_INVALID_PARAMETER_MIX;
+       if (server_id_is_disconnected(&dst)) {
+               return EINVAL;
        }
 
        if (num_fds > INT8_MAX) {
-               return NT_STATUS_INVALID_PARAMETER_MIX;
+               return EINVAL;
        }
 
-       if (!procid_is_local(&server)) {
+       if (server_id_equal(&dst, &msg_ctx->id)) {
+               ret = messaging_post_self(msg_ctx, src, dst, msg_type,
+                                         iov, iovlen, fds, num_fds);
+               return ret;
+       }
+
+       message_hdr_put(hdr, msg_type, src, dst);
+       iov2[0] = (struct iovec){ .iov_base = hdr, .iov_len = sizeof(hdr) };
+       memcpy(&iov2[1], iov, iovlen * sizeof(*iov));
+
+       if (dst.vnn != msg_ctx->id.vnn) {
                if (num_fds > 0) {
-                       return NT_STATUS_NOT_SUPPORTED;
+                       return ENOSYS;
                }
 
-               ret = msg_ctx->remote->send_fn(msg_ctx->id, server,
-                                              msg_type, iov, iovlen,
-                                              NULL, 0,
-                                              msg_ctx->remote);
-               if (ret != 0) {
-                       return map_nt_error_from_unix(ret);
-               }
-               return NT_STATUS_OK;
+               ret = messaging_ctdb_send(dst.vnn, dst.pid, iov2, iovlen+1);
+               return ret;
        }
 
-       if (server_id_same_process(&msg_ctx->id, &server)) {
-               struct messaging_rec rec;
-               uint8_t *buf;
+       ret = messaging_dgm_send(dst.pid, iov2, iovlen+1, fds, num_fds);
 
+       if (ret == EACCES) {
+               become_root();
+               ret = messaging_dgm_send(dst.pid, iov2, iovlen+1,
+                                        fds, num_fds);
+               unbecome_root();
+       }
+
+       if (ret == ECONNREFUSED) {
                /*
-                * Self-send, directly dispatch
+                * Linux returns this when a socket exists in the file
+                * system without a listening process. This is not
+                * documented in susv4 or the linux manpages, but it's
+                * easily testable. For the higher levels this is the
+                * same as "destination does not exist"
                 */
+               ret = ENOENT;
+       }
 
-               if (num_fds > 0) {
-                       return NT_STATUS_NOT_SUPPORTED;
-               }
+       return ret;
+}
 
-               buf = iov_buf(talloc_tos(), iov, iovlen);
-               if (buf == NULL) {
-                       return NT_STATUS_NO_MEMORY;
-               }
+NTSTATUS messaging_send_iov(struct messaging_context *msg_ctx,
+                           struct server_id dst, uint32_t msg_type,
+                           const struct iovec *iov, int iovlen,
+                           const int *fds, size_t num_fds)
+{
+       int ret;
 
-               rec = (struct messaging_rec) {
-                       .msg_version = MESSAGE_VERSION,
-                       .msg_type = msg_type & MSG_TYPE_MASK,
-                       .dest = server,
-                       .src = msg_ctx->id,
-                       .buf = data_blob_const(buf, talloc_get_size(buf)),
-               };
+       ret = messaging_send_iov_from(msg_ctx, msg_ctx->id, dst, msg_type,
+                                     iov, iovlen, fds, num_fds);
+       if (ret != 0) {
+               return map_nt_error_from_unix(ret);
+       }
+       return NT_STATUS_OK;
+}
 
-               messaging_dispatch_rec(msg_ctx, &rec);
-               TALLOC_FREE(buf);
-               return NT_STATUS_OK;
+struct send_all_state {
+       struct messaging_context *msg_ctx;
+       int msg_type;
+       const void *buf;
+       size_t len;
+};
+
+static int send_all_fn(pid_t pid, void *private_data)
+{
+       struct send_all_state *state = private_data;
+       NTSTATUS status;
+
+       if (pid == getpid()) {
+               DBG_DEBUG("Skip ourselves in messaging_send_all\n");
+               return 0;
+       }
+
+       status = messaging_send_buf(state->msg_ctx, pid_to_procid(pid),
+                                   state->msg_type, state->buf, state->len);
+       if (!NT_STATUS_IS_OK(status)) {
+               DBG_WARNING("messaging_send_buf to %ju failed: %s\n",
+                           (uintmax_t)pid, nt_errstr(status));
        }
 
-       ZERO_STRUCT(hdr);
-       hdr = (struct messaging_hdr) {
-               .msg_type = msg_type,
-               .dst = server,
-               .src = msg_ctx->id
+       return 0;
+}
+
+void messaging_send_all(struct messaging_context *msg_ctx,
+                       int msg_type, const void *buf, size_t len)
+{
+       struct send_all_state state = {
+               .msg_ctx = msg_ctx, .msg_type = msg_type,
+               .buf = buf, .len = len
        };
-       iov2[0] = (struct iovec){ .iov_base = &hdr, .iov_len = sizeof(hdr) };
-       memcpy(&iov2[1], iov, iovlen * sizeof(*iov));
+       int ret;
+
+#ifdef CLUSTER_SUPPORT
+       if (lp_clustering()) {
+               struct ctdbd_connection *conn = messaging_ctdb_connection();
+               uint8_t msghdr[MESSAGE_HDR_LENGTH];
+               struct iovec iov[] = {
+                       { .iov_base = msghdr,
+                         .iov_len = sizeof(msghdr) },
+                       { .iov_base = discard_const_p(void, buf),
+                         .iov_len = len }
+               };
+
+               message_hdr_put(msghdr, msg_type, messaging_server_id(msg_ctx),
+                               (struct server_id) {0});
+
+               ret = ctdbd_messaging_send_iov(
+                       conn, CTDB_BROADCAST_CONNECTED,
+                       CTDB_SRVID_SAMBA_PROCESS,
+                       iov, ARRAY_SIZE(iov));
+               if (ret != 0) {
+                       DBG_WARNING("ctdbd_messaging_send_iov failed: %s\n",
+                                   strerror(ret));
+               }
 
-       become_root();
-       ret = messaging_dgm_send(server.pid, iov2, iovlen+1, fds, num_fds);
-       unbecome_root();
+               return;
+       }
+#endif
 
+       ret = messaging_dgm_forall(send_all_fn, &state);
        if (ret != 0) {
-               return map_nt_error_from_unix(ret);
+               DBG_WARNING("messaging_dgm_forall failed: %s\n",
+                           strerror(ret));
        }
-       return NT_STATUS_OK;
 }
 
 static struct messaging_rec *messaging_rec_dup(TALLOC_CTX *mem_ctx,
@@ -544,9 +920,16 @@ static struct messaging_rec *messaging_rec_dup(TALLOC_CTX *mem_ctx,
 {
        struct messaging_rec *result;
        size_t fds_size = sizeof(int64_t) * rec->num_fds;
+       size_t payload_len;
+
+       payload_len = rec->buf.length + fds_size;
+       if (payload_len < rec->buf.length) {
+               /* overflow */
+               return NULL;
+       }
 
        result = talloc_pooled_object(mem_ctx, struct messaging_rec, 2,
-                                     rec->buf.length + fds_size);
+                                     payload_len);
        if (result == NULL) {
                return NULL;
        }
@@ -559,8 +942,7 @@ static struct messaging_rec *messaging_rec_dup(TALLOC_CTX *mem_ctx,
 
        result->fds = NULL;
        if (result->num_fds > 0) {
-               result->fds = talloc_array(result, int64_t, result->num_fds);
-               memcpy(result->fds, rec->fds, fds_size);
+               result->fds = talloc_memdup(result, rec->fds, fds_size);
        }
 
        return result;
@@ -569,7 +951,8 @@ static struct messaging_rec *messaging_rec_dup(TALLOC_CTX *mem_ctx,
 struct messaging_filtered_read_state {
        struct tevent_context *ev;
        struct messaging_context *msg_ctx;
-       void *tevent_handle;
+       struct messaging_dgm_fde *fde;
+       struct messaging_ctdb_fde *cluster_fde;
 
        bool (*filter)(struct messaging_rec *rec, void *private_data);
        void *private_data;
@@ -589,6 +972,7 @@ struct tevent_req *messaging_filtered_read_send(
        struct tevent_req *req;
        struct messaging_filtered_read_state *state;
        size_t new_waiters_len;
+       bool ok;
 
        req = tevent_req_create(mem_ctx, &state,
                                struct messaging_filtered_read_state);
@@ -606,12 +990,19 @@ struct tevent_req *messaging_filtered_read_send(
         */
        tevent_req_defer_callback(req, state->ev);
 
-       state->tevent_handle = messaging_dgm_register_tevent_context(
-               state, ev);
-       if (tevent_req_nomem(state, req)) {
+       state->fde = messaging_dgm_register_tevent_context(state, ev);
+       if (tevent_req_nomem(state->fde, req)) {
                return tevent_req_post(req, ev);
        }
 
+       if (lp_clustering()) {
+               state->cluster_fde =
+                       messaging_ctdb_register_tevent_context(state, ev);
+               if (tevent_req_nomem(state->cluster_fde, req)) {
+                       return tevent_req_post(req, ev);
+               }
+       }
+
        /*
         * We add ourselves to the "new_waiters" array, not the "waiters"
         * array. If we are called from within messaging_read_done,
@@ -637,6 +1028,12 @@ struct tevent_req *messaging_filtered_read_send(
        msg_ctx->num_new_waiters += 1;
        tevent_req_set_cleanup_fn(req, messaging_filtered_read_cleanup);
 
+       ok = messaging_register_event_context(msg_ctx, ev);
+       if (!ok) {
+               tevent_req_oom(req);
+               return tevent_req_post(req, ev);
+       }
+
        return req;
 }
 
@@ -646,11 +1043,18 @@ static void messaging_filtered_read_cleanup(struct tevent_req *req,
        struct messaging_filtered_read_state *state = tevent_req_data(
                req, struct messaging_filtered_read_state);
        struct messaging_context *msg_ctx = state->msg_ctx;
-       unsigned i;
+       size_t i;
+       bool ok;
 
        tevent_req_set_cleanup_fn(req, NULL);
 
-       TALLOC_FREE(state->tevent_handle);
+       TALLOC_FREE(state->fde);
+       TALLOC_FREE(state->cluster_fde);
+
+       ok = messaging_deregister_event_context(msg_ctx, state->ev);
+       if (!ok) {
+               abort();
+       }
 
        /*
         * Just set the [new_]waiters entry to NULL, be careful not to mess
@@ -699,7 +1103,9 @@ int messaging_filtered_read_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
                tevent_req_received(req);
                return err;
        }
-       *presult = talloc_move(mem_ctx, &state->rec);
+       if (presult != NULL) {
+               *presult = talloc_move(mem_ctx, &state->rec);
+       }
        return 0;
 }
 
@@ -809,79 +1215,14 @@ static bool messaging_append_new_waiters(struct messaging_context *msg_ctx)
        return true;
 }
 
-struct messaging_defer_callback_state {
-       struct messaging_context *msg_ctx;
-       struct messaging_rec *rec;
-       void (*fn)(struct messaging_context *msg, void *private_data,
-                  uint32_t msg_type, struct server_id server_id,
-                  DATA_BLOB *data);
-       void *private_data;
-};
-
-static void messaging_defer_callback_trigger(struct tevent_context *ev,
-                                            struct tevent_immediate *im,
-                                            void *private_data);
-
-static void messaging_defer_callback(
-       struct messaging_context *msg_ctx, struct messaging_rec *rec,
-       void (*fn)(struct messaging_context *msg, void *private_data,
-                  uint32_t msg_type, struct server_id server_id,
-                  DATA_BLOB *data),
-       void *private_data)
-{
-       struct messaging_defer_callback_state *state;
-       struct tevent_immediate *im;
-
-       state = talloc(msg_ctx, struct messaging_defer_callback_state);
-       if (state == NULL) {
-               DEBUG(1, ("talloc failed\n"));
-               return;
-       }
-       state->msg_ctx = msg_ctx;
-       state->fn = fn;
-       state->private_data = private_data;
-
-       state->rec = messaging_rec_dup(state, rec);
-       if (state->rec == NULL) {
-               DEBUG(1, ("talloc failed\n"));
-               TALLOC_FREE(state);
-               return;
-       }
-
-       im = tevent_create_immediate(state);
-       if (im == NULL) {
-               DEBUG(1, ("tevent_create_immediate failed\n"));
-               TALLOC_FREE(state);
-               return;
-       }
-       tevent_schedule_immediate(im, msg_ctx->event_ctx,
-                                 messaging_defer_callback_trigger, state);
-}
-
-static void messaging_defer_callback_trigger(struct tevent_context *ev,
-                                            struct tevent_immediate *im,
-                                            void *private_data)
-{
-       struct messaging_defer_callback_state *state = talloc_get_type_abort(
-               private_data, struct messaging_defer_callback_state);
-       struct messaging_rec *rec = state->rec;
-
-       state->fn(state->msg_ctx, state->private_data, rec->msg_type, rec->src,
-                 &rec->buf);
-       TALLOC_FREE(state);
-}
-
-/*
-  Dispatch one messaging_rec
-*/
-void messaging_dispatch_rec(struct messaging_context *msg_ctx,
-                           struct messaging_rec *rec)
+static bool messaging_dispatch_classic(struct messaging_context *msg_ctx,
+                                      struct messaging_rec *rec)
 {
        struct messaging_callback *cb, *next;
-       unsigned i;
-       size_t j;
 
        for (cb = msg_ctx->callbacks; cb != NULL; cb = next) {
+               size_t j;
+
                next = cb->next;
                if (cb->msg_type != rec->msg_type) {
                        continue;
@@ -897,40 +1238,23 @@ void messaging_dispatch_rec(struct messaging_context *msg_ctx,
                rec->num_fds = 0;
                rec->fds = NULL;
 
-               if (server_id_same_process(&rec->src, &rec->dest)) {
-                       /*
-                        * This is a self-send. We are called here from
-                        * messaging_send(), and we don't want to directly
-                        * recurse into the callback but go via a
-                        * tevent_loop_once
-                        */
-                       messaging_defer_callback(msg_ctx, rec, cb->fn,
-                                                cb->private_data);
-               } else {
-                       /*
-                        * This comes from a different process. we are called
-                        * from the event loop, so we should call back
-                        * directly.
-                        */
-                       cb->fn(msg_ctx, cb->private_data, rec->msg_type,
-                              rec->src, &rec->buf);
-               }
-               /*
-                * we continue looking for matching messages after finding
-                * one. This matters for subsystems like the internal notify
-                * code which register more than one handler for the same
-                * message type
-                */
+               cb->fn(msg_ctx, cb->private_data, rec->msg_type,
+                      rec->src, &rec->buf);
+
+               return true;
        }
 
+       return false;
+}
+
+static bool messaging_dispatch_waiters(struct messaging_context *msg_ctx,
+                                      struct tevent_context *ev,
+                                      struct messaging_rec *rec)
+{
+       size_t i;
+
        if (!messaging_append_new_waiters(msg_ctx)) {
-               for (j=0; j < rec->num_fds; j++) {
-                       int fd = rec->fds[j];
-                       close(fd);
-               }
-               rec->num_fds = 0;
-               rec->fds = NULL;
-               return;
+               return false;
        }
 
        i = 0;
@@ -958,24 +1282,71 @@ void messaging_dispatch_rec(struct messaging_context *msg_ctx,
 
                state = tevent_req_data(
                        req, struct messaging_filtered_read_state);
-               if (state->filter(rec, state->private_data)) {
+               if ((ev == state->ev) &&
+                   state->filter(rec, state->private_data)) {
                        messaging_filtered_read_done(req, rec);
-
-                       /*
-                        * Only the first one gets the fd-array
-                        */
-                       rec->num_fds = 0;
-                       rec->fds = NULL;
+                       return true;
                }
 
                i += 1;
        }
 
+       return false;
+}
+
+/*
+  Dispatch one messaging_rec
+*/
+static void messaging_dispatch_rec(struct messaging_context *msg_ctx,
+                                  struct tevent_context *ev,
+                                  struct messaging_rec *rec)
+{
+       bool consumed;
+       size_t i;
+
+       if (ev == msg_ctx->event_ctx) {
+               consumed = messaging_dispatch_classic(msg_ctx, rec);
+               if (consumed) {
+                       return;
+               }
+       }
+
+       consumed = messaging_dispatch_waiters(msg_ctx, ev, rec);
+       if (consumed) {
+               return;
+       }
+
+       if (ev != msg_ctx->event_ctx) {
+               struct iovec iov;
+               int fds[rec->num_fds];
+               int ret;
+
+               /*
+                * We've been listening on a nested event
+                * context. Messages need to be handled in the main
+                * event context, so post to ourselves
+                */
+
+               iov.iov_base = rec->buf.data;
+               iov.iov_len = rec->buf.length;
+
+               for (i=0; i<rec->num_fds; i++) {
+                       fds[i] = rec->fds[i];
+               }
+
+               ret = messaging_post_self(
+                       msg_ctx, rec->src, rec->dest, rec->msg_type,
+                       &iov, 1, fds, rec->num_fds);
+               if (ret == 0) {
+                       return;
+               }
+       }
+
        /*
         * If the fd-array isn't used, just close it.
         */
-       for (j=0; j < rec->num_fds; j++) {
-               int fd = rec->fds[j];
+       for (i=0; i < rec->num_fds; i++) {
+               int fd = rec->fds[i];
                close(fd);
        }
        rec->num_fds = 0;
@@ -995,6 +1366,7 @@ bool messaging_parent_dgm_cleanup_init(struct messaging_context *msg)
                            60*15),
                mess_parent_dgm_cleanup, msg);
        if (req == NULL) {
+               DBG_WARNING("background_job_send failed\n");
                return false;
        }
        tevent_req_set_callback(req, mess_parent_dgm_cleanup_done, msg);
@@ -1030,6 +1402,7 @@ static void mess_parent_dgm_cleanup_done(struct tevent_req *req)
                mess_parent_dgm_cleanup, msg);
        if (req == NULL) {
                DEBUG(1, ("background_job_send failed\n"));
+               return;
        }
        tevent_req_set_callback(req, mess_parent_dgm_cleanup_done, msg);
 }
@@ -1053,4 +1426,9 @@ struct tevent_context *messaging_tevent_context(
        return msg_ctx->event_ctx;
 }
 
+struct server_id_db *messaging_names_db(struct messaging_context *msg_ctx)
+{
+       return msg_ctx->names_db;
+}
+
 /** @} **/