pytdb: Add support for tdb_enable_seqnum, tdb_get_seqnum and tdb_increment_seqnum_non...
[metze/ctdb/wip.git] / server / ctdb_recoverd.c
index 3ee607e5e19a4f00f9ba988d327930b37b6175b3..541eb29b88155696bacf03de6de693211aad095b 100644 (file)
 */
 
 #include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
 #include "system/filesys.h"
 #include "system/time.h"
 #include "system/network.h"
 #include "system/wait.h"
 #include "popt.h"
 #include "cmdline.h"
-#include "../include/ctdb.h"
+#include "../include/ctdb_client.h"
 #include "../include/ctdb_private.h"
 #include "db_wrap.h"
 #include "dlinklist.h"
@@ -887,10 +887,11 @@ static void ctdb_wait_handler(struct event_context *ev, struct timed_event *te,
 /*
   wait for a given number of seconds
  */
-static void ctdb_wait_timeout(struct ctdb_context *ctdb, uint32_t secs)
+static void ctdb_wait_timeout(struct ctdb_context *ctdb, double secs)
 {
        uint32_t timed_out = 0;
-       event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(secs, 0), ctdb_wait_handler, &timed_out);
+       time_t usecs = (secs - (time_t)secs) * 1000000;
+       event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(secs, usecs), ctdb_wait_handler, &timed_out);
        while (!timed_out) {
                event_loop_once(ctdb->ev);
        }
@@ -904,6 +905,7 @@ static void ctdb_election_timeout(struct event_context *ev, struct timed_event *
 {
        struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
        rec->election_timeout = NULL;
+       fast_start = false;
 
        DEBUG(DEBUG_WARNING,(__location__ " Election timed out\n"));
 }
@@ -1302,6 +1304,41 @@ static int ctdb_reload_remote_public_ips(struct ctdb_context *ctdb,
        return 0;
 }
 
+/* when we start a recovery, make sure all nodes use the same reclock file
+   setting
+*/
+static int sync_recovery_lock_file_across_cluster(struct ctdb_recoverd *rec)
+{
+       struct ctdb_context *ctdb = rec->ctdb;
+       TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+       TDB_DATA data;
+       uint32_t *nodes;
+
+       if (ctdb->recovery_lock_file == NULL) {
+               data.dptr  = NULL;
+               data.dsize = 0;
+       } else {
+               data.dsize = strlen(ctdb->recovery_lock_file) + 1;
+               data.dptr  = (uint8_t *)ctdb->recovery_lock_file;
+       }
+
+       nodes = list_of_active_nodes(ctdb, rec->nodemap, tmp_ctx, true);
+       if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECLOCK_FILE,
+                                       nodes, 0,
+                                       CONTROL_TIMEOUT(),
+                                       false, data,
+                                       NULL, NULL,
+                                       rec) != 0) {
+               DEBUG(DEBUG_ERR, (__location__ " Failed to sync reclock file settings\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       talloc_free(tmp_ctx);
+       return 0;
+}
+
+
 /*
   we are the recmaster, and recovery is needed - start a recovery run
  */
@@ -1345,8 +1382,10 @@ static int do_recovery(struct ctdb_recoverd *rec,
                DEBUG(DEBUG_ERR,("Taking out recovery lock from recovery daemon\n"));
                start_time = timeval_current();
                if (!ctdb_recovery_lock(ctdb, true)) {
-                       ctdb_set_culprit(rec, pnn);
-                       DEBUG(DEBUG_ERR,("Unable to get recovery lock - aborting recovery\n"));
+                       DEBUG(DEBUG_ERR,("Unable to get recovery lock - aborting recovery "
+                                        "and ban ourself for %u seconds\n",
+                                        ctdb->tunable.recovery_ban_period));
+                       ctdb_ban_node(rec, pnn, ctdb->tunable.recovery_ban_period);
                        return -1;
                }
                ctdb_ctrl_report_recd_lock_latency(ctdb, CONTROL_TIMEOUT(), timeval_elapsed(&start_time));
@@ -1388,6 +1427,11 @@ static int do_recovery(struct ctdb_recoverd *rec,
        DEBUG(DEBUG_NOTICE, (__location__ " Recovery - updated db priority for all databases\n"));
 
 
+       /* update all other nodes to use the same setting for reclock files
+          as the local recovery master.
+       */
+       sync_recovery_lock_file_across_cluster(rec);
+
        /* set recovery mode to active on all nodes */
        ret = set_recovery_mode(ctdb, rec, nodemap, CTDB_RECOVERY_ACTIVE);
        if (ret != 0) {
@@ -1601,7 +1645,7 @@ static int do_recovery(struct ctdb_recoverd *rec,
 
        /* send a message to all clients telling them that the cluster 
           has been reconfigured */
-       ctdb_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECONFIGURE, tdb_null);
+       ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECONFIGURE, tdb_null);
 
        DEBUG(DEBUG_NOTICE, (__location__ " Recovery complete\n"));
 
@@ -1632,9 +1676,9 @@ static int do_recovery(struct ctdb_recoverd *rec,
           We now wait for rerecovery_timeout before we allow 
           another recovery to take place.
        */
-       DEBUG(DEBUG_NOTICE, (__location__ " New recoveries supressed for the rerecovery timeout\n"));
+       DEBUG(DEBUG_NOTICE, ("Just finished a recovery. New recoveries will now be supressed for the rerecovery timeout (%d seconds)\n", ctdb->tunable.rerecovery_timeout));
        ctdb_wait_timeout(ctdb, ctdb->tunable.rerecovery_timeout);
-       DEBUG(DEBUG_NOTICE, (__location__ " Rerecovery timeout elapsed. Recovery reactivated.\n"));
+       DEBUG(DEBUG_NOTICE, ("The rerecovery timeout has elapsed. We now allow recoveries to trigger again.\n"));
 
        return 0;
 }
@@ -1762,7 +1806,7 @@ static int send_election_request(struct ctdb_recoverd *rec, uint32_t pnn, bool u
 
        /* send an election message to all active nodes */
        DEBUG(DEBUG_INFO,(__location__ " Send election request to all active nodes\n"));
-       ctdb_send_message(ctdb, CTDB_BROADCAST_ALL, srvid, election_data);
+       ctdb_client_send_message(ctdb, CTDB_BROADCAST_ALL, srvid, election_data);
 
 
        /* A new node that is already frozen has entered the cluster.
@@ -1860,7 +1904,7 @@ static void mem_dump_handler(struct ctdb_context *ctdb, uint64_t srvid,
 
 DEBUG(DEBUG_ERR, ("recovery master memory dump\n"));           
 
-       ret = ctdb_send_message(ctdb, rd->pnn, rd->srvid, *dump);
+       ret = ctdb_client_send_message(ctdb, rd->pnn, rd->srvid, *dump);
        if (ret != 0) {
                DEBUG(DEBUG_ERR,("Failed to send rd memdump reply message\n"));
                talloc_free(tmp_ctx);
@@ -2019,7 +2063,7 @@ static void process_ipreallocate_requests(struct ctdb_context *ctdb, struct ctdb
                DEBUG(DEBUG_INFO,("Sending ip reallocate reply message to "
                                  "%u:%llu\n", (unsigned)callers->rd->pnn,
                                  (unsigned long long)callers->rd->srvid));
-               ret = ctdb_send_message(ctdb, callers->rd->pnn, callers->rd->srvid, result);
+               ret = ctdb_client_send_message(ctdb, callers->rd->pnn, callers->rd->srvid, result);
                if (ret != 0) {
                        DEBUG(DEBUG_ERR,("Failed to send ip reallocate reply "
                                         "message to %u:%llu\n",
@@ -2050,6 +2094,8 @@ static void election_handler(struct ctdb_context *ctdb, uint64_t srvid,
        /* we got an election packet - update the timeout for the election */
        talloc_free(rec->election_timeout);
        rec->election_timeout = event_add_timed(ctdb->ev, ctdb, 
+                                               fast_start ?
+                                               timeval_current_ofs(0, 500000) :
                                                timeval_current_ofs(ctdb->tunable.election_timeout, 0), 
                                                ctdb_election_timeout, rec);
 
@@ -2117,6 +2163,8 @@ static void force_election(struct ctdb_recoverd *rec, uint32_t pnn,
 
        talloc_free(rec->election_timeout);
        rec->election_timeout = event_add_timed(ctdb->ev, ctdb, 
+                                               fast_start ?
+                                               timeval_current_ofs(0, 500000) :
                                                timeval_current_ofs(ctdb->tunable.election_timeout, 0), 
                                                ctdb_election_timeout, rec);
 
@@ -2567,7 +2615,7 @@ static int verify_local_ip_allocation(struct ctdb_context *ctdb, struct ctdb_rec
                data.dptr = (uint8_t *)&rd;
                data.dsize = sizeof(rd);
 
-               ret = ctdb_send_message(ctdb, rec->recmaster, CTDB_SRVID_TAKEOVER_RUN, data);
+               ret = ctdb_client_send_message(ctdb, rec->recmaster, CTDB_SRVID_TAKEOVER_RUN, data);
                if (ret != 0) {
                        DEBUG(DEBUG_ERR,(__location__ " Failed to send ipreallocate to recmaster :%d\n", (int)rec->recmaster));
                }
@@ -2728,6 +2776,7 @@ static int check_recovery_lock(struct ctdb_context *ctdb)
                close(state->fd[0]);
                state->fd[0] = -1;
 
+               debug_extra = talloc_asprintf(NULL, "recovery-lock:");
                if (pread(ctdb->recovery_lock_fd, &cc, 1, 0) == -1) {
                        DEBUG(DEBUG_CRIT,("failed read from recovery_lock_fd - %s\n", strerror(errno)));
                        cc = RECLOCK_FAILED;
@@ -2758,7 +2807,7 @@ static int check_recovery_lock(struct ctdb_context *ctdb)
        }
 
        state->fde = event_add_fd(ctdb->ev, state, state->fd[0],
-                               EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+                               EVENT_FD_READ,
                                reclock_child_handler,
                                (void *)state);
 
@@ -2767,6 +2816,7 @@ static int check_recovery_lock(struct ctdb_context *ctdb)
                talloc_free(state);
                return -1;
        }
+       tevent_fd_set_auto_close(state->fde);
 
        while (state->status == RECLOCK_CHECKING) {
                event_loop_once(ctdb->ev);
@@ -2837,14 +2887,11 @@ static int update_recovery_lock_file(struct ctdb_context *ctdb)
        talloc_free(tmp_ctx);
        return 0;
 }
-               
-/*
-  the main monitoring loop
- */
-static void monitor_cluster(struct ctdb_context *ctdb)
+
+static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec,
+                     TALLOC_CTX *mem_ctx)
 {
        uint32_t pnn;
-       TALLOC_CTX *mem_ctx=NULL;
        struct ctdb_node_map *nodemap=NULL;
        struct ctdb_node_map *recmaster_nodemap=NULL;
        struct ctdb_node_map **remote_nodemaps=NULL;
@@ -2852,57 +2899,8 @@ static void monitor_cluster(struct ctdb_context *ctdb)
        struct ctdb_vnn_map *remote_vnnmap=NULL;
        int32_t debug_level;
        int i, j, ret;
-       struct ctdb_recoverd *rec;
-
-       DEBUG(DEBUG_NOTICE,("monitor_cluster starting\n"));
-
-       rec = talloc_zero(ctdb, struct ctdb_recoverd);
-       CTDB_NO_MEMORY_FATAL(ctdb, rec);
 
-       rec->ctdb = ctdb;
-
-       rec->priority_time = timeval_current();
-
-       /* register a message port for sending memory dumps */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_MEM_DUMP, mem_dump_handler, rec);
-
-       /* register a message port for recovery elections */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_RECOVERY, election_handler, rec);
-
-       /* when nodes are disabled/enabled */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_SET_NODE_FLAGS, monitor_handler, rec);
-
-       /* when we are asked to puch out a flag change */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_PUSH_NODE_FLAGS, push_flags_handler, rec);
-
-       /* register a message port for vacuum fetch */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_VACUUM_FETCH, vacuum_fetch_handler, rec);
-
-       /* register a message port for reloadnodes  */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_RELOAD_NODES, reload_nodes_handler, rec);
-
-       /* register a message port for performing a takeover run */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_TAKEOVER_RUN, ip_reallocate_handler, rec);
-
-       /* register a message port for disabling the ip check for a short while */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_DISABLE_IP_CHECK, disable_ip_check_handler, rec);
 
-       /* register a message port for updating the recovery daemons node assignment for an ip */
-       ctdb_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP, recd_update_ip_handler, rec);
-
-again:
-       if (mem_ctx) {
-               talloc_free(mem_ctx);
-               mem_ctx = NULL;
-       }
-       mem_ctx = talloc_new(ctdb);
-       if (!mem_ctx) {
-               DEBUG(DEBUG_CRIT,(__location__ " Failed to create temporary context\n"));
-               exit(-1);
-       }
-
-       /* we only check for recovery once every second */
-       ctdb_wait_timeout(ctdb, ctdb->tunable.recover_interval);
 
        /* verify that the main daemon is still running */
        if (kill(ctdb->ctdbd_pid, 0) != 0) {
@@ -2915,14 +2913,14 @@ again:
 
        if (rec->election_timeout) {
                /* an election is in progress */
-               goto again;
+               return;
        }
 
        /* read the debug level from the parent and update locally */
        ret = ctdb_ctrl_get_debuglevel(ctdb, CTDB_CURRENT_NODE, &debug_level);
        if (ret !=0) {
                DEBUG(DEBUG_ERR, (__location__ " Failed to read debuglevel from parent\n"));
-               goto again;
+               return;
        }
        LogLevel = debug_level;
 
@@ -2952,13 +2950,13 @@ again:
        ret = ctdb_ctrl_get_all_tunables(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &ctdb->tunable);
        if (ret != 0) {
                DEBUG(DEBUG_ERR,("Failed to get tunables - retrying\n"));
-               goto again;
+               return;
        }
 
        /* get the current recovery lock file from the server */
        if (update_recovery_lock_file(ctdb) != 0) {
                DEBUG(DEBUG_ERR,("Failed to update the recovery lock file\n"));
-               goto again;
+               return;
        }
 
        /* Make sure that if recovery lock verification becomes disabled when
@@ -2974,14 +2972,14 @@ again:
        pnn = ctdb_ctrl_getpnn(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
        if (pnn == (uint32_t)-1) {
                DEBUG(DEBUG_ERR,("Failed to get local pnn - retrying\n"));
-               goto again;
+               return;
        }
 
        /* get the vnnmap */
        ret = ctdb_ctrl_getvnnmap(ctdb, CONTROL_TIMEOUT(), pnn, mem_ctx, &vnnmap);
        if (ret != 0) {
                DEBUG(DEBUG_ERR, (__location__ " Unable to get vnnmap from node %u\n", pnn));
-               goto again;
+               return;
        }
 
 
@@ -2994,7 +2992,7 @@ again:
        ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), pnn, rec, &rec->nodemap);
        if (ret != 0) {
                DEBUG(DEBUG_ERR, (__location__ " Unable to get nodemap from node %u\n", pnn));
-               goto again;
+               return;
        }
        nodemap = rec->nodemap;
 
@@ -3002,7 +3000,7 @@ again:
        ret = ctdb_ctrl_getrecmaster(ctdb, mem_ctx, CONTROL_TIMEOUT(), pnn, &rec->recmaster);
        if (ret != 0) {
                DEBUG(DEBUG_ERR, (__location__ " Unable to get recmaster from node %u\n", pnn));
-               goto again;
+               return;
        }
 
        /* if we are not the recmaster we can safely ignore any ip reallocate requests */
@@ -3013,15 +3011,11 @@ again:
                        rec->reallocate_callers = NULL;
                }
        }
-       /* if there are takeovers requested, perform it and notify the waiters */
-       if (rec->reallocate_callers) {
-               process_ipreallocate_requests(ctdb, rec);
-       }
 
        if (rec->recmaster == (uint32_t)-1) {
                DEBUG(DEBUG_NOTICE,(__location__ " Initial recovery master set - forcing election\n"));
                force_election(rec, pnn, nodemap);
-               goto again;
+               return;
        }
 
 
@@ -3039,15 +3033,15 @@ again:
                        ret = ctdb_ctrl_freeze_priority(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, 1);
                        if (ret != 0) {
                                DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to node being STOPPED\n"));
-                               goto again;
+                               return;
                        }
                        ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, CTDB_RECOVERY_ACTIVE);
                        if (ret != 0) {
                                DEBUG(DEBUG_ERR,(__location__ " Failed to activate recovery mode due to node being stopped\n"));
 
-                               goto again;
+                               return;
                        }
-                       goto again;
+                       return;
                }
        }
        /* If the local node is stopped, verify we are not the recmaster 
@@ -3056,7 +3050,7 @@ again:
        if ((nodemap->nodes[pnn].flags & NODE_FLAGS_STOPPED) && (rec->recmaster == pnn)) {
                DEBUG(DEBUG_ERR,("Local node is STOPPED. Yielding recmaster role\n"));
                force_election(rec, pnn, nodemap);
-               goto again;
+               return;
        }
        
        /* check that we (recovery daemon) and the local ctdb daemon
@@ -3090,14 +3084,14 @@ again:
        if (j == nodemap->num) {
                DEBUG(DEBUG_ERR, ("Recmaster node %u not in list. Force reelection\n", rec->recmaster));
                force_election(rec, pnn, nodemap);
-               goto again;
+               return;
        }
 
        /* if recovery master is disconnected we must elect a new recmaster */
        if (nodemap->nodes[j].flags & NODE_FLAGS_DISCONNECTED) {
                DEBUG(DEBUG_NOTICE, ("Recmaster node %u is disconnected. Force reelection\n", nodemap->nodes[j].pnn));
                force_election(rec, pnn, nodemap);
-               goto again;
+               return;
        }
 
        /* grap the nodemap from the recovery master to check if it is banned */
@@ -3106,14 +3100,14 @@ again:
        if (ret != 0) {
                DEBUG(DEBUG_ERR, (__location__ " Unable to get nodemap from recovery master %u\n", 
                          nodemap->nodes[j].pnn));
-               goto again;
+               return;
        }
 
 
        if (recmaster_nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
                DEBUG(DEBUG_NOTICE, ("Recmaster node %u no longer available. Force reelection\n", nodemap->nodes[j].pnn));
                force_election(rec, pnn, nodemap);
-               goto again;
+               return;
        }
 
 
@@ -3133,7 +3127,7 @@ again:
           if recovery is needed
         */
        if (pnn != rec->recmaster) {
-               goto again;
+               return;
        }
 
 
@@ -3142,38 +3136,38 @@ again:
        if (ret == MONITOR_ELECTION_NEEDED) {
                DEBUG(DEBUG_NOTICE,("update_local_flags() called for a re-election.\n"));
                force_election(rec, pnn, nodemap);
-               goto again;
+               return;
        }
        if (ret != MONITOR_OK) {
                DEBUG(DEBUG_ERR,("Unable to update local flags\n"));
-               goto again;
+               return;
        }
 
        if (ctdb->num_nodes != nodemap->num) {
                DEBUG(DEBUG_ERR, (__location__ " ctdb->num_nodes (%d) != nodemap->num (%d) reloading nodes file\n", ctdb->num_nodes, nodemap->num));
                reload_nodes_file(ctdb);
-               goto again;
+               return;
        }
 
        /* verify that all active nodes agree that we are the recmaster */
        switch (verify_recmaster(rec, nodemap, pnn)) {
        case MONITOR_RECOVERY_NEEDED:
                /* can not happen */
-               goto again;
+               return;
        case MONITOR_ELECTION_NEEDED:
                force_election(rec, pnn, nodemap);
-               goto again;
+               return;
        case MONITOR_OK:
                break;
        case MONITOR_FAILED:
-               goto again;
+               return;
        }
 
 
        if (rec->need_recovery) {
                /* a previous recovery didn't finish */
                do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-               goto again;             
+               return;
        }
 
        /* verify that all active nodes are in normal mode 
@@ -3182,9 +3176,9 @@ again:
        switch (verify_recmode(ctdb, nodemap)) {
        case MONITOR_RECOVERY_NEEDED:
                do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-               goto again;
+               return;
        case MONITOR_FAILED:
-               goto again;
+               return;
        case MONITOR_ELECTION_NEEDED:
                /* can not happen */
        case MONITOR_OK:
@@ -3199,23 +3193,28 @@ again:
                        DEBUG(DEBUG_ERR,("Failed check_recovery_lock. Force a recovery\n"));
                        ctdb_set_culprit(rec, ctdb->pnn);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
        }
 
+       /* if there are takeovers requested, perform it and notify the waiters */
+       if (rec->reallocate_callers) {
+               process_ipreallocate_requests(ctdb, rec);
+       }
+
        /* get the nodemap for all active remote nodes
         */
        remote_nodemaps = talloc_array(mem_ctx, struct ctdb_node_map *, nodemap->num);
        if (remote_nodemaps == NULL) {
                DEBUG(DEBUG_ERR, (__location__ " failed to allocate remote nodemap array\n"));
-               goto again;
+               return;
        }
        for(i=0; i<nodemap->num; i++) {
                remote_nodemaps[i] = NULL;
        }
        if (get_remote_nodemaps(ctdb, mem_ctx, nodemap, remote_nodemaps) != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Failed to read remote nodemaps\n"));
-               goto again;
+               return;
        } 
 
        /* verify that all other nodes have the same nodemap as we have
@@ -3229,7 +3228,7 @@ again:
                        DEBUG(DEBUG_ERR,(__location__ " Did not get a remote nodemap for node %d, restarting monitoring\n", j));
                        ctdb_set_culprit(rec, j);
 
-                       goto again;
+                       return;
                }
 
                /* if the nodes disagree on how many nodes there are
@@ -3240,7 +3239,7 @@ again:
                                  nodemap->nodes[j].pnn, remote_nodemaps[j]->num, nodemap->num));
                        ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
 
                /* if the nodes disagree on which nodes exist and are
@@ -3254,7 +3253,7 @@ again:
                                ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                                do_recovery(rec, mem_ctx, pnn, nodemap, 
                                            vnnmap);
-                               goto again;
+                               return;
                        }
                }
 
@@ -3277,14 +3276,14 @@ again:
                                        ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                                        do_recovery(rec, mem_ctx, pnn, nodemap, 
                                                    vnnmap);
-                                       goto again;
+                                       return;
                                } else {
                                        DEBUG(DEBUG_ERR,("Use flags 0x%02x from local recmaster node for cluster update of node %d flags\n", nodemap->nodes[i].flags, i));
                                        update_flags_on_all_nodes(ctdb, nodemap, nodemap->nodes[i].pnn, nodemap->nodes[i].flags);
                                        ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                                        do_recovery(rec, mem_ctx, pnn, nodemap, 
                                                    vnnmap);
-                                       goto again;
+                                       return;
                                }
                        }
                }
@@ -3299,7 +3298,7 @@ again:
                          vnnmap->size, rec->num_active));
                ctdb_set_culprit(rec, ctdb->pnn);
                do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-               goto again;
+               return;
        }
 
        /* verify that all active nodes in the nodemap also exist in 
@@ -3323,7 +3322,7 @@ again:
                                  nodemap->nodes[j].pnn));
                        ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
        }
 
@@ -3344,7 +3343,7 @@ again:
                if (ret != 0) {
                        DEBUG(DEBUG_ERR, (__location__ " Unable to get vnnmap from remote node %u\n", 
                                  nodemap->nodes[j].pnn));
-                       goto again;
+                       return;
                }
 
                /* verify the vnnmap generation is the same */
@@ -3353,7 +3352,7 @@ again:
                                  nodemap->nodes[j].pnn, remote_vnnmap->generation, vnnmap->generation));
                        ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
 
                /* verify the vnnmap size is the same */
@@ -3362,7 +3361,7 @@ again:
                                  nodemap->nodes[j].pnn, remote_vnnmap->size, vnnmap->size));
                        ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
 
                /* verify the vnnmap is the same */
@@ -3373,7 +3372,7 @@ again:
                                ctdb_set_culprit(rec, nodemap->nodes[j].pnn);
                                do_recovery(rec, mem_ctx, pnn, nodemap, 
                                            vnnmap);
-                               goto again;
+                               return;
                        }
                }
        }
@@ -3393,7 +3392,7 @@ again:
                                         culprit));
                        ctdb_set_culprit(rec, culprit);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
 
                /* execute the "startrecovery" event script on all nodes */
@@ -3402,7 +3401,7 @@ again:
                        DEBUG(DEBUG_ERR, (__location__ " Unable to run the 'startrecovery' event on cluster\n"));
                        ctdb_set_culprit(rec, ctdb->pnn);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
 
                ret = ctdb_takeover_run(ctdb, nodemap);
@@ -3410,7 +3409,7 @@ again:
                        DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses - starting recovery\n"));
                        ctdb_set_culprit(rec, ctdb->pnn);
                        do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
-                       goto again;
+                       return;
                }
 
                /* execute the "recovered" event script on all nodes */
@@ -3427,10 +3426,73 @@ again:
                }
 #endif
        }
+}
+
+/*
+  the main monitoring loop
+ */
+static void monitor_cluster(struct ctdb_context *ctdb)
+{
+       struct ctdb_recoverd *rec;
+
+       DEBUG(DEBUG_NOTICE,("monitor_cluster starting\n"));
+
+       rec = talloc_zero(ctdb, struct ctdb_recoverd);
+       CTDB_NO_MEMORY_FATAL(ctdb, rec);
+
+       rec->ctdb = ctdb;
+
+       rec->priority_time = timeval_current();
+
+       /* register a message port for sending memory dumps */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_MEM_DUMP, mem_dump_handler, rec);
+
+       /* register a message port for recovery elections */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RECOVERY, election_handler, rec);
 
+       /* when nodes are disabled/enabled */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_SET_NODE_FLAGS, monitor_handler, rec);
+
+       /* when we are asked to puch out a flag change */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_PUSH_NODE_FLAGS, push_flags_handler, rec);
+
+       /* register a message port for vacuum fetch */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_VACUUM_FETCH, vacuum_fetch_handler, rec);
 
-       goto again;
+       /* register a message port for reloadnodes  */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RELOAD_NODES, reload_nodes_handler, rec);
+
+       /* register a message port for performing a takeover run */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_TAKEOVER_RUN, ip_reallocate_handler, rec);
+
+       /* register a message port for disabling the ip check for a short while */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_DISABLE_IP_CHECK, disable_ip_check_handler, rec);
 
+       /* register a message port for updating the recovery daemons node assignment for an ip */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP, recd_update_ip_handler, rec);
+
+       for (;;) {
+               TALLOC_CTX *mem_ctx = talloc_new(ctdb);
+               struct timeval start;
+               double elapsed;
+
+               if (!mem_ctx) {
+                       DEBUG(DEBUG_CRIT,(__location__
+                                         " Failed to create temp context\n"));
+                       exit(-1);
+               }
+
+               start = timeval_current();
+               main_loop(ctdb, rec, mem_ctx);
+               talloc_free(mem_ctx);
+
+               /* we only check for recovery once every second */
+               elapsed = timeval_elapsed(&start);
+               if (elapsed < ctdb->tunable.recover_interval) {
+                       ctdb_wait_timeout(ctdb, ctdb->tunable.recover_interval
+                                         - elapsed);
+               }
+       }
 }
 
 /*
@@ -3501,6 +3563,7 @@ int ctdb_start_recoverd(struct ctdb_context *ctdb)
 {
        int fd[2];
        struct signal_event *se;
+       struct tevent_fd *fde;
 
        if (pipe(fd) != 0) {
                return -1;
@@ -3525,15 +3588,16 @@ int ctdb_start_recoverd(struct ctdb_context *ctdb)
 
        srandom(getpid() ^ time(NULL));
 
-       if (switch_from_server_to_client(ctdb) != 0) {
+       if (switch_from_server_to_client(ctdb, "recoverd") != 0) {
                DEBUG(DEBUG_CRIT, (__location__ "ERROR: failed to switch recovery daemon into client mode. shutting down.\n"));
                exit(1);
        }
 
        DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to recovery daemon\n", fd[0]));
 
-       event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+       fde = event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ,
                     ctdb_recoverd_parent, &fd[0]);     
+       tevent_fd_set_auto_close(fde);
 
        /* set up a handler to pick up sigchld */
        se = event_add_signal(ctdb->ev, ctdb,