server/recoverd: if we can't get the recovery lock, ban ourself

[metze/ctdb/wip.git] / server / ctdb_recoverd.c

diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index 82fe62ea2db93fe8714e5668a25b86842fdc2ece..30c34b3a64dd7089fb6be94b2c2a7f1e67e72532 100644 (file)
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -18,7 +18,7 @@
 */
 
 #include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
 #include "system/filesys.h"
 #include "system/time.h"
 #include "system/network.h"
@@ -1382,8 +1382,10 @@ static int do_recovery(struct ctdb_recoverd *rec,
                DEBUG(DEBUG_ERR,("Taking out recovery lock from recovery daemon\n"));
                start_time = timeval_current();
                if (!ctdb_recovery_lock(ctdb, true)) {
-                       ctdb_set_culprit(rec, pnn);
-                       DEBUG(DEBUG_ERR,("Unable to get recovery lock - aborting recovery\n"));
+                       DEBUG(DEBUG_ERR,("Unable to get recovery lock - aborting recovery "
+                                        "and ban ourself for %u seconds\n",
+                                        ctdb->tunable.recovery_ban_period));
+                       ctdb_ban_node(rec, pnn, ctdb->tunable.recovery_ban_period);
                        return -1;
                }
                ctdb_ctrl_report_recd_lock_latency(ctdb, CONTROL_TIMEOUT(), timeval_elapsed(&start_time));
@@ -2774,6 +2776,7 @@ static int check_recovery_lock(struct ctdb_context *ctdb)
                close(state->fd[0]);
                state->fd[0] = -1;
 
+               debug_extra = talloc_asprintf(NULL, "recovery-lock:");
                if (pread(ctdb->recovery_lock_fd, &cc, 1, 0) == -1) {
                        DEBUG(DEBUG_CRIT,("failed read from recovery_lock_fd - %s\n", strerror(errno)));
                        cc = RECLOCK_FAILED;
@@ -2804,7 +2807,7 @@ static int check_recovery_lock(struct ctdb_context *ctdb)
        }
 
        state->fde = event_add_fd(ctdb->ev, state, state->fd[0],
-                               EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+                               EVENT_FD_READ,
                                reclock_child_handler,
                                (void *)state);
 
@@ -2813,6 +2816,7 @@ static int check_recovery_lock(struct ctdb_context *ctdb)
                talloc_free(state);
                return -1;
        }
+       tevent_fd_set_auto_close(state->fde);
 
        while (state->status == RECLOCK_CHECKING) {
                event_loop_once(ctdb->ev);
@@ -3007,10 +3011,6 @@ static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec,
                        rec->reallocate_callers = NULL;
                }
        }
-       /* if there are takeovers requested, perform it and notify the waiters */
-       if (rec->reallocate_callers) {
-               process_ipreallocate_requests(ctdb, rec);
-       }
 
        if (rec->recmaster == (uint32_t)-1) {
                DEBUG(DEBUG_NOTICE,(__location__ " Initial recovery master set - forcing election\n"));
@@ -3197,6 +3197,11 @@ static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec,
                }
        }
 
+       /* if there are takeovers requested, perform it and notify the waiters */
+       if (rec->reallocate_callers) {
+               process_ipreallocate_requests(ctdb, rec);
+       }
+
        /* get the nodemap for all active remote nodes
         */
        remote_nodemaps = talloc_array(mem_ctx, struct ctdb_node_map *, nodemap->num);
@@ -3558,6 +3563,7 @@ int ctdb_start_recoverd(struct ctdb_context *ctdb)
 {
        int fd[2];
        struct signal_event *se;
+       struct tevent_fd *fde;
 
        if (pipe(fd) != 0) {
                return -1;
@@ -3582,15 +3588,16 @@ int ctdb_start_recoverd(struct ctdb_context *ctdb)
 
        srandom(getpid() ^ time(NULL));
 
-       if (switch_from_server_to_client(ctdb) != 0) {
+       if (switch_from_server_to_client(ctdb, "recoverd") != 0) {
                DEBUG(DEBUG_CRIT, (__location__ "ERROR: failed to switch recovery daemon into client mode. shutting down.\n"));
                exit(1);
        }
 
        DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to recovery daemon\n", fd[0]));
 
-       event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+       fde = event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ,
                     ctdb_recoverd_parent, &fd[0]);     
+       tevent_fd_set_auto_close(fde);
 
        /* set up a handler to pick up sigchld */
        se = event_add_signal(ctdb->ev, ctdb,