New version 1.2.19
[sahlberg/ctdb.git] / server / ctdb_freeze.c
index 2cc39aa274af438fd8e160b604b2745462e86a83..86cb5edc120e5b2ff4f5838356eefe7b80482b93 100644 (file)
@@ -17,7 +17,7 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 #include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
 #include "lib/tdb/include/tdb.h"
 #include "system/network.h"
 #include "system/filesys.h"
 #include "lib/util/dlinklist.h"
 #include "db_wrap.h"
 
+static bool later_db(const char *name)
+{
+       return (strstr(name, "notify") || strstr(name, "serverid"));
+}
 
 /*
   lock all databases
 static int ctdb_lock_all_databases(struct ctdb_context *ctdb, uint32_t priority)
 {
        struct ctdb_db_context *ctdb_db;
+       /* REMOVE later */
+       /* This double loop is for backward compatibility and deadlock
+          avoidance for old samba versions that not yet support
+          the set prio call.
+          This code shall be removed later
+       */
        for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
                if (ctdb_db->priority != priority) {
                        continue;
                }
+               if (later_db(ctdb_db->db_name)) {
+                       continue;
+               }
                DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
                if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
+                       DEBUG(DEBUG_ERR,(__location__ " Failed to lock database %s\n", ctdb_db->db_name));
+                       return -1;
+               }
+       }
+       for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               if (ctdb_db->priority != priority) {
+                       continue;
+               }
+               if (!later_db(ctdb_db->db_name)) {
+                       continue;
+               }
+               DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
+               if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
+                       DEBUG(DEBUG_ERR,(__location__ " Failed to lock database %s\n", ctdb_db->db_name));
                        return -1;
                }
        }
@@ -130,9 +157,12 @@ static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *
        h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
 
        /* notify the waiters */
-       while ((w = h->ctdb->freeze_handles[h->priority]->waiters)) {
+       if (h != h->ctdb->freeze_handles[h->priority]) {
+               DEBUG(DEBUG_ERR,("lockwait finished but h is not linked\n"));
+       }
+       while ((w = h->waiters)) {
                w->status = status;
-               DLIST_REMOVE(h->ctdb->freeze_handles[h->priority]->waiters, w);
+               DLIST_REMOVE(h->waiters, w);
                talloc_free(w);
        }
 }
@@ -153,8 +183,7 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, ui
        h->ctdb     = ctdb;
        h->priority = priority;
 
-       /* use socketpair() instead of pipe() so we have bi-directional fds */
-       if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != 0) {
+       if (pipe(fd) == -1) {
                DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
                talloc_free(h);
                return NULL;
@@ -169,41 +198,41 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, ui
 
        if (h->child == 0) {
                int ret;
-               int count = 0;
+
                /* in the child */
                close(fd[0]);
+
+               debug_extra = talloc_asprintf(NULL, "freeze_lock-%u:", priority);
                ret = ctdb_lock_all_databases(ctdb, priority);
                if (ret != 0) {
                        _exit(0);
                }
 
-               alarm(30);
-
-               while (count++ < 30) {
-                       ret = write(fd[1], &ret, sizeof(ret));
-                       if (ret == sizeof(ret)) {
-                               break;
-                       }
+               ret = write(fd[1], &ret, sizeof(ret));
+               if (ret != sizeof(ret)) {
                        DEBUG(DEBUG_ERR, (__location__ " Failed to write to socket from freeze child. ret:%d errno:%u\n", ret, errno));
-                       sleep (1);
-               }
-               if (count >= 30) {
-                       DEBUG(DEBUG_ERR, (__location__ " Failed to write to socket from freeze child. Aborting freeze child\n"));
-                       _exit(0);
+                       _exit(1);
                }
 
-               /* the read here means we will die if the parent exits */
-               read(fd[1], &ret, sizeof(ret));
-               _exit(0);
+               while (1) {
+                       sleep(1);
+                       if (kill(ctdb->ctdbd_pid, 0) != 0) {
+                               DEBUG(DEBUG_ERR,("Parent died. Exiting lock wait child\n"));
+
+                               _exit(0);
+                       }
+               }
        }
 
        talloc_set_destructor(h, ctdb_freeze_handle_destructor);
 
        close(fd[1]);
+       set_close_on_exec(fd[0]);
 
        h->fd = fd[0];
 
-       fde = event_add_fd(ctdb->ev, h, h->fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+
+       fde = event_add_fd(ctdb->ev, h, h->fd, EVENT_FD_READ,
                           ctdb_freeze_lock_handler, h);
        if (fde == NULL) {
                DEBUG(DEBUG_ERR,("Failed to setup fd event for ctdb_freeze_lock\n"));
@@ -211,6 +240,7 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, ui
                talloc_free(h);
                return NULL;
        }
+       tevent_fd_set_auto_close(fde);
 
        return h;
 }
@@ -220,7 +250,6 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, ui
  */
 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 {
-       DLIST_REMOVE(w->ctdb->freeze_handles[w->priority]->waiters, w);
        ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
        return 0;
 }
@@ -230,6 +259,11 @@ static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
  */
 int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 {
+       if (priority == 0) {
+               DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
+               priority = 1;
+       }
+
        if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
                DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
                return -1;
@@ -240,8 +274,11 @@ int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
                return 0;
        }
 
+       /* Stop any vacuuming going on: we don't want to wait. */
+       ctdb_stop_vacuuming(ctdb);
+
        /* if there isn't a freeze lock child then create one */
-       if (!ctdb->freeze_handles[priority]) {
+       if (ctdb->freeze_handles[priority] == NULL) {
                ctdb->freeze_handles[priority] = ctdb_freeze_lock(ctdb, priority);
                CTDB_NO_MEMORY(ctdb, ctdb->freeze_handles[priority]);
                ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
@@ -262,6 +299,11 @@ int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *
 
        DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
 
+       if (priority == 0) {
+               DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
+               priority = 1;
+       }
+
        if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
                DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
                return -1;
@@ -278,6 +320,11 @@ int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *
        }
 
        /* add ourselves to list of waiters */
+       if (ctdb->freeze_handles[priority] == NULL) {
+               DEBUG(DEBUG_ERR,("No freeze lock handle when adding a waiter\n"));
+               return -1;
+       }
+
        w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
        CTDB_NO_MEMORY(ctdb, w);
        w->ctdb     = ctdb;
@@ -345,8 +392,10 @@ static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
        system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
 #endif
 
-       talloc_free(ctdb->freeze_handles[priority]);
-       ctdb->freeze_handles[priority] = NULL;
+       if (ctdb->freeze_handles[priority] != NULL) {
+               talloc_free(ctdb->freeze_handles[priority]);
+               ctdb->freeze_handles[priority] = NULL;
+       }
 }
 
 /*
@@ -419,6 +468,31 @@ int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
        return 0;
 }
 
+/*
+  cancel a transaction for all databases - used for recovery
+ */
+int32_t ctdb_control_transaction_cancel(struct ctdb_context *ctdb)
+{
+       struct ctdb_db_context *ctdb_db;
+
+       DEBUG(DEBUG_ERR,(__location__ " recovery transaction cancelled called\n"));
+
+       for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
+
+               if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
+                       DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",  ctdb_db->db_name));
+                       /* not a fatal error */
+               }
+
+               tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
+       }
+
+       ctdb->freeze_transaction_started = false;
+
+       return 0;
+}
+
 /*
   commit transactions on all databases
  */
@@ -426,7 +500,8 @@ int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 {
        struct ctdb_db_context *ctdb_db;
        int i;
-       
+       int healthy_nodes = 0;
+
        for (i=1;i<=NUM_DB_PRIORITIES; i++) {
                if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
                        DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
@@ -444,32 +519,55 @@ int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
                return -1;
        }
 
+       DEBUG(DEBUG_DEBUG,(__location__ " num_nodes[%d]\n", ctdb->num_nodes));
+       for (i=0; i < ctdb->num_nodes; i++) {
+               DEBUG(DEBUG_DEBUG,(__location__ " node[%d].flags[0x%X]\n",
+                                  i, ctdb->nodes[i]->flags));
+               if (ctdb->nodes[i]->flags == 0) {
+                       healthy_nodes++;
+               }
+       }
+       DEBUG(DEBUG_INFO,(__location__ " healthy_nodes[%d]\n", healthy_nodes));
+
        for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               int ret;
+
                tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
-               if (tdb_transaction_commit(ctdb_db->ltdb->tdb) != 0) {
+               ret = tdb_transaction_commit(ctdb_db->ltdb->tdb);
+               if (ret != 0) {
                        DEBUG(DEBUG_ERR,(__location__ " Failed to commit transaction for db '%s'. Cancel all transactions and resetting transaction_started to false.\n",
                                 ctdb_db->db_name));
-
-                       /* cancel any pending transactions */
-                       for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
-                               tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
-                               if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
-                                       DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
-                                                ctdb_db->db_name));
-                               }
-                               tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
-                       }
-                       ctdb->freeze_transaction_started = false;
-
-                       return -1;
+                       goto fail;
                }
                tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
+
+               ret = ctdb_update_persistent_health(ctdb, ctdb_db, NULL, healthy_nodes);
+               if (ret != 0) {
+                       DEBUG(DEBUG_CRIT,(__location__ " Failed to update persistent health for db '%s'. "
+                                        "Cancel all remaining transactions and resetting transaction_started to false.\n",
+                                        ctdb_db->db_name));
+                       goto fail;
+               }
        }
 
        ctdb->freeze_transaction_started = false;
        ctdb->freeze_transaction_id = 0;
 
        return 0;
+
+fail:
+       /* cancel any pending transactions */
+       for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
+               if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
+                       DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
+                                ctdb_db->db_name));
+               }
+               tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
+       }
+       ctdb->freeze_transaction_started = false;
+
+       return -1;
 }
 
 /*