tools/ctdb: Display the locking statistics
[obnox/ctdb.git] / tools / ctdb.c
index 6bc3b6ebf28e4d6a2f7cf47db24af59a18b27d18..fcc44bb2b6fad549de19abf4907061bc02b5f2bf 100644 (file)
@@ -19,7 +19,6 @@
 */
 
 #include "includes.h"
-#include "lib/tevent/tevent.h"
 #include "system/time.h"
 #include "system/filesys.h"
 #include "system/network.h"
@@ -335,6 +334,10 @@ static void show_statistics(struct ctdb_statistics *s, int show_header)
                STATISTICS_FIELD(timeouts.call),
                STATISTICS_FIELD(timeouts.control),
                STATISTICS_FIELD(timeouts.traverse),
+               STATISTICS_FIELD(locks.num_calls),
+               STATISTICS_FIELD(locks.num_current),
+               STATISTICS_FIELD(locks.num_pending),
+               STATISTICS_FIELD(locks.num_failed),
                STATISTICS_FIELD(total_calls),
                STATISTICS_FIELD(pending_calls),
                STATISTICS_FIELD(lockwait_calls),
@@ -346,6 +349,7 @@ static void show_statistics(struct ctdb_statistics *s, int show_header)
                STATISTICS_FIELD(total_ro_delegations),
                STATISTICS_FIELD(total_ro_revokes),
        };
+       
        tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
        seconds = tmp%60;
        tmp    /= 60;
@@ -441,7 +445,19 @@ static void show_statistics(struct ctdb_statistics *s, int show_header)
                               preflen?0:4, "",
                               *(uint32_t *)(fields[i].offset+(uint8_t *)s));
                }
-               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd       MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
+               printf(" hop_count_buckets:");
+               for (i=0;i<MAX_COUNT_BUCKETS;i++) {
+                       printf(" %d", s->hop_count_bucket[i]);
+               }
+               printf("\n");
+               printf(" lock_buckets:");
+               for (i=0; i<MAX_COUNT_BUCKETS; i++) {
+                       printf(" %d", s->locks.buckets[i]);
+               }
+               printf("\n");
+               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
+
+               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
 
                printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
 
@@ -571,7 +587,7 @@ static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
 {
        TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
-       struct ctdb_db_statistics *dbstatistics;
+       struct ctdb_db_statistics *dbstat;
        struct ctdb_dbid_map *dbmap=NULL;
        int i, ret;
 
@@ -600,17 +616,50 @@ static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char
                return -1;
        }
 
-       if (!ctdb_getdbstat(ctdb_connection, options.pnn, dbmap->dbs[i].dbid, &dbstatistics)) {
+       if (!ctdb_getdbstat(ctdb_connection, options.pnn, dbmap->dbs[i].dbid, &dbstat)) {
                DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
                talloc_free(tmp_ctx);
                return -1;
        }
 
        printf("DB Statistics:\n");
-       printf("RO Delegations: %d\n", dbstatistics->db_ro_delegations);
-       printf("RO Revokes:     %d\n", dbstatistics->db_ro_revokes);
+       printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
+               dbstat->db_ro_delegations);
+       printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
+               dbstat->db_ro_delegations);
+       printf(" %s\n", "locks");
+       printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
+               dbstat->locks.num_calls);
+       printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
+               dbstat->locks.num_failed);
+       printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
+               dbstat->locks.num_current);
+       printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
+               dbstat->locks.num_pending);
+       printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
+               "    latency_ctdbd  MIN/AVG/MAX",
+               dbstat->locks.latency.min,
+               (dbstat->locks.latency.num ?
+                dbstat->locks.latency.total /dbstat->locks.latency.num :
+                0.0),
+               dbstat->locks.latency.max,
+               dbstat->locks.latency.num);
+       printf(" %s", "    buckets:");
+       for (i=0; i<MAX_COUNT_BUCKETS; i++) {
+               printf(" %d", dbstat->hop_count_bucket[i]);
+       }
+       printf("\n");
+       printf("Num Hot Keys:     %d\n", dbstat->num_hot_keys);
+       for (i = 0; i < dbstat->num_hot_keys; i++) {
+               int j;
+               printf("Count:%d Key:", dbstat->hot_keys[i].count);
+               for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
+                       printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
+               }
+               printf("\n");
+       }
 
-       ctdb_free_dbstat(dbstatistics);
+       ctdb_free_dbstat(dbstat);
        return 0;
 }
 
@@ -817,6 +866,12 @@ static bool is_partially_online(struct ctdb_node_and_flags *node)
        return ret;
 }
 
+static void control_status_header_machine(void)
+{
+       printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
+              ":Inactive:PartiallyOnline:ThisNode:\n");
+}
+
 static int control_status_1_machine(int mypnn, struct ctdb_node_and_flags *node)
 {
        printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
@@ -863,8 +918,7 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
        }
 
        if (options.machinereadable) {
-               printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
-                      ":Inactive:PartiallyOnline:ThisNode:\n");
+               control_status_header_machine();
                for (i=0;i<nodemap->num;i++) {
                        if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
                                continue;
@@ -930,8 +984,7 @@ static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **
        }
 
        if (options.machinereadable) {
-               printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
-                      ":Inactive:PartiallyOnline:ThisNode:\n");
+               control_status_header_machine();
        } else if (pnn_mode == CTDB_BROADCAST_ALL) {
                printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
        }
@@ -965,23 +1018,62 @@ struct natgw_node {
        const char *addr;
 };
 
+static int find_natgw(struct ctdb_context *ctdb,
+                      struct ctdb_node_map *nodemap, uint32_t flags,
+                      uint32_t *pnn, const char **ip)
+{
+       int i;
+       uint32_t capabilities;
+
+       for (i=0;i<nodemap->num;i++) {
+               if (!(nodemap->nodes[i].flags & flags)) {
+                       if (!ctdb_getcapabilities(ctdb_connection, nodemap->nodes[i].pnn, &capabilities)) {
+                               DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", nodemap->nodes[i].pnn));
+                               return -1;
+                       }
+                       if (!(capabilities&CTDB_CAP_NATGW)) {
+                               continue;
+                       }
+                       *pnn = nodemap->nodes[i].pnn;
+                       *ip = ctdb_addr_to_str(&nodemap->nodes[i].addr);
+                       return 0;
+               }
+       }
+
+       return 2; /* matches ENOENT */
+}
+
 /*
   display the list of nodes belonging to this natgw configuration
  */
 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
 {
        int i, ret;
-       uint32_t capabilities;
        const char *natgw_list;
        int nlines;
        char **lines;
        struct natgw_node *natgw_nodes = NULL;
        struct natgw_node *natgw_node;
        struct ctdb_node_map *nodemap=NULL;
+       uint32_t mypnn, pnn;
+       const char *ip;
 
+       /* When we have some nodes that could be the NATGW, make a
+        * series of attempts to find the first node that doesn't have
+        * certain status flags set.
+        */
+       uint32_t exclude_flags[] = {
+               /* Look for a nice healthy node */
+               NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY,
+               /* If not found, an UNHEALTHY/BANNED node will do */
+               NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED,
+               /* If not found, a STOPPED node will do */
+               NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED,
+               0,
+       };
 
        /* read the natgw nodes file into a linked list */
-       natgw_list = getenv("NATGW_NODES");
+       natgw_list = getenv("CTDB_NATGW_NODES");
        if (natgw_list == NULL) {
                natgw_list = "/etc/ctdb/natgw_nodes";
        }
@@ -990,9 +1082,6 @@ static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **a
                ctdb_set_error(ctdb, "Failed to load natgw node list '%s'\n", natgw_list);
                return -1;
        }
-       while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
-               nlines--;
-       }
        for (i=0;i<nlines;i++) {
                char *node;
 
@@ -1014,12 +1103,14 @@ static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **a
                natgw_nodes = natgw_node;
        }
 
-       ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
-       if (ret != 0) {
+       if (!ctdb_getnodemap(ctdb_connection, CTDB_CURRENT_NODE, &nodemap)) {
                DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
-               return ret;
+               return -1;
        }
 
+       /* Trim the nodemap so it only includes connected nodes in the
+        * current natgw group.
+        */
        i=0;
        while(i<nodemap->num) {
                for(natgw_node=natgw_nodes;natgw_node;natgw_node=natgw_node->next) {
@@ -1043,79 +1134,56 @@ static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **a
                }
 
                i++;
-       }               
+       }
 
-       /* pick a node to be natgwmaster
-        * we dont allow STOPPED, DELETED, BANNED or UNHEALTHY nodes to become the natgwmaster
-        */
-       for(i=0;i<nodemap->num;i++){
-               if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY))) {
-                       ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, &capabilities);
-                       if (ret != 0) {
-                               DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", nodemap->nodes[i].pnn));
-                               return ret;
-                       }
-                       if (!(capabilities&CTDB_CAP_NATGW)) {
-                               continue;
-                       }
-                       printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
-                       break;
+       ret = 2; /* matches ENOENT */
+       pnn = -1;
+       ip = "0.0.0.0";
+       for (i = 0; exclude_flags[i] != 0; i++) {
+               ret = find_natgw(ctdb, nodemap,
+                                exclude_flags[i],
+                                &pnn, &ip);
+               if (ret == -1) {
+                       goto done;
                }
-       }
-       /* we couldnt find any healthy node, try unhealthy ones */
-       if (i == nodemap->num) {
-               for(i=0;i<nodemap->num;i++){
-                       if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED))) {
-                               ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, &capabilities);
-                               if (ret != 0) {
-                                       DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", nodemap->nodes[i].pnn));
-                                       return ret;
-                               }
-                               if (!(capabilities&CTDB_CAP_NATGW)) {
-                                       continue;
-                               }
-                               printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
-                               break;
-                       }
+               if (ret == 0) {
+                       break;
                }
        }
-       /* unless all nodes are STOPPED, when we pick one anyway */
-       if (i == nodemap->num) {
-               for(i=0;i<nodemap->num;i++){
-                       if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED))) {
-                               ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, &capabilities);
-                               if (ret != 0) {
-                                       DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", nodemap->nodes[i].pnn));
-                                       return ret;
-                               }
-                               if (!(capabilities&CTDB_CAP_NATGW)) {
-                                       continue;
-                               }
-                               printf("%d %s\n", nodemap->nodes[i].pnn, ctdb_addr_to_str(&nodemap->nodes[i].addr));
-                               break;
-                       }
-               }
-               /* or if we still can not find any */
-               if (i == nodemap->num) {
-                       printf("-1 0.0.0.0\n");
-                       ret = 2; /* matches ENOENT */
-               }
+
+       if (options.machinereadable) {
+               printf(":Node:IP:\n");
+               printf(":%d:%s:\n", pnn, ip);
+       } else {
+               printf("%d %s\n", pnn, ip);
        }
 
        /* print the pruned list of nodes belonging to this natgw list */
+       if (!ctdb_getpnn(ctdb_connection, options.pnn, &mypnn)) {
+               DEBUG(DEBUG_NOTICE, ("Unable to get PNN from node %u\n", options.pnn));
+               /* This is actually harmless and will only result in
+                * the "this node" indication being missing
+                */
+               mypnn = -1;
+       }
+       if (options.machinereadable) {
+               control_status_header_machine();
+       } else {
+               printf("Number of nodes:%d\n", nodemap->num);
+       }
        for(i=0;i<nodemap->num;i++){
                if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
                        continue;
                }
-               printf(":%d:%s:%d:%d:%d:%d:%d\n", nodemap->nodes[i].pnn,
-                       ctdb_addr_to_str(&nodemap->nodes[i].addr),
-                      !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
-                      !!(nodemap->nodes[i].flags&NODE_FLAGS_BANNED),
-                      !!(nodemap->nodes[i].flags&NODE_FLAGS_PERMANENTLY_DISABLED),
-                      !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
-                      !!(nodemap->nodes[i].flags&NODE_FLAGS_STOPPED));
+               if (options.machinereadable) {
+                       control_status_1_machine(mypnn, &(nodemap->nodes[i]));
+               } else {
+                       control_status_1_human(mypnn, &(nodemap->nodes[i]));
+               }
        }
 
+done:
+       ctdb_free_nodemap(nodemap);
        return ret;
 }
 
@@ -1570,6 +1638,118 @@ static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv
        return 0;
 }
 
+static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
+{
+       uint32_t recmaster;
+       TDB_DATA data;
+
+       if (ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), pnn, &recmaster) != 0) {
+               DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", pnn));
+               return -1;
+       }
+
+       data.dptr  = (uint8_t *)&pnn;
+       data.dsize = sizeof(uint32_t);
+       if (ctdb_client_send_message(ctdb, recmaster, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
+               DEBUG(DEBUG_ERR,("Failed to send message to force node reallocation\n"));
+               return -1;
+       }
+
+       return 0;
+}
+
+
+/*
+  rebalance a node by setting it to allow failback and triggering a
+  takeover run
+ */
+static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       switch (options.pnn) {
+       case CTDB_BROADCAST_ALL:
+       case CTDB_CURRENT_NODE:
+               DEBUG(DEBUG_ERR,("You must specify a node number with -n <pnn> for the node to rebalance\n"));
+               return -1;
+       }
+
+       return rebalance_node(ctdb, options.pnn);
+}
+
+
+static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
+{
+       struct ctdb_public_ip ip;
+       int ret;
+       uint32_t *nodes;
+       uint32_t disable_time;
+       TDB_DATA data;
+       struct ctdb_node_map *nodemap=NULL;
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+
+       disable_time = 30;
+       data.dptr  = (uint8_t*)&disable_time;
+       data.dsize = sizeof(disable_time);
+       ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
+               return -1;
+       }
+
+       ip.pnn  = -1;
+       ip.addr = *addr;
+
+       data.dptr  = (uint8_t *)&ip;
+       data.dsize = sizeof(ip);
+
+       ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
+               talloc_free(tmp_ctx);
+               return ret;
+       }
+
+               nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
+       ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
+                                       nodes, 0,
+                                       LONGTIMELIMIT(),
+                                       false, data,
+                                       NULL, NULL,
+                                       NULL);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       talloc_free(tmp_ctx);
+       return 0;
+}
+
+/*
+  release an ip form all nodes and have it re-assigned by recd
+ */
+static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       ctdb_sock_addr addr;
+
+       if (argc < 1) {
+               usage();
+               return -1;
+       }
+
+       if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
+               DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
+               return -1;
+       }
+
+       if (rebalance_ip(ctdb, &addr) != 0) {
+               DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
+               return -1;
+       }
+
+       return 0;
+}
+
 static int getips_store_callback(void *param, void *data)
 {
        struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
@@ -1601,12 +1781,14 @@ static uint32_t *ip_key(ctdb_sock_addr *ip)
        case AF_INET:
                key[0]  = ip->ip.sin_addr.s_addr;
                break;
-       case AF_INET6:
-               key[0]  = ip->ip6.sin6_addr.s6_addr32[3];
-               key[1]  = ip->ip6.sin6_addr.s6_addr32[2];
-               key[2]  = ip->ip6.sin6_addr.s6_addr32[1];
-               key[3]  = ip->ip6.sin6_addr.s6_addr32[0];
+       case AF_INET6: {
+               uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
+               key[0]  = s6_a32[3];
+               key[1]  = s6_a32[2];
+               key[2]  = s6_a32[1];
+               key[3]  = s6_a32[0];
                break;
+       }
        default:
                DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
                return key;
@@ -1935,17 +2117,8 @@ static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
                return ret;
        }
 
-       do {
-               ret = control_ipreallocate(ctdb, argc, argv);
-               if (ret != 0) {
-                       DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u. Wait 3 seconds and try again.\n", options.pnn));
-                       sleep(3);
-                       retries++;
-               }
-       } while (retries < 5 && ret != 0);
-       if (ret != 0) {
-               DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u. Giving up.\n", options.pnn));
-               talloc_free(tmp_ctx);
+       if (rebalance_node(ctdb, options.pnn) != 0) {
+               DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
                return ret;
        }
 
@@ -1953,6 +2126,27 @@ static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
        return 0;
 }
 
+/*
+  add a public ip address to a node
+ */
+static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       ctdb_sock_addr addr;
+
+       if (argc != 1) {
+               usage();
+       }
+
+       if (!parse_ip(argv[0], NULL, 0, &addr)) {
+               printf("Badly formed ip : %s\n", argv[0]);
+               return -1;
+       }
+
+       printf("IP on interface %s\n", ctdb_sys_find_ifname(&addr));
+
+       return 0;
+}
+
 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
 
 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
@@ -3004,12 +3198,10 @@ static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **
 static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const char **argv)
 {
        uint32_t capabilities;
-       int ret;
 
-       ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), options.pnn, &capabilities);
-       if (ret != 0) {
+       if (!ctdb_getcapabilities(ctdb_connection, options.pnn, &capabilities)) {
                DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", options.pnn));
-               return ret;
+               return -1;
        }
        
        if (!options.machinereadable){
@@ -3038,15 +3230,16 @@ static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
        int i, ret;
        int healthy_count = 0;
 
-       ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
-       if (ret != 0) {
+       if (!ctdb_getnodemap(ctdb_connection, options.pnn, &nodemap)) {
                DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
-               return ret;
+               return -1;
        }
 
        capabilities = talloc_array(ctdb, uint32_t, nodemap->num);
        CTDB_NO_MEMORY(ctdb, capabilities);
        
+       ret = 0;
+
        /* collect capabilities for all connected nodes */
        for (i=0; i<nodemap->num; i++) {
                if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
@@ -3056,10 +3249,10 @@ static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
                        continue;
                }
        
-               ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), i, &capabilities[i]);
-               if (ret != 0) {
+               if (!ctdb_getcapabilities(ctdb_connection, i, &capabilities[i])) {
                        DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", i));
-                       return ret;
+                       ret = -1;
+                       goto done;
                }
 
                if (!(capabilities[i] & CTDB_CAP_LVS)) {
@@ -3093,7 +3286,9 @@ static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
                        ctdb_addr_to_str(&nodemap->nodes[i].addr));
        }
 
-       return 0;
+done:
+       ctdb_free_nodemap(nodemap);
+       return ret;
 }
 
 /*
@@ -3106,14 +3301,15 @@ static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **a
        int i, ret;
        int healthy_count = 0;
 
-       ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
-       if (ret != 0) {
+       if (!ctdb_getnodemap(ctdb_connection, options.pnn, &nodemap)) {
                DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
-               return ret;
+               return -1;
        }
 
        capabilities = talloc_array(ctdb, uint32_t, nodemap->num);
        CTDB_NO_MEMORY(ctdb, capabilities);
+
+       ret = -1;
        
        /* collect capabilities for all connected nodes */
        for (i=0; i<nodemap->num; i++) {
@@ -3124,10 +3320,10 @@ static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **a
                        continue;
                }
        
-               ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), i, &capabilities[i]);
-               if (ret != 0) {
+               if (!ctdb_getcapabilities(ctdb_connection, i, &capabilities[i])) {
                        DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", i));
-                       return ret;
+                       ret = -1;
+                       goto done;
                }
 
                if (!(capabilities[i] & CTDB_CAP_LVS)) {
@@ -3162,11 +3358,14 @@ static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **a
                } else {
                        printf("Node %d is LVS master\n", i);
                }
-               return 0;
+               ret = 0;
+               goto done;
        }
 
        printf("There is no LVS master\n");
-       return -1;
+done:
+       ctdb_free_nodemap(nodemap);
+       return ret;
 }
 
 /*
@@ -3313,7 +3512,7 @@ static int control_cattdb(struct ctdb_context *ctdb, int argc, const char **argv
                return -1;
        }
 
-       ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, false, 0);
+       ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
 
        if (ctdb_db == NULL) {
                DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
@@ -3879,6 +4078,96 @@ static int control_clearlog(struct ctdb_context *ctdb, int argc, const char **ar
 }
 
 
+static uint32_t reloadips_finished;
+
+static void reloadips_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+                            TDB_DATA data, void *private_data)
+{
+       reloadips_finished = 1;
+}
+
+static int reloadips_all(struct ctdb_context *ctdb)
+{
+       struct reloadips_all_reply rips;
+       struct ctdb_node_map *nodemap=NULL;
+       TDB_DATA data;
+       uint32_t recmaster;
+       int ret, i;
+
+       /* check that there are valid nodes available */
+       if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
+               DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
+               return 1;
+       }
+       for (i=0; i<nodemap->num;i++) {
+               if (nodemap->nodes[i].flags != 0) {
+                       DEBUG(DEBUG_ERR,("reloadips -n all  can only be used when all nodes are up and healthy. Aborting due to problem with node %d\n", i));
+                       return 1;
+               }
+       }
+
+
+       rips.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
+       if (rips.pnn == -1) {
+               DEBUG(DEBUG_ERR, ("Failed to get pnn of local node\n"));
+               return 1;
+       }
+       rips.srvid = getpid();
+
+
+       /* register a message port for receiveing the reply so that we
+          can receive the reply
+       */
+       ctdb_client_set_message_handler(ctdb, rips.srvid, reloadips_handler, NULL);
+
+       if (!ctdb_getrecmaster(ctdb_connection, CTDB_CURRENT_NODE, &recmaster)) {
+               DEBUG(DEBUG_ERR, ("Unable to get recmaster from node\n"));
+               return -1;
+       }
+
+
+       data.dptr = (uint8_t *)&rips;
+       data.dsize = sizeof(rips);
+
+       ret = ctdb_client_send_message(ctdb, recmaster, CTDB_SRVID_RELOAD_ALL_IPS, data);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to send reload all ips request message to %u\n", options.pnn));
+               return 1;
+       }
+
+       reloadips_finished = 0;
+       while (reloadips_finished == 0) {
+               event_loop_once(ctdb->ev);
+       }
+
+       return 0;
+}
+
+/*
+  reload public ips on a specific node
+ */
+static int control_reloadips(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       int ret;
+       int32_t res;
+       char *errmsg;
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+
+       if (options.pnn == CTDB_BROADCAST_ALL) {
+               return reloadips_all(ctdb);
+       }
+
+       ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_RELOAD_PUBLIC_IPS,
+                          0, tdb_null, tmp_ctx, NULL, &res, NULL, &errmsg);
+       if (ret != 0 || res != 0) {
+               DEBUG(DEBUG_ERR,("Failed to reload ips\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       talloc_free(tmp_ctx);
+       return 0;
+}
 
 /*
   display a list of the databases on a remote ctdb
@@ -3895,13 +4184,14 @@ static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **ar
        }
 
        if(options.machinereadable){
-               printf(":ID:Name:Path:Persistent:Unhealthy:ReadOnly:\n");
+               printf(":ID:Name:Path:Persistent:Sticky:Unhealthy:ReadOnly:\n");
                for(i=0;i<dbmap->num;i++){
                        const char *path;
                        const char *name;
                        const char *health;
                        bool persistent;
                        bool readonly;
+                       bool sticky;
 
                        ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn,
                                            dbmap->dbs[i].dbid, ctdb, &path);
@@ -3911,9 +4201,11 @@ static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **ar
                                              dbmap->dbs[i].dbid, ctdb, &health);
                        persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
                        readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
-                       printf(":0x%08X:%s:%s:%d:%d:%d:\n",
+                       sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
+                       printf(":0x%08X:%s:%s:%d:%d:%d:%d:\n",
                               dbmap->dbs[i].dbid, name, path,
-                              !!(persistent), !!(health), !!(readonly));
+                              !!(persistent), !!(sticky),
+                              !!(health), !!(readonly));
                }
                return 0;
        }
@@ -3925,15 +4217,18 @@ static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **ar
                const char *health;
                bool persistent;
                bool readonly;
+               bool sticky;
 
                ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
                ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
                ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
                persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
                readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
-               printf("dbid:0x%08x name:%s path:%s%s%s%s\n",
+               sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
+               printf("dbid:0x%08x name:%s path:%s%s%s%s%s\n",
                       dbmap->dbs[i].dbid, name, path,
                       persistent?" PERSISTENT":"",
+                      sticky?" STICKY":"",
                       readonly?" READONLY":"",
                       health?" UNHEALTHY":"");
        }
@@ -3968,6 +4263,7 @@ static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char *
                const char *health;
                bool persistent;
                bool readonly;
+               bool sticky;
 
                ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
                if (strcmp(name, db_name) != 0) {
@@ -3978,9 +4274,11 @@ static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char *
                ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
                persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
                readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
-               printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nREADONLY: %s\nHEALTH: %s\n",
+               sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
+               printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nSTICKY: %s\nREADONLY: %s\nHEALTH: %s\n",
                       dbmap->dbs[i].dbid, name, path,
                       persistent?"yes":"no",
+                      sticky?"yes":"no",
                       readonly?"yes":"no",
                       health?health:"OK");
                return 0;
@@ -4420,26 +4718,107 @@ static int control_getdbprio(struct ctdb_context *ctdb, int argc, const char **a
        return 0;
 }
 
+/*
+  set the sticky records capability for a database
+ */
+static int control_setdbsticky(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+       uint32_t db_id;
+       struct ctdb_dbid_map *dbmap=NULL;
+       int i, ret;
+
+       if (argc < 1) {
+               usage();
+       }
+
+       if (!strncmp(argv[0], "0x", 2)) {
+               db_id = strtoul(argv[0] + 2, NULL, 0);
+       } else {
+               ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
+               if (ret != 0) {
+                       DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
+                       talloc_free(tmp_ctx);
+                       return ret;
+               }
+               for(i=0;i<dbmap->num;i++){
+                       const char *name;
+
+                       ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
+                       if(!strcmp(argv[0], name)){
+                               talloc_free(discard_const(name));
+                               break;
+                       }
+                       talloc_free(discard_const(name));
+               }
+               if (i == dbmap->num) {
+                       DEBUG(DEBUG_ERR,("No database with name '%s' found\n", argv[0]));
+                       talloc_free(tmp_ctx);
+                       return -1;
+               }
+               db_id = dbmap->dbs[i].dbid;
+       }
+
+       ret = ctdb_ctrl_set_db_sticky(ctdb, options.pnn, db_id);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Unable to set db to support sticky records\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       talloc_free(tmp_ctx);
+       return 0;
+}
+
 /*
   set the readonly capability for a database
  */
 static int control_setdbreadonly(struct ctdb_context *ctdb, int argc, const char **argv)
 {
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
        uint32_t db_id;
-       int ret;
+       struct ctdb_dbid_map *dbmap=NULL;
+       int i, ret;
 
        if (argc < 1) {
                usage();
        }
 
-       db_id = strtoul(argv[0], NULL, 0);
+       if (!strncmp(argv[0], "0x", 2)) {
+               db_id = strtoul(argv[0] + 2, NULL, 0);
+       } else {
+               ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
+               if (ret != 0) {
+                       DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
+                       talloc_free(tmp_ctx);
+                       return ret;
+               }
+               for(i=0;i<dbmap->num;i++){
+                       const char *name;
+
+                       ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
+                       if(!strcmp(argv[0], name)){
+                               talloc_free(discard_const(name));
+                               break;
+                       }
+                       talloc_free(discard_const(name));
+               }
+               if (i == dbmap->num) {
+                       DEBUG(DEBUG_ERR,("No database with name '%s' found\n", argv[0]));
+                       talloc_free(tmp_ctx);
+                       return -1;
+               }
+               db_id = dbmap->dbs[i].dbid;
+       }
 
        ret = ctdb_ctrl_set_db_readonly(ctdb, options.pnn, db_id);
        if (ret != 0) {
                DEBUG(DEBUG_ERR,("Unable to set db to support readonly\n"));
+               talloc_free(tmp_ctx);
                return -1;
        }
 
+       talloc_free(tmp_ctx);
        return 0;
 }
 
@@ -5450,6 +5829,7 @@ static const struct {
        { "listnodes",       control_listnodes,         false,  true, "list all nodes in the cluster"},
        { "reloadnodes",     control_reload_nodes_file, false,  false, "reload the nodes file and restart the transport on all nodes"},
        { "moveip",          control_moveip,            false,  false, "move/failover an ip address to another node", "<ip> <node>"},
+       { "rebalanceip",     control_rebalanceip,       false,  false, "release an ip from the node and let recd rebalance it", "<ip>"},
        { "addip",           control_addip,             true,   false, "add a ip address to a node", "<ip/mask> <iface>"},
        { "delip",           control_delip,             false,  false, "delete an ip address from a node", "<ip>"},
        { "eventscript",     control_eventscript,       true,   false, "run the eventscript with the given parameters on a node", "<arguments>"},
@@ -5457,7 +5837,7 @@ static const struct {
        { "restoredb",        control_restoredb,        false,  false, "restore the database from a file.", "<file> [dbname]"},
        { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
        { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname>"},
-       { "recmaster",        control_recmaster,        false,  false, "show the pnn for the recovery master."},
+       { "recmaster",        control_recmaster,        true,   false, "show the pnn for the recovery master."},
        { "scriptstatus",     control_scriptstatus,     true,   false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
        { "enablescript",     control_enablescript,  false,     false, "enable an eventscript", "<script>"},
        { "disablescript",    control_disablescript,  false,    false, "disable an eventscript", "<script>"},
@@ -5470,7 +5850,8 @@ static const struct {
        { "setrecmasterrole", control_setrecmasterrole, false,  false, "Set RECMASTER role to on/off", "{on|off}"},
        { "setdbprio",        control_setdbprio,        false,  false, "Set DB priority", "<dbid> <prio:1-3>"},
        { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbid>"},
-       { "setdbreadonly",    control_setdbreadonly,    false,  false, "Set DB readonly capable", "<dbid>"},
+       { "setdbreadonly",    control_setdbreadonly,    false,  false, "Set DB readonly capable", "<dbid>|<name>"},
+       { "setdbsticky",      control_setdbsticky,      false,  false, "Set DB sticky-records capable", "<dbid>|<name>"},
        { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
        { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
        { "sync",            control_ipreallocate,      false,  false,  "wait until ctdbd has synced all state changes" },
@@ -5481,9 +5862,12 @@ static const struct {
        { "readkey",         control_readkey,           true,   false,  "read the content off a database key", "<tdb-file> <key>" },
        { "writekey",        control_writekey,          true,   false,  "write to a database key", "<tdb-file> <key> <value>" },
        { "checktcpport",    control_chktcpport,        false,  true,  "check if a service is bound to a specific tcp port or not", "<port>" },
+       { "rebalancenode",     control_rebalancenode,   false,  false, "release a node by allowing it to takeover ips", "<pnn>"},
        { "getdbseqnum",     control_getdbseqnum,       false,  false, "get the sequence number off a database", "<dbid>" },
        { "nodestatus",      control_nodestatus,        true,   false,  "show and return node status" },
        { "dbstatistics",    control_dbstatistics,      false,  false, "show db statistics", "<db>" },
+       { "reloadips",       control_reloadips,         false,  false, "reload the public addresses file on a node" },
+       { "ipiface",         control_ipiface,           true,   true,  "Find which interface an ip address is hsoted on", "<ip>" },
 };
 
 /*
@@ -5597,7 +5981,6 @@ int main(int argc, const char *argv[])
                DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
                exit(1);
        }
-       tevent_loop_allow_nesting(ev);
 
        for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
                if (strcmp(control, ctdb_commands[i].name) == 0) {
@@ -5662,6 +6045,7 @@ int main(int argc, const char *argv[])
 
        ctdb_disconnect(ctdb_connection);
        talloc_free(ctdb);
+       talloc_free(ev);
        (void)poptFreeContext(pc);
 
        return ret;