ctdb/tools/ctdb.c

   1 /*
   2    ctdb control tool
   3
   4    Copyright (C) Andrew Tridgell  2007
   5    Copyright (C) Ronnie Sahlberg  2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "includes.h"
  22 #include "system/time.h"
  23 #include "system/filesys.h"
  24 #include "system/network.h"
  25 #include "system/locale.h"
  26 #include "popt.h"
  27 #include "cmdline.h"
  28 #include "../include/ctdb_version.h"
  29 #include "../include/ctdb_client.h"
  30 #include "../include/ctdb_private.h"
  31 #include "../common/rb_tree.h"
  32 #include "db_wrap.h"
  33 #include "lib/util/dlinklist.h"
  34
  35 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
  36 #define ERR_NONODE      21      /* node does not exist */
  37 #define ERR_DISNODE     22      /* node is disconnected */
  38
  39 static void usage(void);
  40
  41 static struct {
  42         int timelimit;
  43         uint32_t pnn;
  44         uint32_t *nodes;
  45         int machinereadable;
  46         int verbose;
  47         int maxruntime;
  48         int printemptyrecords;
  49         int printdatasize;
  50         int printlmaster;
  51         int printhash;
  52         int printrecordflags;
  53 } options;
  54
  55 #define LONGTIMEOUT options.timelimit*10
  56
  57 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
  58 #define LONGTIMELIMIT() timeval_current_ofs(LONGTIMEOUT, 0)
  59
  60 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
  61 {
  62         printf("CTDB version: %s\n", CTDB_VERSION_STRING);
  63         return 0;
  64 }
  65
  66 #define CTDB_NOMEM_ABORT(p) do { if (!(p)) {                            \
  67                 DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n",            \
  68                                    "Out of memory in " __location__ )); \
  69                 abort();                                                \
  70         }} while (0)
  71
  72 static uint32_t getpnn(struct ctdb_context *ctdb)
  73 {
  74         if ((options.pnn == CTDB_BROADCAST_ALL) ||
  75             (options.pnn == CTDB_MULTICAST)) {
  76                 DEBUG(DEBUG_ERR,
  77                       ("Cannot get PNN for node %u\n", options.pnn));
  78                 exit(1);
  79         }
  80
  81         if (options.pnn == CTDB_CURRENT_NODE) {
  82                 return ctdb_get_pnn(ctdb);
  83         } else {
  84                 return options.pnn;
  85         }
  86 }
  87
  88 static void assert_single_node_only(void)
  89 {
  90         if ((options.pnn == CTDB_BROADCAST_ALL) ||
  91             (options.pnn == CTDB_MULTICAST)) {
  92                 DEBUG(DEBUG_ERR,
  93                       ("This control can not be applied to multiple PNNs\n"));
  94                 exit(1);
  95         }
  96 }
  97
  98 /* Pretty print the flags to a static buffer in human-readable format.
  99  * This never returns NULL!
 100  */
 101 static const char *pretty_print_flags(uint32_t flags)
 102 {
 103         int j;
 104         static const struct {
 105                 uint32_t flag;
 106                 const char *name;
 107         } flag_names[] = {
 108                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 109                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 110                 { NODE_FLAGS_BANNED,                "BANNED" },
 111                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 112                 { NODE_FLAGS_DELETED,               "DELETED" },
 113                 { NODE_FLAGS_STOPPED,               "STOPPED" },
 114                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 115         };
 116         static char flags_str[512]; /* Big enough to contain all flag names */
 117
 118         flags_str[0] = '\0';
 119         for (j=0;j<ARRAY_SIZE(flag_names);j++) {
 120                 if (flags & flag_names[j].flag) {
 121                         if (flags_str[0] == '\0') {
 122                                 (void) strcpy(flags_str, flag_names[j].name);
 123                         } else {
 124                                 (void) strncat(flags_str, "|", sizeof(flags_str)-1);
 125                                 (void) strncat(flags_str, flag_names[j].name,
 126                                                sizeof(flags_str)-1);
 127                         }
 128                 }
 129         }
 130         if (flags_str[0] == '\0') {
 131                 (void) strcpy(flags_str, "OK");
 132         }
 133
 134         return flags_str;
 135 }
 136
 137 static int h2i(char h)
 138 {
 139         if (h >= 'a' && h <= 'f') return h - 'a' + 10;
 140         if (h >= 'A' && h <= 'F') return h - 'f' + 10;
 141         return h - '0';
 142 }
 143
 144 static TDB_DATA hextodata(TALLOC_CTX *mem_ctx, const char *str)
 145 {
 146         int i, len;
 147         TDB_DATA key = {NULL, 0};
 148
 149         len = strlen(str);
 150         if (len & 0x01) {
 151                 DEBUG(DEBUG_ERR,("Key specified with odd number of hexadecimal digits\n"));
 152                 return key;
 153         }
 154
 155         key.dsize = len>>1;
 156         key.dptr  = talloc_size(mem_ctx, key.dsize);
 157
 158         for (i=0; i < len/2; i++) {
 159                 key.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
 160         }
 161         return key;
 162 }
 163
 164 /* Parse a nodestring.  Parameter dd_ok controls what happens to nodes
 165  * that are disconnected or deleted.  If dd_ok is true those nodes are
 166  * included in the output list of nodes.  If dd_ok is false, those
 167  * nodes are filtered from the "all" case and cause an error if
 168  * explicitly specified.
 169  */
 170 static bool parse_nodestring(struct ctdb_context *ctdb,
 171                              TALLOC_CTX *mem_ctx,
 172                              const char * nodestring,
 173                              uint32_t current_pnn,
 174                              bool dd_ok,
 175                              uint32_t **nodes,
 176                              uint32_t *pnn_mode)
 177 {
 178         TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
 179         int n;
 180         uint32_t i;
 181         struct ctdb_node_map *nodemap;
 182         int ret;
 183
 184         *nodes = NULL;
 185
 186         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
 187         if (ret != 0) {
 188                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
 189                 talloc_free(tmp_ctx);
 190                 exit(10);
 191         }
 192
 193         if (nodestring != NULL) {
 194                 *nodes = talloc_array(mem_ctx, uint32_t, 0);
 195                 if (*nodes == NULL) {
 196                         goto failed;
 197                 }
 198
 199                 n = 0;
 200
 201                 if (strcmp(nodestring, "all") == 0) {
 202                         *pnn_mode = CTDB_BROADCAST_ALL;
 203
 204                         /* all */
 205                         for (i = 0; i < nodemap->num; i++) {
 206                                 if ((nodemap->nodes[i].flags &
 207                                      (NODE_FLAGS_DISCONNECTED |
 208                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 209                                         continue;
 210                                 }
 211                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 212                                                         uint32_t, n+1);
 213                                 if (*nodes == NULL) {
 214                                         goto failed;
 215                                 }
 216                                 (*nodes)[n] = i;
 217                                 n++;
 218                         }
 219                 } else {
 220                         /* x{,y...} */
 221                         char *ns, *tok;
 222
 223                         ns = talloc_strdup(tmp_ctx, nodestring);
 224                         tok = strtok(ns, ",");
 225                         while (tok != NULL) {
 226                                 uint32_t pnn;
 227                                 char *endptr;
 228                                 i = (uint32_t)strtoul(tok, &endptr, 0);
 229                                 if (i == 0 && tok == endptr) {
 230                                         DEBUG(DEBUG_ERR,
 231                                               ("Invalid node %s\n", tok));
 232                                         talloc_free(tmp_ctx);
 233                                         exit(ERR_NONODE);
 234                                 }
 235                                 if (i >= nodemap->num) {
 236                                         DEBUG(DEBUG_ERR, ("Node %u does not exist\n", i));
 237                                         talloc_free(tmp_ctx);
 238                                         exit(ERR_NONODE);
 239                                 }
 240                                 if ((nodemap->nodes[i].flags &
 241                                      (NODE_FLAGS_DISCONNECTED |
 242                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 243                                         DEBUG(DEBUG_ERR, ("Node %u has status %s\n", i, pretty_print_flags(nodemap->nodes[i].flags)));
 244                                         talloc_free(tmp_ctx);
 245                                         exit(ERR_DISNODE);
 246                                 }
 247                                 if ((pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), i)) < 0) {
 248                                         DEBUG(DEBUG_ERR, ("Can not access node %u. Node is not operational.\n", i));
 249                                         talloc_free(tmp_ctx);
 250                                         exit(10);
 251                                 }
 252
 253                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 254                                                         uint32_t, n+1);
 255                                 if (*nodes == NULL) {
 256                                         goto failed;
 257                                 }
 258
 259                                 (*nodes)[n] = i;
 260                                 n++;
 261
 262                                 tok = strtok(NULL, ",");
 263                         }
 264                         talloc_free(ns);
 265
 266                         if (n == 1) {
 267                                 *pnn_mode = (*nodes)[0];
 268                         } else {
 269                                 *pnn_mode = CTDB_MULTICAST;
 270                         }
 271                 }
 272         } else {
 273                 /* default - no nodes specified */
 274                 *nodes = talloc_array(mem_ctx, uint32_t, 1);
 275                 if (*nodes == NULL) {
 276                         goto failed;
 277                 }
 278                 *pnn_mode = CTDB_CURRENT_NODE;
 279
 280                 if (((*nodes)[0] = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), current_pnn)) < 0) {
 281                         goto failed;
 282                 }
 283         }
 284
 285         talloc_free(tmp_ctx);
 286         return true;
 287
 288 failed:
 289         talloc_free(tmp_ctx);
 290         return false;
 291 }
 292
 293 /*
 294  check if a database exists
 295 */
 296 static bool db_exists(struct ctdb_context *ctdb, const char *dbarg,
 297                       uint32_t *dbid, const char **dbname, uint8_t *flags)
 298 {
 299         int i, ret;
 300         struct ctdb_dbid_map *dbmap=NULL;
 301         bool dbid_given = false, found = false;
 302         uint32_t id;
 303         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 304         const char *name;
 305
 306         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
 307         if (ret != 0) {
 308                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
 309                 goto fail;
 310         }
 311
 312         if (strncmp(dbarg, "0x", 2) == 0) {
 313                 id = strtoul(dbarg, NULL, 0);
 314                 dbid_given = true;
 315         }
 316
 317         for(i=0; i<dbmap->num; i++) {
 318                 if (dbid_given) {
 319                         if (id == dbmap->dbs[i].dbid) {
 320                                 found = true;
 321                                 break;
 322                         }
 323                 } else {
 324                         ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
 325                         if (ret != 0) {
 326                                 DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
 327                                 goto fail;
 328                         }
 329
 330                         if (strcmp(name, dbarg) == 0) {
 331                                 id = dbmap->dbs[i].dbid;
 332                                 found = true;
 333                                 break;
 334                         }
 335                 }
 336         }
 337
 338         if (found && dbid_given && dbname != NULL) {
 339                 ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
 340                 if (ret != 0) {
 341                         DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
 342                         found = false;
 343                         goto fail;
 344                 }
 345         }
 346
 347         if (found) {
 348                 if (dbid) *dbid = id;
 349                 if (dbname) *dbname = talloc_strdup(ctdb, name);
 350                 if (flags) *flags = dbmap->dbs[i].flags;
 351         } else {
 352                 DEBUG(DEBUG_ERR,("No database matching '%s' found\n", dbarg));
 353         }
 354
 355 fail:
 356         talloc_free(tmp_ctx);
 357         return found;
 358 }
 359
 360 /*
 361   see if a process exists
 362  */
 363 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
 364 {
 365         uint32_t pnn, pid;
 366         int ret;
 367         if (argc < 1) {
 368                 usage();
 369         }
 370
 371         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
 372                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
 373                 return -1;
 374         }
 375
 376         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
 377         if (ret == 0) {
 378                 printf("%u:%u exists\n", pnn, pid);
 379         } else {
 380                 printf("%u:%u does not exist\n", pnn, pid);
 381         }
 382         return ret;
 383 }
 384
 385 /*
 386   display statistics structure
 387  */
 388 static void show_statistics(struct ctdb_statistics *s, int show_header)
 389 {
 390         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 391         int i;
 392         const char *prefix=NULL;
 393         int preflen=0;
 394         int tmp, days, hours, minutes, seconds;
 395         const struct {
 396                 const char *name;
 397                 uint32_t offset;
 398         } fields[] = {
 399 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
 400                 STATISTICS_FIELD(num_clients),
 401                 STATISTICS_FIELD(frozen),
 402                 STATISTICS_FIELD(recovering),
 403                 STATISTICS_FIELD(num_recoveries),
 404                 STATISTICS_FIELD(client_packets_sent),
 405                 STATISTICS_FIELD(client_packets_recv),
 406                 STATISTICS_FIELD(node_packets_sent),
 407                 STATISTICS_FIELD(node_packets_recv),
 408                 STATISTICS_FIELD(keepalive_packets_sent),
 409                 STATISTICS_FIELD(keepalive_packets_recv),
 410                 STATISTICS_FIELD(node.req_call),
 411                 STATISTICS_FIELD(node.reply_call),
 412                 STATISTICS_FIELD(node.req_dmaster),
 413                 STATISTICS_FIELD(node.reply_dmaster),
 414                 STATISTICS_FIELD(node.reply_error),
 415                 STATISTICS_FIELD(node.req_message),
 416                 STATISTICS_FIELD(node.req_control),
 417                 STATISTICS_FIELD(node.reply_control),
 418                 STATISTICS_FIELD(client.req_call),
 419                 STATISTICS_FIELD(client.req_message),
 420                 STATISTICS_FIELD(client.req_control),
 421                 STATISTICS_FIELD(timeouts.call),
 422                 STATISTICS_FIELD(timeouts.control),
 423                 STATISTICS_FIELD(timeouts.traverse),
 424                 STATISTICS_FIELD(locks.num_calls),
 425                 STATISTICS_FIELD(locks.num_current),
 426                 STATISTICS_FIELD(locks.num_pending),
 427                 STATISTICS_FIELD(locks.num_failed),
 428                 STATISTICS_FIELD(total_calls),
 429                 STATISTICS_FIELD(pending_calls),
 430                 STATISTICS_FIELD(childwrite_calls),
 431                 STATISTICS_FIELD(pending_childwrite_calls),
 432                 STATISTICS_FIELD(memory_used),
 433                 STATISTICS_FIELD(max_hop_count),
 434                 STATISTICS_FIELD(total_ro_delegations),
 435                 STATISTICS_FIELD(total_ro_revokes),
 436         };
 437
 438         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
 439         seconds = tmp%60;
 440         tmp    /= 60;
 441         minutes = tmp%60;
 442         tmp    /= 60;
 443         hours   = tmp%24;
 444         tmp    /= 24;
 445         days    = tmp;
 446
 447         if (options.machinereadable){
 448                 if (show_header) {
 449                         printf("CTDB version:");
 450                         printf("Current time of statistics:");
 451                         printf("Statistics collected since:");
 452                         for (i=0;i<ARRAY_SIZE(fields);i++) {
 453                                 printf("%s:", fields[i].name);
 454                         }
 455                         printf("num_reclock_ctdbd_latency:");
 456                         printf("min_reclock_ctdbd_latency:");
 457                         printf("avg_reclock_ctdbd_latency:");
 458                         printf("max_reclock_ctdbd_latency:");
 459
 460                         printf("num_reclock_recd_latency:");
 461                         printf("min_reclock_recd_latency:");
 462                         printf("avg_reclock_recd_latency:");
 463                         printf("max_reclock_recd_latency:");
 464
 465                         printf("num_call_latency:");
 466                         printf("min_call_latency:");
 467                         printf("avg_call_latency:");
 468                         printf("max_call_latency:");
 469
 470                         printf("num_lockwait_latency:");
 471                         printf("min_lockwait_latency:");
 472                         printf("avg_lockwait_latency:");
 473                         printf("max_lockwait_latency:");
 474
 475                         printf("num_childwrite_latency:");
 476                         printf("min_childwrite_latency:");
 477                         printf("avg_childwrite_latency:");
 478                         printf("max_childwrite_latency:");
 479                         printf("\n");
 480                 }
 481                 printf("%d:", CTDB_VERSION);
 482                 printf("%d:", (int)s->statistics_current_time.tv_sec);
 483                 printf("%d:", (int)s->statistics_start_time.tv_sec);
 484                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 485                         printf("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 486                 }
 487                 printf("%d:", s->reclock.ctdbd.num);
 488                 printf("%.6f:", s->reclock.ctdbd.min);
 489                 printf("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
 490                 printf("%.6f:", s->reclock.ctdbd.max);
 491
 492                 printf("%d:", s->reclock.recd.num);
 493                 printf("%.6f:", s->reclock.recd.min);
 494                 printf("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
 495                 printf("%.6f:", s->reclock.recd.max);
 496
 497                 printf("%d:", s->call_latency.num);
 498                 printf("%.6f:", s->call_latency.min);
 499                 printf("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
 500                 printf("%.6f:", s->call_latency.max);
 501
 502                 printf("%d:", s->childwrite_latency.num);
 503                 printf("%.6f:", s->childwrite_latency.min);
 504                 printf("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
 505                 printf("%.6f:", s->childwrite_latency.max);
 506                 printf("\n");
 507         } else {
 508                 printf("CTDB version %u\n", CTDB_VERSION);
 509                 printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
 510                 printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
 511
 512                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 513                         if (strchr(fields[i].name, '.')) {
 514                                 preflen = strcspn(fields[i].name, ".")+1;
 515                                 if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
 516                                         prefix = fields[i].name;
 517                                         printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
 518                                 }
 519                         } else {
 520                                 preflen = 0;
 521                         }
 522                         printf(" %*s%-22s%*s%10u\n",
 523                                preflen?4:0, "",
 524                                fields[i].name+preflen,
 525                                preflen?0:4, "",
 526                                *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 527                 }
 528                 printf(" hop_count_buckets:");
 529                 for (i=0;i<MAX_COUNT_BUCKETS;i++) {
 530                         printf(" %d", s->hop_count_bucket[i]);
 531                 }
 532                 printf("\n");
 533                 printf(" lock_buckets:");
 534                 for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 535                         printf(" %d", s->locks.buckets[i]);
 536                 }
 537                 printf("\n");
 538                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
 539
 540                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
 541
 542                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
 543
 544                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
 545                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
 546         }
 547
 548         talloc_free(tmp_ctx);
 549 }
 550
 551 /*
 552   display remote ctdb statistics combined from all nodes
 553  */
 554 static int control_statistics_all(struct ctdb_context *ctdb)
 555 {
 556         int ret, i;
 557         struct ctdb_statistics statistics;
 558         uint32_t *nodes;
 559         uint32_t num_nodes;
 560
 561         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
 562         CTDB_NO_MEMORY(ctdb, nodes);
 563
 564         ZERO_STRUCT(statistics);
 565
 566         for (i=0;i<num_nodes;i++) {
 567                 struct ctdb_statistics s1;
 568                 int j;
 569                 uint32_t *v1 = (uint32_t *)&s1;
 570                 uint32_t *v2 = (uint32_t *)&statistics;
 571                 uint32_t num_ints =
 572                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
 573                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
 574                 if (ret != 0) {
 575                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
 576                         return ret;
 577                 }
 578                 for (j=0;j<num_ints;j++) {
 579                         v2[j] += v1[j];
 580                 }
 581                 statistics.max_hop_count =
 582                         MAX(statistics.max_hop_count, s1.max_hop_count);
 583                 statistics.call_latency.max =
 584                         MAX(statistics.call_latency.max, s1.call_latency.max);
 585         }
 586         talloc_free(nodes);
 587         printf("Gathered statistics for %u nodes\n", num_nodes);
 588         show_statistics(&statistics, 1);
 589         return 0;
 590 }
 591
 592 /*
 593   display remote ctdb statistics
 594  */
 595 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
 596 {
 597         int ret;
 598         struct ctdb_statistics statistics;
 599
 600         if (options.pnn == CTDB_BROADCAST_ALL) {
 601                 return control_statistics_all(ctdb);
 602         }
 603
 604         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
 605         if (ret != 0) {
 606                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
 607                 return ret;
 608         }
 609         show_statistics(&statistics, 1);
 610         return 0;
 611 }
 612
 613
 614 /*
 615   reset remote ctdb statistics
 616  */
 617 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
 618 {
 619         int ret;
 620
 621         ret = ctdb_statistics_reset(ctdb, options.pnn);
 622         if (ret != 0) {
 623                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
 624                 return ret;
 625         }
 626         return 0;
 627 }
 628
 629
 630 /*
 631   display remote ctdb rolling statistics
 632  */
 633 static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
 634 {
 635         int ret;
 636         struct ctdb_statistics_wire *stats;
 637         int i, num_records = -1;
 638
 639         assert_single_node_only();
 640
 641         if (argc ==1) {
 642                 num_records = atoi(argv[0]) - 1;
 643         }
 644
 645         ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
 646         if (ret != 0) {
 647                 DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
 648                 return ret;
 649         }
 650         for (i=0;i<stats->num;i++) {
 651                 if (stats->stats[i].statistics_start_time.tv_sec == 0) {
 652                         continue;
 653                 }
 654                 show_statistics(&stats->stats[i], i==0);
 655                 if (i == num_records) {
 656                         break;
 657                 }
 658         }
 659         return 0;
 660 }
 661
 662
 663 /*
 664   display remote ctdb db statistics
 665  */
 666 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
 667 {
 668         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 669         struct ctdb_db_statistics *dbstat;
 670         int i;
 671         uint32_t db_id;
 672         int num_hot_keys;
 673         int ret;
 674
 675         if (argc < 1) {
 676                 usage();
 677         }
 678
 679         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
 680                 return -1;
 681         }
 682
 683         ret = ctdb_ctrl_dbstatistics(ctdb, options.pnn, db_id, tmp_ctx, &dbstat);
 684         if (ret != 0) {
 685                 DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
 686                 talloc_free(tmp_ctx);
 687                 return -1;
 688         }
 689
 690         printf("DB Statistics: %s\n", argv[0]);
 691         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
 692                 dbstat->db_ro_delegations);
 693         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
 694                 dbstat->db_ro_delegations);
 695         printf(" %s\n", "locks");
 696         printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
 697                 dbstat->locks.num_calls);
 698         printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
 699                 dbstat->locks.num_failed);
 700         printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
 701                 dbstat->locks.num_current);
 702         printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
 703                 dbstat->locks.num_pending);
 704         printf(" %s", "hop_count_buckets:");
 705         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 706                 printf(" %d", dbstat->hop_count_bucket[i]);
 707         }
 708         printf("\n");
 709         printf(" %s", "lock_buckets:");
 710         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 711                 printf(" %d", dbstat->locks.buckets[i]);
 712         }
 713         printf("\n");
 714         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
 715                 "locks_latency      MIN/AVG/MAX",
 716                 dbstat->locks.latency.min,
 717                 (dbstat->locks.latency.num ?
 718                  dbstat->locks.latency.total /dbstat->locks.latency.num :
 719                  0.0),
 720                 dbstat->locks.latency.max,
 721                 dbstat->locks.latency.num);
 722         num_hot_keys = 0;
 723         for (i=0; i<dbstat->num_hot_keys; i++) {
 724                 if (dbstat->hot_keys[i].count > 0) {
 725                         num_hot_keys++;
 726                 }
 727         }
 728         dbstat->num_hot_keys = num_hot_keys;
 729
 730         printf(" Num Hot Keys:     %d\n", dbstat->num_hot_keys);
 731         for (i = 0; i < dbstat->num_hot_keys; i++) {
 732                 int j;
 733                 printf("     Count:%d Key:", dbstat->hot_keys[i].count);
 734                 for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
 735                         printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
 736                 }
 737                 printf("\n");
 738         }
 739
 740         talloc_free(tmp_ctx);
 741         return 0;
 742 }
 743
 744 /*
 745   display uptime of remote node
 746  */
 747 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
 748 {
 749         int ret;
 750         struct ctdb_uptime *uptime = NULL;
 751         int tmp, days, hours, minutes, seconds;
 752
 753         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
 754         if (ret != 0) {
 755                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
 756                 return ret;
 757         }
 758
 759         if (options.machinereadable){
 760                 printf(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
 761                 printf(":%u:%u:%u:%lf\n",
 762                         (unsigned int)uptime->current_time.tv_sec,
 763                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
 764                         (unsigned int)uptime->last_recovery_finished.tv_sec,
 765                         timeval_delta(&uptime->last_recovery_finished,
 766                                       &uptime->last_recovery_started)
 767                 );
 768                 return 0;
 769         }
 770
 771         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
 772
 773         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 774         seconds = tmp%60;
 775         tmp    /= 60;
 776         minutes = tmp%60;
 777         tmp    /= 60;
 778         hours   = tmp%24;
 779         tmp    /= 24;
 780         days    = tmp;
 781         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
 782
 783         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 784         seconds = tmp%60;
 785         tmp    /= 60;
 786         minutes = tmp%60;
 787         tmp    /= 60;
 788         hours   = tmp%24;
 789         tmp    /= 24;
 790         days    = tmp;
 791         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
 792
 793         printf("Duration of last recovery/failover: %lf seconds\n",
 794                 timeval_delta(&uptime->last_recovery_finished,
 795                               &uptime->last_recovery_started));
 796
 797         return 0;
 798 }
 799
 800 /*
 801   show the PNN of the current node
 802  */
 803 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
 804 {
 805         uint32_t mypnn;
 806
 807         mypnn = getpnn(ctdb);
 808
 809         printf("PNN:%d\n", mypnn);
 810         return 0;
 811 }
 812
 813
 814 struct pnn_node {
 815         struct pnn_node *next, *prev;
 816         ctdb_sock_addr addr;
 817         int pnn;
 818 };
 819
 820 static struct pnn_node *read_pnn_node_file(TALLOC_CTX *mem_ctx,
 821                                            const char *file)
 822 {
 823         int nlines;
 824         char **lines;
 825         int i, pnn;
 826         struct pnn_node *pnn_nodes = NULL;
 827         struct pnn_node *pnn_node;
 828
 829         lines = file_lines_load(file, &nlines, mem_ctx);
 830         if (lines == NULL) {
 831                 return NULL;
 832         }
 833         for (i=0, pnn=0; i<nlines; i++) {
 834                 char *node;
 835
 836                 node = lines[i];
 837                 /* strip leading spaces */
 838                 while((*node == ' ') || (*node == '\t')) {
 839                         node++;
 840                 }
 841                 if (*node == '#') {
 842                         pnn++;
 843                         continue;
 844                 }
 845                 if (strcmp(node, "") == 0) {
 846                         continue;
 847                 }
 848                 pnn_node = talloc(mem_ctx, struct pnn_node);
 849                 pnn_node->pnn = pnn++;
 850
 851                 if (!parse_ip(node, NULL, 0, &pnn_node->addr)) {
 852                         DEBUG(DEBUG_ERR,
 853                               ("Invalid IP address '%s' in file %s\n",
 854                                node, file));
 855                         /* Caller will free mem_ctx */
 856                         return NULL;
 857                 }
 858
 859                 DLIST_ADD_END(pnn_nodes, pnn_node, NULL);
 860         }
 861
 862         return pnn_nodes;
 863 }
 864
 865 static struct pnn_node *read_nodes_file(TALLOC_CTX *mem_ctx)
 866 {
 867         const char *nodes_list;
 868
 869         /* read the nodes file */
 870         nodes_list = getenv("CTDB_NODES");
 871         if (nodes_list == NULL) {
 872                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes",
 873                                              getenv("CTDB_BASE"));
 874                 if (nodes_list == NULL) {
 875                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
 876                         exit(1);
 877                 }
 878         }
 879
 880         return read_pnn_node_file(mem_ctx, nodes_list);
 881 }
 882
 883 /*
 884   show the PNN of the current node
 885   discover the pnn by loading the nodes file and try to bind to all
 886   addresses one at a time until the ip address is found.
 887  */
 888 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
 889 {
 890         TALLOC_CTX *mem_ctx = talloc_new(NULL);
 891         struct pnn_node *pnn_nodes;
 892         struct pnn_node *pnn_node;
 893
 894         assert_single_node_only();
 895
 896         pnn_nodes = read_nodes_file(mem_ctx);
 897         if (pnn_nodes == NULL) {
 898                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
 899                 talloc_free(mem_ctx);
 900                 return -1;
 901         }
 902
 903         for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
 904                 if (ctdb_sys_have_ip(&pnn_node->addr)) {
 905                         printf("PNN:%d\n", pnn_node->pnn);
 906                         talloc_free(mem_ctx);
 907                         return 0;
 908                 }
 909         }
 910
 911         printf("Failed to detect which PNN this node is\n");
 912         talloc_free(mem_ctx);
 913         return -1;
 914 }
 915
 916 /* Helpers for ctdb status
 917  */
 918 static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_flags *node)
 919 {
 920         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 921         int j;
 922         bool ret = false;
 923
 924         if (node->flags == 0) {
 925                 struct ctdb_control_get_ifaces *ifaces;
 926
 927                 if (ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), node->pnn,
 928                                          tmp_ctx, &ifaces) == 0) {
 929                         for (j=0; j < ifaces->num; j++) {
 930                                 if (ifaces->ifaces[j].link_state != 0) {
 931                                         continue;
 932                                 }
 933                                 ret = true;
 934                                 break;
 935                         }
 936                 }
 937         }
 938         talloc_free(tmp_ctx);
 939
 940         return ret;
 941 }
 942
 943 static void control_status_header_machine(void)
 944 {
 945         printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
 946                ":Inactive:PartiallyOnline:ThisNode:\n");
 947 }
 948
 949 static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn,
 950                                     struct ctdb_node_and_flags *node)
 951 {
 952         printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
 953                ctdb_addr_to_str(&node->addr),
 954                !!(node->flags&NODE_FLAGS_DISCONNECTED),
 955                !!(node->flags&NODE_FLAGS_BANNED),
 956                !!(node->flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 957                !!(node->flags&NODE_FLAGS_UNHEALTHY),
 958                !!(node->flags&NODE_FLAGS_STOPPED),
 959                !!(node->flags&NODE_FLAGS_INACTIVE),
 960                is_partially_online(ctdb, node) ? 1 : 0,
 961                (node->pnn == mypnn)?'Y':'N');
 962
 963         return node->flags;
 964 }
 965
 966 static int control_status_1_human(struct ctdb_context *ctdb, int mypnn,
 967                                   struct ctdb_node_and_flags *node)
 968 {
 969        printf("pnn:%d %-16s %s%s\n", node->pnn,
 970               ctdb_addr_to_str(&node->addr),
 971               is_partially_online(ctdb, node) ? "PARTIALLYONLINE" : pretty_print_flags(node->flags),
 972               node->pnn == mypnn?" (THIS NODE)":"");
 973
 974        return node->flags;
 975 }
 976
 977 /*
 978   display remote ctdb status
 979  */
 980 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
 981 {
 982         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 983         int i;
 984         struct ctdb_vnn_map *vnnmap=NULL;
 985         struct ctdb_node_map *nodemap=NULL;
 986         uint32_t recmode, recmaster, mypnn;
 987         int num_deleted_nodes = 0;
 988         int ret;
 989
 990         mypnn = getpnn(ctdb);
 991
 992         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
 993         if (ret != 0) {
 994                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
 995                 talloc_free(tmp_ctx);
 996                 return -1;
 997         }
 998
 999         if (options.machinereadable) {
1000                 control_status_header_machine();
1001                 for (i=0;i<nodemap->num;i++) {
1002                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1003                                 continue;
1004                         }
1005                         (void) control_status_1_machine(ctdb, mypnn,
1006                                                         &nodemap->nodes[i]);
1007                 }
1008                 talloc_free(tmp_ctx);
1009                 return 0;
1010         }
1011
1012         for (i=0; i<nodemap->num; i++) {
1013                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1014                         num_deleted_nodes++;
1015                 }
1016         }
1017         if (num_deleted_nodes == 0) {
1018                 printf("Number of nodes:%d\n", nodemap->num);
1019         } else {
1020                 printf("Number of nodes:%d (including %d deleted nodes)\n",
1021                        nodemap->num, num_deleted_nodes);
1022         }
1023         for(i=0;i<nodemap->num;i++){
1024                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1025                         continue;
1026                 }
1027                 (void) control_status_1_human(ctdb, mypnn, &nodemap->nodes[i]);
1028         }
1029
1030         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
1031         if (ret != 0) {
1032                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
1033                 talloc_free(tmp_ctx);
1034                 return -1;
1035         }
1036         if (vnnmap->generation == INVALID_GENERATION) {
1037                 printf("Generation:INVALID\n");
1038         } else {
1039                 printf("Generation:%d\n",vnnmap->generation);
1040         }
1041         printf("Size:%d\n",vnnmap->size);
1042         for(i=0;i<vnnmap->size;i++){
1043                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
1044         }
1045
1046         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmode);
1047         if (ret != 0) {
1048                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
1049                 talloc_free(tmp_ctx);
1050                 return -1;
1051         }
1052         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
1053
1054         ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmaster);
1055         if (ret != 0) {
1056                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1057                 talloc_free(tmp_ctx);
1058                 return -1;
1059         }
1060         printf("Recovery master:%d\n",recmaster);
1061
1062         talloc_free(tmp_ctx);
1063         return 0;
1064 }
1065
1066 static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **argv)
1067 {
1068         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1069         int i, ret;
1070         struct ctdb_node_map *nodemap=NULL;
1071         uint32_t * nodes;
1072         uint32_t pnn_mode, mypnn;
1073
1074         if (argc > 1) {
1075                 usage();
1076         }
1077
1078         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1079                               options.pnn, true, &nodes, &pnn_mode)) {
1080                 return -1;
1081         }
1082
1083         if (options.machinereadable) {
1084                 control_status_header_machine();
1085         } else if (pnn_mode == CTDB_BROADCAST_ALL) {
1086                 printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
1087         }
1088
1089         mypnn = getpnn(ctdb);
1090
1091         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1092         if (ret != 0) {
1093                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1094                 talloc_free(tmp_ctx);
1095                 return -1;
1096         }
1097
1098         ret = 0;
1099
1100         for (i = 0; i < talloc_array_length(nodes); i++) {
1101                 if (options.machinereadable) {
1102                         ret |= control_status_1_machine(ctdb, mypnn,
1103                                                         &nodemap->nodes[nodes[i]]);
1104                 } else {
1105                         ret |= control_status_1_human(ctdb, mypnn,
1106                                                       &nodemap->nodes[nodes[i]]);
1107                 }
1108         }
1109
1110         talloc_free(tmp_ctx);
1111         return ret;
1112 }
1113
1114 static struct pnn_node *read_natgw_nodes_file(struct ctdb_context *ctdb,
1115                                               TALLOC_CTX *mem_ctx)
1116 {
1117         const char *natgw_list;
1118         struct pnn_node *natgw_nodes = NULL;
1119
1120         natgw_list = getenv("CTDB_NATGW_NODES");
1121         if (natgw_list == NULL) {
1122                 natgw_list = talloc_asprintf(mem_ctx, "%s/natgw_nodes",
1123                                              getenv("CTDB_BASE"));
1124                 if (natgw_list == NULL) {
1125                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
1126                         exit(1);
1127                 }
1128         }
1129         /* The PNNs will be junk but they're not used */
1130         natgw_nodes = read_pnn_node_file(mem_ctx, natgw_list);
1131         if (natgw_nodes == NULL) {
1132                 DEBUG(DEBUG_ERR,
1133                       ("Failed to load natgw node list '%s'\n", natgw_list));
1134         }
1135         return natgw_nodes;
1136 }
1137
1138
1139 /* talloc off the existing nodemap... */
1140 static struct ctdb_node_map *talloc_nodemap(struct ctdb_node_map *nodemap)
1141 {
1142         return talloc_zero_size(nodemap,
1143                                 offsetof(struct ctdb_node_map, nodes) +
1144                                 nodemap->num * sizeof(struct ctdb_node_and_flags));
1145 }
1146
1147 static struct ctdb_node_map *
1148 filter_nodemap_by_addrs(struct ctdb_context *ctdb,
1149                         struct ctdb_node_map *nodemap,
1150                         struct pnn_node *nodes)
1151 {
1152         int i;
1153         struct pnn_node *n;
1154         struct ctdb_node_map *ret;
1155
1156         ret = talloc_nodemap(nodemap);
1157         CTDB_NO_MEMORY_NULL(ctdb, ret);
1158
1159         ret->num = 0;
1160
1161         for (i = 0; i < nodemap->num; i++) {
1162                 for(n = nodes; n != NULL ; n = n->next) {
1163                         if (ctdb_same_ip(&n->addr,
1164                                          &nodemap->nodes[i].addr)) {
1165                                 break;
1166                         }
1167                 }
1168                 if (n == NULL) {
1169                         continue;
1170                 }
1171
1172                 ret->nodes[ret->num] = nodemap->nodes[i];
1173                 ret->num++;
1174         }
1175
1176         return ret;
1177 }
1178
1179 static struct ctdb_node_map *
1180 filter_nodemap_by_capabilities(struct ctdb_context *ctdb,
1181                                struct ctdb_node_map *nodemap,
1182                                uint32_t required_capabilities)
1183 {
1184         int i;
1185         uint32_t capabilities;
1186         struct ctdb_node_map *ret;
1187
1188         ret = talloc_nodemap(nodemap);
1189         CTDB_NO_MEMORY_NULL(ctdb, ret);
1190
1191         ret->num = 0;
1192
1193         for (i = 0; i < nodemap->num; i++) {
1194                 int res;
1195
1196                 /* Disconnected nodes have no capabilities! */
1197                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1198                         continue;
1199                 }
1200
1201                 res = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(),
1202                                                 nodemap->nodes[i].pnn,
1203                                                 &capabilities);
1204                 if (res != 0) {
1205                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n",
1206                                           nodemap->nodes[i].pnn));
1207                         talloc_free(ret);
1208                         return NULL;
1209                 }
1210                 if (!(capabilities & required_capabilities)) {
1211                         continue;
1212                 }
1213
1214                 ret->nodes[ret->num] = nodemap->nodes[i];
1215                 ret->num++;
1216         }
1217
1218         return ret;
1219 }
1220
1221 static struct ctdb_node_map *
1222 filter_nodemap_by_flags(struct ctdb_context *ctdb,
1223                         struct ctdb_node_map *nodemap,
1224                         uint32_t flags_mask)
1225 {
1226         int i;
1227         struct ctdb_node_map *ret;
1228
1229         ret = talloc_nodemap(nodemap);
1230         CTDB_NO_MEMORY_NULL(ctdb, ret);
1231
1232         ret->num = 0;
1233
1234         for (i = 0; i < nodemap->num; i++) {
1235                 if (nodemap->nodes[i].flags & flags_mask) {
1236                         continue;
1237                 }
1238
1239                 ret->nodes[ret->num] = nodemap->nodes[i];
1240                 ret->num++;
1241         }
1242
1243         return ret;
1244 }
1245
1246 /*
1247   display the list of nodes belonging to this natgw configuration
1248  */
1249 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
1250 {
1251         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1252         int i, ret;
1253         struct pnn_node *natgw_nodes = NULL;
1254         struct ctdb_node_map *orig_nodemap=NULL;
1255         struct ctdb_node_map *cnodemap, *nodemap;
1256         uint32_t mypnn, pnn;
1257         const char *ip;
1258
1259         /* When we have some nodes that could be the NATGW, make a
1260          * series of attempts to find the first node that doesn't have
1261          * certain status flags set.
1262          */
1263         uint32_t exclude_flags[] = {
1264                 /* Look for a nice healthy node */
1265                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY,
1266                 /* If not found, an UNHEALTHY/BANNED node will do */
1267                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED,
1268                 /* If not found, a STOPPED node will do */
1269                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED,
1270                 0,
1271         };
1272
1273         /* read the natgw nodes file into a linked list */
1274         natgw_nodes = read_natgw_nodes_file(ctdb, tmp_ctx);
1275         if (natgw_nodes == NULL) {
1276                 ret = -1;
1277                 goto done;
1278         }
1279
1280         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
1281                                    tmp_ctx, &orig_nodemap);
1282         if (ret != 0) {
1283                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
1284                 talloc_free(tmp_ctx);
1285                 return -1;
1286         }
1287
1288         /* Get a nodemap that includes only the nodes in the NATGW
1289          * group */
1290         nodemap = filter_nodemap_by_addrs(ctdb, orig_nodemap, natgw_nodes);
1291         if (nodemap == NULL) {
1292                 ret = -1;
1293                 goto done;
1294         }
1295
1296         /* Get a nodemap that includes only the nodes with the NATGW
1297          * capability */
1298         cnodemap = filter_nodemap_by_capabilities(ctdb, nodemap,
1299                                                   CTDB_CAP_NATGW);
1300         if (cnodemap == NULL) {
1301                 ret = -1;
1302                 goto done;
1303         }
1304
1305         ret = 2; /* matches ENOENT */
1306         pnn = -1;
1307         ip = "0.0.0.0";
1308         for (i = 0; exclude_flags[i] != 0; i++) {
1309                 struct ctdb_node_map *t =
1310                         filter_nodemap_by_flags(ctdb, cnodemap,
1311                                                 exclude_flags[i]);
1312                 if (t == NULL) {
1313                         /* No memory */
1314                         ret = -1;
1315                         goto done;
1316                 }
1317                 if (t->num > 0) {
1318                         ret = 0;
1319                         pnn = t->nodes[0].pnn;
1320                         ip = ctdb_addr_to_str(&t->nodes[0].addr);
1321                         break;
1322                 }
1323                 talloc_free(t);
1324         }
1325
1326         if (options.machinereadable) {
1327                 printf(":Node:IP:\n");
1328                 printf(":%d:%s:\n", pnn, ip);
1329         } else {
1330                 printf("%d %s\n", pnn, ip);
1331         }
1332
1333         /* print the pruned list of nodes belonging to this natgw list */
1334         mypnn = getpnn(ctdb);
1335         if (options.machinereadable) {
1336                 control_status_header_machine();
1337         } else {
1338                 printf("Number of nodes:%d\n", nodemap->num);
1339         }
1340         for(i=0;i<nodemap->num;i++){
1341                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1342                         continue;
1343                 }
1344                 if (options.machinereadable) {
1345                         control_status_1_machine(ctdb, mypnn, &(nodemap->nodes[i]));
1346                 } else {
1347                         control_status_1_human(ctdb, mypnn, &(nodemap->nodes[i]));
1348                 }
1349         }
1350
1351 done:
1352         talloc_free(tmp_ctx);
1353         return ret;
1354 }
1355
1356 /*
1357   display the status of the scripts for monitoring (or other events)
1358  */
1359 static int control_one_scriptstatus(struct ctdb_context *ctdb,
1360                                     enum ctdb_eventscript_call type)
1361 {
1362         struct ctdb_scripts_wire *script_status;
1363         int ret, i;
1364
1365         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
1366         if (ret != 0) {
1367                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
1368                 return ret;
1369         }
1370
1371         if (script_status == NULL) {
1372                 if (!options.machinereadable) {
1373                         printf("%s cycle never run\n",
1374                                ctdb_eventscript_call_names[type]);
1375                 }
1376                 return 0;
1377         }
1378
1379         if (!options.machinereadable) {
1380                 printf("%d scripts were executed last %s cycle\n",
1381                        script_status->num_scripts,
1382                        ctdb_eventscript_call_names[type]);
1383         }
1384         for (i=0; i<script_status->num_scripts; i++) {
1385                 const char *status = NULL;
1386
1387                 switch (script_status->scripts[i].status) {
1388                 case -ETIME:
1389                         status = "TIMEDOUT";
1390                         break;
1391                 case -ENOEXEC:
1392                         status = "DISABLED";
1393                         break;
1394                 case 0:
1395                         status = "OK";
1396                         break;
1397                 default:
1398                         if (script_status->scripts[i].status > 0)
1399                                 status = "ERROR";
1400                         break;
1401                 }
1402                 if (options.machinereadable) {
1403                         printf(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
1404                                ctdb_eventscript_call_names[type],
1405                                script_status->scripts[i].name,
1406                                script_status->scripts[i].status,
1407                                status,
1408                                (long)script_status->scripts[i].start.tv_sec,
1409                                (long)script_status->scripts[i].start.tv_usec,
1410                                (long)script_status->scripts[i].finished.tv_sec,
1411                                (long)script_status->scripts[i].finished.tv_usec,
1412                                script_status->scripts[i].output);
1413                         continue;
1414                 }
1415                 if (status)
1416                         printf("%-20s Status:%s    ",
1417                                script_status->scripts[i].name, status);
1418                 else
1419                         /* Some other error, eg from stat. */
1420                         printf("%-20s Status:CANNOT RUN (%s)",
1421                                script_status->scripts[i].name,
1422                                strerror(-script_status->scripts[i].status));
1423
1424                 if (script_status->scripts[i].status >= 0) {
1425                         printf("Duration:%.3lf ",
1426                         timeval_delta(&script_status->scripts[i].finished,
1427                               &script_status->scripts[i].start));
1428                 }
1429                 if (script_status->scripts[i].status != -ENOEXEC) {
1430                         printf("%s",
1431                                ctime(&script_status->scripts[i].start.tv_sec));
1432                         if (script_status->scripts[i].status != 0) {
1433                                 printf("   OUTPUT:%s\n",
1434                                        script_status->scripts[i].output);
1435                         }
1436                 } else {
1437                         printf("\n");
1438                 }
1439         }
1440         return 0;
1441 }
1442
1443
1444 static int control_scriptstatus(struct ctdb_context *ctdb,
1445                                 int argc, const char **argv)
1446 {
1447         int ret;
1448         enum ctdb_eventscript_call type, min, max;
1449         const char *arg;
1450
1451         if (argc > 1) {
1452                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
1453                 return -1;
1454         }
1455
1456         if (argc == 0)
1457                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
1458         else
1459                 arg = argv[0];
1460
1461         for (type = 0; type < CTDB_EVENT_MAX; type++) {
1462                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
1463                         min = type;
1464                         max = type+1;
1465                         break;
1466                 }
1467         }
1468         if (type == CTDB_EVENT_MAX) {
1469                 if (strcmp(arg, "all") == 0) {
1470                         min = 0;
1471                         max = CTDB_EVENT_MAX;
1472                 } else {
1473                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
1474                         return -1;
1475                 }
1476         }
1477
1478         if (options.machinereadable) {
1479                 printf(":Type:Name:Code:Status:Start:End:Error Output...:\n");
1480         }
1481
1482         for (type = min; type < max; type++) {
1483                 ret = control_one_scriptstatus(ctdb, type);
1484                 if (ret != 0) {
1485                         return ret;
1486                 }
1487         }
1488
1489         return 0;
1490 }
1491
1492 /*
1493   enable an eventscript
1494  */
1495 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1496 {
1497         int ret;
1498
1499         if (argc < 1) {
1500                 usage();
1501         }
1502
1503         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1504         if (ret != 0) {
1505           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
1506                 return ret;
1507         }
1508
1509         return 0;
1510 }
1511
1512 /*
1513   disable an eventscript
1514  */
1515 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1516 {
1517         int ret;
1518
1519         if (argc < 1) {
1520                 usage();
1521         }
1522
1523         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1524         if (ret != 0) {
1525           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
1526                 return ret;
1527         }
1528
1529         return 0;
1530 }
1531
1532 /*
1533   display the pnn of the recovery master
1534  */
1535 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
1536 {
1537         uint32_t recmaster;
1538         int ret;
1539
1540         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
1541         if (ret != 0) {
1542                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1543                 return -1;
1544         }
1545         printf("%d\n",recmaster);
1546
1547         return 0;
1548 }
1549
1550 /*
1551   add a tickle to a public address
1552  */
1553 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1554 {
1555         struct ctdb_tcp_connection t;
1556         TDB_DATA data;
1557         int ret;
1558
1559         assert_single_node_only();
1560
1561         if (argc < 2) {
1562                 usage();
1563         }
1564
1565         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1566                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1567                 return -1;
1568         }
1569         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1570                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1571                 return -1;
1572         }
1573
1574         data.dptr = (uint8_t *)&t;
1575         data.dsize = sizeof(t);
1576
1577         /* tell all nodes about this tcp connection */
1578         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1579                            0, data, ctdb, NULL, NULL, NULL, NULL);
1580         if (ret != 0) {
1581                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1582                 return -1;
1583         }
1584
1585         return 0;
1586 }
1587
1588
1589 /*
1590   delete a tickle from a node
1591  */
1592 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1593 {
1594         struct ctdb_tcp_connection t;
1595         TDB_DATA data;
1596         int ret;
1597
1598         assert_single_node_only();
1599
1600         if (argc < 2) {
1601                 usage();
1602         }
1603
1604         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1605                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1606                 return -1;
1607         }
1608         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1609                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1610                 return -1;
1611         }
1612
1613         data.dptr = (uint8_t *)&t;
1614         data.dsize = sizeof(t);
1615
1616         /* tell all nodes about this tcp connection */
1617         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1618                            0, data, ctdb, NULL, NULL, NULL, NULL);
1619         if (ret != 0) {
1620                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1621                 return -1;
1622         }
1623
1624         return 0;
1625 }
1626
1627
1628 /*
1629   get a list of all tickles for this pnn
1630  */
1631 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1632 {
1633         struct ctdb_control_tcp_tickle_list *list;
1634         ctdb_sock_addr addr;
1635         int i, ret;
1636         unsigned port = 0;
1637
1638         assert_single_node_only();
1639
1640         if (argc < 1) {
1641                 usage();
1642         }
1643
1644         if (argc == 2) {
1645                 port = atoi(argv[1]);
1646         }
1647
1648         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1649                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1650                 return -1;
1651         }
1652
1653         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1654         if (ret == -1) {
1655                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1656                 return -1;
1657         }
1658
1659         if (options.machinereadable){
1660                 printf(":source ip:port:destination ip:port:\n");
1661                 for (i=0;i<list->tickles.num;i++) {
1662                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1663                                 continue;
1664                         }
1665                         printf(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1666                         printf(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1667                 }
1668         } else {
1669                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1670                 printf("Num tickles:%u\n", list->tickles.num);
1671                 for (i=0;i<list->tickles.num;i++) {
1672                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1673                                 continue;
1674                         }
1675                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1676                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1677                 }
1678         }
1679
1680         talloc_free(list);
1681
1682         return 0;
1683 }
1684
1685
1686 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1687 {
1688         struct ctdb_all_public_ips *ips;
1689         struct ctdb_public_ip ip;
1690         int i, ret;
1691         uint32_t *nodes;
1692         uint32_t disable_time;
1693         TDB_DATA data;
1694         struct ctdb_node_map *nodemap=NULL;
1695         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1696
1697         disable_time = 30;
1698         data.dptr  = (uint8_t*)&disable_time;
1699         data.dsize = sizeof(disable_time);
1700         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1701         if (ret != 0) {
1702                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1703                 return -1;
1704         }
1705
1706
1707
1708         /* read the public ip list from the node */
1709         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1710         if (ret != 0) {
1711                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1712                 talloc_free(tmp_ctx);
1713                 return -1;
1714         }
1715
1716         for (i=0;i<ips->num;i++) {
1717                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1718                         break;
1719                 }
1720         }
1721         if (i==ips->num) {
1722                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1723                         pnn, ctdb_addr_to_str(addr)));
1724                 talloc_free(tmp_ctx);
1725                 return -1;
1726         }
1727
1728         ip.pnn  = pnn;
1729         ip.addr = *addr;
1730
1731         data.dptr  = (uint8_t *)&ip;
1732         data.dsize = sizeof(ip);
1733
1734         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1735         if (ret != 0) {
1736                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1737                 talloc_free(tmp_ctx);
1738                 return ret;
1739         }
1740
1741         nodes = list_of_nodes(ctdb, nodemap, tmp_ctx, NODE_FLAGS_INACTIVE, pnn);
1742         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1743                                         nodes, 0,
1744                                         LONGTIMELIMIT(),
1745                                         false, data,
1746                                         NULL, NULL,
1747                                         NULL);
1748         if (ret != 0) {
1749                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1750                 talloc_free(tmp_ctx);
1751                 return -1;
1752         }
1753
1754         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1755         if (ret != 0) {
1756                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1757                 talloc_free(tmp_ctx);
1758                 return -1;
1759         }
1760
1761         /* update the recovery daemon so it now knows to expect the new
1762            node assignment for this ip.
1763         */
1764         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1765         if (ret != 0) {
1766                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1767                 return -1;
1768         }
1769
1770         talloc_free(tmp_ctx);
1771         return 0;
1772 }
1773
1774
1775 /*
1776  * scans all other nodes and returns a pnn for another node that can host this
1777  * ip address or -1
1778  */
1779 static int
1780 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1781 {
1782         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1783         struct ctdb_all_public_ips *ips;
1784         struct ctdb_node_map *nodemap=NULL;
1785         int i, j, ret;
1786
1787         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1788         if (ret != 0) {
1789                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1790                 talloc_free(tmp_ctx);
1791                 return ret;
1792         }
1793
1794         for(i=0;i<nodemap->num;i++){
1795                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1796                         continue;
1797                 }
1798                 if (nodemap->nodes[i].pnn == options.pnn) {
1799                         continue;
1800                 }
1801
1802                 /* read the public ip list from this node */
1803                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1804                 if (ret != 0) {
1805                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1806                         return -1;
1807                 }
1808
1809                 for (j=0;j<ips->num;j++) {
1810                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1811                                 talloc_free(tmp_ctx);
1812                                 return nodemap->nodes[i].pnn;
1813                         }
1814                 }
1815                 talloc_free(ips);
1816         }
1817
1818         talloc_free(tmp_ctx);
1819         return -1;
1820 }
1821
1822 /* If pnn is -1 then try to find a node to move IP to... */
1823 static bool try_moveip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1824 {
1825         bool pnn_specified = (pnn == -1 ? false : true);
1826         int retries = 0;
1827
1828         while (retries < 5) {
1829                 if (!pnn_specified) {
1830                         pnn = find_other_host_for_public_ip(ctdb, addr);
1831                         if (pnn == -1) {
1832                                 return false;
1833                         }
1834                         DEBUG(DEBUG_NOTICE,
1835                               ("Trying to move public IP to node %u\n", pnn));
1836                 }
1837
1838                 if (move_ip(ctdb, addr, pnn) == 0) {
1839                         return true;
1840                 }
1841
1842                 sleep(3);
1843                 retries++;
1844         }
1845
1846         return false;
1847 }
1848
1849
1850 /*
1851   move/failover an ip address to a specific node
1852  */
1853 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1854 {
1855         uint32_t pnn;
1856         ctdb_sock_addr addr;
1857
1858         assert_single_node_only();
1859
1860         if (argc < 2) {
1861                 usage();
1862                 return -1;
1863         }
1864
1865         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1866                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1867                 return -1;
1868         }
1869
1870
1871         if (sscanf(argv[1], "%u", &pnn) != 1) {
1872                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1873                 return -1;
1874         }
1875
1876         if (!try_moveip(ctdb, &addr, pnn)) {
1877                 DEBUG(DEBUG_ERR,("Failed to move IP to node %d.\n", pnn));
1878                 return -1;
1879         }
1880
1881         return 0;
1882 }
1883
1884 static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
1885 {
1886         TDB_DATA data;
1887
1888         data.dptr  = (uint8_t *)&pnn;
1889         data.dsize = sizeof(uint32_t);
1890         if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
1891                 DEBUG(DEBUG_ERR,
1892                       ("Failed to send message to force node %u to be a rebalancing target\n",
1893                        pnn));
1894                 return -1;
1895         }
1896
1897         return 0;
1898 }
1899
1900
1901 /*
1902   rebalance a node by setting it to allow failback and triggering a
1903   takeover run
1904  */
1905 static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
1906 {
1907         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1908         uint32_t *nodes;
1909         uint32_t pnn_mode;
1910         int i, ret;
1911
1912         assert_single_node_only();
1913
1914         if (argc > 1) {
1915                 usage();
1916         }
1917
1918         /* Determine the nodes where IPs need to be reloaded */
1919         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1920                               options.pnn, true, &nodes, &pnn_mode)) {
1921                 ret = -1;
1922                 goto done;
1923         }
1924
1925         for (i = 0; i < talloc_array_length(nodes); i++) {
1926                 if (!rebalance_node(ctdb, nodes[i])) {
1927                         ret = -1;
1928                 }
1929         }
1930
1931 done:
1932         talloc_free(tmp_ctx);
1933         return ret;
1934 }
1935
1936 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1937 {
1938         struct ctdb_public_ip ip;
1939         int ret;
1940         uint32_t *nodes;
1941         uint32_t disable_time;
1942         TDB_DATA data;
1943         struct ctdb_node_map *nodemap=NULL;
1944         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1945
1946         disable_time = 30;
1947         data.dptr  = (uint8_t*)&disable_time;
1948         data.dsize = sizeof(disable_time);
1949         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1950         if (ret != 0) {
1951                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1952                 return -1;
1953         }
1954
1955         ip.pnn  = -1;
1956         ip.addr = *addr;
1957
1958         data.dptr  = (uint8_t *)&ip;
1959         data.dsize = sizeof(ip);
1960
1961         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1962         if (ret != 0) {
1963                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1964                 talloc_free(tmp_ctx);
1965                 return ret;
1966         }
1967
1968         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
1969         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1970                                         nodes, 0,
1971                                         LONGTIMELIMIT(),
1972                                         false, data,
1973                                         NULL, NULL,
1974                                         NULL);
1975         if (ret != 0) {
1976                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1977                 talloc_free(tmp_ctx);
1978                 return -1;
1979         }
1980
1981         talloc_free(tmp_ctx);
1982         return 0;
1983 }
1984
1985 /*
1986   release an ip form all nodes and have it re-assigned by recd
1987  */
1988 static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
1989 {
1990         ctdb_sock_addr addr;
1991
1992         assert_single_node_only();
1993
1994         if (argc < 1) {
1995                 usage();
1996                 return -1;
1997         }
1998
1999         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2000                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2001                 return -1;
2002         }
2003
2004         if (rebalance_ip(ctdb, &addr) != 0) {
2005                 DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
2006                 return -1;
2007         }
2008
2009         return 0;
2010 }
2011
2012 static int getips_store_callback(void *param, void *data)
2013 {
2014         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
2015         struct ctdb_all_public_ips *ips = param;
2016         int i;
2017
2018         i = ips->num++;
2019         ips->ips[i].pnn  = node_ip->pnn;
2020         ips->ips[i].addr = node_ip->addr;
2021         return 0;
2022 }
2023
2024 static int getips_count_callback(void *param, void *data)
2025 {
2026         uint32_t *count = param;
2027
2028         (*count)++;
2029         return 0;
2030 }
2031
2032 #define IP_KEYLEN       4
2033 static uint32_t *ip_key(ctdb_sock_addr *ip)
2034 {
2035         static uint32_t key[IP_KEYLEN];
2036
2037         bzero(key, sizeof(key));
2038
2039         switch (ip->sa.sa_family) {
2040         case AF_INET:
2041                 key[0]  = ip->ip.sin_addr.s_addr;
2042                 break;
2043         case AF_INET6: {
2044                 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
2045                 key[0]  = s6_a32[3];
2046                 key[1]  = s6_a32[2];
2047                 key[2]  = s6_a32[1];
2048                 key[3]  = s6_a32[0];
2049                 break;
2050         }
2051         default:
2052                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
2053                 return key;
2054         }
2055
2056         return key;
2057 }
2058
2059 static void *add_ip_callback(void *parm, void *data)
2060 {
2061         return parm;
2062 }
2063
2064 static int
2065 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_all_public_ips **ips)
2066 {
2067         struct ctdb_all_public_ips *tmp_ips;
2068         struct ctdb_node_map *nodemap=NULL;
2069         trbt_tree_t *ip_tree;
2070         int i, j, len, ret;
2071         uint32_t count;
2072
2073         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2074         if (ret != 0) {
2075                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2076                 return ret;
2077         }
2078
2079         ip_tree = trbt_create(tmp_ctx, 0);
2080
2081         for(i=0;i<nodemap->num;i++){
2082                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
2083                         continue;
2084                 }
2085                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2086                         continue;
2087                 }
2088
2089                 /* read the public ip list from this node */
2090                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
2091                 if (ret != 0) {
2092                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
2093                         return -1;
2094                 }
2095
2096                 for (j=0; j<tmp_ips->num;j++) {
2097                         struct ctdb_public_ip *node_ip;
2098
2099                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
2100                         node_ip->pnn  = tmp_ips->ips[j].pnn;
2101                         node_ip->addr = tmp_ips->ips[j].addr;
2102
2103                         trbt_insertarray32_callback(ip_tree,
2104                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
2105                                 add_ip_callback,
2106                                 node_ip);
2107                 }
2108                 talloc_free(tmp_ips);
2109         }
2110
2111         /* traverse */
2112         count = 0;
2113         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
2114
2115         len = offsetof(struct ctdb_all_public_ips, ips) +
2116                 count*sizeof(struct ctdb_public_ip);
2117         tmp_ips = talloc_zero_size(tmp_ctx, len);
2118         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
2119
2120         *ips = tmp_ips;
2121
2122         return 0;
2123 }
2124
2125
2126 static void ctdb_every_second(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
2127 {
2128         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2129
2130         event_add_timed(ctdb->ev, ctdb,
2131                                 timeval_current_ofs(1, 0),
2132                                 ctdb_every_second, ctdb);
2133 }
2134
2135 struct srvid_reply_handler_data {
2136         bool done;
2137         bool wait_for_all;
2138         uint32_t *nodes;
2139         const char *srvid_str;
2140 };
2141
2142 static void srvid_broadcast_reply_handler(struct ctdb_context *ctdb,
2143                                          uint64_t srvid,
2144                                          TDB_DATA data,
2145                                          void *private_data)
2146 {
2147         struct srvid_reply_handler_data *d =
2148                 (struct srvid_reply_handler_data *)private_data;
2149         int i;
2150         int32_t ret;
2151
2152         if (data.dsize != sizeof(ret)) {
2153                 DEBUG(DEBUG_ERR, (__location__ " Wrong reply size\n"));
2154                 return;
2155         }
2156
2157         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
2158         ret = *(int32_t *)data.dptr;
2159         if (ret < 0) {
2160                 DEBUG(DEBUG_ERR,
2161                       ("%s failed with result %d\n", d->srvid_str, ret));
2162                 return;
2163         }
2164
2165         if (!d->wait_for_all) {
2166                 d->done = true;
2167                 return;
2168         }
2169
2170         /* Wait for all replies */
2171         d->done = true;
2172         for (i = 0; i < talloc_array_length(d->nodes); i++) {
2173                 if (d->nodes[i] == ret) {
2174                         DEBUG(DEBUG_INFO,
2175                               ("%s reply received from node %u\n",
2176                                d->srvid_str, ret));
2177                         d->nodes[i] = -1;
2178                 }
2179                 if (d->nodes[i] != -1) {
2180                         /* Found a node that hasn't yet replied */
2181                         d->done = false;
2182                 }
2183         }
2184 }
2185
2186 /* Broadcast the given SRVID to all connected nodes.  Wait for 1 reply
2187  * or replies from all connected nodes.  arg is the data argument to
2188  * pass in the srvid_request structure - pass 0 if this isn't needed.
2189  */
2190 static int srvid_broadcast(struct ctdb_context *ctdb,
2191                            uint64_t srvid, uint32_t *arg,
2192                            const char *srvid_str, bool wait_for_all)
2193 {
2194         int ret;
2195         TDB_DATA data;
2196         uint32_t pnn;
2197         uint64_t reply_srvid;
2198         struct srvid_request request;
2199         struct srvid_request_data request_data;
2200         struct srvid_reply_handler_data reply_data;
2201         struct timeval tv;
2202
2203         ZERO_STRUCT(request);
2204
2205         /* Time ticks to enable timeouts to be processed */
2206         event_add_timed(ctdb->ev, ctdb,
2207                                 timeval_current_ofs(1, 0),
2208                                 ctdb_every_second, ctdb);
2209
2210         pnn = ctdb_get_pnn(ctdb);
2211         reply_srvid = getpid();
2212
2213         if (arg == NULL) {
2214                 request.pnn = pnn;
2215                 request.srvid = reply_srvid;
2216
2217                 data.dptr = (uint8_t *)&request;
2218                 data.dsize = sizeof(request);
2219         } else {
2220                 request_data.pnn = pnn;
2221                 request_data.srvid = reply_srvid;
2222                 request_data.data = *arg;
2223
2224                 data.dptr = (uint8_t *)&request_data;
2225                 data.dsize = sizeof(request_data);
2226         }
2227
2228         /* Register message port for reply from recovery master */
2229         ctdb_client_set_message_handler(ctdb, reply_srvid,
2230                                         srvid_broadcast_reply_handler,
2231                                         &reply_data);
2232
2233         reply_data.wait_for_all = wait_for_all;
2234         reply_data.nodes = NULL;
2235         reply_data.srvid_str = srvid_str;
2236
2237 again:
2238         reply_data.done = false;
2239
2240         if (wait_for_all) {
2241                 struct ctdb_node_map *nodemap;
2242
2243                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(),
2244                                            CTDB_CURRENT_NODE, ctdb, &nodemap);
2245                 if (ret != 0) {
2246                         DEBUG(DEBUG_ERR,
2247                               ("Unable to get nodemap from current node, try again\n"));
2248                         sleep(1);
2249                         goto again;
2250                 }
2251
2252                 if (reply_data.nodes != NULL) {
2253                         talloc_free(reply_data.nodes);
2254                 }
2255                 reply_data.nodes = list_of_connected_nodes(ctdb, nodemap,
2256                                                            NULL, true);
2257
2258                 talloc_free(nodemap);
2259         }
2260
2261         /* Send to all connected nodes. Only recmaster replies */
2262         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2263                                        srvid, data);
2264         if (ret != 0) {
2265                 /* This can only happen if the socket is closed and
2266                  * there's no way to recover from that, so don't try
2267                  * again.
2268                  */
2269                 DEBUG(DEBUG_ERR,
2270                       ("Failed to send %s request to connected nodes\n",
2271                        srvid_str));
2272                 return -1;
2273         }
2274
2275         tv = timeval_current();
2276         /* This loop terminates the reply is received */
2277         while (timeval_elapsed(&tv) < 5.0 && !reply_data.done) {
2278                 event_loop_once(ctdb->ev);
2279         }
2280
2281         if (!reply_data.done) {
2282                 DEBUG(DEBUG_NOTICE,
2283                       ("Still waiting for confirmation of %s\n", srvid_str));
2284                 sleep(1);
2285                 goto again;
2286         }
2287
2288         ctdb_client_remove_message_handler(ctdb, reply_srvid, &reply_data);
2289
2290         talloc_free(reply_data.nodes);
2291
2292         return 0;
2293 }
2294
2295 static int ipreallocate(struct ctdb_context *ctdb)
2296 {
2297         return srvid_broadcast(ctdb, CTDB_SRVID_TAKEOVER_RUN, NULL,
2298                                "IP reallocation", false);
2299 }
2300
2301
2302 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2303 {
2304         return ipreallocate(ctdb);
2305 }
2306
2307 /*
2308   add a public ip address to a node
2309  */
2310 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
2311 {
2312         int i, ret;
2313         int len, retries = 0;
2314         unsigned mask;
2315         ctdb_sock_addr addr;
2316         struct ctdb_control_ip_iface *pub;
2317         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2318         struct ctdb_all_public_ips *ips;
2319
2320
2321         if (argc != 2) {
2322                 talloc_free(tmp_ctx);
2323                 usage();
2324         }
2325
2326         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
2327                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
2328                 talloc_free(tmp_ctx);
2329                 return -1;
2330         }
2331
2332         /* read the public ip list from the node */
2333         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2334         if (ret != 0) {
2335                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", options.pnn));
2336                 talloc_free(tmp_ctx);
2337                 return -1;
2338         }
2339         for (i=0;i<ips->num;i++) {
2340                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2341                         DEBUG(DEBUG_ERR,("Can not add ip to node. Node already hosts this ip\n"));
2342                         return 0;
2343                 }
2344         }
2345
2346
2347
2348         /* Dont timeout. This command waits for an ip reallocation
2349            which sometimes can take wuite a while if there has
2350            been a recent recovery
2351         */
2352         alarm(0);
2353
2354         len = offsetof(struct ctdb_control_ip_iface, iface) + strlen(argv[1]) + 1;
2355         pub = talloc_size(tmp_ctx, len);
2356         CTDB_NO_MEMORY(ctdb, pub);
2357
2358         pub->addr  = addr;
2359         pub->mask  = mask;
2360         pub->len   = strlen(argv[1])+1;
2361         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
2362
2363         do {
2364                 ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
2365                 if (ret != 0) {
2366                         DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
2367                         sleep(3);
2368                         retries++;
2369                 }
2370         } while (retries < 5 && ret != 0);
2371         if (ret != 0) {
2372                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
2373                 talloc_free(tmp_ctx);
2374                 return ret;
2375         }
2376
2377         if (rebalance_node(ctdb, options.pnn) != 0) {
2378                 DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
2379                 return ret;
2380         }
2381
2382         talloc_free(tmp_ctx);
2383         return 0;
2384 }
2385
2386 /*
2387   add a public ip address to a node
2388  */
2389 static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
2390 {
2391         ctdb_sock_addr addr;
2392
2393         if (argc != 1) {
2394                 usage();
2395         }
2396
2397         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2398                 printf("Badly formed ip : %s\n", argv[0]);
2399                 return -1;
2400         }
2401
2402         printf("IP on interface %s\n", ctdb_sys_find_ifname(&addr));
2403
2404         return 0;
2405 }
2406
2407 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
2408
2409 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
2410 {
2411         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2412         struct ctdb_node_map *nodemap=NULL;
2413         struct ctdb_all_public_ips *ips;
2414         int ret, i, j;
2415
2416         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2417         if (ret != 0) {
2418                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
2419                 return ret;
2420         }
2421
2422         /* remove it from the nodes that are not hosting the ip currently */
2423         for(i=0;i<nodemap->num;i++){
2424                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2425                         continue;
2426                 }
2427                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2428                 if (ret != 0) {
2429                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2430                         continue;
2431                 }
2432
2433                 for (j=0;j<ips->num;j++) {
2434                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2435                                 break;
2436                         }
2437                 }
2438                 if (j==ips->num) {
2439                         continue;
2440                 }
2441
2442                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
2443                         continue;
2444                 }
2445
2446                 options.pnn = nodemap->nodes[i].pnn;
2447                 control_delip(ctdb, argc, argv);
2448         }
2449
2450
2451         /* remove it from every node (also the one hosting it) */
2452         for(i=0;i<nodemap->num;i++){
2453                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2454                         continue;
2455                 }
2456                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2457                 if (ret != 0) {
2458                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2459                         continue;
2460                 }
2461
2462                 for (j=0;j<ips->num;j++) {
2463                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2464                                 break;
2465                         }
2466                 }
2467                 if (j==ips->num) {
2468                         continue;
2469                 }
2470
2471                 options.pnn = nodemap->nodes[i].pnn;
2472                 control_delip(ctdb, argc, argv);
2473         }
2474
2475         talloc_free(tmp_ctx);
2476         return 0;
2477 }
2478
2479 /*
2480   delete a public ip address from a node
2481  */
2482 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
2483 {
2484         int i, ret;
2485         ctdb_sock_addr addr;
2486         struct ctdb_control_ip_iface pub;
2487         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2488         struct ctdb_all_public_ips *ips;
2489
2490         if (argc != 1) {
2491                 talloc_free(tmp_ctx);
2492                 usage();
2493         }
2494
2495         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2496                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2497                 return -1;
2498         }
2499
2500         if (options.pnn == CTDB_BROADCAST_ALL) {
2501                 return control_delip_all(ctdb, argc, argv, &addr);
2502         }
2503
2504         pub.addr  = addr;
2505         pub.mask  = 0;
2506         pub.len   = 0;
2507
2508         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2509         if (ret != 0) {
2510                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
2511                 talloc_free(tmp_ctx);
2512                 return ret;
2513         }
2514
2515         for (i=0;i<ips->num;i++) {
2516                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2517                         break;
2518                 }
2519         }
2520
2521         if (i==ips->num) {
2522                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
2523                         ctdb_addr_to_str(&addr)));
2524                 talloc_free(tmp_ctx);
2525                 return -1;
2526         }
2527
2528         /* This is an optimisation.  If this node is hosting the IP
2529          * then try to move it somewhere else without invoking a full
2530          * takeover run.  We don't care if this doesn't work!
2531          */
2532         if (ips->ips[i].pnn == options.pnn) {
2533                 (void) try_moveip(ctdb, &addr, -1);
2534         }
2535
2536         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
2537         if (ret != 0) {
2538                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
2539                 talloc_free(tmp_ctx);
2540                 return ret;
2541         }
2542
2543         talloc_free(tmp_ctx);
2544         return 0;
2545 }
2546
2547 static int kill_tcp_from_file(struct ctdb_context *ctdb,
2548                               int argc, const char **argv)
2549 {
2550         struct ctdb_control_killtcp *killtcp;
2551         int max_entries, current, i;
2552         struct timeval timeout;
2553         char line[128], src[128], dst[128];
2554         int linenum;
2555         TDB_DATA data;
2556         struct client_async_data *async_data;
2557         struct ctdb_client_control_state *state;
2558
2559         if (argc != 0) {
2560                 usage();
2561         }
2562
2563         linenum = 1;
2564         killtcp = NULL;
2565         max_entries = 0;
2566         current = 0;
2567         while (!feof(stdin)) {
2568                 if (fgets(line, sizeof(line), stdin) == NULL) {
2569                         continue;
2570                 }
2571
2572                 /* Silently skip empty lines */
2573                 if (line[0] == '\n') {
2574                         continue;
2575                 }
2576
2577                 if (sscanf(line, "%s %s\n", src, dst) != 2) {
2578                         DEBUG(DEBUG_ERR, ("Bad line [%d]: '%s'\n",
2579                                           linenum, line));
2580                         talloc_free(killtcp);
2581                         return -1;
2582                 }
2583
2584                 if (current >= max_entries) {
2585                         max_entries += 1024;
2586                         killtcp = talloc_realloc(ctdb, killtcp,
2587                                                  struct ctdb_control_killtcp,
2588                                                  max_entries);
2589                         CTDB_NO_MEMORY(ctdb, killtcp);
2590                 }
2591
2592                 if (!parse_ip_port(src, &killtcp[current].src_addr)) {
2593                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2594                                           linenum, src));
2595                         talloc_free(killtcp);
2596                         return -1;
2597                 }
2598
2599                 if (!parse_ip_port(dst, &killtcp[current].dst_addr)) {
2600                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2601                                           linenum, dst));
2602                         talloc_free(killtcp);
2603                         return -1;
2604                 }
2605
2606                 current++;
2607         }
2608
2609         async_data = talloc_zero(ctdb, struct client_async_data);
2610         if (async_data == NULL) {
2611                 talloc_free(killtcp);
2612                 return -1;
2613         }
2614
2615         for (i = 0; i < current; i++) {
2616
2617                 data.dsize = sizeof(struct ctdb_control_killtcp);
2618                 data.dptr  = (unsigned char *)&killtcp[i];
2619
2620                 timeout = TIMELIMIT();
2621                 state = ctdb_control_send(ctdb, options.pnn, 0,
2622                                           CTDB_CONTROL_KILL_TCP, 0, data,
2623                                           async_data, &timeout, NULL);
2624
2625                 if (state == NULL) {
2626                         DEBUG(DEBUG_ERR,
2627                               ("Failed to call async killtcp control to node %u\n",
2628                                options.pnn));
2629                         talloc_free(killtcp);
2630                         return -1;
2631                 }
2632
2633                 ctdb_client_async_add(async_data, state);
2634         }
2635
2636         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2637                 DEBUG(DEBUG_ERR,("killtcp failed\n"));
2638                 talloc_free(killtcp);
2639                 return -1;
2640         }
2641
2642         talloc_free(killtcp);
2643         return 0;
2644 }
2645
2646
2647 /*
2648   kill a tcp connection
2649  */
2650 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2651 {
2652         int ret;
2653         struct ctdb_control_killtcp killtcp;
2654
2655         assert_single_node_only();
2656
2657         if (argc == 0) {
2658                 return kill_tcp_from_file(ctdb, argc, argv);
2659         }
2660
2661         if (argc < 2) {
2662                 usage();
2663         }
2664
2665         if (!parse_ip_port(argv[0], &killtcp.src_addr)) {
2666                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2667                 return -1;
2668         }
2669
2670         if (!parse_ip_port(argv[1], &killtcp.dst_addr)) {
2671                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2672                 return -1;
2673         }
2674
2675         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
2676         if (ret != 0) {
2677                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
2678                 return ret;
2679         }
2680
2681         return 0;
2682 }
2683
2684
2685 /*
2686   send a gratious arp
2687  */
2688 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
2689 {
2690         int ret;
2691         ctdb_sock_addr addr;
2692
2693         assert_single_node_only();
2694
2695         if (argc < 2) {
2696                 usage();
2697         }
2698
2699         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2700                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
2701                 return -1;
2702         }
2703
2704         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
2705         if (ret != 0) {
2706                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
2707                 return ret;
2708         }
2709
2710         return 0;
2711 }
2712
2713 /*
2714   register a server id
2715  */
2716 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2717 {
2718         int ret;
2719         struct ctdb_server_id server_id;
2720
2721         if (argc < 3) {
2722                 usage();
2723         }
2724
2725         server_id.pnn       = strtoul(argv[0], NULL, 0);
2726         server_id.type      = strtoul(argv[1], NULL, 0);
2727         server_id.server_id = strtoul(argv[2], NULL, 0);
2728
2729         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
2730         if (ret != 0) {
2731                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
2732                 return ret;
2733         }
2734         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
2735         sleep(999);
2736         return -1;
2737 }
2738
2739 /*
2740   unregister a server id
2741  */
2742 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2743 {
2744         int ret;
2745         struct ctdb_server_id server_id;
2746
2747         if (argc < 3) {
2748                 usage();
2749         }
2750
2751         server_id.pnn       = strtoul(argv[0], NULL, 0);
2752         server_id.type      = strtoul(argv[1], NULL, 0);
2753         server_id.server_id = strtoul(argv[2], NULL, 0);
2754
2755         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
2756         if (ret != 0) {
2757                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
2758                 return ret;
2759         }
2760         return -1;
2761 }
2762
2763 /*
2764   check if a server id exists
2765  */
2766 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2767 {
2768         uint32_t status;
2769         int ret;
2770         struct ctdb_server_id server_id;
2771
2772         if (argc < 3) {
2773                 usage();
2774         }
2775
2776         server_id.pnn       = strtoul(argv[0], NULL, 0);
2777         server_id.type      = strtoul(argv[1], NULL, 0);
2778         server_id.server_id = strtoul(argv[2], NULL, 0);
2779
2780         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
2781         if (ret != 0) {
2782                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
2783                 return ret;
2784         }
2785
2786         if (status) {
2787                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
2788         } else {
2789                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
2790         }
2791         return 0;
2792 }
2793
2794 /*
2795   get a list of all server ids that are registered on a node
2796  */
2797 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
2798 {
2799         int i, ret;
2800         struct ctdb_server_id_list *server_ids;
2801
2802         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
2803         if (ret != 0) {
2804                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
2805                 return ret;
2806         }
2807
2808         for (i=0; i<server_ids->num; i++) {
2809                 printf("Server id %d:%d:%d\n",
2810                         server_ids->server_ids[i].pnn,
2811                         server_ids->server_ids[i].type,
2812                         server_ids->server_ids[i].server_id);
2813         }
2814
2815         return -1;
2816 }
2817
2818 /*
2819   check if a server id exists
2820  */
2821 static int check_srvids(struct ctdb_context *ctdb, int argc, const char **argv)
2822 {
2823         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
2824         uint64_t *ids;
2825         uint8_t *result;
2826         int i;
2827
2828         if (argc < 1) {
2829                 talloc_free(tmp_ctx);
2830                 usage();
2831         }
2832
2833         ids    = talloc_array(tmp_ctx, uint64_t, argc);
2834         result = talloc_array(tmp_ctx, uint8_t, argc);
2835
2836         for (i = 0; i < argc; i++) {
2837                 ids[i] = strtoull(argv[i], NULL, 0);
2838         }
2839
2840         if (!ctdb_client_check_message_handlers(ctdb, ids, argc, result)) {
2841                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n",
2842                                   options.pnn));
2843                 talloc_free(tmp_ctx);
2844                 return -1;
2845         }
2846
2847         for (i=0; i < argc; i++) {
2848                 printf("Server id %d:%llu %s\n", options.pnn, (long long)ids[i],
2849                        result[i] ? "exists" : "does not exist");
2850         }
2851
2852         talloc_free(tmp_ctx);
2853         return 0;
2854 }
2855
2856 /*
2857   send a tcp tickle ack
2858  */
2859 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2860 {
2861         int ret;
2862         ctdb_sock_addr  src, dst;
2863
2864         if (argc < 2) {
2865                 usage();
2866         }
2867
2868         if (!parse_ip_port(argv[0], &src)) {
2869                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2870                 return -1;
2871         }
2872
2873         if (!parse_ip_port(argv[1], &dst)) {
2874                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2875                 return -1;
2876         }
2877
2878         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
2879         if (ret==0) {
2880                 return 0;
2881         }
2882         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
2883
2884         return -1;
2885 }
2886
2887
2888 /*
2889   display public ip status
2890  */
2891 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
2892 {
2893         int i, ret;
2894         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2895         struct ctdb_all_public_ips *ips;
2896
2897         if (options.pnn == CTDB_BROADCAST_ALL) {
2898                 /* read the list of public ips from all nodes */
2899                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
2900         } else {
2901                 /* read the public ip list from this node */
2902                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2903         }
2904         if (ret != 0) {
2905                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
2906                 talloc_free(tmp_ctx);
2907                 return ret;
2908         }
2909
2910         if (options.machinereadable){
2911                 printf(":Public IP:Node:");
2912                 if (options.verbose){
2913                         printf("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
2914                 }
2915                 printf("\n");
2916         } else {
2917                 if (options.pnn == CTDB_BROADCAST_ALL) {
2918                         printf("Public IPs on ALL nodes\n");
2919                 } else {
2920                         printf("Public IPs on node %u\n", options.pnn);
2921                 }
2922         }
2923
2924         for (i=1;i<=ips->num;i++) {
2925                 struct ctdb_control_public_ip_info *info = NULL;
2926                 int32_t pnn;
2927                 char *aciface = NULL;
2928                 char *avifaces = NULL;
2929                 char *cifaces = NULL;
2930
2931                 if (options.pnn == CTDB_BROADCAST_ALL) {
2932                         pnn = ips->ips[ips->num-i].pnn;
2933                 } else {
2934                         pnn = options.pnn;
2935                 }
2936
2937                 if (pnn != -1) {
2938                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
2939                                                    &ips->ips[ips->num-i].addr, &info);
2940                 } else {
2941                         ret = -1;
2942                 }
2943
2944                 if (ret == 0) {
2945                         int j;
2946                         for (j=0; j < info->num; j++) {
2947                                 if (cifaces == NULL) {
2948                                         cifaces = talloc_strdup(info,
2949                                                                 info->ifaces[j].name);
2950                                 } else {
2951                                         cifaces = talloc_asprintf_append(cifaces,
2952                                                                          ",%s",
2953                                                                          info->ifaces[j].name);
2954                                 }
2955
2956                                 if (info->active_idx == j) {
2957                                         aciface = info->ifaces[j].name;
2958                                 }
2959
2960                                 if (info->ifaces[j].link_state == 0) {
2961                                         continue;
2962                                 }
2963
2964                                 if (avifaces == NULL) {
2965                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
2966                                 } else {
2967                                         avifaces = talloc_asprintf_append(avifaces,
2968                                                                           ",%s",
2969                                                                           info->ifaces[j].name);
2970                                 }
2971                         }
2972                 }
2973
2974                 if (options.machinereadable){
2975                         printf(":%s:%d:",
2976                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2977                                 ips->ips[ips->num-i].pnn);
2978                         if (options.verbose){
2979                                 printf("%s:%s:%s:",
2980                                         aciface?aciface:"",
2981                                         avifaces?avifaces:"",
2982                                         cifaces?cifaces:"");
2983                         }
2984                         printf("\n");
2985                 } else {
2986                         if (options.verbose) {
2987                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
2988                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2989                                         ips->ips[ips->num-i].pnn,
2990                                         aciface?aciface:"",
2991                                         avifaces?avifaces:"",
2992                                         cifaces?cifaces:"");
2993                         } else {
2994                                 printf("%s %d\n",
2995                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2996                                         ips->ips[ips->num-i].pnn);
2997                         }
2998                 }
2999                 talloc_free(info);
3000         }
3001
3002         talloc_free(tmp_ctx);
3003         return 0;
3004 }
3005
3006 /*
3007   public ip info
3008  */
3009 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
3010 {
3011         int i, ret;
3012         ctdb_sock_addr addr;
3013         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3014         struct ctdb_control_public_ip_info *info;
3015
3016         if (argc != 1) {
3017                 talloc_free(tmp_ctx);
3018                 usage();
3019         }
3020
3021         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
3022                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
3023                 return -1;
3024         }
3025
3026         /* read the public ip info from this node */
3027         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
3028                                            tmp_ctx, &addr, &info);
3029         if (ret != 0) {
3030                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
3031                                   argv[0], options.pnn));
3032                 talloc_free(tmp_ctx);
3033                 return ret;
3034         }
3035
3036         printf("Public IP[%s] info on node %u\n",
3037                ctdb_addr_to_str(&info->ip.addr),
3038                options.pnn);
3039
3040         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
3041                ctdb_addr_to_str(&info->ip.addr),
3042                info->ip.pnn, info->num);
3043
3044         for (i=0; i<info->num; i++) {
3045                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
3046
3047                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
3048                        i+1, info->ifaces[i].name,
3049                        info->ifaces[i].link_state?"up":"down",
3050                        (unsigned int)info->ifaces[i].references,
3051                        (i==info->active_idx)?" (active)":"");
3052         }
3053
3054         talloc_free(tmp_ctx);
3055         return 0;
3056 }
3057
3058 /*
3059   display interfaces status
3060  */
3061 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
3062 {
3063         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3064         int i;
3065         struct ctdb_control_get_ifaces *ifaces;
3066         int ret;
3067
3068         /* read the public ip list from this node */
3069         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ifaces);
3070         if (ret != 0) {
3071                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
3072                                   options.pnn));
3073                 talloc_free(tmp_ctx);
3074                 return -1;
3075         }
3076
3077         if (options.machinereadable){
3078                 printf(":Name:LinkStatus:References:\n");
3079         } else {
3080                 printf("Interfaces on node %u\n", options.pnn);
3081         }
3082
3083         for (i=0; i<ifaces->num; i++) {
3084                 if (options.machinereadable){
3085                         printf(":%s:%s:%u\n",
3086                                ifaces->ifaces[i].name,
3087                                ifaces->ifaces[i].link_state?"1":"0",
3088                                (unsigned int)ifaces->ifaces[i].references);
3089                 } else {
3090                         printf("name:%s link:%s references:%u\n",
3091                                ifaces->ifaces[i].name,
3092                                ifaces->ifaces[i].link_state?"up":"down",
3093                                (unsigned int)ifaces->ifaces[i].references);
3094                 }
3095         }
3096
3097         talloc_free(tmp_ctx);
3098         return 0;
3099 }
3100
3101
3102 /*
3103   set link status of an interface
3104  */
3105 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
3106 {
3107         int ret;
3108         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3109         struct ctdb_control_iface_info info;
3110
3111         ZERO_STRUCT(info);
3112
3113         if (argc != 2) {
3114                 usage();
3115         }
3116
3117         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
3118                 DEBUG(DEBUG_ERR, ("interfaces name '%s' too long\n",
3119                                   argv[0]));
3120                 talloc_free(tmp_ctx);
3121                 return -1;
3122         }
3123         strcpy(info.name, argv[0]);
3124
3125         if (strcmp(argv[1], "up") == 0) {
3126                 info.link_state = 1;
3127         } else if (strcmp(argv[1], "down") == 0) {
3128                 info.link_state = 0;
3129         } else {
3130                 DEBUG(DEBUG_ERR, ("link state invalid '%s' should be 'up' or 'down'\n",
3131                                   argv[1]));
3132                 talloc_free(tmp_ctx);
3133                 return -1;
3134         }
3135
3136         /* read the public ip list from this node */
3137         ret = ctdb_ctrl_set_iface_link(ctdb, TIMELIMIT(), options.pnn,
3138                                    tmp_ctx, &info);
3139         if (ret != 0) {
3140                 DEBUG(DEBUG_ERR, ("Unable to set link state for interfaces %s node %u\n",
3141                                   argv[0], options.pnn));
3142                 talloc_free(tmp_ctx);
3143                 return ret;
3144         }
3145
3146         talloc_free(tmp_ctx);
3147         return 0;
3148 }
3149
3150 /*
3151   display pid of a ctdb daemon
3152  */
3153 static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
3154 {
3155         uint32_t pid;
3156         int ret;
3157
3158         ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.pnn, &pid);
3159         if (ret != 0) {
3160                 DEBUG(DEBUG_ERR, ("Unable to get daemon pid from node %u\n", options.pnn));
3161                 return ret;
3162         }
3163         printf("Pid:%d\n", pid);
3164
3165         return 0;
3166 }
3167
3168 typedef bool update_flags_handler_t(struct ctdb_context *ctdb, void *data);
3169
3170 static int update_flags_and_ipreallocate(struct ctdb_context *ctdb,
3171                                               void *data,
3172                                               update_flags_handler_t handler,
3173                                               uint32_t flag,
3174                                               const char *desc,
3175                                               bool set_flag)
3176 {
3177         struct ctdb_node_map *nodemap = NULL;
3178         bool flag_is_set;
3179         int ret;
3180
3181         /* Check if the node is already in the desired state */
3182         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3183         if (ret != 0) {
3184                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3185                 exit(10);
3186         }
3187         flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3188         if (set_flag == flag_is_set) {
3189                 DEBUG(DEBUG_NOTICE, ("Node %d is %s %s\n", options.pnn,
3190                                      (set_flag ? "already" : "not"), desc));
3191                 return 0;
3192         }
3193
3194         do {
3195                 if (!handler(ctdb, data)) {
3196                         DEBUG(DEBUG_WARNING,
3197                               ("Failed to send control to set state %s on node %u, try again\n",
3198                                desc, options.pnn));
3199                 }
3200
3201                 sleep(1);
3202
3203                 /* Read the nodemap and verify the change took effect.
3204                  * Even if the above control/hanlder timed out then it
3205                  * could still have worked!
3206                  */
3207                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
3208                                          ctdb, &nodemap);
3209                 if (ret != 0) {
3210                         DEBUG(DEBUG_WARNING,
3211                               ("Unable to get nodemap from local node, try again\n"));
3212                 }
3213                 flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3214         } while (nodemap == NULL || (set_flag != flag_is_set));
3215
3216         return ipreallocate(ctdb);
3217 }
3218
3219 /* Administratively disable a node */
3220 static bool update_flags_disabled(struct ctdb_context *ctdb, void *data)
3221 {
3222         int ret;
3223
3224         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3225                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
3226         return ret == 0;
3227 }
3228
3229 static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
3230 {
3231         return update_flags_and_ipreallocate(ctdb, NULL,
3232                                                   update_flags_disabled,
3233                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3234                                                   "disabled",
3235                                                   true /* set_flag*/);
3236 }
3237
3238 /* Administratively re-enable a node */
3239 static bool update_flags_not_disabled(struct ctdb_context *ctdb, void *data)
3240 {
3241         int ret;
3242
3243         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3244                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
3245         return ret == 0;
3246 }
3247
3248 static int control_enable(struct ctdb_context *ctdb,  int argc, const char **argv)
3249 {
3250         return update_flags_and_ipreallocate(ctdb, NULL,
3251                                                   update_flags_not_disabled,
3252                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3253                                                   "disabled",
3254                                                   false /* set_flag*/);
3255 }
3256
3257 /* Stop a node */
3258 static bool update_flags_stopped(struct ctdb_context *ctdb, void *data)
3259 {
3260         int ret;
3261
3262         ret = ctdb_ctrl_stop_node(ctdb, TIMELIMIT(), options.pnn);
3263
3264         return ret == 0;
3265 }
3266
3267 static int control_stop(struct ctdb_context *ctdb, int argc, const char **argv)
3268 {
3269         return update_flags_and_ipreallocate(ctdb, NULL,
3270                                                   update_flags_stopped,
3271                                                   NODE_FLAGS_STOPPED,
3272                                                   "stopped",
3273                                                   true /* set_flag*/);
3274 }
3275
3276 /* Continue a stopped node */
3277 static bool update_flags_not_stopped(struct ctdb_context *ctdb, void *data)
3278 {
3279         int ret;
3280
3281         ret = ctdb_ctrl_continue_node(ctdb, TIMELIMIT(), options.pnn);
3282
3283         return ret == 0;
3284 }
3285
3286 static int control_continue(struct ctdb_context *ctdb, int argc, const char **argv)
3287 {
3288         return update_flags_and_ipreallocate(ctdb, NULL,
3289                                                   update_flags_not_stopped,
3290                                                   NODE_FLAGS_STOPPED,
3291                                                   "stopped",
3292                                                   false /* set_flag */);
3293 }
3294
3295 static uint32_t get_generation(struct ctdb_context *ctdb)
3296 {
3297         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3298         struct ctdb_vnn_map *vnnmap=NULL;
3299         int ret;
3300         uint32_t generation;
3301
3302         /* wait until the recmaster is not in recovery mode */
3303         while (1) {
3304                 uint32_t recmode, recmaster;
3305
3306                 if (vnnmap != NULL) {
3307                         talloc_free(vnnmap);
3308                         vnnmap = NULL;
3309                 }
3310
3311                 /* get the recmaster */
3312                 ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), CTDB_CURRENT_NODE, &recmaster);
3313                 if (ret != 0) {
3314                         DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
3315                         talloc_free(tmp_ctx);
3316                         exit(10);
3317                 }
3318
3319                 /* get recovery mode */
3320                 ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), recmaster, &recmode);
3321                 if (ret != 0) {
3322                         DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
3323                         talloc_free(tmp_ctx);
3324                         exit(10);
3325                 }
3326
3327                 /* get the current generation number */
3328                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), recmaster, tmp_ctx, &vnnmap);
3329                 if (ret != 0) {
3330                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from recmaster (%u)\n", recmaster));
3331                         talloc_free(tmp_ctx);
3332                         exit(10);
3333                 }
3334
3335                 if ((recmode == CTDB_RECOVERY_NORMAL) && (vnnmap->generation != 1)) {
3336                         generation = vnnmap->generation;
3337                         talloc_free(tmp_ctx);
3338                         return generation;
3339                 }
3340                 sleep(1);
3341         }
3342 }
3343
3344 /* Ban a node */
3345 static bool update_state_banned(struct ctdb_context *ctdb, void *data)
3346 {
3347         struct ctdb_ban_time *bantime = (struct ctdb_ban_time *)data;
3348         int ret;
3349
3350         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, bantime);
3351
3352         return ret == 0;
3353 }
3354
3355 static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
3356 {
3357         struct ctdb_ban_time bantime;
3358
3359         if (argc < 1) {
3360                 usage();
3361         }
3362
3363         bantime.pnn  = options.pnn;
3364         bantime.time = strtoul(argv[0], NULL, 0);
3365
3366         if (bantime.time == 0) {
3367                 DEBUG(DEBUG_ERR, ("Invalid ban time specified - must be >0\n"));
3368                 return -1;
3369         }
3370
3371         return update_flags_and_ipreallocate(ctdb, &bantime,
3372                                                   update_state_banned,
3373                                                   NODE_FLAGS_BANNED,
3374                                                   "banned",
3375                                                   true /* set_flag*/);
3376 }
3377
3378
3379 /* Unban a node */
3380 static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
3381 {
3382         struct ctdb_ban_time bantime;
3383
3384         bantime.pnn  = options.pnn;
3385         bantime.time = 0;
3386
3387         return update_flags_and_ipreallocate(ctdb, &bantime,
3388                                                   update_state_banned,
3389                                                   NODE_FLAGS_BANNED,
3390                                                   "banned",
3391                                                   false /* set_flag*/);
3392 }
3393
3394 /*
3395   show ban information for a node
3396  */
3397 static int control_showban(struct ctdb_context *ctdb, int argc, const char **argv)
3398 {
3399         int ret;
3400         struct ctdb_node_map *nodemap=NULL;
3401         struct ctdb_ban_time *bantime;
3402
3403         /* verify the node exists */
3404         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3405         if (ret != 0) {
3406                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3407                 return ret;
3408         }
3409
3410         ret = ctdb_ctrl_get_ban(ctdb, TIMELIMIT(), options.pnn, ctdb, &bantime);
3411         if (ret != 0) {
3412                 DEBUG(DEBUG_ERR,("Showing ban info for node %d failed.\n", options.pnn));
3413                 return -1;
3414         }
3415
3416         if (bantime->time == 0) {
3417                 printf("Node %u is not banned\n", bantime->pnn);
3418         } else {
3419                 printf("Node %u is banned, %d seconds remaining\n",
3420                        bantime->pnn, bantime->time);
3421         }
3422
3423         return 0;
3424 }
3425
3426 /*
3427   shutdown a daemon
3428  */
3429 static int control_shutdown(struct ctdb_context *ctdb, int argc, const char **argv)
3430 {
3431         int ret;
3432
3433         ret = ctdb_ctrl_shutdown(ctdb, TIMELIMIT(), options.pnn);
3434         if (ret != 0) {
3435                 DEBUG(DEBUG_ERR, ("Unable to shutdown node %u\n", options.pnn));
3436                 return ret;
3437         }
3438
3439         return 0;
3440 }
3441
3442 /*
3443   trigger a recovery
3444  */
3445 static int control_recover(struct ctdb_context *ctdb, int argc, const char **argv)
3446 {
3447         int ret;
3448         uint32_t generation, next_generation;
3449         bool force;
3450
3451         /* "force" option ignores freeze failure and forces recovery */
3452         force = (argc == 1) && (strcasecmp(argv[0], "force") == 0);
3453
3454         /* record the current generation number */
3455         generation = get_generation(ctdb);
3456
3457         ret = ctdb_ctrl_freeze_priority(ctdb, TIMELIMIT(), options.pnn, 1);
3458         if (ret != 0) {
3459                 if (!force) {
3460                         DEBUG(DEBUG_ERR, ("Unable to freeze node\n"));
3461                         return ret;
3462                 }
3463                 DEBUG(DEBUG_WARNING, ("Unable to freeze node but proceeding because \"force\" option given\n"));
3464         }
3465
3466         ret = ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3467         if (ret != 0) {
3468                 DEBUG(DEBUG_ERR, ("Unable to set recovery mode\n"));
3469                 return ret;
3470         }
3471
3472         /* wait until we are in a new generation */
3473         while (1) {
3474                 next_generation = get_generation(ctdb);
3475                 if (next_generation != generation) {
3476                         return 0;
3477                 }
3478                 sleep(1);
3479         }
3480
3481         return 0;
3482 }
3483
3484
3485 /*
3486   display monitoring mode of a remote node
3487  */
3488 static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
3489 {
3490         uint32_t monmode;
3491         int ret;
3492
3493         ret = ctdb_ctrl_getmonmode(ctdb, TIMELIMIT(), options.pnn, &monmode);
3494         if (ret != 0) {
3495                 DEBUG(DEBUG_ERR, ("Unable to get monmode from node %u\n", options.pnn));
3496                 return ret;
3497         }
3498         if (!options.machinereadable){
3499                 printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode);
3500         } else {
3501                 printf(":mode:\n");
3502                 printf(":%d:\n",monmode);
3503         }
3504         return 0;
3505 }
3506
3507
3508 /*
3509   display capabilities of a remote node
3510  */
3511 static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const char **argv)
3512 {
3513         uint32_t capabilities;
3514         int ret;
3515
3516         ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), options.pnn, &capabilities);
3517         if (ret != 0) {
3518                 DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", options.pnn));
3519                 return -1;
3520         }
3521
3522         if (!options.machinereadable){
3523                 printf("RECMASTER: %s\n", (capabilities&CTDB_CAP_RECMASTER)?"YES":"NO");
3524                 printf("LMASTER: %s\n", (capabilities&CTDB_CAP_LMASTER)?"YES":"NO");
3525                 printf("LVS: %s\n", (capabilities&CTDB_CAP_LVS)?"YES":"NO");
3526                 printf("NATGW: %s\n", (capabilities&CTDB_CAP_NATGW)?"YES":"NO");
3527         } else {
3528                 printf(":RECMASTER:LMASTER:LVS:NATGW:\n");
3529                 printf(":%d:%d:%d:%d:\n",
3530                         !!(capabilities&CTDB_CAP_RECMASTER),
3531                         !!(capabilities&CTDB_CAP_LMASTER),
3532                         !!(capabilities&CTDB_CAP_LVS),
3533                         !!(capabilities&CTDB_CAP_NATGW));
3534         }
3535         return 0;
3536 }
3537
3538 /*
3539   display lvs configuration
3540  */
3541
3542 static uint32_t lvs_exclude_flags[] = {
3543         /* Look for a nice healthy node */
3544         NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED,
3545         /* If not found, an UNHEALTHY node will do */
3546         NODE_FLAGS_INACTIVE|NODE_FLAGS_PERMANENTLY_DISABLED,
3547         0,
3548 };
3549
3550 static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
3551 {
3552         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3553         struct ctdb_node_map *orig_nodemap=NULL;
3554         struct ctdb_node_map *nodemap;
3555         int i, ret;
3556
3557         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3558                                    tmp_ctx, &orig_nodemap);
3559         if (ret != 0) {
3560                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3561                 talloc_free(tmp_ctx);
3562                 return -1;
3563         }
3564
3565         nodemap = filter_nodemap_by_capabilities(ctdb, orig_nodemap,
3566                                                  CTDB_CAP_LVS);
3567         if (nodemap == NULL) {
3568                 /* No memory */
3569                 ret = -1;
3570                 goto done;
3571         }
3572
3573         ret = 0;
3574
3575         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3576                 struct ctdb_node_map *t =
3577                         filter_nodemap_by_flags(ctdb, nodemap,
3578                                                 lvs_exclude_flags[i]);
3579                 if (t == NULL) {
3580                         /* No memory */
3581                         ret = -1;
3582                         goto done;
3583                 }
3584                 if (t->num > 0) {
3585                         /* At least 1 node without excluded flags */
3586                         int j;
3587                         for (j = 0; j < t->num; j++) {
3588                                 printf("%d:%s\n", t->nodes[j].pnn,
3589                                        ctdb_addr_to_str(&t->nodes[j].addr));
3590                         }
3591                         goto done;
3592                 }
3593                 talloc_free(t);
3594         }
3595 done:
3596         talloc_free(tmp_ctx);
3597         return ret;
3598 }
3599
3600 /*
3601   display who is the lvs master
3602  */
3603 static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **argv)
3604 {
3605         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3606         struct ctdb_node_map *orig_nodemap=NULL;
3607         struct ctdb_node_map *nodemap;
3608         int i, ret;
3609
3610         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3611                                    tmp_ctx, &orig_nodemap);
3612         if (ret != 0) {
3613                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3614                 talloc_free(tmp_ctx);
3615                 return -1;
3616         }
3617
3618         nodemap = filter_nodemap_by_capabilities(ctdb, orig_nodemap,
3619                                                  CTDB_CAP_LVS);
3620         if (nodemap == NULL) {
3621                 /* No memory */
3622                 ret = -1;
3623                 goto done;
3624         }
3625
3626         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3627                 struct ctdb_node_map *t =
3628                         filter_nodemap_by_flags(ctdb, nodemap,
3629                                                 lvs_exclude_flags[i]);
3630                 if (t == NULL) {
3631                         /* No memory */
3632                         ret = -1;
3633                         goto done;
3634                 }
3635                 if (t->num > 0) {
3636                         ret = 0;
3637                         printf(options.machinereadable ?
3638                                "%d\n" : "Node %d is LVS master\n",
3639                                 t->nodes[0].pnn);
3640                         goto done;
3641                 }
3642                 talloc_free(t);
3643         }
3644
3645         printf("There is no LVS master\n");
3646         ret = 255;
3647 done:
3648         talloc_free(tmp_ctx);
3649         return ret;
3650 }
3651
3652 /*
3653   disable monitoring on a  node
3654  */
3655 static int control_disable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3656 {
3657
3658         int ret;
3659
3660         ret = ctdb_ctrl_disable_monmode(ctdb, TIMELIMIT(), options.pnn);
3661         if (ret != 0) {
3662                 DEBUG(DEBUG_ERR, ("Unable to disable monmode on node %u\n", options.pnn));
3663                 return ret;
3664         }
3665         printf("Monitoring mode:%s\n","DISABLED");
3666
3667         return 0;
3668 }
3669
3670 /*
3671   enable monitoring on a  node
3672  */
3673 static int control_enable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3674 {
3675
3676         int ret;
3677
3678         ret = ctdb_ctrl_enable_monmode(ctdb, TIMELIMIT(), options.pnn);
3679         if (ret != 0) {
3680                 DEBUG(DEBUG_ERR, ("Unable to enable monmode on node %u\n", options.pnn));
3681                 return ret;
3682         }
3683         printf("Monitoring mode:%s\n","ACTIVE");
3684
3685         return 0;
3686 }
3687
3688 /*
3689   display remote list of keys/data for a db
3690  */
3691 static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
3692 {
3693         const char *db_name;
3694         struct ctdb_db_context *ctdb_db;
3695         int ret;
3696         struct ctdb_dump_db_context c;
3697         uint8_t flags;
3698
3699         if (argc < 1) {
3700                 usage();
3701         }
3702
3703         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3704                 return -1;
3705         }
3706
3707         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3708         if (ctdb_db == NULL) {
3709                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3710                 return -1;
3711         }
3712
3713         if (options.printlmaster) {
3714                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn,
3715                                           ctdb, &ctdb->vnn_map);
3716                 if (ret != 0) {
3717                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
3718                                           options.pnn));
3719                         return ret;
3720                 }
3721         }
3722
3723         ZERO_STRUCT(c);
3724         c.f = stdout;
3725         c.printemptyrecords = (bool)options.printemptyrecords;
3726         c.printdatasize = (bool)options.printdatasize;
3727         c.printlmaster = (bool)options.printlmaster;
3728         c.printhash = (bool)options.printhash;
3729         c.printrecordflags = (bool)options.printrecordflags;
3730
3731         /* traverse and dump the cluster tdb */
3732         ret = ctdb_dump_db(ctdb_db, &c);
3733         if (ret == -1) {
3734                 DEBUG(DEBUG_ERR, ("Unable to dump database\n"));
3735                 DEBUG(DEBUG_ERR, ("Maybe try 'ctdb getdbstatus %s'"
3736                                   " and 'ctdb getvar AllowUnhealthyDBRead'\n",
3737                                   db_name));
3738                 return -1;
3739         }
3740         talloc_free(ctdb_db);
3741
3742         printf("Dumped %d records\n", ret);
3743         return 0;
3744 }
3745
3746 struct cattdb_data {
3747         struct ctdb_context *ctdb;
3748         uint32_t count;
3749 };
3750
3751 static int cattdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private_data)
3752 {
3753         struct cattdb_data *d = private_data;
3754         struct ctdb_dump_db_context c;
3755
3756         d->count++;
3757
3758         ZERO_STRUCT(c);
3759         c.f = stdout;
3760         c.printemptyrecords = (bool)options.printemptyrecords;
3761         c.printdatasize = (bool)options.printdatasize;
3762         c.printlmaster = false;
3763         c.printhash = (bool)options.printhash;
3764         c.printrecordflags = true;
3765
3766         return ctdb_dumpdb_record(d->ctdb, key, data, &c);
3767 }
3768
3769 /*
3770   cat the local tdb database using same format as catdb
3771  */
3772 static int control_cattdb(struct ctdb_context *ctdb, int argc, const char **argv)
3773 {
3774         const char *db_name;
3775         struct ctdb_db_context *ctdb_db;
3776         struct cattdb_data d;
3777         uint8_t flags;
3778
3779         if (argc < 1) {
3780                 usage();
3781         }
3782
3783         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3784                 return -1;
3785         }
3786
3787         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3788         if (ctdb_db == NULL) {
3789                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3790                 return -1;
3791         }
3792
3793         /* traverse the local tdb */
3794         d.count = 0;
3795         d.ctdb  = ctdb;
3796         if (tdb_traverse_read(ctdb_db->ltdb->tdb, cattdb_traverse, &d) == -1) {
3797                 printf("Failed to cattdb data\n");
3798                 exit(10);
3799         }
3800         talloc_free(ctdb_db);
3801
3802         printf("Dumped %d records\n", d.count);
3803         return 0;
3804 }
3805
3806 /*
3807   display the content of a database key
3808  */
3809 static int control_readkey(struct ctdb_context *ctdb, int argc, const char **argv)
3810 {
3811         const char *db_name;
3812         struct ctdb_db_context *ctdb_db;
3813         struct ctdb_record_handle *h;
3814         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3815         TDB_DATA key, data;
3816         uint8_t flags;
3817
3818         if (argc < 2) {
3819                 usage();
3820         }
3821
3822         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3823                 return -1;
3824         }
3825
3826         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3827         if (ctdb_db == NULL) {
3828                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3829                 return -1;
3830         }
3831
3832         key.dptr  = discard_const(argv[1]);
3833         key.dsize = strlen((char *)key.dptr);
3834
3835         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3836         if (h == NULL) {
3837                 printf("Failed to fetch record '%s' on node %d\n",
3838                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3839                 talloc_free(tmp_ctx);
3840                 exit(10);
3841         }
3842
3843         printf("Data: size:%d ptr:[%.*s]\n", (int)data.dsize, (int)data.dsize, data.dptr);
3844
3845         talloc_free(ctdb_db);
3846         talloc_free(tmp_ctx);
3847         return 0;
3848 }
3849
3850 /*
3851   display the content of a database key
3852  */
3853 static int control_writekey(struct ctdb_context *ctdb, int argc, const char **argv)
3854 {
3855         const char *db_name;
3856         struct ctdb_db_context *ctdb_db;
3857         struct ctdb_record_handle *h;
3858         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3859         TDB_DATA key, data;
3860         uint8_t flags;
3861
3862         if (argc < 3) {
3863                 usage();
3864         }
3865
3866         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3867                 return -1;
3868         }
3869
3870         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3871         if (ctdb_db == NULL) {
3872                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3873                 return -1;
3874         }
3875
3876         key.dptr  = discard_const(argv[1]);
3877         key.dsize = strlen((char *)key.dptr);
3878
3879         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3880         if (h == NULL) {
3881                 printf("Failed to fetch record '%s' on node %d\n",
3882                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3883                 talloc_free(tmp_ctx);
3884                 exit(10);
3885         }
3886
3887         data.dptr  = discard_const(argv[2]);
3888         data.dsize = strlen((char *)data.dptr);
3889
3890         if (ctdb_record_store(h, data) != 0) {
3891                 printf("Failed to store record\n");
3892         }
3893
3894         talloc_free(h);
3895         talloc_free(ctdb_db);
3896         talloc_free(tmp_ctx);
3897         return 0;
3898 }
3899
3900 /*
3901   fetch a record from a persistent database
3902  */
3903 static int control_pfetch(struct ctdb_context *ctdb, int argc, const char **argv)
3904 {
3905         const char *db_name;
3906         struct ctdb_db_context *ctdb_db;
3907         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3908         struct ctdb_transaction_handle *h;
3909         TDB_DATA key, data;
3910         int fd, ret;
3911         bool persistent;
3912         uint8_t flags;
3913
3914         if (argc < 2) {
3915                 talloc_free(tmp_ctx);
3916                 usage();
3917         }
3918
3919         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3920                 talloc_free(tmp_ctx);
3921                 return -1;
3922         }
3923
3924         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
3925         if (!persistent) {
3926                 DEBUG(DEBUG_ERR,("Database '%s' is not persistent\n", db_name));
3927                 talloc_free(tmp_ctx);
3928                 return -1;
3929         }
3930
3931         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
3932         if (ctdb_db == NULL) {
3933                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3934                 talloc_free(tmp_ctx);
3935                 return -1;
3936         }
3937
3938         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
3939         if (h == NULL) {
3940                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
3941                 talloc_free(tmp_ctx);
3942                 return -1;
3943         }
3944
3945         key.dptr  = discard_const(argv[1]);
3946         key.dsize = strlen(argv[1]);
3947         ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data);
3948         if (ret != 0) {
3949                 DEBUG(DEBUG_ERR,("Failed to fetch record\n"));
3950                 talloc_free(tmp_ctx);
3951                 return -1;
3952         }
3953
3954         if (data.dsize == 0 || data.dptr == NULL) {
3955                 DEBUG(DEBUG_ERR,("Record is empty\n"));
3956                 talloc_free(tmp_ctx);
3957                 return -1;
3958         }
3959
3960         if (argc == 3) {
3961           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
3962                 if (fd == -1) {
3963                         DEBUG(DEBUG_ERR,("Failed to open output file %s\n", argv[2]));
3964                         talloc_free(tmp_ctx);
3965                         return -1;
3966                 }
3967                 write(fd, data.dptr, data.dsize);
3968                 close(fd);
3969         } else {
3970                 write(1, data.dptr, data.dsize);
3971         }
3972
3973         /* abort the transaction */
3974         talloc_free(h);
3975
3976
3977         talloc_free(tmp_ctx);
3978         return 0;
3979 }
3980
3981 /*
3982   fetch a record from a tdb-file
3983  */
3984 static int control_tfetch(struct ctdb_context *ctdb, int argc, const char **argv)
3985 {
3986         const char *tdb_file;
3987         TDB_CONTEXT *tdb;
3988         TDB_DATA key, data;
3989         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
3990         int fd;
3991
3992         if (argc < 2) {
3993                 usage();
3994         }
3995
3996         tdb_file = argv[0];
3997
3998         tdb = tdb_open(tdb_file, 0, 0, O_RDONLY, 0);
3999         if (tdb == NULL) {
4000                 printf("Failed to open TDB file %s\n", tdb_file);
4001                 return -1;
4002         }
4003
4004         if (!strncmp(argv[1], "0x", 2)) {
4005                 key = hextodata(tmp_ctx, argv[1] + 2);
4006                 if (key.dsize == 0) {
4007                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4008                         return -1;
4009                 }
4010         } else {
4011                 key.dptr  = discard_const(argv[1]);
4012                 key.dsize = strlen(argv[1]);
4013         }
4014
4015         data = tdb_fetch(tdb, key);
4016         if (data.dptr == NULL || data.dsize < sizeof(struct ctdb_ltdb_header)) {
4017                 printf("Failed to read record %s from tdb %s\n", argv[1], tdb_file);
4018                 tdb_close(tdb);
4019                 return -1;
4020         }
4021
4022         tdb_close(tdb);
4023
4024         if (argc == 3) {
4025           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
4026                 if (fd == -1) {
4027                         printf("Failed to open output file %s\n", argv[2]);
4028                         return -1;
4029                 }
4030                 if (options.verbose){
4031                         write(fd, data.dptr, data.dsize);
4032                 } else {
4033                         write(fd, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
4034                 }
4035                 close(fd);
4036         } else {
4037                 if (options.verbose){
4038                         write(1, data.dptr, data.dsize);
4039                 } else {
4040                         write(1, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
4041                 }
4042         }
4043
4044         talloc_free(tmp_ctx);
4045         return 0;
4046 }
4047
4048 /*
4049   store a record and header to a tdb-file
4050  */
4051 static int control_tstore(struct ctdb_context *ctdb, int argc, const char **argv)
4052 {
4053         const char *tdb_file;
4054         TDB_CONTEXT *tdb;
4055         TDB_DATA key, value, data;
4056         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
4057         struct ctdb_ltdb_header header;
4058
4059         if (argc < 3) {
4060                 usage();
4061         }
4062
4063         tdb_file = argv[0];
4064
4065         tdb = tdb_open(tdb_file, 0, 0, O_RDWR, 0);
4066         if (tdb == NULL) {
4067                 printf("Failed to open TDB file %s\n", tdb_file);
4068                 return -1;
4069         }
4070
4071         if (!strncmp(argv[1], "0x", 2)) {
4072                 key = hextodata(tmp_ctx, argv[1] + 2);
4073                 if (key.dsize == 0) {
4074                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4075                         return -1;
4076                 }
4077         } else {
4078                 key.dptr  = discard_const(argv[1]);
4079                 key.dsize = strlen(argv[1]);
4080         }
4081
4082         if (!strncmp(argv[2], "0x", 2)) {
4083                 value = hextodata(tmp_ctx, argv[2] + 2);
4084                 if (value.dsize == 0) {
4085                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[2]);
4086                         return -1;
4087                 }
4088         } else {
4089                 value.dptr  = discard_const(argv[2]);
4090                 value.dsize = strlen(argv[2]);
4091         }
4092
4093         ZERO_STRUCT(header);
4094         if (argc > 3) {
4095                 header.rsn = atoll(argv[3]);
4096         }
4097         if (argc > 4) {
4098                 header.dmaster = atoi(argv[4]);
4099         }
4100         if (argc > 5) {
4101                 header.flags = atoi(argv[5]);
4102         }
4103
4104         data.dsize = sizeof(struct ctdb_ltdb_header) + value.dsize;
4105         data.dptr = talloc_size(tmp_ctx, data.dsize);
4106         if (data.dptr == NULL) {
4107                 printf("Failed to allocate header+value\n");
4108                 return -1;
4109         }
4110
4111         *(struct ctdb_ltdb_header *)data.dptr = header;
4112         memcpy(data.dptr + sizeof(struct ctdb_ltdb_header), value.dptr, value.dsize);
4113
4114         if (tdb_store(tdb, key, data, TDB_REPLACE) != 0) {
4115                 printf("Failed to write record %s to tdb %s\n", argv[1], tdb_file);
4116                 tdb_close(tdb);
4117                 return -1;
4118         }
4119
4120         tdb_close(tdb);
4121
4122         talloc_free(tmp_ctx);
4123         return 0;
4124 }
4125
4126 /*
4127   write a record to a persistent database
4128  */
4129 static int control_pstore(struct ctdb_context *ctdb, int argc, const char **argv)
4130 {
4131         const char *db_name;
4132         struct ctdb_db_context *ctdb_db;
4133         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4134         struct ctdb_transaction_handle *h;
4135         struct stat st;
4136         TDB_DATA key, data;
4137         int fd, ret;
4138
4139         if (argc < 3) {
4140                 talloc_free(tmp_ctx);
4141                 usage();
4142         }
4143
4144         fd = open(argv[2], O_RDONLY);
4145         if (fd == -1) {
4146                 DEBUG(DEBUG_ERR,("Failed to open file containing record data : %s  %s\n", argv[2], strerror(errno)));
4147                 talloc_free(tmp_ctx);
4148                 return -1;
4149         }
4150
4151         ret = fstat(fd, &st);
4152         if (ret == -1) {
4153                 DEBUG(DEBUG_ERR,("fstat of file %s failed: %s\n", argv[2], strerror(errno)));
4154                 close(fd);
4155                 talloc_free(tmp_ctx);
4156                 return -1;
4157         }
4158
4159         if (!S_ISREG(st.st_mode)) {
4160                 DEBUG(DEBUG_ERR,("Not a regular file %s\n", argv[2]));
4161                 close(fd);
4162                 talloc_free(tmp_ctx);
4163                 return -1;
4164         }
4165
4166         data.dsize = st.st_size;
4167         if (data.dsize == 0) {
4168                 data.dptr  = NULL;
4169         } else {
4170                 data.dptr = talloc_size(tmp_ctx, data.dsize);
4171                 if (data.dptr == NULL) {
4172                         DEBUG(DEBUG_ERR,("Failed to talloc %d of memory to store record data\n", (int)data.dsize));
4173                         close(fd);
4174                         talloc_free(tmp_ctx);
4175                         return -1;
4176                 }
4177                 ret = read(fd, data.dptr, data.dsize);
4178                 if (ret != data.dsize) {
4179                         DEBUG(DEBUG_ERR,("Failed to read %d bytes of record data\n", (int)data.dsize));
4180                         close(fd);
4181                         talloc_free(tmp_ctx);
4182                         return -1;
4183                 }
4184         }
4185         close(fd);
4186
4187
4188         db_name = argv[0];
4189
4190         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4191         if (ctdb_db == NULL) {
4192                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4193                 talloc_free(tmp_ctx);
4194                 return -1;
4195         }
4196
4197         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4198         if (h == NULL) {
4199                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4200                 talloc_free(tmp_ctx);
4201                 return -1;
4202         }
4203
4204         key.dptr  = discard_const(argv[1]);
4205         key.dsize = strlen(argv[1]);
4206         ret = ctdb_transaction_store(h, key, data);
4207         if (ret != 0) {
4208                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4209                 talloc_free(tmp_ctx);
4210                 return -1;
4211         }
4212
4213         ret = ctdb_transaction_commit(h);
4214         if (ret != 0) {
4215                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4216                 talloc_free(tmp_ctx);
4217                 return -1;
4218         }
4219
4220
4221         talloc_free(tmp_ctx);
4222         return 0;
4223 }
4224
4225 /*
4226  * delete a record from a persistent database
4227  */
4228 static int control_pdelete(struct ctdb_context *ctdb, int argc, const char **argv)
4229 {
4230         const char *db_name;
4231         struct ctdb_db_context *ctdb_db;
4232         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4233         struct ctdb_transaction_handle *h;
4234         TDB_DATA key;
4235         int ret;
4236         bool persistent;
4237         uint8_t flags;
4238
4239         if (argc < 2) {
4240                 talloc_free(tmp_ctx);
4241                 usage();
4242         }
4243
4244         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
4245                 talloc_free(tmp_ctx);
4246                 return -1;
4247         }
4248
4249         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
4250         if (!persistent) {
4251                 DEBUG(DEBUG_ERR, ("Database '%s' is not persistent\n", db_name));
4252                 talloc_free(tmp_ctx);
4253                 return -1;
4254         }
4255
4256         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
4257         if (ctdb_db == NULL) {
4258                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n", db_name));
4259                 talloc_free(tmp_ctx);
4260                 return -1;
4261         }
4262
4263         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4264         if (h == NULL) {
4265                 DEBUG(DEBUG_ERR, ("Failed to start transaction on database %s\n", db_name));
4266                 talloc_free(tmp_ctx);
4267                 return -1;
4268         }
4269
4270         key.dptr = discard_const(argv[1]);
4271         key.dsize = strlen(argv[1]);
4272         ret = ctdb_transaction_store(h, key, tdb_null);
4273         if (ret != 0) {
4274                 DEBUG(DEBUG_ERR, ("Failed to delete record\n"));
4275                 talloc_free(tmp_ctx);
4276                 return -1;
4277         }
4278
4279         ret = ctdb_transaction_commit(h);
4280         if (ret != 0) {
4281                 DEBUG(DEBUG_ERR, ("Failed to commit transaction\n"));
4282                 talloc_free(tmp_ctx);
4283                 return -1;
4284         }
4285
4286         talloc_free(tmp_ctx);
4287         return 0;
4288 }
4289
4290 static const char *ptrans_parse_string(TALLOC_CTX *mem_ctx, const char *s,
4291                                        TDB_DATA *data)
4292 {
4293         const char *t;
4294         size_t n;
4295         const char *ret; /* Next byte after successfully parsed value */
4296
4297         /* Error, unless someone says otherwise */
4298         ret = NULL;
4299         /* Indicates no value to parse */
4300         *data = tdb_null;
4301
4302         /* Skip whitespace */
4303         n = strspn(s, " \t");
4304         t = s + n;
4305
4306         if (t[0] == '"') {
4307                 /* Quoted ASCII string - no wide characters! */
4308                 t++;
4309                 n = strcspn(t, "\"");
4310                 if (t[n] == '"') {
4311                         if (n > 0) {
4312                                 data->dsize = n;
4313                                 data->dptr = talloc_memdup(mem_ctx, t, n);
4314                                 CTDB_NOMEM_ABORT(data->dptr);
4315                         }
4316                         ret = t + n + 1;
4317                 } else {
4318                         DEBUG(DEBUG_WARNING,("Unmatched \" in input %s\n", s));
4319                 }
4320         } else {
4321                 DEBUG(DEBUG_WARNING,("Unsupported input format in %s\n", s));
4322         }
4323
4324         return ret;
4325 }
4326
4327 static bool ptrans_get_key_value(TALLOC_CTX *mem_ctx, FILE *file,
4328                                  TDB_DATA *key, TDB_DATA *value)
4329 {
4330         char line [1024]; /* FIXME: make this more flexible? */
4331         const char *t;
4332         char *ptr;
4333
4334         ptr = fgets(line, sizeof(line), file);
4335
4336         if (ptr == NULL) {
4337                 return false;
4338         }
4339
4340         /* Get key */
4341         t = ptrans_parse_string(mem_ctx, line, key);
4342         if (t == NULL || key->dptr == NULL) {
4343                 /* Line Ignored but not EOF */
4344                 return true;
4345         }
4346
4347         /* Get value */
4348         t = ptrans_parse_string(mem_ctx, t, value);
4349         if (t == NULL) {
4350                 /* Line Ignored but not EOF */
4351                 talloc_free(key->dptr);
4352                 *key = tdb_null;
4353                 return true;
4354         }
4355
4356         return true;
4357 }
4358
4359 /*
4360  * Update a persistent database as per file/stdin
4361  */
4362 static int control_ptrans(struct ctdb_context *ctdb,
4363                           int argc, const char **argv)
4364 {
4365         const char *db_name;
4366         struct ctdb_db_context *ctdb_db;
4367         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4368         struct ctdb_transaction_handle *h;
4369         TDB_DATA key, value;
4370         FILE *file;
4371         int ret;
4372
4373         if (argc < 1) {
4374                 talloc_free(tmp_ctx);
4375                 usage();
4376         }
4377
4378         file = stdin;
4379         if (argc == 2) {
4380                 file = fopen(argv[1], "r");
4381                 if (file == NULL) {
4382                         DEBUG(DEBUG_ERR,("Unable to open file for reading '%s'\n", argv[1]));
4383                         talloc_free(tmp_ctx);
4384                         return -1;
4385                 }
4386         }
4387
4388         db_name = argv[0];
4389
4390         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4391         if (ctdb_db == NULL) {
4392                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4393                 goto error;
4394         }
4395
4396         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4397         if (h == NULL) {
4398                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4399                 goto error;
4400         }
4401
4402         while (ptrans_get_key_value(tmp_ctx, file, &key, &value)) {
4403                 if (key.dsize != 0) {
4404                         ret = ctdb_transaction_store(h, key, value);
4405                         /* Minimise memory use */
4406                         talloc_free(key.dptr);
4407                         if (value.dptr != NULL) {
4408                                 talloc_free(value.dptr);
4409                         }
4410                         if (ret != 0) {
4411                                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4412                                 ctdb_transaction_cancel(h);
4413                                 goto error;
4414                         }
4415                 }
4416         }
4417
4418         ret = ctdb_transaction_commit(h);
4419         if (ret != 0) {
4420                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4421                 goto error;
4422         }
4423
4424         if (file != stdin) {
4425                 fclose(file);
4426         }
4427         talloc_free(tmp_ctx);
4428         return 0;
4429
4430 error:
4431         if (file != stdin) {
4432                 fclose(file);
4433         }
4434
4435         talloc_free(tmp_ctx);
4436         return -1;
4437 }
4438
4439 /*
4440   check if a service is bound to a port or not
4441  */
4442 static int control_chktcpport(struct ctdb_context *ctdb, int argc, const char **argv)
4443 {
4444         int s, ret;
4445         int v;
4446         int port;
4447         struct sockaddr_in sin;
4448
4449         if (argc != 1) {
4450                 printf("Use: ctdb chktcport <port>\n");
4451                 return EINVAL;
4452         }
4453
4454         port = atoi(argv[0]);
4455
4456         s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
4457         if (s == -1) {
4458                 printf("Failed to open local socket\n");
4459                 return errno;
4460         }
4461
4462         v = fcntl(s, F_GETFL, 0);
4463         if (v == -1 || fcntl(s, F_SETFL, v | O_NONBLOCK) != 0) {
4464                 printf("Unable to set socket non-blocking: %s\n", strerror(errno));
4465         }
4466
4467         bzero(&sin, sizeof(sin));
4468         sin.sin_family = PF_INET;
4469         sin.sin_port   = htons(port);
4470         ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
4471         close(s);
4472         if (ret == -1) {
4473                 printf("Failed to bind to local socket: %d %s\n", errno, strerror(errno));
4474                 return errno;
4475         }
4476
4477         return 0;
4478 }
4479
4480
4481
4482 static void log_handler(struct ctdb_context *ctdb, uint64_t srvid,
4483                              TDB_DATA data, void *private_data)
4484 {
4485         DEBUG(DEBUG_ERR,("Log data received\n"));
4486         if (data.dsize > 0) {
4487                 printf("%s", data.dptr);
4488         }
4489
4490         exit(0);
4491 }
4492
4493 /*
4494   display a list of log messages from the in memory ringbuffer
4495  */
4496 static int control_getlog(struct ctdb_context *ctdb, int argc, const char **argv)
4497 {
4498         int ret, i;
4499         bool main_daemon;
4500         struct ctdb_get_log_addr log_addr;
4501         TDB_DATA data;
4502         struct timeval tv;
4503
4504         /* Process options */
4505         main_daemon = true;
4506         log_addr.pnn = ctdb_get_pnn(ctdb);
4507         log_addr.level = DEBUG_NOTICE;
4508         for (i = 0; i < argc; i++) {
4509                 if (strcmp(argv[i], "recoverd") == 0) {
4510                         main_daemon = false;
4511                 } else {
4512                         if (isalpha(argv[i][0]) || argv[i][0] == '-') {
4513                                 log_addr.level = get_debug_by_desc(argv[i]);
4514                         } else {
4515                                 log_addr.level = strtol(argv[i], NULL, 0);
4516                         }
4517                 }
4518         }
4519
4520         /* Our message port is our PID */
4521         log_addr.srvid = getpid();
4522
4523         data.dptr = (unsigned char *)&log_addr;
4524         data.dsize = sizeof(log_addr);
4525
4526         DEBUG(DEBUG_ERR, ("Pulling logs from node %u\n", options.pnn));
4527
4528         ctdb_client_set_message_handler(ctdb, log_addr.srvid, log_handler, NULL);
4529         sleep(1);
4530
4531         DEBUG(DEBUG_ERR,("Listen for response on %d\n", (int)log_addr.srvid));
4532
4533         if (main_daemon) {
4534                 int32_t res;
4535                 char *errmsg;
4536                 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4537
4538                 ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_GET_LOG,
4539                                    0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
4540                 if (ret != 0 || res != 0) {
4541                         DEBUG(DEBUG_ERR,("Failed to get logs - %s\n", errmsg));
4542                         talloc_free(tmp_ctx);
4543                         return -1;
4544                 }
4545                 talloc_free(tmp_ctx);
4546         } else {
4547                 ret = ctdb_client_send_message(ctdb, options.pnn,
4548                                                CTDB_SRVID_GETLOG, data);
4549                 if (ret != 0) {
4550                         DEBUG(DEBUG_ERR,("Failed to send getlog request message to %u\n", options.pnn));
4551                         return -1;
4552                 }
4553         }
4554
4555         tv = timeval_current();
4556         /* this loop will terminate when we have received the reply */
4557         while (timeval_elapsed(&tv) < (double)options.timelimit) {
4558                 event_loop_once(ctdb->ev);
4559         }
4560
4561         DEBUG(DEBUG_INFO,("Timed out waiting for log data.\n"));
4562
4563         return 0;
4564 }
4565
4566 /*
4567   clear the in memory log area
4568  */
4569 static int control_clearlog(struct ctdb_context *ctdb, int argc, const char **argv)
4570 {
4571         int ret;
4572
4573         if (argc == 0 || (argc >= 1 && strcmp(argv[0], "recoverd") != 0)) {
4574                 /* "recoverd" not given - get logs from main daemon */
4575                 int32_t res;
4576                 char *errmsg;
4577                 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4578
4579                 ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_CLEAR_LOG,
4580                                    0, tdb_null, tmp_ctx, NULL, &res, NULL, &errmsg);
4581                 if (ret != 0 || res != 0) {
4582                         DEBUG(DEBUG_ERR,("Failed to clear logs\n"));
4583                         talloc_free(tmp_ctx);
4584                         return -1;
4585                 }
4586
4587                 talloc_free(tmp_ctx);
4588         } else {
4589                 TDB_DATA data; /* unused in recoverd... */
4590                 data.dsize = 0;
4591
4592                 ret = ctdb_client_send_message(ctdb, options.pnn, CTDB_SRVID_CLEARLOG, data);
4593                 if (ret != 0) {
4594                         DEBUG(DEBUG_ERR,("Failed to send clearlog request message to %u\n", options.pnn));
4595                         return -1;
4596                 }
4597         }
4598
4599         return 0;
4600 }
4601
4602 /* Reload public IPs on a specified nodes */
4603 static int control_reloadips(struct ctdb_context *ctdb, int argc, const char **argv)
4604 {
4605         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4606         uint32_t *nodes;
4607         uint32_t pnn_mode;
4608         uint32_t timeout;
4609         int ret;
4610
4611         assert_single_node_only();
4612
4613         if (argc > 1) {
4614                 usage();
4615         }
4616
4617         /* Determine the nodes where IPs need to be reloaded */
4618         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
4619                               options.pnn, true, &nodes, &pnn_mode)) {
4620                 ret = -1;
4621                 goto done;
4622         }
4623
4624 again:
4625         /* Disable takeover runs on all connected nodes.  A reply
4626          * indicating success is needed from each node so all nodes
4627          * will need to be active.  This will retry until maxruntime
4628          * is exceeded, hence no error handling.
4629          *
4630          * A check could be added to not allow reloading of IPs when
4631          * there are disconnected nodes.  However, this should
4632          * probably be left up to the administrator.
4633          */
4634         timeout = LONGTIMEOUT;
4635         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4636                         "Disable takeover runs", true);
4637
4638         /* Now tell all the desired nodes to reload their public IPs.
4639          * Keep trying this until it succeeds.  This assumes all
4640          * failures are transient, which might not be true...
4641          */
4642         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_RELOAD_PUBLIC_IPS,
4643                                       nodes, 0, LONGTIMELIMIT(),
4644                                       false, tdb_null,
4645                                       NULL, NULL, NULL) != 0) {
4646                 DEBUG(DEBUG_ERR,
4647                       ("Unable to reload IPs on some nodes, try again.\n"));
4648                 goto again;
4649         }
4650
4651         /* It isn't strictly necessary to wait until takeover runs are
4652          * re-enabled but doing so can't hurt.
4653          */
4654         timeout = 0;
4655         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4656                         "Enable takeover runs", true);
4657
4658         ipreallocate(ctdb);
4659
4660         ret = 0;
4661 done:
4662         talloc_free(tmp_ctx);
4663         return ret;
4664 }
4665
4666 /*
4667   display a list of the databases on a remote ctdb
4668  */
4669 static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **argv)
4670 {
4671         int i, ret;
4672         struct ctdb_dbid_map *dbmap=NULL;
4673
4674         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
4675         if (ret != 0) {
4676                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
4677                 return ret;
4678         }
4679
4680         if(options.machinereadable){
4681                 printf(":ID:Name:Path:Persistent:Sticky:Unhealthy:ReadOnly:\n");
4682                 for(i=0;i<dbmap->num;i++){
4683                         const char *path;
4684                         const char *name;
4685                         const char *health;
4686                         bool persistent;
4687                         bool readonly;
4688                         bool sticky;
4689
4690                         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn,
4691                                             dbmap->dbs[i].dbid, ctdb, &path);
4692                         ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
4693                                             dbmap->dbs[i].dbid, ctdb, &name);
4694                         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
4695                                               dbmap->dbs[i].dbid, ctdb, &health);
4696                         persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4697                         readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4698                         sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4699                         printf(":0x%08X:%s:%s:%d:%d:%d:%d:\n",
4700                                dbmap->dbs[i].dbid, name, path,
4701                                !!(persistent), !!(sticky),
4702                                !!(health), !!(readonly));
4703                 }
4704                 return 0;
4705         }
4706
4707         printf("Number of databases:%d\n", dbmap->num);
4708         for(i=0;i<dbmap->num;i++){
4709                 const char *path;
4710                 const char *name;
4711                 const char *health;
4712                 bool persistent;
4713                 bool readonly;
4714                 bool sticky;
4715
4716                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
4717                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
4718                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
4719                 persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4720                 readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4721                 sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4722                 printf("dbid:0x%08x name:%s path:%s%s%s%s%s\n",
4723                        dbmap->dbs[i].dbid, name, path,
4724                        persistent?" PERSISTENT":"",
4725                        sticky?" STICKY":"",
4726                        readonly?" READONLY":"",
4727                        health?" UNHEALTHY":"");
4728         }
4729
4730         return 0;
4731 }
4732
4733 /*
4734   display the status of a database on a remote ctdb
4735  */
4736 static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char **argv)
4737 {
4738         const char *db_name;
4739         uint32_t db_id;
4740         uint8_t flags;
4741         const char *path;
4742         const char *health;
4743
4744         if (argc < 1) {
4745                 usage();
4746         }
4747
4748         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
4749                 return -1;
4750         }
4751
4752         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &path);
4753         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &health);
4754         printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nSTICKY: %s\nREADONLY: %s\nHEALTH: %s\n",
4755                db_id, db_name, path,
4756                (flags & CTDB_DB_FLAGS_PERSISTENT ? "yes" : "no"),
4757                (flags & CTDB_DB_FLAGS_STICKY ? "yes" : "no"),
4758                (flags & CTDB_DB_FLAGS_READONLY ? "yes" : "no"),
4759                (health ? health : "OK"));
4760
4761         return 0;
4762 }
4763
4764 /*
4765   check if the local node is recmaster or not
4766   it will return 1 if this node is the recmaster and 0 if it is not
4767   or if the local ctdb daemon could not be contacted
4768  */
4769 static int control_isnotrecmaster(struct ctdb_context *ctdb, int argc, const char **argv)
4770 {
4771         uint32_t mypnn, recmaster;
4772         int ret;
4773
4774         assert_single_node_only();
4775
4776         mypnn = getpnn(ctdb);
4777
4778         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
4779         if (ret != 0) {
4780                 printf("Failed to get the recmaster\n");
4781                 return 1;
4782         }
4783
4784         if (recmaster != mypnn) {
4785                 printf("this node is not the recmaster\n");
4786                 return 1;
4787         }
4788
4789         printf("this node is the recmaster\n");
4790         return 0;
4791 }
4792
4793 /*
4794   ping a node
4795  */
4796 static int control_ping(struct ctdb_context *ctdb, int argc, const char **argv)
4797 {
4798         int ret;
4799         struct timeval tv = timeval_current();
4800         ret = ctdb_ctrl_ping(ctdb, options.pnn);
4801         if (ret == -1) {
4802                 printf("Unable to get ping response from node %u\n", options.pnn);
4803                 return -1;
4804         } else {
4805                 printf("response from %u time=%.6f sec  (%d clients)\n",
4806                        options.pnn, timeval_elapsed(&tv), ret);
4807         }
4808         return 0;
4809 }
4810
4811
4812 /*
4813   get a node's runstate
4814  */
4815 static int control_runstate(struct ctdb_context *ctdb, int argc, const char **argv)
4816 {
4817         int ret;
4818         enum ctdb_runstate runstate;
4819
4820         ret = ctdb_ctrl_get_runstate(ctdb, TIMELIMIT(), options.pnn, &runstate);
4821         if (ret == -1) {
4822                 printf("Unable to get runstate response from node %u\n",
4823                        options.pnn);
4824                 return -1;
4825         } else {
4826                 bool found = true;
4827                 enum ctdb_runstate t;
4828                 int i;
4829                 for (i=0; i<argc; i++) {
4830                         found = false;
4831                         t = runstate_from_string(argv[i]);
4832                         if (t == CTDB_RUNSTATE_UNKNOWN) {
4833                                 printf("Invalid run state (%s)\n", argv[i]);
4834                                 return -1;
4835                         }
4836
4837                         if (t == runstate) {
4838                                 found = true;
4839                                 break;
4840                         }
4841                 }
4842
4843                 if (!found) {
4844                         printf("CTDB not in required run state (got %s)\n",
4845                                runstate_to_string((enum ctdb_runstate)runstate));
4846                         return -1;
4847                 }
4848         }
4849
4850         printf("%s\n", runstate_to_string(runstate));
4851         return 0;
4852 }
4853
4854
4855 /*
4856   get a tunable
4857  */
4858 static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv)
4859 {
4860         const char *name;
4861         uint32_t value;
4862         int ret;
4863
4864         if (argc < 1) {
4865                 usage();
4866         }
4867
4868         name = argv[0];
4869         ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn, name, &value);
4870         if (ret != 0) {
4871                 DEBUG(DEBUG_ERR, ("Unable to get tunable variable '%s'\n", name));
4872                 return -1;
4873         }
4874
4875         printf("%-23s = %u\n", name, value);
4876         return 0;
4877 }
4878
4879 /*
4880   set a tunable
4881  */
4882 static int control_setvar(struct ctdb_context *ctdb, int argc, const char **argv)
4883 {
4884         const char *name;
4885         uint32_t value;
4886         int ret;
4887
4888         if (argc < 2) {
4889                 usage();
4890         }
4891
4892         name = argv[0];
4893         value = strtoul(argv[1], NULL, 0);
4894
4895         ret = ctdb_ctrl_set_tunable(ctdb, TIMELIMIT(), options.pnn, name, value);
4896         if (ret == -1) {
4897                 DEBUG(DEBUG_ERR, ("Unable to set tunable variable '%s'\n", name));
4898                 return -1;
4899         }
4900         return 0;
4901 }
4902
4903 /*
4904   list all tunables
4905  */
4906 static int control_listvars(struct ctdb_context *ctdb, int argc, const char **argv)
4907 {
4908         uint32_t count;
4909         const char **list;
4910         int ret, i;
4911
4912         ret = ctdb_ctrl_list_tunables(ctdb, TIMELIMIT(), options.pnn, ctdb, &list, &count);
4913         if (ret == -1) {
4914                 DEBUG(DEBUG_ERR, ("Unable to list tunable variables\n"));
4915                 return -1;
4916         }
4917
4918         for (i=0;i<count;i++) {
4919                 control_getvar(ctdb, 1, &list[i]);
4920         }
4921
4922         talloc_free(list);
4923
4924         return 0;
4925 }
4926
4927 /*
4928   display debug level on a node
4929  */
4930 static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **argv)
4931 {
4932         int ret;
4933         int32_t level;
4934
4935         ret = ctdb_ctrl_get_debuglevel(ctdb, options.pnn, &level);
4936         if (ret != 0) {
4937                 DEBUG(DEBUG_ERR, ("Unable to get debuglevel response from node %u\n", options.pnn));
4938                 return ret;
4939         } else {
4940                 if (options.machinereadable){
4941                         printf(":Name:Level:\n");
4942                         printf(":%s:%d:\n",get_debug_by_level(level),level);
4943                 } else {
4944                         printf("Node %u is at debug level %s (%d)\n", options.pnn, get_debug_by_level(level), level);
4945                 }
4946         }
4947         return 0;
4948 }
4949
4950 /*
4951   display reclock file of a node
4952  */
4953 static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4954 {
4955         int ret;
4956         const char *reclock;
4957
4958         ret = ctdb_ctrl_getreclock(ctdb, TIMELIMIT(), options.pnn, ctdb, &reclock);
4959         if (ret != 0) {
4960                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4961                 return ret;
4962         } else {
4963                 if (options.machinereadable){
4964                         if (reclock != NULL) {
4965                                 printf("%s", reclock);
4966                         }
4967                 } else {
4968                         if (reclock == NULL) {
4969                                 printf("No reclock file used.\n");
4970                         } else {
4971                                 printf("Reclock file:%s\n", reclock);
4972                         }
4973                 }
4974         }
4975         return 0;
4976 }
4977
4978 /*
4979   set the reclock file of a node
4980  */
4981 static int control_setreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4982 {
4983         int ret;
4984         const char *reclock;
4985
4986         if (argc == 0) {
4987                 reclock = NULL;
4988         } else if (argc == 1) {
4989                 reclock = argv[0];
4990         } else {
4991                 usage();
4992         }
4993
4994         ret = ctdb_ctrl_setreclock(ctdb, TIMELIMIT(), options.pnn, reclock);
4995         if (ret != 0) {
4996                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4997                 return ret;
4998         }
4999         return 0;
5000 }
5001
5002 /*
5003   set the natgw state on/off
5004  */
5005 static int control_setnatgwstate(struct ctdb_context *ctdb, int argc, const char **argv)
5006 {
5007         int ret;
5008         uint32_t natgwstate;
5009
5010         if (argc == 0) {
5011                 usage();
5012         }
5013
5014         if (!strcmp(argv[0], "on")) {
5015                 natgwstate = 1;
5016         } else if (!strcmp(argv[0], "off")) {
5017                 natgwstate = 0;
5018         } else {
5019                 usage();
5020         }
5021
5022         ret = ctdb_ctrl_setnatgwstate(ctdb, TIMELIMIT(), options.pnn, natgwstate);
5023         if (ret != 0) {
5024                 DEBUG(DEBUG_ERR, ("Unable to set the natgw state for node %u\n", options.pnn));
5025                 return ret;
5026         }
5027
5028         return 0;
5029 }
5030
5031 /*
5032   set the lmaster role on/off
5033  */
5034 static int control_setlmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
5035 {
5036         int ret;
5037         uint32_t lmasterrole;
5038
5039         if (argc == 0) {
5040                 usage();
5041         }
5042
5043         if (!strcmp(argv[0], "on")) {
5044                 lmasterrole = 1;
5045         } else if (!strcmp(argv[0], "off")) {
5046                 lmasterrole = 0;
5047         } else {
5048                 usage();
5049         }
5050
5051         ret = ctdb_ctrl_setlmasterrole(ctdb, TIMELIMIT(), options.pnn, lmasterrole);
5052         if (ret != 0) {
5053                 DEBUG(DEBUG_ERR, ("Unable to set the lmaster role for node %u\n", options.pnn));
5054                 return ret;
5055         }
5056
5057         return 0;
5058 }
5059
5060 /*
5061   set the recmaster role on/off
5062  */
5063 static int control_setrecmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
5064 {
5065         int ret;
5066         uint32_t recmasterrole;
5067
5068         if (argc == 0) {
5069                 usage();
5070         }
5071
5072         if (!strcmp(argv[0], "on")) {
5073                 recmasterrole = 1;
5074         } else if (!strcmp(argv[0], "off")) {
5075                 recmasterrole = 0;
5076         } else {
5077                 usage();
5078         }
5079
5080         ret = ctdb_ctrl_setrecmasterrole(ctdb, TIMELIMIT(), options.pnn, recmasterrole);
5081         if (ret != 0) {
5082                 DEBUG(DEBUG_ERR, ("Unable to set the recmaster role for node %u\n", options.pnn));
5083                 return ret;
5084         }
5085
5086         return 0;
5087 }
5088
5089 /*
5090   set debug level on a node or all nodes
5091  */
5092 static int control_setdebug(struct ctdb_context *ctdb, int argc, const char **argv)
5093 {
5094         int i, ret;
5095         int32_t level;
5096
5097         if (argc == 0) {
5098                 printf("You must specify the debug level. Valid levels are:\n");
5099                 for (i=0; debug_levels[i].description != NULL; i++) {
5100                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
5101                 }
5102
5103                 return 0;
5104         }
5105
5106         if (isalpha(argv[0][0]) || argv[0][0] == '-') {
5107                 level = get_debug_by_desc(argv[0]);
5108         } else {
5109                 level = strtol(argv[0], NULL, 0);
5110         }
5111
5112         for (i=0; debug_levels[i].description != NULL; i++) {
5113                 if (level == debug_levels[i].level) {
5114                         break;
5115                 }
5116         }
5117         if (debug_levels[i].description == NULL) {
5118                 printf("Invalid debug level, must be one of\n");
5119                 for (i=0; debug_levels[i].description != NULL; i++) {
5120                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
5121                 }
5122                 return -1;
5123         }
5124
5125         ret = ctdb_ctrl_set_debuglevel(ctdb, options.pnn, level);
5126         if (ret != 0) {
5127                 DEBUG(DEBUG_ERR, ("Unable to set debug level on node %u\n", options.pnn));
5128         }
5129         return 0;
5130 }
5131
5132
5133 /*
5134   thaw a node
5135  */
5136 static int control_thaw(struct ctdb_context *ctdb, int argc, const char **argv)
5137 {
5138         int ret;
5139         uint32_t priority;
5140
5141         if (argc == 1) {
5142                 priority = strtol(argv[0], NULL, 0);
5143         } else {
5144                 priority = 0;
5145         }
5146         DEBUG(DEBUG_ERR,("Thaw by priority %u\n", priority));
5147
5148         ret = ctdb_ctrl_thaw_priority(ctdb, TIMELIMIT(), options.pnn, priority);
5149         if (ret != 0) {
5150                 DEBUG(DEBUG_ERR, ("Unable to thaw node %u\n", options.pnn));
5151         }
5152         return 0;
5153 }
5154
5155
5156 /*
5157   attach to a database
5158  */
5159 static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv)
5160 {
5161         const char *db_name;
5162         struct ctdb_db_context *ctdb_db;
5163         bool persistent = false;
5164
5165         if (argc < 1) {
5166                 usage();
5167         }
5168         db_name = argv[0];
5169         if (argc > 2) {
5170                 usage();
5171         }
5172         if (argc == 2) {
5173                 if (strcmp(argv[1], "persistent") != 0) {
5174                         usage();
5175                 }
5176                 persistent = true;
5177         }
5178
5179         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
5180         if (ctdb_db == NULL) {
5181                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
5182                 return -1;
5183         }
5184
5185         return 0;
5186 }
5187
5188 /*
5189  * detach from a database
5190  */
5191 static int control_detach(struct ctdb_context *ctdb, int argc,
5192                           const char **argv)
5193 {
5194         uint32_t db_id;
5195         uint8_t flags;
5196         int ret, i, status = 0;
5197         struct ctdb_node_map *nodemap = NULL;
5198         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5199         uint32_t recmode;
5200
5201         if (argc < 1) {
5202                 usage();
5203         }
5204
5205         assert_single_node_only();
5206
5207         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn,
5208                                     &recmode);
5209         if (ret != 0) {
5210                 DEBUG(DEBUG_ERR, ("Database cannot be detached "
5211                                   "when recovery is active\n"));
5212                 talloc_free(tmp_ctx);
5213                 return -1;
5214         }
5215
5216         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5217                                    &nodemap);
5218         if (ret != 0) {
5219                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5220                                   options.pnn));
5221                 talloc_free(tmp_ctx);
5222                 return -1;
5223         }
5224
5225         for (i=0; i<nodemap->num; i++) {
5226                 uint32_t value;
5227
5228                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
5229                         continue;
5230                 }
5231
5232                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
5233                         continue;
5234                 }
5235
5236                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
5237                         DEBUG(DEBUG_ERR, ("Database cannot be detached on "
5238                                           "inactive (stopped or banned) node "
5239                                           "%u\n", nodemap->nodes[i].pnn));
5240                         talloc_free(tmp_ctx);
5241                         return -1;
5242                 }
5243
5244                 ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(),
5245                                             nodemap->nodes[i].pnn,
5246                                             "AllowClientDBAttach",
5247                                             &value);
5248                 if (ret != 0) {
5249                         DEBUG(DEBUG_ERR, ("Unable to get tunable "
5250                                           "AllowClientDBAttach from node %u\n",
5251                                            nodemap->nodes[i].pnn));
5252                         talloc_free(tmp_ctx);
5253                         return -1;
5254                 }
5255
5256                 if (value == 1) {
5257                         DEBUG(DEBUG_ERR, ("Database access is still active on "
5258                                           "node %u. Set AllowClientDBAttach=0 "
5259                                           "on all nodes.\n",
5260                                           nodemap->nodes[i].pnn));
5261                         talloc_free(tmp_ctx);
5262                         return -1;
5263                 }
5264         }
5265
5266         talloc_free(tmp_ctx);
5267
5268         for (i=0; i<argc; i++) {
5269                 if (!db_exists(ctdb, argv[i], &db_id, NULL, &flags)) {
5270                         continue;
5271                 }
5272
5273                 if (flags & CTDB_DB_FLAGS_PERSISTENT) {
5274                         DEBUG(DEBUG_ERR, ("Persistent database '%s' "
5275                                           "cannot be detached\n", argv[i]));
5276                         status = -1;
5277                         continue;
5278                 }
5279
5280                 ret = ctdb_detach(ctdb, db_id);
5281                 if (ret != 0) {
5282                         DEBUG(DEBUG_ERR, ("Database '%s' detach failed\n",
5283                                           argv[i]));
5284                         status = ret;
5285                 }
5286         }
5287
5288         return status;
5289 }
5290
5291 /*
5292   set db priority
5293  */
5294 static int control_setdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5295 {
5296         struct ctdb_db_priority db_prio;
5297         int ret;
5298
5299         if (argc < 2) {
5300                 usage();
5301         }
5302
5303         db_prio.db_id    = strtoul(argv[0], NULL, 0);
5304         db_prio.priority = strtoul(argv[1], NULL, 0);
5305
5306         ret = ctdb_ctrl_set_db_priority(ctdb, TIMELIMIT(), options.pnn, &db_prio);
5307         if (ret != 0) {
5308                 DEBUG(DEBUG_ERR,("Unable to set db prio\n"));
5309                 return -1;
5310         }
5311
5312         return 0;
5313 }
5314
5315 /*
5316   get db priority
5317  */
5318 static int control_getdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5319 {
5320         uint32_t db_id, priority;
5321         int ret;
5322
5323         if (argc < 1) {
5324                 usage();
5325         }
5326
5327         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5328                 return -1;
5329         }
5330
5331         ret = ctdb_ctrl_get_db_priority(ctdb, TIMELIMIT(), options.pnn, db_id, &priority);
5332         if (ret != 0) {
5333                 DEBUG(DEBUG_ERR,("Unable to get db prio\n"));
5334                 return -1;
5335         }
5336
5337         DEBUG(DEBUG_ERR,("Priority:%u\n", priority));
5338
5339         return 0;
5340 }
5341
5342 /*
5343   set the sticky records capability for a database
5344  */
5345 static int control_setdbsticky(struct ctdb_context *ctdb, int argc, const char **argv)
5346 {
5347         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5348         uint32_t db_id;
5349         int ret;
5350
5351         if (argc < 1) {
5352                 usage();
5353         }
5354
5355         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5356                 return -1;
5357         }
5358
5359         ret = ctdb_ctrl_set_db_sticky(ctdb, options.pnn, db_id);
5360         if (ret != 0) {
5361                 DEBUG(DEBUG_ERR,("Unable to set db to support sticky records\n"));
5362                 talloc_free(tmp_ctx);
5363                 return -1;
5364         }
5365
5366         talloc_free(tmp_ctx);
5367         return 0;
5368 }
5369
5370 /*
5371   set the readonly capability for a database
5372  */
5373 static int control_setdbreadonly(struct ctdb_context *ctdb, int argc, const char **argv)
5374 {
5375         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5376         uint32_t db_id;
5377         int ret;
5378
5379         if (argc < 1) {
5380                 usage();
5381         }
5382
5383         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5384                 return -1;
5385         }
5386
5387         ret = ctdb_ctrl_set_db_readonly(ctdb, options.pnn, db_id);
5388         if (ret != 0) {
5389                 DEBUG(DEBUG_ERR,("Unable to set db to support readonly\n"));
5390                 talloc_free(tmp_ctx);
5391                 return -1;
5392         }
5393
5394         talloc_free(tmp_ctx);
5395         return 0;
5396 }
5397
5398 /*
5399   get db seqnum
5400  */
5401 static int control_getdbseqnum(struct ctdb_context *ctdb, int argc, const char **argv)
5402 {
5403         uint32_t db_id;
5404         uint64_t seqnum;
5405         int ret;
5406
5407         if (argc < 1) {
5408                 usage();
5409         }
5410
5411         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5412                 return -1;
5413         }
5414
5415         ret = ctdb_ctrl_getdbseqnum(ctdb, TIMELIMIT(), options.pnn, db_id, &seqnum);
5416         if (ret != 0) {
5417                 DEBUG(DEBUG_ERR, ("Unable to get seqnum from node."));
5418                 return -1;
5419         }
5420
5421         printf("Sequence number:%lld\n", (long long)seqnum);
5422
5423         return 0;
5424 }
5425
5426 /*
5427   run an eventscript on a node
5428  */
5429 static int control_eventscript(struct ctdb_context *ctdb, int argc, const char **argv)
5430 {
5431         TDB_DATA data;
5432         int ret;
5433         int32_t res;
5434         char *errmsg;
5435         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5436
5437         if (argc != 1) {
5438                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5439                 return -1;
5440         }
5441
5442         data.dptr = (unsigned char *)discard_const(argv[0]);
5443         data.dsize = strlen((char *)data.dptr) + 1;
5444
5445         DEBUG(DEBUG_ERR, ("Running eventscripts with arguments \"%s\" on node %u\n", data.dptr, options.pnn));
5446
5447         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_RUN_EVENTSCRIPTS,
5448                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
5449         if (ret != 0 || res != 0) {
5450                 DEBUG(DEBUG_ERR,("Failed to run eventscripts - %s\n", errmsg));
5451                 talloc_free(tmp_ctx);
5452                 return -1;
5453         }
5454         talloc_free(tmp_ctx);
5455         return 0;
5456 }
5457
5458 #define DB_VERSION 1
5459 #define MAX_DB_NAME 64
5460 struct db_file_header {
5461         unsigned long version;
5462         time_t timestamp;
5463         unsigned long persistent;
5464         unsigned long size;
5465         const char name[MAX_DB_NAME];
5466 };
5467
5468 struct backup_data {
5469         struct ctdb_marshall_buffer *records;
5470         uint32_t len;
5471         uint32_t total;
5472         bool traverse_error;
5473 };
5474
5475 static int backup_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private)
5476 {
5477         struct backup_data *bd = talloc_get_type(private, struct backup_data);
5478         struct ctdb_rec_data *rec;
5479
5480         /* add the record */
5481         rec = ctdb_marshall_record(bd->records, 0, key, NULL, data);
5482         if (rec == NULL) {
5483                 bd->traverse_error = true;
5484                 DEBUG(DEBUG_ERR,("Failed to marshall record\n"));
5485                 return -1;
5486         }
5487         bd->records = talloc_realloc_size(NULL, bd->records, rec->length + bd->len);
5488         if (bd->records == NULL) {
5489                 DEBUG(DEBUG_ERR,("Failed to expand marshalling buffer\n"));
5490                 bd->traverse_error = true;
5491                 return -1;
5492         }
5493         bd->records->count++;
5494         memcpy(bd->len+(uint8_t *)bd->records, rec, rec->length);
5495         bd->len += rec->length;
5496         talloc_free(rec);
5497
5498         bd->total++;
5499         return 0;
5500 }
5501
5502 /*
5503  * backup a database to a file
5504  */
5505 static int control_backupdb(struct ctdb_context *ctdb, int argc, const char **argv)
5506 {
5507         const char *db_name;
5508         int ret;
5509         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5510         struct db_file_header dbhdr;
5511         struct ctdb_db_context *ctdb_db;
5512         struct backup_data *bd;
5513         int fh = -1;
5514         int status = -1;
5515         const char *reason = NULL;
5516         uint32_t db_id;
5517         uint8_t flags;
5518
5519         assert_single_node_only();
5520
5521         if (argc != 2) {
5522                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5523                 return -1;
5524         }
5525
5526         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
5527                 return -1;
5528         }
5529
5530         ret = ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
5531                                     db_id, tmp_ctx, &reason);
5532         if (ret != 0) {
5533                 DEBUG(DEBUG_ERR,("Unable to get dbhealth for database '%s'\n",
5534                                  argv[0]));
5535                 talloc_free(tmp_ctx);
5536                 return -1;
5537         }
5538         if (reason) {
5539                 uint32_t allow_unhealthy = 0;
5540
5541                 ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn,
5542                                       "AllowUnhealthyDBRead",
5543                                       &allow_unhealthy);
5544
5545                 if (allow_unhealthy != 1) {
5546                         DEBUG(DEBUG_ERR,("database '%s' is unhealthy: %s\n",
5547                                          argv[0], reason));
5548
5549                         DEBUG(DEBUG_ERR,("disallow backup : tunable AllowUnhealthyDBRead = %u\n",
5550                                          allow_unhealthy));
5551                         talloc_free(tmp_ctx);
5552                         return -1;
5553                 }
5554
5555                 DEBUG(DEBUG_WARNING,("WARNING database '%s' is unhealthy - see 'ctdb getdbstatus %s'\n",
5556                                      argv[0], argv[0]));
5557                 DEBUG(DEBUG_WARNING,("WARNING! allow backup of unhealthy database: "
5558                                      "tunnable AllowUnhealthyDBRead = %u\n",
5559                                      allow_unhealthy));
5560         }
5561
5562         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5563         if (ctdb_db == NULL) {
5564                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", argv[0]));
5565                 talloc_free(tmp_ctx);
5566                 return -1;
5567         }
5568
5569
5570         ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
5571         if (ret == -1) {
5572                 DEBUG(DEBUG_ERR,("Failed to start transaction\n"));
5573                 talloc_free(tmp_ctx);
5574                 return -1;
5575         }
5576
5577
5578         bd = talloc_zero(tmp_ctx, struct backup_data);
5579         if (bd == NULL) {
5580                 DEBUG(DEBUG_ERR,("Failed to allocate backup_data\n"));
5581                 talloc_free(tmp_ctx);
5582                 return -1;
5583         }
5584
5585         bd->records = talloc_zero(bd, struct ctdb_marshall_buffer);
5586         if (bd->records == NULL) {
5587                 DEBUG(DEBUG_ERR,("Failed to allocate ctdb_marshall_buffer\n"));
5588                 talloc_free(tmp_ctx);
5589                 return -1;
5590         }
5591
5592         bd->len = offsetof(struct ctdb_marshall_buffer, data);
5593         bd->records->db_id = ctdb_db->db_id;
5594         /* traverse the database collecting all records */
5595         if (tdb_traverse_read(ctdb_db->ltdb->tdb, backup_traverse, bd) == -1 ||
5596             bd->traverse_error) {
5597                 DEBUG(DEBUG_ERR,("Traverse error\n"));
5598                 talloc_free(tmp_ctx);
5599                 return -1;
5600         }
5601
5602         tdb_transaction_cancel(ctdb_db->ltdb->tdb);
5603
5604
5605         fh = open(argv[1], O_RDWR|O_CREAT, 0600);
5606         if (fh == -1) {
5607                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[1]));
5608                 talloc_free(tmp_ctx);
5609                 return -1;
5610         }
5611
5612         ZERO_STRUCT(dbhdr);
5613         dbhdr.version = DB_VERSION;
5614         dbhdr.timestamp = time(NULL);
5615         dbhdr.persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
5616         dbhdr.size = bd->len;
5617         if (strlen(argv[0]) >= MAX_DB_NAME) {
5618                 DEBUG(DEBUG_ERR,("Too long dbname\n"));
5619                 goto done;
5620         }
5621         strncpy(discard_const(dbhdr.name), argv[0], MAX_DB_NAME-1);
5622         ret = write(fh, &dbhdr, sizeof(dbhdr));
5623         if (ret == -1) {
5624                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5625                 goto done;
5626         }
5627         ret = write(fh, bd->records, bd->len);
5628         if (ret == -1) {
5629                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5630                 goto done;
5631         }
5632
5633         status = 0;
5634 done:
5635         if (fh != -1) {
5636                 ret = close(fh);
5637                 if (ret == -1) {
5638                         DEBUG(DEBUG_ERR,("close failed: %s\n", strerror(errno)));
5639                 }
5640         }
5641
5642         DEBUG(DEBUG_ERR,("Database backed up to %s\n", argv[1]));
5643
5644         talloc_free(tmp_ctx);
5645         return status;
5646 }
5647
5648 /*
5649  * restore a database from a file
5650  */
5651 static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **argv)
5652 {
5653         int ret;
5654         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5655         TDB_DATA outdata;
5656         TDB_DATA data;
5657         struct db_file_header dbhdr;
5658         struct ctdb_db_context *ctdb_db;
5659         struct ctdb_node_map *nodemap=NULL;
5660         struct ctdb_vnn_map *vnnmap=NULL;
5661         int i, fh;
5662         struct ctdb_control_wipe_database w;
5663         uint32_t *nodes;
5664         uint32_t generation;
5665         struct tm *tm;
5666         char tbuf[100];
5667         char *dbname;
5668
5669         assert_single_node_only();
5670
5671         if (argc < 1 || argc > 2) {
5672                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5673                 return -1;
5674         }
5675
5676         fh = open(argv[0], O_RDONLY);
5677         if (fh == -1) {
5678                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5679                 talloc_free(tmp_ctx);
5680                 return -1;
5681         }
5682
5683         read(fh, &dbhdr, sizeof(dbhdr));
5684         if (dbhdr.version != DB_VERSION) {
5685                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5686                 close(fh);
5687                 talloc_free(tmp_ctx);
5688                 return -1;
5689         }
5690
5691         dbname = discard_const(dbhdr.name);
5692         if (argc == 2) {
5693                 dbname = discard_const(argv[1]);
5694         }
5695
5696         outdata.dsize = dbhdr.size;
5697         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5698         if (outdata.dptr == NULL) {
5699                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5700                 close(fh);
5701                 talloc_free(tmp_ctx);
5702                 return -1;
5703         }
5704         read(fh, outdata.dptr, outdata.dsize);
5705         close(fh);
5706
5707         tm = localtime(&dbhdr.timestamp);
5708         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5709         printf("Restoring database '%s' from backup @ %s\n",
5710                 dbname, tbuf);
5711
5712
5713         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), dbname, dbhdr.persistent, 0);
5714         if (ctdb_db == NULL) {
5715                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", dbname));
5716                 talloc_free(tmp_ctx);
5717                 return -1;
5718         }
5719
5720         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
5721         if (ret != 0) {
5722                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
5723                 talloc_free(tmp_ctx);
5724                 return ret;
5725         }
5726
5727
5728         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
5729         if (ret != 0) {
5730                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
5731                 talloc_free(tmp_ctx);
5732                 return ret;
5733         }
5734
5735         /* freeze all nodes */
5736         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5737         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5738                 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5739                                         nodes, i,
5740                                         TIMELIMIT(),
5741                                         false, tdb_null,
5742                                         NULL, NULL,
5743                                         NULL) != 0) {
5744                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5745                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5746                         talloc_free(tmp_ctx);
5747                         return -1;
5748                 }
5749         }
5750
5751         generation = vnnmap->generation;
5752         data.dptr = (void *)&generation;
5753         data.dsize = sizeof(generation);
5754
5755         /* start a cluster wide transaction */
5756         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5757         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
5758                                         nodes, 0,
5759                                         TIMELIMIT(), false, data,
5760                                         NULL, NULL,
5761                                         NULL) != 0) {
5762                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide transactions.\n"));
5763                 return -1;
5764         }
5765
5766
5767         w.db_id = ctdb_db->db_id;
5768         w.transaction_id = generation;
5769
5770         data.dptr = (void *)&w;
5771         data.dsize = sizeof(w);
5772
5773         /* wipe all the remote databases. */
5774         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5775         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
5776                                         nodes, 0,
5777                                         TIMELIMIT(), false, data,
5778                                         NULL, NULL,
5779                                         NULL) != 0) {
5780                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
5781                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5782                 talloc_free(tmp_ctx);
5783                 return -1;
5784         }
5785
5786         /* push the database */
5787         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5788         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
5789                                         nodes, 0,
5790                                         TIMELIMIT(), false, outdata,
5791                                         NULL, NULL,
5792                                         NULL) != 0) {
5793                 DEBUG(DEBUG_ERR, ("Failed to push database.\n"));
5794                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5795                 talloc_free(tmp_ctx);
5796                 return -1;
5797         }
5798
5799         data.dptr = (void *)&ctdb_db->db_id;
5800         data.dsize = sizeof(ctdb_db->db_id);
5801
5802         /* mark the database as healthy */
5803         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5804         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
5805                                         nodes, 0,
5806                                         TIMELIMIT(), false, data,
5807                                         NULL, NULL,
5808                                         NULL) != 0) {
5809                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
5810                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5811                 talloc_free(tmp_ctx);
5812                 return -1;
5813         }
5814
5815         data.dptr = (void *)&generation;
5816         data.dsize = sizeof(generation);
5817
5818         /* commit all the changes */
5819         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
5820                                         nodes, 0,
5821                                         TIMELIMIT(), false, data,
5822                                         NULL, NULL,
5823                                         NULL) != 0) {
5824                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
5825                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5826                 talloc_free(tmp_ctx);
5827                 return -1;
5828         }
5829
5830
5831         /* thaw all nodes */
5832         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5833         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
5834                                         nodes, 0,
5835                                         TIMELIMIT(),
5836                                         false, tdb_null,
5837                                         NULL, NULL,
5838                                         NULL) != 0) {
5839                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
5840                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5841                 talloc_free(tmp_ctx);
5842                 return -1;
5843         }
5844
5845
5846         talloc_free(tmp_ctx);
5847         return 0;
5848 }
5849
5850 /*
5851  * dump a database backup from a file
5852  */
5853 static int control_dumpdbbackup(struct ctdb_context *ctdb, int argc, const char **argv)
5854 {
5855         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5856         TDB_DATA outdata;
5857         struct db_file_header dbhdr;
5858         int i, fh;
5859         struct tm *tm;
5860         char tbuf[100];
5861         struct ctdb_rec_data *rec = NULL;
5862         struct ctdb_marshall_buffer *m;
5863         struct ctdb_dump_db_context c;
5864
5865         assert_single_node_only();
5866
5867         if (argc != 1) {
5868                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5869                 return -1;
5870         }
5871
5872         fh = open(argv[0], O_RDONLY);
5873         if (fh == -1) {
5874                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5875                 talloc_free(tmp_ctx);
5876                 return -1;
5877         }
5878
5879         read(fh, &dbhdr, sizeof(dbhdr));
5880         if (dbhdr.version != DB_VERSION) {
5881                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5882                 close(fh);
5883                 talloc_free(tmp_ctx);
5884                 return -1;
5885         }
5886
5887         outdata.dsize = dbhdr.size;
5888         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5889         if (outdata.dptr == NULL) {
5890                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5891                 close(fh);
5892                 talloc_free(tmp_ctx);
5893                 return -1;
5894         }
5895         read(fh, outdata.dptr, outdata.dsize);
5896         close(fh);
5897         m = (struct ctdb_marshall_buffer *)outdata.dptr;
5898
5899         tm = localtime(&dbhdr.timestamp);
5900         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5901         printf("Backup of database name:'%s' dbid:0x%x08x from @ %s\n",
5902                 dbhdr.name, m->db_id, tbuf);
5903
5904         ZERO_STRUCT(c);
5905         c.f = stdout;
5906         c.printemptyrecords = (bool)options.printemptyrecords;
5907         c.printdatasize = (bool)options.printdatasize;
5908         c.printlmaster = false;
5909         c.printhash = (bool)options.printhash;
5910         c.printrecordflags = (bool)options.printrecordflags;
5911
5912         for (i=0; i < m->count; i++) {
5913                 uint32_t reqid = 0;
5914                 TDB_DATA key, data;
5915
5916                 /* we do not want the header splitted, so we pass NULL*/
5917                 rec = ctdb_marshall_loop_next(m, rec, &reqid,
5918                                               NULL, &key, &data);
5919
5920                 ctdb_dumpdb_record(ctdb, key, data, &c);
5921         }
5922
5923         printf("Dumped %d records\n", i);
5924         talloc_free(tmp_ctx);
5925         return 0;
5926 }
5927
5928 /*
5929  * wipe a database from a file
5930  */
5931 static int control_wipedb(struct ctdb_context *ctdb, int argc,
5932                           const char **argv)
5933 {
5934         const char *db_name;
5935         int ret;
5936         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5937         TDB_DATA data;
5938         struct ctdb_db_context *ctdb_db;
5939         struct ctdb_node_map *nodemap = NULL;
5940         struct ctdb_vnn_map *vnnmap = NULL;
5941         int i;
5942         struct ctdb_control_wipe_database w;
5943         uint32_t *nodes;
5944         uint32_t generation;
5945         uint8_t flags;
5946
5947         assert_single_node_only();
5948
5949         if (argc != 1) {
5950                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5951                 return -1;
5952         }
5953
5954         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
5955                 return -1;
5956         }
5957
5958         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5959         if (ctdb_db == NULL) {
5960                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n",
5961                                   argv[0]));
5962                 talloc_free(tmp_ctx);
5963                 return -1;
5964         }
5965
5966         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb,
5967                                    &nodemap);
5968         if (ret != 0) {
5969                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5970                                   options.pnn));
5971                 talloc_free(tmp_ctx);
5972                 return ret;
5973         }
5974
5975         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5976                                   &vnnmap);
5977         if (ret != 0) {
5978                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
5979                                   options.pnn));
5980                 talloc_free(tmp_ctx);
5981                 return ret;
5982         }
5983
5984         /* freeze all nodes */
5985         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5986         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5987                 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5988                                                 nodes, i,
5989                                                 TIMELIMIT(),
5990                                                 false, tdb_null,
5991                                                 NULL, NULL,
5992                                                 NULL);
5993                 if (ret != 0) {
5994                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5995                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn,
5996                                              CTDB_RECOVERY_ACTIVE);
5997                         talloc_free(tmp_ctx);
5998                         return -1;
5999                 }
6000         }
6001
6002         generation = vnnmap->generation;
6003         data.dptr = (void *)&generation;
6004         data.dsize = sizeof(generation);
6005
6006         /* start a cluster wide transaction */
6007         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
6008         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
6009                                         nodes, 0,
6010                                         TIMELIMIT(), false, data,
6011                                         NULL, NULL,
6012                                         NULL);
6013         if (ret!= 0) {
6014                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide "
6015                                   "transactions.\n"));
6016                 return -1;
6017         }
6018
6019         w.db_id = ctdb_db->db_id;
6020         w.transaction_id = generation;
6021
6022         data.dptr = (void *)&w;
6023         data.dsize = sizeof(w);
6024
6025         /* wipe all the remote databases. */
6026         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
6027         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
6028                                         nodes, 0,
6029                                         TIMELIMIT(), false, data,
6030                                         NULL, NULL,
6031                                         NULL) != 0) {
6032                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
6033                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
6034                 talloc_free(tmp_ctx);
6035                 return -1;
6036         }
6037
6038         data.dptr = (void *)&ctdb_db->db_id;
6039         data.dsize = sizeof(ctdb_db->db_id);
6040
6041         /* mark the database as healthy */
6042         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
6043         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
6044                                         nodes, 0,
6045                                         TIMELIMIT(), false, data,
6046                                         NULL, NULL,
6047                                         NULL) != 0) {
6048                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
6049                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
6050                 talloc_free(tmp_ctx);
6051                 return -1;
6052         }
6053
6054         data.dptr = (void *)&generation;
6055         data.dsize = sizeof(generation);
6056
6057         /* commit all the changes */
6058         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
6059                                         nodes, 0,
6060                                         TIMELIMIT(), false, data,
6061                                         NULL, NULL,
6062                                         NULL) != 0) {
6063                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
6064                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
6065                 talloc_free(tmp_ctx);
6066                 return -1;
6067         }
6068
6069         /* thaw all nodes */
6070         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
6071         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
6072                                         nodes, 0,
6073                                         TIMELIMIT(),
6074                                         false, tdb_null,
6075                                         NULL, NULL,
6076                                         NULL) != 0) {
6077                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
6078                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
6079                 talloc_free(tmp_ctx);
6080                 return -1;
6081         }
6082
6083         DEBUG(DEBUG_ERR, ("Database wiped.\n"));
6084
6085         talloc_free(tmp_ctx);
6086         return 0;
6087 }
6088
6089 /*
6090   dump memory usage
6091  */
6092 static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
6093 {
6094         TDB_DATA data;
6095         int ret;
6096         int32_t res;
6097         char *errmsg;
6098         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
6099         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_DUMP_MEMORY,
6100                            0, tdb_null, tmp_ctx, &data, &res, NULL, &errmsg);
6101         if (ret != 0 || res != 0) {
6102                 DEBUG(DEBUG_ERR,("Failed to dump memory - %s\n", errmsg));
6103                 talloc_free(tmp_ctx);
6104                 return -1;
6105         }
6106         write(1, data.dptr, data.dsize);
6107         talloc_free(tmp_ctx);
6108         return 0;
6109 }
6110
6111 /*
6112   handler for memory dumps
6113 */
6114 static void mem_dump_handler(struct ctdb_context *ctdb, uint64_t srvid,
6115                              TDB_DATA data, void *private_data)
6116 {
6117         write(1, data.dptr, data.dsize);
6118         exit(0);
6119 }
6120
6121 /*
6122   dump memory usage on the recovery daemon
6123  */
6124 static int control_rddumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
6125 {
6126         int ret;
6127         TDB_DATA data;
6128         struct srvid_request rd;
6129
6130         rd.pnn = ctdb_get_pnn(ctdb);
6131         rd.srvid = getpid();
6132
6133         /* register a message port for receiveing the reply so that we
6134            can receive the reply
6135         */
6136         ctdb_client_set_message_handler(ctdb, rd.srvid, mem_dump_handler, NULL);
6137
6138
6139         data.dptr = (uint8_t *)&rd;
6140         data.dsize = sizeof(rd);
6141
6142         ret = ctdb_client_send_message(ctdb, options.pnn, CTDB_SRVID_MEM_DUMP, data);
6143         if (ret != 0) {
6144                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
6145                 return -1;
6146         }
6147
6148         /* this loop will terminate when we have received the reply */
6149         while (1) {
6150                 event_loop_once(ctdb->ev);
6151         }
6152
6153         return 0;
6154 }
6155
6156 /*
6157   send a message to a srvid
6158  */
6159 static int control_msgsend(struct ctdb_context *ctdb, int argc, const char **argv)
6160 {
6161         unsigned long srvid;
6162         int ret;
6163         TDB_DATA data;
6164
6165         if (argc < 2) {
6166                 usage();
6167         }
6168
6169         srvid      = strtoul(argv[0], NULL, 0);
6170
6171         data.dptr = (uint8_t *)discard_const(argv[1]);
6172         data.dsize= strlen(argv[1]);
6173
6174         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, srvid, data);
6175         if (ret != 0) {
6176                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
6177                 return -1;
6178         }
6179
6180         return 0;
6181 }
6182
6183 /*
6184   handler for msglisten
6185 */
6186 static void msglisten_handler(struct ctdb_context *ctdb, uint64_t srvid,
6187                              TDB_DATA data, void *private_data)
6188 {
6189         int i;
6190
6191         printf("Message received: ");
6192         for (i=0;i<data.dsize;i++) {
6193                 printf("%c", data.dptr[i]);
6194         }
6195         printf("\n");
6196 }
6197
6198 /*
6199   listen for messages on a messageport
6200  */
6201 static int control_msglisten(struct ctdb_context *ctdb, int argc, const char **argv)
6202 {
6203         uint64_t srvid;
6204
6205         srvid = getpid();
6206
6207         /* register a message port and listen for messages
6208         */
6209         ctdb_client_set_message_handler(ctdb, srvid, msglisten_handler, NULL);
6210         printf("Listening for messages on srvid:%d\n", (int)srvid);
6211
6212         while (1) {
6213                 event_loop_once(ctdb->ev);
6214         }
6215
6216         return 0;
6217 }
6218
6219 /*
6220   list all nodes in the cluster
6221   we parse the nodes file directly
6222  */
6223 static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
6224 {
6225         TALLOC_CTX *mem_ctx = talloc_new(NULL);
6226         struct pnn_node *pnn_nodes;
6227         struct pnn_node *pnn_node;
6228
6229         assert_single_node_only();
6230
6231         pnn_nodes = read_nodes_file(mem_ctx);
6232         if (pnn_nodes == NULL) {
6233                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
6234                 talloc_free(mem_ctx);
6235                 return -1;
6236         }
6237
6238         for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
6239                 const char *addr = ctdb_addr_to_str(&pnn_node->addr);
6240                 if (options.machinereadable){
6241                         printf(":%d:%s:\n", pnn_node->pnn, addr);
6242                 } else {
6243                         printf("%s\n", addr);
6244                 }
6245         }
6246         talloc_free(mem_ctx);
6247
6248         return 0;
6249 }
6250
6251 /*
6252   reload the nodes file on the local node
6253  */
6254 static int control_reload_nodes_file(struct ctdb_context *ctdb, int argc, const char **argv)
6255 {
6256         int i, ret;
6257         int mypnn;
6258         struct ctdb_node_map *nodemap=NULL;
6259
6260         assert_single_node_only();
6261
6262         mypnn = ctdb_get_pnn(ctdb);
6263
6264         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
6265         if (ret != 0) {
6266                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
6267                 return ret;
6268         }
6269
6270         /* reload the nodes file on all remote nodes */
6271         for (i=0;i<nodemap->num;i++) {
6272                 if (nodemap->nodes[i].pnn == mypnn) {
6273                         continue;
6274                 }
6275                 DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", nodemap->nodes[i].pnn));
6276                 ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(),
6277                         nodemap->nodes[i].pnn);
6278                 if (ret != 0) {
6279                         DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", nodemap->nodes[i].pnn));
6280                 }
6281         }
6282
6283         /* reload the nodes file on the local node */
6284         DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", mypnn));
6285         ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(), mypnn);
6286         if (ret != 0) {
6287                 DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", mypnn));
6288         }
6289
6290         /* initiate a recovery */
6291         control_recover(ctdb, argc, argv);
6292
6293         return 0;
6294 }
6295
6296
6297 static const struct {
6298         const char *name;
6299         int (*fn)(struct ctdb_context *, int, const char **);
6300         bool auto_all;
6301         bool without_daemon; /* can be run without daemon running ? */
6302         const char *msg;
6303         const char *args;
6304 } ctdb_commands[] = {
6305         { "version",         control_version,           true,   true,   "show version of ctdb" },
6306         { "status",          control_status,            true,   false,  "show node status" },
6307         { "uptime",          control_uptime,            true,   false,  "show node uptime" },
6308         { "ping",            control_ping,              true,   false,  "ping all nodes" },
6309         { "runstate",        control_runstate,          true,   false,  "get/check runstate of a node", "[setup|first_recovery|startup|running]" },
6310         { "getvar",          control_getvar,            true,   false,  "get a tunable variable",               "<name>"},
6311         { "setvar",          control_setvar,            true,   false,  "set a tunable variable",               "<name> <value>"},
6312         { "listvars",        control_listvars,          true,   false,  "list tunable variables"},
6313         { "statistics",      control_statistics,        false,  false, "show statistics" },
6314         { "statisticsreset", control_statistics_reset,  true,   false,  "reset statistics"},
6315         { "stats",           control_stats,             false,  false,  "show rolling statistics", "[number of history records]" },
6316         { "ip",              control_ip,                false,  false,  "show which public ip's that ctdb manages" },
6317         { "ipinfo",          control_ipinfo,            true,   false,  "show details about a public ip that ctdb manages", "<ip>" },
6318         { "ifaces",          control_ifaces,            true,   false,  "show which interfaces that ctdb manages" },
6319         { "setifacelink",    control_setifacelink,      true,   false,  "set interface link status", "<iface> <status>" },
6320         { "process-exists",  control_process_exists,    true,   false,  "check if a process exists on a node",  "<pid>"},
6321         { "getdbmap",        control_getdbmap,          true,   false,  "show the database map" },
6322         { "getdbstatus",     control_getdbstatus,       true,   false,  "show the status of a database", "<dbname|dbid>" },
6323         { "catdb",           control_catdb,             true,   false,  "dump a ctdb database" ,                     "<dbname|dbid>"},
6324         { "cattdb",          control_cattdb,            true,   false,  "dump a local tdb database" ,                     "<dbname|dbid>"},
6325         { "getmonmode",      control_getmonmode,        true,   false,  "show monitoring mode" },
6326         { "getcapabilities", control_getcapabilities,   true,   false,  "show node capabilities" },
6327         { "pnn",             control_pnn,               true,   false,  "show the pnn of the currnet node" },
6328         { "lvs",             control_lvs,               true,   false,  "show lvs configuration" },
6329         { "lvsmaster",       control_lvsmaster,         true,   false,  "show which node is the lvs master" },
6330         { "disablemonitor",      control_disable_monmode,true,  false,  "set monitoring mode to DISABLE" },
6331         { "enablemonitor",      control_enable_monmode, true,   false,  "set monitoring mode to ACTIVE" },
6332         { "setdebug",        control_setdebug,          true,   false,  "set debug level",                      "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
6333         { "getdebug",        control_getdebug,          true,   false,  "get debug level" },
6334         { "getlog",          control_getlog,            true,   false,  "get the log data from the in memory ringbuffer", "[<level>] [recoverd]" },
6335         { "clearlog",          control_clearlog,        true,   false,  "clear the log data from the in memory ringbuffer", "[recoverd]" },
6336         { "attach",          control_attach,            true,   false,  "attach to a database",                 "<dbname> [persistent]" },
6337         { "detach",          control_detach,            false,  false,  "detach from a database",                 "<dbname|dbid> [<dbname|dbid> ...]" },
6338         { "dumpmemory",      control_dumpmemory,        true,   false,  "dump memory map to stdout" },
6339         { "rddumpmemory",    control_rddumpmemory,      true,   false,  "dump memory map from the recovery daemon to stdout" },
6340         { "getpid",          control_getpid,            true,   false,  "get ctdbd process ID" },
6341         { "disable",         control_disable,           true,   false,  "disable a nodes public IP" },
6342         { "enable",          control_enable,            true,   false,  "enable a nodes public IP" },
6343         { "stop",            control_stop,              true,   false,  "stop a node" },
6344         { "continue",        control_continue,          true,   false,  "re-start a stopped node" },
6345         { "ban",             control_ban,               true,   false,  "ban a node from the cluster",          "<bantime>"},
6346         { "unban",           control_unban,             true,   false,  "unban a node" },
6347         { "showban",         control_showban,           true,   false,  "show ban information"},
6348         { "shutdown",        control_shutdown,          true,   false,  "shutdown ctdbd" },
6349         { "recover",         control_recover,           true,   false,  "force recovery" },
6350         { "sync",            control_ipreallocate,      false,  false,  "wait until ctdbd has synced all state changes" },
6351         { "ipreallocate",    control_ipreallocate,      false,  false,  "force the recovery daemon to perform a ip reallocation procedure" },
6352         { "thaw",            control_thaw,              true,   false,  "thaw databases", "[priority:1-3]" },
6353         { "isnotrecmaster",  control_isnotrecmaster,    false,  false,  "check if the local node is recmaster or not" },
6354         { "killtcp",         kill_tcp,                  false,  false, "kill a tcp connection.", "[<srcip:port> <dstip:port>]" },
6355         { "gratiousarp",     control_gratious_arp,      false,  false, "send a gratious arp", "<ip> <interface>" },
6356         { "tickle",          tickle_tcp,                false,  false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
6357         { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip> [<port>]" },
6358         { "addtickle",       control_add_tickle,        false,  false, "add a tickle for this ip", "<ip>:<port> <ip>:<port>" },
6359
6360         { "deltickle",       control_del_tickle,        false,  false, "delete a tickle from this ip", "<ip>:<port> <ip>:<port>" },
6361
6362         { "regsrvid",        regsrvid,                  false,  false, "register a server id", "<pnn> <type> <id>" },
6363         { "unregsrvid",      unregsrvid,                false,  false, "unregister a server id", "<pnn> <type> <id>" },
6364         { "chksrvid",        chksrvid,                  false,  false, "check if a server id exists", "<pnn> <type> <id>" },
6365         { "getsrvids",       getsrvids,                 false,  false, "get a list of all server ids"},
6366         { "check_srvids",    check_srvids,              false,  false, "check if a srvid exists", "<id>+" },
6367         { "repack",          ctdb_repack,               false,  false, "repack all databases", "[max_freelist]"},
6368         { "listnodes",       control_listnodes,         false,  true, "list all nodes in the cluster"},
6369         { "reloadnodes",     control_reload_nodes_file, false,  false, "reload the nodes file and restart the transport on all nodes"},
6370         { "moveip",          control_moveip,            false,  false, "move/failover an ip address to another node", "<ip> <node>"},
6371         { "rebalanceip",     control_rebalanceip,       false,  false, "release an ip from the node and let recd rebalance it", "<ip>"},
6372         { "addip",           control_addip,             true,   false, "add a ip address to a node", "<ip/mask> <iface>"},
6373         { "delip",           control_delip,             false,  false, "delete an ip address from a node", "<ip>"},
6374         { "eventscript",     control_eventscript,       true,   false, "run the eventscript with the given parameters on a node", "<arguments>"},
6375         { "backupdb",        control_backupdb,          false,  false, "backup the database into a file.", "<dbname|dbid> <file>"},
6376         { "restoredb",        control_restoredb,        false,  false, "restore the database from a file.", "<file> [dbname]"},
6377         { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
6378         { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname|dbid>"},
6379         { "recmaster",        control_recmaster,        true,   false, "show the pnn for the recovery master."},
6380         { "scriptstatus",     control_scriptstatus,     true,   false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
6381         { "enablescript",     control_enablescript,  true,      false, "enable an eventscript", "<script>"},
6382         { "disablescript",    control_disablescript,  true,     false, "disable an eventscript", "<script>"},
6383         { "natgwlist",        control_natgwlist,        true,   false, "show the nodes belonging to this natgw configuration"},
6384         { "xpnn",             control_xpnn,             false,  true,  "find the pnn of the local node without talking to the daemon (unreliable)" },
6385         { "getreclock",       control_getreclock,       true,   false, "Show the reclock file of a node"},
6386         { "setreclock",       control_setreclock,       true,   false, "Set/clear the reclock file of a node", "[filename]"},
6387         { "setnatgwstate",    control_setnatgwstate,    false,  false, "Set NATGW state to on/off", "{on|off}"},
6388         { "setlmasterrole",   control_setlmasterrole,   false,  false, "Set LMASTER role to on/off", "{on|off}"},
6389         { "setrecmasterrole", control_setrecmasterrole, false,  false, "Set RECMASTER role to on/off", "{on|off}"},
6390         { "setdbprio",        control_setdbprio,        false,  false, "Set DB priority", "<dbname|dbid> <prio:1-3>"},
6391         { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbname|dbid>"},
6392         { "setdbreadonly",    control_setdbreadonly,    false,  false, "Set DB readonly capable", "<dbname|dbid>"},
6393         { "setdbsticky",      control_setdbsticky,      false,  false, "Set DB sticky-records capable", "<dbname|dbid>"},
6394         { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
6395         { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
6396         { "pfetch",          control_pfetch,            false,  false,  "fetch a record from a persistent database", "<dbname|dbid> <key> [<file>]" },
6397         { "pstore",          control_pstore,            false,  false,  "write a record to a persistent database", "<dbname|dbid> <key> <file containing record>" },
6398         { "pdelete",         control_pdelete,           false,  false,  "delete a record from a persistent database", "<dbname|dbid> <key>" },
6399         { "ptrans",          control_ptrans,            false,  false,  "update a persistent database (from stdin)", "<dbname|dbid>" },
6400         { "tfetch",          control_tfetch,            false,  true,  "fetch a record from a [c]tdb-file [-v]", "<tdb-file> <key> [<file>]" },
6401         { "tstore",          control_tstore,            false,  true,  "store a record (including ltdb header)", "<tdb-file> <key> <data> [<rsn> <dmaster> <flags>]" },
6402         { "readkey",         control_readkey,           true,   false,  "read the content off a database key", "<dbname|dbid> <key>" },
6403         { "writekey",        control_writekey,          true,   false,  "write to a database key", "<dbname|dbid> <key> <value>" },
6404         { "checktcpport",    control_chktcpport,        false,  true,  "check if a service is bound to a specific tcp port or not", "<port>" },
6405         { "rebalancenode",     control_rebalancenode,   false,  false, "mark nodes as forced IP rebalancing targets", "[<pnn-list>]"},
6406         { "getdbseqnum",     control_getdbseqnum,       false,  false, "get the sequence number off a database", "<dbname|dbid>" },
6407         { "nodestatus",      control_nodestatus,        true,   false,  "show and return node status", "[<pnn-list>]" },
6408         { "dbstatistics",    control_dbstatistics,      false,  false, "show db statistics", "<dbname|dbid>" },
6409         { "reloadips",       control_reloadips,         false,  false, "reload the public addresses file on specified nodes" , "[<pnn-list>]" },
6410         { "ipiface",         control_ipiface,           false,  true,  "Find which interface an ip address is hosted on", "<ip>" },
6411 };
6412
6413 /*
6414   show usage message
6415  */
6416 static void usage(void)
6417 {
6418         int i;
6419         printf(
6420 "Usage: ctdb [options] <control>\n" \
6421 "Options:\n" \
6422 "   -n <node>          choose node number, or 'all' (defaults to local node)\n"
6423 "   -Y                 generate machinereadable output\n"
6424 "   -v                 generate verbose output\n"
6425 "   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
6426         printf("Controls:\n");
6427         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6428                 printf("  %-15s %-27s  %s\n",
6429                        ctdb_commands[i].name,
6430                        ctdb_commands[i].args?ctdb_commands[i].args:"",
6431                        ctdb_commands[i].msg);
6432         }
6433         exit(1);
6434 }
6435
6436
6437 static void ctdb_alarm(int sig)
6438 {
6439         printf("Maximum runtime exceeded - exiting\n");
6440         _exit(ERR_TIMEOUT);
6441 }
6442
6443 /*
6444   main program
6445 */
6446 int main(int argc, const char *argv[])
6447 {
6448         struct ctdb_context *ctdb;
6449         char *nodestring = NULL;
6450         struct poptOption popt_options[] = {
6451                 POPT_AUTOHELP
6452                 POPT_CTDB_CMDLINE
6453                 { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
6454                 { "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
6455                 { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machinereadable output", NULL },
6456                 { "verbose",    'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL },
6457                 { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" },
6458                 { "print-emptyrecords", 0, POPT_ARG_NONE, &options.printemptyrecords, 0, "print the empty records when dumping databases (catdb, cattdb, dumpdbbackup)", NULL },
6459                 { "print-datasize", 0, POPT_ARG_NONE, &options.printdatasize, 0, "do not print record data when dumping databases, only the data size", NULL },
6460                 { "print-lmaster", 0, POPT_ARG_NONE, &options.printlmaster, 0, "print the record's lmaster in catdb", NULL },
6461                 { "print-hash", 0, POPT_ARG_NONE, &options.printhash, 0, "print the record's hash when dumping databases", NULL },
6462                 { "print-recordflags", 0, POPT_ARG_NONE, &options.printrecordflags, 0, "print the record flags in catdb and dumpdbbackup", NULL },
6463                 POPT_TABLEEND
6464         };
6465         int opt;
6466         const char **extra_argv;
6467         int extra_argc = 0;
6468         int ret=-1, i;
6469         poptContext pc;
6470         struct event_context *ev;
6471         const char *control;
6472
6473         setlinebuf(stdout);
6474
6475         /* set some defaults */
6476         options.maxruntime = 0;
6477         options.timelimit = 10;
6478         options.pnn = CTDB_CURRENT_NODE;
6479
6480         pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
6481
6482         while ((opt = poptGetNextOpt(pc)) != -1) {
6483                 switch (opt) {
6484                 default:
6485                         DEBUG(DEBUG_ERR, ("Invalid option %s: %s\n",
6486                                 poptBadOption(pc, 0), poptStrerror(opt)));
6487                         exit(1);
6488                 }
6489         }
6490
6491         /* setup the remaining options for the main program to use */
6492         extra_argv = poptGetArgs(pc);
6493         if (extra_argv) {
6494                 extra_argv++;
6495                 while (extra_argv[extra_argc]) extra_argc++;
6496         }
6497
6498         if (extra_argc < 1) {
6499                 usage();
6500         }
6501
6502         if (options.maxruntime == 0) {
6503                 const char *ctdb_timeout;
6504                 ctdb_timeout = getenv("CTDB_TIMEOUT");
6505                 if (ctdb_timeout != NULL) {
6506                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
6507                 } else {
6508                         /* default timeout is 120 seconds */
6509                         options.maxruntime = 120;
6510                 }
6511         }
6512
6513         signal(SIGALRM, ctdb_alarm);
6514         alarm(options.maxruntime);
6515
6516         control = extra_argv[0];
6517
6518         /* Default value for CTDB_BASE - don't override */
6519         setenv("CTDB_BASE", ETCDIR "/ctdb", 0);
6520
6521         ev = event_context_init(NULL);
6522         if (!ev) {
6523                 DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
6524                 exit(1);
6525         }
6526
6527         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6528                 if (strcmp(control, ctdb_commands[i].name) == 0) {
6529                         break;
6530                 }
6531         }
6532
6533         if (i == ARRAY_SIZE(ctdb_commands)) {
6534                 DEBUG(DEBUG_ERR, ("Unknown control '%s'\n", control));
6535                 exit(1);
6536         }
6537
6538         if (ctdb_commands[i].without_daemon == true) {
6539                 if (nodestring != NULL) {
6540                         DEBUG(DEBUG_ERR, ("Can't specify node(s) with \"ctdb %s\"\n", control));
6541                         exit(1);
6542                 }
6543                 return ctdb_commands[i].fn(NULL, extra_argc-1, extra_argv+1);
6544         }
6545
6546         /* initialise ctdb */
6547         ctdb = ctdb_cmdline_client(ev, TIMELIMIT());
6548
6549         if (ctdb == NULL) {
6550                 DEBUG(DEBUG_ERR, ("Failed to init ctdb\n"));
6551                 exit(1);
6552         }
6553
6554         /* setup the node number(s) to contact */
6555         if (!parse_nodestring(ctdb, ctdb, nodestring, CTDB_CURRENT_NODE, false,
6556                               &options.nodes, &options.pnn)) {
6557                 usage();
6558         }
6559
6560         if (options.pnn == CTDB_CURRENT_NODE) {
6561                 options.pnn = options.nodes[0];
6562         }
6563
6564         if (ctdb_commands[i].auto_all &&
6565             ((options.pnn == CTDB_BROADCAST_ALL) ||
6566              (options.pnn == CTDB_MULTICAST))) {
6567                 int j;
6568
6569                 ret = 0;
6570                 for (j = 0; j < talloc_array_length(options.nodes); j++) {
6571                         options.pnn = options.nodes[j];
6572                         ret |= ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6573                 }
6574         } else {
6575                 ret = ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6576         }
6577
6578         talloc_free(ctdb);
6579         talloc_free(ev);
6580         (void)poptFreeContext(pc);
6581
6582         return ret;
6583
6584 }