ctdb/tools/ctdb.c

   1 /*
   2    ctdb control tool
   3
   4    Copyright (C) Andrew Tridgell  2007
   5    Copyright (C) Ronnie Sahlberg  2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "includes.h"
  22 #include "system/time.h"
  23 #include "system/filesys.h"
  24 #include "system/network.h"
  25 #include "system/locale.h"
  26 #include "popt.h"
  27 #include "cmdline.h"
  28 #include "../include/ctdb_version.h"
  29 #include "../include/ctdb_client.h"
  30 #include "../include/ctdb_private.h"
  31 #include "../common/rb_tree.h"
  32 #include "lib/tdb_wrap/tdb_wrap.h"
  33 #include "lib/util/dlinklist.h"
  34
  35 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
  36 #define ERR_NONODE      21      /* node does not exist */
  37 #define ERR_DISNODE     22      /* node is disconnected */
  38
  39 static void usage(void);
  40
  41 static struct {
  42         int timelimit;
  43         uint32_t pnn;
  44         uint32_t *nodes;
  45         int machinereadable;
  46         int verbose;
  47         int maxruntime;
  48         int printemptyrecords;
  49         int printdatasize;
  50         int printlmaster;
  51         int printhash;
  52         int printrecordflags;
  53 } options;
  54
  55 #define LONGTIMEOUT options.timelimit*10
  56
  57 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
  58 #define LONGTIMELIMIT() timeval_current_ofs(LONGTIMEOUT, 0)
  59
  60 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
  61 {
  62         return (tv2->tv_sec - tv->tv_sec) +
  63                 (tv2->tv_usec - tv->tv_usec)*1.0e-6;
  64 }
  65
  66 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
  67 {
  68         printf("CTDB version: %s\n", CTDB_VERSION_STRING);
  69         return 0;
  70 }
  71
  72 #define CTDB_NOMEM_ABORT(p) do { if (!(p)) {                            \
  73                 DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n",            \
  74                                    "Out of memory in " __location__ )); \
  75                 abort();                                                \
  76         }} while (0)
  77
  78 static uint32_t getpnn(struct ctdb_context *ctdb)
  79 {
  80         if ((options.pnn == CTDB_BROADCAST_ALL) ||
  81             (options.pnn == CTDB_MULTICAST)) {
  82                 DEBUG(DEBUG_ERR,
  83                       ("Cannot get PNN for node %u\n", options.pnn));
  84                 exit(1);
  85         }
  86
  87         if (options.pnn == CTDB_CURRENT_NODE) {
  88                 return ctdb_get_pnn(ctdb);
  89         } else {
  90                 return options.pnn;
  91         }
  92 }
  93
  94 static void assert_single_node_only(void)
  95 {
  96         if ((options.pnn == CTDB_BROADCAST_ALL) ||
  97             (options.pnn == CTDB_MULTICAST)) {
  98                 DEBUG(DEBUG_ERR,
  99                       ("This control can not be applied to multiple PNNs\n"));
 100                 exit(1);
 101         }
 102 }
 103
 104 /* Pretty print the flags to a static buffer in human-readable format.
 105  * This never returns NULL!
 106  */
 107 static const char *pretty_print_flags(uint32_t flags)
 108 {
 109         int j;
 110         static const struct {
 111                 uint32_t flag;
 112                 const char *name;
 113         } flag_names[] = {
 114                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 115                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 116                 { NODE_FLAGS_BANNED,                "BANNED" },
 117                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 118                 { NODE_FLAGS_DELETED,               "DELETED" },
 119                 { NODE_FLAGS_STOPPED,               "STOPPED" },
 120                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 121         };
 122         static char flags_str[512]; /* Big enough to contain all flag names */
 123
 124         flags_str[0] = '\0';
 125         for (j=0;j<ARRAY_SIZE(flag_names);j++) {
 126                 if (flags & flag_names[j].flag) {
 127                         if (flags_str[0] == '\0') {
 128                                 (void) strcpy(flags_str, flag_names[j].name);
 129                         } else {
 130                                 (void) strncat(flags_str, "|", sizeof(flags_str)-1);
 131                                 (void) strncat(flags_str, flag_names[j].name,
 132                                                sizeof(flags_str)-1);
 133                         }
 134                 }
 135         }
 136         if (flags_str[0] == '\0') {
 137                 (void) strcpy(flags_str, "OK");
 138         }
 139
 140         return flags_str;
 141 }
 142
 143 static int h2i(char h)
 144 {
 145         if (h >= 'a' && h <= 'f') return h - 'a' + 10;
 146         if (h >= 'A' && h <= 'F') return h - 'f' + 10;
 147         return h - '0';
 148 }
 149
 150 static TDB_DATA hextodata(TALLOC_CTX *mem_ctx, const char *str)
 151 {
 152         int i, len;
 153         TDB_DATA key = {NULL, 0};
 154
 155         len = strlen(str);
 156         if (len & 0x01) {
 157                 DEBUG(DEBUG_ERR,("Key specified with odd number of hexadecimal digits\n"));
 158                 return key;
 159         }
 160
 161         key.dsize = len>>1;
 162         key.dptr  = talloc_size(mem_ctx, key.dsize);
 163
 164         for (i=0; i < len/2; i++) {
 165                 key.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
 166         }
 167         return key;
 168 }
 169
 170 /* Parse a nodestring.  Parameter dd_ok controls what happens to nodes
 171  * that are disconnected or deleted.  If dd_ok is true those nodes are
 172  * included in the output list of nodes.  If dd_ok is false, those
 173  * nodes are filtered from the "all" case and cause an error if
 174  * explicitly specified.
 175  */
 176 static bool parse_nodestring(struct ctdb_context *ctdb,
 177                              TALLOC_CTX *mem_ctx,
 178                              const char * nodestring,
 179                              uint32_t current_pnn,
 180                              bool dd_ok,
 181                              uint32_t **nodes,
 182                              uint32_t *pnn_mode)
 183 {
 184         TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
 185         int n;
 186         uint32_t i;
 187         struct ctdb_node_map *nodemap;
 188         int ret;
 189
 190         *nodes = NULL;
 191
 192         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
 193         if (ret != 0) {
 194                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
 195                 talloc_free(tmp_ctx);
 196                 exit(10);
 197         }
 198
 199         if (nodestring != NULL) {
 200                 *nodes = talloc_array(mem_ctx, uint32_t, 0);
 201                 if (*nodes == NULL) {
 202                         goto failed;
 203                 }
 204
 205                 n = 0;
 206
 207                 if (strcmp(nodestring, "all") == 0) {
 208                         *pnn_mode = CTDB_BROADCAST_ALL;
 209
 210                         /* all */
 211                         for (i = 0; i < nodemap->num; i++) {
 212                                 if ((nodemap->nodes[i].flags &
 213                                      (NODE_FLAGS_DISCONNECTED |
 214                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 215                                         continue;
 216                                 }
 217                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 218                                                         uint32_t, n+1);
 219                                 if (*nodes == NULL) {
 220                                         goto failed;
 221                                 }
 222                                 (*nodes)[n] = i;
 223                                 n++;
 224                         }
 225                 } else {
 226                         /* x{,y...} */
 227                         char *ns, *tok;
 228
 229                         ns = talloc_strdup(tmp_ctx, nodestring);
 230                         tok = strtok(ns, ",");
 231                         while (tok != NULL) {
 232                                 uint32_t pnn;
 233                                 char *endptr;
 234                                 i = (uint32_t)strtoul(tok, &endptr, 0);
 235                                 if (i == 0 && tok == endptr) {
 236                                         DEBUG(DEBUG_ERR,
 237                                               ("Invalid node %s\n", tok));
 238                                         talloc_free(tmp_ctx);
 239                                         exit(ERR_NONODE);
 240                                 }
 241                                 if (i >= nodemap->num) {
 242                                         DEBUG(DEBUG_ERR, ("Node %u does not exist\n", i));
 243                                         talloc_free(tmp_ctx);
 244                                         exit(ERR_NONODE);
 245                                 }
 246                                 if ((nodemap->nodes[i].flags &
 247                                      (NODE_FLAGS_DISCONNECTED |
 248                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 249                                         DEBUG(DEBUG_ERR, ("Node %u has status %s\n", i, pretty_print_flags(nodemap->nodes[i].flags)));
 250                                         talloc_free(tmp_ctx);
 251                                         exit(ERR_DISNODE);
 252                                 }
 253                                 if ((pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), i)) < 0) {
 254                                         DEBUG(DEBUG_ERR, ("Can not access node %u. Node is not operational.\n", i));
 255                                         talloc_free(tmp_ctx);
 256                                         exit(10);
 257                                 }
 258
 259                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 260                                                         uint32_t, n+1);
 261                                 if (*nodes == NULL) {
 262                                         goto failed;
 263                                 }
 264
 265                                 (*nodes)[n] = i;
 266                                 n++;
 267
 268                                 tok = strtok(NULL, ",");
 269                         }
 270                         talloc_free(ns);
 271
 272                         if (n == 1) {
 273                                 *pnn_mode = (*nodes)[0];
 274                         } else {
 275                                 *pnn_mode = CTDB_MULTICAST;
 276                         }
 277                 }
 278         } else {
 279                 /* default - no nodes specified */
 280                 *nodes = talloc_array(mem_ctx, uint32_t, 1);
 281                 if (*nodes == NULL) {
 282                         goto failed;
 283                 }
 284                 *pnn_mode = CTDB_CURRENT_NODE;
 285
 286                 if (((*nodes)[0] = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), current_pnn)) < 0) {
 287                         goto failed;
 288                 }
 289         }
 290
 291         talloc_free(tmp_ctx);
 292         return true;
 293
 294 failed:
 295         talloc_free(tmp_ctx);
 296         return false;
 297 }
 298
 299 /*
 300  check if a database exists
 301 */
 302 static bool db_exists(struct ctdb_context *ctdb, const char *dbarg,
 303                       uint32_t *dbid, const char **dbname, uint8_t *flags)
 304 {
 305         int i, ret;
 306         struct ctdb_dbid_map *dbmap=NULL;
 307         bool dbid_given = false, found = false;
 308         uint32_t id;
 309         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 310         const char *name;
 311
 312         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
 313         if (ret != 0) {
 314                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
 315                 goto fail;
 316         }
 317
 318         if (strncmp(dbarg, "0x", 2) == 0) {
 319                 id = strtoul(dbarg, NULL, 0);
 320                 dbid_given = true;
 321         }
 322
 323         for(i=0; i<dbmap->num; i++) {
 324                 if (dbid_given) {
 325                         if (id == dbmap->dbs[i].dbid) {
 326                                 found = true;
 327                                 break;
 328                         }
 329                 } else {
 330                         ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
 331                         if (ret != 0) {
 332                                 DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
 333                                 goto fail;
 334                         }
 335
 336                         if (strcmp(name, dbarg) == 0) {
 337                                 id = dbmap->dbs[i].dbid;
 338                                 found = true;
 339                                 break;
 340                         }
 341                 }
 342         }
 343
 344         if (found && dbid_given && dbname != NULL) {
 345                 ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
 346                 if (ret != 0) {
 347                         DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
 348                         found = false;
 349                         goto fail;
 350                 }
 351         }
 352
 353         if (found) {
 354                 if (dbid) *dbid = id;
 355                 if (dbname) *dbname = talloc_strdup(ctdb, name);
 356                 if (flags) *flags = dbmap->dbs[i].flags;
 357         } else {
 358                 DEBUG(DEBUG_ERR,("No database matching '%s' found\n", dbarg));
 359         }
 360
 361 fail:
 362         talloc_free(tmp_ctx);
 363         return found;
 364 }
 365
 366 /*
 367   see if a process exists
 368  */
 369 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
 370 {
 371         uint32_t pnn, pid;
 372         int ret;
 373         if (argc < 1) {
 374                 usage();
 375         }
 376
 377         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
 378                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
 379                 return -1;
 380         }
 381
 382         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
 383         if (ret == 0) {
 384                 printf("%u:%u exists\n", pnn, pid);
 385         } else {
 386                 printf("%u:%u does not exist\n", pnn, pid);
 387         }
 388         return ret;
 389 }
 390
 391 /*
 392   display statistics structure
 393  */
 394 static void show_statistics(struct ctdb_statistics *s, int show_header)
 395 {
 396         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 397         int i;
 398         const char *prefix=NULL;
 399         int preflen=0;
 400         int tmp, days, hours, minutes, seconds;
 401         const struct {
 402                 const char *name;
 403                 uint32_t offset;
 404         } fields[] = {
 405 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
 406                 STATISTICS_FIELD(num_clients),
 407                 STATISTICS_FIELD(frozen),
 408                 STATISTICS_FIELD(recovering),
 409                 STATISTICS_FIELD(num_recoveries),
 410                 STATISTICS_FIELD(client_packets_sent),
 411                 STATISTICS_FIELD(client_packets_recv),
 412                 STATISTICS_FIELD(node_packets_sent),
 413                 STATISTICS_FIELD(node_packets_recv),
 414                 STATISTICS_FIELD(keepalive_packets_sent),
 415                 STATISTICS_FIELD(keepalive_packets_recv),
 416                 STATISTICS_FIELD(node.req_call),
 417                 STATISTICS_FIELD(node.reply_call),
 418                 STATISTICS_FIELD(node.req_dmaster),
 419                 STATISTICS_FIELD(node.reply_dmaster),
 420                 STATISTICS_FIELD(node.reply_error),
 421                 STATISTICS_FIELD(node.req_message),
 422                 STATISTICS_FIELD(node.req_control),
 423                 STATISTICS_FIELD(node.reply_control),
 424                 STATISTICS_FIELD(client.req_call),
 425                 STATISTICS_FIELD(client.req_message),
 426                 STATISTICS_FIELD(client.req_control),
 427                 STATISTICS_FIELD(timeouts.call),
 428                 STATISTICS_FIELD(timeouts.control),
 429                 STATISTICS_FIELD(timeouts.traverse),
 430                 STATISTICS_FIELD(locks.num_calls),
 431                 STATISTICS_FIELD(locks.num_current),
 432                 STATISTICS_FIELD(locks.num_pending),
 433                 STATISTICS_FIELD(locks.num_failed),
 434                 STATISTICS_FIELD(total_calls),
 435                 STATISTICS_FIELD(pending_calls),
 436                 STATISTICS_FIELD(childwrite_calls),
 437                 STATISTICS_FIELD(pending_childwrite_calls),
 438                 STATISTICS_FIELD(memory_used),
 439                 STATISTICS_FIELD(max_hop_count),
 440                 STATISTICS_FIELD(total_ro_delegations),
 441                 STATISTICS_FIELD(total_ro_revokes),
 442         };
 443
 444         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
 445         seconds = tmp%60;
 446         tmp    /= 60;
 447         minutes = tmp%60;
 448         tmp    /= 60;
 449         hours   = tmp%24;
 450         tmp    /= 24;
 451         days    = tmp;
 452
 453         if (options.machinereadable){
 454                 if (show_header) {
 455                         printf("CTDB version:");
 456                         printf("Current time of statistics:");
 457                         printf("Statistics collected since:");
 458                         for (i=0;i<ARRAY_SIZE(fields);i++) {
 459                                 printf("%s:", fields[i].name);
 460                         }
 461                         printf("num_reclock_ctdbd_latency:");
 462                         printf("min_reclock_ctdbd_latency:");
 463                         printf("avg_reclock_ctdbd_latency:");
 464                         printf("max_reclock_ctdbd_latency:");
 465
 466                         printf("num_reclock_recd_latency:");
 467                         printf("min_reclock_recd_latency:");
 468                         printf("avg_reclock_recd_latency:");
 469                         printf("max_reclock_recd_latency:");
 470
 471                         printf("num_call_latency:");
 472                         printf("min_call_latency:");
 473                         printf("avg_call_latency:");
 474                         printf("max_call_latency:");
 475
 476                         printf("num_lockwait_latency:");
 477                         printf("min_lockwait_latency:");
 478                         printf("avg_lockwait_latency:");
 479                         printf("max_lockwait_latency:");
 480
 481                         printf("num_childwrite_latency:");
 482                         printf("min_childwrite_latency:");
 483                         printf("avg_childwrite_latency:");
 484                         printf("max_childwrite_latency:");
 485                         printf("\n");
 486                 }
 487                 printf("%d:", CTDB_VERSION);
 488                 printf("%d:", (int)s->statistics_current_time.tv_sec);
 489                 printf("%d:", (int)s->statistics_start_time.tv_sec);
 490                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 491                         printf("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 492                 }
 493                 printf("%d:", s->reclock.ctdbd.num);
 494                 printf("%.6f:", s->reclock.ctdbd.min);
 495                 printf("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
 496                 printf("%.6f:", s->reclock.ctdbd.max);
 497
 498                 printf("%d:", s->reclock.recd.num);
 499                 printf("%.6f:", s->reclock.recd.min);
 500                 printf("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
 501                 printf("%.6f:", s->reclock.recd.max);
 502
 503                 printf("%d:", s->call_latency.num);
 504                 printf("%.6f:", s->call_latency.min);
 505                 printf("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
 506                 printf("%.6f:", s->call_latency.max);
 507
 508                 printf("%d:", s->childwrite_latency.num);
 509                 printf("%.6f:", s->childwrite_latency.min);
 510                 printf("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
 511                 printf("%.6f:", s->childwrite_latency.max);
 512                 printf("\n");
 513         } else {
 514                 printf("CTDB version %u\n", CTDB_VERSION);
 515                 printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
 516                 printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
 517
 518                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 519                         if (strchr(fields[i].name, '.')) {
 520                                 preflen = strcspn(fields[i].name, ".")+1;
 521                                 if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
 522                                         prefix = fields[i].name;
 523                                         printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
 524                                 }
 525                         } else {
 526                                 preflen = 0;
 527                         }
 528                         printf(" %*s%-22s%*s%10u\n",
 529                                preflen?4:0, "",
 530                                fields[i].name+preflen,
 531                                preflen?0:4, "",
 532                                *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 533                 }
 534                 printf(" hop_count_buckets:");
 535                 for (i=0;i<MAX_COUNT_BUCKETS;i++) {
 536                         printf(" %d", s->hop_count_bucket[i]);
 537                 }
 538                 printf("\n");
 539                 printf(" lock_buckets:");
 540                 for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 541                         printf(" %d", s->locks.buckets[i]);
 542                 }
 543                 printf("\n");
 544                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
 545
 546                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
 547
 548                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
 549
 550                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
 551                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
 552         }
 553
 554         talloc_free(tmp_ctx);
 555 }
 556
 557 /*
 558   display remote ctdb statistics combined from all nodes
 559  */
 560 static int control_statistics_all(struct ctdb_context *ctdb)
 561 {
 562         int ret, i;
 563         struct ctdb_statistics statistics;
 564         uint32_t *nodes;
 565         uint32_t num_nodes;
 566
 567         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
 568         CTDB_NO_MEMORY(ctdb, nodes);
 569
 570         ZERO_STRUCT(statistics);
 571
 572         for (i=0;i<num_nodes;i++) {
 573                 struct ctdb_statistics s1;
 574                 int j;
 575                 uint32_t *v1 = (uint32_t *)&s1;
 576                 uint32_t *v2 = (uint32_t *)&statistics;
 577                 uint32_t num_ints =
 578                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
 579                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
 580                 if (ret != 0) {
 581                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
 582                         return ret;
 583                 }
 584                 for (j=0;j<num_ints;j++) {
 585                         v2[j] += v1[j];
 586                 }
 587                 statistics.max_hop_count =
 588                         MAX(statistics.max_hop_count, s1.max_hop_count);
 589                 statistics.call_latency.max =
 590                         MAX(statistics.call_latency.max, s1.call_latency.max);
 591         }
 592         talloc_free(nodes);
 593         printf("Gathered statistics for %u nodes\n", num_nodes);
 594         show_statistics(&statistics, 1);
 595         return 0;
 596 }
 597
 598 /*
 599   display remote ctdb statistics
 600  */
 601 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
 602 {
 603         int ret;
 604         struct ctdb_statistics statistics;
 605
 606         if (options.pnn == CTDB_BROADCAST_ALL) {
 607                 return control_statistics_all(ctdb);
 608         }
 609
 610         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
 611         if (ret != 0) {
 612                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
 613                 return ret;
 614         }
 615         show_statistics(&statistics, 1);
 616         return 0;
 617 }
 618
 619
 620 /*
 621   reset remote ctdb statistics
 622  */
 623 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
 624 {
 625         int ret;
 626
 627         ret = ctdb_statistics_reset(ctdb, options.pnn);
 628         if (ret != 0) {
 629                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
 630                 return ret;
 631         }
 632         return 0;
 633 }
 634
 635
 636 /*
 637   display remote ctdb rolling statistics
 638  */
 639 static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
 640 {
 641         int ret;
 642         struct ctdb_statistics_wire *stats;
 643         int i, num_records = -1;
 644
 645         assert_single_node_only();
 646
 647         if (argc ==1) {
 648                 num_records = atoi(argv[0]) - 1;
 649         }
 650
 651         ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
 652         if (ret != 0) {
 653                 DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
 654                 return ret;
 655         }
 656         for (i=0;i<stats->num;i++) {
 657                 if (stats->stats[i].statistics_start_time.tv_sec == 0) {
 658                         continue;
 659                 }
 660                 show_statistics(&stats->stats[i], i==0);
 661                 if (i == num_records) {
 662                         break;
 663                 }
 664         }
 665         return 0;
 666 }
 667
 668
 669 /*
 670   display remote ctdb db statistics
 671  */
 672 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
 673 {
 674         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 675         struct ctdb_db_statistics *dbstat;
 676         int i;
 677         uint32_t db_id;
 678         int num_hot_keys;
 679         int ret;
 680
 681         if (argc < 1) {
 682                 usage();
 683         }
 684
 685         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
 686                 return -1;
 687         }
 688
 689         ret = ctdb_ctrl_dbstatistics(ctdb, options.pnn, db_id, tmp_ctx, &dbstat);
 690         if (ret != 0) {
 691                 DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
 692                 talloc_free(tmp_ctx);
 693                 return -1;
 694         }
 695
 696         printf("DB Statistics: %s\n", argv[0]);
 697         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
 698                 dbstat->db_ro_delegations);
 699         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
 700                 dbstat->db_ro_delegations);
 701         printf(" %s\n", "locks");
 702         printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
 703                 dbstat->locks.num_calls);
 704         printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
 705                 dbstat->locks.num_failed);
 706         printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
 707                 dbstat->locks.num_current);
 708         printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
 709                 dbstat->locks.num_pending);
 710         printf(" %s", "hop_count_buckets:");
 711         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 712                 printf(" %d", dbstat->hop_count_bucket[i]);
 713         }
 714         printf("\n");
 715         printf(" %s", "lock_buckets:");
 716         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 717                 printf(" %d", dbstat->locks.buckets[i]);
 718         }
 719         printf("\n");
 720         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
 721                 "locks_latency      MIN/AVG/MAX",
 722                 dbstat->locks.latency.min,
 723                 (dbstat->locks.latency.num ?
 724                  dbstat->locks.latency.total /dbstat->locks.latency.num :
 725                  0.0),
 726                 dbstat->locks.latency.max,
 727                 dbstat->locks.latency.num);
 728         num_hot_keys = 0;
 729         for (i=0; i<dbstat->num_hot_keys; i++) {
 730                 if (dbstat->hot_keys[i].count > 0) {
 731                         num_hot_keys++;
 732                 }
 733         }
 734         dbstat->num_hot_keys = num_hot_keys;
 735
 736         printf(" Num Hot Keys:     %d\n", dbstat->num_hot_keys);
 737         for (i = 0; i < dbstat->num_hot_keys; i++) {
 738                 int j;
 739                 printf("     Count:%d Key:", dbstat->hot_keys[i].count);
 740                 for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
 741                         printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
 742                 }
 743                 printf("\n");
 744         }
 745
 746         talloc_free(tmp_ctx);
 747         return 0;
 748 }
 749
 750 /*
 751   display uptime of remote node
 752  */
 753 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
 754 {
 755         int ret;
 756         struct ctdb_uptime *uptime = NULL;
 757         int tmp, days, hours, minutes, seconds;
 758
 759         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
 760         if (ret != 0) {
 761                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
 762                 return ret;
 763         }
 764
 765         if (options.machinereadable){
 766                 printf(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
 767                 printf(":%u:%u:%u:%lf\n",
 768                         (unsigned int)uptime->current_time.tv_sec,
 769                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
 770                         (unsigned int)uptime->last_recovery_finished.tv_sec,
 771                         timeval_delta(&uptime->last_recovery_finished,
 772                                       &uptime->last_recovery_started)
 773                 );
 774                 return 0;
 775         }
 776
 777         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
 778
 779         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 780         seconds = tmp%60;
 781         tmp    /= 60;
 782         minutes = tmp%60;
 783         tmp    /= 60;
 784         hours   = tmp%24;
 785         tmp    /= 24;
 786         days    = tmp;
 787         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
 788
 789         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 790         seconds = tmp%60;
 791         tmp    /= 60;
 792         minutes = tmp%60;
 793         tmp    /= 60;
 794         hours   = tmp%24;
 795         tmp    /= 24;
 796         days    = tmp;
 797         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
 798
 799         printf("Duration of last recovery/failover: %lf seconds\n",
 800                 timeval_delta(&uptime->last_recovery_finished,
 801                               &uptime->last_recovery_started));
 802
 803         return 0;
 804 }
 805
 806 /*
 807   show the PNN of the current node
 808  */
 809 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
 810 {
 811         uint32_t mypnn;
 812
 813         mypnn = getpnn(ctdb);
 814
 815         printf("PNN:%d\n", mypnn);
 816         return 0;
 817 }
 818
 819
 820 struct pnn_node {
 821         struct pnn_node *next, *prev;
 822         ctdb_sock_addr addr;
 823         int pnn;
 824 };
 825
 826 static struct pnn_node *read_pnn_node_file(TALLOC_CTX *mem_ctx,
 827                                            const char *file)
 828 {
 829         int nlines;
 830         char **lines;
 831         int i, pnn;
 832         struct pnn_node *pnn_nodes = NULL;
 833         struct pnn_node *pnn_node;
 834
 835         lines = file_lines_load(file, &nlines, 0, mem_ctx);
 836         if (lines == NULL) {
 837                 return NULL;
 838         }
 839         for (i=0, pnn=0; i<nlines; i++) {
 840                 char *node;
 841
 842                 node = lines[i];
 843                 /* strip leading spaces */
 844                 while((*node == ' ') || (*node == '\t')) {
 845                         node++;
 846                 }
 847                 if (*node == '#') {
 848                         pnn++;
 849                         continue;
 850                 }
 851                 if (strcmp(node, "") == 0) {
 852                         continue;
 853                 }
 854                 pnn_node = talloc(mem_ctx, struct pnn_node);
 855                 pnn_node->pnn = pnn++;
 856
 857                 if (!parse_ip(node, NULL, 0, &pnn_node->addr)) {
 858                         DEBUG(DEBUG_ERR,
 859                               ("Invalid IP address '%s' in file %s\n",
 860                                node, file));
 861                         /* Caller will free mem_ctx */
 862                         return NULL;
 863                 }
 864
 865                 DLIST_ADD_END(pnn_nodes, pnn_node, NULL);
 866         }
 867
 868         return pnn_nodes;
 869 }
 870
 871 static struct pnn_node *read_nodes_file(TALLOC_CTX *mem_ctx)
 872 {
 873         const char *nodes_list;
 874
 875         /* read the nodes file */
 876         nodes_list = getenv("CTDB_NODES");
 877         if (nodes_list == NULL) {
 878                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes",
 879                                              getenv("CTDB_BASE"));
 880                 if (nodes_list == NULL) {
 881                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
 882                         exit(1);
 883                 }
 884         }
 885
 886         return read_pnn_node_file(mem_ctx, nodes_list);
 887 }
 888
 889 /*
 890   show the PNN of the current node
 891   discover the pnn by loading the nodes file and try to bind to all
 892   addresses one at a time until the ip address is found.
 893  */
 894 static int find_node_xpnn(void)
 895 {
 896         TALLOC_CTX *mem_ctx = talloc_new(NULL);
 897         struct pnn_node *pnn_nodes;
 898         struct pnn_node *pnn_node;
 899         int pnn;
 900
 901         pnn_nodes = read_nodes_file(mem_ctx);
 902         if (pnn_nodes == NULL) {
 903                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
 904                 talloc_free(mem_ctx);
 905                 return -1;
 906         }
 907
 908         for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
 909                 if (ctdb_sys_have_ip(&pnn_node->addr)) {
 910                         pnn = pnn_node->pnn;
 911                         talloc_free(mem_ctx);
 912                         return pnn;
 913                 }
 914         }
 915
 916         printf("Failed to detect which PNN this node is\n");
 917         talloc_free(mem_ctx);
 918         return -1;
 919 }
 920
 921 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
 922 {
 923         uint32_t pnn;
 924
 925         assert_single_node_only();
 926
 927         pnn = find_node_xpnn();
 928         if (pnn == -1) {
 929                 return -1;
 930         }
 931
 932         printf("PNN:%d\n", pnn);
 933         return 0;
 934 }
 935
 936 /* Helpers for ctdb status
 937  */
 938 static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_flags *node)
 939 {
 940         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 941         int j;
 942         bool ret = false;
 943
 944         if (node->flags == 0) {
 945                 struct ctdb_control_get_ifaces *ifaces;
 946
 947                 if (ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), node->pnn,
 948                                          tmp_ctx, &ifaces) == 0) {
 949                         for (j=0; j < ifaces->num; j++) {
 950                                 if (ifaces->ifaces[j].link_state != 0) {
 951                                         continue;
 952                                 }
 953                                 ret = true;
 954                                 break;
 955                         }
 956                 }
 957         }
 958         talloc_free(tmp_ctx);
 959
 960         return ret;
 961 }
 962
 963 static void control_status_header_machine(void)
 964 {
 965         printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
 966                ":Inactive:PartiallyOnline:ThisNode:\n");
 967 }
 968
 969 static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn,
 970                                     struct ctdb_node_and_flags *node)
 971 {
 972         printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
 973                ctdb_addr_to_str(&node->addr),
 974                !!(node->flags&NODE_FLAGS_DISCONNECTED),
 975                !!(node->flags&NODE_FLAGS_BANNED),
 976                !!(node->flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 977                !!(node->flags&NODE_FLAGS_UNHEALTHY),
 978                !!(node->flags&NODE_FLAGS_STOPPED),
 979                !!(node->flags&NODE_FLAGS_INACTIVE),
 980                is_partially_online(ctdb, node) ? 1 : 0,
 981                (node->pnn == mypnn)?'Y':'N');
 982
 983         return node->flags;
 984 }
 985
 986 static int control_status_1_human(struct ctdb_context *ctdb, int mypnn,
 987                                   struct ctdb_node_and_flags *node)
 988 {
 989        printf("pnn:%d %-16s %s%s\n", node->pnn,
 990               ctdb_addr_to_str(&node->addr),
 991               is_partially_online(ctdb, node) ? "PARTIALLYONLINE" : pretty_print_flags(node->flags),
 992               node->pnn == mypnn?" (THIS NODE)":"");
 993
 994        return node->flags;
 995 }
 996
 997 /*
 998   display remote ctdb status
 999  */
1000 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
1001 {
1002         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1003         int i;
1004         struct ctdb_vnn_map *vnnmap=NULL;
1005         struct ctdb_node_map *nodemap=NULL;
1006         uint32_t recmode, recmaster, mypnn;
1007         int num_deleted_nodes = 0;
1008         int ret;
1009
1010         mypnn = getpnn(ctdb);
1011
1012         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1013         if (ret != 0) {
1014                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1015                 talloc_free(tmp_ctx);
1016                 return -1;
1017         }
1018
1019         if (options.machinereadable) {
1020                 control_status_header_machine();
1021                 for (i=0;i<nodemap->num;i++) {
1022                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1023                                 continue;
1024                         }
1025                         (void) control_status_1_machine(ctdb, mypnn,
1026                                                         &nodemap->nodes[i]);
1027                 }
1028                 talloc_free(tmp_ctx);
1029                 return 0;
1030         }
1031
1032         for (i=0; i<nodemap->num; i++) {
1033                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1034                         num_deleted_nodes++;
1035                 }
1036         }
1037         if (num_deleted_nodes == 0) {
1038                 printf("Number of nodes:%d\n", nodemap->num);
1039         } else {
1040                 printf("Number of nodes:%d (including %d deleted nodes)\n",
1041                        nodemap->num, num_deleted_nodes);
1042         }
1043         for(i=0;i<nodemap->num;i++){
1044                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1045                         continue;
1046                 }
1047                 (void) control_status_1_human(ctdb, mypnn, &nodemap->nodes[i]);
1048         }
1049
1050         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
1051         if (ret != 0) {
1052                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
1053                 talloc_free(tmp_ctx);
1054                 return -1;
1055         }
1056         if (vnnmap->generation == INVALID_GENERATION) {
1057                 printf("Generation:INVALID\n");
1058         } else {
1059                 printf("Generation:%d\n",vnnmap->generation);
1060         }
1061         printf("Size:%d\n",vnnmap->size);
1062         for(i=0;i<vnnmap->size;i++){
1063                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
1064         }
1065
1066         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmode);
1067         if (ret != 0) {
1068                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
1069                 talloc_free(tmp_ctx);
1070                 return -1;
1071         }
1072         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
1073
1074         ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmaster);
1075         if (ret != 0) {
1076                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1077                 talloc_free(tmp_ctx);
1078                 return -1;
1079         }
1080         printf("Recovery master:%d\n",recmaster);
1081
1082         talloc_free(tmp_ctx);
1083         return 0;
1084 }
1085
1086 static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **argv)
1087 {
1088         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1089         int i, ret;
1090         struct ctdb_node_map *nodemap=NULL;
1091         uint32_t * nodes;
1092         uint32_t pnn_mode, mypnn;
1093
1094         if (argc > 1) {
1095                 usage();
1096         }
1097
1098         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1099                               options.pnn, true, &nodes, &pnn_mode)) {
1100                 return -1;
1101         }
1102
1103         if (options.machinereadable) {
1104                 control_status_header_machine();
1105         } else if (pnn_mode == CTDB_BROADCAST_ALL) {
1106                 printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
1107         }
1108
1109         mypnn = getpnn(ctdb);
1110
1111         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1112         if (ret != 0) {
1113                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1114                 talloc_free(tmp_ctx);
1115                 return -1;
1116         }
1117
1118         ret = 0;
1119
1120         for (i = 0; i < talloc_array_length(nodes); i++) {
1121                 if (options.machinereadable) {
1122                         ret |= control_status_1_machine(ctdb, mypnn,
1123                                                         &nodemap->nodes[nodes[i]]);
1124                 } else {
1125                         ret |= control_status_1_human(ctdb, mypnn,
1126                                                       &nodemap->nodes[nodes[i]]);
1127                 }
1128         }
1129
1130         talloc_free(tmp_ctx);
1131         return ret;
1132 }
1133
1134 static struct pnn_node *read_natgw_nodes_file(struct ctdb_context *ctdb,
1135                                               TALLOC_CTX *mem_ctx)
1136 {
1137         const char *natgw_list;
1138         struct pnn_node *natgw_nodes = NULL;
1139
1140         natgw_list = getenv("CTDB_NATGW_NODES");
1141         if (natgw_list == NULL) {
1142                 natgw_list = talloc_asprintf(mem_ctx, "%s/natgw_nodes",
1143                                              getenv("CTDB_BASE"));
1144                 if (natgw_list == NULL) {
1145                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
1146                         exit(1);
1147                 }
1148         }
1149         /* The PNNs will be junk but they're not used */
1150         natgw_nodes = read_pnn_node_file(mem_ctx, natgw_list);
1151         if (natgw_nodes == NULL) {
1152                 DEBUG(DEBUG_ERR,
1153                       ("Failed to load natgw node list '%s'\n", natgw_list));
1154         }
1155         return natgw_nodes;
1156 }
1157
1158
1159 /* talloc off the existing nodemap... */
1160 static struct ctdb_node_map *talloc_nodemap(struct ctdb_node_map *nodemap)
1161 {
1162         return talloc_zero_size(nodemap,
1163                                 offsetof(struct ctdb_node_map, nodes) +
1164                                 nodemap->num * sizeof(struct ctdb_node_and_flags));
1165 }
1166
1167 static struct ctdb_node_map *
1168 filter_nodemap_by_addrs(struct ctdb_context *ctdb,
1169                         struct ctdb_node_map *nodemap,
1170                         struct pnn_node *nodes)
1171 {
1172         int i;
1173         struct pnn_node *n;
1174         struct ctdb_node_map *ret;
1175
1176         ret = talloc_nodemap(nodemap);
1177         CTDB_NO_MEMORY_NULL(ctdb, ret);
1178
1179         ret->num = 0;
1180
1181         for (i = 0; i < nodemap->num; i++) {
1182                 for(n = nodes; n != NULL ; n = n->next) {
1183                         if (ctdb_same_ip(&n->addr,
1184                                          &nodemap->nodes[i].addr)) {
1185                                 break;
1186                         }
1187                 }
1188                 if (n == NULL) {
1189                         continue;
1190                 }
1191
1192                 ret->nodes[ret->num] = nodemap->nodes[i];
1193                 ret->num++;
1194         }
1195
1196         return ret;
1197 }
1198
1199 static struct ctdb_node_map *
1200 filter_nodemap_by_capabilities(struct ctdb_context *ctdb,
1201                                struct ctdb_node_map *nodemap,
1202                                uint32_t required_capabilities,
1203                                bool first_only)
1204 {
1205         int i;
1206         uint32_t capabilities;
1207         struct ctdb_node_map *ret;
1208
1209         ret = talloc_nodemap(nodemap);
1210         CTDB_NO_MEMORY_NULL(ctdb, ret);
1211
1212         ret->num = 0;
1213
1214         for (i = 0; i < nodemap->num; i++) {
1215                 int res;
1216
1217                 /* Disconnected nodes have no capabilities! */
1218                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1219                         continue;
1220                 }
1221
1222                 res = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(),
1223                                                 nodemap->nodes[i].pnn,
1224                                                 &capabilities);
1225                 if (res != 0) {
1226                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n",
1227                                           nodemap->nodes[i].pnn));
1228                         talloc_free(ret);
1229                         return NULL;
1230                 }
1231                 if (!(capabilities & required_capabilities)) {
1232                         continue;
1233                 }
1234
1235                 ret->nodes[ret->num] = nodemap->nodes[i];
1236                 ret->num++;
1237                 if (first_only) {
1238                         break;
1239                 }
1240         }
1241
1242         return ret;
1243 }
1244
1245 static struct ctdb_node_map *
1246 filter_nodemap_by_flags(struct ctdb_context *ctdb,
1247                         struct ctdb_node_map *nodemap,
1248                         uint32_t flags_mask)
1249 {
1250         int i;
1251         struct ctdb_node_map *ret;
1252
1253         ret = talloc_nodemap(nodemap);
1254         CTDB_NO_MEMORY_NULL(ctdb, ret);
1255
1256         ret->num = 0;
1257
1258         for (i = 0; i < nodemap->num; i++) {
1259                 if (nodemap->nodes[i].flags & flags_mask) {
1260                         continue;
1261                 }
1262
1263                 ret->nodes[ret->num] = nodemap->nodes[i];
1264                 ret->num++;
1265         }
1266
1267         return ret;
1268 }
1269
1270 /*
1271   display the list of nodes belonging to this natgw configuration
1272  */
1273 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
1274 {
1275         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1276         int i, ret;
1277         struct pnn_node *natgw_nodes = NULL;
1278         struct ctdb_node_map *orig_nodemap=NULL;
1279         struct ctdb_node_map *nodemap;
1280         uint32_t mypnn, pnn;
1281         const char *ip;
1282
1283         /* When we have some nodes that could be the NATGW, make a
1284          * series of attempts to find the first node that doesn't have
1285          * certain status flags set.
1286          */
1287         uint32_t exclude_flags[] = {
1288                 /* Look for a nice healthy node */
1289                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY,
1290                 /* If not found, an UNHEALTHY/BANNED node will do */
1291                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED,
1292                 /* If not found, a STOPPED node will do */
1293                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED,
1294                 0,
1295         };
1296
1297         /* read the natgw nodes file into a linked list */
1298         natgw_nodes = read_natgw_nodes_file(ctdb, tmp_ctx);
1299         if (natgw_nodes == NULL) {
1300                 ret = -1;
1301                 goto done;
1302         }
1303
1304         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
1305                                    tmp_ctx, &orig_nodemap);
1306         if (ret != 0) {
1307                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
1308                 talloc_free(tmp_ctx);
1309                 return -1;
1310         }
1311
1312         /* Get a nodemap that includes only the nodes in the NATGW
1313          * group */
1314         nodemap = filter_nodemap_by_addrs(ctdb, orig_nodemap, natgw_nodes);
1315         if (nodemap == NULL) {
1316                 ret = -1;
1317                 goto done;
1318         }
1319
1320         ret = 2; /* matches ENOENT */
1321         pnn = -1;
1322         ip = "0.0.0.0";
1323         /* For each flag mask... */
1324         for (i = 0; exclude_flags[i] != 0; i++) {
1325                 /* ... get a nodemap that excludes nodes with with
1326                  * masked flags... */
1327                 struct ctdb_node_map *t =
1328                         filter_nodemap_by_flags(ctdb, nodemap,
1329                                                 exclude_flags[i]);
1330                 if (t == NULL) {
1331                         /* No memory */
1332                         ret = -1;
1333                         goto done;
1334                 }
1335                 if (t->num > 0) {
1336                         /* ... and find the first node with the NATGW
1337                          * capability */
1338                         struct ctdb_node_map *n;
1339                         n = filter_nodemap_by_capabilities(ctdb, t,
1340                                                            CTDB_CAP_NATGW,
1341                                                            true);
1342                         if (n == NULL) {
1343                                 /* No memory */
1344                                 ret = -1;
1345                                 goto done;
1346                         }
1347                         if (n->num > 0) {
1348                                 ret = 0;
1349                                 pnn = n->nodes[0].pnn;
1350                                 ip = ctdb_addr_to_str(&n->nodes[0].addr);
1351                                 break;
1352                         }
1353                 }
1354                 talloc_free(t);
1355         }
1356
1357         if (options.machinereadable) {
1358                 printf(":Node:IP:\n");
1359                 printf(":%d:%s:\n", pnn, ip);
1360         } else {
1361                 printf("%d %s\n", pnn, ip);
1362         }
1363
1364         /* print the pruned list of nodes belonging to this natgw list */
1365         mypnn = getpnn(ctdb);
1366         if (options.machinereadable) {
1367                 control_status_header_machine();
1368         } else {
1369                 printf("Number of nodes:%d\n", nodemap->num);
1370         }
1371         for(i=0;i<nodemap->num;i++){
1372                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1373                         continue;
1374                 }
1375                 if (options.machinereadable) {
1376                         control_status_1_machine(ctdb, mypnn, &(nodemap->nodes[i]));
1377                 } else {
1378                         control_status_1_human(ctdb, mypnn, &(nodemap->nodes[i]));
1379                 }
1380         }
1381
1382 done:
1383         talloc_free(tmp_ctx);
1384         return ret;
1385 }
1386
1387 /*
1388   display the status of the scripts for monitoring (or other events)
1389  */
1390 static int control_one_scriptstatus(struct ctdb_context *ctdb,
1391                                     enum ctdb_eventscript_call type)
1392 {
1393         struct ctdb_scripts_wire *script_status;
1394         int ret, i;
1395
1396         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
1397         if (ret != 0) {
1398                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
1399                 return ret;
1400         }
1401
1402         if (script_status == NULL) {
1403                 if (!options.machinereadable) {
1404                         printf("%s cycle never run\n",
1405                                ctdb_eventscript_call_names[type]);
1406                 }
1407                 return 0;
1408         }
1409
1410         if (!options.machinereadable) {
1411                 int num_run = 0;
1412                 for (i=0; i<script_status->num_scripts; i++) {
1413                         if (script_status->scripts[i].status != -ENOEXEC) {
1414                                 num_run++;
1415                         }
1416                 }
1417                 printf("%d scripts were executed last %s cycle\n",
1418                        num_run,
1419                        ctdb_eventscript_call_names[type]);
1420         }
1421         for (i=0; i<script_status->num_scripts; i++) {
1422                 const char *status = NULL;
1423
1424                 switch (script_status->scripts[i].status) {
1425                 case -ETIME:
1426                         status = "TIMEDOUT";
1427                         break;
1428                 case -ENOEXEC:
1429                         status = "DISABLED";
1430                         break;
1431                 case 0:
1432                         status = "OK";
1433                         break;
1434                 default:
1435                         if (script_status->scripts[i].status > 0)
1436                                 status = "ERROR";
1437                         break;
1438                 }
1439                 if (options.machinereadable) {
1440                         printf(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
1441                                ctdb_eventscript_call_names[type],
1442                                script_status->scripts[i].name,
1443                                script_status->scripts[i].status,
1444                                status,
1445                                (long)script_status->scripts[i].start.tv_sec,
1446                                (long)script_status->scripts[i].start.tv_usec,
1447                                (long)script_status->scripts[i].finished.tv_sec,
1448                                (long)script_status->scripts[i].finished.tv_usec,
1449                                script_status->scripts[i].output);
1450                         continue;
1451                 }
1452                 if (status)
1453                         printf("%-20s Status:%s    ",
1454                                script_status->scripts[i].name, status);
1455                 else
1456                         /* Some other error, eg from stat. */
1457                         printf("%-20s Status:CANNOT RUN (%s)",
1458                                script_status->scripts[i].name,
1459                                strerror(-script_status->scripts[i].status));
1460
1461                 if (script_status->scripts[i].status >= 0) {
1462                         printf("Duration:%.3lf ",
1463                         timeval_delta(&script_status->scripts[i].finished,
1464                               &script_status->scripts[i].start));
1465                 }
1466                 if (script_status->scripts[i].status != -ENOEXEC) {
1467                         printf("%s",
1468                                ctime(&script_status->scripts[i].start.tv_sec));
1469                         if (script_status->scripts[i].status != 0) {
1470                                 printf("   OUTPUT:%s\n",
1471                                        script_status->scripts[i].output);
1472                         }
1473                 } else {
1474                         printf("\n");
1475                 }
1476         }
1477         return 0;
1478 }
1479
1480
1481 static int control_scriptstatus(struct ctdb_context *ctdb,
1482                                 int argc, const char **argv)
1483 {
1484         int ret;
1485         enum ctdb_eventscript_call type, min, max;
1486         const char *arg;
1487
1488         if (argc > 1) {
1489                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
1490                 return -1;
1491         }
1492
1493         if (argc == 0)
1494                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
1495         else
1496                 arg = argv[0];
1497
1498         for (type = 0; type < CTDB_EVENT_MAX; type++) {
1499                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
1500                         min = type;
1501                         max = type+1;
1502                         break;
1503                 }
1504         }
1505         if (type == CTDB_EVENT_MAX) {
1506                 if (strcmp(arg, "all") == 0) {
1507                         min = 0;
1508                         max = CTDB_EVENT_MAX;
1509                 } else {
1510                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
1511                         return -1;
1512                 }
1513         }
1514
1515         if (options.machinereadable) {
1516                 printf(":Type:Name:Code:Status:Start:End:Error Output...:\n");
1517         }
1518
1519         for (type = min; type < max; type++) {
1520                 ret = control_one_scriptstatus(ctdb, type);
1521                 if (ret != 0) {
1522                         return ret;
1523                 }
1524         }
1525
1526         return 0;
1527 }
1528
1529 /*
1530   enable an eventscript
1531  */
1532 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1533 {
1534         int ret;
1535
1536         if (argc < 1) {
1537                 usage();
1538         }
1539
1540         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1541         if (ret != 0) {
1542           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
1543                 return ret;
1544         }
1545
1546         return 0;
1547 }
1548
1549 /*
1550   disable an eventscript
1551  */
1552 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1553 {
1554         int ret;
1555
1556         if (argc < 1) {
1557                 usage();
1558         }
1559
1560         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1561         if (ret != 0) {
1562           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
1563                 return ret;
1564         }
1565
1566         return 0;
1567 }
1568
1569 /*
1570   display the pnn of the recovery master
1571  */
1572 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
1573 {
1574         uint32_t recmaster;
1575         int ret;
1576
1577         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
1578         if (ret != 0) {
1579                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1580                 return -1;
1581         }
1582         printf("%d\n",recmaster);
1583
1584         return 0;
1585 }
1586
1587 /*
1588   add a tickle to a public address
1589  */
1590 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1591 {
1592         struct ctdb_tcp_connection t;
1593         TDB_DATA data;
1594         int ret;
1595
1596         assert_single_node_only();
1597
1598         if (argc < 2) {
1599                 usage();
1600         }
1601
1602         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1603                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1604                 return -1;
1605         }
1606         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1607                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1608                 return -1;
1609         }
1610
1611         data.dptr = (uint8_t *)&t;
1612         data.dsize = sizeof(t);
1613
1614         /* tell all nodes about this tcp connection */
1615         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1616                            0, data, ctdb, NULL, NULL, NULL, NULL);
1617         if (ret != 0) {
1618                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1619                 return -1;
1620         }
1621
1622         return 0;
1623 }
1624
1625
1626 /*
1627   delete a tickle from a node
1628  */
1629 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1630 {
1631         struct ctdb_tcp_connection t;
1632         TDB_DATA data;
1633         int ret;
1634
1635         assert_single_node_only();
1636
1637         if (argc < 2) {
1638                 usage();
1639         }
1640
1641         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1642                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1643                 return -1;
1644         }
1645         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1646                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1647                 return -1;
1648         }
1649
1650         data.dptr = (uint8_t *)&t;
1651         data.dsize = sizeof(t);
1652
1653         /* tell all nodes about this tcp connection */
1654         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1655                            0, data, ctdb, NULL, NULL, NULL, NULL);
1656         if (ret != 0) {
1657                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1658                 return -1;
1659         }
1660
1661         return 0;
1662 }
1663
1664
1665 /*
1666   get a list of all tickles for this pnn
1667  */
1668 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1669 {
1670         struct ctdb_control_tcp_tickle_list *list;
1671         ctdb_sock_addr addr;
1672         int i, ret;
1673         unsigned port = 0;
1674
1675         assert_single_node_only();
1676
1677         if (argc < 1) {
1678                 usage();
1679         }
1680
1681         if (argc == 2) {
1682                 port = atoi(argv[1]);
1683         }
1684
1685         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1686                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1687                 return -1;
1688         }
1689
1690         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1691         if (ret == -1) {
1692                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1693                 return -1;
1694         }
1695
1696         if (options.machinereadable){
1697                 printf(":source ip:port:destination ip:port:\n");
1698                 for (i=0;i<list->tickles.num;i++) {
1699                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1700                                 continue;
1701                         }
1702                         printf(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1703                         printf(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1704                 }
1705         } else {
1706                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1707                 printf("Num tickles:%u\n", list->tickles.num);
1708                 for (i=0;i<list->tickles.num;i++) {
1709                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1710                                 continue;
1711                         }
1712                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1713                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1714                 }
1715         }
1716
1717         talloc_free(list);
1718
1719         return 0;
1720 }
1721
1722
1723 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1724 {
1725         struct ctdb_all_public_ips *ips;
1726         struct ctdb_public_ip ip;
1727         int i, ret;
1728         uint32_t *nodes;
1729         uint32_t disable_time;
1730         TDB_DATA data;
1731         struct ctdb_node_map *nodemap=NULL;
1732         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1733
1734         disable_time = 30;
1735         data.dptr  = (uint8_t*)&disable_time;
1736         data.dsize = sizeof(disable_time);
1737         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1738         if (ret != 0) {
1739                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1740                 return -1;
1741         }
1742
1743
1744
1745         /* read the public ip list from the node */
1746         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1747         if (ret != 0) {
1748                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1749                 talloc_free(tmp_ctx);
1750                 return -1;
1751         }
1752
1753         for (i=0;i<ips->num;i++) {
1754                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1755                         break;
1756                 }
1757         }
1758         if (i==ips->num) {
1759                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1760                         pnn, ctdb_addr_to_str(addr)));
1761                 talloc_free(tmp_ctx);
1762                 return -1;
1763         }
1764
1765         ip.pnn  = pnn;
1766         ip.addr = *addr;
1767
1768         data.dptr  = (uint8_t *)&ip;
1769         data.dsize = sizeof(ip);
1770
1771         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1772         if (ret != 0) {
1773                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1774                 talloc_free(tmp_ctx);
1775                 return ret;
1776         }
1777
1778         nodes = list_of_nodes(ctdb, nodemap, tmp_ctx, NODE_FLAGS_INACTIVE, pnn);
1779         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1780                                         nodes, 0,
1781                                         LONGTIMELIMIT(),
1782                                         false, data,
1783                                         NULL, NULL,
1784                                         NULL);
1785         if (ret != 0) {
1786                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1787                 talloc_free(tmp_ctx);
1788                 return -1;
1789         }
1790
1791         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1792         if (ret != 0) {
1793                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1794                 talloc_free(tmp_ctx);
1795                 return -1;
1796         }
1797
1798         /* update the recovery daemon so it now knows to expect the new
1799            node assignment for this ip.
1800         */
1801         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1802         if (ret != 0) {
1803                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1804                 return -1;
1805         }
1806
1807         talloc_free(tmp_ctx);
1808         return 0;
1809 }
1810
1811
1812 /*
1813  * scans all other nodes and returns a pnn for another node that can host this
1814  * ip address or -1
1815  */
1816 static int
1817 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1818 {
1819         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1820         struct ctdb_all_public_ips *ips;
1821         struct ctdb_node_map *nodemap=NULL;
1822         int i, j, ret;
1823         int pnn;
1824
1825         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1826         if (ret != 0) {
1827                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1828                 talloc_free(tmp_ctx);
1829                 return ret;
1830         }
1831
1832         for(i=0;i<nodemap->num;i++){
1833                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1834                         continue;
1835                 }
1836                 if (nodemap->nodes[i].pnn == options.pnn) {
1837                         continue;
1838                 }
1839
1840                 /* read the public ip list from this node */
1841                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1842                 if (ret != 0) {
1843                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1844                         return -1;
1845                 }
1846
1847                 for (j=0;j<ips->num;j++) {
1848                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1849                                 pnn = nodemap->nodes[i].pnn;
1850                                 talloc_free(tmp_ctx);
1851                                 return pnn;
1852                         }
1853                 }
1854                 talloc_free(ips);
1855         }
1856
1857         talloc_free(tmp_ctx);
1858         return -1;
1859 }
1860
1861 /* If pnn is -1 then try to find a node to move IP to... */
1862 static bool try_moveip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1863 {
1864         bool pnn_specified = (pnn == -1 ? false : true);
1865         int retries = 0;
1866
1867         while (retries < 5) {
1868                 if (!pnn_specified) {
1869                         pnn = find_other_host_for_public_ip(ctdb, addr);
1870                         if (pnn == -1) {
1871                                 return false;
1872                         }
1873                         DEBUG(DEBUG_NOTICE,
1874                               ("Trying to move public IP to node %u\n", pnn));
1875                 }
1876
1877                 if (move_ip(ctdb, addr, pnn) == 0) {
1878                         return true;
1879                 }
1880
1881                 sleep(3);
1882                 retries++;
1883         }
1884
1885         return false;
1886 }
1887
1888
1889 /*
1890   move/failover an ip address to a specific node
1891  */
1892 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1893 {
1894         uint32_t pnn;
1895         ctdb_sock_addr addr;
1896
1897         assert_single_node_only();
1898
1899         if (argc < 2) {
1900                 usage();
1901                 return -1;
1902         }
1903
1904         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1905                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1906                 return -1;
1907         }
1908
1909
1910         if (sscanf(argv[1], "%u", &pnn) != 1) {
1911                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1912                 return -1;
1913         }
1914
1915         if (!try_moveip(ctdb, &addr, pnn)) {
1916                 DEBUG(DEBUG_ERR,("Failed to move IP to node %d.\n", pnn));
1917                 return -1;
1918         }
1919
1920         return 0;
1921 }
1922
1923 static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
1924 {
1925         TDB_DATA data;
1926
1927         data.dptr  = (uint8_t *)&pnn;
1928         data.dsize = sizeof(uint32_t);
1929         if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
1930                 DEBUG(DEBUG_ERR,
1931                       ("Failed to send message to force node %u to be a rebalancing target\n",
1932                        pnn));
1933                 return -1;
1934         }
1935
1936         return 0;
1937 }
1938
1939
1940 /*
1941   rebalance a node by setting it to allow failback and triggering a
1942   takeover run
1943  */
1944 static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
1945 {
1946         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1947         uint32_t *nodes;
1948         uint32_t pnn_mode;
1949         int i, ret;
1950
1951         assert_single_node_only();
1952
1953         if (argc > 1) {
1954                 usage();
1955         }
1956
1957         /* Determine the nodes where IPs need to be reloaded */
1958         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1959                               options.pnn, true, &nodes, &pnn_mode)) {
1960                 ret = -1;
1961                 goto done;
1962         }
1963
1964         for (i = 0; i < talloc_array_length(nodes); i++) {
1965                 if (!rebalance_node(ctdb, nodes[i])) {
1966                         ret = -1;
1967                 }
1968         }
1969
1970 done:
1971         talloc_free(tmp_ctx);
1972         return ret;
1973 }
1974
1975 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1976 {
1977         struct ctdb_public_ip ip;
1978         int ret;
1979         uint32_t *nodes;
1980         uint32_t disable_time;
1981         TDB_DATA data;
1982         struct ctdb_node_map *nodemap=NULL;
1983         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1984
1985         disable_time = 30;
1986         data.dptr  = (uint8_t*)&disable_time;
1987         data.dsize = sizeof(disable_time);
1988         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1989         if (ret != 0) {
1990                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1991                 return -1;
1992         }
1993
1994         ip.pnn  = -1;
1995         ip.addr = *addr;
1996
1997         data.dptr  = (uint8_t *)&ip;
1998         data.dsize = sizeof(ip);
1999
2000         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
2001         if (ret != 0) {
2002                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2003                 talloc_free(tmp_ctx);
2004                 return ret;
2005         }
2006
2007         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
2008         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
2009                                         nodes, 0,
2010                                         LONGTIMELIMIT(),
2011                                         false, data,
2012                                         NULL, NULL,
2013                                         NULL);
2014         if (ret != 0) {
2015                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
2016                 talloc_free(tmp_ctx);
2017                 return -1;
2018         }
2019
2020         talloc_free(tmp_ctx);
2021         return 0;
2022 }
2023
2024 /*
2025   release an ip form all nodes and have it re-assigned by recd
2026  */
2027 static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
2028 {
2029         ctdb_sock_addr addr;
2030
2031         assert_single_node_only();
2032
2033         if (argc < 1) {
2034                 usage();
2035                 return -1;
2036         }
2037
2038         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2039                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2040                 return -1;
2041         }
2042
2043         if (rebalance_ip(ctdb, &addr) != 0) {
2044                 DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
2045                 return -1;
2046         }
2047
2048         return 0;
2049 }
2050
2051 static int getips_store_callback(void *param, void *data)
2052 {
2053         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
2054         struct ctdb_all_public_ips *ips = param;
2055         int i;
2056
2057         i = ips->num++;
2058         ips->ips[i].pnn  = node_ip->pnn;
2059         ips->ips[i].addr = node_ip->addr;
2060         return 0;
2061 }
2062
2063 static int getips_count_callback(void *param, void *data)
2064 {
2065         uint32_t *count = param;
2066
2067         (*count)++;
2068         return 0;
2069 }
2070
2071 #define IP_KEYLEN       4
2072 static uint32_t *ip_key(ctdb_sock_addr *ip)
2073 {
2074         static uint32_t key[IP_KEYLEN];
2075
2076         bzero(key, sizeof(key));
2077
2078         switch (ip->sa.sa_family) {
2079         case AF_INET:
2080                 key[0]  = ip->ip.sin_addr.s_addr;
2081                 break;
2082         case AF_INET6: {
2083                 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
2084                 key[0]  = s6_a32[3];
2085                 key[1]  = s6_a32[2];
2086                 key[2]  = s6_a32[1];
2087                 key[3]  = s6_a32[0];
2088                 break;
2089         }
2090         default:
2091                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
2092                 return key;
2093         }
2094
2095         return key;
2096 }
2097
2098 static void *add_ip_callback(void *parm, void *data)
2099 {
2100         return parm;
2101 }
2102
2103 static int
2104 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_all_public_ips **ips)
2105 {
2106         struct ctdb_all_public_ips *tmp_ips;
2107         struct ctdb_node_map *nodemap=NULL;
2108         trbt_tree_t *ip_tree;
2109         int i, j, len, ret;
2110         uint32_t count;
2111
2112         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2113         if (ret != 0) {
2114                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2115                 return ret;
2116         }
2117
2118         ip_tree = trbt_create(tmp_ctx, 0);
2119
2120         for(i=0;i<nodemap->num;i++){
2121                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
2122                         continue;
2123                 }
2124                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2125                         continue;
2126                 }
2127
2128                 /* read the public ip list from this node */
2129                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
2130                 if (ret != 0) {
2131                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
2132                         return -1;
2133                 }
2134
2135                 for (j=0; j<tmp_ips->num;j++) {
2136                         struct ctdb_public_ip *node_ip;
2137
2138                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
2139                         node_ip->pnn  = tmp_ips->ips[j].pnn;
2140                         node_ip->addr = tmp_ips->ips[j].addr;
2141
2142                         trbt_insertarray32_callback(ip_tree,
2143                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
2144                                 add_ip_callback,
2145                                 node_ip);
2146                 }
2147                 talloc_free(tmp_ips);
2148         }
2149
2150         /* traverse */
2151         count = 0;
2152         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
2153
2154         len = offsetof(struct ctdb_all_public_ips, ips) +
2155                 count*sizeof(struct ctdb_public_ip);
2156         tmp_ips = talloc_zero_size(tmp_ctx, len);
2157         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
2158
2159         *ips = tmp_ips;
2160
2161         return 0;
2162 }
2163
2164
2165 static void ctdb_every_second(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
2166 {
2167         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2168
2169         event_add_timed(ctdb->ev, ctdb,
2170                                 timeval_current_ofs(1, 0),
2171                                 ctdb_every_second, ctdb);
2172 }
2173
2174 struct srvid_reply_handler_data {
2175         bool done;
2176         bool wait_for_all;
2177         uint32_t *nodes;
2178         const char *srvid_str;
2179 };
2180
2181 static void srvid_broadcast_reply_handler(struct ctdb_context *ctdb,
2182                                          uint64_t srvid,
2183                                          TDB_DATA data,
2184                                          void *private_data)
2185 {
2186         struct srvid_reply_handler_data *d =
2187                 (struct srvid_reply_handler_data *)private_data;
2188         int i;
2189         int32_t ret;
2190
2191         if (data.dsize != sizeof(ret)) {
2192                 DEBUG(DEBUG_ERR, (__location__ " Wrong reply size\n"));
2193                 return;
2194         }
2195
2196         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
2197         ret = *(int32_t *)data.dptr;
2198         if (ret < 0) {
2199                 DEBUG(DEBUG_ERR,
2200                       ("%s failed with result %d\n", d->srvid_str, ret));
2201                 return;
2202         }
2203
2204         if (!d->wait_for_all) {
2205                 d->done = true;
2206                 return;
2207         }
2208
2209         /* Wait for all replies */
2210         d->done = true;
2211         for (i = 0; i < talloc_array_length(d->nodes); i++) {
2212                 if (d->nodes[i] == ret) {
2213                         DEBUG(DEBUG_INFO,
2214                               ("%s reply received from node %u\n",
2215                                d->srvid_str, ret));
2216                         d->nodes[i] = -1;
2217                 }
2218                 if (d->nodes[i] != -1) {
2219                         /* Found a node that hasn't yet replied */
2220                         d->done = false;
2221                 }
2222         }
2223 }
2224
2225 /* Broadcast the given SRVID to all connected nodes.  Wait for 1 reply
2226  * or replies from all connected nodes.  arg is the data argument to
2227  * pass in the srvid_request structure - pass 0 if this isn't needed.
2228  */
2229 static int srvid_broadcast(struct ctdb_context *ctdb,
2230                            uint64_t srvid, uint32_t *arg,
2231                            const char *srvid_str, bool wait_for_all)
2232 {
2233         int ret;
2234         TDB_DATA data;
2235         uint32_t pnn;
2236         uint64_t reply_srvid;
2237         struct srvid_request request;
2238         struct srvid_request_data request_data;
2239         struct srvid_reply_handler_data reply_data;
2240         struct timeval tv;
2241
2242         ZERO_STRUCT(request);
2243
2244         /* Time ticks to enable timeouts to be processed */
2245         event_add_timed(ctdb->ev, ctdb,
2246                                 timeval_current_ofs(1, 0),
2247                                 ctdb_every_second, ctdb);
2248
2249         pnn = ctdb_get_pnn(ctdb);
2250         reply_srvid = getpid();
2251
2252         if (arg == NULL) {
2253                 request.pnn = pnn;
2254                 request.srvid = reply_srvid;
2255
2256                 data.dptr = (uint8_t *)&request;
2257                 data.dsize = sizeof(request);
2258         } else {
2259                 request_data.pnn = pnn;
2260                 request_data.srvid = reply_srvid;
2261                 request_data.data = *arg;
2262
2263                 data.dptr = (uint8_t *)&request_data;
2264                 data.dsize = sizeof(request_data);
2265         }
2266
2267         /* Register message port for reply from recovery master */
2268         ctdb_client_set_message_handler(ctdb, reply_srvid,
2269                                         srvid_broadcast_reply_handler,
2270                                         &reply_data);
2271
2272         reply_data.wait_for_all = wait_for_all;
2273         reply_data.nodes = NULL;
2274         reply_data.srvid_str = srvid_str;
2275
2276 again:
2277         reply_data.done = false;
2278
2279         if (wait_for_all) {
2280                 struct ctdb_node_map *nodemap;
2281
2282                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(),
2283                                            CTDB_CURRENT_NODE, ctdb, &nodemap);
2284                 if (ret != 0) {
2285                         DEBUG(DEBUG_ERR,
2286                               ("Unable to get nodemap from current node, try again\n"));
2287                         sleep(1);
2288                         goto again;
2289                 }
2290
2291                 if (reply_data.nodes != NULL) {
2292                         talloc_free(reply_data.nodes);
2293                 }
2294                 reply_data.nodes = list_of_connected_nodes(ctdb, nodemap,
2295                                                            NULL, true);
2296
2297                 talloc_free(nodemap);
2298         }
2299
2300         /* Send to all connected nodes. Only recmaster replies */
2301         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2302                                        srvid, data);
2303         if (ret != 0) {
2304                 /* This can only happen if the socket is closed and
2305                  * there's no way to recover from that, so don't try
2306                  * again.
2307                  */
2308                 DEBUG(DEBUG_ERR,
2309                       ("Failed to send %s request to connected nodes\n",
2310                        srvid_str));
2311                 return -1;
2312         }
2313
2314         tv = timeval_current();
2315         /* This loop terminates the reply is received */
2316         while (timeval_elapsed(&tv) < 5.0 && !reply_data.done) {
2317                 event_loop_once(ctdb->ev);
2318         }
2319
2320         if (!reply_data.done) {
2321                 DEBUG(DEBUG_NOTICE,
2322                       ("Still waiting for confirmation of %s\n", srvid_str));
2323                 sleep(1);
2324                 goto again;
2325         }
2326
2327         ctdb_client_remove_message_handler(ctdb, reply_srvid, &reply_data);
2328
2329         talloc_free(reply_data.nodes);
2330
2331         return 0;
2332 }
2333
2334 static int ipreallocate(struct ctdb_context *ctdb)
2335 {
2336         return srvid_broadcast(ctdb, CTDB_SRVID_TAKEOVER_RUN, NULL,
2337                                "IP reallocation", false);
2338 }
2339
2340
2341 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2342 {
2343         return ipreallocate(ctdb);
2344 }
2345
2346 /*
2347   add a public ip address to a node
2348  */
2349 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
2350 {
2351         int i, ret;
2352         int len, retries = 0;
2353         unsigned mask;
2354         ctdb_sock_addr addr;
2355         struct ctdb_control_ip_iface *pub;
2356         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2357         struct ctdb_all_public_ips *ips;
2358
2359
2360         if (argc != 2) {
2361                 talloc_free(tmp_ctx);
2362                 usage();
2363         }
2364
2365         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
2366                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
2367                 talloc_free(tmp_ctx);
2368                 return -1;
2369         }
2370
2371         /* read the public ip list from the node */
2372         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2373         if (ret != 0) {
2374                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", options.pnn));
2375                 talloc_free(tmp_ctx);
2376                 return -1;
2377         }
2378         for (i=0;i<ips->num;i++) {
2379                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2380                         DEBUG(DEBUG_ERR,("Can not add ip to node. Node already hosts this ip\n"));
2381                         return 0;
2382                 }
2383         }
2384
2385
2386
2387         /* Dont timeout. This command waits for an ip reallocation
2388            which sometimes can take wuite a while if there has
2389            been a recent recovery
2390         */
2391         alarm(0);
2392
2393         len = offsetof(struct ctdb_control_ip_iface, iface) + strlen(argv[1]) + 1;
2394         pub = talloc_size(tmp_ctx, len);
2395         CTDB_NO_MEMORY(ctdb, pub);
2396
2397         pub->addr  = addr;
2398         pub->mask  = mask;
2399         pub->len   = strlen(argv[1])+1;
2400         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
2401
2402         do {
2403                 ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
2404                 if (ret != 0) {
2405                         DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
2406                         sleep(3);
2407                         retries++;
2408                 }
2409         } while (retries < 5 && ret != 0);
2410         if (ret != 0) {
2411                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
2412                 talloc_free(tmp_ctx);
2413                 return ret;
2414         }
2415
2416         if (rebalance_node(ctdb, options.pnn) != 0) {
2417                 DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
2418                 return ret;
2419         }
2420
2421         talloc_free(tmp_ctx);
2422         return 0;
2423 }
2424
2425 /*
2426   add a public ip address to a node
2427  */
2428 static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
2429 {
2430         ctdb_sock_addr addr;
2431
2432         if (argc != 1) {
2433                 usage();
2434         }
2435
2436         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2437                 printf("Badly formed ip : %s\n", argv[0]);
2438                 return -1;
2439         }
2440
2441         printf("IP on interface %s\n", ctdb_sys_find_ifname(&addr));
2442
2443         return 0;
2444 }
2445
2446 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
2447
2448 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
2449 {
2450         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2451         struct ctdb_node_map *nodemap=NULL;
2452         struct ctdb_all_public_ips *ips;
2453         int ret, i, j;
2454
2455         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2456         if (ret != 0) {
2457                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
2458                 return ret;
2459         }
2460
2461         /* remove it from the nodes that are not hosting the ip currently */
2462         for(i=0;i<nodemap->num;i++){
2463                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2464                         continue;
2465                 }
2466                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2467                 if (ret != 0) {
2468                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2469                         continue;
2470                 }
2471
2472                 for (j=0;j<ips->num;j++) {
2473                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2474                                 break;
2475                         }
2476                 }
2477                 if (j==ips->num) {
2478                         continue;
2479                 }
2480
2481                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
2482                         continue;
2483                 }
2484
2485                 options.pnn = nodemap->nodes[i].pnn;
2486                 control_delip(ctdb, argc, argv);
2487         }
2488
2489
2490         /* remove it from every node (also the one hosting it) */
2491         for(i=0;i<nodemap->num;i++){
2492                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2493                         continue;
2494                 }
2495                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2496                 if (ret != 0) {
2497                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2498                         continue;
2499                 }
2500
2501                 for (j=0;j<ips->num;j++) {
2502                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2503                                 break;
2504                         }
2505                 }
2506                 if (j==ips->num) {
2507                         continue;
2508                 }
2509
2510                 options.pnn = nodemap->nodes[i].pnn;
2511                 control_delip(ctdb, argc, argv);
2512         }
2513
2514         talloc_free(tmp_ctx);
2515         return 0;
2516 }
2517
2518 /*
2519   delete a public ip address from a node
2520  */
2521 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
2522 {
2523         int i, ret;
2524         ctdb_sock_addr addr;
2525         struct ctdb_control_ip_iface pub;
2526         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2527         struct ctdb_all_public_ips *ips;
2528
2529         if (argc != 1) {
2530                 talloc_free(tmp_ctx);
2531                 usage();
2532         }
2533
2534         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2535                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2536                 return -1;
2537         }
2538
2539         if (options.pnn == CTDB_BROADCAST_ALL) {
2540                 return control_delip_all(ctdb, argc, argv, &addr);
2541         }
2542
2543         pub.addr  = addr;
2544         pub.mask  = 0;
2545         pub.len   = 0;
2546
2547         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2548         if (ret != 0) {
2549                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
2550                 talloc_free(tmp_ctx);
2551                 return ret;
2552         }
2553
2554         for (i=0;i<ips->num;i++) {
2555                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2556                         break;
2557                 }
2558         }
2559
2560         if (i==ips->num) {
2561                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
2562                         ctdb_addr_to_str(&addr)));
2563                 talloc_free(tmp_ctx);
2564                 return -1;
2565         }
2566
2567         /* This is an optimisation.  If this node is hosting the IP
2568          * then try to move it somewhere else without invoking a full
2569          * takeover run.  We don't care if this doesn't work!
2570          */
2571         if (ips->ips[i].pnn == options.pnn) {
2572                 (void) try_moveip(ctdb, &addr, -1);
2573         }
2574
2575         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
2576         if (ret != 0) {
2577                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
2578                 talloc_free(tmp_ctx);
2579                 return ret;
2580         }
2581
2582         talloc_free(tmp_ctx);
2583         return 0;
2584 }
2585
2586 static int kill_tcp_from_file(struct ctdb_context *ctdb,
2587                               int argc, const char **argv)
2588 {
2589         struct ctdb_control_killtcp *killtcp;
2590         int max_entries, current, i;
2591         struct timeval timeout;
2592         char line[128], src[128], dst[128];
2593         int linenum;
2594         TDB_DATA data;
2595         struct client_async_data *async_data;
2596         struct ctdb_client_control_state *state;
2597
2598         if (argc != 0) {
2599                 usage();
2600         }
2601
2602         linenum = 1;
2603         killtcp = NULL;
2604         max_entries = 0;
2605         current = 0;
2606         while (!feof(stdin)) {
2607                 if (fgets(line, sizeof(line), stdin) == NULL) {
2608                         continue;
2609                 }
2610
2611                 /* Silently skip empty lines */
2612                 if (line[0] == '\n') {
2613                         continue;
2614                 }
2615
2616                 if (sscanf(line, "%s %s\n", src, dst) != 2) {
2617                         DEBUG(DEBUG_ERR, ("Bad line [%d]: '%s'\n",
2618                                           linenum, line));
2619                         talloc_free(killtcp);
2620                         return -1;
2621                 }
2622
2623                 if (current >= max_entries) {
2624                         max_entries += 1024;
2625                         killtcp = talloc_realloc(ctdb, killtcp,
2626                                                  struct ctdb_control_killtcp,
2627                                                  max_entries);
2628                         CTDB_NO_MEMORY(ctdb, killtcp);
2629                 }
2630
2631                 if (!parse_ip_port(src, &killtcp[current].src_addr)) {
2632                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2633                                           linenum, src));
2634                         talloc_free(killtcp);
2635                         return -1;
2636                 }
2637
2638                 if (!parse_ip_port(dst, &killtcp[current].dst_addr)) {
2639                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2640                                           linenum, dst));
2641                         talloc_free(killtcp);
2642                         return -1;
2643                 }
2644
2645                 current++;
2646         }
2647
2648         async_data = talloc_zero(ctdb, struct client_async_data);
2649         if (async_data == NULL) {
2650                 talloc_free(killtcp);
2651                 return -1;
2652         }
2653
2654         for (i = 0; i < current; i++) {
2655
2656                 data.dsize = sizeof(struct ctdb_control_killtcp);
2657                 data.dptr  = (unsigned char *)&killtcp[i];
2658
2659                 timeout = TIMELIMIT();
2660                 state = ctdb_control_send(ctdb, options.pnn, 0,
2661                                           CTDB_CONTROL_KILL_TCP, 0, data,
2662                                           async_data, &timeout, NULL);
2663
2664                 if (state == NULL) {
2665                         DEBUG(DEBUG_ERR,
2666                               ("Failed to call async killtcp control to node %u\n",
2667                                options.pnn));
2668                         talloc_free(killtcp);
2669                         return -1;
2670                 }
2671
2672                 ctdb_client_async_add(async_data, state);
2673         }
2674
2675         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2676                 DEBUG(DEBUG_ERR,("killtcp failed\n"));
2677                 talloc_free(killtcp);
2678                 return -1;
2679         }
2680
2681         talloc_free(killtcp);
2682         return 0;
2683 }
2684
2685
2686 /*
2687   kill a tcp connection
2688  */
2689 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2690 {
2691         int ret;
2692         struct ctdb_control_killtcp killtcp;
2693
2694         assert_single_node_only();
2695
2696         if (argc == 0) {
2697                 return kill_tcp_from_file(ctdb, argc, argv);
2698         }
2699
2700         if (argc < 2) {
2701                 usage();
2702         }
2703
2704         if (!parse_ip_port(argv[0], &killtcp.src_addr)) {
2705                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2706                 return -1;
2707         }
2708
2709         if (!parse_ip_port(argv[1], &killtcp.dst_addr)) {
2710                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2711                 return -1;
2712         }
2713
2714         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
2715         if (ret != 0) {
2716                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
2717                 return ret;
2718         }
2719
2720         return 0;
2721 }
2722
2723
2724 /*
2725   send a gratious arp
2726  */
2727 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
2728 {
2729         int ret;
2730         ctdb_sock_addr addr;
2731
2732         assert_single_node_only();
2733
2734         if (argc < 2) {
2735                 usage();
2736         }
2737
2738         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2739                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
2740                 return -1;
2741         }
2742
2743         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
2744         if (ret != 0) {
2745                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
2746                 return ret;
2747         }
2748
2749         return 0;
2750 }
2751
2752 /*
2753   register a server id
2754  */
2755 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2756 {
2757         int ret;
2758         struct ctdb_server_id server_id;
2759
2760         if (argc < 3) {
2761                 usage();
2762         }
2763
2764         server_id.pnn       = strtoul(argv[0], NULL, 0);
2765         server_id.type      = strtoul(argv[1], NULL, 0);
2766         server_id.server_id = strtoul(argv[2], NULL, 0);
2767
2768         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
2769         if (ret != 0) {
2770                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
2771                 return ret;
2772         }
2773         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
2774         sleep(999);
2775         return -1;
2776 }
2777
2778 /*
2779   unregister a server id
2780  */
2781 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2782 {
2783         int ret;
2784         struct ctdb_server_id server_id;
2785
2786         if (argc < 3) {
2787                 usage();
2788         }
2789
2790         server_id.pnn       = strtoul(argv[0], NULL, 0);
2791         server_id.type      = strtoul(argv[1], NULL, 0);
2792         server_id.server_id = strtoul(argv[2], NULL, 0);
2793
2794         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
2795         if (ret != 0) {
2796                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
2797                 return ret;
2798         }
2799         return -1;
2800 }
2801
2802 /*
2803   check if a server id exists
2804  */
2805 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2806 {
2807         uint32_t status;
2808         int ret;
2809         struct ctdb_server_id server_id;
2810
2811         if (argc < 3) {
2812                 usage();
2813         }
2814
2815         server_id.pnn       = strtoul(argv[0], NULL, 0);
2816         server_id.type      = strtoul(argv[1], NULL, 0);
2817         server_id.server_id = strtoul(argv[2], NULL, 0);
2818
2819         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
2820         if (ret != 0) {
2821                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
2822                 return ret;
2823         }
2824
2825         if (status) {
2826                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
2827         } else {
2828                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
2829         }
2830         return 0;
2831 }
2832
2833 /*
2834   get a list of all server ids that are registered on a node
2835  */
2836 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
2837 {
2838         int i, ret;
2839         struct ctdb_server_id_list *server_ids;
2840
2841         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
2842         if (ret != 0) {
2843                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
2844                 return ret;
2845         }
2846
2847         for (i=0; i<server_ids->num; i++) {
2848                 printf("Server id %d:%d:%d\n",
2849                         server_ids->server_ids[i].pnn,
2850                         server_ids->server_ids[i].type,
2851                         server_ids->server_ids[i].server_id);
2852         }
2853
2854         return -1;
2855 }
2856
2857 /*
2858   check if a server id exists
2859  */
2860 static int check_srvids(struct ctdb_context *ctdb, int argc, const char **argv)
2861 {
2862         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
2863         uint64_t *ids;
2864         uint8_t *result;
2865         int i;
2866
2867         if (argc < 1) {
2868                 talloc_free(tmp_ctx);
2869                 usage();
2870         }
2871
2872         ids    = talloc_array(tmp_ctx, uint64_t, argc);
2873         result = talloc_array(tmp_ctx, uint8_t, argc);
2874
2875         for (i = 0; i < argc; i++) {
2876                 ids[i] = strtoull(argv[i], NULL, 0);
2877         }
2878
2879         if (!ctdb_client_check_message_handlers(ctdb, ids, argc, result)) {
2880                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n",
2881                                   options.pnn));
2882                 talloc_free(tmp_ctx);
2883                 return -1;
2884         }
2885
2886         for (i=0; i < argc; i++) {
2887                 printf("Server id %d:%llu %s\n", options.pnn, (long long)ids[i],
2888                        result[i] ? "exists" : "does not exist");
2889         }
2890
2891         talloc_free(tmp_ctx);
2892         return 0;
2893 }
2894
2895 /*
2896   send a tcp tickle ack
2897  */
2898 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2899 {
2900         int ret;
2901         ctdb_sock_addr  src, dst;
2902
2903         if (argc < 2) {
2904                 usage();
2905         }
2906
2907         if (!parse_ip_port(argv[0], &src)) {
2908                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2909                 return -1;
2910         }
2911
2912         if (!parse_ip_port(argv[1], &dst)) {
2913                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2914                 return -1;
2915         }
2916
2917         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
2918         if (ret==0) {
2919                 return 0;
2920         }
2921         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
2922
2923         return -1;
2924 }
2925
2926
2927 /*
2928   display public ip status
2929  */
2930 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
2931 {
2932         int i, ret;
2933         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2934         struct ctdb_all_public_ips *ips;
2935
2936         if (options.pnn == CTDB_BROADCAST_ALL) {
2937                 /* read the list of public ips from all nodes */
2938                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
2939         } else {
2940                 /* read the public ip list from this node */
2941                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2942         }
2943         if (ret != 0) {
2944                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
2945                 talloc_free(tmp_ctx);
2946                 return ret;
2947         }
2948
2949         if (options.machinereadable){
2950                 printf(":Public IP:Node:");
2951                 if (options.verbose){
2952                         printf("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
2953                 }
2954                 printf("\n");
2955         } else {
2956                 if (options.pnn == CTDB_BROADCAST_ALL) {
2957                         printf("Public IPs on ALL nodes\n");
2958                 } else {
2959                         printf("Public IPs on node %u\n", options.pnn);
2960                 }
2961         }
2962
2963         for (i=1;i<=ips->num;i++) {
2964                 struct ctdb_control_public_ip_info *info = NULL;
2965                 int32_t pnn;
2966                 char *aciface = NULL;
2967                 char *avifaces = NULL;
2968                 char *cifaces = NULL;
2969
2970                 if (options.pnn == CTDB_BROADCAST_ALL) {
2971                         pnn = ips->ips[ips->num-i].pnn;
2972                 } else {
2973                         pnn = options.pnn;
2974                 }
2975
2976                 if (pnn != -1) {
2977                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
2978                                                    &ips->ips[ips->num-i].addr, &info);
2979                 } else {
2980                         ret = -1;
2981                 }
2982
2983                 if (ret == 0) {
2984                         int j;
2985                         for (j=0; j < info->num; j++) {
2986                                 if (cifaces == NULL) {
2987                                         cifaces = talloc_strdup(info,
2988                                                                 info->ifaces[j].name);
2989                                 } else {
2990                                         cifaces = talloc_asprintf_append(cifaces,
2991                                                                          ",%s",
2992                                                                          info->ifaces[j].name);
2993                                 }
2994
2995                                 if (info->active_idx == j) {
2996                                         aciface = info->ifaces[j].name;
2997                                 }
2998
2999                                 if (info->ifaces[j].link_state == 0) {
3000                                         continue;
3001                                 }
3002
3003                                 if (avifaces == NULL) {
3004                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
3005                                 } else {
3006                                         avifaces = talloc_asprintf_append(avifaces,
3007                                                                           ",%s",
3008                                                                           info->ifaces[j].name);
3009                                 }
3010                         }
3011                 }
3012
3013                 if (options.machinereadable){
3014                         printf(":%s:%d:",
3015                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3016                                 ips->ips[ips->num-i].pnn);
3017                         if (options.verbose){
3018                                 printf("%s:%s:%s:",
3019                                         aciface?aciface:"",
3020                                         avifaces?avifaces:"",
3021                                         cifaces?cifaces:"");
3022                         }
3023                         printf("\n");
3024                 } else {
3025                         if (options.verbose) {
3026                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
3027                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3028                                         ips->ips[ips->num-i].pnn,
3029                                         aciface?aciface:"",
3030                                         avifaces?avifaces:"",
3031                                         cifaces?cifaces:"");
3032                         } else {
3033                                 printf("%s %d\n",
3034                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3035                                         ips->ips[ips->num-i].pnn);
3036                         }
3037                 }
3038                 talloc_free(info);
3039         }
3040
3041         talloc_free(tmp_ctx);
3042         return 0;
3043 }
3044
3045 /*
3046   public ip info
3047  */
3048 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
3049 {
3050         int i, ret;
3051         ctdb_sock_addr addr;
3052         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3053         struct ctdb_control_public_ip_info *info;
3054
3055         if (argc != 1) {
3056                 talloc_free(tmp_ctx);
3057                 usage();
3058         }
3059
3060         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
3061                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
3062                 return -1;
3063         }
3064
3065         /* read the public ip info from this node */
3066         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
3067                                            tmp_ctx, &addr, &info);
3068         if (ret != 0) {
3069                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
3070                                   argv[0], options.pnn));
3071                 talloc_free(tmp_ctx);
3072                 return ret;
3073         }
3074
3075         printf("Public IP[%s] info on node %u\n",
3076                ctdb_addr_to_str(&info->ip.addr),
3077                options.pnn);
3078
3079         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
3080                ctdb_addr_to_str(&info->ip.addr),
3081                info->ip.pnn, info->num);
3082
3083         for (i=0; i<info->num; i++) {
3084                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
3085
3086                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
3087                        i+1, info->ifaces[i].name,
3088                        info->ifaces[i].link_state?"up":"down",
3089                        (unsigned int)info->ifaces[i].references,
3090                        (i==info->active_idx)?" (active)":"");
3091         }
3092
3093         talloc_free(tmp_ctx);
3094         return 0;
3095 }
3096
3097 /*
3098   display interfaces status
3099  */
3100 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
3101 {
3102         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3103         int i;
3104         struct ctdb_control_get_ifaces *ifaces;
3105         int ret;
3106
3107         /* read the public ip list from this node */
3108         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ifaces);
3109         if (ret != 0) {
3110                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
3111                                   options.pnn));
3112                 talloc_free(tmp_ctx);
3113                 return -1;
3114         }
3115
3116         if (options.machinereadable){
3117                 printf(":Name:LinkStatus:References:\n");
3118         } else {
3119                 printf("Interfaces on node %u\n", options.pnn);
3120         }
3121
3122         for (i=0; i<ifaces->num; i++) {
3123                 if (options.machinereadable){
3124                         printf(":%s:%s:%u\n",
3125                                ifaces->ifaces[i].name,
3126                                ifaces->ifaces[i].link_state?"1":"0",
3127                                (unsigned int)ifaces->ifaces[i].references);
3128                 } else {
3129                         printf("name:%s link:%s references:%u\n",
3130                                ifaces->ifaces[i].name,
3131                                ifaces->ifaces[i].link_state?"up":"down",
3132                                (unsigned int)ifaces->ifaces[i].references);
3133                 }
3134         }
3135
3136         talloc_free(tmp_ctx);
3137         return 0;
3138 }
3139
3140
3141 /*
3142   set link status of an interface
3143  */
3144 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
3145 {
3146         int ret;
3147         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3148         struct ctdb_control_iface_info info;
3149
3150         ZERO_STRUCT(info);
3151
3152         if (argc != 2) {
3153                 usage();
3154         }
3155
3156         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
3157                 DEBUG(DEBUG_ERR, ("interfaces name '%s' too long\n",
3158                                   argv[0]));
3159                 talloc_free(tmp_ctx);
3160                 return -1;
3161         }
3162         strcpy(info.name, argv[0]);
3163
3164         if (strcmp(argv[1], "up") == 0) {
3165                 info.link_state = 1;
3166         } else if (strcmp(argv[1], "down") == 0) {
3167                 info.link_state = 0;
3168         } else {
3169                 DEBUG(DEBUG_ERR, ("link state invalid '%s' should be 'up' or 'down'\n",
3170                                   argv[1]));
3171                 talloc_free(tmp_ctx);
3172                 return -1;
3173         }
3174
3175         /* read the public ip list from this node */
3176         ret = ctdb_ctrl_set_iface_link(ctdb, TIMELIMIT(), options.pnn,
3177                                    tmp_ctx, &info);
3178         if (ret != 0) {
3179                 DEBUG(DEBUG_ERR, ("Unable to set link state for interfaces %s node %u\n",
3180                                   argv[0], options.pnn));
3181                 talloc_free(tmp_ctx);
3182                 return ret;
3183         }
3184
3185         talloc_free(tmp_ctx);
3186         return 0;
3187 }
3188
3189 /*
3190   display pid of a ctdb daemon
3191  */
3192 static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
3193 {
3194         uint32_t pid;
3195         int ret;
3196
3197         ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.pnn, &pid);
3198         if (ret != 0) {
3199                 DEBUG(DEBUG_ERR, ("Unable to get daemon pid from node %u\n", options.pnn));
3200                 return ret;
3201         }
3202         printf("Pid:%d\n", pid);
3203
3204         return 0;
3205 }
3206
3207 typedef bool update_flags_handler_t(struct ctdb_context *ctdb, void *data);
3208
3209 static int update_flags_and_ipreallocate(struct ctdb_context *ctdb,
3210                                               void *data,
3211                                               update_flags_handler_t handler,
3212                                               uint32_t flag,
3213                                               const char *desc,
3214                                               bool set_flag)
3215 {
3216         struct ctdb_node_map *nodemap = NULL;
3217         bool flag_is_set;
3218         int ret;
3219
3220         /* Check if the node is already in the desired state */
3221         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3222         if (ret != 0) {
3223                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3224                 exit(10);
3225         }
3226         flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3227         if (set_flag == flag_is_set) {
3228                 DEBUG(DEBUG_NOTICE, ("Node %d is %s %s\n", options.pnn,
3229                                      (set_flag ? "already" : "not"), desc));
3230                 return 0;
3231         }
3232
3233         do {
3234                 if (!handler(ctdb, data)) {
3235                         DEBUG(DEBUG_WARNING,
3236                               ("Failed to send control to set state %s on node %u, try again\n",
3237                                desc, options.pnn));
3238                 }
3239
3240                 sleep(1);
3241
3242                 /* Read the nodemap and verify the change took effect.
3243                  * Even if the above control/hanlder timed out then it
3244                  * could still have worked!
3245                  */
3246                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
3247                                          ctdb, &nodemap);
3248                 if (ret != 0) {
3249                         DEBUG(DEBUG_WARNING,
3250                               ("Unable to get nodemap from local node, try again\n"));
3251                 }
3252                 flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3253         } while (nodemap == NULL || (set_flag != flag_is_set));
3254
3255         return ipreallocate(ctdb);
3256 }
3257
3258 /* Administratively disable a node */
3259 static bool update_flags_disabled(struct ctdb_context *ctdb, void *data)
3260 {
3261         int ret;
3262
3263         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3264                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
3265         return ret == 0;
3266 }
3267
3268 static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
3269 {
3270         return update_flags_and_ipreallocate(ctdb, NULL,
3271                                                   update_flags_disabled,
3272                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3273                                                   "disabled",
3274                                                   true /* set_flag*/);
3275 }
3276
3277 /* Administratively re-enable a node */
3278 static bool update_flags_not_disabled(struct ctdb_context *ctdb, void *data)
3279 {
3280         int ret;
3281
3282         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3283                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
3284         return ret == 0;
3285 }
3286
3287 static int control_enable(struct ctdb_context *ctdb,  int argc, const char **argv)
3288 {
3289         return update_flags_and_ipreallocate(ctdb, NULL,
3290                                                   update_flags_not_disabled,
3291                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3292                                                   "disabled",
3293                                                   false /* set_flag*/);
3294 }
3295
3296 /* Stop a node */
3297 static bool update_flags_stopped(struct ctdb_context *ctdb, void *data)
3298 {
3299         int ret;
3300
3301         ret = ctdb_ctrl_stop_node(ctdb, TIMELIMIT(), options.pnn);
3302
3303         return ret == 0;
3304 }
3305
3306 static int control_stop(struct ctdb_context *ctdb, int argc, const char **argv)
3307 {
3308         return update_flags_and_ipreallocate(ctdb, NULL,
3309                                                   update_flags_stopped,
3310                                                   NODE_FLAGS_STOPPED,
3311                                                   "stopped",
3312                                                   true /* set_flag*/);
3313 }
3314
3315 /* Continue a stopped node */
3316 static bool update_flags_not_stopped(struct ctdb_context *ctdb, void *data)
3317 {
3318         int ret;
3319
3320         ret = ctdb_ctrl_continue_node(ctdb, TIMELIMIT(), options.pnn);
3321
3322         return ret == 0;
3323 }
3324
3325 static int control_continue(struct ctdb_context *ctdb, int argc, const char **argv)
3326 {
3327         return update_flags_and_ipreallocate(ctdb, NULL,
3328                                                   update_flags_not_stopped,
3329                                                   NODE_FLAGS_STOPPED,
3330                                                   "stopped",
3331                                                   false /* set_flag */);
3332 }
3333
3334 static uint32_t get_generation(struct ctdb_context *ctdb)
3335 {
3336         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3337         struct ctdb_vnn_map *vnnmap=NULL;
3338         int ret;
3339         uint32_t generation;
3340
3341         /* wait until the recmaster is not in recovery mode */
3342         while (1) {
3343                 uint32_t recmode, recmaster;
3344
3345                 if (vnnmap != NULL) {
3346                         talloc_free(vnnmap);
3347                         vnnmap = NULL;
3348                 }
3349
3350                 /* get the recmaster */
3351                 ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), CTDB_CURRENT_NODE, &recmaster);
3352                 if (ret != 0) {
3353                         DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
3354                         talloc_free(tmp_ctx);
3355                         exit(10);
3356                 }
3357
3358                 /* get recovery mode */
3359                 ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), recmaster, &recmode);
3360                 if (ret != 0) {
3361                         DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
3362                         talloc_free(tmp_ctx);
3363                         exit(10);
3364                 }
3365
3366                 /* get the current generation number */
3367                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), recmaster, tmp_ctx, &vnnmap);
3368                 if (ret != 0) {
3369                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from recmaster (%u)\n", recmaster));
3370                         talloc_free(tmp_ctx);
3371                         exit(10);
3372                 }
3373
3374                 if ((recmode == CTDB_RECOVERY_NORMAL) && (vnnmap->generation != 1)) {
3375                         generation = vnnmap->generation;
3376                         talloc_free(tmp_ctx);
3377                         return generation;
3378                 }
3379                 sleep(1);
3380         }
3381 }
3382
3383 /* Ban a node */
3384 static bool update_state_banned(struct ctdb_context *ctdb, void *data)
3385 {
3386         struct ctdb_ban_time *bantime = (struct ctdb_ban_time *)data;
3387         int ret;
3388
3389         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, bantime);
3390
3391         return ret == 0;
3392 }
3393
3394 static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
3395 {
3396         struct ctdb_ban_time bantime;
3397
3398         if (argc < 1) {
3399                 usage();
3400         }
3401
3402         bantime.pnn  = options.pnn;
3403         bantime.time = strtoul(argv[0], NULL, 0);
3404
3405         if (bantime.time == 0) {
3406                 DEBUG(DEBUG_ERR, ("Invalid ban time specified - must be >0\n"));
3407                 return -1;
3408         }
3409
3410         return update_flags_and_ipreallocate(ctdb, &bantime,
3411                                                   update_state_banned,
3412                                                   NODE_FLAGS_BANNED,
3413                                                   "banned",
3414                                                   true /* set_flag*/);
3415 }
3416
3417
3418 /* Unban a node */
3419 static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
3420 {
3421         struct ctdb_ban_time bantime;
3422
3423         bantime.pnn  = options.pnn;
3424         bantime.time = 0;
3425
3426         return update_flags_and_ipreallocate(ctdb, &bantime,
3427                                                   update_state_banned,
3428                                                   NODE_FLAGS_BANNED,
3429                                                   "banned",
3430                                                   false /* set_flag*/);
3431 }
3432
3433 /*
3434   show ban information for a node
3435  */
3436 static int control_showban(struct ctdb_context *ctdb, int argc, const char **argv)
3437 {
3438         int ret;
3439         struct ctdb_node_map *nodemap=NULL;
3440         struct ctdb_ban_time *bantime;
3441
3442         /* verify the node exists */
3443         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3444         if (ret != 0) {
3445                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3446                 return ret;
3447         }
3448
3449         ret = ctdb_ctrl_get_ban(ctdb, TIMELIMIT(), options.pnn, ctdb, &bantime);
3450         if (ret != 0) {
3451                 DEBUG(DEBUG_ERR,("Showing ban info for node %d failed.\n", options.pnn));
3452                 return -1;
3453         }
3454
3455         if (bantime->time == 0) {
3456                 printf("Node %u is not banned\n", bantime->pnn);
3457         } else {
3458                 printf("Node %u is banned, %d seconds remaining\n",
3459                        bantime->pnn, bantime->time);
3460         }
3461
3462         return 0;
3463 }
3464
3465 /*
3466   shutdown a daemon
3467  */
3468 static int control_shutdown(struct ctdb_context *ctdb, int argc, const char **argv)
3469 {
3470         int ret;
3471
3472         ret = ctdb_ctrl_shutdown(ctdb, TIMELIMIT(), options.pnn);
3473         if (ret != 0) {
3474                 DEBUG(DEBUG_ERR, ("Unable to shutdown node %u\n", options.pnn));
3475                 return ret;
3476         }
3477
3478         return 0;
3479 }
3480
3481 /*
3482   trigger a recovery
3483  */
3484 static int control_recover(struct ctdb_context *ctdb, int argc, const char **argv)
3485 {
3486         int ret;
3487         uint32_t generation, next_generation;
3488
3489         /* record the current generation number */
3490         generation = get_generation(ctdb);
3491
3492         ret = ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3493         if (ret != 0) {
3494                 DEBUG(DEBUG_ERR, ("Unable to set recovery mode\n"));
3495                 return ret;
3496         }
3497
3498         /* wait until we are in a new generation */
3499         while (1) {
3500                 next_generation = get_generation(ctdb);
3501                 if (next_generation != generation) {
3502                         return 0;
3503                 }
3504                 sleep(1);
3505         }
3506
3507         return 0;
3508 }
3509
3510
3511 /*
3512   display monitoring mode of a remote node
3513  */
3514 static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
3515 {
3516         uint32_t monmode;
3517         int ret;
3518
3519         ret = ctdb_ctrl_getmonmode(ctdb, TIMELIMIT(), options.pnn, &monmode);
3520         if (ret != 0) {
3521                 DEBUG(DEBUG_ERR, ("Unable to get monmode from node %u\n", options.pnn));
3522                 return ret;
3523         }
3524         if (!options.machinereadable){
3525                 printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode);
3526         } else {
3527                 printf(":mode:\n");
3528                 printf(":%d:\n",monmode);
3529         }
3530         return 0;
3531 }
3532
3533
3534 /*
3535   display capabilities of a remote node
3536  */
3537 static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const char **argv)
3538 {
3539         uint32_t capabilities;
3540         int ret;
3541
3542         ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), options.pnn, &capabilities);
3543         if (ret != 0) {
3544                 DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", options.pnn));
3545                 return -1;
3546         }
3547
3548         if (!options.machinereadable){
3549                 printf("RECMASTER: %s\n", (capabilities&CTDB_CAP_RECMASTER)?"YES":"NO");
3550                 printf("LMASTER: %s\n", (capabilities&CTDB_CAP_LMASTER)?"YES":"NO");
3551                 printf("LVS: %s\n", (capabilities&CTDB_CAP_LVS)?"YES":"NO");
3552                 printf("NATGW: %s\n", (capabilities&CTDB_CAP_NATGW)?"YES":"NO");
3553         } else {
3554                 printf(":RECMASTER:LMASTER:LVS:NATGW:\n");
3555                 printf(":%d:%d:%d:%d:\n",
3556                         !!(capabilities&CTDB_CAP_RECMASTER),
3557                         !!(capabilities&CTDB_CAP_LMASTER),
3558                         !!(capabilities&CTDB_CAP_LVS),
3559                         !!(capabilities&CTDB_CAP_NATGW));
3560         }
3561         return 0;
3562 }
3563
3564 /*
3565   display lvs configuration
3566  */
3567
3568 static uint32_t lvs_exclude_flags[] = {
3569         /* Look for a nice healthy node */
3570         NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED,
3571         /* If not found, an UNHEALTHY node will do */
3572         NODE_FLAGS_INACTIVE|NODE_FLAGS_PERMANENTLY_DISABLED,
3573         0,
3574 };
3575
3576 static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
3577 {
3578         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3579         struct ctdb_node_map *orig_nodemap=NULL;
3580         struct ctdb_node_map *nodemap;
3581         int i, ret;
3582
3583         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3584                                    tmp_ctx, &orig_nodemap);
3585         if (ret != 0) {
3586                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3587                 talloc_free(tmp_ctx);
3588                 return -1;
3589         }
3590
3591         nodemap = filter_nodemap_by_capabilities(ctdb, orig_nodemap,
3592                                                  CTDB_CAP_LVS, false);
3593         if (nodemap == NULL) {
3594                 /* No memory */
3595                 ret = -1;
3596                 goto done;
3597         }
3598
3599         ret = 0;
3600
3601         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3602                 struct ctdb_node_map *t =
3603                         filter_nodemap_by_flags(ctdb, nodemap,
3604                                                 lvs_exclude_flags[i]);
3605                 if (t == NULL) {
3606                         /* No memory */
3607                         ret = -1;
3608                         goto done;
3609                 }
3610                 if (t->num > 0) {
3611                         /* At least 1 node without excluded flags */
3612                         int j;
3613                         for (j = 0; j < t->num; j++) {
3614                                 printf("%d:%s\n", t->nodes[j].pnn,
3615                                        ctdb_addr_to_str(&t->nodes[j].addr));
3616                         }
3617                         goto done;
3618                 }
3619                 talloc_free(t);
3620         }
3621 done:
3622         talloc_free(tmp_ctx);
3623         return ret;
3624 }
3625
3626 /*
3627   display who is the lvs master
3628  */
3629 static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **argv)
3630 {
3631         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3632         struct ctdb_node_map *nodemap=NULL;
3633         int i, ret;
3634
3635         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3636                                    tmp_ctx, &nodemap);
3637         if (ret != 0) {
3638                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3639                 talloc_free(tmp_ctx);
3640                 return -1;
3641         }
3642
3643         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3644                 struct ctdb_node_map *t =
3645                         filter_nodemap_by_flags(ctdb, nodemap,
3646                                                 lvs_exclude_flags[i]);
3647                 if (t == NULL) {
3648                         /* No memory */
3649                         ret = -1;
3650                         goto done;
3651                 }
3652                 if (t->num > 0) {
3653                         struct ctdb_node_map *n;
3654                         n = filter_nodemap_by_capabilities(ctdb,
3655                                                            t,
3656                                                            CTDB_CAP_LVS,
3657                                                            true);
3658                         if (n == NULL) {
3659                                 /* No memory */
3660                                 ret = -1;
3661                                 goto done;
3662                         }
3663                         if (n->num > 0) {
3664                                 ret = 0;
3665                                 printf(options.machinereadable ?
3666                                        "%d\n" : "Node %d is LVS master\n",
3667                                        n->nodes[0].pnn);
3668                                 goto done;
3669                         }
3670                 }
3671                 talloc_free(t);
3672         }
3673
3674         printf("There is no LVS master\n");
3675         ret = 255;
3676 done:
3677         talloc_free(tmp_ctx);
3678         return ret;
3679 }
3680
3681 /*
3682   disable monitoring on a  node
3683  */
3684 static int control_disable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3685 {
3686
3687         int ret;
3688
3689         ret = ctdb_ctrl_disable_monmode(ctdb, TIMELIMIT(), options.pnn);
3690         if (ret != 0) {
3691                 DEBUG(DEBUG_ERR, ("Unable to disable monmode on node %u\n", options.pnn));
3692                 return ret;
3693         }
3694         printf("Monitoring mode:%s\n","DISABLED");
3695
3696         return 0;
3697 }
3698
3699 /*
3700   enable monitoring on a  node
3701  */
3702 static int control_enable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3703 {
3704
3705         int ret;
3706
3707         ret = ctdb_ctrl_enable_monmode(ctdb, TIMELIMIT(), options.pnn);
3708         if (ret != 0) {
3709                 DEBUG(DEBUG_ERR, ("Unable to enable monmode on node %u\n", options.pnn));
3710                 return ret;
3711         }
3712         printf("Monitoring mode:%s\n","ACTIVE");
3713
3714         return 0;
3715 }
3716
3717 /*
3718   display remote list of keys/data for a db
3719  */
3720 static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
3721 {
3722         const char *db_name;
3723         struct ctdb_db_context *ctdb_db;
3724         int ret;
3725         struct ctdb_dump_db_context c;
3726         uint8_t flags;
3727
3728         if (argc < 1) {
3729                 usage();
3730         }
3731
3732         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3733                 return -1;
3734         }
3735
3736         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3737         if (ctdb_db == NULL) {
3738                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3739                 return -1;
3740         }
3741
3742         if (options.printlmaster) {
3743                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn,
3744                                           ctdb, &ctdb->vnn_map);
3745                 if (ret != 0) {
3746                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
3747                                           options.pnn));
3748                         return ret;
3749                 }
3750         }
3751
3752         ZERO_STRUCT(c);
3753         c.f = stdout;
3754         c.printemptyrecords = (bool)options.printemptyrecords;
3755         c.printdatasize = (bool)options.printdatasize;
3756         c.printlmaster = (bool)options.printlmaster;
3757         c.printhash = (bool)options.printhash;
3758         c.printrecordflags = (bool)options.printrecordflags;
3759
3760         /* traverse and dump the cluster tdb */
3761         ret = ctdb_dump_db(ctdb_db, &c);
3762         if (ret == -1) {
3763                 DEBUG(DEBUG_ERR, ("Unable to dump database\n"));
3764                 DEBUG(DEBUG_ERR, ("Maybe try 'ctdb getdbstatus %s'"
3765                                   " and 'ctdb getvar AllowUnhealthyDBRead'\n",
3766                                   db_name));
3767                 return -1;
3768         }
3769         talloc_free(ctdb_db);
3770
3771         printf("Dumped %d records\n", ret);
3772         return 0;
3773 }
3774
3775 struct cattdb_data {
3776         struct ctdb_context *ctdb;
3777         uint32_t count;
3778 };
3779
3780 static int cattdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private_data)
3781 {
3782         struct cattdb_data *d = private_data;
3783         struct ctdb_dump_db_context c;
3784
3785         d->count++;
3786
3787         ZERO_STRUCT(c);
3788         c.f = stdout;
3789         c.printemptyrecords = (bool)options.printemptyrecords;
3790         c.printdatasize = (bool)options.printdatasize;
3791         c.printlmaster = false;
3792         c.printhash = (bool)options.printhash;
3793         c.printrecordflags = true;
3794
3795         return ctdb_dumpdb_record(d->ctdb, key, data, &c);
3796 }
3797
3798 /*
3799   cat the local tdb database using same format as catdb
3800  */
3801 static int control_cattdb(struct ctdb_context *ctdb, int argc, const char **argv)
3802 {
3803         const char *db_name;
3804         struct ctdb_db_context *ctdb_db;
3805         struct cattdb_data d;
3806         uint8_t flags;
3807
3808         if (argc < 1) {
3809                 usage();
3810         }
3811
3812         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3813                 return -1;
3814         }
3815
3816         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3817         if (ctdb_db == NULL) {
3818                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3819                 return -1;
3820         }
3821
3822         /* traverse the local tdb */
3823         d.count = 0;
3824         d.ctdb  = ctdb;
3825         if (tdb_traverse_read(ctdb_db->ltdb->tdb, cattdb_traverse, &d) == -1) {
3826                 printf("Failed to cattdb data\n");
3827                 exit(10);
3828         }
3829         talloc_free(ctdb_db);
3830
3831         printf("Dumped %d records\n", d.count);
3832         return 0;
3833 }
3834
3835 /*
3836   display the content of a database key
3837  */
3838 static int control_readkey(struct ctdb_context *ctdb, int argc, const char **argv)
3839 {
3840         const char *db_name;
3841         struct ctdb_db_context *ctdb_db;
3842         struct ctdb_record_handle *h;
3843         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3844         TDB_DATA key, data;
3845         uint8_t flags;
3846
3847         if (argc < 2) {
3848                 usage();
3849         }
3850
3851         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3852                 return -1;
3853         }
3854
3855         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3856         if (ctdb_db == NULL) {
3857                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3858                 return -1;
3859         }
3860
3861         key.dptr  = discard_const(argv[1]);
3862         key.dsize = strlen((char *)key.dptr);
3863
3864         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3865         if (h == NULL) {
3866                 printf("Failed to fetch record '%s' on node %d\n",
3867                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3868                 talloc_free(tmp_ctx);
3869                 exit(10);
3870         }
3871
3872         printf("Data: size:%d ptr:[%.*s]\n", (int)data.dsize, (int)data.dsize, data.dptr);
3873
3874         talloc_free(tmp_ctx);
3875         talloc_free(ctdb_db);
3876         return 0;
3877 }
3878
3879 /*
3880   display the content of a database key
3881  */
3882 static int control_writekey(struct ctdb_context *ctdb, int argc, const char **argv)
3883 {
3884         const char *db_name;
3885         struct ctdb_db_context *ctdb_db;
3886         struct ctdb_record_handle *h;
3887         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3888         TDB_DATA key, data;
3889         uint8_t flags;
3890
3891         if (argc < 3) {
3892                 usage();
3893         }
3894
3895         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3896                 return -1;
3897         }
3898
3899         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3900         if (ctdb_db == NULL) {
3901                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3902                 return -1;
3903         }
3904
3905         key.dptr  = discard_const(argv[1]);
3906         key.dsize = strlen((char *)key.dptr);
3907
3908         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3909         if (h == NULL) {
3910                 printf("Failed to fetch record '%s' on node %d\n",
3911                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3912                 talloc_free(tmp_ctx);
3913                 exit(10);
3914         }
3915
3916         data.dptr  = discard_const(argv[2]);
3917         data.dsize = strlen((char *)data.dptr);
3918
3919         if (ctdb_record_store(h, data) != 0) {
3920                 printf("Failed to store record\n");
3921         }
3922
3923         talloc_free(h);
3924         talloc_free(tmp_ctx);
3925         talloc_free(ctdb_db);
3926         return 0;
3927 }
3928
3929 /*
3930   fetch a record from a persistent database
3931  */
3932 static int control_pfetch(struct ctdb_context *ctdb, int argc, const char **argv)
3933 {
3934         const char *db_name;
3935         struct ctdb_db_context *ctdb_db;
3936         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3937         struct ctdb_transaction_handle *h;
3938         TDB_DATA key, data;
3939         int fd, ret;
3940         bool persistent;
3941         uint8_t flags;
3942
3943         if (argc < 2) {
3944                 talloc_free(tmp_ctx);
3945                 usage();
3946         }
3947
3948         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3949                 talloc_free(tmp_ctx);
3950                 return -1;
3951         }
3952
3953         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
3954         if (!persistent) {
3955                 DEBUG(DEBUG_ERR,("Database '%s' is not persistent\n", db_name));
3956                 talloc_free(tmp_ctx);
3957                 return -1;
3958         }
3959
3960         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
3961         if (ctdb_db == NULL) {
3962                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3963                 talloc_free(tmp_ctx);
3964                 return -1;
3965         }
3966
3967         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
3968         if (h == NULL) {
3969                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
3970                 talloc_free(tmp_ctx);
3971                 return -1;
3972         }
3973
3974         key.dptr  = discard_const(argv[1]);
3975         key.dsize = strlen(argv[1]);
3976         ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data);
3977         if (ret != 0) {
3978                 DEBUG(DEBUG_ERR,("Failed to fetch record\n"));
3979                 talloc_free(tmp_ctx);
3980                 return -1;
3981         }
3982
3983         if (data.dsize == 0 || data.dptr == NULL) {
3984                 DEBUG(DEBUG_ERR,("Record is empty\n"));
3985                 talloc_free(tmp_ctx);
3986                 return -1;
3987         }
3988
3989         if (argc == 3) {
3990           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
3991                 if (fd == -1) {
3992                         DEBUG(DEBUG_ERR,("Failed to open output file %s\n", argv[2]));
3993                         talloc_free(tmp_ctx);
3994                         return -1;
3995                 }
3996                 sys_write(fd, data.dptr, data.dsize);
3997                 close(fd);
3998         } else {
3999                 sys_write(1, data.dptr, data.dsize);
4000         }
4001
4002         /* abort the transaction */
4003         talloc_free(h);
4004
4005
4006         talloc_free(tmp_ctx);
4007         return 0;
4008 }
4009
4010 /*
4011   fetch a record from a tdb-file
4012  */
4013 static int control_tfetch(struct ctdb_context *ctdb, int argc, const char **argv)
4014 {
4015         const char *tdb_file;
4016         TDB_CONTEXT *tdb;
4017         TDB_DATA key, data;
4018         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
4019         int fd;
4020
4021         if (argc < 2) {
4022                 usage();
4023         }
4024
4025         tdb_file = argv[0];
4026
4027         tdb = tdb_open(tdb_file, 0, 0, O_RDONLY, 0);
4028         if (tdb == NULL) {
4029                 printf("Failed to open TDB file %s\n", tdb_file);
4030                 return -1;
4031         }
4032
4033         if (!strncmp(argv[1], "0x", 2)) {
4034                 key = hextodata(tmp_ctx, argv[1] + 2);
4035                 if (key.dsize == 0) {
4036                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4037                         return -1;
4038                 }
4039         } else {
4040                 key.dptr  = discard_const(argv[1]);
4041                 key.dsize = strlen(argv[1]);
4042         }
4043
4044         data = tdb_fetch(tdb, key);
4045         if (data.dptr == NULL || data.dsize < sizeof(struct ctdb_ltdb_header)) {
4046                 printf("Failed to read record %s from tdb %s\n", argv[1], tdb_file);
4047                 tdb_close(tdb);
4048                 return -1;
4049         }
4050
4051         tdb_close(tdb);
4052
4053         if (argc == 3) {
4054           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
4055                 if (fd == -1) {
4056                         printf("Failed to open output file %s\n", argv[2]);
4057                         return -1;
4058                 }
4059                 if (options.verbose){
4060                         sys_write(fd, data.dptr, data.dsize);
4061                 } else {
4062                         sys_write(fd, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
4063                 }
4064                 close(fd);
4065         } else {
4066                 if (options.verbose){
4067                         sys_write(1, data.dptr, data.dsize);
4068                 } else {
4069                         sys_write(1, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
4070                 }
4071         }
4072
4073         talloc_free(tmp_ctx);
4074         return 0;
4075 }
4076
4077 /*
4078   store a record and header to a tdb-file
4079  */
4080 static int control_tstore(struct ctdb_context *ctdb, int argc, const char **argv)
4081 {
4082         const char *tdb_file;
4083         TDB_CONTEXT *tdb;
4084         TDB_DATA key, value, data;
4085         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
4086         struct ctdb_ltdb_header header;
4087
4088         if (argc < 3) {
4089                 usage();
4090         }
4091
4092         tdb_file = argv[0];
4093
4094         tdb = tdb_open(tdb_file, 0, 0, O_RDWR, 0);
4095         if (tdb == NULL) {
4096                 printf("Failed to open TDB file %s\n", tdb_file);
4097                 return -1;
4098         }
4099
4100         if (!strncmp(argv[1], "0x", 2)) {
4101                 key = hextodata(tmp_ctx, argv[1] + 2);
4102                 if (key.dsize == 0) {
4103                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4104                         return -1;
4105                 }
4106         } else {
4107                 key.dptr  = discard_const(argv[1]);
4108                 key.dsize = strlen(argv[1]);
4109         }
4110
4111         if (!strncmp(argv[2], "0x", 2)) {
4112                 value = hextodata(tmp_ctx, argv[2] + 2);
4113                 if (value.dsize == 0) {
4114                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[2]);
4115                         return -1;
4116                 }
4117         } else {
4118                 value.dptr  = discard_const(argv[2]);
4119                 value.dsize = strlen(argv[2]);
4120         }
4121
4122         ZERO_STRUCT(header);
4123         if (argc > 3) {
4124                 header.rsn = atoll(argv[3]);
4125         }
4126         if (argc > 4) {
4127                 header.dmaster = atoi(argv[4]);
4128         }
4129         if (argc > 5) {
4130                 header.flags = atoi(argv[5]);
4131         }
4132
4133         data.dsize = sizeof(struct ctdb_ltdb_header) + value.dsize;
4134         data.dptr = talloc_size(tmp_ctx, data.dsize);
4135         if (data.dptr == NULL) {
4136                 printf("Failed to allocate header+value\n");
4137                 return -1;
4138         }
4139
4140         *(struct ctdb_ltdb_header *)data.dptr = header;
4141         memcpy(data.dptr + sizeof(struct ctdb_ltdb_header), value.dptr, value.dsize);
4142
4143         if (tdb_store(tdb, key, data, TDB_REPLACE) != 0) {
4144                 printf("Failed to write record %s to tdb %s\n", argv[1], tdb_file);
4145                 tdb_close(tdb);
4146                 return -1;
4147         }
4148
4149         tdb_close(tdb);
4150
4151         talloc_free(tmp_ctx);
4152         return 0;
4153 }
4154
4155 /*
4156   write a record to a persistent database
4157  */
4158 static int control_pstore(struct ctdb_context *ctdb, int argc, const char **argv)
4159 {
4160         const char *db_name;
4161         struct ctdb_db_context *ctdb_db;
4162         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4163         struct ctdb_transaction_handle *h;
4164         struct stat st;
4165         TDB_DATA key, data;
4166         int fd, ret;
4167
4168         if (argc < 3) {
4169                 talloc_free(tmp_ctx);
4170                 usage();
4171         }
4172
4173         fd = open(argv[2], O_RDONLY);
4174         if (fd == -1) {
4175                 DEBUG(DEBUG_ERR,("Failed to open file containing record data : %s  %s\n", argv[2], strerror(errno)));
4176                 talloc_free(tmp_ctx);
4177                 return -1;
4178         }
4179
4180         ret = fstat(fd, &st);
4181         if (ret == -1) {
4182                 DEBUG(DEBUG_ERR,("fstat of file %s failed: %s\n", argv[2], strerror(errno)));
4183                 close(fd);
4184                 talloc_free(tmp_ctx);
4185                 return -1;
4186         }
4187
4188         if (!S_ISREG(st.st_mode)) {
4189                 DEBUG(DEBUG_ERR,("Not a regular file %s\n", argv[2]));
4190                 close(fd);
4191                 talloc_free(tmp_ctx);
4192                 return -1;
4193         }
4194
4195         data.dsize = st.st_size;
4196         if (data.dsize == 0) {
4197                 data.dptr  = NULL;
4198         } else {
4199                 data.dptr = talloc_size(tmp_ctx, data.dsize);
4200                 if (data.dptr == NULL) {
4201                         DEBUG(DEBUG_ERR,("Failed to talloc %d of memory to store record data\n", (int)data.dsize));
4202                         close(fd);
4203                         talloc_free(tmp_ctx);
4204                         return -1;
4205                 }
4206                 ret = sys_read(fd, data.dptr, data.dsize);
4207                 if (ret != data.dsize) {
4208                         DEBUG(DEBUG_ERR,("Failed to read %d bytes of record data\n", (int)data.dsize));
4209                         close(fd);
4210                         talloc_free(tmp_ctx);
4211                         return -1;
4212                 }
4213         }
4214         close(fd);
4215
4216
4217         db_name = argv[0];
4218
4219         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4220         if (ctdb_db == NULL) {
4221                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4222                 talloc_free(tmp_ctx);
4223                 return -1;
4224         }
4225
4226         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4227         if (h == NULL) {
4228                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4229                 talloc_free(tmp_ctx);
4230                 return -1;
4231         }
4232
4233         key.dptr  = discard_const(argv[1]);
4234         key.dsize = strlen(argv[1]);
4235         ret = ctdb_transaction_store(h, key, data);
4236         if (ret != 0) {
4237                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4238                 talloc_free(tmp_ctx);
4239                 return -1;
4240         }
4241
4242         ret = ctdb_transaction_commit(h);
4243         if (ret != 0) {
4244                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4245                 talloc_free(tmp_ctx);
4246                 return -1;
4247         }
4248
4249
4250         talloc_free(tmp_ctx);
4251         return 0;
4252 }
4253
4254 /*
4255  * delete a record from a persistent database
4256  */
4257 static int control_pdelete(struct ctdb_context *ctdb, int argc, const char **argv)
4258 {
4259         const char *db_name;
4260         struct ctdb_db_context *ctdb_db;
4261         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4262         struct ctdb_transaction_handle *h;
4263         TDB_DATA key;
4264         int ret;
4265         bool persistent;
4266         uint8_t flags;
4267
4268         if (argc < 2) {
4269                 talloc_free(tmp_ctx);
4270                 usage();
4271         }
4272
4273         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
4274                 talloc_free(tmp_ctx);
4275                 return -1;
4276         }
4277
4278         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
4279         if (!persistent) {
4280                 DEBUG(DEBUG_ERR, ("Database '%s' is not persistent\n", db_name));
4281                 talloc_free(tmp_ctx);
4282                 return -1;
4283         }
4284
4285         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
4286         if (ctdb_db == NULL) {
4287                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n", db_name));
4288                 talloc_free(tmp_ctx);
4289                 return -1;
4290         }
4291
4292         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4293         if (h == NULL) {
4294                 DEBUG(DEBUG_ERR, ("Failed to start transaction on database %s\n", db_name));
4295                 talloc_free(tmp_ctx);
4296                 return -1;
4297         }
4298
4299         key.dptr = discard_const(argv[1]);
4300         key.dsize = strlen(argv[1]);
4301         ret = ctdb_transaction_store(h, key, tdb_null);
4302         if (ret != 0) {
4303                 DEBUG(DEBUG_ERR, ("Failed to delete record\n"));
4304                 talloc_free(tmp_ctx);
4305                 return -1;
4306         }
4307
4308         ret = ctdb_transaction_commit(h);
4309         if (ret != 0) {
4310                 DEBUG(DEBUG_ERR, ("Failed to commit transaction\n"));
4311                 talloc_free(tmp_ctx);
4312                 return -1;
4313         }
4314
4315         talloc_free(tmp_ctx);
4316         return 0;
4317 }
4318
4319 static const char *ptrans_parse_string(TALLOC_CTX *mem_ctx, const char *s,
4320                                        TDB_DATA *data)
4321 {
4322         const char *t;
4323         size_t n;
4324         const char *ret; /* Next byte after successfully parsed value */
4325
4326         /* Error, unless someone says otherwise */
4327         ret = NULL;
4328         /* Indicates no value to parse */
4329         *data = tdb_null;
4330
4331         /* Skip whitespace */
4332         n = strspn(s, " \t");
4333         t = s + n;
4334
4335         if (t[0] == '"') {
4336                 /* Quoted ASCII string - no wide characters! */
4337                 t++;
4338                 n = strcspn(t, "\"");
4339                 if (t[n] == '"') {
4340                         if (n > 0) {
4341                                 data->dsize = n;
4342                                 data->dptr = talloc_memdup(mem_ctx, t, n);
4343                                 CTDB_NOMEM_ABORT(data->dptr);
4344                         }
4345                         ret = t + n + 1;
4346                 } else {
4347                         DEBUG(DEBUG_WARNING,("Unmatched \" in input %s\n", s));
4348                 }
4349         } else {
4350                 DEBUG(DEBUG_WARNING,("Unsupported input format in %s\n", s));
4351         }
4352
4353         return ret;
4354 }
4355
4356 static bool ptrans_get_key_value(TALLOC_CTX *mem_ctx, FILE *file,
4357                                  TDB_DATA *key, TDB_DATA *value)
4358 {
4359         char line [1024]; /* FIXME: make this more flexible? */
4360         const char *t;
4361         char *ptr;
4362
4363         ptr = fgets(line, sizeof(line), file);
4364
4365         if (ptr == NULL) {
4366                 return false;
4367         }
4368
4369         /* Get key */
4370         t = ptrans_parse_string(mem_ctx, line, key);
4371         if (t == NULL || key->dptr == NULL) {
4372                 /* Line Ignored but not EOF */
4373                 return true;
4374         }
4375
4376         /* Get value */
4377         t = ptrans_parse_string(mem_ctx, t, value);
4378         if (t == NULL) {
4379                 /* Line Ignored but not EOF */
4380                 talloc_free(key->dptr);
4381                 *key = tdb_null;
4382                 return true;
4383         }
4384
4385         return true;
4386 }
4387
4388 /*
4389  * Update a persistent database as per file/stdin
4390  */
4391 static int control_ptrans(struct ctdb_context *ctdb,
4392                           int argc, const char **argv)
4393 {
4394         const char *db_name;
4395         struct ctdb_db_context *ctdb_db;
4396         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4397         struct ctdb_transaction_handle *h;
4398         TDB_DATA key, value;
4399         FILE *file;
4400         int ret;
4401
4402         if (argc < 1) {
4403                 talloc_free(tmp_ctx);
4404                 usage();
4405         }
4406
4407         file = stdin;
4408         if (argc == 2) {
4409                 file = fopen(argv[1], "r");
4410                 if (file == NULL) {
4411                         DEBUG(DEBUG_ERR,("Unable to open file for reading '%s'\n", argv[1]));
4412                         talloc_free(tmp_ctx);
4413                         return -1;
4414                 }
4415         }
4416
4417         db_name = argv[0];
4418
4419         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4420         if (ctdb_db == NULL) {
4421                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4422                 goto error;
4423         }
4424
4425         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4426         if (h == NULL) {
4427                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4428                 goto error;
4429         }
4430
4431         while (ptrans_get_key_value(tmp_ctx, file, &key, &value)) {
4432                 if (key.dsize != 0) {
4433                         ret = ctdb_transaction_store(h, key, value);
4434                         /* Minimise memory use */
4435                         talloc_free(key.dptr);
4436                         if (value.dptr != NULL) {
4437                                 talloc_free(value.dptr);
4438                         }
4439                         if (ret != 0) {
4440                                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4441                                 ctdb_transaction_cancel(h);
4442                                 goto error;
4443                         }
4444                 }
4445         }
4446
4447         ret = ctdb_transaction_commit(h);
4448         if (ret != 0) {
4449                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4450                 goto error;
4451         }
4452
4453         if (file != stdin) {
4454                 fclose(file);
4455         }
4456         talloc_free(tmp_ctx);
4457         return 0;
4458
4459 error:
4460         if (file != stdin) {
4461                 fclose(file);
4462         }
4463
4464         talloc_free(tmp_ctx);
4465         return -1;
4466 }
4467
4468 /*
4469   check if a service is bound to a port or not
4470  */
4471 static int control_chktcpport(struct ctdb_context *ctdb, int argc, const char **argv)
4472 {
4473         int s, ret;
4474         int v;
4475         int port;
4476         struct sockaddr_in sin;
4477
4478         if (argc != 1) {
4479                 printf("Use: ctdb chktcport <port>\n");
4480                 return EINVAL;
4481         }
4482
4483         port = atoi(argv[0]);
4484
4485         s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
4486         if (s == -1) {
4487                 printf("Failed to open local socket\n");
4488                 return errno;
4489         }
4490
4491         v = fcntl(s, F_GETFL, 0);
4492         if (v == -1 || fcntl(s, F_SETFL, v | O_NONBLOCK) != 0) {
4493                 printf("Unable to set socket non-blocking: %s\n", strerror(errno));
4494         }
4495
4496         bzero(&sin, sizeof(sin));
4497         sin.sin_family = PF_INET;
4498         sin.sin_port   = htons(port);
4499         ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
4500         close(s);
4501         if (ret == -1) {
4502                 printf("Failed to bind to local socket: %d %s\n", errno, strerror(errno));
4503                 return errno;
4504         }
4505
4506         return 0;
4507 }
4508
4509
4510 /* Reload public IPs on a specified nodes */
4511 static int control_reloadips(struct ctdb_context *ctdb, int argc, const char **argv)
4512 {
4513         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4514         uint32_t *nodes;
4515         uint32_t pnn_mode;
4516         uint32_t timeout;
4517         int ret;
4518
4519         assert_single_node_only();
4520
4521         if (argc > 1) {
4522                 usage();
4523         }
4524
4525         /* Determine the nodes where IPs need to be reloaded */
4526         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
4527                               options.pnn, true, &nodes, &pnn_mode)) {
4528                 ret = -1;
4529                 goto done;
4530         }
4531
4532 again:
4533         /* Disable takeover runs on all connected nodes.  A reply
4534          * indicating success is needed from each node so all nodes
4535          * will need to be active.  This will retry until maxruntime
4536          * is exceeded, hence no error handling.
4537          *
4538          * A check could be added to not allow reloading of IPs when
4539          * there are disconnected nodes.  However, this should
4540          * probably be left up to the administrator.
4541          */
4542         timeout = LONGTIMEOUT;
4543         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4544                         "Disable takeover runs", true);
4545
4546         /* Now tell all the desired nodes to reload their public IPs.
4547          * Keep trying this until it succeeds.  This assumes all
4548          * failures are transient, which might not be true...
4549          */
4550         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_RELOAD_PUBLIC_IPS,
4551                                       nodes, 0, LONGTIMELIMIT(),
4552                                       false, tdb_null,
4553                                       NULL, NULL, NULL) != 0) {
4554                 DEBUG(DEBUG_ERR,
4555                       ("Unable to reload IPs on some nodes, try again.\n"));
4556                 goto again;
4557         }
4558
4559         /* It isn't strictly necessary to wait until takeover runs are
4560          * re-enabled but doing so can't hurt.
4561          */
4562         timeout = 0;
4563         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4564                         "Enable takeover runs", true);
4565
4566         ipreallocate(ctdb);
4567
4568         ret = 0;
4569 done:
4570         talloc_free(tmp_ctx);
4571         return ret;
4572 }
4573
4574 /*
4575   display a list of the databases on a remote ctdb
4576  */
4577 static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **argv)
4578 {
4579         int i, ret;
4580         struct ctdb_dbid_map *dbmap=NULL;
4581
4582         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
4583         if (ret != 0) {
4584                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
4585                 return ret;
4586         }
4587
4588         if(options.machinereadable){
4589                 printf(":ID:Name:Path:Persistent:Sticky:Unhealthy:ReadOnly:\n");
4590                 for(i=0;i<dbmap->num;i++){
4591                         const char *path;
4592                         const char *name;
4593                         const char *health;
4594                         bool persistent;
4595                         bool readonly;
4596                         bool sticky;
4597
4598                         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn,
4599                                             dbmap->dbs[i].dbid, ctdb, &path);
4600                         ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
4601                                             dbmap->dbs[i].dbid, ctdb, &name);
4602                         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
4603                                               dbmap->dbs[i].dbid, ctdb, &health);
4604                         persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4605                         readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4606                         sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4607                         printf(":0x%08X:%s:%s:%d:%d:%d:%d:\n",
4608                                dbmap->dbs[i].dbid, name, path,
4609                                !!(persistent), !!(sticky),
4610                                !!(health), !!(readonly));
4611                 }
4612                 return 0;
4613         }
4614
4615         printf("Number of databases:%d\n", dbmap->num);
4616         for(i=0;i<dbmap->num;i++){
4617                 const char *path;
4618                 const char *name;
4619                 const char *health;
4620                 bool persistent;
4621                 bool readonly;
4622                 bool sticky;
4623
4624                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
4625                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
4626                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
4627                 persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4628                 readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4629                 sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4630                 printf("dbid:0x%08x name:%s path:%s%s%s%s%s\n",
4631                        dbmap->dbs[i].dbid, name, path,
4632                        persistent?" PERSISTENT":"",
4633                        sticky?" STICKY":"",
4634                        readonly?" READONLY":"",
4635                        health?" UNHEALTHY":"");
4636         }
4637
4638         return 0;
4639 }
4640
4641 /*
4642   display the status of a database on a remote ctdb
4643  */
4644 static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char **argv)
4645 {
4646         const char *db_name;
4647         uint32_t db_id;
4648         uint8_t flags;
4649         const char *path;
4650         const char *health;
4651
4652         if (argc < 1) {
4653                 usage();
4654         }
4655
4656         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
4657                 return -1;
4658         }
4659
4660         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &path);
4661         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &health);
4662         printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nSTICKY: %s\nREADONLY: %s\nHEALTH: %s\n",
4663                db_id, db_name, path,
4664                (flags & CTDB_DB_FLAGS_PERSISTENT ? "yes" : "no"),
4665                (flags & CTDB_DB_FLAGS_STICKY ? "yes" : "no"),
4666                (flags & CTDB_DB_FLAGS_READONLY ? "yes" : "no"),
4667                (health ? health : "OK"));
4668
4669         return 0;
4670 }
4671
4672 /*
4673   check if the local node is recmaster or not
4674   it will return 1 if this node is the recmaster and 0 if it is not
4675   or if the local ctdb daemon could not be contacted
4676  */
4677 static int control_isnotrecmaster(struct ctdb_context *ctdb, int argc, const char **argv)
4678 {
4679         uint32_t mypnn, recmaster;
4680         int ret;
4681
4682         assert_single_node_only();
4683
4684         mypnn = getpnn(ctdb);
4685
4686         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
4687         if (ret != 0) {
4688                 printf("Failed to get the recmaster\n");
4689                 return 1;
4690         }
4691
4692         if (recmaster != mypnn) {
4693                 printf("this node is not the recmaster\n");
4694                 return 1;
4695         }
4696
4697         printf("this node is the recmaster\n");
4698         return 0;
4699 }
4700
4701 /*
4702   ping a node
4703  */
4704 static int control_ping(struct ctdb_context *ctdb, int argc, const char **argv)
4705 {
4706         int ret;
4707         struct timeval tv = timeval_current();
4708         ret = ctdb_ctrl_ping(ctdb, options.pnn);
4709         if (ret == -1) {
4710                 printf("Unable to get ping response from node %u\n", options.pnn);
4711                 return -1;
4712         } else {
4713                 printf("response from %u time=%.6f sec  (%d clients)\n",
4714                        options.pnn, timeval_elapsed(&tv), ret);
4715         }
4716         return 0;
4717 }
4718
4719
4720 /*
4721   get a node's runstate
4722  */
4723 static int control_runstate(struct ctdb_context *ctdb, int argc, const char **argv)
4724 {
4725         int ret;
4726         enum ctdb_runstate runstate;
4727
4728         ret = ctdb_ctrl_get_runstate(ctdb, TIMELIMIT(), options.pnn, &runstate);
4729         if (ret == -1) {
4730                 printf("Unable to get runstate response from node %u\n",
4731                        options.pnn);
4732                 return -1;
4733         } else {
4734                 bool found = true;
4735                 enum ctdb_runstate t;
4736                 int i;
4737                 for (i=0; i<argc; i++) {
4738                         found = false;
4739                         t = runstate_from_string(argv[i]);
4740                         if (t == CTDB_RUNSTATE_UNKNOWN) {
4741                                 printf("Invalid run state (%s)\n", argv[i]);
4742                                 return -1;
4743                         }
4744
4745                         if (t == runstate) {
4746                                 found = true;
4747                                 break;
4748                         }
4749                 }
4750
4751                 if (!found) {
4752                         printf("CTDB not in required run state (got %s)\n",
4753                                runstate_to_string((enum ctdb_runstate)runstate));
4754                         return -1;
4755                 }
4756         }
4757
4758         printf("%s\n", runstate_to_string(runstate));
4759         return 0;
4760 }
4761
4762
4763 /*
4764   get a tunable
4765  */
4766 static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv)
4767 {
4768         const char *name;
4769         uint32_t value;
4770         int ret;
4771
4772         if (argc < 1) {
4773                 usage();
4774         }
4775
4776         name = argv[0];
4777         ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn, name, &value);
4778         if (ret != 0) {
4779                 DEBUG(DEBUG_ERR, ("Unable to get tunable variable '%s'\n", name));
4780                 return -1;
4781         }
4782
4783         printf("%-23s = %u\n", name, value);
4784         return 0;
4785 }
4786
4787 /*
4788   set a tunable
4789  */
4790 static int control_setvar(struct ctdb_context *ctdb, int argc, const char **argv)
4791 {
4792         const char *name;
4793         uint32_t value;
4794         int ret;
4795
4796         if (argc < 2) {
4797                 usage();
4798         }
4799
4800         name = argv[0];
4801         value = strtoul(argv[1], NULL, 0);
4802
4803         ret = ctdb_ctrl_set_tunable(ctdb, TIMELIMIT(), options.pnn, name, value);
4804         if (ret == -1) {
4805                 DEBUG(DEBUG_ERR, ("Unable to set tunable variable '%s'\n", name));
4806                 return -1;
4807         }
4808         return 0;
4809 }
4810
4811 /*
4812   list all tunables
4813  */
4814 static int control_listvars(struct ctdb_context *ctdb, int argc, const char **argv)
4815 {
4816         uint32_t count;
4817         const char **list;
4818         int ret, i;
4819
4820         ret = ctdb_ctrl_list_tunables(ctdb, TIMELIMIT(), options.pnn, ctdb, &list, &count);
4821         if (ret == -1) {
4822                 DEBUG(DEBUG_ERR, ("Unable to list tunable variables\n"));
4823                 return -1;
4824         }
4825
4826         for (i=0;i<count;i++) {
4827                 control_getvar(ctdb, 1, &list[i]);
4828         }
4829
4830         talloc_free(list);
4831
4832         return 0;
4833 }
4834
4835 /*
4836   display debug level on a node
4837  */
4838 static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **argv)
4839 {
4840         int ret;
4841         int32_t level;
4842
4843         ret = ctdb_ctrl_get_debuglevel(ctdb, options.pnn, &level);
4844         if (ret != 0) {
4845                 DEBUG(DEBUG_ERR, ("Unable to get debuglevel response from node %u\n", options.pnn));
4846                 return ret;
4847         } else {
4848                 if (options.machinereadable){
4849                         printf(":Name:Level:\n");
4850                         printf(":%s:%d:\n",get_debug_by_level(level),level);
4851                 } else {
4852                         printf("Node %u is at debug level %s (%d)\n", options.pnn, get_debug_by_level(level), level);
4853                 }
4854         }
4855         return 0;
4856 }
4857
4858 /*
4859   display reclock file of a node
4860  */
4861 static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4862 {
4863         int ret;
4864         const char *reclock;
4865
4866         ret = ctdb_ctrl_getreclock(ctdb, TIMELIMIT(), options.pnn, ctdb, &reclock);
4867         if (ret != 0) {
4868                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4869                 return ret;
4870         } else {
4871                 if (options.machinereadable){
4872                         if (reclock != NULL) {
4873                                 printf("%s", reclock);
4874                         }
4875                 } else {
4876                         if (reclock == NULL) {
4877                                 printf("No reclock file used.\n");
4878                         } else {
4879                                 printf("Reclock file:%s\n", reclock);
4880                         }
4881                 }
4882         }
4883         return 0;
4884 }
4885
4886 /*
4887   set the reclock file of a node
4888  */
4889 static int control_setreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4890 {
4891         int ret;
4892         const char *reclock;
4893
4894         if (argc == 0) {
4895                 reclock = NULL;
4896         } else if (argc == 1) {
4897                 reclock = argv[0];
4898         } else {
4899                 usage();
4900         }
4901
4902         ret = ctdb_ctrl_setreclock(ctdb, TIMELIMIT(), options.pnn, reclock);
4903         if (ret != 0) {
4904                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4905                 return ret;
4906         }
4907         return 0;
4908 }
4909
4910 /*
4911   set the natgw state on/off
4912  */
4913 static int control_setnatgwstate(struct ctdb_context *ctdb, int argc, const char **argv)
4914 {
4915         int ret;
4916         uint32_t natgwstate;
4917
4918         if (argc == 0) {
4919                 usage();
4920         }
4921
4922         if (!strcmp(argv[0], "on")) {
4923                 natgwstate = 1;
4924         } else if (!strcmp(argv[0], "off")) {
4925                 natgwstate = 0;
4926         } else {
4927                 usage();
4928         }
4929
4930         ret = ctdb_ctrl_setnatgwstate(ctdb, TIMELIMIT(), options.pnn, natgwstate);
4931         if (ret != 0) {
4932                 DEBUG(DEBUG_ERR, ("Unable to set the natgw state for node %u\n", options.pnn));
4933                 return ret;
4934         }
4935
4936         return 0;
4937 }
4938
4939 /*
4940   set the lmaster role on/off
4941  */
4942 static int control_setlmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
4943 {
4944         int ret;
4945         uint32_t lmasterrole;
4946
4947         if (argc == 0) {
4948                 usage();
4949         }
4950
4951         if (!strcmp(argv[0], "on")) {
4952                 lmasterrole = 1;
4953         } else if (!strcmp(argv[0], "off")) {
4954                 lmasterrole = 0;
4955         } else {
4956                 usage();
4957         }
4958
4959         ret = ctdb_ctrl_setlmasterrole(ctdb, TIMELIMIT(), options.pnn, lmasterrole);
4960         if (ret != 0) {
4961                 DEBUG(DEBUG_ERR, ("Unable to set the lmaster role for node %u\n", options.pnn));
4962                 return ret;
4963         }
4964
4965         return 0;
4966 }
4967
4968 /*
4969   set the recmaster role on/off
4970  */
4971 static int control_setrecmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
4972 {
4973         int ret;
4974         uint32_t recmasterrole;
4975
4976         if (argc == 0) {
4977                 usage();
4978         }
4979
4980         if (!strcmp(argv[0], "on")) {
4981                 recmasterrole = 1;
4982         } else if (!strcmp(argv[0], "off")) {
4983                 recmasterrole = 0;
4984         } else {
4985                 usage();
4986         }
4987
4988         ret = ctdb_ctrl_setrecmasterrole(ctdb, TIMELIMIT(), options.pnn, recmasterrole);
4989         if (ret != 0) {
4990                 DEBUG(DEBUG_ERR, ("Unable to set the recmaster role for node %u\n", options.pnn));
4991                 return ret;
4992         }
4993
4994         return 0;
4995 }
4996
4997 /*
4998   set debug level on a node or all nodes
4999  */
5000 static int control_setdebug(struct ctdb_context *ctdb, int argc, const char **argv)
5001 {
5002         int i, ret;
5003         int32_t level;
5004
5005         if (argc == 0) {
5006                 printf("You must specify the debug level. Valid levels are:\n");
5007                 for (i=0; debug_levels[i].description != NULL; i++) {
5008                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
5009                 }
5010
5011                 return 0;
5012         }
5013
5014         if (isalpha(argv[0][0]) || argv[0][0] == '-') {
5015                 level = get_debug_by_desc(argv[0]);
5016         } else {
5017                 level = strtol(argv[0], NULL, 0);
5018         }
5019
5020         for (i=0; debug_levels[i].description != NULL; i++) {
5021                 if (level == debug_levels[i].level) {
5022                         break;
5023                 }
5024         }
5025         if (debug_levels[i].description == NULL) {
5026                 printf("Invalid debug level, must be one of\n");
5027                 for (i=0; debug_levels[i].description != NULL; i++) {
5028                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
5029                 }
5030                 return -1;
5031         }
5032
5033         ret = ctdb_ctrl_set_debuglevel(ctdb, options.pnn, level);
5034         if (ret != 0) {
5035                 DEBUG(DEBUG_ERR, ("Unable to set debug level on node %u\n", options.pnn));
5036         }
5037         return 0;
5038 }
5039
5040
5041 /*
5042   thaw a node
5043  */
5044 static int control_thaw(struct ctdb_context *ctdb, int argc, const char **argv)
5045 {
5046         int ret;
5047         uint32_t priority;
5048
5049         if (argc == 1) {
5050                 priority = strtol(argv[0], NULL, 0);
5051         } else {
5052                 priority = 0;
5053         }
5054         DEBUG(DEBUG_ERR,("Thaw by priority %u\n", priority));
5055
5056         ret = ctdb_ctrl_thaw_priority(ctdb, TIMELIMIT(), options.pnn, priority);
5057         if (ret != 0) {
5058                 DEBUG(DEBUG_ERR, ("Unable to thaw node %u\n", options.pnn));
5059         }
5060         return 0;
5061 }
5062
5063
5064 /*
5065   attach to a database
5066  */
5067 static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv)
5068 {
5069         const char *db_name;
5070         struct ctdb_db_context *ctdb_db;
5071         bool persistent = false;
5072
5073         if (argc < 1) {
5074                 usage();
5075         }
5076         db_name = argv[0];
5077         if (argc > 2) {
5078                 usage();
5079         }
5080         if (argc == 2) {
5081                 if (strcmp(argv[1], "persistent") != 0) {
5082                         usage();
5083                 }
5084                 persistent = true;
5085         }
5086
5087         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
5088         if (ctdb_db == NULL) {
5089                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
5090                 return -1;
5091         }
5092
5093         return 0;
5094 }
5095
5096 /*
5097  * detach from a database
5098  */
5099 static int control_detach(struct ctdb_context *ctdb, int argc,
5100                           const char **argv)
5101 {
5102         uint32_t db_id;
5103         uint8_t flags;
5104         int ret, i, status = 0;
5105         struct ctdb_node_map *nodemap = NULL;
5106         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5107         uint32_t recmode;
5108
5109         if (argc < 1) {
5110                 usage();
5111         }
5112
5113         assert_single_node_only();
5114
5115         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn,
5116                                     &recmode);
5117         if (ret != 0) {
5118                 DEBUG(DEBUG_ERR, ("Database cannot be detached "
5119                                   "when recovery is active\n"));
5120                 talloc_free(tmp_ctx);
5121                 return -1;
5122         }
5123
5124         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5125                                    &nodemap);
5126         if (ret != 0) {
5127                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5128                                   options.pnn));
5129                 talloc_free(tmp_ctx);
5130                 return -1;
5131         }
5132
5133         for (i=0; i<nodemap->num; i++) {
5134                 uint32_t value;
5135
5136                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
5137                         continue;
5138                 }
5139
5140                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
5141                         continue;
5142                 }
5143
5144                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
5145                         DEBUG(DEBUG_ERR, ("Database cannot be detached on "
5146                                           "inactive (stopped or banned) node "
5147                                           "%u\n", nodemap->nodes[i].pnn));
5148                         talloc_free(tmp_ctx);
5149                         return -1;
5150                 }
5151
5152                 ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(),
5153                                             nodemap->nodes[i].pnn,
5154                                             "AllowClientDBAttach",
5155                                             &value);
5156                 if (ret != 0) {
5157                         DEBUG(DEBUG_ERR, ("Unable to get tunable "
5158                                           "AllowClientDBAttach from node %u\n",
5159                                            nodemap->nodes[i].pnn));
5160                         talloc_free(tmp_ctx);
5161                         return -1;
5162                 }
5163
5164                 if (value == 1) {
5165                         DEBUG(DEBUG_ERR, ("Database access is still active on "
5166                                           "node %u. Set AllowClientDBAttach=0 "
5167                                           "on all nodes.\n",
5168                                           nodemap->nodes[i].pnn));
5169                         talloc_free(tmp_ctx);
5170                         return -1;
5171                 }
5172         }
5173
5174         talloc_free(tmp_ctx);
5175
5176         for (i=0; i<argc; i++) {
5177                 if (!db_exists(ctdb, argv[i], &db_id, NULL, &flags)) {
5178                         continue;
5179                 }
5180
5181                 if (flags & CTDB_DB_FLAGS_PERSISTENT) {
5182                         DEBUG(DEBUG_ERR, ("Persistent database '%s' "
5183                                           "cannot be detached\n", argv[i]));
5184                         status = -1;
5185                         continue;
5186                 }
5187
5188                 ret = ctdb_detach(ctdb, db_id);
5189                 if (ret != 0) {
5190                         DEBUG(DEBUG_ERR, ("Database '%s' detach failed\n",
5191                                           argv[i]));
5192                         status = ret;
5193                 }
5194         }
5195
5196         return status;
5197 }
5198
5199 /*
5200   set db priority
5201  */
5202 static int control_setdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5203 {
5204         struct ctdb_db_priority db_prio;
5205         int ret;
5206
5207         if (argc < 2) {
5208                 usage();
5209         }
5210
5211         db_prio.db_id    = strtoul(argv[0], NULL, 0);
5212         db_prio.priority = strtoul(argv[1], NULL, 0);
5213
5214         ret = ctdb_ctrl_set_db_priority(ctdb, TIMELIMIT(), options.pnn, &db_prio);
5215         if (ret != 0) {
5216                 DEBUG(DEBUG_ERR,("Unable to set db prio\n"));
5217                 return -1;
5218         }
5219
5220         return 0;
5221 }
5222
5223 /*
5224   get db priority
5225  */
5226 static int control_getdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5227 {
5228         uint32_t db_id, priority;
5229         int ret;
5230
5231         if (argc < 1) {
5232                 usage();
5233         }
5234
5235         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5236                 return -1;
5237         }
5238
5239         ret = ctdb_ctrl_get_db_priority(ctdb, TIMELIMIT(), options.pnn, db_id, &priority);
5240         if (ret != 0) {
5241                 DEBUG(DEBUG_ERR,("Unable to get db prio\n"));
5242                 return -1;
5243         }
5244
5245         DEBUG(DEBUG_ERR,("Priority:%u\n", priority));
5246
5247         return 0;
5248 }
5249
5250 /*
5251   set the sticky records capability for a database
5252  */
5253 static int control_setdbsticky(struct ctdb_context *ctdb, int argc, const char **argv)
5254 {
5255         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5256         uint32_t db_id;
5257         int ret;
5258
5259         if (argc < 1) {
5260                 usage();
5261         }
5262
5263         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5264                 return -1;
5265         }
5266
5267         ret = ctdb_ctrl_set_db_sticky(ctdb, options.pnn, db_id);
5268         if (ret != 0) {
5269                 DEBUG(DEBUG_ERR,("Unable to set db to support sticky records\n"));
5270                 talloc_free(tmp_ctx);
5271                 return -1;
5272         }
5273
5274         talloc_free(tmp_ctx);
5275         return 0;
5276 }
5277
5278 /*
5279   set the readonly capability for a database
5280  */
5281 static int control_setdbreadonly(struct ctdb_context *ctdb, int argc, const char **argv)
5282 {
5283         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5284         uint32_t db_id;
5285         int ret;
5286
5287         if (argc < 1) {
5288                 usage();
5289         }
5290
5291         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5292                 return -1;
5293         }
5294
5295         ret = ctdb_ctrl_set_db_readonly(ctdb, options.pnn, db_id);
5296         if (ret != 0) {
5297                 DEBUG(DEBUG_ERR,("Unable to set db to support readonly\n"));
5298                 talloc_free(tmp_ctx);
5299                 return -1;
5300         }
5301
5302         talloc_free(tmp_ctx);
5303         return 0;
5304 }
5305
5306 /*
5307   get db seqnum
5308  */
5309 static int control_getdbseqnum(struct ctdb_context *ctdb, int argc, const char **argv)
5310 {
5311         uint32_t db_id;
5312         uint64_t seqnum;
5313         int ret;
5314
5315         if (argc < 1) {
5316                 usage();
5317         }
5318
5319         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5320                 return -1;
5321         }
5322
5323         ret = ctdb_ctrl_getdbseqnum(ctdb, TIMELIMIT(), options.pnn, db_id, &seqnum);
5324         if (ret != 0) {
5325                 DEBUG(DEBUG_ERR, ("Unable to get seqnum from node."));
5326                 return -1;
5327         }
5328
5329         printf("Sequence number:%lld\n", (long long)seqnum);
5330
5331         return 0;
5332 }
5333
5334 /*
5335   run an eventscript on a node
5336  */
5337 static int control_eventscript(struct ctdb_context *ctdb, int argc, const char **argv)
5338 {
5339         TDB_DATA data;
5340         int ret;
5341         int32_t res;
5342         char *errmsg;
5343         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5344
5345         if (argc != 1) {
5346                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5347                 return -1;
5348         }
5349
5350         data.dptr = (unsigned char *)discard_const(argv[0]);
5351         data.dsize = strlen((char *)data.dptr) + 1;
5352
5353         DEBUG(DEBUG_ERR, ("Running eventscripts with arguments \"%s\" on node %u\n", data.dptr, options.pnn));
5354
5355         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_RUN_EVENTSCRIPTS,
5356                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
5357         if (ret != 0 || res != 0) {
5358                 DEBUG(DEBUG_ERR,("Failed to run eventscripts - %s\n", errmsg));
5359                 talloc_free(tmp_ctx);
5360                 return -1;
5361         }
5362         talloc_free(tmp_ctx);
5363         return 0;
5364 }
5365
5366 #define DB_VERSION 1
5367 #define MAX_DB_NAME 64
5368 struct db_file_header {
5369         unsigned long version;
5370         time_t timestamp;
5371         unsigned long persistent;
5372         unsigned long size;
5373         const char name[MAX_DB_NAME];
5374 };
5375
5376 struct backup_data {
5377         struct ctdb_marshall_buffer *records;
5378         uint32_t len;
5379         uint32_t total;
5380         bool traverse_error;
5381 };
5382
5383 static int backup_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private)
5384 {
5385         struct backup_data *bd = talloc_get_type(private, struct backup_data);
5386         struct ctdb_rec_data *rec;
5387
5388         /* add the record */
5389         rec = ctdb_marshall_record(bd->records, 0, key, NULL, data);
5390         if (rec == NULL) {
5391                 bd->traverse_error = true;
5392                 DEBUG(DEBUG_ERR,("Failed to marshall record\n"));
5393                 return -1;
5394         }
5395         bd->records = talloc_realloc_size(NULL, bd->records, rec->length + bd->len);
5396         if (bd->records == NULL) {
5397                 DEBUG(DEBUG_ERR,("Failed to expand marshalling buffer\n"));
5398                 bd->traverse_error = true;
5399                 return -1;
5400         }
5401         bd->records->count++;
5402         memcpy(bd->len+(uint8_t *)bd->records, rec, rec->length);
5403         bd->len += rec->length;
5404         talloc_free(rec);
5405
5406         bd->total++;
5407         return 0;
5408 }
5409
5410 /*
5411  * backup a database to a file
5412  */
5413 static int control_backupdb(struct ctdb_context *ctdb, int argc, const char **argv)
5414 {
5415         const char *db_name;
5416         int ret;
5417         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5418         struct db_file_header dbhdr;
5419         struct ctdb_db_context *ctdb_db;
5420         struct backup_data *bd;
5421         int fh = -1;
5422         int status = -1;
5423         const char *reason = NULL;
5424         uint32_t db_id;
5425         uint8_t flags;
5426
5427         assert_single_node_only();
5428
5429         if (argc != 2) {
5430                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5431                 return -1;
5432         }
5433
5434         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
5435                 return -1;
5436         }
5437
5438         ret = ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
5439                                     db_id, tmp_ctx, &reason);
5440         if (ret != 0) {
5441                 DEBUG(DEBUG_ERR,("Unable to get dbhealth for database '%s'\n",
5442                                  argv[0]));
5443                 talloc_free(tmp_ctx);
5444                 return -1;
5445         }
5446         if (reason) {
5447                 uint32_t allow_unhealthy = 0;
5448
5449                 ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn,
5450                                       "AllowUnhealthyDBRead",
5451                                       &allow_unhealthy);
5452
5453                 if (allow_unhealthy != 1) {
5454                         DEBUG(DEBUG_ERR,("database '%s' is unhealthy: %s\n",
5455                                          argv[0], reason));
5456
5457                         DEBUG(DEBUG_ERR,("disallow backup : tunable AllowUnhealthyDBRead = %u\n",
5458                                          allow_unhealthy));
5459                         talloc_free(tmp_ctx);
5460                         return -1;
5461                 }
5462
5463                 DEBUG(DEBUG_WARNING,("WARNING database '%s' is unhealthy - see 'ctdb getdbstatus %s'\n",
5464                                      argv[0], argv[0]));
5465                 DEBUG(DEBUG_WARNING,("WARNING! allow backup of unhealthy database: "
5466                                      "tunnable AllowUnhealthyDBRead = %u\n",
5467                                      allow_unhealthy));
5468         }
5469
5470         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5471         if (ctdb_db == NULL) {
5472                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", argv[0]));
5473                 talloc_free(tmp_ctx);
5474                 return -1;
5475         }
5476
5477
5478         ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
5479         if (ret == -1) {
5480                 DEBUG(DEBUG_ERR,("Failed to start transaction\n"));
5481                 talloc_free(tmp_ctx);
5482                 return -1;
5483         }
5484
5485
5486         bd = talloc_zero(tmp_ctx, struct backup_data);
5487         if (bd == NULL) {
5488                 DEBUG(DEBUG_ERR,("Failed to allocate backup_data\n"));
5489                 talloc_free(tmp_ctx);
5490                 return -1;
5491         }
5492
5493         bd->records = talloc_zero(bd, struct ctdb_marshall_buffer);
5494         if (bd->records == NULL) {
5495                 DEBUG(DEBUG_ERR,("Failed to allocate ctdb_marshall_buffer\n"));
5496                 talloc_free(tmp_ctx);
5497                 return -1;
5498         }
5499
5500         bd->len = offsetof(struct ctdb_marshall_buffer, data);
5501         bd->records->db_id = ctdb_db->db_id;
5502         /* traverse the database collecting all records */
5503         if (tdb_traverse_read(ctdb_db->ltdb->tdb, backup_traverse, bd) == -1 ||
5504             bd->traverse_error) {
5505                 DEBUG(DEBUG_ERR,("Traverse error\n"));
5506                 talloc_free(tmp_ctx);
5507                 return -1;
5508         }
5509
5510         tdb_transaction_cancel(ctdb_db->ltdb->tdb);
5511
5512
5513         fh = open(argv[1], O_RDWR|O_CREAT, 0600);
5514         if (fh == -1) {
5515                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[1]));
5516                 talloc_free(tmp_ctx);
5517                 return -1;
5518         }
5519
5520         ZERO_STRUCT(dbhdr);
5521         dbhdr.version = DB_VERSION;
5522         dbhdr.timestamp = time(NULL);
5523         dbhdr.persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
5524         dbhdr.size = bd->len;
5525         if (strlen(argv[0]) >= MAX_DB_NAME) {
5526                 DEBUG(DEBUG_ERR,("Too long dbname\n"));
5527                 goto done;
5528         }
5529         strncpy(discard_const(dbhdr.name), argv[0], MAX_DB_NAME-1);
5530         ret = sys_write(fh, &dbhdr, sizeof(dbhdr));
5531         if (ret == -1) {
5532                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5533                 goto done;
5534         }
5535         ret = sys_write(fh, bd->records, bd->len);
5536         if (ret == -1) {
5537                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5538                 goto done;
5539         }
5540
5541         status = 0;
5542 done:
5543         if (fh != -1) {
5544                 ret = close(fh);
5545                 if (ret == -1) {
5546                         DEBUG(DEBUG_ERR,("close failed: %s\n", strerror(errno)));
5547                 }
5548         }
5549
5550         DEBUG(DEBUG_ERR,("Database backed up to %s\n", argv[1]));
5551
5552         talloc_free(tmp_ctx);
5553         return status;
5554 }
5555
5556 /*
5557  * restore a database from a file
5558  */
5559 static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **argv)
5560 {
5561         int ret;
5562         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5563         TDB_DATA outdata;
5564         TDB_DATA data;
5565         struct db_file_header dbhdr;
5566         struct ctdb_db_context *ctdb_db;
5567         struct ctdb_node_map *nodemap=NULL;
5568         struct ctdb_vnn_map *vnnmap=NULL;
5569         int i, fh;
5570         struct ctdb_control_wipe_database w;
5571         uint32_t *nodes;
5572         uint32_t generation;
5573         struct tm *tm;
5574         char tbuf[100];
5575         char *dbname;
5576
5577         assert_single_node_only();
5578
5579         if (argc < 1 || argc > 2) {
5580                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5581                 return -1;
5582         }
5583
5584         fh = open(argv[0], O_RDONLY);
5585         if (fh == -1) {
5586                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5587                 talloc_free(tmp_ctx);
5588                 return -1;
5589         }
5590
5591         sys_read(fh, &dbhdr, sizeof(dbhdr));
5592         if (dbhdr.version != DB_VERSION) {
5593                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5594                 close(fh);
5595                 talloc_free(tmp_ctx);
5596                 return -1;
5597         }
5598
5599         dbname = discard_const(dbhdr.name);
5600         if (argc == 2) {
5601                 dbname = discard_const(argv[1]);
5602         }
5603
5604         outdata.dsize = dbhdr.size;
5605         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5606         if (outdata.dptr == NULL) {
5607                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5608                 close(fh);
5609                 talloc_free(tmp_ctx);
5610                 return -1;
5611         }
5612         sys_read(fh, outdata.dptr, outdata.dsize);
5613         close(fh);
5614
5615         tm = localtime(&dbhdr.timestamp);
5616         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5617         printf("Restoring database '%s' from backup @ %s\n",
5618                 dbname, tbuf);
5619
5620
5621         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), dbname, dbhdr.persistent, 0);
5622         if (ctdb_db == NULL) {
5623                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", dbname));
5624                 talloc_free(tmp_ctx);
5625                 return -1;
5626         }
5627
5628         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
5629         if (ret != 0) {
5630                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
5631                 talloc_free(tmp_ctx);
5632                 return ret;
5633         }
5634
5635
5636         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
5637         if (ret != 0) {
5638                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
5639                 talloc_free(tmp_ctx);
5640                 return ret;
5641         }
5642
5643         /* freeze all nodes */
5644         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5645         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5646                 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5647                                         nodes, i,
5648                                         TIMELIMIT(),
5649                                         false, tdb_null,
5650                                         NULL, NULL,
5651                                         NULL) != 0) {
5652                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5653                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5654                         talloc_free(tmp_ctx);
5655                         return -1;
5656                 }
5657         }
5658
5659         generation = vnnmap->generation;
5660         data.dptr = (void *)&generation;
5661         data.dsize = sizeof(generation);
5662
5663         /* start a cluster wide transaction */
5664         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5665         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
5666                                         nodes, 0,
5667                                         TIMELIMIT(), false, data,
5668                                         NULL, NULL,
5669                                         NULL) != 0) {
5670                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide transactions.\n"));
5671                 return -1;
5672         }
5673
5674
5675         w.db_id = ctdb_db->db_id;
5676         w.transaction_id = generation;
5677
5678         data.dptr = (void *)&w;
5679         data.dsize = sizeof(w);
5680
5681         /* wipe all the remote databases. */
5682         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5683         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
5684                                         nodes, 0,
5685                                         TIMELIMIT(), false, data,
5686                                         NULL, NULL,
5687                                         NULL) != 0) {
5688                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
5689                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5690                 talloc_free(tmp_ctx);
5691                 return -1;
5692         }
5693
5694         /* push the database */
5695         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5696         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
5697                                         nodes, 0,
5698                                         TIMELIMIT(), false, outdata,
5699                                         NULL, NULL,
5700                                         NULL) != 0) {
5701                 DEBUG(DEBUG_ERR, ("Failed to push database.\n"));
5702                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5703                 talloc_free(tmp_ctx);
5704                 return -1;
5705         }
5706
5707         data.dptr = (void *)&ctdb_db->db_id;
5708         data.dsize = sizeof(ctdb_db->db_id);
5709
5710         /* mark the database as healthy */
5711         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5712         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
5713                                         nodes, 0,
5714                                         TIMELIMIT(), false, data,
5715                                         NULL, NULL,
5716                                         NULL) != 0) {
5717                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
5718                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5719                 talloc_free(tmp_ctx);
5720                 return -1;
5721         }
5722
5723         data.dptr = (void *)&generation;
5724         data.dsize = sizeof(generation);
5725
5726         /* commit all the changes */
5727         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
5728                                         nodes, 0,
5729                                         TIMELIMIT(), false, data,
5730                                         NULL, NULL,
5731                                         NULL) != 0) {
5732                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
5733                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5734                 talloc_free(tmp_ctx);
5735                 return -1;
5736         }
5737
5738
5739         /* thaw all nodes */
5740         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5741         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
5742                                         nodes, 0,
5743                                         TIMELIMIT(),
5744                                         false, tdb_null,
5745                                         NULL, NULL,
5746                                         NULL) != 0) {
5747                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
5748                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5749                 talloc_free(tmp_ctx);
5750                 return -1;
5751         }
5752
5753
5754         talloc_free(tmp_ctx);
5755         return 0;
5756 }
5757
5758 /*
5759  * dump a database backup from a file
5760  */
5761 static int control_dumpdbbackup(struct ctdb_context *ctdb, int argc, const char **argv)
5762 {
5763         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5764         TDB_DATA outdata;
5765         struct db_file_header dbhdr;
5766         int i, fh;
5767         struct tm *tm;
5768         char tbuf[100];
5769         struct ctdb_rec_data *rec = NULL;
5770         struct ctdb_marshall_buffer *m;
5771         struct ctdb_dump_db_context c;
5772
5773         assert_single_node_only();
5774
5775         if (argc != 1) {
5776                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5777                 return -1;
5778         }
5779
5780         fh = open(argv[0], O_RDONLY);
5781         if (fh == -1) {
5782                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5783                 talloc_free(tmp_ctx);
5784                 return -1;
5785         }
5786
5787         sys_read(fh, &dbhdr, sizeof(dbhdr));
5788         if (dbhdr.version != DB_VERSION) {
5789                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5790                 close(fh);
5791                 talloc_free(tmp_ctx);
5792                 return -1;
5793         }
5794
5795         outdata.dsize = dbhdr.size;
5796         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5797         if (outdata.dptr == NULL) {
5798                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5799                 close(fh);
5800                 talloc_free(tmp_ctx);
5801                 return -1;
5802         }
5803         sys_read(fh, outdata.dptr, outdata.dsize);
5804         close(fh);
5805         m = (struct ctdb_marshall_buffer *)outdata.dptr;
5806
5807         tm = localtime(&dbhdr.timestamp);
5808         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5809         printf("Backup of database name:'%s' dbid:0x%x08x from @ %s\n",
5810                 dbhdr.name, m->db_id, tbuf);
5811
5812         ZERO_STRUCT(c);
5813         c.f = stdout;
5814         c.printemptyrecords = (bool)options.printemptyrecords;
5815         c.printdatasize = (bool)options.printdatasize;
5816         c.printlmaster = false;
5817         c.printhash = (bool)options.printhash;
5818         c.printrecordflags = (bool)options.printrecordflags;
5819
5820         for (i=0; i < m->count; i++) {
5821                 uint32_t reqid = 0;
5822                 TDB_DATA key, data;
5823
5824                 /* we do not want the header splitted, so we pass NULL*/
5825                 rec = ctdb_marshall_loop_next(m, rec, &reqid,
5826                                               NULL, &key, &data);
5827
5828                 ctdb_dumpdb_record(ctdb, key, data, &c);
5829         }
5830
5831         printf("Dumped %d records\n", i);
5832         talloc_free(tmp_ctx);
5833         return 0;
5834 }
5835
5836 /*
5837  * wipe a database from a file
5838  */
5839 static int control_wipedb(struct ctdb_context *ctdb, int argc,
5840                           const char **argv)
5841 {
5842         const char *db_name;
5843         int ret;
5844         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5845         TDB_DATA data;
5846         struct ctdb_db_context *ctdb_db;
5847         struct ctdb_node_map *nodemap = NULL;
5848         struct ctdb_vnn_map *vnnmap = NULL;
5849         int i;
5850         struct ctdb_control_wipe_database w;
5851         uint32_t *nodes;
5852         uint32_t generation;
5853         uint8_t flags;
5854
5855         assert_single_node_only();
5856
5857         if (argc != 1) {
5858                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5859                 return -1;
5860         }
5861
5862         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
5863                 return -1;
5864         }
5865
5866         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5867         if (ctdb_db == NULL) {
5868                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n",
5869                                   argv[0]));
5870                 talloc_free(tmp_ctx);
5871                 return -1;
5872         }
5873
5874         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb,
5875                                    &nodemap);
5876         if (ret != 0) {
5877                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5878                                   options.pnn));
5879                 talloc_free(tmp_ctx);
5880                 return ret;
5881         }
5882
5883         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5884                                   &vnnmap);
5885         if (ret != 0) {
5886                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
5887                                   options.pnn));
5888                 talloc_free(tmp_ctx);
5889                 return ret;
5890         }
5891
5892         /* freeze all nodes */
5893         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5894         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5895                 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5896                                                 nodes, i,
5897                                                 TIMELIMIT(),
5898                                                 false, tdb_null,
5899                                                 NULL, NULL,
5900                                                 NULL);
5901                 if (ret != 0) {
5902                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5903                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn,
5904                                              CTDB_RECOVERY_ACTIVE);
5905                         talloc_free(tmp_ctx);
5906                         return -1;
5907                 }
5908         }
5909
5910         generation = vnnmap->generation;
5911         data.dptr = (void *)&generation;
5912         data.dsize = sizeof(generation);
5913
5914         /* start a cluster wide transaction */
5915         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5916         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
5917                                         nodes, 0,
5918                                         TIMELIMIT(), false, data,
5919                                         NULL, NULL,
5920                                         NULL);
5921         if (ret!= 0) {
5922                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide "
5923                                   "transactions.\n"));
5924                 return -1;
5925         }
5926
5927         w.db_id = ctdb_db->db_id;
5928         w.transaction_id = generation;
5929
5930         data.dptr = (void *)&w;
5931         data.dsize = sizeof(w);
5932
5933         /* wipe all the remote databases. */
5934         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5935         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
5936                                         nodes, 0,
5937                                         TIMELIMIT(), false, data,
5938                                         NULL, NULL,
5939                                         NULL) != 0) {
5940                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
5941                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5942                 talloc_free(tmp_ctx);
5943                 return -1;
5944         }
5945
5946         data.dptr = (void *)&ctdb_db->db_id;
5947         data.dsize = sizeof(ctdb_db->db_id);
5948
5949         /* mark the database as healthy */
5950         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5951         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
5952                                         nodes, 0,
5953                                         TIMELIMIT(), false, data,
5954                                         NULL, NULL,
5955                                         NULL) != 0) {
5956                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
5957                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5958                 talloc_free(tmp_ctx);
5959                 return -1;
5960         }
5961
5962         data.dptr = (void *)&generation;
5963         data.dsize = sizeof(generation);
5964
5965         /* commit all the changes */
5966         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
5967                                         nodes, 0,
5968                                         TIMELIMIT(), false, data,
5969                                         NULL, NULL,
5970                                         NULL) != 0) {
5971                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
5972                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5973                 talloc_free(tmp_ctx);
5974                 return -1;
5975         }
5976
5977         /* thaw all nodes */
5978         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5979         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
5980                                         nodes, 0,
5981                                         TIMELIMIT(),
5982                                         false, tdb_null,
5983                                         NULL, NULL,
5984                                         NULL) != 0) {
5985                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
5986                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5987                 talloc_free(tmp_ctx);
5988                 return -1;
5989         }
5990
5991         DEBUG(DEBUG_ERR, ("Database wiped.\n"));
5992
5993         talloc_free(tmp_ctx);
5994         return 0;
5995 }
5996
5997 /*
5998   dump memory usage
5999  */
6000 static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
6001 {
6002         TDB_DATA data;
6003         int ret;
6004         int32_t res;
6005         char *errmsg;
6006         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
6007         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_DUMP_MEMORY,
6008                            0, tdb_null, tmp_ctx, &data, &res, NULL, &errmsg);
6009         if (ret != 0 || res != 0) {
6010                 DEBUG(DEBUG_ERR,("Failed to dump memory - %s\n", errmsg));
6011                 talloc_free(tmp_ctx);
6012                 return -1;
6013         }
6014         sys_write(1, data.dptr, data.dsize);
6015         talloc_free(tmp_ctx);
6016         return 0;
6017 }
6018
6019 /*
6020   handler for memory dumps
6021 */
6022 static void mem_dump_handler(struct ctdb_context *ctdb, uint64_t srvid,
6023                              TDB_DATA data, void *private_data)
6024 {
6025         sys_write(1, data.dptr, data.dsize);
6026         exit(0);
6027 }
6028
6029 /*
6030   dump memory usage on the recovery daemon
6031  */
6032 static int control_rddumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
6033 {
6034         int ret;
6035         TDB_DATA data;
6036         struct srvid_request rd;
6037
6038         rd.pnn = ctdb_get_pnn(ctdb);
6039         rd.srvid = getpid();
6040
6041         /* register a message port for receiveing the reply so that we
6042            can receive the reply
6043         */
6044         ctdb_client_set_message_handler(ctdb, rd.srvid, mem_dump_handler, NULL);
6045
6046
6047         data.dptr = (uint8_t *)&rd;
6048         data.dsize = sizeof(rd);
6049
6050         ret = ctdb_client_send_message(ctdb, options.pnn, CTDB_SRVID_MEM_DUMP, data);
6051         if (ret != 0) {
6052                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
6053                 return -1;
6054         }
6055
6056         /* this loop will terminate when we have received the reply */
6057         while (1) {
6058                 event_loop_once(ctdb->ev);
6059         }
6060
6061         return 0;
6062 }
6063
6064 /*
6065   send a message to a srvid
6066  */
6067 static int control_msgsend(struct ctdb_context *ctdb, int argc, const char **argv)
6068 {
6069         unsigned long srvid;
6070         int ret;
6071         TDB_DATA data;
6072
6073         if (argc < 2) {
6074                 usage();
6075         }
6076
6077         srvid      = strtoul(argv[0], NULL, 0);
6078
6079         data.dptr = (uint8_t *)discard_const(argv[1]);
6080         data.dsize= strlen(argv[1]);
6081
6082         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, srvid, data);
6083         if (ret != 0) {
6084                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
6085                 return -1;
6086         }
6087
6088         return 0;
6089 }
6090
6091 /*
6092   handler for msglisten
6093 */
6094 static void msglisten_handler(struct ctdb_context *ctdb, uint64_t srvid,
6095                              TDB_DATA data, void *private_data)
6096 {
6097         int i;
6098
6099         printf("Message received: ");
6100         for (i=0;i<data.dsize;i++) {
6101                 printf("%c", data.dptr[i]);
6102         }
6103         printf("\n");
6104 }
6105
6106 /*
6107   listen for messages on a messageport
6108  */
6109 static int control_msglisten(struct ctdb_context *ctdb, int argc, const char **argv)
6110 {
6111         uint64_t srvid;
6112
6113         srvid = getpid();
6114
6115         /* register a message port and listen for messages
6116         */
6117         ctdb_client_set_message_handler(ctdb, srvid, msglisten_handler, NULL);
6118         printf("Listening for messages on srvid:%d\n", (int)srvid);
6119
6120         while (1) {
6121                 event_loop_once(ctdb->ev);
6122         }
6123
6124         return 0;
6125 }
6126
6127 /*
6128   list all nodes in the cluster
6129   we parse the nodes file directly
6130  */
6131 static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
6132 {
6133         TALLOC_CTX *mem_ctx = talloc_new(NULL);
6134         struct pnn_node *pnn_nodes;
6135         struct pnn_node *pnn_node;
6136
6137         assert_single_node_only();
6138
6139         pnn_nodes = read_nodes_file(mem_ctx);
6140         if (pnn_nodes == NULL) {
6141                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
6142                 talloc_free(mem_ctx);
6143                 return -1;
6144         }
6145
6146         for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
6147                 const char *addr = ctdb_addr_to_str(&pnn_node->addr);
6148                 if (options.machinereadable){
6149                         printf(":%d:%s:\n", pnn_node->pnn, addr);
6150                 } else {
6151                         printf("%s\n", addr);
6152                 }
6153         }
6154         talloc_free(mem_ctx);
6155
6156         return 0;
6157 }
6158
6159 /*
6160   reload the nodes file on the local node
6161  */
6162 static int control_reload_nodes_file(struct ctdb_context *ctdb, int argc, const char **argv)
6163 {
6164         int i, ret;
6165         int mypnn;
6166         struct ctdb_node_map *nodemap=NULL;
6167
6168         assert_single_node_only();
6169
6170         mypnn = ctdb_get_pnn(ctdb);
6171
6172         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
6173         if (ret != 0) {
6174                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
6175                 return ret;
6176         }
6177
6178         /* reload the nodes file on all remote nodes */
6179         for (i=0;i<nodemap->num;i++) {
6180                 if (nodemap->nodes[i].pnn == mypnn) {
6181                         continue;
6182                 }
6183                 DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", nodemap->nodes[i].pnn));
6184                 ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(),
6185                         nodemap->nodes[i].pnn);
6186                 if (ret != 0) {
6187                         DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", nodemap->nodes[i].pnn));
6188                 }
6189         }
6190
6191         /* reload the nodes file on the local node */
6192         DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", mypnn));
6193         ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(), mypnn);
6194         if (ret != 0) {
6195                 DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", mypnn));
6196         }
6197
6198         /* initiate a recovery */
6199         control_recover(ctdb, argc, argv);
6200
6201         return 0;
6202 }
6203
6204
6205 static const struct {
6206         const char *name;
6207         int (*fn)(struct ctdb_context *, int, const char **);
6208         bool auto_all;
6209         bool without_daemon; /* can be run without daemon running ? */
6210         const char *msg;
6211         const char *args;
6212 } ctdb_commands[] = {
6213         { "version",         control_version,           true,   true,   "show version of ctdb" },
6214         { "status",          control_status,            true,   false,  "show node status" },
6215         { "uptime",          control_uptime,            true,   false,  "show node uptime" },
6216         { "ping",            control_ping,              true,   false,  "ping all nodes" },
6217         { "runstate",        control_runstate,          true,   false,  "get/check runstate of a node", "[setup|first_recovery|startup|running]" },
6218         { "getvar",          control_getvar,            true,   false,  "get a tunable variable",               "<name>"},
6219         { "setvar",          control_setvar,            true,   false,  "set a tunable variable",               "<name> <value>"},
6220         { "listvars",        control_listvars,          true,   false,  "list tunable variables"},
6221         { "statistics",      control_statistics,        false,  false, "show statistics" },
6222         { "statisticsreset", control_statistics_reset,  true,   false,  "reset statistics"},
6223         { "stats",           control_stats,             false,  false,  "show rolling statistics", "[number of history records]" },
6224         { "ip",              control_ip,                false,  false,  "show which public ip's that ctdb manages" },
6225         { "ipinfo",          control_ipinfo,            true,   false,  "show details about a public ip that ctdb manages", "<ip>" },
6226         { "ifaces",          control_ifaces,            true,   false,  "show which interfaces that ctdb manages" },
6227         { "setifacelink",    control_setifacelink,      true,   false,  "set interface link status", "<iface> <status>" },
6228         { "process-exists",  control_process_exists,    true,   false,  "check if a process exists on a node",  "<pid>"},
6229         { "getdbmap",        control_getdbmap,          true,   false,  "show the database map" },
6230         { "getdbstatus",     control_getdbstatus,       true,   false,  "show the status of a database", "<dbname|dbid>" },
6231         { "catdb",           control_catdb,             true,   false,  "dump a ctdb database" ,                     "<dbname|dbid>"},
6232         { "cattdb",          control_cattdb,            true,   false,  "dump a local tdb database" ,                     "<dbname|dbid>"},
6233         { "getmonmode",      control_getmonmode,        true,   false,  "show monitoring mode" },
6234         { "getcapabilities", control_getcapabilities,   true,   false,  "show node capabilities" },
6235         { "pnn",             control_pnn,               true,   false,  "show the pnn of the currnet node" },
6236         { "lvs",             control_lvs,               true,   false,  "show lvs configuration" },
6237         { "lvsmaster",       control_lvsmaster,         true,   false,  "show which node is the lvs master" },
6238         { "disablemonitor",      control_disable_monmode,true,  false,  "set monitoring mode to DISABLE" },
6239         { "enablemonitor",      control_enable_monmode, true,   false,  "set monitoring mode to ACTIVE" },
6240         { "setdebug",        control_setdebug,          true,   false,  "set debug level",                      "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
6241         { "getdebug",        control_getdebug,          true,   false,  "get debug level" },
6242         { "attach",          control_attach,            true,   false,  "attach to a database",                 "<dbname> [persistent]" },
6243         { "detach",          control_detach,            false,  false,  "detach from a database",                 "<dbname|dbid> [<dbname|dbid> ...]" },
6244         { "dumpmemory",      control_dumpmemory,        true,   false,  "dump memory map to stdout" },
6245         { "rddumpmemory",    control_rddumpmemory,      true,   false,  "dump memory map from the recovery daemon to stdout" },
6246         { "getpid",          control_getpid,            true,   false,  "get ctdbd process ID" },
6247         { "disable",         control_disable,           true,   false,  "disable a nodes public IP" },
6248         { "enable",          control_enable,            true,   false,  "enable a nodes public IP" },
6249         { "stop",            control_stop,              true,   false,  "stop a node" },
6250         { "continue",        control_continue,          true,   false,  "re-start a stopped node" },
6251         { "ban",             control_ban,               true,   false,  "ban a node from the cluster",          "<bantime>"},
6252         { "unban",           control_unban,             true,   false,  "unban a node" },
6253         { "showban",         control_showban,           true,   false,  "show ban information"},
6254         { "shutdown",        control_shutdown,          true,   false,  "shutdown ctdbd" },
6255         { "recover",         control_recover,           true,   false,  "force recovery" },
6256         { "sync",            control_ipreallocate,      false,  false,  "wait until ctdbd has synced all state changes" },
6257         { "ipreallocate",    control_ipreallocate,      false,  false,  "force the recovery daemon to perform a ip reallocation procedure" },
6258         { "thaw",            control_thaw,              true,   false,  "thaw databases", "[priority:1-3]" },
6259         { "isnotrecmaster",  control_isnotrecmaster,    false,  false,  "check if the local node is recmaster or not" },
6260         { "killtcp",         kill_tcp,                  false,  false, "kill a tcp connection.", "[<srcip:port> <dstip:port>]" },
6261         { "gratiousarp",     control_gratious_arp,      false,  false, "send a gratious arp", "<ip> <interface>" },
6262         { "tickle",          tickle_tcp,                false,  false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
6263         { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip> [<port>]" },
6264         { "addtickle",       control_add_tickle,        false,  false, "add a tickle for this ip", "<ip>:<port> <ip>:<port>" },
6265
6266         { "deltickle",       control_del_tickle,        false,  false, "delete a tickle from this ip", "<ip>:<port> <ip>:<port>" },
6267
6268         { "regsrvid",        regsrvid,                  false,  false, "register a server id", "<pnn> <type> <id>" },
6269         { "unregsrvid",      unregsrvid,                false,  false, "unregister a server id", "<pnn> <type> <id>" },
6270         { "chksrvid",        chksrvid,                  false,  false, "check if a server id exists", "<pnn> <type> <id>" },
6271         { "getsrvids",       getsrvids,                 false,  false, "get a list of all server ids"},
6272         { "check_srvids",    check_srvids,              false,  false, "check if a srvid exists", "<id>+" },
6273         { "repack",          ctdb_repack,               false,  false, "repack all databases", "[max_freelist]"},
6274         { "listnodes",       control_listnodes,         false,  true, "list all nodes in the cluster"},
6275         { "reloadnodes",     control_reload_nodes_file, false,  false, "reload the nodes file and restart the transport on all nodes"},
6276         { "moveip",          control_moveip,            false,  false, "move/failover an ip address to another node", "<ip> <node>"},
6277         { "rebalanceip",     control_rebalanceip,       false,  false, "release an ip from the node and let recd rebalance it", "<ip>"},
6278         { "addip",           control_addip,             true,   false, "add a ip address to a node", "<ip/mask> <iface>"},
6279         { "delip",           control_delip,             false,  false, "delete an ip address from a node", "<ip>"},
6280         { "eventscript",     control_eventscript,       true,   false, "run the eventscript with the given parameters on a node", "<arguments>"},
6281         { "backupdb",        control_backupdb,          false,  false, "backup the database into a file.", "<dbname|dbid> <file>"},
6282         { "restoredb",        control_restoredb,        false,  false, "restore the database from a file.", "<file> [dbname]"},
6283         { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
6284         { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname|dbid>"},
6285         { "recmaster",        control_recmaster,        true,   false, "show the pnn for the recovery master."},
6286         { "scriptstatus",     control_scriptstatus,     true,   false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
6287         { "enablescript",     control_enablescript,  true,      false, "enable an eventscript", "<script>"},
6288         { "disablescript",    control_disablescript,  true,     false, "disable an eventscript", "<script>"},
6289         { "natgwlist",        control_natgwlist,        true,   false, "show the nodes belonging to this natgw configuration"},
6290         { "xpnn",             control_xpnn,             false,  true,  "find the pnn of the local node without talking to the daemon (unreliable)" },
6291         { "getreclock",       control_getreclock,       true,   false, "Show the reclock file of a node"},
6292         { "setreclock",       control_setreclock,       true,   false, "Set/clear the reclock file of a node", "[filename]"},
6293         { "setnatgwstate",    control_setnatgwstate,    false,  false, "Set NATGW state to on/off", "{on|off}"},
6294         { "setlmasterrole",   control_setlmasterrole,   false,  false, "Set LMASTER role to on/off", "{on|off}"},
6295         { "setrecmasterrole", control_setrecmasterrole, false,  false, "Set RECMASTER role to on/off", "{on|off}"},
6296         { "setdbprio",        control_setdbprio,        false,  false, "Set DB priority", "<dbname|dbid> <prio:1-3>"},
6297         { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbname|dbid>"},
6298         { "setdbreadonly",    control_setdbreadonly,    false,  false, "Set DB readonly capable", "<dbname|dbid>"},
6299         { "setdbsticky",      control_setdbsticky,      false,  false, "Set DB sticky-records capable", "<dbname|dbid>"},
6300         { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
6301         { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
6302         { "pfetch",          control_pfetch,            false,  false,  "fetch a record from a persistent database", "<dbname|dbid> <key> [<file>]" },
6303         { "pstore",          control_pstore,            false,  false,  "write a record to a persistent database", "<dbname|dbid> <key> <file containing record>" },
6304         { "pdelete",         control_pdelete,           false,  false,  "delete a record from a persistent database", "<dbname|dbid> <key>" },
6305         { "ptrans",          control_ptrans,            false,  false,  "update a persistent database (from stdin)", "<dbname|dbid>" },
6306         { "tfetch",          control_tfetch,            false,  true,  "fetch a record from a [c]tdb-file [-v]", "<tdb-file> <key> [<file>]" },
6307         { "tstore",          control_tstore,            false,  true,  "store a record (including ltdb header)", "<tdb-file> <key> <data> [<rsn> <dmaster> <flags>]" },
6308         { "readkey",         control_readkey,           true,   false,  "read the content off a database key", "<dbname|dbid> <key>" },
6309         { "writekey",        control_writekey,          true,   false,  "write to a database key", "<dbname|dbid> <key> <value>" },
6310         { "checktcpport",    control_chktcpport,        false,  true,  "check if a service is bound to a specific tcp port or not", "<port>" },
6311         { "rebalancenode",     control_rebalancenode,   false,  false, "mark nodes as forced IP rebalancing targets", "[<pnn-list>]"},
6312         { "getdbseqnum",     control_getdbseqnum,       false,  false, "get the sequence number off a database", "<dbname|dbid>" },
6313         { "nodestatus",      control_nodestatus,        true,   false,  "show and return node status", "[<pnn-list>]" },
6314         { "dbstatistics",    control_dbstatistics,      false,  false, "show db statistics", "<dbname|dbid>" },
6315         { "reloadips",       control_reloadips,         false,  false, "reload the public addresses file on specified nodes" , "[<pnn-list>]" },
6316         { "ipiface",         control_ipiface,           false,  true,  "Find which interface an ip address is hosted on", "<ip>" },
6317 };
6318
6319 /*
6320   show usage message
6321  */
6322 static void usage(void)
6323 {
6324         int i;
6325         printf(
6326 "Usage: ctdb [options] <control>\n" \
6327 "Options:\n" \
6328 "   -n <node>          choose node number, or 'all' (defaults to local node)\n"
6329 "   -Y                 generate machinereadable output\n"
6330 "   -v                 generate verbose output\n"
6331 "   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
6332         printf("Controls:\n");
6333         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6334                 printf("  %-15s %-27s  %s\n",
6335                        ctdb_commands[i].name,
6336                        ctdb_commands[i].args?ctdb_commands[i].args:"",
6337                        ctdb_commands[i].msg);
6338         }
6339         exit(1);
6340 }
6341
6342
6343 static void ctdb_alarm(int sig)
6344 {
6345         printf("Maximum runtime exceeded - exiting\n");
6346         _exit(ERR_TIMEOUT);
6347 }
6348
6349 /*
6350   main program
6351 */
6352 int main(int argc, const char *argv[])
6353 {
6354         struct ctdb_context *ctdb;
6355         char *nodestring = NULL;
6356         struct poptOption popt_options[] = {
6357                 POPT_AUTOHELP
6358                 POPT_CTDB_CMDLINE
6359                 { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
6360                 { "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
6361                 { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machinereadable output", NULL },
6362                 { "verbose",    'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL },
6363                 { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" },
6364                 { "print-emptyrecords", 0, POPT_ARG_NONE, &options.printemptyrecords, 0, "print the empty records when dumping databases (catdb, cattdb, dumpdbbackup)", NULL },
6365                 { "print-datasize", 0, POPT_ARG_NONE, &options.printdatasize, 0, "do not print record data when dumping databases, only the data size", NULL },
6366                 { "print-lmaster", 0, POPT_ARG_NONE, &options.printlmaster, 0, "print the record's lmaster in catdb", NULL },
6367                 { "print-hash", 0, POPT_ARG_NONE, &options.printhash, 0, "print the record's hash when dumping databases", NULL },
6368                 { "print-recordflags", 0, POPT_ARG_NONE, &options.printrecordflags, 0, "print the record flags in catdb and dumpdbbackup", NULL },
6369                 POPT_TABLEEND
6370         };
6371         int opt;
6372         const char **extra_argv;
6373         int extra_argc = 0;
6374         int ret=-1, i;
6375         poptContext pc;
6376         struct event_context *ev;
6377         const char *control;
6378
6379         setlinebuf(stdout);
6380
6381         /* set some defaults */
6382         options.maxruntime = 0;
6383         options.timelimit = 10;
6384         options.pnn = CTDB_CURRENT_NODE;
6385
6386         pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
6387
6388         while ((opt = poptGetNextOpt(pc)) != -1) {
6389                 switch (opt) {
6390                 default:
6391                         DEBUG(DEBUG_ERR, ("Invalid option %s: %s\n",
6392                                 poptBadOption(pc, 0), poptStrerror(opt)));
6393                         exit(1);
6394                 }
6395         }
6396
6397         /* setup the remaining options for the main program to use */
6398         extra_argv = poptGetArgs(pc);
6399         if (extra_argv) {
6400                 extra_argv++;
6401                 while (extra_argv[extra_argc]) extra_argc++;
6402         }
6403
6404         if (extra_argc < 1) {
6405                 usage();
6406         }
6407
6408         if (options.maxruntime == 0) {
6409                 const char *ctdb_timeout;
6410                 ctdb_timeout = getenv("CTDB_TIMEOUT");
6411                 if (ctdb_timeout != NULL) {
6412                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
6413                 } else {
6414                         /* default timeout is 120 seconds */
6415                         options.maxruntime = 120;
6416                 }
6417         }
6418
6419         signal(SIGALRM, ctdb_alarm);
6420         alarm(options.maxruntime);
6421
6422         control = extra_argv[0];
6423
6424         /* Default value for CTDB_BASE - don't override */
6425         setenv("CTDB_BASE", CTDB_ETCDIR, 0);
6426
6427         ev = event_context_init(NULL);
6428         if (!ev) {
6429                 DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
6430                 exit(1);
6431         }
6432
6433         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6434                 if (strcmp(control, ctdb_commands[i].name) == 0) {
6435                         break;
6436                 }
6437         }
6438
6439         if (i == ARRAY_SIZE(ctdb_commands)) {
6440                 DEBUG(DEBUG_ERR, ("Unknown control '%s'\n", control));
6441                 exit(1);
6442         }
6443
6444         if (ctdb_commands[i].without_daemon == true) {
6445                 if (nodestring != NULL) {
6446                         DEBUG(DEBUG_ERR, ("Can't specify node(s) with \"ctdb %s\"\n", control));
6447                         exit(1);
6448                 }
6449                 return ctdb_commands[i].fn(NULL, extra_argc-1, extra_argv+1);
6450         }
6451
6452         /* initialise ctdb */
6453         ctdb = ctdb_cmdline_client(ev, TIMELIMIT());
6454
6455         if (ctdb == NULL) {
6456                 uint32_t pnn;
6457                 DEBUG(DEBUG_ERR, ("Failed to init ctdb\n"));
6458
6459                 pnn = find_node_xpnn();
6460                 if (pnn == -1) {
6461                         DEBUG(DEBUG_ERR,
6462                               ("Is this node part of a CTDB cluster?\n"));
6463                 }
6464                 exit(1);
6465         }
6466
6467         /* setup the node number(s) to contact */
6468         if (!parse_nodestring(ctdb, ctdb, nodestring, CTDB_CURRENT_NODE, false,
6469                               &options.nodes, &options.pnn)) {
6470                 usage();
6471         }
6472
6473         if (options.pnn == CTDB_CURRENT_NODE) {
6474                 options.pnn = options.nodes[0];
6475         }
6476
6477         if (ctdb_commands[i].auto_all &&
6478             ((options.pnn == CTDB_BROADCAST_ALL) ||
6479              (options.pnn == CTDB_MULTICAST))) {
6480                 int j;
6481
6482                 ret = 0;
6483                 for (j = 0; j < talloc_array_length(options.nodes); j++) {
6484                         options.pnn = options.nodes[j];
6485                         ret |= ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6486                 }
6487         } else {
6488                 ret = ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6489         }
6490
6491         talloc_free(ctdb);
6492         talloc_free(ev);
6493         (void)poptFreeContext(pc);
6494
6495         return ret;
6496
6497 }