ctdb/tools/ctdb.c

   1 /*
   2    ctdb control tool
   3
   4    Copyright (C) Andrew Tridgell  2007
   5    Copyright (C) Ronnie Sahlberg  2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "replace.h"
  22 #include "system/time.h"
  23 #include "system/filesys.h"
  24 #include "system/network.h"
  25 #include "system/locale.h"
  26
  27 #include <popt.h>
  28 #include <talloc.h>
  29 /* Allow use of deprecated function tevent_loop_allow_nesting() */
  30 #define TEVENT_DEPRECATED
  31 #include <tevent.h>
  32 #include <tdb.h>
  33
  34 #include "lib/tdb_wrap/tdb_wrap.h"
  35 #include "lib/util/dlinklist.h"
  36 #include "lib/util/debug.h"
  37 #include "lib/util/substitute.h"
  38 #include "lib/util/time.h"
  39
  40 #include "ctdb_version.h"
  41 #include "ctdb_private.h"
  42 #include "ctdb_client.h"
  43
  44 #include "common/cmdline.h"
  45 #include "common/rb_tree.h"
  46 #include "common/system.h"
  47 #include "common/common.h"
  48 #include "common/logging.h"
  49
  50 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
  51 #define ERR_NONODE      21      /* node does not exist */
  52 #define ERR_DISNODE     22      /* node is disconnected */
  53
  54 static void usage(void);
  55
  56 static struct {
  57         int timelimit;
  58         uint32_t pnn;
  59         uint32_t *nodes;
  60         int machinereadable;
  61         const char *machineseparator;
  62         int verbose;
  63         int maxruntime;
  64         int printemptyrecords;
  65         int printdatasize;
  66         int printlmaster;
  67         int printhash;
  68         int printrecordflags;
  69 } options;
  70
  71 #define LONGTIMEOUT options.timelimit*10
  72
  73 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
  74 #define LONGTIMELIMIT() timeval_current_ofs(LONGTIMEOUT, 0)
  75
  76 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
  77 {
  78         return (tv2->tv_sec - tv->tv_sec) +
  79                 (tv2->tv_usec - tv->tv_usec)*1.0e-6;
  80 }
  81
  82 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
  83 {
  84         printf("CTDB version: %s\n", CTDB_VERSION_STRING);
  85         return 0;
  86 }
  87
  88 /* Like printf(3) but substitute for separator in format */
  89 static int printm(const char *format, ...) PRINTF_ATTRIBUTE(1,2);
  90 static int printm(const char *format, ...)
  91 {
  92         va_list ap;
  93         int ret;
  94         size_t len = strlen(format);
  95         char new_format[len+1];
  96
  97         strcpy(new_format, format);
  98
  99         if (options.machineseparator[0] != ':') {
 100                 all_string_sub(new_format,
 101                                ":", options.machineseparator, len + 1);
 102         }
 103
 104         va_start(ap, format);
 105         ret = vprintf(new_format, ap);
 106         va_end(ap);
 107
 108         return ret;
 109 }
 110
 111 #define CTDB_NOMEM_ABORT(p) do { if (!(p)) {                            \
 112                 DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n",            \
 113                                    "Out of memory in " __location__ )); \
 114                 abort();                                                \
 115         }} while (0)
 116
 117 static uint32_t getpnn(struct ctdb_context *ctdb)
 118 {
 119         if ((options.pnn == CTDB_BROADCAST_ALL) ||
 120             (options.pnn == CTDB_MULTICAST)) {
 121                 DEBUG(DEBUG_ERR,
 122                       ("Cannot get PNN for node %u\n", options.pnn));
 123                 exit(1);
 124         }
 125
 126         if (options.pnn == CTDB_CURRENT_NODE) {
 127                 return ctdb_get_pnn(ctdb);
 128         } else {
 129                 return options.pnn;
 130         }
 131 }
 132
 133 static void assert_single_node_only(void)
 134 {
 135         if ((options.pnn == CTDB_BROADCAST_ALL) ||
 136             (options.pnn == CTDB_MULTICAST)) {
 137                 DEBUG(DEBUG_ERR,
 138                       ("This control can not be applied to multiple PNNs\n"));
 139                 exit(1);
 140         }
 141 }
 142
 143 static void assert_current_node_only(struct ctdb_context *ctdb)
 144 {
 145         if (options.pnn != ctdb_get_pnn(ctdb)) {
 146                 DEBUG(DEBUG_ERR,
 147                       ("This control can only be applied to the current node\n"));
 148                 exit(1);
 149         }
 150 }
 151
 152 /* Pretty print the flags to a static buffer in human-readable format.
 153  * This never returns NULL!
 154  */
 155 static const char *pretty_print_flags(uint32_t flags)
 156 {
 157         int j;
 158         static const struct {
 159                 uint32_t flag;
 160                 const char *name;
 161         } flag_names[] = {
 162                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 163                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 164                 { NODE_FLAGS_BANNED,                "BANNED" },
 165                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 166                 { NODE_FLAGS_DELETED,               "DELETED" },
 167                 { NODE_FLAGS_STOPPED,               "STOPPED" },
 168                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 169         };
 170         static char flags_str[512]; /* Big enough to contain all flag names */
 171
 172         flags_str[0] = '\0';
 173         for (j=0;j<ARRAY_SIZE(flag_names);j++) {
 174                 if (flags & flag_names[j].flag) {
 175                         if (flags_str[0] == '\0') {
 176                                 (void) strcpy(flags_str, flag_names[j].name);
 177                         } else {
 178                                 (void) strncat(flags_str, "|", sizeof(flags_str)-1);
 179                                 (void) strncat(flags_str, flag_names[j].name,
 180                                                sizeof(flags_str)-1);
 181                         }
 182                 }
 183         }
 184         if (flags_str[0] == '\0') {
 185                 (void) strcpy(flags_str, "OK");
 186         }
 187
 188         return flags_str;
 189 }
 190
 191 static int h2i(char h)
 192 {
 193         if (h >= 'a' && h <= 'f') return h - 'a' + 10;
 194         if (h >= 'A' && h <= 'F') return h - 'f' + 10;
 195         return h - '0';
 196 }
 197
 198 static TDB_DATA hextodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
 199 {
 200         int i;
 201         TDB_DATA key = {NULL, 0};
 202
 203         if (len & 0x01) {
 204                 DEBUG(DEBUG_ERR,("Key specified with odd number of hexadecimal digits\n"));
 205                 return key;
 206         }
 207
 208         key.dsize = len>>1;
 209         key.dptr  = talloc_size(mem_ctx, key.dsize);
 210
 211         for (i=0; i < len/2; i++) {
 212                 key.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
 213         }
 214         return key;
 215 }
 216
 217 static TDB_DATA strtodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
 218 {
 219         TDB_DATA key;
 220
 221         if (!strncmp(str, "0x", 2)) {
 222                 key = hextodata(mem_ctx, str + 2, len - 2);
 223         } else {
 224                 key.dptr  = talloc_memdup(mem_ctx, str, len);
 225                 key.dsize = len;
 226         }
 227
 228         return key;
 229 }
 230
 231 /* Parse a nodestring.  Parameter dd_ok controls what happens to nodes
 232  * that are disconnected or deleted.  If dd_ok is true those nodes are
 233  * included in the output list of nodes.  If dd_ok is false, those
 234  * nodes are filtered from the "all" case and cause an error if
 235  * explicitly specified.
 236  */
 237 static bool parse_nodestring(struct ctdb_context *ctdb,
 238                              TALLOC_CTX *mem_ctx,
 239                              const char * nodestring,
 240                              uint32_t current_pnn,
 241                              bool dd_ok,
 242                              uint32_t **nodes,
 243                              uint32_t *pnn_mode)
 244 {
 245         TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
 246         int n;
 247         uint32_t i;
 248         struct ctdb_node_map_old *nodemap;
 249         int ret;
 250
 251         *nodes = NULL;
 252
 253         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
 254         if (ret != 0) {
 255                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
 256                 talloc_free(tmp_ctx);
 257                 exit(10);
 258         }
 259
 260         if (nodestring != NULL) {
 261                 *nodes = talloc_array(mem_ctx, uint32_t, 0);
 262                 if (*nodes == NULL) {
 263                         goto failed;
 264                 }
 265
 266                 n = 0;
 267
 268                 if (strcmp(nodestring, "all") == 0) {
 269                         *pnn_mode = CTDB_BROADCAST_ALL;
 270
 271                         /* all */
 272                         for (i = 0; i < nodemap->num; i++) {
 273                                 if ((nodemap->nodes[i].flags &
 274                                      (NODE_FLAGS_DISCONNECTED |
 275                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 276                                         continue;
 277                                 }
 278                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 279                                                         uint32_t, n+1);
 280                                 if (*nodes == NULL) {
 281                                         goto failed;
 282                                 }
 283                                 (*nodes)[n] = i;
 284                                 n++;
 285                         }
 286                 } else {
 287                         /* x{,y...} */
 288                         char *ns, *tok;
 289
 290                         ns = talloc_strdup(tmp_ctx, nodestring);
 291                         tok = strtok(ns, ",");
 292                         while (tok != NULL) {
 293                                 uint32_t pnn;
 294                                 char *endptr;
 295                                 i = (uint32_t)strtoul(tok, &endptr, 0);
 296                                 if (i == 0 && tok == endptr) {
 297                                         DEBUG(DEBUG_ERR,
 298                                               ("Invalid node %s\n", tok));
 299                                         talloc_free(tmp_ctx);
 300                                         exit(ERR_NONODE);
 301                                 }
 302                                 if (i >= nodemap->num) {
 303                                         DEBUG(DEBUG_ERR, ("Node %u does not exist\n", i));
 304                                         talloc_free(tmp_ctx);
 305                                         exit(ERR_NONODE);
 306                                 }
 307                                 if ((nodemap->nodes[i].flags &
 308                                      (NODE_FLAGS_DISCONNECTED |
 309                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 310                                         DEBUG(DEBUG_ERR, ("Node %u has status %s\n", i, pretty_print_flags(nodemap->nodes[i].flags)));
 311                                         talloc_free(tmp_ctx);
 312                                         exit(ERR_DISNODE);
 313                                 }
 314                                 if ((pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), i)) < 0) {
 315                                         DEBUG(DEBUG_ERR, ("Can not access node %u. Node is not operational.\n", i));
 316                                         talloc_free(tmp_ctx);
 317                                         exit(10);
 318                                 }
 319
 320                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 321                                                         uint32_t, n+1);
 322                                 if (*nodes == NULL) {
 323                                         goto failed;
 324                                 }
 325
 326                                 (*nodes)[n] = i;
 327                                 n++;
 328
 329                                 tok = strtok(NULL, ",");
 330                         }
 331                         talloc_free(ns);
 332
 333                         if (n == 1) {
 334                                 *pnn_mode = (*nodes)[0];
 335                         } else {
 336                                 *pnn_mode = CTDB_MULTICAST;
 337                         }
 338                 }
 339         } else {
 340                 /* default - no nodes specified */
 341                 *nodes = talloc_array(mem_ctx, uint32_t, 1);
 342                 if (*nodes == NULL) {
 343                         goto failed;
 344                 }
 345                 *pnn_mode = CTDB_CURRENT_NODE;
 346
 347                 if (((*nodes)[0] = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), current_pnn)) < 0) {
 348                         goto failed;
 349                 }
 350         }
 351
 352         talloc_free(tmp_ctx);
 353         return true;
 354
 355 failed:
 356         talloc_free(tmp_ctx);
 357         return false;
 358 }
 359
 360 /*
 361  check if a database exists
 362 */
 363 static bool db_exists(struct ctdb_context *ctdb, const char *dbarg,
 364                       uint32_t *dbid, const char **dbname, uint8_t *flags)
 365 {
 366         int i, ret;
 367         struct ctdb_dbid_map_old *dbmap=NULL;
 368         bool dbid_given = false, found = false;
 369         uint32_t id;
 370         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 371         const char *name = NULL;
 372
 373         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
 374         if (ret != 0) {
 375                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
 376                 goto fail;
 377         }
 378
 379         if (strncmp(dbarg, "0x", 2) == 0) {
 380                 id = strtoul(dbarg, NULL, 0);
 381                 dbid_given = true;
 382         }
 383
 384         for(i=0; i<dbmap->num; i++) {
 385                 if (dbid_given) {
 386                         if (id == dbmap->dbs[i].db_id) {
 387                                 found = true;
 388                                 break;
 389                         }
 390                 } else {
 391                         ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, tmp_ctx, &name);
 392                         if (ret != 0) {
 393                                 DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].db_id));
 394                                 goto fail;
 395                         }
 396
 397                         if (strcmp(name, dbarg) == 0) {
 398                                 id = dbmap->dbs[i].db_id;
 399                                 found = true;
 400                                 break;
 401                         }
 402                 }
 403         }
 404
 405         if (found && dbid_given && dbname != NULL) {
 406                 ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, tmp_ctx, &name);
 407                 if (ret != 0) {
 408                         DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].db_id));
 409                         found = false;
 410                         goto fail;
 411                 }
 412         }
 413
 414         if (found) {
 415                 if (dbid) *dbid = id;
 416                 if (dbname) *dbname = talloc_strdup(ctdb, name);
 417                 if (flags) *flags = dbmap->dbs[i].flags;
 418         } else {
 419                 DEBUG(DEBUG_ERR,("No database matching '%s' found\n", dbarg));
 420         }
 421
 422 fail:
 423         talloc_free(tmp_ctx);
 424         return found;
 425 }
 426
 427 /*
 428   see if a process exists
 429  */
 430 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
 431 {
 432         uint32_t pnn, pid;
 433         int ret;
 434         if (argc < 1) {
 435                 usage();
 436         }
 437
 438         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
 439                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
 440                 return -1;
 441         }
 442
 443         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
 444         if (ret == 0) {
 445                 printf("%u:%u exists\n", pnn, pid);
 446         } else {
 447                 printf("%u:%u does not exist\n", pnn, pid);
 448         }
 449         return ret;
 450 }
 451
 452 /*
 453   display statistics structure
 454  */
 455 static void show_statistics(struct ctdb_statistics *s, int show_header)
 456 {
 457         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 458         int i;
 459         const char *prefix=NULL;
 460         int preflen=0;
 461         int tmp, days, hours, minutes, seconds;
 462         const struct {
 463                 const char *name;
 464                 uint32_t offset;
 465         } fields[] = {
 466 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
 467                 STATISTICS_FIELD(num_clients),
 468                 STATISTICS_FIELD(frozen),
 469                 STATISTICS_FIELD(recovering),
 470                 STATISTICS_FIELD(num_recoveries),
 471                 STATISTICS_FIELD(client_packets_sent),
 472                 STATISTICS_FIELD(client_packets_recv),
 473                 STATISTICS_FIELD(node_packets_sent),
 474                 STATISTICS_FIELD(node_packets_recv),
 475                 STATISTICS_FIELD(keepalive_packets_sent),
 476                 STATISTICS_FIELD(keepalive_packets_recv),
 477                 STATISTICS_FIELD(node.req_call),
 478                 STATISTICS_FIELD(node.reply_call),
 479                 STATISTICS_FIELD(node.req_dmaster),
 480                 STATISTICS_FIELD(node.reply_dmaster),
 481                 STATISTICS_FIELD(node.reply_error),
 482                 STATISTICS_FIELD(node.req_message),
 483                 STATISTICS_FIELD(node.req_control),
 484                 STATISTICS_FIELD(node.reply_control),
 485                 STATISTICS_FIELD(client.req_call),
 486                 STATISTICS_FIELD(client.req_message),
 487                 STATISTICS_FIELD(client.req_control),
 488                 STATISTICS_FIELD(timeouts.call),
 489                 STATISTICS_FIELD(timeouts.control),
 490                 STATISTICS_FIELD(timeouts.traverse),
 491                 STATISTICS_FIELD(locks.num_calls),
 492                 STATISTICS_FIELD(locks.num_current),
 493                 STATISTICS_FIELD(locks.num_pending),
 494                 STATISTICS_FIELD(locks.num_failed),
 495                 STATISTICS_FIELD(total_calls),
 496                 STATISTICS_FIELD(pending_calls),
 497                 STATISTICS_FIELD(childwrite_calls),
 498                 STATISTICS_FIELD(pending_childwrite_calls),
 499                 STATISTICS_FIELD(memory_used),
 500                 STATISTICS_FIELD(max_hop_count),
 501                 STATISTICS_FIELD(total_ro_delegations),
 502                 STATISTICS_FIELD(total_ro_revokes),
 503         };
 504
 505         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
 506         seconds = tmp%60;
 507         tmp    /= 60;
 508         minutes = tmp%60;
 509         tmp    /= 60;
 510         hours   = tmp%24;
 511         tmp    /= 24;
 512         days    = tmp;
 513
 514         if (options.machinereadable){
 515                 if (show_header) {
 516                         printm("CTDB version:");
 517                         printm("Current time of statistics:");
 518                         printm("Statistics collected since:");
 519                         for (i=0;i<ARRAY_SIZE(fields);i++) {
 520                                 printm("%s:", fields[i].name);
 521                         }
 522                         printm("num_reclock_ctdbd_latency:");
 523                         printm("min_reclock_ctdbd_latency:");
 524                         printm("avg_reclock_ctdbd_latency:");
 525                         printm("max_reclock_ctdbd_latency:");
 526
 527                         printm("num_reclock_recd_latency:");
 528                         printm("min_reclock_recd_latency:");
 529                         printm("avg_reclock_recd_latency:");
 530                         printm("max_reclock_recd_latency:");
 531
 532                         printm("num_call_latency:");
 533                         printm("min_call_latency:");
 534                         printm("avg_call_latency:");
 535                         printm("max_call_latency:");
 536
 537                         printm("num_lockwait_latency:");
 538                         printm("min_lockwait_latency:");
 539                         printm("avg_lockwait_latency:");
 540                         printm("max_lockwait_latency:");
 541
 542                         printm("num_childwrite_latency:");
 543                         printm("min_childwrite_latency:");
 544                         printm("avg_childwrite_latency:");
 545                         printm("max_childwrite_latency:");
 546                         printm("\n");
 547                 }
 548                 printm("%d:", CTDB_PROTOCOL);
 549                 printm("%d:", (int)s->statistics_current_time.tv_sec);
 550                 printm("%d:", (int)s->statistics_start_time.tv_sec);
 551                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 552                         printm("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 553                 }
 554                 printm("%d:", s->reclock.ctdbd.num);
 555                 printm("%.6f:", s->reclock.ctdbd.min);
 556                 printm("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
 557                 printm("%.6f:", s->reclock.ctdbd.max);
 558
 559                 printm("%d:", s->reclock.recd.num);
 560                 printm("%.6f:", s->reclock.recd.min);
 561                 printm("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
 562                 printm("%.6f:", s->reclock.recd.max);
 563
 564                 printm("%d:", s->call_latency.num);
 565                 printm("%.6f:", s->call_latency.min);
 566                 printm("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
 567                 printm("%.6f:", s->call_latency.max);
 568
 569                 printm("%d:", s->childwrite_latency.num);
 570                 printm("%.6f:", s->childwrite_latency.min);
 571                 printm("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
 572                 printm("%.6f:", s->childwrite_latency.max);
 573                 printm("\n");
 574         } else {
 575                 printf("CTDB version %u\n", CTDB_PROTOCOL);
 576                 printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
 577                 printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
 578
 579                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 580                         if (strchr(fields[i].name, '.')) {
 581                                 preflen = strcspn(fields[i].name, ".")+1;
 582                                 if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
 583                                         prefix = fields[i].name;
 584                                         printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
 585                                 }
 586                         } else {
 587                                 preflen = 0;
 588                         }
 589                         printf(" %*s%-22s%*s%10u\n",
 590                                preflen?4:0, "",
 591                                fields[i].name+preflen,
 592                                preflen?0:4, "",
 593                                *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 594                 }
 595                 printf(" hop_count_buckets:");
 596                 for (i=0;i<MAX_COUNT_BUCKETS;i++) {
 597                         printf(" %d", s->hop_count_bucket[i]);
 598                 }
 599                 printf("\n");
 600                 printf(" lock_buckets:");
 601                 for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 602                         printf(" %d", s->locks.buckets[i]);
 603                 }
 604                 printf("\n");
 605                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
 606
 607                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
 608
 609                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
 610
 611                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
 612                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
 613         }
 614
 615         talloc_free(tmp_ctx);
 616 }
 617
 618 /*
 619   display remote ctdb statistics combined from all nodes
 620  */
 621 static int control_statistics_all(struct ctdb_context *ctdb)
 622 {
 623         int ret, i;
 624         struct ctdb_statistics statistics;
 625         uint32_t *nodes;
 626         uint32_t num_nodes;
 627
 628         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
 629         CTDB_NO_MEMORY(ctdb, nodes);
 630
 631         ZERO_STRUCT(statistics);
 632
 633         for (i=0;i<num_nodes;i++) {
 634                 struct ctdb_statistics s1;
 635                 int j;
 636                 uint32_t *v1 = (uint32_t *)&s1;
 637                 uint32_t *v2 = (uint32_t *)&statistics;
 638                 uint32_t num_ints =
 639                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
 640                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
 641                 if (ret != 0) {
 642                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
 643                         return ret;
 644                 }
 645                 for (j=0;j<num_ints;j++) {
 646                         v2[j] += v1[j];
 647                 }
 648                 statistics.max_hop_count =
 649                         MAX(statistics.max_hop_count, s1.max_hop_count);
 650                 statistics.call_latency.max =
 651                         MAX(statistics.call_latency.max, s1.call_latency.max);
 652         }
 653         talloc_free(nodes);
 654         printf("Gathered statistics for %u nodes\n", num_nodes);
 655         show_statistics(&statistics, 1);
 656         return 0;
 657 }
 658
 659 /*
 660   display remote ctdb statistics
 661  */
 662 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
 663 {
 664         int ret;
 665         struct ctdb_statistics statistics;
 666
 667         if (options.pnn == CTDB_BROADCAST_ALL) {
 668                 return control_statistics_all(ctdb);
 669         }
 670
 671         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
 672         if (ret != 0) {
 673                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
 674                 return ret;
 675         }
 676         show_statistics(&statistics, 1);
 677         return 0;
 678 }
 679
 680
 681 /*
 682   reset remote ctdb statistics
 683  */
 684 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
 685 {
 686         int ret;
 687
 688         ret = ctdb_statistics_reset(ctdb, options.pnn);
 689         if (ret != 0) {
 690                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
 691                 return ret;
 692         }
 693         return 0;
 694 }
 695
 696
 697 /*
 698   display remote ctdb rolling statistics
 699  */
 700 static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
 701 {
 702         int ret;
 703         struct ctdb_statistics_list_old *stats;
 704         int i, num_records = -1;
 705
 706         assert_single_node_only();
 707
 708         if (argc ==1) {
 709                 num_records = atoi(argv[0]) - 1;
 710         }
 711
 712         ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
 713         if (ret != 0) {
 714                 DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
 715                 return ret;
 716         }
 717         for (i=0;i<stats->num;i++) {
 718                 if (stats->stats[i].statistics_start_time.tv_sec == 0) {
 719                         continue;
 720                 }
 721                 show_statistics(&stats->stats[i], i==0);
 722                 if (i == num_records) {
 723                         break;
 724                 }
 725         }
 726         return 0;
 727 }
 728
 729
 730 /*
 731   display remote ctdb db statistics
 732  */
 733 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
 734 {
 735         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 736         struct ctdb_db_statistics_old *dbstat;
 737         int i;
 738         uint32_t db_id;
 739         int num_hot_keys;
 740         int ret;
 741
 742         if (argc < 1) {
 743                 usage();
 744         }
 745
 746         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
 747                 return -1;
 748         }
 749
 750         ret = ctdb_ctrl_dbstatistics(ctdb, options.pnn, db_id, tmp_ctx, &dbstat);
 751         if (ret != 0) {
 752                 DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
 753                 talloc_free(tmp_ctx);
 754                 return -1;
 755         }
 756
 757         printf("DB Statistics: %s\n", argv[0]);
 758         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
 759                 dbstat->db_ro_delegations);
 760         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
 761                 dbstat->db_ro_delegations);
 762         printf(" %s\n", "locks");
 763         printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
 764                 dbstat->locks.num_calls);
 765         printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
 766                 dbstat->locks.num_failed);
 767         printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
 768                 dbstat->locks.num_current);
 769         printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
 770                 dbstat->locks.num_pending);
 771         printf(" %s", "hop_count_buckets:");
 772         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 773                 printf(" %d", dbstat->hop_count_bucket[i]);
 774         }
 775         printf("\n");
 776         printf(" %s", "lock_buckets:");
 777         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 778                 printf(" %d", dbstat->locks.buckets[i]);
 779         }
 780         printf("\n");
 781         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
 782                 "locks_latency      MIN/AVG/MAX",
 783                 dbstat->locks.latency.min,
 784                 (dbstat->locks.latency.num ?
 785                  dbstat->locks.latency.total /dbstat->locks.latency.num :
 786                  0.0),
 787                 dbstat->locks.latency.max,
 788                 dbstat->locks.latency.num);
 789         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
 790                 "vacuum_latency     MIN/AVG/MAX",
 791                 dbstat->vacuum.latency.min,
 792                 (dbstat->vacuum.latency.num ?
 793                  dbstat->vacuum.latency.total /dbstat->vacuum.latency.num :
 794                  0.0),
 795                 dbstat->vacuum.latency.max,
 796                 dbstat->vacuum.latency.num);
 797         num_hot_keys = 0;
 798         for (i=0; i<dbstat->num_hot_keys; i++) {
 799                 if (dbstat->hot_keys[i].count > 0) {
 800                         num_hot_keys++;
 801                 }
 802         }
 803         dbstat->num_hot_keys = num_hot_keys;
 804
 805         printf(" Num Hot Keys:     %d\n", dbstat->num_hot_keys);
 806         for (i = 0; i < dbstat->num_hot_keys; i++) {
 807                 int j;
 808                 printf("     Count:%d Key:", dbstat->hot_keys[i].count);
 809                 for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
 810                         printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
 811                 }
 812                 printf("\n");
 813         }
 814
 815         talloc_free(tmp_ctx);
 816         return 0;
 817 }
 818
 819 /*
 820   display uptime of remote node
 821  */
 822 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
 823 {
 824         int ret;
 825         struct ctdb_uptime *uptime = NULL;
 826         int tmp, days, hours, minutes, seconds;
 827
 828         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
 829         if (ret != 0) {
 830                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
 831                 return ret;
 832         }
 833
 834         if (options.machinereadable){
 835                 printm(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
 836                 printm(":%u:%u:%u:%lf\n",
 837                         (unsigned int)uptime->current_time.tv_sec,
 838                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
 839                         (unsigned int)uptime->last_recovery_finished.tv_sec,
 840                         timeval_delta(&uptime->last_recovery_finished,
 841                                       &uptime->last_recovery_started)
 842                 );
 843                 return 0;
 844         }
 845
 846         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
 847
 848         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 849         seconds = tmp%60;
 850         tmp    /= 60;
 851         minutes = tmp%60;
 852         tmp    /= 60;
 853         hours   = tmp%24;
 854         tmp    /= 24;
 855         days    = tmp;
 856         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
 857
 858         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 859         seconds = tmp%60;
 860         tmp    /= 60;
 861         minutes = tmp%60;
 862         tmp    /= 60;
 863         hours   = tmp%24;
 864         tmp    /= 24;
 865         days    = tmp;
 866         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
 867
 868         printf("Duration of last recovery/failover: %lf seconds\n",
 869                 timeval_delta(&uptime->last_recovery_finished,
 870                               &uptime->last_recovery_started));
 871
 872         return 0;
 873 }
 874
 875 /*
 876   show the PNN of the current node
 877  */
 878 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
 879 {
 880         uint32_t mypnn;
 881
 882         mypnn = getpnn(ctdb);
 883
 884         printf("PNN:%d\n", mypnn);
 885         return 0;
 886 }
 887
 888
 889 static struct ctdb_node_map_old *read_nodes_file(TALLOC_CTX *mem_ctx)
 890 {
 891         const char *nodes_list;
 892
 893         /* read the nodes file */
 894         nodes_list = getenv("CTDB_NODES");
 895         if (nodes_list == NULL) {
 896                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes",
 897                                              getenv("CTDB_BASE"));
 898                 if (nodes_list == NULL) {
 899                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
 900                         exit(1);
 901                 }
 902         }
 903
 904         return ctdb_read_nodes_file(mem_ctx, nodes_list);
 905 }
 906
 907 /*
 908   show the PNN of the current node
 909   discover the pnn by loading the nodes file and try to bind to all
 910   addresses one at a time until the ip address is found.
 911  */
 912 static int find_node_xpnn(void)
 913 {
 914         TALLOC_CTX *mem_ctx = talloc_new(NULL);
 915         struct ctdb_node_map_old *node_map;
 916         int i, pnn;
 917
 918         node_map = read_nodes_file(mem_ctx);
 919         if (node_map == NULL) {
 920                 talloc_free(mem_ctx);
 921                 return -1;
 922         }
 923
 924         for (i = 0; i < node_map->num; i++) {
 925                 if (node_map->nodes[i].flags & NODE_FLAGS_DELETED) {
 926                         continue;
 927                 }
 928                 if (ctdb_sys_have_ip(&node_map->nodes[i].addr)) {
 929                         pnn = node_map->nodes[i].pnn;
 930                         talloc_free(mem_ctx);
 931                         return pnn;
 932                 }
 933         }
 934
 935         printf("Failed to detect which PNN this node is\n");
 936         talloc_free(mem_ctx);
 937         return -1;
 938 }
 939
 940 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
 941 {
 942         uint32_t pnn;
 943
 944         assert_single_node_only();
 945
 946         pnn = find_node_xpnn();
 947         if (pnn == -1) {
 948                 return -1;
 949         }
 950
 951         printf("PNN:%d\n", pnn);
 952         return 0;
 953 }
 954
 955 /* Helpers for ctdb status
 956  */
 957 static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_flags *node)
 958 {
 959         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 960         int j;
 961         bool ret = false;
 962
 963         if (node->flags == 0) {
 964                 struct ctdb_iface_list_old *ifaces;
 965
 966                 if (ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), node->pnn,
 967                                          tmp_ctx, &ifaces) == 0) {
 968                         for (j=0; j < ifaces->num; j++) {
 969                                 if (ifaces->ifaces[j].link_state != 0) {
 970                                         continue;
 971                                 }
 972                                 ret = true;
 973                                 break;
 974                         }
 975                 }
 976         }
 977         talloc_free(tmp_ctx);
 978
 979         return ret;
 980 }
 981
 982 static void control_status_header_machine(void)
 983 {
 984         printm(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
 985                ":Inactive:PartiallyOnline:ThisNode:\n");
 986 }
 987
 988 static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn,
 989                                     struct ctdb_node_and_flags *node)
 990 {
 991         printm(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
 992                ctdb_addr_to_str(&node->addr),
 993                !!(node->flags&NODE_FLAGS_DISCONNECTED),
 994                !!(node->flags&NODE_FLAGS_BANNED),
 995                !!(node->flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 996                !!(node->flags&NODE_FLAGS_UNHEALTHY),
 997                !!(node->flags&NODE_FLAGS_STOPPED),
 998                !!(node->flags&NODE_FLAGS_INACTIVE),
 999                is_partially_online(ctdb, node) ? 1 : 0,
1000                (node->pnn == mypnn)?'Y':'N');
1001
1002         return node->flags;
1003 }
1004
1005 static int control_status_1_human(struct ctdb_context *ctdb, int mypnn,
1006                                   struct ctdb_node_and_flags *node)
1007 {
1008        printf("pnn:%d %-16s %s%s\n", node->pnn,
1009               ctdb_addr_to_str(&node->addr),
1010               is_partially_online(ctdb, node) ? "PARTIALLYONLINE" : pretty_print_flags(node->flags),
1011               node->pnn == mypnn?" (THIS NODE)":"");
1012
1013        return node->flags;
1014 }
1015
1016 /*
1017   display remote ctdb status
1018  */
1019 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
1020 {
1021         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1022         int i;
1023         struct ctdb_vnn_map *vnnmap=NULL;
1024         struct ctdb_node_map_old *nodemap=NULL;
1025         uint32_t recmode, recmaster, mypnn;
1026         int num_deleted_nodes = 0;
1027         int ret;
1028
1029         mypnn = getpnn(ctdb);
1030
1031         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1032         if (ret != 0) {
1033                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1034                 talloc_free(tmp_ctx);
1035                 return -1;
1036         }
1037
1038         if (options.machinereadable) {
1039                 control_status_header_machine();
1040                 for (i=0;i<nodemap->num;i++) {
1041                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1042                                 continue;
1043                         }
1044                         (void) control_status_1_machine(ctdb, mypnn,
1045                                                         &nodemap->nodes[i]);
1046                 }
1047                 talloc_free(tmp_ctx);
1048                 return 0;
1049         }
1050
1051         for (i=0; i<nodemap->num; i++) {
1052                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1053                         num_deleted_nodes++;
1054                 }
1055         }
1056         if (num_deleted_nodes == 0) {
1057                 printf("Number of nodes:%d\n", nodemap->num);
1058         } else {
1059                 printf("Number of nodes:%d (including %d deleted nodes)\n",
1060                        nodemap->num, num_deleted_nodes);
1061         }
1062         for(i=0;i<nodemap->num;i++){
1063                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1064                         continue;
1065                 }
1066                 (void) control_status_1_human(ctdb, mypnn, &nodemap->nodes[i]);
1067         }
1068
1069         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
1070         if (ret != 0) {
1071                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
1072                 talloc_free(tmp_ctx);
1073                 return -1;
1074         }
1075         if (vnnmap->generation == INVALID_GENERATION) {
1076                 printf("Generation:INVALID\n");
1077         } else {
1078                 printf("Generation:%d\n",vnnmap->generation);
1079         }
1080         printf("Size:%d\n",vnnmap->size);
1081         for(i=0;i<vnnmap->size;i++){
1082                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
1083         }
1084
1085         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmode);
1086         if (ret != 0) {
1087                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
1088                 talloc_free(tmp_ctx);
1089                 return -1;
1090         }
1091         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
1092
1093         ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmaster);
1094         if (ret != 0) {
1095                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1096                 talloc_free(tmp_ctx);
1097                 return -1;
1098         }
1099         printf("Recovery master:%d\n",recmaster);
1100
1101         talloc_free(tmp_ctx);
1102         return 0;
1103 }
1104
1105 static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **argv)
1106 {
1107         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1108         int i, ret;
1109         struct ctdb_node_map_old *nodemap=NULL;
1110         uint32_t * nodes;
1111         uint32_t pnn_mode, mypnn;
1112
1113         if (argc > 1) {
1114                 usage();
1115         }
1116
1117         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1118                               options.pnn, true, &nodes, &pnn_mode)) {
1119                 return -1;
1120         }
1121
1122         if (options.machinereadable) {
1123                 control_status_header_machine();
1124         } else if (pnn_mode == CTDB_BROADCAST_ALL) {
1125                 printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
1126         }
1127
1128         mypnn = getpnn(ctdb);
1129
1130         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1131         if (ret != 0) {
1132                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1133                 talloc_free(tmp_ctx);
1134                 return -1;
1135         }
1136
1137         ret = 0;
1138
1139         for (i = 0; i < talloc_array_length(nodes); i++) {
1140                 if (options.machinereadable) {
1141                         ret |= control_status_1_machine(ctdb, mypnn,
1142                                                         &nodemap->nodes[nodes[i]]);
1143                 } else {
1144                         ret |= control_status_1_human(ctdb, mypnn,
1145                                                       &nodemap->nodes[nodes[i]]);
1146                 }
1147         }
1148
1149         talloc_free(tmp_ctx);
1150         return ret;
1151 }
1152
1153 /* talloc off the existing nodemap... */
1154 static struct ctdb_node_map_old *talloc_nodemap(struct ctdb_node_map_old *nodemap)
1155 {
1156         return talloc_zero_size(nodemap,
1157                                 offsetof(struct ctdb_node_map_old, nodes) +
1158                                 nodemap->num * sizeof(struct ctdb_node_and_flags));
1159 }
1160
1161 static struct ctdb_node_map_old *
1162 filter_nodemap_by_capabilities(struct ctdb_context *ctdb,
1163                                struct ctdb_node_map_old *nodemap,
1164                                uint32_t required_capabilities,
1165                                bool first_only)
1166 {
1167         int i;
1168         uint32_t capabilities;
1169         struct ctdb_node_map_old *ret;
1170
1171         ret = talloc_nodemap(nodemap);
1172         CTDB_NO_MEMORY_NULL(ctdb, ret);
1173
1174         ret->num = 0;
1175
1176         for (i = 0; i < nodemap->num; i++) {
1177                 int res;
1178
1179                 /* Disconnected nodes have no capabilities! */
1180                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1181                         continue;
1182                 }
1183
1184                 res = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(),
1185                                                 nodemap->nodes[i].pnn,
1186                                                 &capabilities);
1187                 if (res != 0) {
1188                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n",
1189                                           nodemap->nodes[i].pnn));
1190                         talloc_free(ret);
1191                         return NULL;
1192                 }
1193                 if (!(capabilities & required_capabilities)) {
1194                         continue;
1195                 }
1196
1197                 ret->nodes[ret->num] = nodemap->nodes[i];
1198                 ret->num++;
1199                 if (first_only) {
1200                         break;
1201                 }
1202         }
1203
1204         return ret;
1205 }
1206
1207 static struct ctdb_node_map_old *
1208 filter_nodemap_by_flags(struct ctdb_context *ctdb,
1209                         struct ctdb_node_map_old *nodemap,
1210                         uint32_t flags_mask)
1211 {
1212         int i;
1213         struct ctdb_node_map_old *ret;
1214
1215         ret = talloc_nodemap(nodemap);
1216         CTDB_NO_MEMORY_NULL(ctdb, ret);
1217
1218         ret->num = 0;
1219
1220         for (i = 0; i < nodemap->num; i++) {
1221                 if (nodemap->nodes[i].flags & flags_mask) {
1222                         continue;
1223                 }
1224
1225                 ret->nodes[ret->num] = nodemap->nodes[i];
1226                 ret->num++;
1227         }
1228
1229         return ret;
1230 }
1231
1232 /*
1233   display the list of nodes belonging to this natgw configuration
1234  */
1235 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
1236 {
1237         static char prog[PATH_MAX+1] = "";
1238
1239         if (argc != 0) {
1240                 usage();
1241         }
1242
1243         if (!ctdb_set_helper("NAT gateway helper", prog, sizeof(prog),
1244                              "CTDB_NATGW_HELPER", CTDB_HELPER_BINDIR,
1245                              "ctdb_natgw")) {
1246                 DEBUG(DEBUG_ERR, ("Unable to set NAT gateway helper\n"));
1247                 exit(1);
1248         }
1249
1250         execl(prog, prog, "natgwlist", NULL);
1251
1252         DEBUG(DEBUG_ERR,
1253               ("Unable to run NAT gateway helper %s\n", strerror(errno)));
1254         exit(1);
1255 }
1256
1257 /*
1258   display the status of the scripts for monitoring (or other events)
1259  */
1260 static int control_one_scriptstatus(struct ctdb_context *ctdb,
1261                                     enum ctdb_event type)
1262 {
1263         struct ctdb_script_list_old *script_status;
1264         int ret, i;
1265
1266         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
1267         if (ret != 0) {
1268                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
1269                 return ret;
1270         }
1271
1272         if (script_status == NULL) {
1273                 if (!options.machinereadable) {
1274                         printf("%s cycle never run\n",
1275                                ctdb_eventscript_call_names[type]);
1276                 }
1277                 return 0;
1278         }
1279
1280         if (!options.machinereadable) {
1281                 int num_run = 0;
1282                 for (i=0; i<script_status->num_scripts; i++) {
1283                         if (script_status->scripts[i].status != -ENOEXEC) {
1284                                 num_run++;
1285                         }
1286                 }
1287                 printf("%d scripts were executed last %s cycle\n",
1288                        num_run,
1289                        ctdb_eventscript_call_names[type]);
1290         }
1291         for (i=0; i<script_status->num_scripts; i++) {
1292                 const char *status = NULL;
1293
1294                 /* The ETIME status is ignored for certain events.
1295                  * In that case the status is 0, but endtime is not set.
1296                  */
1297                 if (script_status->scripts[i].status == 0 &&
1298                     timeval_is_zero(&script_status->scripts[i].finished)) {
1299                         script_status->scripts[i].status = -ETIME;
1300                 }
1301
1302                 switch (script_status->scripts[i].status) {
1303                 case -ETIME:
1304                         status = "TIMEDOUT";
1305                         break;
1306                 case -ENOEXEC:
1307                         status = "DISABLED";
1308                         break;
1309                 case 0:
1310                         status = "OK";
1311                         break;
1312                 default:
1313                         if (script_status->scripts[i].status > 0)
1314                                 status = "ERROR";
1315                         break;
1316                 }
1317                 if (options.machinereadable) {
1318                         printm(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
1319                                ctdb_eventscript_call_names[type],
1320                                script_status->scripts[i].name,
1321                                script_status->scripts[i].status,
1322                                status,
1323                                (long)script_status->scripts[i].start.tv_sec,
1324                                (long)script_status->scripts[i].start.tv_usec,
1325                                (long)script_status->scripts[i].finished.tv_sec,
1326                                (long)script_status->scripts[i].finished.tv_usec,
1327                                script_status->scripts[i].output);
1328                         continue;
1329                 }
1330                 if (status)
1331                         printf("%-20s Status:%s    ",
1332                                script_status->scripts[i].name, status);
1333                 else
1334                         /* Some other error, eg from stat. */
1335                         printf("%-20s Status:CANNOT RUN (%s)",
1336                                script_status->scripts[i].name,
1337                                strerror(-script_status->scripts[i].status));
1338
1339                 if (script_status->scripts[i].status >= 0) {
1340                         printf("Duration:%.3lf ",
1341                         timeval_delta(&script_status->scripts[i].finished,
1342                               &script_status->scripts[i].start));
1343                 }
1344                 if (script_status->scripts[i].status != -ENOEXEC) {
1345                         printf("%s",
1346                                ctime(&script_status->scripts[i].start.tv_sec));
1347                         if (script_status->scripts[i].status != 0) {
1348                                 printf("   OUTPUT:%s\n",
1349                                        script_status->scripts[i].output);
1350                         }
1351                 } else {
1352                         printf("\n");
1353                 }
1354         }
1355         return 0;
1356 }
1357
1358
1359 static int control_scriptstatus(struct ctdb_context *ctdb,
1360                                 int argc, const char **argv)
1361 {
1362         int ret;
1363         enum ctdb_event type, min, max;
1364         const char *arg;
1365
1366         if (argc > 1) {
1367                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
1368                 return -1;
1369         }
1370
1371         if (argc == 0)
1372                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
1373         else
1374                 arg = argv[0];
1375
1376         for (type = 0; type < CTDB_EVENT_MAX; type++) {
1377                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
1378                         min = type;
1379                         max = type+1;
1380                         break;
1381                 }
1382         }
1383         if (type == CTDB_EVENT_MAX) {
1384                 if (strcmp(arg, "all") == 0) {
1385                         min = 0;
1386                         max = CTDB_EVENT_MAX;
1387                 } else {
1388                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
1389                         return -1;
1390                 }
1391         }
1392
1393         if (options.machinereadable) {
1394                 printm(":Type:Name:Code:Status:Start:End:Error Output...:\n");
1395         }
1396
1397         for (type = min; type < max; type++) {
1398                 ret = control_one_scriptstatus(ctdb, type);
1399                 if (ret != 0) {
1400                         return ret;
1401                 }
1402         }
1403
1404         return 0;
1405 }
1406
1407 /*
1408   enable an eventscript
1409  */
1410 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1411 {
1412         int ret;
1413
1414         if (argc < 1) {
1415                 usage();
1416         }
1417
1418         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1419         if (ret != 0) {
1420           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
1421                 return ret;
1422         }
1423
1424         return 0;
1425 }
1426
1427 /*
1428   disable an eventscript
1429  */
1430 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1431 {
1432         int ret;
1433
1434         if (argc < 1) {
1435                 usage();
1436         }
1437
1438         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1439         if (ret != 0) {
1440           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
1441                 return ret;
1442         }
1443
1444         return 0;
1445 }
1446
1447 /*
1448   display the pnn of the recovery master
1449  */
1450 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
1451 {
1452         uint32_t recmaster;
1453         int ret;
1454
1455         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
1456         if (ret != 0) {
1457                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1458                 return -1;
1459         }
1460         printf("%d\n",recmaster);
1461
1462         return 0;
1463 }
1464
1465 /*
1466   add a tickle to a public address
1467  */
1468 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1469 {
1470         struct ctdb_connection t;
1471         TDB_DATA data;
1472         int ret;
1473
1474         assert_single_node_only();
1475
1476         if (argc < 2) {
1477                 usage();
1478         }
1479
1480         if (parse_ip_port(argv[0], &t.src) == 0) {
1481                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1482                 return -1;
1483         }
1484         if (parse_ip_port(argv[1], &t.dst) == 0) {
1485                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1486                 return -1;
1487         }
1488
1489         data.dptr = (uint8_t *)&t;
1490         data.dsize = sizeof(t);
1491
1492         /* tell all nodes about this tcp connection */
1493         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1494                            0, data, ctdb, NULL, NULL, NULL, NULL);
1495         if (ret != 0) {
1496                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1497                 return -1;
1498         }
1499
1500         return 0;
1501 }
1502
1503
1504 /*
1505   delete a tickle from a node
1506  */
1507 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1508 {
1509         struct ctdb_connection t;
1510         TDB_DATA data;
1511         int ret;
1512
1513         assert_single_node_only();
1514
1515         if (argc < 2) {
1516                 usage();
1517         }
1518
1519         if (parse_ip_port(argv[0], &t.src) == 0) {
1520                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1521                 return -1;
1522         }
1523         if (parse_ip_port(argv[1], &t.dst) == 0) {
1524                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1525                 return -1;
1526         }
1527
1528         data.dptr = (uint8_t *)&t;
1529         data.dsize = sizeof(t);
1530
1531         /* tell all nodes about this tcp connection */
1532         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1533                            0, data, ctdb, NULL, NULL, NULL, NULL);
1534         if (ret != 0) {
1535                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1536                 return -1;
1537         }
1538
1539         return 0;
1540 }
1541
1542
1543 /*
1544   get a list of all tickles for this pnn
1545  */
1546 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1547 {
1548         struct ctdb_tickle_list_old *list;
1549         ctdb_sock_addr addr;
1550         int i, ret;
1551         unsigned port = 0;
1552
1553         assert_single_node_only();
1554
1555         if (argc < 1) {
1556                 usage();
1557         }
1558
1559         if (argc == 2) {
1560                 port = atoi(argv[1]);
1561         }
1562
1563         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1564                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1565                 return -1;
1566         }
1567
1568         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1569         if (ret == -1) {
1570                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1571                 return -1;
1572         }
1573
1574         if (options.machinereadable){
1575                 printm(":source ip:port:destination ip:port:\n");
1576                 for (i=0;i<list->num;i++) {
1577                         if (port && port != ntohs(list->connections[i].dst.ip.sin_port)) {
1578                                 continue;
1579                         }
1580                         printm(":%s:%u", ctdb_addr_to_str(&list->connections[i].src), ntohs(list->connections[i].src.ip.sin_port));
1581                         printm(":%s:%u:\n", ctdb_addr_to_str(&list->connections[i].dst), ntohs(list->connections[i].dst.ip.sin_port));
1582                 }
1583         } else {
1584                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1585                 printf("Num tickles:%u\n", list->num);
1586                 for (i=0;i<list->num;i++) {
1587                         if (port && port != ntohs(list->connections[i].dst.ip.sin_port)) {
1588                                 continue;
1589                         }
1590                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->connections[i].src), ntohs(list->connections[i].src.ip.sin_port));
1591                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->connections[i].dst), ntohs(list->connections[i].dst.ip.sin_port));
1592                 }
1593         }
1594
1595         talloc_free(list);
1596
1597         return 0;
1598 }
1599
1600
1601 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1602 {
1603         struct ctdb_public_ip_list_old *ips;
1604         struct ctdb_public_ip ip;
1605         int i, ret;
1606         uint32_t *nodes;
1607         uint32_t disable_time;
1608         TDB_DATA data;
1609         struct ctdb_node_map_old *nodemap=NULL;
1610         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1611
1612         disable_time = 30;
1613         data.dptr  = (uint8_t*)&disable_time;
1614         data.dsize = sizeof(disable_time);
1615         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1616         if (ret != 0) {
1617                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1618                 return -1;
1619         }
1620
1621
1622
1623         /* read the public ip list from the node */
1624         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1625         if (ret != 0) {
1626                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1627                 talloc_free(tmp_ctx);
1628                 return -1;
1629         }
1630
1631         for (i=0;i<ips->num;i++) {
1632                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1633                         break;
1634                 }
1635         }
1636         if (i==ips->num) {
1637                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1638                         pnn, ctdb_addr_to_str(addr)));
1639                 talloc_free(tmp_ctx);
1640                 return -1;
1641         }
1642
1643         ip.pnn  = pnn;
1644         ip.addr = *addr;
1645
1646         data.dptr  = (uint8_t *)&ip;
1647         data.dsize = sizeof(ip);
1648
1649         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1650         if (ret != 0) {
1651                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1652                 talloc_free(tmp_ctx);
1653                 return ret;
1654         }
1655
1656         nodes = list_of_nodes(ctdb, nodemap, tmp_ctx, NODE_FLAGS_INACTIVE, pnn);
1657         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1658                                         nodes, 0,
1659                                         LONGTIMELIMIT(),
1660                                         false, data,
1661                                         NULL, NULL,
1662                                         NULL);
1663         if (ret != 0) {
1664                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1665                 talloc_free(tmp_ctx);
1666                 return -1;
1667         }
1668
1669         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1670         if (ret != 0) {
1671                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1672                 talloc_free(tmp_ctx);
1673                 return -1;
1674         }
1675
1676         /* update the recovery daemon so it now knows to expect the new
1677            node assignment for this ip.
1678         */
1679         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1680         if (ret != 0) {
1681                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1682                 return -1;
1683         }
1684
1685         talloc_free(tmp_ctx);
1686         return 0;
1687 }
1688
1689
1690 /*
1691  * scans all other nodes and returns a pnn for another node that can host this
1692  * ip address or -1
1693  */
1694 static int
1695 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1696 {
1697         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1698         struct ctdb_public_ip_list_old *ips;
1699         struct ctdb_node_map_old *nodemap=NULL;
1700         int i, j, ret;
1701         int pnn;
1702
1703         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1704         if (ret != 0) {
1705                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1706                 talloc_free(tmp_ctx);
1707                 return ret;
1708         }
1709
1710         for(i=0;i<nodemap->num;i++){
1711                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1712                         continue;
1713                 }
1714                 if (nodemap->nodes[i].pnn == options.pnn) {
1715                         continue;
1716                 }
1717
1718                 /* read the public ip list from this node */
1719                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1720                 if (ret != 0) {
1721                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1722                         return -1;
1723                 }
1724
1725                 for (j=0;j<ips->num;j++) {
1726                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1727                                 pnn = nodemap->nodes[i].pnn;
1728                                 talloc_free(tmp_ctx);
1729                                 return pnn;
1730                         }
1731                 }
1732                 talloc_free(ips);
1733         }
1734
1735         talloc_free(tmp_ctx);
1736         return -1;
1737 }
1738
1739 /* If pnn is -1 then try to find a node to move IP to... */
1740 static bool try_moveip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1741 {
1742         bool pnn_specified = (pnn == -1 ? false : true);
1743         int retries = 0;
1744
1745         while (retries < 5) {
1746                 if (!pnn_specified) {
1747                         pnn = find_other_host_for_public_ip(ctdb, addr);
1748                         if (pnn == -1) {
1749                                 return false;
1750                         }
1751                         DEBUG(DEBUG_NOTICE,
1752                               ("Trying to move public IP to node %u\n", pnn));
1753                 }
1754
1755                 if (move_ip(ctdb, addr, pnn) == 0) {
1756                         return true;
1757                 }
1758
1759                 sleep(3);
1760                 retries++;
1761         }
1762
1763         return false;
1764 }
1765
1766
1767 /*
1768   move/failover an ip address to a specific node
1769  */
1770 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1771 {
1772         uint32_t pnn;
1773         ctdb_sock_addr addr;
1774
1775         assert_single_node_only();
1776
1777         if (argc < 2) {
1778                 usage();
1779                 return -1;
1780         }
1781
1782         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1783                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1784                 return -1;
1785         }
1786
1787
1788         if (sscanf(argv[1], "%u", &pnn) != 1) {
1789                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1790                 return -1;
1791         }
1792
1793         if (!try_moveip(ctdb, &addr, pnn)) {
1794                 DEBUG(DEBUG_ERR,("Failed to move IP to node %d.\n", pnn));
1795                 return -1;
1796         }
1797
1798         return 0;
1799 }
1800
1801 static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
1802 {
1803         TDB_DATA data;
1804
1805         data.dptr  = (uint8_t *)&pnn;
1806         data.dsize = sizeof(uint32_t);
1807         if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
1808                 DEBUG(DEBUG_ERR,
1809                       ("Failed to send message to force node %u to be a rebalancing target\n",
1810                        pnn));
1811                 return -1;
1812         }
1813
1814         return 0;
1815 }
1816
1817
1818 /*
1819   rebalance a node by setting it to allow failback and triggering a
1820   takeover run
1821  */
1822 static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
1823 {
1824         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1825         uint32_t *nodes;
1826         uint32_t pnn_mode;
1827         int i, ret;
1828
1829         assert_single_node_only();
1830
1831         if (argc > 1) {
1832                 usage();
1833         }
1834
1835         /* Determine the nodes where IPs need to be reloaded */
1836         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1837                               options.pnn, true, &nodes, &pnn_mode)) {
1838                 ret = -1;
1839                 goto done;
1840         }
1841
1842         for (i = 0; i < talloc_array_length(nodes); i++) {
1843                 if (!rebalance_node(ctdb, nodes[i])) {
1844                         ret = -1;
1845                 }
1846         }
1847
1848 done:
1849         talloc_free(tmp_ctx);
1850         return ret;
1851 }
1852
1853 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1854 {
1855         struct ctdb_public_ip ip;
1856         int ret;
1857         uint32_t *nodes;
1858         uint32_t disable_time;
1859         TDB_DATA data;
1860         struct ctdb_node_map_old *nodemap=NULL;
1861         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1862
1863         disable_time = 30;
1864         data.dptr  = (uint8_t*)&disable_time;
1865         data.dsize = sizeof(disable_time);
1866         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1867         if (ret != 0) {
1868                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1869                 return -1;
1870         }
1871
1872         ip.pnn  = -1;
1873         ip.addr = *addr;
1874
1875         data.dptr  = (uint8_t *)&ip;
1876         data.dsize = sizeof(ip);
1877
1878         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1879         if (ret != 0) {
1880                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1881                 talloc_free(tmp_ctx);
1882                 return ret;
1883         }
1884
1885         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
1886         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1887                                         nodes, 0,
1888                                         LONGTIMELIMIT(),
1889                                         false, data,
1890                                         NULL, NULL,
1891                                         NULL);
1892         if (ret != 0) {
1893                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1894                 talloc_free(tmp_ctx);
1895                 return -1;
1896         }
1897
1898         talloc_free(tmp_ctx);
1899         return 0;
1900 }
1901
1902 /*
1903   release an ip form all nodes and have it re-assigned by recd
1904  */
1905 static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
1906 {
1907         ctdb_sock_addr addr;
1908
1909         assert_single_node_only();
1910
1911         if (argc < 1) {
1912                 usage();
1913                 return -1;
1914         }
1915
1916         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1917                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1918                 return -1;
1919         }
1920
1921         if (rebalance_ip(ctdb, &addr) != 0) {
1922                 DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
1923                 return -1;
1924         }
1925
1926         return 0;
1927 }
1928
1929 static int getips_store_callback(void *param, void *data)
1930 {
1931         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
1932         struct ctdb_public_ip_list_old *ips = param;
1933         int i;
1934
1935         i = ips->num++;
1936         ips->ips[i].pnn  = node_ip->pnn;
1937         ips->ips[i].addr = node_ip->addr;
1938         return 0;
1939 }
1940
1941 static int getips_count_callback(void *param, void *data)
1942 {
1943         uint32_t *count = param;
1944
1945         (*count)++;
1946         return 0;
1947 }
1948
1949 #define IP_KEYLEN       4
1950 static uint32_t *ip_key(ctdb_sock_addr *ip)
1951 {
1952         static uint32_t key[IP_KEYLEN];
1953
1954         bzero(key, sizeof(key));
1955
1956         switch (ip->sa.sa_family) {
1957         case AF_INET:
1958                 key[0]  = ip->ip.sin_addr.s_addr;
1959                 break;
1960         case AF_INET6: {
1961                 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
1962                 key[0]  = s6_a32[3];
1963                 key[1]  = s6_a32[2];
1964                 key[2]  = s6_a32[1];
1965                 key[3]  = s6_a32[0];
1966                 break;
1967         }
1968         default:
1969                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
1970                 return key;
1971         }
1972
1973         return key;
1974 }
1975
1976 static void *add_ip_callback(void *parm, void *data)
1977 {
1978         return parm;
1979 }
1980
1981 static int
1982 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_public_ip_list_old **ips)
1983 {
1984         struct ctdb_public_ip_list_old *tmp_ips;
1985         struct ctdb_node_map_old *nodemap=NULL;
1986         trbt_tree_t *ip_tree;
1987         int i, j, len, ret;
1988         uint32_t count;
1989
1990         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1991         if (ret != 0) {
1992                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1993                 return ret;
1994         }
1995
1996         ip_tree = trbt_create(tmp_ctx, 0);
1997
1998         for(i=0;i<nodemap->num;i++){
1999                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
2000                         continue;
2001                 }
2002                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2003                         continue;
2004                 }
2005
2006                 /* read the public ip list from this node */
2007                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
2008                 if (ret != 0) {
2009                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
2010                         return -1;
2011                 }
2012
2013                 for (j=0; j<tmp_ips->num;j++) {
2014                         struct ctdb_public_ip *node_ip;
2015
2016                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
2017                         node_ip->pnn  = tmp_ips->ips[j].pnn;
2018                         node_ip->addr = tmp_ips->ips[j].addr;
2019
2020                         trbt_insertarray32_callback(ip_tree,
2021                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
2022                                 add_ip_callback,
2023                                 node_ip);
2024                 }
2025                 talloc_free(tmp_ips);
2026         }
2027
2028         /* traverse */
2029         count = 0;
2030         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
2031
2032         len = offsetof(struct ctdb_public_ip_list_old, ips) +
2033                 count*sizeof(struct ctdb_public_ip);
2034         tmp_ips = talloc_zero_size(tmp_ctx, len);
2035         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
2036
2037         *ips = tmp_ips;
2038
2039         return 0;
2040 }
2041
2042
2043 static void ctdb_every_second(struct tevent_context *ev,
2044                               struct tevent_timer *te,
2045                               struct timeval t, void *p)
2046 {
2047         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2048
2049         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2050                          ctdb_every_second, ctdb);
2051 }
2052
2053 struct srvid_reply_handler_data {
2054         bool done;
2055         bool wait_for_all;
2056         uint32_t *nodes;
2057         const char *srvid_str;
2058 };
2059
2060 static void srvid_broadcast_reply_handler(uint64_t srvid, TDB_DATA data,
2061                                          void *private_data)
2062 {
2063         struct srvid_reply_handler_data *d =
2064                 (struct srvid_reply_handler_data *)private_data;
2065         int i;
2066         int32_t ret;
2067
2068         if (data.dsize != sizeof(ret)) {
2069                 DEBUG(DEBUG_ERR, (__location__ " Wrong reply size\n"));
2070                 return;
2071         }
2072
2073         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
2074         ret = *(int32_t *)data.dptr;
2075         if (ret < 0) {
2076                 DEBUG(DEBUG_ERR,
2077                       ("%s failed with result %d\n", d->srvid_str, ret));
2078                 return;
2079         }
2080
2081         if (!d->wait_for_all) {
2082                 d->done = true;
2083                 return;
2084         }
2085
2086         /* Wait for all replies */
2087         d->done = true;
2088         for (i = 0; i < talloc_array_length(d->nodes); i++) {
2089                 if (d->nodes[i] == ret) {
2090                         DEBUG(DEBUG_DEBUG,
2091                               ("%s reply received from node %u\n",
2092                                d->srvid_str, ret));
2093                         d->nodes[i] = -1;
2094                 }
2095                 if (d->nodes[i] != -1) {
2096                         /* Found a node that hasn't yet replied */
2097                         d->done = false;
2098                 }
2099         }
2100 }
2101
2102 /* Broadcast the given SRVID to all connected nodes.  Wait for 1 reply
2103  * or replies from all connected nodes.  arg is the data argument to
2104  * pass in the srvid_request structure - pass 0 if this isn't needed.
2105  */
2106 static int srvid_broadcast(struct ctdb_context *ctdb,
2107                            uint64_t srvid, uint32_t *arg,
2108                            const char *srvid_str, bool wait_for_all)
2109 {
2110         int ret;
2111         TDB_DATA data;
2112         uint32_t pnn;
2113         uint64_t reply_srvid;
2114         struct ctdb_srvid_message request;
2115         struct ctdb_disable_message request_data;
2116         struct srvid_reply_handler_data reply_data;
2117         struct timeval tv;
2118
2119         ZERO_STRUCT(request);
2120
2121         /* Time ticks to enable timeouts to be processed */
2122         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2123                          ctdb_every_second, ctdb);
2124
2125         pnn = ctdb_get_pnn(ctdb);
2126         reply_srvid = getpid();
2127
2128         if (arg == NULL) {
2129                 request.pnn = pnn;
2130                 request.srvid = reply_srvid;
2131
2132                 data.dptr = (uint8_t *)&request;
2133                 data.dsize = sizeof(request);
2134         } else {
2135                 request_data.pnn = pnn;
2136                 request_data.srvid = reply_srvid;
2137                 request_data.timeout = *arg;
2138
2139                 data.dptr = (uint8_t *)&request_data;
2140                 data.dsize = sizeof(request_data);
2141         }
2142
2143         /* Register message port for reply from recovery master */
2144         ctdb_client_set_message_handler(ctdb, reply_srvid,
2145                                         srvid_broadcast_reply_handler,
2146                                         &reply_data);
2147
2148         reply_data.wait_for_all = wait_for_all;
2149         reply_data.nodes = NULL;
2150         reply_data.srvid_str = srvid_str;
2151
2152 again:
2153         reply_data.done = false;
2154
2155         if (wait_for_all) {
2156                 struct ctdb_node_map_old *nodemap;
2157
2158                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(),
2159                                            CTDB_CURRENT_NODE, ctdb, &nodemap);
2160                 if (ret != 0) {
2161                         DEBUG(DEBUG_ERR,
2162                               ("Unable to get nodemap from current node, try again\n"));
2163                         sleep(1);
2164                         goto again;
2165                 }
2166
2167                 if (reply_data.nodes != NULL) {
2168                         talloc_free(reply_data.nodes);
2169                 }
2170                 reply_data.nodes = list_of_connected_nodes(ctdb, nodemap,
2171                                                            NULL, true);
2172
2173                 talloc_free(nodemap);
2174         }
2175
2176         /* Send to all connected nodes. Only recmaster replies */
2177         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2178                                        srvid, data);
2179         if (ret != 0) {
2180                 /* This can only happen if the socket is closed and
2181                  * there's no way to recover from that, so don't try
2182                  * again.
2183                  */
2184                 DEBUG(DEBUG_ERR,
2185                       ("Failed to send %s request to connected nodes\n",
2186                        srvid_str));
2187                 return -1;
2188         }
2189
2190         tv = timeval_current();
2191         /* This loop terminates the reply is received */
2192         while (timeval_elapsed(&tv) < 5.0 && !reply_data.done) {
2193                 tevent_loop_once(ctdb->ev);
2194         }
2195
2196         if (!reply_data.done) {
2197                 DEBUG(DEBUG_NOTICE,
2198                       ("Still waiting for confirmation of %s\n", srvid_str));
2199                 sleep(1);
2200                 goto again;
2201         }
2202
2203         ctdb_client_remove_message_handler(ctdb, reply_srvid, &reply_data);
2204
2205         talloc_free(reply_data.nodes);
2206
2207         return 0;
2208 }
2209
2210 static int ipreallocate(struct ctdb_context *ctdb)
2211 {
2212         return srvid_broadcast(ctdb, CTDB_SRVID_TAKEOVER_RUN, NULL,
2213                                "IP reallocation", false);
2214 }
2215
2216
2217 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2218 {
2219         return ipreallocate(ctdb);
2220 }
2221
2222 /*
2223   add a public ip address to a node
2224  */
2225 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
2226 {
2227         int i, ret;
2228         int len, retries = 0;
2229         unsigned mask;
2230         ctdb_sock_addr addr;
2231         struct ctdb_addr_info_old *pub;
2232         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2233         struct ctdb_public_ip_list_old *ips;
2234
2235
2236         if (argc != 2) {
2237                 talloc_free(tmp_ctx);
2238                 usage();
2239         }
2240
2241         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
2242                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
2243                 talloc_free(tmp_ctx);
2244                 return -1;
2245         }
2246
2247         /* read the public ip list from the node */
2248         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2249         if (ret != 0) {
2250                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", options.pnn));
2251                 talloc_free(tmp_ctx);
2252                 return -1;
2253         }
2254         for (i=0;i<ips->num;i++) {
2255                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2256                         DEBUG(DEBUG_ERR,("Can not add ip to node. Node already hosts this ip\n"));
2257                         return 0;
2258                 }
2259         }
2260
2261
2262
2263         /* Dont timeout. This command waits for an ip reallocation
2264            which sometimes can take wuite a while if there has
2265            been a recent recovery
2266         */
2267         alarm(0);
2268
2269         len = offsetof(struct ctdb_addr_info_old, iface) + strlen(argv[1]) + 1;
2270         pub = talloc_size(tmp_ctx, len);
2271         CTDB_NO_MEMORY(ctdb, pub);
2272
2273         pub->addr  = addr;
2274         pub->mask  = mask;
2275         pub->len   = strlen(argv[1])+1;
2276         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
2277
2278         do {
2279                 ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
2280                 if (ret != 0) {
2281                         DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
2282                         sleep(3);
2283                         retries++;
2284                 }
2285         } while (retries < 5 && ret != 0);
2286         if (ret != 0) {
2287                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
2288                 talloc_free(tmp_ctx);
2289                 return ret;
2290         }
2291
2292         if (rebalance_node(ctdb, options.pnn) != 0) {
2293                 DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
2294                 return ret;
2295         }
2296
2297         talloc_free(tmp_ctx);
2298         return 0;
2299 }
2300
2301 /*
2302   add a public ip address to a node
2303  */
2304 static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
2305 {
2306         ctdb_sock_addr addr;
2307         char *iface = NULL;
2308
2309         if (argc != 1) {
2310                 usage();
2311         }
2312
2313         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2314                 printf("Badly formed ip : %s\n", argv[0]);
2315                 return -1;
2316         }
2317
2318         iface = ctdb_sys_find_ifname(&addr);
2319         if (iface == NULL) {
2320                 printf("Failed to get interface name for ip: %s", argv[0]);
2321                 return -1;
2322         }
2323
2324         printf("IP on interface %s\n", iface);
2325
2326         free(iface);
2327
2328         return 0;
2329 }
2330
2331 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
2332
2333 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
2334 {
2335         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2336         struct ctdb_node_map_old *nodemap=NULL;
2337         struct ctdb_public_ip_list_old *ips;
2338         int ret, i, j;
2339
2340         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2341         if (ret != 0) {
2342                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
2343                 return ret;
2344         }
2345
2346         /* remove it from the nodes that are not hosting the ip currently */
2347         for(i=0;i<nodemap->num;i++){
2348                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2349                         continue;
2350                 }
2351                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2352                 if (ret != 0) {
2353                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2354                         continue;
2355                 }
2356
2357                 for (j=0;j<ips->num;j++) {
2358                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2359                                 break;
2360                         }
2361                 }
2362                 if (j==ips->num) {
2363                         continue;
2364                 }
2365
2366                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
2367                         continue;
2368                 }
2369
2370                 options.pnn = nodemap->nodes[i].pnn;
2371                 control_delip(ctdb, argc, argv);
2372         }
2373
2374
2375         /* remove it from every node (also the one hosting it) */
2376         for(i=0;i<nodemap->num;i++){
2377                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2378                         continue;
2379                 }
2380                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2381                 if (ret != 0) {
2382                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2383                         continue;
2384                 }
2385
2386                 for (j=0;j<ips->num;j++) {
2387                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2388                                 break;
2389                         }
2390                 }
2391                 if (j==ips->num) {
2392                         continue;
2393                 }
2394
2395                 options.pnn = nodemap->nodes[i].pnn;
2396                 control_delip(ctdb, argc, argv);
2397         }
2398
2399         talloc_free(tmp_ctx);
2400         return 0;
2401 }
2402
2403 /*
2404   delete a public ip address from a node
2405  */
2406 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
2407 {
2408         int i, ret;
2409         ctdb_sock_addr addr;
2410         struct ctdb_addr_info_old pub;
2411         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2412         struct ctdb_public_ip_list_old *ips;
2413
2414         if (argc != 1) {
2415                 talloc_free(tmp_ctx);
2416                 usage();
2417         }
2418
2419         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2420                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2421                 return -1;
2422         }
2423
2424         if (options.pnn == CTDB_BROADCAST_ALL) {
2425                 return control_delip_all(ctdb, argc, argv, &addr);
2426         }
2427
2428         pub.addr  = addr;
2429         pub.mask  = 0;
2430         pub.len   = 0;
2431
2432         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2433         if (ret != 0) {
2434                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
2435                 talloc_free(tmp_ctx);
2436                 return ret;
2437         }
2438
2439         for (i=0;i<ips->num;i++) {
2440                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2441                         break;
2442                 }
2443         }
2444
2445         if (i==ips->num) {
2446                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
2447                         ctdb_addr_to_str(&addr)));
2448                 talloc_free(tmp_ctx);
2449                 return -1;
2450         }
2451
2452         /* This is an optimisation.  If this node is hosting the IP
2453          * then try to move it somewhere else without invoking a full
2454          * takeover run.  We don't care if this doesn't work!
2455          */
2456         if (ips->ips[i].pnn == options.pnn) {
2457                 (void) try_moveip(ctdb, &addr, -1);
2458         }
2459
2460         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
2461         if (ret != 0) {
2462                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
2463                 talloc_free(tmp_ctx);
2464                 return ret;
2465         }
2466
2467         talloc_free(tmp_ctx);
2468         return 0;
2469 }
2470
2471 static int kill_tcp_from_file(struct ctdb_context *ctdb,
2472                               int argc, const char **argv)
2473 {
2474         struct ctdb_connection *killtcp;
2475         int max_entries, current, i;
2476         struct timeval timeout;
2477         char line[128], src[128], dst[128];
2478         int linenum;
2479         TDB_DATA data;
2480         struct client_async_data *async_data;
2481         struct ctdb_client_control_state *state;
2482
2483         if (argc != 0) {
2484                 usage();
2485         }
2486
2487         linenum = 1;
2488         killtcp = NULL;
2489         max_entries = 0;
2490         current = 0;
2491         while (!feof(stdin)) {
2492                 if (fgets(line, sizeof(line), stdin) == NULL) {
2493                         continue;
2494                 }
2495
2496                 /* Silently skip empty lines */
2497                 if (line[0] == '\n') {
2498                         continue;
2499                 }
2500
2501                 if (sscanf(line, "%s %s\n", src, dst) != 2) {
2502                         DEBUG(DEBUG_ERR, ("Bad line [%d]: '%s'\n",
2503                                           linenum, line));
2504                         talloc_free(killtcp);
2505                         return -1;
2506                 }
2507
2508                 if (current >= max_entries) {
2509                         max_entries += 1024;
2510                         killtcp = talloc_realloc(ctdb, killtcp,
2511                                                  struct ctdb_connection,
2512                                                  max_entries);
2513                         CTDB_NO_MEMORY(ctdb, killtcp);
2514                 }
2515
2516                 if (!parse_ip_port(src, &killtcp[current].src)) {
2517                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2518                                           linenum, src));
2519                         talloc_free(killtcp);
2520                         return -1;
2521                 }
2522
2523                 if (!parse_ip_port(dst, &killtcp[current].dst)) {
2524                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2525                                           linenum, dst));
2526                         talloc_free(killtcp);
2527                         return -1;
2528                 }
2529
2530                 current++;
2531         }
2532
2533         async_data = talloc_zero(ctdb, struct client_async_data);
2534         if (async_data == NULL) {
2535                 talloc_free(killtcp);
2536                 return -1;
2537         }
2538
2539         for (i = 0; i < current; i++) {
2540
2541                 data.dsize = sizeof(struct ctdb_connection);
2542                 data.dptr  = (unsigned char *)&killtcp[i];
2543
2544                 timeout = TIMELIMIT();
2545                 state = ctdb_control_send(ctdb, options.pnn, 0,
2546                                           CTDB_CONTROL_KILL_TCP, 0, data,
2547                                           async_data, &timeout, NULL);
2548
2549                 if (state == NULL) {
2550                         DEBUG(DEBUG_ERR,
2551                               ("Failed to call async killtcp control to node %u\n",
2552                                options.pnn));
2553                         talloc_free(killtcp);
2554                         return -1;
2555                 }
2556
2557                 ctdb_client_async_add(async_data, state);
2558         }
2559
2560         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2561                 DEBUG(DEBUG_ERR,("killtcp failed\n"));
2562                 talloc_free(killtcp);
2563                 return -1;
2564         }
2565
2566         talloc_free(killtcp);
2567         return 0;
2568 }
2569
2570
2571 /*
2572   kill a tcp connection
2573  */
2574 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2575 {
2576         int ret;
2577         struct ctdb_connection killtcp;
2578
2579         assert_single_node_only();
2580
2581         if (argc == 0) {
2582                 return kill_tcp_from_file(ctdb, argc, argv);
2583         }
2584
2585         if (argc < 2) {
2586                 usage();
2587         }
2588
2589         if (!parse_ip_port(argv[0], &killtcp.src)) {
2590                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2591                 return -1;
2592         }
2593
2594         if (!parse_ip_port(argv[1], &killtcp.dst)) {
2595                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2596                 return -1;
2597         }
2598
2599         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
2600         if (ret != 0) {
2601                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
2602                 return ret;
2603         }
2604
2605         return 0;
2606 }
2607
2608
2609 /*
2610   send a gratious arp
2611  */
2612 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
2613 {
2614         int ret;
2615         ctdb_sock_addr addr;
2616
2617         assert_single_node_only();
2618
2619         if (argc < 2) {
2620                 usage();
2621         }
2622
2623         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2624                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
2625                 return -1;
2626         }
2627
2628         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
2629         if (ret != 0) {
2630                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
2631                 return ret;
2632         }
2633
2634         return 0;
2635 }
2636
2637 /*
2638   register a server id
2639  */
2640 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2641 {
2642         int ret;
2643         struct ctdb_client_id server_id;
2644
2645         if (argc < 3) {
2646                 usage();
2647         }
2648
2649         server_id.pnn       = strtoul(argv[0], NULL, 0);
2650         server_id.type      = strtoul(argv[1], NULL, 0);
2651         server_id.server_id = strtoul(argv[2], NULL, 0);
2652
2653         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
2654         if (ret != 0) {
2655                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
2656                 return ret;
2657         }
2658         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
2659         sleep(999);
2660         return -1;
2661 }
2662
2663 /*
2664   unregister a server id
2665  */
2666 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2667 {
2668         int ret;
2669         struct ctdb_client_id server_id;
2670
2671         if (argc < 3) {
2672                 usage();
2673         }
2674
2675         server_id.pnn       = strtoul(argv[0], NULL, 0);
2676         server_id.type      = strtoul(argv[1], NULL, 0);
2677         server_id.server_id = strtoul(argv[2], NULL, 0);
2678
2679         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
2680         if (ret != 0) {
2681                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
2682                 return ret;
2683         }
2684         return -1;
2685 }
2686
2687 /*
2688   check if a server id exists
2689  */
2690 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2691 {
2692         uint32_t status = 0;
2693         int ret;
2694         struct ctdb_client_id server_id;
2695
2696         if (argc < 3) {
2697                 usage();
2698         }
2699
2700         server_id.pnn       = strtoul(argv[0], NULL, 0);
2701         server_id.type      = strtoul(argv[1], NULL, 0);
2702         server_id.server_id = strtoul(argv[2], NULL, 0);
2703
2704         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
2705         if (ret != 0) {
2706                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
2707                 return ret;
2708         }
2709
2710         if (status) {
2711                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
2712         } else {
2713                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
2714         }
2715         return 0;
2716 }
2717
2718 /*
2719   get a list of all server ids that are registered on a node
2720  */
2721 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
2722 {
2723         int i, ret;
2724         struct ctdb_client_id_list_old *server_ids;
2725
2726         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
2727         if (ret != 0) {
2728                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
2729                 return ret;
2730         }
2731
2732         for (i=0; i<server_ids->num; i++) {
2733                 printf("Server id %d:%d:%d\n",
2734                         server_ids->server_ids[i].pnn,
2735                         server_ids->server_ids[i].type,
2736                         server_ids->server_ids[i].server_id);
2737         }
2738
2739         return -1;
2740 }
2741
2742 /*
2743   check if a server id exists
2744  */
2745 static int check_srvids(struct ctdb_context *ctdb, int argc, const char **argv)
2746 {
2747         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
2748         uint64_t *ids;
2749         uint8_t *result;
2750         int i;
2751
2752         if (argc < 1) {
2753                 talloc_free(tmp_ctx);
2754                 usage();
2755         }
2756
2757         ids    = talloc_array(tmp_ctx, uint64_t, argc);
2758         result = talloc_array(tmp_ctx, uint8_t, argc);
2759
2760         for (i = 0; i < argc; i++) {
2761                 ids[i] = strtoull(argv[i], NULL, 0);
2762         }
2763
2764         if (!ctdb_client_check_message_handlers(ctdb, ids, argc, result)) {
2765                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n",
2766                                   options.pnn));
2767                 talloc_free(tmp_ctx);
2768                 return -1;
2769         }
2770
2771         for (i=0; i < argc; i++) {
2772                 printf("Server id %d:%llu %s\n", options.pnn, (long long)ids[i],
2773                        result[i] ? "exists" : "does not exist");
2774         }
2775
2776         talloc_free(tmp_ctx);
2777         return 0;
2778 }
2779
2780 /*
2781   send a tcp tickle ack
2782  */
2783 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2784 {
2785         int ret;
2786         ctdb_sock_addr  src, dst;
2787
2788         if (argc < 2) {
2789                 usage();
2790         }
2791
2792         if (!parse_ip_port(argv[0], &src)) {
2793                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2794                 return -1;
2795         }
2796
2797         if (!parse_ip_port(argv[1], &dst)) {
2798                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2799                 return -1;
2800         }
2801
2802         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
2803         if (ret==0) {
2804                 return 0;
2805         }
2806         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
2807
2808         return -1;
2809 }
2810
2811
2812 /*
2813   display public ip status
2814  */
2815 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
2816 {
2817         int i, ret;
2818         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2819         struct ctdb_public_ip_list_old *ips;
2820
2821         if (argc == 1 && strcmp(argv[0], "all") == 0) {
2822                 options.pnn = CTDB_BROADCAST_ALL;
2823         }
2824
2825         if (options.pnn == CTDB_BROADCAST_ALL) {
2826                 /* read the list of public ips from all nodes */
2827                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
2828         } else {
2829                 /* read the public ip list from this node */
2830                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2831         }
2832         if (ret != 0) {
2833                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
2834                 talloc_free(tmp_ctx);
2835                 return ret;
2836         }
2837
2838         if (options.machinereadable){
2839                 printm(":Public IP:Node:");
2840                 if (options.verbose){
2841                         printm("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
2842                 }
2843                 printm("\n");
2844         } else {
2845                 if (options.pnn == CTDB_BROADCAST_ALL) {
2846                         printf("Public IPs on ALL nodes\n");
2847                 } else {
2848                         printf("Public IPs on node %u\n", options.pnn);
2849                 }
2850         }
2851
2852         for (i=1;i<=ips->num;i++) {
2853                 struct ctdb_public_ip_info_old *info = NULL;
2854                 int32_t pnn;
2855                 char *aciface = NULL;
2856                 char *avifaces = NULL;
2857                 char *cifaces = NULL;
2858
2859                 if (options.pnn == CTDB_BROADCAST_ALL) {
2860                         pnn = ips->ips[ips->num-i].pnn;
2861                 } else {
2862                         pnn = options.pnn;
2863                 }
2864
2865                 if (pnn != -1) {
2866                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
2867                                                    &ips->ips[ips->num-i].addr, &info);
2868                 } else {
2869                         ret = -1;
2870                 }
2871
2872                 if (ret == 0) {
2873                         int j;
2874                         for (j=0; j < info->num; j++) {
2875                                 if (cifaces == NULL) {
2876                                         cifaces = talloc_strdup(info,
2877                                                                 info->ifaces[j].name);
2878                                 } else {
2879                                         cifaces = talloc_asprintf_append(cifaces,
2880                                                                          ",%s",
2881                                                                          info->ifaces[j].name);
2882                                 }
2883
2884                                 if (info->active_idx == j) {
2885                                         aciface = info->ifaces[j].name;
2886                                 }
2887
2888                                 if (info->ifaces[j].link_state == 0) {
2889                                         continue;
2890                                 }
2891
2892                                 if (avifaces == NULL) {
2893                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
2894                                 } else {
2895                                         avifaces = talloc_asprintf_append(avifaces,
2896                                                                           ",%s",
2897                                                                           info->ifaces[j].name);
2898                                 }
2899                         }
2900                 }
2901
2902                 if (options.machinereadable){
2903                         printm(":%s:%d:",
2904                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2905                                 ips->ips[ips->num-i].pnn);
2906                         if (options.verbose){
2907                                 printm("%s:%s:%s:",
2908                                         aciface?aciface:"",
2909                                         avifaces?avifaces:"",
2910                                         cifaces?cifaces:"");
2911                         }
2912                         printf("\n");
2913                 } else {
2914                         if (options.verbose) {
2915                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
2916                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2917                                         ips->ips[ips->num-i].pnn,
2918                                         aciface?aciface:"",
2919                                         avifaces?avifaces:"",
2920                                         cifaces?cifaces:"");
2921                         } else {
2922                                 printf("%s %d\n",
2923                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2924                                         ips->ips[ips->num-i].pnn);
2925                         }
2926                 }
2927                 talloc_free(info);
2928         }
2929
2930         talloc_free(tmp_ctx);
2931         return 0;
2932 }
2933
2934 /*
2935   public ip info
2936  */
2937 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
2938 {
2939         int i, ret;
2940         ctdb_sock_addr addr;
2941         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2942         struct ctdb_public_ip_info_old *info;
2943
2944         if (argc != 1) {
2945                 talloc_free(tmp_ctx);
2946                 usage();
2947         }
2948
2949         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2950                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2951                 return -1;
2952         }
2953
2954         /* read the public ip info from this node */
2955         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
2956                                            tmp_ctx, &addr, &info);
2957         if (ret != 0) {
2958                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
2959                                   argv[0], options.pnn));
2960                 talloc_free(tmp_ctx);
2961                 return ret;
2962         }
2963
2964         printf("Public IP[%s] info on node %u\n",
2965                ctdb_addr_to_str(&info->ip.addr),
2966                options.pnn);
2967
2968         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
2969                ctdb_addr_to_str(&info->ip.addr),
2970                info->ip.pnn, info->num);
2971
2972         for (i=0; i<info->num; i++) {
2973                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
2974
2975                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
2976                        i+1, info->ifaces[i].name,
2977                        info->ifaces[i].link_state?"up":"down",
2978                        (unsigned int)info->ifaces[i].references,
2979                        (i==info->active_idx)?" (active)":"");
2980         }
2981
2982         talloc_free(tmp_ctx);
2983         return 0;
2984 }
2985
2986 /*
2987   display interfaces status
2988  */
2989 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
2990 {
2991         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2992         int i;
2993         struct ctdb_iface_list_old *ifaces;
2994         int ret;
2995
2996         /* read the public ip list from this node */
2997         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ifaces);
2998         if (ret != 0) {
2999                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
3000                                   options.pnn));
3001                 talloc_free(tmp_ctx);
3002                 return -1;
3003         }
3004
3005         if (options.machinereadable){
3006                 printm(":Name:LinkStatus:References:\n");
3007         } else {
3008                 printf("Interfaces on node %u\n", options.pnn);
3009         }
3010
3011         for (i=0; i<ifaces->num; i++) {
3012                 if (options.machinereadable){
3013                         printm(":%s:%s:%u:\n",
3014                                ifaces->ifaces[i].name,
3015                                ifaces->ifaces[i].link_state?"1":"0",
3016                                (unsigned int)ifaces->ifaces[i].references);
3017                 } else {
3018                         printf("name:%s link:%s references:%u\n",
3019                                ifaces->ifaces[i].name,
3020                                ifaces->ifaces[i].link_state?"up":"down",
3021                                (unsigned int)ifaces->ifaces[i].references);
3022                 }
3023         }
3024
3025         talloc_free(tmp_ctx);
3026         return 0;
3027 }
3028
3029
3030 /*
3031   set link status of an interface
3032  */
3033 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
3034 {
3035         int ret;
3036         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3037         struct ctdb_iface info;
3038
3039         ZERO_STRUCT(info);
3040
3041         if (argc != 2) {
3042                 usage();
3043         }
3044
3045         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
3046                 DEBUG(DEBUG_ERR, ("interfaces name '%s' too long\n",
3047                                   argv[0]));
3048                 talloc_free(tmp_ctx);
3049                 return -1;
3050         }
3051         strcpy(info.name, argv[0]);
3052
3053         if (strcmp(argv[1], "up") == 0) {
3054                 info.link_state = 1;
3055         } else if (strcmp(argv[1], "down") == 0) {
3056                 info.link_state = 0;
3057         } else {
3058                 DEBUG(DEBUG_ERR, ("link state invalid '%s' should be 'up' or 'down'\n",
3059                                   argv[1]));
3060                 talloc_free(tmp_ctx);
3061                 return -1;
3062         }
3063
3064         /* read the public ip list from this node */
3065         ret = ctdb_ctrl_set_iface_link(ctdb, TIMELIMIT(), options.pnn,
3066                                    tmp_ctx, &info);
3067         if (ret != 0) {
3068                 DEBUG(DEBUG_ERR, ("Unable to set link state for interfaces %s node %u\n",
3069                                   argv[0], options.pnn));
3070                 talloc_free(tmp_ctx);
3071                 return ret;
3072         }
3073
3074         talloc_free(tmp_ctx);
3075         return 0;
3076 }
3077
3078 /*
3079   display pid of a ctdb daemon
3080  */
3081 static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
3082 {
3083         uint32_t pid;
3084         int ret;
3085
3086         ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.pnn, &pid);
3087         if (ret != 0) {
3088                 DEBUG(DEBUG_ERR, ("Unable to get daemon pid from node %u\n", options.pnn));
3089                 return ret;
3090         }
3091         printf("Pid:%d\n", pid);
3092
3093         return 0;
3094 }
3095
3096 typedef bool update_flags_handler_t(struct ctdb_context *ctdb, void *data);
3097
3098 static int update_flags_and_ipreallocate(struct ctdb_context *ctdb,
3099                                               void *data,
3100                                               update_flags_handler_t handler,
3101                                               uint32_t flag,
3102                                               const char *desc,
3103                                               bool set_flag)
3104 {
3105         struct ctdb_node_map_old *nodemap = NULL;
3106         bool flag_is_set;
3107         int ret;
3108
3109         /* Check if the node is already in the desired state */
3110         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3111         if (ret != 0) {
3112                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3113                 exit(10);
3114         }
3115         flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3116         if (set_flag == flag_is_set) {
3117                 DEBUG(DEBUG_NOTICE, ("Node %d is %s %s\n", options.pnn,
3118                                      (set_flag ? "already" : "not"), desc));
3119                 return 0;
3120         }
3121
3122         do {
3123                 if (!handler(ctdb, data)) {
3124                         DEBUG(DEBUG_WARNING,
3125                               ("Failed to send control to set state %s on node %u, try again\n",
3126                                desc, options.pnn));
3127                 }
3128
3129                 sleep(1);
3130
3131                 /* Read the nodemap and verify the change took effect.
3132                  * Even if the above control/hanlder timed out then it
3133                  * could still have worked!
3134                  */
3135                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
3136                                          ctdb, &nodemap);
3137                 if (ret != 0) {
3138                         DEBUG(DEBUG_WARNING,
3139                               ("Unable to get nodemap from local node, try again\n"));
3140                 }
3141                 flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3142         } while (nodemap == NULL || (set_flag != flag_is_set));
3143
3144         return ipreallocate(ctdb);
3145 }
3146
3147 /* Administratively disable a node */
3148 static bool update_flags_disabled(struct ctdb_context *ctdb, void *data)
3149 {
3150         int ret;
3151
3152         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3153                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
3154         return ret == 0;
3155 }
3156
3157 static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
3158 {
3159         return update_flags_and_ipreallocate(ctdb, NULL,
3160                                                   update_flags_disabled,
3161                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3162                                                   "disabled",
3163                                                   true /* set_flag*/);
3164 }
3165
3166 /* Administratively re-enable a node */
3167 static bool update_flags_not_disabled(struct ctdb_context *ctdb, void *data)
3168 {
3169         int ret;
3170
3171         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3172                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
3173         return ret == 0;
3174 }
3175
3176 static int control_enable(struct ctdb_context *ctdb,  int argc, const char **argv)
3177 {
3178         return update_flags_and_ipreallocate(ctdb, NULL,
3179                                                   update_flags_not_disabled,
3180                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3181                                                   "disabled",
3182                                                   false /* set_flag*/);
3183 }
3184
3185 /* Stop a node */
3186 static bool update_flags_stopped(struct ctdb_context *ctdb, void *data)
3187 {
3188         int ret;
3189
3190         ret = ctdb_ctrl_stop_node(ctdb, TIMELIMIT(), options.pnn);
3191
3192         return ret == 0;
3193 }
3194
3195 static int control_stop(struct ctdb_context *ctdb, int argc, const char **argv)
3196 {
3197         return update_flags_and_ipreallocate(ctdb, NULL,
3198                                                   update_flags_stopped,
3199                                                   NODE_FLAGS_STOPPED,
3200                                                   "stopped",
3201                                                   true /* set_flag*/);
3202 }
3203
3204 /* Continue a stopped node */
3205 static bool update_flags_not_stopped(struct ctdb_context *ctdb, void *data)
3206 {
3207         int ret;
3208
3209         ret = ctdb_ctrl_continue_node(ctdb, TIMELIMIT(), options.pnn);
3210
3211         return ret == 0;
3212 }
3213
3214 static int control_continue(struct ctdb_context *ctdb, int argc, const char **argv)
3215 {
3216         return update_flags_and_ipreallocate(ctdb, NULL,
3217                                                   update_flags_not_stopped,
3218                                                   NODE_FLAGS_STOPPED,
3219                                                   "stopped",
3220                                                   false /* set_flag */);
3221 }
3222
3223 static uint32_t get_generation(struct ctdb_context *ctdb)
3224 {
3225         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3226         struct ctdb_vnn_map *vnnmap=NULL;
3227         int ret;
3228         uint32_t generation;
3229
3230         /* wait until the recmaster is not in recovery mode */
3231         while (1) {
3232                 uint32_t recmode, recmaster;
3233
3234                 if (vnnmap != NULL) {
3235                         talloc_free(vnnmap);
3236                         vnnmap = NULL;
3237                 }
3238
3239                 /* get the recmaster */
3240                 ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), CTDB_CURRENT_NODE, &recmaster);
3241                 if (ret != 0) {
3242                         DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
3243                         talloc_free(tmp_ctx);
3244                         exit(10);
3245                 }
3246
3247                 /* get recovery mode */
3248                 ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), recmaster, &recmode);
3249                 if (ret != 0) {
3250                         DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
3251                         talloc_free(tmp_ctx);
3252                         exit(10);
3253                 }
3254
3255                 /* get the current generation number */
3256                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), recmaster, tmp_ctx, &vnnmap);
3257                 if (ret != 0) {
3258                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from recmaster (%u)\n", recmaster));
3259                         talloc_free(tmp_ctx);
3260                         exit(10);
3261                 }
3262
3263                 if ((recmode == CTDB_RECOVERY_NORMAL) && (vnnmap->generation != 1)) {
3264                         generation = vnnmap->generation;
3265                         talloc_free(tmp_ctx);
3266                         return generation;
3267                 }
3268                 sleep(1);
3269         }
3270 }
3271
3272 /* Ban a node */
3273 static bool update_state_banned(struct ctdb_context *ctdb, void *data)
3274 {
3275         struct ctdb_ban_state *bantime = (struct ctdb_ban_state *)data;
3276         int ret;
3277
3278         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, bantime);
3279
3280         return ret == 0;
3281 }
3282
3283 static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
3284 {
3285         struct ctdb_ban_state bantime;
3286
3287         if (argc < 1) {
3288                 usage();
3289         }
3290
3291         bantime.pnn  = options.pnn;
3292         bantime.time = strtoul(argv[0], NULL, 0);
3293
3294         if (bantime.time == 0) {
3295                 DEBUG(DEBUG_ERR, ("Invalid ban time specified - must be >0\n"));
3296                 return -1;
3297         }
3298
3299         return update_flags_and_ipreallocate(ctdb, &bantime,
3300                                                   update_state_banned,
3301                                                   NODE_FLAGS_BANNED,
3302                                                   "banned",
3303                                                   true /* set_flag*/);
3304 }
3305
3306
3307 /* Unban a node */
3308 static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
3309 {
3310         struct ctdb_ban_state bantime;
3311
3312         bantime.pnn  = options.pnn;
3313         bantime.time = 0;
3314
3315         return update_flags_and_ipreallocate(ctdb, &bantime,
3316                                                   update_state_banned,
3317                                                   NODE_FLAGS_BANNED,
3318                                                   "banned",
3319                                                   false /* set_flag*/);
3320 }
3321
3322 /*
3323   show ban information for a node
3324  */
3325 static int control_showban(struct ctdb_context *ctdb, int argc, const char **argv)
3326 {
3327         int ret;
3328         struct ctdb_node_map_old *nodemap=NULL;
3329         struct ctdb_ban_state *bantime;
3330
3331         /* verify the node exists */
3332         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3333         if (ret != 0) {
3334                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3335                 return ret;
3336         }
3337
3338         ret = ctdb_ctrl_get_ban(ctdb, TIMELIMIT(), options.pnn, ctdb, &bantime);
3339         if (ret != 0) {
3340                 DEBUG(DEBUG_ERR,("Showing ban info for node %d failed.\n", options.pnn));
3341                 return -1;
3342         }
3343
3344         if (bantime->time == 0) {
3345                 printf("Node %u is not banned\n", bantime->pnn);
3346         } else {
3347                 printf("Node %u is banned, %d seconds remaining\n",
3348                        bantime->pnn, bantime->time);
3349         }
3350
3351         return 0;
3352 }
3353
3354 /*
3355   shutdown a daemon
3356  */
3357 static int control_shutdown(struct ctdb_context *ctdb, int argc, const char **argv)
3358 {
3359         int ret;
3360
3361         ret = ctdb_ctrl_shutdown(ctdb, TIMELIMIT(), options.pnn);
3362         if (ret != 0) {
3363                 DEBUG(DEBUG_ERR, ("Unable to shutdown node %u\n", options.pnn));
3364                 return ret;
3365         }
3366
3367         return 0;
3368 }
3369
3370 /*
3371   trigger a recovery
3372  */
3373 static int control_recover(struct ctdb_context *ctdb, int argc, const char **argv)
3374 {
3375         int ret;
3376         uint32_t generation, next_generation;
3377
3378         /* record the current generation number */
3379         generation = get_generation(ctdb);
3380
3381         ret = ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3382         if (ret != 0) {
3383                 DEBUG(DEBUG_ERR, ("Unable to set recovery mode\n"));
3384                 return ret;
3385         }
3386
3387         /* wait until we are in a new generation */
3388         while (1) {
3389                 next_generation = get_generation(ctdb);
3390                 if (next_generation != generation) {
3391                         return 0;
3392                 }
3393                 sleep(1);
3394         }
3395
3396         return 0;
3397 }
3398
3399
3400 /*
3401   display monitoring mode of a remote node
3402  */
3403 static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
3404 {
3405         uint32_t monmode;
3406         int ret;
3407
3408         ret = ctdb_ctrl_getmonmode(ctdb, TIMELIMIT(), options.pnn, &monmode);
3409         if (ret != 0) {
3410                 DEBUG(DEBUG_ERR, ("Unable to get monmode from node %u\n", options.pnn));
3411                 return ret;
3412         }
3413         if (!options.machinereadable){
3414                 printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode);
3415         } else {
3416                 printm(":mode:\n");
3417                 printm(":%d:\n",monmode);
3418         }
3419         return 0;
3420 }
3421
3422
3423 /*
3424   display capabilities of a remote node
3425  */
3426 static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const char **argv)
3427 {
3428         uint32_t capabilities;
3429         int ret;
3430
3431         ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), options.pnn, &capabilities);
3432         if (ret != 0) {
3433                 DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", options.pnn));
3434                 return -1;
3435         }
3436
3437         if (!options.machinereadable){
3438                 printf("RECMASTER: %s\n", (capabilities&CTDB_CAP_RECMASTER)?"YES":"NO");
3439                 printf("LMASTER: %s\n", (capabilities&CTDB_CAP_LMASTER)?"YES":"NO");
3440                 printf("LVS: %s\n", (capabilities&CTDB_CAP_LVS)?"YES":"NO");
3441                 printf("NATGW: %s\n", (capabilities&CTDB_CAP_NATGW)?"YES":"NO");
3442         } else {
3443                 printm(":RECMASTER:LMASTER:LVS:NATGW:\n");
3444                 printm(":%d:%d:%d:%d:\n",
3445                         !!(capabilities&CTDB_CAP_RECMASTER),
3446                         !!(capabilities&CTDB_CAP_LMASTER),
3447                         !!(capabilities&CTDB_CAP_LVS),
3448                         !!(capabilities&CTDB_CAP_NATGW));
3449         }
3450         return 0;
3451 }
3452
3453 /*
3454   display lvs configuration
3455  */
3456
3457 static uint32_t lvs_exclude_flags[] = {
3458         /* Look for a nice healthy node */
3459         NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED,
3460         /* If not found, an UNHEALTHY node will do */
3461         NODE_FLAGS_INACTIVE|NODE_FLAGS_PERMANENTLY_DISABLED,
3462         0,
3463 };
3464
3465 static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
3466 {
3467         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3468         struct ctdb_node_map_old *orig_nodemap=NULL;
3469         struct ctdb_node_map_old *nodemap;
3470         int i, ret;
3471
3472         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3473                                    tmp_ctx, &orig_nodemap);
3474         if (ret != 0) {
3475                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3476                 talloc_free(tmp_ctx);
3477                 return -1;
3478         }
3479
3480         nodemap = filter_nodemap_by_capabilities(ctdb, orig_nodemap,
3481                                                  CTDB_CAP_LVS, false);
3482         if (nodemap == NULL) {
3483                 /* No memory */
3484                 ret = -1;
3485                 goto done;
3486         }
3487
3488         ret = 0;
3489
3490         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3491                 struct ctdb_node_map_old *t =
3492                         filter_nodemap_by_flags(ctdb, nodemap,
3493                                                 lvs_exclude_flags[i]);
3494                 if (t == NULL) {
3495                         /* No memory */
3496                         ret = -1;
3497                         goto done;
3498                 }
3499                 if (t->num > 0) {
3500                         /* At least 1 node without excluded flags */
3501                         int j;
3502                         for (j = 0; j < t->num; j++) {
3503                                 printf("%d:%s\n", t->nodes[j].pnn,
3504                                        ctdb_addr_to_str(&t->nodes[j].addr));
3505                         }
3506                         goto done;
3507                 }
3508                 talloc_free(t);
3509         }
3510 done:
3511         talloc_free(tmp_ctx);
3512         return ret;
3513 }
3514
3515 /*
3516   display who is the lvs master
3517  */
3518 static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **argv)
3519 {
3520         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3521         struct ctdb_node_map_old *nodemap=NULL;
3522         int i, ret;
3523
3524         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3525                                    tmp_ctx, &nodemap);
3526         if (ret != 0) {
3527                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3528                 talloc_free(tmp_ctx);
3529                 return -1;
3530         }
3531
3532         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3533                 struct ctdb_node_map_old *t =
3534                         filter_nodemap_by_flags(ctdb, nodemap,
3535                                                 lvs_exclude_flags[i]);
3536                 if (t == NULL) {
3537                         /* No memory */
3538                         ret = -1;
3539                         goto done;
3540                 }
3541                 if (t->num > 0) {
3542                         struct ctdb_node_map_old *n;
3543                         n = filter_nodemap_by_capabilities(ctdb,
3544                                                            t,
3545                                                            CTDB_CAP_LVS,
3546                                                            true);
3547                         if (n == NULL) {
3548                                 /* No memory */
3549                                 ret = -1;
3550                                 goto done;
3551                         }
3552                         if (n->num > 0) {
3553                                 ret = 0;
3554                                 if (options.machinereadable) {
3555                                         printm("%d\n", n->nodes[0].pnn);
3556                                 } else {
3557                                         printf("Node %d is LVS master\n", n->nodes[0].pnn);
3558                                 }
3559                                 goto done;
3560                         }
3561                 }
3562                 talloc_free(t);
3563         }
3564
3565         printf("There is no LVS master\n");
3566         ret = 255;
3567 done:
3568         talloc_free(tmp_ctx);
3569         return ret;
3570 }
3571
3572 /*
3573   disable monitoring on a  node
3574  */
3575 static int control_disable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3576 {
3577
3578         int ret;
3579
3580         ret = ctdb_ctrl_disable_monmode(ctdb, TIMELIMIT(), options.pnn);
3581         if (ret != 0) {
3582                 DEBUG(DEBUG_ERR, ("Unable to disable monmode on node %u\n", options.pnn));
3583                 return ret;
3584         }
3585         printf("Monitoring mode:%s\n","DISABLED");
3586
3587         return 0;
3588 }
3589
3590 /*
3591   enable monitoring on a  node
3592  */
3593 static int control_enable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3594 {
3595
3596         int ret;
3597
3598         ret = ctdb_ctrl_enable_monmode(ctdb, TIMELIMIT(), options.pnn);
3599         if (ret != 0) {
3600                 DEBUG(DEBUG_ERR, ("Unable to enable monmode on node %u\n", options.pnn));
3601                 return ret;
3602         }
3603         printf("Monitoring mode:%s\n","ACTIVE");
3604
3605         return 0;
3606 }
3607
3608 /*
3609   display remote list of keys/data for a db
3610  */
3611 static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
3612 {
3613         const char *db_name;
3614         struct ctdb_db_context *ctdb_db;
3615         int ret;
3616         struct ctdb_dump_db_context c;
3617         uint8_t flags;
3618
3619         if (argc < 1) {
3620                 usage();
3621         }
3622
3623         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3624                 return -1;
3625         }
3626
3627         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3628         if (ctdb_db == NULL) {
3629                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3630                 return -1;
3631         }
3632
3633         if (options.printlmaster) {
3634                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn,
3635                                           ctdb, &ctdb->vnn_map);
3636                 if (ret != 0) {
3637                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
3638                                           options.pnn));
3639                         return ret;
3640                 }
3641         }
3642
3643         ZERO_STRUCT(c);
3644         c.ctdb = ctdb;
3645         c.f = stdout;
3646         c.printemptyrecords = (bool)options.printemptyrecords;
3647         c.printdatasize = (bool)options.printdatasize;
3648         c.printlmaster = (bool)options.printlmaster;
3649         c.printhash = (bool)options.printhash;
3650         c.printrecordflags = (bool)options.printrecordflags;
3651
3652         /* traverse and dump the cluster tdb */
3653         ret = ctdb_dump_db(ctdb_db, &c);
3654         if (ret == -1) {
3655                 DEBUG(DEBUG_ERR, ("Unable to dump database\n"));
3656                 DEBUG(DEBUG_ERR, ("Maybe try 'ctdb getdbstatus %s'"
3657                                   " and 'ctdb getvar AllowUnhealthyDBRead'\n",
3658                                   db_name));
3659                 return -1;
3660         }
3661         talloc_free(ctdb_db);
3662
3663         printf("Dumped %d records\n", ret);
3664         return 0;
3665 }
3666
3667 struct cattdb_data {
3668         struct ctdb_context *ctdb;
3669         uint32_t count;
3670 };
3671
3672 static int cattdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private_data)
3673 {
3674         struct cattdb_data *d = private_data;
3675         struct ctdb_dump_db_context c;
3676
3677         d->count++;
3678
3679         ZERO_STRUCT(c);
3680         c.ctdb = d->ctdb;
3681         c.f = stdout;
3682         c.printemptyrecords = (bool)options.printemptyrecords;
3683         c.printdatasize = (bool)options.printdatasize;
3684         c.printlmaster = false;
3685         c.printhash = (bool)options.printhash;
3686         c.printrecordflags = true;
3687
3688         return ctdb_dumpdb_record(key, data, &c);
3689 }
3690
3691 /*
3692   cat the local tdb database using same format as catdb
3693  */
3694 static int control_cattdb(struct ctdb_context *ctdb, int argc, const char **argv)
3695 {
3696         const char *db_name;
3697         struct ctdb_db_context *ctdb_db;
3698         struct cattdb_data d;
3699         uint8_t flags;
3700
3701         if (argc < 1) {
3702                 usage();
3703         }
3704
3705         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3706                 return -1;
3707         }
3708
3709         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3710         if (ctdb_db == NULL) {
3711                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3712                 return -1;
3713         }
3714
3715         /* traverse the local tdb */
3716         d.count = 0;
3717         d.ctdb  = ctdb;
3718         if (tdb_traverse_read(ctdb_db->ltdb->tdb, cattdb_traverse, &d) == -1) {
3719                 printf("Failed to cattdb data\n");
3720                 exit(10);
3721         }
3722         talloc_free(ctdb_db);
3723
3724         printf("Dumped %d records\n", d.count);
3725         return 0;
3726 }
3727
3728 /*
3729   display the content of a database key
3730  */
3731 static int control_readkey(struct ctdb_context *ctdb, int argc, const char **argv)
3732 {
3733         const char *db_name;
3734         struct ctdb_db_context *ctdb_db;
3735         struct ctdb_record_handle *h;
3736         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3737         TDB_DATA key, data;
3738         uint8_t flags;
3739
3740         if (argc < 2) {
3741                 usage();
3742         }
3743
3744         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3745                 return -1;
3746         }
3747
3748         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3749         if (ctdb_db == NULL) {
3750                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3751                 return -1;
3752         }
3753
3754         key.dptr  = discard_const(argv[1]);
3755         key.dsize = strlen((char *)key.dptr);
3756
3757         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3758         if (h == NULL) {
3759                 printf("Failed to fetch record '%s' on node %d\n",
3760                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3761                 talloc_free(tmp_ctx);
3762                 exit(10);
3763         }
3764
3765         printf("Data: size:%d ptr:[%.*s]\n", (int)data.dsize, (int)data.dsize, data.dptr);
3766
3767         talloc_free(tmp_ctx);
3768         talloc_free(ctdb_db);
3769         return 0;
3770 }
3771
3772 /*
3773   display the content of a database key
3774  */
3775 static int control_writekey(struct ctdb_context *ctdb, int argc, const char **argv)
3776 {
3777         const char *db_name;
3778         struct ctdb_db_context *ctdb_db;
3779         struct ctdb_record_handle *h;
3780         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3781         TDB_DATA key, data;
3782         uint8_t flags;
3783
3784         if (argc < 3) {
3785                 usage();
3786         }
3787
3788         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3789                 return -1;
3790         }
3791
3792         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3793         if (ctdb_db == NULL) {
3794                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3795                 return -1;
3796         }
3797
3798         key.dptr  = discard_const(argv[1]);
3799         key.dsize = strlen((char *)key.dptr);
3800
3801         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3802         if (h == NULL) {
3803                 printf("Failed to fetch record '%s' on node %d\n",
3804                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3805                 talloc_free(tmp_ctx);
3806                 exit(10);
3807         }
3808
3809         data.dptr  = discard_const(argv[2]);
3810         data.dsize = strlen((char *)data.dptr);
3811
3812         if (ctdb_record_store(h, data) != 0) {
3813                 printf("Failed to store record\n");
3814         }
3815
3816         talloc_free(h);
3817         talloc_free(tmp_ctx);
3818         talloc_free(ctdb_db);
3819         return 0;
3820 }
3821
3822 /*
3823   fetch a record from a persistent database
3824  */
3825 static int control_pfetch(struct ctdb_context *ctdb, int argc, const char **argv)
3826 {
3827         const char *db_name;
3828         struct ctdb_db_context *ctdb_db;
3829         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3830         struct ctdb_transaction_handle *h;
3831         TDB_DATA key, data;
3832         int fd, ret;
3833         bool persistent;
3834         uint8_t flags;
3835
3836         if (argc < 2) {
3837                 talloc_free(tmp_ctx);
3838                 usage();
3839         }
3840
3841         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3842                 talloc_free(tmp_ctx);
3843                 return -1;
3844         }
3845
3846         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
3847         if (!persistent) {
3848                 DEBUG(DEBUG_ERR,("Database '%s' is not persistent\n", db_name));
3849                 talloc_free(tmp_ctx);
3850                 return -1;
3851         }
3852
3853         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
3854         if (ctdb_db == NULL) {
3855                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3856                 talloc_free(tmp_ctx);
3857                 return -1;
3858         }
3859
3860         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
3861         if (h == NULL) {
3862                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
3863                 talloc_free(tmp_ctx);
3864                 return -1;
3865         }
3866
3867         key.dptr  = discard_const(argv[1]);
3868         key.dsize = strlen(argv[1]);
3869         ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data);
3870         if (ret != 0) {
3871                 DEBUG(DEBUG_ERR,("Failed to fetch record\n"));
3872                 talloc_free(tmp_ctx);
3873                 return -1;
3874         }
3875
3876         if (data.dsize == 0 || data.dptr == NULL) {
3877                 DEBUG(DEBUG_ERR,("Record is empty\n"));
3878                 talloc_free(tmp_ctx);
3879                 return -1;
3880         }
3881
3882         if (argc == 3) {
3883           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
3884                 if (fd == -1) {
3885                         DEBUG(DEBUG_ERR,("Failed to open output file %s\n", argv[2]));
3886                         talloc_free(tmp_ctx);
3887                         return -1;
3888                 }
3889                 sys_write(fd, data.dptr, data.dsize);
3890                 close(fd);
3891         } else {
3892                 sys_write(1, data.dptr, data.dsize);
3893         }
3894
3895         /* abort the transaction */
3896         talloc_free(h);
3897
3898
3899         talloc_free(tmp_ctx);
3900         return 0;
3901 }
3902
3903 /*
3904   fetch a record from a tdb-file
3905  */
3906 static int control_tfetch(struct ctdb_context *ctdb, int argc, const char **argv)
3907 {
3908         const char *tdb_file;
3909         TDB_CONTEXT *tdb;
3910         TDB_DATA key, data;
3911         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
3912         int fd;
3913
3914         if (argc < 2) {
3915                 usage();
3916         }
3917
3918         tdb_file = argv[0];
3919
3920         tdb = tdb_open(tdb_file, 0, 0, O_RDONLY, 0);
3921         if (tdb == NULL) {
3922                 printf("Failed to open TDB file %s\n", tdb_file);
3923                 return -1;
3924         }
3925
3926         key = strtodata(tmp_ctx, argv[1], strlen(argv[1]));
3927         if (key.dptr == NULL) {
3928                 printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
3929                 return -1;
3930         }
3931
3932         data = tdb_fetch(tdb, key);
3933         if (data.dptr == NULL || data.dsize < sizeof(struct ctdb_ltdb_header)) {
3934                 printf("Failed to read record %s from tdb %s\n", argv[1], tdb_file);
3935                 tdb_close(tdb);
3936                 return -1;
3937         }
3938
3939         tdb_close(tdb);
3940
3941         if (argc == 3) {
3942           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
3943                 if (fd == -1) {
3944                         printf("Failed to open output file %s\n", argv[2]);
3945                         return -1;
3946                 }
3947                 if (options.verbose){
3948                         sys_write(fd, data.dptr, data.dsize);
3949                 } else {
3950                         sys_write(fd, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
3951                 }
3952                 close(fd);
3953         } else {
3954                 if (options.verbose){
3955                         sys_write(1, data.dptr, data.dsize);
3956                 } else {
3957                         sys_write(1, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
3958                 }
3959         }
3960
3961         talloc_free(tmp_ctx);
3962         return 0;
3963 }
3964
3965 /*
3966   store a record and header to a tdb-file
3967  */
3968 static int control_tstore(struct ctdb_context *ctdb, int argc, const char **argv)
3969 {
3970         const char *tdb_file;
3971         TDB_CONTEXT *tdb;
3972         TDB_DATA key, value, data;
3973         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
3974         struct ctdb_ltdb_header header;
3975
3976         if (argc < 3) {
3977                 usage();
3978         }
3979
3980         tdb_file = argv[0];
3981
3982         tdb = tdb_open(tdb_file, 0, 0, O_RDWR, 0);
3983         if (tdb == NULL) {
3984                 printf("Failed to open TDB file %s\n", tdb_file);
3985                 return -1;
3986         }
3987
3988         key = strtodata(tmp_ctx, argv[1], strlen(argv[1]));
3989         if (key.dptr == NULL) {
3990                 printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
3991                 return -1;
3992         }
3993
3994         value = strtodata(tmp_ctx, argv[2], strlen(argv[2]));
3995         if (value.dptr == NULL) {
3996                 printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[2]);
3997                 return -1;
3998         }
3999
4000         ZERO_STRUCT(header);
4001         if (argc > 3) {
4002                 header.rsn = atoll(argv[3]);
4003         }
4004         if (argc > 4) {
4005                 header.dmaster = atoi(argv[4]);
4006         }
4007         if (argc > 5) {
4008                 header.flags = atoi(argv[5]);
4009         }
4010
4011         data.dsize = sizeof(struct ctdb_ltdb_header) + value.dsize;
4012         data.dptr = talloc_size(tmp_ctx, data.dsize);
4013         if (data.dptr == NULL) {
4014                 printf("Failed to allocate header+value\n");
4015                 return -1;
4016         }
4017
4018         *(struct ctdb_ltdb_header *)data.dptr = header;
4019         memcpy(data.dptr + sizeof(struct ctdb_ltdb_header), value.dptr, value.dsize);
4020
4021         if (tdb_store(tdb, key, data, TDB_REPLACE) != 0) {
4022                 printf("Failed to write record %s to tdb %s\n", argv[1], tdb_file);
4023                 tdb_close(tdb);
4024                 return -1;
4025         }
4026
4027         tdb_close(tdb);
4028
4029         talloc_free(tmp_ctx);
4030         return 0;
4031 }
4032
4033 /*
4034   write a record to a persistent database
4035  */
4036 static int control_pstore(struct ctdb_context *ctdb, int argc, const char **argv)
4037 {
4038         const char *db_name;
4039         struct ctdb_db_context *ctdb_db;
4040         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4041         struct ctdb_transaction_handle *h;
4042         struct stat st;
4043         TDB_DATA key, data;
4044         int fd, ret;
4045
4046         if (argc < 3) {
4047                 talloc_free(tmp_ctx);
4048                 usage();
4049         }
4050
4051         fd = open(argv[2], O_RDONLY);
4052         if (fd == -1) {
4053                 DEBUG(DEBUG_ERR,("Failed to open file containing record data : %s  %s\n", argv[2], strerror(errno)));
4054                 talloc_free(tmp_ctx);
4055                 return -1;
4056         }
4057
4058         ret = fstat(fd, &st);
4059         if (ret == -1) {
4060                 DEBUG(DEBUG_ERR,("fstat of file %s failed: %s\n", argv[2], strerror(errno)));
4061                 close(fd);
4062                 talloc_free(tmp_ctx);
4063                 return -1;
4064         }
4065
4066         if (!S_ISREG(st.st_mode)) {
4067                 DEBUG(DEBUG_ERR,("Not a regular file %s\n", argv[2]));
4068                 close(fd);
4069                 talloc_free(tmp_ctx);
4070                 return -1;
4071         }
4072
4073         data.dsize = st.st_size;
4074         if (data.dsize == 0) {
4075                 data.dptr  = NULL;
4076         } else {
4077                 data.dptr = talloc_size(tmp_ctx, data.dsize);
4078                 if (data.dptr == NULL) {
4079                         DEBUG(DEBUG_ERR,("Failed to talloc %d of memory to store record data\n", (int)data.dsize));
4080                         close(fd);
4081                         talloc_free(tmp_ctx);
4082                         return -1;
4083                 }
4084                 ret = sys_read(fd, data.dptr, data.dsize);
4085                 if (ret != data.dsize) {
4086                         DEBUG(DEBUG_ERR,("Failed to read %d bytes of record data\n", (int)data.dsize));
4087                         close(fd);
4088                         talloc_free(tmp_ctx);
4089                         return -1;
4090                 }
4091         }
4092         close(fd);
4093
4094
4095         db_name = argv[0];
4096
4097         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4098         if (ctdb_db == NULL) {
4099                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4100                 talloc_free(tmp_ctx);
4101                 return -1;
4102         }
4103
4104         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4105         if (h == NULL) {
4106                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4107                 talloc_free(tmp_ctx);
4108                 return -1;
4109         }
4110
4111         key = strtodata(tmp_ctx, argv[1], strlen(argv[1]));
4112         if (key.dptr == NULL) {
4113                 printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4114                 return -1;
4115         }
4116
4117         ret = ctdb_transaction_store(h, key, data);
4118         if (ret != 0) {
4119                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4120                 talloc_free(tmp_ctx);
4121                 return -1;
4122         }
4123
4124         ret = ctdb_transaction_commit(h);
4125         if (ret != 0) {
4126                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4127                 talloc_free(tmp_ctx);
4128                 return -1;
4129         }
4130
4131
4132         talloc_free(tmp_ctx);
4133         return 0;
4134 }
4135
4136 /*
4137  * delete a record from a persistent database
4138  */
4139 static int control_pdelete(struct ctdb_context *ctdb, int argc, const char **argv)
4140 {
4141         const char *db_name;
4142         struct ctdb_db_context *ctdb_db;
4143         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4144         struct ctdb_transaction_handle *h;
4145         TDB_DATA key;
4146         int ret;
4147         bool persistent;
4148         uint8_t flags;
4149
4150         if (argc < 2) {
4151                 talloc_free(tmp_ctx);
4152                 usage();
4153         }
4154
4155         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
4156                 talloc_free(tmp_ctx);
4157                 return -1;
4158         }
4159
4160         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
4161         if (!persistent) {
4162                 DEBUG(DEBUG_ERR, ("Database '%s' is not persistent\n", db_name));
4163                 talloc_free(tmp_ctx);
4164                 return -1;
4165         }
4166
4167         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
4168         if (ctdb_db == NULL) {
4169                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n", db_name));
4170                 talloc_free(tmp_ctx);
4171                 return -1;
4172         }
4173
4174         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4175         if (h == NULL) {
4176                 DEBUG(DEBUG_ERR, ("Failed to start transaction on database %s\n", db_name));
4177                 talloc_free(tmp_ctx);
4178                 return -1;
4179         }
4180
4181         key = strtodata(tmp_ctx, argv[1], strlen(argv[1]));
4182         if (key.dptr == NULL) {
4183                 printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4184                 return -1;
4185         }
4186
4187         ret = ctdb_transaction_store(h, key, tdb_null);
4188         if (ret != 0) {
4189                 DEBUG(DEBUG_ERR, ("Failed to delete record\n"));
4190                 talloc_free(tmp_ctx);
4191                 return -1;
4192         }
4193
4194         ret = ctdb_transaction_commit(h);
4195         if (ret != 0) {
4196                 DEBUG(DEBUG_ERR, ("Failed to commit transaction\n"));
4197                 talloc_free(tmp_ctx);
4198                 return -1;
4199         }
4200
4201         talloc_free(tmp_ctx);
4202         return 0;
4203 }
4204
4205 static const char *ptrans_parse_string(TALLOC_CTX *mem_ctx, const char *s,
4206                                        TDB_DATA *data)
4207 {
4208         const char *t;
4209         size_t n;
4210         const char *ret; /* Next byte after successfully parsed value */
4211
4212         /* Error, unless someone says otherwise */
4213         ret = NULL;
4214         /* Indicates no value to parse */
4215         *data = tdb_null;
4216
4217         /* Skip whitespace */
4218         n = strspn(s, " \t");
4219         t = s + n;
4220
4221         if (t[0] == '"') {
4222                 /* Quoted ASCII string - no wide characters! */
4223                 t++;
4224                 n = strcspn(t, "\"");
4225                 if (t[n] == '"') {
4226                         if (n > 0) {
4227                                 *data = strtodata(mem_ctx, t, n);
4228                                 CTDB_NOMEM_ABORT(data->dptr);
4229                         }
4230                         ret = t + n + 1;
4231                 } else {
4232                         DEBUG(DEBUG_WARNING,("Unmatched \" in input %s\n", s));
4233                 }
4234         } else {
4235                 DEBUG(DEBUG_WARNING,("Unsupported input format in %s\n", s));
4236         }
4237
4238         return ret;
4239 }
4240
4241 static bool ptrans_get_key_value(TALLOC_CTX *mem_ctx, FILE *file,
4242                                  TDB_DATA *key, TDB_DATA *value)
4243 {
4244         char line [1024]; /* FIXME: make this more flexible? */
4245         const char *t;
4246         char *ptr;
4247
4248         ptr = fgets(line, sizeof(line), file);
4249
4250         if (ptr == NULL) {
4251                 return false;
4252         }
4253
4254         /* Get key */
4255         t = ptrans_parse_string(mem_ctx, line, key);
4256         if (t == NULL || key->dptr == NULL) {
4257                 /* Line Ignored but not EOF */
4258                 return true;
4259         }
4260
4261         /* Get value */
4262         t = ptrans_parse_string(mem_ctx, t, value);
4263         if (t == NULL) {
4264                 /* Line Ignored but not EOF */
4265                 talloc_free(key->dptr);
4266                 *key = tdb_null;
4267                 return true;
4268         }
4269
4270         return true;
4271 }
4272
4273 /*
4274  * Update a persistent database as per file/stdin
4275  */
4276 static int control_ptrans(struct ctdb_context *ctdb,
4277                           int argc, const char **argv)
4278 {
4279         const char *db_name;
4280         struct ctdb_db_context *ctdb_db;
4281         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4282         struct ctdb_transaction_handle *h;
4283         TDB_DATA key, value;
4284         FILE *file;
4285         int ret;
4286
4287         if (argc < 1) {
4288                 talloc_free(tmp_ctx);
4289                 usage();
4290         }
4291
4292         file = stdin;
4293         if (argc == 2) {
4294                 file = fopen(argv[1], "r");
4295                 if (file == NULL) {
4296                         DEBUG(DEBUG_ERR,("Unable to open file for reading '%s'\n", argv[1]));
4297                         talloc_free(tmp_ctx);
4298                         return -1;
4299                 }
4300         }
4301
4302         db_name = argv[0];
4303
4304         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4305         if (ctdb_db == NULL) {
4306                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4307                 goto error;
4308         }
4309
4310         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4311         if (h == NULL) {
4312                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4313                 goto error;
4314         }
4315
4316         while (ptrans_get_key_value(tmp_ctx, file, &key, &value)) {
4317                 if (key.dsize != 0) {
4318                         ret = ctdb_transaction_store(h, key, value);
4319                         /* Minimise memory use */
4320                         talloc_free(key.dptr);
4321                         if (value.dptr != NULL) {
4322                                 talloc_free(value.dptr);
4323                         }
4324                         if (ret != 0) {
4325                                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4326                                 ctdb_transaction_cancel(h);
4327                                 goto error;
4328                         }
4329                 }
4330         }
4331
4332         ret = ctdb_transaction_commit(h);
4333         if (ret != 0) {
4334                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4335                 goto error;
4336         }
4337
4338         if (file != stdin) {
4339                 fclose(file);
4340         }
4341         talloc_free(tmp_ctx);
4342         return 0;
4343
4344 error:
4345         if (file != stdin) {
4346                 fclose(file);
4347         }
4348
4349         talloc_free(tmp_ctx);
4350         return -1;
4351 }
4352
4353 /*
4354   check if a service is bound to a port or not
4355  */
4356 static int control_chktcpport(struct ctdb_context *ctdb, int argc, const char **argv)
4357 {
4358         int s, ret;
4359         int v;
4360         int port;
4361         struct sockaddr_in sin;
4362
4363         if (argc != 1) {
4364                 printf("Use: ctdb chktcport <port>\n");
4365                 return EINVAL;
4366         }
4367
4368         port = atoi(argv[0]);
4369
4370         s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
4371         if (s == -1) {
4372                 printf("Failed to open local socket\n");
4373                 return errno;
4374         }
4375
4376         v = fcntl(s, F_GETFL, 0);
4377         if (v == -1 || fcntl(s, F_SETFL, v | O_NONBLOCK) != 0) {
4378                 printf("Unable to set socket non-blocking: %s\n", strerror(errno));
4379         }
4380
4381         bzero(&sin, sizeof(sin));
4382         sin.sin_family = PF_INET;
4383         sin.sin_port   = htons(port);
4384         ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
4385         close(s);
4386         if (ret == -1) {
4387                 printf("Failed to bind to local socket: %d %s\n", errno, strerror(errno));
4388                 return errno;
4389         }
4390
4391         return 0;
4392 }
4393
4394
4395 /* Reload public IPs on a specified nodes */
4396 static int control_reloadips(struct ctdb_context *ctdb, int argc, const char **argv)
4397 {
4398         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4399         uint32_t *nodes;
4400         uint32_t pnn_mode;
4401         uint32_t timeout;
4402         int ret;
4403
4404         assert_single_node_only();
4405
4406         if (argc > 1) {
4407                 usage();
4408         }
4409
4410         /* Determine the nodes where IPs need to be reloaded */
4411         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
4412                               options.pnn, true, &nodes, &pnn_mode)) {
4413                 ret = -1;
4414                 goto done;
4415         }
4416
4417 again:
4418         /* Disable takeover runs on all connected nodes.  A reply
4419          * indicating success is needed from each node so all nodes
4420          * will need to be active.  This will retry until maxruntime
4421          * is exceeded, hence no error handling.
4422          *
4423          * A check could be added to not allow reloading of IPs when
4424          * there are disconnected nodes.  However, this should
4425          * probably be left up to the administrator.
4426          */
4427         timeout = LONGTIMEOUT;
4428         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4429                         "Disable takeover runs", true);
4430
4431         /* Now tell all the desired nodes to reload their public IPs.
4432          * Keep trying this until it succeeds.  This assumes all
4433          * failures are transient, which might not be true...
4434          */
4435         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_RELOAD_PUBLIC_IPS,
4436                                       nodes, 0, LONGTIMELIMIT(),
4437                                       false, tdb_null,
4438                                       NULL, NULL, NULL) != 0) {
4439                 DEBUG(DEBUG_ERR,
4440                       ("Unable to reload IPs on some nodes, try again.\n"));
4441                 goto again;
4442         }
4443
4444         /* It isn't strictly necessary to wait until takeover runs are
4445          * re-enabled but doing so can't hurt.
4446          */
4447         timeout = 0;
4448         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4449                         "Enable takeover runs", true);
4450
4451         ipreallocate(ctdb);
4452
4453         ret = 0;
4454 done:
4455         talloc_free(tmp_ctx);
4456         return ret;
4457 }
4458
4459 /*
4460   display a list of the databases on a remote ctdb
4461  */
4462 static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **argv)
4463 {
4464         int i, ret;
4465         struct ctdb_dbid_map_old *dbmap=NULL;
4466
4467         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
4468         if (ret != 0) {
4469                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
4470                 return ret;
4471         }
4472
4473         if(options.machinereadable){
4474                 printm(":ID:Name:Path:Persistent:Sticky:Unhealthy:ReadOnly:\n");
4475                 for(i=0;i<dbmap->num;i++){
4476                         const char *path = NULL;
4477                         const char *name = NULL;
4478                         const char *health = NULL;
4479                         bool persistent;
4480                         bool readonly;
4481                         bool sticky;
4482
4483                         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn,
4484                                             dbmap->dbs[i].db_id, ctdb, &path);
4485                         ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
4486                                             dbmap->dbs[i].db_id, ctdb, &name);
4487                         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
4488                                               dbmap->dbs[i].db_id, ctdb, &health);
4489                         persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4490                         readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4491                         sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4492                         printm(":0x%08X:%s:%s:%d:%d:%d:%d:\n",
4493                                dbmap->dbs[i].db_id, name, path,
4494                                !!(persistent), !!(sticky),
4495                                !!(health), !!(readonly));
4496                 }
4497                 return 0;
4498         }
4499
4500         printf("Number of databases:%d\n", dbmap->num);
4501         for(i=0;i<dbmap->num;i++){
4502                 const char *path = NULL;
4503                 const char *name = NULL;
4504                 const char *health = NULL;
4505                 bool persistent;
4506                 bool readonly;
4507                 bool sticky;
4508
4509                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, ctdb, &path);
4510                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, ctdb, &name);
4511                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, ctdb, &health);
4512                 persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4513                 readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4514                 sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4515                 printf("dbid:0x%08x name:%s path:%s%s%s%s%s\n",
4516                        dbmap->dbs[i].db_id, name, path,
4517                        persistent?" PERSISTENT":"",
4518                        sticky?" STICKY":"",
4519                        readonly?" READONLY":"",
4520                        health?" UNHEALTHY":"");
4521         }
4522
4523         return 0;
4524 }
4525
4526 /*
4527   display the status of a database on a remote ctdb
4528  */
4529 static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char **argv)
4530 {
4531         const char *db_name;
4532         uint32_t db_id;
4533         uint8_t flags;
4534         const char *path = NULL;
4535         const char *health = NULL;
4536
4537         if (argc < 1) {
4538                 usage();
4539         }
4540
4541         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
4542                 return -1;
4543         }
4544
4545         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &path);
4546         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &health);
4547         printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nSTICKY: %s\nREADONLY: %s\nHEALTH: %s\n",
4548                db_id, db_name, path,
4549                (flags & CTDB_DB_FLAGS_PERSISTENT ? "yes" : "no"),
4550                (flags & CTDB_DB_FLAGS_STICKY ? "yes" : "no"),
4551                (flags & CTDB_DB_FLAGS_READONLY ? "yes" : "no"),
4552                (health ? health : "OK"));
4553
4554         return 0;
4555 }
4556
4557 /*
4558   check if the local node is recmaster or not
4559   it will return 1 if this node is the recmaster and 0 if it is not
4560   or if the local ctdb daemon could not be contacted
4561  */
4562 static int control_isnotrecmaster(struct ctdb_context *ctdb, int argc, const char **argv)
4563 {
4564         uint32_t mypnn, recmaster;
4565         int ret;
4566
4567         assert_single_node_only();
4568
4569         mypnn = getpnn(ctdb);
4570
4571         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
4572         if (ret != 0) {
4573                 printf("Failed to get the recmaster\n");
4574                 return 1;
4575         }
4576
4577         if (recmaster != mypnn) {
4578                 printf("this node is not the recmaster\n");
4579                 return 1;
4580         }
4581
4582         printf("this node is the recmaster\n");
4583         return 0;
4584 }
4585
4586 /*
4587   ping a node
4588  */
4589 static int control_ping(struct ctdb_context *ctdb, int argc, const char **argv)
4590 {
4591         int ret;
4592         struct timeval tv = timeval_current();
4593         ret = ctdb_ctrl_ping(ctdb, options.pnn);
4594         if (ret == -1) {
4595                 printf("Unable to get ping response from node %u\n", options.pnn);
4596                 return -1;
4597         } else {
4598                 printf("response from %u time=%.6f sec  (%d clients)\n",
4599                        options.pnn, timeval_elapsed(&tv), ret);
4600         }
4601         return 0;
4602 }
4603
4604
4605 /*
4606   get a node's runstate
4607  */
4608 static int control_runstate(struct ctdb_context *ctdb, int argc, const char **argv)
4609 {
4610         int ret;
4611         enum ctdb_runstate runstate;
4612
4613         ret = ctdb_ctrl_get_runstate(ctdb, TIMELIMIT(), options.pnn, &runstate);
4614         if (ret == -1) {
4615                 printf("Unable to get runstate response from node %u\n",
4616                        options.pnn);
4617                 return -1;
4618         } else {
4619                 bool found = true;
4620                 enum ctdb_runstate t;
4621                 int i;
4622                 for (i=0; i<argc; i++) {
4623                         found = false;
4624                         t = runstate_from_string(argv[i]);
4625                         if (t == CTDB_RUNSTATE_UNKNOWN) {
4626                                 printf("Invalid run state (%s)\n", argv[i]);
4627                                 return -1;
4628                         }
4629
4630                         if (t == runstate) {
4631                                 found = true;
4632                                 break;
4633                         }
4634                 }
4635
4636                 if (!found) {
4637                         printf("CTDB not in required run state (got %s)\n",
4638                                runstate_to_string((enum ctdb_runstate)runstate));
4639                         return -1;
4640                 }
4641         }
4642
4643         printf("%s\n", runstate_to_string(runstate));
4644         return 0;
4645 }
4646
4647
4648 /*
4649   get a tunable
4650  */
4651 static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv)
4652 {
4653         const char *name;
4654         uint32_t value;
4655         int ret;
4656
4657         if (argc < 1) {
4658                 usage();
4659         }
4660
4661         name = argv[0];
4662         ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn, name, &value);
4663         if (ret != 0) {
4664                 DEBUG(DEBUG_ERR, ("Unable to get tunable variable '%s'\n", name));
4665                 return -1;
4666         }
4667
4668         printf("%-23s = %u\n", name, value);
4669         return 0;
4670 }
4671
4672 /*
4673   set a tunable
4674  */
4675 static int control_setvar(struct ctdb_context *ctdb, int argc, const char **argv)
4676 {
4677         const char *name;
4678         uint32_t value;
4679         int ret;
4680
4681         if (argc < 2) {
4682                 usage();
4683         }
4684
4685         name = argv[0];
4686         value = strtoul(argv[1], NULL, 0);
4687
4688         ret = ctdb_ctrl_set_tunable(ctdb, TIMELIMIT(), options.pnn, name, value);
4689         if (ret == -1) {
4690                 DEBUG(DEBUG_ERR, ("Unable to set tunable variable '%s'\n", name));
4691                 return -1;
4692         }
4693         if (ret == 1) {
4694                 DEBUG(DEBUG_WARNING,
4695                       ("Setting obsolete tunable variable '%s'\n",
4696                        name));
4697         }
4698         return 0;
4699 }
4700
4701 /*
4702   list all tunables
4703  */
4704 static int control_listvars(struct ctdb_context *ctdb, int argc, const char **argv)
4705 {
4706         uint32_t count;
4707         const char **list;
4708         int ret, i;
4709
4710         ret = ctdb_ctrl_list_tunables(ctdb, TIMELIMIT(), options.pnn, ctdb, &list, &count);
4711         if (ret == -1) {
4712                 DEBUG(DEBUG_ERR, ("Unable to list tunable variables\n"));
4713                 return -1;
4714         }
4715
4716         for (i=0;i<count;i++) {
4717                 control_getvar(ctdb, 1, &list[i]);
4718         }
4719
4720         talloc_free(list);
4721
4722         return 0;
4723 }
4724
4725 /*
4726   display debug level on a node
4727  */
4728 static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **argv)
4729 {
4730         int ret;
4731         int32_t level;
4732
4733         ret = ctdb_ctrl_get_debuglevel(ctdb, options.pnn, &level);
4734         if (ret != 0) {
4735                 DEBUG(DEBUG_ERR, ("Unable to get debuglevel response from node %u\n", options.pnn));
4736                 return ret;
4737         } else {
4738                 enum debug_level log_level = debug_level_from_int(level);
4739                 const char *desc = debug_level_to_string(log_level);
4740                 if (desc == NULL) {
4741                         /* This should never happen */
4742                         desc = "Unknown";
4743                 }
4744                 if (options.machinereadable){
4745                         printm(":Name:Level:\n");
4746                         printm(":%s:%d:\n", desc, level);
4747                 } else {
4748                         printf("Node %u is at debug level %s (%d)\n",
4749                                options.pnn, desc, level);
4750                 }
4751         }
4752         return 0;
4753 }
4754
4755 /*
4756   display reclock file of a node
4757  */
4758 static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4759 {
4760         int ret;
4761         const char *reclock;
4762
4763         ret = ctdb_ctrl_getreclock(ctdb, TIMELIMIT(), options.pnn, ctdb, &reclock);
4764         if (ret != 0) {
4765                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4766                 return ret;
4767         } else {
4768                 if (options.machinereadable){
4769                         if (reclock != NULL) {
4770                                 printm("%s", reclock);
4771                         }
4772                 } else {
4773                         if (reclock == NULL) {
4774                                 printf("No reclock file used.\n");
4775                         } else {
4776                                 printf("Reclock file:%s\n", reclock);
4777                         }
4778                 }
4779         }
4780         return 0;
4781 }
4782
4783 /*
4784   set the reclock file of a node
4785  */
4786 static int control_setreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4787 {
4788         int ret;
4789         const char *reclock = NULL;
4790
4791         if (argc == 0) {
4792                 reclock = NULL;
4793         } else if (argc == 1) {
4794                 reclock = argv[0];
4795         } else {
4796                 usage();
4797         }
4798
4799         ret = ctdb_ctrl_setreclock(ctdb, TIMELIMIT(), options.pnn, reclock);
4800         if (ret != 0) {
4801                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4802                 return ret;
4803         }
4804         return 0;
4805 }
4806
4807 /*
4808   set the natgw state on/off
4809  */
4810 static int control_setnatgwstate(struct ctdb_context *ctdb, int argc, const char **argv)
4811 {
4812         int ret;
4813         uint32_t natgwstate = 0;
4814
4815         if (argc == 0) {
4816                 usage();
4817         }
4818
4819         if (!strcmp(argv[0], "on")) {
4820                 natgwstate = 1;
4821         } else if (!strcmp(argv[0], "off")) {
4822                 natgwstate = 0;
4823         } else {
4824                 usage();
4825         }
4826
4827         ret = ctdb_ctrl_setnatgwstate(ctdb, TIMELIMIT(), options.pnn, natgwstate);
4828         if (ret != 0) {
4829                 DEBUG(DEBUG_ERR, ("Unable to set the natgw state for node %u\n", options.pnn));
4830                 return ret;
4831         }
4832
4833         return 0;
4834 }
4835
4836 /*
4837   set the lmaster role on/off
4838  */
4839 static int control_setlmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
4840 {
4841         int ret;
4842         uint32_t lmasterrole = 0;
4843
4844         if (argc == 0) {
4845                 usage();
4846         }
4847
4848         if (!strcmp(argv[0], "on")) {
4849                 lmasterrole = 1;
4850         } else if (!strcmp(argv[0], "off")) {
4851                 lmasterrole = 0;
4852         } else {
4853                 usage();
4854         }
4855
4856         ret = ctdb_ctrl_setlmasterrole(ctdb, TIMELIMIT(), options.pnn, lmasterrole);
4857         if (ret != 0) {
4858                 DEBUG(DEBUG_ERR, ("Unable to set the lmaster role for node %u\n", options.pnn));
4859                 return ret;
4860         }
4861
4862         return 0;
4863 }
4864
4865 /*
4866   set the recmaster role on/off
4867  */
4868 static int control_setrecmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
4869 {
4870         int ret;
4871         uint32_t recmasterrole = 0;
4872
4873         if (argc == 0) {
4874                 usage();
4875         }
4876
4877         if (!strcmp(argv[0], "on")) {
4878                 recmasterrole = 1;
4879         } else if (!strcmp(argv[0], "off")) {
4880                 recmasterrole = 0;
4881         } else {
4882                 usage();
4883         }
4884
4885         ret = ctdb_ctrl_setrecmasterrole(ctdb, TIMELIMIT(), options.pnn, recmasterrole);
4886         if (ret != 0) {
4887                 DEBUG(DEBUG_ERR, ("Unable to set the recmaster role for node %u\n", options.pnn));
4888                 return ret;
4889         }
4890
4891         return 0;
4892 }
4893
4894 /*
4895   set debug level on a node or all nodes
4896  */
4897 static int control_setdebug(struct ctdb_context *ctdb, int argc, const char **argv)
4898 {
4899         int ret;
4900         int32_t level;
4901         enum debug_level log_level;
4902
4903         if (argc == 0) {
4904                 printf("You must specify the debug level. Valid levels are:\n");
4905                 printf("\tERROR | WARNING | NOTICE | INFO | DEBUG\n");
4906                 return 0;
4907         }
4908
4909         if (!debug_level_parse(argv[0], &log_level)) {
4910                 printf("Invalid debug level, must be one of\n");
4911                 printf("\tERROR | WARNING | NOTICE | INFO | DEBUG\n");
4912                 return -1;
4913         }
4914
4915         level = debug_level_to_int(log_level);
4916
4917         ret = ctdb_ctrl_set_debuglevel(ctdb, options.pnn, level);
4918         if (ret != 0) {
4919                 DEBUG(DEBUG_ERR, ("Unable to set debug level on node %u\n", options.pnn));
4920         }
4921         return 0;
4922 }
4923
4924
4925 /*
4926   thaw a node
4927  */
4928 static int control_thaw(struct ctdb_context *ctdb, int argc, const char **argv)
4929 {
4930         int ret;
4931         uint32_t priority;
4932
4933         if (argc == 1) {
4934                 priority = strtol(argv[0], NULL, 0);
4935         } else {
4936                 priority = 0;
4937         }
4938         DEBUG(DEBUG_ERR,("Thaw by priority %u\n", priority));
4939
4940         ret = ctdb_ctrl_thaw_priority(ctdb, TIMELIMIT(), options.pnn, priority);
4941         if (ret != 0) {
4942                 DEBUG(DEBUG_ERR, ("Unable to thaw node %u\n", options.pnn));
4943         }
4944         return 0;
4945 }
4946
4947
4948 /*
4949   attach to a database
4950  */
4951 static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv)
4952 {
4953         const char *db_name;
4954         struct ctdb_db_context *ctdb_db;
4955         bool persistent = false;
4956
4957         if (argc < 1) {
4958                 usage();
4959         }
4960         db_name = argv[0];
4961         if (argc > 2) {
4962                 usage();
4963         }
4964         if (argc == 2) {
4965                 if (strcmp(argv[1], "persistent") != 0) {
4966                         usage();
4967                 }
4968                 persistent = true;
4969         }
4970
4971         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
4972         if (ctdb_db == NULL) {
4973                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4974                 return -1;
4975         }
4976
4977         return 0;
4978 }
4979
4980 /*
4981  * detach from a database
4982  */
4983 static int control_detach(struct ctdb_context *ctdb, int argc,
4984                           const char **argv)
4985 {
4986         uint32_t db_id;
4987         uint8_t flags;
4988         int ret, i, status = 0;
4989         struct ctdb_node_map_old *nodemap = NULL;
4990         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4991         uint32_t recmode;
4992
4993         if (argc < 1) {
4994                 usage();
4995         }
4996
4997         assert_single_node_only();
4998
4999         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn,
5000                                     &recmode);
5001         if (ret != 0) {
5002                 DEBUG(DEBUG_ERR, ("Database cannot be detached "
5003                                   "when recovery is active\n"));
5004                 talloc_free(tmp_ctx);
5005                 return -1;
5006         }
5007
5008         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5009                                    &nodemap);
5010         if (ret != 0) {
5011                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5012                                   options.pnn));
5013                 talloc_free(tmp_ctx);
5014                 return -1;
5015         }
5016
5017         for (i=0; i<nodemap->num; i++) {
5018                 uint32_t value;
5019
5020                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
5021                         continue;
5022                 }
5023
5024                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
5025                         continue;
5026                 }
5027
5028                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
5029                         DEBUG(DEBUG_ERR, ("Database cannot be detached on "
5030                                           "inactive (stopped or banned) node "
5031                                           "%u\n", nodemap->nodes[i].pnn));
5032                         talloc_free(tmp_ctx);
5033                         return -1;
5034                 }
5035
5036                 ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(),
5037                                             nodemap->nodes[i].pnn,
5038                                             "AllowClientDBAttach",
5039                                             &value);
5040                 if (ret != 0) {
5041                         DEBUG(DEBUG_ERR, ("Unable to get tunable "
5042                                           "AllowClientDBAttach from node %u\n",
5043                                            nodemap->nodes[i].pnn));
5044                         talloc_free(tmp_ctx);
5045                         return -1;
5046                 }
5047
5048                 if (value == 1) {
5049                         DEBUG(DEBUG_ERR, ("Database access is still active on "
5050                                           "node %u. Set AllowClientDBAttach=0 "
5051                                           "on all nodes.\n",
5052                                           nodemap->nodes[i].pnn));
5053                         talloc_free(tmp_ctx);
5054                         return -1;
5055                 }
5056         }
5057
5058         talloc_free(tmp_ctx);
5059
5060         for (i=0; i<argc; i++) {
5061                 if (!db_exists(ctdb, argv[i], &db_id, NULL, &flags)) {
5062                         continue;
5063                 }
5064
5065                 if (flags & CTDB_DB_FLAGS_PERSISTENT) {
5066                         DEBUG(DEBUG_ERR, ("Persistent database '%s' "
5067                                           "cannot be detached\n", argv[i]));
5068                         status = -1;
5069                         continue;
5070                 }
5071
5072                 ret = ctdb_detach(ctdb, db_id);
5073                 if (ret != 0) {
5074                         DEBUG(DEBUG_ERR, ("Database '%s' detach failed\n",
5075                                           argv[i]));
5076                         status = ret;
5077                 }
5078         }
5079
5080         return status;
5081 }
5082
5083 /*
5084   set db priority
5085  */
5086 static int control_setdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5087 {
5088         struct ctdb_db_priority db_prio;
5089         int ret;
5090
5091         if (argc < 2) {
5092                 usage();
5093         }
5094
5095         db_prio.db_id    = strtoul(argv[0], NULL, 0);
5096         db_prio.priority = strtoul(argv[1], NULL, 0);
5097
5098         ret = ctdb_ctrl_set_db_priority(ctdb, TIMELIMIT(), options.pnn, &db_prio);
5099         if (ret != 0) {
5100                 DEBUG(DEBUG_ERR,("Unable to set db prio\n"));
5101                 return -1;
5102         }
5103
5104         return 0;
5105 }
5106
5107 /*
5108   get db priority
5109  */
5110 static int control_getdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5111 {
5112         uint32_t db_id, priority;
5113         int ret;
5114
5115         if (argc < 1) {
5116                 usage();
5117         }
5118
5119         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5120                 return -1;
5121         }
5122
5123         ret = ctdb_ctrl_get_db_priority(ctdb, TIMELIMIT(), options.pnn, db_id, &priority);
5124         if (ret != 0) {
5125                 DEBUG(DEBUG_ERR,("Unable to get db prio\n"));
5126                 return -1;
5127         }
5128
5129         DEBUG(DEBUG_ERR,("Priority:%u\n", priority));
5130
5131         return 0;
5132 }
5133
5134 /*
5135   set the sticky records capability for a database
5136  */
5137 static int control_setdbsticky(struct ctdb_context *ctdb, int argc, const char **argv)
5138 {
5139         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5140         uint32_t db_id;
5141         int ret;
5142
5143         if (argc < 1) {
5144                 usage();
5145         }
5146
5147         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5148                 return -1;
5149         }
5150
5151         ret = ctdb_ctrl_set_db_sticky(ctdb, options.pnn, db_id);
5152         if (ret != 0) {
5153                 DEBUG(DEBUG_ERR,("Unable to set db to support sticky records\n"));
5154                 talloc_free(tmp_ctx);
5155                 return -1;
5156         }
5157
5158         talloc_free(tmp_ctx);
5159         return 0;
5160 }
5161
5162 /*
5163   set the readonly capability for a database
5164  */
5165 static int control_setdbreadonly(struct ctdb_context *ctdb, int argc, const char **argv)
5166 {
5167         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5168         uint32_t db_id;
5169         int ret;
5170
5171         if (argc < 1) {
5172                 usage();
5173         }
5174
5175         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5176                 return -1;
5177         }
5178
5179         ret = ctdb_ctrl_set_db_readonly(ctdb, options.pnn, db_id);
5180         if (ret != 0) {
5181                 DEBUG(DEBUG_ERR,("Unable to set db to support readonly\n"));
5182                 talloc_free(tmp_ctx);
5183                 return -1;
5184         }
5185
5186         talloc_free(tmp_ctx);
5187         return 0;
5188 }
5189
5190 /*
5191   get db seqnum
5192  */
5193 static int control_getdbseqnum(struct ctdb_context *ctdb, int argc, const char **argv)
5194 {
5195         uint32_t db_id;
5196         uint64_t seqnum;
5197         int ret;
5198
5199         if (argc < 1) {
5200                 usage();
5201         }
5202
5203         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5204                 return -1;
5205         }
5206
5207         ret = ctdb_ctrl_getdbseqnum(ctdb, TIMELIMIT(), options.pnn, db_id, &seqnum);
5208         if (ret != 0) {
5209                 DEBUG(DEBUG_ERR, ("Unable to get seqnum from node."));
5210                 return -1;
5211         }
5212
5213         printf("Sequence number:%lld\n", (long long)seqnum);
5214
5215         return 0;
5216 }
5217
5218 /*
5219   run an eventscript on a node
5220  */
5221 static int control_eventscript(struct ctdb_context *ctdb, int argc, const char **argv)
5222 {
5223         TDB_DATA data;
5224         int ret;
5225         int32_t res;
5226         char *errmsg;
5227         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5228
5229         if (argc != 1) {
5230                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5231                 return -1;
5232         }
5233
5234         data.dptr = (unsigned char *)discard_const(argv[0]);
5235         data.dsize = strlen((char *)data.dptr) + 1;
5236
5237         DEBUG(DEBUG_ERR, ("Running eventscripts with arguments \"%s\" on node %u\n", data.dptr, options.pnn));
5238
5239         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_RUN_EVENTSCRIPTS,
5240                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
5241         if (ret != 0 || res != 0) {
5242                 if (errmsg != NULL) {
5243                         DEBUG(DEBUG_ERR,
5244                               ("Failed to run eventscripts - %s\n", errmsg));
5245                 } else {
5246                         DEBUG(DEBUG_ERR, ("Failed to run eventscripts\n"));
5247                 }
5248                 talloc_free(tmp_ctx);
5249                 return -1;
5250         }
5251         talloc_free(tmp_ctx);
5252         return 0;
5253 }
5254
5255 #define DB_VERSION 1
5256 #define MAX_DB_NAME 64
5257 struct db_file_header {
5258         unsigned long version;
5259         time_t timestamp;
5260         unsigned long persistent;
5261         unsigned long size;
5262         const char name[MAX_DB_NAME];
5263 };
5264
5265 struct backup_data {
5266         struct ctdb_marshall_buffer *records;
5267         uint32_t len;
5268         uint32_t total;
5269         bool traverse_error;
5270 };
5271
5272 static int backup_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private)
5273 {
5274         struct backup_data *bd = talloc_get_type(private, struct backup_data);
5275         struct ctdb_rec_data_old *rec;
5276
5277         /* add the record */
5278         rec = ctdb_marshall_record(bd->records, 0, key, NULL, data);
5279         if (rec == NULL) {
5280                 bd->traverse_error = true;
5281                 DEBUG(DEBUG_ERR,("Failed to marshall record\n"));
5282                 return -1;
5283         }
5284         bd->records = talloc_realloc_size(NULL, bd->records, rec->length + bd->len);
5285         if (bd->records == NULL) {
5286                 DEBUG(DEBUG_ERR,("Failed to expand marshalling buffer\n"));
5287                 bd->traverse_error = true;
5288                 return -1;
5289         }
5290         bd->records->count++;
5291         memcpy(bd->len+(uint8_t *)bd->records, rec, rec->length);
5292         bd->len += rec->length;
5293         talloc_free(rec);
5294
5295         bd->total++;
5296         return 0;
5297 }
5298
5299 /*
5300  * backup a database to a file
5301  */
5302 static int control_backupdb(struct ctdb_context *ctdb, int argc, const char **argv)
5303 {
5304         const char *db_name;
5305         int ret;
5306         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5307         struct db_file_header dbhdr;
5308         struct ctdb_db_context *ctdb_db;
5309         struct backup_data *bd;
5310         int fh = -1;
5311         int status = -1;
5312         const char *reason = NULL;
5313         uint32_t db_id;
5314         uint8_t flags;
5315
5316         assert_single_node_only();
5317
5318         if (argc != 2) {
5319                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5320                 return -1;
5321         }
5322
5323         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
5324                 return -1;
5325         }
5326
5327         ret = ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
5328                                     db_id, tmp_ctx, &reason);
5329         if (ret != 0) {
5330                 DEBUG(DEBUG_ERR,("Unable to get dbhealth for database '%s'\n",
5331                                  argv[0]));
5332                 talloc_free(tmp_ctx);
5333                 return -1;
5334         }
5335         if (reason) {
5336                 uint32_t allow_unhealthy = 0;
5337
5338                 ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn,
5339                                       "AllowUnhealthyDBRead",
5340                                       &allow_unhealthy);
5341
5342                 if (allow_unhealthy != 1) {
5343                         DEBUG(DEBUG_ERR,("database '%s' is unhealthy: %s\n",
5344                                          argv[0], reason));
5345
5346                         DEBUG(DEBUG_ERR,("disallow backup : tunable AllowUnhealthyDBRead = %u\n",
5347                                          allow_unhealthy));
5348                         talloc_free(tmp_ctx);
5349                         return -1;
5350                 }
5351
5352                 DEBUG(DEBUG_WARNING,("WARNING database '%s' is unhealthy - see 'ctdb getdbstatus %s'\n",
5353                                      argv[0], argv[0]));
5354                 DEBUG(DEBUG_WARNING,("WARNING! allow backup of unhealthy database: "
5355                                      "tunnable AllowUnhealthyDBRead = %u\n",
5356                                      allow_unhealthy));
5357         }
5358
5359         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5360         if (ctdb_db == NULL) {
5361                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", argv[0]));
5362                 talloc_free(tmp_ctx);
5363                 return -1;
5364         }
5365
5366
5367         ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
5368         if (ret == -1) {
5369                 DEBUG(DEBUG_ERR,("Failed to start transaction\n"));
5370                 talloc_free(tmp_ctx);
5371                 return -1;
5372         }
5373
5374
5375         bd = talloc_zero(tmp_ctx, struct backup_data);
5376         if (bd == NULL) {
5377                 DEBUG(DEBUG_ERR,("Failed to allocate backup_data\n"));
5378                 talloc_free(tmp_ctx);
5379                 return -1;
5380         }
5381
5382         bd->records = talloc_zero(bd, struct ctdb_marshall_buffer);
5383         if (bd->records == NULL) {
5384                 DEBUG(DEBUG_ERR,("Failed to allocate ctdb_marshall_buffer\n"));
5385                 talloc_free(tmp_ctx);
5386                 return -1;
5387         }
5388
5389         bd->len = offsetof(struct ctdb_marshall_buffer, data);
5390         bd->records->db_id = ctdb_db->db_id;
5391         /* traverse the database collecting all records */
5392         if (tdb_traverse_read(ctdb_db->ltdb->tdb, backup_traverse, bd) == -1 ||
5393             bd->traverse_error) {
5394                 DEBUG(DEBUG_ERR,("Traverse error\n"));
5395                 talloc_free(tmp_ctx);
5396                 return -1;
5397         }
5398
5399         tdb_transaction_cancel(ctdb_db->ltdb->tdb);
5400
5401
5402         fh = open(argv[1], O_RDWR|O_CREAT, 0600);
5403         if (fh == -1) {
5404                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[1]));
5405                 talloc_free(tmp_ctx);
5406                 return -1;
5407         }
5408
5409         ZERO_STRUCT(dbhdr);
5410         dbhdr.version = DB_VERSION;
5411         dbhdr.timestamp = time(NULL);
5412         dbhdr.persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
5413         dbhdr.size = bd->len;
5414         if (strlen(argv[0]) >= MAX_DB_NAME) {
5415                 DEBUG(DEBUG_ERR,("Too long dbname\n"));
5416                 goto done;
5417         }
5418         strncpy(discard_const(dbhdr.name), argv[0], MAX_DB_NAME-1);
5419         ret = sys_write(fh, &dbhdr, sizeof(dbhdr));
5420         if (ret == -1) {
5421                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5422                 goto done;
5423         }
5424         ret = sys_write(fh, bd->records, bd->len);
5425         if (ret == -1) {
5426                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5427                 goto done;
5428         }
5429
5430         status = 0;
5431 done:
5432         if (fh != -1) {
5433                 ret = close(fh);
5434                 if (ret == -1) {
5435                         DEBUG(DEBUG_ERR,("close failed: %s\n", strerror(errno)));
5436                 }
5437         }
5438
5439         DEBUG(DEBUG_ERR,("Database backed up to %s\n", argv[1]));
5440
5441         talloc_free(tmp_ctx);
5442         return status;
5443 }
5444
5445 /*
5446  * restore a database from a file
5447  */
5448 static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **argv)
5449 {
5450         int ret;
5451         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5452         TDB_DATA outdata;
5453         TDB_DATA data;
5454         struct db_file_header dbhdr;
5455         struct ctdb_db_context *ctdb_db;
5456         struct ctdb_node_map_old *nodemap=NULL;
5457         struct ctdb_vnn_map *vnnmap=NULL;
5458         int i, fh;
5459         struct ctdb_transdb w;
5460         uint32_t *nodes;
5461         uint32_t generation;
5462         struct tm *tm;
5463         char tbuf[100];
5464         char *dbname;
5465
5466         assert_single_node_only();
5467
5468         if (argc < 1 || argc > 2) {
5469                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5470                 return -1;
5471         }
5472
5473         fh = open(argv[0], O_RDONLY);
5474         if (fh == -1) {
5475                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5476                 talloc_free(tmp_ctx);
5477                 return -1;
5478         }
5479
5480         sys_read(fh, &dbhdr, sizeof(dbhdr));
5481         if (dbhdr.version != DB_VERSION) {
5482                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5483                 close(fh);
5484                 talloc_free(tmp_ctx);
5485                 return -1;
5486         }
5487
5488         dbname = discard_const(dbhdr.name);
5489         if (argc == 2) {
5490                 dbname = discard_const(argv[1]);
5491         }
5492
5493         outdata.dsize = dbhdr.size;
5494         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5495         if (outdata.dptr == NULL) {
5496                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5497                 close(fh);
5498                 talloc_free(tmp_ctx);
5499                 return -1;
5500         }
5501         sys_read(fh, outdata.dptr, outdata.dsize);
5502         close(fh);
5503
5504         tm = localtime(&dbhdr.timestamp);
5505         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5506         printf("Restoring database '%s' from backup @ %s\n",
5507                 dbname, tbuf);
5508
5509
5510         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), dbname, dbhdr.persistent, 0);
5511         if (ctdb_db == NULL) {
5512                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", dbname));
5513                 talloc_free(tmp_ctx);
5514                 return -1;
5515         }
5516
5517         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
5518         if (ret != 0) {
5519                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
5520                 talloc_free(tmp_ctx);
5521                 return ret;
5522         }
5523
5524
5525         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
5526         if (ret != 0) {
5527                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
5528                 talloc_free(tmp_ctx);
5529                 return ret;
5530         }
5531
5532         /* freeze all nodes */
5533         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5534         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5535                 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5536                                         nodes, i,
5537                                         TIMELIMIT(),
5538                                         false, tdb_null,
5539                                         NULL, NULL,
5540                                         NULL) != 0) {
5541                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5542                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5543                         talloc_free(tmp_ctx);
5544                         return -1;
5545                 }
5546         }
5547
5548         generation = vnnmap->generation;
5549         data.dptr = (void *)&generation;
5550         data.dsize = sizeof(generation);
5551
5552         /* start a cluster wide transaction */
5553         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5554         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
5555                                         nodes, 0,
5556                                         TIMELIMIT(), false, data,
5557                                         NULL, NULL,
5558                                         NULL) != 0) {
5559                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide transactions.\n"));
5560                 return -1;
5561         }
5562
5563
5564         w.db_id = ctdb_db->db_id;
5565         w.tid = generation;
5566
5567         data.dptr = (void *)&w;
5568         data.dsize = sizeof(w);
5569
5570         /* wipe all the remote databases. */
5571         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5572         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
5573                                         nodes, 0,
5574                                         TIMELIMIT(), false, data,
5575                                         NULL, NULL,
5576                                         NULL) != 0) {
5577                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
5578                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5579                 talloc_free(tmp_ctx);
5580                 return -1;
5581         }
5582
5583         /* push the database */
5584         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5585         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
5586                                         nodes, 0,
5587                                         TIMELIMIT(), false, outdata,
5588                                         NULL, NULL,
5589                                         NULL) != 0) {
5590                 DEBUG(DEBUG_ERR, ("Failed to push database.\n"));
5591                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5592                 talloc_free(tmp_ctx);
5593                 return -1;
5594         }
5595
5596         data.dptr = (void *)&ctdb_db->db_id;
5597         data.dsize = sizeof(ctdb_db->db_id);
5598
5599         /* mark the database as healthy */
5600         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5601         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
5602                                         nodes, 0,
5603                                         TIMELIMIT(), false, data,
5604                                         NULL, NULL,
5605                                         NULL) != 0) {
5606                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
5607                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5608                 talloc_free(tmp_ctx);
5609                 return -1;
5610         }
5611
5612         data.dptr = (void *)&generation;
5613         data.dsize = sizeof(generation);
5614
5615         /* commit all the changes */
5616         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
5617                                         nodes, 0,
5618                                         TIMELIMIT(), false, data,
5619                                         NULL, NULL,
5620                                         NULL) != 0) {
5621                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
5622                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5623                 talloc_free(tmp_ctx);
5624                 return -1;
5625         }
5626
5627
5628         /* thaw all nodes */
5629         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5630         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
5631                                         nodes, 0,
5632                                         TIMELIMIT(),
5633                                         false, tdb_null,
5634                                         NULL, NULL,
5635                                         NULL) != 0) {
5636                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
5637                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5638                 talloc_free(tmp_ctx);
5639                 return -1;
5640         }
5641
5642
5643         talloc_free(tmp_ctx);
5644         return 0;
5645 }
5646
5647 /*
5648  * dump a database backup from a file
5649  */
5650 static int control_dumpdbbackup(struct ctdb_context *ctdb, int argc, const char **argv)
5651 {
5652         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5653         TDB_DATA outdata;
5654         struct db_file_header dbhdr;
5655         int i, fh;
5656         struct tm *tm;
5657         char tbuf[100];
5658         struct ctdb_rec_data_old *rec = NULL;
5659         struct ctdb_marshall_buffer *m;
5660         struct ctdb_dump_db_context c;
5661
5662         assert_single_node_only();
5663
5664         if (argc != 1) {
5665                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5666                 return -1;
5667         }
5668
5669         fh = open(argv[0], O_RDONLY);
5670         if (fh == -1) {
5671                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5672                 talloc_free(tmp_ctx);
5673                 return -1;
5674         }
5675
5676         sys_read(fh, &dbhdr, sizeof(dbhdr));
5677         if (dbhdr.version != DB_VERSION) {
5678                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5679                 close(fh);
5680                 talloc_free(tmp_ctx);
5681                 return -1;
5682         }
5683
5684         outdata.dsize = dbhdr.size;
5685         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5686         if (outdata.dptr == NULL) {
5687                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5688                 close(fh);
5689                 talloc_free(tmp_ctx);
5690                 return -1;
5691         }
5692         sys_read(fh, outdata.dptr, outdata.dsize);
5693         close(fh);
5694         m = (struct ctdb_marshall_buffer *)outdata.dptr;
5695
5696         tm = localtime(&dbhdr.timestamp);
5697         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5698         printf("Backup of database name:'%s' dbid:0x%x08x from @ %s\n",
5699                 dbhdr.name, m->db_id, tbuf);
5700
5701         ZERO_STRUCT(c);
5702         c.ctdb = ctdb;
5703         c.f = stdout;
5704         c.printemptyrecords = (bool)options.printemptyrecords;
5705         c.printdatasize = (bool)options.printdatasize;
5706         c.printlmaster = false;
5707         c.printhash = (bool)options.printhash;
5708         c.printrecordflags = (bool)options.printrecordflags;
5709
5710         for (i=0; i < m->count; i++) {
5711                 uint32_t reqid = 0;
5712                 TDB_DATA key, data;
5713
5714                 /* we do not want the header splitted, so we pass NULL*/
5715                 rec = ctdb_marshall_loop_next(m, rec, &reqid,
5716                                               NULL, &key, &data);
5717
5718                 ctdb_dumpdb_record(key, data, &c);
5719         }
5720
5721         printf("Dumped %d records\n", i);
5722         talloc_free(tmp_ctx);
5723         return 0;
5724 }
5725
5726 /*
5727  * wipe a database from a file
5728  */
5729 static int control_wipedb(struct ctdb_context *ctdb, int argc,
5730                           const char **argv)
5731 {
5732         const char *db_name;
5733         int ret;
5734         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5735         TDB_DATA data;
5736         struct ctdb_db_context *ctdb_db;
5737         struct ctdb_node_map_old *nodemap = NULL;
5738         struct ctdb_vnn_map *vnnmap = NULL;
5739         int i;
5740         struct ctdb_transdb w;
5741         uint32_t *nodes;
5742         uint32_t generation;
5743         uint8_t flags;
5744
5745         assert_single_node_only();
5746
5747         if (argc != 1) {
5748                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5749                 return -1;
5750         }
5751
5752         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
5753                 return -1;
5754         }
5755
5756         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5757         if (ctdb_db == NULL) {
5758                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n",
5759                                   argv[0]));
5760                 talloc_free(tmp_ctx);
5761                 return -1;
5762         }
5763
5764         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb,
5765                                    &nodemap);
5766         if (ret != 0) {
5767                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5768                                   options.pnn));
5769                 talloc_free(tmp_ctx);
5770                 return ret;
5771         }
5772
5773         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5774                                   &vnnmap);
5775         if (ret != 0) {
5776                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
5777                                   options.pnn));
5778                 talloc_free(tmp_ctx);
5779                 return ret;
5780         }
5781
5782         /* freeze all nodes */
5783         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5784         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5785                 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5786                                                 nodes, i,
5787                                                 TIMELIMIT(),
5788                                                 false, tdb_null,
5789                                                 NULL, NULL,
5790                                                 NULL);
5791                 if (ret != 0) {
5792                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5793                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn,
5794                                              CTDB_RECOVERY_ACTIVE);
5795                         talloc_free(tmp_ctx);
5796                         return -1;
5797                 }
5798         }
5799
5800         generation = vnnmap->generation;
5801         data.dptr = (void *)&generation;
5802         data.dsize = sizeof(generation);
5803
5804         /* start a cluster wide transaction */
5805         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5806         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
5807                                         nodes, 0,
5808                                         TIMELIMIT(), false, data,
5809                                         NULL, NULL,
5810                                         NULL);
5811         if (ret!= 0) {
5812                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide "
5813                                   "transactions.\n"));
5814                 return -1;
5815         }
5816
5817         w.db_id = ctdb_db->db_id;
5818         w.tid = generation;
5819
5820         data.dptr = (void *)&w;
5821         data.dsize = sizeof(w);
5822
5823         /* wipe all the remote databases. */
5824         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5825         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
5826                                         nodes, 0,
5827                                         TIMELIMIT(), false, data,
5828                                         NULL, NULL,
5829                                         NULL) != 0) {
5830                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
5831                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5832                 talloc_free(tmp_ctx);
5833                 return -1;
5834         }
5835
5836         data.dptr = (void *)&ctdb_db->db_id;
5837         data.dsize = sizeof(ctdb_db->db_id);
5838
5839         /* mark the database as healthy */
5840         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5841         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
5842                                         nodes, 0,
5843                                         TIMELIMIT(), false, data,
5844                                         NULL, NULL,
5845                                         NULL) != 0) {
5846                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
5847                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5848                 talloc_free(tmp_ctx);
5849                 return -1;
5850         }
5851
5852         data.dptr = (void *)&generation;
5853         data.dsize = sizeof(generation);
5854
5855         /* commit all the changes */
5856         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
5857                                         nodes, 0,
5858                                         TIMELIMIT(), false, data,
5859                                         NULL, NULL,
5860                                         NULL) != 0) {
5861                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
5862                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5863                 talloc_free(tmp_ctx);
5864                 return -1;
5865         }
5866
5867         /* thaw all nodes */
5868         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5869         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
5870                                         nodes, 0,
5871                                         TIMELIMIT(),
5872                                         false, tdb_null,
5873                                         NULL, NULL,
5874                                         NULL) != 0) {
5875                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
5876                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5877                 talloc_free(tmp_ctx);
5878                 return -1;
5879         }
5880
5881         DEBUG(DEBUG_ERR, ("Database wiped.\n"));
5882
5883         talloc_free(tmp_ctx);
5884         return 0;
5885 }
5886
5887 /*
5888   dump memory usage
5889  */
5890 static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
5891 {
5892         TDB_DATA data;
5893         int ret;
5894         int32_t res;
5895         char *errmsg;
5896         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5897         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_DUMP_MEMORY,
5898                            0, tdb_null, tmp_ctx, &data, &res, NULL, &errmsg);
5899         if (ret != 0 || res != 0) {
5900                 DEBUG(DEBUG_ERR,("Failed to dump memory - %s\n", errmsg));
5901                 talloc_free(tmp_ctx);
5902                 return -1;
5903         }
5904         sys_write(1, data.dptr, data.dsize);
5905         talloc_free(tmp_ctx);
5906         return 0;
5907 }
5908
5909 /*
5910   handler for memory dumps
5911 */
5912 static void mem_dump_handler(uint64_t srvid, TDB_DATA data, void *private_data)
5913 {
5914         sys_write(1, data.dptr, data.dsize);
5915         exit(0);
5916 }
5917
5918 /*
5919   dump memory usage on the recovery daemon
5920  */
5921 static int control_rddumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
5922 {
5923         int ret;
5924         TDB_DATA data;
5925         struct ctdb_srvid_message rd;
5926
5927         rd.pnn = ctdb_get_pnn(ctdb);
5928         rd.srvid = getpid();
5929
5930         /* register a message port for receiveing the reply so that we
5931            can receive the reply
5932         */
5933         ctdb_client_set_message_handler(ctdb, rd.srvid, mem_dump_handler, NULL);
5934
5935
5936         data.dptr = (uint8_t *)&rd;
5937         data.dsize = sizeof(rd);
5938
5939         ret = ctdb_client_send_message(ctdb, options.pnn, CTDB_SRVID_MEM_DUMP, data);
5940         if (ret != 0) {
5941                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
5942                 return -1;
5943         }
5944
5945         /* this loop will terminate when we have received the reply */
5946         while (1) {
5947                 tevent_loop_once(ctdb->ev);
5948         }
5949
5950         return 0;
5951 }
5952
5953 /*
5954   send a message to a srvid
5955  */
5956 static int control_msgsend(struct ctdb_context *ctdb, int argc, const char **argv)
5957 {
5958         unsigned long srvid;
5959         int ret;
5960         TDB_DATA data;
5961
5962         if (argc < 2) {
5963                 usage();
5964         }
5965
5966         srvid      = strtoul(argv[0], NULL, 0);
5967
5968         data.dptr = (uint8_t *)discard_const(argv[1]);
5969         data.dsize= strlen(argv[1]);
5970
5971         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, srvid, data);
5972         if (ret != 0) {
5973                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
5974                 return -1;
5975         }
5976
5977         return 0;
5978 }
5979
5980 /*
5981   handler for msglisten
5982 */
5983 static void msglisten_handler(uint64_t srvid, TDB_DATA data,
5984                               void *private_data)
5985 {
5986         int i;
5987
5988         printf("Message received: ");
5989         for (i=0;i<data.dsize;i++) {
5990                 printf("%c", data.dptr[i]);
5991         }
5992         printf("\n");
5993 }
5994
5995 /*
5996   listen for messages on a messageport
5997  */
5998 static int control_msglisten(struct ctdb_context *ctdb, int argc, const char **argv)
5999 {
6000         uint64_t srvid;
6001
6002         srvid = getpid();
6003
6004         /* register a message port and listen for messages
6005         */
6006         ctdb_client_set_message_handler(ctdb, srvid, msglisten_handler, NULL);
6007         printf("Listening for messages on srvid:%d\n", (int)srvid);
6008
6009         while (1) {
6010                 tevent_loop_once(ctdb->ev);
6011         }
6012
6013         return 0;
6014 }
6015
6016 /*
6017   list all nodes in the cluster
6018   we parse the nodes file directly
6019  */
6020 static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
6021 {
6022         TALLOC_CTX *mem_ctx = talloc_new(NULL);
6023         struct ctdb_node_map_old *node_map;
6024         int i;
6025
6026         assert_single_node_only();
6027
6028         node_map = read_nodes_file(mem_ctx);
6029         if (node_map == NULL) {
6030                 talloc_free(mem_ctx);
6031                 return -1;
6032         }
6033
6034         for (i = 0; i < node_map->num; i++) {
6035                 const char *addr;
6036
6037                 if (node_map->nodes[i].flags & NODE_FLAGS_DELETED) {
6038                         continue;
6039                 }
6040                 addr = ctdb_addr_to_str(&node_map->nodes[i].addr);
6041                 if (options.machinereadable){
6042                         printm(":%d:%s:\n", node_map->nodes[i].pnn, addr);
6043                 } else {
6044                         printf("%s\n", addr);
6045                 }
6046         }
6047         talloc_free(mem_ctx);
6048
6049         return 0;
6050 }
6051
6052 /**********************************************************************/
6053 /* reload the nodes file on all nodes */
6054
6055 static void get_nodes_files_callback(struct ctdb_context *ctdb,
6056                                      uint32_t node_pnn, int32_t res,
6057                                      TDB_DATA outdata, void *callback_data)
6058 {
6059         struct ctdb_node_map_old **maps =
6060                 talloc_get_type(callback_data, struct ctdb_node_map_old *);
6061
6062         if (outdata.dsize < offsetof(struct ctdb_node_map_old, nodes) ||
6063             outdata.dptr == NULL) {
6064                 DEBUG(DEBUG_ERR,
6065                       (__location__ " Invalid return data: %u %p\n",
6066                        (unsigned)outdata.dsize, outdata.dptr));
6067                 return;
6068         }
6069
6070         if (node_pnn >= talloc_array_length(maps)) {
6071                 DEBUG(DEBUG_ERR,
6072                       (__location__ " unexpected PNN %u\n", node_pnn));
6073                 return;
6074         }
6075
6076         maps[node_pnn] = talloc_memdup(maps, outdata.dptr, outdata.dsize);
6077 }
6078
6079 static void get_nodes_files_fail_callback(struct ctdb_context *ctdb,
6080                                           uint32_t node_pnn, int32_t res,
6081                                           TDB_DATA outdata, void *callback_data)
6082 {
6083         DEBUG(DEBUG_ERR,
6084               ("ERROR: Failed to get nodes file from node %u\n", node_pnn));
6085 }
6086
6087 static struct ctdb_node_map_old **
6088 ctdb_get_nodes_files(struct ctdb_context *ctdb,
6089                      TALLOC_CTX *mem_ctx,
6090                      struct timeval timeout,
6091                      struct ctdb_node_map_old *nodemap)
6092 {
6093         uint32_t *nodes;
6094         int ret;
6095         struct ctdb_node_map_old **maps;
6096
6097         maps = talloc_zero_array(mem_ctx, struct ctdb_node_map_old *, nodemap->num);
6098         CTDB_NO_MEMORY_NULL(ctdb, maps);
6099
6100         nodes = list_of_connected_nodes(ctdb, nodemap, mem_ctx, true);
6101
6102         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_NODES_FILE,
6103                                         nodes, 0, TIMELIMIT(),
6104                                         true, tdb_null,
6105                                         get_nodes_files_callback,
6106                                         get_nodes_files_fail_callback,
6107                                         maps);
6108         if (ret != 0) {
6109                 talloc_free(maps);
6110                 return NULL;
6111         }
6112
6113         return maps;
6114 }
6115
6116 static bool node_files_are_identical(struct ctdb_node_map_old *nm1,
6117                                      struct ctdb_node_map_old *nm2)
6118 {
6119         int i;
6120
6121         if (nm1->num != nm2->num) {
6122                 return false;
6123         }
6124         for (i = 0; i < nm1->num; i++) {
6125                 if (memcmp(&nm1->nodes[i], &nm2->nodes[i],
6126                            sizeof(struct ctdb_node_and_flags)) != 0) {
6127                         return false;
6128                 }
6129         }
6130
6131         return true;
6132 }
6133
6134 static bool check_all_node_files_are_identical(struct ctdb_context *ctdb,
6135                                                TALLOC_CTX *mem_ctx,
6136                                                struct timeval timeout,
6137                                                struct ctdb_node_map_old *nodemap,
6138                                                struct ctdb_node_map_old *file_nodemap)
6139 {
6140         static struct ctdb_node_map_old **maps;
6141         int i;
6142         bool ret = true;
6143
6144         maps = ctdb_get_nodes_files(ctdb, mem_ctx, timeout, nodemap);
6145         if (maps == NULL) {
6146                 return false;
6147         }
6148
6149         for (i = 0; i < talloc_array_length(maps); i++) {
6150                 if (maps[i] == NULL) {
6151                         continue;
6152                 }
6153                 if (!node_files_are_identical(file_nodemap, maps[i])) {
6154                         DEBUG(DEBUG_ERR,
6155                               ("ERROR: Node file on node %u differs from current node (%u)\n",
6156                                i, ctdb_get_pnn(ctdb)));
6157                         ret = false;
6158                 }
6159         }
6160
6161         return ret;
6162 }
6163
6164 /*
6165   reload the nodes file on the local node
6166  */
6167 static bool sanity_check_nodes_file_changes(TALLOC_CTX *mem_ctx,
6168                                             struct ctdb_node_map_old *nodemap,
6169                                             struct ctdb_node_map_old *file_nodemap)
6170 {
6171         int i;
6172         bool should_abort = false;
6173         bool have_changes = false;
6174
6175         for (i=0; i<nodemap->num; i++) {
6176                 if (i >= file_nodemap->num) {
6177                         DEBUG(DEBUG_ERR,
6178                               ("ERROR: Node %u (%s) missing from nodes file\n",
6179                                nodemap->nodes[i].pnn,
6180                                ctdb_addr_to_str(&nodemap->nodes[i].addr)));
6181                         should_abort = true;
6182                         continue;
6183                 }
6184                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED &&
6185                     file_nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
6186                         /* Node remains deleted */
6187                         DEBUG(DEBUG_INFO,
6188                               ("Node %u is unchanged (DELETED)\n",
6189                                nodemap->nodes[i].pnn));
6190                 } else if (!(nodemap->nodes[i].flags & NODE_FLAGS_DELETED) &&
6191                            !(file_nodemap->nodes[i].flags & NODE_FLAGS_DELETED)) {
6192                         /* Node not newly nor previously deleted */
6193                         if (!ctdb_same_ip(&nodemap->nodes[i].addr,
6194                                           &file_nodemap->nodes[i].addr)) {
6195                                 DEBUG(DEBUG_ERR,
6196                                       ("ERROR: Node %u has changed IP address (was %s, now %s)\n",
6197                                        nodemap->nodes[i].pnn,
6198                                        /* ctdb_addr_to_str() returns a static */
6199                                        talloc_strdup(mem_ctx,
6200                                                      ctdb_addr_to_str(&nodemap->nodes[i].addr)),
6201                                        ctdb_addr_to_str(&file_nodemap->nodes[i].addr)));
6202                                 should_abort = true;
6203                         } else {
6204                                 DEBUG(DEBUG_INFO,
6205                                       ("Node %u is unchanged\n",
6206                                        nodemap->nodes[i].pnn));
6207                                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
6208                                         DEBUG(DEBUG_WARNING,
6209                                               ("WARNING: Node %u is disconnected. You MUST fix this node manually!\n",
6210                                                nodemap->nodes[i].pnn));
6211                                 }
6212                         }
6213                 } else if (file_nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
6214                         /* Node is being deleted */
6215                         DEBUG(DEBUG_NOTICE,
6216                               ("Node %u is DELETED\n",
6217                                nodemap->nodes[i].pnn));
6218                         have_changes = true;
6219                         if (!(nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED)) {
6220                                 DEBUG(DEBUG_ERR,
6221                                       ("ERROR: Node %u is still connected\n",
6222                                        nodemap->nodes[i].pnn));
6223                                 should_abort = true;
6224                         }
6225                 } else if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
6226                         /* Node was previously deleted */
6227                         DEBUG(DEBUG_NOTICE,
6228                               ("Node %u is UNDELETED\n", nodemap->nodes[i].pnn));
6229                         have_changes = true;
6230                 }
6231         }
6232
6233         if (should_abort) {
6234                 DEBUG(DEBUG_ERR,
6235                       ("ERROR: Nodes will not be reloaded due to previous error\n"));
6236                 talloc_free(mem_ctx);
6237                 exit(1);
6238         }
6239
6240         /* Leftover nodes in file are NEW */
6241         for (; i < file_nodemap->num; i++) {
6242                 DEBUG(DEBUG_NOTICE, ("Node %u is NEW\n",
6243                                      file_nodemap->nodes[i].pnn));
6244                 have_changes = true;
6245         }
6246
6247         return have_changes;
6248 }
6249
6250 static void reload_nodes_fail_callback(struct ctdb_context *ctdb,
6251                                        uint32_t node_pnn, int32_t res,
6252                                        TDB_DATA outdata, void *callback_data)
6253 {
6254         DEBUG(DEBUG_WARNING,
6255               ("WARNING: Node %u failed to reload nodes. You MUST fix this node manually!\n",
6256                node_pnn));
6257 }
6258
6259 static int control_reload_nodes_file(struct ctdb_context *ctdb, int argc, const char **argv)
6260 {
6261         int i, ret;
6262         struct ctdb_node_map_old *nodemap=NULL;
6263         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
6264         struct ctdb_node_map_old *file_nodemap;
6265         uint32_t *conn;
6266         uint32_t timeout;
6267
6268         assert_current_node_only(ctdb);
6269
6270         /* Load both the current nodemap and the contents of the local
6271          * nodes file.  Compare and sanity check them before doing
6272          * anything. */
6273
6274         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
6275         if (ret != 0) {
6276                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
6277                 return ret;
6278         }
6279
6280         file_nodemap = read_nodes_file(tmp_ctx);
6281         if (file_nodemap == NULL) {
6282                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
6283                 talloc_free(tmp_ctx);
6284                 return -1;
6285         }
6286
6287         if (!check_all_node_files_are_identical(ctdb, tmp_ctx, TIMELIMIT(),
6288                                                 nodemap, file_nodemap)) {
6289                 return -1;
6290         }
6291
6292         if (!sanity_check_nodes_file_changes(tmp_ctx, nodemap, file_nodemap)) {
6293                 DEBUG(DEBUG_NOTICE,
6294                       ("No change in nodes file, skipping unnecessary reload\n"));
6295                 talloc_free(tmp_ctx);
6296                 return 0;
6297         }
6298
6299         /* Now make the changes */
6300         conn = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
6301         for (i = 0; i < talloc_array_length(conn); i++) {
6302                 DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n",
6303                                      conn[i]));
6304         }
6305
6306         /* Another timeout could be used, such as ReRecoveryTimeout or
6307          * a new one for this purpose.  However, this is the simplest
6308          * option. */
6309         timeout = options.timelimit;
6310         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_RECOVERIES, &timeout,
6311                         "Disable recoveries", true);
6312
6313
6314         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELOAD_NODES_FILE,
6315                                         conn, 0, TIMELIMIT(),
6316                                         true, tdb_null,
6317                                         NULL, reload_nodes_fail_callback,
6318                                         NULL);
6319
6320         timeout = 0;
6321         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_RECOVERIES, &timeout,
6322                         "Enable recoveries", true);
6323
6324         talloc_free(tmp_ctx);
6325
6326         return 0;
6327 }
6328
6329
6330 static const struct {
6331         const char *name;
6332         int (*fn)(struct ctdb_context *, int, const char **);
6333         bool auto_all;
6334         bool without_daemon; /* can be run without daemon running ? */
6335         const char *msg;
6336         const char *args;
6337 } ctdb_commands[] = {
6338         { "version",         control_version,           true,   true,   "show version of ctdb" },
6339         { "status",          control_status,            true,   false,  "show node status" },
6340         { "uptime",          control_uptime,            true,   false,  "show node uptime" },
6341         { "ping",            control_ping,              true,   false,  "ping all nodes" },
6342         { "runstate",        control_runstate,          true,   false,  "get/check runstate of a node", "[setup|first_recovery|startup|running]" },
6343         { "getvar",          control_getvar,            true,   false,  "get a tunable variable",               "<name>"},
6344         { "setvar",          control_setvar,            true,   false,  "set a tunable variable",               "<name> <value>"},
6345         { "listvars",        control_listvars,          true,   false,  "list tunable variables"},
6346         { "statistics",      control_statistics,        false,  false, "show statistics" },
6347         { "statisticsreset", control_statistics_reset,  true,   false,  "reset statistics"},
6348         { "stats",           control_stats,             false,  false,  "show rolling statistics", "[number of history records]" },
6349         { "ip",              control_ip,                false,  false,  "show which public ip's that ctdb manages" },
6350         { "ipinfo",          control_ipinfo,            true,   false,  "show details about a public ip that ctdb manages", "<ip>" },
6351         { "ifaces",          control_ifaces,            true,   false,  "show which interfaces that ctdb manages" },
6352         { "setifacelink",    control_setifacelink,      true,   false,  "set interface link status", "<iface> <status>" },
6353         { "process-exists",  control_process_exists,    true,   false,  "check if a process exists on a node",  "<pid>"},
6354         { "getdbmap",        control_getdbmap,          true,   false,  "show the database map" },
6355         { "getdbstatus",     control_getdbstatus,       true,   false,  "show the status of a database", "<dbname|dbid>" },
6356         { "catdb",           control_catdb,             true,   false,  "dump a ctdb database" ,                     "<dbname|dbid>"},
6357         { "cattdb",          control_cattdb,            true,   false,  "dump a local tdb database" ,                     "<dbname|dbid>"},
6358         { "getmonmode",      control_getmonmode,        true,   false,  "show monitoring mode" },
6359         { "getcapabilities", control_getcapabilities,   true,   false,  "show node capabilities" },
6360         { "pnn",             control_pnn,               true,   false,  "show the pnn of the currnet node" },
6361         { "lvs",             control_lvs,               true,   false,  "show lvs configuration" },
6362         { "lvsmaster",       control_lvsmaster,         true,   false,  "show which node is the lvs master" },
6363         { "disablemonitor",      control_disable_monmode,true,  false,  "set monitoring mode to DISABLE" },
6364         { "enablemonitor",      control_enable_monmode, true,   false,  "set monitoring mode to ACTIVE" },
6365         { "setdebug",        control_setdebug,          true,   false,  "set debug level",                      "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
6366         { "getdebug",        control_getdebug,          true,   false,  "get debug level" },
6367         { "attach",          control_attach,            true,   false,  "attach to a database",                 "<dbname> [persistent]" },
6368         { "detach",          control_detach,            false,  false,  "detach from a database",                 "<dbname|dbid> [<dbname|dbid> ...]" },
6369         { "dumpmemory",      control_dumpmemory,        true,   false,  "dump memory map to stdout" },
6370         { "rddumpmemory",    control_rddumpmemory,      true,   false,  "dump memory map from the recovery daemon to stdout" },
6371         { "getpid",          control_getpid,            true,   false,  "get ctdbd process ID" },
6372         { "disable",         control_disable,           true,   false,  "disable a nodes public IP" },
6373         { "enable",          control_enable,            true,   false,  "enable a nodes public IP" },
6374         { "stop",            control_stop,              true,   false,  "stop a node" },
6375         { "continue",        control_continue,          true,   false,  "re-start a stopped node" },
6376         { "ban",             control_ban,               true,   false,  "ban a node from the cluster",          "<bantime>"},
6377         { "unban",           control_unban,             true,   false,  "unban a node" },
6378         { "showban",         control_showban,           true,   false,  "show ban information"},
6379         { "shutdown",        control_shutdown,          true,   false,  "shutdown ctdbd" },
6380         { "recover",         control_recover,           true,   false,  "force recovery" },
6381         { "sync",            control_ipreallocate,      false,  false,  "wait until ctdbd has synced all state changes" },
6382         { "ipreallocate",    control_ipreallocate,      false,  false,  "force the recovery daemon to perform a ip reallocation procedure" },
6383         { "thaw",            control_thaw,              true,   false,  "thaw databases", "[priority:1-3]" },
6384         { "isnotrecmaster",  control_isnotrecmaster,    false,  false,  "check if the local node is recmaster or not" },
6385         { "killtcp",         kill_tcp,                  false,  false, "kill a tcp connection.", "[<srcip:port> <dstip:port>]" },
6386         { "gratiousarp",     control_gratious_arp,      false,  false, "send a gratious arp", "<ip> <interface>" },
6387         { "tickle",          tickle_tcp,                false,  false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
6388         { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip> [<port>]" },
6389         { "addtickle",       control_add_tickle,        false,  false, "add a tickle for this ip", "<ip>:<port> <ip>:<port>" },
6390
6391         { "deltickle",       control_del_tickle,        false,  false, "delete a tickle from this ip", "<ip>:<port> <ip>:<port>" },
6392
6393         { "regsrvid",        regsrvid,                  false,  false, "register a server id", "<pnn> <type> <id>" },
6394         { "unregsrvid",      unregsrvid,                false,  false, "unregister a server id", "<pnn> <type> <id>" },
6395         { "chksrvid",        chksrvid,                  false,  false, "check if a server id exists", "<pnn> <type> <id>" },
6396         { "getsrvids",       getsrvids,                 false,  false, "get a list of all server ids"},
6397         { "check_srvids",    check_srvids,              false,  false, "check if a srvid exists", "<id>+" },
6398         { "listnodes",       control_listnodes,         false,  true, "list all nodes in the cluster"},
6399         { "reloadnodes",     control_reload_nodes_file, false,  false, "reload the nodes file and restart the transport on all nodes"},
6400         { "moveip",          control_moveip,            false,  false, "move/failover an ip address to another node", "<ip> <node>"},
6401         { "rebalanceip",     control_rebalanceip,       false,  false, "release an ip from the node and let recd rebalance it", "<ip>"},
6402         { "addip",           control_addip,             true,   false, "add a ip address to a node", "<ip/mask> <iface>"},
6403         { "delip",           control_delip,             false,  false, "delete an ip address from a node", "<ip>"},
6404         { "eventscript",     control_eventscript,       true,   false, "run the eventscript with the given parameters on a node", "<arguments>"},
6405         { "backupdb",        control_backupdb,          false,  false, "backup the database into a file.", "<dbname|dbid> <file>"},
6406         { "restoredb",        control_restoredb,        false,  false, "restore the database from a file.", "<file> [dbname]"},
6407         { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
6408         { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname|dbid>"},
6409         { "recmaster",        control_recmaster,        true,   false, "show the pnn for the recovery master."},
6410         { "scriptstatus",     control_scriptstatus,     true,   false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
6411         { "enablescript",     control_enablescript,  true,      false, "enable an eventscript", "<script>"},
6412         { "disablescript",    control_disablescript,  true,     false, "disable an eventscript", "<script>"},
6413         { "natgwlist",        control_natgwlist,        false,  true, "show the nodes belonging to this natgw configuration"},
6414         { "xpnn",             control_xpnn,             false,  true,  "find the pnn of the local node without talking to the daemon (unreliable)" },
6415         { "getreclock",       control_getreclock,       true,   false, "Show the reclock file of a node"},
6416         { "setreclock",       control_setreclock,       true,   false, "Set/clear the reclock file of a node", "[filename]"},
6417         { "setnatgwstate",    control_setnatgwstate,    false,  false, "Set NATGW state to on/off", "{on|off}"},
6418         { "setlmasterrole",   control_setlmasterrole,   false,  false, "Set LMASTER role to on/off", "{on|off}"},
6419         { "setrecmasterrole", control_setrecmasterrole, false,  false, "Set RECMASTER role to on/off", "{on|off}"},
6420         { "setdbprio",        control_setdbprio,        false,  false, "Set DB priority", "<dbname|dbid> <prio:1-3>"},
6421         { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbname|dbid>"},
6422         { "setdbreadonly",    control_setdbreadonly,    false,  false, "Set DB readonly capable", "<dbname|dbid>"},
6423         { "setdbsticky",      control_setdbsticky,      false,  false, "Set DB sticky-records capable", "<dbname|dbid>"},
6424         { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
6425         { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
6426         { "pfetch",          control_pfetch,            false,  false,  "fetch a record from a persistent database", "<dbname|dbid> <key> [<file>]" },
6427         { "pstore",          control_pstore,            false,  false,  "write a record to a persistent database", "<dbname|dbid> <key> <file containing record>" },
6428         { "pdelete",         control_pdelete,           false,  false,  "delete a record from a persistent database", "<dbname|dbid> <key>" },
6429         { "ptrans",          control_ptrans,            false,  false,  "update a persistent database (from stdin)", "<dbname|dbid>" },
6430         { "tfetch",          control_tfetch,            false,  true,  "fetch a record from a [c]tdb-file [-v]", "<tdb-file> <key> [<file>]" },
6431         { "tstore",          control_tstore,            false,  true,  "store a record (including ltdb header)", "<tdb-file> <key> <data> [<rsn> <dmaster> <flags>]" },
6432         { "readkey",         control_readkey,           true,   false,  "read the content off a database key", "<dbname|dbid> <key>" },
6433         { "writekey",        control_writekey,          true,   false,  "write to a database key", "<dbname|dbid> <key> <value>" },
6434         { "checktcpport",    control_chktcpport,        false,  true,  "check if a service is bound to a specific tcp port or not", "<port>" },
6435         { "rebalancenode",     control_rebalancenode,   false,  false, "mark nodes as forced IP rebalancing targets", "[<pnn-list>]"},
6436         { "getdbseqnum",     control_getdbseqnum,       false,  false, "get the sequence number off a database", "<dbname|dbid>" },
6437         { "nodestatus",      control_nodestatus,        true,   false,  "show and return node status", "[<pnn-list>]" },
6438         { "dbstatistics",    control_dbstatistics,      false,  false, "show db statistics", "<dbname|dbid>" },
6439         { "reloadips",       control_reloadips,         false,  false, "reload the public addresses file on specified nodes" , "[<pnn-list>]" },
6440         { "ipiface",         control_ipiface,           false,  true,  "Find which interface an ip address is hosted on", "<ip>" },
6441 };
6442
6443 /*
6444   show usage message
6445  */
6446 static void usage(void)
6447 {
6448         int i;
6449         printf(
6450 "Usage: ctdb [options] <control>\n" \
6451 "Options:\n" \
6452 "   -n <node>          choose node number, or 'all' (defaults to local node)\n"
6453 "   -Y                 generate machine readable output\n"
6454 "   -x <char>          specify delimiter for machine readable output\n"
6455 "   -v                 generate verbose output\n"
6456 "   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
6457         printf("Controls:\n");
6458         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6459                 printf("  %-15s %-27s  %s\n",
6460                        ctdb_commands[i].name,
6461                        ctdb_commands[i].args?ctdb_commands[i].args:"",
6462                        ctdb_commands[i].msg);
6463         }
6464         exit(1);
6465 }
6466
6467
6468 static void ctdb_alarm(int sig)
6469 {
6470         printf("Maximum runtime exceeded - exiting\n");
6471         _exit(ERR_TIMEOUT);
6472 }
6473
6474 /*
6475   main program
6476 */
6477 int main(int argc, const char *argv[])
6478 {
6479         struct ctdb_context *ctdb;
6480         char *nodestring = NULL;
6481         int machineparsable = 0;
6482         struct poptOption popt_options[] = {
6483                 POPT_AUTOHELP
6484                 POPT_CTDB_CMDLINE
6485                 { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
6486                 { "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
6487                 { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machine readable output", NULL },
6488                 { NULL, 'x', POPT_ARG_STRING, &options.machineseparator, 0, "specify separator for machine readable output", "char" },
6489                 { NULL, 'X', POPT_ARG_NONE, &machineparsable, 0, "enable machine parsable output with separator |", NULL },
6490                 { "verbose",    'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL },
6491                 { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" },
6492                 { "print-emptyrecords", 0, POPT_ARG_NONE, &options.printemptyrecords, 0, "print the empty records when dumping databases (catdb, cattdb, dumpdbbackup)", NULL },
6493                 { "print-datasize", 0, POPT_ARG_NONE, &options.printdatasize, 0, "do not print record data when dumping databases, only the data size", NULL },
6494                 { "print-lmaster", 0, POPT_ARG_NONE, &options.printlmaster, 0, "print the record's lmaster in catdb", NULL },
6495                 { "print-hash", 0, POPT_ARG_NONE, &options.printhash, 0, "print the record's hash when dumping databases", NULL },
6496                 { "print-recordflags", 0, POPT_ARG_NONE, &options.printrecordflags, 0, "print the record flags in catdb and dumpdbbackup", NULL },
6497                 POPT_TABLEEND
6498         };
6499         int opt;
6500         const char **extra_argv;
6501         int extra_argc = 0;
6502         int ret=-1, i;
6503         poptContext pc;
6504         struct tevent_context *ev;
6505         const char *control;
6506
6507         setlinebuf(stdout);
6508
6509         /* set some defaults */
6510         options.maxruntime = 0;
6511         options.timelimit = 10;
6512         options.pnn = CTDB_CURRENT_NODE;
6513
6514         pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
6515
6516         while ((opt = poptGetNextOpt(pc)) != -1) {
6517                 switch (opt) {
6518                 default:
6519                         DEBUG(DEBUG_ERR, ("Invalid option %s: %s\n",
6520                                 poptBadOption(pc, 0), poptStrerror(opt)));
6521                         exit(1);
6522                 }
6523         }
6524
6525         /* setup the remaining options for the main program to use */
6526         extra_argv = poptGetArgs(pc);
6527         if (extra_argv) {
6528                 extra_argv++;
6529                 while (extra_argv[extra_argc]) extra_argc++;
6530         }
6531
6532         if (extra_argc < 1) {
6533                 usage();
6534         }
6535
6536         if (options.maxruntime == 0) {
6537                 const char *ctdb_timeout;
6538                 ctdb_timeout = getenv("CTDB_TIMEOUT");
6539                 if (ctdb_timeout != NULL) {
6540                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
6541                 } else {
6542                         /* default timeout is 120 seconds */
6543                         options.maxruntime = 120;
6544                 }
6545         }
6546
6547         if (machineparsable) {
6548                 options.machineseparator = "|";
6549         }
6550         if (options.machineseparator != NULL) {
6551                 if (strlen(options.machineseparator) != 1) {
6552                         printf("Invalid separator \"%s\" - "
6553                                "must be single character\n",
6554                                options.machineseparator);
6555                         exit(1);
6556                 }
6557
6558                 /* -x implies -Y */
6559                 options.machinereadable = true;
6560         } else if (options.machinereadable) {
6561                 options.machineseparator = ":";
6562         }
6563
6564         signal(SIGALRM, ctdb_alarm);
6565         alarm(options.maxruntime);
6566
6567         control = extra_argv[0];
6568
6569         /* Default value for CTDB_BASE - don't override */
6570         setenv("CTDB_BASE", CTDB_ETCDIR, 0);
6571
6572         ev = tevent_context_init(NULL);
6573         if (!ev) {
6574                 DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
6575                 exit(1);
6576         }
6577
6578         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6579                 if (strcmp(control, ctdb_commands[i].name) == 0) {
6580                         break;
6581                 }
6582         }
6583
6584         if (i == ARRAY_SIZE(ctdb_commands)) {
6585                 DEBUG(DEBUG_ERR, ("Unknown control '%s'\n", control));
6586                 exit(1);
6587         }
6588
6589         if (ctdb_commands[i].without_daemon == true) {
6590                 if (nodestring != NULL) {
6591                         DEBUG(DEBUG_ERR, ("Can't specify node(s) with \"ctdb %s\"\n", control));
6592                         exit(1);
6593                 }
6594                 return ctdb_commands[i].fn(NULL, extra_argc-1, extra_argv+1);
6595         }
6596
6597         /* initialise ctdb */
6598         ctdb = ctdb_cmdline_client(ev, TIMELIMIT());
6599
6600         if (ctdb == NULL) {
6601                 uint32_t pnn;
6602                 DEBUG(DEBUG_ERR, ("Failed to init ctdb\n"));
6603
6604                 pnn = find_node_xpnn();
6605                 if (pnn == -1) {
6606                         DEBUG(DEBUG_ERR,
6607                               ("Is this node part of a CTDB cluster?\n"));
6608                 }
6609                 exit(1);
6610         }
6611
6612         /* setup the node number(s) to contact */
6613         if (!parse_nodestring(ctdb, ctdb, nodestring, CTDB_CURRENT_NODE, false,
6614                               &options.nodes, &options.pnn)) {
6615                 usage();
6616         }
6617
6618         if (options.pnn == CTDB_CURRENT_NODE) {
6619                 options.pnn = options.nodes[0];
6620         }
6621
6622         if (ctdb_commands[i].auto_all &&
6623             ((options.pnn == CTDB_BROADCAST_ALL) ||
6624              (options.pnn == CTDB_MULTICAST))) {
6625                 int j;
6626
6627                 ret = 0;
6628                 for (j = 0; j < talloc_array_length(options.nodes); j++) {
6629                         options.pnn = options.nodes[j];
6630                         ret |= ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6631                 }
6632         } else {
6633                 ret = ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6634         }
6635
6636         talloc_free(ctdb);
6637         talloc_free(ev);
6638         (void)poptFreeContext(pc);
6639
6640         return ret;
6641
6642 }