ctdb/tools/ctdb.c

   1 /*
   2    ctdb control tool
   3
   4    Copyright (C) Andrew Tridgell  2007
   5    Copyright (C) Ronnie Sahlberg  2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "includes.h"
  22 #include "system/time.h"
  23 #include "system/filesys.h"
  24 #include "system/network.h"
  25 #include "system/locale.h"
  26 #include "popt.h"
  27 #include "cmdline.h"
  28 #include "../include/ctdb_version.h"
  29 #include "../include/ctdb_client.h"
  30 #include "../include/ctdb_private.h"
  31 #include "../common/rb_tree.h"
  32 #include "lib/tdb_wrap/tdb_wrap.h"
  33 #include "lib/util/dlinklist.h"
  34
  35 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
  36 #define ERR_NONODE      21      /* node does not exist */
  37 #define ERR_DISNODE     22      /* node is disconnected */
  38
  39 static void usage(void);
  40
  41 static struct {
  42         int timelimit;
  43         uint32_t pnn;
  44         uint32_t *nodes;
  45         int machinereadable;
  46         const char *machineseparator;
  47         int verbose;
  48         int maxruntime;
  49         int printemptyrecords;
  50         int printdatasize;
  51         int printlmaster;
  52         int printhash;
  53         int printrecordflags;
  54 } options;
  55
  56 #define LONGTIMEOUT options.timelimit*10
  57
  58 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
  59 #define LONGTIMELIMIT() timeval_current_ofs(LONGTIMEOUT, 0)
  60
  61 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
  62 {
  63         return (tv2->tv_sec - tv->tv_sec) +
  64                 (tv2->tv_usec - tv->tv_usec)*1.0e-6;
  65 }
  66
  67 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
  68 {
  69         printf("CTDB version: %s\n", CTDB_VERSION_STRING);
  70         return 0;
  71 }
  72
  73 /* Like printf(3) but substitute for separator in format */
  74 static int printm(const char *format, ...) PRINTF_ATTRIBUTE(1,2);
  75 static int printm(const char *format, ...)
  76 {
  77         va_list ap;
  78         int ret;
  79         size_t len = strlen(format);
  80         char new_format[len+1];
  81
  82         strcpy(new_format, format);
  83
  84         if (options.machineseparator[0] != ':') {
  85                 all_string_sub(new_format,
  86                                ":", options.machineseparator, len + 1);
  87         }
  88
  89         va_start(ap, format);
  90         ret = vprintf(new_format, ap);
  91         va_end(ap);
  92
  93         return ret;
  94 }
  95
  96 #define CTDB_NOMEM_ABORT(p) do { if (!(p)) {                            \
  97                 DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n",            \
  98                                    "Out of memory in " __location__ )); \
  99                 abort();                                                \
 100         }} while (0)
 101
 102 static uint32_t getpnn(struct ctdb_context *ctdb)
 103 {
 104         if ((options.pnn == CTDB_BROADCAST_ALL) ||
 105             (options.pnn == CTDB_MULTICAST)) {
 106                 DEBUG(DEBUG_ERR,
 107                       ("Cannot get PNN for node %u\n", options.pnn));
 108                 exit(1);
 109         }
 110
 111         if (options.pnn == CTDB_CURRENT_NODE) {
 112                 return ctdb_get_pnn(ctdb);
 113         } else {
 114                 return options.pnn;
 115         }
 116 }
 117
 118 static void assert_single_node_only(void)
 119 {
 120         if ((options.pnn == CTDB_BROADCAST_ALL) ||
 121             (options.pnn == CTDB_MULTICAST)) {
 122                 DEBUG(DEBUG_ERR,
 123                       ("This control can not be applied to multiple PNNs\n"));
 124                 exit(1);
 125         }
 126 }
 127
 128 static void assert_current_node_only(struct ctdb_context *ctdb)
 129 {
 130         if (options.pnn != ctdb_get_pnn(ctdb)) {
 131                 DEBUG(DEBUG_ERR,
 132                       ("This control can only be applied to the current node\n"));
 133                 exit(1);
 134         }
 135 }
 136
 137 /* Pretty print the flags to a static buffer in human-readable format.
 138  * This never returns NULL!
 139  */
 140 static const char *pretty_print_flags(uint32_t flags)
 141 {
 142         int j;
 143         static const struct {
 144                 uint32_t flag;
 145                 const char *name;
 146         } flag_names[] = {
 147                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 148                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 149                 { NODE_FLAGS_BANNED,                "BANNED" },
 150                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 151                 { NODE_FLAGS_DELETED,               "DELETED" },
 152                 { NODE_FLAGS_STOPPED,               "STOPPED" },
 153                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 154         };
 155         static char flags_str[512]; /* Big enough to contain all flag names */
 156
 157         flags_str[0] = '\0';
 158         for (j=0;j<ARRAY_SIZE(flag_names);j++) {
 159                 if (flags & flag_names[j].flag) {
 160                         if (flags_str[0] == '\0') {
 161                                 (void) strcpy(flags_str, flag_names[j].name);
 162                         } else {
 163                                 (void) strncat(flags_str, "|", sizeof(flags_str)-1);
 164                                 (void) strncat(flags_str, flag_names[j].name,
 165                                                sizeof(flags_str)-1);
 166                         }
 167                 }
 168         }
 169         if (flags_str[0] == '\0') {
 170                 (void) strcpy(flags_str, "OK");
 171         }
 172
 173         return flags_str;
 174 }
 175
 176 static int h2i(char h)
 177 {
 178         if (h >= 'a' && h <= 'f') return h - 'a' + 10;
 179         if (h >= 'A' && h <= 'F') return h - 'f' + 10;
 180         return h - '0';
 181 }
 182
 183 static TDB_DATA hextodata(TALLOC_CTX *mem_ctx, const char *str)
 184 {
 185         int i, len;
 186         TDB_DATA key = {NULL, 0};
 187
 188         len = strlen(str);
 189         if (len & 0x01) {
 190                 DEBUG(DEBUG_ERR,("Key specified with odd number of hexadecimal digits\n"));
 191                 return key;
 192         }
 193
 194         key.dsize = len>>1;
 195         key.dptr  = talloc_size(mem_ctx, key.dsize);
 196
 197         for (i=0; i < len/2; i++) {
 198                 key.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
 199         }
 200         return key;
 201 }
 202
 203 /* Parse a nodestring.  Parameter dd_ok controls what happens to nodes
 204  * that are disconnected or deleted.  If dd_ok is true those nodes are
 205  * included in the output list of nodes.  If dd_ok is false, those
 206  * nodes are filtered from the "all" case and cause an error if
 207  * explicitly specified.
 208  */
 209 static bool parse_nodestring(struct ctdb_context *ctdb,
 210                              TALLOC_CTX *mem_ctx,
 211                              const char * nodestring,
 212                              uint32_t current_pnn,
 213                              bool dd_ok,
 214                              uint32_t **nodes,
 215                              uint32_t *pnn_mode)
 216 {
 217         TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
 218         int n;
 219         uint32_t i;
 220         struct ctdb_node_map *nodemap;
 221         int ret;
 222
 223         *nodes = NULL;
 224
 225         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
 226         if (ret != 0) {
 227                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
 228                 talloc_free(tmp_ctx);
 229                 exit(10);
 230         }
 231
 232         if (nodestring != NULL) {
 233                 *nodes = talloc_array(mem_ctx, uint32_t, 0);
 234                 if (*nodes == NULL) {
 235                         goto failed;
 236                 }
 237
 238                 n = 0;
 239
 240                 if (strcmp(nodestring, "all") == 0) {
 241                         *pnn_mode = CTDB_BROADCAST_ALL;
 242
 243                         /* all */
 244                         for (i = 0; i < nodemap->num; i++) {
 245                                 if ((nodemap->nodes[i].flags &
 246                                      (NODE_FLAGS_DISCONNECTED |
 247                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 248                                         continue;
 249                                 }
 250                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 251                                                         uint32_t, n+1);
 252                                 if (*nodes == NULL) {
 253                                         goto failed;
 254                                 }
 255                                 (*nodes)[n] = i;
 256                                 n++;
 257                         }
 258                 } else {
 259                         /* x{,y...} */
 260                         char *ns, *tok;
 261
 262                         ns = talloc_strdup(tmp_ctx, nodestring);
 263                         tok = strtok(ns, ",");
 264                         while (tok != NULL) {
 265                                 uint32_t pnn;
 266                                 char *endptr;
 267                                 i = (uint32_t)strtoul(tok, &endptr, 0);
 268                                 if (i == 0 && tok == endptr) {
 269                                         DEBUG(DEBUG_ERR,
 270                                               ("Invalid node %s\n", tok));
 271                                         talloc_free(tmp_ctx);
 272                                         exit(ERR_NONODE);
 273                                 }
 274                                 if (i >= nodemap->num) {
 275                                         DEBUG(DEBUG_ERR, ("Node %u does not exist\n", i));
 276                                         talloc_free(tmp_ctx);
 277                                         exit(ERR_NONODE);
 278                                 }
 279                                 if ((nodemap->nodes[i].flags &
 280                                      (NODE_FLAGS_DISCONNECTED |
 281                                       NODE_FLAGS_DELETED)) && !dd_ok) {
 282                                         DEBUG(DEBUG_ERR, ("Node %u has status %s\n", i, pretty_print_flags(nodemap->nodes[i].flags)));
 283                                         talloc_free(tmp_ctx);
 284                                         exit(ERR_DISNODE);
 285                                 }
 286                                 if ((pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), i)) < 0) {
 287                                         DEBUG(DEBUG_ERR, ("Can not access node %u. Node is not operational.\n", i));
 288                                         talloc_free(tmp_ctx);
 289                                         exit(10);
 290                                 }
 291
 292                                 *nodes = talloc_realloc(mem_ctx, *nodes,
 293                                                         uint32_t, n+1);
 294                                 if (*nodes == NULL) {
 295                                         goto failed;
 296                                 }
 297
 298                                 (*nodes)[n] = i;
 299                                 n++;
 300
 301                                 tok = strtok(NULL, ",");
 302                         }
 303                         talloc_free(ns);
 304
 305                         if (n == 1) {
 306                                 *pnn_mode = (*nodes)[0];
 307                         } else {
 308                                 *pnn_mode = CTDB_MULTICAST;
 309                         }
 310                 }
 311         } else {
 312                 /* default - no nodes specified */
 313                 *nodes = talloc_array(mem_ctx, uint32_t, 1);
 314                 if (*nodes == NULL) {
 315                         goto failed;
 316                 }
 317                 *pnn_mode = CTDB_CURRENT_NODE;
 318
 319                 if (((*nodes)[0] = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), current_pnn)) < 0) {
 320                         goto failed;
 321                 }
 322         }
 323
 324         talloc_free(tmp_ctx);
 325         return true;
 326
 327 failed:
 328         talloc_free(tmp_ctx);
 329         return false;
 330 }
 331
 332 /*
 333  check if a database exists
 334 */
 335 static bool db_exists(struct ctdb_context *ctdb, const char *dbarg,
 336                       uint32_t *dbid, const char **dbname, uint8_t *flags)
 337 {
 338         int i, ret;
 339         struct ctdb_dbid_map *dbmap=NULL;
 340         bool dbid_given = false, found = false;
 341         uint32_t id;
 342         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 343         const char *name = NULL;
 344
 345         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
 346         if (ret != 0) {
 347                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
 348                 goto fail;
 349         }
 350
 351         if (strncmp(dbarg, "0x", 2) == 0) {
 352                 id = strtoul(dbarg, NULL, 0);
 353                 dbid_given = true;
 354         }
 355
 356         for(i=0; i<dbmap->num; i++) {
 357                 if (dbid_given) {
 358                         if (id == dbmap->dbs[i].dbid) {
 359                                 found = true;
 360                                 break;
 361                         }
 362                 } else {
 363                         ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
 364                         if (ret != 0) {
 365                                 DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
 366                                 goto fail;
 367                         }
 368
 369                         if (strcmp(name, dbarg) == 0) {
 370                                 id = dbmap->dbs[i].dbid;
 371                                 found = true;
 372                                 break;
 373                         }
 374                 }
 375         }
 376
 377         if (found && dbid_given && dbname != NULL) {
 378                 ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
 379                 if (ret != 0) {
 380                         DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
 381                         found = false;
 382                         goto fail;
 383                 }
 384         }
 385
 386         if (found) {
 387                 if (dbid) *dbid = id;
 388                 if (dbname) *dbname = talloc_strdup(ctdb, name);
 389                 if (flags) *flags = dbmap->dbs[i].flags;
 390         } else {
 391                 DEBUG(DEBUG_ERR,("No database matching '%s' found\n", dbarg));
 392         }
 393
 394 fail:
 395         talloc_free(tmp_ctx);
 396         return found;
 397 }
 398
 399 /*
 400   see if a process exists
 401  */
 402 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
 403 {
 404         uint32_t pnn, pid;
 405         int ret;
 406         if (argc < 1) {
 407                 usage();
 408         }
 409
 410         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
 411                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
 412                 return -1;
 413         }
 414
 415         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
 416         if (ret == 0) {
 417                 printf("%u:%u exists\n", pnn, pid);
 418         } else {
 419                 printf("%u:%u does not exist\n", pnn, pid);
 420         }
 421         return ret;
 422 }
 423
 424 /*
 425   display statistics structure
 426  */
 427 static void show_statistics(struct ctdb_statistics *s, int show_header)
 428 {
 429         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 430         int i;
 431         const char *prefix=NULL;
 432         int preflen=0;
 433         int tmp, days, hours, minutes, seconds;
 434         const struct {
 435                 const char *name;
 436                 uint32_t offset;
 437         } fields[] = {
 438 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
 439                 STATISTICS_FIELD(num_clients),
 440                 STATISTICS_FIELD(frozen),
 441                 STATISTICS_FIELD(recovering),
 442                 STATISTICS_FIELD(num_recoveries),
 443                 STATISTICS_FIELD(client_packets_sent),
 444                 STATISTICS_FIELD(client_packets_recv),
 445                 STATISTICS_FIELD(node_packets_sent),
 446                 STATISTICS_FIELD(node_packets_recv),
 447                 STATISTICS_FIELD(keepalive_packets_sent),
 448                 STATISTICS_FIELD(keepalive_packets_recv),
 449                 STATISTICS_FIELD(node.req_call),
 450                 STATISTICS_FIELD(node.reply_call),
 451                 STATISTICS_FIELD(node.req_dmaster),
 452                 STATISTICS_FIELD(node.reply_dmaster),
 453                 STATISTICS_FIELD(node.reply_error),
 454                 STATISTICS_FIELD(node.req_message),
 455                 STATISTICS_FIELD(node.req_control),
 456                 STATISTICS_FIELD(node.reply_control),
 457                 STATISTICS_FIELD(client.req_call),
 458                 STATISTICS_FIELD(client.req_message),
 459                 STATISTICS_FIELD(client.req_control),
 460                 STATISTICS_FIELD(timeouts.call),
 461                 STATISTICS_FIELD(timeouts.control),
 462                 STATISTICS_FIELD(timeouts.traverse),
 463                 STATISTICS_FIELD(locks.num_calls),
 464                 STATISTICS_FIELD(locks.num_current),
 465                 STATISTICS_FIELD(locks.num_pending),
 466                 STATISTICS_FIELD(locks.num_failed),
 467                 STATISTICS_FIELD(total_calls),
 468                 STATISTICS_FIELD(pending_calls),
 469                 STATISTICS_FIELD(childwrite_calls),
 470                 STATISTICS_FIELD(pending_childwrite_calls),
 471                 STATISTICS_FIELD(memory_used),
 472                 STATISTICS_FIELD(max_hop_count),
 473                 STATISTICS_FIELD(total_ro_delegations),
 474                 STATISTICS_FIELD(total_ro_revokes),
 475         };
 476
 477         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
 478         seconds = tmp%60;
 479         tmp    /= 60;
 480         minutes = tmp%60;
 481         tmp    /= 60;
 482         hours   = tmp%24;
 483         tmp    /= 24;
 484         days    = tmp;
 485
 486         if (options.machinereadable){
 487                 if (show_header) {
 488                         printm("CTDB version:");
 489                         printm("Current time of statistics:");
 490                         printm("Statistics collected since:");
 491                         for (i=0;i<ARRAY_SIZE(fields);i++) {
 492                                 printm("%s:", fields[i].name);
 493                         }
 494                         printm("num_reclock_ctdbd_latency:");
 495                         printm("min_reclock_ctdbd_latency:");
 496                         printm("avg_reclock_ctdbd_latency:");
 497                         printm("max_reclock_ctdbd_latency:");
 498
 499                         printm("num_reclock_recd_latency:");
 500                         printm("min_reclock_recd_latency:");
 501                         printm("avg_reclock_recd_latency:");
 502                         printm("max_reclock_recd_latency:");
 503
 504                         printm("num_call_latency:");
 505                         printm("min_call_latency:");
 506                         printm("avg_call_latency:");
 507                         printm("max_call_latency:");
 508
 509                         printm("num_lockwait_latency:");
 510                         printm("min_lockwait_latency:");
 511                         printm("avg_lockwait_latency:");
 512                         printm("max_lockwait_latency:");
 513
 514                         printm("num_childwrite_latency:");
 515                         printm("min_childwrite_latency:");
 516                         printm("avg_childwrite_latency:");
 517                         printm("max_childwrite_latency:");
 518                         printm("\n");
 519                 }
 520                 printm("%d:", CTDB_PROTOCOL);
 521                 printm("%d:", (int)s->statistics_current_time.tv_sec);
 522                 printm("%d:", (int)s->statistics_start_time.tv_sec);
 523                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 524                         printm("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 525                 }
 526                 printm("%d:", s->reclock.ctdbd.num);
 527                 printm("%.6f:", s->reclock.ctdbd.min);
 528                 printm("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
 529                 printm("%.6f:", s->reclock.ctdbd.max);
 530
 531                 printm("%d:", s->reclock.recd.num);
 532                 printm("%.6f:", s->reclock.recd.min);
 533                 printm("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
 534                 printm("%.6f:", s->reclock.recd.max);
 535
 536                 printm("%d:", s->call_latency.num);
 537                 printm("%.6f:", s->call_latency.min);
 538                 printm("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
 539                 printm("%.6f:", s->call_latency.max);
 540
 541                 printm("%d:", s->childwrite_latency.num);
 542                 printm("%.6f:", s->childwrite_latency.min);
 543                 printm("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
 544                 printm("%.6f:", s->childwrite_latency.max);
 545                 printm("\n");
 546         } else {
 547                 printf("CTDB version %u\n", CTDB_PROTOCOL);
 548                 printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
 549                 printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
 550
 551                 for (i=0;i<ARRAY_SIZE(fields);i++) {
 552                         if (strchr(fields[i].name, '.')) {
 553                                 preflen = strcspn(fields[i].name, ".")+1;
 554                                 if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
 555                                         prefix = fields[i].name;
 556                                         printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
 557                                 }
 558                         } else {
 559                                 preflen = 0;
 560                         }
 561                         printf(" %*s%-22s%*s%10u\n",
 562                                preflen?4:0, "",
 563                                fields[i].name+preflen,
 564                                preflen?0:4, "",
 565                                *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 566                 }
 567                 printf(" hop_count_buckets:");
 568                 for (i=0;i<MAX_COUNT_BUCKETS;i++) {
 569                         printf(" %d", s->hop_count_bucket[i]);
 570                 }
 571                 printf("\n");
 572                 printf(" lock_buckets:");
 573                 for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 574                         printf(" %d", s->locks.buckets[i]);
 575                 }
 576                 printf("\n");
 577                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
 578
 579                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
 580
 581                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
 582
 583                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
 584                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
 585         }
 586
 587         talloc_free(tmp_ctx);
 588 }
 589
 590 /*
 591   display remote ctdb statistics combined from all nodes
 592  */
 593 static int control_statistics_all(struct ctdb_context *ctdb)
 594 {
 595         int ret, i;
 596         struct ctdb_statistics statistics;
 597         uint32_t *nodes;
 598         uint32_t num_nodes;
 599
 600         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
 601         CTDB_NO_MEMORY(ctdb, nodes);
 602
 603         ZERO_STRUCT(statistics);
 604
 605         for (i=0;i<num_nodes;i++) {
 606                 struct ctdb_statistics s1;
 607                 int j;
 608                 uint32_t *v1 = (uint32_t *)&s1;
 609                 uint32_t *v2 = (uint32_t *)&statistics;
 610                 uint32_t num_ints =
 611                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
 612                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
 613                 if (ret != 0) {
 614                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
 615                         return ret;
 616                 }
 617                 for (j=0;j<num_ints;j++) {
 618                         v2[j] += v1[j];
 619                 }
 620                 statistics.max_hop_count =
 621                         MAX(statistics.max_hop_count, s1.max_hop_count);
 622                 statistics.call_latency.max =
 623                         MAX(statistics.call_latency.max, s1.call_latency.max);
 624         }
 625         talloc_free(nodes);
 626         printf("Gathered statistics for %u nodes\n", num_nodes);
 627         show_statistics(&statistics, 1);
 628         return 0;
 629 }
 630
 631 /*
 632   display remote ctdb statistics
 633  */
 634 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
 635 {
 636         int ret;
 637         struct ctdb_statistics statistics;
 638
 639         if (options.pnn == CTDB_BROADCAST_ALL) {
 640                 return control_statistics_all(ctdb);
 641         }
 642
 643         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
 644         if (ret != 0) {
 645                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
 646                 return ret;
 647         }
 648         show_statistics(&statistics, 1);
 649         return 0;
 650 }
 651
 652
 653 /*
 654   reset remote ctdb statistics
 655  */
 656 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
 657 {
 658         int ret;
 659
 660         ret = ctdb_statistics_reset(ctdb, options.pnn);
 661         if (ret != 0) {
 662                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
 663                 return ret;
 664         }
 665         return 0;
 666 }
 667
 668
 669 /*
 670   display remote ctdb rolling statistics
 671  */
 672 static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
 673 {
 674         int ret;
 675         struct ctdb_statistics_wire *stats;
 676         int i, num_records = -1;
 677
 678         assert_single_node_only();
 679
 680         if (argc ==1) {
 681                 num_records = atoi(argv[0]) - 1;
 682         }
 683
 684         ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
 685         if (ret != 0) {
 686                 DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
 687                 return ret;
 688         }
 689         for (i=0;i<stats->num;i++) {
 690                 if (stats->stats[i].statistics_start_time.tv_sec == 0) {
 691                         continue;
 692                 }
 693                 show_statistics(&stats->stats[i], i==0);
 694                 if (i == num_records) {
 695                         break;
 696                 }
 697         }
 698         return 0;
 699 }
 700
 701
 702 /*
 703   display remote ctdb db statistics
 704  */
 705 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
 706 {
 707         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 708         struct ctdb_db_statistics *dbstat;
 709         int i;
 710         uint32_t db_id;
 711         int num_hot_keys;
 712         int ret;
 713
 714         if (argc < 1) {
 715                 usage();
 716         }
 717
 718         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
 719                 return -1;
 720         }
 721
 722         ret = ctdb_ctrl_dbstatistics(ctdb, options.pnn, db_id, tmp_ctx, &dbstat);
 723         if (ret != 0) {
 724                 DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
 725                 talloc_free(tmp_ctx);
 726                 return -1;
 727         }
 728
 729         printf("DB Statistics: %s\n", argv[0]);
 730         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
 731                 dbstat->db_ro_delegations);
 732         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
 733                 dbstat->db_ro_delegations);
 734         printf(" %s\n", "locks");
 735         printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
 736                 dbstat->locks.num_calls);
 737         printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
 738                 dbstat->locks.num_failed);
 739         printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
 740                 dbstat->locks.num_current);
 741         printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
 742                 dbstat->locks.num_pending);
 743         printf(" %s", "hop_count_buckets:");
 744         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 745                 printf(" %d", dbstat->hop_count_bucket[i]);
 746         }
 747         printf("\n");
 748         printf(" %s", "lock_buckets:");
 749         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
 750                 printf(" %d", dbstat->locks.buckets[i]);
 751         }
 752         printf("\n");
 753         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
 754                 "locks_latency      MIN/AVG/MAX",
 755                 dbstat->locks.latency.min,
 756                 (dbstat->locks.latency.num ?
 757                  dbstat->locks.latency.total /dbstat->locks.latency.num :
 758                  0.0),
 759                 dbstat->locks.latency.max,
 760                 dbstat->locks.latency.num);
 761         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
 762                 "vacuum_latency     MIN/AVG/MAX",
 763                 dbstat->vacuum.latency.min,
 764                 (dbstat->vacuum.latency.num ?
 765                  dbstat->vacuum.latency.total /dbstat->vacuum.latency.num :
 766                  0.0),
 767                 dbstat->vacuum.latency.max,
 768                 dbstat->vacuum.latency.num);
 769         num_hot_keys = 0;
 770         for (i=0; i<dbstat->num_hot_keys; i++) {
 771                 if (dbstat->hot_keys[i].count > 0) {
 772                         num_hot_keys++;
 773                 }
 774         }
 775         dbstat->num_hot_keys = num_hot_keys;
 776
 777         printf(" Num Hot Keys:     %d\n", dbstat->num_hot_keys);
 778         for (i = 0; i < dbstat->num_hot_keys; i++) {
 779                 int j;
 780                 printf("     Count:%d Key:", dbstat->hot_keys[i].count);
 781                 for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
 782                         printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
 783                 }
 784                 printf("\n");
 785         }
 786
 787         talloc_free(tmp_ctx);
 788         return 0;
 789 }
 790
 791 /*
 792   display uptime of remote node
 793  */
 794 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
 795 {
 796         int ret;
 797         struct ctdb_uptime *uptime = NULL;
 798         int tmp, days, hours, minutes, seconds;
 799
 800         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
 801         if (ret != 0) {
 802                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
 803                 return ret;
 804         }
 805
 806         if (options.machinereadable){
 807                 printm(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
 808                 printm(":%u:%u:%u:%lf\n",
 809                         (unsigned int)uptime->current_time.tv_sec,
 810                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
 811                         (unsigned int)uptime->last_recovery_finished.tv_sec,
 812                         timeval_delta(&uptime->last_recovery_finished,
 813                                       &uptime->last_recovery_started)
 814                 );
 815                 return 0;
 816         }
 817
 818         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
 819
 820         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 821         seconds = tmp%60;
 822         tmp    /= 60;
 823         minutes = tmp%60;
 824         tmp    /= 60;
 825         hours   = tmp%24;
 826         tmp    /= 24;
 827         days    = tmp;
 828         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
 829
 830         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 831         seconds = tmp%60;
 832         tmp    /= 60;
 833         minutes = tmp%60;
 834         tmp    /= 60;
 835         hours   = tmp%24;
 836         tmp    /= 24;
 837         days    = tmp;
 838         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
 839
 840         printf("Duration of last recovery/failover: %lf seconds\n",
 841                 timeval_delta(&uptime->last_recovery_finished,
 842                               &uptime->last_recovery_started));
 843
 844         return 0;
 845 }
 846
 847 /*
 848   show the PNN of the current node
 849  */
 850 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
 851 {
 852         uint32_t mypnn;
 853
 854         mypnn = getpnn(ctdb);
 855
 856         printf("PNN:%d\n", mypnn);
 857         return 0;
 858 }
 859
 860
 861 static struct ctdb_node_map *read_nodes_file(TALLOC_CTX *mem_ctx)
 862 {
 863         const char *nodes_list;
 864
 865         /* read the nodes file */
 866         nodes_list = getenv("CTDB_NODES");
 867         if (nodes_list == NULL) {
 868                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes",
 869                                              getenv("CTDB_BASE"));
 870                 if (nodes_list == NULL) {
 871                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
 872                         exit(1);
 873                 }
 874         }
 875
 876         return ctdb_read_nodes_file(mem_ctx, nodes_list);
 877 }
 878
 879 /*
 880   show the PNN of the current node
 881   discover the pnn by loading the nodes file and try to bind to all
 882   addresses one at a time until the ip address is found.
 883  */
 884 static int find_node_xpnn(void)
 885 {
 886         TALLOC_CTX *mem_ctx = talloc_new(NULL);
 887         struct ctdb_node_map *node_map;
 888         int i, pnn;
 889
 890         node_map = read_nodes_file(mem_ctx);
 891         if (node_map == NULL) {
 892                 talloc_free(mem_ctx);
 893                 return -1;
 894         }
 895
 896         for (i = 0; i < node_map->num; i++) {
 897                 if (node_map->nodes[i].flags & NODE_FLAGS_DELETED) {
 898                         continue;
 899                 }
 900                 if (ctdb_sys_have_ip(&node_map->nodes[i].addr)) {
 901                         pnn = node_map->nodes[i].pnn;
 902                         talloc_free(mem_ctx);
 903                         return pnn;
 904                 }
 905         }
 906
 907         printf("Failed to detect which PNN this node is\n");
 908         talloc_free(mem_ctx);
 909         return -1;
 910 }
 911
 912 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
 913 {
 914         uint32_t pnn;
 915
 916         assert_single_node_only();
 917
 918         pnn = find_node_xpnn();
 919         if (pnn == -1) {
 920                 return -1;
 921         }
 922
 923         printf("PNN:%d\n", pnn);
 924         return 0;
 925 }
 926
 927 /* Helpers for ctdb status
 928  */
 929 static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_flags *node)
 930 {
 931         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 932         int j;
 933         bool ret = false;
 934
 935         if (node->flags == 0) {
 936                 struct ctdb_control_get_ifaces *ifaces;
 937
 938                 if (ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), node->pnn,
 939                                          tmp_ctx, &ifaces) == 0) {
 940                         for (j=0; j < ifaces->num; j++) {
 941                                 if (ifaces->ifaces[j].link_state != 0) {
 942                                         continue;
 943                                 }
 944                                 ret = true;
 945                                 break;
 946                         }
 947                 }
 948         }
 949         talloc_free(tmp_ctx);
 950
 951         return ret;
 952 }
 953
 954 static void control_status_header_machine(void)
 955 {
 956         printm(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
 957                ":Inactive:PartiallyOnline:ThisNode:\n");
 958 }
 959
 960 static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn,
 961                                     struct ctdb_node_and_flags *node)
 962 {
 963         printm(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
 964                ctdb_addr_to_str(&node->addr),
 965                !!(node->flags&NODE_FLAGS_DISCONNECTED),
 966                !!(node->flags&NODE_FLAGS_BANNED),
 967                !!(node->flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 968                !!(node->flags&NODE_FLAGS_UNHEALTHY),
 969                !!(node->flags&NODE_FLAGS_STOPPED),
 970                !!(node->flags&NODE_FLAGS_INACTIVE),
 971                is_partially_online(ctdb, node) ? 1 : 0,
 972                (node->pnn == mypnn)?'Y':'N');
 973
 974         return node->flags;
 975 }
 976
 977 static int control_status_1_human(struct ctdb_context *ctdb, int mypnn,
 978                                   struct ctdb_node_and_flags *node)
 979 {
 980        printf("pnn:%d %-16s %s%s\n", node->pnn,
 981               ctdb_addr_to_str(&node->addr),
 982               is_partially_online(ctdb, node) ? "PARTIALLYONLINE" : pretty_print_flags(node->flags),
 983               node->pnn == mypnn?" (THIS NODE)":"");
 984
 985        return node->flags;
 986 }
 987
 988 /*
 989   display remote ctdb status
 990  */
 991 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
 992 {
 993         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 994         int i;
 995         struct ctdb_vnn_map *vnnmap=NULL;
 996         struct ctdb_node_map *nodemap=NULL;
 997         uint32_t recmode, recmaster, mypnn;
 998         int num_deleted_nodes = 0;
 999         int ret;
1000
1001         mypnn = getpnn(ctdb);
1002
1003         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1004         if (ret != 0) {
1005                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1006                 talloc_free(tmp_ctx);
1007                 return -1;
1008         }
1009
1010         if (options.machinereadable) {
1011                 control_status_header_machine();
1012                 for (i=0;i<nodemap->num;i++) {
1013                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1014                                 continue;
1015                         }
1016                         (void) control_status_1_machine(ctdb, mypnn,
1017                                                         &nodemap->nodes[i]);
1018                 }
1019                 talloc_free(tmp_ctx);
1020                 return 0;
1021         }
1022
1023         for (i=0; i<nodemap->num; i++) {
1024                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1025                         num_deleted_nodes++;
1026                 }
1027         }
1028         if (num_deleted_nodes == 0) {
1029                 printf("Number of nodes:%d\n", nodemap->num);
1030         } else {
1031                 printf("Number of nodes:%d (including %d deleted nodes)\n",
1032                        nodemap->num, num_deleted_nodes);
1033         }
1034         for(i=0;i<nodemap->num;i++){
1035                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1036                         continue;
1037                 }
1038                 (void) control_status_1_human(ctdb, mypnn, &nodemap->nodes[i]);
1039         }
1040
1041         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
1042         if (ret != 0) {
1043                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
1044                 talloc_free(tmp_ctx);
1045                 return -1;
1046         }
1047         if (vnnmap->generation == INVALID_GENERATION) {
1048                 printf("Generation:INVALID\n");
1049         } else {
1050                 printf("Generation:%d\n",vnnmap->generation);
1051         }
1052         printf("Size:%d\n",vnnmap->size);
1053         for(i=0;i<vnnmap->size;i++){
1054                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
1055         }
1056
1057         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmode);
1058         if (ret != 0) {
1059                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
1060                 talloc_free(tmp_ctx);
1061                 return -1;
1062         }
1063         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
1064
1065         ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmaster);
1066         if (ret != 0) {
1067                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1068                 talloc_free(tmp_ctx);
1069                 return -1;
1070         }
1071         printf("Recovery master:%d\n",recmaster);
1072
1073         talloc_free(tmp_ctx);
1074         return 0;
1075 }
1076
1077 static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **argv)
1078 {
1079         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1080         int i, ret;
1081         struct ctdb_node_map *nodemap=NULL;
1082         uint32_t * nodes;
1083         uint32_t pnn_mode, mypnn;
1084
1085         if (argc > 1) {
1086                 usage();
1087         }
1088
1089         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1090                               options.pnn, true, &nodes, &pnn_mode)) {
1091                 return -1;
1092         }
1093
1094         if (options.machinereadable) {
1095                 control_status_header_machine();
1096         } else if (pnn_mode == CTDB_BROADCAST_ALL) {
1097                 printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
1098         }
1099
1100         mypnn = getpnn(ctdb);
1101
1102         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1103         if (ret != 0) {
1104                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1105                 talloc_free(tmp_ctx);
1106                 return -1;
1107         }
1108
1109         ret = 0;
1110
1111         for (i = 0; i < talloc_array_length(nodes); i++) {
1112                 if (options.machinereadable) {
1113                         ret |= control_status_1_machine(ctdb, mypnn,
1114                                                         &nodemap->nodes[nodes[i]]);
1115                 } else {
1116                         ret |= control_status_1_human(ctdb, mypnn,
1117                                                       &nodemap->nodes[nodes[i]]);
1118                 }
1119         }
1120
1121         talloc_free(tmp_ctx);
1122         return ret;
1123 }
1124
1125 static struct ctdb_node_map *read_natgw_nodes_file(struct ctdb_context *ctdb,
1126                                                    TALLOC_CTX *mem_ctx)
1127 {
1128         const char *natgw_list;
1129         struct ctdb_node_map *natgw_nodes = NULL;
1130
1131         natgw_list = getenv("CTDB_NATGW_NODES");
1132         if (natgw_list == NULL) {
1133                 natgw_list = talloc_asprintf(mem_ctx, "%s/natgw_nodes",
1134                                              getenv("CTDB_BASE"));
1135                 if (natgw_list == NULL) {
1136                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
1137                         exit(1);
1138                 }
1139         }
1140         /* The PNNs/flags will be junk but they're not used */
1141         natgw_nodes = ctdb_read_nodes_file(mem_ctx, natgw_list);
1142         if (natgw_nodes == NULL) {
1143                 DEBUG(DEBUG_ERR,
1144                       ("Failed to load natgw node list '%s'\n", natgw_list));
1145         }
1146         return natgw_nodes;
1147 }
1148
1149
1150 /* talloc off the existing nodemap... */
1151 static struct ctdb_node_map *talloc_nodemap(struct ctdb_node_map *nodemap)
1152 {
1153         return talloc_zero_size(nodemap,
1154                                 offsetof(struct ctdb_node_map, nodes) +
1155                                 nodemap->num * sizeof(struct ctdb_node_and_flags));
1156 }
1157
1158 static struct ctdb_node_map *
1159 filter_nodemap_by_addrs(struct ctdb_context *ctdb,
1160                         struct ctdb_node_map *nodemap,
1161                         struct ctdb_node_map *natgw_nodes)
1162 {
1163         int i, j;
1164         struct ctdb_node_map *ret;
1165
1166         ret = talloc_nodemap(nodemap);
1167         CTDB_NO_MEMORY_NULL(ctdb, ret);
1168
1169         ret->num = 0;
1170
1171         for (i = 0; i < nodemap->num; i++) {
1172                 for(j = 0; j < natgw_nodes->num ; j++) {
1173                         if (nodemap->nodes[j].flags & NODE_FLAGS_DELETED) {
1174                                 continue;
1175                         }
1176                         if (ctdb_same_ip(&natgw_nodes->nodes[j].addr,
1177                                          &nodemap->nodes[i].addr)) {
1178
1179                                 ret->nodes[ret->num] = nodemap->nodes[i];
1180                                 ret->num++;
1181                                 break;
1182                         }
1183                 }
1184         }
1185
1186         return ret;
1187 }
1188
1189 static struct ctdb_node_map *
1190 filter_nodemap_by_capabilities(struct ctdb_context *ctdb,
1191                                struct ctdb_node_map *nodemap,
1192                                uint32_t required_capabilities,
1193                                bool first_only)
1194 {
1195         int i;
1196         uint32_t capabilities;
1197         struct ctdb_node_map *ret;
1198
1199         ret = talloc_nodemap(nodemap);
1200         CTDB_NO_MEMORY_NULL(ctdb, ret);
1201
1202         ret->num = 0;
1203
1204         for (i = 0; i < nodemap->num; i++) {
1205                 int res;
1206
1207                 /* Disconnected nodes have no capabilities! */
1208                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1209                         continue;
1210                 }
1211
1212                 res = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(),
1213                                                 nodemap->nodes[i].pnn,
1214                                                 &capabilities);
1215                 if (res != 0) {
1216                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n",
1217                                           nodemap->nodes[i].pnn));
1218                         talloc_free(ret);
1219                         return NULL;
1220                 }
1221                 if (!(capabilities & required_capabilities)) {
1222                         continue;
1223                 }
1224
1225                 ret->nodes[ret->num] = nodemap->nodes[i];
1226                 ret->num++;
1227                 if (first_only) {
1228                         break;
1229                 }
1230         }
1231
1232         return ret;
1233 }
1234
1235 static struct ctdb_node_map *
1236 filter_nodemap_by_flags(struct ctdb_context *ctdb,
1237                         struct ctdb_node_map *nodemap,
1238                         uint32_t flags_mask)
1239 {
1240         int i;
1241         struct ctdb_node_map *ret;
1242
1243         ret = talloc_nodemap(nodemap);
1244         CTDB_NO_MEMORY_NULL(ctdb, ret);
1245
1246         ret->num = 0;
1247
1248         for (i = 0; i < nodemap->num; i++) {
1249                 if (nodemap->nodes[i].flags & flags_mask) {
1250                         continue;
1251                 }
1252
1253                 ret->nodes[ret->num] = nodemap->nodes[i];
1254                 ret->num++;
1255         }
1256
1257         return ret;
1258 }
1259
1260 /*
1261   display the list of nodes belonging to this natgw configuration
1262  */
1263 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
1264 {
1265         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1266         int i, ret;
1267         struct ctdb_node_map *natgw_nodes = NULL;
1268         struct ctdb_node_map *orig_nodemap=NULL;
1269         struct ctdb_node_map *nodemap;
1270         uint32_t mypnn, pnn;
1271         const char *ip;
1272
1273         /* When we have some nodes that could be the NATGW, make a
1274          * series of attempts to find the first node that doesn't have
1275          * certain status flags set.
1276          */
1277         uint32_t exclude_flags[] = {
1278                 /* Look for a nice healthy node */
1279                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY,
1280                 /* If not found, an UNHEALTHY/BANNED node will do */
1281                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED,
1282                 /* If not found, a STOPPED node will do */
1283                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED,
1284                 0,
1285         };
1286
1287         /* read the natgw nodes file into a linked list */
1288         natgw_nodes = read_natgw_nodes_file(ctdb, tmp_ctx);
1289         if (natgw_nodes == NULL) {
1290                 ret = -1;
1291                 goto done;
1292         }
1293
1294         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
1295                                    tmp_ctx, &orig_nodemap);
1296         if (ret != 0) {
1297                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
1298                 talloc_free(tmp_ctx);
1299                 return -1;
1300         }
1301
1302         /* Get a nodemap that includes only the nodes in the NATGW
1303          * group */
1304         nodemap = filter_nodemap_by_addrs(ctdb, orig_nodemap, natgw_nodes);
1305         if (nodemap == NULL) {
1306                 ret = -1;
1307                 goto done;
1308         }
1309
1310         ret = 2; /* matches ENOENT */
1311         pnn = -1;
1312         ip = "0.0.0.0";
1313         /* For each flag mask... */
1314         for (i = 0; exclude_flags[i] != 0; i++) {
1315                 /* ... get a nodemap that excludes nodes with with
1316                  * masked flags... */
1317                 struct ctdb_node_map *t =
1318                         filter_nodemap_by_flags(ctdb, nodemap,
1319                                                 exclude_flags[i]);
1320                 if (t == NULL) {
1321                         /* No memory */
1322                         ret = -1;
1323                         goto done;
1324                 }
1325                 if (t->num > 0) {
1326                         /* ... and find the first node with the NATGW
1327                          * capability */
1328                         struct ctdb_node_map *n;
1329                         n = filter_nodemap_by_capabilities(ctdb, t,
1330                                                            CTDB_CAP_NATGW,
1331                                                            true);
1332                         if (n == NULL) {
1333                                 /* No memory */
1334                                 ret = -1;
1335                                 goto done;
1336                         }
1337                         if (n->num > 0) {
1338                                 ret = 0;
1339                                 pnn = n->nodes[0].pnn;
1340                                 ip = ctdb_addr_to_str(&n->nodes[0].addr);
1341                                 break;
1342                         }
1343                 }
1344                 talloc_free(t);
1345         }
1346
1347         if (options.machinereadable) {
1348                 printm(":Node:IP:\n");
1349                 printm(":%d:%s:\n", pnn, ip);
1350         } else {
1351                 printf("%d %s\n", pnn, ip);
1352         }
1353
1354         /* print the pruned list of nodes belonging to this natgw list */
1355         mypnn = getpnn(ctdb);
1356         if (options.machinereadable) {
1357                 control_status_header_machine();
1358         } else {
1359                 printf("Number of nodes:%d\n", nodemap->num);
1360         }
1361         for(i=0;i<nodemap->num;i++){
1362                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1363                         continue;
1364                 }
1365                 if (options.machinereadable) {
1366                         control_status_1_machine(ctdb, mypnn, &(nodemap->nodes[i]));
1367                 } else {
1368                         control_status_1_human(ctdb, mypnn, &(nodemap->nodes[i]));
1369                 }
1370         }
1371
1372 done:
1373         talloc_free(tmp_ctx);
1374         return ret;
1375 }
1376
1377 /*
1378   display the status of the scripts for monitoring (or other events)
1379  */
1380 static int control_one_scriptstatus(struct ctdb_context *ctdb,
1381                                     enum ctdb_eventscript_call type)
1382 {
1383         struct ctdb_scripts_wire *script_status;
1384         int ret, i;
1385
1386         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
1387         if (ret != 0) {
1388                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
1389                 return ret;
1390         }
1391
1392         if (script_status == NULL) {
1393                 if (!options.machinereadable) {
1394                         printf("%s cycle never run\n",
1395                                ctdb_eventscript_call_names[type]);
1396                 }
1397                 return 0;
1398         }
1399
1400         if (!options.machinereadable) {
1401                 int num_run = 0;
1402                 for (i=0; i<script_status->num_scripts; i++) {
1403                         if (script_status->scripts[i].status != -ENOEXEC) {
1404                                 num_run++;
1405                         }
1406                 }
1407                 printf("%d scripts were executed last %s cycle\n",
1408                        num_run,
1409                        ctdb_eventscript_call_names[type]);
1410         }
1411         for (i=0; i<script_status->num_scripts; i++) {
1412                 const char *status = NULL;
1413
1414                 switch (script_status->scripts[i].status) {
1415                 case -ETIME:
1416                         status = "TIMEDOUT";
1417                         break;
1418                 case -ENOEXEC:
1419                         status = "DISABLED";
1420                         break;
1421                 case 0:
1422                         status = "OK";
1423                         break;
1424                 default:
1425                         if (script_status->scripts[i].status > 0)
1426                                 status = "ERROR";
1427                         break;
1428                 }
1429                 if (options.machinereadable) {
1430                         printm(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
1431                                ctdb_eventscript_call_names[type],
1432                                script_status->scripts[i].name,
1433                                script_status->scripts[i].status,
1434                                status,
1435                                (long)script_status->scripts[i].start.tv_sec,
1436                                (long)script_status->scripts[i].start.tv_usec,
1437                                (long)script_status->scripts[i].finished.tv_sec,
1438                                (long)script_status->scripts[i].finished.tv_usec,
1439                                script_status->scripts[i].output);
1440                         continue;
1441                 }
1442                 if (status)
1443                         printf("%-20s Status:%s    ",
1444                                script_status->scripts[i].name, status);
1445                 else
1446                         /* Some other error, eg from stat. */
1447                         printf("%-20s Status:CANNOT RUN (%s)",
1448                                script_status->scripts[i].name,
1449                                strerror(-script_status->scripts[i].status));
1450
1451                 if (script_status->scripts[i].status >= 0) {
1452                         printf("Duration:%.3lf ",
1453                         timeval_delta(&script_status->scripts[i].finished,
1454                               &script_status->scripts[i].start));
1455                 }
1456                 if (script_status->scripts[i].status != -ENOEXEC) {
1457                         printf("%s",
1458                                ctime(&script_status->scripts[i].start.tv_sec));
1459                         if (script_status->scripts[i].status != 0) {
1460                                 printf("   OUTPUT:%s\n",
1461                                        script_status->scripts[i].output);
1462                         }
1463                 } else {
1464                         printf("\n");
1465                 }
1466         }
1467         return 0;
1468 }
1469
1470
1471 static int control_scriptstatus(struct ctdb_context *ctdb,
1472                                 int argc, const char **argv)
1473 {
1474         int ret;
1475         enum ctdb_eventscript_call type, min, max;
1476         const char *arg;
1477
1478         if (argc > 1) {
1479                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
1480                 return -1;
1481         }
1482
1483         if (argc == 0)
1484                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
1485         else
1486                 arg = argv[0];
1487
1488         for (type = 0; type < CTDB_EVENT_MAX; type++) {
1489                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
1490                         min = type;
1491                         max = type+1;
1492                         break;
1493                 }
1494         }
1495         if (type == CTDB_EVENT_MAX) {
1496                 if (strcmp(arg, "all") == 0) {
1497                         min = 0;
1498                         max = CTDB_EVENT_MAX;
1499                 } else {
1500                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
1501                         return -1;
1502                 }
1503         }
1504
1505         if (options.machinereadable) {
1506                 printm(":Type:Name:Code:Status:Start:End:Error Output...:\n");
1507         }
1508
1509         for (type = min; type < max; type++) {
1510                 ret = control_one_scriptstatus(ctdb, type);
1511                 if (ret != 0) {
1512                         return ret;
1513                 }
1514         }
1515
1516         return 0;
1517 }
1518
1519 /*
1520   enable an eventscript
1521  */
1522 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1523 {
1524         int ret;
1525
1526         if (argc < 1) {
1527                 usage();
1528         }
1529
1530         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1531         if (ret != 0) {
1532           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
1533                 return ret;
1534         }
1535
1536         return 0;
1537 }
1538
1539 /*
1540   disable an eventscript
1541  */
1542 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1543 {
1544         int ret;
1545
1546         if (argc < 1) {
1547                 usage();
1548         }
1549
1550         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1551         if (ret != 0) {
1552           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
1553                 return ret;
1554         }
1555
1556         return 0;
1557 }
1558
1559 /*
1560   display the pnn of the recovery master
1561  */
1562 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
1563 {
1564         uint32_t recmaster;
1565         int ret;
1566
1567         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
1568         if (ret != 0) {
1569                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1570                 return -1;
1571         }
1572         printf("%d\n",recmaster);
1573
1574         return 0;
1575 }
1576
1577 /*
1578   add a tickle to a public address
1579  */
1580 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1581 {
1582         struct ctdb_tcp_connection t;
1583         TDB_DATA data;
1584         int ret;
1585
1586         assert_single_node_only();
1587
1588         if (argc < 2) {
1589                 usage();
1590         }
1591
1592         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1593                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1594                 return -1;
1595         }
1596         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1597                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1598                 return -1;
1599         }
1600
1601         data.dptr = (uint8_t *)&t;
1602         data.dsize = sizeof(t);
1603
1604         /* tell all nodes about this tcp connection */
1605         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1606                            0, data, ctdb, NULL, NULL, NULL, NULL);
1607         if (ret != 0) {
1608                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1609                 return -1;
1610         }
1611
1612         return 0;
1613 }
1614
1615
1616 /*
1617   delete a tickle from a node
1618  */
1619 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1620 {
1621         struct ctdb_tcp_connection t;
1622         TDB_DATA data;
1623         int ret;
1624
1625         assert_single_node_only();
1626
1627         if (argc < 2) {
1628                 usage();
1629         }
1630
1631         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1632                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1633                 return -1;
1634         }
1635         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1636                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1637                 return -1;
1638         }
1639
1640         data.dptr = (uint8_t *)&t;
1641         data.dsize = sizeof(t);
1642
1643         /* tell all nodes about this tcp connection */
1644         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1645                            0, data, ctdb, NULL, NULL, NULL, NULL);
1646         if (ret != 0) {
1647                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1648                 return -1;
1649         }
1650
1651         return 0;
1652 }
1653
1654
1655 /*
1656   get a list of all tickles for this pnn
1657  */
1658 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1659 {
1660         struct ctdb_control_tcp_tickle_list *list;
1661         ctdb_sock_addr addr;
1662         int i, ret;
1663         unsigned port = 0;
1664
1665         assert_single_node_only();
1666
1667         if (argc < 1) {
1668                 usage();
1669         }
1670
1671         if (argc == 2) {
1672                 port = atoi(argv[1]);
1673         }
1674
1675         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1676                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1677                 return -1;
1678         }
1679
1680         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1681         if (ret == -1) {
1682                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1683                 return -1;
1684         }
1685
1686         if (options.machinereadable){
1687                 printm(":source ip:port:destination ip:port:\n");
1688                 for (i=0;i<list->tickles.num;i++) {
1689                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1690                                 continue;
1691                         }
1692                         printm(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1693                         printm(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1694                 }
1695         } else {
1696                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1697                 printf("Num tickles:%u\n", list->tickles.num);
1698                 for (i=0;i<list->tickles.num;i++) {
1699                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1700                                 continue;
1701                         }
1702                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1703                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1704                 }
1705         }
1706
1707         talloc_free(list);
1708
1709         return 0;
1710 }
1711
1712
1713 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1714 {
1715         struct ctdb_all_public_ips *ips;
1716         struct ctdb_public_ip ip;
1717         int i, ret;
1718         uint32_t *nodes;
1719         uint32_t disable_time;
1720         TDB_DATA data;
1721         struct ctdb_node_map *nodemap=NULL;
1722         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1723
1724         disable_time = 30;
1725         data.dptr  = (uint8_t*)&disable_time;
1726         data.dsize = sizeof(disable_time);
1727         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1728         if (ret != 0) {
1729                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1730                 return -1;
1731         }
1732
1733
1734
1735         /* read the public ip list from the node */
1736         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1737         if (ret != 0) {
1738                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1739                 talloc_free(tmp_ctx);
1740                 return -1;
1741         }
1742
1743         for (i=0;i<ips->num;i++) {
1744                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1745                         break;
1746                 }
1747         }
1748         if (i==ips->num) {
1749                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1750                         pnn, ctdb_addr_to_str(addr)));
1751                 talloc_free(tmp_ctx);
1752                 return -1;
1753         }
1754
1755         ip.pnn  = pnn;
1756         ip.addr = *addr;
1757
1758         data.dptr  = (uint8_t *)&ip;
1759         data.dsize = sizeof(ip);
1760
1761         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1762         if (ret != 0) {
1763                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1764                 talloc_free(tmp_ctx);
1765                 return ret;
1766         }
1767
1768         nodes = list_of_nodes(ctdb, nodemap, tmp_ctx, NODE_FLAGS_INACTIVE, pnn);
1769         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1770                                         nodes, 0,
1771                                         LONGTIMELIMIT(),
1772                                         false, data,
1773                                         NULL, NULL,
1774                                         NULL);
1775         if (ret != 0) {
1776                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1777                 talloc_free(tmp_ctx);
1778                 return -1;
1779         }
1780
1781         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1782         if (ret != 0) {
1783                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1784                 talloc_free(tmp_ctx);
1785                 return -1;
1786         }
1787
1788         /* update the recovery daemon so it now knows to expect the new
1789            node assignment for this ip.
1790         */
1791         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1792         if (ret != 0) {
1793                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1794                 return -1;
1795         }
1796
1797         talloc_free(tmp_ctx);
1798         return 0;
1799 }
1800
1801
1802 /*
1803  * scans all other nodes and returns a pnn for another node that can host this
1804  * ip address or -1
1805  */
1806 static int
1807 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1808 {
1809         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1810         struct ctdb_all_public_ips *ips;
1811         struct ctdb_node_map *nodemap=NULL;
1812         int i, j, ret;
1813         int pnn;
1814
1815         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1816         if (ret != 0) {
1817                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1818                 talloc_free(tmp_ctx);
1819                 return ret;
1820         }
1821
1822         for(i=0;i<nodemap->num;i++){
1823                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1824                         continue;
1825                 }
1826                 if (nodemap->nodes[i].pnn == options.pnn) {
1827                         continue;
1828                 }
1829
1830                 /* read the public ip list from this node */
1831                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1832                 if (ret != 0) {
1833                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1834                         return -1;
1835                 }
1836
1837                 for (j=0;j<ips->num;j++) {
1838                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1839                                 pnn = nodemap->nodes[i].pnn;
1840                                 talloc_free(tmp_ctx);
1841                                 return pnn;
1842                         }
1843                 }
1844                 talloc_free(ips);
1845         }
1846
1847         talloc_free(tmp_ctx);
1848         return -1;
1849 }
1850
1851 /* If pnn is -1 then try to find a node to move IP to... */
1852 static bool try_moveip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1853 {
1854         bool pnn_specified = (pnn == -1 ? false : true);
1855         int retries = 0;
1856
1857         while (retries < 5) {
1858                 if (!pnn_specified) {
1859                         pnn = find_other_host_for_public_ip(ctdb, addr);
1860                         if (pnn == -1) {
1861                                 return false;
1862                         }
1863                         DEBUG(DEBUG_NOTICE,
1864                               ("Trying to move public IP to node %u\n", pnn));
1865                 }
1866
1867                 if (move_ip(ctdb, addr, pnn) == 0) {
1868                         return true;
1869                 }
1870
1871                 sleep(3);
1872                 retries++;
1873         }
1874
1875         return false;
1876 }
1877
1878
1879 /*
1880   move/failover an ip address to a specific node
1881  */
1882 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1883 {
1884         uint32_t pnn;
1885         ctdb_sock_addr addr;
1886
1887         assert_single_node_only();
1888
1889         if (argc < 2) {
1890                 usage();
1891                 return -1;
1892         }
1893
1894         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1895                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1896                 return -1;
1897         }
1898
1899
1900         if (sscanf(argv[1], "%u", &pnn) != 1) {
1901                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1902                 return -1;
1903         }
1904
1905         if (!try_moveip(ctdb, &addr, pnn)) {
1906                 DEBUG(DEBUG_ERR,("Failed to move IP to node %d.\n", pnn));
1907                 return -1;
1908         }
1909
1910         return 0;
1911 }
1912
1913 static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
1914 {
1915         TDB_DATA data;
1916
1917         data.dptr  = (uint8_t *)&pnn;
1918         data.dsize = sizeof(uint32_t);
1919         if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
1920                 DEBUG(DEBUG_ERR,
1921                       ("Failed to send message to force node %u to be a rebalancing target\n",
1922                        pnn));
1923                 return -1;
1924         }
1925
1926         return 0;
1927 }
1928
1929
1930 /*
1931   rebalance a node by setting it to allow failback and triggering a
1932   takeover run
1933  */
1934 static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
1935 {
1936         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1937         uint32_t *nodes;
1938         uint32_t pnn_mode;
1939         int i, ret;
1940
1941         assert_single_node_only();
1942
1943         if (argc > 1) {
1944                 usage();
1945         }
1946
1947         /* Determine the nodes where IPs need to be reloaded */
1948         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1949                               options.pnn, true, &nodes, &pnn_mode)) {
1950                 ret = -1;
1951                 goto done;
1952         }
1953
1954         for (i = 0; i < talloc_array_length(nodes); i++) {
1955                 if (!rebalance_node(ctdb, nodes[i])) {
1956                         ret = -1;
1957                 }
1958         }
1959
1960 done:
1961         talloc_free(tmp_ctx);
1962         return ret;
1963 }
1964
1965 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1966 {
1967         struct ctdb_public_ip ip;
1968         int ret;
1969         uint32_t *nodes;
1970         uint32_t disable_time;
1971         TDB_DATA data;
1972         struct ctdb_node_map *nodemap=NULL;
1973         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1974
1975         disable_time = 30;
1976         data.dptr  = (uint8_t*)&disable_time;
1977         data.dsize = sizeof(disable_time);
1978         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1979         if (ret != 0) {
1980                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1981                 return -1;
1982         }
1983
1984         ip.pnn  = -1;
1985         ip.addr = *addr;
1986
1987         data.dptr  = (uint8_t *)&ip;
1988         data.dsize = sizeof(ip);
1989
1990         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1991         if (ret != 0) {
1992                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1993                 talloc_free(tmp_ctx);
1994                 return ret;
1995         }
1996
1997         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
1998         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1999                                         nodes, 0,
2000                                         LONGTIMELIMIT(),
2001                                         false, data,
2002                                         NULL, NULL,
2003                                         NULL);
2004         if (ret != 0) {
2005                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
2006                 talloc_free(tmp_ctx);
2007                 return -1;
2008         }
2009
2010         talloc_free(tmp_ctx);
2011         return 0;
2012 }
2013
2014 /*
2015   release an ip form all nodes and have it re-assigned by recd
2016  */
2017 static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
2018 {
2019         ctdb_sock_addr addr;
2020
2021         assert_single_node_only();
2022
2023         if (argc < 1) {
2024                 usage();
2025                 return -1;
2026         }
2027
2028         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2029                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2030                 return -1;
2031         }
2032
2033         if (rebalance_ip(ctdb, &addr) != 0) {
2034                 DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
2035                 return -1;
2036         }
2037
2038         return 0;
2039 }
2040
2041 static int getips_store_callback(void *param, void *data)
2042 {
2043         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
2044         struct ctdb_all_public_ips *ips = param;
2045         int i;
2046
2047         i = ips->num++;
2048         ips->ips[i].pnn  = node_ip->pnn;
2049         ips->ips[i].addr = node_ip->addr;
2050         return 0;
2051 }
2052
2053 static int getips_count_callback(void *param, void *data)
2054 {
2055         uint32_t *count = param;
2056
2057         (*count)++;
2058         return 0;
2059 }
2060
2061 #define IP_KEYLEN       4
2062 static uint32_t *ip_key(ctdb_sock_addr *ip)
2063 {
2064         static uint32_t key[IP_KEYLEN];
2065
2066         bzero(key, sizeof(key));
2067
2068         switch (ip->sa.sa_family) {
2069         case AF_INET:
2070                 key[0]  = ip->ip.sin_addr.s_addr;
2071                 break;
2072         case AF_INET6: {
2073                 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
2074                 key[0]  = s6_a32[3];
2075                 key[1]  = s6_a32[2];
2076                 key[2]  = s6_a32[1];
2077                 key[3]  = s6_a32[0];
2078                 break;
2079         }
2080         default:
2081                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
2082                 return key;
2083         }
2084
2085         return key;
2086 }
2087
2088 static void *add_ip_callback(void *parm, void *data)
2089 {
2090         return parm;
2091 }
2092
2093 static int
2094 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_all_public_ips **ips)
2095 {
2096         struct ctdb_all_public_ips *tmp_ips;
2097         struct ctdb_node_map *nodemap=NULL;
2098         trbt_tree_t *ip_tree;
2099         int i, j, len, ret;
2100         uint32_t count;
2101
2102         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2103         if (ret != 0) {
2104                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2105                 return ret;
2106         }
2107
2108         ip_tree = trbt_create(tmp_ctx, 0);
2109
2110         for(i=0;i<nodemap->num;i++){
2111                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
2112                         continue;
2113                 }
2114                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2115                         continue;
2116                 }
2117
2118                 /* read the public ip list from this node */
2119                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
2120                 if (ret != 0) {
2121                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
2122                         return -1;
2123                 }
2124
2125                 for (j=0; j<tmp_ips->num;j++) {
2126                         struct ctdb_public_ip *node_ip;
2127
2128                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
2129                         node_ip->pnn  = tmp_ips->ips[j].pnn;
2130                         node_ip->addr = tmp_ips->ips[j].addr;
2131
2132                         trbt_insertarray32_callback(ip_tree,
2133                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
2134                                 add_ip_callback,
2135                                 node_ip);
2136                 }
2137                 talloc_free(tmp_ips);
2138         }
2139
2140         /* traverse */
2141         count = 0;
2142         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
2143
2144         len = offsetof(struct ctdb_all_public_ips, ips) +
2145                 count*sizeof(struct ctdb_public_ip);
2146         tmp_ips = talloc_zero_size(tmp_ctx, len);
2147         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
2148
2149         *ips = tmp_ips;
2150
2151         return 0;
2152 }
2153
2154
2155 static void ctdb_every_second(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
2156 {
2157         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2158
2159         event_add_timed(ctdb->ev, ctdb,
2160                                 timeval_current_ofs(1, 0),
2161                                 ctdb_every_second, ctdb);
2162 }
2163
2164 struct srvid_reply_handler_data {
2165         bool done;
2166         bool wait_for_all;
2167         uint32_t *nodes;
2168         const char *srvid_str;
2169 };
2170
2171 static void srvid_broadcast_reply_handler(struct ctdb_context *ctdb,
2172                                          uint64_t srvid,
2173                                          TDB_DATA data,
2174                                          void *private_data)
2175 {
2176         struct srvid_reply_handler_data *d =
2177                 (struct srvid_reply_handler_data *)private_data;
2178         int i;
2179         int32_t ret;
2180
2181         if (data.dsize != sizeof(ret)) {
2182                 DEBUG(DEBUG_ERR, (__location__ " Wrong reply size\n"));
2183                 return;
2184         }
2185
2186         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
2187         ret = *(int32_t *)data.dptr;
2188         if (ret < 0) {
2189                 DEBUG(DEBUG_ERR,
2190                       ("%s failed with result %d\n", d->srvid_str, ret));
2191                 return;
2192         }
2193
2194         if (!d->wait_for_all) {
2195                 d->done = true;
2196                 return;
2197         }
2198
2199         /* Wait for all replies */
2200         d->done = true;
2201         for (i = 0; i < talloc_array_length(d->nodes); i++) {
2202                 if (d->nodes[i] == ret) {
2203                         DEBUG(DEBUG_DEBUG,
2204                               ("%s reply received from node %u\n",
2205                                d->srvid_str, ret));
2206                         d->nodes[i] = -1;
2207                 }
2208                 if (d->nodes[i] != -1) {
2209                         /* Found a node that hasn't yet replied */
2210                         d->done = false;
2211                 }
2212         }
2213 }
2214
2215 /* Broadcast the given SRVID to all connected nodes.  Wait for 1 reply
2216  * or replies from all connected nodes.  arg is the data argument to
2217  * pass in the srvid_request structure - pass 0 if this isn't needed.
2218  */
2219 static int srvid_broadcast(struct ctdb_context *ctdb,
2220                            uint64_t srvid, uint32_t *arg,
2221                            const char *srvid_str, bool wait_for_all)
2222 {
2223         int ret;
2224         TDB_DATA data;
2225         uint32_t pnn;
2226         uint64_t reply_srvid;
2227         struct srvid_request request;
2228         struct srvid_request_data request_data;
2229         struct srvid_reply_handler_data reply_data;
2230         struct timeval tv;
2231
2232         ZERO_STRUCT(request);
2233
2234         /* Time ticks to enable timeouts to be processed */
2235         event_add_timed(ctdb->ev, ctdb,
2236                                 timeval_current_ofs(1, 0),
2237                                 ctdb_every_second, ctdb);
2238
2239         pnn = ctdb_get_pnn(ctdb);
2240         reply_srvid = getpid();
2241
2242         if (arg == NULL) {
2243                 request.pnn = pnn;
2244                 request.srvid = reply_srvid;
2245
2246                 data.dptr = (uint8_t *)&request;
2247                 data.dsize = sizeof(request);
2248         } else {
2249                 request_data.pnn = pnn;
2250                 request_data.srvid = reply_srvid;
2251                 request_data.data = *arg;
2252
2253                 data.dptr = (uint8_t *)&request_data;
2254                 data.dsize = sizeof(request_data);
2255         }
2256
2257         /* Register message port for reply from recovery master */
2258         ctdb_client_set_message_handler(ctdb, reply_srvid,
2259                                         srvid_broadcast_reply_handler,
2260                                         &reply_data);
2261
2262         reply_data.wait_for_all = wait_for_all;
2263         reply_data.nodes = NULL;
2264         reply_data.srvid_str = srvid_str;
2265
2266 again:
2267         reply_data.done = false;
2268
2269         if (wait_for_all) {
2270                 struct ctdb_node_map *nodemap;
2271
2272                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(),
2273                                            CTDB_CURRENT_NODE, ctdb, &nodemap);
2274                 if (ret != 0) {
2275                         DEBUG(DEBUG_ERR,
2276                               ("Unable to get nodemap from current node, try again\n"));
2277                         sleep(1);
2278                         goto again;
2279                 }
2280
2281                 if (reply_data.nodes != NULL) {
2282                         talloc_free(reply_data.nodes);
2283                 }
2284                 reply_data.nodes = list_of_connected_nodes(ctdb, nodemap,
2285                                                            NULL, true);
2286
2287                 talloc_free(nodemap);
2288         }
2289
2290         /* Send to all connected nodes. Only recmaster replies */
2291         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2292                                        srvid, data);
2293         if (ret != 0) {
2294                 /* This can only happen if the socket is closed and
2295                  * there's no way to recover from that, so don't try
2296                  * again.
2297                  */
2298                 DEBUG(DEBUG_ERR,
2299                       ("Failed to send %s request to connected nodes\n",
2300                        srvid_str));
2301                 return -1;
2302         }
2303
2304         tv = timeval_current();
2305         /* This loop terminates the reply is received */
2306         while (timeval_elapsed(&tv) < 5.0 && !reply_data.done) {
2307                 event_loop_once(ctdb->ev);
2308         }
2309
2310         if (!reply_data.done) {
2311                 DEBUG(DEBUG_NOTICE,
2312                       ("Still waiting for confirmation of %s\n", srvid_str));
2313                 sleep(1);
2314                 goto again;
2315         }
2316
2317         ctdb_client_remove_message_handler(ctdb, reply_srvid, &reply_data);
2318
2319         talloc_free(reply_data.nodes);
2320
2321         return 0;
2322 }
2323
2324 static int ipreallocate(struct ctdb_context *ctdb)
2325 {
2326         return srvid_broadcast(ctdb, CTDB_SRVID_TAKEOVER_RUN, NULL,
2327                                "IP reallocation", false);
2328 }
2329
2330
2331 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2332 {
2333         return ipreallocate(ctdb);
2334 }
2335
2336 /*
2337   add a public ip address to a node
2338  */
2339 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
2340 {
2341         int i, ret;
2342         int len, retries = 0;
2343         unsigned mask;
2344         ctdb_sock_addr addr;
2345         struct ctdb_control_ip_iface *pub;
2346         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2347         struct ctdb_all_public_ips *ips;
2348
2349
2350         if (argc != 2) {
2351                 talloc_free(tmp_ctx);
2352                 usage();
2353         }
2354
2355         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
2356                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
2357                 talloc_free(tmp_ctx);
2358                 return -1;
2359         }
2360
2361         /* read the public ip list from the node */
2362         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2363         if (ret != 0) {
2364                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", options.pnn));
2365                 talloc_free(tmp_ctx);
2366                 return -1;
2367         }
2368         for (i=0;i<ips->num;i++) {
2369                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2370                         DEBUG(DEBUG_ERR,("Can not add ip to node. Node already hosts this ip\n"));
2371                         return 0;
2372                 }
2373         }
2374
2375
2376
2377         /* Dont timeout. This command waits for an ip reallocation
2378            which sometimes can take wuite a while if there has
2379            been a recent recovery
2380         */
2381         alarm(0);
2382
2383         len = offsetof(struct ctdb_control_ip_iface, iface) + strlen(argv[1]) + 1;
2384         pub = talloc_size(tmp_ctx, len);
2385         CTDB_NO_MEMORY(ctdb, pub);
2386
2387         pub->addr  = addr;
2388         pub->mask  = mask;
2389         pub->len   = strlen(argv[1])+1;
2390         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
2391
2392         do {
2393                 ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
2394                 if (ret != 0) {
2395                         DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
2396                         sleep(3);
2397                         retries++;
2398                 }
2399         } while (retries < 5 && ret != 0);
2400         if (ret != 0) {
2401                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
2402                 talloc_free(tmp_ctx);
2403                 return ret;
2404         }
2405
2406         if (rebalance_node(ctdb, options.pnn) != 0) {
2407                 DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
2408                 return ret;
2409         }
2410
2411         talloc_free(tmp_ctx);
2412         return 0;
2413 }
2414
2415 /*
2416   add a public ip address to a node
2417  */
2418 static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
2419 {
2420         ctdb_sock_addr addr;
2421         char *iface = NULL;
2422
2423         if (argc != 1) {
2424                 usage();
2425         }
2426
2427         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2428                 printf("Badly formed ip : %s\n", argv[0]);
2429                 return -1;
2430         }
2431
2432         iface = ctdb_sys_find_ifname(&addr);
2433         if (iface == NULL) {
2434                 printf("Failed to get interface name for ip: %s", argv[0]);
2435                 return -1;
2436         }
2437
2438         printf("IP on interface %s\n", iface);
2439
2440         free(iface);
2441
2442         return 0;
2443 }
2444
2445 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
2446
2447 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
2448 {
2449         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2450         struct ctdb_node_map *nodemap=NULL;
2451         struct ctdb_all_public_ips *ips;
2452         int ret, i, j;
2453
2454         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2455         if (ret != 0) {
2456                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
2457                 return ret;
2458         }
2459
2460         /* remove it from the nodes that are not hosting the ip currently */
2461         for(i=0;i<nodemap->num;i++){
2462                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2463                         continue;
2464                 }
2465                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2466                 if (ret != 0) {
2467                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2468                         continue;
2469                 }
2470
2471                 for (j=0;j<ips->num;j++) {
2472                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2473                                 break;
2474                         }
2475                 }
2476                 if (j==ips->num) {
2477                         continue;
2478                 }
2479
2480                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
2481                         continue;
2482                 }
2483
2484                 options.pnn = nodemap->nodes[i].pnn;
2485                 control_delip(ctdb, argc, argv);
2486         }
2487
2488
2489         /* remove it from every node (also the one hosting it) */
2490         for(i=0;i<nodemap->num;i++){
2491                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2492                         continue;
2493                 }
2494                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2495                 if (ret != 0) {
2496                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2497                         continue;
2498                 }
2499
2500                 for (j=0;j<ips->num;j++) {
2501                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2502                                 break;
2503                         }
2504                 }
2505                 if (j==ips->num) {
2506                         continue;
2507                 }
2508
2509                 options.pnn = nodemap->nodes[i].pnn;
2510                 control_delip(ctdb, argc, argv);
2511         }
2512
2513         talloc_free(tmp_ctx);
2514         return 0;
2515 }
2516
2517 /*
2518   delete a public ip address from a node
2519  */
2520 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
2521 {
2522         int i, ret;
2523         ctdb_sock_addr addr;
2524         struct ctdb_control_ip_iface pub;
2525         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2526         struct ctdb_all_public_ips *ips;
2527
2528         if (argc != 1) {
2529                 talloc_free(tmp_ctx);
2530                 usage();
2531         }
2532
2533         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2534                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2535                 return -1;
2536         }
2537
2538         if (options.pnn == CTDB_BROADCAST_ALL) {
2539                 return control_delip_all(ctdb, argc, argv, &addr);
2540         }
2541
2542         pub.addr  = addr;
2543         pub.mask  = 0;
2544         pub.len   = 0;
2545
2546         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2547         if (ret != 0) {
2548                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
2549                 talloc_free(tmp_ctx);
2550                 return ret;
2551         }
2552
2553         for (i=0;i<ips->num;i++) {
2554                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2555                         break;
2556                 }
2557         }
2558
2559         if (i==ips->num) {
2560                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
2561                         ctdb_addr_to_str(&addr)));
2562                 talloc_free(tmp_ctx);
2563                 return -1;
2564         }
2565
2566         /* This is an optimisation.  If this node is hosting the IP
2567          * then try to move it somewhere else without invoking a full
2568          * takeover run.  We don't care if this doesn't work!
2569          */
2570         if (ips->ips[i].pnn == options.pnn) {
2571                 (void) try_moveip(ctdb, &addr, -1);
2572         }
2573
2574         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
2575         if (ret != 0) {
2576                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
2577                 talloc_free(tmp_ctx);
2578                 return ret;
2579         }
2580
2581         talloc_free(tmp_ctx);
2582         return 0;
2583 }
2584
2585 static int kill_tcp_from_file(struct ctdb_context *ctdb,
2586                               int argc, const char **argv)
2587 {
2588         struct ctdb_control_killtcp *killtcp;
2589         int max_entries, current, i;
2590         struct timeval timeout;
2591         char line[128], src[128], dst[128];
2592         int linenum;
2593         TDB_DATA data;
2594         struct client_async_data *async_data;
2595         struct ctdb_client_control_state *state;
2596
2597         if (argc != 0) {
2598                 usage();
2599         }
2600
2601         linenum = 1;
2602         killtcp = NULL;
2603         max_entries = 0;
2604         current = 0;
2605         while (!feof(stdin)) {
2606                 if (fgets(line, sizeof(line), stdin) == NULL) {
2607                         continue;
2608                 }
2609
2610                 /* Silently skip empty lines */
2611                 if (line[0] == '\n') {
2612                         continue;
2613                 }
2614
2615                 if (sscanf(line, "%s %s\n", src, dst) != 2) {
2616                         DEBUG(DEBUG_ERR, ("Bad line [%d]: '%s'\n",
2617                                           linenum, line));
2618                         talloc_free(killtcp);
2619                         return -1;
2620                 }
2621
2622                 if (current >= max_entries) {
2623                         max_entries += 1024;
2624                         killtcp = talloc_realloc(ctdb, killtcp,
2625                                                  struct ctdb_control_killtcp,
2626                                                  max_entries);
2627                         CTDB_NO_MEMORY(ctdb, killtcp);
2628                 }
2629
2630                 if (!parse_ip_port(src, &killtcp[current].src_addr)) {
2631                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2632                                           linenum, src));
2633                         talloc_free(killtcp);
2634                         return -1;
2635                 }
2636
2637                 if (!parse_ip_port(dst, &killtcp[current].dst_addr)) {
2638                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2639                                           linenum, dst));
2640                         talloc_free(killtcp);
2641                         return -1;
2642                 }
2643
2644                 current++;
2645         }
2646
2647         async_data = talloc_zero(ctdb, struct client_async_data);
2648         if (async_data == NULL) {
2649                 talloc_free(killtcp);
2650                 return -1;
2651         }
2652
2653         for (i = 0; i < current; i++) {
2654
2655                 data.dsize = sizeof(struct ctdb_control_killtcp);
2656                 data.dptr  = (unsigned char *)&killtcp[i];
2657
2658                 timeout = TIMELIMIT();
2659                 state = ctdb_control_send(ctdb, options.pnn, 0,
2660                                           CTDB_CONTROL_KILL_TCP, 0, data,
2661                                           async_data, &timeout, NULL);
2662
2663                 if (state == NULL) {
2664                         DEBUG(DEBUG_ERR,
2665                               ("Failed to call async killtcp control to node %u\n",
2666                                options.pnn));
2667                         talloc_free(killtcp);
2668                         return -1;
2669                 }
2670
2671                 ctdb_client_async_add(async_data, state);
2672         }
2673
2674         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2675                 DEBUG(DEBUG_ERR,("killtcp failed\n"));
2676                 talloc_free(killtcp);
2677                 return -1;
2678         }
2679
2680         talloc_free(killtcp);
2681         return 0;
2682 }
2683
2684
2685 /*
2686   kill a tcp connection
2687  */
2688 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2689 {
2690         int ret;
2691         struct ctdb_control_killtcp killtcp;
2692
2693         assert_single_node_only();
2694
2695         if (argc == 0) {
2696                 return kill_tcp_from_file(ctdb, argc, argv);
2697         }
2698
2699         if (argc < 2) {
2700                 usage();
2701         }
2702
2703         if (!parse_ip_port(argv[0], &killtcp.src_addr)) {
2704                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2705                 return -1;
2706         }
2707
2708         if (!parse_ip_port(argv[1], &killtcp.dst_addr)) {
2709                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2710                 return -1;
2711         }
2712
2713         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
2714         if (ret != 0) {
2715                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
2716                 return ret;
2717         }
2718
2719         return 0;
2720 }
2721
2722
2723 /*
2724   send a gratious arp
2725  */
2726 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
2727 {
2728         int ret;
2729         ctdb_sock_addr addr;
2730
2731         assert_single_node_only();
2732
2733         if (argc < 2) {
2734                 usage();
2735         }
2736
2737         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2738                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
2739                 return -1;
2740         }
2741
2742         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
2743         if (ret != 0) {
2744                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
2745                 return ret;
2746         }
2747
2748         return 0;
2749 }
2750
2751 /*
2752   register a server id
2753  */
2754 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2755 {
2756         int ret;
2757         struct ctdb_server_id server_id;
2758
2759         if (argc < 3) {
2760                 usage();
2761         }
2762
2763         server_id.pnn       = strtoul(argv[0], NULL, 0);
2764         server_id.type      = strtoul(argv[1], NULL, 0);
2765         server_id.server_id = strtoul(argv[2], NULL, 0);
2766
2767         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
2768         if (ret != 0) {
2769                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
2770                 return ret;
2771         }
2772         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
2773         sleep(999);
2774         return -1;
2775 }
2776
2777 /*
2778   unregister a server id
2779  */
2780 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2781 {
2782         int ret;
2783         struct ctdb_server_id server_id;
2784
2785         if (argc < 3) {
2786                 usage();
2787         }
2788
2789         server_id.pnn       = strtoul(argv[0], NULL, 0);
2790         server_id.type      = strtoul(argv[1], NULL, 0);
2791         server_id.server_id = strtoul(argv[2], NULL, 0);
2792
2793         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
2794         if (ret != 0) {
2795                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
2796                 return ret;
2797         }
2798         return -1;
2799 }
2800
2801 /*
2802   check if a server id exists
2803  */
2804 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2805 {
2806         uint32_t status = 0;
2807         int ret;
2808         struct ctdb_server_id server_id;
2809
2810         if (argc < 3) {
2811                 usage();
2812         }
2813
2814         server_id.pnn       = strtoul(argv[0], NULL, 0);
2815         server_id.type      = strtoul(argv[1], NULL, 0);
2816         server_id.server_id = strtoul(argv[2], NULL, 0);
2817
2818         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
2819         if (ret != 0) {
2820                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
2821                 return ret;
2822         }
2823
2824         if (status) {
2825                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
2826         } else {
2827                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
2828         }
2829         return 0;
2830 }
2831
2832 /*
2833   get a list of all server ids that are registered on a node
2834  */
2835 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
2836 {
2837         int i, ret;
2838         struct ctdb_server_id_list *server_ids;
2839
2840         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
2841         if (ret != 0) {
2842                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
2843                 return ret;
2844         }
2845
2846         for (i=0; i<server_ids->num; i++) {
2847                 printf("Server id %d:%d:%d\n",
2848                         server_ids->server_ids[i].pnn,
2849                         server_ids->server_ids[i].type,
2850                         server_ids->server_ids[i].server_id);
2851         }
2852
2853         return -1;
2854 }
2855
2856 /*
2857   check if a server id exists
2858  */
2859 static int check_srvids(struct ctdb_context *ctdb, int argc, const char **argv)
2860 {
2861         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
2862         uint64_t *ids;
2863         uint8_t *result;
2864         int i;
2865
2866         if (argc < 1) {
2867                 talloc_free(tmp_ctx);
2868                 usage();
2869         }
2870
2871         ids    = talloc_array(tmp_ctx, uint64_t, argc);
2872         result = talloc_array(tmp_ctx, uint8_t, argc);
2873
2874         for (i = 0; i < argc; i++) {
2875                 ids[i] = strtoull(argv[i], NULL, 0);
2876         }
2877
2878         if (!ctdb_client_check_message_handlers(ctdb, ids, argc, result)) {
2879                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n",
2880                                   options.pnn));
2881                 talloc_free(tmp_ctx);
2882                 return -1;
2883         }
2884
2885         for (i=0; i < argc; i++) {
2886                 printf("Server id %d:%llu %s\n", options.pnn, (long long)ids[i],
2887                        result[i] ? "exists" : "does not exist");
2888         }
2889
2890         talloc_free(tmp_ctx);
2891         return 0;
2892 }
2893
2894 /*
2895   send a tcp tickle ack
2896  */
2897 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2898 {
2899         int ret;
2900         ctdb_sock_addr  src, dst;
2901
2902         if (argc < 2) {
2903                 usage();
2904         }
2905
2906         if (!parse_ip_port(argv[0], &src)) {
2907                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2908                 return -1;
2909         }
2910
2911         if (!parse_ip_port(argv[1], &dst)) {
2912                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2913                 return -1;
2914         }
2915
2916         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
2917         if (ret==0) {
2918                 return 0;
2919         }
2920         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
2921
2922         return -1;
2923 }
2924
2925
2926 /*
2927   display public ip status
2928  */
2929 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
2930 {
2931         int i, ret;
2932         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2933         struct ctdb_all_public_ips *ips;
2934
2935         if (options.pnn == CTDB_BROADCAST_ALL) {
2936                 /* read the list of public ips from all nodes */
2937                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
2938         } else {
2939                 /* read the public ip list from this node */
2940                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2941         }
2942         if (ret != 0) {
2943                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
2944                 talloc_free(tmp_ctx);
2945                 return ret;
2946         }
2947
2948         if (options.machinereadable){
2949                 printm(":Public IP:Node:");
2950                 if (options.verbose){
2951                         printm("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
2952                 }
2953                 printm("\n");
2954         } else {
2955                 if (options.pnn == CTDB_BROADCAST_ALL) {
2956                         printf("Public IPs on ALL nodes\n");
2957                 } else {
2958                         printf("Public IPs on node %u\n", options.pnn);
2959                 }
2960         }
2961
2962         for (i=1;i<=ips->num;i++) {
2963                 struct ctdb_control_public_ip_info *info = NULL;
2964                 int32_t pnn;
2965                 char *aciface = NULL;
2966                 char *avifaces = NULL;
2967                 char *cifaces = NULL;
2968
2969                 if (options.pnn == CTDB_BROADCAST_ALL) {
2970                         pnn = ips->ips[ips->num-i].pnn;
2971                 } else {
2972                         pnn = options.pnn;
2973                 }
2974
2975                 if (pnn != -1) {
2976                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
2977                                                    &ips->ips[ips->num-i].addr, &info);
2978                 } else {
2979                         ret = -1;
2980                 }
2981
2982                 if (ret == 0) {
2983                         int j;
2984                         for (j=0; j < info->num; j++) {
2985                                 if (cifaces == NULL) {
2986                                         cifaces = talloc_strdup(info,
2987                                                                 info->ifaces[j].name);
2988                                 } else {
2989                                         cifaces = talloc_asprintf_append(cifaces,
2990                                                                          ",%s",
2991                                                                          info->ifaces[j].name);
2992                                 }
2993
2994                                 if (info->active_idx == j) {
2995                                         aciface = info->ifaces[j].name;
2996                                 }
2997
2998                                 if (info->ifaces[j].link_state == 0) {
2999                                         continue;
3000                                 }
3001
3002                                 if (avifaces == NULL) {
3003                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
3004                                 } else {
3005                                         avifaces = talloc_asprintf_append(avifaces,
3006                                                                           ",%s",
3007                                                                           info->ifaces[j].name);
3008                                 }
3009                         }
3010                 }
3011
3012                 if (options.machinereadable){
3013                         printm(":%s:%d:",
3014                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3015                                 ips->ips[ips->num-i].pnn);
3016                         if (options.verbose){
3017                                 printm("%s:%s:%s:",
3018                                         aciface?aciface:"",
3019                                         avifaces?avifaces:"",
3020                                         cifaces?cifaces:"");
3021                         }
3022                         printf("\n");
3023                 } else {
3024                         if (options.verbose) {
3025                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
3026                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3027                                         ips->ips[ips->num-i].pnn,
3028                                         aciface?aciface:"",
3029                                         avifaces?avifaces:"",
3030                                         cifaces?cifaces:"");
3031                         } else {
3032                                 printf("%s %d\n",
3033                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3034                                         ips->ips[ips->num-i].pnn);
3035                         }
3036                 }
3037                 talloc_free(info);
3038         }
3039
3040         talloc_free(tmp_ctx);
3041         return 0;
3042 }
3043
3044 /*
3045   public ip info
3046  */
3047 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
3048 {
3049         int i, ret;
3050         ctdb_sock_addr addr;
3051         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3052         struct ctdb_control_public_ip_info *info;
3053
3054         if (argc != 1) {
3055                 talloc_free(tmp_ctx);
3056                 usage();
3057         }
3058
3059         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
3060                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
3061                 return -1;
3062         }
3063
3064         /* read the public ip info from this node */
3065         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
3066                                            tmp_ctx, &addr, &info);
3067         if (ret != 0) {
3068                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
3069                                   argv[0], options.pnn));
3070                 talloc_free(tmp_ctx);
3071                 return ret;
3072         }
3073
3074         printf("Public IP[%s] info on node %u\n",
3075                ctdb_addr_to_str(&info->ip.addr),
3076                options.pnn);
3077
3078         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
3079                ctdb_addr_to_str(&info->ip.addr),
3080                info->ip.pnn, info->num);
3081
3082         for (i=0; i<info->num; i++) {
3083                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
3084
3085                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
3086                        i+1, info->ifaces[i].name,
3087                        info->ifaces[i].link_state?"up":"down",
3088                        (unsigned int)info->ifaces[i].references,
3089                        (i==info->active_idx)?" (active)":"");
3090         }
3091
3092         talloc_free(tmp_ctx);
3093         return 0;
3094 }
3095
3096 /*
3097   display interfaces status
3098  */
3099 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
3100 {
3101         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3102         int i;
3103         struct ctdb_control_get_ifaces *ifaces;
3104         int ret;
3105
3106         /* read the public ip list from this node */
3107         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ifaces);
3108         if (ret != 0) {
3109                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
3110                                   options.pnn));
3111                 talloc_free(tmp_ctx);
3112                 return -1;
3113         }
3114
3115         if (options.machinereadable){
3116                 printm(":Name:LinkStatus:References:\n");
3117         } else {
3118                 printf("Interfaces on node %u\n", options.pnn);
3119         }
3120
3121         for (i=0; i<ifaces->num; i++) {
3122                 if (options.machinereadable){
3123                         printm(":%s:%s:%u:\n",
3124                                ifaces->ifaces[i].name,
3125                                ifaces->ifaces[i].link_state?"1":"0",
3126                                (unsigned int)ifaces->ifaces[i].references);
3127                 } else {
3128                         printf("name:%s link:%s references:%u\n",
3129                                ifaces->ifaces[i].name,
3130                                ifaces->ifaces[i].link_state?"up":"down",
3131                                (unsigned int)ifaces->ifaces[i].references);
3132                 }
3133         }
3134
3135         talloc_free(tmp_ctx);
3136         return 0;
3137 }
3138
3139
3140 /*
3141   set link status of an interface
3142  */
3143 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
3144 {
3145         int ret;
3146         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3147         struct ctdb_control_iface_info info;
3148
3149         ZERO_STRUCT(info);
3150
3151         if (argc != 2) {
3152                 usage();
3153         }
3154
3155         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
3156                 DEBUG(DEBUG_ERR, ("interfaces name '%s' too long\n",
3157                                   argv[0]));
3158                 talloc_free(tmp_ctx);
3159                 return -1;
3160         }
3161         strcpy(info.name, argv[0]);
3162
3163         if (strcmp(argv[1], "up") == 0) {
3164                 info.link_state = 1;
3165         } else if (strcmp(argv[1], "down") == 0) {
3166                 info.link_state = 0;
3167         } else {
3168                 DEBUG(DEBUG_ERR, ("link state invalid '%s' should be 'up' or 'down'\n",
3169                                   argv[1]));
3170                 talloc_free(tmp_ctx);
3171                 return -1;
3172         }
3173
3174         /* read the public ip list from this node */
3175         ret = ctdb_ctrl_set_iface_link(ctdb, TIMELIMIT(), options.pnn,
3176                                    tmp_ctx, &info);
3177         if (ret != 0) {
3178                 DEBUG(DEBUG_ERR, ("Unable to set link state for interfaces %s node %u\n",
3179                                   argv[0], options.pnn));
3180                 talloc_free(tmp_ctx);
3181                 return ret;
3182         }
3183
3184         talloc_free(tmp_ctx);
3185         return 0;
3186 }
3187
3188 /*
3189   display pid of a ctdb daemon
3190  */
3191 static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
3192 {
3193         uint32_t pid;
3194         int ret;
3195
3196         ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.pnn, &pid);
3197         if (ret != 0) {
3198                 DEBUG(DEBUG_ERR, ("Unable to get daemon pid from node %u\n", options.pnn));
3199                 return ret;
3200         }
3201         printf("Pid:%d\n", pid);
3202
3203         return 0;
3204 }
3205
3206 typedef bool update_flags_handler_t(struct ctdb_context *ctdb, void *data);
3207
3208 static int update_flags_and_ipreallocate(struct ctdb_context *ctdb,
3209                                               void *data,
3210                                               update_flags_handler_t handler,
3211                                               uint32_t flag,
3212                                               const char *desc,
3213                                               bool set_flag)
3214 {
3215         struct ctdb_node_map *nodemap = NULL;
3216         bool flag_is_set;
3217         int ret;
3218
3219         /* Check if the node is already in the desired state */
3220         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3221         if (ret != 0) {
3222                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3223                 exit(10);
3224         }
3225         flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3226         if (set_flag == flag_is_set) {
3227                 DEBUG(DEBUG_NOTICE, ("Node %d is %s %s\n", options.pnn,
3228                                      (set_flag ? "already" : "not"), desc));
3229                 return 0;
3230         }
3231
3232         do {
3233                 if (!handler(ctdb, data)) {
3234                         DEBUG(DEBUG_WARNING,
3235                               ("Failed to send control to set state %s on node %u, try again\n",
3236                                desc, options.pnn));
3237                 }
3238
3239                 sleep(1);
3240
3241                 /* Read the nodemap and verify the change took effect.
3242                  * Even if the above control/hanlder timed out then it
3243                  * could still have worked!
3244                  */
3245                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
3246                                          ctdb, &nodemap);
3247                 if (ret != 0) {
3248                         DEBUG(DEBUG_WARNING,
3249                               ("Unable to get nodemap from local node, try again\n"));
3250                 }
3251                 flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3252         } while (nodemap == NULL || (set_flag != flag_is_set));
3253
3254         return ipreallocate(ctdb);
3255 }
3256
3257 /* Administratively disable a node */
3258 static bool update_flags_disabled(struct ctdb_context *ctdb, void *data)
3259 {
3260         int ret;
3261
3262         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3263                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
3264         return ret == 0;
3265 }
3266
3267 static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
3268 {
3269         return update_flags_and_ipreallocate(ctdb, NULL,
3270                                                   update_flags_disabled,
3271                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3272                                                   "disabled",
3273                                                   true /* set_flag*/);
3274 }
3275
3276 /* Administratively re-enable a node */
3277 static bool update_flags_not_disabled(struct ctdb_context *ctdb, void *data)
3278 {
3279         int ret;
3280
3281         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3282                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
3283         return ret == 0;
3284 }
3285
3286 static int control_enable(struct ctdb_context *ctdb,  int argc, const char **argv)
3287 {
3288         return update_flags_and_ipreallocate(ctdb, NULL,
3289                                                   update_flags_not_disabled,
3290                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3291                                                   "disabled",
3292                                                   false /* set_flag*/);
3293 }
3294
3295 /* Stop a node */
3296 static bool update_flags_stopped(struct ctdb_context *ctdb, void *data)
3297 {
3298         int ret;
3299
3300         ret = ctdb_ctrl_stop_node(ctdb, TIMELIMIT(), options.pnn);
3301
3302         return ret == 0;
3303 }
3304
3305 static int control_stop(struct ctdb_context *ctdb, int argc, const char **argv)
3306 {
3307         return update_flags_and_ipreallocate(ctdb, NULL,
3308                                                   update_flags_stopped,
3309                                                   NODE_FLAGS_STOPPED,
3310                                                   "stopped",
3311                                                   true /* set_flag*/);
3312 }
3313
3314 /* Continue a stopped node */
3315 static bool update_flags_not_stopped(struct ctdb_context *ctdb, void *data)
3316 {
3317         int ret;
3318
3319         ret = ctdb_ctrl_continue_node(ctdb, TIMELIMIT(), options.pnn);
3320
3321         return ret == 0;
3322 }
3323
3324 static int control_continue(struct ctdb_context *ctdb, int argc, const char **argv)
3325 {
3326         return update_flags_and_ipreallocate(ctdb, NULL,
3327                                                   update_flags_not_stopped,
3328                                                   NODE_FLAGS_STOPPED,
3329                                                   "stopped",
3330                                                   false /* set_flag */);
3331 }
3332
3333 static uint32_t get_generation(struct ctdb_context *ctdb)
3334 {
3335         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3336         struct ctdb_vnn_map *vnnmap=NULL;
3337         int ret;
3338         uint32_t generation;
3339
3340         /* wait until the recmaster is not in recovery mode */
3341         while (1) {
3342                 uint32_t recmode, recmaster;
3343
3344                 if (vnnmap != NULL) {
3345                         talloc_free(vnnmap);
3346                         vnnmap = NULL;
3347                 }
3348
3349                 /* get the recmaster */
3350                 ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), CTDB_CURRENT_NODE, &recmaster);
3351                 if (ret != 0) {
3352                         DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
3353                         talloc_free(tmp_ctx);
3354                         exit(10);
3355                 }
3356
3357                 /* get recovery mode */
3358                 ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), recmaster, &recmode);
3359                 if (ret != 0) {
3360                         DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
3361                         talloc_free(tmp_ctx);
3362                         exit(10);
3363                 }
3364
3365                 /* get the current generation number */
3366                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), recmaster, tmp_ctx, &vnnmap);
3367                 if (ret != 0) {
3368                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from recmaster (%u)\n", recmaster));
3369                         talloc_free(tmp_ctx);
3370                         exit(10);
3371                 }
3372
3373                 if ((recmode == CTDB_RECOVERY_NORMAL) && (vnnmap->generation != 1)) {
3374                         generation = vnnmap->generation;
3375                         talloc_free(tmp_ctx);
3376                         return generation;
3377                 }
3378                 sleep(1);
3379         }
3380 }
3381
3382 /* Ban a node */
3383 static bool update_state_banned(struct ctdb_context *ctdb, void *data)
3384 {
3385         struct ctdb_ban_time *bantime = (struct ctdb_ban_time *)data;
3386         int ret;
3387
3388         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, bantime);
3389
3390         return ret == 0;
3391 }
3392
3393 static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
3394 {
3395         struct ctdb_ban_time bantime;
3396
3397         if (argc < 1) {
3398                 usage();
3399         }
3400
3401         bantime.pnn  = options.pnn;
3402         bantime.time = strtoul(argv[0], NULL, 0);
3403
3404         if (bantime.time == 0) {
3405                 DEBUG(DEBUG_ERR, ("Invalid ban time specified - must be >0\n"));
3406                 return -1;
3407         }
3408
3409         return update_flags_and_ipreallocate(ctdb, &bantime,
3410                                                   update_state_banned,
3411                                                   NODE_FLAGS_BANNED,
3412                                                   "banned",
3413                                                   true /* set_flag*/);
3414 }
3415
3416
3417 /* Unban a node */
3418 static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
3419 {
3420         struct ctdb_ban_time bantime;
3421
3422         bantime.pnn  = options.pnn;
3423         bantime.time = 0;
3424
3425         return update_flags_and_ipreallocate(ctdb, &bantime,
3426                                                   update_state_banned,
3427                                                   NODE_FLAGS_BANNED,
3428                                                   "banned",
3429                                                   false /* set_flag*/);
3430 }
3431
3432 /*
3433   show ban information for a node
3434  */
3435 static int control_showban(struct ctdb_context *ctdb, int argc, const char **argv)
3436 {
3437         int ret;
3438         struct ctdb_node_map *nodemap=NULL;
3439         struct ctdb_ban_time *bantime;
3440
3441         /* verify the node exists */
3442         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3443         if (ret != 0) {
3444                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3445                 return ret;
3446         }
3447
3448         ret = ctdb_ctrl_get_ban(ctdb, TIMELIMIT(), options.pnn, ctdb, &bantime);
3449         if (ret != 0) {
3450                 DEBUG(DEBUG_ERR,("Showing ban info for node %d failed.\n", options.pnn));
3451                 return -1;
3452         }
3453
3454         if (bantime->time == 0) {
3455                 printf("Node %u is not banned\n", bantime->pnn);
3456         } else {
3457                 printf("Node %u is banned, %d seconds remaining\n",
3458                        bantime->pnn, bantime->time);
3459         }
3460
3461         return 0;
3462 }
3463
3464 /*
3465   shutdown a daemon
3466  */
3467 static int control_shutdown(struct ctdb_context *ctdb, int argc, const char **argv)
3468 {
3469         int ret;
3470
3471         ret = ctdb_ctrl_shutdown(ctdb, TIMELIMIT(), options.pnn);
3472         if (ret != 0) {
3473                 DEBUG(DEBUG_ERR, ("Unable to shutdown node %u\n", options.pnn));
3474                 return ret;
3475         }
3476
3477         return 0;
3478 }
3479
3480 /*
3481   trigger a recovery
3482  */
3483 static int control_recover(struct ctdb_context *ctdb, int argc, const char **argv)
3484 {
3485         int ret;
3486         uint32_t generation, next_generation;
3487
3488         /* record the current generation number */
3489         generation = get_generation(ctdb);
3490
3491         ret = ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3492         if (ret != 0) {
3493                 DEBUG(DEBUG_ERR, ("Unable to set recovery mode\n"));
3494                 return ret;
3495         }
3496
3497         /* wait until we are in a new generation */
3498         while (1) {
3499                 next_generation = get_generation(ctdb);
3500                 if (next_generation != generation) {
3501                         return 0;
3502                 }
3503                 sleep(1);
3504         }
3505
3506         return 0;
3507 }
3508
3509
3510 /*
3511   display monitoring mode of a remote node
3512  */
3513 static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
3514 {
3515         uint32_t monmode;
3516         int ret;
3517
3518         ret = ctdb_ctrl_getmonmode(ctdb, TIMELIMIT(), options.pnn, &monmode);
3519         if (ret != 0) {
3520                 DEBUG(DEBUG_ERR, ("Unable to get monmode from node %u\n", options.pnn));
3521                 return ret;
3522         }
3523         if (!options.machinereadable){
3524                 printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode);
3525         } else {
3526                 printm(":mode:\n");
3527                 printm(":%d:\n",monmode);
3528         }
3529         return 0;
3530 }
3531
3532
3533 /*
3534   display capabilities of a remote node
3535  */
3536 static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const char **argv)
3537 {
3538         uint32_t capabilities;
3539         int ret;
3540
3541         ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), options.pnn, &capabilities);
3542         if (ret != 0) {
3543                 DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", options.pnn));
3544                 return -1;
3545         }
3546
3547         if (!options.machinereadable){
3548                 printf("RECMASTER: %s\n", (capabilities&CTDB_CAP_RECMASTER)?"YES":"NO");
3549                 printf("LMASTER: %s\n", (capabilities&CTDB_CAP_LMASTER)?"YES":"NO");
3550                 printf("LVS: %s\n", (capabilities&CTDB_CAP_LVS)?"YES":"NO");
3551                 printf("NATGW: %s\n", (capabilities&CTDB_CAP_NATGW)?"YES":"NO");
3552         } else {
3553                 printm(":RECMASTER:LMASTER:LVS:NATGW:\n");
3554                 printm(":%d:%d:%d:%d:\n",
3555                         !!(capabilities&CTDB_CAP_RECMASTER),
3556                         !!(capabilities&CTDB_CAP_LMASTER),
3557                         !!(capabilities&CTDB_CAP_LVS),
3558                         !!(capabilities&CTDB_CAP_NATGW));
3559         }
3560         return 0;
3561 }
3562
3563 /*
3564   display lvs configuration
3565  */
3566
3567 static uint32_t lvs_exclude_flags[] = {
3568         /* Look for a nice healthy node */
3569         NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED,
3570         /* If not found, an UNHEALTHY node will do */
3571         NODE_FLAGS_INACTIVE|NODE_FLAGS_PERMANENTLY_DISABLED,
3572         0,
3573 };
3574
3575 static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
3576 {
3577         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3578         struct ctdb_node_map *orig_nodemap=NULL;
3579         struct ctdb_node_map *nodemap;
3580         int i, ret;
3581
3582         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3583                                    tmp_ctx, &orig_nodemap);
3584         if (ret != 0) {
3585                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3586                 talloc_free(tmp_ctx);
3587                 return -1;
3588         }
3589
3590         nodemap = filter_nodemap_by_capabilities(ctdb, orig_nodemap,
3591                                                  CTDB_CAP_LVS, false);
3592         if (nodemap == NULL) {
3593                 /* No memory */
3594                 ret = -1;
3595                 goto done;
3596         }
3597
3598         ret = 0;
3599
3600         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3601                 struct ctdb_node_map *t =
3602                         filter_nodemap_by_flags(ctdb, nodemap,
3603                                                 lvs_exclude_flags[i]);
3604                 if (t == NULL) {
3605                         /* No memory */
3606                         ret = -1;
3607                         goto done;
3608                 }
3609                 if (t->num > 0) {
3610                         /* At least 1 node without excluded flags */
3611                         int j;
3612                         for (j = 0; j < t->num; j++) {
3613                                 printf("%d:%s\n", t->nodes[j].pnn,
3614                                        ctdb_addr_to_str(&t->nodes[j].addr));
3615                         }
3616                         goto done;
3617                 }
3618                 talloc_free(t);
3619         }
3620 done:
3621         talloc_free(tmp_ctx);
3622         return ret;
3623 }
3624
3625 /*
3626   display who is the lvs master
3627  */
3628 static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **argv)
3629 {
3630         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3631         struct ctdb_node_map *nodemap=NULL;
3632         int i, ret;
3633
3634         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn,
3635                                    tmp_ctx, &nodemap);
3636         if (ret != 0) {
3637                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3638                 talloc_free(tmp_ctx);
3639                 return -1;
3640         }
3641
3642         for (i = 0; lvs_exclude_flags[i] != 0; i++) {
3643                 struct ctdb_node_map *t =
3644                         filter_nodemap_by_flags(ctdb, nodemap,
3645                                                 lvs_exclude_flags[i]);
3646                 if (t == NULL) {
3647                         /* No memory */
3648                         ret = -1;
3649                         goto done;
3650                 }
3651                 if (t->num > 0) {
3652                         struct ctdb_node_map *n;
3653                         n = filter_nodemap_by_capabilities(ctdb,
3654                                                            t,
3655                                                            CTDB_CAP_LVS,
3656                                                            true);
3657                         if (n == NULL) {
3658                                 /* No memory */
3659                                 ret = -1;
3660                                 goto done;
3661                         }
3662                         if (n->num > 0) {
3663                                 ret = 0;
3664                                 if (options.machinereadable) {
3665                                         printm("%d\n", n->nodes[0].pnn);
3666                                 } else {
3667                                         printf("Node %d is LVS master\n", n->nodes[0].pnn);
3668                                 }
3669                                 goto done;
3670                         }
3671                 }
3672                 talloc_free(t);
3673         }
3674
3675         printf("There is no LVS master\n");
3676         ret = 255;
3677 done:
3678         talloc_free(tmp_ctx);
3679         return ret;
3680 }
3681
3682 /*
3683   disable monitoring on a  node
3684  */
3685 static int control_disable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3686 {
3687
3688         int ret;
3689
3690         ret = ctdb_ctrl_disable_monmode(ctdb, TIMELIMIT(), options.pnn);
3691         if (ret != 0) {
3692                 DEBUG(DEBUG_ERR, ("Unable to disable monmode on node %u\n", options.pnn));
3693                 return ret;
3694         }
3695         printf("Monitoring mode:%s\n","DISABLED");
3696
3697         return 0;
3698 }
3699
3700 /*
3701   enable monitoring on a  node
3702  */
3703 static int control_enable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
3704 {
3705
3706         int ret;
3707
3708         ret = ctdb_ctrl_enable_monmode(ctdb, TIMELIMIT(), options.pnn);
3709         if (ret != 0) {
3710                 DEBUG(DEBUG_ERR, ("Unable to enable monmode on node %u\n", options.pnn));
3711                 return ret;
3712         }
3713         printf("Monitoring mode:%s\n","ACTIVE");
3714
3715         return 0;
3716 }
3717
3718 /*
3719   display remote list of keys/data for a db
3720  */
3721 static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
3722 {
3723         const char *db_name;
3724         struct ctdb_db_context *ctdb_db;
3725         int ret;
3726         struct ctdb_dump_db_context c;
3727         uint8_t flags;
3728
3729         if (argc < 1) {
3730                 usage();
3731         }
3732
3733         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3734                 return -1;
3735         }
3736
3737         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3738         if (ctdb_db == NULL) {
3739                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3740                 return -1;
3741         }
3742
3743         if (options.printlmaster) {
3744                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn,
3745                                           ctdb, &ctdb->vnn_map);
3746                 if (ret != 0) {
3747                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
3748                                           options.pnn));
3749                         return ret;
3750                 }
3751         }
3752
3753         ZERO_STRUCT(c);
3754         c.f = stdout;
3755         c.printemptyrecords = (bool)options.printemptyrecords;
3756         c.printdatasize = (bool)options.printdatasize;
3757         c.printlmaster = (bool)options.printlmaster;
3758         c.printhash = (bool)options.printhash;
3759         c.printrecordflags = (bool)options.printrecordflags;
3760
3761         /* traverse and dump the cluster tdb */
3762         ret = ctdb_dump_db(ctdb_db, &c);
3763         if (ret == -1) {
3764                 DEBUG(DEBUG_ERR, ("Unable to dump database\n"));
3765                 DEBUG(DEBUG_ERR, ("Maybe try 'ctdb getdbstatus %s'"
3766                                   " and 'ctdb getvar AllowUnhealthyDBRead'\n",
3767                                   db_name));
3768                 return -1;
3769         }
3770         talloc_free(ctdb_db);
3771
3772         printf("Dumped %d records\n", ret);
3773         return 0;
3774 }
3775
3776 struct cattdb_data {
3777         struct ctdb_context *ctdb;
3778         uint32_t count;
3779 };
3780
3781 static int cattdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private_data)
3782 {
3783         struct cattdb_data *d = private_data;
3784         struct ctdb_dump_db_context c;
3785
3786         d->count++;
3787
3788         ZERO_STRUCT(c);
3789         c.f = stdout;
3790         c.printemptyrecords = (bool)options.printemptyrecords;
3791         c.printdatasize = (bool)options.printdatasize;
3792         c.printlmaster = false;
3793         c.printhash = (bool)options.printhash;
3794         c.printrecordflags = true;
3795
3796         return ctdb_dumpdb_record(d->ctdb, key, data, &c);
3797 }
3798
3799 /*
3800   cat the local tdb database using same format as catdb
3801  */
3802 static int control_cattdb(struct ctdb_context *ctdb, int argc, const char **argv)
3803 {
3804         const char *db_name;
3805         struct ctdb_db_context *ctdb_db;
3806         struct cattdb_data d;
3807         uint8_t flags;
3808
3809         if (argc < 1) {
3810                 usage();
3811         }
3812
3813         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3814                 return -1;
3815         }
3816
3817         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3818         if (ctdb_db == NULL) {
3819                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3820                 return -1;
3821         }
3822
3823         /* traverse the local tdb */
3824         d.count = 0;
3825         d.ctdb  = ctdb;
3826         if (tdb_traverse_read(ctdb_db->ltdb->tdb, cattdb_traverse, &d) == -1) {
3827                 printf("Failed to cattdb data\n");
3828                 exit(10);
3829         }
3830         talloc_free(ctdb_db);
3831
3832         printf("Dumped %d records\n", d.count);
3833         return 0;
3834 }
3835
3836 /*
3837   display the content of a database key
3838  */
3839 static int control_readkey(struct ctdb_context *ctdb, int argc, const char **argv)
3840 {
3841         const char *db_name;
3842         struct ctdb_db_context *ctdb_db;
3843         struct ctdb_record_handle *h;
3844         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3845         TDB_DATA key, data;
3846         uint8_t flags;
3847
3848         if (argc < 2) {
3849                 usage();
3850         }
3851
3852         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3853                 return -1;
3854         }
3855
3856         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3857         if (ctdb_db == NULL) {
3858                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3859                 return -1;
3860         }
3861
3862         key.dptr  = discard_const(argv[1]);
3863         key.dsize = strlen((char *)key.dptr);
3864
3865         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3866         if (h == NULL) {
3867                 printf("Failed to fetch record '%s' on node %d\n",
3868                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3869                 talloc_free(tmp_ctx);
3870                 exit(10);
3871         }
3872
3873         printf("Data: size:%d ptr:[%.*s]\n", (int)data.dsize, (int)data.dsize, data.dptr);
3874
3875         talloc_free(tmp_ctx);
3876         talloc_free(ctdb_db);
3877         return 0;
3878 }
3879
3880 /*
3881   display the content of a database key
3882  */
3883 static int control_writekey(struct ctdb_context *ctdb, int argc, const char **argv)
3884 {
3885         const char *db_name;
3886         struct ctdb_db_context *ctdb_db;
3887         struct ctdb_record_handle *h;
3888         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3889         TDB_DATA key, data;
3890         uint8_t flags;
3891
3892         if (argc < 3) {
3893                 usage();
3894         }
3895
3896         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3897                 return -1;
3898         }
3899
3900         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
3901         if (ctdb_db == NULL) {
3902                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3903                 return -1;
3904         }
3905
3906         key.dptr  = discard_const(argv[1]);
3907         key.dsize = strlen((char *)key.dptr);
3908
3909         h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
3910         if (h == NULL) {
3911                 printf("Failed to fetch record '%s' on node %d\n",
3912                         (const char *)key.dptr, ctdb_get_pnn(ctdb));
3913                 talloc_free(tmp_ctx);
3914                 exit(10);
3915         }
3916
3917         data.dptr  = discard_const(argv[2]);
3918         data.dsize = strlen((char *)data.dptr);
3919
3920         if (ctdb_record_store(h, data) != 0) {
3921                 printf("Failed to store record\n");
3922         }
3923
3924         talloc_free(h);
3925         talloc_free(tmp_ctx);
3926         talloc_free(ctdb_db);
3927         return 0;
3928 }
3929
3930 /*
3931   fetch a record from a persistent database
3932  */
3933 static int control_pfetch(struct ctdb_context *ctdb, int argc, const char **argv)
3934 {
3935         const char *db_name;
3936         struct ctdb_db_context *ctdb_db;
3937         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3938         struct ctdb_transaction_handle *h;
3939         TDB_DATA key, data;
3940         int fd, ret;
3941         bool persistent;
3942         uint8_t flags;
3943
3944         if (argc < 2) {
3945                 talloc_free(tmp_ctx);
3946                 usage();
3947         }
3948
3949         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
3950                 talloc_free(tmp_ctx);
3951                 return -1;
3952         }
3953
3954         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
3955         if (!persistent) {
3956                 DEBUG(DEBUG_ERR,("Database '%s' is not persistent\n", db_name));
3957                 talloc_free(tmp_ctx);
3958                 return -1;
3959         }
3960
3961         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
3962         if (ctdb_db == NULL) {
3963                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3964                 talloc_free(tmp_ctx);
3965                 return -1;
3966         }
3967
3968         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
3969         if (h == NULL) {
3970                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
3971                 talloc_free(tmp_ctx);
3972                 return -1;
3973         }
3974
3975         key.dptr  = discard_const(argv[1]);
3976         key.dsize = strlen(argv[1]);
3977         ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data);
3978         if (ret != 0) {
3979                 DEBUG(DEBUG_ERR,("Failed to fetch record\n"));
3980                 talloc_free(tmp_ctx);
3981                 return -1;
3982         }
3983
3984         if (data.dsize == 0 || data.dptr == NULL) {
3985                 DEBUG(DEBUG_ERR,("Record is empty\n"));
3986                 talloc_free(tmp_ctx);
3987                 return -1;
3988         }
3989
3990         if (argc == 3) {
3991           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
3992                 if (fd == -1) {
3993                         DEBUG(DEBUG_ERR,("Failed to open output file %s\n", argv[2]));
3994                         talloc_free(tmp_ctx);
3995                         return -1;
3996                 }
3997                 sys_write(fd, data.dptr, data.dsize);
3998                 close(fd);
3999         } else {
4000                 sys_write(1, data.dptr, data.dsize);
4001         }
4002
4003         /* abort the transaction */
4004         talloc_free(h);
4005
4006
4007         talloc_free(tmp_ctx);
4008         return 0;
4009 }
4010
4011 /*
4012   fetch a record from a tdb-file
4013  */
4014 static int control_tfetch(struct ctdb_context *ctdb, int argc, const char **argv)
4015 {
4016         const char *tdb_file;
4017         TDB_CONTEXT *tdb;
4018         TDB_DATA key, data;
4019         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
4020         int fd;
4021
4022         if (argc < 2) {
4023                 usage();
4024         }
4025
4026         tdb_file = argv[0];
4027
4028         tdb = tdb_open(tdb_file, 0, 0, O_RDONLY, 0);
4029         if (tdb == NULL) {
4030                 printf("Failed to open TDB file %s\n", tdb_file);
4031                 return -1;
4032         }
4033
4034         if (!strncmp(argv[1], "0x", 2)) {
4035                 key = hextodata(tmp_ctx, argv[1] + 2);
4036                 if (key.dsize == 0) {
4037                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4038                         return -1;
4039                 }
4040         } else {
4041                 key.dptr  = discard_const(argv[1]);
4042                 key.dsize = strlen(argv[1]);
4043         }
4044
4045         data = tdb_fetch(tdb, key);
4046         if (data.dptr == NULL || data.dsize < sizeof(struct ctdb_ltdb_header)) {
4047                 printf("Failed to read record %s from tdb %s\n", argv[1], tdb_file);
4048                 tdb_close(tdb);
4049                 return -1;
4050         }
4051
4052         tdb_close(tdb);
4053
4054         if (argc == 3) {
4055           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
4056                 if (fd == -1) {
4057                         printf("Failed to open output file %s\n", argv[2]);
4058                         return -1;
4059                 }
4060                 if (options.verbose){
4061                         sys_write(fd, data.dptr, data.dsize);
4062                 } else {
4063                         sys_write(fd, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
4064                 }
4065                 close(fd);
4066         } else {
4067                 if (options.verbose){
4068                         sys_write(1, data.dptr, data.dsize);
4069                 } else {
4070                         sys_write(1, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
4071                 }
4072         }
4073
4074         talloc_free(tmp_ctx);
4075         return 0;
4076 }
4077
4078 /*
4079   store a record and header to a tdb-file
4080  */
4081 static int control_tstore(struct ctdb_context *ctdb, int argc, const char **argv)
4082 {
4083         const char *tdb_file;
4084         TDB_CONTEXT *tdb;
4085         TDB_DATA key, value, data;
4086         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
4087         struct ctdb_ltdb_header header;
4088
4089         if (argc < 3) {
4090                 usage();
4091         }
4092
4093         tdb_file = argv[0];
4094
4095         tdb = tdb_open(tdb_file, 0, 0, O_RDWR, 0);
4096         if (tdb == NULL) {
4097                 printf("Failed to open TDB file %s\n", tdb_file);
4098                 return -1;
4099         }
4100
4101         if (!strncmp(argv[1], "0x", 2)) {
4102                 key = hextodata(tmp_ctx, argv[1] + 2);
4103                 if (key.dsize == 0) {
4104                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4105                         return -1;
4106                 }
4107         } else {
4108                 key.dptr  = discard_const(argv[1]);
4109                 key.dsize = strlen(argv[1]);
4110         }
4111
4112         if (!strncmp(argv[2], "0x", 2)) {
4113                 value = hextodata(tmp_ctx, argv[2] + 2);
4114                 if (value.dsize == 0) {
4115                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[2]);
4116                         return -1;
4117                 }
4118         } else {
4119                 value.dptr  = discard_const(argv[2]);
4120                 value.dsize = strlen(argv[2]);
4121         }
4122
4123         ZERO_STRUCT(header);
4124         if (argc > 3) {
4125                 header.rsn = atoll(argv[3]);
4126         }
4127         if (argc > 4) {
4128                 header.dmaster = atoi(argv[4]);
4129         }
4130         if (argc > 5) {
4131                 header.flags = atoi(argv[5]);
4132         }
4133
4134         data.dsize = sizeof(struct ctdb_ltdb_header) + value.dsize;
4135         data.dptr = talloc_size(tmp_ctx, data.dsize);
4136         if (data.dptr == NULL) {
4137                 printf("Failed to allocate header+value\n");
4138                 return -1;
4139         }
4140
4141         *(struct ctdb_ltdb_header *)data.dptr = header;
4142         memcpy(data.dptr + sizeof(struct ctdb_ltdb_header), value.dptr, value.dsize);
4143
4144         if (tdb_store(tdb, key, data, TDB_REPLACE) != 0) {
4145                 printf("Failed to write record %s to tdb %s\n", argv[1], tdb_file);
4146                 tdb_close(tdb);
4147                 return -1;
4148         }
4149
4150         tdb_close(tdb);
4151
4152         talloc_free(tmp_ctx);
4153         return 0;
4154 }
4155
4156 /*
4157   write a record to a persistent database
4158  */
4159 static int control_pstore(struct ctdb_context *ctdb, int argc, const char **argv)
4160 {
4161         const char *db_name;
4162         struct ctdb_db_context *ctdb_db;
4163         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4164         struct ctdb_transaction_handle *h;
4165         struct stat st;
4166         TDB_DATA key, data;
4167         int fd, ret;
4168
4169         if (argc < 3) {
4170                 talloc_free(tmp_ctx);
4171                 usage();
4172         }
4173
4174         fd = open(argv[2], O_RDONLY);
4175         if (fd == -1) {
4176                 DEBUG(DEBUG_ERR,("Failed to open file containing record data : %s  %s\n", argv[2], strerror(errno)));
4177                 talloc_free(tmp_ctx);
4178                 return -1;
4179         }
4180
4181         ret = fstat(fd, &st);
4182         if (ret == -1) {
4183                 DEBUG(DEBUG_ERR,("fstat of file %s failed: %s\n", argv[2], strerror(errno)));
4184                 close(fd);
4185                 talloc_free(tmp_ctx);
4186                 return -1;
4187         }
4188
4189         if (!S_ISREG(st.st_mode)) {
4190                 DEBUG(DEBUG_ERR,("Not a regular file %s\n", argv[2]));
4191                 close(fd);
4192                 talloc_free(tmp_ctx);
4193                 return -1;
4194         }
4195
4196         data.dsize = st.st_size;
4197         if (data.dsize == 0) {
4198                 data.dptr  = NULL;
4199         } else {
4200                 data.dptr = talloc_size(tmp_ctx, data.dsize);
4201                 if (data.dptr == NULL) {
4202                         DEBUG(DEBUG_ERR,("Failed to talloc %d of memory to store record data\n", (int)data.dsize));
4203                         close(fd);
4204                         talloc_free(tmp_ctx);
4205                         return -1;
4206                 }
4207                 ret = sys_read(fd, data.dptr, data.dsize);
4208                 if (ret != data.dsize) {
4209                         DEBUG(DEBUG_ERR,("Failed to read %d bytes of record data\n", (int)data.dsize));
4210                         close(fd);
4211                         talloc_free(tmp_ctx);
4212                         return -1;
4213                 }
4214         }
4215         close(fd);
4216
4217
4218         db_name = argv[0];
4219
4220         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4221         if (ctdb_db == NULL) {
4222                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4223                 talloc_free(tmp_ctx);
4224                 return -1;
4225         }
4226
4227         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4228         if (h == NULL) {
4229                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4230                 talloc_free(tmp_ctx);
4231                 return -1;
4232         }
4233
4234         if (!strncmp(argv[1], "0x", 2)) {
4235                 key = hextodata(tmp_ctx, argv[1] + 2);
4236                 if (key.dsize == 0) {
4237                         printf("Failed to convert \"%s\" into a TDB_DATA\n", argv[1]);
4238                         return -1;
4239                 }
4240         } else {
4241                 key.dptr  = discard_const(argv[1]);
4242                 key.dsize = strlen(argv[1]);
4243         }
4244
4245         ret = ctdb_transaction_store(h, key, data);
4246         if (ret != 0) {
4247                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4248                 talloc_free(tmp_ctx);
4249                 return -1;
4250         }
4251
4252         ret = ctdb_transaction_commit(h);
4253         if (ret != 0) {
4254                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4255                 talloc_free(tmp_ctx);
4256                 return -1;
4257         }
4258
4259
4260         talloc_free(tmp_ctx);
4261         return 0;
4262 }
4263
4264 /*
4265  * delete a record from a persistent database
4266  */
4267 static int control_pdelete(struct ctdb_context *ctdb, int argc, const char **argv)
4268 {
4269         const char *db_name;
4270         struct ctdb_db_context *ctdb_db;
4271         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4272         struct ctdb_transaction_handle *h;
4273         TDB_DATA key;
4274         int ret;
4275         bool persistent;
4276         uint8_t flags;
4277
4278         if (argc < 2) {
4279                 talloc_free(tmp_ctx);
4280                 usage();
4281         }
4282
4283         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
4284                 talloc_free(tmp_ctx);
4285                 return -1;
4286         }
4287
4288         persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
4289         if (!persistent) {
4290                 DEBUG(DEBUG_ERR, ("Database '%s' is not persistent\n", db_name));
4291                 talloc_free(tmp_ctx);
4292                 return -1;
4293         }
4294
4295         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
4296         if (ctdb_db == NULL) {
4297                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n", db_name));
4298                 talloc_free(tmp_ctx);
4299                 return -1;
4300         }
4301
4302         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4303         if (h == NULL) {
4304                 DEBUG(DEBUG_ERR, ("Failed to start transaction on database %s\n", db_name));
4305                 talloc_free(tmp_ctx);
4306                 return -1;
4307         }
4308
4309         key.dptr = discard_const(argv[1]);
4310         key.dsize = strlen(argv[1]);
4311         ret = ctdb_transaction_store(h, key, tdb_null);
4312         if (ret != 0) {
4313                 DEBUG(DEBUG_ERR, ("Failed to delete record\n"));
4314                 talloc_free(tmp_ctx);
4315                 return -1;
4316         }
4317
4318         ret = ctdb_transaction_commit(h);
4319         if (ret != 0) {
4320                 DEBUG(DEBUG_ERR, ("Failed to commit transaction\n"));
4321                 talloc_free(tmp_ctx);
4322                 return -1;
4323         }
4324
4325         talloc_free(tmp_ctx);
4326         return 0;
4327 }
4328
4329 static const char *ptrans_parse_string(TALLOC_CTX *mem_ctx, const char *s,
4330                                        TDB_DATA *data)
4331 {
4332         const char *t;
4333         size_t n;
4334         const char *ret; /* Next byte after successfully parsed value */
4335
4336         /* Error, unless someone says otherwise */
4337         ret = NULL;
4338         /* Indicates no value to parse */
4339         *data = tdb_null;
4340
4341         /* Skip whitespace */
4342         n = strspn(s, " \t");
4343         t = s + n;
4344
4345         if (t[0] == '"') {
4346                 /* Quoted ASCII string - no wide characters! */
4347                 t++;
4348                 n = strcspn(t, "\"");
4349                 if (t[n] == '"') {
4350                         if (n > 0) {
4351                                 data->dsize = n;
4352                                 data->dptr = talloc_memdup(mem_ctx, t, n);
4353                                 CTDB_NOMEM_ABORT(data->dptr);
4354                         }
4355                         ret = t + n + 1;
4356                 } else {
4357                         DEBUG(DEBUG_WARNING,("Unmatched \" in input %s\n", s));
4358                 }
4359         } else {
4360                 DEBUG(DEBUG_WARNING,("Unsupported input format in %s\n", s));
4361         }
4362
4363         return ret;
4364 }
4365
4366 static bool ptrans_get_key_value(TALLOC_CTX *mem_ctx, FILE *file,
4367                                  TDB_DATA *key, TDB_DATA *value)
4368 {
4369         char line [1024]; /* FIXME: make this more flexible? */
4370         const char *t;
4371         char *ptr;
4372
4373         ptr = fgets(line, sizeof(line), file);
4374
4375         if (ptr == NULL) {
4376                 return false;
4377         }
4378
4379         /* Get key */
4380         t = ptrans_parse_string(mem_ctx, line, key);
4381         if (t == NULL || key->dptr == NULL) {
4382                 /* Line Ignored but not EOF */
4383                 return true;
4384         }
4385
4386         /* Get value */
4387         t = ptrans_parse_string(mem_ctx, t, value);
4388         if (t == NULL) {
4389                 /* Line Ignored but not EOF */
4390                 talloc_free(key->dptr);
4391                 *key = tdb_null;
4392                 return true;
4393         }
4394
4395         return true;
4396 }
4397
4398 /*
4399  * Update a persistent database as per file/stdin
4400  */
4401 static int control_ptrans(struct ctdb_context *ctdb,
4402                           int argc, const char **argv)
4403 {
4404         const char *db_name;
4405         struct ctdb_db_context *ctdb_db;
4406         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4407         struct ctdb_transaction_handle *h;
4408         TDB_DATA key, value;
4409         FILE *file;
4410         int ret;
4411
4412         if (argc < 1) {
4413                 talloc_free(tmp_ctx);
4414                 usage();
4415         }
4416
4417         file = stdin;
4418         if (argc == 2) {
4419                 file = fopen(argv[1], "r");
4420                 if (file == NULL) {
4421                         DEBUG(DEBUG_ERR,("Unable to open file for reading '%s'\n", argv[1]));
4422                         talloc_free(tmp_ctx);
4423                         return -1;
4424                 }
4425         }
4426
4427         db_name = argv[0];
4428
4429         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
4430         if (ctdb_db == NULL) {
4431                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
4432                 goto error;
4433         }
4434
4435         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
4436         if (h == NULL) {
4437                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
4438                 goto error;
4439         }
4440
4441         while (ptrans_get_key_value(tmp_ctx, file, &key, &value)) {
4442                 if (key.dsize != 0) {
4443                         ret = ctdb_transaction_store(h, key, value);
4444                         /* Minimise memory use */
4445                         talloc_free(key.dptr);
4446                         if (value.dptr != NULL) {
4447                                 talloc_free(value.dptr);
4448                         }
4449                         if (ret != 0) {
4450                                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
4451                                 ctdb_transaction_cancel(h);
4452                                 goto error;
4453                         }
4454                 }
4455         }
4456
4457         ret = ctdb_transaction_commit(h);
4458         if (ret != 0) {
4459                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
4460                 goto error;
4461         }
4462
4463         if (file != stdin) {
4464                 fclose(file);
4465         }
4466         talloc_free(tmp_ctx);
4467         return 0;
4468
4469 error:
4470         if (file != stdin) {
4471                 fclose(file);
4472         }
4473
4474         talloc_free(tmp_ctx);
4475         return -1;
4476 }
4477
4478 /*
4479   check if a service is bound to a port or not
4480  */
4481 static int control_chktcpport(struct ctdb_context *ctdb, int argc, const char **argv)
4482 {
4483         int s, ret;
4484         int v;
4485         int port;
4486         struct sockaddr_in sin;
4487
4488         if (argc != 1) {
4489                 printf("Use: ctdb chktcport <port>\n");
4490                 return EINVAL;
4491         }
4492
4493         port = atoi(argv[0]);
4494
4495         s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
4496         if (s == -1) {
4497                 printf("Failed to open local socket\n");
4498                 return errno;
4499         }
4500
4501         v = fcntl(s, F_GETFL, 0);
4502         if (v == -1 || fcntl(s, F_SETFL, v | O_NONBLOCK) != 0) {
4503                 printf("Unable to set socket non-blocking: %s\n", strerror(errno));
4504         }
4505
4506         bzero(&sin, sizeof(sin));
4507         sin.sin_family = PF_INET;
4508         sin.sin_port   = htons(port);
4509         ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
4510         close(s);
4511         if (ret == -1) {
4512                 printf("Failed to bind to local socket: %d %s\n", errno, strerror(errno));
4513                 return errno;
4514         }
4515
4516         return 0;
4517 }
4518
4519
4520 /* Reload public IPs on a specified nodes */
4521 static int control_reloadips(struct ctdb_context *ctdb, int argc, const char **argv)
4522 {
4523         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4524         uint32_t *nodes;
4525         uint32_t pnn_mode;
4526         uint32_t timeout;
4527         int ret;
4528
4529         assert_single_node_only();
4530
4531         if (argc > 1) {
4532                 usage();
4533         }
4534
4535         /* Determine the nodes where IPs need to be reloaded */
4536         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
4537                               options.pnn, true, &nodes, &pnn_mode)) {
4538                 ret = -1;
4539                 goto done;
4540         }
4541
4542 again:
4543         /* Disable takeover runs on all connected nodes.  A reply
4544          * indicating success is needed from each node so all nodes
4545          * will need to be active.  This will retry until maxruntime
4546          * is exceeded, hence no error handling.
4547          *
4548          * A check could be added to not allow reloading of IPs when
4549          * there are disconnected nodes.  However, this should
4550          * probably be left up to the administrator.
4551          */
4552         timeout = LONGTIMEOUT;
4553         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4554                         "Disable takeover runs", true);
4555
4556         /* Now tell all the desired nodes to reload their public IPs.
4557          * Keep trying this until it succeeds.  This assumes all
4558          * failures are transient, which might not be true...
4559          */
4560         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_RELOAD_PUBLIC_IPS,
4561                                       nodes, 0, LONGTIMELIMIT(),
4562                                       false, tdb_null,
4563                                       NULL, NULL, NULL) != 0) {
4564                 DEBUG(DEBUG_ERR,
4565                       ("Unable to reload IPs on some nodes, try again.\n"));
4566                 goto again;
4567         }
4568
4569         /* It isn't strictly necessary to wait until takeover runs are
4570          * re-enabled but doing so can't hurt.
4571          */
4572         timeout = 0;
4573         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_TAKEOVER_RUNS, &timeout,
4574                         "Enable takeover runs", true);
4575
4576         ipreallocate(ctdb);
4577
4578         ret = 0;
4579 done:
4580         talloc_free(tmp_ctx);
4581         return ret;
4582 }
4583
4584 /*
4585   display a list of the databases on a remote ctdb
4586  */
4587 static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **argv)
4588 {
4589         int i, ret;
4590         struct ctdb_dbid_map *dbmap=NULL;
4591
4592         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
4593         if (ret != 0) {
4594                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
4595                 return ret;
4596         }
4597
4598         if(options.machinereadable){
4599                 printm(":ID:Name:Path:Persistent:Sticky:Unhealthy:ReadOnly:\n");
4600                 for(i=0;i<dbmap->num;i++){
4601                         const char *path = NULL;
4602                         const char *name = NULL;
4603                         const char *health = NULL;
4604                         bool persistent;
4605                         bool readonly;
4606                         bool sticky;
4607
4608                         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn,
4609                                             dbmap->dbs[i].dbid, ctdb, &path);
4610                         ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
4611                                             dbmap->dbs[i].dbid, ctdb, &name);
4612                         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
4613                                               dbmap->dbs[i].dbid, ctdb, &health);
4614                         persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4615                         readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4616                         sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4617                         printm(":0x%08X:%s:%s:%d:%d:%d:%d:\n",
4618                                dbmap->dbs[i].dbid, name, path,
4619                                !!(persistent), !!(sticky),
4620                                !!(health), !!(readonly));
4621                 }
4622                 return 0;
4623         }
4624
4625         printf("Number of databases:%d\n", dbmap->num);
4626         for(i=0;i<dbmap->num;i++){
4627                 const char *path = NULL;
4628                 const char *name = NULL;
4629                 const char *health = NULL;
4630                 bool persistent;
4631                 bool readonly;
4632                 bool sticky;
4633
4634                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
4635                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
4636                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
4637                 persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
4638                 readonly   = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
4639                 sticky     = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
4640                 printf("dbid:0x%08x name:%s path:%s%s%s%s%s\n",
4641                        dbmap->dbs[i].dbid, name, path,
4642                        persistent?" PERSISTENT":"",
4643                        sticky?" STICKY":"",
4644                        readonly?" READONLY":"",
4645                        health?" UNHEALTHY":"");
4646         }
4647
4648         return 0;
4649 }
4650
4651 /*
4652   display the status of a database on a remote ctdb
4653  */
4654 static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char **argv)
4655 {
4656         const char *db_name;
4657         uint32_t db_id;
4658         uint8_t flags;
4659         const char *path = NULL;
4660         const char *health = NULL;
4661
4662         if (argc < 1) {
4663                 usage();
4664         }
4665
4666         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
4667                 return -1;
4668         }
4669
4670         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &path);
4671         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, db_id, ctdb, &health);
4672         printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nSTICKY: %s\nREADONLY: %s\nHEALTH: %s\n",
4673                db_id, db_name, path,
4674                (flags & CTDB_DB_FLAGS_PERSISTENT ? "yes" : "no"),
4675                (flags & CTDB_DB_FLAGS_STICKY ? "yes" : "no"),
4676                (flags & CTDB_DB_FLAGS_READONLY ? "yes" : "no"),
4677                (health ? health : "OK"));
4678
4679         return 0;
4680 }
4681
4682 /*
4683   check if the local node is recmaster or not
4684   it will return 1 if this node is the recmaster and 0 if it is not
4685   or if the local ctdb daemon could not be contacted
4686  */
4687 static int control_isnotrecmaster(struct ctdb_context *ctdb, int argc, const char **argv)
4688 {
4689         uint32_t mypnn, recmaster;
4690         int ret;
4691
4692         assert_single_node_only();
4693
4694         mypnn = getpnn(ctdb);
4695
4696         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
4697         if (ret != 0) {
4698                 printf("Failed to get the recmaster\n");
4699                 return 1;
4700         }
4701
4702         if (recmaster != mypnn) {
4703                 printf("this node is not the recmaster\n");
4704                 return 1;
4705         }
4706
4707         printf("this node is the recmaster\n");
4708         return 0;
4709 }
4710
4711 /*
4712   ping a node
4713  */
4714 static int control_ping(struct ctdb_context *ctdb, int argc, const char **argv)
4715 {
4716         int ret;
4717         struct timeval tv = timeval_current();
4718         ret = ctdb_ctrl_ping(ctdb, options.pnn);
4719         if (ret == -1) {
4720                 printf("Unable to get ping response from node %u\n", options.pnn);
4721                 return -1;
4722         } else {
4723                 printf("response from %u time=%.6f sec  (%d clients)\n",
4724                        options.pnn, timeval_elapsed(&tv), ret);
4725         }
4726         return 0;
4727 }
4728
4729
4730 /*
4731   get a node's runstate
4732  */
4733 static int control_runstate(struct ctdb_context *ctdb, int argc, const char **argv)
4734 {
4735         int ret;
4736         enum ctdb_runstate runstate;
4737
4738         ret = ctdb_ctrl_get_runstate(ctdb, TIMELIMIT(), options.pnn, &runstate);
4739         if (ret == -1) {
4740                 printf("Unable to get runstate response from node %u\n",
4741                        options.pnn);
4742                 return -1;
4743         } else {
4744                 bool found = true;
4745                 enum ctdb_runstate t;
4746                 int i;
4747                 for (i=0; i<argc; i++) {
4748                         found = false;
4749                         t = runstate_from_string(argv[i]);
4750                         if (t == CTDB_RUNSTATE_UNKNOWN) {
4751                                 printf("Invalid run state (%s)\n", argv[i]);
4752                                 return -1;
4753                         }
4754
4755                         if (t == runstate) {
4756                                 found = true;
4757                                 break;
4758                         }
4759                 }
4760
4761                 if (!found) {
4762                         printf("CTDB not in required run state (got %s)\n",
4763                                runstate_to_string((enum ctdb_runstate)runstate));
4764                         return -1;
4765                 }
4766         }
4767
4768         printf("%s\n", runstate_to_string(runstate));
4769         return 0;
4770 }
4771
4772
4773 /*
4774   get a tunable
4775  */
4776 static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv)
4777 {
4778         const char *name;
4779         uint32_t value;
4780         int ret;
4781
4782         if (argc < 1) {
4783                 usage();
4784         }
4785
4786         name = argv[0];
4787         ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn, name, &value);
4788         if (ret != 0) {
4789                 DEBUG(DEBUG_ERR, ("Unable to get tunable variable '%s'\n", name));
4790                 return -1;
4791         }
4792
4793         printf("%-23s = %u\n", name, value);
4794         return 0;
4795 }
4796
4797 /*
4798   set a tunable
4799  */
4800 static int control_setvar(struct ctdb_context *ctdb, int argc, const char **argv)
4801 {
4802         const char *name;
4803         uint32_t value;
4804         int ret;
4805
4806         if (argc < 2) {
4807                 usage();
4808         }
4809
4810         name = argv[0];
4811         value = strtoul(argv[1], NULL, 0);
4812
4813         ret = ctdb_ctrl_set_tunable(ctdb, TIMELIMIT(), options.pnn, name, value);
4814         if (ret == -1) {
4815                 DEBUG(DEBUG_ERR, ("Unable to set tunable variable '%s'\n", name));
4816                 return -1;
4817         }
4818         if (ret == 1) {
4819                 DEBUG(DEBUG_WARNING,
4820                       ("Setting obsolete tunable variable '%s'\n",
4821                        name));
4822         }
4823         return 0;
4824 }
4825
4826 /*
4827   list all tunables
4828  */
4829 static int control_listvars(struct ctdb_context *ctdb, int argc, const char **argv)
4830 {
4831         uint32_t count;
4832         const char **list;
4833         int ret, i;
4834
4835         ret = ctdb_ctrl_list_tunables(ctdb, TIMELIMIT(), options.pnn, ctdb, &list, &count);
4836         if (ret == -1) {
4837                 DEBUG(DEBUG_ERR, ("Unable to list tunable variables\n"));
4838                 return -1;
4839         }
4840
4841         for (i=0;i<count;i++) {
4842                 control_getvar(ctdb, 1, &list[i]);
4843         }
4844
4845         talloc_free(list);
4846
4847         return 0;
4848 }
4849
4850 /*
4851   display debug level on a node
4852  */
4853 static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **argv)
4854 {
4855         int ret;
4856         int32_t level;
4857
4858         ret = ctdb_ctrl_get_debuglevel(ctdb, options.pnn, &level);
4859         if (ret != 0) {
4860                 DEBUG(DEBUG_ERR, ("Unable to get debuglevel response from node %u\n", options.pnn));
4861                 return ret;
4862         } else {
4863                 const char *desc = get_debug_by_level(level);
4864                 if (desc == NULL) {
4865                         /* This should never happen */
4866                         desc = "Unknown";
4867                 }
4868                 if (options.machinereadable){
4869                         printm(":Name:Level:\n");
4870                         printm(":%s:%d:\n", desc, level);
4871                 } else {
4872                         printf("Node %u is at debug level %s (%d)\n",
4873                                options.pnn, desc, level);
4874                 }
4875         }
4876         return 0;
4877 }
4878
4879 /*
4880   display reclock file of a node
4881  */
4882 static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4883 {
4884         int ret;
4885         const char *reclock;
4886
4887         ret = ctdb_ctrl_getreclock(ctdb, TIMELIMIT(), options.pnn, ctdb, &reclock);
4888         if (ret != 0) {
4889                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4890                 return ret;
4891         } else {
4892                 if (options.machinereadable){
4893                         if (reclock != NULL) {
4894                                 printm("%s", reclock);
4895                         }
4896                 } else {
4897                         if (reclock == NULL) {
4898                                 printf("No reclock file used.\n");
4899                         } else {
4900                                 printf("Reclock file:%s\n", reclock);
4901                         }
4902                 }
4903         }
4904         return 0;
4905 }
4906
4907 /*
4908   set the reclock file of a node
4909  */
4910 static int control_setreclock(struct ctdb_context *ctdb, int argc, const char **argv)
4911 {
4912         int ret;
4913         const char *reclock;
4914
4915         if (argc == 0) {
4916                 reclock = NULL;
4917         } else if (argc == 1) {
4918                 reclock = argv[0];
4919         } else {
4920                 usage();
4921         }
4922
4923         ret = ctdb_ctrl_setreclock(ctdb, TIMELIMIT(), options.pnn, reclock);
4924         if (ret != 0) {
4925                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
4926                 return ret;
4927         }
4928         return 0;
4929 }
4930
4931 /*
4932   set the natgw state on/off
4933  */
4934 static int control_setnatgwstate(struct ctdb_context *ctdb, int argc, const char **argv)
4935 {
4936         int ret;
4937         uint32_t natgwstate;
4938
4939         if (argc == 0) {
4940                 usage();
4941         }
4942
4943         if (!strcmp(argv[0], "on")) {
4944                 natgwstate = 1;
4945         } else if (!strcmp(argv[0], "off")) {
4946                 natgwstate = 0;
4947         } else {
4948                 usage();
4949         }
4950
4951         ret = ctdb_ctrl_setnatgwstate(ctdb, TIMELIMIT(), options.pnn, natgwstate);
4952         if (ret != 0) {
4953                 DEBUG(DEBUG_ERR, ("Unable to set the natgw state for node %u\n", options.pnn));
4954                 return ret;
4955         }
4956
4957         return 0;
4958 }
4959
4960 /*
4961   set the lmaster role on/off
4962  */
4963 static int control_setlmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
4964 {
4965         int ret;
4966         uint32_t lmasterrole;
4967
4968         if (argc == 0) {
4969                 usage();
4970         }
4971
4972         if (!strcmp(argv[0], "on")) {
4973                 lmasterrole = 1;
4974         } else if (!strcmp(argv[0], "off")) {
4975                 lmasterrole = 0;
4976         } else {
4977                 usage();
4978         }
4979
4980         ret = ctdb_ctrl_setlmasterrole(ctdb, TIMELIMIT(), options.pnn, lmasterrole);
4981         if (ret != 0) {
4982                 DEBUG(DEBUG_ERR, ("Unable to set the lmaster role for node %u\n", options.pnn));
4983                 return ret;
4984         }
4985
4986         return 0;
4987 }
4988
4989 /*
4990   set the recmaster role on/off
4991  */
4992 static int control_setrecmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
4993 {
4994         int ret;
4995         uint32_t recmasterrole;
4996
4997         if (argc == 0) {
4998                 usage();
4999         }
5000
5001         if (!strcmp(argv[0], "on")) {
5002                 recmasterrole = 1;
5003         } else if (!strcmp(argv[0], "off")) {
5004                 recmasterrole = 0;
5005         } else {
5006                 usage();
5007         }
5008
5009         ret = ctdb_ctrl_setrecmasterrole(ctdb, TIMELIMIT(), options.pnn, recmasterrole);
5010         if (ret != 0) {
5011                 DEBUG(DEBUG_ERR, ("Unable to set the recmaster role for node %u\n", options.pnn));
5012                 return ret;
5013         }
5014
5015         return 0;
5016 }
5017
5018 /*
5019   set debug level on a node or all nodes
5020  */
5021 static int control_setdebug(struct ctdb_context *ctdb, int argc, const char **argv)
5022 {
5023         int ret;
5024         int32_t level;
5025
5026         if (argc == 0) {
5027                 printf("You must specify the debug level. Valid levels are:\n");
5028                 print_debug_levels(stdout);
5029                 return 0;
5030         }
5031
5032         if (!parse_debug(argv[0], &level)) {
5033                 printf("Invalid debug level, must be one of\n");
5034                 print_debug_levels(stdout);
5035                 return -1;
5036         }
5037
5038         ret = ctdb_ctrl_set_debuglevel(ctdb, options.pnn, level);
5039         if (ret != 0) {
5040                 DEBUG(DEBUG_ERR, ("Unable to set debug level on node %u\n", options.pnn));
5041         }
5042         return 0;
5043 }
5044
5045
5046 /*
5047   thaw a node
5048  */
5049 static int control_thaw(struct ctdb_context *ctdb, int argc, const char **argv)
5050 {
5051         int ret;
5052         uint32_t priority;
5053
5054         if (argc == 1) {
5055                 priority = strtol(argv[0], NULL, 0);
5056         } else {
5057                 priority = 0;
5058         }
5059         DEBUG(DEBUG_ERR,("Thaw by priority %u\n", priority));
5060
5061         ret = ctdb_ctrl_thaw_priority(ctdb, TIMELIMIT(), options.pnn, priority);
5062         if (ret != 0) {
5063                 DEBUG(DEBUG_ERR, ("Unable to thaw node %u\n", options.pnn));
5064         }
5065         return 0;
5066 }
5067
5068
5069 /*
5070   attach to a database
5071  */
5072 static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv)
5073 {
5074         const char *db_name;
5075         struct ctdb_db_context *ctdb_db;
5076         bool persistent = false;
5077
5078         if (argc < 1) {
5079                 usage();
5080         }
5081         db_name = argv[0];
5082         if (argc > 2) {
5083                 usage();
5084         }
5085         if (argc == 2) {
5086                 if (strcmp(argv[1], "persistent") != 0) {
5087                         usage();
5088                 }
5089                 persistent = true;
5090         }
5091
5092         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
5093         if (ctdb_db == NULL) {
5094                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
5095                 return -1;
5096         }
5097
5098         return 0;
5099 }
5100
5101 /*
5102  * detach from a database
5103  */
5104 static int control_detach(struct ctdb_context *ctdb, int argc,
5105                           const char **argv)
5106 {
5107         uint32_t db_id;
5108         uint8_t flags;
5109         int ret, i, status = 0;
5110         struct ctdb_node_map *nodemap = NULL;
5111         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5112         uint32_t recmode;
5113
5114         if (argc < 1) {
5115                 usage();
5116         }
5117
5118         assert_single_node_only();
5119
5120         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn,
5121                                     &recmode);
5122         if (ret != 0) {
5123                 DEBUG(DEBUG_ERR, ("Database cannot be detached "
5124                                   "when recovery is active\n"));
5125                 talloc_free(tmp_ctx);
5126                 return -1;
5127         }
5128
5129         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5130                                    &nodemap);
5131         if (ret != 0) {
5132                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5133                                   options.pnn));
5134                 talloc_free(tmp_ctx);
5135                 return -1;
5136         }
5137
5138         for (i=0; i<nodemap->num; i++) {
5139                 uint32_t value;
5140
5141                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
5142                         continue;
5143                 }
5144
5145                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
5146                         continue;
5147                 }
5148
5149                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
5150                         DEBUG(DEBUG_ERR, ("Database cannot be detached on "
5151                                           "inactive (stopped or banned) node "
5152                                           "%u\n", nodemap->nodes[i].pnn));
5153                         talloc_free(tmp_ctx);
5154                         return -1;
5155                 }
5156
5157                 ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(),
5158                                             nodemap->nodes[i].pnn,
5159                                             "AllowClientDBAttach",
5160                                             &value);
5161                 if (ret != 0) {
5162                         DEBUG(DEBUG_ERR, ("Unable to get tunable "
5163                                           "AllowClientDBAttach from node %u\n",
5164                                            nodemap->nodes[i].pnn));
5165                         talloc_free(tmp_ctx);
5166                         return -1;
5167                 }
5168
5169                 if (value == 1) {
5170                         DEBUG(DEBUG_ERR, ("Database access is still active on "
5171                                           "node %u. Set AllowClientDBAttach=0 "
5172                                           "on all nodes.\n",
5173                                           nodemap->nodes[i].pnn));
5174                         talloc_free(tmp_ctx);
5175                         return -1;
5176                 }
5177         }
5178
5179         talloc_free(tmp_ctx);
5180
5181         for (i=0; i<argc; i++) {
5182                 if (!db_exists(ctdb, argv[i], &db_id, NULL, &flags)) {
5183                         continue;
5184                 }
5185
5186                 if (flags & CTDB_DB_FLAGS_PERSISTENT) {
5187                         DEBUG(DEBUG_ERR, ("Persistent database '%s' "
5188                                           "cannot be detached\n", argv[i]));
5189                         status = -1;
5190                         continue;
5191                 }
5192
5193                 ret = ctdb_detach(ctdb, db_id);
5194                 if (ret != 0) {
5195                         DEBUG(DEBUG_ERR, ("Database '%s' detach failed\n",
5196                                           argv[i]));
5197                         status = ret;
5198                 }
5199         }
5200
5201         return status;
5202 }
5203
5204 /*
5205   set db priority
5206  */
5207 static int control_setdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5208 {
5209         struct ctdb_db_priority db_prio;
5210         int ret;
5211
5212         if (argc < 2) {
5213                 usage();
5214         }
5215
5216         db_prio.db_id    = strtoul(argv[0], NULL, 0);
5217         db_prio.priority = strtoul(argv[1], NULL, 0);
5218
5219         ret = ctdb_ctrl_set_db_priority(ctdb, TIMELIMIT(), options.pnn, &db_prio);
5220         if (ret != 0) {
5221                 DEBUG(DEBUG_ERR,("Unable to set db prio\n"));
5222                 return -1;
5223         }
5224
5225         return 0;
5226 }
5227
5228 /*
5229   get db priority
5230  */
5231 static int control_getdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
5232 {
5233         uint32_t db_id, priority;
5234         int ret;
5235
5236         if (argc < 1) {
5237                 usage();
5238         }
5239
5240         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5241                 return -1;
5242         }
5243
5244         ret = ctdb_ctrl_get_db_priority(ctdb, TIMELIMIT(), options.pnn, db_id, &priority);
5245         if (ret != 0) {
5246                 DEBUG(DEBUG_ERR,("Unable to get db prio\n"));
5247                 return -1;
5248         }
5249
5250         DEBUG(DEBUG_ERR,("Priority:%u\n", priority));
5251
5252         return 0;
5253 }
5254
5255 /*
5256   set the sticky records capability for a database
5257  */
5258 static int control_setdbsticky(struct ctdb_context *ctdb, int argc, const char **argv)
5259 {
5260         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5261         uint32_t db_id;
5262         int ret;
5263
5264         if (argc < 1) {
5265                 usage();
5266         }
5267
5268         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5269                 return -1;
5270         }
5271
5272         ret = ctdb_ctrl_set_db_sticky(ctdb, options.pnn, db_id);
5273         if (ret != 0) {
5274                 DEBUG(DEBUG_ERR,("Unable to set db to support sticky records\n"));
5275                 talloc_free(tmp_ctx);
5276                 return -1;
5277         }
5278
5279         talloc_free(tmp_ctx);
5280         return 0;
5281 }
5282
5283 /*
5284   set the readonly capability for a database
5285  */
5286 static int control_setdbreadonly(struct ctdb_context *ctdb, int argc, const char **argv)
5287 {
5288         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5289         uint32_t db_id;
5290         int ret;
5291
5292         if (argc < 1) {
5293                 usage();
5294         }
5295
5296         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5297                 return -1;
5298         }
5299
5300         ret = ctdb_ctrl_set_db_readonly(ctdb, options.pnn, db_id);
5301         if (ret != 0) {
5302                 DEBUG(DEBUG_ERR,("Unable to set db to support readonly\n"));
5303                 talloc_free(tmp_ctx);
5304                 return -1;
5305         }
5306
5307         talloc_free(tmp_ctx);
5308         return 0;
5309 }
5310
5311 /*
5312   get db seqnum
5313  */
5314 static int control_getdbseqnum(struct ctdb_context *ctdb, int argc, const char **argv)
5315 {
5316         uint32_t db_id;
5317         uint64_t seqnum;
5318         int ret;
5319
5320         if (argc < 1) {
5321                 usage();
5322         }
5323
5324         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
5325                 return -1;
5326         }
5327
5328         ret = ctdb_ctrl_getdbseqnum(ctdb, TIMELIMIT(), options.pnn, db_id, &seqnum);
5329         if (ret != 0) {
5330                 DEBUG(DEBUG_ERR, ("Unable to get seqnum from node."));
5331                 return -1;
5332         }
5333
5334         printf("Sequence number:%lld\n", (long long)seqnum);
5335
5336         return 0;
5337 }
5338
5339 /*
5340   run an eventscript on a node
5341  */
5342 static int control_eventscript(struct ctdb_context *ctdb, int argc, const char **argv)
5343 {
5344         TDB_DATA data;
5345         int ret;
5346         int32_t res;
5347         char *errmsg;
5348         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5349
5350         if (argc != 1) {
5351                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5352                 return -1;
5353         }
5354
5355         data.dptr = (unsigned char *)discard_const(argv[0]);
5356         data.dsize = strlen((char *)data.dptr) + 1;
5357
5358         DEBUG(DEBUG_ERR, ("Running eventscripts with arguments \"%s\" on node %u\n", data.dptr, options.pnn));
5359
5360         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_RUN_EVENTSCRIPTS,
5361                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
5362         if (ret != 0 || res != 0) {
5363                 DEBUG(DEBUG_ERR,("Failed to run eventscripts - %s\n", errmsg));
5364                 talloc_free(tmp_ctx);
5365                 return -1;
5366         }
5367         talloc_free(tmp_ctx);
5368         return 0;
5369 }
5370
5371 #define DB_VERSION 1
5372 #define MAX_DB_NAME 64
5373 struct db_file_header {
5374         unsigned long version;
5375         time_t timestamp;
5376         unsigned long persistent;
5377         unsigned long size;
5378         const char name[MAX_DB_NAME];
5379 };
5380
5381 struct backup_data {
5382         struct ctdb_marshall_buffer *records;
5383         uint32_t len;
5384         uint32_t total;
5385         bool traverse_error;
5386 };
5387
5388 static int backup_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private)
5389 {
5390         struct backup_data *bd = talloc_get_type(private, struct backup_data);
5391         struct ctdb_rec_data *rec;
5392
5393         /* add the record */
5394         rec = ctdb_marshall_record(bd->records, 0, key, NULL, data);
5395         if (rec == NULL) {
5396                 bd->traverse_error = true;
5397                 DEBUG(DEBUG_ERR,("Failed to marshall record\n"));
5398                 return -1;
5399         }
5400         bd->records = talloc_realloc_size(NULL, bd->records, rec->length + bd->len);
5401         if (bd->records == NULL) {
5402                 DEBUG(DEBUG_ERR,("Failed to expand marshalling buffer\n"));
5403                 bd->traverse_error = true;
5404                 return -1;
5405         }
5406         bd->records->count++;
5407         memcpy(bd->len+(uint8_t *)bd->records, rec, rec->length);
5408         bd->len += rec->length;
5409         talloc_free(rec);
5410
5411         bd->total++;
5412         return 0;
5413 }
5414
5415 /*
5416  * backup a database to a file
5417  */
5418 static int control_backupdb(struct ctdb_context *ctdb, int argc, const char **argv)
5419 {
5420         const char *db_name;
5421         int ret;
5422         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5423         struct db_file_header dbhdr;
5424         struct ctdb_db_context *ctdb_db;
5425         struct backup_data *bd;
5426         int fh = -1;
5427         int status = -1;
5428         const char *reason = NULL;
5429         uint32_t db_id;
5430         uint8_t flags;
5431
5432         assert_single_node_only();
5433
5434         if (argc != 2) {
5435                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5436                 return -1;
5437         }
5438
5439         if (!db_exists(ctdb, argv[0], &db_id, &db_name, &flags)) {
5440                 return -1;
5441         }
5442
5443         ret = ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
5444                                     db_id, tmp_ctx, &reason);
5445         if (ret != 0) {
5446                 DEBUG(DEBUG_ERR,("Unable to get dbhealth for database '%s'\n",
5447                                  argv[0]));
5448                 talloc_free(tmp_ctx);
5449                 return -1;
5450         }
5451         if (reason) {
5452                 uint32_t allow_unhealthy = 0;
5453
5454                 ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn,
5455                                       "AllowUnhealthyDBRead",
5456                                       &allow_unhealthy);
5457
5458                 if (allow_unhealthy != 1) {
5459                         DEBUG(DEBUG_ERR,("database '%s' is unhealthy: %s\n",
5460                                          argv[0], reason));
5461
5462                         DEBUG(DEBUG_ERR,("disallow backup : tunable AllowUnhealthyDBRead = %u\n",
5463                                          allow_unhealthy));
5464                         talloc_free(tmp_ctx);
5465                         return -1;
5466                 }
5467
5468                 DEBUG(DEBUG_WARNING,("WARNING database '%s' is unhealthy - see 'ctdb getdbstatus %s'\n",
5469                                      argv[0], argv[0]));
5470                 DEBUG(DEBUG_WARNING,("WARNING! allow backup of unhealthy database: "
5471                                      "tunnable AllowUnhealthyDBRead = %u\n",
5472                                      allow_unhealthy));
5473         }
5474
5475         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5476         if (ctdb_db == NULL) {
5477                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", argv[0]));
5478                 talloc_free(tmp_ctx);
5479                 return -1;
5480         }
5481
5482
5483         ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
5484         if (ret == -1) {
5485                 DEBUG(DEBUG_ERR,("Failed to start transaction\n"));
5486                 talloc_free(tmp_ctx);
5487                 return -1;
5488         }
5489
5490
5491         bd = talloc_zero(tmp_ctx, struct backup_data);
5492         if (bd == NULL) {
5493                 DEBUG(DEBUG_ERR,("Failed to allocate backup_data\n"));
5494                 talloc_free(tmp_ctx);
5495                 return -1;
5496         }
5497
5498         bd->records = talloc_zero(bd, struct ctdb_marshall_buffer);
5499         if (bd->records == NULL) {
5500                 DEBUG(DEBUG_ERR,("Failed to allocate ctdb_marshall_buffer\n"));
5501                 talloc_free(tmp_ctx);
5502                 return -1;
5503         }
5504
5505         bd->len = offsetof(struct ctdb_marshall_buffer, data);
5506         bd->records->db_id = ctdb_db->db_id;
5507         /* traverse the database collecting all records */
5508         if (tdb_traverse_read(ctdb_db->ltdb->tdb, backup_traverse, bd) == -1 ||
5509             bd->traverse_error) {
5510                 DEBUG(DEBUG_ERR,("Traverse error\n"));
5511                 talloc_free(tmp_ctx);
5512                 return -1;
5513         }
5514
5515         tdb_transaction_cancel(ctdb_db->ltdb->tdb);
5516
5517
5518         fh = open(argv[1], O_RDWR|O_CREAT, 0600);
5519         if (fh == -1) {
5520                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[1]));
5521                 talloc_free(tmp_ctx);
5522                 return -1;
5523         }
5524
5525         ZERO_STRUCT(dbhdr);
5526         dbhdr.version = DB_VERSION;
5527         dbhdr.timestamp = time(NULL);
5528         dbhdr.persistent = flags & CTDB_DB_FLAGS_PERSISTENT;
5529         dbhdr.size = bd->len;
5530         if (strlen(argv[0]) >= MAX_DB_NAME) {
5531                 DEBUG(DEBUG_ERR,("Too long dbname\n"));
5532                 goto done;
5533         }
5534         strncpy(discard_const(dbhdr.name), argv[0], MAX_DB_NAME-1);
5535         ret = sys_write(fh, &dbhdr, sizeof(dbhdr));
5536         if (ret == -1) {
5537                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5538                 goto done;
5539         }
5540         ret = sys_write(fh, bd->records, bd->len);
5541         if (ret == -1) {
5542                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
5543                 goto done;
5544         }
5545
5546         status = 0;
5547 done:
5548         if (fh != -1) {
5549                 ret = close(fh);
5550                 if (ret == -1) {
5551                         DEBUG(DEBUG_ERR,("close failed: %s\n", strerror(errno)));
5552                 }
5553         }
5554
5555         DEBUG(DEBUG_ERR,("Database backed up to %s\n", argv[1]));
5556
5557         talloc_free(tmp_ctx);
5558         return status;
5559 }
5560
5561 /*
5562  * restore a database from a file
5563  */
5564 static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **argv)
5565 {
5566         int ret;
5567         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5568         TDB_DATA outdata;
5569         TDB_DATA data;
5570         struct db_file_header dbhdr;
5571         struct ctdb_db_context *ctdb_db;
5572         struct ctdb_node_map *nodemap=NULL;
5573         struct ctdb_vnn_map *vnnmap=NULL;
5574         int i, fh;
5575         struct ctdb_control_wipe_database w;
5576         uint32_t *nodes;
5577         uint32_t generation;
5578         struct tm *tm;
5579         char tbuf[100];
5580         char *dbname;
5581
5582         assert_single_node_only();
5583
5584         if (argc < 1 || argc > 2) {
5585                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5586                 return -1;
5587         }
5588
5589         fh = open(argv[0], O_RDONLY);
5590         if (fh == -1) {
5591                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5592                 talloc_free(tmp_ctx);
5593                 return -1;
5594         }
5595
5596         sys_read(fh, &dbhdr, sizeof(dbhdr));
5597         if (dbhdr.version != DB_VERSION) {
5598                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5599                 close(fh);
5600                 talloc_free(tmp_ctx);
5601                 return -1;
5602         }
5603
5604         dbname = discard_const(dbhdr.name);
5605         if (argc == 2) {
5606                 dbname = discard_const(argv[1]);
5607         }
5608
5609         outdata.dsize = dbhdr.size;
5610         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5611         if (outdata.dptr == NULL) {
5612                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5613                 close(fh);
5614                 talloc_free(tmp_ctx);
5615                 return -1;
5616         }
5617         sys_read(fh, outdata.dptr, outdata.dsize);
5618         close(fh);
5619
5620         tm = localtime(&dbhdr.timestamp);
5621         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5622         printf("Restoring database '%s' from backup @ %s\n",
5623                 dbname, tbuf);
5624
5625
5626         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), dbname, dbhdr.persistent, 0);
5627         if (ctdb_db == NULL) {
5628                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", dbname));
5629                 talloc_free(tmp_ctx);
5630                 return -1;
5631         }
5632
5633         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
5634         if (ret != 0) {
5635                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
5636                 talloc_free(tmp_ctx);
5637                 return ret;
5638         }
5639
5640
5641         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
5642         if (ret != 0) {
5643                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
5644                 talloc_free(tmp_ctx);
5645                 return ret;
5646         }
5647
5648         /* freeze all nodes */
5649         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5650         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5651                 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5652                                         nodes, i,
5653                                         TIMELIMIT(),
5654                                         false, tdb_null,
5655                                         NULL, NULL,
5656                                         NULL) != 0) {
5657                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5658                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5659                         talloc_free(tmp_ctx);
5660                         return -1;
5661                 }
5662         }
5663
5664         generation = vnnmap->generation;
5665         data.dptr = (void *)&generation;
5666         data.dsize = sizeof(generation);
5667
5668         /* start a cluster wide transaction */
5669         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5670         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
5671                                         nodes, 0,
5672                                         TIMELIMIT(), false, data,
5673                                         NULL, NULL,
5674                                         NULL) != 0) {
5675                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide transactions.\n"));
5676                 return -1;
5677         }
5678
5679
5680         w.db_id = ctdb_db->db_id;
5681         w.transaction_id = generation;
5682
5683         data.dptr = (void *)&w;
5684         data.dsize = sizeof(w);
5685
5686         /* wipe all the remote databases. */
5687         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5688         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
5689                                         nodes, 0,
5690                                         TIMELIMIT(), false, data,
5691                                         NULL, NULL,
5692                                         NULL) != 0) {
5693                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
5694                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5695                 talloc_free(tmp_ctx);
5696                 return -1;
5697         }
5698
5699         /* push the database */
5700         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5701         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
5702                                         nodes, 0,
5703                                         TIMELIMIT(), false, outdata,
5704                                         NULL, NULL,
5705                                         NULL) != 0) {
5706                 DEBUG(DEBUG_ERR, ("Failed to push database.\n"));
5707                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5708                 talloc_free(tmp_ctx);
5709                 return -1;
5710         }
5711
5712         data.dptr = (void *)&ctdb_db->db_id;
5713         data.dsize = sizeof(ctdb_db->db_id);
5714
5715         /* mark the database as healthy */
5716         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5717         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
5718                                         nodes, 0,
5719                                         TIMELIMIT(), false, data,
5720                                         NULL, NULL,
5721                                         NULL) != 0) {
5722                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
5723                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5724                 talloc_free(tmp_ctx);
5725                 return -1;
5726         }
5727
5728         data.dptr = (void *)&generation;
5729         data.dsize = sizeof(generation);
5730
5731         /* commit all the changes */
5732         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
5733                                         nodes, 0,
5734                                         TIMELIMIT(), false, data,
5735                                         NULL, NULL,
5736                                         NULL) != 0) {
5737                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
5738                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5739                 talloc_free(tmp_ctx);
5740                 return -1;
5741         }
5742
5743
5744         /* thaw all nodes */
5745         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5746         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
5747                                         nodes, 0,
5748                                         TIMELIMIT(),
5749                                         false, tdb_null,
5750                                         NULL, NULL,
5751                                         NULL) != 0) {
5752                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
5753                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5754                 talloc_free(tmp_ctx);
5755                 return -1;
5756         }
5757
5758
5759         talloc_free(tmp_ctx);
5760         return 0;
5761 }
5762
5763 /*
5764  * dump a database backup from a file
5765  */
5766 static int control_dumpdbbackup(struct ctdb_context *ctdb, int argc, const char **argv)
5767 {
5768         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5769         TDB_DATA outdata;
5770         struct db_file_header dbhdr;
5771         int i, fh;
5772         struct tm *tm;
5773         char tbuf[100];
5774         struct ctdb_rec_data *rec = NULL;
5775         struct ctdb_marshall_buffer *m;
5776         struct ctdb_dump_db_context c;
5777
5778         assert_single_node_only();
5779
5780         if (argc != 1) {
5781                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5782                 return -1;
5783         }
5784
5785         fh = open(argv[0], O_RDONLY);
5786         if (fh == -1) {
5787                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
5788                 talloc_free(tmp_ctx);
5789                 return -1;
5790         }
5791
5792         sys_read(fh, &dbhdr, sizeof(dbhdr));
5793         if (dbhdr.version != DB_VERSION) {
5794                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
5795                 close(fh);
5796                 talloc_free(tmp_ctx);
5797                 return -1;
5798         }
5799
5800         outdata.dsize = dbhdr.size;
5801         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
5802         if (outdata.dptr == NULL) {
5803                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
5804                 close(fh);
5805                 talloc_free(tmp_ctx);
5806                 return -1;
5807         }
5808         sys_read(fh, outdata.dptr, outdata.dsize);
5809         close(fh);
5810         m = (struct ctdb_marshall_buffer *)outdata.dptr;
5811
5812         tm = localtime(&dbhdr.timestamp);
5813         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
5814         printf("Backup of database name:'%s' dbid:0x%x08x from @ %s\n",
5815                 dbhdr.name, m->db_id, tbuf);
5816
5817         ZERO_STRUCT(c);
5818         c.f = stdout;
5819         c.printemptyrecords = (bool)options.printemptyrecords;
5820         c.printdatasize = (bool)options.printdatasize;
5821         c.printlmaster = false;
5822         c.printhash = (bool)options.printhash;
5823         c.printrecordflags = (bool)options.printrecordflags;
5824
5825         for (i=0; i < m->count; i++) {
5826                 uint32_t reqid = 0;
5827                 TDB_DATA key, data;
5828
5829                 /* we do not want the header splitted, so we pass NULL*/
5830                 rec = ctdb_marshall_loop_next(m, rec, &reqid,
5831                                               NULL, &key, &data);
5832
5833                 ctdb_dumpdb_record(ctdb, key, data, &c);
5834         }
5835
5836         printf("Dumped %d records\n", i);
5837         talloc_free(tmp_ctx);
5838         return 0;
5839 }
5840
5841 /*
5842  * wipe a database from a file
5843  */
5844 static int control_wipedb(struct ctdb_context *ctdb, int argc,
5845                           const char **argv)
5846 {
5847         const char *db_name;
5848         int ret;
5849         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
5850         TDB_DATA data;
5851         struct ctdb_db_context *ctdb_db;
5852         struct ctdb_node_map *nodemap = NULL;
5853         struct ctdb_vnn_map *vnnmap = NULL;
5854         int i;
5855         struct ctdb_control_wipe_database w;
5856         uint32_t *nodes;
5857         uint32_t generation;
5858         uint8_t flags;
5859
5860         assert_single_node_only();
5861
5862         if (argc != 1) {
5863                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
5864                 return -1;
5865         }
5866
5867         if (!db_exists(ctdb, argv[0], NULL, &db_name, &flags)) {
5868                 return -1;
5869         }
5870
5871         ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, flags & CTDB_DB_FLAGS_PERSISTENT, 0);
5872         if (ctdb_db == NULL) {
5873                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n",
5874                                   argv[0]));
5875                 talloc_free(tmp_ctx);
5876                 return -1;
5877         }
5878
5879         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb,
5880                                    &nodemap);
5881         if (ret != 0) {
5882                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
5883                                   options.pnn));
5884                 talloc_free(tmp_ctx);
5885                 return ret;
5886         }
5887
5888         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
5889                                   &vnnmap);
5890         if (ret != 0) {
5891                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
5892                                   options.pnn));
5893                 talloc_free(tmp_ctx);
5894                 return ret;
5895         }
5896
5897         /* freeze all nodes */
5898         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5899         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
5900                 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
5901                                                 nodes, i,
5902                                                 TIMELIMIT(),
5903                                                 false, tdb_null,
5904                                                 NULL, NULL,
5905                                                 NULL);
5906                 if (ret != 0) {
5907                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
5908                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn,
5909                                              CTDB_RECOVERY_ACTIVE);
5910                         talloc_free(tmp_ctx);
5911                         return -1;
5912                 }
5913         }
5914
5915         generation = vnnmap->generation;
5916         data.dptr = (void *)&generation;
5917         data.dsize = sizeof(generation);
5918
5919         /* start a cluster wide transaction */
5920         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5921         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
5922                                         nodes, 0,
5923                                         TIMELIMIT(), false, data,
5924                                         NULL, NULL,
5925                                         NULL);
5926         if (ret!= 0) {
5927                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide "
5928                                   "transactions.\n"));
5929                 return -1;
5930         }
5931
5932         w.db_id = ctdb_db->db_id;
5933         w.transaction_id = generation;
5934
5935         data.dptr = (void *)&w;
5936         data.dsize = sizeof(w);
5937
5938         /* wipe all the remote databases. */
5939         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5940         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
5941                                         nodes, 0,
5942                                         TIMELIMIT(), false, data,
5943                                         NULL, NULL,
5944                                         NULL) != 0) {
5945                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
5946                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5947                 talloc_free(tmp_ctx);
5948                 return -1;
5949         }
5950
5951         data.dptr = (void *)&ctdb_db->db_id;
5952         data.dsize = sizeof(ctdb_db->db_id);
5953
5954         /* mark the database as healthy */
5955         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5956         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
5957                                         nodes, 0,
5958                                         TIMELIMIT(), false, data,
5959                                         NULL, NULL,
5960                                         NULL) != 0) {
5961                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
5962                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5963                 talloc_free(tmp_ctx);
5964                 return -1;
5965         }
5966
5967         data.dptr = (void *)&generation;
5968         data.dsize = sizeof(generation);
5969
5970         /* commit all the changes */
5971         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
5972                                         nodes, 0,
5973                                         TIMELIMIT(), false, data,
5974                                         NULL, NULL,
5975                                         NULL) != 0) {
5976                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
5977                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5978                 talloc_free(tmp_ctx);
5979                 return -1;
5980         }
5981
5982         /* thaw all nodes */
5983         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
5984         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
5985                                         nodes, 0,
5986                                         TIMELIMIT(),
5987                                         false, tdb_null,
5988                                         NULL, NULL,
5989                                         NULL) != 0) {
5990                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
5991                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
5992                 talloc_free(tmp_ctx);
5993                 return -1;
5994         }
5995
5996         DEBUG(DEBUG_ERR, ("Database wiped.\n"));
5997
5998         talloc_free(tmp_ctx);
5999         return 0;
6000 }
6001
6002 /*
6003   dump memory usage
6004  */
6005 static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
6006 {
6007         TDB_DATA data;
6008         int ret;
6009         int32_t res;
6010         char *errmsg;
6011         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
6012         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_DUMP_MEMORY,
6013                            0, tdb_null, tmp_ctx, &data, &res, NULL, &errmsg);
6014         if (ret != 0 || res != 0) {
6015                 DEBUG(DEBUG_ERR,("Failed to dump memory - %s\n", errmsg));
6016                 talloc_free(tmp_ctx);
6017                 return -1;
6018         }
6019         sys_write(1, data.dptr, data.dsize);
6020         talloc_free(tmp_ctx);
6021         return 0;
6022 }
6023
6024 /*
6025   handler for memory dumps
6026 */
6027 static void mem_dump_handler(struct ctdb_context *ctdb, uint64_t srvid,
6028                              TDB_DATA data, void *private_data)
6029 {
6030         sys_write(1, data.dptr, data.dsize);
6031         exit(0);
6032 }
6033
6034 /*
6035   dump memory usage on the recovery daemon
6036  */
6037 static int control_rddumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
6038 {
6039         int ret;
6040         TDB_DATA data;
6041         struct srvid_request rd;
6042
6043         rd.pnn = ctdb_get_pnn(ctdb);
6044         rd.srvid = getpid();
6045
6046         /* register a message port for receiveing the reply so that we
6047            can receive the reply
6048         */
6049         ctdb_client_set_message_handler(ctdb, rd.srvid, mem_dump_handler, NULL);
6050
6051
6052         data.dptr = (uint8_t *)&rd;
6053         data.dsize = sizeof(rd);
6054
6055         ret = ctdb_client_send_message(ctdb, options.pnn, CTDB_SRVID_MEM_DUMP, data);
6056         if (ret != 0) {
6057                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
6058                 return -1;
6059         }
6060
6061         /* this loop will terminate when we have received the reply */
6062         while (1) {
6063                 event_loop_once(ctdb->ev);
6064         }
6065
6066         return 0;
6067 }
6068
6069 /*
6070   send a message to a srvid
6071  */
6072 static int control_msgsend(struct ctdb_context *ctdb, int argc, const char **argv)
6073 {
6074         unsigned long srvid;
6075         int ret;
6076         TDB_DATA data;
6077
6078         if (argc < 2) {
6079                 usage();
6080         }
6081
6082         srvid      = strtoul(argv[0], NULL, 0);
6083
6084         data.dptr = (uint8_t *)discard_const(argv[1]);
6085         data.dsize= strlen(argv[1]);
6086
6087         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, srvid, data);
6088         if (ret != 0) {
6089                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
6090                 return -1;
6091         }
6092
6093         return 0;
6094 }
6095
6096 /*
6097   handler for msglisten
6098 */
6099 static void msglisten_handler(struct ctdb_context *ctdb, uint64_t srvid,
6100                              TDB_DATA data, void *private_data)
6101 {
6102         int i;
6103
6104         printf("Message received: ");
6105         for (i=0;i<data.dsize;i++) {
6106                 printf("%c", data.dptr[i]);
6107         }
6108         printf("\n");
6109 }
6110
6111 /*
6112   listen for messages on a messageport
6113  */
6114 static int control_msglisten(struct ctdb_context *ctdb, int argc, const char **argv)
6115 {
6116         uint64_t srvid;
6117
6118         srvid = getpid();
6119
6120         /* register a message port and listen for messages
6121         */
6122         ctdb_client_set_message_handler(ctdb, srvid, msglisten_handler, NULL);
6123         printf("Listening for messages on srvid:%d\n", (int)srvid);
6124
6125         while (1) {
6126                 event_loop_once(ctdb->ev);
6127         }
6128
6129         return 0;
6130 }
6131
6132 /*
6133   list all nodes in the cluster
6134   we parse the nodes file directly
6135  */
6136 static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
6137 {
6138         TALLOC_CTX *mem_ctx = talloc_new(NULL);
6139         struct ctdb_node_map *node_map;
6140         int i;
6141
6142         assert_single_node_only();
6143
6144         node_map = read_nodes_file(mem_ctx);
6145         if (node_map == NULL) {
6146                 talloc_free(mem_ctx);
6147                 return -1;
6148         }
6149
6150         for (i = 0; i < node_map->num; i++) {
6151                 const char *addr;
6152
6153                 if (node_map->nodes[i].flags & NODE_FLAGS_DELETED) {
6154                         continue;
6155                 }
6156                 addr = ctdb_addr_to_str(&node_map->nodes[i].addr);
6157                 if (options.machinereadable){
6158                         printm(":%d:%s:\n", node_map->nodes[i].pnn, addr);
6159                 } else {
6160                         printf("%s\n", addr);
6161                 }
6162         }
6163         talloc_free(mem_ctx);
6164
6165         return 0;
6166 }
6167
6168 /**********************************************************************/
6169 /* reload the nodes file on all nodes */
6170
6171 static void get_nodes_files_callback(struct ctdb_context *ctdb,
6172                                      uint32_t node_pnn, int32_t res,
6173                                      TDB_DATA outdata, void *callback_data)
6174 {
6175         struct ctdb_node_map **maps =
6176                 talloc_get_type(callback_data, struct ctdb_node_map *);
6177
6178         if (outdata.dsize < offsetof(struct ctdb_node_map, nodes) ||
6179             outdata.dptr == NULL) {
6180                 DEBUG(DEBUG_ERR,
6181                       (__location__ " Invalid return data: %u %p\n",
6182                        (unsigned)outdata.dsize, outdata.dptr));
6183                 return;
6184         }
6185
6186         if (node_pnn >= talloc_array_length(maps)) {
6187                 DEBUG(DEBUG_ERR,
6188                       (__location__ " unexpected PNN %u\n", node_pnn));
6189                 return;
6190         }
6191
6192         maps[node_pnn] = talloc_memdup(maps, outdata.dptr, outdata.dsize);
6193 }
6194
6195 static void get_nodes_files_fail_callback(struct ctdb_context *ctdb,
6196                                           uint32_t node_pnn, int32_t res,
6197                                           TDB_DATA outdata, void *callback_data)
6198 {
6199         DEBUG(DEBUG_ERR,
6200               ("ERROR: Failed to get nodes file from node %u\n", node_pnn));
6201 }
6202
6203 static struct ctdb_node_map **
6204 ctdb_get_nodes_files(struct ctdb_context *ctdb,
6205                      TALLOC_CTX *mem_ctx,
6206                      struct timeval timeout,
6207                      struct ctdb_node_map *nodemap)
6208 {
6209         uint32_t *nodes;
6210         int ret;
6211         struct ctdb_node_map **maps;
6212
6213         maps = talloc_zero_array(mem_ctx, struct ctdb_node_map *, nodemap->num);
6214         CTDB_NO_MEMORY_NULL(ctdb, maps);
6215
6216         nodes = list_of_connected_nodes(ctdb, nodemap, mem_ctx, true);
6217
6218         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_NODES_FILE,
6219                                         nodes, 0, TIMELIMIT(),
6220                                         true, tdb_null,
6221                                         get_nodes_files_callback,
6222                                         get_nodes_files_fail_callback,
6223                                         maps);
6224         if (ret != 0) {
6225                 talloc_free(maps);
6226                 return NULL;
6227         }
6228
6229         return maps;
6230 }
6231
6232 static bool node_files_are_identical(struct ctdb_node_map *nm1,
6233                                      struct ctdb_node_map *nm2)
6234 {
6235         int i;
6236
6237         if (nm1->num != nm2->num) {
6238                 return false;
6239         }
6240         for (i = 0; i < nm1->num; i++) {
6241                 if (memcmp(&nm1->nodes[i], &nm2->nodes[i],
6242                            sizeof(struct ctdb_node_and_flags)) != 0) {
6243                         return false;
6244                 }
6245         }
6246
6247         return true;
6248 }
6249
6250 static bool check_all_node_files_are_identical(struct ctdb_context *ctdb,
6251                                                TALLOC_CTX *mem_ctx,
6252                                                struct timeval timeout,
6253                                                struct ctdb_node_map *nodemap,
6254                                                struct ctdb_node_map *file_nodemap)
6255 {
6256         static struct ctdb_node_map **maps;
6257         int i;
6258         bool ret = true;
6259
6260         maps = ctdb_get_nodes_files(ctdb, mem_ctx, timeout, nodemap);
6261         if (maps == NULL) {
6262                 return false;
6263         }
6264
6265         for (i = 0; i < talloc_array_length(maps); i++) {
6266                 if (maps[i] == NULL) {
6267                         continue;
6268                 }
6269                 if (!node_files_are_identical(file_nodemap, maps[i])) {
6270                         DEBUG(DEBUG_ERR,
6271                               ("ERROR: Node file on node %u differs from current node (%u)\n",
6272                                i, ctdb_get_pnn(ctdb)));
6273                         ret = false;
6274                 }
6275         }
6276
6277         return ret;
6278 }
6279
6280 /*
6281   reload the nodes file on the local node
6282  */
6283 static bool sanity_check_nodes_file_changes(TALLOC_CTX *mem_ctx,
6284                                             struct ctdb_node_map *nodemap,
6285                                             struct ctdb_node_map *file_nodemap)
6286 {
6287         int i;
6288         bool should_abort = false;
6289         bool have_changes = false;
6290
6291         for (i=0; i<nodemap->num; i++) {
6292                 if (i >= file_nodemap->num) {
6293                         DEBUG(DEBUG_ERR,
6294                               ("ERROR: Node %u (%s) missing from nodes file\n",
6295                                nodemap->nodes[i].pnn,
6296                                ctdb_addr_to_str(&nodemap->nodes[i].addr)));
6297                         should_abort = true;
6298                         continue;
6299                 }
6300                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED &&
6301                     file_nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
6302                         /* Node remains deleted */
6303                         DEBUG(DEBUG_INFO,
6304                               ("Node %u is unchanged (DELETED)\n",
6305                                nodemap->nodes[i].pnn));
6306                 } else if (!(nodemap->nodes[i].flags & NODE_FLAGS_DELETED) &&
6307                            !(file_nodemap->nodes[i].flags & NODE_FLAGS_DELETED)) {
6308                         /* Node not newly nor previously deleted */
6309                         if (!ctdb_same_ip(&nodemap->nodes[i].addr,
6310                                           &file_nodemap->nodes[i].addr)) {
6311                                 DEBUG(DEBUG_ERR,
6312                                       ("ERROR: Node %u has changed IP address (was %s, now %s)\n",
6313                                        nodemap->nodes[i].pnn,
6314                                        /* ctdb_addr_to_str() returns a static */
6315                                        talloc_strdup(mem_ctx,
6316                                                      ctdb_addr_to_str(&nodemap->nodes[i].addr)),
6317                                        ctdb_addr_to_str(&file_nodemap->nodes[i].addr)));
6318                                 should_abort = true;
6319                         } else {
6320                                 DEBUG(DEBUG_INFO,
6321                                       ("Node %u is unchanged\n",
6322                                        nodemap->nodes[i].pnn));
6323                                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
6324                                         DEBUG(DEBUG_WARNING,
6325                                               ("WARNING: Node %u is disconnected. You MUST fix this node manually!\n",
6326                                                nodemap->nodes[i].pnn));
6327                                 }
6328                         }
6329                 } else if (file_nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
6330                         /* Node is being deleted */
6331                         DEBUG(DEBUG_NOTICE,
6332                               ("Node %u is DELETED\n",
6333                                nodemap->nodes[i].pnn));
6334                         have_changes = true;
6335                         if (!(nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED)) {
6336                                 DEBUG(DEBUG_ERR,
6337                                       ("ERROR: Node %u is still connected\n",
6338                                        nodemap->nodes[i].pnn));
6339                                 should_abort = true;
6340                         }
6341                 } else if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
6342                         /* Node was previously deleted */
6343                         DEBUG(DEBUG_NOTICE,
6344                               ("Node %u is UNDELETED\n", nodemap->nodes[i].pnn));
6345                         have_changes = true;
6346                 }
6347         }
6348
6349         if (should_abort) {
6350                 DEBUG(DEBUG_ERR,
6351                       ("ERROR: Nodes will not be reloaded due to previous error\n"));
6352                 talloc_free(mem_ctx);
6353                 exit(1);
6354         }
6355
6356         /* Leftover nodes in file are NEW */
6357         for (; i < file_nodemap->num; i++) {
6358                 DEBUG(DEBUG_NOTICE, ("Node %u is NEW\n",
6359                                      file_nodemap->nodes[i].pnn));
6360                 have_changes = true;
6361         }
6362
6363         return have_changes;
6364 }
6365
6366 static void reload_nodes_fail_callback(struct ctdb_context *ctdb,
6367                                        uint32_t node_pnn, int32_t res,
6368                                        TDB_DATA outdata, void *callback_data)
6369 {
6370         DEBUG(DEBUG_WARNING,
6371               ("WARNING: Node %u failed to reload nodes. You MUST fix this node manually!\n",
6372                node_pnn));
6373 }
6374
6375 static int control_reload_nodes_file(struct ctdb_context *ctdb, int argc, const char **argv)
6376 {
6377         int i, ret;
6378         struct ctdb_node_map *nodemap=NULL;
6379         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
6380         struct ctdb_node_map *file_nodemap;
6381         uint32_t *conn;
6382         uint32_t timeout;
6383
6384         assert_current_node_only(ctdb);
6385
6386         /* Load both the current nodemap and the contents of the local
6387          * nodes file.  Compare and sanity check them before doing
6388          * anything. */
6389
6390         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
6391         if (ret != 0) {
6392                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
6393                 return ret;
6394         }
6395
6396         file_nodemap = read_nodes_file(tmp_ctx);
6397         if (file_nodemap == NULL) {
6398                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
6399                 talloc_free(tmp_ctx);
6400                 return -1;
6401         }
6402
6403         if (!check_all_node_files_are_identical(ctdb, tmp_ctx, TIMELIMIT(),
6404                                                 nodemap, file_nodemap)) {
6405                 return -1;
6406         }
6407
6408         if (!sanity_check_nodes_file_changes(tmp_ctx, nodemap, file_nodemap)) {
6409                 DEBUG(DEBUG_NOTICE,
6410                       ("No change in nodes file, skipping unnecessary reload\n"));
6411                 talloc_free(tmp_ctx);
6412                 return 0;
6413         }
6414
6415         /* Now make the changes */
6416         conn = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
6417         for (i = 0; i < talloc_array_length(conn); i++) {
6418                 DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n",
6419                                      conn[i]));
6420         }
6421
6422         /* Another timeout could be used, such as ReRecoveryTimeout or
6423          * a new one for this purpose.  However, this is the simplest
6424          * option. */
6425         timeout = options.timelimit;
6426         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_RECOVERIES, &timeout,
6427                         "Disable recoveries", true);
6428
6429
6430         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELOAD_NODES_FILE,
6431                                         conn, 0, TIMELIMIT(),
6432                                         true, tdb_null,
6433                                         NULL, reload_nodes_fail_callback,
6434                                         NULL);
6435
6436         timeout = 0;
6437         srvid_broadcast(ctdb, CTDB_SRVID_DISABLE_RECOVERIES, &timeout,
6438                         "Enable recoveries", true);
6439
6440         talloc_free(tmp_ctx);
6441
6442         return 0;
6443 }
6444
6445
6446 static const struct {
6447         const char *name;
6448         int (*fn)(struct ctdb_context *, int, const char **);
6449         bool auto_all;
6450         bool without_daemon; /* can be run without daemon running ? */
6451         const char *msg;
6452         const char *args;
6453 } ctdb_commands[] = {
6454         { "version",         control_version,           true,   true,   "show version of ctdb" },
6455         { "status",          control_status,            true,   false,  "show node status" },
6456         { "uptime",          control_uptime,            true,   false,  "show node uptime" },
6457         { "ping",            control_ping,              true,   false,  "ping all nodes" },
6458         { "runstate",        control_runstate,          true,   false,  "get/check runstate of a node", "[setup|first_recovery|startup|running]" },
6459         { "getvar",          control_getvar,            true,   false,  "get a tunable variable",               "<name>"},
6460         { "setvar",          control_setvar,            true,   false,  "set a tunable variable",               "<name> <value>"},
6461         { "listvars",        control_listvars,          true,   false,  "list tunable variables"},
6462         { "statistics",      control_statistics,        false,  false, "show statistics" },
6463         { "statisticsreset", control_statistics_reset,  true,   false,  "reset statistics"},
6464         { "stats",           control_stats,             false,  false,  "show rolling statistics", "[number of history records]" },
6465         { "ip",              control_ip,                false,  false,  "show which public ip's that ctdb manages" },
6466         { "ipinfo",          control_ipinfo,            true,   false,  "show details about a public ip that ctdb manages", "<ip>" },
6467         { "ifaces",          control_ifaces,            true,   false,  "show which interfaces that ctdb manages" },
6468         { "setifacelink",    control_setifacelink,      true,   false,  "set interface link status", "<iface> <status>" },
6469         { "process-exists",  control_process_exists,    true,   false,  "check if a process exists on a node",  "<pid>"},
6470         { "getdbmap",        control_getdbmap,          true,   false,  "show the database map" },
6471         { "getdbstatus",     control_getdbstatus,       true,   false,  "show the status of a database", "<dbname|dbid>" },
6472         { "catdb",           control_catdb,             true,   false,  "dump a ctdb database" ,                     "<dbname|dbid>"},
6473         { "cattdb",          control_cattdb,            true,   false,  "dump a local tdb database" ,                     "<dbname|dbid>"},
6474         { "getmonmode",      control_getmonmode,        true,   false,  "show monitoring mode" },
6475         { "getcapabilities", control_getcapabilities,   true,   false,  "show node capabilities" },
6476         { "pnn",             control_pnn,               true,   false,  "show the pnn of the currnet node" },
6477         { "lvs",             control_lvs,               true,   false,  "show lvs configuration" },
6478         { "lvsmaster",       control_lvsmaster,         true,   false,  "show which node is the lvs master" },
6479         { "disablemonitor",      control_disable_monmode,true,  false,  "set monitoring mode to DISABLE" },
6480         { "enablemonitor",      control_enable_monmode, true,   false,  "set monitoring mode to ACTIVE" },
6481         { "setdebug",        control_setdebug,          true,   false,  "set debug level",                      "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
6482         { "getdebug",        control_getdebug,          true,   false,  "get debug level" },
6483         { "attach",          control_attach,            true,   false,  "attach to a database",                 "<dbname> [persistent]" },
6484         { "detach",          control_detach,            false,  false,  "detach from a database",                 "<dbname|dbid> [<dbname|dbid> ...]" },
6485         { "dumpmemory",      control_dumpmemory,        true,   false,  "dump memory map to stdout" },
6486         { "rddumpmemory",    control_rddumpmemory,      true,   false,  "dump memory map from the recovery daemon to stdout" },
6487         { "getpid",          control_getpid,            true,   false,  "get ctdbd process ID" },
6488         { "disable",         control_disable,           true,   false,  "disable a nodes public IP" },
6489         { "enable",          control_enable,            true,   false,  "enable a nodes public IP" },
6490         { "stop",            control_stop,              true,   false,  "stop a node" },
6491         { "continue",        control_continue,          true,   false,  "re-start a stopped node" },
6492         { "ban",             control_ban,               true,   false,  "ban a node from the cluster",          "<bantime>"},
6493         { "unban",           control_unban,             true,   false,  "unban a node" },
6494         { "showban",         control_showban,           true,   false,  "show ban information"},
6495         { "shutdown",        control_shutdown,          true,   false,  "shutdown ctdbd" },
6496         { "recover",         control_recover,           true,   false,  "force recovery" },
6497         { "sync",            control_ipreallocate,      false,  false,  "wait until ctdbd has synced all state changes" },
6498         { "ipreallocate",    control_ipreallocate,      false,  false,  "force the recovery daemon to perform a ip reallocation procedure" },
6499         { "thaw",            control_thaw,              true,   false,  "thaw databases", "[priority:1-3]" },
6500         { "isnotrecmaster",  control_isnotrecmaster,    false,  false,  "check if the local node is recmaster or not" },
6501         { "killtcp",         kill_tcp,                  false,  false, "kill a tcp connection.", "[<srcip:port> <dstip:port>]" },
6502         { "gratiousarp",     control_gratious_arp,      false,  false, "send a gratious arp", "<ip> <interface>" },
6503         { "tickle",          tickle_tcp,                false,  false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
6504         { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip> [<port>]" },
6505         { "addtickle",       control_add_tickle,        false,  false, "add a tickle for this ip", "<ip>:<port> <ip>:<port>" },
6506
6507         { "deltickle",       control_del_tickle,        false,  false, "delete a tickle from this ip", "<ip>:<port> <ip>:<port>" },
6508
6509         { "regsrvid",        regsrvid,                  false,  false, "register a server id", "<pnn> <type> <id>" },
6510         { "unregsrvid",      unregsrvid,                false,  false, "unregister a server id", "<pnn> <type> <id>" },
6511         { "chksrvid",        chksrvid,                  false,  false, "check if a server id exists", "<pnn> <type> <id>" },
6512         { "getsrvids",       getsrvids,                 false,  false, "get a list of all server ids"},
6513         { "check_srvids",    check_srvids,              false,  false, "check if a srvid exists", "<id>+" },
6514         { "repack",          ctdb_repack,               false,  false, "repack all databases", "[max_freelist]"},
6515         { "listnodes",       control_listnodes,         false,  true, "list all nodes in the cluster"},
6516         { "reloadnodes",     control_reload_nodes_file, false,  false, "reload the nodes file and restart the transport on all nodes"},
6517         { "moveip",          control_moveip,            false,  false, "move/failover an ip address to another node", "<ip> <node>"},
6518         { "rebalanceip",     control_rebalanceip,       false,  false, "release an ip from the node and let recd rebalance it", "<ip>"},
6519         { "addip",           control_addip,             true,   false, "add a ip address to a node", "<ip/mask> <iface>"},
6520         { "delip",           control_delip,             false,  false, "delete an ip address from a node", "<ip>"},
6521         { "eventscript",     control_eventscript,       true,   false, "run the eventscript with the given parameters on a node", "<arguments>"},
6522         { "backupdb",        control_backupdb,          false,  false, "backup the database into a file.", "<dbname|dbid> <file>"},
6523         { "restoredb",        control_restoredb,        false,  false, "restore the database from a file.", "<file> [dbname]"},
6524         { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
6525         { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname|dbid>"},
6526         { "recmaster",        control_recmaster,        true,   false, "show the pnn for the recovery master."},
6527         { "scriptstatus",     control_scriptstatus,     true,   false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
6528         { "enablescript",     control_enablescript,  true,      false, "enable an eventscript", "<script>"},
6529         { "disablescript",    control_disablescript,  true,     false, "disable an eventscript", "<script>"},
6530         { "natgwlist",        control_natgwlist,        true,   false, "show the nodes belonging to this natgw configuration"},
6531         { "xpnn",             control_xpnn,             false,  true,  "find the pnn of the local node without talking to the daemon (unreliable)" },
6532         { "getreclock",       control_getreclock,       true,   false, "Show the reclock file of a node"},
6533         { "setreclock",       control_setreclock,       true,   false, "Set/clear the reclock file of a node", "[filename]"},
6534         { "setnatgwstate",    control_setnatgwstate,    false,  false, "Set NATGW state to on/off", "{on|off}"},
6535         { "setlmasterrole",   control_setlmasterrole,   false,  false, "Set LMASTER role to on/off", "{on|off}"},
6536         { "setrecmasterrole", control_setrecmasterrole, false,  false, "Set RECMASTER role to on/off", "{on|off}"},
6537         { "setdbprio",        control_setdbprio,        false,  false, "Set DB priority", "<dbname|dbid> <prio:1-3>"},
6538         { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbname|dbid>"},
6539         { "setdbreadonly",    control_setdbreadonly,    false,  false, "Set DB readonly capable", "<dbname|dbid>"},
6540         { "setdbsticky",      control_setdbsticky,      false,  false, "Set DB sticky-records capable", "<dbname|dbid>"},
6541         { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
6542         { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
6543         { "pfetch",          control_pfetch,            false,  false,  "fetch a record from a persistent database", "<dbname|dbid> <key> [<file>]" },
6544         { "pstore",          control_pstore,            false,  false,  "write a record to a persistent database", "<dbname|dbid> <key> <file containing record>" },
6545         { "pdelete",         control_pdelete,           false,  false,  "delete a record from a persistent database", "<dbname|dbid> <key>" },
6546         { "ptrans",          control_ptrans,            false,  false,  "update a persistent database (from stdin)", "<dbname|dbid>" },
6547         { "tfetch",          control_tfetch,            false,  true,  "fetch a record from a [c]tdb-file [-v]", "<tdb-file> <key> [<file>]" },
6548         { "tstore",          control_tstore,            false,  true,  "store a record (including ltdb header)", "<tdb-file> <key> <data> [<rsn> <dmaster> <flags>]" },
6549         { "readkey",         control_readkey,           true,   false,  "read the content off a database key", "<dbname|dbid> <key>" },
6550         { "writekey",        control_writekey,          true,   false,  "write to a database key", "<dbname|dbid> <key> <value>" },
6551         { "checktcpport",    control_chktcpport,        false,  true,  "check if a service is bound to a specific tcp port or not", "<port>" },
6552         { "rebalancenode",     control_rebalancenode,   false,  false, "mark nodes as forced IP rebalancing targets", "[<pnn-list>]"},
6553         { "getdbseqnum",     control_getdbseqnum,       false,  false, "get the sequence number off a database", "<dbname|dbid>" },
6554         { "nodestatus",      control_nodestatus,        true,   false,  "show and return node status", "[<pnn-list>]" },
6555         { "dbstatistics",    control_dbstatistics,      false,  false, "show db statistics", "<dbname|dbid>" },
6556         { "reloadips",       control_reloadips,         false,  false, "reload the public addresses file on specified nodes" , "[<pnn-list>]" },
6557         { "ipiface",         control_ipiface,           false,  true,  "Find which interface an ip address is hosted on", "<ip>" },
6558 };
6559
6560 /*
6561   show usage message
6562  */
6563 static void usage(void)
6564 {
6565         int i;
6566         printf(
6567 "Usage: ctdb [options] <control>\n" \
6568 "Options:\n" \
6569 "   -n <node>          choose node number, or 'all' (defaults to local node)\n"
6570 "   -Y                 generate machine readable output\n"
6571 "   -x <char>          specify delimiter for machine readable output\n"
6572 "   -v                 generate verbose output\n"
6573 "   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
6574         printf("Controls:\n");
6575         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6576                 printf("  %-15s %-27s  %s\n",
6577                        ctdb_commands[i].name,
6578                        ctdb_commands[i].args?ctdb_commands[i].args:"",
6579                        ctdb_commands[i].msg);
6580         }
6581         exit(1);
6582 }
6583
6584
6585 static void ctdb_alarm(int sig)
6586 {
6587         printf("Maximum runtime exceeded - exiting\n");
6588         _exit(ERR_TIMEOUT);
6589 }
6590
6591 /*
6592   main program
6593 */
6594 int main(int argc, const char *argv[])
6595 {
6596         struct ctdb_context *ctdb;
6597         char *nodestring = NULL;
6598         int machineparsable = 0;
6599         struct poptOption popt_options[] = {
6600                 POPT_AUTOHELP
6601                 POPT_CTDB_CMDLINE
6602                 { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
6603                 { "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
6604                 { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machine readable output", NULL },
6605                 { NULL, 'x', POPT_ARG_STRING, &options.machineseparator, 0, "specify separator for machine readable output", "char" },
6606                 { NULL, 'X', POPT_ARG_NONE, &machineparsable, 0, "enable machine parsable output with separator |", NULL },
6607                 { "verbose",    'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL },
6608                 { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" },
6609                 { "print-emptyrecords", 0, POPT_ARG_NONE, &options.printemptyrecords, 0, "print the empty records when dumping databases (catdb, cattdb, dumpdbbackup)", NULL },
6610                 { "print-datasize", 0, POPT_ARG_NONE, &options.printdatasize, 0, "do not print record data when dumping databases, only the data size", NULL },
6611                 { "print-lmaster", 0, POPT_ARG_NONE, &options.printlmaster, 0, "print the record's lmaster in catdb", NULL },
6612                 { "print-hash", 0, POPT_ARG_NONE, &options.printhash, 0, "print the record's hash when dumping databases", NULL },
6613                 { "print-recordflags", 0, POPT_ARG_NONE, &options.printrecordflags, 0, "print the record flags in catdb and dumpdbbackup", NULL },
6614                 POPT_TABLEEND
6615         };
6616         int opt;
6617         const char **extra_argv;
6618         int extra_argc = 0;
6619         int ret=-1, i;
6620         poptContext pc;
6621         struct event_context *ev;
6622         const char *control;
6623
6624         setlinebuf(stdout);
6625
6626         /* set some defaults */
6627         options.maxruntime = 0;
6628         options.timelimit = 10;
6629         options.pnn = CTDB_CURRENT_NODE;
6630
6631         pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
6632
6633         while ((opt = poptGetNextOpt(pc)) != -1) {
6634                 switch (opt) {
6635                 default:
6636                         DEBUG(DEBUG_ERR, ("Invalid option %s: %s\n",
6637                                 poptBadOption(pc, 0), poptStrerror(opt)));
6638                         exit(1);
6639                 }
6640         }
6641
6642         /* setup the remaining options for the main program to use */
6643         extra_argv = poptGetArgs(pc);
6644         if (extra_argv) {
6645                 extra_argv++;
6646                 while (extra_argv[extra_argc]) extra_argc++;
6647         }
6648
6649         if (extra_argc < 1) {
6650                 usage();
6651         }
6652
6653         if (options.maxruntime == 0) {
6654                 const char *ctdb_timeout;
6655                 ctdb_timeout = getenv("CTDB_TIMEOUT");
6656                 if (ctdb_timeout != NULL) {
6657                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
6658                 } else {
6659                         /* default timeout is 120 seconds */
6660                         options.maxruntime = 120;
6661                 }
6662         }
6663
6664         if (machineparsable) {
6665                 options.machineseparator = "|";
6666         }
6667         if (options.machineseparator != NULL) {
6668                 if (strlen(options.machineseparator) != 1) {
6669                         printf("Invalid separator \"%s\" - "
6670                                "must be single character\n",
6671                                options.machineseparator);
6672                         exit(1);
6673                 }
6674
6675                 /* -x implies -Y */
6676                 options.machinereadable = true;
6677         } else if (options.machinereadable) {
6678                 options.machineseparator = ":";
6679         }
6680
6681         signal(SIGALRM, ctdb_alarm);
6682         alarm(options.maxruntime);
6683
6684         control = extra_argv[0];
6685
6686         /* Default value for CTDB_BASE - don't override */
6687         setenv("CTDB_BASE", CTDB_ETCDIR, 0);
6688
6689         ev = event_context_init(NULL);
6690         if (!ev) {
6691                 DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
6692                 exit(1);
6693         }
6694
6695         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
6696                 if (strcmp(control, ctdb_commands[i].name) == 0) {
6697                         break;
6698                 }
6699         }
6700
6701         if (i == ARRAY_SIZE(ctdb_commands)) {
6702                 DEBUG(DEBUG_ERR, ("Unknown control '%s'\n", control));
6703                 exit(1);
6704         }
6705
6706         if (ctdb_commands[i].without_daemon == true) {
6707                 if (nodestring != NULL) {
6708                         DEBUG(DEBUG_ERR, ("Can't specify node(s) with \"ctdb %s\"\n", control));
6709                         exit(1);
6710                 }
6711                 return ctdb_commands[i].fn(NULL, extra_argc-1, extra_argv+1);
6712         }
6713
6714         /* initialise ctdb */
6715         ctdb = ctdb_cmdline_client(ev, TIMELIMIT());
6716
6717         if (ctdb == NULL) {
6718                 uint32_t pnn;
6719                 DEBUG(DEBUG_ERR, ("Failed to init ctdb\n"));
6720
6721                 pnn = find_node_xpnn();
6722                 if (pnn == -1) {
6723                         DEBUG(DEBUG_ERR,
6724                               ("Is this node part of a CTDB cluster?\n"));
6725                 }
6726                 exit(1);
6727         }
6728
6729         /* setup the node number(s) to contact */
6730         if (!parse_nodestring(ctdb, ctdb, nodestring, CTDB_CURRENT_NODE, false,
6731                               &options.nodes, &options.pnn)) {
6732                 usage();
6733         }
6734
6735         if (options.pnn == CTDB_CURRENT_NODE) {
6736                 options.pnn = options.nodes[0];
6737         }
6738
6739         if (ctdb_commands[i].auto_all &&
6740             ((options.pnn == CTDB_BROADCAST_ALL) ||
6741              (options.pnn == CTDB_MULTICAST))) {
6742                 int j;
6743
6744                 ret = 0;
6745                 for (j = 0; j < talloc_array_length(options.nodes); j++) {
6746                         options.pnn = options.nodes[j];
6747                         ret |= ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6748                 }
6749         } else {
6750                 ret = ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
6751         }
6752
6753         talloc_free(ctdb);
6754         talloc_free(ev);
6755         (void)poptFreeContext(pc);
6756
6757         return ret;
6758
6759 }