ctdb/tools/ctdb.c

   1 /*
   2    CTDB control tool
   3
   4    Copyright (C) Amitay Isaacs  2015
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20 #include "replace.h"
  21 #include "system/network.h"
  22 #include "system/filesys.h"
  23 #include "system/time.h"
  24 #include "system/wait.h"
  25
  26 #include <ctype.h>
  27 #include <popt.h>
  28 #include <talloc.h>
  29 #include <tevent.h>
  30 #include <tdb.h>
  31
  32 #include "ctdb_version.h"
  33 #include "lib/util/debug.h"
  34 #include "lib/util/samba_util.h"
  35 #include "lib/util/sys_rw.h"
  36
  37 #include "common/db_hash.h"
  38 #include "common/logging.h"
  39 #include "protocol/protocol.h"
  40 #include "protocol/protocol_api.h"
  41 #include "common/system.h"
  42 #include "client/client.h"
  43
  44 #define TIMEOUT()       timeval_current_ofs(options.timelimit, 0)
  45
  46 #define SRVID_CTDB_TOOL    (CTDB_SRVID_TOOL_RANGE | 0x0001000000000000LL)
  47 #define SRVID_CTDB_PUSHDB  (CTDB_SRVID_TOOL_RANGE | 0x0002000000000000LL)
  48
  49 static struct {
  50         const char *socket;
  51         const char *debuglevelstr;
  52         int timelimit;
  53         int pnn;
  54         int machinereadable;
  55         const char *sep;
  56         int machineparsable;
  57         int verbose;
  58         int maxruntime;
  59         int printemptyrecords;
  60         int printdatasize;
  61         int printlmaster;
  62         int printhash;
  63         int printrecordflags;
  64 } options;
  65
  66 static poptContext pc;
  67
  68 struct ctdb_context {
  69         struct tevent_context *ev;
  70         struct ctdb_client_context *client;
  71         struct ctdb_node_map *nodemap;
  72         uint32_t pnn, cmd_pnn;
  73         uint64_t srvid;
  74 };
  75
  76 static void usage(const char *command);
  77
  78 /*
  79  * Utility Functions
  80  */
  81
  82 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
  83 {
  84         return (tv2->tv_sec - tv->tv_sec) +
  85                (tv2->tv_usec - tv->tv_usec) * 1.0e-6;
  86 }
  87
  88 static struct ctdb_node_and_flags *get_node_by_pnn(
  89                                         struct ctdb_node_map *nodemap,
  90                                         uint32_t pnn)
  91 {
  92         int i;
  93
  94         for (i=0; i<nodemap->num; i++) {
  95                 if (nodemap->node[i].pnn == pnn) {
  96                         return &nodemap->node[i];
  97                 }
  98         }
  99         return NULL;
 100 }
 101
 102 static const char *pretty_print_flags(TALLOC_CTX *mem_ctx, uint32_t flags)
 103 {
 104         static const struct {
 105                 uint32_t flag;
 106                 const char *name;
 107         } flag_names[] = {
 108                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 109                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 110                 { NODE_FLAGS_BANNED,                "BANNED" },
 111                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 112                 { NODE_FLAGS_DELETED,               "DELETED" },
 113                 { NODE_FLAGS_STOPPED,               "STOPPED" },
 114                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 115         };
 116         char *flags_str = NULL;
 117         int i;
 118
 119         for (i=0; i<ARRAY_SIZE(flag_names); i++) {
 120                 if (flags & flag_names[i].flag) {
 121                         if (flags_str == NULL) {
 122                                 flags_str = talloc_asprintf(mem_ctx,
 123                                                 "%s", flag_names[i].name);
 124                         } else {
 125                                 flags_str = talloc_asprintf_append(flags_str,
 126                                                 "|%s", flag_names[i].name);
 127                         }
 128                         if (flags_str == NULL) {
 129                                 return "OUT-OF-MEMORY";
 130                         }
 131                 }
 132         }
 133         if (flags_str == NULL) {
 134                 return "OK";
 135         }
 136
 137         return flags_str;
 138 }
 139
 140 static uint64_t next_srvid(struct ctdb_context *ctdb)
 141 {
 142         ctdb->srvid += 1;
 143         return ctdb->srvid;
 144 }
 145
 146 /*
 147  * Get consistent nodemap information.
 148  *
 149  * If nodemap is already cached, use that. If not get it.
 150  * If the current node is BANNED, then get nodemap from "better" node.
 151  */
 152 static struct ctdb_node_map *get_nodemap(struct ctdb_context *ctdb, bool force)
 153 {
 154         TALLOC_CTX *tmp_ctx;
 155         struct ctdb_node_map *nodemap;
 156         struct ctdb_node_and_flags *node;
 157         uint32_t current_node;
 158         int ret;
 159
 160         if (force) {
 161                 TALLOC_FREE(ctdb->nodemap);
 162         }
 163
 164         if (ctdb->nodemap != NULL) {
 165                 return ctdb->nodemap;
 166         }
 167
 168         tmp_ctx = talloc_new(ctdb);
 169         if (tmp_ctx == NULL) {
 170                 return false;
 171         }
 172
 173         current_node = ctdb->pnn;
 174 again:
 175         ret = ctdb_ctrl_get_nodemap(tmp_ctx, ctdb->ev, ctdb->client,
 176                                     current_node, TIMEOUT(), &nodemap);
 177         if (ret != 0) {
 178                 fprintf(stderr, "Failed to get nodemap from node %u\n",
 179                         current_node);
 180                 goto failed;
 181         }
 182
 183         node = get_node_by_pnn(nodemap, current_node);
 184         if (node->flags & NODE_FLAGS_BANNED) {
 185                 /* Pick next node */
 186                 do {
 187                         current_node = (current_node + 1) % nodemap->num;
 188                         node = get_node_by_pnn(nodemap, current_node);
 189                         if (! (node->flags &
 190                               (NODE_FLAGS_DELETED|NODE_FLAGS_DISCONNECTED))) {
 191                                 break;
 192                         }
 193                 } while (current_node != ctdb->pnn);
 194
 195                 if (current_node == ctdb->pnn) {
 196                         /* Tried all nodes in the cluster */
 197                         fprintf(stderr, "Warning: All nodes are banned.\n");
 198                         goto failed;
 199                 }
 200
 201                 goto again;
 202         }
 203
 204         ctdb->nodemap = talloc_steal(ctdb, nodemap);
 205         return nodemap;
 206
 207 failed:
 208         talloc_free(tmp_ctx);
 209         return NULL;
 210 }
 211
 212 static bool verify_pnn(struct ctdb_context *ctdb, int pnn)
 213 {
 214         struct ctdb_node_map *nodemap;
 215         bool found;
 216         int i;
 217
 218         if (pnn == -1) {
 219                 return false;
 220         }
 221
 222         nodemap = get_nodemap(ctdb, false);
 223         if (nodemap == NULL) {
 224                 return false;
 225         }
 226
 227         found = false;
 228         for (i=0; i<nodemap->num; i++) {
 229                 if (nodemap->node[i].pnn == pnn) {
 230                         found = true;
 231                         break;
 232                 }
 233         }
 234         if (! found) {
 235                 fprintf(stderr, "Node %u does not exist\n", pnn);
 236                 return false;
 237         }
 238
 239         if (nodemap->node[i].flags &
 240             (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED)) {
 241                 fprintf(stderr, "Node %u has status %s\n", pnn,
 242                         pretty_print_flags(ctdb, nodemap->node[i].flags));
 243                 return false;
 244         }
 245
 246         return true;
 247 }
 248
 249 static struct ctdb_node_map *talloc_nodemap(TALLOC_CTX *mem_ctx,
 250                                             struct ctdb_node_map *nodemap)
 251 {
 252         struct ctdb_node_map *nodemap2;
 253
 254         nodemap2 = talloc_zero(mem_ctx, struct ctdb_node_map);
 255         if (nodemap2 == NULL) {
 256                 return NULL;
 257         }
 258
 259         nodemap2->node = talloc_array(nodemap2, struct ctdb_node_and_flags,
 260                                       nodemap->num);
 261         if (nodemap2->node == NULL) {
 262                 talloc_free(nodemap2);
 263                 return NULL;
 264         }
 265
 266         return nodemap2;
 267 }
 268
 269 /*
 270  * Get the number and the list of matching nodes
 271  *
 272  *   nodestring :=  NULL | all | pnn,[pnn,...]
 273  *
 274  * If nodestring is NULL, use the current node.
 275  */
 276 static bool parse_nodestring(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 277                              const char *nodestring,
 278                              struct ctdb_node_map **out)
 279 {
 280         struct ctdb_node_map *nodemap, *nodemap2;
 281         struct ctdb_node_and_flags *node;
 282         int i;
 283
 284         nodemap = get_nodemap(ctdb, false);
 285         if (nodemap == NULL) {
 286                 return false;
 287         }
 288
 289         nodemap2 = talloc_nodemap(mem_ctx, nodemap);
 290         if (nodemap2 == NULL) {
 291                 return false;
 292         }
 293
 294         if (nodestring == NULL) {
 295                 for (i=0; i<nodemap->num; i++) {
 296                         if (nodemap->node[i].pnn == ctdb->cmd_pnn) {
 297                                 nodemap2->node[0] = nodemap->node[i];
 298                                 break;
 299                         }
 300                 }
 301                 nodemap2->num = 1;
 302
 303                 goto done;
 304         }
 305
 306         if (strcmp(nodestring, "all") == 0) {
 307                 for (i=0; i<nodemap->num; i++) {
 308                         nodemap2->node[i] = nodemap->node[i];
 309                 }
 310                 nodemap2->num = nodemap->num;
 311
 312                 goto done;
 313         } else {
 314                 char *ns, *tok;
 315
 316                 ns = talloc_strdup(mem_ctx, nodestring);
 317                 if (ns == NULL) {
 318                         return false;
 319                 }
 320
 321                 tok = strtok(ns, ",");
 322                 while (tok != NULL) {
 323                         uint32_t pnn;
 324                         char *endptr;
 325
 326                         pnn = (uint32_t)strtoul(tok, &endptr, 0);
 327                         if (pnn == 0 && tok == endptr) {
 328                                 fprintf(stderr, "Invalid node %s\n", tok);
 329                                         return false;
 330                         }
 331
 332                         node = get_node_by_pnn(nodemap, pnn);
 333                         if (node == NULL) {
 334                                 fprintf(stderr, "Node %u does not exist\n",
 335                                         pnn);
 336                                 return false;
 337                         }
 338
 339                         nodemap2->node[nodemap2->num] = *node;
 340                         nodemap2->num += 1;
 341
 342                         tok = strtok(NULL, ",");
 343                 }
 344         }
 345
 346 done:
 347         *out = nodemap2;
 348         return true;
 349 }
 350
 351 /* Compare IP address */
 352 static bool ctdb_same_ip(ctdb_sock_addr *ip1, ctdb_sock_addr *ip2)
 353 {
 354         bool ret = false;
 355
 356         if (ip1->sa.sa_family != ip2->sa.sa_family) {
 357                 return false;
 358         }
 359
 360         switch (ip1->sa.sa_family) {
 361         case AF_INET:
 362                 ret = (memcmp(&ip1->ip.sin_addr, &ip2->ip.sin_addr,
 363                               sizeof(struct in_addr)) == 0);
 364                 break;
 365
 366         case AF_INET6:
 367                 ret = (memcmp(&ip1->ip6.sin6_addr, &ip2->ip6.sin6_addr,
 368                               sizeof(struct in6_addr)) == 0);
 369                 break;
 370         }
 371
 372         return ret;
 373 }
 374
 375 /* Append a node to a node map with given address and flags */
 376 static bool node_map_add(struct ctdb_node_map *nodemap,
 377                          const char *nstr, uint32_t flags)
 378 {
 379         ctdb_sock_addr addr;
 380         uint32_t num;
 381         struct ctdb_node_and_flags *n;
 382
 383         if (! parse_ip(nstr, NULL, 0, &addr)) {
 384                 fprintf(stderr, "Invalid IP address %s\n", nstr);
 385                 return false;
 386         }
 387
 388         num = nodemap->num;
 389         nodemap->node = talloc_realloc(nodemap, nodemap->node,
 390                                        struct ctdb_node_and_flags, num+1);
 391         if (nodemap->node == NULL) {
 392                 return false;
 393         }
 394
 395         n = &nodemap->node[num];
 396         n->addr = addr;
 397         n->pnn = num;
 398         n->flags = flags;
 399
 400         nodemap->num = num+1;
 401         return true;
 402 }
 403
 404 /* Read a nodes file into a node map */
 405 static struct ctdb_node_map *ctdb_read_nodes_file(TALLOC_CTX *mem_ctx,
 406                                                   const char *nlist)
 407 {
 408         char **lines;
 409         int nlines;
 410         int i;
 411         struct ctdb_node_map *nodemap;
 412
 413         nodemap = talloc_zero(mem_ctx, struct ctdb_node_map);
 414         if (nodemap == NULL) {
 415                 return NULL;
 416         }
 417
 418         lines = file_lines_load(nlist, &nlines, 0, mem_ctx);
 419         if (lines == NULL) {
 420                 return NULL;
 421         }
 422
 423         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 424                 nlines--;
 425         }
 426
 427         for (i=0; i<nlines; i++) {
 428                 char *node;
 429                 uint32_t flags;
 430                 size_t len;
 431
 432                 node = lines[i];
 433                 /* strip leading spaces */
 434                 while((*node == ' ') || (*node == '\t')) {
 435                         node++;
 436                 }
 437
 438                 len = strlen(node);
 439
 440                 /* strip trailing spaces */
 441                 while ((len > 1) &&
 442                        ((node[len-1] == ' ') || (node[len-1] == '\t')))
 443                 {
 444                         node[len-1] = '\0';
 445                         len--;
 446                 }
 447
 448                 if (len == 0) {
 449                         continue;
 450                 }
 451                 if (*node == '#') {
 452                         /* A "deleted" node is a node that is
 453                            commented out in the nodes file.  This is
 454                            used instead of removing a line, which
 455                            would cause subsequent nodes to change
 456                            their PNN. */
 457                         flags = NODE_FLAGS_DELETED;
 458                         node = discard_const("0.0.0.0");
 459                 } else {
 460                         flags = 0;
 461                 }
 462                 if (! node_map_add(nodemap, node, flags)) {
 463                         talloc_free(lines);
 464                         TALLOC_FREE(nodemap);
 465                         return NULL;
 466                 }
 467         }
 468
 469         talloc_free(lines);
 470         return nodemap;
 471 }
 472
 473 static struct ctdb_node_map *read_nodes_file(TALLOC_CTX *mem_ctx, uint32_t pnn)
 474 {
 475         struct ctdb_node_map *nodemap;
 476         char *nodepath;
 477         const char *nodes_list = NULL;
 478
 479         if (pnn != CTDB_UNKNOWN_PNN) {
 480                 nodepath = talloc_asprintf(mem_ctx, "CTDB_NODES_%u", pnn);
 481                 if (nodepath != NULL) {
 482                         nodes_list = getenv(nodepath);
 483                 }
 484         }
 485         if (nodes_list == NULL) {
 486                 nodes_list = getenv("CTDB_NODES");
 487         }
 488         if (nodes_list == NULL) {
 489                 const char *basedir = getenv("CTDB_BASE");
 490                 if (basedir == NULL) {
 491                         basedir = CTDB_ETCDIR;
 492                 }
 493                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes", basedir);
 494                 if (nodes_list == NULL) {
 495                         fprintf(stderr, "Memory allocation error\n");
 496                         return NULL;
 497                 }
 498         }
 499
 500         nodemap = ctdb_read_nodes_file(mem_ctx, nodes_list);
 501         if (nodemap == NULL) {
 502                 fprintf(stderr, "Failed to read nodes file \"%s\"\n",
 503                         nodes_list);
 504                 return NULL;
 505         }
 506
 507         return nodemap;
 508 }
 509
 510 static struct ctdb_dbid *db_find(TALLOC_CTX *mem_ctx,
 511                                  struct ctdb_context *ctdb,
 512                                  struct ctdb_dbid_map *dbmap,
 513                                  const char *db_name)
 514 {
 515         struct ctdb_dbid *db = NULL;
 516         const char *name;
 517         int ret, i;
 518
 519         for (i=0; i<dbmap->num; i++) {
 520                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
 521                                            ctdb->pnn, TIMEOUT(),
 522                                            dbmap->dbs[i].db_id, &name);
 523                 if (ret != 0) {
 524                         return false;
 525                 }
 526
 527                 if (strcmp(db_name, name) == 0) {
 528                         talloc_free(discard_const(name));
 529                         db = &dbmap->dbs[i];
 530                         break;
 531                 }
 532         }
 533
 534         return db;
 535 }
 536
 537 static bool db_exists(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 538                       const char *db_arg, uint32_t *db_id,
 539                       const char **db_name, uint8_t *db_flags)
 540 {
 541         struct ctdb_dbid_map *dbmap;
 542         struct ctdb_dbid *db = NULL;
 543         uint32_t id = 0;
 544         const char *name = NULL;
 545         int ret, i;
 546
 547         ret = ctdb_ctrl_get_dbmap(mem_ctx, ctdb->ev, ctdb->client,
 548                                   ctdb->pnn, TIMEOUT(), &dbmap);
 549         if (ret != 0) {
 550                 return false;
 551         }
 552
 553         if (strncmp(db_arg, "0x", 2) == 0) {
 554                 id = strtoul(db_arg, NULL, 0);
 555                 for (i=0; i<dbmap->num; i++) {
 556                         if (id == dbmap->dbs[i].db_id) {
 557                                 db = &dbmap->dbs[i];
 558                                 break;
 559                         }
 560                 }
 561         } else {
 562                 name = db_arg;
 563                 db = db_find(mem_ctx, ctdb, dbmap, name);
 564         }
 565
 566         if (db == NULL) {
 567                 fprintf(stderr, "No database matching '%s' found\n", db_arg);
 568                 return false;
 569         }
 570
 571         if (name == NULL) {
 572                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
 573                                            ctdb->pnn, TIMEOUT(), id, &name);
 574                 if (ret != 0) {
 575                         return false;
 576                 }
 577         }
 578
 579         if (db_id != NULL) {
 580                 *db_id = db->db_id;
 581         }
 582         if (db_name != NULL) {
 583                 *db_name = talloc_strdup(mem_ctx, name);
 584         }
 585         if (db_flags != NULL) {
 586                 *db_flags = db->flags;
 587         }
 588         return true;
 589 }
 590
 591 static int h2i(char h)
 592 {
 593         if (h >= 'a' && h <= 'f') {
 594                 return h - 'a' + 10;
 595         }
 596         if (h >= 'A' && h <= 'F') {
 597                 return h - 'f' + 10;
 598         }
 599         return h - '0';
 600 }
 601
 602 static int hex_to_data(const char *str, size_t len, TALLOC_CTX *mem_ctx,
 603                        TDB_DATA *out)
 604 {
 605         int i;
 606         TDB_DATA data;
 607
 608         if (len & 0x01) {
 609                 fprintf(stderr, "Key (%s) contains odd number of hex digits\n",
 610                         str);
 611                 return EINVAL;
 612         }
 613
 614         data.dsize = len / 2;
 615         data.dptr = talloc_size(mem_ctx, data.dsize);
 616         if (data.dptr == NULL) {
 617                 return ENOMEM;
 618         }
 619
 620         for (i=0; i<data.dsize; i++) {
 621                 data.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
 622         }
 623
 624         *out = data;
 625         return 0;
 626 }
 627
 628 static int str_to_data(const char *str, size_t len, TALLOC_CTX *mem_ctx,
 629                        TDB_DATA *out)
 630 {
 631         TDB_DATA data;
 632         int ret = 0;
 633
 634         if (strncmp(str, "0x", 2) == 0) {
 635                 ret = hex_to_data(str+2, len-2, mem_ctx, &data);
 636         } else {
 637                 data.dptr = talloc_memdup(mem_ctx, str, len);
 638                 if (data.dptr == NULL) {
 639                         return ENOMEM;
 640                 }
 641                 data.dsize = len;
 642         }
 643
 644         *out = data;
 645         return ret;
 646 }
 647
 648 static int run_helper(TALLOC_CTX *mem_ctx, const char *command,
 649                       const char *path, int argc, const char **argv)
 650 {
 651         pid_t pid;
 652         int save_errno, status, ret;
 653         const char **new_argv;
 654         int i;
 655
 656         new_argv = talloc_array(mem_ctx, const char *, argc + 2);
 657         if (new_argv == NULL) {
 658                 return ENOMEM;
 659         }
 660
 661         new_argv[0] = path;
 662         for (i=0; i<argc; i++) {
 663                 new_argv[i+1] = argv[i];
 664         }
 665         new_argv[argc+1] = NULL;
 666
 667         pid = fork();
 668         if (pid < 0) {
 669                 save_errno = errno;
 670                 talloc_free(new_argv);
 671                 fprintf(stderr, "Failed to fork %s (%s) - %s\n",
 672                         command, path, strerror(save_errno));
 673                 return save_errno;
 674         }
 675
 676         if (pid == 0) {
 677                 ret = execv(path, discard_const(new_argv));
 678                 if (ret == -1) {
 679                         _exit(errno);
 680                 }
 681                 /* Should not happen */
 682                 _exit(ENOEXEC);
 683         }
 684
 685         talloc_free(new_argv);
 686
 687         ret = waitpid(pid, &status, 0);
 688         if (ret == -1) {
 689                 save_errno = errno;
 690                 fprintf(stderr, "waitpid() failed for %s - %s\n",
 691                         command, strerror(save_errno));
 692                 return save_errno;
 693         }
 694
 695         if (WIFEXITED(status)) {
 696                 ret = WEXITSTATUS(status);
 697                 return ret;
 698         }
 699
 700         if (WIFSIGNALED(status)) {
 701                 fprintf(stderr, "%s terminated with signal %d\n",
 702                         command, WTERMSIG(status));
 703                 return EINTR;
 704         }
 705
 706         return 0;
 707 }
 708
 709 /*
 710  * Command Functions
 711  */
 712
 713 static int control_version(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 714                            int argc, const char **argv)
 715 {
 716         printf("%s\n", CTDB_VERSION_STRING);
 717         return 0;
 718 }
 719
 720 static bool partially_online(TALLOC_CTX *mem_ctx,
 721                              struct ctdb_context *ctdb,
 722                              struct ctdb_node_and_flags *node)
 723 {
 724         struct ctdb_iface_list *iface_list;
 725         int ret, i;
 726         bool status = false;
 727
 728         if (node->flags != 0) {
 729                 return false;
 730         }
 731
 732         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
 733                                    node->pnn, TIMEOUT(), &iface_list);
 734         if (ret != 0) {
 735                 return false;
 736         }
 737
 738         status = false;
 739         for (i=0; i < iface_list->num; i++) {
 740                 if (iface_list->iface[i].link_state == 0) {
 741                         status = true;
 742                         break;
 743                 }
 744         }
 745
 746         return status;
 747 }
 748
 749 static void print_nodemap_machine(TALLOC_CTX *mem_ctx,
 750                                   struct ctdb_context *ctdb,
 751                                   struct ctdb_node_map *nodemap,
 752                                   uint32_t mypnn)
 753 {
 754         struct ctdb_node_and_flags *node;
 755         int i;
 756
 757         printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
 758                options.sep,
 759                "Node", options.sep,
 760                "IP", options.sep,
 761                "Disconnected", options.sep,
 762                "Banned", options.sep,
 763                "Disabled", options.sep,
 764                "Unhealthy", options.sep,
 765                "Stopped", options.sep,
 766                "Inactive", options.sep,
 767                "PartiallyOnline", options.sep,
 768                "ThisNode", options.sep);
 769
 770         for (i=0; i<nodemap->num; i++) {
 771                 node = &nodemap->node[i];
 772                 if (node->flags & NODE_FLAGS_DELETED) {
 773                         continue;
 774                 }
 775
 776                 printf("%s%u%s%s%s%d%s%d%s%d%s%d%s%d%s%d%s%d%s%c%s\n",
 777                        options.sep,
 778                        node->pnn, options.sep,
 779                        ctdb_sock_addr_to_string(mem_ctx, &node->addr),
 780                        options.sep,
 781                        !! (node->flags & NODE_FLAGS_DISCONNECTED), options.sep,
 782                        !! (node->flags & NODE_FLAGS_BANNED), options.sep,
 783                        !! (node->flags & NODE_FLAGS_PERMANENTLY_DISABLED),
 784                        options.sep,
 785                        !! (node->flags & NODE_FLAGS_UNHEALTHY), options.sep,
 786                        !! (node->flags & NODE_FLAGS_STOPPED), options.sep,
 787                        !! (node->flags & NODE_FLAGS_INACTIVE), options.sep,
 788                        partially_online(mem_ctx, ctdb, node), options.sep,
 789                        (node->pnn == mypnn)?'Y':'N', options.sep);
 790         }
 791
 792 }
 793
 794 static void print_nodemap(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 795                           struct ctdb_node_map *nodemap, uint32_t mypnn)
 796 {
 797         struct ctdb_node_and_flags *node;
 798         int num_deleted_nodes = 0;
 799         int i;
 800
 801         for (i=0; i<nodemap->num; i++) {
 802                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
 803                         num_deleted_nodes++;
 804                 }
 805         }
 806
 807         if (num_deleted_nodes == 0) {
 808                 printf("Number of nodes:%d\n", nodemap->num);
 809         } else {
 810                 printf("Number of nodes:%d (including %d deleted nodes)\n",
 811                        nodemap->num, num_deleted_nodes);
 812         }
 813
 814         for (i=0; i<nodemap->num; i++) {
 815                 node = &nodemap->node[i];
 816                 if (node->flags & NODE_FLAGS_DELETED) {
 817                         continue;
 818                 }
 819
 820                 printf("pnn:%u %-16s %s%s\n",
 821                        node->pnn,
 822                        ctdb_sock_addr_to_string(mem_ctx, &node->addr),
 823                        partially_online(mem_ctx, ctdb, node) ?
 824                                 "PARTIALLYONLINE" :
 825                                 pretty_print_flags(mem_ctx, node->flags),
 826                        node->pnn == mypnn ? " (THIS NODE)" : "");
 827         }
 828 }
 829
 830 static void print_status(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 831                          struct ctdb_node_map *nodemap, uint32_t mypnn,
 832                          struct ctdb_vnn_map *vnnmap, int recmode,
 833                          uint32_t recmaster)
 834 {
 835         int i;
 836
 837         print_nodemap(mem_ctx, ctdb, nodemap, mypnn);
 838
 839         if (vnnmap->generation == INVALID_GENERATION) {
 840                 printf("Generation:INVALID\n");
 841         } else {
 842                 printf("Generation:%u\n", vnnmap->generation);
 843         }
 844         printf("Size:%d\n", vnnmap->size);
 845         for (i=0; i<vnnmap->size; i++) {
 846                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
 847         }
 848
 849         printf("Recovery mode:%s (%d)\n",
 850                recmode == CTDB_RECOVERY_NORMAL ? "NORMAL" : "RECOVERY",
 851                recmode);
 852         printf("Recovery master:%d\n", recmaster);
 853 }
 854
 855 static int control_status(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 856                           int argc, const char **argv)
 857 {
 858         struct ctdb_node_map *nodemap;
 859         struct ctdb_vnn_map *vnnmap;
 860         int recmode;
 861         uint32_t recmaster;
 862         int ret;
 863
 864         if (argc != 0) {
 865                 usage("status");
 866         }
 867
 868         nodemap = get_nodemap(ctdb, false);
 869         if (nodemap == NULL) {
 870                 return 1;
 871         }
 872
 873         if (options.machinereadable == 1) {
 874                 print_nodemap_machine(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn);
 875                 return 0;
 876         }
 877
 878         ret = ctdb_ctrl_getvnnmap(mem_ctx, ctdb->ev, ctdb->client,
 879                                   ctdb->cmd_pnn, TIMEOUT(), &vnnmap);
 880         if (ret != 0) {
 881                 return ret;
 882         }
 883
 884         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
 885                                     ctdb->cmd_pnn, TIMEOUT(), &recmode);
 886         if (ret != 0) {
 887                 return ret;
 888         }
 889
 890         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
 891                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
 892         if (ret != 0) {
 893                 return ret;
 894         }
 895
 896         print_status(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn, vnnmap,
 897                      recmode, recmaster);
 898         return 0;
 899 }
 900
 901 static int control_uptime(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 902                           int argc, const char **argv)
 903 {
 904         struct ctdb_uptime *uptime;
 905         int ret, tmp, days, hours, minutes, seconds;
 906
 907         ret = ctdb_ctrl_uptime(mem_ctx, ctdb->ev, ctdb->client,
 908                                ctdb->cmd_pnn, TIMEOUT(), &uptime);
 909         if (ret != 0) {
 910                 return ret;
 911         }
 912
 913         printf("Current time of node %-4u     :                %s",
 914                ctdb->cmd_pnn, ctime(&uptime->current_time.tv_sec));
 915
 916         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 917         seconds = tmp % 60; tmp /= 60;
 918         minutes = tmp % 60; tmp /= 60;
 919         hours = tmp % 24; tmp /= 24;
 920         days = tmp;
 921
 922         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s",
 923                days, hours, minutes, seconds,
 924                ctime(&uptime->ctdbd_start_time.tv_sec));
 925
 926         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 927         seconds = tmp % 60; tmp /= 60;
 928         minutes = tmp % 60; tmp /= 60;
 929         hours = tmp % 24; tmp /= 24;
 930         days = tmp;
 931
 932         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s",
 933                days, hours, minutes, seconds,
 934                ctime(&uptime->last_recovery_finished.tv_sec));
 935
 936         printf("Duration of last recovery/failover: %lf seconds\n",
 937                timeval_delta(&uptime->last_recovery_finished,
 938                              &uptime->last_recovery_started));
 939
 940         return 0;
 941 }
 942
 943 static int control_ping(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 944                         int argc, const char **argv)
 945 {
 946         struct timeval tv;
 947         int ret, num_clients;
 948
 949         tv = timeval_current();
 950         ret = ctdb_ctrl_ping(mem_ctx, ctdb->ev, ctdb->client,
 951                              ctdb->cmd_pnn, TIMEOUT(), &num_clients);
 952         if (ret != 0) {
 953                 return ret;
 954         }
 955
 956         printf("response from %u time=%.6f sec  (%d clients)\n",
 957                ctdb->cmd_pnn, timeval_elapsed(&tv), num_clients);
 958         return 0;
 959 }
 960
 961 const char *runstate_to_string(enum ctdb_runstate runstate);
 962 enum ctdb_runstate runstate_from_string(const char *runstate_str);
 963
 964 static int control_runstate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 965                             int argc, const char **argv)
 966 {
 967         enum ctdb_runstate runstate;
 968         bool found;
 969         int ret, i;
 970
 971         ret = ctdb_ctrl_get_runstate(mem_ctx, ctdb->ev, ctdb->client,
 972                                      ctdb->cmd_pnn, TIMEOUT(), &runstate);
 973         if (ret != 0) {
 974                 return ret;
 975         }
 976
 977         found = true;
 978         for (i=0; i<argc; i++) {
 979                 enum ctdb_runstate t;
 980
 981                 found = false;
 982                 t = ctdb_runstate_from_string(argv[i]);
 983                 if (t == CTDB_RUNSTATE_UNKNOWN) {
 984                         printf("Invalid run state (%s)\n", argv[i]);
 985                         return 1;
 986                 }
 987
 988                 if (t == runstate) {
 989                         found = true;
 990                         break;
 991                 }
 992         }
 993
 994         if (! found) {
 995                 printf("CTDB not in required run state (got %s)\n",
 996                        ctdb_runstate_to_string(runstate));
 997                 return 1;
 998         }
 999
1000         printf("%s\n", ctdb_runstate_to_string(runstate));
1001         return 0;
1002 }
1003
1004 static int control_getvar(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1005                           int argc, const char **argv)
1006 {
1007         struct ctdb_var_list *tun_var_list;
1008         uint32_t value;
1009         int ret, i;
1010         bool found;
1011
1012         if (argc != 1) {
1013                 usage("getvar");
1014         }
1015
1016         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1017                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1018         if (ret != 0) {
1019                 fprintf(stderr,
1020                         "Failed to get list of variables from node %u\n",
1021                         ctdb->cmd_pnn);
1022                 return ret;
1023         }
1024
1025         found = false;
1026         for (i=0; i<tun_var_list->count; i++) {
1027                 if (strcasecmp(tun_var_list->var[i], argv[0]) == 0) {
1028                         found = true;
1029                         break;
1030                 }
1031         }
1032
1033         if (! found) {
1034                 printf("No such tunable %s\n", argv[0]);
1035                 return 1;
1036         }
1037
1038         ret = ctdb_ctrl_get_tunable(mem_ctx, ctdb->ev, ctdb->client,
1039                                     ctdb->cmd_pnn, TIMEOUT(), argv[0], &value);
1040         if (ret != 0) {
1041                 return ret;
1042         }
1043
1044         printf("%-26s = %u\n", argv[0], value);
1045         return 0;
1046 }
1047
1048 static int control_setvar(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1049                           int argc, const char **argv)
1050 {
1051         struct ctdb_var_list *tun_var_list;
1052         struct ctdb_tunable tunable;
1053         int ret, i;
1054         bool found;
1055
1056         if (argc != 2) {
1057                 usage("setvar");
1058         }
1059
1060         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1061                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1062         if (ret != 0) {
1063                 fprintf(stderr,
1064                         "Failed to get list of variables from node %u\n",
1065                         ctdb->cmd_pnn);
1066                 return ret;
1067         }
1068
1069         found = false;
1070         for (i=0; i<tun_var_list->count; i++) {
1071                 if (strcasecmp(tun_var_list->var[i], argv[0]) == 0) {
1072                         found = true;
1073                         break;
1074                 }
1075         }
1076
1077         if (! found) {
1078                 printf("No such tunable %s\n", argv[0]);
1079                 return 1;
1080         }
1081
1082         tunable.name = argv[0];
1083         tunable.value = strtoul(argv[1], NULL, 0);
1084
1085         ret = ctdb_ctrl_set_tunable(mem_ctx, ctdb->ev, ctdb->client,
1086                                     ctdb->cmd_pnn, TIMEOUT(), &tunable);
1087         if (ret != 0) {
1088                 if (ret == 1) {
1089                         fprintf(stderr,
1090                                 "Setting obsolete tunable variable '%s'\n",
1091                                tunable.name);
1092                         return 0;
1093                 }
1094         }
1095
1096         return ret;
1097 }
1098
1099 static int control_listvars(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1100                             int argc, const char **argv)
1101 {
1102         struct ctdb_var_list *tun_var_list;
1103         int ret, i;
1104
1105         if (argc != 0) {
1106                 usage("listvars");
1107         }
1108
1109         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1110                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1111         if (ret != 0) {
1112                 return ret;
1113         }
1114
1115         for (i=0; i<tun_var_list->count; i++) {
1116                 control_getvar(mem_ctx, ctdb, 1, &tun_var_list->var[i]);
1117         }
1118
1119         return 0;
1120 }
1121
1122 const struct {
1123         const char *name;
1124         uint32_t offset;
1125 } stats_fields[] = {
1126 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
1127         STATISTICS_FIELD(num_clients),
1128         STATISTICS_FIELD(frozen),
1129         STATISTICS_FIELD(recovering),
1130         STATISTICS_FIELD(num_recoveries),
1131         STATISTICS_FIELD(client_packets_sent),
1132         STATISTICS_FIELD(client_packets_recv),
1133         STATISTICS_FIELD(node_packets_sent),
1134         STATISTICS_FIELD(node_packets_recv),
1135         STATISTICS_FIELD(keepalive_packets_sent),
1136         STATISTICS_FIELD(keepalive_packets_recv),
1137         STATISTICS_FIELD(node.req_call),
1138         STATISTICS_FIELD(node.reply_call),
1139         STATISTICS_FIELD(node.req_dmaster),
1140         STATISTICS_FIELD(node.reply_dmaster),
1141         STATISTICS_FIELD(node.reply_error),
1142         STATISTICS_FIELD(node.req_message),
1143         STATISTICS_FIELD(node.req_control),
1144         STATISTICS_FIELD(node.reply_control),
1145         STATISTICS_FIELD(client.req_call),
1146         STATISTICS_FIELD(client.req_message),
1147         STATISTICS_FIELD(client.req_control),
1148         STATISTICS_FIELD(timeouts.call),
1149         STATISTICS_FIELD(timeouts.control),
1150         STATISTICS_FIELD(timeouts.traverse),
1151         STATISTICS_FIELD(locks.num_calls),
1152         STATISTICS_FIELD(locks.num_current),
1153         STATISTICS_FIELD(locks.num_pending),
1154         STATISTICS_FIELD(locks.num_failed),
1155         STATISTICS_FIELD(total_calls),
1156         STATISTICS_FIELD(pending_calls),
1157         STATISTICS_FIELD(childwrite_calls),
1158         STATISTICS_FIELD(pending_childwrite_calls),
1159         STATISTICS_FIELD(memory_used),
1160         STATISTICS_FIELD(max_hop_count),
1161         STATISTICS_FIELD(total_ro_delegations),
1162         STATISTICS_FIELD(total_ro_revokes),
1163 };
1164
1165 #define LATENCY_AVG(v)  ((v).num ? (v).total / (v).num : 0.0 )
1166
1167 static void print_statistics_machine(struct ctdb_statistics *s,
1168                                      bool show_header)
1169 {
1170         int i;
1171
1172         if (show_header) {
1173                 printf("CTDB version%s", options.sep);
1174                 printf("Current time of statistics%s", options.sep);
1175                 printf("Statistics collected since%s", options.sep);
1176                 for (i=0; i<ARRAY_SIZE(stats_fields); i++) {
1177                         printf("%s%s", stats_fields[i].name, options.sep);
1178                 }
1179                 printf("num_reclock_ctdbd_latency%s", options.sep);
1180                 printf("min_reclock_ctdbd_latency%s", options.sep);
1181                 printf("avg_reclock_ctdbd_latency%s", options.sep);
1182                 printf("max_reclock_ctdbd_latency%s", options.sep);
1183
1184                 printf("num_reclock_recd_latency%s", options.sep);
1185                 printf("min_reclock_recd_latency%s", options.sep);
1186                 printf("avg_reclock_recd_latency%s", options.sep);
1187                 printf("max_reclock_recd_latency%s", options.sep);
1188
1189                 printf("num_call_latency%s", options.sep);
1190                 printf("min_call_latency%s", options.sep);
1191                 printf("avg_call_latency%s", options.sep);
1192                 printf("max_call_latency%s", options.sep);
1193
1194                 printf("num_lockwait_latency%s", options.sep);
1195                 printf("min_lockwait_latency%s", options.sep);
1196                 printf("avg_lockwait_latency%s", options.sep);
1197                 printf("max_lockwait_latency%s", options.sep);
1198
1199                 printf("num_childwrite_latency%s", options.sep);
1200                 printf("min_childwrite_latency%s", options.sep);
1201                 printf("avg_childwrite_latency%s", options.sep);
1202                 printf("max_childwrite_latency%s", options.sep);
1203                 printf("\n");
1204         }
1205
1206         printf("%u%s", CTDB_PROTOCOL, options.sep);
1207         printf("%u%s", (uint32_t)s->statistics_current_time.tv_sec, options.sep);
1208         printf("%u%s", (uint32_t)s->statistics_start_time.tv_sec, options.sep);
1209         for (i=0;i<ARRAY_SIZE(stats_fields);i++) {
1210                 printf("%u%s",
1211                        *(uint32_t *)(stats_fields[i].offset+(uint8_t *)s),
1212                        options.sep);
1213         }
1214         printf("%u%s", s->reclock.ctdbd.num, options.sep);
1215         printf("%.6f%s", s->reclock.ctdbd.min, options.sep);
1216         printf("%.6f%s", LATENCY_AVG(s->reclock.ctdbd), options.sep);
1217         printf("%.6f%s", s->reclock.ctdbd.max, options.sep);
1218
1219         printf("%u%s", s->reclock.recd.num, options.sep);
1220         printf("%.6f%s", s->reclock.recd.min, options.sep);
1221         printf("%.6f%s", LATENCY_AVG(s->reclock.recd), options.sep);
1222         printf("%.6f%s", s->reclock.recd.max, options.sep);
1223
1224         printf("%d%s", s->call_latency.num, options.sep);
1225         printf("%.6f%s", s->call_latency.min, options.sep);
1226         printf("%.6f%s", LATENCY_AVG(s->call_latency), options.sep);
1227         printf("%.6f%s", s->call_latency.max, options.sep);
1228
1229         printf("%d%s", s->childwrite_latency.num, options.sep);
1230         printf("%.6f%s", s->childwrite_latency.min, options.sep);
1231         printf("%.6f%s", LATENCY_AVG(s->childwrite_latency), options.sep);
1232         printf("%.6f%s", s->childwrite_latency.max, options.sep);
1233         printf("\n");
1234 }
1235
1236 static void print_statistics(struct ctdb_statistics *s)
1237 {
1238         int tmp, days, hours, minutes, seconds;
1239         int i;
1240         const char *prefix = NULL;
1241         int preflen = 0;
1242
1243         tmp = s->statistics_current_time.tv_sec -
1244               s->statistics_start_time.tv_sec;
1245         seconds = tmp % 60; tmp /= 60;
1246         minutes = tmp % 60; tmp /= 60;
1247         hours   = tmp % 24; tmp /= 24;
1248         days    = tmp;
1249
1250         printf("CTDB version %u\n", CTDB_PROTOCOL);
1251         printf("Current time of statistics  :                %s",
1252                ctime(&s->statistics_current_time.tv_sec));
1253         printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s",
1254                days, hours, minutes, seconds,
1255                ctime(&s->statistics_start_time.tv_sec));
1256
1257         for (i=0; i<ARRAY_SIZE(stats_fields); i++) {
1258                 if (strchr(stats_fields[i].name, '.') != NULL) {
1259                         preflen = strcspn(stats_fields[i].name, ".") + 1;
1260                         if (! prefix ||
1261                             strncmp(prefix, stats_fields[i].name, preflen) != 0) {
1262                                 prefix = stats_fields[i].name;
1263                                 printf(" %*.*s\n", preflen-1, preflen-1,
1264                                        stats_fields[i].name);
1265                         }
1266                 } else {
1267                         preflen = 0;
1268                 }
1269                 printf(" %*s%-22s%*s%10u\n", preflen ? 4 : 0, "",
1270                        stats_fields[i].name+preflen, preflen ? 0 : 4, "",
1271                        *(uint32_t *)(stats_fields[i].offset+(uint8_t *)s));
1272         }
1273
1274         printf(" hop_count_buckets:");
1275         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
1276                 printf(" %d", s->hop_count_bucket[i]);
1277         }
1278         printf("\n");
1279         printf(" lock_buckets:");
1280         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
1281                 printf(" %d", s->locks.buckets[i]);
1282         }
1283         printf("\n");
1284         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1285                "locks_latency      MIN/AVG/MAX",
1286                s->locks.latency.min, LATENCY_AVG(s->locks.latency),
1287                s->locks.latency.max, s->locks.latency.num);
1288
1289         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1290                "reclock_ctdbd      MIN/AVG/MAX",
1291                s->reclock.ctdbd.min, LATENCY_AVG(s->reclock.ctdbd),
1292                s->reclock.ctdbd.max, s->reclock.ctdbd.num);
1293
1294         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1295                "reclock_recd       MIN/AVG/MAX",
1296                s->reclock.recd.min, LATENCY_AVG(s->reclock.recd),
1297                s->reclock.recd.max, s->reclock.recd.num);
1298
1299         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1300                "call_latency       MIN/AVG/MAX",
1301                s->call_latency.min, LATENCY_AVG(s->call_latency),
1302                s->call_latency.max, s->call_latency.num);
1303
1304         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1305                "childwrite_latency MIN/AVG/MAX",
1306                s->childwrite_latency.min,
1307                LATENCY_AVG(s->childwrite_latency),
1308                s->childwrite_latency.max, s->childwrite_latency.num);
1309 }
1310
1311 static int control_statistics(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1312                               int argc, const char **argv)
1313 {
1314         struct ctdb_statistics *stats;
1315         int ret;
1316
1317         if (argc != 0) {
1318                 usage("statistics");
1319         }
1320
1321         ret = ctdb_ctrl_statistics(mem_ctx, ctdb->ev, ctdb->client,
1322                                    ctdb->cmd_pnn, TIMEOUT(), &stats);
1323         if (ret != 0) {
1324                 return ret;
1325         }
1326
1327         if (options.machinereadable) {
1328                 print_statistics_machine(stats, true);
1329         } else {
1330                 print_statistics(stats);
1331         }
1332
1333         return 0;
1334 }
1335
1336 static int control_statistics_reset(TALLOC_CTX *mem_ctx,
1337                                     struct ctdb_context *ctdb,
1338                                     int argc, const char **argv)
1339 {
1340         int ret;
1341
1342         if (argc != 0) {
1343                 usage("statisticsreset");
1344         }
1345
1346         ret = ctdb_ctrl_statistics_reset(mem_ctx, ctdb->ev, ctdb->client,
1347                                          ctdb->cmd_pnn, TIMEOUT());
1348         if (ret != 0) {
1349                 return ret;
1350         }
1351
1352         return 0;
1353 }
1354
1355 static int control_stats(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1356                          int argc, const char **argv)
1357 {
1358         struct ctdb_statistics_list *slist;
1359         int ret, count = 0, i;
1360         bool show_header = true;
1361
1362         if (argc > 1) {
1363                 usage("stats");
1364         }
1365
1366         if (argc == 1) {
1367                 count = atoi(argv[0]);
1368         }
1369
1370         ret = ctdb_ctrl_get_stat_history(mem_ctx, ctdb->ev, ctdb->client,
1371                                          ctdb->cmd_pnn, TIMEOUT(), &slist);
1372         if (ret != 0) {
1373                 return ret;
1374         }
1375
1376         for (i=0; i<slist->num; i++) {
1377                 if (slist->stats[i].statistics_start_time.tv_sec == 0) {
1378                         continue;
1379                 }
1380                 if (options.machinereadable == 1) {
1381                         print_statistics_machine(&slist->stats[i],
1382                                                  show_header);
1383                         show_header = false;
1384                 } else {
1385                         print_statistics(&slist->stats[i]);
1386                 }
1387                 if (count > 0 && i == count) {
1388                         break;
1389                 }
1390         }
1391
1392         return 0;
1393 }
1394
1395 static int ctdb_public_ip_cmp(const void *a, const void *b)
1396 {
1397         const struct ctdb_public_ip *ip_a = a;
1398         const struct ctdb_public_ip *ip_b = b;
1399
1400         return ctdb_sock_addr_cmp(&ip_a->addr, &ip_b->addr);
1401 }
1402
1403 static void print_ip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1404                      struct ctdb_public_ip_list *ips,
1405                      struct ctdb_public_ip_info **ipinfo,
1406                      bool all_nodes)
1407 {
1408         int i, j;
1409         char *conf, *avail, *active;
1410
1411         if (options.machinereadable == 1) {
1412                 printf("%s%s%s%s%s", options.sep,
1413                        "Public IP", options.sep,
1414                        "Node", options.sep);
1415                 if (options.verbose == 1) {
1416                         printf("%s%s%s%s%s%s\n",
1417                                "ActiveInterfaces", options.sep,
1418                                "AvailableInterfaces", options.sep,
1419                                "ConfiguredInterfaces", options.sep);
1420                 } else {
1421                         printf("\n");
1422                 }
1423         } else {
1424                 if (all_nodes) {
1425                         printf("Public IPs on ALL nodes\n");
1426                 } else {
1427                         printf("Public IPs on node %u\n", ctdb->cmd_pnn);
1428                 }
1429         }
1430
1431         for (i = 0; i < ips->num; i++) {
1432
1433                 if (options.machinereadable == 1) {
1434                         printf("%s%s%s%d%s", options.sep,
1435                                ctdb_sock_addr_to_string(
1436                                         mem_ctx, &ips->ip[i].addr),
1437                                options.sep,
1438                                (int)ips->ip[i].pnn, options.sep);
1439                 } else {
1440                         printf("%s", ctdb_sock_addr_to_string(
1441                                                 mem_ctx, &ips->ip[i].addr));
1442                 }
1443
1444                 if (options.verbose == 0) {
1445                         if (options.machinereadable == 1) {
1446                                 printf("\n");
1447                         } else {
1448                                 printf(" %d\n", (int)ips->ip[i].pnn);
1449                         }
1450                         continue;
1451                 }
1452
1453                 conf = NULL;
1454                 avail = NULL;
1455                 active = NULL;
1456
1457                 if (ipinfo[i] == NULL) {
1458                         goto skip_ipinfo;
1459                 }
1460
1461                 for (j=0; j<ipinfo[i]->ifaces->num; j++) {
1462                         struct ctdb_iface *iface;
1463
1464                         iface = &ipinfo[i]->ifaces->iface[j];
1465                         if (conf == NULL) {
1466                                 conf = talloc_strdup(mem_ctx, iface->name);
1467                         } else {
1468                                 conf = talloc_asprintf_append(
1469                                                 conf, ",%s", iface->name);
1470                         }
1471
1472                         if (ipinfo[i]->active_idx == j) {
1473                                 active = iface->name;
1474                         }
1475
1476                         if (iface->link_state == 0) {
1477                                 continue;
1478                         }
1479
1480                         if (avail == NULL) {
1481                                 avail = talloc_strdup(mem_ctx, iface->name);
1482                         } else {
1483                                 avail = talloc_asprintf_append(
1484                                                 avail, ",%s", iface->name);
1485                         }
1486                 }
1487
1488         skip_ipinfo:
1489
1490                 if (options.machinereadable == 1) {
1491                         printf("%s%s%s%s%s%s\n",
1492                                active ? active : "", options.sep,
1493                                avail ? avail : "", options.sep,
1494                                conf ? conf : "", options.sep);
1495                 } else {
1496                         printf(" node[%d] active[%s] available[%s]"
1497                                " configured[%s]\n",
1498                                (int)ips->ip[i].pnn, active ? active : "",
1499                                avail ? avail : "", conf ? conf : "");
1500                 }
1501         }
1502 }
1503
1504 static int collect_ips(uint8_t *keybuf, size_t keylen, uint8_t *databuf,
1505                        size_t datalen, void *private_data)
1506 {
1507         struct ctdb_public_ip_list *ips = talloc_get_type_abort(
1508                 private_data, struct ctdb_public_ip_list);
1509         struct ctdb_public_ip *ip;
1510
1511         ip = (struct ctdb_public_ip *)databuf;
1512         ips->ip[ips->num] = *ip;
1513         ips->num += 1;
1514
1515         return 0;
1516 }
1517
1518 static int get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
1519                               struct ctdb_public_ip_list **out)
1520 {
1521         struct ctdb_node_map *nodemap;
1522         struct ctdb_public_ip_list *ips;
1523         struct db_hash_context *ipdb;
1524         uint32_t *pnn_list;
1525         int ret, count, i, j;
1526
1527         nodemap = get_nodemap(ctdb, false);
1528         if (nodemap == NULL) {
1529                 return 1;
1530         }
1531
1532         ret = db_hash_init(mem_ctx, "ips", 101, DB_HASH_COMPLEX, &ipdb);
1533         if (ret != 0) {
1534                 goto failed;
1535         }
1536
1537         count = list_of_active_nodes(nodemap, CTDB_UNKNOWN_PNN, mem_ctx,
1538                                      &pnn_list);
1539         if (count <= 0) {
1540                 goto failed;
1541         }
1542
1543         for (i=0; i<count; i++) {
1544                 ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
1545                                                pnn_list[i], TIMEOUT(), &ips);
1546                 if (ret != 0) {
1547                         goto failed;
1548                 }
1549
1550                 for (j=0; j<ips->num; j++) {
1551                         struct ctdb_public_ip ip;
1552
1553                         ip.pnn = ips->ip[j].pnn;
1554                         ip.addr = ips->ip[j].addr;
1555
1556                         if (pnn_list[i] == ip.pnn) {
1557                                 /* Node claims IP is hosted on it, so
1558                                  * save that information
1559                                  */
1560                                 ret = db_hash_add(ipdb, (uint8_t *)&ip.addr,
1561                                                   sizeof(ip.addr),
1562                                                   (uint8_t *)&ip, sizeof(ip));
1563                                 if (ret != 0) {
1564                                         goto failed;
1565                                 }
1566                         } else {
1567                                 /* Node thinks IP is hosted elsewhere,
1568                                  * so overwrite with CTDB_UNKNOWN_PNN
1569                                  * if there's no existing entry
1570                                  */
1571                                 ret = db_hash_exists(ipdb, (uint8_t *)&ip.addr,
1572                                                      sizeof(ip.addr));
1573                                 if (ret == ENOENT) {
1574                                         ip.pnn = CTDB_UNKNOWN_PNN;
1575                                         ret = db_hash_add(ipdb,
1576                                                           (uint8_t *)&ip.addr,
1577                                                           sizeof(ip.addr),
1578                                                           (uint8_t *)&ip,
1579                                                           sizeof(ip));
1580                                         if (ret != 0) {
1581                                                 goto failed;
1582                                         }
1583                                 }
1584                         }
1585                 }
1586
1587                 TALLOC_FREE(ips);
1588         }
1589
1590         talloc_free(pnn_list);
1591
1592         ret = db_hash_traverse(ipdb, NULL, NULL, &count);
1593         if (ret != 0) {
1594                 goto failed;
1595         }
1596
1597         ips = talloc_zero(mem_ctx, struct ctdb_public_ip_list);
1598         if (ips == NULL) {
1599                 goto failed;
1600         }
1601
1602         ips->ip = talloc_array(ips, struct ctdb_public_ip, count);
1603         if (ips->ip == NULL) {
1604                 goto failed;
1605         }
1606
1607         ret = db_hash_traverse(ipdb, collect_ips, ips, &count);
1608         if (ret != 0) {
1609                 goto failed;
1610         }
1611
1612         if (count != ips->num) {
1613                 goto failed;
1614         }
1615
1616         talloc_free(ipdb);
1617
1618         *out = ips;
1619         return 0;
1620
1621 failed:
1622         talloc_free(ipdb);
1623         return 1;
1624 }
1625
1626 static int control_ip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1627                       int argc, const char **argv)
1628 {
1629         struct ctdb_public_ip_list *ips;
1630         struct ctdb_public_ip_info **ipinfo;
1631         int ret, i;
1632         bool do_all = false;
1633
1634         if (argc > 1) {
1635                 usage("ip");
1636         }
1637
1638         if (argc == 1) {
1639                 if (strcmp(argv[0], "all") == 0) {
1640                         do_all = true;
1641                 } else {
1642                         usage("ip");
1643                 }
1644         }
1645
1646         if (do_all) {
1647                 ret = get_all_public_ips(ctdb, mem_ctx, &ips);
1648         } else {
1649                 ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
1650                                                ctdb->cmd_pnn, TIMEOUT(), &ips);
1651         }
1652         if (ret != 0) {
1653                 return ret;
1654         }
1655
1656         qsort(ips->ip, ips->num, sizeof(struct ctdb_public_ip),
1657               ctdb_public_ip_cmp);
1658
1659         ipinfo = talloc_array(mem_ctx, struct ctdb_public_ip_info *, ips->num);
1660         if (ipinfo == NULL) {
1661                 return 1;
1662         }
1663
1664         for (i=0; i<ips->num; i++) {
1665                 uint32_t pnn;
1666                 if (do_all) {
1667                         pnn = ips->ip[i].pnn;
1668                 } else {
1669                         pnn = ctdb->cmd_pnn;
1670                 }
1671                 if (pnn == CTDB_UNKNOWN_PNN) {
1672                         ipinfo[i] = NULL;
1673                         continue;
1674                 }
1675                 ret = ctdb_ctrl_get_public_ip_info(mem_ctx, ctdb->ev,
1676                                                    ctdb->client, pnn,
1677                                                    TIMEOUT(), &ips->ip[i].addr,
1678                                                    &ipinfo[i]);
1679                 if (ret != 0) {
1680                         return ret;
1681                 }
1682         }
1683
1684         print_ip(mem_ctx, ctdb, ips, ipinfo, do_all);
1685         return 0;
1686 }
1687
1688 static int control_ipinfo(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1689                           int argc, const char **argv)
1690 {
1691         struct ctdb_public_ip_info *ipinfo;
1692         ctdb_sock_addr addr;
1693         int ret, i;
1694
1695         if (argc != 1) {
1696                 usage("ipinfo");
1697         }
1698
1699         if (! parse_ip(argv[0], NULL, 0, &addr)) {
1700                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
1701                 return 1;
1702         }
1703
1704         ret = ctdb_ctrl_get_public_ip_info(mem_ctx, ctdb->ev, ctdb->client,
1705                                            ctdb->cmd_pnn, TIMEOUT(), &addr,
1706                                            &ipinfo);
1707         if (ret != 0) {
1708                 if (ret == -1) {
1709                         printf("Node %u does not know about IP %s\n",
1710                                ctdb->cmd_pnn, argv[0]);
1711                 }
1712                 return ret;
1713         }
1714
1715         printf("Public IP[%s] info on node %u\n",
1716                ctdb_sock_addr_to_string(mem_ctx, &ipinfo->ip.addr),
1717                                         ctdb->cmd_pnn);
1718
1719         printf("IP:%s\nCurrentNode:%u\nNumInterfaces:%u\n",
1720                ctdb_sock_addr_to_string(mem_ctx, &ipinfo->ip.addr),
1721                ipinfo->ip.pnn, ipinfo->ifaces->num);
1722
1723         for (i=0; i<ipinfo->ifaces->num; i++) {
1724                 struct ctdb_iface *iface;
1725
1726                 iface = &ipinfo->ifaces->iface[i];
1727                 iface->name[CTDB_IFACE_SIZE] = '\0';
1728                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
1729                        i+1, iface->name,
1730                        iface->link_state == 0 ? "down" : "up",
1731                        iface->references,
1732                        (i == ipinfo->active_idx) ? " (active)" : "");
1733         }
1734
1735         return 0;
1736 }
1737
1738 static int control_ifaces(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1739                           int argc, const char **argv)
1740 {
1741         struct ctdb_iface_list *ifaces;
1742         int ret, i;
1743
1744         if (argc != 0) {
1745                 usage("ifaces");
1746         }
1747
1748         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
1749                                    ctdb->cmd_pnn, TIMEOUT(), &ifaces);
1750         if (ret != 0) {
1751                 return ret;
1752         }
1753
1754         if (ifaces->num == 0) {
1755                 printf("No interfaces configured on node %u\n",
1756                        ctdb->cmd_pnn);
1757                 return 0;
1758         }
1759
1760         if (options.machinereadable) {
1761                 printf("%s%s%s%s%s%s%s\n", options.sep,
1762                        "Name", options.sep,
1763                        "LinkStatus", options.sep,
1764                        "References", options.sep);
1765         } else {
1766                 printf("Interfaces on node %u\n", ctdb->cmd_pnn);
1767         }
1768
1769         for (i=0; i<ifaces->num; i++) {
1770                 if (options.machinereadable) {
1771                         printf("%s%s%s%u%s%u%s\n", options.sep,
1772                                ifaces->iface[i].name, options.sep,
1773                                ifaces->iface[i].link_state, options.sep,
1774                                ifaces->iface[i].references, options.sep);
1775                 } else {
1776                         printf("name:%s link:%s references:%u\n",
1777                                ifaces->iface[i].name,
1778                                ifaces->iface[i].link_state ? "up" : "down",
1779                                ifaces->iface[i].references);
1780                 }
1781         }
1782
1783         return 0;
1784 }
1785
1786 static int control_setifacelink(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1787                                 int argc, const char **argv)
1788 {
1789         struct ctdb_iface_list *ifaces;
1790         struct ctdb_iface *iface;
1791         int ret, i;
1792
1793         if (argc != 2) {
1794                 usage("setifacelink");
1795         }
1796
1797         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
1798                 fprintf(stderr, "Interface name '%s' too long\n", argv[0]);
1799                 return 1;
1800         }
1801
1802         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
1803                                    ctdb->cmd_pnn, TIMEOUT(), &ifaces);
1804         if (ret != 0) {
1805                 fprintf(stderr,
1806                         "Failed to get interface information from node %u\n",
1807                         ctdb->cmd_pnn);
1808                 return ret;
1809         }
1810
1811         iface = NULL;
1812         for (i=0; i<ifaces->num; i++) {
1813                 if (strcmp(ifaces->iface[i].name, argv[0]) == 0) {
1814                         iface = &ifaces->iface[i];
1815                         break;
1816                 }
1817         }
1818
1819         if (iface == NULL) {
1820                 printf("Interface %s not configured on node %u\n",
1821                        argv[0], ctdb->cmd_pnn);
1822                 return 1;
1823         }
1824
1825         if (strcmp(argv[1], "up") == 0) {
1826                 iface->link_state = 1;
1827         } else if (strcmp(argv[1], "down") == 0) {
1828                 iface->link_state = 0;
1829         } else {
1830                 usage("setifacelink");
1831                 return 1;
1832         }
1833
1834         iface->references = 0;
1835
1836         ret = ctdb_ctrl_set_iface_link_state(mem_ctx, ctdb->ev, ctdb->client,
1837                                              ctdb->cmd_pnn, TIMEOUT(), iface);
1838         if (ret != 0) {
1839                 return ret;
1840         }
1841
1842         return 0;
1843 }
1844
1845 static int control_process_exists(TALLOC_CTX *mem_ctx,
1846                                   struct ctdb_context *ctdb,
1847                                   int argc, const char **argv)
1848 {
1849         pid_t pid;
1850         int ret, status;
1851
1852         if (argc != 1) {
1853                 usage("process-exists");
1854         }
1855
1856         pid = atoi(argv[0]);
1857         ret = ctdb_ctrl_process_exists(mem_ctx, ctdb->ev, ctdb->client,
1858                                        ctdb->cmd_pnn, TIMEOUT(), pid, &status);
1859         if (ret != 0) {
1860                 return ret;
1861         }
1862
1863         if (status == 0) {
1864                 printf("PID %u exists\n", pid);
1865         } else {
1866                 printf("PID %u does not exist\n", pid);
1867         }
1868         return status;
1869 }
1870
1871 static int control_getdbmap(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1872                             int argc, const char **argv)
1873 {
1874         struct ctdb_dbid_map *dbmap;
1875         int ret, i;
1876
1877         if (argc != 0) {
1878                 usage("getdbmap");
1879         }
1880
1881         ret = ctdb_ctrl_get_dbmap(mem_ctx, ctdb->ev, ctdb->client,
1882                                   ctdb->cmd_pnn, TIMEOUT(), &dbmap);
1883         if (ret != 0) {
1884                 return ret;
1885         }
1886
1887         if (options.machinereadable == 1) {
1888                 printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
1889                        options.sep,
1890                        "ID", options.sep,
1891                        "Name", options.sep,
1892                        "Path", options.sep,
1893                        "Persistent", options.sep,
1894                        "Sticky", options.sep,
1895                        "Unhealthy", options.sep,
1896                        "Readonly", options.sep);
1897         } else {
1898                 printf("Number of databases:%d\n", dbmap->num);
1899         }
1900
1901         for (i=0; i<dbmap->num; i++) {
1902                 const char *name;
1903                 const char *path;
1904                 const char *health;
1905                 bool persistent;
1906                 bool readonly;
1907                 bool sticky;
1908                 uint32_t db_id;
1909
1910                 db_id = dbmap->dbs[i].db_id;
1911
1912                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
1913                                            ctdb->cmd_pnn, TIMEOUT(), db_id,
1914                                            &name);
1915                 if (ret != 0) {
1916                         return ret;
1917                 }
1918
1919                 ret = ctdb_ctrl_getdbpath(mem_ctx, ctdb->ev, ctdb->client,
1920                                           ctdb->cmd_pnn, TIMEOUT(), db_id,
1921                                           &path);
1922                 if (ret != 0) {
1923                         return ret;
1924                 }
1925
1926                 ret = ctdb_ctrl_db_get_health(mem_ctx, ctdb->ev, ctdb->client,
1927                                               ctdb->cmd_pnn, TIMEOUT(), db_id,
1928                                               &health);
1929                 if (ret != 0) {
1930                         return ret;
1931                 }
1932
1933                 persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
1934                 readonly = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
1935                 sticky = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
1936
1937                 if (options.machinereadable == 1) {
1938                         printf("%s0x%08X%s%s%s%s%s%d%s%d%s%d%s%d%s\n",
1939                                options.sep,
1940                                db_id, options.sep,
1941                                name, options.sep,
1942                                path, options.sep,
1943                                !! (persistent), options.sep,
1944                                !! (sticky), options.sep,
1945                                !! (health), options.sep,
1946                                !! (readonly), options.sep);
1947                 } else {
1948                         printf("dbid:0x%08x name:%s path:%s%s%s%s%s\n",
1949                                db_id, name, path,
1950                                persistent ? " PERSISTENT" : "",
1951                                sticky ? " STICKY" : "",
1952                                readonly ? " READONLY" : "",
1953                                health ? " UNHEALTHY" : "");
1954                 }
1955
1956                 talloc_free(discard_const(name));
1957                 talloc_free(discard_const(path));
1958                 talloc_free(discard_const(health));
1959         }
1960
1961         return 0;
1962 }
1963
1964 static int control_getdbstatus(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1965                                int argc, const char **argv)
1966 {
1967         uint32_t db_id;
1968         const char *db_name, *db_path, *db_health;
1969         uint8_t db_flags;
1970         int ret;
1971
1972         if (argc != 1) {
1973                 usage("getdbstatus");
1974         }
1975
1976         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
1977                 return 1;
1978         }
1979
1980         ret = ctdb_ctrl_getdbpath(mem_ctx, ctdb->ev, ctdb->client,
1981                                   ctdb->cmd_pnn, TIMEOUT(), db_id,
1982                                   &db_path);
1983         if (ret != 0) {
1984                 return ret;
1985         }
1986
1987         ret = ctdb_ctrl_db_get_health(mem_ctx, ctdb->ev, ctdb->client,
1988                                       ctdb->cmd_pnn, TIMEOUT(), db_id,
1989                                       &db_health);
1990         if (ret != 0) {
1991                 return ret;
1992         }
1993
1994         printf("dbid: 0x%08x\nname: %s\npath: %s\n", db_id, db_name, db_path);
1995         printf("PERSISTENT: %s\nSTICKY: %s\nREADONLY: %s\nHEALTH: %s\n",
1996                (db_flags & CTDB_DB_FLAGS_PERSISTENT ? "yes" : "no"),
1997                (db_flags & CTDB_DB_FLAGS_STICKY ? "yes" : "no"),
1998                (db_flags & CTDB_DB_FLAGS_READONLY ? "yes" : "no"),
1999                (db_health ? db_health : "OK"));
2000         return 0;
2001 }
2002
2003 struct dump_record_state {
2004         uint32_t count;
2005 };
2006
2007 #define ISASCII(x) (isprint(x) && ! strchr("\"\\", (x)))
2008
2009 static void dump_tdb_data(const char *name, TDB_DATA val)
2010 {
2011         int i;
2012
2013         fprintf(stdout, "%s(%zu) = \"", name, val.dsize);
2014         for (i=0; i<val.dsize; i++) {
2015                 if (ISASCII(val.dptr[i])) {
2016                         fprintf(stdout, "%c", val.dptr[i]);
2017                 } else {
2018                         fprintf(stdout, "\\%02X", val.dptr[i]);
2019                 }
2020         }
2021         fprintf(stdout, "\"\n");
2022 }
2023
2024 static void dump_ltdb_header(struct ctdb_ltdb_header *header)
2025 {
2026         fprintf(stdout, "dmaster: %u\n", header->dmaster);
2027         fprintf(stdout, "rsn: %" PRIu64 "\n", header->rsn);
2028         fprintf(stdout, "flags: 0x%08x", header->flags);
2029         if (header->flags & CTDB_REC_FLAG_MIGRATED_WITH_DATA) {
2030                 fprintf(stdout, " MIGRATED_WITH_DATA");
2031         }
2032         if (header->flags & CTDB_REC_FLAG_VACUUM_MIGRATED) {
2033                 fprintf(stdout, " VACUUM_MIGRATED");
2034         }
2035         if (header->flags & CTDB_REC_FLAG_AUTOMATIC) {
2036                 fprintf(stdout, " AUTOMATIC");
2037         }
2038         if (header->flags & CTDB_REC_RO_HAVE_DELEGATIONS) {
2039                 fprintf(stdout, " RO_HAVE_DELEGATIONS");
2040         }
2041         if (header->flags & CTDB_REC_RO_HAVE_READONLY) {
2042                 fprintf(stdout, " RO_HAVE_READONLY");
2043         }
2044         if (header->flags & CTDB_REC_RO_REVOKING_READONLY) {
2045                 fprintf(stdout, " RO_REVOKING_READONLY");
2046         }
2047         if (header->flags & CTDB_REC_RO_REVOKE_COMPLETE) {
2048                 fprintf(stdout, " RO_REVOKE_COMPLETE");
2049         }
2050         fprintf(stdout, "\n");
2051
2052 }
2053
2054 static int dump_record(uint32_t reqid, struct ctdb_ltdb_header *header,
2055                        TDB_DATA key, TDB_DATA data, void *private_data)
2056 {
2057         struct dump_record_state *state =
2058                 (struct dump_record_state *)private_data;
2059
2060         state->count += 1;
2061
2062         dump_tdb_data("key", key);
2063         dump_ltdb_header(header);
2064         dump_tdb_data("data", data);
2065         fprintf(stdout, "\n");
2066
2067         return 0;
2068 }
2069
2070 struct traverse_state {
2071         TALLOC_CTX *mem_ctx;
2072         bool done;
2073         ctdb_rec_parser_func_t func;
2074         struct dump_record_state sub_state;
2075 };
2076
2077 static void traverse_handler(uint64_t srvid, TDB_DATA data, void *private_data)
2078 {
2079         struct traverse_state *state = (struct traverse_state *)private_data;
2080         struct ctdb_rec_data *rec;
2081         struct ctdb_ltdb_header header;
2082         int ret;
2083
2084         ret = ctdb_rec_data_pull(data.dptr, data.dsize, state->mem_ctx, &rec);
2085         if (ret != 0) {
2086                 return;
2087         }
2088
2089         if (rec->key.dsize == 0 && rec->data.dsize == 0) {
2090                 talloc_free(rec);
2091                 /* end of traverse */
2092                 state->done = true;
2093                 return;
2094         }
2095
2096         ret = ctdb_ltdb_header_extract(&rec->data, &header);
2097         if (ret != 0) {
2098                 talloc_free(rec);
2099                 return;
2100         }
2101
2102         if (rec->data.dsize == 0) {
2103                 talloc_free(rec);
2104                 return;
2105         }
2106
2107         ret = state->func(rec->reqid, &header, rec->key, rec->data,
2108                           &state->sub_state);
2109         talloc_free(rec);
2110         if (ret != 0) {
2111                 state->done = true;
2112         }
2113 }
2114
2115 static int control_catdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2116                          int argc, const char **argv)
2117 {
2118         struct ctdb_db_context *db;
2119         const char *db_name;
2120         uint32_t db_id;
2121         uint8_t db_flags;
2122         struct ctdb_traverse_start_ext traverse;
2123         struct traverse_state state;
2124         int ret;
2125
2126         if (argc != 1) {
2127                 usage("catdb");
2128         }
2129
2130         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
2131                 return 1;
2132         }
2133
2134         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2135                           db_flags, &db);
2136         if (ret != 0) {
2137                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
2138                 return ret;
2139         }
2140
2141         /* Valgrind fix */
2142         ZERO_STRUCT(traverse);
2143
2144         traverse.db_id = db_id;
2145         traverse.reqid = 0;
2146         traverse.srvid = next_srvid(ctdb);
2147         traverse.withemptyrecords = false;
2148
2149         state.mem_ctx = mem_ctx;
2150         state.done = false;
2151         state.func = dump_record;
2152         state.sub_state.count = 0;
2153
2154         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2155                                               traverse.srvid,
2156                                               traverse_handler, &state);
2157         if (ret != 0) {
2158                 return ret;
2159         }
2160
2161         ret = ctdb_ctrl_traverse_start_ext(mem_ctx, ctdb->ev, ctdb->client,
2162                                            ctdb->cmd_pnn, TIMEOUT(),
2163                                            &traverse);
2164         if (ret != 0) {
2165                 return ret;
2166         }
2167
2168         ctdb_client_wait(ctdb->ev, &state.done);
2169
2170         printf("Dumped %u records\n", state.sub_state.count);
2171
2172         ret = ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
2173                                                  traverse.srvid, &state);
2174         if (ret != 0) {
2175                 return ret;
2176         }
2177
2178         return 0;
2179 }
2180
2181 static int control_cattdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2182                           int argc, const char **argv)
2183 {
2184         struct ctdb_db_context *db;
2185         const char *db_name;
2186         uint32_t db_id;
2187         uint8_t db_flags;
2188         struct dump_record_state state;
2189         int ret;
2190
2191         if (argc != 1) {
2192                 usage("catdb");
2193         }
2194
2195         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
2196                 return 1;
2197         }
2198
2199         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2200                           db_flags, &db);
2201         if (ret != 0) {
2202                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
2203                 return ret;
2204         }
2205
2206         state.count = 0;
2207         ret = ctdb_db_traverse(db, true, true, dump_record, &state);
2208
2209         printf("Dumped %u record(s)\n", state.count);
2210
2211         return ret;
2212 }
2213
2214 static int control_getmonmode(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2215                               int argc, const char **argv)
2216 {
2217         int mode, ret;
2218
2219         if (argc != 0) {
2220                 usage("getmonmode");
2221         }
2222
2223         ret = ctdb_ctrl_get_monmode(mem_ctx, ctdb->ev, ctdb->client,
2224                                     ctdb->cmd_pnn, TIMEOUT(), &mode);
2225         if (ret != 0) {
2226                 return ret;
2227         }
2228
2229         printf("%s\n",
2230                (mode == CTDB_MONITORING_ENABLED) ? "ENABLED" : "DISABLED");
2231         return 0;
2232 }
2233
2234 static int control_getcapabilities(TALLOC_CTX *mem_ctx,
2235                                    struct ctdb_context *ctdb,
2236                                    int argc, const char **argv)
2237 {
2238         uint32_t caps;
2239         int ret;
2240
2241         if (argc != 0) {
2242                 usage("getcapabilities");
2243         }
2244
2245         ret = ctdb_ctrl_get_capabilities(mem_ctx, ctdb->ev, ctdb->client,
2246                                          ctdb->cmd_pnn, TIMEOUT(), &caps);
2247         if (ret != 0) {
2248                 return ret;
2249         }
2250
2251         if (options.machinereadable == 1) {
2252                 printf("%s%s%s%s%s\n",
2253                        options.sep,
2254                        "RECMASTER", options.sep,
2255                        "LMASTER", options.sep);
2256                 printf("%s%d%s%d%s\n", options.sep,
2257                        !! (caps & CTDB_CAP_RECMASTER), options.sep,
2258                        !! (caps & CTDB_CAP_LMASTER), options.sep);
2259         } else {
2260                 printf("RECMASTER: %s\n",
2261                        (caps & CTDB_CAP_RECMASTER) ? "YES" : "NO");
2262                 printf("LMASTER: %s\n",
2263                        (caps & CTDB_CAP_LMASTER) ? "YES" : "NO");
2264         }
2265
2266         return 0;
2267 }
2268
2269 static int control_pnn(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2270                        int argc, const char **argv)
2271 {
2272         printf("%u\n", ctdb_client_pnn(ctdb->client));
2273         return 0;
2274 }
2275
2276 static int control_lvs(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2277                        int argc, const char **argv)
2278 {
2279         char *t, *lvs_helper = NULL;
2280
2281         if (argc != 1) {
2282                 usage("lvs");
2283         }
2284
2285         t = getenv("CTDB_LVS_HELPER");
2286         if (t != NULL) {
2287                 lvs_helper = talloc_strdup(mem_ctx, t);
2288         } else {
2289                 lvs_helper = talloc_asprintf(mem_ctx, "%s/ctdb_lvs",
2290                                              CTDB_HELPER_BINDIR);
2291         }
2292
2293         if (lvs_helper == NULL) {
2294                 fprintf(stderr, "Unable to set LVS helper\n");
2295                 return 1;
2296         }
2297
2298         return run_helper(mem_ctx, "LVS helper", lvs_helper, argc, argv);
2299 }
2300
2301 static int control_disable_monitor(TALLOC_CTX *mem_ctx,
2302                                    struct ctdb_context *ctdb,
2303                                    int argc, const char **argv)
2304 {
2305         int ret;
2306
2307         if (argc != 0) {
2308                 usage("disablemonitor");
2309         }
2310
2311         ret = ctdb_ctrl_disable_monitor(mem_ctx, ctdb->ev, ctdb->client,
2312                                         ctdb->cmd_pnn, TIMEOUT());
2313         if (ret != 0) {
2314                 return ret;
2315         }
2316
2317         return 0;
2318 }
2319
2320 static int control_enable_monitor(TALLOC_CTX *mem_ctx,
2321                                   struct ctdb_context *ctdb,
2322                                   int argc, const char **argv)
2323 {
2324         int ret;
2325
2326         if (argc != 0) {
2327                 usage("enablemonitor");
2328         }
2329
2330         ret = ctdb_ctrl_enable_monitor(mem_ctx, ctdb->ev, ctdb->client,
2331                                        ctdb->cmd_pnn, TIMEOUT());
2332         if (ret != 0) {
2333                 return ret;
2334         }
2335
2336         return 0;
2337 }
2338
2339 static int control_setdebug(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2340                             int argc, const char **argv)
2341 {
2342         int log_level;
2343         int ret;
2344         bool found;
2345
2346         if (argc != 1) {
2347                 usage("setdebug");
2348         }
2349
2350         found = debug_level_parse(argv[0], &log_level);
2351         if (! found) {
2352                 fprintf(stderr,
2353                         "Invalid debug level '%s'. Valid levels are:\n",
2354                         argv[0]);
2355                 fprintf(stderr, "\tERROR | WARNING | NOTICE | INFO | DEBUG\n");
2356                 return 1;
2357         }
2358
2359         ret = ctdb_ctrl_setdebug(mem_ctx, ctdb->ev, ctdb->client,
2360                                  ctdb->cmd_pnn, TIMEOUT(), log_level);
2361         if (ret != 0) {
2362                 return ret;
2363         }
2364
2365         return 0;
2366 }
2367
2368 static int control_getdebug(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2369                             int argc, const char **argv)
2370 {
2371         int loglevel;
2372         const char *log_str;
2373         int ret;
2374
2375         if (argc != 0) {
2376                 usage("getdebug");
2377         }
2378
2379         ret = ctdb_ctrl_getdebug(mem_ctx, ctdb->ev, ctdb->client,
2380                                  ctdb->cmd_pnn, TIMEOUT(), &loglevel);
2381         if (ret != 0) {
2382                 return ret;
2383         }
2384
2385         log_str = debug_level_to_string(loglevel);
2386         printf("%s\n", log_str);
2387
2388         return 0;
2389 }
2390
2391 static int control_attach(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2392                           int argc, const char **argv)
2393 {
2394         const char *db_name;
2395         uint8_t db_flags = 0;
2396         int ret;
2397
2398         if (argc < 1 || argc > 2) {
2399                 usage("attach");
2400         }
2401
2402         db_name = argv[0];
2403         if (argc == 2) {
2404                 if (strcmp(argv[1], "persistent") == 0) {
2405                         db_flags = CTDB_DB_FLAGS_PERSISTENT;
2406                 } else if (strcmp(argv[1], "readonly") == 0) {
2407                         db_flags = CTDB_DB_FLAGS_READONLY;
2408                 } else if (strcmp(argv[1], "sticky") == 0) {
2409                         db_flags = CTDB_DB_FLAGS_STICKY;
2410                 } else {
2411                         usage("attach");
2412                 }
2413         }
2414
2415         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2416                           db_flags, NULL);
2417         if (ret != 0) {
2418                 return ret;
2419         }
2420
2421         return 0;
2422 }
2423
2424 static int control_detach(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2425                           int argc, const char **argv)
2426 {
2427         const char *db_name;
2428         uint32_t db_id;
2429         uint8_t db_flags;
2430         struct ctdb_node_map *nodemap;
2431         int recmode;
2432         int ret, ret2, i;
2433
2434         if (argc < 1) {
2435                 usage("detach");
2436         }
2437
2438         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
2439                                     ctdb->cmd_pnn, TIMEOUT(), &recmode);
2440         if (ret != 0) {
2441                 return ret;
2442         }
2443
2444         if (recmode == CTDB_RECOVERY_ACTIVE) {
2445                 fprintf(stderr, "Database cannot be detached"
2446                                 " when recovery is active\n");
2447                 return 1;
2448         }
2449
2450         nodemap = get_nodemap(ctdb, false);
2451         if (nodemap == NULL) {
2452                 return 1;
2453         }
2454
2455         for (i=0; i<nodemap->num; i++) {
2456                 uint32_t value;
2457
2458                 if (nodemap->node[i].flags & NODE_FLAGS_DISCONNECTED) {
2459                         continue;
2460                 }
2461                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
2462                         continue;
2463                 }
2464                 if (nodemap->node[i].flags & NODE_FLAGS_INACTIVE) {
2465                         fprintf(stderr, "Database cannot be detached on"
2466                                 " inactive (stopped or banned) node %u\n",
2467                                 nodemap->node[i].pnn);
2468                         return 1;
2469                 }
2470
2471                 ret = ctdb_ctrl_get_tunable(mem_ctx, ctdb->ev, ctdb->client,
2472                                             nodemap->node[i].pnn, TIMEOUT(),
2473                                             "AllowClientDBAttach", &value);
2474                 if (ret != 0) {
2475                         fprintf(stderr,
2476                                 "Unable to get tunable AllowClientDBAttach"
2477                                 " from node %u\n", nodemap->node[i].pnn);
2478                         return ret;
2479                 }
2480
2481                 if (value == 1) {
2482                         fprintf(stderr,
2483                                 "Database access is still active on node %u."
2484                                 " Set AllowclientDBAttach=0 on all nodes.\n",
2485                                 nodemap->node[i].pnn);
2486                         return 1;
2487                 }
2488         }
2489
2490         ret2 = 0;
2491         for (i=0; i<argc; i++) {
2492                 if (! db_exists(mem_ctx, ctdb, argv[i], &db_id, &db_name,
2493                                 &db_flags)) {
2494                         continue;
2495                 }
2496
2497                 if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
2498                         fprintf(stderr,
2499                                 "Persistent database %s cannot be detached\n",
2500                                 argv[0]);
2501                         return 1;
2502                 }
2503
2504                 ret = ctdb_detach(mem_ctx, ctdb->ev, ctdb->client,
2505                                   TIMEOUT(), db_id);
2506                 if (ret != 0) {
2507                         fprintf(stderr, "Database %s detach failed\n", db_name);
2508                         ret2 = ret;
2509                 }
2510         }
2511
2512         return ret2;
2513 }
2514
2515 static int control_dumpmemory(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2516                               int argc, const char **argv)
2517 {
2518         const char *mem_str;
2519         ssize_t n;
2520         int ret;
2521
2522         ret = ctdb_ctrl_dump_memory(mem_ctx, ctdb->ev, ctdb->client,
2523                                     ctdb->cmd_pnn, TIMEOUT(), &mem_str);
2524         if (ret != 0) {
2525                 return ret;
2526         }
2527
2528         n = write(1, mem_str, strlen(mem_str)+1);
2529         if (n < 0 || n != strlen(mem_str)+1) {
2530                 fprintf(stderr, "Failed to write talloc summary\n");
2531                 return 1;
2532         }
2533
2534         return 0;
2535 }
2536
2537 static void dump_memory(uint64_t srvid, TDB_DATA data, void *private_data)
2538 {
2539         bool *done = (bool *)private_data;
2540         ssize_t n;
2541
2542         n = write(1, data.dptr, data.dsize);
2543         if (n < 0 || n != data.dsize) {
2544                 fprintf(stderr, "Failed to write talloc summary\n");
2545         }
2546
2547         *done = true;
2548 }
2549
2550 static int control_rddumpmemory(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2551                                 int argc, const char **argv)
2552 {
2553         struct ctdb_srvid_message msg = { 0 };
2554         int ret;
2555         bool done = false;
2556
2557         msg.pnn = ctdb->pnn;
2558         msg.srvid = next_srvid(ctdb);
2559
2560         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2561                                               msg.srvid, dump_memory, &done);
2562         if (ret != 0) {
2563                 return ret;
2564         }
2565
2566         ret = ctdb_message_mem_dump(mem_ctx, ctdb->ev, ctdb->client,
2567                                     ctdb->cmd_pnn, &msg);
2568         if (ret != 0) {
2569                 return ret;
2570         }
2571
2572         ctdb_client_wait(ctdb->ev, &done);
2573         return 0;
2574 }
2575
2576 static int control_getpid(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2577                           int argc, const char **argv)
2578 {
2579         pid_t pid;
2580         int ret;
2581
2582         ret = ctdb_ctrl_get_pid(mem_ctx, ctdb->ev, ctdb->client,
2583                                 ctdb->cmd_pnn, TIMEOUT(), &pid);
2584         if (ret != 0) {
2585                 return ret;
2586         }
2587
2588         printf("%u\n", pid);
2589         return 0;
2590 }
2591
2592 static int check_flags(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2593                        const char *desc, uint32_t flag, bool set_flag)
2594 {
2595         struct ctdb_node_map *nodemap;
2596         bool flag_is_set;
2597
2598         nodemap = get_nodemap(ctdb, false);
2599         if (nodemap == NULL) {
2600                 return 1;
2601         }
2602
2603         flag_is_set = nodemap->node[ctdb->cmd_pnn].flags & flag;
2604         if (set_flag == flag_is_set) {
2605                 if (set_flag) {
2606                         fprintf(stderr, "Node %u is already %s\n",
2607                                 ctdb->cmd_pnn, desc);
2608                 } else {
2609                         fprintf(stderr, "Node %u is not %s\n",
2610                                 ctdb->cmd_pnn, desc);
2611                 }
2612                 return 0;
2613         }
2614
2615         return 1;
2616 }
2617
2618 static void wait_for_flags(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2619                            uint32_t flag, bool set_flag)
2620 {
2621         struct ctdb_node_map *nodemap;
2622         bool flag_is_set;
2623
2624         while (1) {
2625                 nodemap = get_nodemap(ctdb, true);
2626                 if (nodemap == NULL) {
2627                         fprintf(stderr,
2628                                 "Failed to get nodemap, trying again\n");
2629                         sleep(1);
2630                         continue;
2631                 }
2632
2633                 flag_is_set = nodemap->node[ctdb->cmd_pnn].flags & flag;
2634                 if (flag_is_set == set_flag) {
2635                         break;
2636                 }
2637
2638                 sleep(1);
2639         }
2640 }
2641
2642 struct ipreallocate_state {
2643         int status;
2644         bool done;
2645 };
2646
2647 static void ipreallocate_handler(uint64_t srvid, TDB_DATA data,
2648                                  void *private_data)
2649 {
2650         struct ipreallocate_state *state =
2651                 (struct ipreallocate_state *)private_data;
2652
2653         if (data.dsize != sizeof(int)) {
2654                 /* Ignore packet */
2655                 return;
2656         }
2657
2658         state->status = *(int *)data.dptr;
2659         state->done = true;
2660 }
2661
2662 static int ipreallocate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb)
2663 {
2664         struct ctdb_srvid_message msg = { 0 };
2665         struct ipreallocate_state state;
2666         int ret;
2667
2668         msg.pnn = ctdb->pnn;
2669         msg.srvid = next_srvid(ctdb);
2670
2671         state.done = false;
2672         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2673                                               msg.srvid,
2674                                               ipreallocate_handler, &state);
2675         if (ret != 0) {
2676                 return ret;
2677         }
2678
2679         while (true) {
2680                 ret = ctdb_message_takeover_run(mem_ctx, ctdb->ev,
2681                                                 ctdb->client,
2682                                                 CTDB_BROADCAST_CONNECTED,
2683                                                 &msg);
2684                 if (ret != 0) {
2685                         goto fail;
2686                 }
2687
2688                 ret = ctdb_client_wait_timeout(ctdb->ev, &state.done,
2689                                                TIMEOUT());
2690                 if (ret != 0) {
2691                         continue;
2692                 }
2693
2694                 if (state.status >= 0) {
2695                         ret = 0;
2696                 } else {
2697                         ret = state.status;
2698                 }
2699                 break;
2700         }
2701
2702 fail:
2703         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
2704                                            msg.srvid, &state);
2705         return ret;
2706 }
2707
2708 static int control_disable(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2709                            int argc, const char **argv)
2710 {
2711         int ret;
2712
2713         if (argc != 0) {
2714                 usage("disable");
2715         }
2716
2717         ret = check_flags(mem_ctx, ctdb, "disabled",
2718                           NODE_FLAGS_PERMANENTLY_DISABLED, true);
2719         if (ret == 0) {
2720                 return 0;
2721         }
2722
2723         ret = ctdb_ctrl_modflags(mem_ctx, ctdb->ev, ctdb->client,
2724                                  ctdb->cmd_pnn, TIMEOUT(),
2725                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
2726         if (ret != 0) {
2727                 fprintf(stderr,
2728                         "Failed to set DISABLED flag on node %u\n",
2729                         ctdb->cmd_pnn);
2730                 return ret;
2731         }
2732
2733         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_PERMANENTLY_DISABLED, true);
2734         return ipreallocate(mem_ctx, ctdb);
2735 }
2736
2737 static int control_enable(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2738                           int argc, const char **argv)
2739 {
2740         int ret;
2741
2742         if (argc != 0) {
2743                 usage("enable");
2744         }
2745
2746         ret = check_flags(mem_ctx, ctdb, "disabled",
2747                           NODE_FLAGS_PERMANENTLY_DISABLED, false);
2748         if (ret == 0) {
2749                 return 0;
2750         }
2751
2752         ret = ctdb_ctrl_modflags(mem_ctx, ctdb->ev, ctdb->client,
2753                                  ctdb->cmd_pnn, TIMEOUT(),
2754                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
2755         if (ret != 0) {
2756                 fprintf(stderr, "Failed to reset DISABLED flag on node %u\n",
2757                         ctdb->cmd_pnn);
2758                 return ret;
2759         }
2760
2761         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_PERMANENTLY_DISABLED, false);
2762         return ipreallocate(mem_ctx, ctdb);
2763 }
2764
2765 static int control_stop(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2766                         int argc, const char **argv)
2767 {
2768         int ret;
2769
2770         if (argc != 0) {
2771                 usage("stop");
2772         }
2773
2774         ret = check_flags(mem_ctx, ctdb, "stopped",
2775                           NODE_FLAGS_STOPPED, true);
2776         if (ret == 0) {
2777                 return 0;
2778         }
2779
2780         ret = ctdb_ctrl_stop_node(mem_ctx, ctdb->ev, ctdb->client,
2781                                   ctdb->cmd_pnn, TIMEOUT());
2782         if (ret != 0) {
2783                 fprintf(stderr, "Failed to stop node %u\n", ctdb->cmd_pnn);
2784                 return ret;
2785         }
2786
2787         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_STOPPED, true);
2788         return ipreallocate(mem_ctx, ctdb);
2789 }
2790
2791 static int control_continue(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2792                             int argc, const char **argv)
2793 {
2794         int ret;
2795
2796         if (argc != 0) {
2797                 usage("continue");
2798         }
2799
2800         ret = check_flags(mem_ctx, ctdb, "stopped",
2801                           NODE_FLAGS_STOPPED, false);
2802         if (ret == 0) {
2803                 return 0;
2804         }
2805
2806         ret = ctdb_ctrl_continue_node(mem_ctx, ctdb->ev, ctdb->client,
2807                                       ctdb->cmd_pnn, TIMEOUT());
2808         if (ret != 0) {
2809                 fprintf(stderr, "Failed to continue stopped node %u\n",
2810                         ctdb->cmd_pnn);
2811                 return ret;
2812         }
2813
2814         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_STOPPED, false);
2815         return ipreallocate(mem_ctx, ctdb);
2816 }
2817
2818 static int control_ban(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2819                        int argc, const char **argv)
2820 {
2821         struct ctdb_ban_state ban_state;
2822         int ret;
2823
2824         if (argc != 1) {
2825                 usage("ban");
2826         }
2827
2828         ret = check_flags(mem_ctx, ctdb, "banned",
2829                           NODE_FLAGS_BANNED, true);
2830         if (ret == 0) {
2831                 return 0;
2832         }
2833
2834         ban_state.pnn = ctdb->cmd_pnn;
2835         ban_state.time = strtoul(argv[0], NULL, 0);
2836
2837         if (ban_state.time == 0) {
2838                 fprintf(stderr, "Ban time cannot be zero\n");
2839                 return EINVAL;
2840         }
2841
2842         ret = ctdb_ctrl_set_ban_state(mem_ctx, ctdb->ev, ctdb->client,
2843                                       ctdb->cmd_pnn, TIMEOUT(), &ban_state);
2844         if (ret != 0) {
2845                 fprintf(stderr, "Failed to ban node %u\n", ctdb->cmd_pnn);
2846                 return ret;
2847         }
2848
2849         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_BANNED, true);
2850         return ipreallocate(mem_ctx, ctdb);
2851
2852 }
2853
2854 static int control_unban(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2855                          int argc, const char **argv)
2856 {
2857         struct ctdb_ban_state ban_state;
2858         int ret;
2859
2860         if (argc != 0) {
2861                 usage("unban");
2862         }
2863
2864         ret = check_flags(mem_ctx, ctdb, "banned",
2865                           NODE_FLAGS_BANNED, false);
2866         if (ret == 0) {
2867                 return 0;
2868         }
2869
2870         ban_state.pnn = ctdb->cmd_pnn;
2871         ban_state.time = 0;
2872
2873         ret = ctdb_ctrl_set_ban_state(mem_ctx, ctdb->ev, ctdb->client,
2874                                       ctdb->cmd_pnn, TIMEOUT(), &ban_state);
2875         if (ret != 0) {
2876                 fprintf(stderr, "Failed to unban node %u\n", ctdb->cmd_pnn);
2877                 return ret;
2878         }
2879
2880         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_BANNED, false);
2881         return ipreallocate(mem_ctx, ctdb);
2882
2883 }
2884
2885 static int control_shutdown(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2886                             int argc, const char **argv)
2887 {
2888         int ret;
2889
2890         if (argc != 0) {
2891                 usage("shutdown");
2892         }
2893
2894         ret = ctdb_ctrl_shutdown(mem_ctx, ctdb->ev, ctdb->client,
2895                                  ctdb->cmd_pnn, TIMEOUT());
2896         if (ret != 0) {
2897                 fprintf(stderr, "Unable to shutdown node %u\n", ctdb->cmd_pnn);
2898                 return ret;
2899         }
2900
2901         return 0;
2902 }
2903
2904 static int get_generation(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2905                           uint32_t *generation)
2906 {
2907         uint32_t recmaster;
2908         int recmode;
2909         struct ctdb_vnn_map *vnnmap;
2910         int ret;
2911
2912 again:
2913         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
2914                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
2915         if (ret != 0) {
2916                 fprintf(stderr, "Failed to find recovery master\n");
2917                 return ret;
2918         }
2919
2920         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
2921                                     recmaster, TIMEOUT(), &recmode);
2922         if (ret != 0) {
2923                 fprintf(stderr, "Failed to get recovery mode from node %u\n",
2924                         recmaster);
2925                 return ret;
2926         }
2927
2928         if (recmode == CTDB_RECOVERY_ACTIVE) {
2929                 sleep(1);
2930                 goto again;
2931         }
2932
2933         ret = ctdb_ctrl_getvnnmap(mem_ctx, ctdb->ev, ctdb->client,
2934                                   recmaster, TIMEOUT(), &vnnmap);
2935         if (ret != 0) {
2936                 fprintf(stderr, "Failed to get generation from node %u\n",
2937                         recmaster);
2938                 return ret;
2939         }
2940
2941         if (vnnmap->generation == INVALID_GENERATION) {
2942                 talloc_free(vnnmap);
2943                 sleep(1);
2944                 goto again;
2945         }
2946
2947         *generation = vnnmap->generation;
2948         talloc_free(vnnmap);
2949         return 0;
2950 }
2951
2952
2953 static int control_recover(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2954                            int argc, const char **argv)
2955 {
2956         uint32_t generation, next_generation;
2957         int ret;
2958
2959         if (argc != 0) {
2960                 usage("recover");
2961         }
2962
2963         ret = get_generation(mem_ctx, ctdb, &generation);
2964         if (ret != 0) {
2965                 return ret;
2966         }
2967
2968         ret = ctdb_ctrl_set_recmode(mem_ctx, ctdb->ev, ctdb->client,
2969                                     ctdb->cmd_pnn, TIMEOUT(),
2970                                     CTDB_RECOVERY_ACTIVE);
2971         if (ret != 0) {
2972                 fprintf(stderr, "Failed to set recovery mode active\n");
2973                 return ret;
2974         }
2975
2976         while (1) {
2977                 ret = get_generation(mem_ctx, ctdb, &next_generation);
2978                 if (ret != 0) {
2979                         fprintf(stderr,
2980                                 "Failed to confirm end of recovery\n");
2981                         return ret;
2982                 }
2983
2984                 if (next_generation != generation) {
2985                         break;
2986                 }
2987
2988                 sleep (1);
2989         }
2990
2991         return 0;
2992 }
2993
2994 static int control_ipreallocate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2995                                 int argc, const char **argv)
2996 {
2997         if (argc != 0) {
2998                 usage("ipreallocate");
2999         }
3000
3001         return ipreallocate(mem_ctx, ctdb);
3002 }
3003
3004 static int control_isnotrecmaster(TALLOC_CTX *mem_ctx,
3005                                   struct ctdb_context *ctdb,
3006                                   int argc, const char **argv)
3007 {
3008         uint32_t recmaster;
3009         int ret;
3010
3011         if (argc != 0) {
3012                 usage("isnotrecmaster");
3013         }
3014
3015         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
3016                                       ctdb->pnn, TIMEOUT(), &recmaster);
3017         if (ret != 0) {
3018                 fprintf(stderr, "Failed to get recmaster\n");
3019                 return ret;
3020         }
3021
3022         if (recmaster != ctdb->pnn) {
3023                 printf("this node is not the recmaster\n");
3024                 return 1;
3025         }
3026
3027         printf("this node is the recmaster\n");
3028         return 0;
3029 }
3030
3031 static int control_gratarp(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3032                            int argc, const char **argv)
3033 {
3034         struct ctdb_addr_info addr_info;
3035         int ret;
3036
3037         if (argc != 2) {
3038                 usage("gratarp");
3039         }
3040
3041         if (! parse_ip(argv[0], NULL, 0, &addr_info.addr)) {
3042                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3043                 return 1;
3044         }
3045         addr_info.iface = argv[1];
3046
3047         ret = ctdb_ctrl_send_gratuitous_arp(mem_ctx, ctdb->ev, ctdb->client,
3048                                             ctdb->cmd_pnn, TIMEOUT(),
3049                                             &addr_info);
3050         if (ret != 0) {
3051                 fprintf(stderr, "Unable to send gratuitous arp from node %u\n",
3052                         ctdb->cmd_pnn);
3053                 return ret;
3054         }
3055
3056         return 0;
3057 }
3058
3059 static int control_tickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3060                            int argc, const char **argv)
3061 {
3062         ctdb_sock_addr src, dst;
3063         int ret;
3064
3065         if (argc != 0 && argc != 2) {
3066                 usage("tickle");
3067         }
3068
3069         if (argc == 0) {
3070                 struct ctdb_connection *clist;
3071                 int count;
3072                 int i, num_failed;
3073
3074                 ret = ctdb_parse_connections(stdin, mem_ctx, &count, &clist);
3075                 if (ret != 0) {
3076                         return ret;
3077                 }
3078
3079                 num_failed = 0;
3080                 for (i=0; i<count; i++) {
3081                         ret = ctdb_sys_send_tcp(&clist[i].src,
3082                                                 &clist[i].dst,
3083                                                 0, 0, 0);
3084                         if (ret != 0) {
3085                                 num_failed += 1;
3086                         }
3087                 }
3088
3089                 TALLOC_FREE(clist);
3090
3091                 if (num_failed > 0) {
3092                         fprintf(stderr, "Failed to send %d tickles\n",
3093                                 num_failed);
3094                         return 1;
3095                 }
3096
3097                 return 0;
3098         }
3099
3100
3101         if (! parse_ip_port(argv[0], &src)) {
3102                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3103                 return 1;
3104         }
3105
3106         if (! parse_ip_port(argv[1], &dst)) {
3107                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3108                 return 1;
3109         }
3110
3111         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
3112         if (ret != 0) {
3113                 fprintf(stderr, "Failed to send tickle ack\n");
3114                 return ret;
3115         }
3116
3117         return 0;
3118 }
3119
3120 static int control_gettickles(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3121                               int argc, const char **argv)
3122 {
3123         ctdb_sock_addr addr;
3124         struct ctdb_tickle_list *tickles;
3125         unsigned port = 0;
3126         int ret, i;
3127
3128         if (argc < 1 || argc > 2) {
3129                 usage("gettickles");
3130         }
3131
3132         if (argc == 2) {
3133                 port = strtoul(argv[1], NULL, 10);
3134         }
3135
3136         if (! parse_ip(argv[0], NULL, port, &addr)) {
3137                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3138                 return 1;
3139         }
3140
3141         ret = ctdb_ctrl_get_tcp_tickle_list(mem_ctx, ctdb->ev, ctdb->client,
3142                                             ctdb->cmd_pnn, TIMEOUT(), &addr,
3143                                             &tickles);
3144         if (ret != 0) {
3145                 fprintf(stderr, "Failed to get list of connections\n");
3146                 return ret;
3147         }
3148
3149         if (options.machinereadable) {
3150                 printf("%s%s%s%s%s%s%s%s%s\n",
3151                        options.sep,
3152                        "Source IP", options.sep,
3153                        "Port", options.sep,
3154                        "Destiation IP", options.sep,
3155                        "Port", options.sep);
3156                 for (i=0; i<tickles->num; i++) {
3157                         printf("%s%s%s%u%s%s%s%u%s\n", options.sep,
3158                                ctdb_sock_addr_to_string(
3159                                        mem_ctx, &tickles->conn[i].src),
3160                                options.sep,
3161                                ntohs(tickles->conn[i].src.ip.sin_port),
3162                                options.sep,
3163                                ctdb_sock_addr_to_string(
3164                                        mem_ctx, &tickles->conn[i].dst),
3165                                options.sep,
3166                                ntohs(tickles->conn[i].dst.ip.sin_port),
3167                                options.sep);
3168                 }
3169         } else {
3170                 printf("Connections for IP: %s\n",
3171                        ctdb_sock_addr_to_string(mem_ctx, &tickles->addr));
3172                 printf("Num connections: %u\n", tickles->num);
3173                 for (i=0; i<tickles->num; i++) {
3174                         printf("SRC: %s:%u   DST: %s:%u\n",
3175                                ctdb_sock_addr_to_string(
3176                                        mem_ctx, &tickles->conn[i].src),
3177                                ntohs(tickles->conn[i].src.ip.sin_port),
3178                                ctdb_sock_addr_to_string(
3179                                        mem_ctx, &tickles->conn[i].dst),
3180                                ntohs(tickles->conn[i].dst.ip.sin_port));
3181                 }
3182         }
3183
3184         talloc_free(tickles);
3185         return 0;
3186 }
3187
3188 typedef void (*clist_request_func)(struct ctdb_req_control *request,
3189                                    struct ctdb_connection *conn);
3190
3191 typedef int (*clist_reply_func)(struct ctdb_reply_control *reply);
3192
3193 struct process_clist_state {
3194         struct ctdb_connection *clist;
3195         int count;
3196         int num_failed, num_total;
3197         clist_reply_func reply_func;
3198 };
3199
3200 static void process_clist_done(struct tevent_req *subreq);
3201
3202 static struct tevent_req *process_clist_send(
3203                                         TALLOC_CTX *mem_ctx,
3204                                         struct ctdb_context *ctdb,
3205                                         struct ctdb_connection *clist,
3206                                         int count,
3207                                         clist_request_func request_func,
3208                                         clist_reply_func reply_func)
3209 {
3210         struct tevent_req *req, *subreq;
3211         struct process_clist_state *state;
3212         struct ctdb_req_control request;
3213         int i;
3214
3215         req = tevent_req_create(mem_ctx, &state, struct process_clist_state);
3216         if (req == NULL) {
3217                 return NULL;
3218         }
3219
3220         state->clist = clist;
3221         state->count = count;
3222         state->reply_func = reply_func;
3223
3224         for (i=0; i<count; i++) {
3225                 request_func(&request, &clist[i]);
3226                 subreq = ctdb_client_control_send(state, ctdb->ev,
3227                                                   ctdb->client, ctdb->cmd_pnn,
3228                                                   TIMEOUT(), &request);
3229                 if (tevent_req_nomem(subreq, req)) {
3230                         return tevent_req_post(req, ctdb->ev);
3231                 }
3232                 tevent_req_set_callback(subreq, process_clist_done, req);
3233         }
3234
3235         return req;
3236 }
3237
3238 static void process_clist_done(struct tevent_req *subreq)
3239 {
3240         struct tevent_req *req = tevent_req_callback_data(
3241                 subreq, struct tevent_req);
3242         struct process_clist_state *state = tevent_req_data(
3243                 req, struct process_clist_state);
3244         struct ctdb_reply_control *reply;
3245         int ret;
3246         bool status;
3247
3248         status = ctdb_client_control_recv(subreq, NULL, state, &reply);
3249         TALLOC_FREE(subreq);
3250         if (! status) {
3251                 state->num_failed += 1;
3252                 goto done;
3253         }
3254
3255         ret = state->reply_func(reply);
3256         if (ret != 0) {
3257                 state->num_failed += 1;
3258                 goto done;
3259         }
3260
3261 done:
3262         state->num_total += 1;
3263         if (state->num_total == state->count) {
3264                 tevent_req_done(req);
3265         }
3266 }
3267
3268 static int process_clist_recv(struct tevent_req *req)
3269 {
3270         struct process_clist_state *state = tevent_req_data(
3271                 req, struct process_clist_state);
3272
3273         return state->num_failed;
3274 }
3275
3276 static int control_addtickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3277                              int argc, const char **argv)
3278 {
3279         struct ctdb_connection conn;
3280         int ret;
3281
3282         if (argc != 0 && argc != 2) {
3283                 usage("addtickle");
3284         }
3285
3286         if (argc == 0) {
3287                 struct ctdb_connection *clist;
3288                 struct tevent_req *req;
3289                 int count;
3290
3291                 ret = ctdb_parse_connections(stdin, mem_ctx, &count, &clist);
3292                 if (ret != 0) {
3293                         return ret;
3294                 }
3295                 if (count == 0) {
3296                         return 0;
3297                 }
3298
3299                 req = process_clist_send(mem_ctx, ctdb, clist, count,
3300                                  ctdb_req_control_tcp_add_delayed_update,
3301                                  ctdb_reply_control_tcp_add_delayed_update);
3302                 if (req == NULL) {
3303                         talloc_free(clist);
3304                         return ENOMEM;
3305                 }
3306
3307                 tevent_req_poll(req, ctdb->ev);
3308                 talloc_free(clist);
3309
3310                 ret = process_clist_recv(req);
3311                 if (ret != 0) {
3312                         fprintf(stderr, "Failed to add %d tickles\n", ret);
3313                         return 1;
3314                 }
3315
3316                 return 0;
3317         }
3318
3319         if (! parse_ip_port(argv[0], &conn.src)) {
3320                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3321                 return 1;
3322         }
3323         if (! parse_ip_port(argv[1], &conn.dst)) {
3324                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3325                 return 1;
3326         }
3327
3328         ret = ctdb_ctrl_tcp_add_delayed_update(mem_ctx, ctdb->ev,
3329                                                ctdb->client, ctdb->cmd_pnn,
3330                                                TIMEOUT(), &conn);
3331         if (ret != 0) {
3332                 fprintf(stderr, "Failed to register connection\n");
3333                 return ret;
3334         }
3335
3336         return 0;
3337 }
3338
3339 static int control_deltickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3340                              int argc, const char **argv)
3341 {
3342         struct ctdb_connection conn;
3343         int ret;
3344
3345         if (argc != 0 && argc != 2) {
3346                 usage("deltickle");
3347         }
3348
3349         if (argc == 0) {
3350                 struct ctdb_connection *clist;
3351                 struct tevent_req *req;
3352                 int count;
3353
3354                 ret = ctdb_parse_connections(stdin, mem_ctx, &count, &clist);
3355                 if (ret != 0) {
3356                         return ret;
3357                 }
3358                 if (count == 0) {
3359                         return 0;
3360                 }
3361
3362                 req = process_clist_send(mem_ctx, ctdb, clist, count,
3363                                          ctdb_req_control_tcp_remove,
3364                                          ctdb_reply_control_tcp_remove);
3365                 if (req == NULL) {
3366                         talloc_free(clist);
3367                         return ENOMEM;
3368                 }
3369
3370                 tevent_req_poll(req, ctdb->ev);
3371                 talloc_free(clist);
3372
3373                 ret = process_clist_recv(req);
3374                 if (ret != 0) {
3375                         fprintf(stderr, "Failed to remove %d tickles\n", ret);
3376                         return 1;
3377                 }
3378
3379                 return 0;
3380         }
3381
3382         if (! parse_ip_port(argv[0], &conn.src)) {
3383                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3384                 return 1;
3385         }
3386         if (! parse_ip_port(argv[1], &conn.dst)) {
3387                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3388                 return 1;
3389         }
3390
3391         ret = ctdb_ctrl_tcp_remove(mem_ctx, ctdb->ev, ctdb->client,
3392                                    ctdb->cmd_pnn, TIMEOUT(), &conn);
3393         if (ret != 0) {
3394                 fprintf(stderr, "Failed to unregister connection\n");
3395                 return ret;
3396         }
3397
3398         return 0;
3399 }
3400
3401 static int control_check_srvids(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3402                                 int argc, const char **argv)
3403 {
3404         uint64_t *srvid;
3405         uint8_t *result;
3406         int ret, i;
3407
3408         if (argc == 0) {
3409                 usage("check_srvids");
3410         }
3411
3412         srvid = talloc_array(mem_ctx, uint64_t, argc);
3413         if (srvid == NULL) {
3414                 fprintf(stderr, "Memory allocation error\n");
3415                 return 1;
3416         }
3417
3418         for (i=0; i<argc; i++) {
3419                 srvid[i] = strtoull(argv[i], NULL, 0);
3420         }
3421
3422         ret = ctdb_ctrl_check_srvids(mem_ctx, ctdb->ev, ctdb->client,
3423                                      ctdb->cmd_pnn, TIMEOUT(), srvid, argc,
3424                                      &result);
3425         if (ret != 0) {
3426                 fprintf(stderr, "Failed to check srvids on node %u\n",
3427                         ctdb->cmd_pnn);
3428                 return ret;
3429         }
3430
3431         for (i=0; i<argc; i++) {
3432                 printf("SRVID 0x%" PRIx64 " %s\n", srvid[i],
3433                        (result[i] ? "exists" : "does not exist"));
3434         }
3435
3436         return 0;
3437 }
3438
3439 static int control_listnodes(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3440                              int argc, const char **argv)
3441 {
3442         struct ctdb_node_map *nodemap;
3443         int i;
3444
3445         if (argc != 0) {
3446                 usage("listnodes");
3447         }
3448
3449         nodemap = read_nodes_file(mem_ctx, CTDB_UNKNOWN_PNN);
3450         if (nodemap == NULL) {
3451                 return 1;
3452         }
3453
3454         for (i=0; i<nodemap->num; i++) {
3455                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
3456                         continue;
3457                 }
3458
3459                 if (options.machinereadable) {
3460                         printf("%s%u%s%s%s\n", options.sep,
3461                                nodemap->node[i].pnn, options.sep,
3462                                ctdb_sock_addr_to_string(
3463                                         mem_ctx, &nodemap->node[i].addr),
3464                                options.sep);
3465                 } else {
3466                         printf("%s\n",
3467                                ctdb_sock_addr_to_string(
3468                                         mem_ctx, &nodemap->node[i].addr));
3469                 }
3470         }
3471
3472         return 0;
3473 }
3474
3475 static bool nodemap_identical(struct ctdb_node_map *nodemap1,
3476                               struct ctdb_node_map *nodemap2)
3477 {
3478         int i;
3479
3480         if (nodemap1->num != nodemap2->num) {
3481                 return false;
3482         }
3483
3484         for (i=0; i<nodemap1->num; i++) {
3485                 struct ctdb_node_and_flags *n1, *n2;
3486
3487                 n1 = &nodemap1->node[i];
3488                 n2 = &nodemap2->node[i];
3489
3490                 if ((n1->pnn != n2->pnn) ||
3491                     (n1->flags != n2->flags) ||
3492                     ! ctdb_sock_addr_same_ip(&n1->addr, &n2->addr)) {
3493                         return false;
3494                 }
3495         }
3496
3497         return true;
3498 }
3499
3500 static int check_node_file_changes(TALLOC_CTX *mem_ctx,
3501                                    struct ctdb_node_map *nm,
3502                                    struct ctdb_node_map *fnm,
3503                                    bool *reload)
3504 {
3505         int i;
3506         bool check_failed = false;
3507
3508         *reload = false;
3509
3510         for (i=0; i<nm->num; i++) {
3511                 if (i >= fnm->num) {
3512                         fprintf(stderr,
3513                                 "Node %u (%s) missing from nodes file\n",
3514                                 nm->node[i].pnn,
3515                                 ctdb_sock_addr_to_string(
3516                                         mem_ctx, &nm->node[i].addr));
3517                         check_failed = true;
3518                         continue;
3519                 }
3520                 if (nm->node[i].flags & NODE_FLAGS_DELETED &&
3521                     fnm->node[i].flags & NODE_FLAGS_DELETED) {
3522                         /* Node remains deleted */
3523                         continue;
3524                 }
3525
3526                 if (! (nm->node[i].flags & NODE_FLAGS_DELETED) &&
3527                     ! (fnm->node[i].flags & NODE_FLAGS_DELETED)) {
3528                         /* Node not newly nor previously deleted */
3529                         if (! ctdb_same_ip(&nm->node[i].addr,
3530                                            &fnm->node[i].addr)) {
3531                                 fprintf(stderr,
3532                                         "Node %u has changed IP address"
3533                                         " (was %s, now %s)\n",
3534                                         nm->node[i].pnn,
3535                                         ctdb_sock_addr_to_string(
3536                                                 mem_ctx, &nm->node[i].addr),
3537                                         ctdb_sock_addr_to_string(
3538                                                 mem_ctx, &fnm->node[i].addr));
3539                                 check_failed = true;
3540                         } else {
3541                                 if (nm->node[i].flags & NODE_FLAGS_DISCONNECTED) {
3542                                         fprintf(stderr,
3543                                                 "WARNING: Node %u is disconnected."
3544                                                 " You MUST fix this node manually!\n",
3545                                                 nm->node[i].pnn);
3546                                 }
3547                         }
3548                         continue;
3549                 }
3550
3551                 if (fnm->node[i].flags & NODE_FLAGS_DELETED) {
3552                         /* Node is being deleted */
3553                         printf("Node %u is DELETED\n", nm->node[i].pnn);
3554                         *reload = true;
3555                         if (! (nm->node[i].flags & NODE_FLAGS_DISCONNECTED)) {
3556                                 fprintf(stderr,
3557                                         "ERROR: Node %u is still connected\n",
3558                                         nm->node[i].pnn);
3559                                 check_failed = true;
3560                         }
3561                         continue;
3562                 }
3563
3564                 if (nm->node[i].flags & NODE_FLAGS_DELETED) {
3565                         /* Node was previously deleted */
3566                         printf("Node %u is UNDELETED\n", nm->node[i].pnn);
3567                         *reload = true;
3568                 }
3569         }
3570
3571         if (check_failed) {
3572                 fprintf(stderr,
3573                         "ERROR: Nodes will not be reloaded due to previous error\n");
3574                 return 1;
3575         }
3576
3577         /* Leftover nodes in file are NEW */
3578         for (; i < fnm->num; i++) {
3579                 printf("Node %u is NEW\n", fnm->node[i].pnn);
3580                 *reload = true;
3581         }
3582
3583         return 0;
3584 }
3585
3586 struct disable_recoveries_state {
3587         uint32_t *pnn_list;
3588         int node_count;
3589         bool *reply;
3590         int status;
3591         bool done;
3592 };
3593
3594 static void disable_recoveries_handler(uint64_t srvid, TDB_DATA data,
3595                                        void *private_data)
3596 {
3597         struct disable_recoveries_state *state =
3598                 (struct disable_recoveries_state *)private_data;
3599         int ret, i;
3600
3601         if (data.dsize != sizeof(int)) {
3602                 /* Ignore packet */
3603                 return;
3604         }
3605
3606         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
3607         ret = *(int *)data.dptr;
3608         if (ret < 0) {
3609                 state->status = ret;
3610                 state->done = true;
3611                 return;
3612         }
3613         for (i=0; i<state->node_count; i++) {
3614                 if (state->pnn_list[i] == ret) {
3615                         state->reply[i] = true;
3616                         break;
3617                 }
3618         }
3619
3620         state->done = true;
3621         for (i=0; i<state->node_count; i++) {
3622                 if (! state->reply[i]) {
3623                         state->done = false;
3624                         break;
3625                 }
3626         }
3627 }
3628
3629 static int disable_recoveries(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3630                               uint32_t timeout, uint32_t *pnn_list, int count)
3631 {
3632         struct ctdb_disable_message disable = { 0 };
3633         struct disable_recoveries_state state;
3634         int ret, i;
3635
3636         disable.pnn = ctdb->pnn;
3637         disable.srvid = next_srvid(ctdb);
3638         disable.timeout = timeout;
3639
3640         state.pnn_list = pnn_list;
3641         state.node_count = count;
3642         state.done = false;
3643         state.status = 0;
3644         state.reply = talloc_zero_array(mem_ctx, bool, count);
3645         if (state.reply == NULL) {
3646                 return ENOMEM;
3647         }
3648
3649         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
3650                                               disable.srvid,
3651                                               disable_recoveries_handler,
3652                                               &state);
3653         if (ret != 0) {
3654                 return ret;
3655         }
3656
3657         for (i=0; i<count; i++) {
3658                 ret = ctdb_message_disable_recoveries(mem_ctx, ctdb->ev,
3659                                                       ctdb->client,
3660                                                       pnn_list[i],
3661                                                       &disable);
3662                 if (ret != 0) {
3663                         goto fail;
3664                 }
3665         }
3666
3667         ret = ctdb_client_wait_timeout(ctdb->ev, &state.done, TIMEOUT());
3668         if (ret == ETIME) {
3669                 fprintf(stderr, "Timed out waiting to disable recoveries\n");
3670         } else {
3671                 ret = (state.status >= 0 ? 0 : 1);
3672         }
3673
3674 fail:
3675         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
3676                                            disable.srvid, &state);
3677         return ret;
3678 }
3679
3680 static int control_reloadnodes(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3681                                int argc, const char **argv)
3682 {
3683         struct ctdb_node_map *nodemap = NULL;
3684         struct ctdb_node_map *file_nodemap;
3685         struct ctdb_node_map *remote_nodemap;
3686         struct ctdb_req_control request;
3687         struct ctdb_reply_control **reply;
3688         bool reload;
3689         int ret, i;
3690         uint32_t *pnn_list;
3691         int count;
3692
3693         nodemap = get_nodemap(ctdb, false);
3694         if (nodemap == NULL) {
3695                 return 1;
3696         }
3697
3698         file_nodemap = read_nodes_file(mem_ctx, ctdb->pnn);
3699         if (file_nodemap == NULL) {
3700                 return 1;
3701         }
3702
3703         for (i=0; i<nodemap->num; i++) {
3704                 if (nodemap->node[i].flags & NODE_FLAGS_DISCONNECTED) {
3705                         continue;
3706                 }
3707
3708                 ret = ctdb_ctrl_get_nodes_file(mem_ctx, ctdb->ev, ctdb->client,
3709                                                nodemap->node[i].pnn, TIMEOUT(),
3710                                                &remote_nodemap);
3711                 if (ret != 0) {
3712                         fprintf(stderr,
3713                                 "ERROR: Failed to get nodes file from node %u\n",
3714                                 nodemap->node[i].pnn);
3715                         return ret;
3716                 }
3717
3718                 if (! nodemap_identical(file_nodemap, remote_nodemap)) {
3719                         fprintf(stderr,
3720                                 "ERROR: Nodes file on node %u differs"
3721                                  " from current node (%u)\n",
3722                                  nodemap->node[i].pnn, ctdb->pnn);
3723                         return 1;
3724                 }
3725         }
3726
3727         ret = check_node_file_changes(mem_ctx, nodemap, file_nodemap, &reload);
3728         if (ret != 0) {
3729                 return ret;
3730         }
3731
3732         if (! reload) {
3733                 fprintf(stderr, "No change in nodes file,"
3734                                 " skipping unnecessary reload\n");
3735                 return 0;
3736         }
3737
3738         count = list_of_connected_nodes(nodemap, CTDB_UNKNOWN_PNN,
3739                                         mem_ctx, &pnn_list);
3740         if (count <= 0) {
3741                 fprintf(stderr, "Memory allocation error\n");
3742                 return 1;
3743         }
3744
3745         ret = disable_recoveries(mem_ctx, ctdb, 2*options.timelimit,
3746                                  pnn_list, count);
3747         if (ret != 0) {
3748                 fprintf(stderr, "Failed to disable recoveries\n");
3749                 return ret;
3750         }
3751
3752         ctdb_req_control_reload_nodes_file(&request);
3753         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
3754                                         pnn_list, count, TIMEOUT(),
3755                                         &request, NULL, &reply);
3756         if (ret != 0) {
3757                 bool failed = false;
3758
3759                 for (i=0; i<count; i++) {
3760                         ret = ctdb_reply_control_reload_nodes_file(reply[i]);
3761                         if (ret != 0) {
3762                                 fprintf(stderr,
3763                                         "Node %u failed to reload nodes\n",
3764                                         pnn_list[i]);
3765                                 failed = true;
3766                         }
3767                 }
3768                 if (failed) {
3769                         fprintf(stderr,
3770                                 "You MUST fix failed nodes manually!\n");
3771                 }
3772         }
3773
3774         ret = disable_recoveries(mem_ctx, ctdb, 0, pnn_list, count);
3775         if (ret != 0) {
3776                 fprintf(stderr, "Failed to enable recoveries\n");
3777                 return ret;
3778         }
3779
3780         return 0;
3781 }
3782
3783 static int moveip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3784                   ctdb_sock_addr *addr, uint32_t pnn)
3785 {
3786         struct ctdb_public_ip_list *pubip_list;
3787         struct ctdb_public_ip pubip;
3788         struct ctdb_node_map *nodemap;
3789         struct ctdb_req_control request;
3790         uint32_t *pnn_list;
3791         int ret, i, count;
3792
3793         ret = ctdb_message_disable_ip_check(mem_ctx, ctdb->ev, ctdb->client,
3794                                             CTDB_BROADCAST_CONNECTED,
3795                                             2*options.timelimit);
3796         if (ret != 0) {
3797                 fprintf(stderr, "Failed to disable IP check\n");
3798                 return ret;
3799         }
3800
3801         ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
3802                                        pnn, TIMEOUT(), &pubip_list);
3803         if (ret != 0) {
3804                 fprintf(stderr, "Failed to get Public IPs from node %u\n",
3805                         pnn);
3806                 return ret;
3807         }
3808
3809         for (i=0; i<pubip_list->num; i++) {
3810                 if (ctdb_same_ip(addr, &pubip_list->ip[i].addr)) {
3811                         break;
3812                 }
3813         }
3814
3815         if (i == pubip_list->num) {
3816                 fprintf(stderr, "Node %u CANNOT host IP address %s\n",
3817                         pnn, ctdb_sock_addr_to_string(mem_ctx, addr));
3818                 return 1;
3819         }
3820
3821         nodemap = get_nodemap(ctdb, false);
3822         if (nodemap == NULL) {
3823                 return 1;
3824         }
3825
3826         count = list_of_active_nodes(nodemap, pnn, mem_ctx, &pnn_list);
3827         if (count <= 0) {
3828                 fprintf(stderr, "Memory allocation error\n");
3829                 return 1;
3830         }
3831
3832         pubip.pnn = pnn;
3833         pubip.addr = *addr;
3834         ctdb_req_control_release_ip(&request, &pubip);
3835
3836         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
3837                                         pnn_list, count, TIMEOUT(),
3838                                         &request, NULL, NULL);
3839         if (ret != 0) {
3840                 fprintf(stderr, "Failed to release IP on nodes\n");
3841                 return ret;
3842         }
3843
3844         ret = ctdb_ctrl_takeover_ip(mem_ctx, ctdb->ev, ctdb->client,
3845                                     pnn, TIMEOUT(), &pubip);
3846         if (ret != 0) {
3847                 fprintf(stderr, "Failed to takeover IP on node %u\n", pnn);
3848                 return ret;
3849         }
3850
3851         return 0;
3852 }
3853
3854 static int control_moveip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3855                           int argc, const char **argv)
3856 {
3857         ctdb_sock_addr addr;
3858         uint32_t pnn;
3859         int ret, retries = 0;
3860
3861         if (argc != 2) {
3862                 usage("moveip");
3863         }
3864
3865         if (! parse_ip(argv[0], NULL, 0, &addr)) {
3866                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3867                 return 1;
3868         }
3869
3870         pnn = strtoul(argv[1], NULL, 10);
3871         if (pnn == CTDB_UNKNOWN_PNN) {
3872                 fprintf(stderr, "Invalid PNN %s\n", argv[1]);
3873                 return 1;
3874         }
3875
3876         while (retries < 5) {
3877                 ret = moveip(mem_ctx, ctdb, &addr, pnn);
3878                 if (ret == 0) {
3879                         break;
3880                 }
3881
3882                 sleep(3);
3883                 retries++;
3884         }
3885
3886         if (ret != 0) {
3887                 fprintf(stderr, "Failed to move IP %s to node %u\n",
3888                         argv[0], pnn);
3889                 return ret;
3890         }
3891
3892         return 0;
3893 }
3894
3895 static int rebalancenode(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3896                          uint32_t pnn)
3897 {
3898         int ret;
3899
3900         ret = ctdb_message_rebalance_node(mem_ctx, ctdb->ev, ctdb->client,
3901                                           CTDB_BROADCAST_CONNECTED, pnn);
3902         if (ret != 0) {
3903                 fprintf(stderr,
3904                         "Failed to ask recovery master to distribute IPs\n");
3905                 return ret;
3906         }
3907
3908         return 0;
3909 }
3910
3911 static int control_addip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3912                          int argc, const char **argv)
3913 {
3914         ctdb_sock_addr addr;
3915         struct ctdb_public_ip_list *pubip_list;
3916         struct ctdb_addr_info addr_info;
3917         unsigned int mask;
3918         int ret, i, retries = 0;
3919
3920         if (argc != 2) {
3921                 usage("addip");
3922         }
3923
3924         if (! parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
3925                 fprintf(stderr, "Invalid IP/Mask %s\n", argv[0]);
3926                 return 1;
3927         }
3928
3929         ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
3930                                        ctdb->cmd_pnn, TIMEOUT(), &pubip_list);
3931         if (ret != 0) {
3932                 fprintf(stderr, "Failed to get Public IPs from node %u\n",
3933                         ctdb->cmd_pnn);
3934                 return 1;
3935         }
3936
3937         for (i=0; i<pubip_list->num; i++) {
3938                 if (ctdb_same_ip(&addr, &pubip_list->ip[i].addr)) {
3939                         fprintf(stderr, "Node already knows about IP %s\n",
3940                                 ctdb_sock_addr_to_string(mem_ctx, &addr));
3941                         return 0;
3942                 }
3943         }
3944
3945         addr_info.addr = addr;
3946         addr_info.mask = mask;
3947         addr_info.iface = argv[1];
3948
3949         while (retries < 5) {
3950                 ret = ctdb_ctrl_add_public_ip(mem_ctx, ctdb->ev, ctdb->client,
3951                                               ctdb->cmd_pnn, TIMEOUT(),
3952                                               &addr_info);
3953                 if (ret == 0) {
3954                         break;
3955                 }
3956
3957                 sleep(3);
3958                 retries++;
3959         }
3960
3961         if (ret != 0) {
3962                 fprintf(stderr, "Failed to add public IP to node %u."
3963                                 " Giving up\n", ctdb->cmd_pnn);
3964                 return ret;
3965         }
3966
3967         ret = rebalancenode(mem_ctx, ctdb, ctdb->cmd_pnn);
3968         if (ret != 0) {
3969                 return ret;
3970         }
3971
3972         return 0;
3973 }
3974
3975 static int control_delip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3976                          int argc, const char **argv)
3977 {
3978         ctdb_sock_addr addr;
3979         struct ctdb_public_ip_list *pubip_list;
3980         struct ctdb_addr_info addr_info;
3981         int ret, i;
3982
3983         if (argc != 1) {
3984                 usage("delip");
3985         }
3986
3987         if (! parse_ip(argv[0], NULL, 0, &addr)) {
3988                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3989                 return 1;
3990         }
3991
3992         ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
3993                                        ctdb->cmd_pnn, TIMEOUT(), &pubip_list);
3994         if (ret != 0) {
3995                 fprintf(stderr, "Failed to get Public IPs from node %u\n",
3996                         ctdb->cmd_pnn);
3997                 return 1;
3998         }
3999
4000         for (i=0; i<pubip_list->num; i++) {
4001                 if (ctdb_same_ip(&addr, &pubip_list->ip[i].addr)) {
4002                         break;
4003                 }
4004         }
4005
4006         if (i == pubip_list->num) {
4007                 fprintf(stderr, "Node does not know about IP address %s\n",
4008                         ctdb_sock_addr_to_string(mem_ctx, &addr));
4009                 return 0;
4010         }
4011
4012         addr_info.addr = addr;
4013         addr_info.mask = 0;
4014         addr_info.iface = NULL;
4015
4016         ret = ctdb_ctrl_del_public_ip(mem_ctx, ctdb->ev, ctdb->client,
4017                                       ctdb->cmd_pnn, TIMEOUT(), &addr_info);
4018         if (ret != 0) {
4019                 fprintf(stderr, "Failed to delete public IP from node %u\n",
4020                         ctdb->cmd_pnn);
4021                 return ret;
4022         }
4023
4024         return 0;
4025 }
4026
4027 static int control_eventscript(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4028                                int argc, const char **argv)
4029 {
4030         int ret;
4031
4032         if (argc != 1) {
4033                 usage("eventscript");
4034         }
4035
4036         if (strcmp(argv[0], "monitor") != 0) {
4037                 fprintf(stderr, "Only monitor event can be run\n");
4038                 return 1;
4039         }
4040
4041         ret = ctdb_ctrl_run_eventscripts(mem_ctx, ctdb->ev, ctdb->client,
4042                                          ctdb->cmd_pnn, TIMEOUT(), argv[0]);
4043         if (ret != 0) {
4044                 fprintf(stderr, "Failed to run monitor event on node %u\n",
4045                         ctdb->cmd_pnn);
4046                 return ret;
4047         }
4048
4049         return 0;
4050 }
4051
4052 #define DB_VERSION      3
4053 #define MAX_DB_NAME     64
4054 #define MAX_REC_BUFFER_SIZE     (100*1000)
4055
4056 struct db_header {
4057         unsigned long version;
4058         time_t timestamp;
4059         unsigned long flags;
4060         unsigned long nbuf;
4061         unsigned long nrec;
4062         char name[MAX_DB_NAME];
4063 };
4064
4065 struct backup_state {
4066         TALLOC_CTX *mem_ctx;
4067         struct ctdb_rec_buffer *recbuf;
4068         uint32_t db_id;
4069         int fd;
4070         unsigned int nbuf, nrec;
4071 };
4072
4073 static int backup_handler(uint32_t reqid, struct ctdb_ltdb_header *header,
4074                           TDB_DATA key, TDB_DATA data, void *private_data)
4075 {
4076         struct backup_state *state = (struct backup_state *)private_data;
4077         size_t len;
4078         int ret;
4079
4080         if (state->recbuf == NULL) {
4081                 state->recbuf = ctdb_rec_buffer_init(state->mem_ctx,
4082                                                      state->db_id);
4083                 if (state->recbuf == NULL) {
4084                         return ENOMEM;
4085                 }
4086         }
4087
4088         ret = ctdb_rec_buffer_add(state->recbuf, state->recbuf, reqid,
4089                                   header, key, data);
4090         if (ret != 0) {
4091                 return ret;
4092         }
4093
4094         len = ctdb_rec_buffer_len(state->recbuf);
4095         if (len < MAX_REC_BUFFER_SIZE) {
4096                 return 0;
4097         }
4098
4099         ret = ctdb_rec_buffer_write(state->recbuf, state->fd);
4100         if (ret != 0) {
4101                 fprintf(stderr, "Failed to write records to backup file\n");
4102                 return ret;
4103         }
4104
4105         state->nbuf += 1;
4106         state->nrec += state->recbuf->count;
4107         TALLOC_FREE(state->recbuf);
4108
4109         return 0;
4110 }
4111
4112 static int control_backupdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4113                             int argc, const char **argv)
4114 {
4115         const char *db_name;
4116         struct ctdb_db_context *db;
4117         uint32_t db_id;
4118         uint8_t db_flags;
4119         struct backup_state state;
4120         struct db_header db_hdr;
4121         int fd, ret;
4122
4123         if (argc != 2) {
4124                 usage("backupdb");
4125         }
4126
4127         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
4128                 return 1;
4129         }
4130
4131         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
4132                           db_flags, &db);
4133         if (ret != 0) {
4134                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
4135                 return ret;
4136         }
4137
4138         fd = open(argv[1], O_RDWR|O_CREAT, 0600);
4139         if (fd == -1) {
4140                 ret = errno;
4141                 fprintf(stderr, "Failed to open file %s for writing\n",
4142                         argv[1]);
4143                 return ret;
4144         }
4145
4146         /* Write empty header first */
4147         ZERO_STRUCT(db_hdr);
4148         ret = write(fd, &db_hdr, sizeof(struct db_header));
4149         if (ret == -1) {
4150                 ret = errno;
4151                 close(fd);
4152                 fprintf(stderr, "Failed to write header to file %s\n", argv[1]);
4153                 return ret;
4154         }
4155
4156         state.mem_ctx = mem_ctx;
4157         state.recbuf = NULL;
4158         state.fd = fd;
4159         state.nbuf = 0;
4160         state.nrec = 0;
4161
4162         ret = ctdb_db_traverse(db, true, false, backup_handler, &state);
4163         if (ret != 0) {
4164                 fprintf(stderr, "Failed to collect records from DB %s\n",
4165                         db_name);
4166                 close(fd);
4167                 return ret;
4168         }
4169
4170         if (state.recbuf != NULL) {
4171                 ret = ctdb_rec_buffer_write(state.recbuf, state.fd);
4172                 if (ret != 0) {
4173                         fprintf(stderr,
4174                                 "Failed to write records to backup file\n");
4175                         close(fd);
4176                         return ret;
4177                 }
4178
4179                 state.nbuf += 1;
4180                 state.nrec += state.recbuf->count;
4181                 TALLOC_FREE(state.recbuf);
4182         }
4183
4184         db_hdr.version = DB_VERSION;
4185         db_hdr.timestamp = time(NULL);
4186         db_hdr.flags = db_flags;
4187         db_hdr.nbuf = state.nbuf;
4188         db_hdr.nrec = state.nrec;
4189         strncpy(db_hdr.name, db_name, MAX_DB_NAME-1);
4190
4191         lseek(fd, 0, SEEK_SET);
4192         ret = write(fd, &db_hdr, sizeof(struct db_header));
4193         if (ret == -1) {
4194                 ret = errno;
4195                 close(fd);
4196                 fprintf(stderr, "Failed to write header to file %s\n", argv[1]);
4197                 return ret;
4198         }
4199
4200         close(fd);
4201         printf("Database backed up to %s\n", argv[1]);
4202         return 0;
4203 }
4204
4205 static int control_restoredb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4206                              int argc, const char **argv)
4207 {
4208         const char *db_name = NULL;
4209         struct ctdb_db_context *db;
4210         struct db_header db_hdr;
4211         struct ctdb_node_map *nodemap;
4212         struct ctdb_req_control request;
4213         struct ctdb_reply_control **reply;
4214         struct ctdb_transdb wipedb;
4215         struct ctdb_pulldb_ext pulldb;
4216         struct ctdb_rec_buffer *recbuf;
4217         uint32_t generation;
4218         uint32_t *pnn_list;
4219         char timebuf[128];
4220         int fd, i;
4221         int count, ret;
4222
4223         if (argc < 1 || argc > 2) {
4224                 usage("restoredb");
4225         }
4226
4227         fd = open(argv[0], O_RDONLY, 0600);
4228         if (fd == -1) {
4229                 ret = errno;
4230                 fprintf(stderr, "Failed to open file %s for reading\n",
4231                         argv[0]);
4232                 return ret;
4233         }
4234
4235         if (argc == 2) {
4236                 db_name = argv[1];
4237         }
4238
4239         ret = read(fd, &db_hdr, sizeof(struct db_header));
4240         if (ret == -1) {
4241                 ret = errno;
4242                 close(fd);
4243                 fprintf(stderr, "Failed to read db header from file %s\n",
4244                         argv[0]);
4245                 return ret;
4246         }
4247         db_hdr.name[sizeof(db_hdr.name)-1] = '\0';
4248
4249         if (db_hdr.version != DB_VERSION) {
4250                 fprintf(stderr,
4251                         "Wrong version of backup file, expected %u, got %lu\n",
4252                         DB_VERSION, db_hdr.version);
4253                 close(fd);
4254                 return EINVAL;
4255         }
4256
4257         if (db_name == NULL) {
4258                 db_name = db_hdr.name;
4259         }
4260
4261         strftime(timebuf, sizeof(timebuf)-1, "%Y/%m/%d %H:%M:%S",
4262                  localtime(&db_hdr.timestamp));
4263         printf("Restoring database %s from backup @ %s\n", db_name, timebuf);
4264
4265         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
4266                           db_hdr.flags, &db);
4267         if (ret != 0) {
4268                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
4269                 close(fd);
4270                 return ret;
4271         }
4272
4273         nodemap = get_nodemap(ctdb, false);
4274         if (nodemap == NULL) {
4275                 fprintf(stderr, "Failed to get nodemap\n");
4276                 close(fd);
4277                 return ENOMEM;
4278         }
4279
4280         ret = get_generation(mem_ctx, ctdb, &generation);
4281         if (ret != 0) {
4282                 fprintf(stderr, "Failed to get current generation\n");
4283                 close(fd);
4284                 return ret;
4285         }
4286
4287         count = list_of_active_nodes(nodemap, CTDB_UNKNOWN_PNN, mem_ctx,
4288                                      &pnn_list);
4289         if (count <= 0) {
4290                 close(fd);
4291                 return ENOMEM;
4292         }
4293
4294         wipedb.db_id = ctdb_db_id(db);
4295         wipedb.tid = generation;
4296
4297         ctdb_req_control_db_freeze(&request, wipedb.db_id);
4298         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4299                                         ctdb->client, pnn_list, count,
4300                                         TIMEOUT(), &request, NULL, NULL);
4301         if (ret != 0) {
4302                 goto failed;
4303         }
4304
4305
4306         ctdb_req_control_db_transaction_start(&request, &wipedb);
4307         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4308                                         pnn_list, count, TIMEOUT(),
4309                                         &request, NULL, NULL);
4310         if (ret != 0) {
4311                 goto failed;
4312         }
4313
4314         ctdb_req_control_wipe_database(&request, &wipedb);
4315         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4316                                         pnn_list, count, TIMEOUT(),
4317                                         &request, NULL, NULL);
4318         if (ret != 0) {
4319                 goto failed;
4320         }
4321
4322         pulldb.db_id = ctdb_db_id(db);
4323         pulldb.lmaster = 0;
4324         pulldb.srvid = SRVID_CTDB_PUSHDB;
4325
4326         ctdb_req_control_db_push_start(&request, &pulldb);
4327         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4328                                         pnn_list, count, TIMEOUT(),
4329                                         &request, NULL, NULL);
4330         if (ret != 0) {
4331                 goto failed;
4332         }
4333
4334         for (i=0; i<db_hdr.nbuf; i++) {
4335                 struct ctdb_req_message message;
4336                 TDB_DATA data;
4337
4338                 ret = ctdb_rec_buffer_read(fd, mem_ctx, &recbuf);
4339                 if (ret != 0) {
4340                         goto failed;
4341                 }
4342
4343                 data.dsize = ctdb_rec_buffer_len(recbuf);
4344                 data.dptr = talloc_size(mem_ctx, data.dsize);
4345                 if (data.dptr == NULL) {
4346                         goto failed;
4347                 }
4348
4349                 ctdb_rec_buffer_push(recbuf, data.dptr);
4350
4351                 message.srvid = pulldb.srvid;
4352                 message.data.data = data;
4353
4354                 ret = ctdb_client_message_multi(mem_ctx, ctdb->ev,
4355                                                 ctdb->client,
4356                                                 pnn_list, count,
4357                                                 &message, NULL);
4358                 if (ret != 0) {
4359                         goto failed;
4360                 }
4361
4362                 talloc_free(recbuf);
4363                 talloc_free(data.dptr);
4364         }
4365
4366         ctdb_req_control_db_push_confirm(&request, pulldb.db_id);
4367         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4368                                         pnn_list, count, TIMEOUT(),
4369                                         &request, NULL, &reply);
4370         if (ret != 0) {
4371                 goto failed;
4372         }
4373
4374         for (i=0; i<count; i++) {
4375                 uint32_t num_records;
4376
4377                 ret = ctdb_reply_control_db_push_confirm(reply[i],
4378                                                          &num_records);
4379                 if (ret != 0) {
4380                         fprintf(stderr, "Invalid response from node %u\n",
4381                                 pnn_list[i]);
4382                         goto failed;
4383                 }
4384
4385                 if (num_records != db_hdr.nrec) {
4386                         fprintf(stderr, "Node %u received %u of %lu records\n",
4387                                 pnn_list[i], num_records, db_hdr.nrec);
4388                         goto failed;
4389                 }
4390         }
4391
4392         ctdb_req_control_db_set_healthy(&request, wipedb.db_id);
4393         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4394                                         pnn_list, count, TIMEOUT(),
4395                                         &request, NULL, NULL);
4396         if (ret != 0) {
4397                 goto failed;
4398         }
4399
4400         ctdb_req_control_db_transaction_commit(&request, &wipedb);
4401         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4402                                         pnn_list, count, TIMEOUT(),
4403                                         &request, NULL, NULL);
4404         if (ret != 0) {
4405                 goto failed;
4406         }
4407
4408         ctdb_req_control_db_thaw(&request, wipedb.db_id);
4409         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4410                                         ctdb->client, pnn_list, count,
4411                                         TIMEOUT(), &request, NULL, NULL);
4412         if (ret != 0) {
4413                 goto failed;
4414         }
4415
4416         printf("Database %s restored\n", db_name);
4417         return 0;
4418
4419
4420 failed:
4421         close(fd);
4422         ctdb_ctrl_set_recmode(mem_ctx, ctdb->ev, ctdb->client,
4423                               ctdb->pnn, TIMEOUT(), CTDB_RECOVERY_ACTIVE);
4424         return ret;
4425 }
4426
4427 struct dumpdbbackup_state {
4428         ctdb_rec_parser_func_t parser;
4429         struct dump_record_state sub_state;
4430 };
4431
4432 static int dumpdbbackup_handler(uint32_t reqid,
4433                                 struct ctdb_ltdb_header *header,
4434                                 TDB_DATA key, TDB_DATA data,
4435                                 void *private_data)
4436 {
4437         struct dumpdbbackup_state *state =
4438                 (struct dumpdbbackup_state *)private_data;
4439         struct ctdb_ltdb_header hdr;
4440         int ret;
4441
4442         ret = ctdb_ltdb_header_extract(&data, &hdr);
4443         if (ret != 0) {
4444                 return ret;
4445         }
4446
4447         return state->parser(reqid, &hdr, key, data, &state->sub_state);
4448 }
4449
4450 static int control_dumpdbbackup(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4451                                 int argc, const char **argv)
4452 {
4453         struct db_header db_hdr;
4454         char timebuf[128];
4455         struct dumpdbbackup_state state;
4456         int fd, ret, i;
4457
4458         if (argc != 1) {
4459                 usage("dumpbackup");
4460         }
4461
4462         fd = open(argv[0], O_RDONLY, 0600);
4463         if (fd == -1) {
4464                 ret = errno;
4465                 fprintf(stderr, "Failed to open file %s for reading\n",
4466                         argv[0]);
4467                 return ret;
4468         }
4469
4470         ret = read(fd, &db_hdr, sizeof(struct db_header));
4471         if (ret == -1) {
4472                 ret = errno;
4473                 close(fd);
4474                 fprintf(stderr, "Failed to read db header from file %s\n",
4475                         argv[0]);
4476                 return ret;
4477         }
4478         db_hdr.name[sizeof(db_hdr.name)-1] = '\0';
4479
4480         if (db_hdr.version != DB_VERSION) {
4481                 fprintf(stderr,
4482                         "Wrong version of backup file, expected %u, got %lu\n",
4483                         DB_VERSION, db_hdr.version);
4484                 close(fd);
4485                 return EINVAL;
4486         }
4487
4488         strftime(timebuf, sizeof(timebuf)-1, "%Y/%m/%d %H:%M:%S",
4489                  localtime(&db_hdr.timestamp));
4490         printf("Dumping database %s from backup @ %s\n",
4491                db_hdr.name, timebuf);
4492
4493         state.parser = dump_record;
4494         state.sub_state.count = 0;
4495
4496         for (i=0; i<db_hdr.nbuf; i++) {
4497                 struct ctdb_rec_buffer *recbuf;
4498
4499                 ret = ctdb_rec_buffer_read(fd, mem_ctx, &recbuf);
4500                 if (ret != 0) {
4501                         fprintf(stderr, "Failed to read records\n");
4502                         close(fd);
4503                         return ret;
4504                 }
4505
4506                 ret = ctdb_rec_buffer_traverse(recbuf, dumpdbbackup_handler,
4507                                                &state);
4508                 if (ret != 0) {
4509                         fprintf(stderr, "Failed to dump records\n");
4510                         close(fd);
4511                         return ret;
4512                 }
4513         }
4514
4515         close(fd);
4516         printf("Dumped %u record(s)\n", state.sub_state.count);
4517         return 0;
4518 }
4519
4520 static int control_wipedb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4521                           int argc, const char **argv)
4522 {
4523         const char *db_name;
4524         struct ctdb_db_context *db;
4525         uint32_t db_id;
4526         uint8_t db_flags;
4527         struct ctdb_node_map *nodemap;
4528         struct ctdb_req_control request;
4529         struct ctdb_transdb wipedb;
4530         uint32_t generation;
4531         uint32_t *pnn_list;
4532         int count, ret;
4533
4534         if (argc != 1) {
4535                 usage("wipedb");
4536         }
4537
4538         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
4539                 return 1;
4540         }
4541
4542         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
4543                           db_flags, &db);
4544         if (ret != 0) {
4545                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
4546                 return ret;
4547         }
4548
4549         nodemap = get_nodemap(ctdb, false);
4550         if (nodemap == NULL) {
4551                 fprintf(stderr, "Failed to get nodemap\n");
4552                 return ENOMEM;
4553         }
4554
4555         ret = get_generation(mem_ctx, ctdb, &generation);
4556         if (ret != 0) {
4557                 fprintf(stderr, "Failed to get current generation\n");
4558                 return ret;
4559         }
4560
4561         count = list_of_active_nodes(nodemap, CTDB_UNKNOWN_PNN, mem_ctx,
4562                                      &pnn_list);
4563         if (count <= 0) {
4564                 return ENOMEM;
4565         }
4566
4567         ctdb_req_control_db_freeze(&request, db_id);
4568         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4569                                         ctdb->client, pnn_list, count,
4570                                         TIMEOUT(), &request, NULL, NULL);
4571         if (ret != 0) {
4572                 goto failed;
4573         }
4574
4575         wipedb.db_id = db_id;
4576         wipedb.tid = generation;
4577
4578         ctdb_req_control_db_transaction_start(&request, &wipedb);
4579         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4580                                         pnn_list, count, TIMEOUT(),
4581                                         &request, NULL, NULL);
4582         if (ret != 0) {
4583                 goto failed;
4584         }
4585
4586         ctdb_req_control_wipe_database(&request, &wipedb);
4587         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4588                                         pnn_list, count, TIMEOUT(),
4589                                         &request, NULL, NULL);
4590         if (ret != 0) {
4591                 goto failed;
4592         }
4593
4594         ctdb_req_control_db_set_healthy(&request, db_id);
4595         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4596                                         pnn_list, count, TIMEOUT(),
4597                                         &request, NULL, NULL);
4598         if (ret != 0) {
4599                 goto failed;
4600         }
4601
4602         ctdb_req_control_db_transaction_commit(&request, &wipedb);
4603         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4604                                         pnn_list, count, TIMEOUT(),
4605                                         &request, NULL, NULL);
4606         if (ret != 0) {
4607                 goto failed;
4608         }
4609
4610         ctdb_req_control_db_thaw(&request, db_id);
4611         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4612                                         ctdb->client, pnn_list, count,
4613                                         TIMEOUT(), &request, NULL, NULL);
4614         if (ret != 0) {
4615                 goto failed;
4616         }
4617
4618         printf("Database %s wiped\n", db_name);
4619         return 0;
4620
4621
4622 failed:
4623         ctdb_ctrl_set_recmode(mem_ctx, ctdb->ev, ctdb->client,
4624                               ctdb->pnn, TIMEOUT(), CTDB_RECOVERY_ACTIVE);
4625         return ret;
4626 }
4627
4628 static int control_recmaster(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4629                              int argc, const char **argv)
4630 {
4631         uint32_t recmaster;
4632         int ret;
4633
4634         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
4635                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
4636         if (ret != 0) {
4637                 return ret;
4638         }
4639
4640         printf("%u\n", recmaster);
4641         return 0;
4642 }
4643
4644 static void print_scriptstatus_one(struct ctdb_script_list *slist,
4645                                    const char *event_str)
4646 {
4647         int i;
4648         int num_run = 0;
4649
4650         if (slist == NULL) {
4651                 if (! options.machinereadable) {
4652                         printf("%s cycle never run\n", event_str);
4653                 }
4654                 return;
4655         }
4656
4657         for (i=0; i<slist->num_scripts; i++) {
4658                 if (slist->script[i].status != -ENOEXEC) {
4659                         num_run++;
4660                 }
4661         }
4662
4663         if (! options.machinereadable) {
4664                 printf("%d scripts were executed last %s cycle\n",
4665                        num_run, event_str);
4666         }
4667
4668         for (i=0; i<slist->num_scripts; i++) {
4669                 const char *status = NULL;
4670
4671                 switch (slist->script[i].status) {
4672                 case -ETIME:
4673                         status = "TIMEDOUT";
4674                         break;
4675                 case -ENOEXEC:
4676                         status = "DISABLED";
4677                         break;
4678                 case 0:
4679                         status = "OK";
4680                         break;
4681                 default:
4682                         if (slist->script[i].status > 0) {
4683                                 status = "ERROR";
4684                         }
4685                         break;
4686                 }
4687
4688                 if (options.machinereadable) {
4689                         printf("%s%s%s%s%s%i%s%s%s%lu.%06lu%s%lu.%06lu%s%s%s\n",
4690                                options.sep,
4691                                event_str, options.sep,
4692                                slist->script[i].name, options.sep,
4693                                slist->script[i].status, options.sep,
4694                                status, options.sep,
4695                                (unsigned long)slist->script[i].start.tv_sec,
4696                                slist->script[i].start.tv_usec, options.sep,
4697                                (unsigned long)slist->script[i].finished.tv_sec,
4698                                slist->script[i].finished.tv_usec, options.sep,
4699                                slist->script[i].output, options.sep);
4700                         continue;
4701                 }
4702
4703                 if (status) {
4704                         printf("%-20s Status:%s    ",
4705                                slist->script[i].name, status);
4706                 } else {
4707                         /* Some other error, eg from stat. */
4708                         printf("%-20s Status:CANNOT RUN (%s)",
4709                                slist->script[i].name,
4710                                strerror(-slist->script[i].status));
4711                 }
4712
4713                 if (slist->script[i].status >= 0) {
4714                         printf("Duration:%.3lf ",
4715                                timeval_delta(&slist->script[i].finished,
4716                                              &slist->script[i].start));
4717                 }
4718                 if (slist->script[i].status != -ENOEXEC) {
4719                         printf("%s", ctime(&slist->script[i].start.tv_sec));
4720                         if (slist->script[i].status != 0) {
4721                                 printf("   OUTPUT:%s\n",
4722                                        slist->script[i].output);
4723                         }
4724                 } else {
4725                         printf("\n");
4726                 }
4727         }
4728 }
4729
4730 static void print_scriptstatus(struct ctdb_script_list **slist,
4731                                int count, const char **event_str)
4732 {
4733         int i;
4734
4735         if (options.machinereadable) {
4736                 printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
4737                        options.sep,
4738                        "Type", options.sep,
4739                        "Name", options.sep,
4740                        "Code", options.sep,
4741                        "Status", options.sep,
4742                        "Start", options.sep,
4743                        "End", options.sep,
4744                        "Error Output", options.sep);
4745         }
4746
4747         for (i=0; i<count; i++) {
4748                 print_scriptstatus_one(slist[i], event_str[i]);
4749         }
4750 }
4751
4752 static int control_scriptstatus(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4753                                 int argc, const char **argv)
4754 {
4755         struct ctdb_script_list **slist;
4756         const char *event_str;
4757         enum ctdb_event event;
4758         const char *all_events[] = {
4759                 "init", "setup", "startup", "monitor",
4760                 "takeip", "releaseip", "updateip", "ipreallocated" };
4761         bool valid;
4762         int ret, i, j;
4763         int count, start, end, num;
4764
4765         if (argc > 1) {
4766                 usage("scriptstatus");
4767         }
4768
4769         if (argc == 0) {
4770                 event_str = "monitor";
4771         } else {
4772                 event_str = argv[0];
4773         }
4774
4775         valid = false;
4776         end = 0;
4777
4778         for (i=0; i<ARRAY_SIZE(all_events); i++) {
4779                 if (strcmp(event_str, all_events[i]) == 0) {
4780                         valid = true;
4781                         count = 1;
4782                         start = i;
4783                         end = i+1;
4784                         break;
4785                 }
4786         }
4787
4788         if (strcmp(event_str, "all") == 0) {
4789                 valid = true;
4790                 count = ARRAY_SIZE(all_events);
4791                 start = 0;
4792                 end = count-1;
4793         }
4794
4795         if (! valid) {
4796                 fprintf(stderr, "Unknown event name %s\n", argv[0]);
4797                 usage("scriptstatus");
4798         }
4799
4800         slist = talloc_array(mem_ctx, struct ctdb_script_list *, count);
4801         if (slist == NULL) {
4802                 fprintf(stderr, "Memory allocation error\n");
4803                 return 1;
4804         }
4805
4806         num = 0;
4807         for (i=start; i<end; i++) {
4808                 event = ctdb_event_from_string(all_events[i]);
4809
4810                 ret = ctdb_ctrl_get_event_script_status(mem_ctx, ctdb->ev,
4811                                                         ctdb->client,
4812                                                         ctdb->cmd_pnn,
4813                                                         TIMEOUT(), event,
4814                                                         &slist[num]);
4815                 if (ret != 0) {
4816                         fprintf(stderr,
4817                                 "failed to get script status for %s event\n",
4818                                 all_events[i]);
4819                         return 1;
4820                 }
4821
4822                 if (slist[num] == NULL) {
4823                         num++;
4824                         continue;
4825                 }
4826
4827                 /* The ETIME status is ignored for certain events.
4828                  * In that case the status is 0, but endtime is not set.
4829                  */
4830                 for (j=0; j<slist[num]->num_scripts; j++) {
4831                         if (slist[num]->script[j].status == 0 &&
4832                             timeval_is_zero(&slist[num]->script[j].finished)) {
4833                                 slist[num]->script[j].status = -ETIME;
4834                         }
4835                 }
4836
4837                 num++;
4838         }
4839
4840         print_scriptstatus(slist, count, &all_events[start]);
4841         return 0;
4842 }
4843
4844 static int control_enablescript(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4845                                 int argc, const char **argv)
4846 {
4847         int ret;
4848
4849         if (argc != 1) {
4850                 usage("enablescript");
4851         }
4852
4853         ret = ctdb_ctrl_enable_script(mem_ctx, ctdb->ev, ctdb->client,
4854                                       ctdb->cmd_pnn, TIMEOUT(), argv[0]);
4855         if (ret != 0) {
4856                 fprintf(stderr, "Failed to enable script %s\n", argv[0]);
4857                 return ret;
4858         }
4859
4860         return 0;
4861 }
4862
4863 static int control_disablescript(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4864                                  int argc, const char **argv)
4865 {
4866         int ret;
4867
4868         if (argc != 1) {
4869                 usage("disablescript");
4870         }
4871
4872         ret = ctdb_ctrl_disable_script(mem_ctx, ctdb->ev, ctdb->client,
4873                                        ctdb->cmd_pnn, TIMEOUT(), argv[0]);
4874         if (ret != 0) {
4875                 fprintf(stderr, "Failed to disable script %s\n", argv[0]);
4876                 return ret;
4877         }
4878
4879         return 0;
4880 }
4881
4882 static int control_natgw(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4883                          int argc, const char **argv)
4884 {
4885         char *t, *natgw_helper = NULL;
4886
4887         if (argc != 1) {
4888                 usage("natgw");
4889         }
4890
4891         t = getenv("CTDB_NATGW_HELPER");
4892         if (t != NULL) {
4893                 natgw_helper = talloc_strdup(mem_ctx, t);
4894         } else {
4895                 natgw_helper = talloc_asprintf(mem_ctx, "%s/ctdb_natgw",
4896                                                CTDB_HELPER_BINDIR);
4897         }
4898
4899         if (natgw_helper == NULL) {
4900                 fprintf(stderr, "Unable to set NAT gateway helper\n");
4901                 return 1;
4902         }
4903
4904         return run_helper(mem_ctx, "NAT gateway helper", natgw_helper,
4905                           argc, argv);
4906 }
4907
4908 static int control_natgwlist(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4909                              int argc, const char **argv)
4910 {
4911         char *t, *natgw_helper = NULL;
4912         const char *cmd_argv[] = { "natgwlist", NULL };
4913
4914         if (argc != 0) {
4915                 usage("natgwlist");
4916         }
4917
4918         t = getenv("CTDB_NATGW_HELPER");
4919         if (t != NULL) {
4920                 natgw_helper = talloc_strdup(mem_ctx, t);
4921         } else {
4922                 natgw_helper = talloc_asprintf(mem_ctx, "%s/ctdb_natgw",
4923                                                CTDB_HELPER_BINDIR);
4924         }
4925
4926         if (natgw_helper == NULL) {
4927                 fprintf(stderr, "Unable to set NAT gateway helper\n");
4928                 return 1;
4929         }
4930
4931         return run_helper(mem_ctx, "NAT gateway helper", natgw_helper,
4932                           1, cmd_argv);
4933 }
4934
4935 /*
4936  * Find the PNN of the current node
4937  * discover the pnn by loading the nodes file and try to bind
4938  * to all addresses one at a time until the ip address is found.
4939  */
4940 static bool find_node_xpnn(TALLOC_CTX *mem_ctx, uint32_t *pnn)
4941 {
4942         struct ctdb_node_map *nodemap;
4943         int i;
4944
4945         nodemap = read_nodes_file(mem_ctx, CTDB_UNKNOWN_PNN);
4946         if (nodemap == NULL) {
4947                 return false;
4948         }
4949
4950         for (i=0; i<nodemap->num; i++) {
4951                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
4952                         continue;
4953                 }
4954                 if (ctdb_sys_have_ip(&nodemap->node[i].addr)) {
4955                         if (pnn != NULL) {
4956                                 *pnn = nodemap->node[i].pnn;
4957                         }
4958                         talloc_free(nodemap);
4959                         return true;
4960                 }
4961         }
4962
4963         fprintf(stderr, "Failed to detect PNN of the current node.\n");
4964         talloc_free(nodemap);
4965         return false;
4966 }
4967
4968 static int control_getreclock(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4969                               int argc, const char **argv)
4970 {
4971         const char *reclock;
4972         int ret;
4973
4974         if (argc != 0) {
4975                 usage("getreclock");
4976         }
4977
4978         ret = ctdb_ctrl_get_reclock_file(mem_ctx, ctdb->ev, ctdb->client,
4979                                          ctdb->cmd_pnn, TIMEOUT(), &reclock);
4980         if (ret != 0) {
4981                 return ret;
4982         }
4983
4984         if (reclock != NULL) {
4985                 printf("%s\n", reclock);
4986         }
4987
4988         return 0;
4989 }
4990
4991 static int control_setlmasterrole(TALLOC_CTX *mem_ctx,
4992                                   struct ctdb_context *ctdb,
4993                                   int argc, const char **argv)
4994 {
4995         uint32_t lmasterrole = 0;
4996         int ret;
4997
4998         if (argc != 1) {
4999                 usage("setlmasterrole");
5000         }
5001
5002         if (strcmp(argv[0], "on") == 0) {
5003                 lmasterrole = 1;
5004         } else if (strcmp(argv[0], "off") == 0) {
5005                 lmasterrole = 0;
5006         } else {
5007                 usage("setlmasterrole");
5008         }
5009
5010         ret = ctdb_ctrl_set_lmasterrole(mem_ctx, ctdb->ev, ctdb->client,
5011                                         ctdb->cmd_pnn, TIMEOUT(), lmasterrole);
5012         if (ret != 0) {
5013                 return ret;
5014         }
5015
5016         return 0;
5017 }
5018
5019 static int control_setrecmasterrole(TALLOC_CTX *mem_ctx,
5020                                     struct ctdb_context *ctdb,
5021                                     int argc, const char **argv)
5022 {
5023         uint32_t recmasterrole = 0;
5024         int ret;
5025
5026         if (argc != 1) {
5027                 usage("setrecmasterrole");
5028         }
5029
5030         if (strcmp(argv[0], "on") == 0) {
5031                 recmasterrole = 1;
5032         } else if (strcmp(argv[0], "off") == 0) {
5033                 recmasterrole = 0;
5034         } else {
5035                 usage("setrecmasterrole");
5036         }
5037
5038         ret = ctdb_ctrl_set_recmasterrole(mem_ctx, ctdb->ev, ctdb->client,
5039                                           ctdb->cmd_pnn, TIMEOUT(),
5040                                           recmasterrole);
5041         if (ret != 0) {
5042                 return ret;
5043         }
5044
5045         return 0;
5046 }
5047
5048 static int control_setdbreadonly(TALLOC_CTX *mem_ctx,
5049                                  struct ctdb_context *ctdb,
5050                                  int argc, const char **argv)
5051 {
5052         uint32_t db_id;
5053         uint8_t db_flags;
5054         int ret;
5055
5056         if (argc != 1) {
5057                 usage("setdbreadonly");
5058         }
5059
5060         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, NULL, &db_flags)) {
5061                 return 1;
5062         }
5063
5064         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5065                 fprintf(stderr, "Cannot set READONLY on persistent DB\n");
5066                 return 1;
5067         }
5068
5069         ret = ctdb_ctrl_set_db_readonly(mem_ctx, ctdb->ev, ctdb->client,
5070                                         ctdb->cmd_pnn, TIMEOUT(), db_id);
5071         if (ret != 0) {
5072                 return ret;
5073         }
5074
5075         return 0;
5076 }
5077
5078 static int control_setdbsticky(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5079                                int argc, const char **argv)
5080 {
5081         uint32_t db_id;
5082         uint8_t db_flags;
5083         int ret;
5084
5085         if (argc != 1) {
5086                 usage("setdbsticky");
5087         }
5088
5089         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, NULL, &db_flags)) {
5090                 return 1;
5091         }
5092
5093         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5094                 fprintf(stderr, "Cannot set STICKY on persistent DB\n");
5095                 return 1;
5096         }
5097
5098         ret = ctdb_ctrl_set_db_sticky(mem_ctx, ctdb->ev, ctdb->client,
5099                                       ctdb->cmd_pnn, TIMEOUT(), db_id);
5100         if (ret != 0) {
5101                 return ret;
5102         }
5103
5104         return 0;
5105 }
5106
5107 static int control_pfetch(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5108                           int argc, const char **argv)
5109 {
5110         const char *db_name;
5111         struct ctdb_db_context *db;
5112         struct ctdb_transaction_handle *h;
5113         uint8_t db_flags;
5114         TDB_DATA key, data;
5115         int ret;
5116
5117         if (argc < 2 || argc > 3) {
5118                 usage("pfetch");
5119         }
5120
5121         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5122                 return 1;
5123         }
5124
5125         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5126                 fprintf(stderr, "DB %s is not a persistent database\n",
5127                         db_name);
5128                 return 1;
5129         }
5130
5131         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5132                           db_flags, &db);
5133         if (ret != 0) {
5134                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5135                 return ret;
5136         }
5137
5138         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5139         if (ret != 0) {
5140                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5141                 return ret;
5142         }
5143
5144         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5145                                      TIMEOUT(), db, true, &h);
5146         if (ret != 0) {
5147                 fprintf(stderr, "Failed to start transaction on db %s\n",
5148                         db_name);
5149                 return ret;
5150         }
5151
5152         ret = ctdb_transaction_fetch_record(h, key, mem_ctx, &data);
5153         if (ret != 0) {
5154                 fprintf(stderr, "Failed to read record for key %s\n",
5155                         argv[1]);
5156                 ctdb_transaction_cancel(h);
5157                 return ret;
5158         }
5159
5160         printf("%.*s\n", (int)data.dsize, data.dptr);
5161
5162         ctdb_transaction_cancel(h);
5163         return 0;
5164 }
5165
5166 static int control_pstore(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5167                           int argc, const char **argv)
5168 {
5169         const char *db_name;
5170         struct ctdb_db_context *db;
5171         struct ctdb_transaction_handle *h;
5172         uint8_t db_flags;
5173         TDB_DATA key, data;
5174         int ret;
5175
5176         if (argc != 3) {
5177                 usage("pstore");
5178         }
5179
5180         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5181                 return 1;
5182         }
5183
5184         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5185                 fprintf(stderr, "DB %s is not a persistent database\n",
5186                         db_name);
5187                 return 1;
5188         }
5189
5190         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5191                           db_flags, &db);
5192         if (ret != 0) {
5193                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5194                 return ret;
5195         }
5196
5197         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5198         if (ret != 0) {
5199                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5200                 return ret;
5201         }
5202
5203         ret = str_to_data(argv[2], strlen(argv[2]), mem_ctx, &data);
5204         if (ret != 0) {
5205                 fprintf(stderr, "Failed to parse value %s\n", argv[2]);
5206                 return ret;
5207         }
5208
5209         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5210                                      TIMEOUT(), db, false, &h);
5211         if (ret != 0) {
5212                 fprintf(stderr, "Failed to start transaction on db %s\n",
5213                         db_name);
5214                 return ret;
5215         }
5216
5217         ret = ctdb_transaction_store_record(h, key, data);
5218         if (ret != 0) {
5219                 fprintf(stderr, "Failed to store record for key %s\n",
5220                         argv[1]);
5221                 ctdb_transaction_cancel(h);
5222                 return ret;
5223         }
5224
5225         ret = ctdb_transaction_commit(h);
5226         if (ret != 0) {
5227                 fprintf(stderr, "Failed to commit transaction on db %s\n",
5228                         db_name);
5229                 ctdb_transaction_cancel(h);
5230                 return ret;
5231         }
5232
5233         return 0;
5234 }
5235
5236 static int control_pdelete(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5237                            int argc, const char **argv)
5238 {
5239         const char *db_name;
5240         struct ctdb_db_context *db;
5241         struct ctdb_transaction_handle *h;
5242         uint8_t db_flags;
5243         TDB_DATA key;
5244         int ret;
5245
5246         if (argc != 2) {
5247                 usage("pdelete");
5248         }
5249
5250         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5251                 return 1;
5252         }
5253
5254         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5255                 fprintf(stderr, "DB %s is not a persistent database\n",
5256                         db_name);
5257                 return 1;
5258         }
5259
5260         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5261                           db_flags, &db);
5262         if (ret != 0) {
5263                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5264                 return ret;
5265         }
5266
5267         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5268         if (ret != 0) {
5269                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5270                 return ret;
5271         }
5272
5273         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5274                                      TIMEOUT(), db, false, &h);
5275         if (ret != 0) {
5276                 fprintf(stderr, "Failed to start transaction on db %s\n",
5277                         db_name);
5278                 return ret;
5279         }
5280
5281         ret = ctdb_transaction_delete_record(h, key);
5282         if (ret != 0) {
5283                 fprintf(stderr, "Failed to delete record for key %s\n",
5284                         argv[1]);
5285                 ctdb_transaction_cancel(h);
5286                 return ret;
5287         }
5288
5289         ret = ctdb_transaction_commit(h);
5290         if (ret != 0) {
5291                 fprintf(stderr, "Failed to commit transaction on db %s\n",
5292                         db_name);
5293                 ctdb_transaction_cancel(h);
5294                 return ret;
5295         }
5296
5297         return 0;
5298 }
5299
5300 static int ptrans_parse_string(TALLOC_CTX *mem_ctx, const char **ptr, TDB_DATA *data)
5301 {
5302         const char *t;
5303         size_t n;
5304         int ret;
5305
5306         *data = tdb_null;
5307
5308         /* Skip whitespace */
5309         n = strspn(*ptr, " \t");
5310         t = *ptr + n;
5311
5312         if (t[0] == '"') {
5313                 /* Quoted ASCII string - no wide characters! */
5314                 t++;
5315                 n = strcspn(t, "\"");
5316                 if (t[n] == '"') {
5317                         if (n > 0) {
5318                                 ret = str_to_data(t, n, mem_ctx, data);
5319                                 if (ret != 0) {
5320                                         return ret;
5321                                 }
5322                         }
5323                         *ptr = t + n + 1;
5324                 } else {
5325                         fprintf(stderr, "Unmatched \" in input %s\n", *ptr);
5326                         return 1;
5327                 }
5328         } else {
5329                 fprintf(stderr, "Unsupported input format in %s\n", *ptr);
5330                 return 1;
5331         }
5332
5333         return 0;
5334 }
5335
5336 #define MAX_LINE_SIZE   1024
5337
5338 static bool ptrans_get_key_value(TALLOC_CTX *mem_ctx, FILE *file,
5339                                  TDB_DATA *key, TDB_DATA *value)
5340 {
5341         char line [MAX_LINE_SIZE]; /* FIXME: make this more flexible? */
5342         const char *ptr;
5343         int ret;
5344
5345         ptr = fgets(line, MAX_LINE_SIZE, file);
5346         if (ptr == NULL) {
5347                 return false;
5348         }
5349
5350         /* Get key */
5351         ret = ptrans_parse_string(mem_ctx, &ptr, key);
5352         if (ret != 0 || ptr == NULL || key->dptr == NULL) {
5353                 /* Line Ignored but not EOF */
5354                 *key = tdb_null;
5355                 return true;
5356         }
5357
5358         /* Get value */
5359         ret = ptrans_parse_string(mem_ctx, &ptr, value);
5360         if (ret != 0) {
5361                 /* Line Ignored but not EOF */
5362                 talloc_free(key->dptr);
5363                 *key = tdb_null;
5364                 return true;
5365         }
5366
5367         return true;
5368 }
5369
5370 static int control_ptrans(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5371                           int argc, const char **argv)
5372 {
5373         const char *db_name;
5374         struct ctdb_db_context *db;
5375         struct ctdb_transaction_handle *h;
5376         uint8_t db_flags;
5377         FILE *file;
5378         TDB_DATA key, value;
5379         int ret;
5380
5381         if (argc < 1 || argc > 2) {
5382                 usage("ptrans");
5383         }
5384
5385         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5386                 return 1;
5387         }
5388
5389         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5390                 fprintf(stderr, "DB %s is not a persistent database\n",
5391                         db_name);
5392                 return 1;
5393         }
5394
5395         if (argc == 2) {
5396                 file = fopen(argv[1], "r");
5397                 if (file == NULL) {
5398                         fprintf(stderr, "Failed to open file %s\n", argv[1]);
5399                         return 1;
5400                 }
5401         } else {
5402                 file = stdin;
5403         }
5404
5405         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5406                           db_flags, &db);
5407         if (ret != 0) {
5408                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5409                 goto done;
5410         }
5411
5412         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5413                                      TIMEOUT(), db, false, &h);
5414         if (ret != 0) {
5415                 fprintf(stderr, "Failed to start transaction on db %s\n",
5416                         db_name);
5417                 goto done;
5418         }
5419
5420         while (ptrans_get_key_value(mem_ctx, file, &key, &value)) {
5421                 if (key.dsize != 0) {
5422                         ret = ctdb_transaction_store_record(h, key, value);
5423                         if (ret != 0) {
5424                                 fprintf(stderr, "Failed to store record\n");
5425                                 ctdb_transaction_cancel(h);
5426                                 goto done;
5427                         }
5428                         talloc_free(key.dptr);
5429                         talloc_free(value.dptr);
5430                 }
5431         }
5432
5433         ret = ctdb_transaction_commit(h);
5434         if (ret != 0) {
5435                 fprintf(stderr, "Failed to commit transaction on db %s\n",
5436                         db_name);
5437                 ctdb_transaction_cancel(h);
5438         }
5439
5440 done:
5441         if (file != stdin) {
5442                 fclose(file);
5443         }
5444         return ret;
5445 }
5446
5447 static int control_tfetch(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5448                           int argc, const char **argv)
5449 {
5450         struct tdb_context *tdb;
5451         TDB_DATA key, data;
5452         struct ctdb_ltdb_header header;
5453         int ret;
5454
5455         if (argc < 2 || argc > 3) {
5456                 usage("tfetch");
5457         }
5458
5459         tdb = tdb_open(argv[0], 0, 0, O_RDWR, 0);
5460         if (tdb == NULL) {
5461                 fprintf(stderr, "Failed to open TDB file %s\n", argv[0]);
5462                 return 1;
5463         }
5464
5465         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5466         if (ret != 0) {
5467                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5468                 tdb_close(tdb);
5469                 return ret;
5470         }
5471
5472         data = tdb_fetch(tdb, key);
5473         if (data.dptr == NULL) {
5474                 fprintf(stderr, "No record for key %s\n", argv[1]);
5475                 tdb_close(tdb);
5476                 return 1;
5477         }
5478
5479         if (data.dsize < sizeof(struct ctdb_ltdb_header)) {
5480                 fprintf(stderr, "Invalid record for key %s\n", argv[1]);
5481                 tdb_close(tdb);
5482                 return 1;
5483         }
5484
5485         tdb_close(tdb);
5486
5487         if (argc == 3) {
5488                 int fd;
5489                 ssize_t nwritten;
5490
5491                 fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
5492                 if (fd == -1) {
5493                         fprintf(stderr, "Failed to open output file %s\n",
5494                                 argv[2]);
5495                         goto fail;
5496                 }
5497
5498                 nwritten = sys_write(fd, data.dptr, data.dsize);
5499                 if (nwritten != data.dsize) {
5500                         fprintf(stderr, "Failed to write record to file\n");
5501                         close(fd);
5502                         goto fail;
5503                 }
5504
5505                 close(fd);
5506         }
5507
5508 fail:
5509         ret = ctdb_ltdb_header_extract(&data, &header);
5510         if (ret != 0) {
5511                 fprintf(stderr, "Failed to parse header from data\n");
5512                 return 1;
5513         }
5514
5515         dump_ltdb_header(&header);
5516         dump_tdb_data("data", data);
5517
5518         return 0;
5519 }
5520
5521 static int control_tstore(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5522                           int argc, const char **argv)
5523 {
5524         struct tdb_context *tdb;
5525         TDB_DATA key, data[2], value;
5526         struct ctdb_ltdb_header header;
5527         uint8_t header_buf[sizeof(struct ctdb_ltdb_header)];
5528         int ret;
5529
5530         if (argc < 3 || argc > 5) {
5531                 usage("tstore");
5532         }
5533
5534         tdb = tdb_open(argv[0], 0, 0, O_RDWR, 0);
5535         if (tdb == NULL) {
5536                 fprintf(stderr, "Failed to open TDB file %s\n", argv[0]);
5537                 return 1;
5538         }
5539
5540         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5541         if (ret != 0) {
5542                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5543                 tdb_close(tdb);
5544                 return ret;
5545         }
5546
5547         ret = str_to_data(argv[2], strlen(argv[2]), mem_ctx, &value);
5548         if (ret != 0) {
5549                 fprintf(stderr, "Failed to parse value %s\n", argv[2]);
5550                 tdb_close(tdb);
5551                 return ret;
5552         }
5553
5554         ZERO_STRUCT(header);
5555
5556         if (argc > 3) {
5557                 header.rsn = (uint64_t)strtoull(argv[3], NULL, 0);
5558         }
5559         if (argc > 4) {
5560                 header.dmaster = (uint32_t)atol(argv[4]);
5561         }
5562         if (argc > 5) {
5563                 header.flags = (uint32_t)atol(argv[5]);
5564         }
5565
5566         ctdb_ltdb_header_push(&header, header_buf);
5567
5568         data[0].dsize = ctdb_ltdb_header_len(&header);
5569         data[0].dptr = header_buf;
5570
5571         data[1].dsize = value.dsize;
5572         data[1].dptr = value.dptr;
5573
5574         ret = tdb_storev(tdb, key, data, 2, TDB_REPLACE);
5575         if (ret != 0) {
5576                 fprintf(stderr, "Failed to write record %s to file %s\n",
5577                         argv[1], argv[0]);
5578         }
5579
5580         tdb_close(tdb);
5581
5582         return ret;
5583 }
5584
5585 static int control_readkey(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5586                            int argc, const char **argv)
5587 {
5588         const char *db_name;
5589         struct ctdb_db_context *db;
5590         struct ctdb_record_handle *h;
5591         uint8_t db_flags;
5592         TDB_DATA key, data;
5593         bool readonly = false;
5594         int ret;
5595
5596         if (argc < 2 || argc > 3) {
5597                 usage("readkey");
5598         }
5599
5600         if (argc == 3) {
5601                 if (strcmp(argv[2], "readonly") == 0) {
5602                         readonly = true;
5603                 } else {
5604                         usage("readkey");
5605                 }
5606         }
5607
5608         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5609                 return 1;
5610         }
5611
5612         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5613                 fprintf(stderr, "DB %s is not a volatile database\n",
5614                         db_name);
5615                 return 1;
5616         }
5617
5618         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5619                           db_flags, &db);
5620         if (ret != 0) {
5621                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5622                 return ret;
5623         }
5624
5625         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5626         if (ret != 0) {
5627                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5628                 return ret;
5629         }
5630
5631         ret = ctdb_fetch_lock(mem_ctx, ctdb->ev, ctdb->client,
5632                               db, key, readonly, &h, NULL, &data);
5633         if (ret != 0) {
5634                 fprintf(stderr, "Failed to read record for key %s\n",
5635                         argv[1]);
5636         } else {
5637                 printf("Data: size:%zu ptr:[%.*s]\n", data.dsize,
5638                        (int)data.dsize, data.dptr);
5639         }
5640
5641         talloc_free(h);
5642         return ret;
5643 }
5644
5645 static int control_writekey(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5646                             int argc, const char **argv)
5647 {
5648         const char *db_name;
5649         struct ctdb_db_context *db;
5650         struct ctdb_record_handle *h;
5651         uint8_t db_flags;
5652         TDB_DATA key, data;
5653         int ret;
5654
5655         if (argc != 3) {
5656                 usage("writekey");
5657         }
5658
5659         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5660                 return 1;
5661         }
5662
5663         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5664                 fprintf(stderr, "DB %s is not a volatile database\n",
5665                         db_name);
5666                 return 1;
5667         }
5668
5669         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5670                           db_flags, &db);
5671         if (ret != 0) {
5672                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5673                 return ret;
5674         }
5675
5676         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5677         if (ret != 0) {
5678                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5679                 return ret;
5680         }
5681
5682         ret = str_to_data(argv[2], strlen(argv[2]), mem_ctx, &data);
5683         if (ret != 0) {
5684                 fprintf(stderr, "Failed to parse value %s\n", argv[2]);
5685                 return ret;
5686         }
5687
5688         ret = ctdb_fetch_lock(mem_ctx, ctdb->ev, ctdb->client,
5689                               db, key, false, &h, NULL, NULL);
5690         if (ret != 0) {
5691                 fprintf(stderr, "Failed to lock record for key %s\n", argv[0]);
5692                 return ret;
5693         }
5694
5695         ret = ctdb_store_record(h, data);
5696         if (ret != 0) {
5697                 fprintf(stderr, "Failed to store record for key %s\n",
5698                         argv[1]);
5699         }
5700
5701         talloc_free(h);
5702         return ret;
5703 }
5704
5705 static int control_deletekey(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5706                              int argc, const char **argv)
5707 {
5708         const char *db_name;
5709         struct ctdb_db_context *db;
5710         struct ctdb_record_handle *h;
5711         uint8_t db_flags;
5712         TDB_DATA key, data;
5713         int ret;
5714
5715         if (argc != 2) {
5716                 usage("deletekey");
5717         }
5718
5719         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5720                 return 1;
5721         }
5722
5723         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5724                 fprintf(stderr, "DB %s is not a volatile database\n",
5725                         db_name);
5726                 return 1;
5727         }
5728
5729         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5730                           db_flags, &db);
5731         if (ret != 0) {
5732                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5733                 return ret;
5734         }
5735
5736         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5737         if (ret != 0) {
5738                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5739                 return ret;
5740         }
5741
5742         ret = ctdb_fetch_lock(mem_ctx, ctdb->ev, ctdb->client,
5743                               db, key, false, &h, NULL, &data);
5744         if (ret != 0) {
5745                 fprintf(stderr, "Failed to fetch record for key %s\n",
5746                         argv[1]);
5747                 return ret;
5748         }
5749
5750         ret = ctdb_delete_record(h);
5751         if (ret != 0) {
5752                 fprintf(stderr, "Failed to delete record for key %s\n",
5753                         argv[1]);
5754         }
5755
5756         talloc_free(h);
5757         return ret;
5758 }
5759
5760 static int control_checktcpport(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5761                                 int argc, const char **argv)
5762 {
5763         struct sockaddr_in sin;
5764         unsigned int port;
5765         int s, v;
5766         int ret;
5767
5768         if (argc != 1) {
5769                 usage("chktcpport");
5770         }
5771
5772         port = atoi(argv[0]);
5773
5774         s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
5775         if (s == -1) {
5776                 fprintf(stderr, "Failed to open local socket\n");
5777                 return errno;
5778         }
5779
5780         v = fcntl(s, F_GETFL, 0);
5781         if (v == -1 || fcntl(s, F_SETFL, v | O_NONBLOCK)) {
5782                 fprintf(stderr, "Unable to set socket non-blocking\n");
5783                 close(s);
5784                 return errno;
5785         }
5786
5787         bzero(&sin, sizeof(sin));
5788         sin.sin_family = AF_INET;
5789         sin.sin_port = htons(port);
5790         ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
5791         close(s);
5792         if (ret == -1) {
5793                 fprintf(stderr, "Failed to bind to TCP port %u\n", port);
5794                 return errno;
5795         }
5796
5797         return 0;
5798 }
5799
5800 static int control_getdbseqnum(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5801                                int argc, const char **argv)
5802 {
5803         uint32_t db_id;
5804         const char *db_name;
5805         uint64_t seqnum;
5806         int ret;
5807
5808         if (argc != 1) {
5809                 usage("getdbseqnum");
5810         }
5811
5812         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, NULL)) {
5813                 return 1;
5814         }
5815
5816         ret = ctdb_ctrl_get_db_seqnum(mem_ctx, ctdb->ev, ctdb->client,
5817                                       ctdb->cmd_pnn, TIMEOUT(), db_id,
5818                                       &seqnum);
5819         if (ret != 0) {
5820                 fprintf(stderr, "Failed to get sequence number for DB %s\n",
5821                         db_name);
5822                 return ret;
5823         }
5824
5825         printf("0x%"PRIx64"\n", seqnum);
5826         return 0;
5827 }
5828
5829 static int control_nodestatus(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5830                               int argc, const char **argv)
5831 {
5832         const char *nodestring = NULL;
5833         struct ctdb_node_map *nodemap;
5834         int ret, i;
5835
5836         if (argc > 1) {
5837                 usage("nodestatus");
5838         }
5839
5840         if (argc == 1) {
5841                 nodestring = argv[0];
5842         }
5843
5844         if (! parse_nodestring(mem_ctx, ctdb, nodestring, &nodemap)) {
5845                 return 1;
5846         }
5847
5848         nodemap = get_nodemap(ctdb, false);
5849         if (nodemap == NULL) {
5850                 return 1;
5851         }
5852
5853         if (options.machinereadable) {
5854                 print_nodemap_machine(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn);
5855         } else {
5856                 print_nodemap(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn);
5857         }
5858
5859         ret = 0;
5860         for (i=0; i<nodemap->num; i++) {
5861                 ret |= nodemap->node[i].flags;
5862         }
5863
5864         return ret;
5865 }
5866
5867 const struct {
5868         const char *name;
5869         uint32_t offset;
5870 } db_stats_fields[] = {
5871 #define DBSTATISTICS_FIELD(n) { #n, offsetof(struct ctdb_db_statistics, n) }
5872         DBSTATISTICS_FIELD(db_ro_delegations),
5873         DBSTATISTICS_FIELD(db_ro_revokes),
5874         DBSTATISTICS_FIELD(locks.num_calls),
5875         DBSTATISTICS_FIELD(locks.num_current),
5876         DBSTATISTICS_FIELD(locks.num_pending),
5877         DBSTATISTICS_FIELD(locks.num_failed),
5878         DBSTATISTICS_FIELD(db_ro_delegations),
5879 };
5880
5881 static void print_dbstatistics(const char *db_name,
5882                                struct ctdb_db_statistics *s)
5883 {
5884         int i;
5885         const char *prefix = NULL;
5886         int preflen = 0;
5887
5888         printf("DB Statistics %s\n", db_name);
5889
5890         for (i=0; i<ARRAY_SIZE(db_stats_fields); i++) {
5891                 if (strchr(db_stats_fields[i].name, '.') != NULL) {
5892                         preflen = strcspn(db_stats_fields[i].name, ".") + 1;
5893                         if (! prefix ||
5894                             strncmp(prefix, db_stats_fields[i].name, preflen) != 0) {
5895                                 prefix = db_stats_fields[i].name;
5896                                 printf(" %*.*s\n", preflen-1, preflen-1,
5897                                        db_stats_fields[i].name);
5898                         }
5899                 } else {
5900                         preflen = 0;
5901                 }
5902                 printf(" %*s%-22s%*s%10u\n", preflen ? 4 : 0, "",
5903                        db_stats_fields[i].name+preflen, preflen ? 0 : 4, "",
5904                        *(uint32_t *)(db_stats_fields[i].offset+(uint8_t *)s));
5905         }
5906
5907         printf(" hop_count_buckets:");
5908         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
5909                 printf(" %d", s->hop_count_bucket[i]);
5910         }
5911         printf("\n");
5912
5913         printf(" lock_buckets:");
5914         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
5915                 printf(" %d", s->locks.buckets[i]);
5916         }
5917         printf("\n");
5918
5919         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
5920                "locks_latency      MIN/AVG/MAX",
5921                s->locks.latency.min, LATENCY_AVG(s->locks.latency),
5922                s->locks.latency.max, s->locks.latency.num);
5923
5924         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
5925                "vacuum_latency     MIN/AVG/MAX",
5926                s->vacuum.latency.min, LATENCY_AVG(s->vacuum.latency),
5927                s->vacuum.latency.max, s->vacuum.latency.num);
5928
5929         printf(" Num Hot Keys:     %d\n", s->num_hot_keys);
5930         for (i=0; i<s->num_hot_keys; i++) {
5931                 int j;
5932                 printf("     Count:%d Key:", s->hot_keys[i].count);
5933                 for (j=0; j<s->hot_keys[i].key.dsize; j++) {
5934                         printf("%02x", s->hot_keys[i].key.dptr[j] & 0xff);
5935                 }
5936                 printf("\n");
5937         }
5938 }
5939
5940 static int control_dbstatistics(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5941                                 int argc, const char **argv)
5942 {
5943         uint32_t db_id;
5944         const char *db_name;
5945         struct ctdb_db_statistics *dbstats;
5946         int ret;
5947
5948         if (argc != 1) {
5949                 usage("dbstatistics");
5950         }
5951
5952         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, NULL)) {
5953                 return 1;
5954         }
5955
5956         ret = ctdb_ctrl_get_db_statistics(mem_ctx, ctdb->ev, ctdb->client,
5957                                           ctdb->cmd_pnn, TIMEOUT(), db_id,
5958                                           &dbstats);
5959         if (ret != 0) {
5960                 fprintf(stderr, "Failed to get statistics for DB %s\n",
5961                         db_name);
5962                 return ret;
5963         }
5964
5965         print_dbstatistics(db_name, dbstats);
5966         return 0;
5967 }
5968
5969 struct disable_takeover_runs_state {
5970         uint32_t *pnn_list;
5971         int node_count;
5972         bool *reply;
5973         int status;
5974         bool done;
5975 };
5976
5977 static void disable_takeover_run_handler(uint64_t srvid, TDB_DATA data,
5978                                          void *private_data)
5979 {
5980         struct disable_takeover_runs_state *state =
5981                 (struct disable_takeover_runs_state *)private_data;
5982         int ret, i;
5983
5984         if (data.dsize != sizeof(int)) {
5985                 /* Ignore packet */
5986                 return;
5987         }
5988
5989         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
5990         ret = *(int *)data.dptr;
5991         if (ret < 0) {
5992                 state->status = ret;
5993                 state->done = true;
5994                 return;
5995         }
5996         for (i=0; i<state->node_count; i++) {
5997                 if (state->pnn_list[i] == ret) {
5998                         state->reply[i] = true;
5999                         break;
6000                 }
6001         }
6002
6003         state->done = true;
6004         for (i=0; i<state->node_count; i++) {
6005                 if (! state->reply[i]) {
6006                         state->done = false;
6007                         break;
6008                 }
6009         }
6010 }
6011
6012 static int disable_takeover_runs(TALLOC_CTX *mem_ctx,
6013                                  struct ctdb_context *ctdb, uint32_t timeout,
6014                                  uint32_t *pnn_list, int count)
6015 {
6016         struct ctdb_disable_message disable = { 0 };
6017         struct disable_takeover_runs_state state;
6018         int ret, i;
6019
6020         disable.pnn = ctdb->pnn;
6021         disable.srvid = next_srvid(ctdb);
6022         disable.timeout = timeout;
6023
6024         state.pnn_list = pnn_list;
6025         state.node_count = count;
6026         state.done = false;
6027         state.status = 0;
6028         state.reply = talloc_zero_array(mem_ctx, bool, count);
6029         if (state.reply == NULL) {
6030                 return ENOMEM;
6031         }
6032
6033         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
6034                                               disable.srvid,
6035                                               disable_takeover_run_handler,
6036                                               &state);
6037         if (ret != 0) {
6038                 return ret;
6039         }
6040
6041         for (i=0; i<count; i++) {
6042                 ret = ctdb_message_disable_takeover_runs(mem_ctx, ctdb->ev,
6043                                                          ctdb->client,
6044                                                          pnn_list[i],
6045                                                          &disable);
6046                 if (ret != 0) {
6047                         goto fail;
6048                 }
6049         }
6050
6051         ret = ctdb_client_wait_timeout(ctdb->ev, &state.done, TIMEOUT());
6052         if (ret == ETIME) {
6053                 fprintf(stderr, "Timed out waiting to disable takeover runs\n");
6054         } else {
6055                 ret = (state.status >= 0 ? 0 : 1);
6056         }
6057
6058 fail:
6059         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
6060                                            disable.srvid, &state);
6061         return ret;
6062 }
6063
6064 static int control_reloadips(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
6065                              int argc, const char **argv)
6066 {
6067         const char *nodestring = NULL;
6068         struct ctdb_node_map *nodemap, *nodemap2;
6069         struct ctdb_req_control request;
6070         uint32_t *pnn_list, *pnn_list2;
6071         int ret, count, count2;
6072
6073         if (argc > 1) {
6074                 usage("reloadips");
6075         }
6076
6077         if (argc == 1) {
6078                 nodestring = argv[0];
6079         }
6080
6081         nodemap = get_nodemap(ctdb, false);
6082         if (nodemap == NULL) {
6083                 return 1;
6084         }
6085
6086         if (! parse_nodestring(mem_ctx, ctdb, nodestring, &nodemap2)) {
6087                 return 1;
6088         }
6089
6090         count = list_of_connected_nodes(nodemap, CTDB_UNKNOWN_PNN,
6091                                         mem_ctx, &pnn_list);
6092         if (count <= 0) {
6093                 fprintf(stderr, "Memory allocation error\n");
6094                 return 1;
6095         }
6096
6097         count2 = list_of_active_nodes(nodemap2, CTDB_UNKNOWN_PNN,
6098                                       mem_ctx, &pnn_list2);
6099         if (count2 <= 0) {
6100                 fprintf(stderr, "Memory allocation error\n");
6101                 return 1;
6102         }
6103
6104         /* Disable takeover runs on all connected nodes.  A reply
6105          * indicating success is needed from each node so all nodes
6106          * will need to be active.
6107          *
6108          * A check could be added to not allow reloading of IPs when
6109          * there are disconnected nodes.  However, this should
6110          * probably be left up to the administrator.
6111          */
6112         ret = disable_takeover_runs(mem_ctx, ctdb, 2*options.timelimit,
6113                                     pnn_list, count);
6114         if (ret != 0) {
6115                 fprintf(stderr, "Failed to disable takeover runs\n");
6116                 return ret;
6117         }
6118
6119         /* Now tell all the desired nodes to reload their public IPs.
6120          * Keep trying this until it succeeds.  This assumes all
6121          * failures are transient, which might not be true...
6122          */
6123         ctdb_req_control_reload_public_ips(&request);
6124         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
6125                                         pnn_list2, count2, TIMEOUT(),
6126                                         &request, NULL, NULL);
6127         if (ret != 0) {
6128                 fprintf(stderr, "Failed to reload IPs on some nodes.\n");
6129         }
6130
6131         /* It isn't strictly necessary to wait until takeover runs are
6132          * re-enabled but doing so can't hurt.
6133          */
6134         ret = disable_takeover_runs(mem_ctx, ctdb, 0, pnn_list, count);
6135         if (ret != 0) {
6136                 fprintf(stderr, "Failed to enable takeover runs\n");
6137                 return ret;
6138         }
6139
6140         return ipreallocate(mem_ctx, ctdb);
6141 }
6142
6143 static int control_ipiface(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
6144                            int argc, const char **argv)
6145 {
6146         ctdb_sock_addr addr;
6147         char *iface;
6148
6149         if (argc != 1) {
6150                 usage("ipiface");
6151         }
6152
6153         if (! parse_ip(argv[0], NULL, 0, &addr)) {
6154                 fprintf(stderr, "Failed to Parse IP %s\n", argv[0]);
6155                 return 1;
6156         }
6157
6158         iface = ctdb_sys_find_ifname(&addr);
6159         if (iface == NULL) {
6160                 fprintf(stderr, "Failed to find interface for IP %s\n",
6161                         argv[0]);
6162                 return 1;
6163         }
6164         free(iface);
6165
6166         return 0;
6167 }
6168
6169
6170 static const struct ctdb_cmd {
6171         const char *name;
6172         int (*fn)(TALLOC_CTX *, struct ctdb_context *, int, const char **);
6173         bool without_daemon; /* can be run without daemon running ? */
6174         bool remote; /* can be run on remote nodes */
6175         const char *msg;
6176         const char *args;
6177 } ctdb_commands[] = {
6178         { "version", control_version, true, false,
6179                 "show version of ctdb", NULL },
6180         { "status", control_status, false, true,
6181                 "show node status", NULL },
6182         { "uptime", control_uptime, false, true,
6183                 "show node uptime", NULL },
6184         { "ping", control_ping, false, true,
6185                 "ping all nodes", NULL },
6186         { "runstate", control_runstate, false, true,
6187                 "get/check runstate of a node",
6188                 "[setup|first_recovery|startup|running]" },
6189         { "getvar", control_getvar, false, true,
6190                 "get a tunable variable", "<name>" },
6191         { "setvar", control_setvar, false, true,
6192                 "set a tunable variable", "<name> <value>" },
6193         { "listvars", control_listvars, false, true,
6194                 "list tunable variables", NULL },
6195         { "statistics", control_statistics, false, true,
6196                 "show ctdb statistics", NULL },
6197         { "statisticsreset", control_statistics_reset, false, true,
6198                 "reset ctdb statistics", NULL },
6199         { "stats", control_stats, false, true,
6200                 "show rolling statistics", "[count]" },
6201         { "ip", control_ip, false, true,
6202                 "show public ips", "[all]" },
6203         { "ipinfo", control_ipinfo, false, true,
6204                 "show public ip details", "<ip>" },
6205         { "ifaces", control_ifaces, false, true,
6206                 "show interfaces", NULL },
6207         { "setifacelink", control_setifacelink, false, true,
6208                 "set interface link status", "<iface> up|down" },
6209         { "process-exists", control_process_exists, false, true,
6210                 "check if a process exists on a node",  "<pid>" },
6211         { "getdbmap", control_getdbmap, false, true,
6212                 "show attached databases", NULL },
6213         { "getdbstatus", control_getdbstatus, false, true,
6214                 "show database status", "<dbname|dbid>" },
6215         { "catdb", control_catdb, false, false,
6216                 "dump cluster-wide ctdb database", "<dbname|dbid>" },
6217         { "cattdb", control_cattdb, false, false,
6218                 "dump local ctdb database", "<dbname|dbid>" },
6219         { "getmonmode", control_getmonmode, false, true,
6220                 "show monitoring mode", NULL },
6221         { "getcapabilities", control_getcapabilities, false, true,
6222                 "show node capabilities", NULL },
6223         { "pnn", control_pnn, false, false,
6224                 "show the pnn of the currnet node", NULL },
6225         { "lvs", control_lvs, false, false,
6226                 "show lvs configuration", "master|list|status" },
6227         { "disablemonitor", control_disable_monitor, false, true,
6228                 "disable monitoring", NULL },
6229         { "enablemonitor", control_enable_monitor, false, true,
6230                 "enable monitoring", NULL },
6231         { "setdebug", control_setdebug, false, true,
6232                 "set debug level", "ERROR|WARNING|NOTICE|INFO|DEBUG" },
6233         { "getdebug", control_getdebug, false, true,
6234                 "get debug level", NULL },
6235         { "attach", control_attach, false, false,
6236                 "attach a database", "<dbname> [persistent]" },
6237         { "detach", control_detach, false, false,
6238                 "detach database(s)", "<dbname|dbid> ..." },
6239         { "dumpmemory", control_dumpmemory, false, true,
6240                 "dump ctdbd memory map", NULL },
6241         { "rddumpmemory", control_rddumpmemory, false, true,
6242                 "dump recoverd memory map", NULL },
6243         { "getpid", control_getpid, false, true,
6244                 "get ctdbd process ID", NULL },
6245         { "disable", control_disable, false, true,
6246                 "disable a node", NULL },
6247         { "enable", control_enable, false, true,
6248                 "enable a node", NULL },
6249         { "stop", control_stop, false, true,
6250                 "stop a node", NULL },
6251         { "continue", control_continue, false, true,
6252                 "continue a stopped node", NULL },
6253         { "ban", control_ban, false, true,
6254                 "ban a node", "<bantime>"},
6255         { "unban", control_unban, false, true,
6256                 "unban a node", NULL },
6257         { "shutdown", control_shutdown, false, true,
6258                 "shutdown ctdb daemon", NULL },
6259         { "recover", control_recover, false, true,
6260                 "force recovery", NULL },
6261         { "sync", control_ipreallocate, false, true,
6262                 "run ip reallocation (deprecated)", NULL },
6263         { "ipreallocate", control_ipreallocate, false, true,
6264                 "run ip reallocation", NULL },
6265         { "isnotrecmaster", control_isnotrecmaster, false, false,
6266                 "check if local node is the recmaster", NULL },
6267         { "gratarp", control_gratarp, false, true,
6268                 "send a gratuitous arp", "<ip> <interface>" },
6269         { "tickle", control_tickle, true, false,
6270                 "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
6271         { "gettickles", control_gettickles, false, true,
6272                 "get the list of tickles", "<ip> [<port>]" },
6273         { "addtickle", control_addtickle, false, true,
6274                 "add a tickle", "<ip>:<port> <ip>:<port>" },
6275         { "deltickle", control_deltickle, false, true,
6276                 "delete a tickle", "<ip>:<port> <ip>:<port>" },
6277         { "check_srvids", control_check_srvids, false, true,
6278                 "check if srvid is registered", "<id> [<id> ...]" },
6279         { "listnodes", control_listnodes, true, true,
6280                 "list nodes in the cluster", NULL },
6281         { "reloadnodes", control_reloadnodes, false, false,
6282                 "reload the nodes file all nodes", NULL },
6283         { "moveip", control_moveip, false, false,
6284                 "move an ip address to another node", "<ip> <node>" },
6285         { "addip", control_addip, false, true,
6286                 "add an ip address to a node", "<ip/mask> <iface>" },
6287         { "delip", control_delip, false, true,
6288                 "delete an ip address from a node", "<ip>" },
6289         { "eventscript", control_eventscript, false, true,
6290                 "run an event", "monitor" },
6291         { "backupdb", control_backupdb, false, false,
6292                 "backup a database into a file", "<dbname|dbid> <file>" },
6293         { "restoredb", control_restoredb, false, false,
6294                 "restore a database from a file", "<file> [dbname]" },
6295         { "dumpdbbackup", control_dumpdbbackup, true, false,
6296                 "dump database from a backup file", "<file>" },
6297         { "wipedb", control_wipedb, false, false,
6298                 "wipe the contents of a database.", "<dbname|dbid>"},
6299         { "recmaster", control_recmaster, false, true,
6300                 "show the pnn for the recovery master", NULL },
6301         { "scriptstatus", control_scriptstatus, false, true,
6302                 "show event script status",
6303                 "[init|setup|startup|monitor|takeip|releaseip|ipreallocated]" },
6304         { "enablescript", control_enablescript, false, true,
6305                 "enable an eventscript", "<script>"},
6306         { "disablescript", control_disablescript, false, true,
6307                 "disable an eventscript", "<script>"},
6308         { "natgw", control_natgw, false, false,
6309                 "show natgw configuration", "master|list|status" },
6310         { "natgwlist", control_natgwlist, false, false,
6311                 "show the nodes belonging to this natgw configuration", NULL },
6312         { "getreclock", control_getreclock, false, true,
6313                 "get recovery lock file", NULL },
6314         { "setlmasterrole", control_setlmasterrole, false, true,
6315                 "set LMASTER role", "on|off" },
6316         { "setrecmasterrole", control_setrecmasterrole, false, true,
6317                 "set RECMASTER role", "on|off"},
6318         { "setdbreadonly", control_setdbreadonly, false, true,
6319                 "enable readonly records", "<dbname|dbid>" },
6320         { "setdbsticky", control_setdbsticky, false, true,
6321                 "enable sticky records", "<dbname|dbid>"},
6322         { "pfetch", control_pfetch, false, false,
6323                 "fetch record from persistent database", "<dbname|dbid> <key> [<file>]" },
6324         { "pstore", control_pstore, false, false,
6325                 "write record to persistent database", "<dbname|dbid> <key> <value>" },
6326         { "pdelete", control_pdelete, false, false,
6327                 "delete record from persistent database", "<dbname|dbid> <key>" },
6328         { "ptrans", control_ptrans, false, false,
6329                 "update a persistent database (from file or stdin)", "<dbname|dbid> [<file>]" },
6330         { "tfetch", control_tfetch, false, true,
6331                 "fetch a record", "<tdb-file> <key> [<file>]" },
6332         { "tstore", control_tstore, false, true,
6333                 "store a record", "<tdb-file> <key> <data> [<rsn> <dmaster> <flags>]" },
6334         { "readkey", control_readkey, false, false,
6335                 "read value of a database key", "<dbname|dbid> <key> [readonly]" },
6336         { "writekey", control_writekey, false, false,
6337                 "write value for a database key", "<dbname|dbid> <key> <value>" },
6338         { "deletekey", control_deletekey, false, false,
6339                 "delete a database key", "<dbname|dbid> <key>" },
6340         { "checktcpport", control_checktcpport, true, false,
6341                 "check if a service is bound to a specific tcp port or not", "<port>" },
6342         { "getdbseqnum", control_getdbseqnum, false, false,
6343                 "get database sequence number", "<dbname|dbid>" },
6344         { "nodestatus", control_nodestatus, false, true,
6345                 "show and return node status", "[all|<pnn-list>]" },
6346         { "dbstatistics", control_dbstatistics, false, true,
6347                 "show database statistics", "<dbname|dbid>" },
6348         { "reloadips", control_reloadips, false, false,
6349                 "reload the public addresses file", "[all|<pnn-list>]" },
6350         { "ipiface", control_ipiface, true, false,
6351                 "Find the interface an ip address is hosted on", "<ip>" },
6352 };
6353
6354 static const struct ctdb_cmd *match_command(const char *command)
6355 {
6356         const struct ctdb_cmd *cmd;
6357         int i;
6358
6359         for (i=0; i<ARRAY_SIZE(ctdb_commands); i++) {
6360                 cmd = &ctdb_commands[i];
6361                 if (strlen(command) == strlen(cmd->name) &&
6362                     strncmp(command, cmd->name, strlen(command)) == 0) {
6363                         return cmd;
6364                 }
6365         }
6366
6367         return NULL;
6368 }
6369
6370
6371 /**
6372  * Show usage message
6373  */
6374 static void usage_full(void)
6375 {
6376         int i;
6377
6378         poptPrintHelp(pc, stdout, 0);
6379         printf("\nCommands:\n");
6380         for (i=0; i<ARRAY_SIZE(ctdb_commands); i++) {
6381                 printf("  %-15s %-27s  %s\n",
6382                        ctdb_commands[i].name,
6383                        ctdb_commands[i].args ? ctdb_commands[i].args : "",
6384                        ctdb_commands[i].msg);
6385         }
6386 }
6387
6388 static void usage(const char *command)
6389 {
6390         const struct ctdb_cmd *cmd;
6391
6392         if (command == NULL) {
6393                 usage_full();
6394                 exit(1);
6395         }
6396
6397         cmd = match_command(command);
6398         if (cmd == NULL) {
6399                 usage_full();
6400         } else {
6401                 poptPrintUsage(pc, stdout, 0);
6402                 printf("\nCommands:\n");
6403                 printf("  %-15s %-27s  %s\n",
6404                        cmd->name, cmd->args ? cmd->args : "", cmd->msg);
6405         }
6406
6407         exit(1);
6408 }
6409
6410 struct poptOption cmdline_options[] = {
6411         POPT_AUTOHELP
6412         { "socket", 's', POPT_ARG_STRING, &options.socket, 0,
6413                 "CTDB socket path", "filename" },
6414         { "debug", 'd', POPT_ARG_STRING, &options.debuglevelstr, 0,
6415                 "debug level"},
6416         { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0,
6417                 "timelimit (in seconds)" },
6418         { "node", 'n', POPT_ARG_INT, &options.pnn, 0,
6419                 "node specification - integer" },
6420         { NULL, 'Y', POPT_ARG_NONE, &options.machinereadable, 0,
6421                 "enable machine readable output", NULL },
6422         { "separator", 'x', POPT_ARG_STRING, &options.sep, 0,
6423                 "specify separator for machine readable output", "CHAR" },
6424         { NULL, 'X', POPT_ARG_NONE, &options.machineparsable, 0,
6425                 "enable machine parsable output with separator |", NULL },
6426         { "verbose", 'v', POPT_ARG_NONE, &options.verbose, 0,
6427                 "enable verbose output", NULL },
6428         { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0,
6429                 "die if runtime exceeds this limit (in seconds)" },
6430         POPT_TABLEEND
6431 };
6432
6433 static int process_command(const struct ctdb_cmd *cmd, int argc,
6434                            const char **argv)
6435 {
6436         TALLOC_CTX *tmp_ctx;
6437         struct ctdb_context *ctdb;
6438         int ret;
6439         bool status;
6440         uint64_t srvid_offset;
6441
6442         tmp_ctx = talloc_new(NULL);
6443         if (tmp_ctx == NULL) {
6444                 fprintf(stderr, "Memory allocation error\n");
6445                 goto fail;
6446         }
6447
6448         if (cmd->without_daemon) {
6449                 if (options.pnn != -1) {
6450                         fprintf(stderr,
6451                                 "Cannot specify node for command %s\n",
6452                                 cmd->name);
6453                         goto fail;
6454                 }
6455
6456                 ret = cmd->fn(tmp_ctx, NULL, argc-1, argv+1);
6457                 talloc_free(tmp_ctx);
6458                 return ret;
6459         }
6460
6461         ctdb = talloc_zero(tmp_ctx, struct ctdb_context);
6462         if (ctdb == NULL) {
6463                 fprintf(stderr, "Memory allocation error\n");
6464                 goto fail;
6465         }
6466
6467         ctdb->ev = tevent_context_init(ctdb);
6468         if (ctdb->ev == NULL) {
6469                 fprintf(stderr, "Failed to initialize tevent\n");
6470                 goto fail;
6471         }
6472
6473         ret = ctdb_client_init(ctdb, ctdb->ev, options.socket, &ctdb->client);
6474         if (ret != 0) {
6475                 fprintf(stderr, "Failed to connect to CTDB daemon (%s)\n",
6476                         options.socket);
6477
6478                 if (!find_node_xpnn(ctdb, NULL)) {
6479                         fprintf(stderr, "Is this node part of CTDB cluster?\n");
6480                 }
6481                 goto fail;
6482         }
6483
6484         ctdb->pnn = ctdb_client_pnn(ctdb->client);
6485         srvid_offset = getpid() & 0xFFFF;
6486         ctdb->srvid = SRVID_CTDB_TOOL | (srvid_offset << 16);
6487
6488         if (options.pnn != -1) {
6489                 status = verify_pnn(ctdb, options.pnn);
6490                 if (! status) {
6491                         goto fail;
6492                 }
6493
6494                 ctdb->cmd_pnn = options.pnn;
6495         } else {
6496                 ctdb->cmd_pnn = ctdb->pnn;
6497         }
6498
6499         if (! cmd->remote && ctdb->pnn != ctdb->cmd_pnn) {
6500                 fprintf(stderr, "Node cannot be specified for command %s\n",
6501                         cmd->name);
6502                 goto fail;
6503         }
6504
6505         ret = cmd->fn(tmp_ctx, ctdb, argc-1, argv+1);
6506         talloc_free(tmp_ctx);
6507         return ret;
6508
6509 fail:
6510         talloc_free(tmp_ctx);
6511         return 1;
6512 }
6513
6514 static void signal_handler(int sig)
6515 {
6516         fprintf(stderr, "Maximum runtime exceeded - exiting\n");
6517 }
6518
6519 static void alarm_handler(int sig)
6520 {
6521         /* Kill any child processes */
6522         signal(SIGTERM, signal_handler);
6523         kill(0, SIGTERM);
6524
6525         _exit(1);
6526 }
6527
6528 int main(int argc, const char *argv[])
6529 {
6530         int opt;
6531         const char **extra_argv;
6532         int extra_argc;
6533         const struct ctdb_cmd *cmd;
6534         const char *ctdb_socket;
6535         int loglevel;
6536         int ret;
6537
6538         setlinebuf(stdout);
6539
6540         /* Set default options */
6541         options.socket = CTDB_SOCKET;
6542         options.debuglevelstr = NULL;
6543         options.timelimit = 10;
6544         options.sep = "|";
6545         options.maxruntime = 0;
6546         options.pnn = -1;
6547
6548         ctdb_socket = getenv("CTDB_SOCKET");
6549         if (ctdb_socket != NULL) {
6550                 options.socket = ctdb_socket;
6551         }
6552
6553         pc = poptGetContext(argv[0], argc, argv, cmdline_options,
6554                             POPT_CONTEXT_KEEP_FIRST);
6555         while ((opt = poptGetNextOpt(pc)) != -1) {
6556                 fprintf(stderr, "Invalid option %s: %s\n",
6557                         poptBadOption(pc, 0), poptStrerror(opt));
6558                 exit(1);
6559         }
6560
6561         if (options.maxruntime == 0) {
6562                 const char *ctdb_timeout;
6563
6564                 ctdb_timeout = getenv("CTDB_TIMEOUT");
6565                 if (ctdb_timeout != NULL) {
6566                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
6567                 } else {
6568                         options.maxruntime = 120;
6569                 }
6570         }
6571         if (options.maxruntime <= 120) {
6572                 /* default timeout is 120 seconds */
6573                 options.maxruntime = 120;
6574         }
6575
6576         if (options.machineparsable) {
6577                 options.machinereadable = 1;
6578         }
6579
6580         /* setup the remaining options for the commands */
6581         extra_argc = 0;
6582         extra_argv = poptGetArgs(pc);
6583         if (extra_argv) {
6584                 extra_argv++;
6585                 while (extra_argv[extra_argc]) extra_argc++;
6586         }
6587
6588         if (extra_argc < 1) {
6589                 usage(NULL);
6590         }
6591
6592         cmd = match_command(extra_argv[0]);
6593         if (cmd == NULL) {
6594                 fprintf(stderr, "Unknown command '%s'\n", extra_argv[0]);
6595                 exit(1);
6596         }
6597
6598         /* Enable logging */
6599         setup_logging("ctdb", DEBUG_STDERR);
6600         if (debug_level_parse(options.debuglevelstr, &loglevel)) {
6601                 DEBUGLEVEL = loglevel;
6602         } else {
6603                 DEBUGLEVEL = DEBUG_ERR;
6604         }
6605
6606         signal(SIGALRM, alarm_handler);
6607         alarm(options.maxruntime);
6608
6609         ret = process_command(cmd, extra_argc, extra_argv);
6610         if (ret == -1) {
6611                 ret = 1;
6612         }
6613
6614         (void)poptFreeContext(pc);
6615
6616         return ret;
6617 }