ctdb/tools/ctdb.c

   1 /*
   2    CTDB control tool
   3
   4    Copyright (C) Amitay Isaacs  2015
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20 #include "replace.h"
  21 #include "system/network.h"
  22 #include "system/filesys.h"
  23 #include "system/time.h"
  24 #include "system/wait.h"
  25 #include "system/dir.h"
  26
  27 #include <ctype.h>
  28 #include <popt.h>
  29 #include <talloc.h>
  30 #include <tevent.h>
  31 #include <tdb.h>
  32
  33 #include "ctdb_version.h"
  34 #include "lib/util/debug.h"
  35 #include "lib/util/samba_util.h"
  36 #include "lib/util/sys_rw.h"
  37
  38 #include "common/db_hash.h"
  39 #include "common/logging.h"
  40 #include "protocol/protocol.h"
  41 #include "protocol/protocol_api.h"
  42 #include "common/system.h"
  43 #include "client/client.h"
  44
  45 #define TIMEOUT()       timeval_current_ofs(options.timelimit, 0)
  46
  47 #define SRVID_CTDB_TOOL    (CTDB_SRVID_TOOL_RANGE | 0x0001000000000000LL)
  48 #define SRVID_CTDB_PUSHDB  (CTDB_SRVID_TOOL_RANGE | 0x0002000000000000LL)
  49
  50 static struct {
  51         const char *socket;
  52         const char *debuglevelstr;
  53         int timelimit;
  54         int pnn;
  55         int machinereadable;
  56         const char *sep;
  57         int machineparsable;
  58         int verbose;
  59         int maxruntime;
  60         int printemptyrecords;
  61         int printdatasize;
  62         int printlmaster;
  63         int printhash;
  64         int printrecordflags;
  65 } options;
  66
  67 static poptContext pc;
  68
  69 struct ctdb_context {
  70         struct tevent_context *ev;
  71         struct ctdb_client_context *client;
  72         struct ctdb_node_map *nodemap;
  73         uint32_t pnn, cmd_pnn;
  74         uint64_t srvid;
  75 };
  76
  77 static void usage(const char *command);
  78
  79 /*
  80  * Utility Functions
  81  */
  82
  83 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
  84 {
  85         return (tv2->tv_sec - tv->tv_sec) +
  86                (tv2->tv_usec - tv->tv_usec) * 1.0e-6;
  87 }
  88
  89 static struct ctdb_node_and_flags *get_node_by_pnn(
  90                                         struct ctdb_node_map *nodemap,
  91                                         uint32_t pnn)
  92 {
  93         int i;
  94
  95         for (i=0; i<nodemap->num; i++) {
  96                 if (nodemap->node[i].pnn == pnn) {
  97                         return &nodemap->node[i];
  98                 }
  99         }
 100         return NULL;
 101 }
 102
 103 static const char *pretty_print_flags(TALLOC_CTX *mem_ctx, uint32_t flags)
 104 {
 105         static const struct {
 106                 uint32_t flag;
 107                 const char *name;
 108         } flag_names[] = {
 109                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 110                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 111                 { NODE_FLAGS_BANNED,                "BANNED" },
 112                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 113                 { NODE_FLAGS_DELETED,               "DELETED" },
 114                 { NODE_FLAGS_STOPPED,               "STOPPED" },
 115                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 116         };
 117         char *flags_str = NULL;
 118         int i;
 119
 120         for (i=0; i<ARRAY_SIZE(flag_names); i++) {
 121                 if (flags & flag_names[i].flag) {
 122                         if (flags_str == NULL) {
 123                                 flags_str = talloc_asprintf(mem_ctx,
 124                                                 "%s", flag_names[i].name);
 125                         } else {
 126                                 flags_str = talloc_asprintf_append(flags_str,
 127                                                 "|%s", flag_names[i].name);
 128                         }
 129                         if (flags_str == NULL) {
 130                                 return "OUT-OF-MEMORY";
 131                         }
 132                 }
 133         }
 134         if (flags_str == NULL) {
 135                 return "OK";
 136         }
 137
 138         return flags_str;
 139 }
 140
 141 static uint64_t next_srvid(struct ctdb_context *ctdb)
 142 {
 143         ctdb->srvid += 1;
 144         return ctdb->srvid;
 145 }
 146
 147 /*
 148  * Get consistent nodemap information.
 149  *
 150  * If nodemap is already cached, use that. If not get it.
 151  * If the current node is BANNED, then get nodemap from "better" node.
 152  */
 153 static struct ctdb_node_map *get_nodemap(struct ctdb_context *ctdb, bool force)
 154 {
 155         TALLOC_CTX *tmp_ctx;
 156         struct ctdb_node_map *nodemap;
 157         struct ctdb_node_and_flags *node;
 158         uint32_t current_node;
 159         int ret;
 160
 161         if (force) {
 162                 TALLOC_FREE(ctdb->nodemap);
 163         }
 164
 165         if (ctdb->nodemap != NULL) {
 166                 return ctdb->nodemap;
 167         }
 168
 169         tmp_ctx = talloc_new(ctdb);
 170         if (tmp_ctx == NULL) {
 171                 return false;
 172         }
 173
 174         current_node = ctdb->pnn;
 175 again:
 176         ret = ctdb_ctrl_get_nodemap(tmp_ctx, ctdb->ev, ctdb->client,
 177                                     current_node, TIMEOUT(), &nodemap);
 178         if (ret != 0) {
 179                 fprintf(stderr, "Failed to get nodemap from node %u\n",
 180                         current_node);
 181                 goto failed;
 182         }
 183
 184         node = get_node_by_pnn(nodemap, current_node);
 185         if (node->flags & NODE_FLAGS_BANNED) {
 186                 /* Pick next node */
 187                 do {
 188                         current_node = (current_node + 1) % nodemap->num;
 189                         node = get_node_by_pnn(nodemap, current_node);
 190                         if (! (node->flags &
 191                               (NODE_FLAGS_DELETED|NODE_FLAGS_DISCONNECTED))) {
 192                                 break;
 193                         }
 194                 } while (current_node != ctdb->pnn);
 195
 196                 if (current_node == ctdb->pnn) {
 197                         /* Tried all nodes in the cluster */
 198                         fprintf(stderr, "Warning: All nodes are banned.\n");
 199                         goto failed;
 200                 }
 201
 202                 goto again;
 203         }
 204
 205         ctdb->nodemap = talloc_steal(ctdb, nodemap);
 206         return nodemap;
 207
 208 failed:
 209         talloc_free(tmp_ctx);
 210         return NULL;
 211 }
 212
 213 static bool verify_pnn(struct ctdb_context *ctdb, int pnn)
 214 {
 215         struct ctdb_node_map *nodemap;
 216         bool found;
 217         int i;
 218
 219         if (pnn == -1) {
 220                 return false;
 221         }
 222
 223         nodemap = get_nodemap(ctdb, false);
 224         if (nodemap == NULL) {
 225                 return false;
 226         }
 227
 228         found = false;
 229         for (i=0; i<nodemap->num; i++) {
 230                 if (nodemap->node[i].pnn == pnn) {
 231                         found = true;
 232                         break;
 233                 }
 234         }
 235         if (! found) {
 236                 fprintf(stderr, "Node %u does not exist\n", pnn);
 237                 return false;
 238         }
 239
 240         if (nodemap->node[i].flags &
 241             (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED)) {
 242                 fprintf(stderr, "Node %u has status %s\n", pnn,
 243                         pretty_print_flags(ctdb, nodemap->node[i].flags));
 244                 return false;
 245         }
 246
 247         return true;
 248 }
 249
 250 static struct ctdb_node_map *talloc_nodemap(TALLOC_CTX *mem_ctx,
 251                                             struct ctdb_node_map *nodemap)
 252 {
 253         struct ctdb_node_map *nodemap2;
 254
 255         nodemap2 = talloc_zero(mem_ctx, struct ctdb_node_map);
 256         if (nodemap2 == NULL) {
 257                 return NULL;
 258         }
 259
 260         nodemap2->node = talloc_array(nodemap2, struct ctdb_node_and_flags,
 261                                       nodemap->num);
 262         if (nodemap2->node == NULL) {
 263                 talloc_free(nodemap2);
 264                 return NULL;
 265         }
 266
 267         return nodemap2;
 268 }
 269
 270 /*
 271  * Get the number and the list of matching nodes
 272  *
 273  *   nodestring :=  NULL | all | pnn,[pnn,...]
 274  *
 275  * If nodestring is NULL, use the current node.
 276  */
 277 static bool parse_nodestring(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 278                              const char *nodestring,
 279                              struct ctdb_node_map **out)
 280 {
 281         struct ctdb_node_map *nodemap, *nodemap2;
 282         struct ctdb_node_and_flags *node;
 283         int i;
 284
 285         nodemap = get_nodemap(ctdb, false);
 286         if (nodemap == NULL) {
 287                 return false;
 288         }
 289
 290         nodemap2 = talloc_nodemap(mem_ctx, nodemap);
 291         if (nodemap2 == NULL) {
 292                 return false;
 293         }
 294
 295         if (nodestring == NULL) {
 296                 for (i=0; i<nodemap->num; i++) {
 297                         if (nodemap->node[i].pnn == ctdb->cmd_pnn) {
 298                                 nodemap2->node[0] = nodemap->node[i];
 299                                 break;
 300                         }
 301                 }
 302                 nodemap2->num = 1;
 303
 304                 goto done;
 305         }
 306
 307         if (strcmp(nodestring, "all") == 0) {
 308                 for (i=0; i<nodemap->num; i++) {
 309                         nodemap2->node[i] = nodemap->node[i];
 310                 }
 311                 nodemap2->num = nodemap->num;
 312
 313                 goto done;
 314         } else {
 315                 char *ns, *tok;
 316
 317                 ns = talloc_strdup(mem_ctx, nodestring);
 318                 if (ns == NULL) {
 319                         return false;
 320                 }
 321
 322                 tok = strtok(ns, ",");
 323                 while (tok != NULL) {
 324                         uint32_t pnn;
 325                         char *endptr;
 326
 327                         pnn = (uint32_t)strtoul(tok, &endptr, 0);
 328                         if (pnn == 0 && tok == endptr) {
 329                                 fprintf(stderr, "Invalid node %s\n", tok);
 330                                         return false;
 331                         }
 332
 333                         node = get_node_by_pnn(nodemap, pnn);
 334                         if (node == NULL) {
 335                                 fprintf(stderr, "Node %u does not exist\n",
 336                                         pnn);
 337                                 return false;
 338                         }
 339
 340                         nodemap2->node[nodemap2->num] = *node;
 341                         nodemap2->num += 1;
 342
 343                         tok = strtok(NULL, ",");
 344                 }
 345         }
 346
 347 done:
 348         *out = nodemap2;
 349         return true;
 350 }
 351
 352 /* Compare IP address */
 353 static bool ctdb_same_ip(ctdb_sock_addr *ip1, ctdb_sock_addr *ip2)
 354 {
 355         bool ret = false;
 356
 357         if (ip1->sa.sa_family != ip2->sa.sa_family) {
 358                 return false;
 359         }
 360
 361         switch (ip1->sa.sa_family) {
 362         case AF_INET:
 363                 ret = (memcmp(&ip1->ip.sin_addr, &ip2->ip.sin_addr,
 364                               sizeof(struct in_addr)) == 0);
 365                 break;
 366
 367         case AF_INET6:
 368                 ret = (memcmp(&ip1->ip6.sin6_addr, &ip2->ip6.sin6_addr,
 369                               sizeof(struct in6_addr)) == 0);
 370                 break;
 371         }
 372
 373         return ret;
 374 }
 375
 376 /* Append a node to a node map with given address and flags */
 377 static bool node_map_add(struct ctdb_node_map *nodemap,
 378                          const char *nstr, uint32_t flags)
 379 {
 380         ctdb_sock_addr addr;
 381         uint32_t num;
 382         struct ctdb_node_and_flags *n;
 383
 384         if (! parse_ip(nstr, NULL, 0, &addr)) {
 385                 fprintf(stderr, "Invalid IP address %s\n", nstr);
 386                 return false;
 387         }
 388
 389         num = nodemap->num;
 390         nodemap->node = talloc_realloc(nodemap, nodemap->node,
 391                                        struct ctdb_node_and_flags, num+1);
 392         if (nodemap->node == NULL) {
 393                 return false;
 394         }
 395
 396         n = &nodemap->node[num];
 397         n->addr = addr;
 398         n->pnn = num;
 399         n->flags = flags;
 400
 401         nodemap->num = num+1;
 402         return true;
 403 }
 404
 405 /* Read a nodes file into a node map */
 406 static struct ctdb_node_map *ctdb_read_nodes_file(TALLOC_CTX *mem_ctx,
 407                                                   const char *nlist)
 408 {
 409         char **lines;
 410         int nlines;
 411         int i;
 412         struct ctdb_node_map *nodemap;
 413
 414         nodemap = talloc_zero(mem_ctx, struct ctdb_node_map);
 415         if (nodemap == NULL) {
 416                 return NULL;
 417         }
 418
 419         lines = file_lines_load(nlist, &nlines, 0, mem_ctx);
 420         if (lines == NULL) {
 421                 return NULL;
 422         }
 423
 424         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 425                 nlines--;
 426         }
 427
 428         for (i=0; i<nlines; i++) {
 429                 char *node;
 430                 uint32_t flags;
 431                 size_t len;
 432
 433                 node = lines[i];
 434                 /* strip leading spaces */
 435                 while((*node == ' ') || (*node == '\t')) {
 436                         node++;
 437                 }
 438
 439                 len = strlen(node);
 440
 441                 /* strip trailing spaces */
 442                 while ((len > 1) &&
 443                        ((node[len-1] == ' ') || (node[len-1] == '\t')))
 444                 {
 445                         node[len-1] = '\0';
 446                         len--;
 447                 }
 448
 449                 if (len == 0) {
 450                         continue;
 451                 }
 452                 if (*node == '#') {
 453                         /* A "deleted" node is a node that is
 454                            commented out in the nodes file.  This is
 455                            used instead of removing a line, which
 456                            would cause subsequent nodes to change
 457                            their PNN. */
 458                         flags = NODE_FLAGS_DELETED;
 459                         node = discard_const("0.0.0.0");
 460                 } else {
 461                         flags = 0;
 462                 }
 463                 if (! node_map_add(nodemap, node, flags)) {
 464                         talloc_free(lines);
 465                         TALLOC_FREE(nodemap);
 466                         return NULL;
 467                 }
 468         }
 469
 470         talloc_free(lines);
 471         return nodemap;
 472 }
 473
 474 static struct ctdb_node_map *read_nodes_file(TALLOC_CTX *mem_ctx, uint32_t pnn)
 475 {
 476         struct ctdb_node_map *nodemap;
 477         char *nodepath;
 478         const char *nodes_list = NULL;
 479
 480         if (pnn != CTDB_UNKNOWN_PNN) {
 481                 nodepath = talloc_asprintf(mem_ctx, "CTDB_NODES_%u", pnn);
 482                 if (nodepath != NULL) {
 483                         nodes_list = getenv(nodepath);
 484                 }
 485         }
 486         if (nodes_list == NULL) {
 487                 nodes_list = getenv("CTDB_NODES");
 488         }
 489         if (nodes_list == NULL) {
 490                 const char *basedir = getenv("CTDB_BASE");
 491                 if (basedir == NULL) {
 492                         basedir = CTDB_ETCDIR;
 493                 }
 494                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes", basedir);
 495                 if (nodes_list == NULL) {
 496                         fprintf(stderr, "Memory allocation error\n");
 497                         return NULL;
 498                 }
 499         }
 500
 501         nodemap = ctdb_read_nodes_file(mem_ctx, nodes_list);
 502         if (nodemap == NULL) {
 503                 fprintf(stderr, "Failed to read nodes file \"%s\"\n",
 504                         nodes_list);
 505                 return NULL;
 506         }
 507
 508         return nodemap;
 509 }
 510
 511 static struct ctdb_dbid *db_find(TALLOC_CTX *mem_ctx,
 512                                  struct ctdb_context *ctdb,
 513                                  struct ctdb_dbid_map *dbmap,
 514                                  const char *db_name)
 515 {
 516         struct ctdb_dbid *db = NULL;
 517         const char *name;
 518         int ret, i;
 519
 520         for (i=0; i<dbmap->num; i++) {
 521                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
 522                                            ctdb->pnn, TIMEOUT(),
 523                                            dbmap->dbs[i].db_id, &name);
 524                 if (ret != 0) {
 525                         return false;
 526                 }
 527
 528                 if (strcmp(db_name, name) == 0) {
 529                         talloc_free(discard_const(name));
 530                         db = &dbmap->dbs[i];
 531                         break;
 532                 }
 533         }
 534
 535         return db;
 536 }
 537
 538 static bool db_exists(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 539                       const char *db_arg, uint32_t *db_id,
 540                       const char **db_name, uint8_t *db_flags)
 541 {
 542         struct ctdb_dbid_map *dbmap;
 543         struct ctdb_dbid *db = NULL;
 544         uint32_t id = 0;
 545         const char *name = NULL;
 546         int ret, i;
 547
 548         ret = ctdb_ctrl_get_dbmap(mem_ctx, ctdb->ev, ctdb->client,
 549                                   ctdb->pnn, TIMEOUT(), &dbmap);
 550         if (ret != 0) {
 551                 return false;
 552         }
 553
 554         if (strncmp(db_arg, "0x", 2) == 0) {
 555                 id = strtoul(db_arg, NULL, 0);
 556                 for (i=0; i<dbmap->num; i++) {
 557                         if (id == dbmap->dbs[i].db_id) {
 558                                 db = &dbmap->dbs[i];
 559                                 break;
 560                         }
 561                 }
 562         } else {
 563                 name = db_arg;
 564                 db = db_find(mem_ctx, ctdb, dbmap, name);
 565         }
 566
 567         if (db == NULL) {
 568                 fprintf(stderr, "No database matching '%s' found\n", db_arg);
 569                 return false;
 570         }
 571
 572         if (name == NULL) {
 573                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
 574                                            ctdb->pnn, TIMEOUT(), id, &name);
 575                 if (ret != 0) {
 576                         return false;
 577                 }
 578         }
 579
 580         if (db_id != NULL) {
 581                 *db_id = db->db_id;
 582         }
 583         if (db_name != NULL) {
 584                 *db_name = talloc_strdup(mem_ctx, name);
 585         }
 586         if (db_flags != NULL) {
 587                 *db_flags = db->flags;
 588         }
 589         return true;
 590 }
 591
 592 static int h2i(char h)
 593 {
 594         if (h >= 'a' && h <= 'f') {
 595                 return h - 'a' + 10;
 596         }
 597         if (h >= 'A' && h <= 'F') {
 598                 return h - 'f' + 10;
 599         }
 600         return h - '0';
 601 }
 602
 603 static int hex_to_data(const char *str, size_t len, TALLOC_CTX *mem_ctx,
 604                        TDB_DATA *out)
 605 {
 606         int i;
 607         TDB_DATA data;
 608
 609         if (len & 0x01) {
 610                 fprintf(stderr, "Key (%s) contains odd number of hex digits\n",
 611                         str);
 612                 return EINVAL;
 613         }
 614
 615         data.dsize = len / 2;
 616         data.dptr = talloc_size(mem_ctx, data.dsize);
 617         if (data.dptr == NULL) {
 618                 return ENOMEM;
 619         }
 620
 621         for (i=0; i<data.dsize; i++) {
 622                 data.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
 623         }
 624
 625         *out = data;
 626         return 0;
 627 }
 628
 629 static int str_to_data(const char *str, size_t len, TALLOC_CTX *mem_ctx,
 630                        TDB_DATA *out)
 631 {
 632         TDB_DATA data;
 633         int ret = 0;
 634
 635         if (strncmp(str, "0x", 2) == 0) {
 636                 ret = hex_to_data(str+2, len-2, mem_ctx, &data);
 637         } else {
 638                 data.dptr = talloc_memdup(mem_ctx, str, len);
 639                 if (data.dptr == NULL) {
 640                         return ENOMEM;
 641                 }
 642                 data.dsize = len;
 643         }
 644
 645         *out = data;
 646         return ret;
 647 }
 648
 649 static int run_helper(TALLOC_CTX *mem_ctx, const char *command,
 650                       const char *path, int argc, const char **argv)
 651 {
 652         pid_t pid;
 653         int save_errno, status, ret;
 654         const char **new_argv;
 655         int i;
 656
 657         new_argv = talloc_array(mem_ctx, const char *, argc + 2);
 658         if (new_argv == NULL) {
 659                 return ENOMEM;
 660         }
 661
 662         new_argv[0] = path;
 663         for (i=0; i<argc; i++) {
 664                 new_argv[i+1] = argv[i];
 665         }
 666         new_argv[argc+1] = NULL;
 667
 668         pid = fork();
 669         if (pid < 0) {
 670                 save_errno = errno;
 671                 talloc_free(new_argv);
 672                 fprintf(stderr, "Failed to fork %s (%s) - %s\n",
 673                         command, path, strerror(save_errno));
 674                 return save_errno;
 675         }
 676
 677         if (pid == 0) {
 678                 ret = execv(path, discard_const(new_argv));
 679                 if (ret == -1) {
 680                         _exit(64+errno);
 681                 }
 682                 /* Should not happen */
 683                 _exit(64+ENOEXEC);
 684         }
 685
 686         talloc_free(new_argv);
 687
 688         ret = waitpid(pid, &status, 0);
 689         if (ret == -1) {
 690                 save_errno = errno;
 691                 fprintf(stderr, "waitpid() failed for %s - %s\n",
 692                         command, strerror(save_errno));
 693                 return save_errno;
 694         }
 695
 696         if (WIFEXITED(status)) {
 697                 int pstatus = WEXITSTATUS(status);
 698                 if (WIFSIGNALED(status)) {
 699                         fprintf(stderr, "%s terminated with signal %d\n",
 700                                 command, WTERMSIG(status));
 701                         ret = EINTR;
 702                 } else if (pstatus >= 64 && pstatus < 255) {
 703                         fprintf(stderr, "%s failed with error %d\n",
 704                                 command, pstatus-64);
 705                         ret = pstatus - 64;
 706                 } else {
 707                         ret = pstatus;
 708                 }
 709                 return ret;
 710         } else if (WIFSIGNALED(status)) {
 711                 fprintf(stderr, "%s terminated with signal %d\n",
 712                         command, WTERMSIG(status));
 713                 return EINTR;
 714         }
 715
 716         return 0;
 717 }
 718
 719 /*
 720  * Command Functions
 721  */
 722
 723 static int control_version(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 724                            int argc, const char **argv)
 725 {
 726         printf("%s\n", CTDB_VERSION_STRING);
 727         return 0;
 728 }
 729
 730 static bool partially_online(TALLOC_CTX *mem_ctx,
 731                              struct ctdb_context *ctdb,
 732                              struct ctdb_node_and_flags *node)
 733 {
 734         struct ctdb_iface_list *iface_list;
 735         int ret, i;
 736         bool status = false;
 737
 738         if (node->flags != 0) {
 739                 return false;
 740         }
 741
 742         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
 743                                    node->pnn, TIMEOUT(), &iface_list);
 744         if (ret != 0) {
 745                 return false;
 746         }
 747
 748         status = false;
 749         for (i=0; i < iface_list->num; i++) {
 750                 if (iface_list->iface[i].link_state == 0) {
 751                         status = true;
 752                         break;
 753                 }
 754         }
 755
 756         return status;
 757 }
 758
 759 static void print_nodemap_machine(TALLOC_CTX *mem_ctx,
 760                                   struct ctdb_context *ctdb,
 761                                   struct ctdb_node_map *nodemap,
 762                                   uint32_t mypnn)
 763 {
 764         struct ctdb_node_and_flags *node;
 765         int i;
 766
 767         printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
 768                options.sep,
 769                "Node", options.sep,
 770                "IP", options.sep,
 771                "Disconnected", options.sep,
 772                "Banned", options.sep,
 773                "Disabled", options.sep,
 774                "Unhealthy", options.sep,
 775                "Stopped", options.sep,
 776                "Inactive", options.sep,
 777                "PartiallyOnline", options.sep,
 778                "ThisNode", options.sep);
 779
 780         for (i=0; i<nodemap->num; i++) {
 781                 node = &nodemap->node[i];
 782                 if (node->flags & NODE_FLAGS_DELETED) {
 783                         continue;
 784                 }
 785
 786                 printf("%s%u%s%s%s%d%s%d%s%d%s%d%s%d%s%d%s%d%s%c%s\n",
 787                        options.sep,
 788                        node->pnn, options.sep,
 789                        ctdb_sock_addr_to_string(mem_ctx, &node->addr),
 790                        options.sep,
 791                        !! (node->flags & NODE_FLAGS_DISCONNECTED), options.sep,
 792                        !! (node->flags & NODE_FLAGS_BANNED), options.sep,
 793                        !! (node->flags & NODE_FLAGS_PERMANENTLY_DISABLED),
 794                        options.sep,
 795                        !! (node->flags & NODE_FLAGS_UNHEALTHY), options.sep,
 796                        !! (node->flags & NODE_FLAGS_STOPPED), options.sep,
 797                        !! (node->flags & NODE_FLAGS_INACTIVE), options.sep,
 798                        partially_online(mem_ctx, ctdb, node), options.sep,
 799                        (node->pnn == mypnn)?'Y':'N', options.sep);
 800         }
 801
 802 }
 803
 804 static void print_nodemap(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 805                           struct ctdb_node_map *nodemap, uint32_t mypnn)
 806 {
 807         struct ctdb_node_and_flags *node;
 808         int num_deleted_nodes = 0;
 809         int i;
 810
 811         for (i=0; i<nodemap->num; i++) {
 812                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
 813                         num_deleted_nodes++;
 814                 }
 815         }
 816
 817         if (num_deleted_nodes == 0) {
 818                 printf("Number of nodes:%d\n", nodemap->num);
 819         } else {
 820                 printf("Number of nodes:%d (including %d deleted nodes)\n",
 821                        nodemap->num, num_deleted_nodes);
 822         }
 823
 824         for (i=0; i<nodemap->num; i++) {
 825                 node = &nodemap->node[i];
 826                 if (node->flags & NODE_FLAGS_DELETED) {
 827                         continue;
 828                 }
 829
 830                 printf("pnn:%u %-16s %s%s\n",
 831                        node->pnn,
 832                        ctdb_sock_addr_to_string(mem_ctx, &node->addr),
 833                        partially_online(mem_ctx, ctdb, node) ?
 834                                 "PARTIALLYONLINE" :
 835                                 pretty_print_flags(mem_ctx, node->flags),
 836                        node->pnn == mypnn ? " (THIS NODE)" : "");
 837         }
 838 }
 839
 840 static void print_status(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 841                          struct ctdb_node_map *nodemap, uint32_t mypnn,
 842                          struct ctdb_vnn_map *vnnmap, int recmode,
 843                          uint32_t recmaster)
 844 {
 845         int i;
 846
 847         print_nodemap(mem_ctx, ctdb, nodemap, mypnn);
 848
 849         if (vnnmap->generation == INVALID_GENERATION) {
 850                 printf("Generation:INVALID\n");
 851         } else {
 852                 printf("Generation:%u\n", vnnmap->generation);
 853         }
 854         printf("Size:%d\n", vnnmap->size);
 855         for (i=0; i<vnnmap->size; i++) {
 856                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
 857         }
 858
 859         printf("Recovery mode:%s (%d)\n",
 860                recmode == CTDB_RECOVERY_NORMAL ? "NORMAL" : "RECOVERY",
 861                recmode);
 862         printf("Recovery master:%d\n", recmaster);
 863 }
 864
 865 static int control_status(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 866                           int argc, const char **argv)
 867 {
 868         struct ctdb_node_map *nodemap;
 869         struct ctdb_vnn_map *vnnmap;
 870         int recmode;
 871         uint32_t recmaster;
 872         int ret;
 873
 874         if (argc != 0) {
 875                 usage("status");
 876         }
 877
 878         nodemap = get_nodemap(ctdb, false);
 879         if (nodemap == NULL) {
 880                 return 1;
 881         }
 882
 883         if (options.machinereadable == 1) {
 884                 print_nodemap_machine(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn);
 885                 return 0;
 886         }
 887
 888         ret = ctdb_ctrl_getvnnmap(mem_ctx, ctdb->ev, ctdb->client,
 889                                   ctdb->cmd_pnn, TIMEOUT(), &vnnmap);
 890         if (ret != 0) {
 891                 return ret;
 892         }
 893
 894         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
 895                                     ctdb->cmd_pnn, TIMEOUT(), &recmode);
 896         if (ret != 0) {
 897                 return ret;
 898         }
 899
 900         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
 901                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
 902         if (ret != 0) {
 903                 return ret;
 904         }
 905
 906         print_status(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn, vnnmap,
 907                      recmode, recmaster);
 908         return 0;
 909 }
 910
 911 static int control_uptime(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 912                           int argc, const char **argv)
 913 {
 914         struct ctdb_uptime *uptime;
 915         int ret, tmp, days, hours, minutes, seconds;
 916
 917         ret = ctdb_ctrl_uptime(mem_ctx, ctdb->ev, ctdb->client,
 918                                ctdb->cmd_pnn, TIMEOUT(), &uptime);
 919         if (ret != 0) {
 920                 return ret;
 921         }
 922
 923         printf("Current time of node %-4u     :                %s",
 924                ctdb->cmd_pnn, ctime(&uptime->current_time.tv_sec));
 925
 926         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 927         seconds = tmp % 60; tmp /= 60;
 928         minutes = tmp % 60; tmp /= 60;
 929         hours = tmp % 24; tmp /= 24;
 930         days = tmp;
 931
 932         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s",
 933                days, hours, minutes, seconds,
 934                ctime(&uptime->ctdbd_start_time.tv_sec));
 935
 936         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 937         seconds = tmp % 60; tmp /= 60;
 938         minutes = tmp % 60; tmp /= 60;
 939         hours = tmp % 24; tmp /= 24;
 940         days = tmp;
 941
 942         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s",
 943                days, hours, minutes, seconds,
 944                ctime(&uptime->last_recovery_finished.tv_sec));
 945
 946         printf("Duration of last recovery/failover: %lf seconds\n",
 947                timeval_delta(&uptime->last_recovery_finished,
 948                              &uptime->last_recovery_started));
 949
 950         return 0;
 951 }
 952
 953 static int control_ping(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 954                         int argc, const char **argv)
 955 {
 956         struct timeval tv;
 957         int ret, num_clients;
 958
 959         tv = timeval_current();
 960         ret = ctdb_ctrl_ping(mem_ctx, ctdb->ev, ctdb->client,
 961                              ctdb->cmd_pnn, TIMEOUT(), &num_clients);
 962         if (ret != 0) {
 963                 return ret;
 964         }
 965
 966         printf("response from %u time=%.6f sec  (%d clients)\n",
 967                ctdb->cmd_pnn, timeval_elapsed(&tv), num_clients);
 968         return 0;
 969 }
 970
 971 const char *runstate_to_string(enum ctdb_runstate runstate);
 972 enum ctdb_runstate runstate_from_string(const char *runstate_str);
 973
 974 static int control_runstate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
 975                             int argc, const char **argv)
 976 {
 977         enum ctdb_runstate runstate;
 978         bool found;
 979         int ret, i;
 980
 981         ret = ctdb_ctrl_get_runstate(mem_ctx, ctdb->ev, ctdb->client,
 982                                      ctdb->cmd_pnn, TIMEOUT(), &runstate);
 983         if (ret != 0) {
 984                 return ret;
 985         }
 986
 987         found = true;
 988         for (i=0; i<argc; i++) {
 989                 enum ctdb_runstate t;
 990
 991                 found = false;
 992                 t = ctdb_runstate_from_string(argv[i]);
 993                 if (t == CTDB_RUNSTATE_UNKNOWN) {
 994                         printf("Invalid run state (%s)\n", argv[i]);
 995                         return 1;
 996                 }
 997
 998                 if (t == runstate) {
 999                         found = true;
1000                         break;
1001                 }
1002         }
1003
1004         if (! found) {
1005                 printf("CTDB not in required run state (got %s)\n",
1006                        ctdb_runstate_to_string(runstate));
1007                 return 1;
1008         }
1009
1010         printf("%s\n", ctdb_runstate_to_string(runstate));
1011         return 0;
1012 }
1013
1014 static int control_getvar(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1015                           int argc, const char **argv)
1016 {
1017         struct ctdb_var_list *tun_var_list;
1018         uint32_t value;
1019         int ret, i;
1020         bool found;
1021
1022         if (argc != 1) {
1023                 usage("getvar");
1024         }
1025
1026         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1027                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1028         if (ret != 0) {
1029                 fprintf(stderr,
1030                         "Failed to get list of variables from node %u\n",
1031                         ctdb->cmd_pnn);
1032                 return ret;
1033         }
1034
1035         found = false;
1036         for (i=0; i<tun_var_list->count; i++) {
1037                 if (strcasecmp(tun_var_list->var[i], argv[0]) == 0) {
1038                         found = true;
1039                         break;
1040                 }
1041         }
1042
1043         if (! found) {
1044                 printf("No such tunable %s\n", argv[0]);
1045                 return 1;
1046         }
1047
1048         ret = ctdb_ctrl_get_tunable(mem_ctx, ctdb->ev, ctdb->client,
1049                                     ctdb->cmd_pnn, TIMEOUT(), argv[0], &value);
1050         if (ret != 0) {
1051                 return ret;
1052         }
1053
1054         printf("%-26s = %u\n", argv[0], value);
1055         return 0;
1056 }
1057
1058 static int control_setvar(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1059                           int argc, const char **argv)
1060 {
1061         struct ctdb_var_list *tun_var_list;
1062         struct ctdb_tunable tunable;
1063         int ret, i;
1064         bool found;
1065
1066         if (argc != 2) {
1067                 usage("setvar");
1068         }
1069
1070         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1071                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1072         if (ret != 0) {
1073                 fprintf(stderr,
1074                         "Failed to get list of variables from node %u\n",
1075                         ctdb->cmd_pnn);
1076                 return ret;
1077         }
1078
1079         found = false;
1080         for (i=0; i<tun_var_list->count; i++) {
1081                 if (strcasecmp(tun_var_list->var[i], argv[0]) == 0) {
1082                         found = true;
1083                         break;
1084                 }
1085         }
1086
1087         if (! found) {
1088                 printf("No such tunable %s\n", argv[0]);
1089                 return 1;
1090         }
1091
1092         tunable.name = argv[0];
1093         tunable.value = strtoul(argv[1], NULL, 0);
1094
1095         ret = ctdb_ctrl_set_tunable(mem_ctx, ctdb->ev, ctdb->client,
1096                                     ctdb->cmd_pnn, TIMEOUT(), &tunable);
1097         if (ret != 0) {
1098                 if (ret == 1) {
1099                         fprintf(stderr,
1100                                 "Setting obsolete tunable variable '%s'\n",
1101                                tunable.name);
1102                         return 0;
1103                 }
1104         }
1105
1106         return ret;
1107 }
1108
1109 static int control_listvars(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1110                             int argc, const char **argv)
1111 {
1112         struct ctdb_var_list *tun_var_list;
1113         int ret, i;
1114
1115         if (argc != 0) {
1116                 usage("listvars");
1117         }
1118
1119         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1120                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1121         if (ret != 0) {
1122                 return ret;
1123         }
1124
1125         for (i=0; i<tun_var_list->count; i++) {
1126                 control_getvar(mem_ctx, ctdb, 1, &tun_var_list->var[i]);
1127         }
1128
1129         return 0;
1130 }
1131
1132 const struct {
1133         const char *name;
1134         uint32_t offset;
1135 } stats_fields[] = {
1136 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
1137         STATISTICS_FIELD(num_clients),
1138         STATISTICS_FIELD(frozen),
1139         STATISTICS_FIELD(recovering),
1140         STATISTICS_FIELD(num_recoveries),
1141         STATISTICS_FIELD(client_packets_sent),
1142         STATISTICS_FIELD(client_packets_recv),
1143         STATISTICS_FIELD(node_packets_sent),
1144         STATISTICS_FIELD(node_packets_recv),
1145         STATISTICS_FIELD(keepalive_packets_sent),
1146         STATISTICS_FIELD(keepalive_packets_recv),
1147         STATISTICS_FIELD(node.req_call),
1148         STATISTICS_FIELD(node.reply_call),
1149         STATISTICS_FIELD(node.req_dmaster),
1150         STATISTICS_FIELD(node.reply_dmaster),
1151         STATISTICS_FIELD(node.reply_error),
1152         STATISTICS_FIELD(node.req_message),
1153         STATISTICS_FIELD(node.req_control),
1154         STATISTICS_FIELD(node.reply_control),
1155         STATISTICS_FIELD(client.req_call),
1156         STATISTICS_FIELD(client.req_message),
1157         STATISTICS_FIELD(client.req_control),
1158         STATISTICS_FIELD(timeouts.call),
1159         STATISTICS_FIELD(timeouts.control),
1160         STATISTICS_FIELD(timeouts.traverse),
1161         STATISTICS_FIELD(locks.num_calls),
1162         STATISTICS_FIELD(locks.num_current),
1163         STATISTICS_FIELD(locks.num_pending),
1164         STATISTICS_FIELD(locks.num_failed),
1165         STATISTICS_FIELD(total_calls),
1166         STATISTICS_FIELD(pending_calls),
1167         STATISTICS_FIELD(childwrite_calls),
1168         STATISTICS_FIELD(pending_childwrite_calls),
1169         STATISTICS_FIELD(memory_used),
1170         STATISTICS_FIELD(max_hop_count),
1171         STATISTICS_FIELD(total_ro_delegations),
1172         STATISTICS_FIELD(total_ro_revokes),
1173 };
1174
1175 #define LATENCY_AVG(v)  ((v).num ? (v).total / (v).num : 0.0 )
1176
1177 static void print_statistics_machine(struct ctdb_statistics *s,
1178                                      bool show_header)
1179 {
1180         int i;
1181
1182         if (show_header) {
1183                 printf("CTDB version%s", options.sep);
1184                 printf("Current time of statistics%s", options.sep);
1185                 printf("Statistics collected since%s", options.sep);
1186                 for (i=0; i<ARRAY_SIZE(stats_fields); i++) {
1187                         printf("%s%s", stats_fields[i].name, options.sep);
1188                 }
1189                 printf("num_reclock_ctdbd_latency%s", options.sep);
1190                 printf("min_reclock_ctdbd_latency%s", options.sep);
1191                 printf("avg_reclock_ctdbd_latency%s", options.sep);
1192                 printf("max_reclock_ctdbd_latency%s", options.sep);
1193
1194                 printf("num_reclock_recd_latency%s", options.sep);
1195                 printf("min_reclock_recd_latency%s", options.sep);
1196                 printf("avg_reclock_recd_latency%s", options.sep);
1197                 printf("max_reclock_recd_latency%s", options.sep);
1198
1199                 printf("num_call_latency%s", options.sep);
1200                 printf("min_call_latency%s", options.sep);
1201                 printf("avg_call_latency%s", options.sep);
1202                 printf("max_call_latency%s", options.sep);
1203
1204                 printf("num_lockwait_latency%s", options.sep);
1205                 printf("min_lockwait_latency%s", options.sep);
1206                 printf("avg_lockwait_latency%s", options.sep);
1207                 printf("max_lockwait_latency%s", options.sep);
1208
1209                 printf("num_childwrite_latency%s", options.sep);
1210                 printf("min_childwrite_latency%s", options.sep);
1211                 printf("avg_childwrite_latency%s", options.sep);
1212                 printf("max_childwrite_latency%s", options.sep);
1213                 printf("\n");
1214         }
1215
1216         printf("%u%s", CTDB_PROTOCOL, options.sep);
1217         printf("%u%s", (uint32_t)s->statistics_current_time.tv_sec, options.sep);
1218         printf("%u%s", (uint32_t)s->statistics_start_time.tv_sec, options.sep);
1219         for (i=0;i<ARRAY_SIZE(stats_fields);i++) {
1220                 printf("%u%s",
1221                        *(uint32_t *)(stats_fields[i].offset+(uint8_t *)s),
1222                        options.sep);
1223         }
1224         printf("%u%s", s->reclock.ctdbd.num, options.sep);
1225         printf("%.6f%s", s->reclock.ctdbd.min, options.sep);
1226         printf("%.6f%s", LATENCY_AVG(s->reclock.ctdbd), options.sep);
1227         printf("%.6f%s", s->reclock.ctdbd.max, options.sep);
1228
1229         printf("%u%s", s->reclock.recd.num, options.sep);
1230         printf("%.6f%s", s->reclock.recd.min, options.sep);
1231         printf("%.6f%s", LATENCY_AVG(s->reclock.recd), options.sep);
1232         printf("%.6f%s", s->reclock.recd.max, options.sep);
1233
1234         printf("%d%s", s->call_latency.num, options.sep);
1235         printf("%.6f%s", s->call_latency.min, options.sep);
1236         printf("%.6f%s", LATENCY_AVG(s->call_latency), options.sep);
1237         printf("%.6f%s", s->call_latency.max, options.sep);
1238
1239         printf("%d%s", s->childwrite_latency.num, options.sep);
1240         printf("%.6f%s", s->childwrite_latency.min, options.sep);
1241         printf("%.6f%s", LATENCY_AVG(s->childwrite_latency), options.sep);
1242         printf("%.6f%s", s->childwrite_latency.max, options.sep);
1243         printf("\n");
1244 }
1245
1246 static void print_statistics(struct ctdb_statistics *s)
1247 {
1248         int tmp, days, hours, minutes, seconds;
1249         int i;
1250         const char *prefix = NULL;
1251         int preflen = 0;
1252
1253         tmp = s->statistics_current_time.tv_sec -
1254               s->statistics_start_time.tv_sec;
1255         seconds = tmp % 60; tmp /= 60;
1256         minutes = tmp % 60; tmp /= 60;
1257         hours   = tmp % 24; tmp /= 24;
1258         days    = tmp;
1259
1260         printf("CTDB version %u\n", CTDB_PROTOCOL);
1261         printf("Current time of statistics  :                %s",
1262                ctime(&s->statistics_current_time.tv_sec));
1263         printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s",
1264                days, hours, minutes, seconds,
1265                ctime(&s->statistics_start_time.tv_sec));
1266
1267         for (i=0; i<ARRAY_SIZE(stats_fields); i++) {
1268                 if (strchr(stats_fields[i].name, '.') != NULL) {
1269                         preflen = strcspn(stats_fields[i].name, ".") + 1;
1270                         if (! prefix ||
1271                             strncmp(prefix, stats_fields[i].name, preflen) != 0) {
1272                                 prefix = stats_fields[i].name;
1273                                 printf(" %*.*s\n", preflen-1, preflen-1,
1274                                        stats_fields[i].name);
1275                         }
1276                 } else {
1277                         preflen = 0;
1278                 }
1279                 printf(" %*s%-22s%*s%10u\n", preflen ? 4 : 0, "",
1280                        stats_fields[i].name+preflen, preflen ? 0 : 4, "",
1281                        *(uint32_t *)(stats_fields[i].offset+(uint8_t *)s));
1282         }
1283
1284         printf(" hop_count_buckets:");
1285         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
1286                 printf(" %d", s->hop_count_bucket[i]);
1287         }
1288         printf("\n");
1289         printf(" lock_buckets:");
1290         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
1291                 printf(" %d", s->locks.buckets[i]);
1292         }
1293         printf("\n");
1294         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1295                "locks_latency      MIN/AVG/MAX",
1296                s->locks.latency.min, LATENCY_AVG(s->locks.latency),
1297                s->locks.latency.max, s->locks.latency.num);
1298
1299         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1300                "reclock_ctdbd      MIN/AVG/MAX",
1301                s->reclock.ctdbd.min, LATENCY_AVG(s->reclock.ctdbd),
1302                s->reclock.ctdbd.max, s->reclock.ctdbd.num);
1303
1304         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1305                "reclock_recd       MIN/AVG/MAX",
1306                s->reclock.recd.min, LATENCY_AVG(s->reclock.recd),
1307                s->reclock.recd.max, s->reclock.recd.num);
1308
1309         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1310                "call_latency       MIN/AVG/MAX",
1311                s->call_latency.min, LATENCY_AVG(s->call_latency),
1312                s->call_latency.max, s->call_latency.num);
1313
1314         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1315                "childwrite_latency MIN/AVG/MAX",
1316                s->childwrite_latency.min,
1317                LATENCY_AVG(s->childwrite_latency),
1318                s->childwrite_latency.max, s->childwrite_latency.num);
1319 }
1320
1321 static int control_statistics(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1322                               int argc, const char **argv)
1323 {
1324         struct ctdb_statistics *stats;
1325         int ret;
1326
1327         if (argc != 0) {
1328                 usage("statistics");
1329         }
1330
1331         ret = ctdb_ctrl_statistics(mem_ctx, ctdb->ev, ctdb->client,
1332                                    ctdb->cmd_pnn, TIMEOUT(), &stats);
1333         if (ret != 0) {
1334                 return ret;
1335         }
1336
1337         if (options.machinereadable) {
1338                 print_statistics_machine(stats, true);
1339         } else {
1340                 print_statistics(stats);
1341         }
1342
1343         return 0;
1344 }
1345
1346 static int control_statistics_reset(TALLOC_CTX *mem_ctx,
1347                                     struct ctdb_context *ctdb,
1348                                     int argc, const char **argv)
1349 {
1350         int ret;
1351
1352         if (argc != 0) {
1353                 usage("statisticsreset");
1354         }
1355
1356         ret = ctdb_ctrl_statistics_reset(mem_ctx, ctdb->ev, ctdb->client,
1357                                          ctdb->cmd_pnn, TIMEOUT());
1358         if (ret != 0) {
1359                 return ret;
1360         }
1361
1362         return 0;
1363 }
1364
1365 static int control_stats(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1366                          int argc, const char **argv)
1367 {
1368         struct ctdb_statistics_list *slist;
1369         int ret, count = 0, i;
1370         bool show_header = true;
1371
1372         if (argc > 1) {
1373                 usage("stats");
1374         }
1375
1376         if (argc == 1) {
1377                 count = atoi(argv[0]);
1378         }
1379
1380         ret = ctdb_ctrl_get_stat_history(mem_ctx, ctdb->ev, ctdb->client,
1381                                          ctdb->cmd_pnn, TIMEOUT(), &slist);
1382         if (ret != 0) {
1383                 return ret;
1384         }
1385
1386         for (i=0; i<slist->num; i++) {
1387                 if (slist->stats[i].statistics_start_time.tv_sec == 0) {
1388                         continue;
1389                 }
1390                 if (options.machinereadable == 1) {
1391                         print_statistics_machine(&slist->stats[i],
1392                                                  show_header);
1393                         show_header = false;
1394                 } else {
1395                         print_statistics(&slist->stats[i]);
1396                 }
1397                 if (count > 0 && i == count) {
1398                         break;
1399                 }
1400         }
1401
1402         return 0;
1403 }
1404
1405 static int ctdb_public_ip_cmp(const void *a, const void *b)
1406 {
1407         const struct ctdb_public_ip *ip_a = a;
1408         const struct ctdb_public_ip *ip_b = b;
1409
1410         return ctdb_sock_addr_cmp(&ip_a->addr, &ip_b->addr);
1411 }
1412
1413 static void print_ip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1414                      struct ctdb_public_ip_list *ips,
1415                      struct ctdb_public_ip_info **ipinfo,
1416                      bool all_nodes)
1417 {
1418         int i, j;
1419         char *conf, *avail, *active;
1420
1421         if (options.machinereadable == 1) {
1422                 printf("%s%s%s%s%s", options.sep,
1423                        "Public IP", options.sep,
1424                        "Node", options.sep);
1425                 if (options.verbose == 1) {
1426                         printf("%s%s%s%s%s%s\n",
1427                                "ActiveInterfaces", options.sep,
1428                                "AvailableInterfaces", options.sep,
1429                                "ConfiguredInterfaces", options.sep);
1430                 } else {
1431                         printf("\n");
1432                 }
1433         } else {
1434                 if (all_nodes) {
1435                         printf("Public IPs on ALL nodes\n");
1436                 } else {
1437                         printf("Public IPs on node %u\n", ctdb->cmd_pnn);
1438                 }
1439         }
1440
1441         for (i = 0; i < ips->num; i++) {
1442
1443                 if (options.machinereadable == 1) {
1444                         printf("%s%s%s%d%s", options.sep,
1445                                ctdb_sock_addr_to_string(
1446                                         mem_ctx, &ips->ip[i].addr),
1447                                options.sep,
1448                                (int)ips->ip[i].pnn, options.sep);
1449                 } else {
1450                         printf("%s", ctdb_sock_addr_to_string(
1451                                                 mem_ctx, &ips->ip[i].addr));
1452                 }
1453
1454                 if (options.verbose == 0) {
1455                         if (options.machinereadable == 1) {
1456                                 printf("\n");
1457                         } else {
1458                                 printf(" %d\n", (int)ips->ip[i].pnn);
1459                         }
1460                         continue;
1461                 }
1462
1463                 conf = NULL;
1464                 avail = NULL;
1465                 active = NULL;
1466
1467                 if (ipinfo[i] == NULL) {
1468                         goto skip_ipinfo;
1469                 }
1470
1471                 for (j=0; j<ipinfo[i]->ifaces->num; j++) {
1472                         struct ctdb_iface *iface;
1473
1474                         iface = &ipinfo[i]->ifaces->iface[j];
1475                         if (conf == NULL) {
1476                                 conf = talloc_strdup(mem_ctx, iface->name);
1477                         } else {
1478                                 conf = talloc_asprintf_append(
1479                                                 conf, ",%s", iface->name);
1480                         }
1481
1482                         if (ipinfo[i]->active_idx == j) {
1483                                 active = iface->name;
1484                         }
1485
1486                         if (iface->link_state == 0) {
1487                                 continue;
1488                         }
1489
1490                         if (avail == NULL) {
1491                                 avail = talloc_strdup(mem_ctx, iface->name);
1492                         } else {
1493                                 avail = talloc_asprintf_append(
1494                                                 avail, ",%s", iface->name);
1495                         }
1496                 }
1497
1498         skip_ipinfo:
1499
1500                 if (options.machinereadable == 1) {
1501                         printf("%s%s%s%s%s%s\n",
1502                                active ? active : "", options.sep,
1503                                avail ? avail : "", options.sep,
1504                                conf ? conf : "", options.sep);
1505                 } else {
1506                         printf(" node[%d] active[%s] available[%s]"
1507                                " configured[%s]\n",
1508                                (int)ips->ip[i].pnn, active ? active : "",
1509                                avail ? avail : "", conf ? conf : "");
1510                 }
1511         }
1512 }
1513
1514 static int collect_ips(uint8_t *keybuf, size_t keylen, uint8_t *databuf,
1515                        size_t datalen, void *private_data)
1516 {
1517         struct ctdb_public_ip_list *ips = talloc_get_type_abort(
1518                 private_data, struct ctdb_public_ip_list);
1519         struct ctdb_public_ip *ip;
1520
1521         ip = (struct ctdb_public_ip *)databuf;
1522         ips->ip[ips->num] = *ip;
1523         ips->num += 1;
1524
1525         return 0;
1526 }
1527
1528 static int get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
1529                               struct ctdb_public_ip_list **out)
1530 {
1531         struct ctdb_node_map *nodemap;
1532         struct ctdb_public_ip_list *ips;
1533         struct db_hash_context *ipdb;
1534         uint32_t *pnn_list;
1535         int ret, count, i, j;
1536
1537         nodemap = get_nodemap(ctdb, false);
1538         if (nodemap == NULL) {
1539                 return 1;
1540         }
1541
1542         ret = db_hash_init(mem_ctx, "ips", 101, DB_HASH_COMPLEX, &ipdb);
1543         if (ret != 0) {
1544                 goto failed;
1545         }
1546
1547         count = list_of_active_nodes(nodemap, CTDB_UNKNOWN_PNN, mem_ctx,
1548                                      &pnn_list);
1549         if (count <= 0) {
1550                 goto failed;
1551         }
1552
1553         for (i=0; i<count; i++) {
1554                 ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
1555                                                pnn_list[i], TIMEOUT(), &ips);
1556                 if (ret != 0) {
1557                         goto failed;
1558                 }
1559
1560                 for (j=0; j<ips->num; j++) {
1561                         struct ctdb_public_ip ip;
1562
1563                         ip.pnn = ips->ip[j].pnn;
1564                         ip.addr = ips->ip[j].addr;
1565
1566                         if (pnn_list[i] == ip.pnn) {
1567                                 /* Node claims IP is hosted on it, so
1568                                  * save that information
1569                                  */
1570                                 ret = db_hash_add(ipdb, (uint8_t *)&ip.addr,
1571                                                   sizeof(ip.addr),
1572                                                   (uint8_t *)&ip, sizeof(ip));
1573                                 if (ret != 0) {
1574                                         goto failed;
1575                                 }
1576                         } else {
1577                                 /* Node thinks IP is hosted elsewhere,
1578                                  * so overwrite with CTDB_UNKNOWN_PNN
1579                                  * if there's no existing entry
1580                                  */
1581                                 ret = db_hash_exists(ipdb, (uint8_t *)&ip.addr,
1582                                                      sizeof(ip.addr));
1583                                 if (ret == ENOENT) {
1584                                         ip.pnn = CTDB_UNKNOWN_PNN;
1585                                         ret = db_hash_add(ipdb,
1586                                                           (uint8_t *)&ip.addr,
1587                                                           sizeof(ip.addr),
1588                                                           (uint8_t *)&ip,
1589                                                           sizeof(ip));
1590                                         if (ret != 0) {
1591                                                 goto failed;
1592                                         }
1593                                 }
1594                         }
1595                 }
1596
1597                 TALLOC_FREE(ips);
1598         }
1599
1600         talloc_free(pnn_list);
1601
1602         ret = db_hash_traverse(ipdb, NULL, NULL, &count);
1603         if (ret != 0) {
1604                 goto failed;
1605         }
1606
1607         ips = talloc_zero(mem_ctx, struct ctdb_public_ip_list);
1608         if (ips == NULL) {
1609                 goto failed;
1610         }
1611
1612         ips->ip = talloc_array(ips, struct ctdb_public_ip, count);
1613         if (ips->ip == NULL) {
1614                 goto failed;
1615         }
1616
1617         ret = db_hash_traverse(ipdb, collect_ips, ips, &count);
1618         if (ret != 0) {
1619                 goto failed;
1620         }
1621
1622         if (count != ips->num) {
1623                 goto failed;
1624         }
1625
1626         talloc_free(ipdb);
1627
1628         *out = ips;
1629         return 0;
1630
1631 failed:
1632         talloc_free(ipdb);
1633         return 1;
1634 }
1635
1636 static int control_ip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1637                       int argc, const char **argv)
1638 {
1639         struct ctdb_public_ip_list *ips;
1640         struct ctdb_public_ip_info **ipinfo;
1641         int ret, i;
1642         bool do_all = false;
1643
1644         if (argc > 1) {
1645                 usage("ip");
1646         }
1647
1648         if (argc == 1) {
1649                 if (strcmp(argv[0], "all") == 0) {
1650                         do_all = true;
1651                 } else {
1652                         usage("ip");
1653                 }
1654         }
1655
1656         if (do_all) {
1657                 ret = get_all_public_ips(ctdb, mem_ctx, &ips);
1658         } else {
1659                 ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
1660                                                ctdb->cmd_pnn, TIMEOUT(), &ips);
1661         }
1662         if (ret != 0) {
1663                 return ret;
1664         }
1665
1666         qsort(ips->ip, ips->num, sizeof(struct ctdb_public_ip),
1667               ctdb_public_ip_cmp);
1668
1669         ipinfo = talloc_array(mem_ctx, struct ctdb_public_ip_info *, ips->num);
1670         if (ipinfo == NULL) {
1671                 return 1;
1672         }
1673
1674         for (i=0; i<ips->num; i++) {
1675                 uint32_t pnn;
1676                 if (do_all) {
1677                         pnn = ips->ip[i].pnn;
1678                 } else {
1679                         pnn = ctdb->cmd_pnn;
1680                 }
1681                 if (pnn == CTDB_UNKNOWN_PNN) {
1682                         ipinfo[i] = NULL;
1683                         continue;
1684                 }
1685                 ret = ctdb_ctrl_get_public_ip_info(mem_ctx, ctdb->ev,
1686                                                    ctdb->client, pnn,
1687                                                    TIMEOUT(), &ips->ip[i].addr,
1688                                                    &ipinfo[i]);
1689                 if (ret != 0) {
1690                         return ret;
1691                 }
1692         }
1693
1694         print_ip(mem_ctx, ctdb, ips, ipinfo, do_all);
1695         return 0;
1696 }
1697
1698 static int control_ipinfo(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1699                           int argc, const char **argv)
1700 {
1701         struct ctdb_public_ip_info *ipinfo;
1702         ctdb_sock_addr addr;
1703         int ret, i;
1704
1705         if (argc != 1) {
1706                 usage("ipinfo");
1707         }
1708
1709         if (! parse_ip(argv[0], NULL, 0, &addr)) {
1710                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
1711                 return 1;
1712         }
1713
1714         ret = ctdb_ctrl_get_public_ip_info(mem_ctx, ctdb->ev, ctdb->client,
1715                                            ctdb->cmd_pnn, TIMEOUT(), &addr,
1716                                            &ipinfo);
1717         if (ret != 0) {
1718                 if (ret == -1) {
1719                         printf("Node %u does not know about IP %s\n",
1720                                ctdb->cmd_pnn, argv[0]);
1721                 }
1722                 return ret;
1723         }
1724
1725         printf("Public IP[%s] info on node %u\n",
1726                ctdb_sock_addr_to_string(mem_ctx, &ipinfo->ip.addr),
1727                                         ctdb->cmd_pnn);
1728
1729         printf("IP:%s\nCurrentNode:%u\nNumInterfaces:%u\n",
1730                ctdb_sock_addr_to_string(mem_ctx, &ipinfo->ip.addr),
1731                ipinfo->ip.pnn, ipinfo->ifaces->num);
1732
1733         for (i=0; i<ipinfo->ifaces->num; i++) {
1734                 struct ctdb_iface *iface;
1735
1736                 iface = &ipinfo->ifaces->iface[i];
1737                 iface->name[CTDB_IFACE_SIZE] = '\0';
1738                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
1739                        i+1, iface->name,
1740                        iface->link_state == 0 ? "down" : "up",
1741                        iface->references,
1742                        (i == ipinfo->active_idx) ? " (active)" : "");
1743         }
1744
1745         return 0;
1746 }
1747
1748 static int control_ifaces(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1749                           int argc, const char **argv)
1750 {
1751         struct ctdb_iface_list *ifaces;
1752         int ret, i;
1753
1754         if (argc != 0) {
1755                 usage("ifaces");
1756         }
1757
1758         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
1759                                    ctdb->cmd_pnn, TIMEOUT(), &ifaces);
1760         if (ret != 0) {
1761                 return ret;
1762         }
1763
1764         if (ifaces->num == 0) {
1765                 printf("No interfaces configured on node %u\n",
1766                        ctdb->cmd_pnn);
1767                 return 0;
1768         }
1769
1770         if (options.machinereadable) {
1771                 printf("%s%s%s%s%s%s%s\n", options.sep,
1772                        "Name", options.sep,
1773                        "LinkStatus", options.sep,
1774                        "References", options.sep);
1775         } else {
1776                 printf("Interfaces on node %u\n", ctdb->cmd_pnn);
1777         }
1778
1779         for (i=0; i<ifaces->num; i++) {
1780                 if (options.machinereadable) {
1781                         printf("%s%s%s%u%s%u%s\n", options.sep,
1782                                ifaces->iface[i].name, options.sep,
1783                                ifaces->iface[i].link_state, options.sep,
1784                                ifaces->iface[i].references, options.sep);
1785                 } else {
1786                         printf("name:%s link:%s references:%u\n",
1787                                ifaces->iface[i].name,
1788                                ifaces->iface[i].link_state ? "up" : "down",
1789                                ifaces->iface[i].references);
1790                 }
1791         }
1792
1793         return 0;
1794 }
1795
1796 static int control_setifacelink(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1797                                 int argc, const char **argv)
1798 {
1799         struct ctdb_iface_list *ifaces;
1800         struct ctdb_iface *iface;
1801         int ret, i;
1802
1803         if (argc != 2) {
1804                 usage("setifacelink");
1805         }
1806
1807         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
1808                 fprintf(stderr, "Interface name '%s' too long\n", argv[0]);
1809                 return 1;
1810         }
1811
1812         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
1813                                    ctdb->cmd_pnn, TIMEOUT(), &ifaces);
1814         if (ret != 0) {
1815                 fprintf(stderr,
1816                         "Failed to get interface information from node %u\n",
1817                         ctdb->cmd_pnn);
1818                 return ret;
1819         }
1820
1821         iface = NULL;
1822         for (i=0; i<ifaces->num; i++) {
1823                 if (strcmp(ifaces->iface[i].name, argv[0]) == 0) {
1824                         iface = &ifaces->iface[i];
1825                         break;
1826                 }
1827         }
1828
1829         if (iface == NULL) {
1830                 printf("Interface %s not configured on node %u\n",
1831                        argv[0], ctdb->cmd_pnn);
1832                 return 1;
1833         }
1834
1835         if (strcmp(argv[1], "up") == 0) {
1836                 iface->link_state = 1;
1837         } else if (strcmp(argv[1], "down") == 0) {
1838                 iface->link_state = 0;
1839         } else {
1840                 usage("setifacelink");
1841                 return 1;
1842         }
1843
1844         iface->references = 0;
1845
1846         ret = ctdb_ctrl_set_iface_link_state(mem_ctx, ctdb->ev, ctdb->client,
1847                                              ctdb->cmd_pnn, TIMEOUT(), iface);
1848         if (ret != 0) {
1849                 return ret;
1850         }
1851
1852         return 0;
1853 }
1854
1855 static int control_process_exists(TALLOC_CTX *mem_ctx,
1856                                   struct ctdb_context *ctdb,
1857                                   int argc, const char **argv)
1858 {
1859         pid_t pid;
1860         int ret, status;
1861
1862         if (argc != 1) {
1863                 usage("process-exists");
1864         }
1865
1866         pid = atoi(argv[0]);
1867         ret = ctdb_ctrl_process_exists(mem_ctx, ctdb->ev, ctdb->client,
1868                                        ctdb->cmd_pnn, TIMEOUT(), pid, &status);
1869         if (ret != 0) {
1870                 return ret;
1871         }
1872
1873         if (status == 0) {
1874                 printf("PID %u exists\n", pid);
1875         } else {
1876                 printf("PID %u does not exist\n", pid);
1877         }
1878         return status;
1879 }
1880
1881 static int control_getdbmap(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1882                             int argc, const char **argv)
1883 {
1884         struct ctdb_dbid_map *dbmap;
1885         int ret, i;
1886
1887         if (argc != 0) {
1888                 usage("getdbmap");
1889         }
1890
1891         ret = ctdb_ctrl_get_dbmap(mem_ctx, ctdb->ev, ctdb->client,
1892                                   ctdb->cmd_pnn, TIMEOUT(), &dbmap);
1893         if (ret != 0) {
1894                 return ret;
1895         }
1896
1897         if (options.machinereadable == 1) {
1898                 printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
1899                        options.sep,
1900                        "ID", options.sep,
1901                        "Name", options.sep,
1902                        "Path", options.sep,
1903                        "Persistent", options.sep,
1904                        "Sticky", options.sep,
1905                        "Unhealthy", options.sep,
1906                        "Readonly", options.sep);
1907         } else {
1908                 printf("Number of databases:%d\n", dbmap->num);
1909         }
1910
1911         for (i=0; i<dbmap->num; i++) {
1912                 const char *name;
1913                 const char *path;
1914                 const char *health;
1915                 bool persistent;
1916                 bool readonly;
1917                 bool sticky;
1918                 uint32_t db_id;
1919
1920                 db_id = dbmap->dbs[i].db_id;
1921
1922                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
1923                                            ctdb->cmd_pnn, TIMEOUT(), db_id,
1924                                            &name);
1925                 if (ret != 0) {
1926                         return ret;
1927                 }
1928
1929                 ret = ctdb_ctrl_getdbpath(mem_ctx, ctdb->ev, ctdb->client,
1930                                           ctdb->cmd_pnn, TIMEOUT(), db_id,
1931                                           &path);
1932                 if (ret != 0) {
1933                         return ret;
1934                 }
1935
1936                 ret = ctdb_ctrl_db_get_health(mem_ctx, ctdb->ev, ctdb->client,
1937                                               ctdb->cmd_pnn, TIMEOUT(), db_id,
1938                                               &health);
1939                 if (ret != 0) {
1940                         return ret;
1941                 }
1942
1943                 persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
1944                 readonly = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
1945                 sticky = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
1946
1947                 if (options.machinereadable == 1) {
1948                         printf("%s0x%08X%s%s%s%s%s%d%s%d%s%d%s%d%s\n",
1949                                options.sep,
1950                                db_id, options.sep,
1951                                name, options.sep,
1952                                path, options.sep,
1953                                !! (persistent), options.sep,
1954                                !! (sticky), options.sep,
1955                                !! (health), options.sep,
1956                                !! (readonly), options.sep);
1957                 } else {
1958                         printf("dbid:0x%08x name:%s path:%s%s%s%s%s\n",
1959                                db_id, name, path,
1960                                persistent ? " PERSISTENT" : "",
1961                                sticky ? " STICKY" : "",
1962                                readonly ? " READONLY" : "",
1963                                health ? " UNHEALTHY" : "");
1964                 }
1965
1966                 talloc_free(discard_const(name));
1967                 talloc_free(discard_const(path));
1968                 talloc_free(discard_const(health));
1969         }
1970
1971         return 0;
1972 }
1973
1974 static int control_getdbstatus(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1975                                int argc, const char **argv)
1976 {
1977         uint32_t db_id;
1978         const char *db_name, *db_path, *db_health;
1979         uint8_t db_flags;
1980         int ret;
1981
1982         if (argc != 1) {
1983                 usage("getdbstatus");
1984         }
1985
1986         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
1987                 return 1;
1988         }
1989
1990         ret = ctdb_ctrl_getdbpath(mem_ctx, ctdb->ev, ctdb->client,
1991                                   ctdb->cmd_pnn, TIMEOUT(), db_id,
1992                                   &db_path);
1993         if (ret != 0) {
1994                 return ret;
1995         }
1996
1997         ret = ctdb_ctrl_db_get_health(mem_ctx, ctdb->ev, ctdb->client,
1998                                       ctdb->cmd_pnn, TIMEOUT(), db_id,
1999                                       &db_health);
2000         if (ret != 0) {
2001                 return ret;
2002         }
2003
2004         printf("dbid: 0x%08x\nname: %s\npath: %s\n", db_id, db_name, db_path);
2005         printf("PERSISTENT: %s\nSTICKY: %s\nREADONLY: %s\nHEALTH: %s\n",
2006                (db_flags & CTDB_DB_FLAGS_PERSISTENT ? "yes" : "no"),
2007                (db_flags & CTDB_DB_FLAGS_STICKY ? "yes" : "no"),
2008                (db_flags & CTDB_DB_FLAGS_READONLY ? "yes" : "no"),
2009                (db_health ? db_health : "OK"));
2010         return 0;
2011 }
2012
2013 struct dump_record_state {
2014         uint32_t count;
2015 };
2016
2017 #define ISASCII(x) (isprint(x) && ! strchr("\"\\", (x)))
2018
2019 static void dump_tdb_data(const char *name, TDB_DATA val)
2020 {
2021         int i;
2022
2023         fprintf(stdout, "%s(%zu) = \"", name, val.dsize);
2024         for (i=0; i<val.dsize; i++) {
2025                 if (ISASCII(val.dptr[i])) {
2026                         fprintf(stdout, "%c", val.dptr[i]);
2027                 } else {
2028                         fprintf(stdout, "\\%02X", val.dptr[i]);
2029                 }
2030         }
2031         fprintf(stdout, "\"\n");
2032 }
2033
2034 static void dump_ltdb_header(struct ctdb_ltdb_header *header)
2035 {
2036         fprintf(stdout, "dmaster: %u\n", header->dmaster);
2037         fprintf(stdout, "rsn: %" PRIu64 "\n", header->rsn);
2038         fprintf(stdout, "flags: 0x%08x", header->flags);
2039         if (header->flags & CTDB_REC_FLAG_MIGRATED_WITH_DATA) {
2040                 fprintf(stdout, " MIGRATED_WITH_DATA");
2041         }
2042         if (header->flags & CTDB_REC_FLAG_VACUUM_MIGRATED) {
2043                 fprintf(stdout, " VACUUM_MIGRATED");
2044         }
2045         if (header->flags & CTDB_REC_FLAG_AUTOMATIC) {
2046                 fprintf(stdout, " AUTOMATIC");
2047         }
2048         if (header->flags & CTDB_REC_RO_HAVE_DELEGATIONS) {
2049                 fprintf(stdout, " RO_HAVE_DELEGATIONS");
2050         }
2051         if (header->flags & CTDB_REC_RO_HAVE_READONLY) {
2052                 fprintf(stdout, " RO_HAVE_READONLY");
2053         }
2054         if (header->flags & CTDB_REC_RO_REVOKING_READONLY) {
2055                 fprintf(stdout, " RO_REVOKING_READONLY");
2056         }
2057         if (header->flags & CTDB_REC_RO_REVOKE_COMPLETE) {
2058                 fprintf(stdout, " RO_REVOKE_COMPLETE");
2059         }
2060         fprintf(stdout, "\n");
2061
2062 }
2063
2064 static int dump_record(uint32_t reqid, struct ctdb_ltdb_header *header,
2065                        TDB_DATA key, TDB_DATA data, void *private_data)
2066 {
2067         struct dump_record_state *state =
2068                 (struct dump_record_state *)private_data;
2069
2070         state->count += 1;
2071
2072         dump_tdb_data("key", key);
2073         dump_ltdb_header(header);
2074         dump_tdb_data("data", data);
2075         fprintf(stdout, "\n");
2076
2077         return 0;
2078 }
2079
2080 struct traverse_state {
2081         TALLOC_CTX *mem_ctx;
2082         bool done;
2083         ctdb_rec_parser_func_t func;
2084         struct dump_record_state sub_state;
2085 };
2086
2087 static void traverse_handler(uint64_t srvid, TDB_DATA data, void *private_data)
2088 {
2089         struct traverse_state *state = (struct traverse_state *)private_data;
2090         struct ctdb_rec_data *rec;
2091         struct ctdb_ltdb_header header;
2092         int ret;
2093
2094         ret = ctdb_rec_data_pull(data.dptr, data.dsize, state->mem_ctx, &rec);
2095         if (ret != 0) {
2096                 return;
2097         }
2098
2099         if (rec->key.dsize == 0 && rec->data.dsize == 0) {
2100                 talloc_free(rec);
2101                 /* end of traverse */
2102                 state->done = true;
2103                 return;
2104         }
2105
2106         ret = ctdb_ltdb_header_extract(&rec->data, &header);
2107         if (ret != 0) {
2108                 talloc_free(rec);
2109                 return;
2110         }
2111
2112         if (rec->data.dsize == 0) {
2113                 talloc_free(rec);
2114                 return;
2115         }
2116
2117         ret = state->func(rec->reqid, &header, rec->key, rec->data,
2118                           &state->sub_state);
2119         talloc_free(rec);
2120         if (ret != 0) {
2121                 state->done = true;
2122         }
2123 }
2124
2125 static int control_catdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2126                          int argc, const char **argv)
2127 {
2128         struct ctdb_db_context *db;
2129         const char *db_name;
2130         uint32_t db_id;
2131         uint8_t db_flags;
2132         struct ctdb_traverse_start_ext traverse;
2133         struct traverse_state state;
2134         int ret;
2135
2136         if (argc != 1) {
2137                 usage("catdb");
2138         }
2139
2140         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
2141                 return 1;
2142         }
2143
2144         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2145                           db_flags, &db);
2146         if (ret != 0) {
2147                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
2148                 return ret;
2149         }
2150
2151         /* Valgrind fix */
2152         ZERO_STRUCT(traverse);
2153
2154         traverse.db_id = db_id;
2155         traverse.reqid = 0;
2156         traverse.srvid = next_srvid(ctdb);
2157         traverse.withemptyrecords = false;
2158
2159         state.mem_ctx = mem_ctx;
2160         state.done = false;
2161         state.func = dump_record;
2162         state.sub_state.count = 0;
2163
2164         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2165                                               traverse.srvid,
2166                                               traverse_handler, &state);
2167         if (ret != 0) {
2168                 return ret;
2169         }
2170
2171         ret = ctdb_ctrl_traverse_start_ext(mem_ctx, ctdb->ev, ctdb->client,
2172                                            ctdb->cmd_pnn, TIMEOUT(),
2173                                            &traverse);
2174         if (ret != 0) {
2175                 return ret;
2176         }
2177
2178         ctdb_client_wait(ctdb->ev, &state.done);
2179
2180         printf("Dumped %u records\n", state.sub_state.count);
2181
2182         ret = ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
2183                                                  traverse.srvid, &state);
2184         if (ret != 0) {
2185                 return ret;
2186         }
2187
2188         return 0;
2189 }
2190
2191 static int control_cattdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2192                           int argc, const char **argv)
2193 {
2194         struct ctdb_db_context *db;
2195         const char *db_name;
2196         uint32_t db_id;
2197         uint8_t db_flags;
2198         struct dump_record_state state;
2199         int ret;
2200
2201         if (argc != 1) {
2202                 usage("catdb");
2203         }
2204
2205         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
2206                 return 1;
2207         }
2208
2209         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2210                           db_flags, &db);
2211         if (ret != 0) {
2212                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
2213                 return ret;
2214         }
2215
2216         state.count = 0;
2217         ret = ctdb_db_traverse(db, true, true, dump_record, &state);
2218
2219         printf("Dumped %u record(s)\n", state.count);
2220
2221         return ret;
2222 }
2223
2224 static int control_getmonmode(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2225                               int argc, const char **argv)
2226 {
2227         int mode, ret;
2228
2229         if (argc != 0) {
2230                 usage("getmonmode");
2231         }
2232
2233         ret = ctdb_ctrl_get_monmode(mem_ctx, ctdb->ev, ctdb->client,
2234                                     ctdb->cmd_pnn, TIMEOUT(), &mode);
2235         if (ret != 0) {
2236                 return ret;
2237         }
2238
2239         printf("%s\n",
2240                (mode == CTDB_MONITORING_ENABLED) ? "ENABLED" : "DISABLED");
2241         return 0;
2242 }
2243
2244 static int control_getcapabilities(TALLOC_CTX *mem_ctx,
2245                                    struct ctdb_context *ctdb,
2246                                    int argc, const char **argv)
2247 {
2248         uint32_t caps;
2249         int ret;
2250
2251         if (argc != 0) {
2252                 usage("getcapabilities");
2253         }
2254
2255         ret = ctdb_ctrl_get_capabilities(mem_ctx, ctdb->ev, ctdb->client,
2256                                          ctdb->cmd_pnn, TIMEOUT(), &caps);
2257         if (ret != 0) {
2258                 return ret;
2259         }
2260
2261         if (options.machinereadable == 1) {
2262                 printf("%s%s%s%s%s\n",
2263                        options.sep,
2264                        "RECMASTER", options.sep,
2265                        "LMASTER", options.sep);
2266                 printf("%s%d%s%d%s\n", options.sep,
2267                        !! (caps & CTDB_CAP_RECMASTER), options.sep,
2268                        !! (caps & CTDB_CAP_LMASTER), options.sep);
2269         } else {
2270                 printf("RECMASTER: %s\n",
2271                        (caps & CTDB_CAP_RECMASTER) ? "YES" : "NO");
2272                 printf("LMASTER: %s\n",
2273                        (caps & CTDB_CAP_LMASTER) ? "YES" : "NO");
2274         }
2275
2276         return 0;
2277 }
2278
2279 static int control_pnn(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2280                        int argc, const char **argv)
2281 {
2282         printf("%u\n", ctdb_client_pnn(ctdb->client));
2283         return 0;
2284 }
2285
2286 static int control_lvs(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2287                        int argc, const char **argv)
2288 {
2289         char *t, *lvs_helper = NULL;
2290
2291         if (argc != 1) {
2292                 usage("lvs");
2293         }
2294
2295         t = getenv("CTDB_LVS_HELPER");
2296         if (t != NULL) {
2297                 lvs_helper = talloc_strdup(mem_ctx, t);
2298         } else {
2299                 lvs_helper = talloc_asprintf(mem_ctx, "%s/ctdb_lvs",
2300                                              CTDB_HELPER_BINDIR);
2301         }
2302
2303         if (lvs_helper == NULL) {
2304                 fprintf(stderr, "Unable to set LVS helper\n");
2305                 return 1;
2306         }
2307
2308         return run_helper(mem_ctx, "LVS helper", lvs_helper, argc, argv);
2309 }
2310
2311 static int control_disable_monitor(TALLOC_CTX *mem_ctx,
2312                                    struct ctdb_context *ctdb,
2313                                    int argc, const char **argv)
2314 {
2315         int ret;
2316
2317         if (argc != 0) {
2318                 usage("disablemonitor");
2319         }
2320
2321         ret = ctdb_ctrl_disable_monitor(mem_ctx, ctdb->ev, ctdb->client,
2322                                         ctdb->cmd_pnn, TIMEOUT());
2323         if (ret != 0) {
2324                 return ret;
2325         }
2326
2327         return 0;
2328 }
2329
2330 static int control_enable_monitor(TALLOC_CTX *mem_ctx,
2331                                   struct ctdb_context *ctdb,
2332                                   int argc, const char **argv)
2333 {
2334         int ret;
2335
2336         if (argc != 0) {
2337                 usage("enablemonitor");
2338         }
2339
2340         ret = ctdb_ctrl_enable_monitor(mem_ctx, ctdb->ev, ctdb->client,
2341                                        ctdb->cmd_pnn, TIMEOUT());
2342         if (ret != 0) {
2343                 return ret;
2344         }
2345
2346         return 0;
2347 }
2348
2349 static int control_setdebug(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2350                             int argc, const char **argv)
2351 {
2352         int log_level;
2353         int ret;
2354         bool found;
2355
2356         if (argc != 1) {
2357                 usage("setdebug");
2358         }
2359
2360         found = debug_level_parse(argv[0], &log_level);
2361         if (! found) {
2362                 fprintf(stderr,
2363                         "Invalid debug level '%s'. Valid levels are:\n",
2364                         argv[0]);
2365                 fprintf(stderr, "\tERROR | WARNING | NOTICE | INFO | DEBUG\n");
2366                 return 1;
2367         }
2368
2369         ret = ctdb_ctrl_setdebug(mem_ctx, ctdb->ev, ctdb->client,
2370                                  ctdb->cmd_pnn, TIMEOUT(), log_level);
2371         if (ret != 0) {
2372                 return ret;
2373         }
2374
2375         return 0;
2376 }
2377
2378 static int control_getdebug(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2379                             int argc, const char **argv)
2380 {
2381         int loglevel;
2382         const char *log_str;
2383         int ret;
2384
2385         if (argc != 0) {
2386                 usage("getdebug");
2387         }
2388
2389         ret = ctdb_ctrl_getdebug(mem_ctx, ctdb->ev, ctdb->client,
2390                                  ctdb->cmd_pnn, TIMEOUT(), &loglevel);
2391         if (ret != 0) {
2392                 return ret;
2393         }
2394
2395         log_str = debug_level_to_string(loglevel);
2396         printf("%s\n", log_str);
2397
2398         return 0;
2399 }
2400
2401 static int control_attach(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2402                           int argc, const char **argv)
2403 {
2404         const char *db_name;
2405         uint8_t db_flags = 0;
2406         int ret;
2407
2408         if (argc < 1 || argc > 2) {
2409                 usage("attach");
2410         }
2411
2412         db_name = argv[0];
2413         if (argc == 2) {
2414                 if (strcmp(argv[1], "persistent") == 0) {
2415                         db_flags = CTDB_DB_FLAGS_PERSISTENT;
2416                 } else if (strcmp(argv[1], "readonly") == 0) {
2417                         db_flags = CTDB_DB_FLAGS_READONLY;
2418                 } else if (strcmp(argv[1], "sticky") == 0) {
2419                         db_flags = CTDB_DB_FLAGS_STICKY;
2420                 } else {
2421                         usage("attach");
2422                 }
2423         }
2424
2425         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2426                           db_flags, NULL);
2427         if (ret != 0) {
2428                 return ret;
2429         }
2430
2431         return 0;
2432 }
2433
2434 static int control_detach(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2435                           int argc, const char **argv)
2436 {
2437         const char *db_name;
2438         uint32_t db_id;
2439         uint8_t db_flags;
2440         struct ctdb_node_map *nodemap;
2441         int recmode;
2442         int ret, ret2, i;
2443
2444         if (argc < 1) {
2445                 usage("detach");
2446         }
2447
2448         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
2449                                     ctdb->cmd_pnn, TIMEOUT(), &recmode);
2450         if (ret != 0) {
2451                 return ret;
2452         }
2453
2454         if (recmode == CTDB_RECOVERY_ACTIVE) {
2455                 fprintf(stderr, "Database cannot be detached"
2456                                 " when recovery is active\n");
2457                 return 1;
2458         }
2459
2460         nodemap = get_nodemap(ctdb, false);
2461         if (nodemap == NULL) {
2462                 return 1;
2463         }
2464
2465         for (i=0; i<nodemap->num; i++) {
2466                 uint32_t value;
2467
2468                 if (nodemap->node[i].flags & NODE_FLAGS_DISCONNECTED) {
2469                         continue;
2470                 }
2471                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
2472                         continue;
2473                 }
2474                 if (nodemap->node[i].flags & NODE_FLAGS_INACTIVE) {
2475                         fprintf(stderr, "Database cannot be detached on"
2476                                 " inactive (stopped or banned) node %u\n",
2477                                 nodemap->node[i].pnn);
2478                         return 1;
2479                 }
2480
2481                 ret = ctdb_ctrl_get_tunable(mem_ctx, ctdb->ev, ctdb->client,
2482                                             nodemap->node[i].pnn, TIMEOUT(),
2483                                             "AllowClientDBAttach", &value);
2484                 if (ret != 0) {
2485                         fprintf(stderr,
2486                                 "Unable to get tunable AllowClientDBAttach"
2487                                 " from node %u\n", nodemap->node[i].pnn);
2488                         return ret;
2489                 }
2490
2491                 if (value == 1) {
2492                         fprintf(stderr,
2493                                 "Database access is still active on node %u."
2494                                 " Set AllowclientDBAttach=0 on all nodes.\n",
2495                                 nodemap->node[i].pnn);
2496                         return 1;
2497                 }
2498         }
2499
2500         ret2 = 0;
2501         for (i=0; i<argc; i++) {
2502                 if (! db_exists(mem_ctx, ctdb, argv[i], &db_id, &db_name,
2503                                 &db_flags)) {
2504                         continue;
2505                 }
2506
2507                 if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
2508                         fprintf(stderr,
2509                                 "Persistent database %s cannot be detached\n",
2510                                 argv[0]);
2511                         return 1;
2512                 }
2513
2514                 ret = ctdb_detach(mem_ctx, ctdb->ev, ctdb->client,
2515                                   TIMEOUT(), db_id);
2516                 if (ret != 0) {
2517                         fprintf(stderr, "Database %s detach failed\n", db_name);
2518                         ret2 = ret;
2519                 }
2520         }
2521
2522         return ret2;
2523 }
2524
2525 static int control_dumpmemory(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2526                               int argc, const char **argv)
2527 {
2528         const char *mem_str;
2529         ssize_t n;
2530         int ret;
2531
2532         ret = ctdb_ctrl_dump_memory(mem_ctx, ctdb->ev, ctdb->client,
2533                                     ctdb->cmd_pnn, TIMEOUT(), &mem_str);
2534         if (ret != 0) {
2535                 return ret;
2536         }
2537
2538         n = write(1, mem_str, strlen(mem_str)+1);
2539         if (n < 0 || n != strlen(mem_str)+1) {
2540                 fprintf(stderr, "Failed to write talloc summary\n");
2541                 return 1;
2542         }
2543
2544         return 0;
2545 }
2546
2547 static void dump_memory(uint64_t srvid, TDB_DATA data, void *private_data)
2548 {
2549         bool *done = (bool *)private_data;
2550         ssize_t n;
2551
2552         n = write(1, data.dptr, data.dsize);
2553         if (n < 0 || n != data.dsize) {
2554                 fprintf(stderr, "Failed to write talloc summary\n");
2555         }
2556
2557         *done = true;
2558 }
2559
2560 static int control_rddumpmemory(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2561                                 int argc, const char **argv)
2562 {
2563         struct ctdb_srvid_message msg = { 0 };
2564         int ret;
2565         bool done = false;
2566
2567         msg.pnn = ctdb->pnn;
2568         msg.srvid = next_srvid(ctdb);
2569
2570         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2571                                               msg.srvid, dump_memory, &done);
2572         if (ret != 0) {
2573                 return ret;
2574         }
2575
2576         ret = ctdb_message_mem_dump(mem_ctx, ctdb->ev, ctdb->client,
2577                                     ctdb->cmd_pnn, &msg);
2578         if (ret != 0) {
2579                 return ret;
2580         }
2581
2582         ctdb_client_wait(ctdb->ev, &done);
2583         return 0;
2584 }
2585
2586 static int control_getpid(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2587                           int argc, const char **argv)
2588 {
2589         pid_t pid;
2590         int ret;
2591
2592         ret = ctdb_ctrl_get_pid(mem_ctx, ctdb->ev, ctdb->client,
2593                                 ctdb->cmd_pnn, TIMEOUT(), &pid);
2594         if (ret != 0) {
2595                 return ret;
2596         }
2597
2598         printf("%u\n", pid);
2599         return 0;
2600 }
2601
2602 static int check_flags(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2603                        const char *desc, uint32_t flag, bool set_flag)
2604 {
2605         struct ctdb_node_map *nodemap;
2606         bool flag_is_set;
2607
2608         nodemap = get_nodemap(ctdb, false);
2609         if (nodemap == NULL) {
2610                 return 1;
2611         }
2612
2613         flag_is_set = nodemap->node[ctdb->cmd_pnn].flags & flag;
2614         if (set_flag == flag_is_set) {
2615                 if (set_flag) {
2616                         fprintf(stderr, "Node %u is already %s\n",
2617                                 ctdb->cmd_pnn, desc);
2618                 } else {
2619                         fprintf(stderr, "Node %u is not %s\n",
2620                                 ctdb->cmd_pnn, desc);
2621                 }
2622                 return 0;
2623         }
2624
2625         return 1;
2626 }
2627
2628 static void wait_for_flags(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2629                            uint32_t flag, bool set_flag)
2630 {
2631         struct ctdb_node_map *nodemap;
2632         bool flag_is_set;
2633
2634         while (1) {
2635                 nodemap = get_nodemap(ctdb, true);
2636                 if (nodemap == NULL) {
2637                         fprintf(stderr,
2638                                 "Failed to get nodemap, trying again\n");
2639                         sleep(1);
2640                         continue;
2641                 }
2642
2643                 flag_is_set = nodemap->node[ctdb->cmd_pnn].flags & flag;
2644                 if (flag_is_set == set_flag) {
2645                         break;
2646                 }
2647
2648                 sleep(1);
2649         }
2650 }
2651
2652 struct ipreallocate_state {
2653         int status;
2654         bool done;
2655 };
2656
2657 static void ipreallocate_handler(uint64_t srvid, TDB_DATA data,
2658                                  void *private_data)
2659 {
2660         struct ipreallocate_state *state =
2661                 (struct ipreallocate_state *)private_data;
2662
2663         if (data.dsize != sizeof(int)) {
2664                 /* Ignore packet */
2665                 return;
2666         }
2667
2668         state->status = *(int *)data.dptr;
2669         state->done = true;
2670 }
2671
2672 static int ipreallocate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb)
2673 {
2674         struct ctdb_srvid_message msg = { 0 };
2675         struct ipreallocate_state state;
2676         int ret;
2677
2678         msg.pnn = ctdb->pnn;
2679         msg.srvid = next_srvid(ctdb);
2680
2681         state.done = false;
2682         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2683                                               msg.srvid,
2684                                               ipreallocate_handler, &state);
2685         if (ret != 0) {
2686                 return ret;
2687         }
2688
2689         while (true) {
2690                 ret = ctdb_message_takeover_run(mem_ctx, ctdb->ev,
2691                                                 ctdb->client,
2692                                                 CTDB_BROADCAST_CONNECTED,
2693                                                 &msg);
2694                 if (ret != 0) {
2695                         goto fail;
2696                 }
2697
2698                 ret = ctdb_client_wait_timeout(ctdb->ev, &state.done,
2699                                                TIMEOUT());
2700                 if (ret != 0) {
2701                         continue;
2702                 }
2703
2704                 if (state.status >= 0) {
2705                         ret = 0;
2706                 } else {
2707                         ret = state.status;
2708                 }
2709                 break;
2710         }
2711
2712 fail:
2713         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
2714                                            msg.srvid, &state);
2715         return ret;
2716 }
2717
2718 static int control_disable(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2719                            int argc, const char **argv)
2720 {
2721         int ret;
2722
2723         if (argc != 0) {
2724                 usage("disable");
2725         }
2726
2727         ret = check_flags(mem_ctx, ctdb, "disabled",
2728                           NODE_FLAGS_PERMANENTLY_DISABLED, true);
2729         if (ret == 0) {
2730                 return 0;
2731         }
2732
2733         ret = ctdb_ctrl_modflags(mem_ctx, ctdb->ev, ctdb->client,
2734                                  ctdb->cmd_pnn, TIMEOUT(),
2735                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
2736         if (ret != 0) {
2737                 fprintf(stderr,
2738                         "Failed to set DISABLED flag on node %u\n",
2739                         ctdb->cmd_pnn);
2740                 return ret;
2741         }
2742
2743         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_PERMANENTLY_DISABLED, true);
2744         return ipreallocate(mem_ctx, ctdb);
2745 }
2746
2747 static int control_enable(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2748                           int argc, const char **argv)
2749 {
2750         int ret;
2751
2752         if (argc != 0) {
2753                 usage("enable");
2754         }
2755
2756         ret = check_flags(mem_ctx, ctdb, "disabled",
2757                           NODE_FLAGS_PERMANENTLY_DISABLED, false);
2758         if (ret == 0) {
2759                 return 0;
2760         }
2761
2762         ret = ctdb_ctrl_modflags(mem_ctx, ctdb->ev, ctdb->client,
2763                                  ctdb->cmd_pnn, TIMEOUT(),
2764                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
2765         if (ret != 0) {
2766                 fprintf(stderr, "Failed to reset DISABLED flag on node %u\n",
2767                         ctdb->cmd_pnn);
2768                 return ret;
2769         }
2770
2771         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_PERMANENTLY_DISABLED, false);
2772         return ipreallocate(mem_ctx, ctdb);
2773 }
2774
2775 static int control_stop(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2776                         int argc, const char **argv)
2777 {
2778         int ret;
2779
2780         if (argc != 0) {
2781                 usage("stop");
2782         }
2783
2784         ret = check_flags(mem_ctx, ctdb, "stopped",
2785                           NODE_FLAGS_STOPPED, true);
2786         if (ret == 0) {
2787                 return 0;
2788         }
2789
2790         ret = ctdb_ctrl_stop_node(mem_ctx, ctdb->ev, ctdb->client,
2791                                   ctdb->cmd_pnn, TIMEOUT());
2792         if (ret != 0) {
2793                 fprintf(stderr, "Failed to stop node %u\n", ctdb->cmd_pnn);
2794                 return ret;
2795         }
2796
2797         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_STOPPED, true);
2798         return ipreallocate(mem_ctx, ctdb);
2799 }
2800
2801 static int control_continue(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2802                             int argc, const char **argv)
2803 {
2804         int ret;
2805
2806         if (argc != 0) {
2807                 usage("continue");
2808         }
2809
2810         ret = check_flags(mem_ctx, ctdb, "stopped",
2811                           NODE_FLAGS_STOPPED, false);
2812         if (ret == 0) {
2813                 return 0;
2814         }
2815
2816         ret = ctdb_ctrl_continue_node(mem_ctx, ctdb->ev, ctdb->client,
2817                                       ctdb->cmd_pnn, TIMEOUT());
2818         if (ret != 0) {
2819                 fprintf(stderr, "Failed to continue stopped node %u\n",
2820                         ctdb->cmd_pnn);
2821                 return ret;
2822         }
2823
2824         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_STOPPED, false);
2825         return ipreallocate(mem_ctx, ctdb);
2826 }
2827
2828 static int control_ban(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2829                        int argc, const char **argv)
2830 {
2831         struct ctdb_ban_state ban_state;
2832         int ret;
2833
2834         if (argc != 1) {
2835                 usage("ban");
2836         }
2837
2838         ret = check_flags(mem_ctx, ctdb, "banned",
2839                           NODE_FLAGS_BANNED, true);
2840         if (ret == 0) {
2841                 return 0;
2842         }
2843
2844         ban_state.pnn = ctdb->cmd_pnn;
2845         ban_state.time = strtoul(argv[0], NULL, 0);
2846
2847         if (ban_state.time == 0) {
2848                 fprintf(stderr, "Ban time cannot be zero\n");
2849                 return EINVAL;
2850         }
2851
2852         ret = ctdb_ctrl_set_ban_state(mem_ctx, ctdb->ev, ctdb->client,
2853                                       ctdb->cmd_pnn, TIMEOUT(), &ban_state);
2854         if (ret != 0) {
2855                 fprintf(stderr, "Failed to ban node %u\n", ctdb->cmd_pnn);
2856                 return ret;
2857         }
2858
2859         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_BANNED, true);
2860         return ipreallocate(mem_ctx, ctdb);
2861
2862 }
2863
2864 static int control_unban(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2865                          int argc, const char **argv)
2866 {
2867         struct ctdb_ban_state ban_state;
2868         int ret;
2869
2870         if (argc != 0) {
2871                 usage("unban");
2872         }
2873
2874         ret = check_flags(mem_ctx, ctdb, "banned",
2875                           NODE_FLAGS_BANNED, false);
2876         if (ret == 0) {
2877                 return 0;
2878         }
2879
2880         ban_state.pnn = ctdb->cmd_pnn;
2881         ban_state.time = 0;
2882
2883         ret = ctdb_ctrl_set_ban_state(mem_ctx, ctdb->ev, ctdb->client,
2884                                       ctdb->cmd_pnn, TIMEOUT(), &ban_state);
2885         if (ret != 0) {
2886                 fprintf(stderr, "Failed to unban node %u\n", ctdb->cmd_pnn);
2887                 return ret;
2888         }
2889
2890         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_BANNED, false);
2891         return ipreallocate(mem_ctx, ctdb);
2892
2893 }
2894
2895 static int control_shutdown(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2896                             int argc, const char **argv)
2897 {
2898         int ret;
2899
2900         if (argc != 0) {
2901                 usage("shutdown");
2902         }
2903
2904         ret = ctdb_ctrl_shutdown(mem_ctx, ctdb->ev, ctdb->client,
2905                                  ctdb->cmd_pnn, TIMEOUT());
2906         if (ret != 0) {
2907                 fprintf(stderr, "Unable to shutdown node %u\n", ctdb->cmd_pnn);
2908                 return ret;
2909         }
2910
2911         return 0;
2912 }
2913
2914 static int get_generation(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2915                           uint32_t *generation)
2916 {
2917         uint32_t recmaster;
2918         int recmode;
2919         struct ctdb_vnn_map *vnnmap;
2920         int ret;
2921
2922 again:
2923         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
2924                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
2925         if (ret != 0) {
2926                 fprintf(stderr, "Failed to find recovery master\n");
2927                 return ret;
2928         }
2929
2930         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
2931                                     recmaster, TIMEOUT(), &recmode);
2932         if (ret != 0) {
2933                 fprintf(stderr, "Failed to get recovery mode from node %u\n",
2934                         recmaster);
2935                 return ret;
2936         }
2937
2938         if (recmode == CTDB_RECOVERY_ACTIVE) {
2939                 sleep(1);
2940                 goto again;
2941         }
2942
2943         ret = ctdb_ctrl_getvnnmap(mem_ctx, ctdb->ev, ctdb->client,
2944                                   recmaster, TIMEOUT(), &vnnmap);
2945         if (ret != 0) {
2946                 fprintf(stderr, "Failed to get generation from node %u\n",
2947                         recmaster);
2948                 return ret;
2949         }
2950
2951         if (vnnmap->generation == INVALID_GENERATION) {
2952                 talloc_free(vnnmap);
2953                 sleep(1);
2954                 goto again;
2955         }
2956
2957         *generation = vnnmap->generation;
2958         talloc_free(vnnmap);
2959         return 0;
2960 }
2961
2962
2963 static int control_recover(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2964                            int argc, const char **argv)
2965 {
2966         uint32_t generation, next_generation;
2967         int ret;
2968
2969         if (argc != 0) {
2970                 usage("recover");
2971         }
2972
2973         ret = get_generation(mem_ctx, ctdb, &generation);
2974         if (ret != 0) {
2975                 return ret;
2976         }
2977
2978         ret = ctdb_ctrl_set_recmode(mem_ctx, ctdb->ev, ctdb->client,
2979                                     ctdb->cmd_pnn, TIMEOUT(),
2980                                     CTDB_RECOVERY_ACTIVE);
2981         if (ret != 0) {
2982                 fprintf(stderr, "Failed to set recovery mode active\n");
2983                 return ret;
2984         }
2985
2986         while (1) {
2987                 ret = get_generation(mem_ctx, ctdb, &next_generation);
2988                 if (ret != 0) {
2989                         fprintf(stderr,
2990                                 "Failed to confirm end of recovery\n");
2991                         return ret;
2992                 }
2993
2994                 if (next_generation != generation) {
2995                         break;
2996                 }
2997
2998                 sleep (1);
2999         }
3000
3001         return 0;
3002 }
3003
3004 static int control_ipreallocate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3005                                 int argc, const char **argv)
3006 {
3007         if (argc != 0) {
3008                 usage("ipreallocate");
3009         }
3010
3011         return ipreallocate(mem_ctx, ctdb);
3012 }
3013
3014 static int control_isnotrecmaster(TALLOC_CTX *mem_ctx,
3015                                   struct ctdb_context *ctdb,
3016                                   int argc, const char **argv)
3017 {
3018         uint32_t recmaster;
3019         int ret;
3020
3021         if (argc != 0) {
3022                 usage("isnotrecmaster");
3023         }
3024
3025         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
3026                                       ctdb->pnn, TIMEOUT(), &recmaster);
3027         if (ret != 0) {
3028                 fprintf(stderr, "Failed to get recmaster\n");
3029                 return ret;
3030         }
3031
3032         if (recmaster != ctdb->pnn) {
3033                 printf("this node is not the recmaster\n");
3034                 return 1;
3035         }
3036
3037         printf("this node is the recmaster\n");
3038         return 0;
3039 }
3040
3041 static int control_gratarp(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3042                            int argc, const char **argv)
3043 {
3044         struct ctdb_addr_info addr_info;
3045         int ret;
3046
3047         if (argc != 2) {
3048                 usage("gratarp");
3049         }
3050
3051         if (! parse_ip(argv[0], NULL, 0, &addr_info.addr)) {
3052                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3053                 return 1;
3054         }
3055         addr_info.iface = argv[1];
3056
3057         ret = ctdb_ctrl_send_gratuitous_arp(mem_ctx, ctdb->ev, ctdb->client,
3058                                             ctdb->cmd_pnn, TIMEOUT(),
3059                                             &addr_info);
3060         if (ret != 0) {
3061                 fprintf(stderr, "Unable to send gratuitous arp from node %u\n",
3062                         ctdb->cmd_pnn);
3063                 return ret;
3064         }
3065
3066         return 0;
3067 }
3068
3069 static int control_tickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3070                            int argc, const char **argv)
3071 {
3072         ctdb_sock_addr src, dst;
3073         int ret;
3074
3075         if (argc != 0 && argc != 2) {
3076                 usage("tickle");
3077         }
3078
3079         if (argc == 0) {
3080                 struct ctdb_connection *clist;
3081                 int count;
3082                 int i, num_failed;
3083
3084                 ret = ctdb_parse_connections(stdin, mem_ctx, &count, &clist);
3085                 if (ret != 0) {
3086                         return ret;
3087                 }
3088
3089                 num_failed = 0;
3090                 for (i=0; i<count; i++) {
3091                         ret = ctdb_sys_send_tcp(&clist[i].src,
3092                                                 &clist[i].dst,
3093                                                 0, 0, 0);
3094                         if (ret != 0) {
3095                                 num_failed += 1;
3096                         }
3097                 }
3098
3099                 TALLOC_FREE(clist);
3100
3101                 if (num_failed > 0) {
3102                         fprintf(stderr, "Failed to send %d tickles\n",
3103                                 num_failed);
3104                         return 1;
3105                 }
3106
3107                 return 0;
3108         }
3109
3110
3111         if (! parse_ip_port(argv[0], &src)) {
3112                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3113                 return 1;
3114         }
3115
3116         if (! parse_ip_port(argv[1], &dst)) {
3117                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3118                 return 1;
3119         }
3120
3121         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
3122         if (ret != 0) {
3123                 fprintf(stderr, "Failed to send tickle ack\n");
3124                 return ret;
3125         }
3126
3127         return 0;
3128 }
3129
3130 static int control_gettickles(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3131                               int argc, const char **argv)
3132 {
3133         ctdb_sock_addr addr;
3134         struct ctdb_tickle_list *tickles;
3135         unsigned port = 0;
3136         int ret, i;
3137
3138         if (argc < 1 || argc > 2) {
3139                 usage("gettickles");
3140         }
3141
3142         if (argc == 2) {
3143                 port = strtoul(argv[1], NULL, 10);
3144         }
3145
3146         if (! parse_ip(argv[0], NULL, port, &addr)) {
3147                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3148                 return 1;
3149         }
3150
3151         ret = ctdb_ctrl_get_tcp_tickle_list(mem_ctx, ctdb->ev, ctdb->client,
3152                                             ctdb->cmd_pnn, TIMEOUT(), &addr,
3153                                             &tickles);
3154         if (ret != 0) {
3155                 fprintf(stderr, "Failed to get list of connections\n");
3156                 return ret;
3157         }
3158
3159         if (options.machinereadable) {
3160                 printf("%s%s%s%s%s%s%s%s%s\n",
3161                        options.sep,
3162                        "Source IP", options.sep,
3163                        "Port", options.sep,
3164                        "Destiation IP", options.sep,
3165                        "Port", options.sep);
3166                 for (i=0; i<tickles->num; i++) {
3167                         printf("%s%s%s%u%s%s%s%u%s\n", options.sep,
3168                                ctdb_sock_addr_to_string(
3169                                        mem_ctx, &tickles->conn[i].src),
3170                                options.sep,
3171                                ntohs(tickles->conn[i].src.ip.sin_port),
3172                                options.sep,
3173                                ctdb_sock_addr_to_string(
3174                                        mem_ctx, &tickles->conn[i].dst),
3175                                options.sep,
3176                                ntohs(tickles->conn[i].dst.ip.sin_port),
3177                                options.sep);
3178                 }
3179         } else {
3180                 printf("Connections for IP: %s\n",
3181                        ctdb_sock_addr_to_string(mem_ctx, &tickles->addr));
3182                 printf("Num connections: %u\n", tickles->num);
3183                 for (i=0; i<tickles->num; i++) {
3184                         printf("SRC: %s:%u   DST: %s:%u\n",
3185                                ctdb_sock_addr_to_string(
3186                                        mem_ctx, &tickles->conn[i].src),
3187                                ntohs(tickles->conn[i].src.ip.sin_port),
3188                                ctdb_sock_addr_to_string(
3189                                        mem_ctx, &tickles->conn[i].dst),
3190                                ntohs(tickles->conn[i].dst.ip.sin_port));
3191                 }
3192         }
3193
3194         talloc_free(tickles);
3195         return 0;
3196 }
3197
3198 typedef void (*clist_request_func)(struct ctdb_req_control *request,
3199                                    struct ctdb_connection *conn);
3200
3201 typedef int (*clist_reply_func)(struct ctdb_reply_control *reply);
3202
3203 struct process_clist_state {
3204         struct ctdb_connection *clist;
3205         int count;
3206         int num_failed, num_total;
3207         clist_reply_func reply_func;
3208 };
3209
3210 static void process_clist_done(struct tevent_req *subreq);
3211
3212 static struct tevent_req *process_clist_send(
3213                                         TALLOC_CTX *mem_ctx,
3214                                         struct ctdb_context *ctdb,
3215                                         struct ctdb_connection *clist,
3216                                         int count,
3217                                         clist_request_func request_func,
3218                                         clist_reply_func reply_func)
3219 {
3220         struct tevent_req *req, *subreq;
3221         struct process_clist_state *state;
3222         struct ctdb_req_control request;
3223         int i;
3224
3225         req = tevent_req_create(mem_ctx, &state, struct process_clist_state);
3226         if (req == NULL) {
3227                 return NULL;
3228         }
3229
3230         state->clist = clist;
3231         state->count = count;
3232         state->reply_func = reply_func;
3233
3234         for (i=0; i<count; i++) {
3235                 request_func(&request, &clist[i]);
3236                 subreq = ctdb_client_control_send(state, ctdb->ev,
3237                                                   ctdb->client, ctdb->cmd_pnn,
3238                                                   TIMEOUT(), &request);
3239                 if (tevent_req_nomem(subreq, req)) {
3240                         return tevent_req_post(req, ctdb->ev);
3241                 }
3242                 tevent_req_set_callback(subreq, process_clist_done, req);
3243         }
3244
3245         return req;
3246 }
3247
3248 static void process_clist_done(struct tevent_req *subreq)
3249 {
3250         struct tevent_req *req = tevent_req_callback_data(
3251                 subreq, struct tevent_req);
3252         struct process_clist_state *state = tevent_req_data(
3253                 req, struct process_clist_state);
3254         struct ctdb_reply_control *reply;
3255         int ret;
3256         bool status;
3257
3258         status = ctdb_client_control_recv(subreq, NULL, state, &reply);
3259         TALLOC_FREE(subreq);
3260         if (! status) {
3261                 state->num_failed += 1;
3262                 goto done;
3263         }
3264
3265         ret = state->reply_func(reply);
3266         if (ret != 0) {
3267                 state->num_failed += 1;
3268                 goto done;
3269         }
3270
3271 done:
3272         state->num_total += 1;
3273         if (state->num_total == state->count) {
3274                 tevent_req_done(req);
3275         }
3276 }
3277
3278 static int process_clist_recv(struct tevent_req *req)
3279 {
3280         struct process_clist_state *state = tevent_req_data(
3281                 req, struct process_clist_state);
3282
3283         return state->num_failed;
3284 }
3285
3286 static int control_addtickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3287                              int argc, const char **argv)
3288 {
3289         struct ctdb_connection conn;
3290         int ret;
3291
3292         if (argc != 0 && argc != 2) {
3293                 usage("addtickle");
3294         }
3295
3296         if (argc == 0) {
3297                 struct ctdb_connection *clist;
3298                 struct tevent_req *req;
3299                 int count;
3300
3301                 ret = ctdb_parse_connections(stdin, mem_ctx, &count, &clist);
3302                 if (ret != 0) {
3303                         return ret;
3304                 }
3305                 if (count == 0) {
3306                         return 0;
3307                 }
3308
3309                 req = process_clist_send(mem_ctx, ctdb, clist, count,
3310                                  ctdb_req_control_tcp_add_delayed_update,
3311                                  ctdb_reply_control_tcp_add_delayed_update);
3312                 if (req == NULL) {
3313                         talloc_free(clist);
3314                         return ENOMEM;
3315                 }
3316
3317                 tevent_req_poll(req, ctdb->ev);
3318                 talloc_free(clist);
3319
3320                 ret = process_clist_recv(req);
3321                 if (ret != 0) {
3322                         fprintf(stderr, "Failed to add %d tickles\n", ret);
3323                         return 1;
3324                 }
3325
3326                 return 0;
3327         }
3328
3329         if (! parse_ip_port(argv[0], &conn.src)) {
3330                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3331                 return 1;
3332         }
3333         if (! parse_ip_port(argv[1], &conn.dst)) {
3334                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3335                 return 1;
3336         }
3337
3338         ret = ctdb_ctrl_tcp_add_delayed_update(mem_ctx, ctdb->ev,
3339                                                ctdb->client, ctdb->cmd_pnn,
3340                                                TIMEOUT(), &conn);
3341         if (ret != 0) {
3342                 fprintf(stderr, "Failed to register connection\n");
3343                 return ret;
3344         }
3345
3346         return 0;
3347 }
3348
3349 static int control_deltickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3350                              int argc, const char **argv)
3351 {
3352         struct ctdb_connection conn;
3353         int ret;
3354
3355         if (argc != 0 && argc != 2) {
3356                 usage("deltickle");
3357         }
3358
3359         if (argc == 0) {
3360                 struct ctdb_connection *clist;
3361                 struct tevent_req *req;
3362                 int count;
3363
3364                 ret = ctdb_parse_connections(stdin, mem_ctx, &count, &clist);
3365                 if (ret != 0) {
3366                         return ret;
3367                 }
3368                 if (count == 0) {
3369                         return 0;
3370                 }
3371
3372                 req = process_clist_send(mem_ctx, ctdb, clist, count,
3373                                          ctdb_req_control_tcp_remove,
3374                                          ctdb_reply_control_tcp_remove);
3375                 if (req == NULL) {
3376                         talloc_free(clist);
3377                         return ENOMEM;
3378                 }
3379
3380                 tevent_req_poll(req, ctdb->ev);
3381                 talloc_free(clist);
3382
3383                 ret = process_clist_recv(req);
3384                 if (ret != 0) {
3385                         fprintf(stderr, "Failed to remove %d tickles\n", ret);
3386                         return 1;
3387                 }
3388
3389                 return 0;
3390         }
3391
3392         if (! parse_ip_port(argv[0], &conn.src)) {
3393                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3394                 return 1;
3395         }
3396         if (! parse_ip_port(argv[1], &conn.dst)) {
3397                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3398                 return 1;
3399         }
3400
3401         ret = ctdb_ctrl_tcp_remove(mem_ctx, ctdb->ev, ctdb->client,
3402                                    ctdb->cmd_pnn, TIMEOUT(), &conn);
3403         if (ret != 0) {
3404                 fprintf(stderr, "Failed to unregister connection\n");
3405                 return ret;
3406         }
3407
3408         return 0;
3409 }
3410
3411 static int control_check_srvids(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3412                                 int argc, const char **argv)
3413 {
3414         uint64_t *srvid;
3415         uint8_t *result;
3416         int ret, i;
3417
3418         if (argc == 0) {
3419                 usage("check_srvids");
3420         }
3421
3422         srvid = talloc_array(mem_ctx, uint64_t, argc);
3423         if (srvid == NULL) {
3424                 fprintf(stderr, "Memory allocation error\n");
3425                 return 1;
3426         }
3427
3428         for (i=0; i<argc; i++) {
3429                 srvid[i] = strtoull(argv[i], NULL, 0);
3430         }
3431
3432         ret = ctdb_ctrl_check_srvids(mem_ctx, ctdb->ev, ctdb->client,
3433                                      ctdb->cmd_pnn, TIMEOUT(), srvid, argc,
3434                                      &result);
3435         if (ret != 0) {
3436                 fprintf(stderr, "Failed to check srvids on node %u\n",
3437                         ctdb->cmd_pnn);
3438                 return ret;
3439         }
3440
3441         for (i=0; i<argc; i++) {
3442                 printf("SRVID 0x%" PRIx64 " %s\n", srvid[i],
3443                        (result[i] ? "exists" : "does not exist"));
3444         }
3445
3446         return 0;
3447 }
3448
3449 static int control_listnodes(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3450                              int argc, const char **argv)
3451 {
3452         struct ctdb_node_map *nodemap;
3453         int i;
3454
3455         if (argc != 0) {
3456                 usage("listnodes");
3457         }
3458
3459         nodemap = read_nodes_file(mem_ctx, CTDB_UNKNOWN_PNN);
3460         if (nodemap == NULL) {
3461                 return 1;
3462         }
3463
3464         for (i=0; i<nodemap->num; i++) {
3465                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
3466                         continue;
3467                 }
3468
3469                 if (options.machinereadable) {
3470                         printf("%s%u%s%s%s\n", options.sep,
3471                                nodemap->node[i].pnn, options.sep,
3472                                ctdb_sock_addr_to_string(
3473                                         mem_ctx, &nodemap->node[i].addr),
3474                                options.sep);
3475                 } else {
3476                         printf("%s\n",
3477                                ctdb_sock_addr_to_string(
3478                                         mem_ctx, &nodemap->node[i].addr));
3479                 }
3480         }
3481
3482         return 0;
3483 }
3484
3485 static bool nodemap_identical(struct ctdb_node_map *nodemap1,
3486                               struct ctdb_node_map *nodemap2)
3487 {
3488         int i;
3489
3490         if (nodemap1->num != nodemap2->num) {
3491                 return false;
3492         }
3493
3494         for (i=0; i<nodemap1->num; i++) {
3495                 struct ctdb_node_and_flags *n1, *n2;
3496
3497                 n1 = &nodemap1->node[i];
3498                 n2 = &nodemap2->node[i];
3499
3500                 if ((n1->pnn != n2->pnn) ||
3501                     (n1->flags != n2->flags) ||
3502                     ! ctdb_sock_addr_same_ip(&n1->addr, &n2->addr)) {
3503                         return false;
3504                 }
3505         }
3506
3507         return true;
3508 }
3509
3510 static int check_node_file_changes(TALLOC_CTX *mem_ctx,
3511                                    struct ctdb_node_map *nm,
3512                                    struct ctdb_node_map *fnm,
3513                                    bool *reload)
3514 {
3515         int i;
3516         bool check_failed = false;
3517
3518         *reload = false;
3519
3520         for (i=0; i<nm->num; i++) {
3521                 if (i >= fnm->num) {
3522                         fprintf(stderr,
3523                                 "Node %u (%s) missing from nodes file\n",
3524                                 nm->node[i].pnn,
3525                                 ctdb_sock_addr_to_string(
3526                                         mem_ctx, &nm->node[i].addr));
3527                         check_failed = true;
3528                         continue;
3529                 }
3530                 if (nm->node[i].flags & NODE_FLAGS_DELETED &&
3531                     fnm->node[i].flags & NODE_FLAGS_DELETED) {
3532                         /* Node remains deleted */
3533                         continue;
3534                 }
3535
3536                 if (! (nm->node[i].flags & NODE_FLAGS_DELETED) &&
3537                     ! (fnm->node[i].flags & NODE_FLAGS_DELETED)) {
3538                         /* Node not newly nor previously deleted */
3539                         if (! ctdb_same_ip(&nm->node[i].addr,
3540                                            &fnm->node[i].addr)) {
3541                                 fprintf(stderr,
3542                                         "Node %u has changed IP address"
3543                                         " (was %s, now %s)\n",
3544                                         nm->node[i].pnn,
3545                                         ctdb_sock_addr_to_string(
3546                                                 mem_ctx, &nm->node[i].addr),
3547                                         ctdb_sock_addr_to_string(
3548                                                 mem_ctx, &fnm->node[i].addr));
3549                                 check_failed = true;
3550                         } else {
3551                                 if (nm->node[i].flags & NODE_FLAGS_DISCONNECTED) {
3552                                         fprintf(stderr,
3553                                                 "WARNING: Node %u is disconnected."
3554                                                 " You MUST fix this node manually!\n",
3555                                                 nm->node[i].pnn);
3556                                 }
3557                         }
3558                         continue;
3559                 }
3560
3561                 if (fnm->node[i].flags & NODE_FLAGS_DELETED) {
3562                         /* Node is being deleted */
3563                         printf("Node %u is DELETED\n", nm->node[i].pnn);
3564                         *reload = true;
3565                         if (! (nm->node[i].flags & NODE_FLAGS_DISCONNECTED)) {
3566                                 fprintf(stderr,
3567                                         "ERROR: Node %u is still connected\n",
3568                                         nm->node[i].pnn);
3569                                 check_failed = true;
3570                         }
3571                         continue;
3572                 }
3573
3574                 if (nm->node[i].flags & NODE_FLAGS_DELETED) {
3575                         /* Node was previously deleted */
3576                         printf("Node %u is UNDELETED\n", nm->node[i].pnn);
3577                         *reload = true;
3578                 }
3579         }
3580
3581         if (check_failed) {
3582                 fprintf(stderr,
3583                         "ERROR: Nodes will not be reloaded due to previous error\n");
3584                 return 1;
3585         }
3586
3587         /* Leftover nodes in file are NEW */
3588         for (; i < fnm->num; i++) {
3589                 printf("Node %u is NEW\n", fnm->node[i].pnn);
3590                 *reload = true;
3591         }
3592
3593         return 0;
3594 }
3595
3596 struct disable_recoveries_state {
3597         uint32_t *pnn_list;
3598         int node_count;
3599         bool *reply;
3600         int status;
3601         bool done;
3602 };
3603
3604 static void disable_recoveries_handler(uint64_t srvid, TDB_DATA data,
3605                                        void *private_data)
3606 {
3607         struct disable_recoveries_state *state =
3608                 (struct disable_recoveries_state *)private_data;
3609         int ret, i;
3610
3611         if (data.dsize != sizeof(int)) {
3612                 /* Ignore packet */
3613                 return;
3614         }
3615
3616         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
3617         ret = *(int *)data.dptr;
3618         if (ret < 0) {
3619                 state->status = ret;
3620                 state->done = true;
3621                 return;
3622         }
3623         for (i=0; i<state->node_count; i++) {
3624                 if (state->pnn_list[i] == ret) {
3625                         state->reply[i] = true;
3626                         break;
3627                 }
3628         }
3629
3630         state->done = true;
3631         for (i=0; i<state->node_count; i++) {
3632                 if (! state->reply[i]) {
3633                         state->done = false;
3634                         break;
3635                 }
3636         }
3637 }
3638
3639 static int disable_recoveries(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3640                               uint32_t timeout, uint32_t *pnn_list, int count)
3641 {
3642         struct ctdb_disable_message disable = { 0 };
3643         struct disable_recoveries_state state;
3644         int ret, i;
3645
3646         disable.pnn = ctdb->pnn;
3647         disable.srvid = next_srvid(ctdb);
3648         disable.timeout = timeout;
3649
3650         state.pnn_list = pnn_list;
3651         state.node_count = count;
3652         state.done = false;
3653         state.status = 0;
3654         state.reply = talloc_zero_array(mem_ctx, bool, count);
3655         if (state.reply == NULL) {
3656                 return ENOMEM;
3657         }
3658
3659         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
3660                                               disable.srvid,
3661                                               disable_recoveries_handler,
3662                                               &state);
3663         if (ret != 0) {
3664                 return ret;
3665         }
3666
3667         for (i=0; i<count; i++) {
3668                 ret = ctdb_message_disable_recoveries(mem_ctx, ctdb->ev,
3669                                                       ctdb->client,
3670                                                       pnn_list[i],
3671                                                       &disable);
3672                 if (ret != 0) {
3673                         goto fail;
3674                 }
3675         }
3676
3677         ret = ctdb_client_wait_timeout(ctdb->ev, &state.done, TIMEOUT());
3678         if (ret == ETIME) {
3679                 fprintf(stderr, "Timed out waiting to disable recoveries\n");
3680         } else {
3681                 ret = (state.status >= 0 ? 0 : 1);
3682         }
3683
3684 fail:
3685         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
3686                                            disable.srvid, &state);
3687         return ret;
3688 }
3689
3690 static int control_reloadnodes(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3691                                int argc, const char **argv)
3692 {
3693         struct ctdb_node_map *nodemap = NULL;
3694         struct ctdb_node_map *file_nodemap;
3695         struct ctdb_node_map *remote_nodemap;
3696         struct ctdb_req_control request;
3697         struct ctdb_reply_control **reply;
3698         bool reload;
3699         int ret, i;
3700         uint32_t *pnn_list;
3701         int count;
3702
3703         nodemap = get_nodemap(ctdb, false);
3704         if (nodemap == NULL) {
3705                 return 1;
3706         }
3707
3708         file_nodemap = read_nodes_file(mem_ctx, ctdb->pnn);
3709         if (file_nodemap == NULL) {
3710                 return 1;
3711         }
3712
3713         for (i=0; i<nodemap->num; i++) {
3714                 if (nodemap->node[i].flags & NODE_FLAGS_DISCONNECTED) {
3715                         continue;
3716                 }
3717
3718                 ret = ctdb_ctrl_get_nodes_file(mem_ctx, ctdb->ev, ctdb->client,
3719                                                nodemap->node[i].pnn, TIMEOUT(),
3720                                                &remote_nodemap);
3721                 if (ret != 0) {
3722                         fprintf(stderr,
3723                                 "ERROR: Failed to get nodes file from node %u\n",
3724                                 nodemap->node[i].pnn);
3725                         return ret;
3726                 }
3727
3728                 if (! nodemap_identical(file_nodemap, remote_nodemap)) {
3729                         fprintf(stderr,
3730                                 "ERROR: Nodes file on node %u differs"
3731                                  " from current node (%u)\n",
3732                                  nodemap->node[i].pnn, ctdb->pnn);
3733                         return 1;
3734                 }
3735         }
3736
3737         ret = check_node_file_changes(mem_ctx, nodemap, file_nodemap, &reload);
3738         if (ret != 0) {
3739                 return ret;
3740         }
3741
3742         if (! reload) {
3743                 fprintf(stderr, "No change in nodes file,"
3744                                 " skipping unnecessary reload\n");
3745                 return 0;
3746         }
3747
3748         count = list_of_connected_nodes(nodemap, CTDB_UNKNOWN_PNN,
3749                                         mem_ctx, &pnn_list);
3750         if (count <= 0) {
3751                 fprintf(stderr, "Memory allocation error\n");
3752                 return 1;
3753         }
3754
3755         ret = disable_recoveries(mem_ctx, ctdb, 2*options.timelimit,
3756                                  pnn_list, count);
3757         if (ret != 0) {
3758                 fprintf(stderr, "Failed to disable recoveries\n");
3759                 return ret;
3760         }
3761
3762         ctdb_req_control_reload_nodes_file(&request);
3763         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
3764                                         pnn_list, count, TIMEOUT(),
3765                                         &request, NULL, &reply);
3766         if (ret != 0) {
3767                 bool failed = false;
3768
3769                 for (i=0; i<count; i++) {
3770                         ret = ctdb_reply_control_reload_nodes_file(reply[i]);
3771                         if (ret != 0) {
3772                                 fprintf(stderr,
3773                                         "Node %u failed to reload nodes\n",
3774                                         pnn_list[i]);
3775                                 failed = true;
3776                         }
3777                 }
3778                 if (failed) {
3779                         fprintf(stderr,
3780                                 "You MUST fix failed nodes manually!\n");
3781                 }
3782         }
3783
3784         ret = disable_recoveries(mem_ctx, ctdb, 0, pnn_list, count);
3785         if (ret != 0) {
3786                 fprintf(stderr, "Failed to enable recoveries\n");
3787                 return ret;
3788         }
3789
3790         return 0;
3791 }
3792
3793 static int moveip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3794                   ctdb_sock_addr *addr, uint32_t pnn)
3795 {
3796         struct ctdb_public_ip_list *pubip_list;
3797         struct ctdb_public_ip pubip;
3798         struct ctdb_node_map *nodemap;
3799         struct ctdb_req_control request;
3800         uint32_t *pnn_list;
3801         int ret, i, count;
3802
3803         ret = ctdb_message_disable_ip_check(mem_ctx, ctdb->ev, ctdb->client,
3804                                             CTDB_BROADCAST_CONNECTED,
3805                                             2*options.timelimit);
3806         if (ret != 0) {
3807                 fprintf(stderr, "Failed to disable IP check\n");
3808                 return ret;
3809         }
3810
3811         ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
3812                                        pnn, TIMEOUT(), &pubip_list);
3813         if (ret != 0) {
3814                 fprintf(stderr, "Failed to get Public IPs from node %u\n",
3815                         pnn);
3816                 return ret;
3817         }
3818
3819         for (i=0; i<pubip_list->num; i++) {
3820                 if (ctdb_same_ip(addr, &pubip_list->ip[i].addr)) {
3821                         break;
3822                 }
3823         }
3824
3825         if (i == pubip_list->num) {
3826                 fprintf(stderr, "Node %u CANNOT host IP address %s\n",
3827                         pnn, ctdb_sock_addr_to_string(mem_ctx, addr));
3828                 return 1;
3829         }
3830
3831         nodemap = get_nodemap(ctdb, false);
3832         if (nodemap == NULL) {
3833                 return 1;
3834         }
3835
3836         count = list_of_active_nodes(nodemap, pnn, mem_ctx, &pnn_list);
3837         if (count <= 0) {
3838                 fprintf(stderr, "Memory allocation error\n");
3839                 return 1;
3840         }
3841
3842         pubip.pnn = pnn;
3843         pubip.addr = *addr;
3844         ctdb_req_control_release_ip(&request, &pubip);
3845
3846         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
3847                                         pnn_list, count, TIMEOUT(),
3848                                         &request, NULL, NULL);
3849         if (ret != 0) {
3850                 fprintf(stderr, "Failed to release IP on nodes\n");
3851                 return ret;
3852         }
3853
3854         ret = ctdb_ctrl_takeover_ip(mem_ctx, ctdb->ev, ctdb->client,
3855                                     pnn, TIMEOUT(), &pubip);
3856         if (ret != 0) {
3857                 fprintf(stderr, "Failed to takeover IP on node %u\n", pnn);
3858                 return ret;
3859         }
3860
3861         return 0;
3862 }
3863
3864 static int control_moveip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3865                           int argc, const char **argv)
3866 {
3867         ctdb_sock_addr addr;
3868         uint32_t pnn;
3869         int ret, retries = 0;
3870
3871         if (argc != 2) {
3872                 usage("moveip");
3873         }
3874
3875         if (! parse_ip(argv[0], NULL, 0, &addr)) {
3876                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3877                 return 1;
3878         }
3879
3880         pnn = strtoul(argv[1], NULL, 10);
3881         if (pnn == CTDB_UNKNOWN_PNN) {
3882                 fprintf(stderr, "Invalid PNN %s\n", argv[1]);
3883                 return 1;
3884         }
3885
3886         while (retries < 5) {
3887                 ret = moveip(mem_ctx, ctdb, &addr, pnn);
3888                 if (ret == 0) {
3889                         break;
3890                 }
3891
3892                 sleep(3);
3893                 retries++;
3894         }
3895
3896         if (ret != 0) {
3897                 fprintf(stderr, "Failed to move IP %s to node %u\n",
3898                         argv[0], pnn);
3899                 return ret;
3900         }
3901
3902         return 0;
3903 }
3904
3905 static int rebalancenode(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3906                          uint32_t pnn)
3907 {
3908         int ret;
3909
3910         ret = ctdb_message_rebalance_node(mem_ctx, ctdb->ev, ctdb->client,
3911                                           CTDB_BROADCAST_CONNECTED, pnn);
3912         if (ret != 0) {
3913                 fprintf(stderr,
3914                         "Failed to ask recovery master to distribute IPs\n");
3915                 return ret;
3916         }
3917
3918         return 0;
3919 }
3920
3921 static int control_addip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3922                          int argc, const char **argv)
3923 {
3924         ctdb_sock_addr addr;
3925         struct ctdb_public_ip_list *pubip_list;
3926         struct ctdb_addr_info addr_info;
3927         unsigned int mask;
3928         int ret, i, retries = 0;
3929
3930         if (argc != 2) {
3931                 usage("addip");
3932         }
3933
3934         if (! parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
3935                 fprintf(stderr, "Invalid IP/Mask %s\n", argv[0]);
3936                 return 1;
3937         }
3938
3939         ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
3940                                        ctdb->cmd_pnn, TIMEOUT(), &pubip_list);
3941         if (ret != 0) {
3942                 fprintf(stderr, "Failed to get Public IPs from node %u\n",
3943                         ctdb->cmd_pnn);
3944                 return 1;
3945         }
3946
3947         for (i=0; i<pubip_list->num; i++) {
3948                 if (ctdb_same_ip(&addr, &pubip_list->ip[i].addr)) {
3949                         fprintf(stderr, "Node already knows about IP %s\n",
3950                                 ctdb_sock_addr_to_string(mem_ctx, &addr));
3951                         return 0;
3952                 }
3953         }
3954
3955         addr_info.addr = addr;
3956         addr_info.mask = mask;
3957         addr_info.iface = argv[1];
3958
3959         while (retries < 5) {
3960                 ret = ctdb_ctrl_add_public_ip(mem_ctx, ctdb->ev, ctdb->client,
3961                                               ctdb->cmd_pnn, TIMEOUT(),
3962                                               &addr_info);
3963                 if (ret == 0) {
3964                         break;
3965                 }
3966
3967                 sleep(3);
3968                 retries++;
3969         }
3970
3971         if (ret != 0) {
3972                 fprintf(stderr, "Failed to add public IP to node %u."
3973                                 " Giving up\n", ctdb->cmd_pnn);
3974                 return ret;
3975         }
3976
3977         ret = rebalancenode(mem_ctx, ctdb, ctdb->cmd_pnn);
3978         if (ret != 0) {
3979                 return ret;
3980         }
3981
3982         return 0;
3983 }
3984
3985 static int control_delip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3986                          int argc, const char **argv)
3987 {
3988         ctdb_sock_addr addr;
3989         struct ctdb_public_ip_list *pubip_list;
3990         struct ctdb_addr_info addr_info;
3991         int ret, i;
3992
3993         if (argc != 1) {
3994                 usage("delip");
3995         }
3996
3997         if (! parse_ip(argv[0], NULL, 0, &addr)) {
3998                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3999                 return 1;
4000         }
4001
4002         ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
4003                                        ctdb->cmd_pnn, TIMEOUT(), &pubip_list);
4004         if (ret != 0) {
4005                 fprintf(stderr, "Failed to get Public IPs from node %u\n",
4006                         ctdb->cmd_pnn);
4007                 return 1;
4008         }
4009
4010         for (i=0; i<pubip_list->num; i++) {
4011                 if (ctdb_same_ip(&addr, &pubip_list->ip[i].addr)) {
4012                         break;
4013                 }
4014         }
4015
4016         if (i == pubip_list->num) {
4017                 fprintf(stderr, "Node does not know about IP address %s\n",
4018                         ctdb_sock_addr_to_string(mem_ctx, &addr));
4019                 return 0;
4020         }
4021
4022         addr_info.addr = addr;
4023         addr_info.mask = 0;
4024         addr_info.iface = NULL;
4025
4026         ret = ctdb_ctrl_del_public_ip(mem_ctx, ctdb->ev, ctdb->client,
4027                                       ctdb->cmd_pnn, TIMEOUT(), &addr_info);
4028         if (ret != 0) {
4029                 fprintf(stderr, "Failed to delete public IP from node %u\n",
4030                         ctdb->cmd_pnn);
4031                 return ret;
4032         }
4033
4034         return 0;
4035 }
4036
4037 static int control_eventscript(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4038                                int argc, const char **argv)
4039 {
4040         int ret;
4041
4042         if (argc != 1) {
4043                 usage("eventscript");
4044         }
4045
4046         if (strcmp(argv[0], "monitor") != 0) {
4047                 fprintf(stderr, "Only monitor event can be run\n");
4048                 return 1;
4049         }
4050
4051         ret = ctdb_ctrl_run_eventscripts(mem_ctx, ctdb->ev, ctdb->client,
4052                                          ctdb->cmd_pnn, TIMEOUT(), argv[0]);
4053         if (ret != 0) {
4054                 fprintf(stderr, "Failed to run monitor event on node %u\n",
4055                         ctdb->cmd_pnn);
4056                 return ret;
4057         }
4058
4059         return 0;
4060 }
4061
4062 #define DB_VERSION      3
4063 #define MAX_DB_NAME     64
4064 #define MAX_REC_BUFFER_SIZE     (100*1000)
4065
4066 struct db_header {
4067         unsigned long version;
4068         time_t timestamp;
4069         unsigned long flags;
4070         unsigned long nbuf;
4071         unsigned long nrec;
4072         char name[MAX_DB_NAME];
4073 };
4074
4075 struct backup_state {
4076         TALLOC_CTX *mem_ctx;
4077         struct ctdb_rec_buffer *recbuf;
4078         uint32_t db_id;
4079         int fd;
4080         unsigned int nbuf, nrec;
4081 };
4082
4083 static int backup_handler(uint32_t reqid, struct ctdb_ltdb_header *header,
4084                           TDB_DATA key, TDB_DATA data, void *private_data)
4085 {
4086         struct backup_state *state = (struct backup_state *)private_data;
4087         size_t len;
4088         int ret;
4089
4090         if (state->recbuf == NULL) {
4091                 state->recbuf = ctdb_rec_buffer_init(state->mem_ctx,
4092                                                      state->db_id);
4093                 if (state->recbuf == NULL) {
4094                         return ENOMEM;
4095                 }
4096         }
4097
4098         ret = ctdb_rec_buffer_add(state->recbuf, state->recbuf, reqid,
4099                                   header, key, data);
4100         if (ret != 0) {
4101                 return ret;
4102         }
4103
4104         len = ctdb_rec_buffer_len(state->recbuf);
4105         if (len < MAX_REC_BUFFER_SIZE) {
4106                 return 0;
4107         }
4108
4109         ret = ctdb_rec_buffer_write(state->recbuf, state->fd);
4110         if (ret != 0) {
4111                 fprintf(stderr, "Failed to write records to backup file\n");
4112                 return ret;
4113         }
4114
4115         state->nbuf += 1;
4116         state->nrec += state->recbuf->count;
4117         TALLOC_FREE(state->recbuf);
4118
4119         return 0;
4120 }
4121
4122 static int control_backupdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4123                             int argc, const char **argv)
4124 {
4125         const char *db_name;
4126         struct ctdb_db_context *db;
4127         uint32_t db_id;
4128         uint8_t db_flags;
4129         struct backup_state state;
4130         struct db_header db_hdr;
4131         int fd, ret;
4132
4133         if (argc != 2) {
4134                 usage("backupdb");
4135         }
4136
4137         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
4138                 return 1;
4139         }
4140
4141         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
4142                           db_flags, &db);
4143         if (ret != 0) {
4144                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
4145                 return ret;
4146         }
4147
4148         fd = open(argv[1], O_RDWR|O_CREAT, 0600);
4149         if (fd == -1) {
4150                 ret = errno;
4151                 fprintf(stderr, "Failed to open file %s for writing\n",
4152                         argv[1]);
4153                 return ret;
4154         }
4155
4156         /* Write empty header first */
4157         ZERO_STRUCT(db_hdr);
4158         ret = write(fd, &db_hdr, sizeof(struct db_header));
4159         if (ret == -1) {
4160                 ret = errno;
4161                 close(fd);
4162                 fprintf(stderr, "Failed to write header to file %s\n", argv[1]);
4163                 return ret;
4164         }
4165
4166         state.mem_ctx = mem_ctx;
4167         state.recbuf = NULL;
4168         state.fd = fd;
4169         state.nbuf = 0;
4170         state.nrec = 0;
4171
4172         ret = ctdb_db_traverse(db, true, false, backup_handler, &state);
4173         if (ret != 0) {
4174                 fprintf(stderr, "Failed to collect records from DB %s\n",
4175                         db_name);
4176                 close(fd);
4177                 return ret;
4178         }
4179
4180         if (state.recbuf != NULL) {
4181                 ret = ctdb_rec_buffer_write(state.recbuf, state.fd);
4182                 if (ret != 0) {
4183                         fprintf(stderr,
4184                                 "Failed to write records to backup file\n");
4185                         close(fd);
4186                         return ret;
4187                 }
4188
4189                 state.nbuf += 1;
4190                 state.nrec += state.recbuf->count;
4191                 TALLOC_FREE(state.recbuf);
4192         }
4193
4194         db_hdr.version = DB_VERSION;
4195         db_hdr.timestamp = time(NULL);
4196         db_hdr.flags = db_flags;
4197         db_hdr.nbuf = state.nbuf;
4198         db_hdr.nrec = state.nrec;
4199         strncpy(db_hdr.name, db_name, MAX_DB_NAME-1);
4200
4201         lseek(fd, 0, SEEK_SET);
4202         ret = write(fd, &db_hdr, sizeof(struct db_header));
4203         if (ret == -1) {
4204                 ret = errno;
4205                 close(fd);
4206                 fprintf(stderr, "Failed to write header to file %s\n", argv[1]);
4207                 return ret;
4208         }
4209
4210         close(fd);
4211         printf("Database backed up to %s\n", argv[1]);
4212         return 0;
4213 }
4214
4215 static int control_restoredb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4216                              int argc, const char **argv)
4217 {
4218         const char *db_name = NULL;
4219         struct ctdb_db_context *db;
4220         struct db_header db_hdr;
4221         struct ctdb_node_map *nodemap;
4222         struct ctdb_req_control request;
4223         struct ctdb_reply_control **reply;
4224         struct ctdb_transdb wipedb;
4225         struct ctdb_pulldb_ext pulldb;
4226         struct ctdb_rec_buffer *recbuf;
4227         uint32_t generation;
4228         uint32_t *pnn_list;
4229         char timebuf[128];
4230         int fd, i;
4231         int count, ret;
4232
4233         if (argc < 1 || argc > 2) {
4234                 usage("restoredb");
4235         }
4236
4237         fd = open(argv[0], O_RDONLY, 0600);
4238         if (fd == -1) {
4239                 ret = errno;
4240                 fprintf(stderr, "Failed to open file %s for reading\n",
4241                         argv[0]);
4242                 return ret;
4243         }
4244
4245         if (argc == 2) {
4246                 db_name = argv[1];
4247         }
4248
4249         ret = read(fd, &db_hdr, sizeof(struct db_header));
4250         if (ret == -1) {
4251                 ret = errno;
4252                 close(fd);
4253                 fprintf(stderr, "Failed to read db header from file %s\n",
4254                         argv[0]);
4255                 return ret;
4256         }
4257         db_hdr.name[sizeof(db_hdr.name)-1] = '\0';
4258
4259         if (db_hdr.version != DB_VERSION) {
4260                 fprintf(stderr,
4261                         "Wrong version of backup file, expected %u, got %lu\n",
4262                         DB_VERSION, db_hdr.version);
4263                 close(fd);
4264                 return EINVAL;
4265         }
4266
4267         if (db_name == NULL) {
4268                 db_name = db_hdr.name;
4269         }
4270
4271         strftime(timebuf, sizeof(timebuf)-1, "%Y/%m/%d %H:%M:%S",
4272                  localtime(&db_hdr.timestamp));
4273         printf("Restoring database %s from backup @ %s\n", db_name, timebuf);
4274
4275         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
4276                           db_hdr.flags, &db);
4277         if (ret != 0) {
4278                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
4279                 close(fd);
4280                 return ret;
4281         }
4282
4283         nodemap = get_nodemap(ctdb, false);
4284         if (nodemap == NULL) {
4285                 fprintf(stderr, "Failed to get nodemap\n");
4286                 close(fd);
4287                 return ENOMEM;
4288         }
4289
4290         ret = get_generation(mem_ctx, ctdb, &generation);
4291         if (ret != 0) {
4292                 fprintf(stderr, "Failed to get current generation\n");
4293                 close(fd);
4294                 return ret;
4295         }
4296
4297         count = list_of_active_nodes(nodemap, CTDB_UNKNOWN_PNN, mem_ctx,
4298                                      &pnn_list);
4299         if (count <= 0) {
4300                 close(fd);
4301                 return ENOMEM;
4302         }
4303
4304         wipedb.db_id = ctdb_db_id(db);
4305         wipedb.tid = generation;
4306
4307         ctdb_req_control_db_freeze(&request, wipedb.db_id);
4308         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4309                                         ctdb->client, pnn_list, count,
4310                                         TIMEOUT(), &request, NULL, NULL);
4311         if (ret != 0) {
4312                 goto failed;
4313         }
4314
4315
4316         ctdb_req_control_db_transaction_start(&request, &wipedb);
4317         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4318                                         pnn_list, count, TIMEOUT(),
4319                                         &request, NULL, NULL);
4320         if (ret != 0) {
4321                 goto failed;
4322         }
4323
4324         ctdb_req_control_wipe_database(&request, &wipedb);
4325         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4326                                         pnn_list, count, TIMEOUT(),
4327                                         &request, NULL, NULL);
4328         if (ret != 0) {
4329                 goto failed;
4330         }
4331
4332         pulldb.db_id = ctdb_db_id(db);
4333         pulldb.lmaster = 0;
4334         pulldb.srvid = SRVID_CTDB_PUSHDB;
4335
4336         ctdb_req_control_db_push_start(&request, &pulldb);
4337         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4338                                         pnn_list, count, TIMEOUT(),
4339                                         &request, NULL, NULL);
4340         if (ret != 0) {
4341                 goto failed;
4342         }
4343
4344         for (i=0; i<db_hdr.nbuf; i++) {
4345                 struct ctdb_req_message message;
4346                 TDB_DATA data;
4347
4348                 ret = ctdb_rec_buffer_read(fd, mem_ctx, &recbuf);
4349                 if (ret != 0) {
4350                         goto failed;
4351                 }
4352
4353                 data.dsize = ctdb_rec_buffer_len(recbuf);
4354                 data.dptr = talloc_size(mem_ctx, data.dsize);
4355                 if (data.dptr == NULL) {
4356                         goto failed;
4357                 }
4358
4359                 ctdb_rec_buffer_push(recbuf, data.dptr);
4360
4361                 message.srvid = pulldb.srvid;
4362                 message.data.data = data;
4363
4364                 ret = ctdb_client_message_multi(mem_ctx, ctdb->ev,
4365                                                 ctdb->client,
4366                                                 pnn_list, count,
4367                                                 &message, NULL);
4368                 if (ret != 0) {
4369                         goto failed;
4370                 }
4371
4372                 talloc_free(recbuf);
4373                 talloc_free(data.dptr);
4374         }
4375
4376         ctdb_req_control_db_push_confirm(&request, pulldb.db_id);
4377         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4378                                         pnn_list, count, TIMEOUT(),
4379                                         &request, NULL, &reply);
4380         if (ret != 0) {
4381                 goto failed;
4382         }
4383
4384         for (i=0; i<count; i++) {
4385                 uint32_t num_records;
4386
4387                 ret = ctdb_reply_control_db_push_confirm(reply[i],
4388                                                          &num_records);
4389                 if (ret != 0) {
4390                         fprintf(stderr, "Invalid response from node %u\n",
4391                                 pnn_list[i]);
4392                         goto failed;
4393                 }
4394
4395                 if (num_records != db_hdr.nrec) {
4396                         fprintf(stderr, "Node %u received %u of %lu records\n",
4397                                 pnn_list[i], num_records, db_hdr.nrec);
4398                         goto failed;
4399                 }
4400         }
4401
4402         ctdb_req_control_db_set_healthy(&request, wipedb.db_id);
4403         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4404                                         pnn_list, count, TIMEOUT(),
4405                                         &request, NULL, NULL);
4406         if (ret != 0) {
4407                 goto failed;
4408         }
4409
4410         ctdb_req_control_db_transaction_commit(&request, &wipedb);
4411         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4412                                         pnn_list, count, TIMEOUT(),
4413                                         &request, NULL, NULL);
4414         if (ret != 0) {
4415                 goto failed;
4416         }
4417
4418         ctdb_req_control_db_thaw(&request, wipedb.db_id);
4419         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4420                                         ctdb->client, pnn_list, count,
4421                                         TIMEOUT(), &request, NULL, NULL);
4422         if (ret != 0) {
4423                 goto failed;
4424         }
4425
4426         printf("Database %s restored\n", db_name);
4427         return 0;
4428
4429
4430 failed:
4431         close(fd);
4432         ctdb_ctrl_set_recmode(mem_ctx, ctdb->ev, ctdb->client,
4433                               ctdb->pnn, TIMEOUT(), CTDB_RECOVERY_ACTIVE);
4434         return ret;
4435 }
4436
4437 struct dumpdbbackup_state {
4438         ctdb_rec_parser_func_t parser;
4439         struct dump_record_state sub_state;
4440 };
4441
4442 static int dumpdbbackup_handler(uint32_t reqid,
4443                                 struct ctdb_ltdb_header *header,
4444                                 TDB_DATA key, TDB_DATA data,
4445                                 void *private_data)
4446 {
4447         struct dumpdbbackup_state *state =
4448                 (struct dumpdbbackup_state *)private_data;
4449         struct ctdb_ltdb_header hdr;
4450         int ret;
4451
4452         ret = ctdb_ltdb_header_extract(&data, &hdr);
4453         if (ret != 0) {
4454                 return ret;
4455         }
4456
4457         return state->parser(reqid, &hdr, key, data, &state->sub_state);
4458 }
4459
4460 static int control_dumpdbbackup(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4461                                 int argc, const char **argv)
4462 {
4463         struct db_header db_hdr;
4464         char timebuf[128];
4465         struct dumpdbbackup_state state;
4466         int fd, ret, i;
4467
4468         if (argc != 1) {
4469                 usage("dumpbackup");
4470         }
4471
4472         fd = open(argv[0], O_RDONLY, 0600);
4473         if (fd == -1) {
4474                 ret = errno;
4475                 fprintf(stderr, "Failed to open file %s for reading\n",
4476                         argv[0]);
4477                 return ret;
4478         }
4479
4480         ret = read(fd, &db_hdr, sizeof(struct db_header));
4481         if (ret == -1) {
4482                 ret = errno;
4483                 close(fd);
4484                 fprintf(stderr, "Failed to read db header from file %s\n",
4485                         argv[0]);
4486                 return ret;
4487         }
4488         db_hdr.name[sizeof(db_hdr.name)-1] = '\0';
4489
4490         if (db_hdr.version != DB_VERSION) {
4491                 fprintf(stderr,
4492                         "Wrong version of backup file, expected %u, got %lu\n",
4493                         DB_VERSION, db_hdr.version);
4494                 close(fd);
4495                 return EINVAL;
4496         }
4497
4498         strftime(timebuf, sizeof(timebuf)-1, "%Y/%m/%d %H:%M:%S",
4499                  localtime(&db_hdr.timestamp));
4500         printf("Dumping database %s from backup @ %s\n",
4501                db_hdr.name, timebuf);
4502
4503         state.parser = dump_record;
4504         state.sub_state.count = 0;
4505
4506         for (i=0; i<db_hdr.nbuf; i++) {
4507                 struct ctdb_rec_buffer *recbuf;
4508
4509                 ret = ctdb_rec_buffer_read(fd, mem_ctx, &recbuf);
4510                 if (ret != 0) {
4511                         fprintf(stderr, "Failed to read records\n");
4512                         close(fd);
4513                         return ret;
4514                 }
4515
4516                 ret = ctdb_rec_buffer_traverse(recbuf, dumpdbbackup_handler,
4517                                                &state);
4518                 if (ret != 0) {
4519                         fprintf(stderr, "Failed to dump records\n");
4520                         close(fd);
4521                         return ret;
4522                 }
4523         }
4524
4525         close(fd);
4526         printf("Dumped %u record(s)\n", state.sub_state.count);
4527         return 0;
4528 }
4529
4530 static int control_wipedb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4531                           int argc, const char **argv)
4532 {
4533         const char *db_name;
4534         struct ctdb_db_context *db;
4535         uint32_t db_id;
4536         uint8_t db_flags;
4537         struct ctdb_node_map *nodemap;
4538         struct ctdb_req_control request;
4539         struct ctdb_transdb wipedb;
4540         uint32_t generation;
4541         uint32_t *pnn_list;
4542         int count, ret;
4543
4544         if (argc != 1) {
4545                 usage("wipedb");
4546         }
4547
4548         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
4549                 return 1;
4550         }
4551
4552         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
4553                           db_flags, &db);
4554         if (ret != 0) {
4555                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
4556                 return ret;
4557         }
4558
4559         nodemap = get_nodemap(ctdb, false);
4560         if (nodemap == NULL) {
4561                 fprintf(stderr, "Failed to get nodemap\n");
4562                 return ENOMEM;
4563         }
4564
4565         ret = get_generation(mem_ctx, ctdb, &generation);
4566         if (ret != 0) {
4567                 fprintf(stderr, "Failed to get current generation\n");
4568                 return ret;
4569         }
4570
4571         count = list_of_active_nodes(nodemap, CTDB_UNKNOWN_PNN, mem_ctx,
4572                                      &pnn_list);
4573         if (count <= 0) {
4574                 return ENOMEM;
4575         }
4576
4577         ctdb_req_control_db_freeze(&request, db_id);
4578         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4579                                         ctdb->client, pnn_list, count,
4580                                         TIMEOUT(), &request, NULL, NULL);
4581         if (ret != 0) {
4582                 goto failed;
4583         }
4584
4585         wipedb.db_id = db_id;
4586         wipedb.tid = generation;
4587
4588         ctdb_req_control_db_transaction_start(&request, &wipedb);
4589         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4590                                         pnn_list, count, TIMEOUT(),
4591                                         &request, NULL, NULL);
4592         if (ret != 0) {
4593                 goto failed;
4594         }
4595
4596         ctdb_req_control_wipe_database(&request, &wipedb);
4597         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4598                                         pnn_list, count, TIMEOUT(),
4599                                         &request, NULL, NULL);
4600         if (ret != 0) {
4601                 goto failed;
4602         }
4603
4604         ctdb_req_control_db_set_healthy(&request, db_id);
4605         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4606                                         pnn_list, count, TIMEOUT(),
4607                                         &request, NULL, NULL);
4608         if (ret != 0) {
4609                 goto failed;
4610         }
4611
4612         ctdb_req_control_db_transaction_commit(&request, &wipedb);
4613         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
4614                                         pnn_list, count, TIMEOUT(),
4615                                         &request, NULL, NULL);
4616         if (ret != 0) {
4617                 goto failed;
4618         }
4619
4620         ctdb_req_control_db_thaw(&request, db_id);
4621         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev,
4622                                         ctdb->client, pnn_list, count,
4623                                         TIMEOUT(), &request, NULL, NULL);
4624         if (ret != 0) {
4625                 goto failed;
4626         }
4627
4628         printf("Database %s wiped\n", db_name);
4629         return 0;
4630
4631
4632 failed:
4633         ctdb_ctrl_set_recmode(mem_ctx, ctdb->ev, ctdb->client,
4634                               ctdb->pnn, TIMEOUT(), CTDB_RECOVERY_ACTIVE);
4635         return ret;
4636 }
4637
4638 static int control_recmaster(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4639                              int argc, const char **argv)
4640 {
4641         uint32_t recmaster;
4642         int ret;
4643
4644         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
4645                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
4646         if (ret != 0) {
4647                 return ret;
4648         }
4649
4650         printf("%u\n", recmaster);
4651         return 0;
4652 }
4653
4654 static void print_scriptstatus_one(struct ctdb_script_list *slist,
4655                                    const char *event_str)
4656 {
4657         int i;
4658         int num_run = 0;
4659
4660         if (slist == NULL) {
4661                 if (! options.machinereadable) {
4662                         printf("%s cycle never run\n", event_str);
4663                 }
4664                 return;
4665         }
4666
4667         for (i=0; i<slist->num_scripts; i++) {
4668                 if (slist->script[i].status != -ENOEXEC) {
4669                         num_run++;
4670                 }
4671         }
4672
4673         if (! options.machinereadable) {
4674                 printf("%d scripts were executed last %s cycle\n",
4675                        num_run, event_str);
4676         }
4677
4678         for (i=0; i<slist->num_scripts; i++) {
4679                 const char *status = NULL;
4680
4681                 switch (slist->script[i].status) {
4682                 case -ETIME:
4683                         status = "TIMEDOUT";
4684                         break;
4685                 case -ENOEXEC:
4686                         status = "DISABLED";
4687                         break;
4688                 case 0:
4689                         status = "OK";
4690                         break;
4691                 default:
4692                         if (slist->script[i].status > 0) {
4693                                 status = "ERROR";
4694                         }
4695                         break;
4696                 }
4697
4698                 if (options.machinereadable) {
4699                         printf("%s%s%s%s%s%i%s%s%s%lu.%06lu%s%lu.%06lu%s%s%s\n",
4700                                options.sep,
4701                                event_str, options.sep,
4702                                slist->script[i].name, options.sep,
4703                                slist->script[i].status, options.sep,
4704                                status, options.sep,
4705                                (unsigned long)slist->script[i].start.tv_sec,
4706                                slist->script[i].start.tv_usec, options.sep,
4707                                (unsigned long)slist->script[i].finished.tv_sec,
4708                                slist->script[i].finished.tv_usec, options.sep,
4709                                slist->script[i].output, options.sep);
4710                         continue;
4711                 }
4712
4713                 if (status) {
4714                         printf("%-20s Status:%s    ",
4715                                slist->script[i].name, status);
4716                 } else {
4717                         /* Some other error, eg from stat. */
4718                         printf("%-20s Status:CANNOT RUN (%s)",
4719                                slist->script[i].name,
4720                                strerror(-slist->script[i].status));
4721                 }
4722
4723                 if (slist->script[i].status >= 0) {
4724                         printf("Duration:%.3lf ",
4725                                timeval_delta(&slist->script[i].finished,
4726                                              &slist->script[i].start));
4727                 }
4728                 if (slist->script[i].status != -ENOEXEC) {
4729                         printf("%s", ctime(&slist->script[i].start.tv_sec));
4730                         if (slist->script[i].status != 0) {
4731                                 printf("   OUTPUT:%s\n",
4732                                        slist->script[i].output);
4733                         }
4734                 } else {
4735                         printf("\n");
4736                 }
4737         }
4738 }
4739
4740 static void print_scriptstatus(struct ctdb_script_list **slist,
4741                                int count, const char **event_str)
4742 {
4743         int i;
4744
4745         if (options.machinereadable) {
4746                 printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
4747                        options.sep,
4748                        "Type", options.sep,
4749                        "Name", options.sep,
4750                        "Code", options.sep,
4751                        "Status", options.sep,
4752                        "Start", options.sep,
4753                        "End", options.sep,
4754                        "Error Output", options.sep);
4755         }
4756
4757         for (i=0; i<count; i++) {
4758                 print_scriptstatus_one(slist[i], event_str[i]);
4759         }
4760 }
4761
4762 static int control_event(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4763                          int argc, const char **argv)
4764 {
4765         char *t, *event_helper = NULL;
4766         char *eventd_socket = NULL;
4767         const char **new_argv;
4768         int i;
4769
4770         t = getenv("CTDB_EVENT_HELPER");
4771         if (t != NULL) {
4772                 event_helper = talloc_strdup(mem_ctx, t);
4773         } else {
4774                 event_helper = talloc_asprintf(mem_ctx, "%s/ctdb_event",
4775                                                CTDB_HELPER_BINDIR);
4776         }
4777
4778         if (event_helper == NULL) {
4779                 fprintf(stderr, "Unable to set event daemon helper\n");
4780                 return 1;
4781         }
4782
4783         t = getenv("CTDB_SOCKET");
4784         if (t != NULL) {
4785                 eventd_socket = talloc_asprintf(mem_ctx, "%s/eventd.sock",
4786                                                 dirname(t));
4787         } else {
4788                 eventd_socket = talloc_asprintf(mem_ctx, "%s/eventd.sock",
4789                                                 CTDB_RUNDIR);
4790         }
4791
4792         if (eventd_socket == NULL) {
4793                 fprintf(stderr, "Unable to set event daemon socket\n");
4794                 return 1;
4795         }
4796
4797         new_argv = talloc_array(mem_ctx, const char *, argc + 1);
4798         if (new_argv == NULL) {
4799                 fprintf(stderr, "Memory allocation error\n");
4800                 return 1;
4801         }
4802
4803         new_argv[0] = eventd_socket;
4804         for (i=0; i<argc; i++) {
4805                 new_argv[i+1] = argv[i];
4806         }
4807
4808         return run_helper(mem_ctx, "event daemon helper", event_helper,
4809                           argc+1, new_argv);
4810 }
4811
4812 static int control_scriptstatus(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4813                                 int argc, const char **argv)
4814 {
4815         struct ctdb_script_list **slist;
4816         const char *event_str;
4817         enum ctdb_event event;
4818         const char *all_events[] = {
4819                 "init", "setup", "startup", "monitor",
4820                 "takeip", "releaseip", "updateip", "ipreallocated" };
4821         bool valid;
4822         int ret, i, j;
4823         int count, start, end, num;
4824
4825         if (argc > 1) {
4826                 usage("scriptstatus");
4827         }
4828
4829         if (argc == 0) {
4830                 event_str = "monitor";
4831         } else {
4832                 event_str = argv[0];
4833         }
4834
4835         valid = false;
4836         end = 0;
4837
4838         for (i=0; i<ARRAY_SIZE(all_events); i++) {
4839                 if (strcmp(event_str, all_events[i]) == 0) {
4840                         valid = true;
4841                         count = 1;
4842                         start = i;
4843                         end = i+1;
4844                         break;
4845                 }
4846         }
4847
4848         if (strcmp(event_str, "all") == 0) {
4849                 valid = true;
4850                 count = ARRAY_SIZE(all_events);
4851                 start = 0;
4852                 end = count-1;
4853         }
4854
4855         if (! valid) {
4856                 fprintf(stderr, "Unknown event name %s\n", argv[0]);
4857                 usage("scriptstatus");
4858         }
4859
4860         slist = talloc_array(mem_ctx, struct ctdb_script_list *, count);
4861         if (slist == NULL) {
4862                 fprintf(stderr, "Memory allocation error\n");
4863                 return 1;
4864         }
4865
4866         num = 0;
4867         for (i=start; i<end; i++) {
4868                 event = ctdb_event_from_string(all_events[i]);
4869
4870                 ret = ctdb_ctrl_get_event_script_status(mem_ctx, ctdb->ev,
4871                                                         ctdb->client,
4872                                                         ctdb->cmd_pnn,
4873                                                         TIMEOUT(), event,
4874                                                         &slist[num]);
4875                 if (ret != 0) {
4876                         fprintf(stderr,
4877                                 "failed to get script status for %s event\n",
4878                                 all_events[i]);
4879                         return 1;
4880                 }
4881
4882                 if (slist[num] == NULL) {
4883                         num++;
4884                         continue;
4885                 }
4886
4887                 /* The ETIME status is ignored for certain events.
4888                  * In that case the status is 0, but endtime is not set.
4889                  */
4890                 for (j=0; j<slist[num]->num_scripts; j++) {
4891                         if (slist[num]->script[j].status == 0 &&
4892                             timeval_is_zero(&slist[num]->script[j].finished)) {
4893                                 slist[num]->script[j].status = -ETIME;
4894                         }
4895                 }
4896
4897                 num++;
4898         }
4899
4900         print_scriptstatus(slist, count, &all_events[start]);
4901         return 0;
4902 }
4903
4904 static int control_enablescript(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4905                                 int argc, const char **argv)
4906 {
4907         int ret;
4908
4909         if (argc != 1) {
4910                 usage("enablescript");
4911         }
4912
4913         ret = ctdb_ctrl_enable_script(mem_ctx, ctdb->ev, ctdb->client,
4914                                       ctdb->cmd_pnn, TIMEOUT(), argv[0]);
4915         if (ret != 0) {
4916                 fprintf(stderr, "Failed to enable script %s\n", argv[0]);
4917                 return ret;
4918         }
4919
4920         return 0;
4921 }
4922
4923 static int control_disablescript(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4924                                  int argc, const char **argv)
4925 {
4926         int ret;
4927
4928         if (argc != 1) {
4929                 usage("disablescript");
4930         }
4931
4932         ret = ctdb_ctrl_disable_script(mem_ctx, ctdb->ev, ctdb->client,
4933                                        ctdb->cmd_pnn, TIMEOUT(), argv[0]);
4934         if (ret != 0) {
4935                 fprintf(stderr, "Failed to disable script %s\n", argv[0]);
4936                 return ret;
4937         }
4938
4939         return 0;
4940 }
4941
4942 static int control_natgw(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4943                          int argc, const char **argv)
4944 {
4945         char *t, *natgw_helper = NULL;
4946
4947         if (argc != 1) {
4948                 usage("natgw");
4949         }
4950
4951         t = getenv("CTDB_NATGW_HELPER");
4952         if (t != NULL) {
4953                 natgw_helper = talloc_strdup(mem_ctx, t);
4954         } else {
4955                 natgw_helper = talloc_asprintf(mem_ctx, "%s/ctdb_natgw",
4956                                                CTDB_HELPER_BINDIR);
4957         }
4958
4959         if (natgw_helper == NULL) {
4960                 fprintf(stderr, "Unable to set NAT gateway helper\n");
4961                 return 1;
4962         }
4963
4964         return run_helper(mem_ctx, "NAT gateway helper", natgw_helper,
4965                           argc, argv);
4966 }
4967
4968 static int control_natgwlist(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
4969                              int argc, const char **argv)
4970 {
4971         char *t, *natgw_helper = NULL;
4972         const char *cmd_argv[] = { "natgwlist", NULL };
4973
4974         if (argc != 0) {
4975                 usage("natgwlist");
4976         }
4977
4978         t = getenv("CTDB_NATGW_HELPER");
4979         if (t != NULL) {
4980                 natgw_helper = talloc_strdup(mem_ctx, t);
4981         } else {
4982                 natgw_helper = talloc_asprintf(mem_ctx, "%s/ctdb_natgw",
4983                                                CTDB_HELPER_BINDIR);
4984         }
4985
4986         if (natgw_helper == NULL) {
4987                 fprintf(stderr, "Unable to set NAT gateway helper\n");
4988                 return 1;
4989         }
4990
4991         return run_helper(mem_ctx, "NAT gateway helper", natgw_helper,
4992                           1, cmd_argv);
4993 }
4994
4995 /*
4996  * Find the PNN of the current node
4997  * discover the pnn by loading the nodes file and try to bind
4998  * to all addresses one at a time until the ip address is found.
4999  */
5000 static bool find_node_xpnn(TALLOC_CTX *mem_ctx, uint32_t *pnn)
5001 {
5002         struct ctdb_node_map *nodemap;
5003         int i;
5004
5005         nodemap = read_nodes_file(mem_ctx, CTDB_UNKNOWN_PNN);
5006         if (nodemap == NULL) {
5007                 return false;
5008         }
5009
5010         for (i=0; i<nodemap->num; i++) {
5011                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
5012                         continue;
5013                 }
5014                 if (ctdb_sys_have_ip(&nodemap->node[i].addr)) {
5015                         if (pnn != NULL) {
5016                                 *pnn = nodemap->node[i].pnn;
5017                         }
5018                         talloc_free(nodemap);
5019                         return true;
5020                 }
5021         }
5022
5023         fprintf(stderr, "Failed to detect PNN of the current node.\n");
5024         talloc_free(nodemap);
5025         return false;
5026 }
5027
5028 static int control_getreclock(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5029                               int argc, const char **argv)
5030 {
5031         const char *reclock;
5032         int ret;
5033
5034         if (argc != 0) {
5035                 usage("getreclock");
5036         }
5037
5038         ret = ctdb_ctrl_get_reclock_file(mem_ctx, ctdb->ev, ctdb->client,
5039                                          ctdb->cmd_pnn, TIMEOUT(), &reclock);
5040         if (ret != 0) {
5041                 return ret;
5042         }
5043
5044         if (reclock != NULL) {
5045                 printf("%s\n", reclock);
5046         }
5047
5048         return 0;
5049 }
5050
5051 static int control_setlmasterrole(TALLOC_CTX *mem_ctx,
5052                                   struct ctdb_context *ctdb,
5053                                   int argc, const char **argv)
5054 {
5055         uint32_t lmasterrole = 0;
5056         int ret;
5057
5058         if (argc != 1) {
5059                 usage("setlmasterrole");
5060         }
5061
5062         if (strcmp(argv[0], "on") == 0) {
5063                 lmasterrole = 1;
5064         } else if (strcmp(argv[0], "off") == 0) {
5065                 lmasterrole = 0;
5066         } else {
5067                 usage("setlmasterrole");
5068         }
5069
5070         ret = ctdb_ctrl_set_lmasterrole(mem_ctx, ctdb->ev, ctdb->client,
5071                                         ctdb->cmd_pnn, TIMEOUT(), lmasterrole);
5072         if (ret != 0) {
5073                 return ret;
5074         }
5075
5076         return 0;
5077 }
5078
5079 static int control_setrecmasterrole(TALLOC_CTX *mem_ctx,
5080                                     struct ctdb_context *ctdb,
5081                                     int argc, const char **argv)
5082 {
5083         uint32_t recmasterrole = 0;
5084         int ret;
5085
5086         if (argc != 1) {
5087                 usage("setrecmasterrole");
5088         }
5089
5090         if (strcmp(argv[0], "on") == 0) {
5091                 recmasterrole = 1;
5092         } else if (strcmp(argv[0], "off") == 0) {
5093                 recmasterrole = 0;
5094         } else {
5095                 usage("setrecmasterrole");
5096         }
5097
5098         ret = ctdb_ctrl_set_recmasterrole(mem_ctx, ctdb->ev, ctdb->client,
5099                                           ctdb->cmd_pnn, TIMEOUT(),
5100                                           recmasterrole);
5101         if (ret != 0) {
5102                 return ret;
5103         }
5104
5105         return 0;
5106 }
5107
5108 static int control_setdbreadonly(TALLOC_CTX *mem_ctx,
5109                                  struct ctdb_context *ctdb,
5110                                  int argc, const char **argv)
5111 {
5112         uint32_t db_id;
5113         uint8_t db_flags;
5114         int ret;
5115
5116         if (argc != 1) {
5117                 usage("setdbreadonly");
5118         }
5119
5120         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, NULL, &db_flags)) {
5121                 return 1;
5122         }
5123
5124         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5125                 fprintf(stderr, "Cannot set READONLY on persistent DB\n");
5126                 return 1;
5127         }
5128
5129         ret = ctdb_ctrl_set_db_readonly(mem_ctx, ctdb->ev, ctdb->client,
5130                                         ctdb->cmd_pnn, TIMEOUT(), db_id);
5131         if (ret != 0) {
5132                 return ret;
5133         }
5134
5135         return 0;
5136 }
5137
5138 static int control_setdbsticky(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5139                                int argc, const char **argv)
5140 {
5141         uint32_t db_id;
5142         uint8_t db_flags;
5143         int ret;
5144
5145         if (argc != 1) {
5146                 usage("setdbsticky");
5147         }
5148
5149         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, NULL, &db_flags)) {
5150                 return 1;
5151         }
5152
5153         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5154                 fprintf(stderr, "Cannot set STICKY on persistent DB\n");
5155                 return 1;
5156         }
5157
5158         ret = ctdb_ctrl_set_db_sticky(mem_ctx, ctdb->ev, ctdb->client,
5159                                       ctdb->cmd_pnn, TIMEOUT(), db_id);
5160         if (ret != 0) {
5161                 return ret;
5162         }
5163
5164         return 0;
5165 }
5166
5167 static int control_pfetch(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5168                           int argc, const char **argv)
5169 {
5170         const char *db_name;
5171         struct ctdb_db_context *db;
5172         struct ctdb_transaction_handle *h;
5173         uint8_t db_flags;
5174         TDB_DATA key, data;
5175         int ret;
5176
5177         if (argc < 2 || argc > 3) {
5178                 usage("pfetch");
5179         }
5180
5181         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5182                 return 1;
5183         }
5184
5185         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5186                 fprintf(stderr, "DB %s is not a persistent database\n",
5187                         db_name);
5188                 return 1;
5189         }
5190
5191         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5192                           db_flags, &db);
5193         if (ret != 0) {
5194                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5195                 return ret;
5196         }
5197
5198         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5199         if (ret != 0) {
5200                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5201                 return ret;
5202         }
5203
5204         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5205                                      TIMEOUT(), db, true, &h);
5206         if (ret != 0) {
5207                 fprintf(stderr, "Failed to start transaction on db %s\n",
5208                         db_name);
5209                 return ret;
5210         }
5211
5212         ret = ctdb_transaction_fetch_record(h, key, mem_ctx, &data);
5213         if (ret != 0) {
5214                 fprintf(stderr, "Failed to read record for key %s\n",
5215                         argv[1]);
5216                 ctdb_transaction_cancel(h);
5217                 return ret;
5218         }
5219
5220         printf("%.*s\n", (int)data.dsize, data.dptr);
5221
5222         ctdb_transaction_cancel(h);
5223         return 0;
5224 }
5225
5226 static int control_pstore(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5227                           int argc, const char **argv)
5228 {
5229         const char *db_name;
5230         struct ctdb_db_context *db;
5231         struct ctdb_transaction_handle *h;
5232         uint8_t db_flags;
5233         TDB_DATA key, data;
5234         int ret;
5235
5236         if (argc != 3) {
5237                 usage("pstore");
5238         }
5239
5240         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5241                 return 1;
5242         }
5243
5244         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5245                 fprintf(stderr, "DB %s is not a persistent database\n",
5246                         db_name);
5247                 return 1;
5248         }
5249
5250         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5251                           db_flags, &db);
5252         if (ret != 0) {
5253                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5254                 return ret;
5255         }
5256
5257         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5258         if (ret != 0) {
5259                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5260                 return ret;
5261         }
5262
5263         ret = str_to_data(argv[2], strlen(argv[2]), mem_ctx, &data);
5264         if (ret != 0) {
5265                 fprintf(stderr, "Failed to parse value %s\n", argv[2]);
5266                 return ret;
5267         }
5268
5269         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5270                                      TIMEOUT(), db, false, &h);
5271         if (ret != 0) {
5272                 fprintf(stderr, "Failed to start transaction on db %s\n",
5273                         db_name);
5274                 return ret;
5275         }
5276
5277         ret = ctdb_transaction_store_record(h, key, data);
5278         if (ret != 0) {
5279                 fprintf(stderr, "Failed to store record for key %s\n",
5280                         argv[1]);
5281                 ctdb_transaction_cancel(h);
5282                 return ret;
5283         }
5284
5285         ret = ctdb_transaction_commit(h);
5286         if (ret != 0) {
5287                 fprintf(stderr, "Failed to commit transaction on db %s\n",
5288                         db_name);
5289                 ctdb_transaction_cancel(h);
5290                 return ret;
5291         }
5292
5293         return 0;
5294 }
5295
5296 static int control_pdelete(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5297                            int argc, const char **argv)
5298 {
5299         const char *db_name;
5300         struct ctdb_db_context *db;
5301         struct ctdb_transaction_handle *h;
5302         uint8_t db_flags;
5303         TDB_DATA key;
5304         int ret;
5305
5306         if (argc != 2) {
5307                 usage("pdelete");
5308         }
5309
5310         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5311                 return 1;
5312         }
5313
5314         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5315                 fprintf(stderr, "DB %s is not a persistent database\n",
5316                         db_name);
5317                 return 1;
5318         }
5319
5320         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5321                           db_flags, &db);
5322         if (ret != 0) {
5323                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5324                 return ret;
5325         }
5326
5327         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5328         if (ret != 0) {
5329                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5330                 return ret;
5331         }
5332
5333         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5334                                      TIMEOUT(), db, false, &h);
5335         if (ret != 0) {
5336                 fprintf(stderr, "Failed to start transaction on db %s\n",
5337                         db_name);
5338                 return ret;
5339         }
5340
5341         ret = ctdb_transaction_delete_record(h, key);
5342         if (ret != 0) {
5343                 fprintf(stderr, "Failed to delete record for key %s\n",
5344                         argv[1]);
5345                 ctdb_transaction_cancel(h);
5346                 return ret;
5347         }
5348
5349         ret = ctdb_transaction_commit(h);
5350         if (ret != 0) {
5351                 fprintf(stderr, "Failed to commit transaction on db %s\n",
5352                         db_name);
5353                 ctdb_transaction_cancel(h);
5354                 return ret;
5355         }
5356
5357         return 0;
5358 }
5359
5360 static int ptrans_parse_string(TALLOC_CTX *mem_ctx, const char **ptr, TDB_DATA *data)
5361 {
5362         const char *t;
5363         size_t n;
5364         int ret;
5365
5366         *data = tdb_null;
5367
5368         /* Skip whitespace */
5369         n = strspn(*ptr, " \t");
5370         t = *ptr + n;
5371
5372         if (t[0] == '"') {
5373                 /* Quoted ASCII string - no wide characters! */
5374                 t++;
5375                 n = strcspn(t, "\"");
5376                 if (t[n] == '"') {
5377                         if (n > 0) {
5378                                 ret = str_to_data(t, n, mem_ctx, data);
5379                                 if (ret != 0) {
5380                                         return ret;
5381                                 }
5382                         }
5383                         *ptr = t + n + 1;
5384                 } else {
5385                         fprintf(stderr, "Unmatched \" in input %s\n", *ptr);
5386                         return 1;
5387                 }
5388         } else {
5389                 fprintf(stderr, "Unsupported input format in %s\n", *ptr);
5390                 return 1;
5391         }
5392
5393         return 0;
5394 }
5395
5396 #define MAX_LINE_SIZE   1024
5397
5398 static bool ptrans_get_key_value(TALLOC_CTX *mem_ctx, FILE *file,
5399                                  TDB_DATA *key, TDB_DATA *value)
5400 {
5401         char line [MAX_LINE_SIZE]; /* FIXME: make this more flexible? */
5402         const char *ptr;
5403         int ret;
5404
5405         ptr = fgets(line, MAX_LINE_SIZE, file);
5406         if (ptr == NULL) {
5407                 return false;
5408         }
5409
5410         /* Get key */
5411         ret = ptrans_parse_string(mem_ctx, &ptr, key);
5412         if (ret != 0 || ptr == NULL || key->dptr == NULL) {
5413                 /* Line Ignored but not EOF */
5414                 *key = tdb_null;
5415                 return true;
5416         }
5417
5418         /* Get value */
5419         ret = ptrans_parse_string(mem_ctx, &ptr, value);
5420         if (ret != 0) {
5421                 /* Line Ignored but not EOF */
5422                 talloc_free(key->dptr);
5423                 *key = tdb_null;
5424                 return true;
5425         }
5426
5427         return true;
5428 }
5429
5430 static int control_ptrans(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5431                           int argc, const char **argv)
5432 {
5433         const char *db_name;
5434         struct ctdb_db_context *db;
5435         struct ctdb_transaction_handle *h;
5436         uint8_t db_flags;
5437         FILE *file;
5438         TDB_DATA key, value;
5439         int ret;
5440
5441         if (argc < 1 || argc > 2) {
5442                 usage("ptrans");
5443         }
5444
5445         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5446                 return 1;
5447         }
5448
5449         if (! (db_flags & CTDB_DB_FLAGS_PERSISTENT)) {
5450                 fprintf(stderr, "DB %s is not a persistent database\n",
5451                         db_name);
5452                 return 1;
5453         }
5454
5455         if (argc == 2) {
5456                 file = fopen(argv[1], "r");
5457                 if (file == NULL) {
5458                         fprintf(stderr, "Failed to open file %s\n", argv[1]);
5459                         return 1;
5460                 }
5461         } else {
5462                 file = stdin;
5463         }
5464
5465         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5466                           db_flags, &db);
5467         if (ret != 0) {
5468                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5469                 goto done;
5470         }
5471
5472         ret = ctdb_transaction_start(mem_ctx, ctdb->ev, ctdb->client,
5473                                      TIMEOUT(), db, false, &h);
5474         if (ret != 0) {
5475                 fprintf(stderr, "Failed to start transaction on db %s\n",
5476                         db_name);
5477                 goto done;
5478         }
5479
5480         while (ptrans_get_key_value(mem_ctx, file, &key, &value)) {
5481                 if (key.dsize != 0) {
5482                         ret = ctdb_transaction_store_record(h, key, value);
5483                         if (ret != 0) {
5484                                 fprintf(stderr, "Failed to store record\n");
5485                                 ctdb_transaction_cancel(h);
5486                                 goto done;
5487                         }
5488                         talloc_free(key.dptr);
5489                         talloc_free(value.dptr);
5490                 }
5491         }
5492
5493         ret = ctdb_transaction_commit(h);
5494         if (ret != 0) {
5495                 fprintf(stderr, "Failed to commit transaction on db %s\n",
5496                         db_name);
5497                 ctdb_transaction_cancel(h);
5498         }
5499
5500 done:
5501         if (file != stdin) {
5502                 fclose(file);
5503         }
5504         return ret;
5505 }
5506
5507 static int control_tfetch(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5508                           int argc, const char **argv)
5509 {
5510         struct tdb_context *tdb;
5511         TDB_DATA key, data;
5512         struct ctdb_ltdb_header header;
5513         int ret;
5514
5515         if (argc < 2 || argc > 3) {
5516                 usage("tfetch");
5517         }
5518
5519         tdb = tdb_open(argv[0], 0, 0, O_RDWR, 0);
5520         if (tdb == NULL) {
5521                 fprintf(stderr, "Failed to open TDB file %s\n", argv[0]);
5522                 return 1;
5523         }
5524
5525         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5526         if (ret != 0) {
5527                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5528                 tdb_close(tdb);
5529                 return ret;
5530         }
5531
5532         data = tdb_fetch(tdb, key);
5533         if (data.dptr == NULL) {
5534                 fprintf(stderr, "No record for key %s\n", argv[1]);
5535                 tdb_close(tdb);
5536                 return 1;
5537         }
5538
5539         if (data.dsize < sizeof(struct ctdb_ltdb_header)) {
5540                 fprintf(stderr, "Invalid record for key %s\n", argv[1]);
5541                 tdb_close(tdb);
5542                 return 1;
5543         }
5544
5545         tdb_close(tdb);
5546
5547         if (argc == 3) {
5548                 int fd;
5549                 ssize_t nwritten;
5550
5551                 fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
5552                 if (fd == -1) {
5553                         fprintf(stderr, "Failed to open output file %s\n",
5554                                 argv[2]);
5555                         goto fail;
5556                 }
5557
5558                 nwritten = sys_write(fd, data.dptr, data.dsize);
5559                 if (nwritten != data.dsize) {
5560                         fprintf(stderr, "Failed to write record to file\n");
5561                         close(fd);
5562                         goto fail;
5563                 }
5564
5565                 close(fd);
5566         }
5567
5568 fail:
5569         ret = ctdb_ltdb_header_extract(&data, &header);
5570         if (ret != 0) {
5571                 fprintf(stderr, "Failed to parse header from data\n");
5572                 return 1;
5573         }
5574
5575         dump_ltdb_header(&header);
5576         dump_tdb_data("data", data);
5577
5578         return 0;
5579 }
5580
5581 static int control_tstore(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5582                           int argc, const char **argv)
5583 {
5584         struct tdb_context *tdb;
5585         TDB_DATA key, data[2], value;
5586         struct ctdb_ltdb_header header;
5587         uint8_t header_buf[sizeof(struct ctdb_ltdb_header)];
5588         int ret;
5589
5590         if (argc < 3 || argc > 5) {
5591                 usage("tstore");
5592         }
5593
5594         tdb = tdb_open(argv[0], 0, 0, O_RDWR, 0);
5595         if (tdb == NULL) {
5596                 fprintf(stderr, "Failed to open TDB file %s\n", argv[0]);
5597                 return 1;
5598         }
5599
5600         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5601         if (ret != 0) {
5602                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5603                 tdb_close(tdb);
5604                 return ret;
5605         }
5606
5607         ret = str_to_data(argv[2], strlen(argv[2]), mem_ctx, &value);
5608         if (ret != 0) {
5609                 fprintf(stderr, "Failed to parse value %s\n", argv[2]);
5610                 tdb_close(tdb);
5611                 return ret;
5612         }
5613
5614         ZERO_STRUCT(header);
5615
5616         if (argc > 3) {
5617                 header.rsn = (uint64_t)strtoull(argv[3], NULL, 0);
5618         }
5619         if (argc > 4) {
5620                 header.dmaster = (uint32_t)atol(argv[4]);
5621         }
5622         if (argc > 5) {
5623                 header.flags = (uint32_t)atol(argv[5]);
5624         }
5625
5626         ctdb_ltdb_header_push(&header, header_buf);
5627
5628         data[0].dsize = ctdb_ltdb_header_len(&header);
5629         data[0].dptr = header_buf;
5630
5631         data[1].dsize = value.dsize;
5632         data[1].dptr = value.dptr;
5633
5634         ret = tdb_storev(tdb, key, data, 2, TDB_REPLACE);
5635         if (ret != 0) {
5636                 fprintf(stderr, "Failed to write record %s to file %s\n",
5637                         argv[1], argv[0]);
5638         }
5639
5640         tdb_close(tdb);
5641
5642         return ret;
5643 }
5644
5645 static int control_readkey(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5646                            int argc, const char **argv)
5647 {
5648         const char *db_name;
5649         struct ctdb_db_context *db;
5650         struct ctdb_record_handle *h;
5651         uint8_t db_flags;
5652         TDB_DATA key, data;
5653         bool readonly = false;
5654         int ret;
5655
5656         if (argc < 2 || argc > 3) {
5657                 usage("readkey");
5658         }
5659
5660         if (argc == 3) {
5661                 if (strcmp(argv[2], "readonly") == 0) {
5662                         readonly = true;
5663                 } else {
5664                         usage("readkey");
5665                 }
5666         }
5667
5668         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5669                 return 1;
5670         }
5671
5672         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5673                 fprintf(stderr, "DB %s is not a volatile database\n",
5674                         db_name);
5675                 return 1;
5676         }
5677
5678         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5679                           db_flags, &db);
5680         if (ret != 0) {
5681                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5682                 return ret;
5683         }
5684
5685         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5686         if (ret != 0) {
5687                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5688                 return ret;
5689         }
5690
5691         ret = ctdb_fetch_lock(mem_ctx, ctdb->ev, ctdb->client,
5692                               db, key, readonly, &h, NULL, &data);
5693         if (ret != 0) {
5694                 fprintf(stderr, "Failed to read record for key %s\n",
5695                         argv[1]);
5696         } else {
5697                 printf("Data: size:%zu ptr:[%.*s]\n", data.dsize,
5698                        (int)data.dsize, data.dptr);
5699         }
5700
5701         talloc_free(h);
5702         return ret;
5703 }
5704
5705 static int control_writekey(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5706                             int argc, const char **argv)
5707 {
5708         const char *db_name;
5709         struct ctdb_db_context *db;
5710         struct ctdb_record_handle *h;
5711         uint8_t db_flags;
5712         TDB_DATA key, data;
5713         int ret;
5714
5715         if (argc != 3) {
5716                 usage("writekey");
5717         }
5718
5719         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5720                 return 1;
5721         }
5722
5723         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5724                 fprintf(stderr, "DB %s is not a volatile database\n",
5725                         db_name);
5726                 return 1;
5727         }
5728
5729         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5730                           db_flags, &db);
5731         if (ret != 0) {
5732                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5733                 return ret;
5734         }
5735
5736         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5737         if (ret != 0) {
5738                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5739                 return ret;
5740         }
5741
5742         ret = str_to_data(argv[2], strlen(argv[2]), mem_ctx, &data);
5743         if (ret != 0) {
5744                 fprintf(stderr, "Failed to parse value %s\n", argv[2]);
5745                 return ret;
5746         }
5747
5748         ret = ctdb_fetch_lock(mem_ctx, ctdb->ev, ctdb->client,
5749                               db, key, false, &h, NULL, NULL);
5750         if (ret != 0) {
5751                 fprintf(stderr, "Failed to lock record for key %s\n", argv[0]);
5752                 return ret;
5753         }
5754
5755         ret = ctdb_store_record(h, data);
5756         if (ret != 0) {
5757                 fprintf(stderr, "Failed to store record for key %s\n",
5758                         argv[1]);
5759         }
5760
5761         talloc_free(h);
5762         return ret;
5763 }
5764
5765 static int control_deletekey(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5766                              int argc, const char **argv)
5767 {
5768         const char *db_name;
5769         struct ctdb_db_context *db;
5770         struct ctdb_record_handle *h;
5771         uint8_t db_flags;
5772         TDB_DATA key, data;
5773         int ret;
5774
5775         if (argc != 2) {
5776                 usage("deletekey");
5777         }
5778
5779         if (! db_exists(mem_ctx, ctdb, argv[0], NULL, &db_name, &db_flags)) {
5780                 return 1;
5781         }
5782
5783         if (db_flags & CTDB_DB_FLAGS_PERSISTENT) {
5784                 fprintf(stderr, "DB %s is not a volatile database\n",
5785                         db_name);
5786                 return 1;
5787         }
5788
5789         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
5790                           db_flags, &db);
5791         if (ret != 0) {
5792                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
5793                 return ret;
5794         }
5795
5796         ret = str_to_data(argv[1], strlen(argv[1]), mem_ctx, &key);
5797         if (ret != 0) {
5798                 fprintf(stderr, "Failed to parse key %s\n", argv[1]);
5799                 return ret;
5800         }
5801
5802         ret = ctdb_fetch_lock(mem_ctx, ctdb->ev, ctdb->client,
5803                               db, key, false, &h, NULL, &data);
5804         if (ret != 0) {
5805                 fprintf(stderr, "Failed to fetch record for key %s\n",
5806                         argv[1]);
5807                 return ret;
5808         }
5809
5810         ret = ctdb_delete_record(h);
5811         if (ret != 0) {
5812                 fprintf(stderr, "Failed to delete record for key %s\n",
5813                         argv[1]);
5814         }
5815
5816         talloc_free(h);
5817         return ret;
5818 }
5819
5820 static int control_checktcpport(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5821                                 int argc, const char **argv)
5822 {
5823         struct sockaddr_in sin;
5824         unsigned int port;
5825         int s, v;
5826         int ret;
5827
5828         if (argc != 1) {
5829                 usage("chktcpport");
5830         }
5831
5832         port = atoi(argv[0]);
5833
5834         s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
5835         if (s == -1) {
5836                 fprintf(stderr, "Failed to open local socket\n");
5837                 return errno;
5838         }
5839
5840         v = fcntl(s, F_GETFL, 0);
5841         if (v == -1 || fcntl(s, F_SETFL, v | O_NONBLOCK)) {
5842                 fprintf(stderr, "Unable to set socket non-blocking\n");
5843                 close(s);
5844                 return errno;
5845         }
5846
5847         bzero(&sin, sizeof(sin));
5848         sin.sin_family = AF_INET;
5849         sin.sin_port = htons(port);
5850         ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
5851         close(s);
5852         if (ret == -1) {
5853                 fprintf(stderr, "Failed to bind to TCP port %u\n", port);
5854                 return errno;
5855         }
5856
5857         return 0;
5858 }
5859
5860 static int control_getdbseqnum(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5861                                int argc, const char **argv)
5862 {
5863         uint32_t db_id;
5864         const char *db_name;
5865         uint64_t seqnum;
5866         int ret;
5867
5868         if (argc != 1) {
5869                 usage("getdbseqnum");
5870         }
5871
5872         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, NULL)) {
5873                 return 1;
5874         }
5875
5876         ret = ctdb_ctrl_get_db_seqnum(mem_ctx, ctdb->ev, ctdb->client,
5877                                       ctdb->cmd_pnn, TIMEOUT(), db_id,
5878                                       &seqnum);
5879         if (ret != 0) {
5880                 fprintf(stderr, "Failed to get sequence number for DB %s\n",
5881                         db_name);
5882                 return ret;
5883         }
5884
5885         printf("0x%"PRIx64"\n", seqnum);
5886         return 0;
5887 }
5888
5889 static int control_nodestatus(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
5890                               int argc, const char **argv)
5891 {
5892         const char *nodestring = NULL;
5893         struct ctdb_node_map *nodemap;
5894         int ret, i;
5895
5896         if (argc > 1) {
5897                 usage("nodestatus");
5898         }
5899
5900         if (argc == 1) {
5901                 nodestring = argv[0];
5902         }
5903
5904         if (! parse_nodestring(mem_ctx, ctdb, nodestring, &nodemap)) {
5905                 return 1;
5906         }
5907
5908         nodemap = get_nodemap(ctdb, false);
5909         if (nodemap == NULL) {
5910                 return 1;
5911         }
5912
5913         if (options.machinereadable) {
5914                 print_nodemap_machine(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn);
5915         } else {
5916                 print_nodemap(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn);
5917         }
5918
5919         ret = 0;
5920         for (i=0; i<nodemap->num; i++) {
5921                 ret |= nodemap->node[i].flags;
5922         }
5923
5924         return ret;
5925 }
5926
5927 const struct {
5928         const char *name;
5929         uint32_t offset;
5930 } db_stats_fields[] = {
5931 #define DBSTATISTICS_FIELD(n) { #n, offsetof(struct ctdb_db_statistics, n) }
5932         DBSTATISTICS_FIELD(db_ro_delegations),
5933         DBSTATISTICS_FIELD(db_ro_revokes),
5934         DBSTATISTICS_FIELD(locks.num_calls),
5935         DBSTATISTICS_FIELD(locks.num_current),
5936         DBSTATISTICS_FIELD(locks.num_pending),
5937         DBSTATISTICS_FIELD(locks.num_failed),
5938         DBSTATISTICS_FIELD(db_ro_delegations),
5939 };
5940
5941 static void print_dbstatistics(const char *db_name,
5942                                struct ctdb_db_statistics *s)
5943 {
5944         int i;
5945         const char *prefix = NULL;
5946         int preflen = 0;
5947
5948         printf("DB Statistics %s\n", db_name);
5949
5950         for (i=0; i<ARRAY_SIZE(db_stats_fields); i++) {
5951                 if (strchr(db_stats_fields[i].name, '.') != NULL) {
5952                         preflen = strcspn(db_stats_fields[i].name, ".") + 1;
5953                         if (! prefix ||
5954                             strncmp(prefix, db_stats_fields[i].name, preflen) != 0) {
5955                                 prefix = db_stats_fields[i].name;
5956                                 printf(" %*.*s\n", preflen-1, preflen-1,
5957                                        db_stats_fields[i].name);
5958                         }
5959                 } else {
5960                         preflen = 0;
5961                 }
5962                 printf(" %*s%-22s%*s%10u\n", preflen ? 4 : 0, "",
5963                        db_stats_fields[i].name+preflen, preflen ? 0 : 4, "",
5964                        *(uint32_t *)(db_stats_fields[i].offset+(uint8_t *)s));
5965         }
5966
5967         printf(" hop_count_buckets:");
5968         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
5969                 printf(" %d", s->hop_count_bucket[i]);
5970         }
5971         printf("\n");
5972
5973         printf(" lock_buckets:");
5974         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
5975                 printf(" %d", s->locks.buckets[i]);
5976         }
5977         printf("\n");
5978
5979         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
5980                "locks_latency      MIN/AVG/MAX",
5981                s->locks.latency.min, LATENCY_AVG(s->locks.latency),
5982                s->locks.latency.max, s->locks.latency.num);
5983
5984         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
5985                "vacuum_latency     MIN/AVG/MAX",
5986                s->vacuum.latency.min, LATENCY_AVG(s->vacuum.latency),
5987                s->vacuum.latency.max, s->vacuum.latency.num);
5988
5989         printf(" Num Hot Keys:     %d\n", s->num_hot_keys);
5990         for (i=0; i<s->num_hot_keys; i++) {
5991                 int j;
5992                 printf("     Count:%d Key:", s->hot_keys[i].count);
5993                 for (j=0; j<s->hot_keys[i].key.dsize; j++) {
5994                         printf("%02x", s->hot_keys[i].key.dptr[j] & 0xff);
5995                 }
5996                 printf("\n");
5997         }
5998 }
5999
6000 static int control_dbstatistics(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
6001                                 int argc, const char **argv)
6002 {
6003         uint32_t db_id;
6004         const char *db_name;
6005         struct ctdb_db_statistics *dbstats;
6006         int ret;
6007
6008         if (argc != 1) {
6009                 usage("dbstatistics");
6010         }
6011
6012         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, NULL)) {
6013                 return 1;
6014         }
6015
6016         ret = ctdb_ctrl_get_db_statistics(mem_ctx, ctdb->ev, ctdb->client,
6017                                           ctdb->cmd_pnn, TIMEOUT(), db_id,
6018                                           &dbstats);
6019         if (ret != 0) {
6020                 fprintf(stderr, "Failed to get statistics for DB %s\n",
6021                         db_name);
6022                 return ret;
6023         }
6024
6025         print_dbstatistics(db_name, dbstats);
6026         return 0;
6027 }
6028
6029 struct disable_takeover_runs_state {
6030         uint32_t *pnn_list;
6031         int node_count;
6032         bool *reply;
6033         int status;
6034         bool done;
6035 };
6036
6037 static void disable_takeover_run_handler(uint64_t srvid, TDB_DATA data,
6038                                          void *private_data)
6039 {
6040         struct disable_takeover_runs_state *state =
6041                 (struct disable_takeover_runs_state *)private_data;
6042         int ret, i;
6043
6044         if (data.dsize != sizeof(int)) {
6045                 /* Ignore packet */
6046                 return;
6047         }
6048
6049         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
6050         ret = *(int *)data.dptr;
6051         if (ret < 0) {
6052                 state->status = ret;
6053                 state->done = true;
6054                 return;
6055         }
6056         for (i=0; i<state->node_count; i++) {
6057                 if (state->pnn_list[i] == ret) {
6058                         state->reply[i] = true;
6059                         break;
6060                 }
6061         }
6062
6063         state->done = true;
6064         for (i=0; i<state->node_count; i++) {
6065                 if (! state->reply[i]) {
6066                         state->done = false;
6067                         break;
6068                 }
6069         }
6070 }
6071
6072 static int disable_takeover_runs(TALLOC_CTX *mem_ctx,
6073                                  struct ctdb_context *ctdb, uint32_t timeout,
6074                                  uint32_t *pnn_list, int count)
6075 {
6076         struct ctdb_disable_message disable = { 0 };
6077         struct disable_takeover_runs_state state;
6078         int ret, i;
6079
6080         disable.pnn = ctdb->pnn;
6081         disable.srvid = next_srvid(ctdb);
6082         disable.timeout = timeout;
6083
6084         state.pnn_list = pnn_list;
6085         state.node_count = count;
6086         state.done = false;
6087         state.status = 0;
6088         state.reply = talloc_zero_array(mem_ctx, bool, count);
6089         if (state.reply == NULL) {
6090                 return ENOMEM;
6091         }
6092
6093         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
6094                                               disable.srvid,
6095                                               disable_takeover_run_handler,
6096                                               &state);
6097         if (ret != 0) {
6098                 return ret;
6099         }
6100
6101         for (i=0; i<count; i++) {
6102                 ret = ctdb_message_disable_takeover_runs(mem_ctx, ctdb->ev,
6103                                                          ctdb->client,
6104                                                          pnn_list[i],
6105                                                          &disable);
6106                 if (ret != 0) {
6107                         goto fail;
6108                 }
6109         }
6110
6111         ret = ctdb_client_wait_timeout(ctdb->ev, &state.done, TIMEOUT());
6112         if (ret == ETIME) {
6113                 fprintf(stderr, "Timed out waiting to disable takeover runs\n");
6114         } else {
6115                 ret = (state.status >= 0 ? 0 : 1);
6116         }
6117
6118 fail:
6119         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
6120                                            disable.srvid, &state);
6121         return ret;
6122 }
6123
6124 static int control_reloadips(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
6125                              int argc, const char **argv)
6126 {
6127         const char *nodestring = NULL;
6128         struct ctdb_node_map *nodemap, *nodemap2;
6129         struct ctdb_req_control request;
6130         uint32_t *pnn_list, *pnn_list2;
6131         int ret, count, count2;
6132
6133         if (argc > 1) {
6134                 usage("reloadips");
6135         }
6136
6137         if (argc == 1) {
6138                 nodestring = argv[0];
6139         }
6140
6141         nodemap = get_nodemap(ctdb, false);
6142         if (nodemap == NULL) {
6143                 return 1;
6144         }
6145
6146         if (! parse_nodestring(mem_ctx, ctdb, nodestring, &nodemap2)) {
6147                 return 1;
6148         }
6149
6150         count = list_of_connected_nodes(nodemap, CTDB_UNKNOWN_PNN,
6151                                         mem_ctx, &pnn_list);
6152         if (count <= 0) {
6153                 fprintf(stderr, "Memory allocation error\n");
6154                 return 1;
6155         }
6156
6157         count2 = list_of_active_nodes(nodemap2, CTDB_UNKNOWN_PNN,
6158                                       mem_ctx, &pnn_list2);
6159         if (count2 <= 0) {
6160                 fprintf(stderr, "Memory allocation error\n");
6161                 return 1;
6162         }
6163
6164         /* Disable takeover runs on all connected nodes.  A reply
6165          * indicating success is needed from each node so all nodes
6166          * will need to be active.
6167          *
6168          * A check could be added to not allow reloading of IPs when
6169          * there are disconnected nodes.  However, this should
6170          * probably be left up to the administrator.
6171          */
6172         ret = disable_takeover_runs(mem_ctx, ctdb, 2*options.timelimit,
6173                                     pnn_list, count);
6174         if (ret != 0) {
6175                 fprintf(stderr, "Failed to disable takeover runs\n");
6176                 return ret;
6177         }
6178
6179         /* Now tell all the desired nodes to reload their public IPs.
6180          * Keep trying this until it succeeds.  This assumes all
6181          * failures are transient, which might not be true...
6182          */
6183         ctdb_req_control_reload_public_ips(&request);
6184         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
6185                                         pnn_list2, count2, TIMEOUT(),
6186                                         &request, NULL, NULL);
6187         if (ret != 0) {
6188                 fprintf(stderr, "Failed to reload IPs on some nodes.\n");
6189         }
6190
6191         /* It isn't strictly necessary to wait until takeover runs are
6192          * re-enabled but doing so can't hurt.
6193          */
6194         ret = disable_takeover_runs(mem_ctx, ctdb, 0, pnn_list, count);
6195         if (ret != 0) {
6196                 fprintf(stderr, "Failed to enable takeover runs\n");
6197                 return ret;
6198         }
6199
6200         return ipreallocate(mem_ctx, ctdb);
6201 }
6202
6203 static int control_ipiface(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
6204                            int argc, const char **argv)
6205 {
6206         ctdb_sock_addr addr;
6207         char *iface;
6208
6209         if (argc != 1) {
6210                 usage("ipiface");
6211         }
6212
6213         if (! parse_ip(argv[0], NULL, 0, &addr)) {
6214                 fprintf(stderr, "Failed to Parse IP %s\n", argv[0]);
6215                 return 1;
6216         }
6217
6218         iface = ctdb_sys_find_ifname(&addr);
6219         if (iface == NULL) {
6220                 fprintf(stderr, "Failed to find interface for IP %s\n",
6221                         argv[0]);
6222                 return 1;
6223         }
6224         free(iface);
6225
6226         return 0;
6227 }
6228
6229
6230 static const struct ctdb_cmd {
6231         const char *name;
6232         int (*fn)(TALLOC_CTX *, struct ctdb_context *, int, const char **);
6233         bool without_daemon; /* can be run without daemon running ? */
6234         bool remote; /* can be run on remote nodes */
6235         const char *msg;
6236         const char *args;
6237 } ctdb_commands[] = {
6238         { "version", control_version, true, false,
6239                 "show version of ctdb", NULL },
6240         { "status", control_status, false, true,
6241                 "show node status", NULL },
6242         { "uptime", control_uptime, false, true,
6243                 "show node uptime", NULL },
6244         { "ping", control_ping, false, true,
6245                 "ping all nodes", NULL },
6246         { "runstate", control_runstate, false, true,
6247                 "get/check runstate of a node",
6248                 "[setup|first_recovery|startup|running]" },
6249         { "getvar", control_getvar, false, true,
6250                 "get a tunable variable", "<name>" },
6251         { "setvar", control_setvar, false, true,
6252                 "set a tunable variable", "<name> <value>" },
6253         { "listvars", control_listvars, false, true,
6254                 "list tunable variables", NULL },
6255         { "statistics", control_statistics, false, true,
6256                 "show ctdb statistics", NULL },
6257         { "statisticsreset", control_statistics_reset, false, true,
6258                 "reset ctdb statistics", NULL },
6259         { "stats", control_stats, false, true,
6260                 "show rolling statistics", "[count]" },
6261         { "ip", control_ip, false, true,
6262                 "show public ips", "[all]" },
6263         { "ipinfo", control_ipinfo, false, true,
6264                 "show public ip details", "<ip>" },
6265         { "ifaces", control_ifaces, false, true,
6266                 "show interfaces", NULL },
6267         { "setifacelink", control_setifacelink, false, true,
6268                 "set interface link status", "<iface> up|down" },
6269         { "process-exists", control_process_exists, false, true,
6270                 "check if a process exists on a node",  "<pid>" },
6271         { "getdbmap", control_getdbmap, false, true,
6272                 "show attached databases", NULL },
6273         { "getdbstatus", control_getdbstatus, false, true,
6274                 "show database status", "<dbname|dbid>" },
6275         { "catdb", control_catdb, false, false,
6276                 "dump cluster-wide ctdb database", "<dbname|dbid>" },
6277         { "cattdb", control_cattdb, false, false,
6278                 "dump local ctdb database", "<dbname|dbid>" },
6279         { "getmonmode", control_getmonmode, false, true,
6280                 "show monitoring mode", NULL },
6281         { "getcapabilities", control_getcapabilities, false, true,
6282                 "show node capabilities", NULL },
6283         { "pnn", control_pnn, false, false,
6284                 "show the pnn of the currnet node", NULL },
6285         { "lvs", control_lvs, false, false,
6286                 "show lvs configuration", "master|list|status" },
6287         { "disablemonitor", control_disable_monitor, false, true,
6288                 "disable monitoring", NULL },
6289         { "enablemonitor", control_enable_monitor, false, true,
6290                 "enable monitoring", NULL },
6291         { "setdebug", control_setdebug, false, true,
6292                 "set debug level", "ERROR|WARNING|NOTICE|INFO|DEBUG" },
6293         { "getdebug", control_getdebug, false, true,
6294                 "get debug level", NULL },
6295         { "attach", control_attach, false, false,
6296                 "attach a database", "<dbname> [persistent]" },
6297         { "detach", control_detach, false, false,
6298                 "detach database(s)", "<dbname|dbid> ..." },
6299         { "dumpmemory", control_dumpmemory, false, true,
6300                 "dump ctdbd memory map", NULL },
6301         { "rddumpmemory", control_rddumpmemory, false, true,
6302                 "dump recoverd memory map", NULL },
6303         { "getpid", control_getpid, false, true,
6304                 "get ctdbd process ID", NULL },
6305         { "disable", control_disable, false, true,
6306                 "disable a node", NULL },
6307         { "enable", control_enable, false, true,
6308                 "enable a node", NULL },
6309         { "stop", control_stop, false, true,
6310                 "stop a node", NULL },
6311         { "continue", control_continue, false, true,
6312                 "continue a stopped node", NULL },
6313         { "ban", control_ban, false, true,
6314                 "ban a node", "<bantime>"},
6315         { "unban", control_unban, false, true,
6316                 "unban a node", NULL },
6317         { "shutdown", control_shutdown, false, true,
6318                 "shutdown ctdb daemon", NULL },
6319         { "recover", control_recover, false, true,
6320                 "force recovery", NULL },
6321         { "sync", control_ipreallocate, false, true,
6322                 "run ip reallocation (deprecated)", NULL },
6323         { "ipreallocate", control_ipreallocate, false, true,
6324                 "run ip reallocation", NULL },
6325         { "isnotrecmaster", control_isnotrecmaster, false, false,
6326                 "check if local node is the recmaster", NULL },
6327         { "gratarp", control_gratarp, false, true,
6328                 "send a gratuitous arp", "<ip> <interface>" },
6329         { "tickle", control_tickle, true, false,
6330                 "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
6331         { "gettickles", control_gettickles, false, true,
6332                 "get the list of tickles", "<ip> [<port>]" },
6333         { "addtickle", control_addtickle, false, true,
6334                 "add a tickle", "<ip>:<port> <ip>:<port>" },
6335         { "deltickle", control_deltickle, false, true,
6336                 "delete a tickle", "<ip>:<port> <ip>:<port>" },
6337         { "check_srvids", control_check_srvids, false, true,
6338                 "check if srvid is registered", "<id> [<id> ...]" },
6339         { "listnodes", control_listnodes, true, true,
6340                 "list nodes in the cluster", NULL },
6341         { "reloadnodes", control_reloadnodes, false, false,
6342                 "reload the nodes file all nodes", NULL },
6343         { "moveip", control_moveip, false, false,
6344                 "move an ip address to another node", "<ip> <node>" },
6345         { "addip", control_addip, false, true,
6346                 "add an ip address to a node", "<ip/mask> <iface>" },
6347         { "delip", control_delip, false, true,
6348                 "delete an ip address from a node", "<ip>" },
6349         { "eventscript", control_eventscript, false, true,
6350                 "run an event", "monitor" },
6351         { "backupdb", control_backupdb, false, false,
6352                 "backup a database into a file", "<dbname|dbid> <file>" },
6353         { "restoredb", control_restoredb, false, false,
6354                 "restore a database from a file", "<file> [dbname]" },
6355         { "dumpdbbackup", control_dumpdbbackup, true, false,
6356                 "dump database from a backup file", "<file>" },
6357         { "wipedb", control_wipedb, false, false,
6358                 "wipe the contents of a database.", "<dbname|dbid>"},
6359         { "recmaster", control_recmaster, false, true,
6360                 "show the pnn for the recovery master", NULL },
6361         { "event", control_event, true, false,
6362                 "event and event script commands", NULL },
6363         { "scriptstatus", control_scriptstatus, false, true,
6364                 "show event script status",
6365                 "[init|setup|startup|monitor|takeip|releaseip|ipreallocated]" },
6366         { "enablescript", control_enablescript, false, true,
6367                 "enable an eventscript", "<script>"},
6368         { "disablescript", control_disablescript, false, true,
6369                 "disable an eventscript", "<script>"},
6370         { "natgw", control_natgw, false, false,
6371                 "show natgw configuration", "master|list|status" },
6372         { "natgwlist", control_natgwlist, false, false,
6373                 "show the nodes belonging to this natgw configuration", NULL },
6374         { "getreclock", control_getreclock, false, true,
6375                 "get recovery lock file", NULL },
6376         { "setlmasterrole", control_setlmasterrole, false, true,
6377                 "set LMASTER role", "on|off" },
6378         { "setrecmasterrole", control_setrecmasterrole, false, true,
6379                 "set RECMASTER role", "on|off"},
6380         { "setdbreadonly", control_setdbreadonly, false, true,
6381                 "enable readonly records", "<dbname|dbid>" },
6382         { "setdbsticky", control_setdbsticky, false, true,
6383                 "enable sticky records", "<dbname|dbid>"},
6384         { "pfetch", control_pfetch, false, false,
6385                 "fetch record from persistent database", "<dbname|dbid> <key> [<file>]" },
6386         { "pstore", control_pstore, false, false,
6387                 "write record to persistent database", "<dbname|dbid> <key> <value>" },
6388         { "pdelete", control_pdelete, false, false,
6389                 "delete record from persistent database", "<dbname|dbid> <key>" },
6390         { "ptrans", control_ptrans, false, false,
6391                 "update a persistent database (from file or stdin)", "<dbname|dbid> [<file>]" },
6392         { "tfetch", control_tfetch, false, true,
6393                 "fetch a record", "<tdb-file> <key> [<file>]" },
6394         { "tstore", control_tstore, false, true,
6395                 "store a record", "<tdb-file> <key> <data> [<rsn> <dmaster> <flags>]" },
6396         { "readkey", control_readkey, false, false,
6397                 "read value of a database key", "<dbname|dbid> <key> [readonly]" },
6398         { "writekey", control_writekey, false, false,
6399                 "write value for a database key", "<dbname|dbid> <key> <value>" },
6400         { "deletekey", control_deletekey, false, false,
6401                 "delete a database key", "<dbname|dbid> <key>" },
6402         { "checktcpport", control_checktcpport, true, false,
6403                 "check if a service is bound to a specific tcp port or not", "<port>" },
6404         { "getdbseqnum", control_getdbseqnum, false, false,
6405                 "get database sequence number", "<dbname|dbid>" },
6406         { "nodestatus", control_nodestatus, false, true,
6407                 "show and return node status", "[all|<pnn-list>]" },
6408         { "dbstatistics", control_dbstatistics, false, true,
6409                 "show database statistics", "<dbname|dbid>" },
6410         { "reloadips", control_reloadips, false, false,
6411                 "reload the public addresses file", "[all|<pnn-list>]" },
6412         { "ipiface", control_ipiface, true, false,
6413                 "Find the interface an ip address is hosted on", "<ip>" },
6414 };
6415
6416 static const struct ctdb_cmd *match_command(const char *command)
6417 {
6418         const struct ctdb_cmd *cmd;
6419         int i;
6420
6421         for (i=0; i<ARRAY_SIZE(ctdb_commands); i++) {
6422                 cmd = &ctdb_commands[i];
6423                 if (strlen(command) == strlen(cmd->name) &&
6424                     strncmp(command, cmd->name, strlen(command)) == 0) {
6425                         return cmd;
6426                 }
6427         }
6428
6429         return NULL;
6430 }
6431
6432
6433 /**
6434  * Show usage message
6435  */
6436 static void usage_full(void)
6437 {
6438         int i;
6439
6440         poptPrintHelp(pc, stdout, 0);
6441         printf("\nCommands:\n");
6442         for (i=0; i<ARRAY_SIZE(ctdb_commands); i++) {
6443                 printf("  %-15s %-27s  %s\n",
6444                        ctdb_commands[i].name,
6445                        ctdb_commands[i].args ? ctdb_commands[i].args : "",
6446                        ctdb_commands[i].msg);
6447         }
6448 }
6449
6450 static void usage(const char *command)
6451 {
6452         const struct ctdb_cmd *cmd;
6453
6454         if (command == NULL) {
6455                 usage_full();
6456                 exit(1);
6457         }
6458
6459         cmd = match_command(command);
6460         if (cmd == NULL) {
6461                 usage_full();
6462         } else {
6463                 poptPrintUsage(pc, stdout, 0);
6464                 printf("\nCommands:\n");
6465                 printf("  %-15s %-27s  %s\n",
6466                        cmd->name, cmd->args ? cmd->args : "", cmd->msg);
6467         }
6468
6469         exit(1);
6470 }
6471
6472 struct poptOption cmdline_options[] = {
6473         POPT_AUTOHELP
6474         { "socket", 's', POPT_ARG_STRING, &options.socket, 0,
6475                 "CTDB socket path", "filename" },
6476         { "debug", 'd', POPT_ARG_STRING, &options.debuglevelstr, 0,
6477                 "debug level"},
6478         { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0,
6479                 "timelimit (in seconds)" },
6480         { "node", 'n', POPT_ARG_INT, &options.pnn, 0,
6481                 "node specification - integer" },
6482         { NULL, 'Y', POPT_ARG_NONE, &options.machinereadable, 0,
6483                 "enable machine readable output", NULL },
6484         { "separator", 'x', POPT_ARG_STRING, &options.sep, 0,
6485                 "specify separator for machine readable output", "CHAR" },
6486         { NULL, 'X', POPT_ARG_NONE, &options.machineparsable, 0,
6487                 "enable machine parsable output with separator |", NULL },
6488         { "verbose", 'v', POPT_ARG_NONE, &options.verbose, 0,
6489                 "enable verbose output", NULL },
6490         { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0,
6491                 "die if runtime exceeds this limit (in seconds)" },
6492         POPT_TABLEEND
6493 };
6494
6495 static int process_command(const struct ctdb_cmd *cmd, int argc,
6496                            const char **argv)
6497 {
6498         TALLOC_CTX *tmp_ctx;
6499         struct ctdb_context *ctdb;
6500         int ret;
6501         bool status;
6502         uint64_t srvid_offset;
6503
6504         tmp_ctx = talloc_new(NULL);
6505         if (tmp_ctx == NULL) {
6506                 fprintf(stderr, "Memory allocation error\n");
6507                 goto fail;
6508         }
6509
6510         if (cmd->without_daemon) {
6511                 if (options.pnn != -1) {
6512                         fprintf(stderr,
6513                                 "Cannot specify node for command %s\n",
6514                                 cmd->name);
6515                         goto fail;
6516                 }
6517
6518                 ret = cmd->fn(tmp_ctx, NULL, argc-1, argv+1);
6519                 talloc_free(tmp_ctx);
6520                 return ret;
6521         }
6522
6523         ctdb = talloc_zero(tmp_ctx, struct ctdb_context);
6524         if (ctdb == NULL) {
6525                 fprintf(stderr, "Memory allocation error\n");
6526                 goto fail;
6527         }
6528
6529         ctdb->ev = tevent_context_init(ctdb);
6530         if (ctdb->ev == NULL) {
6531                 fprintf(stderr, "Failed to initialize tevent\n");
6532                 goto fail;
6533         }
6534
6535         ret = ctdb_client_init(ctdb, ctdb->ev, options.socket, &ctdb->client);
6536         if (ret != 0) {
6537                 fprintf(stderr, "Failed to connect to CTDB daemon (%s)\n",
6538                         options.socket);
6539
6540                 if (!find_node_xpnn(ctdb, NULL)) {
6541                         fprintf(stderr, "Is this node part of CTDB cluster?\n");
6542                 }
6543                 goto fail;
6544         }
6545
6546         ctdb->pnn = ctdb_client_pnn(ctdb->client);
6547         srvid_offset = getpid() & 0xFFFF;
6548         ctdb->srvid = SRVID_CTDB_TOOL | (srvid_offset << 16);
6549
6550         if (options.pnn != -1) {
6551                 status = verify_pnn(ctdb, options.pnn);
6552                 if (! status) {
6553                         goto fail;
6554                 }
6555
6556                 ctdb->cmd_pnn = options.pnn;
6557         } else {
6558                 ctdb->cmd_pnn = ctdb->pnn;
6559         }
6560
6561         if (! cmd->remote && ctdb->pnn != ctdb->cmd_pnn) {
6562                 fprintf(stderr, "Node cannot be specified for command %s\n",
6563                         cmd->name);
6564                 goto fail;
6565         }
6566
6567         ret = cmd->fn(tmp_ctx, ctdb, argc-1, argv+1);
6568         talloc_free(tmp_ctx);
6569         return ret;
6570
6571 fail:
6572         talloc_free(tmp_ctx);
6573         return 1;
6574 }
6575
6576 static void signal_handler(int sig)
6577 {
6578         fprintf(stderr, "Maximum runtime exceeded - exiting\n");
6579 }
6580
6581 static void alarm_handler(int sig)
6582 {
6583         /* Kill any child processes */
6584         signal(SIGTERM, signal_handler);
6585         kill(0, SIGTERM);
6586
6587         _exit(1);
6588 }
6589
6590 int main(int argc, const char *argv[])
6591 {
6592         int opt;
6593         const char **extra_argv;
6594         int extra_argc;
6595         const struct ctdb_cmd *cmd;
6596         const char *ctdb_socket;
6597         int loglevel;
6598         int ret;
6599
6600         setlinebuf(stdout);
6601
6602         /* Set default options */
6603         options.socket = CTDB_SOCKET;
6604         options.debuglevelstr = NULL;
6605         options.timelimit = 10;
6606         options.sep = "|";
6607         options.maxruntime = 0;
6608         options.pnn = -1;
6609
6610         ctdb_socket = getenv("CTDB_SOCKET");
6611         if (ctdb_socket != NULL) {
6612                 options.socket = ctdb_socket;
6613         }
6614
6615         pc = poptGetContext(argv[0], argc, argv, cmdline_options,
6616                             POPT_CONTEXT_KEEP_FIRST);
6617         while ((opt = poptGetNextOpt(pc)) != -1) {
6618                 fprintf(stderr, "Invalid option %s: %s\n",
6619                         poptBadOption(pc, 0), poptStrerror(opt));
6620                 exit(1);
6621         }
6622
6623         if (options.maxruntime == 0) {
6624                 const char *ctdb_timeout;
6625
6626                 ctdb_timeout = getenv("CTDB_TIMEOUT");
6627                 if (ctdb_timeout != NULL) {
6628                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
6629                 } else {
6630                         options.maxruntime = 120;
6631                 }
6632         }
6633         if (options.maxruntime <= 120) {
6634                 /* default timeout is 120 seconds */
6635                 options.maxruntime = 120;
6636         }
6637
6638         if (options.machineparsable) {
6639                 options.machinereadable = 1;
6640         }
6641
6642         /* setup the remaining options for the commands */
6643         extra_argc = 0;
6644         extra_argv = poptGetArgs(pc);
6645         if (extra_argv) {
6646                 extra_argv++;
6647                 while (extra_argv[extra_argc]) extra_argc++;
6648         }
6649
6650         if (extra_argc < 1) {
6651                 usage(NULL);
6652         }
6653
6654         cmd = match_command(extra_argv[0]);
6655         if (cmd == NULL) {
6656                 fprintf(stderr, "Unknown command '%s'\n", extra_argv[0]);
6657                 exit(1);
6658         }
6659
6660         /* Enable logging */
6661         setup_logging("ctdb", DEBUG_STDERR);
6662         if (debug_level_parse(options.debuglevelstr, &loglevel)) {
6663                 DEBUGLEVEL = loglevel;
6664         } else {
6665                 DEBUGLEVEL = DEBUG_ERR;
6666         }
6667
6668         signal(SIGALRM, alarm_handler);
6669         alarm(options.maxruntime);
6670
6671         ret = process_command(cmd, extra_argc, extra_argv);
6672         if (ret == -1) {
6673                 ret = 1;
6674         }
6675
6676         (void)poptFreeContext(pc);
6677
6678         return ret;
6679 }