ctdb/tests/src/ctdb_takeover_tests.c

   1 /*
   2    Tests for ctdb_takeover.c
   3
   4    Copyright (C) Martin Schwenke 2011
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20 #include "ctdbd_test.c"
  21
  22 /* This is lazy... but it is test code! */
  23 #define CTDB_TEST_MAX_NODES 256
  24 #define CTDB_TEST_MAX_IPS 1024
  25
  26 /* Format of each line is "IP pnn" - the separator has to be at least
  27  * 1 space (not a tab or whatever - a space!).
  28  */
  29 static struct ctdb_public_ip_list *
  30 read_ctdb_public_ip_list(TALLOC_CTX *ctx)
  31 {
  32         char line[1024];
  33         ctdb_sock_addr addr;
  34         char *t;
  35         int pnn;
  36         struct ctdb_public_ip_list *last = NULL;
  37
  38         struct ctdb_public_ip_list *ret = NULL;
  39
  40         while (fgets(line, sizeof(line), stdin) != NULL) {
  41
  42                 if ((t = strchr(line, ' ')) != NULL) {
  43                         /* Make line contain just the address */
  44                         *t = '\0';
  45                         /* Point to PNN or leading whitespace...  */
  46                         t++;
  47                         pnn = (int) strtol(t, (char **) NULL, 10);
  48                 } else {
  49                         /* Assume just an IP address, default to PNN -1 */
  50                         if ((t = strchr(line, '\n')) != NULL) {
  51                                 *t = '\0';
  52                         }
  53                         pnn = -1;
  54                 }
  55
  56                 if (parse_ip(line, NULL, 0, &addr)) {
  57                         if (last == NULL) {
  58                                 last = talloc(ctx, struct ctdb_public_ip_list);
  59                         } else {
  60                                 last->next = talloc(ctx, struct ctdb_public_ip_list);
  61                                 last = last->next;
  62                         }
  63                         last->next = NULL;
  64                         last->pnn = pnn;
  65                         memcpy(&(last->addr), &addr, sizeof(addr));
  66                         if (ret == NULL) {
  67                                 ret = last;
  68                         }
  69                 } else {
  70                         DEBUG(DEBUG_ERR, (__location__ " ERROR, bad address :%s\n", line));
  71                 }
  72         }
  73
  74         return ret;
  75 }
  76
  77 void print_ctdb_public_ip_list(struct ctdb_public_ip_list * ips)
  78 {
  79         while (ips) {
  80                 printf("%s %d\n", ctdb_addr_to_str(&(ips->addr)), ips->pnn);
  81                 ips = ips->next;
  82         }
  83 }
  84
  85 /* Read some IPs from stdin, 1 per line, parse them and then print
  86  * them back out. */
  87 void ctdb_test_read_ctdb_public_ip_list(void)
  88 {
  89         struct ctdb_public_ip_list *l;
  90
  91         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
  92
  93         l = read_ctdb_public_ip_list(tmp_ctx);
  94
  95         print_ctdb_public_ip_list(l);
  96
  97         talloc_free(tmp_ctx);
  98 }
  99
 100 /* Format of each line is "IP CURRENT_PNN ALLOWED_PNN,...".
 101  */
 102 static bool
 103 read_ctdb_public_ip_info(TALLOC_CTX *ctx,
 104                          int numnodes,
 105                          struct ctdb_public_ip_list ** all_ips,
 106                          struct ctdb_all_public_ips *** avail)
 107 {
 108         char line[1024];
 109         ctdb_sock_addr addr;
 110         char *t, *tok;
 111         struct ctdb_public_ip_list * ta;
 112         int pnn, numips, curr, n, i;
 113         struct ctdb_all_public_ips * a;
 114
 115         struct ctdb_public_ip_list *last = NULL;
 116
 117         *avail = talloc_array_size(ctx, sizeof(struct ctdb_all_public_ips *), CTDB_TEST_MAX_NODES);
 118         memset(*avail, 0,
 119                sizeof(struct ctdb_all_public_ips *) * CTDB_TEST_MAX_NODES);
 120
 121         numips = 0;
 122         *all_ips = NULL;
 123         while (fgets(line, sizeof(line), stdin) != NULL) {
 124
 125                 /* Get rid of pesky newline */
 126                 if ((t = strchr(line, '\n')) != NULL) {
 127                         *t = '\0';
 128                 }
 129
 130                 /* Exit on an empty line */
 131                 if (line[0] == '\0') {
 132                         break;
 133                 }
 134
 135                 /* Get the IP address */
 136                 tok = strtok(line, " \t");
 137                 if (tok == NULL) {
 138                         DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line ignored :%s\n", line));
 139                         continue;
 140                 }
 141
 142                 if (!parse_ip(tok, NULL, 0, &addr)) {
 143                         DEBUG(DEBUG_ERR, (__location__ " ERROR, bad address :%s\n", tok));
 144                         continue;
 145                 }
 146
 147                 numips++;
 148                 if (numips > CTDB_TEST_MAX_IPS) {
 149                         DEBUG(DEBUG_ERR, ("ERROR: Exceeding CTDB_TEST_MAX_IPS: %d\n", CTDB_TEST_MAX_IPS));
 150                         exit(1);
 151                 }
 152
 153                 /* Get the PNN */
 154                 pnn = -1;
 155                 tok = strtok(NULL, " \t");
 156                 if (tok != NULL) {
 157                         pnn = (int) strtol(tok, (char **) NULL, 10);
 158                 }
 159
 160                 /* Add address + pnn to all_ips */
 161                 if (last == NULL) {
 162                         last = talloc(ctx, struct ctdb_public_ip_list);
 163                 } else {
 164                         last->next = talloc(ctx, struct ctdb_public_ip_list);
 165                         last = last->next;
 166                 }
 167                 last->next = NULL;
 168                 last->pnn = pnn;
 169                 memcpy(&(last->addr), &addr, sizeof(addr));
 170                 if (*all_ips == NULL) {
 171                         *all_ips = last;
 172                 }
 173
 174                 tok = strtok(NULL, " \t#");
 175                 if (tok == NULL) {
 176                         continue;
 177                 }
 178
 179                 /* Handle allowed nodes for addr */
 180                 t = strtok(tok, ",");
 181                 while (t != NULL) {
 182                         n = (int) strtol(t, (char **) NULL, 10);
 183                         if ((*avail)[n] == NULL) {
 184                                 (*avail)[n] = talloc_array(ctx, struct ctdb_all_public_ips, CTDB_TEST_MAX_IPS);
 185                                 (*avail)[n]->num = 0;
 186                         }
 187                         curr = (*avail)[n]->num;
 188                         (*avail)[n]->ips[curr].pnn = pnn;
 189                         memcpy(&((*avail)[n]->ips[curr].addr),
 190                                &addr, sizeof(addr));
 191                         (*avail)[n]->num++;
 192                         t = strtok(NULL, ",");
 193                 }
 194
 195         }
 196
 197         /* Build list of all allowed IPs */
 198         a = talloc_array(ctx, struct ctdb_all_public_ips, CTDB_TEST_MAX_IPS);
 199         a->num = numips;
 200         for (ta = *all_ips, i=0; ta != NULL && i < numips ; ta = ta->next, i++) {
 201                 a->ips[i].pnn = ta->pnn;
 202                 memcpy(&(a->ips[i].addr), &(ta->addr), sizeof(ta->addr));
 203         }
 204
 205         /* Assign it to any nodes that don't have a list assigned */
 206         for (n = 0; n < numnodes; n++) {
 207                 if ((*avail)[n] == NULL) {
 208                         (*avail)[n] = a;
 209                 }
 210         }
 211
 212         return true;
 213 }
 214
 215 void print_ctdb_available_ips(int numnodes, struct ctdb_all_public_ips **avail)
 216 {
 217         int n, i;
 218
 219         for (n = 0; n < numnodes; n++) {
 220                 if ((avail[n] != NULL) && (avail[n]->num > 0)) {
 221                         printf("%d:", n);
 222                         for (i = 0; i < avail[n]->num; i++) {
 223                                 printf("%s%s",
 224                                        (i == 0) ? " " : ", ",
 225                                        ctdb_addr_to_str(&(avail[n]->ips[i].addr)));
 226                         }
 227                         printf("\n");
 228                 }
 229         }
 230 }
 231
 232 void ctdb_test_read_ctdb_public_ip_info(const char nodestates[])
 233 {
 234         int numnodes;
 235         struct ctdb_public_ip_list *l;
 236         struct ctdb_all_public_ips **avail;
 237         char *tok, *ns;
 238
 239         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 240
 241         /* Avoid that const */
 242         ns = talloc_strdup(tmp_ctx, nodestates);
 243
 244         numnodes = 0;
 245         tok = strtok(ns, ",");
 246         while (tok != NULL) {
 247                 numnodes++;
 248                 if (numnodes > CTDB_TEST_MAX_NODES) {
 249                         DEBUG(DEBUG_ERR, ("ERROR: Exceeding CTDB_TEST_MAX_NODES: %d\n", CTDB_TEST_MAX_NODES));
 250                         exit(1);
 251                 }
 252                 tok = strtok(NULL, ",");
 253         }
 254
 255         read_ctdb_public_ip_info(tmp_ctx, numnodes, &l, &avail);
 256
 257         print_ctdb_public_ip_list(l);
 258         print_ctdb_available_ips(numnodes, avail);
 259
 260         talloc_free(tmp_ctx);
 261 }
 262
 263 /* Read 2 IPs from stdin, calculate the IP distance and print it. */
 264 void ctdb_test_ip_distance(void)
 265 {
 266         struct ctdb_public_ip_list *l;
 267         uint32_t distance;
 268
 269         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 270
 271         l = read_ctdb_public_ip_list(tmp_ctx);
 272
 273         if (l && l->next) {
 274                 distance = ip_distance(&(l->addr), &(l->next->addr));
 275                 printf ("%lu\n", (unsigned long) distance);
 276         }
 277
 278         talloc_free(tmp_ctx);
 279 }
 280
 281 /* Read some IPs from stdin, calculate the sum of the squares of the
 282  * IP distances between the 1st argument and those read that are on
 283  * the given node. The given IP must one of the ones in the list.  */
 284 void ctdb_test_ip_distance_2_sum(const char ip[], int pnn)
 285 {
 286         struct ctdb_public_ip_list *l;
 287         struct ctdb_public_ip_list *t;
 288         ctdb_sock_addr addr;
 289         uint32_t distance;
 290
 291         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 292
 293
 294         l = read_ctdb_public_ip_list(tmp_ctx);
 295
 296         if (l && parse_ip(ip, NULL, 0, &addr)) {
 297                 /* find the entry for the specified IP */
 298                 for (t=l; t!=NULL; t=t->next) {
 299                         if (ctdb_same_ip(&(t->addr), &addr)) {
 300                                 break;
 301                         }
 302                 }
 303
 304                 if (t == NULL) {
 305                         fprintf(stderr, "IP NOT PRESENT IN LIST");
 306                         exit(1);
 307                 }
 308
 309                 distance = ip_distance_2_sum(&(t->addr), l, pnn);
 310                 printf ("%lu\n", (unsigned long) distance);
 311         } else {
 312                 fprintf(stderr, "BAD INPUT");
 313                 exit(1);
 314         }
 315
 316         talloc_free(tmp_ctx);
 317 }
 318
 319 /* Read some IPs from stdin, calculate the sume of the squares of the
 320  * IP distances between the first and the rest, and print it. */
 321 void ctdb_test_lcp2_imbalance(int pnn)
 322 {
 323         struct ctdb_public_ip_list *l;
 324         uint32_t imbalance;
 325
 326         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 327
 328         l = read_ctdb_public_ip_list(tmp_ctx);
 329
 330         imbalance = lcp2_imbalance(l, pnn);
 331         printf ("%lu\n", (unsigned long) imbalance);
 332
 333         talloc_free(tmp_ctx);
 334 }
 335
 336 static uint32_t *get_tunable_values(TALLOC_CTX *tmp_ctx,
 337                                     int numnodes,
 338                                     const char *tunable)
 339 {
 340         int i;
 341         char *tok;
 342         uint32_t *tvals = talloc_zero_array(tmp_ctx, uint32_t, numnodes);
 343         char *t = getenv(tunable);
 344
 345         if (t) {
 346                 if (strcmp(t, "1") == 0) {
 347                         for (i=0; i<numnodes; i++) {
 348                                 tvals[i] = 1;
 349                         }
 350                 } else {
 351                         tok = strtok(t, ",");
 352                         i = 0;
 353                         while (tok != NULL) {
 354                                 tvals[i] =
 355                                         (uint32_t) strtol(tok, NULL, 0);
 356                                 i++;
 357                                 tok = strtok(NULL, ",");
 358                         }
 359                         if (i != numnodes) {
 360                                 fprintf(stderr, "ERROR: Wrong number of values in %s\n", tunable);
 361                                 exit(1);
 362                         }
 363                 }
 364         }
 365
 366         return tvals;
 367 }
 368
 369 static enum ctdb_runstate *get_runstate(TALLOC_CTX *tmp_ctx,
 370                                         int numnodes)
 371 {
 372         int i;
 373         uint32_t *tvals;
 374         enum ctdb_runstate *runstate =
 375                 talloc_zero_array(tmp_ctx, enum ctdb_runstate, numnodes);
 376         char *t = getenv("CTDB_TEST_RUNSTATE");
 377
 378         if (t == NULL) {
 379                 for (i=0; i<numnodes; i++) {
 380                         runstate[i] = CTDB_RUNSTATE_RUNNING;
 381                 }
 382         } else {
 383                 tvals = get_tunable_values(tmp_ctx, numnodes, "CTDB_TEST_RUNSTATE");
 384                 for (i=0; i<numnodes; i++) {
 385                         runstate[i] = (enum ctdb_runstate) tvals[i];
 386                 }
 387                 talloc_free(tvals);
 388         }
 389
 390         return runstate;
 391 }
 392
 393 /* Fake up enough CTDB state to be able to run the IP allocation
 394  * algorithm.  Usually this sets up some standard state, sets the node
 395  * states from the command-line and reads the current IP layout from
 396  * stdin.
 397  *
 398  * However, if read_ips_for_multiple_nodes is true then each node's
 399  * idea of the IP layout is read separately from stdin.  In this mode
 400  * is doesn't make much sense to use read_ctdb_public_ip_info's
 401  * optional ALLOWED_PNN,... list in the input, since each node is
 402  * being handled separately anyway.  IPs for each node are separated
 403  * by a blank line.  This mode is for testing weird behaviours where
 404  * the IP layouts differs across nodes and we want to improve
 405  * create_merged_ip_list(), so should only be used in tests of
 406  * ctdb_takeover_run_core().  Yes, it is a hack...  :-)
 407  */
 408 void ctdb_test_init(const char nodestates[],
 409                     struct ctdb_context **ctdb,
 410                     struct ctdb_public_ip_list **all_ips,
 411                     struct ctdb_ipflags **ipflags,
 412                     bool read_ips_for_multiple_nodes)
 413 {
 414         struct ctdb_all_public_ips **avail;
 415         int i, numnodes;
 416         uint32_t nodeflags[CTDB_TEST_MAX_NODES];
 417         char *tok, *ns, *t;
 418         struct ctdb_node_map *nodemap;
 419         uint32_t *tval_noiptakeover;
 420         uint32_t *tval_noiptakeoverondisabled;
 421         enum ctdb_runstate *runstate;
 422
 423         *ctdb = talloc_zero(NULL, struct ctdb_context);
 424
 425         /* Avoid that const */
 426         ns = talloc_strdup(*ctdb, nodestates);
 427
 428         numnodes = 0;
 429         tok = strtok(ns, ",");
 430         while (tok != NULL) {
 431                 nodeflags[numnodes] = (uint32_t) strtol(tok, NULL, 0);
 432                 numnodes++;
 433                 if (numnodes > CTDB_TEST_MAX_NODES) {
 434                         DEBUG(DEBUG_ERR, ("ERROR: Exceeding CTDB_TEST_MAX_NODES: %d\n", CTDB_TEST_MAX_NODES));
 435                         exit(1);
 436                 }
 437                 tok = strtok(NULL, ",");
 438         }
 439
 440         /* Fake things up... */
 441         (*ctdb)->num_nodes = numnodes;
 442
 443         /* Default to LCP2 */
 444         (*ctdb)->tunable.lcp2_public_ip_assignment = 1;
 445         (*ctdb)->tunable.deterministic_public_ips = 0;
 446         (*ctdb)->tunable.disable_ip_failover = 0;
 447         (*ctdb)->tunable.no_ip_failback = 0;
 448
 449         if ((t = getenv("CTDB_IP_ALGORITHM"))) {
 450                 if (strcmp(t, "lcp2") == 0) {
 451                         (*ctdb)->tunable.lcp2_public_ip_assignment = 1;
 452                 } else if (strcmp(t, "nondet") == 0) {
 453                         (*ctdb)->tunable.lcp2_public_ip_assignment = 0;
 454                 } else if (strcmp(t, "det") == 0) {
 455                         (*ctdb)->tunable.lcp2_public_ip_assignment = 0;
 456                         (*ctdb)->tunable.deterministic_public_ips = 1;
 457                 } else {
 458                         fprintf(stderr, "ERROR: unknown IP algorithm %s\n", t);
 459                         exit(1);
 460                 }
 461         }
 462
 463         tval_noiptakeover = get_tunable_values(*ctdb, numnodes,
 464                                                "CTDB_SET_NoIPTakeover");
 465         tval_noiptakeoverondisabled =
 466                 get_tunable_values(*ctdb, numnodes,
 467                                    "CTDB_SET_NoIPHostOnAllDisabled");
 468
 469         runstate = get_runstate(*ctdb, numnodes);
 470
 471         nodemap =  talloc_array(*ctdb, struct ctdb_node_map, numnodes);
 472         nodemap->num = numnodes;
 473
 474         if (!read_ips_for_multiple_nodes) {
 475                 read_ctdb_public_ip_info(*ctdb, numnodes, all_ips, &avail);
 476         }
 477
 478         (*ctdb)->nodes = talloc_array(*ctdb, struct ctdb_node *, numnodes); // FIXME: bogus size, overkill
 479
 480         for (i=0; i < numnodes; i++) {
 481                 nodemap->nodes[i].pnn = i;
 482                 nodemap->nodes[i].flags = nodeflags[i];
 483                 /* nodemap->nodes[i].sockaddr is uninitialised */
 484
 485                 if (read_ips_for_multiple_nodes) {
 486                         read_ctdb_public_ip_info(*ctdb, numnodes,
 487                                                  all_ips, &avail);
 488                 }
 489
 490                 (*ctdb)->nodes[i] = talloc(*ctdb, struct ctdb_node);
 491                 (*ctdb)->nodes[i]->pnn = i;
 492                 (*ctdb)->nodes[i]->flags = nodeflags[i];
 493                 (*ctdb)->nodes[i]->available_public_ips = avail[i];
 494                 (*ctdb)->nodes[i]->known_public_ips = avail[i];
 495         }
 496
 497         *ipflags = set_ipflags_internal(*ctdb, *ctdb, nodemap,
 498                                         tval_noiptakeover,
 499                                         tval_noiptakeoverondisabled,
 500                                         runstate);
 501 }
 502
 503 /* IP layout is read from stdin. */
 504 void ctdb_test_lcp2_allocate_unassigned(const char nodestates[])
 505 {
 506         struct ctdb_context *ctdb;
 507         struct ctdb_public_ip_list *all_ips;
 508         struct ctdb_ipflags *ipflags;
 509
 510         uint32_t *lcp2_imbalances;
 511         bool *newly_healthy;
 512
 513         ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
 514
 515         lcp2_init(ctdb, ipflags, all_ips, NULL,
 516                   &lcp2_imbalances, &newly_healthy);
 517
 518         lcp2_allocate_unassigned(ctdb, ipflags,
 519                                  all_ips, lcp2_imbalances);
 520
 521         print_ctdb_public_ip_list(all_ips);
 522
 523         talloc_free(ctdb);
 524 }
 525
 526 /* IP layout is read from stdin. */
 527 void ctdb_test_lcp2_failback(const char nodestates[])
 528 {
 529         struct ctdb_context *ctdb;
 530         struct ctdb_public_ip_list *all_ips;
 531         struct ctdb_ipflags *ipflags;
 532
 533         uint32_t *lcp2_imbalances;
 534         bool *newly_healthy;
 535
 536         ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
 537
 538         lcp2_init(ctdb, ipflags, all_ips, NULL,
 539                   &lcp2_imbalances, &newly_healthy);
 540
 541         lcp2_failback(ctdb, ipflags,
 542                       all_ips, lcp2_imbalances, newly_healthy);
 543
 544         print_ctdb_public_ip_list(all_ips);
 545
 546         talloc_free(ctdb);
 547 }
 548
 549 /* IP layout is read from stdin. */
 550 void ctdb_test_lcp2_failback_loop(const char nodestates[])
 551 {
 552         struct ctdb_context *ctdb;
 553         struct ctdb_public_ip_list *all_ips;
 554         struct ctdb_ipflags *ipflags;
 555
 556         uint32_t *lcp2_imbalances;
 557         bool *newly_healthy;
 558
 559         ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false);
 560
 561         lcp2_init(ctdb, ipflags, all_ips, NULL,
 562                   &lcp2_imbalances, &newly_healthy);
 563
 564         lcp2_failback(ctdb, ipflags,
 565                       all_ips, lcp2_imbalances, newly_healthy);
 566
 567         print_ctdb_public_ip_list(all_ips);
 568
 569         talloc_free(ctdb);
 570 }
 571
 572 /* IP layout is read from stdin.  See comment for ctdb_test_init() for
 573  * explanation of read_ips_for_multiple_nodes.
 574  */
 575 void ctdb_test_ctdb_takeover_run_core(const char nodestates[],
 576                                       bool read_ips_for_multiple_nodes)
 577 {
 578         struct ctdb_context *ctdb;
 579         struct ctdb_public_ip_list *all_ips;
 580         struct ctdb_ipflags *ipflags;
 581
 582         ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags,
 583                        read_ips_for_multiple_nodes);
 584
 585         ctdb_takeover_run_core(ctdb, ipflags, &all_ips, NULL);
 586
 587         print_ctdb_public_ip_list(all_ips);
 588
 589         talloc_free(ctdb);
 590 }
 591
 592 void usage(void)
 593 {
 594         fprintf(stderr, "usage: ctdb_takeover_tests <op>\n");
 595         exit(1);
 596 }
 597
 598 int main(int argc, const char *argv[])
 599 {
 600         LogLevel = DEBUG_DEBUG;
 601         if (getenv("CTDB_TEST_LOGLEVEL")) {
 602                 LogLevel = atoi(getenv("CTDB_TEST_LOGLEVEL"));
 603         }
 604
 605         if (argc < 2) {
 606                 usage();
 607         }
 608
 609         if (strcmp(argv[1], "ip_list") == 0) {
 610                 ctdb_test_read_ctdb_public_ip_list();
 611         } else if (argc == 3 && strcmp(argv[1], "ip_info") == 0) {
 612                 ctdb_test_read_ctdb_public_ip_info(argv[2]);
 613         } else if (strcmp(argv[1], "ip_distance") == 0) {
 614                 ctdb_test_ip_distance();
 615         } else if (argc == 4 && strcmp(argv[1], "ip_distance_2_sum") == 0) {
 616                 ctdb_test_ip_distance_2_sum(argv[2], atoi(argv[3]));
 617         } else if (argc >= 3 && strcmp(argv[1], "lcp2_imbalance") == 0) {
 618                 ctdb_test_lcp2_imbalance(atoi(argv[2]));
 619         } else if (argc == 3 && strcmp(argv[1], "lcp2_allocate_unassigned") == 0) {
 620                 ctdb_test_lcp2_allocate_unassigned(argv[2]);
 621         } else if (argc == 3 && strcmp(argv[1], "lcp2_failback") == 0) {
 622                 ctdb_test_lcp2_failback(argv[2]);
 623         } else if (argc == 3 && strcmp(argv[1], "lcp2_failback_loop") == 0) {
 624                 ctdb_test_lcp2_failback_loop(argv[2]);
 625         } else if (argc == 3 &&
 626                    strcmp(argv[1], "ctdb_takeover_run_core") == 0) {
 627                 ctdb_test_ctdb_takeover_run_core(argv[2], false);
 628         } else if (argc == 4 &&
 629                    strcmp(argv[1], "ctdb_takeover_run_core") == 0 &&
 630                    strcmp(argv[3], "multi") == 0) {
 631                 ctdb_test_ctdb_takeover_run_core(argv[2], true);
 632         } else {
 633                 usage();
 634         }
 635
 636         return 0;
 637 }