1844f1ff1966de1978e922d28d355540e1b064b2
[amitay/samba.git] / ctdb / tools / ctdb.c
1 /*
2    CTDB control tool
3
4    Copyright (C) Amitay Isaacs  2015
5
6    This program is free software; you can redistribute it and/or modify
7    it under the terms of the GNU General Public License as published by
8    the Free Software Foundation; either version 3 of the License, or
9    (at your option) any later version.
10
11    This program is distributed in the hope that it will be useful,
12    but WITHOUT ANY WARRANTY; without even the implied warranty of
13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14    GNU General Public License for more details.
15
16    You should have received a copy of the GNU General Public License
17    along with this program; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "replace.h"
21 #include "system/network.h"
22 #include "system/filesys.h"
23 #include "system/time.h"
24 #include "system/wait.h"
25 #include "system/dir.h"
26
27 #include <ctype.h>
28 #include <popt.h>
29 #include <talloc.h>
30 #include <tevent.h>
31 #include <tdb.h>
32
33 #include "common/version.h"
34 #include "lib/util/debug.h"
35 #include "lib/util/samba_util.h"
36 #include "lib/util/sys_rw.h"
37
38 #include "common/db_hash.h"
39 #include "common/logging.h"
40 #include "common/path.h"
41 #include "protocol/protocol.h"
42 #include "protocol/protocol_api.h"
43 #include "protocol/protocol_util.h"
44 #include "common/system_socket.h"
45 #include "client/client.h"
46 #include "client/client_sync.h"
47
48 #define TIMEOUT()       timeval_current_ofs(options.timelimit, 0)
49
50 #define SRVID_CTDB_TOOL    (CTDB_SRVID_TOOL_RANGE | 0x0001000000000000LL)
51 #define SRVID_CTDB_PUSHDB  (CTDB_SRVID_TOOL_RANGE | 0x0002000000000000LL)
52
53 static struct {
54         const char *debuglevelstr;
55         int timelimit;
56         int pnn;
57         int machinereadable;
58         const char *sep;
59         int machineparsable;
60         int verbose;
61         int maxruntime;
62         int printemptyrecords;
63         int printdatasize;
64         int printlmaster;
65         int printhash;
66         int printrecordflags;
67 } options;
68
69 static poptContext pc;
70
71 struct ctdb_context {
72         struct tevent_context *ev;
73         struct ctdb_client_context *client;
74         struct ctdb_node_map *nodemap;
75         uint32_t pnn, cmd_pnn;
76         uint64_t srvid;
77 };
78
79 static void usage(const char *command);
80
81 /*
82  * Utility Functions
83  */
84
85 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
86 {
87         return (tv2->tv_sec - tv->tv_sec) +
88                (tv2->tv_usec - tv->tv_usec) * 1.0e-6;
89 }
90
91 static struct ctdb_node_and_flags *get_node_by_pnn(
92                                         struct ctdb_node_map *nodemap,
93                                         uint32_t pnn)
94 {
95         int i;
96
97         for (i=0; i<nodemap->num; i++) {
98                 if (nodemap->node[i].pnn == pnn) {
99                         return &nodemap->node[i];
100                 }
101         }
102         return NULL;
103 }
104
105 static const char *pretty_print_flags(TALLOC_CTX *mem_ctx, uint32_t flags)
106 {
107         static const struct {
108                 uint32_t flag;
109                 const char *name;
110         } flag_names[] = {
111                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
112                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
113                 { NODE_FLAGS_BANNED,                "BANNED" },
114                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
115                 { NODE_FLAGS_DELETED,               "DELETED" },
116                 { NODE_FLAGS_STOPPED,               "STOPPED" },
117                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
118         };
119         char *flags_str = NULL;
120         int i;
121
122         for (i=0; i<ARRAY_SIZE(flag_names); i++) {
123                 if (flags & flag_names[i].flag) {
124                         if (flags_str == NULL) {
125                                 flags_str = talloc_asprintf(mem_ctx,
126                                                 "%s", flag_names[i].name);
127                         } else {
128                                 flags_str = talloc_asprintf_append(flags_str,
129                                                 "|%s", flag_names[i].name);
130                         }
131                         if (flags_str == NULL) {
132                                 return "OUT-OF-MEMORY";
133                         }
134                 }
135         }
136         if (flags_str == NULL) {
137                 return "OK";
138         }
139
140         return flags_str;
141 }
142
143 static uint64_t next_srvid(struct ctdb_context *ctdb)
144 {
145         ctdb->srvid += 1;
146         return ctdb->srvid;
147 }
148
149 /*
150  * Get consistent nodemap information.
151  *
152  * If nodemap is already cached, use that. If not get it.
153  * If the current node is BANNED, then get nodemap from "better" node.
154  */
155 static struct ctdb_node_map *get_nodemap(struct ctdb_context *ctdb, bool force)
156 {
157         TALLOC_CTX *tmp_ctx;
158         struct ctdb_node_map *nodemap;
159         struct ctdb_node_and_flags *node;
160         uint32_t current_node;
161         int ret;
162
163         if (force) {
164                 TALLOC_FREE(ctdb->nodemap);
165         }
166
167         if (ctdb->nodemap != NULL) {
168                 return ctdb->nodemap;
169         }
170
171         tmp_ctx = talloc_new(ctdb);
172         if (tmp_ctx == NULL) {
173                 return false;
174         }
175
176         current_node = ctdb->pnn;
177 again:
178         ret = ctdb_ctrl_get_nodemap(tmp_ctx, ctdb->ev, ctdb->client,
179                                     current_node, TIMEOUT(), &nodemap);
180         if (ret != 0) {
181                 fprintf(stderr, "Failed to get nodemap from node %u\n",
182                         current_node);
183                 goto failed;
184         }
185
186         node = get_node_by_pnn(nodemap, current_node);
187         if (node->flags & NODE_FLAGS_BANNED) {
188                 /* Pick next node */
189                 do {
190                         current_node = (current_node + 1) % nodemap->num;
191                         node = get_node_by_pnn(nodemap, current_node);
192                         if (! (node->flags &
193                               (NODE_FLAGS_DELETED|NODE_FLAGS_DISCONNECTED))) {
194                                 break;
195                         }
196                 } while (current_node != ctdb->pnn);
197
198                 if (current_node == ctdb->pnn) {
199                         /* Tried all nodes in the cluster */
200                         fprintf(stderr, "Warning: All nodes are banned.\n");
201                         goto failed;
202                 }
203
204                 goto again;
205         }
206
207         ctdb->nodemap = talloc_steal(ctdb, nodemap);
208         return nodemap;
209
210 failed:
211         talloc_free(tmp_ctx);
212         return NULL;
213 }
214
215 static bool verify_pnn(struct ctdb_context *ctdb, int pnn)
216 {
217         struct ctdb_node_map *nodemap;
218         bool found;
219         int i;
220
221         if (pnn == -1) {
222                 return false;
223         }
224
225         nodemap = get_nodemap(ctdb, false);
226         if (nodemap == NULL) {
227                 return false;
228         }
229
230         found = false;
231         for (i=0; i<nodemap->num; i++) {
232                 if (nodemap->node[i].pnn == pnn) {
233                         found = true;
234                         break;
235                 }
236         }
237         if (! found) {
238                 fprintf(stderr, "Node %u does not exist\n", pnn);
239                 return false;
240         }
241
242         if (nodemap->node[i].flags &
243             (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED)) {
244                 fprintf(stderr, "Node %u has status %s\n", pnn,
245                         pretty_print_flags(ctdb, nodemap->node[i].flags));
246                 return false;
247         }
248
249         return true;
250 }
251
252 static struct ctdb_node_map *talloc_nodemap(TALLOC_CTX *mem_ctx,
253                                             struct ctdb_node_map *nodemap)
254 {
255         struct ctdb_node_map *nodemap2;
256
257         nodemap2 = talloc_zero(mem_ctx, struct ctdb_node_map);
258         if (nodemap2 == NULL) {
259                 return NULL;
260         }
261
262         nodemap2->node = talloc_array(nodemap2, struct ctdb_node_and_flags,
263                                       nodemap->num);
264         if (nodemap2->node == NULL) {
265                 talloc_free(nodemap2);
266                 return NULL;
267         }
268
269         return nodemap2;
270 }
271
272 /*
273  * Get the number and the list of matching nodes
274  *
275  *   nodestring :=  NULL | all | pnn,[pnn,...]
276  *
277  * If nodestring is NULL, use the current node.
278  */
279 static bool parse_nodestring(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
280                              const char *nodestring,
281                              struct ctdb_node_map **out)
282 {
283         struct ctdb_node_map *nodemap, *nodemap2;
284         struct ctdb_node_and_flags *node;
285         int i;
286
287         nodemap = get_nodemap(ctdb, false);
288         if (nodemap == NULL) {
289                 return false;
290         }
291
292         nodemap2 = talloc_nodemap(mem_ctx, nodemap);
293         if (nodemap2 == NULL) {
294                 return false;
295         }
296
297         if (nodestring == NULL) {
298                 for (i=0; i<nodemap->num; i++) {
299                         if (nodemap->node[i].pnn == ctdb->cmd_pnn) {
300                                 nodemap2->node[0] = nodemap->node[i];
301                                 break;
302                         }
303                 }
304                 nodemap2->num = 1;
305
306                 goto done;
307         }
308
309         if (strcmp(nodestring, "all") == 0) {
310                 for (i=0; i<nodemap->num; i++) {
311                         nodemap2->node[i] = nodemap->node[i];
312                 }
313                 nodemap2->num = nodemap->num;
314
315                 goto done;
316         } else {
317                 char *ns, *tok;
318
319                 ns = talloc_strdup(mem_ctx, nodestring);
320                 if (ns == NULL) {
321                         return false;
322                 }
323
324                 tok = strtok(ns, ",");
325                 while (tok != NULL) {
326                         uint32_t pnn;
327                         char *endptr;
328
329                         pnn = (uint32_t)strtoul(tok, &endptr, 0);
330                         if (pnn == 0 && tok == endptr) {
331                                 fprintf(stderr, "Invalid node %s\n", tok);
332                                         return false;
333                         }
334
335                         node = get_node_by_pnn(nodemap, pnn);
336                         if (node == NULL) {
337                                 fprintf(stderr, "Node %u does not exist\n",
338                                         pnn);
339                                 return false;
340                         }
341
342                         nodemap2->node[nodemap2->num] = *node;
343                         nodemap2->num += 1;
344
345                         tok = strtok(NULL, ",");
346                 }
347         }
348
349 done:
350         *out = nodemap2;
351         return true;
352 }
353
354 /* Compare IP address */
355 static bool ctdb_same_ip(ctdb_sock_addr *ip1, ctdb_sock_addr *ip2)
356 {
357         bool ret = false;
358
359         if (ip1->sa.sa_family != ip2->sa.sa_family) {
360                 return false;
361         }
362
363         switch (ip1->sa.sa_family) {
364         case AF_INET:
365                 ret = (memcmp(&ip1->ip.sin_addr, &ip2->ip.sin_addr,
366                               sizeof(struct in_addr)) == 0);
367                 break;
368
369         case AF_INET6:
370                 ret = (memcmp(&ip1->ip6.sin6_addr, &ip2->ip6.sin6_addr,
371                               sizeof(struct in6_addr)) == 0);
372                 break;
373         }
374
375         return ret;
376 }
377
378 /* Append a node to a node map with given address and flags */
379 static bool node_map_add(struct ctdb_node_map *nodemap,
380                          const char *nstr, uint32_t flags)
381 {
382         ctdb_sock_addr addr;
383         uint32_t num;
384         struct ctdb_node_and_flags *n;
385         int ret;
386
387         ret = ctdb_sock_addr_from_string(nstr, &addr, false);
388         if (ret != 0) {
389                 fprintf(stderr, "Invalid IP address %s\n", nstr);
390                 return false;
391         }
392
393         num = nodemap->num;
394         nodemap->node = talloc_realloc(nodemap, nodemap->node,
395                                        struct ctdb_node_and_flags, num+1);
396         if (nodemap->node == NULL) {
397                 return false;
398         }
399
400         n = &nodemap->node[num];
401         n->addr = addr;
402         n->pnn = num;
403         n->flags = flags;
404
405         nodemap->num = num+1;
406         return true;
407 }
408
409 /* Read a nodes file into a node map */
410 static struct ctdb_node_map *ctdb_read_nodes_file(TALLOC_CTX *mem_ctx,
411                                                   const char *nlist)
412 {
413         char **lines;
414         int nlines;
415         int i;
416         struct ctdb_node_map *nodemap;
417
418         nodemap = talloc_zero(mem_ctx, struct ctdb_node_map);
419         if (nodemap == NULL) {
420                 return NULL;
421         }
422
423         lines = file_lines_load(nlist, &nlines, 0, mem_ctx);
424         if (lines == NULL) {
425                 return NULL;
426         }
427
428         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
429                 nlines--;
430         }
431
432         for (i=0; i<nlines; i++) {
433                 char *node;
434                 uint32_t flags;
435                 size_t len;
436
437                 node = lines[i];
438                 /* strip leading spaces */
439                 while((*node == ' ') || (*node == '\t')) {
440                         node++;
441                 }
442
443                 len = strlen(node);
444
445                 /* strip trailing spaces */
446                 while ((len > 1) &&
447                        ((node[len-1] == ' ') || (node[len-1] == '\t')))
448                 {
449                         node[len-1] = '\0';
450                         len--;
451                 }
452
453                 if (len == 0) {
454                         continue;
455                 }
456                 if (*node == '#') {
457                         /* A "deleted" node is a node that is
458                            commented out in the nodes file.  This is
459                            used instead of removing a line, which
460                            would cause subsequent nodes to change
461                            their PNN. */
462                         flags = NODE_FLAGS_DELETED;
463                         node = discard_const("0.0.0.0");
464                 } else {
465                         flags = 0;
466                 }
467                 if (! node_map_add(nodemap, node, flags)) {
468                         talloc_free(lines);
469                         TALLOC_FREE(nodemap);
470                         return NULL;
471                 }
472         }
473
474         talloc_free(lines);
475         return nodemap;
476 }
477
478 static struct ctdb_node_map *read_nodes_file(TALLOC_CTX *mem_ctx, uint32_t pnn)
479 {
480         struct ctdb_node_map *nodemap;
481         const char *nodes_list = NULL;
482
483         const char *basedir = getenv("CTDB_BASE");
484         if (basedir == NULL) {
485                 basedir = CTDB_ETCDIR;
486         }
487         nodes_list = talloc_asprintf(mem_ctx, "%s/nodes", basedir);
488         if (nodes_list == NULL) {
489                 fprintf(stderr, "Memory allocation error\n");
490                 return NULL;
491         }
492
493         nodemap = ctdb_read_nodes_file(mem_ctx, nodes_list);
494         if (nodemap == NULL) {
495                 fprintf(stderr, "Failed to read nodes file \"%s\"\n",
496                         nodes_list);
497                 return NULL;
498         }
499
500         return nodemap;
501 }
502
503 static struct ctdb_dbid *db_find(TALLOC_CTX *mem_ctx,
504                                  struct ctdb_context *ctdb,
505                                  struct ctdb_dbid_map *dbmap,
506                                  const char *db_name)
507 {
508         struct ctdb_dbid *db = NULL;
509         const char *name;
510         int ret, i;
511
512         for (i=0; i<dbmap->num; i++) {
513                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
514                                            ctdb->pnn, TIMEOUT(),
515                                            dbmap->dbs[i].db_id, &name);
516                 if (ret != 0) {
517                         return false;
518                 }
519
520                 if (strcmp(db_name, name) == 0) {
521                         talloc_free(discard_const(name));
522                         db = &dbmap->dbs[i];
523                         break;
524                 }
525         }
526
527         return db;
528 }
529
530 static bool db_exists(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
531                       const char *db_arg, uint32_t *db_id,
532                       const char **db_name, uint8_t *db_flags)
533 {
534         struct ctdb_dbid_map *dbmap;
535         struct ctdb_dbid *db = NULL;
536         uint32_t id = 0;
537         const char *name = NULL;
538         int ret, i;
539
540         ret = ctdb_ctrl_get_dbmap(mem_ctx, ctdb->ev, ctdb->client,
541                                   ctdb->pnn, TIMEOUT(), &dbmap);
542         if (ret != 0) {
543                 return false;
544         }
545
546         if (strncmp(db_arg, "0x", 2) == 0) {
547                 id = strtoul(db_arg, NULL, 0);
548                 for (i=0; i<dbmap->num; i++) {
549                         if (id == dbmap->dbs[i].db_id) {
550                                 db = &dbmap->dbs[i];
551                                 break;
552                         }
553                 }
554         } else {
555                 name = db_arg;
556                 db = db_find(mem_ctx, ctdb, dbmap, name);
557         }
558
559         if (db == NULL) {
560                 fprintf(stderr, "No database matching '%s' found\n", db_arg);
561                 return false;
562         }
563
564         if (name == NULL) {
565                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
566                                            ctdb->pnn, TIMEOUT(), id, &name);
567                 if (ret != 0) {
568                         return false;
569                 }
570         }
571
572         if (db_id != NULL) {
573                 *db_id = db->db_id;
574         }
575         if (db_name != NULL) {
576                 *db_name = talloc_strdup(mem_ctx, name);
577         }
578         if (db_flags != NULL) {
579                 *db_flags = db->flags;
580         }
581         return true;
582 }
583
584 static int h2i(char h)
585 {
586         if (h >= 'a' && h <= 'f') {
587                 return h - 'a' + 10;
588         }
589         if (h >= 'A' && h <= 'F') {
590                 return h - 'A' + 10;
591         }
592         return h - '0';
593 }
594
595 static int hex_to_data(const char *str, size_t len, TALLOC_CTX *mem_ctx,
596                        TDB_DATA *out)
597 {
598         int i;
599         TDB_DATA data;
600
601         if (len & 0x01) {
602                 fprintf(stderr, "Key (%s) contains odd number of hex digits\n",
603                         str);
604                 return EINVAL;
605         }
606
607         data.dsize = len / 2;
608         data.dptr = talloc_size(mem_ctx, data.dsize);
609         if (data.dptr == NULL) {
610                 return ENOMEM;
611         }
612
613         for (i=0; i<data.dsize; i++) {
614                 data.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
615         }
616
617         *out = data;
618         return 0;
619 }
620
621 static int str_to_data(const char *str, size_t len, TALLOC_CTX *mem_ctx,
622                        TDB_DATA *out)
623 {
624         TDB_DATA data;
625         int ret = 0;
626
627         if (strncmp(str, "0x", 2) == 0) {
628                 ret = hex_to_data(str+2, len-2, mem_ctx, &data);
629         } else {
630                 data.dptr = talloc_memdup(mem_ctx, str, len);
631                 if (data.dptr == NULL) {
632                         return ENOMEM;
633                 }
634                 data.dsize = len;
635         }
636
637         *out = data;
638         return ret;
639 }
640
641 static int run_helper(TALLOC_CTX *mem_ctx, const char *command,
642                       const char *path, int argc, const char **argv)
643 {
644         pid_t pid;
645         int save_errno, status, ret;
646         const char **new_argv;
647         int i;
648
649         new_argv = talloc_array(mem_ctx, const char *, argc + 2);
650         if (new_argv == NULL) {
651                 return ENOMEM;
652         }
653
654         new_argv[0] = path;
655         for (i=0; i<argc; i++) {
656                 new_argv[i+1] = argv[i];
657         }
658         new_argv[argc+1] = NULL;
659
660         pid = fork();
661         if (pid < 0) {
662                 save_errno = errno;
663                 talloc_free(new_argv);
664                 fprintf(stderr, "Failed to fork %s (%s) - %s\n",
665                         command, path, strerror(save_errno));
666                 return save_errno;
667         }
668
669         if (pid == 0) {
670                 ret = execv(path, discard_const(new_argv));
671                 if (ret == -1) {
672                         _exit(64+errno);
673                 }
674                 /* Should not happen */
675                 _exit(64+ENOEXEC);
676         }
677
678         talloc_free(new_argv);
679
680         ret = waitpid(pid, &status, 0);
681         if (ret == -1) {
682                 save_errno = errno;
683                 fprintf(stderr, "waitpid() failed for %s - %s\n",
684                         command, strerror(save_errno));
685                 return save_errno;
686         }
687
688         if (WIFEXITED(status)) {
689                 int pstatus = WEXITSTATUS(status);
690                 if (WIFSIGNALED(status)) {
691                         fprintf(stderr, "%s terminated with signal %d\n",
692                                 command, WTERMSIG(status));
693                         ret = EINTR;
694                 } else if (pstatus >= 64 && pstatus < 255) {
695                         fprintf(stderr, "%s failed with error %d\n",
696                                 command, pstatus-64);
697                         ret = pstatus - 64;
698                 } else {
699                         ret = pstatus;
700                 }
701                 return ret;
702         } else if (WIFSIGNALED(status)) {
703                 fprintf(stderr, "%s terminated with signal %d\n",
704                         command, WTERMSIG(status));
705                 return EINTR;
706         }
707
708         return 0;
709 }
710
711 /*
712  * Command Functions
713  */
714
715 static int control_version(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
716                            int argc, const char **argv)
717 {
718         printf("%s\n", ctdb_version_string);
719         return 0;
720 }
721
722 static bool partially_online(TALLOC_CTX *mem_ctx,
723                              struct ctdb_context *ctdb,
724                              struct ctdb_node_and_flags *node)
725 {
726         struct ctdb_iface_list *iface_list;
727         int ret, i;
728         bool status = false;
729
730         if (node->flags != 0) {
731                 return false;
732         }
733
734         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
735                                    node->pnn, TIMEOUT(), &iface_list);
736         if (ret != 0) {
737                 return false;
738         }
739
740         status = false;
741         for (i=0; i < iface_list->num; i++) {
742                 if (iface_list->iface[i].link_state == 0) {
743                         status = true;
744                         break;
745                 }
746         }
747
748         return status;
749 }
750
751 static void print_nodemap_machine(TALLOC_CTX *mem_ctx,
752                                   struct ctdb_context *ctdb,
753                                   struct ctdb_node_map *nodemap,
754                                   uint32_t mypnn)
755 {
756         struct ctdb_node_and_flags *node;
757         int i;
758
759         printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
760                options.sep,
761                "Node", options.sep,
762                "IP", options.sep,
763                "Disconnected", options.sep,
764                "Banned", options.sep,
765                "Disabled", options.sep,
766                "Unhealthy", options.sep,
767                "Stopped", options.sep,
768                "Inactive", options.sep,
769                "PartiallyOnline", options.sep,
770                "ThisNode", options.sep);
771
772         for (i=0; i<nodemap->num; i++) {
773                 node = &nodemap->node[i];
774                 if (node->flags & NODE_FLAGS_DELETED) {
775                         continue;
776                 }
777
778                 printf("%s%u%s%s%s%d%s%d%s%d%s%d%s%d%s%d%s%d%s%c%s\n",
779                        options.sep,
780                        node->pnn, options.sep,
781                        ctdb_sock_addr_to_string(mem_ctx, &node->addr, false),
782                        options.sep,
783                        !! (node->flags & NODE_FLAGS_DISCONNECTED), options.sep,
784                        !! (node->flags & NODE_FLAGS_BANNED), options.sep,
785                        !! (node->flags & NODE_FLAGS_PERMANENTLY_DISABLED),
786                        options.sep,
787                        !! (node->flags & NODE_FLAGS_UNHEALTHY), options.sep,
788                        !! (node->flags & NODE_FLAGS_STOPPED), options.sep,
789                        !! (node->flags & NODE_FLAGS_INACTIVE), options.sep,
790                        partially_online(mem_ctx, ctdb, node), options.sep,
791                        (node->pnn == mypnn)?'Y':'N', options.sep);
792         }
793
794 }
795
796 static void print_nodemap(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
797                           struct ctdb_node_map *nodemap, uint32_t mypnn,
798                           bool print_header)
799 {
800         struct ctdb_node_and_flags *node;
801         int num_deleted_nodes = 0;
802         int i;
803
804         for (i=0; i<nodemap->num; i++) {
805                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
806                         num_deleted_nodes++;
807                 }
808         }
809
810         if (print_header) {
811                 if (num_deleted_nodes == 0) {
812                         printf("Number of nodes:%d\n", nodemap->num);
813                 } else {
814                         printf("Number of nodes:%d "
815                                "(including %d deleted nodes)\n",
816                                nodemap->num, num_deleted_nodes);
817                 }
818         }
819
820         for (i=0; i<nodemap->num; i++) {
821                 node = &nodemap->node[i];
822                 if (node->flags & NODE_FLAGS_DELETED) {
823                         continue;
824                 }
825
826                 printf("pnn:%u %-16s %s%s\n",
827                        node->pnn,
828                        ctdb_sock_addr_to_string(mem_ctx, &node->addr, false),
829                        partially_online(mem_ctx, ctdb, node) ?
830                                 "PARTIALLYONLINE" :
831                                 pretty_print_flags(mem_ctx, node->flags),
832                        node->pnn == mypnn ? " (THIS NODE)" : "");
833         }
834 }
835
836 static void print_status(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
837                          struct ctdb_node_map *nodemap, uint32_t mypnn,
838                          struct ctdb_vnn_map *vnnmap, int recmode,
839                          uint32_t recmaster)
840 {
841         int i;
842
843         print_nodemap(mem_ctx, ctdb, nodemap, mypnn, true);
844
845         if (vnnmap->generation == INVALID_GENERATION) {
846                 printf("Generation:INVALID\n");
847         } else {
848                 printf("Generation:%u\n", vnnmap->generation);
849         }
850         printf("Size:%d\n", vnnmap->size);
851         for (i=0; i<vnnmap->size; i++) {
852                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
853         }
854
855         printf("Recovery mode:%s (%d)\n",
856                recmode == CTDB_RECOVERY_NORMAL ? "NORMAL" : "RECOVERY",
857                recmode);
858         printf("Recovery master:%d\n", recmaster);
859 }
860
861 static int control_status(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
862                           int argc, const char **argv)
863 {
864         struct ctdb_node_map *nodemap;
865         struct ctdb_vnn_map *vnnmap;
866         int recmode;
867         uint32_t recmaster;
868         int ret;
869
870         if (argc != 0) {
871                 usage("status");
872         }
873
874         nodemap = get_nodemap(ctdb, false);
875         if (nodemap == NULL) {
876                 return 1;
877         }
878
879         if (options.machinereadable == 1) {
880                 print_nodemap_machine(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn);
881                 return 0;
882         }
883
884         ret = ctdb_ctrl_getvnnmap(mem_ctx, ctdb->ev, ctdb->client,
885                                   ctdb->cmd_pnn, TIMEOUT(), &vnnmap);
886         if (ret != 0) {
887                 return ret;
888         }
889
890         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
891                                     ctdb->cmd_pnn, TIMEOUT(), &recmode);
892         if (ret != 0) {
893                 return ret;
894         }
895
896         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
897                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
898         if (ret != 0) {
899                 return ret;
900         }
901
902         print_status(mem_ctx, ctdb, nodemap, ctdb->cmd_pnn, vnnmap,
903                      recmode, recmaster);
904         return 0;
905 }
906
907 static int control_uptime(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
908                           int argc, const char **argv)
909 {
910         struct ctdb_uptime *uptime;
911         int ret, tmp, days, hours, minutes, seconds;
912
913         ret = ctdb_ctrl_uptime(mem_ctx, ctdb->ev, ctdb->client,
914                                ctdb->cmd_pnn, TIMEOUT(), &uptime);
915         if (ret != 0) {
916                 return ret;
917         }
918
919         printf("Current time of node %-4u     :                %s",
920                ctdb->cmd_pnn, ctime(&uptime->current_time.tv_sec));
921
922         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
923         seconds = tmp % 60; tmp /= 60;
924         minutes = tmp % 60; tmp /= 60;
925         hours = tmp % 24; tmp /= 24;
926         days = tmp;
927
928         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s",
929                days, hours, minutes, seconds,
930                ctime(&uptime->ctdbd_start_time.tv_sec));
931
932         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
933         seconds = tmp % 60; tmp /= 60;
934         minutes = tmp % 60; tmp /= 60;
935         hours = tmp % 24; tmp /= 24;
936         days = tmp;
937
938         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s",
939                days, hours, minutes, seconds,
940                ctime(&uptime->last_recovery_finished.tv_sec));
941
942         printf("Duration of last recovery/failover: %lf seconds\n",
943                timeval_delta(&uptime->last_recovery_finished,
944                              &uptime->last_recovery_started));
945
946         return 0;
947 }
948
949 static int control_ping(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
950                         int argc, const char **argv)
951 {
952         struct timeval tv;
953         int ret, num_clients;
954
955         tv = timeval_current();
956         ret = ctdb_ctrl_ping(mem_ctx, ctdb->ev, ctdb->client,
957                              ctdb->cmd_pnn, TIMEOUT(), &num_clients);
958         if (ret != 0) {
959                 return ret;
960         }
961
962         printf("response from %u time=%.6f sec  (%d clients)\n",
963                ctdb->cmd_pnn, timeval_elapsed(&tv), num_clients);
964         return 0;
965 }
966
967 const char *runstate_to_string(enum ctdb_runstate runstate);
968 enum ctdb_runstate runstate_from_string(const char *runstate_str);
969
970 static int control_runstate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
971                             int argc, const char **argv)
972 {
973         enum ctdb_runstate runstate;
974         bool found;
975         int ret, i;
976
977         ret = ctdb_ctrl_get_runstate(mem_ctx, ctdb->ev, ctdb->client,
978                                      ctdb->cmd_pnn, TIMEOUT(), &runstate);
979         if (ret != 0) {
980                 return ret;
981         }
982
983         found = true;
984         for (i=0; i<argc; i++) {
985                 enum ctdb_runstate t;
986
987                 found = false;
988                 t = ctdb_runstate_from_string(argv[i]);
989                 if (t == CTDB_RUNSTATE_UNKNOWN) {
990                         printf("Invalid run state (%s)\n", argv[i]);
991                         return 1;
992                 }
993
994                 if (t == runstate) {
995                         found = true;
996                         break;
997                 }
998         }
999
1000         if (! found) {
1001                 printf("CTDB not in required run state (got %s)\n",
1002                        ctdb_runstate_to_string(runstate));
1003                 return 1;
1004         }
1005
1006         printf("%s\n", ctdb_runstate_to_string(runstate));
1007         return 0;
1008 }
1009
1010 static int control_getvar(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1011                           int argc, const char **argv)
1012 {
1013         struct ctdb_var_list *tun_var_list;
1014         uint32_t value;
1015         int ret, i;
1016         bool found;
1017
1018         if (argc != 1) {
1019                 usage("getvar");
1020         }
1021
1022         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1023                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1024         if (ret != 0) {
1025                 fprintf(stderr,
1026                         "Failed to get list of variables from node %u\n",
1027                         ctdb->cmd_pnn);
1028                 return ret;
1029         }
1030
1031         found = false;
1032         for (i=0; i<tun_var_list->count; i++) {
1033                 if (strcasecmp(tun_var_list->var[i], argv[0]) == 0) {
1034                         found = true;
1035                         break;
1036                 }
1037         }
1038
1039         if (! found) {
1040                 printf("No such tunable %s\n", argv[0]);
1041                 return 1;
1042         }
1043
1044         ret = ctdb_ctrl_get_tunable(mem_ctx, ctdb->ev, ctdb->client,
1045                                     ctdb->cmd_pnn, TIMEOUT(), argv[0], &value);
1046         if (ret != 0) {
1047                 return ret;
1048         }
1049
1050         printf("%-26s = %u\n", argv[0], value);
1051         return 0;
1052 }
1053
1054 static int control_setvar(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1055                           int argc, const char **argv)
1056 {
1057         struct ctdb_var_list *tun_var_list;
1058         struct ctdb_tunable tunable;
1059         int ret, i;
1060         bool found;
1061
1062         if (argc != 2) {
1063                 usage("setvar");
1064         }
1065
1066         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1067                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1068         if (ret != 0) {
1069                 fprintf(stderr,
1070                         "Failed to get list of variables from node %u\n",
1071                         ctdb->cmd_pnn);
1072                 return ret;
1073         }
1074
1075         found = false;
1076         for (i=0; i<tun_var_list->count; i++) {
1077                 if (strcasecmp(tun_var_list->var[i], argv[0]) == 0) {
1078                         found = true;
1079                         break;
1080                 }
1081         }
1082
1083         if (! found) {
1084                 printf("No such tunable %s\n", argv[0]);
1085                 return 1;
1086         }
1087
1088         tunable.name = argv[0];
1089         tunable.value = strtoul(argv[1], NULL, 0);
1090
1091         ret = ctdb_ctrl_set_tunable(mem_ctx, ctdb->ev, ctdb->client,
1092                                     ctdb->cmd_pnn, TIMEOUT(), &tunable);
1093         if (ret != 0) {
1094                 if (ret == 1) {
1095                         fprintf(stderr,
1096                                 "Setting obsolete tunable variable '%s'\n",
1097                                tunable.name);
1098                         return 0;
1099                 }
1100         }
1101
1102         return ret;
1103 }
1104
1105 static int control_listvars(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1106                             int argc, const char **argv)
1107 {
1108         struct ctdb_var_list *tun_var_list;
1109         int ret, i;
1110
1111         if (argc != 0) {
1112                 usage("listvars");
1113         }
1114
1115         ret = ctdb_ctrl_list_tunables(mem_ctx, ctdb->ev, ctdb->client,
1116                                       ctdb->cmd_pnn, TIMEOUT(), &tun_var_list);
1117         if (ret != 0) {
1118                 return ret;
1119         }
1120
1121         for (i=0; i<tun_var_list->count; i++) {
1122                 control_getvar(mem_ctx, ctdb, 1, &tun_var_list->var[i]);
1123         }
1124
1125         return 0;
1126 }
1127
1128 const struct {
1129         const char *name;
1130         uint32_t offset;
1131 } stats_fields[] = {
1132 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
1133         STATISTICS_FIELD(num_clients),
1134         STATISTICS_FIELD(frozen),
1135         STATISTICS_FIELD(recovering),
1136         STATISTICS_FIELD(num_recoveries),
1137         STATISTICS_FIELD(client_packets_sent),
1138         STATISTICS_FIELD(client_packets_recv),
1139         STATISTICS_FIELD(node_packets_sent),
1140         STATISTICS_FIELD(node_packets_recv),
1141         STATISTICS_FIELD(keepalive_packets_sent),
1142         STATISTICS_FIELD(keepalive_packets_recv),
1143         STATISTICS_FIELD(node.req_call),
1144         STATISTICS_FIELD(node.reply_call),
1145         STATISTICS_FIELD(node.req_dmaster),
1146         STATISTICS_FIELD(node.reply_dmaster),
1147         STATISTICS_FIELD(node.reply_error),
1148         STATISTICS_FIELD(node.req_message),
1149         STATISTICS_FIELD(node.req_control),
1150         STATISTICS_FIELD(node.reply_control),
1151         STATISTICS_FIELD(node.req_tunnel),
1152         STATISTICS_FIELD(client.req_call),
1153         STATISTICS_FIELD(client.req_message),
1154         STATISTICS_FIELD(client.req_control),
1155         STATISTICS_FIELD(client.req_tunnel),
1156         STATISTICS_FIELD(timeouts.call),
1157         STATISTICS_FIELD(timeouts.control),
1158         STATISTICS_FIELD(timeouts.traverse),
1159         STATISTICS_FIELD(locks.num_calls),
1160         STATISTICS_FIELD(locks.num_current),
1161         STATISTICS_FIELD(locks.num_pending),
1162         STATISTICS_FIELD(locks.num_failed),
1163         STATISTICS_FIELD(total_calls),
1164         STATISTICS_FIELD(pending_calls),
1165         STATISTICS_FIELD(childwrite_calls),
1166         STATISTICS_FIELD(pending_childwrite_calls),
1167         STATISTICS_FIELD(memory_used),
1168         STATISTICS_FIELD(max_hop_count),
1169         STATISTICS_FIELD(total_ro_delegations),
1170         STATISTICS_FIELD(total_ro_revokes),
1171 };
1172
1173 #define LATENCY_AVG(v)  ((v).num ? (v).total / (v).num : 0.0 )
1174
1175 static void print_statistics_machine(struct ctdb_statistics *s,
1176                                      bool show_header)
1177 {
1178         int i;
1179
1180         if (show_header) {
1181                 printf("CTDB version%s", options.sep);
1182                 printf("Current time of statistics%s", options.sep);
1183                 printf("Statistics collected since%s", options.sep);
1184                 for (i=0; i<ARRAY_SIZE(stats_fields); i++) {
1185                         printf("%s%s", stats_fields[i].name, options.sep);
1186                 }
1187                 printf("num_reclock_ctdbd_latency%s", options.sep);
1188                 printf("min_reclock_ctdbd_latency%s", options.sep);
1189                 printf("avg_reclock_ctdbd_latency%s", options.sep);
1190                 printf("max_reclock_ctdbd_latency%s", options.sep);
1191
1192                 printf("num_reclock_recd_latency%s", options.sep);
1193                 printf("min_reclock_recd_latency%s", options.sep);
1194                 printf("avg_reclock_recd_latency%s", options.sep);
1195                 printf("max_reclock_recd_latency%s", options.sep);
1196
1197                 printf("num_call_latency%s", options.sep);
1198                 printf("min_call_latency%s", options.sep);
1199                 printf("avg_call_latency%s", options.sep);
1200                 printf("max_call_latency%s", options.sep);
1201
1202                 printf("num_lockwait_latency%s", options.sep);
1203                 printf("min_lockwait_latency%s", options.sep);
1204                 printf("avg_lockwait_latency%s", options.sep);
1205                 printf("max_lockwait_latency%s", options.sep);
1206
1207                 printf("num_childwrite_latency%s", options.sep);
1208                 printf("min_childwrite_latency%s", options.sep);
1209                 printf("avg_childwrite_latency%s", options.sep);
1210                 printf("max_childwrite_latency%s", options.sep);
1211                 printf("\n");
1212         }
1213
1214         printf("%u%s", CTDB_PROTOCOL, options.sep);
1215         printf("%u%s", (uint32_t)s->statistics_current_time.tv_sec, options.sep);
1216         printf("%u%s", (uint32_t)s->statistics_start_time.tv_sec, options.sep);
1217         for (i=0;i<ARRAY_SIZE(stats_fields);i++) {
1218                 printf("%u%s",
1219                        *(uint32_t *)(stats_fields[i].offset+(uint8_t *)s),
1220                        options.sep);
1221         }
1222         printf("%u%s", s->reclock.ctdbd.num, options.sep);
1223         printf("%.6f%s", s->reclock.ctdbd.min, options.sep);
1224         printf("%.6f%s", LATENCY_AVG(s->reclock.ctdbd), options.sep);
1225         printf("%.6f%s", s->reclock.ctdbd.max, options.sep);
1226
1227         printf("%u%s", s->reclock.recd.num, options.sep);
1228         printf("%.6f%s", s->reclock.recd.min, options.sep);
1229         printf("%.6f%s", LATENCY_AVG(s->reclock.recd), options.sep);
1230         printf("%.6f%s", s->reclock.recd.max, options.sep);
1231
1232         printf("%d%s", s->call_latency.num, options.sep);
1233         printf("%.6f%s", s->call_latency.min, options.sep);
1234         printf("%.6f%s", LATENCY_AVG(s->call_latency), options.sep);
1235         printf("%.6f%s", s->call_latency.max, options.sep);
1236
1237         printf("%d%s", s->childwrite_latency.num, options.sep);
1238         printf("%.6f%s", s->childwrite_latency.min, options.sep);
1239         printf("%.6f%s", LATENCY_AVG(s->childwrite_latency), options.sep);
1240         printf("%.6f%s", s->childwrite_latency.max, options.sep);
1241         printf("\n");
1242 }
1243
1244 static void print_statistics(struct ctdb_statistics *s)
1245 {
1246         int tmp, days, hours, minutes, seconds;
1247         int i;
1248         const char *prefix = NULL;
1249         int preflen = 0;
1250
1251         tmp = s->statistics_current_time.tv_sec -
1252               s->statistics_start_time.tv_sec;
1253         seconds = tmp % 60; tmp /= 60;
1254         minutes = tmp % 60; tmp /= 60;
1255         hours   = tmp % 24; tmp /= 24;
1256         days    = tmp;
1257
1258         printf("CTDB version %u\n", CTDB_PROTOCOL);
1259         printf("Current time of statistics  :                %s",
1260                ctime(&s->statistics_current_time.tv_sec));
1261         printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s",
1262                days, hours, minutes, seconds,
1263                ctime(&s->statistics_start_time.tv_sec));
1264
1265         for (i=0; i<ARRAY_SIZE(stats_fields); i++) {
1266                 if (strchr(stats_fields[i].name, '.') != NULL) {
1267                         preflen = strcspn(stats_fields[i].name, ".") + 1;
1268                         if (! prefix ||
1269                             strncmp(prefix, stats_fields[i].name, preflen) != 0) {
1270                                 prefix = stats_fields[i].name;
1271                                 printf(" %*.*s\n", preflen-1, preflen-1,
1272                                        stats_fields[i].name);
1273                         }
1274                 } else {
1275                         preflen = 0;
1276                 }
1277                 printf(" %*s%-22s%*s%10u\n", preflen ? 4 : 0, "",
1278                        stats_fields[i].name+preflen, preflen ? 0 : 4, "",
1279                        *(uint32_t *)(stats_fields[i].offset+(uint8_t *)s));
1280         }
1281
1282         printf(" hop_count_buckets:");
1283         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
1284                 printf(" %d", s->hop_count_bucket[i]);
1285         }
1286         printf("\n");
1287         printf(" lock_buckets:");
1288         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
1289                 printf(" %d", s->locks.buckets[i]);
1290         }
1291         printf("\n");
1292         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1293                "locks_latency      MIN/AVG/MAX",
1294                s->locks.latency.min, LATENCY_AVG(s->locks.latency),
1295                s->locks.latency.max, s->locks.latency.num);
1296
1297         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1298                "reclock_ctdbd      MIN/AVG/MAX",
1299                s->reclock.ctdbd.min, LATENCY_AVG(s->reclock.ctdbd),
1300                s->reclock.ctdbd.max, s->reclock.ctdbd.num);
1301
1302         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1303                "reclock_recd       MIN/AVG/MAX",
1304                s->reclock.recd.min, LATENCY_AVG(s->reclock.recd),
1305                s->reclock.recd.max, s->reclock.recd.num);
1306
1307         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1308                "call_latency       MIN/AVG/MAX",
1309                s->call_latency.min, LATENCY_AVG(s->call_latency),
1310                s->call_latency.max, s->call_latency.num);
1311
1312         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
1313                "childwrite_latency MIN/AVG/MAX",
1314                s->childwrite_latency.min,
1315                LATENCY_AVG(s->childwrite_latency),
1316                s->childwrite_latency.max, s->childwrite_latency.num);
1317 }
1318
1319 static int control_statistics(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1320                               int argc, const char **argv)
1321 {
1322         struct ctdb_statistics *stats;
1323         int ret;
1324
1325         if (argc != 0) {
1326                 usage("statistics");
1327         }
1328
1329         ret = ctdb_ctrl_statistics(mem_ctx, ctdb->ev, ctdb->client,
1330                                    ctdb->cmd_pnn, TIMEOUT(), &stats);
1331         if (ret != 0) {
1332                 return ret;
1333         }
1334
1335         if (options.machinereadable) {
1336                 print_statistics_machine(stats, true);
1337         } else {
1338                 print_statistics(stats);
1339         }
1340
1341         return 0;
1342 }
1343
1344 static int control_statistics_reset(TALLOC_CTX *mem_ctx,
1345                                     struct ctdb_context *ctdb,
1346                                     int argc, const char **argv)
1347 {
1348         int ret;
1349
1350         if (argc != 0) {
1351                 usage("statisticsreset");
1352         }
1353
1354         ret = ctdb_ctrl_statistics_reset(mem_ctx, ctdb->ev, ctdb->client,
1355                                          ctdb->cmd_pnn, TIMEOUT());
1356         if (ret != 0) {
1357                 return ret;
1358         }
1359
1360         return 0;
1361 }
1362
1363 static int control_stats(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1364                          int argc, const char **argv)
1365 {
1366         struct ctdb_statistics_list *slist;
1367         int ret, count = 0, i;
1368         bool show_header = true;
1369
1370         if (argc > 1) {
1371                 usage("stats");
1372         }
1373
1374         if (argc == 1) {
1375                 count = atoi(argv[0]);
1376         }
1377
1378         ret = ctdb_ctrl_get_stat_history(mem_ctx, ctdb->ev, ctdb->client,
1379                                          ctdb->cmd_pnn, TIMEOUT(), &slist);
1380         if (ret != 0) {
1381                 return ret;
1382         }
1383
1384         for (i=0; i<slist->num; i++) {
1385                 if (slist->stats[i].statistics_start_time.tv_sec == 0) {
1386                         continue;
1387                 }
1388                 if (options.machinereadable == 1) {
1389                         print_statistics_machine(&slist->stats[i],
1390                                                  show_header);
1391                         show_header = false;
1392                 } else {
1393                         print_statistics(&slist->stats[i]);
1394                 }
1395                 if (count > 0 && i == count) {
1396                         break;
1397                 }
1398         }
1399
1400         return 0;
1401 }
1402
1403 static int ctdb_public_ip_cmp(const void *a, const void *b)
1404 {
1405         const struct ctdb_public_ip *ip_a = a;
1406         const struct ctdb_public_ip *ip_b = b;
1407
1408         return ctdb_sock_addr_cmp(&ip_a->addr, &ip_b->addr);
1409 }
1410
1411 static void print_ip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1412                      struct ctdb_public_ip_list *ips,
1413                      struct ctdb_public_ip_info **ipinfo,
1414                      bool all_nodes)
1415 {
1416         int i, j;
1417         char *conf, *avail, *active;
1418
1419         if (options.machinereadable == 1) {
1420                 printf("%s%s%s%s%s", options.sep,
1421                        "Public IP", options.sep,
1422                        "Node", options.sep);
1423                 if (options.verbose == 1) {
1424                         printf("%s%s%s%s%s%s\n",
1425                                "ActiveInterfaces", options.sep,
1426                                "AvailableInterfaces", options.sep,
1427                                "ConfiguredInterfaces", options.sep);
1428                 } else {
1429                         printf("\n");
1430                 }
1431         } else {
1432                 if (all_nodes) {
1433                         printf("Public IPs on ALL nodes\n");
1434                 } else {
1435                         printf("Public IPs on node %u\n", ctdb->cmd_pnn);
1436                 }
1437         }
1438
1439         for (i = 0; i < ips->num; i++) {
1440
1441                 if (options.machinereadable == 1) {
1442                         printf("%s%s%s%d%s", options.sep,
1443                                ctdb_sock_addr_to_string(
1444                                        mem_ctx, &ips->ip[i].addr, false),
1445                                options.sep,
1446                                (int)ips->ip[i].pnn, options.sep);
1447                 } else {
1448                         printf("%s", ctdb_sock_addr_to_string(
1449                                        mem_ctx, &ips->ip[i].addr, false));
1450                 }
1451
1452                 if (options.verbose == 0) {
1453                         if (options.machinereadable == 1) {
1454                                 printf("\n");
1455                         } else {
1456                                 printf(" %d\n", (int)ips->ip[i].pnn);
1457                         }
1458                         continue;
1459                 }
1460
1461                 conf = NULL;
1462                 avail = NULL;
1463                 active = NULL;
1464
1465                 if (ipinfo[i] == NULL) {
1466                         goto skip_ipinfo;
1467                 }
1468
1469                 for (j=0; j<ipinfo[i]->ifaces->num; j++) {
1470                         struct ctdb_iface *iface;
1471
1472                         iface = &ipinfo[i]->ifaces->iface[j];
1473                         if (conf == NULL) {
1474                                 conf = talloc_strdup(mem_ctx, iface->name);
1475                         } else {
1476                                 conf = talloc_asprintf_append(
1477                                                 conf, ",%s", iface->name);
1478                         }
1479
1480                         if (ipinfo[i]->active_idx == j) {
1481                                 active = iface->name;
1482                         }
1483
1484                         if (iface->link_state == 0) {
1485                                 continue;
1486                         }
1487
1488                         if (avail == NULL) {
1489                                 avail = talloc_strdup(mem_ctx, iface->name);
1490                         } else {
1491                                 avail = talloc_asprintf_append(
1492                                                 avail, ",%s", iface->name);
1493                         }
1494                 }
1495
1496         skip_ipinfo:
1497
1498                 if (options.machinereadable == 1) {
1499                         printf("%s%s%s%s%s%s\n",
1500                                active ? active : "", options.sep,
1501                                avail ? avail : "", options.sep,
1502                                conf ? conf : "", options.sep);
1503                 } else {
1504                         printf(" node[%d] active[%s] available[%s]"
1505                                " configured[%s]\n",
1506                                (int)ips->ip[i].pnn, active ? active : "",
1507                                avail ? avail : "", conf ? conf : "");
1508                 }
1509         }
1510 }
1511
1512 static int collect_ips(uint8_t *keybuf, size_t keylen, uint8_t *databuf,
1513                        size_t datalen, void *private_data)
1514 {
1515         struct ctdb_public_ip_list *ips = talloc_get_type_abort(
1516                 private_data, struct ctdb_public_ip_list);
1517         struct ctdb_public_ip *ip;
1518
1519         ip = (struct ctdb_public_ip *)databuf;
1520         ips->ip[ips->num] = *ip;
1521         ips->num += 1;
1522
1523         return 0;
1524 }
1525
1526 static int get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
1527                               struct ctdb_public_ip_list **out)
1528 {
1529         struct ctdb_node_map *nodemap;
1530         struct ctdb_public_ip_list *ips;
1531         struct db_hash_context *ipdb;
1532         uint32_t *pnn_list;
1533         int ret, count, i, j;
1534
1535         nodemap = get_nodemap(ctdb, false);
1536         if (nodemap == NULL) {
1537                 return 1;
1538         }
1539
1540         ret = db_hash_init(mem_ctx, "ips", 101, DB_HASH_COMPLEX, &ipdb);
1541         if (ret != 0) {
1542                 goto failed;
1543         }
1544
1545         count = list_of_active_nodes(nodemap, CTDB_UNKNOWN_PNN, mem_ctx,
1546                                      &pnn_list);
1547         if (count <= 0) {
1548                 goto failed;
1549         }
1550
1551         for (i=0; i<count; i++) {
1552                 ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
1553                                                pnn_list[i], TIMEOUT(),
1554                                                false, &ips);
1555                 if (ret != 0) {
1556                         goto failed;
1557                 }
1558
1559                 for (j=0; j<ips->num; j++) {
1560                         struct ctdb_public_ip ip;
1561
1562                         ip.pnn = ips->ip[j].pnn;
1563                         ip.addr = ips->ip[j].addr;
1564
1565                         if (pnn_list[i] == ip.pnn) {
1566                                 /* Node claims IP is hosted on it, so
1567                                  * save that information
1568                                  */
1569                                 ret = db_hash_add(ipdb, (uint8_t *)&ip.addr,
1570                                                   sizeof(ip.addr),
1571                                                   (uint8_t *)&ip, sizeof(ip));
1572                                 if (ret != 0) {
1573                                         goto failed;
1574                                 }
1575                         } else {
1576                                 /* Node thinks IP is hosted elsewhere,
1577                                  * so overwrite with CTDB_UNKNOWN_PNN
1578                                  * if there's no existing entry
1579                                  */
1580                                 ret = db_hash_exists(ipdb, (uint8_t *)&ip.addr,
1581                                                      sizeof(ip.addr));
1582                                 if (ret == ENOENT) {
1583                                         ip.pnn = CTDB_UNKNOWN_PNN;
1584                                         ret = db_hash_add(ipdb,
1585                                                           (uint8_t *)&ip.addr,
1586                                                           sizeof(ip.addr),
1587                                                           (uint8_t *)&ip,
1588                                                           sizeof(ip));
1589                                         if (ret != 0) {
1590                                                 goto failed;
1591                                         }
1592                                 }
1593                         }
1594                 }
1595
1596                 TALLOC_FREE(ips);
1597         }
1598
1599         talloc_free(pnn_list);
1600
1601         ret = db_hash_traverse(ipdb, NULL, NULL, &count);
1602         if (ret != 0) {
1603                 goto failed;
1604         }
1605
1606         ips = talloc_zero(mem_ctx, struct ctdb_public_ip_list);
1607         if (ips == NULL) {
1608                 goto failed;
1609         }
1610
1611         ips->ip = talloc_array(ips, struct ctdb_public_ip, count);
1612         if (ips->ip == NULL) {
1613                 goto failed;
1614         }
1615
1616         ret = db_hash_traverse(ipdb, collect_ips, ips, &count);
1617         if (ret != 0) {
1618                 goto failed;
1619         }
1620
1621         if (count != ips->num) {
1622                 goto failed;
1623         }
1624
1625         talloc_free(ipdb);
1626
1627         *out = ips;
1628         return 0;
1629
1630 failed:
1631         talloc_free(ipdb);
1632         return 1;
1633 }
1634
1635 static int control_ip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1636                       int argc, const char **argv)
1637 {
1638         struct ctdb_public_ip_list *ips;
1639         struct ctdb_public_ip_info **ipinfo;
1640         int ret, i;
1641         bool do_all = false;
1642
1643         if (argc > 1) {
1644                 usage("ip");
1645         }
1646
1647         if (argc == 1) {
1648                 if (strcmp(argv[0], "all") == 0) {
1649                         do_all = true;
1650                 } else {
1651                         usage("ip");
1652                 }
1653         }
1654
1655         if (do_all) {
1656                 ret = get_all_public_ips(ctdb, mem_ctx, &ips);
1657         } else {
1658                 ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
1659                                                ctdb->cmd_pnn, TIMEOUT(),
1660                                                false, &ips);
1661         }
1662         if (ret != 0) {
1663                 return ret;
1664         }
1665
1666         qsort(ips->ip, ips->num, sizeof(struct ctdb_public_ip),
1667               ctdb_public_ip_cmp);
1668
1669         ipinfo = talloc_array(mem_ctx, struct ctdb_public_ip_info *, ips->num);
1670         if (ipinfo == NULL) {
1671                 return 1;
1672         }
1673
1674         for (i=0; i<ips->num; i++) {
1675                 uint32_t pnn;
1676                 if (do_all) {
1677                         pnn = ips->ip[i].pnn;
1678                 } else {
1679                         pnn = ctdb->cmd_pnn;
1680                 }
1681                 if (pnn == CTDB_UNKNOWN_PNN) {
1682                         ipinfo[i] = NULL;
1683                         continue;
1684                 }
1685                 ret = ctdb_ctrl_get_public_ip_info(mem_ctx, ctdb->ev,
1686                                                    ctdb->client, pnn,
1687                                                    TIMEOUT(), &ips->ip[i].addr,
1688                                                    &ipinfo[i]);
1689                 if (ret != 0) {
1690                         return ret;
1691                 }
1692         }
1693
1694         print_ip(mem_ctx, ctdb, ips, ipinfo, do_all);
1695         return 0;
1696 }
1697
1698 static int control_ipinfo(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1699                           int argc, const char **argv)
1700 {
1701         struct ctdb_public_ip_info *ipinfo;
1702         ctdb_sock_addr addr;
1703         int ret, i;
1704
1705         if (argc != 1) {
1706                 usage("ipinfo");
1707         }
1708
1709         ret = ctdb_sock_addr_from_string(argv[0], &addr, false);
1710         if (ret != 0) {
1711                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
1712                 return 1;
1713         }
1714
1715         ret = ctdb_ctrl_get_public_ip_info(mem_ctx, ctdb->ev, ctdb->client,
1716                                            ctdb->cmd_pnn, TIMEOUT(), &addr,
1717                                            &ipinfo);
1718         if (ret != 0) {
1719                 if (ret == -1) {
1720                         printf("Node %u does not know about IP %s\n",
1721                                ctdb->cmd_pnn, argv[0]);
1722                 }
1723                 return ret;
1724         }
1725
1726         printf("Public IP[%s] info on node %u\n",
1727                ctdb_sock_addr_to_string(mem_ctx, &ipinfo->ip.addr, false),
1728                                         ctdb->cmd_pnn);
1729
1730         printf("IP:%s\nCurrentNode:%u\nNumInterfaces:%u\n",
1731                ctdb_sock_addr_to_string(mem_ctx, &ipinfo->ip.addr, false),
1732                ipinfo->ip.pnn, ipinfo->ifaces->num);
1733
1734         for (i=0; i<ipinfo->ifaces->num; i++) {
1735                 struct ctdb_iface *iface;
1736
1737                 iface = &ipinfo->ifaces->iface[i];
1738                 iface->name[CTDB_IFACE_SIZE] = '\0';
1739                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
1740                        i+1, iface->name,
1741                        iface->link_state == 0 ? "down" : "up",
1742                        iface->references,
1743                        (i == ipinfo->active_idx) ? " (active)" : "");
1744         }
1745
1746         return 0;
1747 }
1748
1749 static int control_ifaces(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1750                           int argc, const char **argv)
1751 {
1752         struct ctdb_iface_list *ifaces;
1753         int ret, i;
1754
1755         if (argc != 0) {
1756                 usage("ifaces");
1757         }
1758
1759         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
1760                                    ctdb->cmd_pnn, TIMEOUT(), &ifaces);
1761         if (ret != 0) {
1762                 return ret;
1763         }
1764
1765         if (ifaces->num == 0) {
1766                 printf("No interfaces configured on node %u\n",
1767                        ctdb->cmd_pnn);
1768                 return 0;
1769         }
1770
1771         if (options.machinereadable) {
1772                 printf("%s%s%s%s%s%s%s\n", options.sep,
1773                        "Name", options.sep,
1774                        "LinkStatus", options.sep,
1775                        "References", options.sep);
1776         } else {
1777                 printf("Interfaces on node %u\n", ctdb->cmd_pnn);
1778         }
1779
1780         for (i=0; i<ifaces->num; i++) {
1781                 if (options.machinereadable) {
1782                         printf("%s%s%s%u%s%u%s\n", options.sep,
1783                                ifaces->iface[i].name, options.sep,
1784                                ifaces->iface[i].link_state, options.sep,
1785                                ifaces->iface[i].references, options.sep);
1786                 } else {
1787                         printf("name:%s link:%s references:%u\n",
1788                                ifaces->iface[i].name,
1789                                ifaces->iface[i].link_state ? "up" : "down",
1790                                ifaces->iface[i].references);
1791                 }
1792         }
1793
1794         return 0;
1795 }
1796
1797 static int control_setifacelink(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1798                                 int argc, const char **argv)
1799 {
1800         struct ctdb_iface_list *ifaces;
1801         struct ctdb_iface *iface;
1802         int ret, i;
1803
1804         if (argc != 2) {
1805                 usage("setifacelink");
1806         }
1807
1808         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
1809                 fprintf(stderr, "Interface name '%s' too long\n", argv[0]);
1810                 return 1;
1811         }
1812
1813         ret = ctdb_ctrl_get_ifaces(mem_ctx, ctdb->ev, ctdb->client,
1814                                    ctdb->cmd_pnn, TIMEOUT(), &ifaces);
1815         if (ret != 0) {
1816                 fprintf(stderr,
1817                         "Failed to get interface information from node %u\n",
1818                         ctdb->cmd_pnn);
1819                 return ret;
1820         }
1821
1822         iface = NULL;
1823         for (i=0; i<ifaces->num; i++) {
1824                 if (strcmp(ifaces->iface[i].name, argv[0]) == 0) {
1825                         iface = &ifaces->iface[i];
1826                         break;
1827                 }
1828         }
1829
1830         if (iface == NULL) {
1831                 printf("Interface %s not configured on node %u\n",
1832                        argv[0], ctdb->cmd_pnn);
1833                 return 1;
1834         }
1835
1836         if (strcmp(argv[1], "up") == 0) {
1837                 iface->link_state = 1;
1838         } else if (strcmp(argv[1], "down") == 0) {
1839                 iface->link_state = 0;
1840         } else {
1841                 usage("setifacelink");
1842                 return 1;
1843         }
1844
1845         iface->references = 0;
1846
1847         ret = ctdb_ctrl_set_iface_link_state(mem_ctx, ctdb->ev, ctdb->client,
1848                                              ctdb->cmd_pnn, TIMEOUT(), iface);
1849         if (ret != 0) {
1850                 return ret;
1851         }
1852
1853         return 0;
1854 }
1855
1856 static int control_process_exists(TALLOC_CTX *mem_ctx,
1857                                   struct ctdb_context *ctdb,
1858                                   int argc, const char **argv)
1859 {
1860         pid_t pid;
1861         uint64_t srvid = 0;
1862         int ret, status;
1863
1864         if (argc != 1 && argc != 2) {
1865                 usage("process-exists");
1866         }
1867
1868         pid = atoi(argv[0]);
1869         if (argc == 2) {
1870                 srvid = strtoull(argv[1], NULL, 0);
1871         }
1872
1873         if (srvid == 0) {
1874                 ret = ctdb_ctrl_process_exists(mem_ctx, ctdb->ev, ctdb->client,
1875                                        ctdb->cmd_pnn, TIMEOUT(), pid, &status);
1876         } else {
1877                 struct ctdb_pid_srvid pid_srvid;
1878
1879                 pid_srvid.pid = pid;
1880                 pid_srvid.srvid = srvid;
1881
1882                 ret = ctdb_ctrl_check_pid_srvid(mem_ctx, ctdb->ev,
1883                                                 ctdb->client, ctdb->cmd_pnn,
1884                                                 TIMEOUT(), &pid_srvid,
1885                                                 &status);
1886         }
1887
1888         if (ret != 0) {
1889                 return ret;
1890         }
1891
1892         if (srvid == 0) {
1893                 printf("PID %d %s\n", pid,
1894                        (status == 0 ? "exists" : "does not exist"));
1895         } else {
1896                 printf("PID %d with SRVID 0x%"PRIx64" %s\n", pid, srvid,
1897                        (status == 0 ? "exists" : "does not exist"));
1898         }
1899         return status;
1900 }
1901
1902 static int control_getdbmap(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
1903                             int argc, const char **argv)
1904 {
1905         struct ctdb_dbid_map *dbmap;
1906         int ret, i;
1907
1908         if (argc != 0) {
1909                 usage("getdbmap");
1910         }
1911
1912         ret = ctdb_ctrl_get_dbmap(mem_ctx, ctdb->ev, ctdb->client,
1913                                   ctdb->cmd_pnn, TIMEOUT(), &dbmap);
1914         if (ret != 0) {
1915                 return ret;
1916         }
1917
1918         if (options.machinereadable == 1) {
1919                 printf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
1920                        options.sep,
1921                        "ID", options.sep,
1922                        "Name", options.sep,
1923                        "Path", options.sep,
1924                        "Persistent", options.sep,
1925                        "Sticky", options.sep,
1926                        "Unhealthy", options.sep,
1927                        "Readonly", options.sep,
1928                        "Replicated", options.sep);
1929         } else {
1930                 printf("Number of databases:%d\n", dbmap->num);
1931         }
1932
1933         for (i=0; i<dbmap->num; i++) {
1934                 const char *name;
1935                 const char *path;
1936                 const char *health;
1937                 bool persistent;
1938                 bool readonly;
1939                 bool sticky;
1940                 bool replicated;
1941                 uint32_t db_id;
1942
1943                 db_id = dbmap->dbs[i].db_id;
1944
1945                 ret = ctdb_ctrl_get_dbname(mem_ctx, ctdb->ev, ctdb->client,
1946                                            ctdb->cmd_pnn, TIMEOUT(), db_id,
1947                                            &name);
1948                 if (ret != 0) {
1949                         return ret;
1950                 }
1951
1952                 ret = ctdb_ctrl_getdbpath(mem_ctx, ctdb->ev, ctdb->client,
1953                                           ctdb->cmd_pnn, TIMEOUT(), db_id,
1954                                           &path);
1955                 if (ret != 0) {
1956                         return ret;
1957                 }
1958
1959                 ret = ctdb_ctrl_db_get_health(mem_ctx, ctdb->ev, ctdb->client,
1960                                               ctdb->cmd_pnn, TIMEOUT(), db_id,
1961                                               &health);
1962                 if (ret != 0) {
1963                         return ret;
1964                 }
1965
1966                 persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT;
1967                 readonly = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY;
1968                 sticky = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY;
1969                 replicated = dbmap->dbs[i].flags & CTDB_DB_FLAGS_REPLICATED;
1970
1971                 if (options.machinereadable == 1) {
1972                         printf("%s0x%08X%s%s%s%s%s%d%s%d%s%d%s%d%s%d%s\n",
1973                                options.sep,
1974                                db_id, options.sep,
1975                                name, options.sep,
1976                                path, options.sep,
1977                                !! (persistent), options.sep,
1978                                !! (sticky), options.sep,
1979                                !! (health), options.sep,
1980                                !! (readonly), options.sep,
1981                                !! (replicated), options.sep);
1982                 } else {
1983                         printf("dbid:0x%08x name:%s path:%s%s%s%s%s%s\n",
1984                                db_id, name, path,
1985                                persistent ? " PERSISTENT" : "",
1986                                sticky ? " STICKY" : "",
1987                                readonly ? " READONLY" : "",
1988                                replicated ? " REPLICATED" : "",
1989                                health ? " UNHEALTHY" : "");
1990                 }
1991
1992                 talloc_free(discard_const(name));
1993                 talloc_free(discard_const(path));
1994                 talloc_free(discard_const(health));
1995         }
1996
1997         return 0;
1998 }
1999
2000 static int control_getdbstatus(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2001                                int argc, const char **argv)
2002 {
2003         uint32_t db_id;
2004         const char *db_name, *db_path, *db_health;
2005         uint8_t db_flags;
2006         int ret;
2007
2008         if (argc != 1) {
2009                 usage("getdbstatus");
2010         }
2011
2012         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
2013                 return 1;
2014         }
2015
2016         ret = ctdb_ctrl_getdbpath(mem_ctx, ctdb->ev, ctdb->client,
2017                                   ctdb->cmd_pnn, TIMEOUT(), db_id,
2018                                   &db_path);
2019         if (ret != 0) {
2020                 return ret;
2021         }
2022
2023         ret = ctdb_ctrl_db_get_health(mem_ctx, ctdb->ev, ctdb->client,
2024                                       ctdb->cmd_pnn, TIMEOUT(), db_id,
2025                                       &db_health);
2026         if (ret != 0) {
2027                 return ret;
2028         }
2029
2030         printf("dbid: 0x%08x\nname: %s\npath: %s\n", db_id, db_name, db_path);
2031         printf("PERSISTENT: %s\nREPLICATED: %s\nSTICKY: %s\nREADONLY: %s\n",
2032                (db_flags & CTDB_DB_FLAGS_PERSISTENT ? "yes" : "no"),
2033                (db_flags & CTDB_DB_FLAGS_REPLICATED ? "yes" : "no"),
2034                (db_flags & CTDB_DB_FLAGS_STICKY ? "yes" : "no"),
2035                (db_flags & CTDB_DB_FLAGS_READONLY ? "yes" : "no"));
2036         printf("HEALTH: %s\n", (db_health ? db_health : "OK"));
2037         return 0;
2038 }
2039
2040 struct dump_record_state {
2041         uint32_t count;
2042 };
2043
2044 #define ISASCII(x) (isprint(x) && ! strchr("\"\\", (x)))
2045
2046 static void dump_tdb_data(const char *name, TDB_DATA val)
2047 {
2048         int i;
2049
2050         fprintf(stdout, "%s(%zu) = \"", name, val.dsize);
2051         for (i=0; i<val.dsize; i++) {
2052                 if (ISASCII(val.dptr[i])) {
2053                         fprintf(stdout, "%c", val.dptr[i]);
2054                 } else {
2055                         fprintf(stdout, "\\%02X", val.dptr[i]);
2056                 }
2057         }
2058         fprintf(stdout, "\"\n");
2059 }
2060
2061 static void dump_ltdb_header(struct ctdb_ltdb_header *header)
2062 {
2063         fprintf(stdout, "dmaster: %u\n", header->dmaster);
2064         fprintf(stdout, "rsn: %" PRIu64 "\n", header->rsn);
2065         fprintf(stdout, "flags: 0x%08x", header->flags);
2066         if (header->flags & CTDB_REC_FLAG_MIGRATED_WITH_DATA) {
2067                 fprintf(stdout, " MIGRATED_WITH_DATA");
2068         }
2069         if (header->flags & CTDB_REC_FLAG_VACUUM_MIGRATED) {
2070                 fprintf(stdout, " VACUUM_MIGRATED");
2071         }
2072         if (header->flags & CTDB_REC_FLAG_AUTOMATIC) {
2073                 fprintf(stdout, " AUTOMATIC");
2074         }
2075         if (header->flags & CTDB_REC_RO_HAVE_DELEGATIONS) {
2076                 fprintf(stdout, " RO_HAVE_DELEGATIONS");
2077         }
2078         if (header->flags & CTDB_REC_RO_HAVE_READONLY) {
2079                 fprintf(stdout, " RO_HAVE_READONLY");
2080         }
2081         if (header->flags & CTDB_REC_RO_REVOKING_READONLY) {
2082                 fprintf(stdout, " RO_REVOKING_READONLY");
2083         }
2084         if (header->flags & CTDB_REC_RO_REVOKE_COMPLETE) {
2085                 fprintf(stdout, " RO_REVOKE_COMPLETE");
2086         }
2087         fprintf(stdout, "\n");
2088
2089 }
2090
2091 static int dump_record(uint32_t reqid, struct ctdb_ltdb_header *header,
2092                        TDB_DATA key, TDB_DATA data, void *private_data)
2093 {
2094         struct dump_record_state *state =
2095                 (struct dump_record_state *)private_data;
2096
2097         state->count += 1;
2098
2099         dump_tdb_data("key", key);
2100         dump_ltdb_header(header);
2101         dump_tdb_data("data", data);
2102         fprintf(stdout, "\n");
2103
2104         return 0;
2105 }
2106
2107 static int control_catdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2108                          int argc, const char **argv)
2109 {
2110         struct ctdb_db_context *db;
2111         const char *db_name;
2112         uint32_t db_id;
2113         uint8_t db_flags;
2114         struct dump_record_state state;
2115         int ret;
2116
2117         if (argc != 1) {
2118                 usage("catdb");
2119         }
2120
2121         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
2122                 return 1;
2123         }
2124
2125         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2126                           db_flags, &db);
2127         if (ret != 0) {
2128                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
2129                 return ret;
2130         }
2131
2132         state.count = 0;
2133
2134         ret = ctdb_db_traverse(mem_ctx, ctdb->ev, ctdb->client, db,
2135                                ctdb->cmd_pnn, TIMEOUT(),
2136                                dump_record, &state);
2137
2138         printf("Dumped %u records\n", state.count);
2139
2140         return ret;
2141 }
2142
2143 static int control_cattdb(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2144                           int argc, const char **argv)
2145 {
2146         struct ctdb_db_context *db;
2147         const char *db_name;
2148         uint32_t db_id;
2149         uint8_t db_flags;
2150         struct dump_record_state state;
2151         int ret;
2152
2153         if (argc != 1) {
2154                 usage("catdb");
2155         }
2156
2157         if (! db_exists(mem_ctx, ctdb, argv[0], &db_id, &db_name, &db_flags)) {
2158                 return 1;
2159         }
2160
2161         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2162                           db_flags, &db);
2163         if (ret != 0) {
2164                 fprintf(stderr, "Failed to attach to DB %s\n", db_name);
2165                 return ret;
2166         }
2167
2168         state.count = 0;
2169         ret = ctdb_db_traverse_local(db, true, true, dump_record, &state);
2170
2171         printf("Dumped %u record(s)\n", state.count);
2172
2173         return ret;
2174 }
2175
2176 static int control_getcapabilities(TALLOC_CTX *mem_ctx,
2177                                    struct ctdb_context *ctdb,
2178                                    int argc, const char **argv)
2179 {
2180         uint32_t caps;
2181         int ret;
2182
2183         if (argc != 0) {
2184                 usage("getcapabilities");
2185         }
2186
2187         ret = ctdb_ctrl_get_capabilities(mem_ctx, ctdb->ev, ctdb->client,
2188                                          ctdb->cmd_pnn, TIMEOUT(), &caps);
2189         if (ret != 0) {
2190                 return ret;
2191         }
2192
2193         if (options.machinereadable == 1) {
2194                 printf("%s%s%s%s%s\n",
2195                        options.sep,
2196                        "RECMASTER", options.sep,
2197                        "LMASTER", options.sep);
2198                 printf("%s%d%s%d%s\n", options.sep,
2199                        !! (caps & CTDB_CAP_RECMASTER), options.sep,
2200                        !! (caps & CTDB_CAP_LMASTER), options.sep);
2201         } else {
2202                 printf("RECMASTER: %s\n",
2203                        (caps & CTDB_CAP_RECMASTER) ? "YES" : "NO");
2204                 printf("LMASTER: %s\n",
2205                        (caps & CTDB_CAP_LMASTER) ? "YES" : "NO");
2206         }
2207
2208         return 0;
2209 }
2210
2211 static int control_pnn(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2212                        int argc, const char **argv)
2213 {
2214         printf("%u\n", ctdb_client_pnn(ctdb->client));
2215         return 0;
2216 }
2217
2218 static int control_lvs(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2219                        int argc, const char **argv)
2220 {
2221         char *t, *lvs_helper = NULL;
2222
2223         if (argc != 1) {
2224                 usage("lvs");
2225         }
2226
2227         t = getenv("CTDB_LVS_HELPER");
2228         if (t != NULL) {
2229                 lvs_helper = talloc_strdup(mem_ctx, t);
2230         } else {
2231                 lvs_helper = talloc_asprintf(mem_ctx, "%s/ctdb_lvs",
2232                                              CTDB_HELPER_BINDIR);
2233         }
2234
2235         if (lvs_helper == NULL) {
2236                 fprintf(stderr, "Unable to set LVS helper\n");
2237                 return 1;
2238         }
2239
2240         return run_helper(mem_ctx, "LVS helper", lvs_helper, argc, argv);
2241 }
2242
2243 static int control_setdebug(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2244                             int argc, const char **argv)
2245 {
2246         int log_level;
2247         int ret;
2248         bool found;
2249
2250         if (argc != 1) {
2251                 usage("setdebug");
2252         }
2253
2254         found = debug_level_parse(argv[0], &log_level);
2255         if (! found) {
2256                 fprintf(stderr,
2257                         "Invalid debug level '%s'. Valid levels are:\n",
2258                         argv[0]);
2259                 fprintf(stderr, "\tERROR | WARNING | NOTICE | INFO | DEBUG\n");
2260                 return 1;
2261         }
2262
2263         ret = ctdb_ctrl_setdebug(mem_ctx, ctdb->ev, ctdb->client,
2264                                  ctdb->cmd_pnn, TIMEOUT(), log_level);
2265         if (ret != 0) {
2266                 return ret;
2267         }
2268
2269         return 0;
2270 }
2271
2272 static int control_getdebug(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2273                             int argc, const char **argv)
2274 {
2275         int loglevel;
2276         const char *log_str;
2277         int ret;
2278
2279         if (argc != 0) {
2280                 usage("getdebug");
2281         }
2282
2283         ret = ctdb_ctrl_getdebug(mem_ctx, ctdb->ev, ctdb->client,
2284                                  ctdb->cmd_pnn, TIMEOUT(), &loglevel);
2285         if (ret != 0) {
2286                 return ret;
2287         }
2288
2289         log_str = debug_level_to_string(loglevel);
2290         printf("%s\n", log_str);
2291
2292         return 0;
2293 }
2294
2295 static int control_attach(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2296                           int argc, const char **argv)
2297 {
2298         const char *db_name;
2299         uint8_t db_flags = 0;
2300         int ret;
2301
2302         if (argc < 1 || argc > 2) {
2303                 usage("attach");
2304         }
2305
2306         db_name = argv[0];
2307         if (argc == 2) {
2308                 if (strcmp(argv[1], "persistent") == 0) {
2309                         db_flags = CTDB_DB_FLAGS_PERSISTENT;
2310                 } else if (strcmp(argv[1], "readonly") == 0) {
2311                         db_flags = CTDB_DB_FLAGS_READONLY;
2312                 } else if (strcmp(argv[1], "sticky") == 0) {
2313                         db_flags = CTDB_DB_FLAGS_STICKY;
2314                 } else if (strcmp(argv[1], "replicated") == 0) {
2315                         db_flags = CTDB_DB_FLAGS_REPLICATED;
2316                 } else {
2317                         usage("attach");
2318                 }
2319         }
2320
2321         ret = ctdb_attach(ctdb->ev, ctdb->client, TIMEOUT(), db_name,
2322                           db_flags, NULL);
2323         if (ret != 0) {
2324                 return ret;
2325         }
2326
2327         return 0;
2328 }
2329
2330 static int control_detach(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2331                           int argc, const char **argv)
2332 {
2333         const char *db_name;
2334         uint32_t db_id;
2335         uint8_t db_flags;
2336         struct ctdb_node_map *nodemap;
2337         int recmode;
2338         int ret, ret2, i;
2339
2340         if (argc < 1) {
2341                 usage("detach");
2342         }
2343
2344         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
2345                                     ctdb->cmd_pnn, TIMEOUT(), &recmode);
2346         if (ret != 0) {
2347                 return ret;
2348         }
2349
2350         if (recmode == CTDB_RECOVERY_ACTIVE) {
2351                 fprintf(stderr, "Database cannot be detached"
2352                                 " when recovery is active\n");
2353                 return 1;
2354         }
2355
2356         nodemap = get_nodemap(ctdb, false);
2357         if (nodemap == NULL) {
2358                 return 1;
2359         }
2360
2361         for (i=0; i<nodemap->num; i++) {
2362                 uint32_t value;
2363
2364                 if (nodemap->node[i].flags & NODE_FLAGS_DISCONNECTED) {
2365                         continue;
2366                 }
2367                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
2368                         continue;
2369                 }
2370                 if (nodemap->node[i].flags & NODE_FLAGS_INACTIVE) {
2371                         fprintf(stderr, "Database cannot be detached on"
2372                                 " inactive (stopped or banned) node %u\n",
2373                                 nodemap->node[i].pnn);
2374                         return 1;
2375                 }
2376
2377                 ret = ctdb_ctrl_get_tunable(mem_ctx, ctdb->ev, ctdb->client,
2378                                             nodemap->node[i].pnn, TIMEOUT(),
2379                                             "AllowClientDBAttach", &value);
2380                 if (ret != 0) {
2381                         fprintf(stderr,
2382                                 "Unable to get tunable AllowClientDBAttach"
2383                                 " from node %u\n", nodemap->node[i].pnn);
2384                         return ret;
2385                 }
2386
2387                 if (value == 1) {
2388                         fprintf(stderr,
2389                                 "Database access is still active on node %u."
2390                                 " Set AllowclientDBAttach=0 on all nodes.\n",
2391                                 nodemap->node[i].pnn);
2392                         return 1;
2393                 }
2394         }
2395
2396         ret2 = 0;
2397         for (i=0; i<argc; i++) {
2398                 if (! db_exists(mem_ctx, ctdb, argv[i], &db_id, &db_name,
2399                                 &db_flags)) {
2400                         continue;
2401                 }
2402
2403                 if (db_flags &
2404                     (CTDB_DB_FLAGS_PERSISTENT | CTDB_DB_FLAGS_REPLICATED)) {
2405                         fprintf(stderr,
2406                                 "Only volatile databases can be detached\n");
2407                         return 1;
2408                 }
2409
2410                 ret = ctdb_detach(ctdb->ev, ctdb->client, TIMEOUT(), db_id);
2411                 if (ret != 0) {
2412                         fprintf(stderr, "Database %s detach failed\n", db_name);
2413                         ret2 = ret;
2414                 }
2415         }
2416
2417         return ret2;
2418 }
2419
2420 static int control_dumpmemory(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2421                               int argc, const char **argv)
2422 {
2423         const char *mem_str;
2424         ssize_t n;
2425         int ret;
2426
2427         ret = ctdb_ctrl_dump_memory(mem_ctx, ctdb->ev, ctdb->client,
2428                                     ctdb->cmd_pnn, TIMEOUT(), &mem_str);
2429         if (ret != 0) {
2430                 return ret;
2431         }
2432
2433         n = write(1, mem_str, strlen(mem_str)+1);
2434         if (n < 0 || n != strlen(mem_str)+1) {
2435                 fprintf(stderr, "Failed to write talloc summary\n");
2436                 return 1;
2437         }
2438
2439         return 0;
2440 }
2441
2442 static void dump_memory(uint64_t srvid, TDB_DATA data, void *private_data)
2443 {
2444         bool *done = (bool *)private_data;
2445         ssize_t n;
2446
2447         n = write(1, data.dptr, data.dsize);
2448         if (n < 0 || n != data.dsize) {
2449                 fprintf(stderr, "Failed to write talloc summary\n");
2450         }
2451
2452         *done = true;
2453 }
2454
2455 static int control_rddumpmemory(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2456                                 int argc, const char **argv)
2457 {
2458         struct ctdb_srvid_message msg = { 0 };
2459         int ret;
2460         bool done = false;
2461
2462         msg.pnn = ctdb->pnn;
2463         msg.srvid = next_srvid(ctdb);
2464
2465         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2466                                               msg.srvid, dump_memory, &done);
2467         if (ret != 0) {
2468                 return ret;
2469         }
2470
2471         ret = ctdb_message_mem_dump(mem_ctx, ctdb->ev, ctdb->client,
2472                                     ctdb->cmd_pnn, &msg);
2473         if (ret != 0) {
2474                 return ret;
2475         }
2476
2477         ctdb_client_wait(ctdb->ev, &done);
2478         return 0;
2479 }
2480
2481 static int control_getpid(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2482                           int argc, const char **argv)
2483 {
2484         pid_t pid;
2485         int ret;
2486
2487         ret = ctdb_ctrl_get_pid(mem_ctx, ctdb->ev, ctdb->client,
2488                                 ctdb->cmd_pnn, TIMEOUT(), &pid);
2489         if (ret != 0) {
2490                 return ret;
2491         }
2492
2493         printf("%u\n", pid);
2494         return 0;
2495 }
2496
2497 static int check_flags(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2498                        const char *desc, uint32_t flag, bool set_flag)
2499 {
2500         struct ctdb_node_map *nodemap;
2501         bool flag_is_set;
2502
2503         nodemap = get_nodemap(ctdb, false);
2504         if (nodemap == NULL) {
2505                 return 1;
2506         }
2507
2508         flag_is_set = nodemap->node[ctdb->cmd_pnn].flags & flag;
2509         if (set_flag == flag_is_set) {
2510                 if (set_flag) {
2511                         fprintf(stderr, "Node %u is already %s\n",
2512                                 ctdb->cmd_pnn, desc);
2513                 } else {
2514                         fprintf(stderr, "Node %u is not %s\n",
2515                                 ctdb->cmd_pnn, desc);
2516                 }
2517                 return 0;
2518         }
2519
2520         return 1;
2521 }
2522
2523 static void wait_for_flags(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2524                            uint32_t flag, bool set_flag)
2525 {
2526         struct ctdb_node_map *nodemap;
2527         bool flag_is_set;
2528
2529         while (1) {
2530                 nodemap = get_nodemap(ctdb, true);
2531                 if (nodemap == NULL) {
2532                         fprintf(stderr,
2533                                 "Failed to get nodemap, trying again\n");
2534                         sleep(1);
2535                         continue;
2536                 }
2537
2538                 flag_is_set = nodemap->node[ctdb->cmd_pnn].flags & flag;
2539                 if (flag_is_set == set_flag) {
2540                         break;
2541                 }
2542
2543                 sleep(1);
2544         }
2545 }
2546
2547 static int ctdb_ctrl_modflags(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
2548                               struct ctdb_client_context *client,
2549                               uint32_t destnode, struct timeval timeout,
2550                               uint32_t set, uint32_t clear)
2551 {
2552         struct ctdb_node_map *nodemap;
2553         struct ctdb_node_flag_change flag_change;
2554         struct ctdb_req_control request;
2555         uint32_t *pnn_list;
2556         int ret, count;
2557
2558         ret = ctdb_ctrl_get_nodemap(mem_ctx, ev, client, destnode,
2559                                     tevent_timeval_zero(), &nodemap);
2560         if (ret != 0) {
2561                 return ret;
2562         }
2563
2564         flag_change.pnn = destnode;
2565         flag_change.old_flags = nodemap->node[destnode].flags;
2566         flag_change.new_flags = flag_change.old_flags | set;
2567         flag_change.new_flags &= ~clear;
2568
2569         count = list_of_connected_nodes(nodemap, -1, mem_ctx, &pnn_list);
2570         if (count == -1) {
2571                 return ENOMEM;
2572         }
2573
2574         ctdb_req_control_modify_flags(&request, &flag_change);
2575         ret = ctdb_client_control_multi(mem_ctx, ev, client, pnn_list, count,
2576                                         tevent_timeval_zero(), &request,
2577                                         NULL, NULL);
2578         return ret;
2579 }
2580
2581 struct ipreallocate_state {
2582         int status;
2583         bool done;
2584 };
2585
2586 static void ipreallocate_handler(uint64_t srvid, TDB_DATA data,
2587                                  void *private_data)
2588 {
2589         struct ipreallocate_state *state =
2590                 (struct ipreallocate_state *)private_data;
2591
2592         if (data.dsize != sizeof(int)) {
2593                 /* Ignore packet */
2594                 return;
2595         }
2596
2597         state->status = *(int *)data.dptr;
2598         state->done = true;
2599 }
2600
2601 static int ipreallocate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb)
2602 {
2603         struct ctdb_srvid_message msg = { 0 };
2604         struct ipreallocate_state state;
2605         int ret;
2606
2607         msg.pnn = ctdb->pnn;
2608         msg.srvid = next_srvid(ctdb);
2609
2610         state.done = false;
2611         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
2612                                               msg.srvid,
2613                                               ipreallocate_handler, &state);
2614         if (ret != 0) {
2615                 return ret;
2616         }
2617
2618         while (true) {
2619                 ret = ctdb_message_takeover_run(mem_ctx, ctdb->ev,
2620                                                 ctdb->client,
2621                                                 CTDB_BROADCAST_CONNECTED,
2622                                                 &msg);
2623                 if (ret != 0) {
2624                         goto fail;
2625                 }
2626
2627                 ret = ctdb_client_wait_timeout(ctdb->ev, &state.done,
2628                                                TIMEOUT());
2629                 if (ret != 0) {
2630                         continue;
2631                 }
2632
2633                 if (state.status >= 0) {
2634                         ret = 0;
2635                 } else {
2636                         ret = state.status;
2637                 }
2638                 break;
2639         }
2640
2641 fail:
2642         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
2643                                            msg.srvid, &state);
2644         return ret;
2645 }
2646
2647 static int control_disable(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2648                            int argc, const char **argv)
2649 {
2650         int ret;
2651
2652         if (argc != 0) {
2653                 usage("disable");
2654         }
2655
2656         ret = check_flags(mem_ctx, ctdb, "disabled",
2657                           NODE_FLAGS_PERMANENTLY_DISABLED, true);
2658         if (ret == 0) {
2659                 return 0;
2660         }
2661
2662         ret = ctdb_ctrl_modflags(mem_ctx, ctdb->ev, ctdb->client,
2663                                  ctdb->cmd_pnn, TIMEOUT(),
2664                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
2665         if (ret != 0) {
2666                 fprintf(stderr,
2667                         "Failed to set DISABLED flag on node %u\n",
2668                         ctdb->cmd_pnn);
2669                 return ret;
2670         }
2671
2672         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_PERMANENTLY_DISABLED, true);
2673         return ipreallocate(mem_ctx, ctdb);
2674 }
2675
2676 static int control_enable(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2677                           int argc, const char **argv)
2678 {
2679         int ret;
2680
2681         if (argc != 0) {
2682                 usage("enable");
2683         }
2684
2685         ret = check_flags(mem_ctx, ctdb, "disabled",
2686                           NODE_FLAGS_PERMANENTLY_DISABLED, false);
2687         if (ret == 0) {
2688                 return 0;
2689         }
2690
2691         ret = ctdb_ctrl_modflags(mem_ctx, ctdb->ev, ctdb->client,
2692                                  ctdb->cmd_pnn, TIMEOUT(),
2693                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
2694         if (ret != 0) {
2695                 fprintf(stderr, "Failed to reset DISABLED flag on node %u\n",
2696                         ctdb->cmd_pnn);
2697                 return ret;
2698         }
2699
2700         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_PERMANENTLY_DISABLED, false);
2701         return ipreallocate(mem_ctx, ctdb);
2702 }
2703
2704 static int control_stop(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2705                         int argc, const char **argv)
2706 {
2707         int ret;
2708
2709         if (argc != 0) {
2710                 usage("stop");
2711         }
2712
2713         ret = check_flags(mem_ctx, ctdb, "stopped",
2714                           NODE_FLAGS_STOPPED, true);
2715         if (ret == 0) {
2716                 return 0;
2717         }
2718
2719         ret = ctdb_ctrl_stop_node(mem_ctx, ctdb->ev, ctdb->client,
2720                                   ctdb->cmd_pnn, TIMEOUT());
2721         if (ret != 0) {
2722                 fprintf(stderr, "Failed to stop node %u\n", ctdb->cmd_pnn);
2723                 return ret;
2724         }
2725
2726         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_STOPPED, true);
2727         return ipreallocate(mem_ctx, ctdb);
2728 }
2729
2730 static int control_continue(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2731                             int argc, const char **argv)
2732 {
2733         int ret;
2734
2735         if (argc != 0) {
2736                 usage("continue");
2737         }
2738
2739         ret = check_flags(mem_ctx, ctdb, "stopped",
2740                           NODE_FLAGS_STOPPED, false);
2741         if (ret == 0) {
2742                 return 0;
2743         }
2744
2745         ret = ctdb_ctrl_continue_node(mem_ctx, ctdb->ev, ctdb->client,
2746                                       ctdb->cmd_pnn, TIMEOUT());
2747         if (ret != 0) {
2748                 fprintf(stderr, "Failed to continue stopped node %u\n",
2749                         ctdb->cmd_pnn);
2750                 return ret;
2751         }
2752
2753         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_STOPPED, false);
2754         return ipreallocate(mem_ctx, ctdb);
2755 }
2756
2757 static int control_ban(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2758                        int argc, const char **argv)
2759 {
2760         struct ctdb_ban_state ban_state;
2761         int ret;
2762
2763         if (argc != 1) {
2764                 usage("ban");
2765         }
2766
2767         ret = check_flags(mem_ctx, ctdb, "banned",
2768                           NODE_FLAGS_BANNED, true);
2769         if (ret == 0) {
2770                 return 0;
2771         }
2772
2773         ban_state.pnn = ctdb->cmd_pnn;
2774         ban_state.time = strtoul(argv[0], NULL, 0);
2775
2776         if (ban_state.time == 0) {
2777                 fprintf(stderr, "Ban time cannot be zero\n");
2778                 return EINVAL;
2779         }
2780
2781         ret = ctdb_ctrl_set_ban_state(mem_ctx, ctdb->ev, ctdb->client,
2782                                       ctdb->cmd_pnn, TIMEOUT(), &ban_state);
2783         if (ret != 0) {
2784                 fprintf(stderr, "Failed to ban node %u\n", ctdb->cmd_pnn);
2785                 return ret;
2786         }
2787
2788         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_BANNED, true);
2789         return ipreallocate(mem_ctx, ctdb);
2790
2791 }
2792
2793 static int control_unban(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2794                          int argc, const char **argv)
2795 {
2796         struct ctdb_ban_state ban_state;
2797         int ret;
2798
2799         if (argc != 0) {
2800                 usage("unban");
2801         }
2802
2803         ret = check_flags(mem_ctx, ctdb, "banned",
2804                           NODE_FLAGS_BANNED, false);
2805         if (ret == 0) {
2806                 return 0;
2807         }
2808
2809         ban_state.pnn = ctdb->cmd_pnn;
2810         ban_state.time = 0;
2811
2812         ret = ctdb_ctrl_set_ban_state(mem_ctx, ctdb->ev, ctdb->client,
2813                                       ctdb->cmd_pnn, TIMEOUT(), &ban_state);
2814         if (ret != 0) {
2815                 fprintf(stderr, "Failed to unban node %u\n", ctdb->cmd_pnn);
2816                 return ret;
2817         }
2818
2819         wait_for_flags(mem_ctx, ctdb, NODE_FLAGS_BANNED, false);
2820         return ipreallocate(mem_ctx, ctdb);
2821
2822 }
2823
2824 static void wait_for_shutdown(void *private_data)
2825 {
2826         bool *done = (bool *)private_data;
2827
2828         *done = true;
2829 }
2830
2831 static int control_shutdown(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2832                             int argc, const char **argv)
2833 {
2834         int ret;
2835         bool done = false;
2836
2837         if (argc != 0) {
2838                 usage("shutdown");
2839         }
2840
2841         if (ctdb->pnn == ctdb->cmd_pnn) {
2842                 ctdb_client_set_disconnect_callback(ctdb->client,
2843                                                     wait_for_shutdown,
2844                                                     &done);
2845         }
2846
2847         ret = ctdb_ctrl_shutdown(mem_ctx, ctdb->ev, ctdb->client,
2848                                  ctdb->cmd_pnn, TIMEOUT());
2849         if (ret != 0) {
2850                 fprintf(stderr, "Unable to shutdown node %u\n", ctdb->cmd_pnn);
2851                 return ret;
2852         }
2853
2854         if (ctdb->pnn == ctdb->cmd_pnn) {
2855                 ctdb_client_wait(ctdb->ev, &done);
2856         }
2857
2858         return 0;
2859 }
2860
2861 static int get_generation(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2862                           uint32_t *generation)
2863 {
2864         uint32_t recmaster;
2865         int recmode;
2866         struct ctdb_vnn_map *vnnmap;
2867         int ret;
2868
2869 again:
2870         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
2871                                       ctdb->cmd_pnn, TIMEOUT(), &recmaster);
2872         if (ret != 0) {
2873                 fprintf(stderr, "Failed to find recovery master\n");
2874                 return ret;
2875         }
2876
2877         ret = ctdb_ctrl_get_recmode(mem_ctx, ctdb->ev, ctdb->client,
2878                                     recmaster, TIMEOUT(), &recmode);
2879         if (ret != 0) {
2880                 fprintf(stderr, "Failed to get recovery mode from node %u\n",
2881                         recmaster);
2882                 return ret;
2883         }
2884
2885         if (recmode == CTDB_RECOVERY_ACTIVE) {
2886                 sleep(1);
2887                 goto again;
2888         }
2889
2890         ret = ctdb_ctrl_getvnnmap(mem_ctx, ctdb->ev, ctdb->client,
2891                                   recmaster, TIMEOUT(), &vnnmap);
2892         if (ret != 0) {
2893                 fprintf(stderr, "Failed to get generation from node %u\n",
2894                         recmaster);
2895                 return ret;
2896         }
2897
2898         if (vnnmap->generation == INVALID_GENERATION) {
2899                 talloc_free(vnnmap);
2900                 sleep(1);
2901                 goto again;
2902         }
2903
2904         *generation = vnnmap->generation;
2905         talloc_free(vnnmap);
2906         return 0;
2907 }
2908
2909
2910 static int control_recover(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2911                            int argc, const char **argv)
2912 {
2913         uint32_t generation, next_generation;
2914         int ret;
2915
2916         if (argc != 0) {
2917                 usage("recover");
2918         }
2919
2920         ret = get_generation(mem_ctx, ctdb, &generation);
2921         if (ret != 0) {
2922                 return ret;
2923         }
2924
2925         ret = ctdb_ctrl_set_recmode(mem_ctx, ctdb->ev, ctdb->client,
2926                                     ctdb->cmd_pnn, TIMEOUT(),
2927                                     CTDB_RECOVERY_ACTIVE);
2928         if (ret != 0) {
2929                 fprintf(stderr, "Failed to set recovery mode active\n");
2930                 return ret;
2931         }
2932
2933         while (1) {
2934                 ret = get_generation(mem_ctx, ctdb, &next_generation);
2935                 if (ret != 0) {
2936                         fprintf(stderr,
2937                                 "Failed to confirm end of recovery\n");
2938                         return ret;
2939                 }
2940
2941                 if (next_generation != generation) {
2942                         break;
2943                 }
2944
2945                 sleep (1);
2946         }
2947
2948         return 0;
2949 }
2950
2951 static int control_ipreallocate(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2952                                 int argc, const char **argv)
2953 {
2954         if (argc != 0) {
2955                 usage("ipreallocate");
2956         }
2957
2958         return ipreallocate(mem_ctx, ctdb);
2959 }
2960
2961 static int control_isnotrecmaster(TALLOC_CTX *mem_ctx,
2962                                   struct ctdb_context *ctdb,
2963                                   int argc, const char **argv)
2964 {
2965         uint32_t recmaster;
2966         int ret;
2967
2968         if (argc != 0) {
2969                 usage("isnotrecmaster");
2970         }
2971
2972         ret = ctdb_ctrl_get_recmaster(mem_ctx, ctdb->ev, ctdb->client,
2973                                       ctdb->pnn, TIMEOUT(), &recmaster);
2974         if (ret != 0) {
2975                 fprintf(stderr, "Failed to get recmaster\n");
2976                 return ret;
2977         }
2978
2979         if (recmaster != ctdb->pnn) {
2980                 printf("this node is not the recmaster\n");
2981                 return 1;
2982         }
2983
2984         printf("this node is the recmaster\n");
2985         return 0;
2986 }
2987
2988 static int control_gratarp(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
2989                            int argc, const char **argv)
2990 {
2991         struct ctdb_addr_info addr_info;
2992         int ret;
2993
2994         if (argc != 2) {
2995                 usage("gratarp");
2996         }
2997
2998         ret = ctdb_sock_addr_from_string(argv[0], &addr_info.addr, false);
2999         if (ret != 0) {
3000                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3001                 return 1;
3002         }
3003         addr_info.iface = argv[1];
3004
3005         ret = ctdb_ctrl_send_gratuitous_arp(mem_ctx, ctdb->ev, ctdb->client,
3006                                             ctdb->cmd_pnn, TIMEOUT(),
3007                                             &addr_info);
3008         if (ret != 0) {
3009                 fprintf(stderr, "Unable to send gratuitous arp from node %u\n",
3010                         ctdb->cmd_pnn);
3011                 return ret;
3012         }
3013
3014         return 0;
3015 }
3016
3017 static int control_tickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3018                            int argc, const char **argv)
3019 {
3020         ctdb_sock_addr src, dst;
3021         int ret;
3022
3023         if (argc != 0 && argc != 2) {
3024                 usage("tickle");
3025         }
3026
3027         if (argc == 0) {
3028                 struct ctdb_connection_list *clist;
3029                 int i;
3030                 unsigned int num_failed;
3031
3032                 /* Client first but the src/dst logic is confused */
3033                 ret = ctdb_connection_list_read(mem_ctx, 0, false, &clist);
3034                 if (ret != 0) {
3035                         return ret;
3036                 }
3037
3038                 num_failed = 0;
3039                 for (i = 0; i < clist->num; i++) {
3040                         ret = ctdb_sys_send_tcp(&clist->conn[i].src,
3041                                                 &clist->conn[i].dst,
3042                                                 0, 0, 0);
3043                         if (ret != 0) {
3044                                 num_failed += 1;
3045                         }
3046                 }
3047
3048                 TALLOC_FREE(clist);
3049
3050                 if (num_failed > 0) {
3051                         fprintf(stderr, "Failed to send %d tickles\n",
3052                                 num_failed);
3053                         return 1;
3054                 }
3055
3056                 return 0;
3057         }
3058
3059
3060         ret = ctdb_sock_addr_from_string(argv[0], &src, true);
3061         if (ret != 0) {
3062                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3063                 return 1;
3064         }
3065
3066         ret = ctdb_sock_addr_from_string(argv[1], &dst, true);
3067         if (ret != 0) {
3068                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3069                 return 1;
3070         }
3071
3072         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
3073         if (ret != 0) {
3074                 fprintf(stderr, "Failed to send tickle ack\n");
3075                 return ret;
3076         }
3077
3078         return 0;
3079 }
3080
3081 static int control_gettickles(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3082                               int argc, const char **argv)
3083 {
3084         ctdb_sock_addr addr;
3085         struct ctdb_tickle_list *tickles;
3086         unsigned port = 0;
3087         int ret, i;
3088
3089         if (argc < 1 || argc > 2) {
3090                 usage("gettickles");
3091         }
3092
3093         if (argc == 2) {
3094                 port = strtoul(argv[1], NULL, 10);
3095         }
3096
3097         ret = ctdb_sock_addr_from_string(argv[0], &addr, false);
3098         if (ret != 0) {
3099                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3100                 return 1;
3101         }
3102         ctdb_sock_addr_set_port(&addr, port);
3103
3104         ret = ctdb_ctrl_get_tcp_tickle_list(mem_ctx, ctdb->ev, ctdb->client,
3105                                             ctdb->cmd_pnn, TIMEOUT(), &addr,
3106                                             &tickles);
3107         if (ret != 0) {
3108                 fprintf(stderr, "Failed to get list of connections\n");
3109                 return ret;
3110         }
3111
3112         if (options.machinereadable) {
3113                 printf("%s%s%s%s%s%s%s%s%s\n",
3114                        options.sep,
3115                        "Source IP", options.sep,
3116                        "Port", options.sep,
3117                        "Destiation IP", options.sep,
3118                        "Port", options.sep);
3119                 for (i=0; i<tickles->num; i++) {
3120                         printf("%s%s%s%u%s%s%s%u%s\n", options.sep,
3121                                ctdb_sock_addr_to_string(
3122                                        mem_ctx, &tickles->conn[i].src, false),
3123                                options.sep,
3124                                ntohs(tickles->conn[i].src.ip.sin_port),
3125                                options.sep,
3126                                ctdb_sock_addr_to_string(
3127                                        mem_ctx, &tickles->conn[i].dst, false),
3128                                options.sep,
3129                                ntohs(tickles->conn[i].dst.ip.sin_port),
3130                                options.sep);
3131                 }
3132         } else {
3133                 printf("Connections for IP: %s\n",
3134                        ctdb_sock_addr_to_string(mem_ctx,
3135                                                 &tickles->addr, false));
3136                 printf("Num connections: %u\n", tickles->num);
3137                 for (i=0; i<tickles->num; i++) {
3138                         printf("SRC: %s   DST: %s\n",
3139                                ctdb_sock_addr_to_string(
3140                                        mem_ctx, &tickles->conn[i].src, true),
3141                                ctdb_sock_addr_to_string(
3142                                        mem_ctx, &tickles->conn[i].dst, true));
3143                 }
3144         }
3145
3146         talloc_free(tickles);
3147         return 0;
3148 }
3149
3150 typedef void (*clist_request_func)(struct ctdb_req_control *request,
3151                                    struct ctdb_connection *conn);
3152
3153 typedef int (*clist_reply_func)(struct ctdb_reply_control *reply);
3154
3155 struct process_clist_state {
3156         struct ctdb_connection_list *clist;
3157         int count;
3158         int num_failed, num_total;
3159         clist_reply_func reply_func;
3160 };
3161
3162 static void process_clist_done(struct tevent_req *subreq);
3163
3164 static struct tevent_req *process_clist_send(
3165                                         TALLOC_CTX *mem_ctx,
3166                                         struct ctdb_context *ctdb,
3167                                         struct ctdb_connection_list *clist,
3168                                         clist_request_func request_func,
3169                                         clist_reply_func reply_func)
3170 {
3171         struct tevent_req *req, *subreq;
3172         struct process_clist_state *state;
3173         struct ctdb_req_control request;
3174         int i;
3175
3176         req = tevent_req_create(mem_ctx, &state, struct process_clist_state);
3177         if (req == NULL) {
3178                 return NULL;
3179         }
3180
3181         state->clist = clist;
3182         state->reply_func = reply_func;
3183
3184         for (i = 0; i < clist->num; i++) {
3185                 request_func(&request, &clist->conn[i]);
3186                 subreq = ctdb_client_control_send(state, ctdb->ev,
3187                                                   ctdb->client, ctdb->cmd_pnn,
3188                                                   TIMEOUT(), &request);
3189                 if (tevent_req_nomem(subreq, req)) {
3190                         return tevent_req_post(req, ctdb->ev);
3191                 }
3192                 tevent_req_set_callback(subreq, process_clist_done, req);
3193         }
3194
3195         return req;
3196 }
3197
3198 static void process_clist_done(struct tevent_req *subreq)
3199 {
3200         struct tevent_req *req = tevent_req_callback_data(
3201                 subreq, struct tevent_req);
3202         struct process_clist_state *state = tevent_req_data(
3203                 req, struct process_clist_state);
3204         struct ctdb_reply_control *reply;
3205         int ret;
3206         bool status;
3207
3208         status = ctdb_client_control_recv(subreq, NULL, state, &reply);
3209         TALLOC_FREE(subreq);
3210         if (! status) {
3211                 state->num_failed += 1;
3212                 goto done;
3213         }
3214
3215         ret = state->reply_func(reply);
3216         if (ret != 0) {
3217                 state->num_failed += 1;
3218                 goto done;
3219         }
3220
3221 done:
3222         state->num_total += 1;
3223         if (state->num_total == state->clist->num) {
3224                 tevent_req_done(req);
3225         }
3226 }
3227
3228 static int process_clist_recv(struct tevent_req *req)
3229 {
3230         struct process_clist_state *state = tevent_req_data(
3231                 req, struct process_clist_state);
3232
3233         return state->num_failed;
3234 }
3235
3236 static int control_addtickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3237                              int argc, const char **argv)
3238 {
3239         struct ctdb_connection conn;
3240         int ret;
3241
3242         if (argc != 0 && argc != 2) {
3243                 usage("addtickle");
3244         }
3245
3246         if (argc == 0) {
3247                 struct ctdb_connection_list *clist;
3248                 struct tevent_req *req;
3249
3250                 /* Client first but the src/dst logic is confused */
3251                 ret = ctdb_connection_list_read(mem_ctx, 0, false, &clist);
3252                 if (ret != 0) {
3253                         return ret;
3254                 }
3255                 if (clist->num == 0) {
3256                         return 0;
3257                 }
3258
3259                 req = process_clist_send(mem_ctx, ctdb, clist,
3260                                  ctdb_req_control_tcp_add_delayed_update,
3261                                  ctdb_reply_control_tcp_add_delayed_update);
3262                 if (req == NULL) {
3263                         talloc_free(clist);
3264                         return ENOMEM;
3265                 }
3266
3267                 tevent_req_poll(req, ctdb->ev);
3268                 talloc_free(clist);
3269
3270                 ret = process_clist_recv(req);
3271                 if (ret != 0) {
3272                         fprintf(stderr, "Failed to add %d tickles\n", ret);
3273                         return 1;
3274                 }
3275
3276                 return 0;
3277         }
3278
3279         ret = ctdb_sock_addr_from_string(argv[0], &conn.src, true);
3280         if (ret != 0) {
3281                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3282                 return 1;
3283         }
3284         ret = ctdb_sock_addr_from_string(argv[1], &conn.dst, true);
3285         if (ret != 0) {
3286                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3287                 return 1;
3288         }
3289
3290         ret = ctdb_ctrl_tcp_add_delayed_update(mem_ctx, ctdb->ev,
3291                                                ctdb->client, ctdb->cmd_pnn,
3292                                                TIMEOUT(), &conn);
3293         if (ret != 0) {
3294                 fprintf(stderr, "Failed to register connection\n");
3295                 return ret;
3296         }
3297
3298         return 0;
3299 }
3300
3301 static int control_deltickle(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3302                              int argc, const char **argv)
3303 {
3304         struct ctdb_connection conn;
3305         int ret;
3306
3307         if (argc != 0 && argc != 2) {
3308                 usage("deltickle");
3309         }
3310
3311         if (argc == 0) {
3312                 struct ctdb_connection_list *clist;
3313                 struct tevent_req *req;
3314
3315                 /* Client first but the src/dst logic is confused */
3316                 ret = ctdb_connection_list_read(mem_ctx, 0, false, &clist);
3317                 if (ret != 0) {
3318                         return ret;
3319                 }
3320                 if (clist->num == 0) {
3321                         return 0;
3322                 }
3323
3324                 req = process_clist_send(mem_ctx, ctdb, clist,
3325                                          ctdb_req_control_tcp_remove,
3326                                          ctdb_reply_control_tcp_remove);
3327                 if (req == NULL) {
3328                         talloc_free(clist);
3329                         return ENOMEM;
3330                 }
3331
3332                 tevent_req_poll(req, ctdb->ev);
3333                 talloc_free(clist);
3334
3335                 ret = process_clist_recv(req);
3336                 if (ret != 0) {
3337                         fprintf(stderr, "Failed to remove %d tickles\n", ret);
3338                         return 1;
3339                 }
3340
3341                 return 0;
3342         }
3343
3344         ret = ctdb_sock_addr_from_string(argv[0], &conn.src, true);
3345         if (ret != 0) {
3346                 fprintf(stderr, "Invalid IP address %s\n", argv[0]);
3347                 return 1;
3348         }
3349         ret = ctdb_sock_addr_from_string(argv[1], &conn.dst, true);
3350         if (ret != 0) {
3351                 fprintf(stderr, "Invalid IP address %s\n", argv[1]);
3352                 return 1;
3353         }
3354
3355         ret = ctdb_ctrl_tcp_remove(mem_ctx, ctdb->ev, ctdb->client,
3356                                    ctdb->cmd_pnn, TIMEOUT(), &conn);
3357         if (ret != 0) {
3358                 fprintf(stderr, "Failed to unregister connection\n");
3359                 return ret;
3360         }
3361
3362         return 0;
3363 }
3364
3365 static int control_listnodes(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3366                              int argc, const char **argv)
3367 {
3368         struct ctdb_node_map *nodemap;
3369         int i;
3370
3371         if (argc != 0) {
3372                 usage("listnodes");
3373         }
3374
3375         nodemap = read_nodes_file(mem_ctx, CTDB_UNKNOWN_PNN);
3376         if (nodemap == NULL) {
3377                 return 1;
3378         }
3379
3380         for (i=0; i<nodemap->num; i++) {
3381                 if (nodemap->node[i].flags & NODE_FLAGS_DELETED) {
3382                         continue;
3383                 }
3384
3385                 if (options.machinereadable) {
3386                         printf("%s%u%s%s%s\n", options.sep,
3387                                nodemap->node[i].pnn, options.sep,
3388                                ctdb_sock_addr_to_string(
3389                                        mem_ctx, &nodemap->node[i].addr, false),
3390                                options.sep);
3391                 } else {
3392                         printf("%s\n",
3393                                ctdb_sock_addr_to_string(
3394                                        mem_ctx, &nodemap->node[i].addr, false));
3395                 }
3396         }
3397
3398         return 0;
3399 }
3400
3401 static bool nodemap_identical(struct ctdb_node_map *nodemap1,
3402                               struct ctdb_node_map *nodemap2)
3403 {
3404         int i;
3405
3406         if (nodemap1->num != nodemap2->num) {
3407                 return false;
3408         }
3409
3410         for (i=0; i<nodemap1->num; i++) {
3411                 struct ctdb_node_and_flags *n1, *n2;
3412
3413                 n1 = &nodemap1->node[i];
3414                 n2 = &nodemap2->node[i];
3415
3416                 if ((n1->pnn != n2->pnn) ||
3417                     (n1->flags != n2->flags) ||
3418                     ! ctdb_sock_addr_same_ip(&n1->addr, &n2->addr)) {
3419                         return false;
3420                 }
3421         }
3422
3423         return true;
3424 }
3425
3426 static int check_node_file_changes(TALLOC_CTX *mem_ctx,
3427                                    struct ctdb_node_map *nm,
3428                                    struct ctdb_node_map *fnm,
3429                                    bool *reload)
3430 {
3431         int i;
3432         bool check_failed = false;
3433
3434         *reload = false;
3435
3436         for (i=0; i<nm->num; i++) {
3437                 if (i >= fnm->num) {
3438                         fprintf(stderr,
3439                                 "Node %u (%s) missing from nodes file\n",
3440                                 nm->node[i].pnn,
3441                                 ctdb_sock_addr_to_string(
3442                                         mem_ctx, &nm->node[i].addr, false));
3443                         check_failed = true;
3444                         continue;
3445                 }
3446                 if (nm->node[i].flags & NODE_FLAGS_DELETED &&
3447                     fnm->node[i].flags & NODE_FLAGS_DELETED) {
3448                         /* Node remains deleted */
3449                         continue;
3450                 }
3451
3452                 if (! (nm->node[i].flags & NODE_FLAGS_DELETED) &&
3453                     ! (fnm->node[i].flags & NODE_FLAGS_DELETED)) {
3454                         /* Node not newly nor previously deleted */
3455                         if (! ctdb_same_ip(&nm->node[i].addr,
3456                                            &fnm->node[i].addr)) {
3457                                 fprintf(stderr,
3458                                         "Node %u has changed IP address"
3459                                         " (was %s, now %s)\n",
3460                                         nm->node[i].pnn,
3461                                         ctdb_sock_addr_to_string(
3462                                                 mem_ctx,
3463                                                 &nm->node[i].addr, false),
3464                                         ctdb_sock_addr_to_string(
3465                                                 mem_ctx,
3466                                                 &fnm->node[i].addr, false));
3467                                 check_failed = true;
3468                         } else {
3469                                 if (nm->node[i].flags & NODE_FLAGS_DISCONNECTED) {
3470                                         fprintf(stderr,
3471                                                 "WARNING: Node %u is disconnected."
3472                                                 " You MUST fix this node manually!\n",
3473                                                 nm->node[i].pnn);
3474                                 }
3475                         }
3476                         continue;
3477                 }
3478
3479                 if (fnm->node[i].flags & NODE_FLAGS_DELETED) {
3480                         /* Node is being deleted */
3481                         printf("Node %u is DELETED\n", nm->node[i].pnn);
3482                         *reload = true;
3483                         if (! (nm->node[i].flags & NODE_FLAGS_DISCONNECTED)) {
3484                                 fprintf(stderr,
3485                                         "ERROR: Node %u is still connected\n",
3486                                         nm->node[i].pnn);
3487                                 check_failed = true;
3488                         }
3489                         continue;
3490                 }
3491
3492                 if (nm->node[i].flags & NODE_FLAGS_DELETED) {
3493                         /* Node was previously deleted */
3494                         printf("Node %u is UNDELETED\n", nm->node[i].pnn);
3495                         *reload = true;
3496                 }
3497         }
3498
3499         if (check_failed) {
3500                 fprintf(stderr,
3501                         "ERROR: Nodes will not be reloaded due to previous error\n");
3502                 return 1;
3503         }
3504
3505         /* Leftover nodes in file are NEW */
3506         for (; i < fnm->num; i++) {
3507                 printf("Node %u is NEW\n", fnm->node[i].pnn);
3508                 *reload = true;
3509         }
3510
3511         return 0;
3512 }
3513
3514 struct disable_recoveries_state {
3515         uint32_t *pnn_list;
3516         int node_count;
3517         bool *reply;
3518         int status;
3519         bool done;
3520 };
3521
3522 static void disable_recoveries_handler(uint64_t srvid, TDB_DATA data,
3523                                        void *private_data)
3524 {
3525         struct disable_recoveries_state *state =
3526                 (struct disable_recoveries_state *)private_data;
3527         int ret, i;
3528
3529         if (data.dsize != sizeof(int)) {
3530                 /* Ignore packet */
3531                 return;
3532         }
3533
3534         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
3535         ret = *(int *)data.dptr;
3536         if (ret < 0) {
3537                 state->status = ret;
3538                 state->done = true;
3539                 return;
3540         }
3541         for (i=0; i<state->node_count; i++) {
3542                 if (state->pnn_list[i] == ret) {
3543                         state->reply[i] = true;
3544                         break;
3545                 }
3546         }
3547
3548         state->done = true;
3549         for (i=0; i<state->node_count; i++) {
3550                 if (! state->reply[i]) {
3551                         state->done = false;
3552                         break;
3553                 }
3554         }
3555 }
3556
3557 static int disable_recoveries(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3558                               uint32_t timeout, uint32_t *pnn_list, int count)
3559 {
3560         struct ctdb_disable_message disable = { 0 };
3561         struct disable_recoveries_state state;
3562         int ret, i;
3563
3564         disable.pnn = ctdb->pnn;
3565         disable.srvid = next_srvid(ctdb);
3566         disable.timeout = timeout;
3567
3568         state.pnn_list = pnn_list;
3569         state.node_count = count;
3570         state.done = false;
3571         state.status = 0;
3572         state.reply = talloc_zero_array(mem_ctx, bool, count);
3573         if (state.reply == NULL) {
3574                 return ENOMEM;
3575         }
3576
3577         ret = ctdb_client_set_message_handler(ctdb->ev, ctdb->client,
3578                                               disable.srvid,
3579                                               disable_recoveries_handler,
3580                                               &state);
3581         if (ret != 0) {
3582                 return ret;
3583         }
3584
3585         for (i=0; i<count; i++) {
3586                 ret = ctdb_message_disable_recoveries(mem_ctx, ctdb->ev,
3587                                                       ctdb->client,
3588                                                       pnn_list[i],
3589                                                       &disable);
3590                 if (ret != 0) {
3591                         goto fail;
3592                 }
3593         }
3594
3595         ret = ctdb_client_wait_timeout(ctdb->ev, &state.done, TIMEOUT());
3596         if (ret == ETIME) {
3597                 fprintf(stderr, "Timed out waiting to disable recoveries\n");
3598         } else {
3599                 ret = (state.status >= 0 ? 0 : 1);
3600         }
3601
3602 fail:
3603         ctdb_client_remove_message_handler(ctdb->ev, ctdb->client,
3604                                            disable.srvid, &state);
3605         return ret;
3606 }
3607
3608 static int control_reloadnodes(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3609                                int argc, const char **argv)
3610 {
3611         struct ctdb_node_map *nodemap = NULL;
3612         struct ctdb_node_map *file_nodemap;
3613         struct ctdb_node_map *remote_nodemap;
3614         struct ctdb_req_control request;
3615         struct ctdb_reply_control **reply;
3616         bool reload;
3617         int ret, i;
3618         uint32_t *pnn_list;
3619         int count;
3620
3621         nodemap = get_nodemap(ctdb, false);
3622         if (nodemap == NULL) {
3623                 return 1;
3624         }
3625
3626         file_nodemap = read_nodes_file(mem_ctx, ctdb->pnn);
3627         if (file_nodemap == NULL) {
3628                 return 1;
3629         }
3630
3631         for (i=0; i<nodemap->num; i++) {
3632                 if (nodemap->node[i].flags & NODE_FLAGS_DISCONNECTED) {
3633                         continue;
3634                 }
3635
3636                 ret = ctdb_ctrl_get_nodes_file(mem_ctx, ctdb->ev, ctdb->client,
3637                                                nodemap->node[i].pnn, TIMEOUT(),
3638                                                &remote_nodemap);
3639                 if (ret != 0) {
3640                         fprintf(stderr,
3641                                 "ERROR: Failed to get nodes file from node %u\n",
3642                                 nodemap->node[i].pnn);
3643                         return ret;
3644                 }
3645
3646                 if (! nodemap_identical(file_nodemap, remote_nodemap)) {
3647                         fprintf(stderr,
3648                                 "ERROR: Nodes file on node %u differs"
3649                                  " from current node (%u)\n",
3650                                  nodemap->node[i].pnn, ctdb->pnn);
3651                         return 1;
3652                 }
3653         }
3654
3655         ret = check_node_file_changes(mem_ctx, nodemap, file_nodemap, &reload);
3656         if (ret != 0) {
3657                 return ret;
3658         }
3659
3660         if (! reload) {
3661                 fprintf(stderr, "No change in nodes file,"
3662                                 " skipping unnecessary reload\n");
3663                 return 0;
3664         }
3665
3666         count = list_of_connected_nodes(nodemap, CTDB_UNKNOWN_PNN,
3667                                         mem_ctx, &pnn_list);
3668         if (count <= 0) {
3669                 fprintf(stderr, "Memory allocation error\n");
3670                 return 1;
3671         }
3672
3673         ret = disable_recoveries(mem_ctx, ctdb, 2*options.timelimit,
3674                                  pnn_list, count);
3675         if (ret != 0) {
3676                 fprintf(stderr, "Failed to disable recoveries\n");
3677                 return ret;
3678         }
3679
3680         ctdb_req_control_reload_nodes_file(&request);
3681         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
3682                                         pnn_list, count, TIMEOUT(),
3683                                         &request, NULL, &reply);
3684         if (ret != 0) {
3685                 bool failed = false;
3686
3687                 for (i=0; i<count; i++) {
3688                         ret = ctdb_reply_control_reload_nodes_file(reply[i]);
3689                         if (ret != 0) {
3690                                 fprintf(stderr,
3691                                         "Node %u failed to reload nodes\n",
3692                                         pnn_list[i]);
3693                                 failed = true;
3694                         }
3695                 }
3696                 if (failed) {
3697                         fprintf(stderr,
3698                                 "You MUST fix failed nodes manually!\n");
3699                 }
3700         }
3701
3702         ret = disable_recoveries(mem_ctx, ctdb, 0, pnn_list, count);
3703         if (ret != 0) {
3704                 fprintf(stderr, "Failed to enable recoveries\n");
3705                 return ret;
3706         }
3707
3708         return 0;
3709 }
3710
3711 static int moveip(TALLOC_CTX *mem_ctx, struct ctdb_context *ctdb,
3712                   ctdb_sock_addr *addr, uint32_t pnn)
3713 {
3714         struct ctdb_public_ip_list *pubip_list;
3715         struct ctdb_public_ip pubip;
3716         struct ctdb_node_map *nodemap;
3717         struct ctdb_req_control request;
3718         uint32_t *pnn_list;
3719         int ret, i, count;
3720
3721         ret = ctdb_message_disable_ip_check(mem_ctx, ctdb->ev, ctdb->client,
3722                                             CTDB_BROADCAST_CONNECTED,
3723                                             2*options.timelimit);
3724         if (ret != 0) {
3725                 fprintf(stderr, "Failed to disable IP check\n");
3726                 return ret;
3727         }
3728
3729         ret = ctdb_ctrl_get_public_ips(mem_ctx, ctdb->ev, ctdb->client,
3730                                        pnn, TIMEOUT(), false, &pubip_list);
3731         if (ret != 0) {
3732                 fprintf(stderr, "Failed to get Public IPs from node %u\n",
3733                         pnn);
3734                 return ret;
3735         }
3736
3737         for (i=0; i<pubip_list->num; i++) {
3738                 if (ctdb_same_ip(addr, &pubip_list->ip[i].addr)) {
3739                         break;
3740                 }
3741         }
3742
3743         if (i == pubip_list->num) {
3744                 fprintf(stderr, "Node %u CANNOT host IP address %s\n",
3745                         pnn, ctdb_sock_addr_to_string(mem_ctx, addr, false));
3746                 return 1;
3747         }
3748
3749         nodemap = get_nodemap(ctdb, false);
3750         if (nodemap == NULL) {
3751                 return 1;
3752         }
3753
3754         count = list_of_active_nodes(nodemap, pnn, mem_ctx, &pnn_list);
3755         if (count <= 0) {
3756                 fprintf(stderr, "Memory allocation error\n");
3757                 return 1;
3758         }
3759
3760         pubip.pnn = pnn;
3761         pubip.addr = *addr;
3762         ctdb_req_control_release_ip(&request, &pubip);
3763
3764         ret = ctdb_client_control_multi(mem_ctx, ctdb->ev, ctdb->client,
3765              &n