ctdb-daemon: Stop using tevent compatibility definitions
[vlendec/samba-autobuild/.git] / ctdb / tools / ctdb.c
1 /* 
2    ctdb control tool
3
4    Copyright (C) Andrew Tridgell  2007
5    Copyright (C) Ronnie Sahlberg  2007
6
7    This program is free software; you can redistribute it and/or modify
8    it under the terms of the GNU General Public License as published by
9    the Free Software Foundation; either version 3 of the License, or
10    (at your option) any later version.
11    
12    This program is distributed in the hope that it will be useful,
13    but WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15    GNU General Public License for more details.
16    
17    You should have received a copy of the GNU General Public License
18    along with this program; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "includes.h"
22 #include "system/time.h"
23 #include "system/filesys.h"
24 #include "system/network.h"
25 #include "system/locale.h"
26 #include "popt.h"
27 #include "common/cmdline.h"
28 #include "../include/ctdb_version.h"
29 #include "../include/ctdb_client.h"
30 #include "../include/ctdb_private.h"
31 #include "../common/rb_tree.h"
32 #include "lib/tdb_wrap/tdb_wrap.h"
33 #include "lib/util/dlinklist.h"
34 #include "common/system.h"
35
36 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
37 #define ERR_NONODE      21      /* node does not exist */
38 #define ERR_DISNODE     22      /* node is disconnected */
39
40 static void usage(void);
41
42 static struct {
43         int timelimit;
44         uint32_t pnn;
45         uint32_t *nodes;
46         int machinereadable;
47         const char *machineseparator;
48         int verbose;
49         int maxruntime;
50         int printemptyrecords;
51         int printdatasize;
52         int printlmaster;
53         int printhash;
54         int printrecordflags;
55 } options;
56
57 #define LONGTIMEOUT options.timelimit*10
58
59 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
60 #define LONGTIMELIMIT() timeval_current_ofs(LONGTIMEOUT, 0)
61
62 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
63 {
64         return (tv2->tv_sec - tv->tv_sec) +
65                 (tv2->tv_usec - tv->tv_usec)*1.0e-6;
66 }
67
68 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
69 {
70         printf("CTDB version: %s\n", CTDB_VERSION_STRING);
71         return 0;
72 }
73
74 /* Like printf(3) but substitute for separator in format */
75 static int printm(const char *format, ...) PRINTF_ATTRIBUTE(1,2);
76 static int printm(const char *format, ...)
77 {
78         va_list ap;
79         int ret;
80         size_t len = strlen(format);
81         char new_format[len+1];
82
83         strcpy(new_format, format);
84
85         if (options.machineseparator[0] != ':') {
86                 all_string_sub(new_format,
87                                ":", options.machineseparator, len + 1);
88         }
89
90         va_start(ap, format);
91         ret = vprintf(new_format, ap);
92         va_end(ap);
93
94         return ret;
95 }
96
97 #define CTDB_NOMEM_ABORT(p) do { if (!(p)) {                            \
98                 DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n",            \
99                                    "Out of memory in " __location__ )); \
100                 abort();                                                \
101         }} while (0)
102
103 static uint32_t getpnn(struct ctdb_context *ctdb)
104 {
105         if ((options.pnn == CTDB_BROADCAST_ALL) ||
106             (options.pnn == CTDB_MULTICAST)) {
107                 DEBUG(DEBUG_ERR,
108                       ("Cannot get PNN for node %u\n", options.pnn));
109                 exit(1);
110         }
111
112         if (options.pnn == CTDB_CURRENT_NODE) {
113                 return ctdb_get_pnn(ctdb);
114         } else {
115                 return options.pnn;
116         }
117 }
118
119 static void assert_single_node_only(void)
120 {
121         if ((options.pnn == CTDB_BROADCAST_ALL) ||
122             (options.pnn == CTDB_MULTICAST)) {
123                 DEBUG(DEBUG_ERR,
124                       ("This control can not be applied to multiple PNNs\n"));
125                 exit(1);
126         }
127 }
128
129 static void assert_current_node_only(struct ctdb_context *ctdb)
130 {
131         if (options.pnn != ctdb_get_pnn(ctdb)) {
132                 DEBUG(DEBUG_ERR,
133                       ("This control can only be applied to the current node\n"));
134                 exit(1);
135         }
136 }
137
138 /* Pretty print the flags to a static buffer in human-readable format.
139  * This never returns NULL!
140  */
141 static const char *pretty_print_flags(uint32_t flags)
142 {
143         int j;
144         static const struct {
145                 uint32_t flag;
146                 const char *name;
147         } flag_names[] = {
148                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
149                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
150                 { NODE_FLAGS_BANNED,                "BANNED" },
151                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
152                 { NODE_FLAGS_DELETED,               "DELETED" },
153                 { NODE_FLAGS_STOPPED,               "STOPPED" },
154                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
155         };
156         static char flags_str[512]; /* Big enough to contain all flag names */
157
158         flags_str[0] = '\0';
159         for (j=0;j<ARRAY_SIZE(flag_names);j++) {
160                 if (flags & flag_names[j].flag) {
161                         if (flags_str[0] == '\0') {
162                                 (void) strcpy(flags_str, flag_names[j].name);
163                         } else {
164                                 (void) strncat(flags_str, "|", sizeof(flags_str)-1);
165                                 (void) strncat(flags_str, flag_names[j].name,
166                                                sizeof(flags_str)-1);
167                         }
168                 }
169         }
170         if (flags_str[0] == '\0') {
171                 (void) strcpy(flags_str, "OK");
172         }
173
174         return flags_str;
175 }
176
177 static int h2i(char h)
178 {
179         if (h >= 'a' && h <= 'f') return h - 'a' + 10;
180         if (h >= 'A' && h <= 'F') return h - 'f' + 10;
181         return h - '0';
182 }
183
184 static TDB_DATA hextodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
185 {
186         int i;
187         TDB_DATA key = {NULL, 0};
188
189         if (len & 0x01) {
190                 DEBUG(DEBUG_ERR,("Key specified with odd number of hexadecimal digits\n"));
191                 return key;
192         }
193
194         key.dsize = len>>1;
195         key.dptr  = talloc_size(mem_ctx, key.dsize);
196
197         for (i=0; i < len/2; i++) {
198                 key.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
199         }
200         return key;
201 }
202
203 static TDB_DATA strtodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
204 {
205         TDB_DATA key;
206
207         if (!strncmp(str, "0x", 2)) {
208                 key = hextodata(mem_ctx, str + 2, len - 2);
209         } else {
210                 key.dptr  = talloc_memdup(mem_ctx, str, len);
211                 key.dsize = len;
212         }
213
214         return key;
215 }
216
217 /* Parse a nodestring.  Parameter dd_ok controls what happens to nodes
218  * that are disconnected or deleted.  If dd_ok is true those nodes are
219  * included in the output list of nodes.  If dd_ok is false, those
220  * nodes are filtered from the "all" case and cause an error if
221  * explicitly specified.
222  */
223 static bool parse_nodestring(struct ctdb_context *ctdb,
224                              TALLOC_CTX *mem_ctx,
225                              const char * nodestring,
226                              uint32_t current_pnn,
227                              bool dd_ok,
228                              uint32_t **nodes,
229                              uint32_t *pnn_mode)
230 {
231         TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
232         int n;
233         uint32_t i;
234         struct ctdb_node_map *nodemap;
235         int ret;
236
237         *nodes = NULL;
238
239         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
240         if (ret != 0) {
241                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
242                 talloc_free(tmp_ctx);
243                 exit(10);
244         }
245
246         if (nodestring != NULL) {
247                 *nodes = talloc_array(mem_ctx, uint32_t, 0);
248                 if (*nodes == NULL) {
249                         goto failed;
250                 }
251
252                 n = 0;
253
254                 if (strcmp(nodestring, "all") == 0) {
255                         *pnn_mode = CTDB_BROADCAST_ALL;
256
257                         /* all */
258                         for (i = 0; i < nodemap->num; i++) {
259                                 if ((nodemap->nodes[i].flags &
260                                      (NODE_FLAGS_DISCONNECTED |
261                                       NODE_FLAGS_DELETED)) && !dd_ok) {
262                                         continue;
263                                 }
264                                 *nodes = talloc_realloc(mem_ctx, *nodes,
265                                                         uint32_t, n+1);
266                                 if (*nodes == NULL) {
267                                         goto failed;
268                                 }
269                                 (*nodes)[n] = i;
270                                 n++;
271                         }
272                 } else {
273                         /* x{,y...} */
274                         char *ns, *tok;
275
276                         ns = talloc_strdup(tmp_ctx, nodestring);
277                         tok = strtok(ns, ",");
278                         while (tok != NULL) {
279                                 uint32_t pnn;
280                                 char *endptr;
281                                 i = (uint32_t)strtoul(tok, &endptr, 0);
282                                 if (i == 0 && tok == endptr) {
283                                         DEBUG(DEBUG_ERR,
284                                               ("Invalid node %s\n", tok));
285                                         talloc_free(tmp_ctx);
286                                         exit(ERR_NONODE);
287                                 }
288                                 if (i >= nodemap->num) {
289                                         DEBUG(DEBUG_ERR, ("Node %u does not exist\n", i));
290                                         talloc_free(tmp_ctx);
291                                         exit(ERR_NONODE);
292                                 }
293                                 if ((nodemap->nodes[i].flags & 
294                                      (NODE_FLAGS_DISCONNECTED |
295                                       NODE_FLAGS_DELETED)) && !dd_ok) {
296                                         DEBUG(DEBUG_ERR, ("Node %u has status %s\n", i, pretty_print_flags(nodemap->nodes[i].flags)));
297                                         talloc_free(tmp_ctx);
298                                         exit(ERR_DISNODE);
299                                 }
300                                 if ((pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), i)) < 0) {
301                                         DEBUG(DEBUG_ERR, ("Can not access node %u. Node is not operational.\n", i));
302                                         talloc_free(tmp_ctx);
303                                         exit(10);
304                                 }
305
306                                 *nodes = talloc_realloc(mem_ctx, *nodes,
307                                                         uint32_t, n+1);
308                                 if (*nodes == NULL) {
309                                         goto failed;
310                                 }
311
312                                 (*nodes)[n] = i;
313                                 n++;
314
315                                 tok = strtok(NULL, ",");
316                         }
317                         talloc_free(ns);
318
319                         if (n == 1) {
320                                 *pnn_mode = (*nodes)[0];
321                         } else {
322                                 *pnn_mode = CTDB_MULTICAST;
323                         }
324                 }
325         } else {
326                 /* default - no nodes specified */
327                 *nodes = talloc_array(mem_ctx, uint32_t, 1);
328                 if (*nodes == NULL) {
329                         goto failed;
330                 }
331                 *pnn_mode = CTDB_CURRENT_NODE;
332
333                 if (((*nodes)[0] = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), current_pnn)) < 0) {
334                         goto failed;
335                 }
336         }
337
338         talloc_free(tmp_ctx);
339         return true;
340
341 failed:
342         talloc_free(tmp_ctx);
343         return false;
344 }
345
346 /*
347  check if a database exists
348 */
349 static bool db_exists(struct ctdb_context *ctdb, const char *dbarg,
350                       uint32_t *dbid, const char **dbname, uint8_t *flags)
351 {
352         int i, ret;
353         struct ctdb_dbid_map *dbmap=NULL;
354         bool dbid_given = false, found = false;
355         uint32_t id;
356         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
357         const char *name = NULL;
358
359         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
360         if (ret != 0) {
361                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
362                 goto fail;
363         }
364
365         if (strncmp(dbarg, "0x", 2) == 0) {
366                 id = strtoul(dbarg, NULL, 0);
367                 dbid_given = true;
368         }
369
370         for(i=0; i<dbmap->num; i++) {
371                 if (dbid_given) {
372                         if (id == dbmap->dbs[i].dbid) {
373                                 found = true;
374                                 break;
375                         }
376                 } else {
377                         ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
378                         if (ret != 0) {
379                                 DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
380                                 goto fail;
381                         }
382
383                         if (strcmp(name, dbarg) == 0) {
384                                 id = dbmap->dbs[i].dbid;
385                                 found = true;
386                                 break;
387                         }
388                 }
389         }
390
391         if (found && dbid_given && dbname != NULL) {
392                 ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
393                 if (ret != 0) {
394                         DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].dbid));
395                         found = false;
396                         goto fail;
397                 }
398         }
399
400         if (found) {
401                 if (dbid) *dbid = id;
402                 if (dbname) *dbname = talloc_strdup(ctdb, name);
403                 if (flags) *flags = dbmap->dbs[i].flags;
404         } else {
405                 DEBUG(DEBUG_ERR,("No database matching '%s' found\n", dbarg));
406         }
407
408 fail:
409         talloc_free(tmp_ctx);
410         return found;
411 }
412
413 /*
414   see if a process exists
415  */
416 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
417 {
418         uint32_t pnn, pid;
419         int ret;
420         if (argc < 1) {
421                 usage();
422         }
423
424         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
425                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
426                 return -1;
427         }
428
429         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
430         if (ret == 0) {
431                 printf("%u:%u exists\n", pnn, pid);
432         } else {
433                 printf("%u:%u does not exist\n", pnn, pid);
434         }
435         return ret;
436 }
437
438 /*
439   display statistics structure
440  */
441 static void show_statistics(struct ctdb_statistics *s, int show_header)
442 {
443         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
444         int i;
445         const char *prefix=NULL;
446         int preflen=0;
447         int tmp, days, hours, minutes, seconds;
448         const struct {
449                 const char *name;
450                 uint32_t offset;
451         } fields[] = {
452 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
453                 STATISTICS_FIELD(num_clients),
454                 STATISTICS_FIELD(frozen),
455                 STATISTICS_FIELD(recovering),
456                 STATISTICS_FIELD(num_recoveries),
457                 STATISTICS_FIELD(client_packets_sent),
458                 STATISTICS_FIELD(client_packets_recv),
459                 STATISTICS_FIELD(node_packets_sent),
460                 STATISTICS_FIELD(node_packets_recv),
461                 STATISTICS_FIELD(keepalive_packets_sent),
462                 STATISTICS_FIELD(keepalive_packets_recv),
463                 STATISTICS_FIELD(node.req_call),
464                 STATISTICS_FIELD(node.reply_call),
465                 STATISTICS_FIELD(node.req_dmaster),
466                 STATISTICS_FIELD(node.reply_dmaster),
467                 STATISTICS_FIELD(node.reply_error),
468                 STATISTICS_FIELD(node.req_message),
469                 STATISTICS_FIELD(node.req_control),
470                 STATISTICS_FIELD(node.reply_control),
471                 STATISTICS_FIELD(client.req_call),
472                 STATISTICS_FIELD(client.req_message),
473                 STATISTICS_FIELD(client.req_control),
474                 STATISTICS_FIELD(timeouts.call),
475                 STATISTICS_FIELD(timeouts.control),
476                 STATISTICS_FIELD(timeouts.traverse),
477                 STATISTICS_FIELD(locks.num_calls),
478                 STATISTICS_FIELD(locks.num_current),
479                 STATISTICS_FIELD(locks.num_pending),
480                 STATISTICS_FIELD(locks.num_failed),
481                 STATISTICS_FIELD(total_calls),
482                 STATISTICS_FIELD(pending_calls),
483                 STATISTICS_FIELD(childwrite_calls),
484                 STATISTICS_FIELD(pending_childwrite_calls),
485                 STATISTICS_FIELD(memory_used),
486                 STATISTICS_FIELD(max_hop_count),
487                 STATISTICS_FIELD(total_ro_delegations),
488                 STATISTICS_FIELD(total_ro_revokes),
489         };
490         
491         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
492         seconds = tmp%60;
493         tmp    /= 60;
494         minutes = tmp%60;
495         tmp    /= 60;
496         hours   = tmp%24;
497         tmp    /= 24;
498         days    = tmp;
499
500         if (options.machinereadable){
501                 if (show_header) {
502                         printm("CTDB version:");
503                         printm("Current time of statistics:");
504                         printm("Statistics collected since:");
505                         for (i=0;i<ARRAY_SIZE(fields);i++) {
506                                 printm("%s:", fields[i].name);
507                         }
508                         printm("num_reclock_ctdbd_latency:");
509                         printm("min_reclock_ctdbd_latency:");
510                         printm("avg_reclock_ctdbd_latency:");
511                         printm("max_reclock_ctdbd_latency:");
512
513                         printm("num_reclock_recd_latency:");
514                         printm("min_reclock_recd_latency:");
515                         printm("avg_reclock_recd_latency:");
516                         printm("max_reclock_recd_latency:");
517
518                         printm("num_call_latency:");
519                         printm("min_call_latency:");
520                         printm("avg_call_latency:");
521                         printm("max_call_latency:");
522
523                         printm("num_lockwait_latency:");
524                         printm("min_lockwait_latency:");
525                         printm("avg_lockwait_latency:");
526                         printm("max_lockwait_latency:");
527
528                         printm("num_childwrite_latency:");
529                         printm("min_childwrite_latency:");
530                         printm("avg_childwrite_latency:");
531                         printm("max_childwrite_latency:");
532                         printm("\n");
533                 }
534                 printm("%d:", CTDB_PROTOCOL);
535                 printm("%d:", (int)s->statistics_current_time.tv_sec);
536                 printm("%d:", (int)s->statistics_start_time.tv_sec);
537                 for (i=0;i<ARRAY_SIZE(fields);i++) {
538                         printm("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
539                 }
540                 printm("%d:", s->reclock.ctdbd.num);
541                 printm("%.6f:", s->reclock.ctdbd.min);
542                 printm("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
543                 printm("%.6f:", s->reclock.ctdbd.max);
544
545                 printm("%d:", s->reclock.recd.num);
546                 printm("%.6f:", s->reclock.recd.min);
547                 printm("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
548                 printm("%.6f:", s->reclock.recd.max);
549
550                 printm("%d:", s->call_latency.num);
551                 printm("%.6f:", s->call_latency.min);
552                 printm("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
553                 printm("%.6f:", s->call_latency.max);
554
555                 printm("%d:", s->childwrite_latency.num);
556                 printm("%.6f:", s->childwrite_latency.min);
557                 printm("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
558                 printm("%.6f:", s->childwrite_latency.max);
559                 printm("\n");
560         } else {
561                 printf("CTDB version %u\n", CTDB_PROTOCOL);
562                 printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
563                 printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
564
565                 for (i=0;i<ARRAY_SIZE(fields);i++) {
566                         if (strchr(fields[i].name, '.')) {
567                                 preflen = strcspn(fields[i].name, ".")+1;
568                                 if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
569                                         prefix = fields[i].name;
570                                         printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
571                                 }
572                         } else {
573                                 preflen = 0;
574                         }
575                         printf(" %*s%-22s%*s%10u\n", 
576                                preflen?4:0, "",
577                                fields[i].name+preflen, 
578                                preflen?0:4, "",
579                                *(uint32_t *)(fields[i].offset+(uint8_t *)s));
580                 }
581                 printf(" hop_count_buckets:");
582                 for (i=0;i<MAX_COUNT_BUCKETS;i++) {
583                         printf(" %d", s->hop_count_bucket[i]);
584                 }
585                 printf("\n");
586                 printf(" lock_buckets:");
587                 for (i=0; i<MAX_COUNT_BUCKETS; i++) {
588                         printf(" %d", s->locks.buckets[i]);
589                 }
590                 printf("\n");
591                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
592
593                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
594
595                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
596
597                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
598                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
599         }
600
601         talloc_free(tmp_ctx);
602 }
603
604 /*
605   display remote ctdb statistics combined from all nodes
606  */
607 static int control_statistics_all(struct ctdb_context *ctdb)
608 {
609         int ret, i;
610         struct ctdb_statistics statistics;
611         uint32_t *nodes;
612         uint32_t num_nodes;
613
614         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
615         CTDB_NO_MEMORY(ctdb, nodes);
616         
617         ZERO_STRUCT(statistics);
618
619         for (i=0;i<num_nodes;i++) {
620                 struct ctdb_statistics s1;
621                 int j;
622                 uint32_t *v1 = (uint32_t *)&s1;
623                 uint32_t *v2 = (uint32_t *)&statistics;
624                 uint32_t num_ints = 
625                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
626                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
627                 if (ret != 0) {
628                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
629                         return ret;
630                 }
631                 for (j=0;j<num_ints;j++) {
632                         v2[j] += v1[j];
633                 }
634                 statistics.max_hop_count = 
635                         MAX(statistics.max_hop_count, s1.max_hop_count);
636                 statistics.call_latency.max = 
637                         MAX(statistics.call_latency.max, s1.call_latency.max);
638         }
639         talloc_free(nodes);
640         printf("Gathered statistics for %u nodes\n", num_nodes);
641         show_statistics(&statistics, 1);
642         return 0;
643 }
644
645 /*
646   display remote ctdb statistics
647  */
648 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
649 {
650         int ret;
651         struct ctdb_statistics statistics;
652
653         if (options.pnn == CTDB_BROADCAST_ALL) {
654                 return control_statistics_all(ctdb);
655         }
656
657         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
658         if (ret != 0) {
659                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
660                 return ret;
661         }
662         show_statistics(&statistics, 1);
663         return 0;
664 }
665
666
667 /*
668   reset remote ctdb statistics
669  */
670 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
671 {
672         int ret;
673
674         ret = ctdb_statistics_reset(ctdb, options.pnn);
675         if (ret != 0) {
676                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
677                 return ret;
678         }
679         return 0;
680 }
681
682
683 /*
684   display remote ctdb rolling statistics
685  */
686 static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
687 {
688         int ret;
689         struct ctdb_statistics_wire *stats;
690         int i, num_records = -1;
691
692         assert_single_node_only();
693
694         if (argc ==1) {
695                 num_records = atoi(argv[0]) - 1;
696         }
697
698         ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
699         if (ret != 0) {
700                 DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
701                 return ret;
702         }
703         for (i=0;i<stats->num;i++) {
704                 if (stats->stats[i].statistics_start_time.tv_sec == 0) {
705                         continue;
706                 }
707                 show_statistics(&stats->stats[i], i==0);
708                 if (i == num_records) {
709                         break;
710                 }
711         }
712         return 0;
713 }
714
715
716 /*
717   display remote ctdb db statistics
718  */
719 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
720 {
721         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
722         struct ctdb_db_statistics *dbstat;
723         int i;
724         uint32_t db_id;
725         int num_hot_keys;
726         int ret;
727
728         if (argc < 1) {
729                 usage();
730         }
731
732         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
733                 return -1;
734         }
735
736         ret = ctdb_ctrl_dbstatistics(ctdb, options.pnn, db_id, tmp_ctx, &dbstat);
737         if (ret != 0) {
738                 DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
739                 talloc_free(tmp_ctx);
740                 return -1;
741         }
742
743         printf("DB Statistics: %s\n", argv[0]);
744         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
745                 dbstat->db_ro_delegations);
746         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
747                 dbstat->db_ro_delegations);
748         printf(" %s\n", "locks");
749         printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
750                 dbstat->locks.num_calls);
751         printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
752                 dbstat->locks.num_failed);
753         printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
754                 dbstat->locks.num_current);
755         printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
756                 dbstat->locks.num_pending);
757         printf(" %s", "hop_count_buckets:");
758         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
759                 printf(" %d", dbstat->hop_count_bucket[i]);
760         }
761         printf("\n");
762         printf(" %s", "lock_buckets:");
763         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
764                 printf(" %d", dbstat->locks.buckets[i]);
765         }
766         printf("\n");
767         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
768                 "locks_latency      MIN/AVG/MAX",
769                 dbstat->locks.latency.min,
770                 (dbstat->locks.latency.num ?
771                  dbstat->locks.latency.total /dbstat->locks.latency.num :
772                  0.0),
773                 dbstat->locks.latency.max,
774                 dbstat->locks.latency.num);
775         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
776                 "vacuum_latency     MIN/AVG/MAX",
777                 dbstat->vacuum.latency.min,
778                 (dbstat->vacuum.latency.num ?
779                  dbstat->vacuum.latency.total /dbstat->vacuum.latency.num :
780                  0.0),
781                 dbstat->vacuum.latency.max,
782                 dbstat->vacuum.latency.num);
783         num_hot_keys = 0;
784         for (i=0; i<dbstat->num_hot_keys; i++) {
785                 if (dbstat->hot_keys[i].count > 0) {
786                         num_hot_keys++;
787                 }
788         }
789         dbstat->num_hot_keys = num_hot_keys;
790
791         printf(" Num Hot Keys:     %d\n", dbstat->num_hot_keys);
792         for (i = 0; i < dbstat->num_hot_keys; i++) {
793                 int j;
794                 printf("     Count:%d Key:", dbstat->hot_keys[i].count);
795                 for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
796                         printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
797                 }
798                 printf("\n");
799         }
800
801         talloc_free(tmp_ctx);
802         return 0;
803 }
804
805 /*
806   display uptime of remote node
807  */
808 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
809 {
810         int ret;
811         struct ctdb_uptime *uptime = NULL;
812         int tmp, days, hours, minutes, seconds;
813
814         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
815         if (ret != 0) {
816                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
817                 return ret;
818         }
819
820         if (options.machinereadable){
821                 printm(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
822                 printm(":%u:%u:%u:%lf\n",
823                         (unsigned int)uptime->current_time.tv_sec,
824                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
825                         (unsigned int)uptime->last_recovery_finished.tv_sec,
826                         timeval_delta(&uptime->last_recovery_finished,
827                                       &uptime->last_recovery_started)
828                 );
829                 return 0;
830         }
831
832         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
833
834         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
835         seconds = tmp%60;
836         tmp    /= 60;
837         minutes = tmp%60;
838         tmp    /= 60;
839         hours   = tmp%24;
840         tmp    /= 24;
841         days    = tmp;
842         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
843
844         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
845         seconds = tmp%60;
846         tmp    /= 60;
847         minutes = tmp%60;
848         tmp    /= 60;
849         hours   = tmp%24;
850         tmp    /= 24;
851         days    = tmp;
852         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
853         
854         printf("Duration of last recovery/failover: %lf seconds\n",
855                 timeval_delta(&uptime->last_recovery_finished,
856                               &uptime->last_recovery_started));
857
858         return 0;
859 }
860
861 /*
862   show the PNN of the current node
863  */
864 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
865 {
866         uint32_t mypnn;
867
868         mypnn = getpnn(ctdb);
869
870         printf("PNN:%d\n", mypnn);
871         return 0;
872 }
873
874
875 static struct ctdb_node_map *read_nodes_file(TALLOC_CTX *mem_ctx)
876 {
877         const char *nodes_list;
878
879         /* read the nodes file */
880         nodes_list = getenv("CTDB_NODES");
881         if (nodes_list == NULL) {
882                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes",
883                                              getenv("CTDB_BASE"));
884                 if (nodes_list == NULL) {
885                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
886                         exit(1);
887                 }
888         }
889
890         return ctdb_read_nodes_file(mem_ctx, nodes_list);
891 }
892
893 /*
894   show the PNN of the current node
895   discover the pnn by loading the nodes file and try to bind to all
896   addresses one at a time until the ip address is found.
897  */
898 static int find_node_xpnn(void)
899 {
900         TALLOC_CTX *mem_ctx = talloc_new(NULL);
901         struct ctdb_node_map *node_map;
902         int i, pnn;
903
904         node_map = read_nodes_file(mem_ctx);
905         if (node_map == NULL) {
906                 talloc_free(mem_ctx);
907                 return -1;
908         }
909
910         for (i = 0; i < node_map->num; i++) {
911                 if (node_map->nodes[i].flags & NODE_FLAGS_DELETED) {
912                         continue;
913                 }
914                 if (ctdb_sys_have_ip(&node_map->nodes[i].addr)) {
915                         pnn = node_map->nodes[i].pnn;
916                         talloc_free(mem_ctx);
917                         return pnn;
918                 }
919         }
920
921         printf("Failed to detect which PNN this node is\n");
922         talloc_free(mem_ctx);
923         return -1;
924 }
925
926 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
927 {
928         uint32_t pnn;
929
930         assert_single_node_only();
931
932         pnn = find_node_xpnn();
933         if (pnn == -1) {
934                 return -1;
935         }
936
937         printf("PNN:%d\n", pnn);
938         return 0;
939 }
940
941 /* Helpers for ctdb status
942  */
943 static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_flags *node)
944 {
945         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
946         int j;
947         bool ret = false;
948
949         if (node->flags == 0) {
950                 struct ctdb_control_get_ifaces *ifaces;
951
952                 if (ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), node->pnn,
953                                          tmp_ctx, &ifaces) == 0) {
954                         for (j=0; j < ifaces->num; j++) {
955                                 if (ifaces->ifaces[j].link_state != 0) {
956                                         continue;
957                                 }
958                                 ret = true;
959                                 break;
960                         }
961                 }
962         }
963         talloc_free(tmp_ctx);
964
965         return ret;
966 }
967
968 static void control_status_header_machine(void)
969 {
970         printm(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
971                ":Inactive:PartiallyOnline:ThisNode:\n");
972 }
973
974 static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn,
975                                     struct ctdb_node_and_flags *node)
976 {
977         printm(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
978                ctdb_addr_to_str(&node->addr),
979                !!(node->flags&NODE_FLAGS_DISCONNECTED),
980                !!(node->flags&NODE_FLAGS_BANNED),
981                !!(node->flags&NODE_FLAGS_PERMANENTLY_DISABLED),
982                !!(node->flags&NODE_FLAGS_UNHEALTHY),
983                !!(node->flags&NODE_FLAGS_STOPPED),
984                !!(node->flags&NODE_FLAGS_INACTIVE),
985                is_partially_online(ctdb, node) ? 1 : 0,
986                (node->pnn == mypnn)?'Y':'N');
987
988         return node->flags;
989 }
990
991 static int control_status_1_human(struct ctdb_context *ctdb, int mypnn,
992                                   struct ctdb_node_and_flags *node)
993 {
994        printf("pnn:%d %-16s %s%s\n", node->pnn,
995               ctdb_addr_to_str(&node->addr),
996               is_partially_online(ctdb, node) ? "PARTIALLYONLINE" : pretty_print_flags(node->flags),
997               node->pnn == mypnn?" (THIS NODE)":"");
998
999        return node->flags;
1000 }
1001
1002 /*
1003   display remote ctdb status
1004  */
1005 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
1006 {
1007         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1008         int i;
1009         struct ctdb_vnn_map *vnnmap=NULL;
1010         struct ctdb_node_map *nodemap=NULL;
1011         uint32_t recmode, recmaster, mypnn;
1012         int num_deleted_nodes = 0;
1013         int ret;
1014
1015         mypnn = getpnn(ctdb);
1016
1017         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1018         if (ret != 0) {
1019                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1020                 talloc_free(tmp_ctx);
1021                 return -1;
1022         }
1023
1024         if (options.machinereadable) {
1025                 control_status_header_machine();
1026                 for (i=0;i<nodemap->num;i++) {
1027                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1028                                 continue;
1029                         }
1030                         (void) control_status_1_machine(ctdb, mypnn,
1031                                                         &nodemap->nodes[i]);
1032                 }
1033                 talloc_free(tmp_ctx);
1034                 return 0;
1035         }
1036
1037         for (i=0; i<nodemap->num; i++) {
1038                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1039                         num_deleted_nodes++;
1040                 }
1041         }
1042         if (num_deleted_nodes == 0) {
1043                 printf("Number of nodes:%d\n", nodemap->num);
1044         } else {
1045                 printf("Number of nodes:%d (including %d deleted nodes)\n",
1046                        nodemap->num, num_deleted_nodes);
1047         }
1048         for(i=0;i<nodemap->num;i++){
1049                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1050                         continue;
1051                 }
1052                 (void) control_status_1_human(ctdb, mypnn, &nodemap->nodes[i]);
1053         }
1054
1055         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
1056         if (ret != 0) {
1057                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
1058                 talloc_free(tmp_ctx);
1059                 return -1;
1060         }
1061         if (vnnmap->generation == INVALID_GENERATION) {
1062                 printf("Generation:INVALID\n");
1063         } else {
1064                 printf("Generation:%d\n",vnnmap->generation);
1065         }
1066         printf("Size:%d\n",vnnmap->size);
1067         for(i=0;i<vnnmap->size;i++){
1068                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
1069         }
1070
1071         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmode);
1072         if (ret != 0) {
1073                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
1074                 talloc_free(tmp_ctx);
1075                 return -1;
1076         }
1077         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
1078
1079         ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmaster);
1080         if (ret != 0) {
1081                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1082                 talloc_free(tmp_ctx);
1083                 return -1;
1084         }
1085         printf("Recovery master:%d\n",recmaster);
1086
1087         talloc_free(tmp_ctx);
1088         return 0;
1089 }
1090
1091 static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **argv)
1092 {
1093         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1094         int i, ret;
1095         struct ctdb_node_map *nodemap=NULL;
1096         uint32_t * nodes;
1097         uint32_t pnn_mode, mypnn;
1098
1099         if (argc > 1) {
1100                 usage();
1101         }
1102
1103         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1104                               options.pnn, true, &nodes, &pnn_mode)) {
1105                 return -1;
1106         }
1107
1108         if (options.machinereadable) {
1109                 control_status_header_machine();
1110         } else if (pnn_mode == CTDB_BROADCAST_ALL) {
1111                 printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
1112         }
1113
1114         mypnn = getpnn(ctdb);
1115
1116         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1117         if (ret != 0) {
1118                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1119                 talloc_free(tmp_ctx);
1120                 return -1;
1121         }
1122
1123         ret = 0;
1124
1125         for (i = 0; i < talloc_array_length(nodes); i++) {
1126                 if (options.machinereadable) {
1127                         ret |= control_status_1_machine(ctdb, mypnn,
1128                                                         &nodemap->nodes[nodes[i]]);
1129                 } else {
1130                         ret |= control_status_1_human(ctdb, mypnn,
1131                                                       &nodemap->nodes[nodes[i]]);
1132                 }
1133         }
1134
1135         talloc_free(tmp_ctx);
1136         return ret;
1137 }
1138
1139 static struct ctdb_node_map *read_natgw_nodes_file(struct ctdb_context *ctdb,
1140                                                    TALLOC_CTX *mem_ctx)
1141 {
1142         const char *natgw_list;
1143         struct ctdb_node_map *natgw_nodes = NULL;
1144
1145         natgw_list = getenv("CTDB_NATGW_NODES");
1146         if (natgw_list == NULL) {
1147                 natgw_list = talloc_asprintf(mem_ctx, "%s/natgw_nodes",
1148                                              getenv("CTDB_BASE"));
1149                 if (natgw_list == NULL) {
1150                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
1151                         exit(1);
1152                 }
1153         }
1154         /* The PNNs/flags will be junk but they're not used */
1155         natgw_nodes = ctdb_read_nodes_file(mem_ctx, natgw_list);
1156         if (natgw_nodes == NULL) {
1157                 DEBUG(DEBUG_ERR,
1158                       ("Failed to load natgw node list '%s'\n", natgw_list));
1159         }
1160         return natgw_nodes;
1161 }
1162
1163
1164 /* talloc off the existing nodemap... */
1165 static struct ctdb_node_map *talloc_nodemap(struct ctdb_node_map *nodemap)
1166 {
1167         return talloc_zero_size(nodemap,
1168                                 offsetof(struct ctdb_node_map, nodes) +
1169                                 nodemap->num * sizeof(struct ctdb_node_and_flags));
1170 }
1171
1172 static struct ctdb_node_map *
1173 filter_nodemap_by_addrs(struct ctdb_context *ctdb,
1174                         struct ctdb_node_map *nodemap,
1175                         struct ctdb_node_map *natgw_nodes)
1176 {
1177         int i, j;
1178         struct ctdb_node_map *ret;
1179
1180         ret = talloc_nodemap(nodemap);
1181         CTDB_NO_MEMORY_NULL(ctdb, ret);
1182
1183         ret->num = 0;
1184
1185         for (i = 0; i < nodemap->num; i++) {
1186                 for(j = 0; j < natgw_nodes->num ; j++) {
1187                         if (nodemap->nodes[j].flags & NODE_FLAGS_DELETED) {
1188                                 continue;
1189                         }
1190                         if (ctdb_same_ip(&natgw_nodes->nodes[j].addr,
1191                                          &nodemap->nodes[i].addr)) {
1192
1193                                 ret->nodes[ret->num] = nodemap->nodes[i];
1194                                 ret->num++;
1195                                 break;
1196                         }
1197                 }
1198         }
1199
1200         return ret;
1201 }
1202
1203 static struct ctdb_node_map *
1204 filter_nodemap_by_capabilities(struct ctdb_context *ctdb,
1205                                struct ctdb_node_map *nodemap,
1206                                uint32_t required_capabilities,
1207                                bool first_only)
1208 {
1209         int i;
1210         uint32_t capabilities;
1211         struct ctdb_node_map *ret;
1212
1213         ret = talloc_nodemap(nodemap);
1214         CTDB_NO_MEMORY_NULL(ctdb, ret);
1215
1216         ret->num = 0;
1217
1218         for (i = 0; i < nodemap->num; i++) {
1219                 int res;
1220
1221                 /* Disconnected nodes have no capabilities! */
1222                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1223                         continue;
1224                 }
1225
1226                 res = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(),
1227                                                 nodemap->nodes[i].pnn,
1228                                                 &capabilities);
1229                 if (res != 0) {
1230                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n",
1231                                           nodemap->nodes[i].pnn));
1232                         talloc_free(ret);
1233                         return NULL;
1234                 }
1235                 if (!(capabilities & required_capabilities)) {
1236                         continue;
1237                 }
1238
1239                 ret->nodes[ret->num] = nodemap->nodes[i];
1240                 ret->num++;
1241                 if (first_only) {
1242                         break;
1243                 }
1244         }
1245
1246         return ret;
1247 }
1248
1249 static struct ctdb_node_map *
1250 filter_nodemap_by_flags(struct ctdb_context *ctdb,
1251                         struct ctdb_node_map *nodemap,
1252                         uint32_t flags_mask)
1253 {
1254         int i;
1255         struct ctdb_node_map *ret;
1256
1257         ret = talloc_nodemap(nodemap);
1258         CTDB_NO_MEMORY_NULL(ctdb, ret);
1259
1260         ret->num = 0;
1261
1262         for (i = 0; i < nodemap->num; i++) {
1263                 if (nodemap->nodes[i].flags & flags_mask) {
1264                         continue;
1265                 }
1266
1267                 ret->nodes[ret->num] = nodemap->nodes[i];
1268                 ret->num++;
1269         }
1270
1271         return ret;
1272 }
1273
1274 /*
1275   display the list of nodes belonging to this natgw configuration
1276  */
1277 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
1278 {
1279         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1280         int i, ret;
1281         struct ctdb_node_map *natgw_nodes = NULL;
1282         struct ctdb_node_map *orig_nodemap=NULL;
1283         struct ctdb_node_map *nodemap;
1284         uint32_t mypnn, pnn;
1285         const char *ip;
1286
1287         /* When we have some nodes that could be the NATGW, make a
1288          * series of attempts to find the first node that doesn't have
1289          * certain status flags set.
1290          */
1291         uint32_t exclude_flags[] = {
1292                 /* Look for a nice healthy node */
1293                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY,
1294                 /* If not found, an UNHEALTHY/BANNED node will do */
1295                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED,
1296                 /* If not found, a STOPPED node will do */
1297                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED,
1298                 0,
1299         };
1300
1301         /* read the natgw nodes file into a linked list */
1302         natgw_nodes = read_natgw_nodes_file(ctdb, tmp_ctx);
1303         if (natgw_nodes == NULL) {
1304                 ret = -1;
1305                 goto done;
1306         }
1307
1308         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
1309                                    tmp_ctx, &orig_nodemap);
1310         if (ret != 0) {
1311                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
1312                 talloc_free(tmp_ctx);
1313                 return -1;
1314         }
1315
1316         /* Get a nodemap that includes only the nodes in the NATGW
1317          * group */
1318         nodemap = filter_nodemap_by_addrs(ctdb, orig_nodemap, natgw_nodes);
1319         if (nodemap == NULL) {
1320                 ret = -1;
1321                 goto done;
1322         }
1323
1324         ret = 2; /* matches ENOENT */
1325         pnn = -1;
1326         ip = "0.0.0.0";
1327         /* For each flag mask... */
1328         for (i = 0; exclude_flags[i] != 0; i++) {
1329                 /* ... get a nodemap that excludes nodes with with
1330                  * masked flags... */
1331                 struct ctdb_node_map *t =
1332                         filter_nodemap_by_flags(ctdb, nodemap,
1333                                                 exclude_flags[i]);
1334                 if (t == NULL) {
1335                         /* No memory */
1336                         ret = -1;
1337                         goto done;
1338                 }
1339                 if (t->num > 0) {
1340                         /* ... and find the first node with the NATGW
1341                          * capability */
1342                         struct ctdb_node_map *n;
1343                         n = filter_nodemap_by_capabilities(ctdb, t,
1344                                                            CTDB_CAP_NATGW,
1345                                                            true);
1346                         if (n == NULL) {
1347                                 /* No memory */
1348                                 ret = -1;
1349                                 goto done;
1350                         }
1351                         if (n->num > 0) {
1352                                 ret = 0;
1353                                 pnn = n->nodes[0].pnn;
1354                                 ip = ctdb_addr_to_str(&n->nodes[0].addr);
1355                                 break;
1356                         }
1357                 }
1358                 talloc_free(t);
1359         }
1360
1361         if (options.machinereadable) {
1362                 printm(":Node:IP:\n");
1363                 printm(":%d:%s:\n", pnn, ip);
1364         } else {
1365                 printf("%d %s\n", pnn, ip);
1366         }
1367
1368         /* print the pruned list of nodes belonging to this natgw list */
1369         mypnn = getpnn(ctdb);
1370         if (options.machinereadable) {
1371                 control_status_header_machine();
1372         } else {
1373                 printf("Number of nodes:%d\n", nodemap->num);
1374         }
1375         for(i=0;i<nodemap->num;i++){
1376                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1377                         continue;
1378                 }
1379                 if (options.machinereadable) {
1380                         control_status_1_machine(ctdb, mypnn, &(nodemap->nodes[i]));
1381                 } else {
1382                         control_status_1_human(ctdb, mypnn, &(nodemap->nodes[i]));
1383                 }
1384         }
1385
1386 done:
1387         talloc_free(tmp_ctx);
1388         return ret;
1389 }
1390
1391 /*
1392   display the status of the scripts for monitoring (or other events)
1393  */
1394 static int control_one_scriptstatus(struct ctdb_context *ctdb,
1395                                     enum ctdb_eventscript_call type)
1396 {
1397         struct ctdb_scripts_wire *script_status;
1398         int ret, i;
1399
1400         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
1401         if (ret != 0) {
1402                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
1403                 return ret;
1404         }
1405
1406         if (script_status == NULL) {
1407                 if (!options.machinereadable) {
1408                         printf("%s cycle never run\n",
1409                                ctdb_eventscript_call_names[type]);
1410                 }
1411                 return 0;
1412         }
1413
1414         if (!options.machinereadable) {
1415                 int num_run = 0;
1416                 for (i=0; i<script_status->num_scripts; i++) {
1417                         if (script_status->scripts[i].status != -ENOEXEC) {
1418                                 num_run++;
1419                         }
1420                 }
1421                 printf("%d scripts were executed last %s cycle\n",
1422                        num_run,
1423                        ctdb_eventscript_call_names[type]);
1424         }
1425         for (i=0; i<script_status->num_scripts; i++) {
1426                 const char *status = NULL;
1427
1428                 /* The ETIME status is ignored for certain events.
1429                  * In that case the status is 0, but endtime is not set.
1430                  */
1431                 if (script_status->scripts[i].status == 0 &&
1432                     timeval_is_zero(&script_status->scripts[i].finished)) {
1433                         script_status->scripts[i].status = -ETIME;
1434                 }
1435
1436                 switch (script_status->scripts[i].status) {
1437                 case -ETIME:
1438                         status = "TIMEDOUT";
1439                         break;
1440                 case -ENOEXEC:
1441                         status = "DISABLED";
1442                         break;
1443                 case 0:
1444                         status = "OK";
1445                         break;
1446                 default:
1447                         if (script_status->scripts[i].status > 0)
1448                                 status = "ERROR";
1449                         break;
1450                 }
1451                 if (options.machinereadable) {
1452                         printm(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
1453                                ctdb_eventscript_call_names[type],
1454                                script_status->scripts[i].name,
1455                                script_status->scripts[i].status,
1456                                status,
1457                                (long)script_status->scripts[i].start.tv_sec,
1458                                (long)script_status->scripts[i].start.tv_usec,
1459                                (long)script_status->scripts[i].finished.tv_sec,
1460                                (long)script_status->scripts[i].finished.tv_usec,
1461                                script_status->scripts[i].output);
1462                         continue;
1463                 }
1464                 if (status)
1465                         printf("%-20s Status:%s    ",
1466                                script_status->scripts[i].name, status);
1467                 else
1468                         /* Some other error, eg from stat. */
1469                         printf("%-20s Status:CANNOT RUN (%s)",
1470                                script_status->scripts[i].name,
1471                                strerror(-script_status->scripts[i].status));
1472
1473                 if (script_status->scripts[i].status >= 0) {
1474                         printf("Duration:%.3lf ",
1475                         timeval_delta(&script_status->scripts[i].finished,
1476                               &script_status->scripts[i].start));
1477                 }
1478                 if (script_status->scripts[i].status != -ENOEXEC) {
1479                         printf("%s",
1480                                ctime(&script_status->scripts[i].start.tv_sec));
1481                         if (script_status->scripts[i].status != 0) {
1482                                 printf("   OUTPUT:%s\n",
1483                                        script_status->scripts[i].output);
1484                         }
1485                 } else {
1486                         printf("\n");
1487                 }
1488         }
1489         return 0;
1490 }
1491
1492
1493 static int control_scriptstatus(struct ctdb_context *ctdb,
1494                                 int argc, const char **argv)
1495 {
1496         int ret;
1497         enum ctdb_eventscript_call type, min, max;
1498         const char *arg;
1499
1500         if (argc > 1) {
1501                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
1502                 return -1;
1503         }
1504
1505         if (argc == 0)
1506                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
1507         else
1508                 arg = argv[0];
1509
1510         for (type = 0; type < CTDB_EVENT_MAX; type++) {
1511                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
1512                         min = type;
1513                         max = type+1;
1514                         break;
1515                 }
1516         }
1517         if (type == CTDB_EVENT_MAX) {
1518                 if (strcmp(arg, "all") == 0) {
1519                         min = 0;
1520                         max = CTDB_EVENT_MAX;
1521                 } else {
1522                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
1523                         return -1;
1524                 }
1525         }
1526
1527         if (options.machinereadable) {
1528                 printm(":Type:Name:Code:Status:Start:End:Error Output...:\n");
1529         }
1530
1531         for (type = min; type < max; type++) {
1532                 ret = control_one_scriptstatus(ctdb, type);
1533                 if (ret != 0) {
1534                         return ret;
1535                 }
1536         }
1537
1538         return 0;
1539 }
1540
1541 /*
1542   enable an eventscript
1543  */
1544 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1545 {
1546         int ret;
1547
1548         if (argc < 1) {
1549                 usage();
1550         }
1551
1552         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1553         if (ret != 0) {
1554           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
1555                 return ret;
1556         }
1557
1558         return 0;
1559 }
1560
1561 /*
1562   disable an eventscript
1563  */
1564 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1565 {
1566         int ret;
1567
1568         if (argc < 1) {
1569                 usage();
1570         }
1571
1572         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1573         if (ret != 0) {
1574           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
1575                 return ret;
1576         }
1577
1578         return 0;
1579 }
1580
1581 /*
1582   display the pnn of the recovery master
1583  */
1584 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
1585 {
1586         uint32_t recmaster;
1587         int ret;
1588
1589         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
1590         if (ret != 0) {
1591                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1592                 return -1;
1593         }
1594         printf("%d\n",recmaster);
1595
1596         return 0;
1597 }
1598
1599 /*
1600   add a tickle to a public address
1601  */
1602 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1603 {
1604         struct ctdb_tcp_connection t;
1605         TDB_DATA data;
1606         int ret;
1607
1608         assert_single_node_only();
1609
1610         if (argc < 2) {
1611                 usage();
1612         }
1613
1614         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1615                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1616                 return -1;
1617         }
1618         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1619                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1620                 return -1;
1621         }
1622
1623         data.dptr = (uint8_t *)&t;
1624         data.dsize = sizeof(t);
1625
1626         /* tell all nodes about this tcp connection */
1627         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1628                            0, data, ctdb, NULL, NULL, NULL, NULL);
1629         if (ret != 0) {
1630                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1631                 return -1;
1632         }
1633         
1634         return 0;
1635 }
1636
1637
1638 /*
1639   delete a tickle from a node
1640  */
1641 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1642 {
1643         struct ctdb_tcp_connection t;
1644         TDB_DATA data;
1645         int ret;
1646
1647         assert_single_node_only();
1648
1649         if (argc < 2) {
1650                 usage();
1651         }
1652
1653         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1654                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1655                 return -1;
1656         }
1657         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1658                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1659                 return -1;
1660         }
1661
1662         data.dptr = (uint8_t *)&t;
1663         data.dsize = sizeof(t);
1664
1665         /* tell all nodes about this tcp connection */
1666         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1667                            0, data, ctdb, NULL, NULL, NULL, NULL);
1668         if (ret != 0) {
1669                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1670                 return -1;
1671         }
1672         
1673         return 0;
1674 }
1675
1676
1677 /*
1678   get a list of all tickles for this pnn
1679  */
1680 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1681 {
1682         struct ctdb_control_tcp_tickle_list *list;
1683         ctdb_sock_addr addr;
1684         int i, ret;
1685         unsigned port = 0;
1686
1687         assert_single_node_only();
1688
1689         if (argc < 1) {
1690                 usage();
1691         }
1692
1693         if (argc == 2) {
1694                 port = atoi(argv[1]);
1695         }
1696
1697         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1698                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1699                 return -1;
1700         }
1701
1702         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1703         if (ret == -1) {
1704                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1705                 return -1;
1706         }
1707
1708         if (options.machinereadable){
1709                 printm(":source ip:port:destination ip:port:\n");
1710                 for (i=0;i<list->tickles.num;i++) {
1711                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1712                                 continue;
1713                         }
1714                         printm(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1715                         printm(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1716                 }
1717         } else {
1718                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1719                 printf("Num tickles:%u\n", list->tickles.num);
1720                 for (i=0;i<list->tickles.num;i++) {
1721                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1722                                 continue;
1723                         }
1724                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1725                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1726                 }
1727         }
1728
1729         talloc_free(list);
1730         
1731         return 0;
1732 }
1733
1734
1735 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1736 {
1737         struct ctdb_all_public_ips *ips;
1738         struct ctdb_public_ip ip;
1739         int i, ret;
1740         uint32_t *nodes;
1741         uint32_t disable_time;
1742         TDB_DATA data;
1743         struct ctdb_node_map *nodemap=NULL;
1744         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1745
1746         disable_time = 30;
1747         data.dptr  = (uint8_t*)&disable_time;
1748         data.dsize = sizeof(disable_time);
1749         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1750         if (ret != 0) {
1751                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1752                 return -1;
1753         }
1754
1755
1756
1757         /* read the public ip list from the node */
1758         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1759         if (ret != 0) {
1760                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1761                 talloc_free(tmp_ctx);
1762                 return -1;
1763         }
1764
1765         for (i=0;i<ips->num;i++) {
1766                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1767                         break;
1768                 }
1769         }
1770         if (i==ips->num) {
1771                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1772                         pnn, ctdb_addr_to_str(addr)));
1773                 talloc_free(tmp_ctx);
1774                 return -1;
1775         }
1776
1777         ip.pnn  = pnn;
1778         ip.addr = *addr;
1779
1780         data.dptr  = (uint8_t *)&ip;
1781         data.dsize = sizeof(ip);
1782
1783         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1784         if (ret != 0) {
1785                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1786                 talloc_free(tmp_ctx);
1787                 return ret;
1788         }
1789
1790         nodes = list_of_nodes(ctdb, nodemap, tmp_ctx, NODE_FLAGS_INACTIVE, pnn);
1791         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1792                                         nodes, 0,
1793                                         LONGTIMELIMIT(),
1794                                         false, data,
1795                                         NULL, NULL,
1796                                         NULL);
1797         if (ret != 0) {
1798                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1799                 talloc_free(tmp_ctx);
1800                 return -1;
1801         }
1802
1803         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1804         if (ret != 0) {
1805                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1806                 talloc_free(tmp_ctx);
1807                 return -1;
1808         }
1809
1810         /* update the recovery daemon so it now knows to expect the new
1811            node assignment for this ip.
1812         */
1813         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1814         if (ret != 0) {
1815                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1816                 return -1;
1817         }
1818
1819         talloc_free(tmp_ctx);
1820         return 0;
1821 }
1822
1823
1824 /* 
1825  * scans all other nodes and returns a pnn for another node that can host this 
1826  * ip address or -1
1827  */
1828 static int
1829 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1830 {
1831         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1832         struct ctdb_all_public_ips *ips;
1833         struct ctdb_node_map *nodemap=NULL;
1834         int i, j, ret;
1835         int pnn;
1836
1837         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1838         if (ret != 0) {
1839                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1840                 talloc_free(tmp_ctx);
1841                 return ret;
1842         }
1843
1844         for(i=0;i<nodemap->num;i++){
1845                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1846                         continue;
1847                 }
1848                 if (nodemap->nodes[i].pnn == options.pnn) {
1849                         continue;
1850                 }
1851
1852                 /* read the public ip list from this node */
1853                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1854                 if (ret != 0) {
1855                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1856                         return -1;
1857                 }
1858
1859                 for (j=0;j<ips->num;j++) {
1860                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1861                                 pnn = nodemap->nodes[i].pnn;
1862                                 talloc_free(tmp_ctx);
1863                                 return pnn;
1864                         }
1865                 }
1866                 talloc_free(ips);
1867         }
1868
1869         talloc_free(tmp_ctx);
1870         return -1;
1871 }
1872
1873 /* If pnn is -1 then try to find a node to move IP to... */
1874 static bool try_moveip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1875 {
1876         bool pnn_specified = (pnn == -1 ? false : true);
1877         int retries = 0;
1878
1879         while (retries < 5) {
1880                 if (!pnn_specified) {
1881                         pnn = find_other_host_for_public_ip(ctdb, addr);
1882                         if (pnn == -1) {
1883                                 return false;
1884                         }
1885                         DEBUG(DEBUG_NOTICE,
1886                               ("Trying to move public IP to node %u\n", pnn));
1887                 }
1888
1889                 if (move_ip(ctdb, addr, pnn) == 0) {
1890                         return true;
1891                 }
1892
1893                 sleep(3);
1894                 retries++;
1895         }
1896
1897         return false;
1898 }
1899
1900
1901 /*
1902   move/failover an ip address to a specific node
1903  */
1904 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1905 {
1906         uint32_t pnn;
1907         ctdb_sock_addr addr;
1908
1909         assert_single_node_only();
1910
1911         if (argc < 2) {
1912                 usage();
1913                 return -1;
1914         }
1915
1916         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1917                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1918                 return -1;
1919         }
1920
1921
1922         if (sscanf(argv[1], "%u", &pnn) != 1) {
1923                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1924                 return -1;
1925         }
1926
1927         if (!try_moveip(ctdb, &addr, pnn)) {
1928                 DEBUG(DEBUG_ERR,("Failed to move IP to node %d.\n", pnn));
1929                 return -1;
1930         }
1931
1932         return 0;
1933 }
1934
1935 static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
1936 {
1937         TDB_DATA data;
1938
1939         data.dptr  = (uint8_t *)&pnn;
1940         data.dsize = sizeof(uint32_t);
1941         if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
1942                 DEBUG(DEBUG_ERR,
1943                       ("Failed to send message to force node %u to be a rebalancing target\n",
1944                        pnn));
1945                 return -1;
1946         }
1947
1948         return 0;
1949 }
1950
1951
1952 /*
1953   rebalance a node by setting it to allow failback and triggering a
1954   takeover run
1955  */
1956 static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
1957 {
1958         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1959         uint32_t *nodes;
1960         uint32_t pnn_mode;
1961         int i, ret;
1962
1963         assert_single_node_only();
1964
1965         if (argc > 1) {
1966                 usage();
1967         }
1968
1969         /* Determine the nodes where IPs need to be reloaded */
1970         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1971                               options.pnn, true, &nodes, &pnn_mode)) {
1972                 ret = -1;
1973                 goto done;
1974         }
1975
1976         for (i = 0; i < talloc_array_length(nodes); i++) {
1977                 if (!rebalance_node(ctdb, nodes[i])) {
1978                         ret = -1;
1979                 }
1980         }
1981
1982 done:
1983         talloc_free(tmp_ctx);
1984         return ret;
1985 }
1986
1987 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1988 {
1989         struct ctdb_public_ip ip;
1990         int ret;
1991         uint32_t *nodes;
1992         uint32_t disable_time;
1993         TDB_DATA data;
1994         struct ctdb_node_map *nodemap=NULL;
1995         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1996
1997         disable_time = 30;
1998         data.dptr  = (uint8_t*)&disable_time;
1999         data.dsize = sizeof(disable_time);
2000         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
2001         if (ret != 0) {
2002                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
2003                 return -1;
2004         }
2005
2006         ip.pnn  = -1;
2007         ip.addr = *addr;
2008
2009         data.dptr  = (uint8_t *)&ip;
2010         data.dsize = sizeof(ip);
2011
2012         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
2013         if (ret != 0) {
2014                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2015                 talloc_free(tmp_ctx);
2016                 return ret;
2017         }
2018
2019         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
2020         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
2021                                         nodes, 0,
2022                                         LONGTIMELIMIT(),
2023                                         false, data,
2024                                         NULL, NULL,
2025                                         NULL);
2026         if (ret != 0) {
2027                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
2028                 talloc_free(tmp_ctx);
2029                 return -1;
2030         }
2031
2032         talloc_free(tmp_ctx);
2033         return 0;
2034 }
2035
2036 /*
2037   release an ip form all nodes and have it re-assigned by recd
2038  */
2039 static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
2040 {
2041         ctdb_sock_addr addr;
2042
2043         assert_single_node_only();
2044
2045         if (argc < 1) {
2046                 usage();
2047                 return -1;
2048         }
2049
2050         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2051                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2052                 return -1;
2053         }
2054
2055         if (rebalance_ip(ctdb, &addr) != 0) {
2056                 DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
2057                 return -1;
2058         }
2059
2060         return 0;
2061 }
2062
2063 static int getips_store_callback(void *param, void *data)
2064 {
2065         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
2066         struct ctdb_all_public_ips *ips = param;
2067         int i;
2068
2069         i = ips->num++;
2070         ips->ips[i].pnn  = node_ip->pnn;
2071         ips->ips[i].addr = node_ip->addr;
2072         return 0;
2073 }
2074
2075 static int getips_count_callback(void *param, void *data)
2076 {
2077         uint32_t *count = param;
2078
2079         (*count)++;
2080         return 0;
2081 }
2082
2083 #define IP_KEYLEN       4
2084 static uint32_t *ip_key(ctdb_sock_addr *ip)
2085 {
2086         static uint32_t key[IP_KEYLEN];
2087
2088         bzero(key, sizeof(key));
2089
2090         switch (ip->sa.sa_family) {
2091         case AF_INET:
2092                 key[0]  = ip->ip.sin_addr.s_addr;
2093                 break;
2094         case AF_INET6: {
2095                 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
2096                 key[0]  = s6_a32[3];
2097                 key[1]  = s6_a32[2];
2098                 key[2]  = s6_a32[1];
2099                 key[3]  = s6_a32[0];
2100                 break;
2101         }
2102         default:
2103                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
2104                 return key;
2105         }
2106
2107         return key;
2108 }
2109
2110 static void *add_ip_callback(void *parm, void *data)
2111 {
2112         return parm;
2113 }
2114
2115 static int
2116 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_all_public_ips **ips)
2117 {
2118         struct ctdb_all_public_ips *tmp_ips;
2119         struct ctdb_node_map *nodemap=NULL;
2120         trbt_tree_t *ip_tree;
2121         int i, j, len, ret;
2122         uint32_t count;
2123
2124         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2125         if (ret != 0) {
2126                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2127                 return ret;
2128         }
2129
2130         ip_tree = trbt_create(tmp_ctx, 0);
2131
2132         for(i=0;i<nodemap->num;i++){
2133                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
2134                         continue;
2135                 }
2136                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2137                         continue;
2138                 }
2139
2140                 /* read the public ip list from this node */
2141                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
2142                 if (ret != 0) {
2143                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
2144                         return -1;
2145                 }
2146         
2147                 for (j=0; j<tmp_ips->num;j++) {
2148                         struct ctdb_public_ip *node_ip;
2149
2150                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
2151                         node_ip->pnn  = tmp_ips->ips[j].pnn;
2152                         node_ip->addr = tmp_ips->ips[j].addr;
2153
2154                         trbt_insertarray32_callback(ip_tree,
2155                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
2156                                 add_ip_callback,
2157                                 node_ip);
2158                 }
2159                 talloc_free(tmp_ips);
2160         }
2161
2162         /* traverse */
2163         count = 0;
2164         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
2165
2166         len = offsetof(struct ctdb_all_public_ips, ips) + 
2167                 count*sizeof(struct ctdb_public_ip);
2168         tmp_ips = talloc_zero_size(tmp_ctx, len);
2169         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
2170
2171         *ips = tmp_ips;
2172
2173         return 0;
2174 }
2175
2176
2177 static void ctdb_every_second(struct tevent_context *ev,
2178                               struct tevent_timer *te,
2179                               struct timeval t, void *p)
2180 {
2181         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2182
2183         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2184                          ctdb_every_second, ctdb);
2185 }
2186
2187 struct srvid_reply_handler_data {
2188         bool done;
2189         bool wait_for_all;
2190         uint32_t *nodes;
2191         const char *srvid_str;
2192 };
2193
2194 static void srvid_broadcast_reply_handler(uint64_t srvid, TDB_DATA data,
2195                                          void *private_data)
2196 {
2197         struct srvid_reply_handler_data *d =
2198                 (struct srvid_reply_handler_data *)private_data;
2199         int i;
2200         int32_t ret;
2201
2202         if (data.dsize != sizeof(ret)) {
2203                 DEBUG(DEBUG_ERR, (__location__ " Wrong reply size\n"));
2204                 return;
2205         }
2206
2207         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
2208         ret = *(int32_t *)data.dptr;
2209         if (ret < 0) {
2210                 DEBUG(DEBUG_ERR,
2211                       ("%s failed with result %d\n", d->srvid_str, ret));
2212                 return;
2213         }
2214
2215         if (!d->wait_for_all) {
2216                 d->done = true;
2217                 return;
2218         }
2219
2220         /* Wait for all replies */
2221         d->done = true;
2222         for (i = 0; i < talloc_array_length(d->nodes); i++) {
2223                 if (d->nodes[i] == ret) {
2224                         DEBUG(DEBUG_DEBUG,
2225                               ("%s reply received from node %u\n",
2226                                d->srvid_str, ret));
2227                         d->nodes[i] = -1;
2228                 }
2229                 if (d->nodes[i] != -1) {
2230                         /* Found a node that hasn't yet replied */
2231                         d->done = false;
2232                 }
2233         }
2234 }
2235
2236 /* Broadcast the given SRVID to all connected nodes.  Wait for 1 reply
2237  * or replies from all connected nodes.  arg is the data argument to
2238  * pass in the srvid_request structure - pass 0 if this isn't needed.
2239  */
2240 static int srvid_broadcast(struct ctdb_context *ctdb,
2241                            uint64_t srvid, uint32_t *arg,
2242                            const char *srvid_str, bool wait_for_all)
2243 {
2244         int ret;
2245         TDB_DATA data;
2246         uint32_t pnn;
2247         uint64_t reply_srvid;
2248         struct srvid_request request;
2249         struct srvid_request_data request_data;
2250         struct srvid_reply_handler_data reply_data;
2251         struct timeval tv;
2252
2253         ZERO_STRUCT(request);
2254
2255         /* Time ticks to enable timeouts to be processed */
2256         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2257                          ctdb_every_second, ctdb);
2258
2259         pnn = ctdb_get_pnn(ctdb);
2260         reply_srvid = getpid();
2261
2262         if (arg == NULL) {
2263                 request.pnn = pnn;
2264                 request.srvid = reply_srvid;
2265
2266                 data.dptr = (uint8_t *)&request;
2267                 data.dsize = sizeof(request);
2268         } else {
2269                 request_data.pnn = pnn;
2270                 request_data.srvid = reply_srvid;
2271                 request_data.data = *arg;
2272
2273                 data.dptr = (uint8_t *)&request_data;
2274                 data.dsize = sizeof(request_data);
2275         }
2276
2277         /* Register message port for reply from recovery master */
2278         ctdb_client_set_message_handler(ctdb, reply_srvid,
2279                                         srvid_broadcast_reply_handler,
2280                                         &reply_data);
2281
2282         reply_data.wait_for_all = wait_for_all;
2283         reply_data.nodes = NULL;
2284         reply_data.srvid_str = srvid_str;
2285
2286 again:
2287         reply_data.done = false;
2288
2289         if (wait_for_all) {
2290                 struct ctdb_node_map *nodemap;
2291
2292                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(),
2293                                            CTDB_CURRENT_NODE, ctdb, &nodemap);
2294                 if (ret != 0) {
2295                         DEBUG(DEBUG_ERR,
2296                               ("Unable to get nodemap from current node, try again\n"));
2297                         sleep(1);
2298                         goto again;
2299                 }
2300
2301                 if (reply_data.nodes != NULL) {
2302                         talloc_free(reply_data.nodes);
2303                 }
2304                 reply_data.nodes = list_of_connected_nodes(ctdb, nodemap,
2305                                                            NULL, true);
2306
2307                 talloc_free(nodemap);
2308         }
2309
2310         /* Send to all connected nodes. Only recmaster replies */
2311         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2312                                        srvid, data);
2313         if (ret != 0) {
2314                 /* This can only happen if the socket is closed and
2315                  * there's no way to recover from that, so don't try
2316                  * again.
2317                  */
2318                 DEBUG(DEBUG_ERR,
2319                       ("Failed to send %s request to connected nodes\n",
2320                        srvid_str));
2321                 return -1;
2322         }
2323
2324         tv = timeval_current();
2325         /* This loop terminates the reply is received */
2326         while (timeval_elapsed(&tv) < 5.0 && !reply_data.done) {
2327                 tevent_loop_once(ctdb->ev);
2328         }
2329
2330         if (!reply_data.done) {
2331                 DEBUG(DEBUG_NOTICE,
2332                       ("Still waiting for confirmation of %s\n", srvid_str));
2333                 sleep(1);
2334                 goto again;
2335         }
2336
2337         ctdb_client_remove_message_handler(ctdb, reply_srvid, &reply_data);
2338
2339         talloc_free(reply_data.nodes);
2340
2341         return 0;
2342 }
2343
2344 static int ipreallocate(struct ctdb_context *ctdb)
2345 {
2346         return srvid_broadcast(ctdb, CTDB_SRVID_TAKEOVER_RUN, NULL,
2347                                "IP reallocation", false);
2348 }
2349
2350
2351 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2352 {
2353         return ipreallocate(ctdb);
2354 }
2355
2356 /*
2357   add a public ip address to a node
2358  */
2359 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
2360 {
2361         int i, ret;
2362         int len, retries = 0;
2363         unsigned mask;
2364         ctdb_sock_addr addr;
2365         struct ctdb_control_ip_iface *pub;
2366         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2367         struct ctdb_all_public_ips *ips;
2368
2369
2370         if (argc != 2) {
2371                 talloc_free(tmp_ctx);
2372                 usage();
2373         }
2374
2375         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
2376                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
2377                 talloc_free(tmp_ctx);
2378                 return -1;
2379         }
2380
2381         /* read the public ip list from the node */
2382         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2383         if (ret != 0) {
2384                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", options.pnn));
2385                 talloc_free(tmp_ctx);
2386                 return -1;
2387         }
2388         for (i=0;i<ips->num;i++) {
2389                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2390                         DEBUG(DEBUG_ERR,("Can not add ip to node. Node already hosts this ip\n"));
2391                         return 0;
2392                 }
2393         }
2394
2395
2396
2397         /* Dont timeout. This command waits for an ip reallocation
2398            which sometimes can take wuite a while if there has
2399            been a recent recovery
2400         */
2401         alarm(0);
2402
2403         len = offsetof(struct ctdb_control_ip_iface, iface) + strlen(argv[1]) + 1;
2404         pub = talloc_size(tmp_ctx, len); 
2405         CTDB_NO_MEMORY(ctdb, pub);
2406
2407         pub->addr  = addr;
2408         pub->mask  = mask;
2409         pub->len   = strlen(argv[1])+1;
2410         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
2411
2412         do {
2413                 ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
2414                 if (ret != 0) {
2415                         DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
2416                         sleep(3);
2417                         retries++;
2418                 }
2419         } while (retries < 5 && ret != 0);
2420         if (ret != 0) {
2421                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
2422                 talloc_free(tmp_ctx);
2423                 return ret;
2424         }
2425
2426         if (rebalance_node(ctdb, options.pnn) != 0) {
2427                 DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
2428                 return ret;
2429         }
2430
2431         talloc_free(tmp_ctx);
2432         return 0;
2433 }
2434
2435 /*
2436   add a public ip address to a node
2437  */
2438 static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
2439 {
2440         ctdb_sock_addr addr;
2441         char *iface = NULL;
2442
2443         if (argc != 1) {
2444                 usage();
2445         }
2446
2447         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2448                 printf("Badly formed ip : %s\n", argv[0]);
2449                 return -1;
2450         }
2451
2452         iface = ctdb_sys_find_ifname(&addr);
2453         if (iface == NULL) {
2454                 printf("Failed to get interface name for ip: %s", argv[0]);
2455                 return -1;
2456         }
2457
2458         printf("IP on interface %s\n", iface);
2459
2460         free(iface);
2461
2462         return 0;
2463 }
2464
2465 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
2466
2467 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
2468 {
2469         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2470         struct ctdb_node_map *nodemap=NULL;
2471         struct ctdb_all_public_ips *ips;
2472         int ret, i, j;
2473
2474         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2475         if (ret != 0) {
2476                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
2477                 return ret;
2478         }
2479
2480         /* remove it from the nodes that are not hosting the ip currently */
2481         for(i=0;i<nodemap->num;i++){
2482                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2483                         continue;
2484                 }
2485                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2486                 if (ret != 0) {
2487                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2488                         continue;
2489                 }
2490
2491                 for (j=0;j<ips->num;j++) {
2492                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2493                                 break;
2494                         }
2495                 }
2496                 if (j==ips->num) {
2497                         continue;
2498                 }
2499
2500                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
2501                         continue;
2502                 }
2503
2504                 options.pnn = nodemap->nodes[i].pnn;
2505                 control_delip(ctdb, argc, argv);
2506         }
2507
2508
2509         /* remove it from every node (also the one hosting it) */
2510         for(i=0;i<nodemap->num;i++){
2511                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2512                         continue;
2513                 }
2514                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2515                 if (ret != 0) {
2516                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2517                         continue;
2518                 }
2519
2520                 for (j=0;j<ips->num;j++) {
2521                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2522                                 break;
2523                         }
2524                 }
2525                 if (j==ips->num) {
2526                         continue;
2527                 }
2528
2529                 options.pnn = nodemap->nodes[i].pnn;
2530                 control_delip(ctdb, argc, argv);
2531         }
2532
2533         talloc_free(tmp_ctx);
2534         return 0;
2535 }
2536         
2537 /*
2538   delete a public ip address from a node
2539  */
2540 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
2541 {
2542         int i, ret;
2543         ctdb_sock_addr addr;
2544         struct ctdb_control_ip_iface pub;
2545         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2546         struct ctdb_all_public_ips *ips;
2547
2548         if (argc != 1) {
2549                 talloc_free(tmp_ctx);
2550                 usage();
2551         }
2552
2553         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2554                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2555                 return -1;
2556         }
2557
2558         if (options.pnn == CTDB_BROADCAST_ALL) {
2559                 return control_delip_all(ctdb, argc, argv, &addr);
2560         }
2561
2562         pub.addr  = addr;
2563         pub.mask  = 0;
2564         pub.len   = 0;
2565
2566         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2567         if (ret != 0) {
2568                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
2569                 talloc_free(tmp_ctx);
2570                 return ret;
2571         }
2572         
2573         for (i=0;i<ips->num;i++) {
2574                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2575                         break;
2576                 }
2577         }
2578
2579         if (i==ips->num) {
2580                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
2581                         ctdb_addr_to_str(&addr)));
2582                 talloc_free(tmp_ctx);
2583                 return -1;
2584         }
2585
2586         /* This is an optimisation.  If this node is hosting the IP
2587          * then try to move it somewhere else without invoking a full
2588          * takeover run.  We don't care if this doesn't work!
2589          */
2590         if (ips->ips[i].pnn == options.pnn) {
2591                 (void) try_moveip(ctdb, &addr, -1);
2592         }
2593
2594         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
2595         if (ret != 0) {
2596                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
2597                 talloc_free(tmp_ctx);
2598                 return ret;
2599         }
2600
2601         talloc_free(tmp_ctx);
2602         return 0;
2603 }
2604
2605 static int kill_tcp_from_file(struct ctdb_context *ctdb,
2606                               int argc, const char **argv)
2607 {
2608         struct ctdb_tcp_connection *killtcp;
2609         int max_entries, current, i;
2610         struct timeval timeout;
2611         char line[128], src[128], dst[128];
2612         int linenum;
2613         TDB_DATA data;
2614         struct client_async_data *async_data;
2615         struct ctdb_client_control_state *state;
2616
2617         if (argc != 0) {
2618                 usage();
2619         }
2620
2621         linenum = 1;
2622         killtcp = NULL;
2623         max_entries = 0;
2624         current = 0;
2625         while (!feof(stdin)) {
2626                 if (fgets(line, sizeof(line), stdin) == NULL) {
2627                         continue;
2628                 }
2629
2630                 /* Silently skip empty lines */
2631                 if (line[0] == '\n') {
2632                         continue;
2633                 }
2634
2635                 if (sscanf(line, "%s %s\n", src, dst) != 2) {
2636                         DEBUG(DEBUG_ERR, ("Bad line [%d]: '%s'\n",
2637                                           linenum, line));
2638                         talloc_free(killtcp);
2639                         return -1;
2640                 }
2641
2642                 if (current >= max_entries) {
2643                         max_entries += 1024;
2644                         killtcp = talloc_realloc(ctdb, killtcp,
2645                                                  struct ctdb_tcp_connection,
2646                                                  max_entries);
2647                         CTDB_NO_MEMORY(ctdb, killtcp);
2648                 }
2649
2650                 if (!parse_ip_port(src, &killtcp[current].src_addr)) {
2651                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2652                                           linenum, src));
2653                         talloc_free(killtcp);
2654                         return -1;
2655                 }
2656
2657                 if (!parse_ip_port(dst, &killtcp[current].dst_addr)) {
2658                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2659                                           linenum, dst));
2660                         talloc_free(killtcp);
2661                         return -1;
2662                 }
2663
2664                 current++;
2665         }
2666
2667         async_data = talloc_zero(ctdb, struct client_async_data);
2668         if (async_data == NULL) {
2669                 talloc_free(killtcp);
2670                 return -1;
2671         }
2672
2673         for (i = 0; i < current; i++) {
2674
2675                 data.dsize = sizeof(struct ctdb_tcp_connection);
2676                 data.dptr  = (unsigned char *)&killtcp[i];
2677
2678                 timeout = TIMELIMIT();
2679                 state = ctdb_control_send(ctdb, options.pnn, 0,
2680                                           CTDB_CONTROL_KILL_TCP, 0, data,
2681                                           async_data, &timeout, NULL);
2682
2683                 if (state == NULL) {
2684                         DEBUG(DEBUG_ERR,
2685                               ("Failed to call async killtcp control to node %u\n",
2686                                options.pnn));
2687                         talloc_free(killtcp);
2688                         return -1;
2689                 }
2690                 
2691                 ctdb_client_async_add(async_data, state);
2692         }
2693
2694         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2695                 DEBUG(DEBUG_ERR,("killtcp failed\n"));
2696                 talloc_free(killtcp);
2697                 return -1;
2698         }
2699
2700         talloc_free(killtcp);
2701         return 0;
2702 }
2703
2704
2705 /*
2706   kill a tcp connection
2707  */
2708 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2709 {
2710         int ret;
2711         struct ctdb_tcp_connection killtcp;
2712
2713         assert_single_node_only();
2714
2715         if (argc == 0) {
2716                 return kill_tcp_from_file(ctdb, argc, argv);
2717         }
2718
2719         if (argc < 2) {
2720                 usage();
2721         }
2722
2723         if (!parse_ip_port(argv[0], &killtcp.src_addr)) {
2724                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2725                 return -1;
2726         }
2727
2728         if (!parse_ip_port(argv[1], &killtcp.dst_addr)) {
2729                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2730                 return -1;
2731         }
2732
2733         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
2734         if (ret != 0) {
2735                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
2736                 return ret;
2737         }
2738
2739         return 0;
2740 }
2741
2742
2743 /*
2744   send a gratious arp
2745  */
2746 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
2747 {
2748         int ret;
2749         ctdb_sock_addr addr;
2750
2751         assert_single_node_only();
2752
2753         if (argc < 2) {
2754                 usage();
2755         }
2756
2757         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2758                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
2759                 return -1;
2760         }
2761
2762         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
2763         if (ret != 0) {
2764                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
2765                 return ret;
2766         }
2767
2768         return 0;
2769 }
2770
2771 /*
2772   register a server id
2773  */
2774 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2775 {
2776         int ret;
2777         struct ctdb_server_id server_id;
2778
2779         if (argc < 3) {
2780                 usage();
2781         }
2782
2783         server_id.pnn       = strtoul(argv[0], NULL, 0);
2784         server_id.type      = strtoul(argv[1], NULL, 0);
2785         server_id.server_id = strtoul(argv[2], NULL, 0);
2786
2787         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
2788         if (ret != 0) {
2789                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
2790                 return ret;
2791         }
2792         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
2793         sleep(999);
2794         return -1;
2795 }
2796
2797 /*
2798   unregister a server id
2799  */
2800 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2801 {
2802         int ret;
2803         struct ctdb_server_id server_id;
2804
2805         if (argc < 3) {
2806                 usage();
2807         }
2808
2809         server_id.pnn       = strtoul(argv[0], NULL, 0);
2810         server_id.type      = strtoul(argv[1], NULL, 0);
2811         server_id.server_id = strtoul(argv[2], NULL, 0);
2812
2813         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
2814         if (ret != 0) {
2815                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
2816                 return ret;
2817         }
2818         return -1;
2819 }
2820
2821 /*
2822   check if a server id exists
2823  */
2824 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2825 {
2826         uint32_t status = 0;
2827         int ret;
2828         struct ctdb_server_id server_id;
2829
2830         if (argc < 3) {
2831                 usage();
2832         }
2833
2834         server_id.pnn       = strtoul(argv[0], NULL, 0);
2835         server_id.type      = strtoul(argv[1], NULL, 0);
2836         server_id.server_id = strtoul(argv[2], NULL, 0);
2837
2838         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
2839         if (ret != 0) {
2840                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
2841                 return ret;
2842         }
2843
2844         if (status) {
2845                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
2846         } else {
2847                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
2848         }
2849         return 0;
2850 }
2851
2852 /*
2853   get a list of all server ids that are registered on a node
2854  */
2855 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
2856 {
2857         int i, ret;
2858         struct ctdb_server_id_list *server_ids;
2859
2860         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
2861         if (ret != 0) {
2862                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
2863                 return ret;
2864         }
2865
2866         for (i=0; i<server_ids->num; i++) {
2867                 printf("Server id %d:%d:%d\n", 
2868                         server_ids->server_ids[i].pnn, 
2869                         server_ids->server_ids[i].type, 
2870                         server_ids->server_ids[i].server_id); 
2871         }
2872
2873         return -1;
2874 }
2875
2876 /*
2877   check if a server id exists
2878  */
2879 static int check_srvids(struct ctdb_context *ctdb, int argc, const char **argv)
2880 {
2881         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
2882         uint64_t *ids;
2883         uint8_t *result;
2884         int i;
2885
2886         if (argc < 1) {
2887                 talloc_free(tmp_ctx);
2888                 usage();
2889         }
2890
2891         ids    = talloc_array(tmp_ctx, uint64_t, argc);
2892         result = talloc_array(tmp_ctx, uint8_t, argc);
2893
2894         for (i = 0; i < argc; i++) {
2895                 ids[i] = strtoull(argv[i], NULL, 0);
2896         }
2897
2898         if (!ctdb_client_check_message_handlers(ctdb, ids, argc, result)) {
2899                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n",
2900                                   options.pnn));
2901                 talloc_free(tmp_ctx);
2902                 return -1;
2903         }
2904
2905         for (i=0; i < argc; i++) {
2906                 printf("Server id %d:%llu %s\n", options.pnn, (long long)ids[i],
2907                        result[i] ? "exists" : "does not exist");
2908         }
2909
2910         talloc_free(tmp_ctx);
2911         return 0;
2912 }
2913
2914 /*
2915   send a tcp tickle ack
2916  */
2917 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2918 {
2919         int ret;
2920         ctdb_sock_addr  src, dst;
2921
2922         if (argc < 2) {
2923                 usage();
2924         }
2925
2926         if (!parse_ip_port(argv[0], &src)) {
2927                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2928                 return -1;
2929         }
2930
2931         if (!parse_ip_port(argv[1], &dst)) {
2932                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2933                 return -1;
2934         }
2935
2936         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
2937         if (ret==0) {
2938                 return 0;
2939         }
2940         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
2941
2942         return -1;
2943 }
2944
2945
2946 /*
2947   display public ip status
2948  */
2949 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
2950 {
2951         int i, ret;
2952         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2953         struct ctdb_all_public_ips *ips;
2954
2955         if (argc == 1 && strcmp(argv[0], "all") == 0) {
2956                 options.pnn = CTDB_BROADCAST_ALL;
2957         }
2958
2959         if (options.pnn == CTDB_BROADCAST_ALL) {
2960                 /* read the list of public ips from all nodes */
2961                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
2962         } else {
2963                 /* read the public ip list from this node */
2964                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2965         }
2966         if (ret != 0) {
2967                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
2968                 talloc_free(tmp_ctx);
2969                 return ret;
2970         }
2971
2972         if (options.machinereadable){
2973                 printm(":Public IP:Node:");
2974                 if (options.verbose){
2975                         printm("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
2976                 }
2977                 printm("\n");
2978         } else {
2979                 if (options.pnn == CTDB_BROADCAST_ALL) {
2980                         printf("Public IPs on ALL nodes\n");
2981                 } else {
2982                         printf("Public IPs on node %u\n", options.pnn);
2983                 }
2984         }
2985
2986         for (i=1;i<=ips->num;i++) {
2987                 struct ctdb_control_public_ip_info *info = NULL;
2988                 int32_t pnn;
2989                 char *aciface = NULL;
2990                 char *avifaces = NULL;
2991                 char *cifaces = NULL;
2992
2993                 if (options.pnn == CTDB_BROADCAST_ALL) {
2994                         pnn = ips->ips[ips->num-i].pnn;
2995                 } else {
2996                         pnn = options.pnn;
2997                 }
2998
2999                 if (pnn != -1) {
3000                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
3001                                                    &ips->ips[ips->num-i].addr, &info);
3002                 } else {
3003                         ret = -1;
3004                 }
3005
3006                 if (ret == 0) {
3007                         int j;
3008                         for (j=0; j < info->num; j++) {
3009                                 if (cifaces == NULL) {
3010                                         cifaces = talloc_strdup(info,
3011                                                                 info->ifaces[j].name);
3012                                 } else {
3013                                         cifaces = talloc_asprintf_append(cifaces,
3014                                                                          ",%s",
3015                                                                          info->ifaces[j].name);
3016                                 }
3017
3018                                 if (info->active_idx == j) {
3019                                         aciface = info->ifaces[j].name;
3020                                 }
3021
3022                                 if (info->ifaces[j].link_state == 0) {
3023                                         continue;
3024                                 }
3025
3026                                 if (avifaces == NULL) {
3027                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
3028                                 } else {
3029                                         avifaces = talloc_asprintf_append(avifaces,
3030                                                                           ",%s",
3031                                                                           info->ifaces[j].name);
3032                                 }
3033                         }
3034                 }
3035
3036                 if (options.machinereadable){
3037                         printm(":%s:%d:",
3038                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3039                                 ips->ips[ips->num-i].pnn);
3040                         if (options.verbose){
3041                                 printm("%s:%s:%s:",
3042                                         aciface?aciface:"",
3043                                         avifaces?avifaces:"",
3044                                         cifaces?cifaces:"");
3045                         }
3046                         printf("\n");
3047                 } else {
3048                         if (options.verbose) {
3049                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
3050                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3051                                         ips->ips[ips->num-i].pnn,
3052                                         aciface?aciface:"",
3053                                         avifaces?avifaces:"",
3054                                         cifaces?cifaces:"");
3055                         } else {
3056                                 printf("%s %d\n",
3057                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3058                                         ips->ips[ips->num-i].pnn);
3059                         }
3060                 }
3061                 talloc_free(info);
3062         }
3063
3064         talloc_free(tmp_ctx);
3065         return 0;
3066 }
3067
3068 /*
3069   public ip info
3070  */
3071 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
3072 {
3073         int i, ret;
3074         ctdb_sock_addr addr;
3075         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3076         struct ctdb_control_public_ip_info *info;
3077
3078         if (argc != 1) {
3079                 talloc_free(tmp_ctx);
3080                 usage();
3081         }
3082
3083         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
3084                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
3085                 return -1;
3086         }
3087
3088         /* read the public ip info from this node */
3089         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
3090                                            tmp_ctx, &addr, &info);
3091         if (ret != 0) {
3092                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
3093                                   argv[0], options.pnn));
3094                 talloc_free(tmp_ctx);
3095                 return ret;
3096         }
3097
3098         printf("Public IP[%s] info on node %u\n",
3099                ctdb_addr_to_str(&info->ip.addr),
3100                options.pnn);
3101
3102         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
3103                ctdb_addr_to_str(&info->ip.addr),
3104                info->ip.pnn, info->num);
3105
3106         for (i=0; i<info->num; i++) {
3107                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
3108
3109                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
3110                        i+1, info->ifaces[i].name,
3111                        info->ifaces[i].link_state?"up":"down",
3112                        (unsigned int)info->ifaces[i].references,
3113                        (i==info->active_idx)?" (active)":"");
3114         }
3115
3116         talloc_free(tmp_ctx);
3117         return 0;
3118 }
3119
3120 /*
3121   display interfaces status
3122  */
3123 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
3124 {
3125         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3126         int i;
3127         struct ctdb_control_get_ifaces *ifaces;
3128         int ret;
3129
3130         /* read the public ip list from this node */
3131         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ifaces);
3132         if (ret != 0) {
3133                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
3134                                   options.pnn));
3135                 talloc_free(tmp_ctx);
3136                 return -1;
3137         }
3138
3139         if (options.machinereadable){
3140                 printm(":Name:LinkStatus:References:\n");
3141         } else {
3142    &n