ctdb-tool: Use new debug level API
[kai/samba-autobuild/.git] / ctdb / tools / ctdb.c
1 /* 
2    ctdb control tool
3
4    Copyright (C) Andrew Tridgell  2007
5    Copyright (C) Ronnie Sahlberg  2007
6
7    This program is free software; you can redistribute it and/or modify
8    it under the terms of the GNU General Public License as published by
9    the Free Software Foundation; either version 3 of the License, or
10    (at your option) any later version.
11    
12    This program is distributed in the hope that it will be useful,
13    but WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15    GNU General Public License for more details.
16    
17    You should have received a copy of the GNU General Public License
18    along with this program; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "replace.h"
22 #include "system/time.h"
23 #include "system/filesys.h"
24 #include "system/network.h"
25 #include "system/locale.h"
26
27 #include <popt.h>
28 #include <talloc.h>
29 /* Allow use of deprecated function tevent_loop_allow_nesting() */
30 #define TEVENT_DEPRECATED
31 #include <tevent.h>
32 #include <tdb.h>
33
34 #include "lib/tdb_wrap/tdb_wrap.h"
35 #include "lib/util/dlinklist.h"
36 #include "lib/util/debug.h"
37 #include "lib/util/substitute.h"
38 #include "lib/util/time.h"
39
40 #include "ctdb_version.h"
41 #include "ctdb_private.h"
42 #include "ctdb_client.h"
43
44 #include "common/cmdline.h"
45 #include "common/rb_tree.h"
46 #include "common/system.h"
47 #include "common/common.h"
48 #include "common/logging.h"
49
50 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
51 #define ERR_NONODE      21      /* node does not exist */
52 #define ERR_DISNODE     22      /* node is disconnected */
53
54 static void usage(void);
55
56 static struct {
57         int timelimit;
58         uint32_t pnn;
59         uint32_t *nodes;
60         int machinereadable;
61         const char *machineseparator;
62         int verbose;
63         int maxruntime;
64         int printemptyrecords;
65         int printdatasize;
66         int printlmaster;
67         int printhash;
68         int printrecordflags;
69 } options;
70
71 #define LONGTIMEOUT options.timelimit*10
72
73 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
74 #define LONGTIMELIMIT() timeval_current_ofs(LONGTIMEOUT, 0)
75
76 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
77 {
78         return (tv2->tv_sec - tv->tv_sec) +
79                 (tv2->tv_usec - tv->tv_usec)*1.0e-6;
80 }
81
82 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
83 {
84         printf("CTDB version: %s\n", CTDB_VERSION_STRING);
85         return 0;
86 }
87
88 /* Like printf(3) but substitute for separator in format */
89 static int printm(const char *format, ...) PRINTF_ATTRIBUTE(1,2);
90 static int printm(const char *format, ...)
91 {
92         va_list ap;
93         int ret;
94         size_t len = strlen(format);
95         char new_format[len+1];
96
97         strcpy(new_format, format);
98
99         if (options.machineseparator[0] != ':') {
100                 all_string_sub(new_format,
101                                ":", options.machineseparator, len + 1);
102         }
103
104         va_start(ap, format);
105         ret = vprintf(new_format, ap);
106         va_end(ap);
107
108         return ret;
109 }
110
111 #define CTDB_NOMEM_ABORT(p) do { if (!(p)) {                            \
112                 DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n",            \
113                                    "Out of memory in " __location__ )); \
114                 abort();                                                \
115         }} while (0)
116
117 static uint32_t getpnn(struct ctdb_context *ctdb)
118 {
119         if ((options.pnn == CTDB_BROADCAST_ALL) ||
120             (options.pnn == CTDB_MULTICAST)) {
121                 DEBUG(DEBUG_ERR,
122                       ("Cannot get PNN for node %u\n", options.pnn));
123                 exit(1);
124         }
125
126         if (options.pnn == CTDB_CURRENT_NODE) {
127                 return ctdb_get_pnn(ctdb);
128         } else {
129                 return options.pnn;
130         }
131 }
132
133 static void assert_single_node_only(void)
134 {
135         if ((options.pnn == CTDB_BROADCAST_ALL) ||
136             (options.pnn == CTDB_MULTICAST)) {
137                 DEBUG(DEBUG_ERR,
138                       ("This control can not be applied to multiple PNNs\n"));
139                 exit(1);
140         }
141 }
142
143 static void assert_current_node_only(struct ctdb_context *ctdb)
144 {
145         if (options.pnn != ctdb_get_pnn(ctdb)) {
146                 DEBUG(DEBUG_ERR,
147                       ("This control can only be applied to the current node\n"));
148                 exit(1);
149         }
150 }
151
152 /* Pretty print the flags to a static buffer in human-readable format.
153  * This never returns NULL!
154  */
155 static const char *pretty_print_flags(uint32_t flags)
156 {
157         int j;
158         static const struct {
159                 uint32_t flag;
160                 const char *name;
161         } flag_names[] = {
162                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
163                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
164                 { NODE_FLAGS_BANNED,                "BANNED" },
165                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
166                 { NODE_FLAGS_DELETED,               "DELETED" },
167                 { NODE_FLAGS_STOPPED,               "STOPPED" },
168                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
169         };
170         static char flags_str[512]; /* Big enough to contain all flag names */
171
172         flags_str[0] = '\0';
173         for (j=0;j<ARRAY_SIZE(flag_names);j++) {
174                 if (flags & flag_names[j].flag) {
175                         if (flags_str[0] == '\0') {
176                                 (void) strcpy(flags_str, flag_names[j].name);
177                         } else {
178                                 (void) strncat(flags_str, "|", sizeof(flags_str)-1);
179                                 (void) strncat(flags_str, flag_names[j].name,
180                                                sizeof(flags_str)-1);
181                         }
182                 }
183         }
184         if (flags_str[0] == '\0') {
185                 (void) strcpy(flags_str, "OK");
186         }
187
188         return flags_str;
189 }
190
191 static int h2i(char h)
192 {
193         if (h >= 'a' && h <= 'f') return h - 'a' + 10;
194         if (h >= 'A' && h <= 'F') return h - 'f' + 10;
195         return h - '0';
196 }
197
198 static TDB_DATA hextodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
199 {
200         int i;
201         TDB_DATA key = {NULL, 0};
202
203         if (len & 0x01) {
204                 DEBUG(DEBUG_ERR,("Key specified with odd number of hexadecimal digits\n"));
205                 return key;
206         }
207
208         key.dsize = len>>1;
209         key.dptr  = talloc_size(mem_ctx, key.dsize);
210
211         for (i=0; i < len/2; i++) {
212                 key.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
213         }
214         return key;
215 }
216
217 static TDB_DATA strtodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
218 {
219         TDB_DATA key;
220
221         if (!strncmp(str, "0x", 2)) {
222                 key = hextodata(mem_ctx, str + 2, len - 2);
223         } else {
224                 key.dptr  = talloc_memdup(mem_ctx, str, len);
225                 key.dsize = len;
226         }
227
228         return key;
229 }
230
231 /* Parse a nodestring.  Parameter dd_ok controls what happens to nodes
232  * that are disconnected or deleted.  If dd_ok is true those nodes are
233  * included in the output list of nodes.  If dd_ok is false, those
234  * nodes are filtered from the "all" case and cause an error if
235  * explicitly specified.
236  */
237 static bool parse_nodestring(struct ctdb_context *ctdb,
238                              TALLOC_CTX *mem_ctx,
239                              const char * nodestring,
240                              uint32_t current_pnn,
241                              bool dd_ok,
242                              uint32_t **nodes,
243                              uint32_t *pnn_mode)
244 {
245         TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
246         int n;
247         uint32_t i;
248         struct ctdb_node_map_old *nodemap;
249         int ret;
250
251         *nodes = NULL;
252
253         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
254         if (ret != 0) {
255                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
256                 talloc_free(tmp_ctx);
257                 exit(10);
258         }
259
260         if (nodestring != NULL) {
261                 *nodes = talloc_array(mem_ctx, uint32_t, 0);
262                 if (*nodes == NULL) {
263                         goto failed;
264                 }
265
266                 n = 0;
267
268                 if (strcmp(nodestring, "all") == 0) {
269                         *pnn_mode = CTDB_BROADCAST_ALL;
270
271                         /* all */
272                         for (i = 0; i < nodemap->num; i++) {
273                                 if ((nodemap->nodes[i].flags &
274                                      (NODE_FLAGS_DISCONNECTED |
275                                       NODE_FLAGS_DELETED)) && !dd_ok) {
276                                         continue;
277                                 }
278                                 *nodes = talloc_realloc(mem_ctx, *nodes,
279                                                         uint32_t, n+1);
280                                 if (*nodes == NULL) {
281                                         goto failed;
282                                 }
283                                 (*nodes)[n] = i;
284                                 n++;
285                         }
286                 } else {
287                         /* x{,y...} */
288                         char *ns, *tok;
289
290                         ns = talloc_strdup(tmp_ctx, nodestring);
291                         tok = strtok(ns, ",");
292                         while (tok != NULL) {
293                                 uint32_t pnn;
294                                 char *endptr;
295                                 i = (uint32_t)strtoul(tok, &endptr, 0);
296                                 if (i == 0 && tok == endptr) {
297                                         DEBUG(DEBUG_ERR,
298                                               ("Invalid node %s\n", tok));
299                                         talloc_free(tmp_ctx);
300                                         exit(ERR_NONODE);
301                                 }
302                                 if (i >= nodemap->num) {
303                                         DEBUG(DEBUG_ERR, ("Node %u does not exist\n", i));
304                                         talloc_free(tmp_ctx);
305                                         exit(ERR_NONODE);
306                                 }
307                                 if ((nodemap->nodes[i].flags & 
308                                      (NODE_FLAGS_DISCONNECTED |
309                                       NODE_FLAGS_DELETED)) && !dd_ok) {
310                                         DEBUG(DEBUG_ERR, ("Node %u has status %s\n", i, pretty_print_flags(nodemap->nodes[i].flags)));
311                                         talloc_free(tmp_ctx);
312                                         exit(ERR_DISNODE);
313                                 }
314                                 if ((pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), i)) < 0) {
315                                         DEBUG(DEBUG_ERR, ("Can not access node %u. Node is not operational.\n", i));
316                                         talloc_free(tmp_ctx);
317                                         exit(10);
318                                 }
319
320                                 *nodes = talloc_realloc(mem_ctx, *nodes,
321                                                         uint32_t, n+1);
322                                 if (*nodes == NULL) {
323                                         goto failed;
324                                 }
325
326                                 (*nodes)[n] = i;
327                                 n++;
328
329                                 tok = strtok(NULL, ",");
330                         }
331                         talloc_free(ns);
332
333                         if (n == 1) {
334                                 *pnn_mode = (*nodes)[0];
335                         } else {
336                                 *pnn_mode = CTDB_MULTICAST;
337                         }
338                 }
339         } else {
340                 /* default - no nodes specified */
341                 *nodes = talloc_array(mem_ctx, uint32_t, 1);
342                 if (*nodes == NULL) {
343                         goto failed;
344                 }
345                 *pnn_mode = CTDB_CURRENT_NODE;
346
347                 if (((*nodes)[0] = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), current_pnn)) < 0) {
348                         goto failed;
349                 }
350         }
351
352         talloc_free(tmp_ctx);
353         return true;
354
355 failed:
356         talloc_free(tmp_ctx);
357         return false;
358 }
359
360 /*
361  check if a database exists
362 */
363 static bool db_exists(struct ctdb_context *ctdb, const char *dbarg,
364                       uint32_t *dbid, const char **dbname, uint8_t *flags)
365 {
366         int i, ret;
367         struct ctdb_dbid_map_old *dbmap=NULL;
368         bool dbid_given = false, found = false;
369         uint32_t id;
370         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
371         const char *name = NULL;
372
373         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
374         if (ret != 0) {
375                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
376                 goto fail;
377         }
378
379         if (strncmp(dbarg, "0x", 2) == 0) {
380                 id = strtoul(dbarg, NULL, 0);
381                 dbid_given = true;
382         }
383
384         for(i=0; i<dbmap->num; i++) {
385                 if (dbid_given) {
386                         if (id == dbmap->dbs[i].db_id) {
387                                 found = true;
388                                 break;
389                         }
390                 } else {
391                         ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, tmp_ctx, &name);
392                         if (ret != 0) {
393                                 DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].db_id));
394                                 goto fail;
395                         }
396
397                         if (strcmp(name, dbarg) == 0) {
398                                 id = dbmap->dbs[i].db_id;
399                                 found = true;
400                                 break;
401                         }
402                 }
403         }
404
405         if (found && dbid_given && dbname != NULL) {
406                 ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, tmp_ctx, &name);
407                 if (ret != 0) {
408                         DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].db_id));
409                         found = false;
410                         goto fail;
411                 }
412         }
413
414         if (found) {
415                 if (dbid) *dbid = id;
416                 if (dbname) *dbname = talloc_strdup(ctdb, name);
417                 if (flags) *flags = dbmap->dbs[i].flags;
418         } else {
419                 DEBUG(DEBUG_ERR,("No database matching '%s' found\n", dbarg));
420         }
421
422 fail:
423         talloc_free(tmp_ctx);
424         return found;
425 }
426
427 /*
428   see if a process exists
429  */
430 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
431 {
432         uint32_t pnn, pid;
433         int ret;
434         if (argc < 1) {
435                 usage();
436         }
437
438         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
439                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
440                 return -1;
441         }
442
443         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
444         if (ret == 0) {
445                 printf("%u:%u exists\n", pnn, pid);
446         } else {
447                 printf("%u:%u does not exist\n", pnn, pid);
448         }
449         return ret;
450 }
451
452 /*
453   display statistics structure
454  */
455 static void show_statistics(struct ctdb_statistics *s, int show_header)
456 {
457         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
458         int i;
459         const char *prefix=NULL;
460         int preflen=0;
461         int tmp, days, hours, minutes, seconds;
462         const struct {
463                 const char *name;
464                 uint32_t offset;
465         } fields[] = {
466 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
467                 STATISTICS_FIELD(num_clients),
468                 STATISTICS_FIELD(frozen),
469                 STATISTICS_FIELD(recovering),
470                 STATISTICS_FIELD(num_recoveries),
471                 STATISTICS_FIELD(client_packets_sent),
472                 STATISTICS_FIELD(client_packets_recv),
473                 STATISTICS_FIELD(node_packets_sent),
474                 STATISTICS_FIELD(node_packets_recv),
475                 STATISTICS_FIELD(keepalive_packets_sent),
476                 STATISTICS_FIELD(keepalive_packets_recv),
477                 STATISTICS_FIELD(node.req_call),
478                 STATISTICS_FIELD(node.reply_call),
479                 STATISTICS_FIELD(node.req_dmaster),
480                 STATISTICS_FIELD(node.reply_dmaster),
481                 STATISTICS_FIELD(node.reply_error),
482                 STATISTICS_FIELD(node.req_message),
483                 STATISTICS_FIELD(node.req_control),
484                 STATISTICS_FIELD(node.reply_control),
485                 STATISTICS_FIELD(client.req_call),
486                 STATISTICS_FIELD(client.req_message),
487                 STATISTICS_FIELD(client.req_control),
488                 STATISTICS_FIELD(timeouts.call),
489                 STATISTICS_FIELD(timeouts.control),
490                 STATISTICS_FIELD(timeouts.traverse),
491                 STATISTICS_FIELD(locks.num_calls),
492                 STATISTICS_FIELD(locks.num_current),
493                 STATISTICS_FIELD(locks.num_pending),
494                 STATISTICS_FIELD(locks.num_failed),
495                 STATISTICS_FIELD(total_calls),
496                 STATISTICS_FIELD(pending_calls),
497                 STATISTICS_FIELD(childwrite_calls),
498                 STATISTICS_FIELD(pending_childwrite_calls),
499                 STATISTICS_FIELD(memory_used),
500                 STATISTICS_FIELD(max_hop_count),
501                 STATISTICS_FIELD(total_ro_delegations),
502                 STATISTICS_FIELD(total_ro_revokes),
503         };
504         
505         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
506         seconds = tmp%60;
507         tmp    /= 60;
508         minutes = tmp%60;
509         tmp    /= 60;
510         hours   = tmp%24;
511         tmp    /= 24;
512         days    = tmp;
513
514         if (options.machinereadable){
515                 if (show_header) {
516                         printm("CTDB version:");
517                         printm("Current time of statistics:");
518                         printm("Statistics collected since:");
519                         for (i=0;i<ARRAY_SIZE(fields);i++) {
520                                 printm("%s:", fields[i].name);
521                         }
522                         printm("num_reclock_ctdbd_latency:");
523                         printm("min_reclock_ctdbd_latency:");
524                         printm("avg_reclock_ctdbd_latency:");
525                         printm("max_reclock_ctdbd_latency:");
526
527                         printm("num_reclock_recd_latency:");
528                         printm("min_reclock_recd_latency:");
529                         printm("avg_reclock_recd_latency:");
530                         printm("max_reclock_recd_latency:");
531
532                         printm("num_call_latency:");
533                         printm("min_call_latency:");
534                         printm("avg_call_latency:");
535                         printm("max_call_latency:");
536
537                         printm("num_lockwait_latency:");
538                         printm("min_lockwait_latency:");
539                         printm("avg_lockwait_latency:");
540                         printm("max_lockwait_latency:");
541
542                         printm("num_childwrite_latency:");
543                         printm("min_childwrite_latency:");
544                         printm("avg_childwrite_latency:");
545                         printm("max_childwrite_latency:");
546                         printm("\n");
547                 }
548                 printm("%d:", CTDB_PROTOCOL);
549                 printm("%d:", (int)s->statistics_current_time.tv_sec);
550                 printm("%d:", (int)s->statistics_start_time.tv_sec);
551                 for (i=0;i<ARRAY_SIZE(fields);i++) {
552                         printm("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
553                 }
554                 printm("%d:", s->reclock.ctdbd.num);
555                 printm("%.6f:", s->reclock.ctdbd.min);
556                 printm("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
557                 printm("%.6f:", s->reclock.ctdbd.max);
558
559                 printm("%d:", s->reclock.recd.num);
560                 printm("%.6f:", s->reclock.recd.min);
561                 printm("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
562                 printm("%.6f:", s->reclock.recd.max);
563
564                 printm("%d:", s->call_latency.num);
565                 printm("%.6f:", s->call_latency.min);
566                 printm("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
567                 printm("%.6f:", s->call_latency.max);
568
569                 printm("%d:", s->childwrite_latency.num);
570                 printm("%.6f:", s->childwrite_latency.min);
571                 printm("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
572                 printm("%.6f:", s->childwrite_latency.max);
573                 printm("\n");
574         } else {
575                 printf("CTDB version %u\n", CTDB_PROTOCOL);
576                 printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
577                 printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
578
579                 for (i=0;i<ARRAY_SIZE(fields);i++) {
580                         if (strchr(fields[i].name, '.')) {
581                                 preflen = strcspn(fields[i].name, ".")+1;
582                                 if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
583                                         prefix = fields[i].name;
584                                         printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
585                                 }
586                         } else {
587                                 preflen = 0;
588                         }
589                         printf(" %*s%-22s%*s%10u\n", 
590                                preflen?4:0, "",
591                                fields[i].name+preflen, 
592                                preflen?0:4, "",
593                                *(uint32_t *)(fields[i].offset+(uint8_t *)s));
594                 }
595                 printf(" hop_count_buckets:");
596                 for (i=0;i<MAX_COUNT_BUCKETS;i++) {
597                         printf(" %d", s->hop_count_bucket[i]);
598                 }
599                 printf("\n");
600                 printf(" lock_buckets:");
601                 for (i=0; i<MAX_COUNT_BUCKETS; i++) {
602                         printf(" %d", s->locks.buckets[i]);
603                 }
604                 printf("\n");
605                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
606
607                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
608
609                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
610
611                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
612                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
613         }
614
615         talloc_free(tmp_ctx);
616 }
617
618 /*
619   display remote ctdb statistics combined from all nodes
620  */
621 static int control_statistics_all(struct ctdb_context *ctdb)
622 {
623         int ret, i;
624         struct ctdb_statistics statistics;
625         uint32_t *nodes;
626         uint32_t num_nodes;
627
628         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
629         CTDB_NO_MEMORY(ctdb, nodes);
630         
631         ZERO_STRUCT(statistics);
632
633         for (i=0;i<num_nodes;i++) {
634                 struct ctdb_statistics s1;
635                 int j;
636                 uint32_t *v1 = (uint32_t *)&s1;
637                 uint32_t *v2 = (uint32_t *)&statistics;
638                 uint32_t num_ints = 
639                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
640                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
641                 if (ret != 0) {
642                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
643                         return ret;
644                 }
645                 for (j=0;j<num_ints;j++) {
646                         v2[j] += v1[j];
647                 }
648                 statistics.max_hop_count = 
649                         MAX(statistics.max_hop_count, s1.max_hop_count);
650                 statistics.call_latency.max = 
651                         MAX(statistics.call_latency.max, s1.call_latency.max);
652         }
653         talloc_free(nodes);
654         printf("Gathered statistics for %u nodes\n", num_nodes);
655         show_statistics(&statistics, 1);
656         return 0;
657 }
658
659 /*
660   display remote ctdb statistics
661  */
662 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
663 {
664         int ret;
665         struct ctdb_statistics statistics;
666
667         if (options.pnn == CTDB_BROADCAST_ALL) {
668                 return control_statistics_all(ctdb);
669         }
670
671         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
672         if (ret != 0) {
673                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
674                 return ret;
675         }
676         show_statistics(&statistics, 1);
677         return 0;
678 }
679
680
681 /*
682   reset remote ctdb statistics
683  */
684 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
685 {
686         int ret;
687
688         ret = ctdb_statistics_reset(ctdb, options.pnn);
689         if (ret != 0) {
690                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
691                 return ret;
692         }
693         return 0;
694 }
695
696
697 /*
698   display remote ctdb rolling statistics
699  */
700 static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
701 {
702         int ret;
703         struct ctdb_statistics_list_old *stats;
704         int i, num_records = -1;
705
706         assert_single_node_only();
707
708         if (argc ==1) {
709                 num_records = atoi(argv[0]) - 1;
710         }
711
712         ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
713         if (ret != 0) {
714                 DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
715                 return ret;
716         }
717         for (i=0;i<stats->num;i++) {
718                 if (stats->stats[i].statistics_start_time.tv_sec == 0) {
719                         continue;
720                 }
721                 show_statistics(&stats->stats[i], i==0);
722                 if (i == num_records) {
723                         break;
724                 }
725         }
726         return 0;
727 }
728
729
730 /*
731   display remote ctdb db statistics
732  */
733 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
734 {
735         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
736         struct ctdb_db_statistics_old *dbstat;
737         int i;
738         uint32_t db_id;
739         int num_hot_keys;
740         int ret;
741
742         if (argc < 1) {
743                 usage();
744         }
745
746         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
747                 return -1;
748         }
749
750         ret = ctdb_ctrl_dbstatistics(ctdb, options.pnn, db_id, tmp_ctx, &dbstat);
751         if (ret != 0) {
752                 DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
753                 talloc_free(tmp_ctx);
754                 return -1;
755         }
756
757         printf("DB Statistics: %s\n", argv[0]);
758         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
759                 dbstat->db_ro_delegations);
760         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
761                 dbstat->db_ro_delegations);
762         printf(" %s\n", "locks");
763         printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
764                 dbstat->locks.num_calls);
765         printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
766                 dbstat->locks.num_failed);
767         printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
768                 dbstat->locks.num_current);
769         printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
770                 dbstat->locks.num_pending);
771         printf(" %s", "hop_count_buckets:");
772         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
773                 printf(" %d", dbstat->hop_count_bucket[i]);
774         }
775         printf("\n");
776         printf(" %s", "lock_buckets:");
777         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
778                 printf(" %d", dbstat->locks.buckets[i]);
779         }
780         printf("\n");
781         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
782                 "locks_latency      MIN/AVG/MAX",
783                 dbstat->locks.latency.min,
784                 (dbstat->locks.latency.num ?
785                  dbstat->locks.latency.total /dbstat->locks.latency.num :
786                  0.0),
787                 dbstat->locks.latency.max,
788                 dbstat->locks.latency.num);
789         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
790                 "vacuum_latency     MIN/AVG/MAX",
791                 dbstat->vacuum.latency.min,
792                 (dbstat->vacuum.latency.num ?
793                  dbstat->vacuum.latency.total /dbstat->vacuum.latency.num :
794                  0.0),
795                 dbstat->vacuum.latency.max,
796                 dbstat->vacuum.latency.num);
797         num_hot_keys = 0;
798         for (i=0; i<dbstat->num_hot_keys; i++) {
799                 if (dbstat->hot_keys[i].count > 0) {
800                         num_hot_keys++;
801                 }
802         }
803         dbstat->num_hot_keys = num_hot_keys;
804
805         printf(" Num Hot Keys:     %d\n", dbstat->num_hot_keys);
806         for (i = 0; i < dbstat->num_hot_keys; i++) {
807                 int j;
808                 printf("     Count:%d Key:", dbstat->hot_keys[i].count);
809                 for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
810                         printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
811                 }
812                 printf("\n");
813         }
814
815         talloc_free(tmp_ctx);
816         return 0;
817 }
818
819 /*
820   display uptime of remote node
821  */
822 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
823 {
824         int ret;
825         struct ctdb_uptime *uptime = NULL;
826         int tmp, days, hours, minutes, seconds;
827
828         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
829         if (ret != 0) {
830                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
831                 return ret;
832         }
833
834         if (options.machinereadable){
835                 printm(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
836                 printm(":%u:%u:%u:%lf\n",
837                         (unsigned int)uptime->current_time.tv_sec,
838                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
839                         (unsigned int)uptime->last_recovery_finished.tv_sec,
840                         timeval_delta(&uptime->last_recovery_finished,
841                                       &uptime->last_recovery_started)
842                 );
843                 return 0;
844         }
845
846         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
847
848         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
849         seconds = tmp%60;
850         tmp    /= 60;
851         minutes = tmp%60;
852         tmp    /= 60;
853         hours   = tmp%24;
854         tmp    /= 24;
855         days    = tmp;
856         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
857
858         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
859         seconds = tmp%60;
860         tmp    /= 60;
861         minutes = tmp%60;
862         tmp    /= 60;
863         hours   = tmp%24;
864         tmp    /= 24;
865         days    = tmp;
866         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
867         
868         printf("Duration of last recovery/failover: %lf seconds\n",
869                 timeval_delta(&uptime->last_recovery_finished,
870                               &uptime->last_recovery_started));
871
872         return 0;
873 }
874
875 /*
876   show the PNN of the current node
877  */
878 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
879 {
880         uint32_t mypnn;
881
882         mypnn = getpnn(ctdb);
883
884         printf("PNN:%d\n", mypnn);
885         return 0;
886 }
887
888
889 static struct ctdb_node_map_old *read_nodes_file(TALLOC_CTX *mem_ctx)
890 {
891         const char *nodes_list;
892
893         /* read the nodes file */
894         nodes_list = getenv("CTDB_NODES");
895         if (nodes_list == NULL) {
896                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes",
897                                              getenv("CTDB_BASE"));
898                 if (nodes_list == NULL) {
899                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
900                         exit(1);
901                 }
902         }
903
904         return ctdb_read_nodes_file(mem_ctx, nodes_list);
905 }
906
907 /*
908   show the PNN of the current node
909   discover the pnn by loading the nodes file and try to bind to all
910   addresses one at a time until the ip address is found.
911  */
912 static int find_node_xpnn(void)
913 {
914         TALLOC_CTX *mem_ctx = talloc_new(NULL);
915         struct ctdb_node_map_old *node_map;
916         int i, pnn;
917
918         node_map = read_nodes_file(mem_ctx);
919         if (node_map == NULL) {
920                 talloc_free(mem_ctx);
921                 return -1;
922         }
923
924         for (i = 0; i < node_map->num; i++) {
925                 if (node_map->nodes[i].flags & NODE_FLAGS_DELETED) {
926                         continue;
927                 }
928                 if (ctdb_sys_have_ip(&node_map->nodes[i].addr)) {
929                         pnn = node_map->nodes[i].pnn;
930                         talloc_free(mem_ctx);
931                         return pnn;
932                 }
933         }
934
935         printf("Failed to detect which PNN this node is\n");
936         talloc_free(mem_ctx);
937         return -1;
938 }
939
940 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
941 {
942         uint32_t pnn;
943
944         assert_single_node_only();
945
946         pnn = find_node_xpnn();
947         if (pnn == -1) {
948                 return -1;
949         }
950
951         printf("PNN:%d\n", pnn);
952         return 0;
953 }
954
955 /* Helpers for ctdb status
956  */
957 static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_flags *node)
958 {
959         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
960         int j;
961         bool ret = false;
962
963         if (node->flags == 0) {
964                 struct ctdb_iface_list_old *ifaces;
965
966                 if (ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), node->pnn,
967                                          tmp_ctx, &ifaces) == 0) {
968                         for (j=0; j < ifaces->num; j++) {
969                                 if (ifaces->ifaces[j].link_state != 0) {
970                                         continue;
971                                 }
972                                 ret = true;
973                                 break;
974                         }
975                 }
976         }
977         talloc_free(tmp_ctx);
978
979         return ret;
980 }
981
982 static void control_status_header_machine(void)
983 {
984         printm(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
985                ":Inactive:PartiallyOnline:ThisNode:\n");
986 }
987
988 static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn,
989                                     struct ctdb_node_and_flags *node)
990 {
991         printm(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
992                ctdb_addr_to_str(&node->addr),
993                !!(node->flags&NODE_FLAGS_DISCONNECTED),
994                !!(node->flags&NODE_FLAGS_BANNED),
995                !!(node->flags&NODE_FLAGS_PERMANENTLY_DISABLED),
996                !!(node->flags&NODE_FLAGS_UNHEALTHY),
997                !!(node->flags&NODE_FLAGS_STOPPED),
998                !!(node->flags&NODE_FLAGS_INACTIVE),
999                is_partially_online(ctdb, node) ? 1 : 0,
1000                (node->pnn == mypnn)?'Y':'N');
1001
1002         return node->flags;
1003 }
1004
1005 static int control_status_1_human(struct ctdb_context *ctdb, int mypnn,
1006                                   struct ctdb_node_and_flags *node)
1007 {
1008        printf("pnn:%d %-16s %s%s\n", node->pnn,
1009               ctdb_addr_to_str(&node->addr),
1010               is_partially_online(ctdb, node) ? "PARTIALLYONLINE" : pretty_print_flags(node->flags),
1011               node->pnn == mypnn?" (THIS NODE)":"");
1012
1013        return node->flags;
1014 }
1015
1016 /*
1017   display remote ctdb status
1018  */
1019 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
1020 {
1021         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1022         int i;
1023         struct ctdb_vnn_map *vnnmap=NULL;
1024         struct ctdb_node_map_old *nodemap=NULL;
1025         uint32_t recmode, recmaster, mypnn;
1026         int num_deleted_nodes = 0;
1027         int ret;
1028
1029         mypnn = getpnn(ctdb);
1030
1031         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1032         if (ret != 0) {
1033                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1034                 talloc_free(tmp_ctx);
1035                 return -1;
1036         }
1037
1038         if (options.machinereadable) {
1039                 control_status_header_machine();
1040                 for (i=0;i<nodemap->num;i++) {
1041                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1042                                 continue;
1043                         }
1044                         (void) control_status_1_machine(ctdb, mypnn,
1045                                                         &nodemap->nodes[i]);
1046                 }
1047                 talloc_free(tmp_ctx);
1048                 return 0;
1049         }
1050
1051         for (i=0; i<nodemap->num; i++) {
1052                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1053                         num_deleted_nodes++;
1054                 }
1055         }
1056         if (num_deleted_nodes == 0) {
1057                 printf("Number of nodes:%d\n", nodemap->num);
1058         } else {
1059                 printf("Number of nodes:%d (including %d deleted nodes)\n",
1060                        nodemap->num, num_deleted_nodes);
1061         }
1062         for(i=0;i<nodemap->num;i++){
1063                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1064                         continue;
1065                 }
1066                 (void) control_status_1_human(ctdb, mypnn, &nodemap->nodes[i]);
1067         }
1068
1069         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
1070         if (ret != 0) {
1071                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
1072                 talloc_free(tmp_ctx);
1073                 return -1;
1074         }
1075         if (vnnmap->generation == INVALID_GENERATION) {
1076                 printf("Generation:INVALID\n");
1077         } else {
1078                 printf("Generation:%d\n",vnnmap->generation);
1079         }
1080         printf("Size:%d\n",vnnmap->size);
1081         for(i=0;i<vnnmap->size;i++){
1082                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
1083         }
1084
1085         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmode);
1086         if (ret != 0) {
1087                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
1088                 talloc_free(tmp_ctx);
1089                 return -1;
1090         }
1091         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
1092
1093         ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmaster);
1094         if (ret != 0) {
1095                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1096                 talloc_free(tmp_ctx);
1097                 return -1;
1098         }
1099         printf("Recovery master:%d\n",recmaster);
1100
1101         talloc_free(tmp_ctx);
1102         return 0;
1103 }
1104
1105 static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **argv)
1106 {
1107         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1108         int i, ret;
1109         struct ctdb_node_map_old *nodemap=NULL;
1110         uint32_t * nodes;
1111         uint32_t pnn_mode, mypnn;
1112
1113         if (argc > 1) {
1114                 usage();
1115         }
1116
1117         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1118                               options.pnn, true, &nodes, &pnn_mode)) {
1119                 return -1;
1120         }
1121
1122         if (options.machinereadable) {
1123                 control_status_header_machine();
1124         } else if (pnn_mode == CTDB_BROADCAST_ALL) {
1125                 printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
1126         }
1127
1128         mypnn = getpnn(ctdb);
1129
1130         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1131         if (ret != 0) {
1132                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1133                 talloc_free(tmp_ctx);
1134                 return -1;
1135         }
1136
1137         ret = 0;
1138
1139         for (i = 0; i < talloc_array_length(nodes); i++) {
1140                 if (options.machinereadable) {
1141                         ret |= control_status_1_machine(ctdb, mypnn,
1142                                                         &nodemap->nodes[nodes[i]]);
1143                 } else {
1144                         ret |= control_status_1_human(ctdb, mypnn,
1145                                                       &nodemap->nodes[nodes[i]]);
1146                 }
1147         }
1148
1149         talloc_free(tmp_ctx);
1150         return ret;
1151 }
1152
1153 static struct ctdb_node_map_old *read_natgw_nodes_file(struct ctdb_context *ctdb,
1154                                                    TALLOC_CTX *mem_ctx)
1155 {
1156         const char *natgw_list;
1157         struct ctdb_node_map_old *natgw_nodes = NULL;
1158
1159         natgw_list = getenv("CTDB_NATGW_NODES");
1160         if (natgw_list == NULL) {
1161                 natgw_list = talloc_asprintf(mem_ctx, "%s/natgw_nodes",
1162                                              getenv("CTDB_BASE"));
1163                 if (natgw_list == NULL) {
1164                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
1165                         exit(1);
1166                 }
1167         }
1168         /* The PNNs/flags will be junk but they're not used */
1169         natgw_nodes = ctdb_read_nodes_file(mem_ctx, natgw_list);
1170         if (natgw_nodes == NULL) {
1171                 DEBUG(DEBUG_ERR,
1172                       ("Failed to load natgw node list '%s'\n", natgw_list));
1173         }
1174         return natgw_nodes;
1175 }
1176
1177
1178 /* talloc off the existing nodemap... */
1179 static struct ctdb_node_map_old *talloc_nodemap(struct ctdb_node_map_old *nodemap)
1180 {
1181         return talloc_zero_size(nodemap,
1182                                 offsetof(struct ctdb_node_map_old, nodes) +
1183                                 nodemap->num * sizeof(struct ctdb_node_and_flags));
1184 }
1185
1186 static struct ctdb_node_map_old *
1187 filter_nodemap_by_addrs(struct ctdb_context *ctdb,
1188                         struct ctdb_node_map_old *nodemap,
1189                         struct ctdb_node_map_old *natgw_nodes)
1190 {
1191         int i, j;
1192         struct ctdb_node_map_old *ret;
1193
1194         ret = talloc_nodemap(nodemap);
1195         CTDB_NO_MEMORY_NULL(ctdb, ret);
1196
1197         ret->num = 0;
1198
1199         for (i = 0; i < nodemap->num; i++) {
1200                 for(j = 0; j < natgw_nodes->num ; j++) {
1201                         if (nodemap->nodes[j].flags & NODE_FLAGS_DELETED) {
1202                                 continue;
1203                         }
1204                         if (ctdb_same_ip(&natgw_nodes->nodes[j].addr,
1205                                          &nodemap->nodes[i].addr)) {
1206
1207                                 ret->nodes[ret->num] = nodemap->nodes[i];
1208                                 ret->num++;
1209                                 break;
1210                         }
1211                 }
1212         }
1213
1214         return ret;
1215 }
1216
1217 static struct ctdb_node_map_old *
1218 filter_nodemap_by_capabilities(struct ctdb_context *ctdb,
1219                                struct ctdb_node_map_old *nodemap,
1220                                uint32_t required_capabilities,
1221                                bool first_only)
1222 {
1223         int i;
1224         uint32_t capabilities;
1225         struct ctdb_node_map_old *ret;
1226
1227         ret = talloc_nodemap(nodemap);
1228         CTDB_NO_MEMORY_NULL(ctdb, ret);
1229
1230         ret->num = 0;
1231
1232         for (i = 0; i < nodemap->num; i++) {
1233                 int res;
1234
1235                 /* Disconnected nodes have no capabilities! */
1236                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1237                         continue;
1238                 }
1239
1240                 res = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(),
1241                                                 nodemap->nodes[i].pnn,
1242                                                 &capabilities);
1243                 if (res != 0) {
1244                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n",
1245                                           nodemap->nodes[i].pnn));
1246                         talloc_free(ret);
1247                         return NULL;
1248                 }
1249                 if (!(capabilities & required_capabilities)) {
1250                         continue;
1251                 }
1252
1253                 ret->nodes[ret->num] = nodemap->nodes[i];
1254                 ret->num++;
1255                 if (first_only) {
1256                         break;
1257                 }
1258         }
1259
1260         return ret;
1261 }
1262
1263 static struct ctdb_node_map_old *
1264 filter_nodemap_by_flags(struct ctdb_context *ctdb,
1265                         struct ctdb_node_map_old *nodemap,
1266                         uint32_t flags_mask)
1267 {
1268         int i;
1269         struct ctdb_node_map_old *ret;
1270
1271         ret = talloc_nodemap(nodemap);
1272         CTDB_NO_MEMORY_NULL(ctdb, ret);
1273
1274         ret->num = 0;
1275
1276         for (i = 0; i < nodemap->num; i++) {
1277                 if (nodemap->nodes[i].flags & flags_mask) {
1278                         continue;
1279                 }
1280
1281                 ret->nodes[ret->num] = nodemap->nodes[i];
1282                 ret->num++;
1283         }
1284
1285         return ret;
1286 }
1287
1288 /*
1289   display the list of nodes belonging to this natgw configuration
1290  */
1291 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
1292 {
1293         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1294         int i, ret;
1295         struct ctdb_node_map_old *natgw_nodes = NULL;
1296         struct ctdb_node_map_old *orig_nodemap=NULL;
1297         struct ctdb_node_map_old *nodemap;
1298         uint32_t mypnn, pnn;
1299         const char *ip;
1300
1301         /* When we have some nodes that could be the NATGW, make a
1302          * series of attempts to find the first node that doesn't have
1303          * certain status flags set.
1304          */
1305         uint32_t exclude_flags[] = {
1306                 /* Look for a nice healthy node */
1307                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY,
1308                 /* If not found, an UNHEALTHY/BANNED node will do */
1309                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED,
1310                 /* If not found, a STOPPED node will do */
1311                 NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED,
1312                 0,
1313         };
1314
1315         /* read the natgw nodes file into a linked list */
1316         natgw_nodes = read_natgw_nodes_file(ctdb, tmp_ctx);
1317         if (natgw_nodes == NULL) {
1318                 ret = -1;
1319                 goto done;
1320         }
1321
1322         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
1323                                    tmp_ctx, &orig_nodemap);
1324         if (ret != 0) {
1325                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
1326                 talloc_free(tmp_ctx);
1327                 return -1;
1328         }
1329
1330         /* Get a nodemap that includes only the nodes in the NATGW
1331          * group */
1332         nodemap = filter_nodemap_by_addrs(ctdb, orig_nodemap, natgw_nodes);
1333         if (nodemap == NULL) {
1334                 ret = -1;
1335                 goto done;
1336         }
1337
1338         ret = 2; /* matches ENOENT */
1339         pnn = -1;
1340         ip = "0.0.0.0";
1341         /* For each flag mask... */
1342         for (i = 0; exclude_flags[i] != 0; i++) {
1343                 /* ... get a nodemap that excludes nodes with with
1344                  * masked flags... */
1345                 struct ctdb_node_map_old *t =
1346                         filter_nodemap_by_flags(ctdb, nodemap,
1347                                                 exclude_flags[i]);
1348                 if (t == NULL) {
1349                         /* No memory */
1350                         ret = -1;
1351                         goto done;
1352                 }
1353                 if (t->num > 0) {
1354                         /* ... and find the first node with the NATGW
1355                          * capability */
1356                         struct ctdb_node_map_old *n;
1357                         n = filter_nodemap_by_capabilities(ctdb, t,
1358                                                            CTDB_CAP_NATGW,
1359                                                            true);
1360                         if (n == NULL) {
1361                                 /* No memory */
1362                                 ret = -1;
1363                                 goto done;
1364                         }
1365                         if (n->num > 0) {
1366                                 ret = 0;
1367                                 pnn = n->nodes[0].pnn;
1368                                 ip = ctdb_addr_to_str(&n->nodes[0].addr);
1369                                 break;
1370                         }
1371                 }
1372                 talloc_free(t);
1373         }
1374
1375         if (options.machinereadable) {
1376                 printm(":Node:IP:\n");
1377                 printm(":%d:%s:\n", pnn, ip);
1378         } else {
1379                 printf("%d %s\n", pnn, ip);
1380         }
1381
1382         /* print the pruned list of nodes belonging to this natgw list */
1383         mypnn = getpnn(ctdb);
1384         if (options.machinereadable) {
1385                 control_status_header_machine();
1386         } else {
1387                 printf("Number of nodes:%d\n", nodemap->num);
1388         }
1389         for(i=0;i<nodemap->num;i++){
1390                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1391                         continue;
1392                 }
1393                 if (options.machinereadable) {
1394                         control_status_1_machine(ctdb, mypnn, &(nodemap->nodes[i]));
1395                 } else {
1396                         control_status_1_human(ctdb, mypnn, &(nodemap->nodes[i]));
1397                 }
1398         }
1399
1400 done:
1401         talloc_free(tmp_ctx);
1402         return ret;
1403 }
1404
1405 /*
1406   display the status of the scripts for monitoring (or other events)
1407  */
1408 static int control_one_scriptstatus(struct ctdb_context *ctdb,
1409                                     enum ctdb_event type)
1410 {
1411         struct ctdb_script_list_old *script_status;
1412         int ret, i;
1413
1414         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
1415         if (ret != 0) {
1416                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
1417                 return ret;
1418         }
1419
1420         if (script_status == NULL) {
1421                 if (!options.machinereadable) {
1422                         printf("%s cycle never run\n",
1423                                ctdb_eventscript_call_names[type]);
1424                 }
1425                 return 0;
1426         }
1427
1428         if (!options.machinereadable) {
1429                 int num_run = 0;
1430                 for (i=0; i<script_status->num_scripts; i++) {
1431                         if (script_status->scripts[i].status != -ENOEXEC) {
1432                                 num_run++;
1433                         }
1434                 }
1435                 printf("%d scripts were executed last %s cycle\n",
1436                        num_run,
1437                        ctdb_eventscript_call_names[type]);
1438         }
1439         for (i=0; i<script_status->num_scripts; i++) {
1440                 const char *status = NULL;
1441
1442                 /* The ETIME status is ignored for certain events.
1443                  * In that case the status is 0, but endtime is not set.
1444                  */
1445                 if (script_status->scripts[i].status == 0 &&
1446                     timeval_is_zero(&script_status->scripts[i].finished)) {
1447                         script_status->scripts[i].status = -ETIME;
1448                 }
1449
1450                 switch (script_status->scripts[i].status) {
1451                 case -ETIME:
1452                         status = "TIMEDOUT";
1453                         break;
1454                 case -ENOEXEC:
1455                         status = "DISABLED";
1456                         break;
1457                 case 0:
1458                         status = "OK";
1459                         break;
1460                 default:
1461                         if (script_status->scripts[i].status > 0)
1462                                 status = "ERROR";
1463                         break;
1464                 }
1465                 if (options.machinereadable) {
1466                         printm(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
1467                                ctdb_eventscript_call_names[type],
1468                                script_status->scripts[i].name,
1469                                script_status->scripts[i].status,
1470                                status,
1471                                (long)script_status->scripts[i].start.tv_sec,
1472                                (long)script_status->scripts[i].start.tv_usec,
1473                                (long)script_status->scripts[i].finished.tv_sec,
1474                                (long)script_status->scripts[i].finished.tv_usec,
1475                                script_status->scripts[i].output);
1476                         continue;
1477                 }
1478                 if (status)
1479                         printf("%-20s Status:%s    ",
1480                                script_status->scripts[i].name, status);
1481                 else
1482                         /* Some other error, eg from stat. */
1483                         printf("%-20s Status:CANNOT RUN (%s)",
1484                                script_status->scripts[i].name,
1485                                strerror(-script_status->scripts[i].status));
1486
1487                 if (script_status->scripts[i].status >= 0) {
1488                         printf("Duration:%.3lf ",
1489                         timeval_delta(&script_status->scripts[i].finished,
1490                               &script_status->scripts[i].start));
1491                 }
1492                 if (script_status->scripts[i].status != -ENOEXEC) {
1493                         printf("%s",
1494                                ctime(&script_status->scripts[i].start.tv_sec));
1495                         if (script_status->scripts[i].status != 0) {
1496                                 printf("   OUTPUT:%s\n",
1497                                        script_status->scripts[i].output);
1498                         }
1499                 } else {
1500                         printf("\n");
1501                 }
1502         }
1503         return 0;
1504 }
1505
1506
1507 static int control_scriptstatus(struct ctdb_context *ctdb,
1508                                 int argc, const char **argv)
1509 {
1510         int ret;
1511         enum ctdb_event type, min, max;
1512         const char *arg;
1513
1514         if (argc > 1) {
1515                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
1516                 return -1;
1517         }
1518
1519         if (argc == 0)
1520                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
1521         else
1522                 arg = argv[0];
1523
1524         for (type = 0; type < CTDB_EVENT_MAX; type++) {
1525                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
1526                         min = type;
1527                         max = type+1;
1528                         break;
1529                 }
1530         }
1531         if (type == CTDB_EVENT_MAX) {
1532                 if (strcmp(arg, "all") == 0) {
1533                         min = 0;
1534                         max = CTDB_EVENT_MAX;
1535                 } else {
1536                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
1537                         return -1;
1538                 }
1539         }
1540
1541         if (options.machinereadable) {
1542                 printm(":Type:Name:Code:Status:Start:End:Error Output...:\n");
1543         }
1544
1545         for (type = min; type < max; type++) {
1546                 ret = control_one_scriptstatus(ctdb, type);
1547                 if (ret != 0) {
1548                         return ret;
1549                 }
1550         }
1551
1552         return 0;
1553 }
1554
1555 /*
1556   enable an eventscript
1557  */
1558 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1559 {
1560         int ret;
1561
1562         if (argc < 1) {
1563                 usage();
1564         }
1565
1566         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1567         if (ret != 0) {
1568           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
1569                 return ret;
1570         }
1571
1572         return 0;
1573 }
1574
1575 /*
1576   disable an eventscript
1577  */
1578 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1579 {
1580         int ret;
1581
1582         if (argc < 1) {
1583                 usage();
1584         }
1585
1586         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1587         if (ret != 0) {
1588           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
1589                 return ret;
1590         }
1591
1592         return 0;
1593 }
1594
1595 /*
1596   display the pnn of the recovery master
1597  */
1598 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
1599 {
1600         uint32_t recmaster;
1601         int ret;
1602
1603         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
1604         if (ret != 0) {
1605                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1606                 return -1;
1607         }
1608         printf("%d\n",recmaster);
1609
1610         return 0;
1611 }
1612
1613 /*
1614   add a tickle to a public address
1615  */
1616 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1617 {
1618         struct ctdb_connection t;
1619         TDB_DATA data;
1620         int ret;
1621
1622         assert_single_node_only();
1623
1624         if (argc < 2) {
1625                 usage();
1626         }
1627
1628         if (parse_ip_port(argv[0], &t.src) == 0) {
1629                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1630                 return -1;
1631         }
1632         if (parse_ip_port(argv[1], &t.dst) == 0) {
1633                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1634                 return -1;
1635         }
1636
1637         data.dptr = (uint8_t *)&t;
1638         data.dsize = sizeof(t);
1639
1640         /* tell all nodes about this tcp connection */
1641         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1642                            0, data, ctdb, NULL, NULL, NULL, NULL);
1643         if (ret != 0) {
1644                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1645                 return -1;
1646         }
1647         
1648         return 0;
1649 }
1650
1651
1652 /*
1653   delete a tickle from a node
1654  */
1655 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1656 {
1657         struct ctdb_connection t;
1658         TDB_DATA data;
1659         int ret;
1660
1661         assert_single_node_only();
1662
1663         if (argc < 2) {
1664                 usage();
1665         }
1666
1667         if (parse_ip_port(argv[0], &t.src) == 0) {
1668                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1669                 return -1;
1670         }
1671         if (parse_ip_port(argv[1], &t.dst) == 0) {
1672                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1673                 return -1;
1674         }
1675
1676         data.dptr = (uint8_t *)&t;
1677         data.dsize = sizeof(t);
1678
1679         /* tell all nodes about this tcp connection */
1680         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1681                            0, data, ctdb, NULL, NULL, NULL, NULL);
1682         if (ret != 0) {
1683                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1684                 return -1;
1685         }
1686         
1687         return 0;
1688 }
1689
1690
1691 /*
1692   get a list of all tickles for this pnn
1693  */
1694 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1695 {
1696         struct ctdb_tickle_list_old *list;
1697         ctdb_sock_addr addr;
1698         int i, ret;
1699         unsigned port = 0;
1700
1701         assert_single_node_only();
1702
1703         if (argc < 1) {
1704                 usage();
1705         }
1706
1707         if (argc == 2) {
1708                 port = atoi(argv[1]);
1709         }
1710
1711         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1712                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1713                 return -1;
1714         }
1715
1716         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1717         if (ret == -1) {
1718                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1719                 return -1;
1720         }
1721
1722         if (options.machinereadable){
1723                 printm(":source ip:port:destination ip:port:\n");
1724                 for (i=0;i<list->num;i++) {
1725                         if (port && port != ntohs(list->connections[i].dst.ip.sin_port)) {
1726                                 continue;
1727                         }
1728                         printm(":%s:%u", ctdb_addr_to_str(&list->connections[i].src), ntohs(list->connections[i].src.ip.sin_port));
1729                         printm(":%s:%u:\n", ctdb_addr_to_str(&list->connections[i].dst), ntohs(list->connections[i].dst.ip.sin_port));
1730                 }
1731         } else {
1732                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1733                 printf("Num tickles:%u\n", list->num);
1734                 for (i=0;i<list->num;i++) {
1735                         if (port && port != ntohs(list->connections[i].dst.ip.sin_port)) {
1736                                 continue;
1737                         }
1738                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->connections[i].src), ntohs(list->connections[i].src.ip.sin_port));
1739                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->connections[i].dst), ntohs(list->connections[i].dst.ip.sin_port));
1740                 }
1741         }
1742
1743         talloc_free(list);
1744         
1745         return 0;
1746 }
1747
1748
1749 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1750 {
1751         struct ctdb_public_ip_list_old *ips;
1752         struct ctdb_public_ip ip;
1753         int i, ret;
1754         uint32_t *nodes;
1755         uint32_t disable_time;
1756         TDB_DATA data;
1757         struct ctdb_node_map_old *nodemap=NULL;
1758         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1759
1760         disable_time = 30;
1761         data.dptr  = (uint8_t*)&disable_time;
1762         data.dsize = sizeof(disable_time);
1763         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1764         if (ret != 0) {
1765                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1766                 return -1;
1767         }
1768
1769
1770
1771         /* read the public ip list from the node */
1772         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1773         if (ret != 0) {
1774                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1775                 talloc_free(tmp_ctx);
1776                 return -1;
1777         }
1778
1779         for (i=0;i<ips->num;i++) {
1780                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1781                         break;
1782                 }
1783         }
1784         if (i==ips->num) {
1785                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1786                         pnn, ctdb_addr_to_str(addr)));
1787                 talloc_free(tmp_ctx);
1788                 return -1;
1789         }
1790
1791         ip.pnn  = pnn;
1792         ip.addr = *addr;
1793
1794         data.dptr  = (uint8_t *)&ip;
1795         data.dsize = sizeof(ip);
1796
1797         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1798         if (ret != 0) {
1799                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1800                 talloc_free(tmp_ctx);
1801                 return ret;
1802         }
1803
1804         nodes = list_of_nodes(ctdb, nodemap, tmp_ctx, NODE_FLAGS_INACTIVE, pnn);
1805         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1806                                         nodes, 0,
1807                                         LONGTIMELIMIT(),
1808                                         false, data,
1809                                         NULL, NULL,
1810                                         NULL);
1811         if (ret != 0) {
1812                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1813                 talloc_free(tmp_ctx);
1814                 return -1;
1815         }
1816
1817         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1818         if (ret != 0) {
1819                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1820                 talloc_free(tmp_ctx);
1821                 return -1;
1822         }
1823
1824         /* update the recovery daemon so it now knows to expect the new
1825            node assignment for this ip.
1826         */
1827         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1828         if (ret != 0) {
1829                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1830                 return -1;
1831         }
1832
1833         talloc_free(tmp_ctx);
1834         return 0;
1835 }
1836
1837
1838 /* 
1839  * scans all other nodes and returns a pnn for another node that can host this 
1840  * ip address or -1
1841  */
1842 static int
1843 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1844 {
1845         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1846         struct ctdb_public_ip_list_old *ips;
1847         struct ctdb_node_map_old *nodemap=NULL;
1848         int i, j, ret;
1849         int pnn;
1850
1851         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1852         if (ret != 0) {
1853                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1854                 talloc_free(tmp_ctx);
1855                 return ret;
1856         }
1857
1858         for(i=0;i<nodemap->num;i++){
1859                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1860                         continue;
1861                 }
1862                 if (nodemap->nodes[i].pnn == options.pnn) {
1863                         continue;
1864                 }
1865
1866                 /* read the public ip list from this node */
1867                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1868                 if (ret != 0) {
1869                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1870                         return -1;
1871                 }
1872
1873                 for (j=0;j<ips->num;j++) {
1874                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1875                                 pnn = nodemap->nodes[i].pnn;
1876                                 talloc_free(tmp_ctx);
1877                                 return pnn;
1878                         }
1879                 }
1880                 talloc_free(ips);
1881         }
1882
1883         talloc_free(tmp_ctx);
1884         return -1;
1885 }
1886
1887 /* If pnn is -1 then try to find a node to move IP to... */
1888 static bool try_moveip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1889 {
1890         bool pnn_specified = (pnn == -1 ? false : true);
1891         int retries = 0;
1892
1893         while (retries < 5) {
1894                 if (!pnn_specified) {
1895                         pnn = find_other_host_for_public_ip(ctdb, addr);
1896                         if (pnn == -1) {
1897                                 return false;
1898                         }
1899                         DEBUG(DEBUG_NOTICE,
1900                               ("Trying to move public IP to node %u\n", pnn));
1901                 }
1902
1903                 if (move_ip(ctdb, addr, pnn) == 0) {
1904                         return true;
1905                 }
1906
1907                 sleep(3);
1908                 retries++;
1909         }
1910
1911         return false;
1912 }
1913
1914
1915 /*
1916   move/failover an ip address to a specific node
1917  */
1918 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1919 {
1920         uint32_t pnn;
1921         ctdb_sock_addr addr;
1922
1923         assert_single_node_only();
1924
1925         if (argc < 2) {
1926                 usage();
1927                 return -1;
1928         }
1929
1930         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1931                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1932                 return -1;
1933         }
1934
1935
1936         if (sscanf(argv[1], "%u", &pnn) != 1) {
1937                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1938                 return -1;
1939         }
1940
1941         if (!try_moveip(ctdb, &addr, pnn)) {
1942                 DEBUG(DEBUG_ERR,("Failed to move IP to node %d.\n", pnn));
1943                 return -1;
1944         }
1945
1946         return 0;
1947 }
1948
1949 static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
1950 {
1951         TDB_DATA data;
1952
1953         data.dptr  = (uint8_t *)&pnn;
1954         data.dsize = sizeof(uint32_t);
1955         if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
1956                 DEBUG(DEBUG_ERR,
1957                       ("Failed to send message to force node %u to be a rebalancing target\n",
1958                        pnn));
1959                 return -1;
1960         }
1961
1962         return 0;
1963 }
1964
1965
1966 /*
1967   rebalance a node by setting it to allow failback and triggering a
1968   takeover run
1969  */
1970 static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
1971 {
1972         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1973         uint32_t *nodes;
1974         uint32_t pnn_mode;
1975         int i, ret;
1976
1977         assert_single_node_only();
1978
1979         if (argc > 1) {
1980                 usage();
1981         }
1982
1983         /* Determine the nodes where IPs need to be reloaded */
1984         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1985                               options.pnn, true, &nodes, &pnn_mode)) {
1986                 ret = -1;
1987                 goto done;
1988         }
1989
1990         for (i = 0; i < talloc_array_length(nodes); i++) {
1991                 if (!rebalance_node(ctdb, nodes[i])) {
1992                         ret = -1;
1993                 }
1994         }
1995
1996 done:
1997         talloc_free(tmp_ctx);
1998         return ret;
1999 }
2000
2001 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
2002 {
2003         struct ctdb_public_ip ip;
2004         int ret;
2005         uint32_t *nodes;
2006         uint32_t disable_time;
2007         TDB_DATA data;
2008         struct ctdb_node_map_old *nodemap=NULL;
2009         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2010
2011         disable_time = 30;
2012         data.dptr  = (uint8_t*)&disable_time;
2013         data.dsize = sizeof(disable_time);
2014         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
2015         if (ret != 0) {
2016                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
2017                 return -1;
2018         }
2019
2020         ip.pnn  = -1;
2021         ip.addr = *addr;
2022
2023         data.dptr  = (uint8_t *)&ip;
2024         data.dsize = sizeof(ip);
2025
2026         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
2027         if (ret != 0) {
2028                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2029                 talloc_free(tmp_ctx);
2030                 return ret;
2031         }
2032
2033         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
2034         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
2035                                         nodes, 0,
2036                                         LONGTIMELIMIT(),
2037                                         false, data,
2038                                         NULL, NULL,
2039                                         NULL);
2040         if (ret != 0) {
2041                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
2042                 talloc_free(tmp_ctx);
2043                 return -1;
2044         }
2045
2046         talloc_free(tmp_ctx);
2047         return 0;
2048 }
2049
2050 /*
2051   release an ip form all nodes and have it re-assigned by recd
2052  */
2053 static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
2054 {
2055         ctdb_sock_addr addr;
2056
2057         assert_single_node_only();
2058
2059         if (argc < 1) {
2060                 usage();
2061                 return -1;
2062         }
2063
2064         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2065                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2066                 return -1;
2067         }
2068
2069         if (rebalance_ip(ctdb, &addr) != 0) {
2070                 DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
2071                 return -1;
2072         }
2073
2074         return 0;
2075 }
2076
2077 static int getips_store_callback(void *param, void *data)
2078 {
2079         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
2080         struct ctdb_public_ip_list_old *ips = param;
2081         int i;
2082
2083         i = ips->num++;
2084         ips->ips[i].pnn  = node_ip->pnn;
2085         ips->ips[i].addr = node_ip->addr;
2086         return 0;
2087 }
2088
2089 static int getips_count_callback(void *param, void *data)
2090 {
2091         uint32_t *count = param;
2092
2093         (*count)++;
2094         return 0;
2095 }
2096
2097 #define IP_KEYLEN       4
2098 static uint32_t *ip_key(ctdb_sock_addr *ip)
2099 {
2100         static uint32_t key[IP_KEYLEN];
2101
2102         bzero(key, sizeof(key));
2103
2104         switch (ip->sa.sa_family) {
2105         case AF_INET:
2106                 key[0]  = ip->ip.sin_addr.s_addr;
2107                 break;
2108         case AF_INET6: {
2109                 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
2110                 key[0]  = s6_a32[3];
2111                 key[1]  = s6_a32[2];
2112                 key[2]  = s6_a32[1];
2113                 key[3]  = s6_a32[0];
2114                 break;
2115         }
2116         default:
2117                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
2118                 return key;
2119         }
2120
2121         return key;
2122 }
2123
2124 static void *add_ip_callback(void *parm, void *data)
2125 {
2126         return parm;
2127 }
2128
2129 static int
2130 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_public_ip_list_old **ips)
2131 {
2132         struct ctdb_public_ip_list_old *tmp_ips;
2133         struct ctdb_node_map_old *nodemap=NULL;
2134         trbt_tree_t *ip_tree;
2135         int i, j, len, ret;
2136         uint32_t count;
2137
2138         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2139         if (ret != 0) {
2140                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2141                 return ret;
2142         }
2143
2144         ip_tree = trbt_create(tmp_ctx, 0);
2145
2146         for(i=0;i<nodemap->num;i++){
2147                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
2148                         continue;
2149                 }
2150                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2151                         continue;
2152                 }
2153
2154                 /* read the public ip list from this node */
2155                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
2156                 if (ret != 0) {
2157                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
2158                         return -1;
2159                 }
2160         
2161                 for (j=0; j<tmp_ips->num;j++) {
2162                         struct ctdb_public_ip *node_ip;
2163
2164                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
2165                         node_ip->pnn  = tmp_ips->ips[j].pnn;
2166                         node_ip->addr = tmp_ips->ips[j].addr;
2167
2168                         trbt_insertarray32_callback(ip_tree,
2169                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
2170                                 add_ip_callback,
2171                                 node_ip);
2172                 }
2173                 talloc_free(tmp_ips);
2174         }
2175
2176         /* traverse */
2177         count = 0;
2178         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
2179
2180         len = offsetof(struct ctdb_public_ip_list_old, ips) +
2181                 count*sizeof(struct ctdb_public_ip);
2182         tmp_ips = talloc_zero_size(tmp_ctx, len);
2183         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
2184
2185         *ips = tmp_ips;
2186
2187         return 0;
2188 }
2189
2190
2191 static void ctdb_every_second(struct tevent_context *ev,
2192                               struct tevent_timer *te,
2193                               struct timeval t, void *p)
2194 {
2195         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2196
2197         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2198                          ctdb_every_second, ctdb);
2199 }
2200
2201 struct srvid_reply_handler_data {
2202         bool done;
2203         bool wait_for_all;
2204         uint32_t *nodes;
2205         const char *srvid_str;
2206 };
2207
2208 static void srvid_broadcast_reply_handler(uint64_t srvid, TDB_DATA data,
2209                                          void *private_data)
2210 {
2211         struct srvid_reply_handler_data *d =
2212                 (struct srvid_reply_handler_data *)private_data;
2213         int i;
2214         int32_t ret;
2215
2216         if (data.dsize != sizeof(ret)) {
2217                 DEBUG(DEBUG_ERR, (__location__ " Wrong reply size\n"));
2218                 return;
2219         }
2220
2221         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
2222         ret = *(int32_t *)data.dptr;
2223         if (ret < 0) {
2224                 DEBUG(DEBUG_ERR,
2225                       ("%s failed with result %d\n", d->srvid_str, ret));
2226                 return;
2227         }
2228
2229         if (!d->wait_for_all) {
2230                 d->done = true;
2231                 return;
2232         }
2233
2234         /* Wait for all replies */
2235         d->done = true;
2236         for (i = 0; i < talloc_array_length(d->nodes); i++) {
2237                 if (d->nodes[i] == ret) {
2238                         DEBUG(DEBUG_DEBUG,
2239                               ("%s reply received from node %u\n",
2240                                d->srvid_str, ret));
2241                         d->nodes[i] = -1;
2242                 }
2243                 if (d->nodes[i] != -1) {
2244                         /* Found a node that hasn't yet replied */
2245                         d->done = false;
2246                 }
2247         }
2248 }
2249
2250 /* Broadcast the given SRVID to all connected nodes.  Wait for 1 reply
2251  * or replies from all connected nodes.  arg is the data argument to
2252  * pass in the srvid_request structure - pass 0 if this isn't needed.
2253  */
2254 static int srvid_broadcast(struct ctdb_context *ctdb,
2255                            uint64_t srvid, uint32_t *arg,
2256                            const char *srvid_str, bool wait_for_all)
2257 {
2258         int ret;
2259         TDB_DATA data;
2260         uint32_t pnn;
2261         uint64_t reply_srvid;
2262         struct ctdb_srvid_message request;
2263         struct ctdb_disable_message request_data;
2264         struct srvid_reply_handler_data reply_data;
2265         struct timeval tv;
2266
2267         ZERO_STRUCT(request);
2268
2269         /* Time ticks to enable timeouts to be processed */
2270         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2271                          ctdb_every_second, ctdb);
2272
2273         pnn = ctdb_get_pnn(ctdb);
2274         reply_srvid = getpid();
2275
2276         if (arg == NULL) {
2277                 request.pnn = pnn;
2278                 request.srvid = reply_srvid;
2279
2280                 data.dptr = (uint8_t *)&request;
2281                 data.dsize = sizeof(request);
2282         } else {
2283                 request_data.pnn = pnn;
2284                 request_data.srvid = reply_srvid;
2285                 request_data.timeout = *arg;
2286
2287                 data.dptr = (uint8_t *)&request_data;
2288                 data.dsize = sizeof(request_data);
2289         }
2290
2291         /* Register message port for reply from recovery master */
2292         ctdb_client_set_message_handler(ctdb, reply_srvid,
2293                                         srvid_broadcast_reply_handler,
2294                                         &reply_data);
2295
2296         reply_data.wait_for_all = wait_for_all;
2297         reply_data.nodes = NULL;
2298         reply_data.srvid_str = srvid_str;
2299
2300 again:
2301         reply_data.done = false;
2302
2303         if (wait_for_all) {
2304                 struct ctdb_node_map_old *nodemap;
2305
2306                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(),
2307                                            CTDB_CURRENT_NODE, ctdb, &nodemap);
2308                 if (ret != 0) {
2309                         DEBUG(DEBUG_ERR,
2310                               ("Unable to get nodemap from current node, try again\n"));
2311                         sleep(1);
2312                         goto again;
2313                 }
2314
2315                 if (reply_data.nodes != NULL) {
2316                         talloc_free(reply_data.nodes);
2317                 }
2318                 reply_data.nodes = list_of_connected_nodes(ctdb, nodemap,
2319                                                            NULL, true);
2320
2321                 talloc_free(nodemap);
2322         }
2323
2324         /* Send to all connected nodes. Only recmaster replies */
2325         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2326                                        srvid, data);
2327         if (ret != 0) {
2328                 /* This can only happen if the socket is closed and
2329                  * there's no way to recover from that, so don't try
2330                  * again.
2331                  */
2332                 DEBUG(DEBUG_ERR,
2333                       ("Failed to send %s request to connected nodes\n",
2334                        srvid_str));
2335                 return -1;
2336         }
2337
2338         tv = timeval_current();
2339         /* This loop terminates the reply is received */
2340         while (timeval_elapsed(&tv) < 5.0 && !reply_data.done) {
2341                 tevent_loop_once(ctdb->ev);
2342         }
2343
2344         if (!reply_data.done) {
2345                 DEBUG(DEBUG_NOTICE,
2346                       ("Still waiting for confirmation of %s\n", srvid_str));
2347                 sleep(1);
2348                 goto again;
2349         }
2350
2351         ctdb_client_remove_message_handler(ctdb, reply_srvid, &reply_data);
2352
2353         talloc_free(reply_data.nodes);
2354
2355         return 0;
2356 }
2357
2358 static int ipreallocate(struct ctdb_context *ctdb)
2359 {
2360         return srvid_broadcast(ctdb, CTDB_SRVID_TAKEOVER_RUN, NULL,
2361                                "IP reallocation", false);
2362 }
2363
2364
2365 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2366 {
2367         return ipreallocate(ctdb);
2368 }
2369
2370 /*
2371   add a public ip address to a node
2372  */
2373 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
2374 {
2375         int i, ret;
2376         int len, retries = 0;
2377         unsigned mask;
2378         ctdb_sock_addr addr;
2379         struct ctdb_addr_info_old *pub;
2380         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2381         struct ctdb_public_ip_list_old *ips;
2382
2383
2384         if (argc != 2) {
2385                 talloc_free(tmp_ctx);
2386                 usage();
2387         }
2388
2389         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
2390                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
2391                 talloc_free(tmp_ctx);
2392                 return -1;
2393         }
2394
2395         /* read the public ip list from the node */
2396         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2397         if (ret != 0) {
2398                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", options.pnn));
2399                 talloc_free(tmp_ctx);
2400                 return -1;
2401         }
2402         for (i=0;i<ips->num;i++) {
2403                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2404                         DEBUG(DEBUG_ERR,("Can not add ip to node. Node already hosts this ip\n"));
2405                         return 0;
2406                 }
2407         }
2408
2409
2410
2411         /* Dont timeout. This command waits for an ip reallocation
2412            which sometimes can take wuite a while if there has
2413            been a recent recovery
2414         */
2415         alarm(0);
2416
2417         len = offsetof(struct ctdb_addr_info_old, iface) + strlen(argv[1]) + 1;
2418         pub = talloc_size(tmp_ctx, len); 
2419         CTDB_NO_MEMORY(ctdb, pub);
2420
2421         pub->addr  = addr;
2422         pub->mask  = mask;
2423         pub->len   = strlen(argv[1])+1;
2424         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
2425
2426         do {
2427                 ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
2428                 if (ret != 0) {
2429                         DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
2430                         sleep(3);
2431                         retries++;
2432                 }
2433         } while (retries < 5 && ret != 0);
2434         if (ret != 0) {
2435                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
2436                 talloc_free(tmp_ctx);
2437                 return ret;
2438         }
2439
2440         if (rebalance_node(ctdb, options.pnn) != 0) {
2441                 DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
2442                 return ret;
2443         }
2444
2445         talloc_free(tmp_ctx);
2446         return 0;
2447 }
2448
2449 /*
2450   add a public ip address to a node
2451  */
2452 static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
2453 {
2454         ctdb_sock_addr addr;
2455         char *iface = NULL;
2456
2457         if (argc != 1) {
2458                 usage();
2459         }
2460
2461         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2462                 printf("Badly formed ip : %s\n", argv[0]);
2463                 return -1;
2464         }
2465
2466         iface = ctdb_sys_find_ifname(&addr);
2467         if (iface == NULL) {
2468                 printf("Failed to get interface name for ip: %s", argv[0]);
2469                 return -1;
2470         }
2471
2472         printf("IP on interface %s\n", iface);
2473
2474         free(iface);
2475
2476         return 0;
2477 }
2478
2479 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
2480
2481 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
2482 {
2483         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2484         struct ctdb_node_map_old *nodemap=NULL;
2485         struct ctdb_public_ip_list_old *ips;
2486         int ret, i, j;
2487
2488         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2489         if (ret != 0) {
2490                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
2491                 return ret;
2492         }
2493
2494         /* remove it from the nodes that are not hosting the ip currently */
2495         for(i=0;i<nodemap->num;i++){
2496                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2497                         continue;
2498                 }
2499                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2500                 if (ret != 0) {
2501                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2502                         continue;
2503                 }
2504
2505                 for (j=0;j<ips->num;j++) {
2506                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2507                                 break;
2508                         }
2509                 }
2510                 if (j==ips->num) {
2511                         continue;
2512                 }
2513
2514                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
2515                         continue;
2516                 }
2517
2518                 options.pnn = nodemap->nodes[i].pnn;
2519                 control_delip(ctdb, argc, argv);
2520         }
2521
2522
2523         /* remove it from every node (also the one hosting it) */
2524         for(i=0;i<nodemap->num;i++){
2525                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2526                         continue;
2527                 }
2528                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2529                 if (ret != 0) {
2530                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2531                         continue;
2532                 }
2533
2534                 for (j=0;j<ips->num;j++) {
2535                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2536                                 break;
2537                         }
2538                 }
2539                 if (j==ips->num) {
2540                         continue;
2541                 }
2542
2543                 options.pnn = nodemap->nodes[i].pnn;
2544                 control_delip(ctdb, argc, argv);
2545         }
2546
2547         talloc_free(tmp_ctx);
2548         return 0;
2549 }
2550         
2551 /*
2552   delete a public ip address from a node
2553  */
2554 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
2555 {
2556         int i, ret;
2557         ctdb_sock_addr addr;
2558         struct ctdb_addr_info_old pub;
2559         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2560         struct ctdb_public_ip_list_old *ips;
2561
2562         if (argc != 1) {
2563                 talloc_free(tmp_ctx);
2564                 usage();
2565         }
2566
2567         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2568                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2569                 return -1;
2570         }
2571
2572         if (options.pnn == CTDB_BROADCAST_ALL) {
2573                 return control_delip_all(ctdb, argc, argv, &addr);
2574         }
2575
2576         pub.addr  = addr;
2577         pub.mask  = 0;
2578         pub.len   = 0;
2579
2580         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2581         if (ret != 0) {
2582                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
2583                 talloc_free(tmp_ctx);
2584                 return ret;
2585         }
2586         
2587         for (i=0;i<ips->num;i++) {
2588                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2589                         break;
2590                 }
2591         }
2592
2593         if (i==ips->num) {
2594                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
2595                         ctdb_addr_to_str(&addr)));
2596                 talloc_free(tmp_ctx);
2597                 return -1;
2598         }
2599
2600         /* This is an optimisation.  If this node is hosting the IP
2601          * then try to move it somewhere else without invoking a full
2602          * takeover run.  We don't care if this doesn't work!
2603          */
2604         if (ips->ips[i].pnn == options.pnn) {
2605                 (void) try_moveip(ctdb, &addr, -1);
2606         }
2607
2608         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
2609         if (ret != 0) {
2610                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
2611                 talloc_free(tmp_ctx);
2612                 return ret;
2613         }
2614
2615         talloc_free(tmp_ctx);
2616         return 0;
2617 }
2618
2619 static int kill_tcp_from_file(struct ctdb_context *ctdb,
2620                               int argc, const char **argv)
2621 {
2622         struct ctdb_connection *killtcp;
2623         int max_entries, current, i;
2624         struct timeval timeout;
2625         char line[128], src[128], dst[128];
2626         int linenum;
2627         TDB_DATA data;
2628         struct client_async_data *async_data;
2629         struct ctdb_client_control_state *state;
2630
2631         if (argc != 0) {
2632                 usage();
2633         }
2634
2635         linenum = 1;
2636         killtcp = NULL;
2637         max_entries = 0;
2638         current = 0;
2639         while (!feof(stdin)) {
2640                 if (fgets(line, sizeof(line), stdin) == NULL) {
2641                         continue;
2642                 }
2643
2644                 /* Silently skip empty lines */
2645                 if (line[0] == '\n') {
2646                         continue;
2647                 }
2648
2649                 if (sscanf(line, "%s %s\n", src, dst) != 2) {
2650                         DEBUG(DEBUG_ERR, ("Bad line [%d]: '%s'\n",
2651                                           linenum, line));
2652                         talloc_free(killtcp);
2653                         return -1;
2654                 }
2655
2656                 if (current >= max_entries) {
2657                         max_entries += 1024;
2658                         killtcp = talloc_realloc(ctdb, killtcp,
2659                                                  struct ctdb_connection,
2660                                                  max_entries);
2661                         CTDB_NO_MEMORY(ctdb, killtcp);
2662                 }
2663
2664                 if (!parse_ip_port(src, &killtcp[current].src)) {
2665                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2666                                           linenum, src));
2667                         talloc_free(killtcp);
2668                         return -1;
2669                 }
2670
2671                 if (!parse_ip_port(dst, &killtcp[current].dst)) {
2672                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2673                                           linenum, dst));
2674                         talloc_free(killtcp);
2675                         return -1;
2676                 }
2677
2678                 current++;
2679         }
2680
2681         async_data = talloc_zero(ctdb, struct client_async_data);
2682         if (async_data == NULL) {
2683                 talloc_free(killtcp);
2684                 return -1;
2685         }
2686
2687         for (i = 0; i < current; i++) {
2688
2689                 data.dsize = sizeof(struct ctdb_connection);
2690                 data.dptr  = (unsigned char *)&killtcp[i];
2691
2692                 timeout = TIMELIMIT();
2693                 state = ctdb_control_send(ctdb, options.pnn, 0,
2694                                           CTDB_CONTROL_KILL_TCP, 0, data,
2695                                           async_data, &timeout, NULL);
2696
2697                 if (state == NULL) {
2698                         DEBUG(DEBUG_ERR,
2699                               ("Failed to call async killtcp control to node %u\n",
2700                                options.pnn));
2701                         talloc_free(killtcp);
2702                         return -1;
2703                 }
2704                 
2705                 ctdb_client_async_add(async_data, state);
2706         }
2707
2708         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2709                 DEBUG(DEBUG_ERR,("killtcp failed\n"));
2710                 talloc_free(killtcp);
2711                 return -1;
2712         }
2713
2714         talloc_free(killtcp);
2715         return 0;
2716 }
2717
2718
2719 /*
2720   kill a tcp connection
2721  */
2722 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2723 {
2724         int ret;
2725         struct ctdb_connection killtcp;
2726
2727         assert_single_node_only();
2728
2729         if (argc == 0) {
2730                 return kill_tcp_from_file(ctdb, argc, argv);
2731         }
2732
2733         if (argc < 2) {
2734                 usage();
2735         }
2736
2737         if (!parse_ip_port(argv[0], &killtcp.src)) {
2738                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2739                 return -1;
2740         }
2741
2742         if (!parse_ip_port(argv[1], &killtcp.dst)) {
2743                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2744                 return -1;
2745         }
2746
2747         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
2748         if (ret != 0) {
2749                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
2750                 return ret;
2751         }
2752
2753         return 0;
2754 }
2755
2756
2757 /*
2758   send a gratious arp
2759  */
2760 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
2761 {
2762         int ret;
2763         ctdb_sock_addr addr;
2764
2765         assert_single_node_only();
2766
2767         if (argc < 2) {
2768                 usage();
2769         }
2770
2771         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2772                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
2773                 return -1;
2774         }
2775
2776         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
2777         if (ret != 0) {
2778                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
2779                 return ret;
2780         }
2781
2782         return 0;
2783 }
2784
2785 /*
2786   register a server id
2787  */
2788 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2789 {
2790         int ret;
2791         struct ctdb_client_id server_id;
2792
2793         if (argc < 3) {
2794                 usage();
2795         }
2796
2797         server_id.pnn       = strtoul(argv[0], NULL, 0);
2798         server_id.type      = strtoul(argv[1], NULL, 0);
2799         server_id.server_id = strtoul(argv[2], NULL, 0);
2800
2801         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
2802         if (ret != 0) {
2803                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
2804                 return ret;
2805         }
2806         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
2807         sleep(999);
2808         return -1;
2809 }
2810
2811 /*
2812   unregister a server id
2813  */
2814 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2815 {
2816         int ret;
2817         struct ctdb_client_id server_id;
2818
2819         if (argc < 3) {
2820                 usage();
2821         }
2822
2823         server_id.pnn       = strtoul(argv[0], NULL, 0);
2824         server_id.type      = strtoul(argv[1], NULL, 0);
2825         server_id.server_id = strtoul(argv[2], NULL, 0);
2826
2827         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
2828         if (ret != 0) {
2829                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
2830                 return ret;
2831         }
2832         return -1;
2833 }
2834
2835 /*
2836   check if a server id exists
2837  */
2838 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2839 {
2840         uint32_t status = 0;
2841         int ret;
2842         struct ctdb_client_id server_id;
2843
2844         if (argc < 3) {
2845                 usage();
2846         }
2847
2848         server_id.pnn       = strtoul(argv[0], NULL, 0);
2849         server_id.type      = strtoul(argv[1], NULL, 0);
2850         server_id.server_id = strtoul(argv[2], NULL, 0);
2851
2852         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
2853         if (ret != 0) {
2854                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
2855                 return ret;
2856         }
2857
2858         if (status) {
2859                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
2860         } else {
2861                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
2862         }
2863         return 0;
2864 }
2865
2866 /*
2867   get a list of all server ids that are registered on a node
2868  */
2869 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
2870 {
2871         int i, ret;
2872         struct ctdb_client_id_list_old *server_ids;
2873
2874         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
2875         if (ret != 0) {
2876                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
2877                 return ret;
2878         }
2879
2880         for (i=0; i<server_ids->num; i++) {
2881                 printf("Server id %d:%d:%d\n", 
2882                         server_ids->server_ids[i].pnn, 
2883                         server_ids->server_ids[i].type, 
2884                         server_ids->server_ids[i].server_id); 
2885         }
2886
2887         return -1;
2888 }
2889
2890 /*
2891   check if a server id exists
2892  */
2893 static int check_srvids(struct ctdb_context *ctdb, int argc, const char **argv)
2894 {
2895         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
2896         uint64_t *ids;
2897         uint8_t *result;
2898         int i;
2899
2900         if (argc < 1) {
2901                 talloc_free(tmp_ctx);
2902                 usage();
2903         }
2904
2905         ids    = talloc_array(tmp_ctx, uint64_t, argc);
2906         result = talloc_array(tmp_ctx, uint8_t, argc);
2907
2908         for (i = 0; i < argc; i++) {
2909                 ids[i] = strtoull(argv[i], NULL, 0);
2910         }
2911
2912         if (!ctdb_client_check_message_handlers(ctdb, ids, argc, result)) {
2913                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n",
2914                                   options.pnn));
2915                 talloc_free(tmp_ctx);
2916                 return -1;
2917         }
2918
2919         for (i=0; i < argc; i++) {
2920                 printf("Server id %d:%llu %s\n", options.pnn, (long long)ids[i],
2921                        result[i] ? "exists" : "does not exist");
2922         }
2923
2924         talloc_free(tmp_ctx);
2925         return 0;
2926 }
2927
2928 /*
2929   send a tcp tickle ack
2930  */
2931 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2932 {
2933         int ret;
2934         ctdb_sock_addr  src, dst;
2935
2936         if (argc < 2) {
2937                 usage();
2938         }
2939
2940         if (!parse_ip_port(argv[0], &src)) {
2941                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2942                 return -1;
2943         }
2944
2945         if (!parse_ip_port(argv[1], &dst)) {
2946                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2947                 return -1;
2948         }
2949
2950         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
2951         if (ret==0) {
2952                 return 0;
2953         }
2954         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
2955
2956         return -1;
2957 }
2958
2959
2960 /*
2961   display public ip status
2962  */
2963 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
2964 {
2965         int i, ret;
2966         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2967         struct ctdb_public_ip_list_old *ips;
2968
2969         if (argc == 1 && strcmp(argv[0], "all") == 0) {
2970                 options.pnn = CTDB_BROADCAST_ALL;
2971         }
2972
2973         if (options.pnn == CTDB_BROADCAST_ALL) {
2974                 /* read the list of public ips from all nodes */
2975                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
2976         } else {
2977                 /* read the public ip list from this node */
2978                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2979         }
2980         if (ret != 0) {
2981                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
2982                 talloc_free(tmp_ctx);
2983                 return ret;
2984         }
2985
2986         if (options.machinereadable){
2987                 printm(":Public IP:Node:");
2988                 if (options.verbose){
2989                         printm("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
2990                 }
2991                 printm("\n");
2992         } else {
2993                 if (options.pnn == CTDB_BROADCAST_ALL) {
2994                         printf("Public IPs on ALL nodes\n");
2995                 } else {
2996                         printf("Public IPs on node %u\n", options.pnn);
2997                 }
2998         }
2999
3000         for (i=1;i<=ips->num;i++) {
3001                 struct ctdb_public_ip_info_old *info = NULL;
3002                 int32_t pnn;
3003                 char *aciface = NULL;
3004                 char *avifaces = NULL;
3005                 char *cifaces = NULL;
3006
3007                 if (options.pnn == CTDB_BROADCAST_ALL) {
3008                         pnn = ips->ips[ips->num-i].pnn;
3009                 } else {
3010                         pnn = options.pnn;
3011                 }
3012
3013                 if (pnn != -1) {
3014                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
3015                                                    &ips->ips[ips->num-i].addr, &info);
3016                 } else {
3017                         ret = -1;
3018                 }
3019
3020                 if (ret == 0) {
3021                         int j;
3022                         for (j=0; j < info->num; j++) {
3023                                 if (cifaces == NULL) {
3024                                         cifaces = talloc_strdup(info,
3025                                                                 info->ifaces[j].name);
3026                                 } else {
3027                                         cifaces = talloc_asprintf_append(cifaces,
3028                                                                          ",%s",
3029                                                                          info->ifaces[j].name);
3030                                 }
3031
3032                                 if (info->active_idx == j) {
3033                                         aciface = info->ifaces[j].name;
3034                                 }
3035
3036                                 if (info->ifaces[j].link_state == 0) {
3037                                         continue;
3038                                 }
3039
3040                                 if (avifaces == NULL) {
3041                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
3042                                 } else {
3043                                         avifaces = talloc_asprintf_append(avifaces,
3044                                                                           ",%s",
3045                                                                           info->ifaces[j].name);
3046                                 }
3047                         }
3048                 }
3049
3050                 if (options.machinereadable){
3051                         printm(":%s:%d:",
3052                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3053                                 ips->ips[ips->num-i].pnn);
3054                         if (options.verbose){
3055                                 printm("%s:%s:%s:",
3056                                         aciface?aciface:"",
3057                                         avifaces?avifaces:"",
3058                                         cifaces?cifaces:"");
3059                         }
3060                         printf("\n");
3061                 } else {
3062                         if (options.verbose) {
3063                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
3064                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3065                                         ips->ips[ips->num-i].pnn,
3066                                         aciface?aciface:"",
3067                                         avifaces?avifaces:"",
3068                                         cifaces?cifaces:"");
3069                         } else {
3070                                 printf("%s %d\n",
3071                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
3072                                         ips->ips[ips->num-i].pnn);
3073                         }
3074                 }
3075                 talloc_free(info);
3076         }
3077
3078         talloc_free(tmp_ctx);
3079         return 0;
3080 }
3081
3082 /*
3083   public ip info
3084  */
3085 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
3086 {
3087         int i, ret;
3088         ctdb_sock_addr addr;
3089         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3090         struct ctdb_public_ip_info_old *info;
3091
3092         if (argc != 1) {
3093                 talloc_free(tmp_ctx);
3094                 usage();
3095         }
3096
3097         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
3098                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
3099                 return -1;
3100         }
3101
3102         /* read the public ip info from this node */
3103         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
3104                                            tmp_ctx, &addr, &info);
3105         if (ret != 0) {
3106                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
3107                                   argv[0], options.pnn));
3108                 talloc_free(tmp_ctx);
3109                 return ret;
3110         }
3111
3112         printf("Public IP[%s] info on node %u\n",
3113                ctdb_addr_to_str(&info->ip.addr),
3114                options.pnn);
3115
3116         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
3117                ctdb_addr_to_str(&info->ip.addr),
3118                info->ip.pnn, info->num);
3119
3120         for (i=0; i<info->num; i++) {
3121                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
3122
3123                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
3124                        i+1, info->ifaces[i].name,
3125                        info->ifaces[i].link_state?"up":"down",
3126                        (unsigned int)info->ifaces[i].references,
3127                        (i==info->active_idx)?" (active)":"");
3128         }
3129
3130         talloc_free(tmp_ctx);
3131         return 0;
3132 }
3133
3134 /*
3135   display interfaces status
3136  */
3137 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
3138 {
3139         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3140         int i;
3141         struct ctdb_iface_list_old *ifaces;
3142         int ret;
3143
3144         /* read the public ip list from this node */
3145         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ifaces);
3146         if (ret != 0) {
3147                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
3148                                   options.pnn));
3149                 talloc_free(tmp_ctx);
3150                 return -1;
3151         }
3152
3153         if (options.machinereadable){
3154                 printm(":Name:LinkStatus:References:\n");
3155         } else {
3156                 printf("Interfaces on node %u\n", options.pnn);
3157         }
3158
3159         for (i=0; i<ifaces->num; i++) {
3160                 if (options.machinereadable){
3161                         printm(":%s:%s:%u:\n",
3162                                ifaces->ifaces[i].name,
3163                                ifaces->ifaces[i].link_state?"1":"0",
3164                                (unsigned int)ifaces->ifaces[i].references);
3165                 } else {
3166                         printf("name:%s link:%s references:%u\n",
3167                                ifaces->ifaces[i].name,
3168                                ifaces->ifaces[i].link_state?"up":"down",
3169                                (unsigned int)ifaces->ifaces[i].references);
3170                 }
3171         }
3172
3173         talloc_free(tmp_ctx);
3174         return 0;
3175 }
3176
3177
3178 /*
3179   set link status of an interface
3180  */
3181 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
3182 {
3183         int ret;