ctdb-tools: Drop support for setting and viewing NAT gateway capability
[ambi/samba-autobuild/.git] / ctdb / tools / ctdb.c
1 /* 
2    ctdb control tool
3
4    Copyright (C) Andrew Tridgell  2007
5    Copyright (C) Ronnie Sahlberg  2007
6
7    This program is free software; you can redistribute it and/or modify
8    it under the terms of the GNU General Public License as published by
9    the Free Software Foundation; either version 3 of the License, or
10    (at your option) any later version.
11    
12    This program is distributed in the hope that it will be useful,
13    but WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15    GNU General Public License for more details.
16    
17    You should have received a copy of the GNU General Public License
18    along with this program; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "replace.h"
22 #include "system/time.h"
23 #include "system/filesys.h"
24 #include "system/network.h"
25 #include "system/locale.h"
26
27 #include <popt.h>
28 #include <talloc.h>
29 /* Allow use of deprecated function tevent_loop_allow_nesting() */
30 #define TEVENT_DEPRECATED
31 #include <tevent.h>
32 #include <tdb.h>
33
34 #include "lib/tdb_wrap/tdb_wrap.h"
35 #include "lib/util/dlinklist.h"
36 #include "lib/util/debug.h"
37 #include "lib/util/substitute.h"
38 #include "lib/util/time.h"
39
40 #include "ctdb_version.h"
41 #include "ctdb_private.h"
42 #include "ctdb_client.h"
43
44 #include "common/cmdline.h"
45 #include "common/rb_tree.h"
46 #include "common/system.h"
47 #include "common/common.h"
48 #include "common/logging.h"
49
50 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
51 #define ERR_NONODE      21      /* node does not exist */
52 #define ERR_DISNODE     22      /* node is disconnected */
53
54 static void usage(void);
55
56 static struct {
57         int timelimit;
58         uint32_t pnn;
59         uint32_t *nodes;
60         int machinereadable;
61         const char *machineseparator;
62         int verbose;
63         int maxruntime;
64         int printemptyrecords;
65         int printdatasize;
66         int printlmaster;
67         int printhash;
68         int printrecordflags;
69 } options;
70
71 #define LONGTIMEOUT options.timelimit*10
72
73 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
74 #define LONGTIMELIMIT() timeval_current_ofs(LONGTIMEOUT, 0)
75
76 static double timeval_delta(struct timeval *tv2, struct timeval *tv)
77 {
78         return (tv2->tv_sec - tv->tv_sec) +
79                 (tv2->tv_usec - tv->tv_usec)*1.0e-6;
80 }
81
82 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
83 {
84         printf("CTDB version: %s\n", CTDB_VERSION_STRING);
85         return 0;
86 }
87
88 /* Like printf(3) but substitute for separator in format */
89 static int printm(const char *format, ...) PRINTF_ATTRIBUTE(1,2);
90 static int printm(const char *format, ...)
91 {
92         va_list ap;
93         int ret;
94         size_t len = strlen(format);
95         char new_format[len+1];
96
97         strcpy(new_format, format);
98
99         if (options.machineseparator[0] != ':') {
100                 all_string_sub(new_format,
101                                ":", options.machineseparator, len + 1);
102         }
103
104         va_start(ap, format);
105         ret = vprintf(new_format, ap);
106         va_end(ap);
107
108         return ret;
109 }
110
111 #define CTDB_NOMEM_ABORT(p) do { if (!(p)) {                            \
112                 DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n",            \
113                                    "Out of memory in " __location__ )); \
114                 abort();                                                \
115         }} while (0)
116
117 static uint32_t getpnn(struct ctdb_context *ctdb)
118 {
119         if ((options.pnn == CTDB_BROADCAST_ALL) ||
120             (options.pnn == CTDB_MULTICAST)) {
121                 DEBUG(DEBUG_ERR,
122                       ("Cannot get PNN for node %u\n", options.pnn));
123                 exit(1);
124         }
125
126         if (options.pnn == CTDB_CURRENT_NODE) {
127                 return ctdb_get_pnn(ctdb);
128         } else {
129                 return options.pnn;
130         }
131 }
132
133 static void assert_single_node_only(void)
134 {
135         if ((options.pnn == CTDB_BROADCAST_ALL) ||
136             (options.pnn == CTDB_MULTICAST)) {
137                 DEBUG(DEBUG_ERR,
138                       ("This control can not be applied to multiple PNNs\n"));
139                 exit(1);
140         }
141 }
142
143 static void assert_current_node_only(struct ctdb_context *ctdb)
144 {
145         if (options.pnn != ctdb_get_pnn(ctdb)) {
146                 DEBUG(DEBUG_ERR,
147                       ("This control can only be applied to the current node\n"));
148                 exit(1);
149         }
150 }
151
152 /* Pretty print the flags to a static buffer in human-readable format.
153  * This never returns NULL!
154  */
155 static const char *pretty_print_flags(uint32_t flags)
156 {
157         int j;
158         static const struct {
159                 uint32_t flag;
160                 const char *name;
161         } flag_names[] = {
162                 { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
163                 { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
164                 { NODE_FLAGS_BANNED,                "BANNED" },
165                 { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
166                 { NODE_FLAGS_DELETED,               "DELETED" },
167                 { NODE_FLAGS_STOPPED,               "STOPPED" },
168                 { NODE_FLAGS_INACTIVE,              "INACTIVE" },
169         };
170         static char flags_str[512]; /* Big enough to contain all flag names */
171
172         flags_str[0] = '\0';
173         for (j=0;j<ARRAY_SIZE(flag_names);j++) {
174                 if (flags & flag_names[j].flag) {
175                         if (flags_str[0] == '\0') {
176                                 (void) strcpy(flags_str, flag_names[j].name);
177                         } else {
178                                 (void) strncat(flags_str, "|", sizeof(flags_str)-1);
179                                 (void) strncat(flags_str, flag_names[j].name,
180                                                sizeof(flags_str)-1);
181                         }
182                 }
183         }
184         if (flags_str[0] == '\0') {
185                 (void) strcpy(flags_str, "OK");
186         }
187
188         return flags_str;
189 }
190
191 static int h2i(char h)
192 {
193         if (h >= 'a' && h <= 'f') return h - 'a' + 10;
194         if (h >= 'A' && h <= 'F') return h - 'f' + 10;
195         return h - '0';
196 }
197
198 static TDB_DATA hextodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
199 {
200         int i;
201         TDB_DATA key = {NULL, 0};
202
203         if (len & 0x01) {
204                 DEBUG(DEBUG_ERR,("Key specified with odd number of hexadecimal digits\n"));
205                 return key;
206         }
207
208         key.dsize = len>>1;
209         key.dptr  = talloc_size(mem_ctx, key.dsize);
210
211         for (i=0; i < len/2; i++) {
212                 key.dptr[i] = h2i(str[i*2]) << 4 | h2i(str[i*2+1]);
213         }
214         return key;
215 }
216
217 static TDB_DATA strtodata(TALLOC_CTX *mem_ctx, const char *str, size_t len)
218 {
219         TDB_DATA key;
220
221         if (!strncmp(str, "0x", 2)) {
222                 key = hextodata(mem_ctx, str + 2, len - 2);
223         } else {
224                 key.dptr  = talloc_memdup(mem_ctx, str, len);
225                 key.dsize = len;
226         }
227
228         return key;
229 }
230
231 /* Parse a nodestring.  Parameter dd_ok controls what happens to nodes
232  * that are disconnected or deleted.  If dd_ok is true those nodes are
233  * included in the output list of nodes.  If dd_ok is false, those
234  * nodes are filtered from the "all" case and cause an error if
235  * explicitly specified.
236  */
237 static bool parse_nodestring(struct ctdb_context *ctdb,
238                              TALLOC_CTX *mem_ctx,
239                              const char * nodestring,
240                              uint32_t current_pnn,
241                              bool dd_ok,
242                              uint32_t **nodes,
243                              uint32_t *pnn_mode)
244 {
245         TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
246         int n;
247         uint32_t i;
248         struct ctdb_node_map_old *nodemap;
249         int ret;
250
251         *nodes = NULL;
252
253         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
254         if (ret != 0) {
255                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
256                 talloc_free(tmp_ctx);
257                 exit(10);
258         }
259
260         if (nodestring != NULL) {
261                 *nodes = talloc_array(mem_ctx, uint32_t, 0);
262                 if (*nodes == NULL) {
263                         goto failed;
264                 }
265
266                 n = 0;
267
268                 if (strcmp(nodestring, "all") == 0) {
269                         *pnn_mode = CTDB_BROADCAST_ALL;
270
271                         /* all */
272                         for (i = 0; i < nodemap->num; i++) {
273                                 if ((nodemap->nodes[i].flags &
274                                      (NODE_FLAGS_DISCONNECTED |
275                                       NODE_FLAGS_DELETED)) && !dd_ok) {
276                                         continue;
277                                 }
278                                 *nodes = talloc_realloc(mem_ctx, *nodes,
279                                                         uint32_t, n+1);
280                                 if (*nodes == NULL) {
281                                         goto failed;
282                                 }
283                                 (*nodes)[n] = i;
284                                 n++;
285                         }
286                 } else {
287                         /* x{,y...} */
288                         char *ns, *tok;
289
290                         ns = talloc_strdup(tmp_ctx, nodestring);
291                         tok = strtok(ns, ",");
292                         while (tok != NULL) {
293                                 uint32_t pnn;
294                                 char *endptr;
295                                 i = (uint32_t)strtoul(tok, &endptr, 0);
296                                 if (i == 0 && tok == endptr) {
297                                         DEBUG(DEBUG_ERR,
298                                               ("Invalid node %s\n", tok));
299                                         talloc_free(tmp_ctx);
300                                         exit(ERR_NONODE);
301                                 }
302                                 if (i >= nodemap->num) {
303                                         DEBUG(DEBUG_ERR, ("Node %u does not exist\n", i));
304                                         talloc_free(tmp_ctx);
305                                         exit(ERR_NONODE);
306                                 }
307                                 if ((nodemap->nodes[i].flags & 
308                                      (NODE_FLAGS_DISCONNECTED |
309                                       NODE_FLAGS_DELETED)) && !dd_ok) {
310                                         DEBUG(DEBUG_ERR, ("Node %u has status %s\n", i, pretty_print_flags(nodemap->nodes[i].flags)));
311                                         talloc_free(tmp_ctx);
312                                         exit(ERR_DISNODE);
313                                 }
314                                 if ((pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), i)) < 0) {
315                                         DEBUG(DEBUG_ERR, ("Can not access node %u. Node is not operational.\n", i));
316                                         talloc_free(tmp_ctx);
317                                         exit(10);
318                                 }
319
320                                 *nodes = talloc_realloc(mem_ctx, *nodes,
321                                                         uint32_t, n+1);
322                                 if (*nodes == NULL) {
323                                         goto failed;
324                                 }
325
326                                 (*nodes)[n] = i;
327                                 n++;
328
329                                 tok = strtok(NULL, ",");
330                         }
331                         talloc_free(ns);
332
333                         if (n == 1) {
334                                 *pnn_mode = (*nodes)[0];
335                         } else {
336                                 *pnn_mode = CTDB_MULTICAST;
337                         }
338                 }
339         } else {
340                 /* default - no nodes specified */
341                 *nodes = talloc_array(mem_ctx, uint32_t, 1);
342                 if (*nodes == NULL) {
343                         goto failed;
344                 }
345                 *pnn_mode = CTDB_CURRENT_NODE;
346
347                 if (((*nodes)[0] = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), current_pnn)) < 0) {
348                         goto failed;
349                 }
350         }
351
352         talloc_free(tmp_ctx);
353         return true;
354
355 failed:
356         talloc_free(tmp_ctx);
357         return false;
358 }
359
360 /*
361  check if a database exists
362 */
363 static bool db_exists(struct ctdb_context *ctdb, const char *dbarg,
364                       uint32_t *dbid, const char **dbname, uint8_t *flags)
365 {
366         int i, ret;
367         struct ctdb_dbid_map_old *dbmap=NULL;
368         bool dbid_given = false, found = false;
369         uint32_t id;
370         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
371         const char *name = NULL;
372
373         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
374         if (ret != 0) {
375                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
376                 goto fail;
377         }
378
379         if (strncmp(dbarg, "0x", 2) == 0) {
380                 id = strtoul(dbarg, NULL, 0);
381                 dbid_given = true;
382         }
383
384         for(i=0; i<dbmap->num; i++) {
385                 if (dbid_given) {
386                         if (id == dbmap->dbs[i].db_id) {
387                                 found = true;
388                                 break;
389                         }
390                 } else {
391                         ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, tmp_ctx, &name);
392                         if (ret != 0) {
393                                 DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].db_id));
394                                 goto fail;
395                         }
396
397                         if (strcmp(name, dbarg) == 0) {
398                                 id = dbmap->dbs[i].db_id;
399                                 found = true;
400                                 break;
401                         }
402                 }
403         }
404
405         if (found && dbid_given && dbname != NULL) {
406                 ret = ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].db_id, tmp_ctx, &name);
407                 if (ret != 0) {
408                         DEBUG(DEBUG_ERR, ("Unable to get dbname from dbid %u\n", dbmap->dbs[i].db_id));
409                         found = false;
410                         goto fail;
411                 }
412         }
413
414         if (found) {
415                 if (dbid) *dbid = id;
416                 if (dbname) *dbname = talloc_strdup(ctdb, name);
417                 if (flags) *flags = dbmap->dbs[i].flags;
418         } else {
419                 DEBUG(DEBUG_ERR,("No database matching '%s' found\n", dbarg));
420         }
421
422 fail:
423         talloc_free(tmp_ctx);
424         return found;
425 }
426
427 /*
428   see if a process exists
429  */
430 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
431 {
432         uint32_t pnn, pid;
433         int ret;
434         if (argc < 1) {
435                 usage();
436         }
437
438         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
439                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
440                 return -1;
441         }
442
443         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
444         if (ret == 0) {
445                 printf("%u:%u exists\n", pnn, pid);
446         } else {
447                 printf("%u:%u does not exist\n", pnn, pid);
448         }
449         return ret;
450 }
451
452 /*
453   display statistics structure
454  */
455 static void show_statistics(struct ctdb_statistics *s, int show_header)
456 {
457         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
458         int i;
459         const char *prefix=NULL;
460         int preflen=0;
461         int tmp, days, hours, minutes, seconds;
462         const struct {
463                 const char *name;
464                 uint32_t offset;
465         } fields[] = {
466 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
467                 STATISTICS_FIELD(num_clients),
468                 STATISTICS_FIELD(frozen),
469                 STATISTICS_FIELD(recovering),
470                 STATISTICS_FIELD(num_recoveries),
471                 STATISTICS_FIELD(client_packets_sent),
472                 STATISTICS_FIELD(client_packets_recv),
473                 STATISTICS_FIELD(node_packets_sent),
474                 STATISTICS_FIELD(node_packets_recv),
475                 STATISTICS_FIELD(keepalive_packets_sent),
476                 STATISTICS_FIELD(keepalive_packets_recv),
477                 STATISTICS_FIELD(node.req_call),
478                 STATISTICS_FIELD(node.reply_call),
479                 STATISTICS_FIELD(node.req_dmaster),
480                 STATISTICS_FIELD(node.reply_dmaster),
481                 STATISTICS_FIELD(node.reply_error),
482                 STATISTICS_FIELD(node.req_message),
483                 STATISTICS_FIELD(node.req_control),
484                 STATISTICS_FIELD(node.reply_control),
485                 STATISTICS_FIELD(client.req_call),
486                 STATISTICS_FIELD(client.req_message),
487                 STATISTICS_FIELD(client.req_control),
488                 STATISTICS_FIELD(timeouts.call),
489                 STATISTICS_FIELD(timeouts.control),
490                 STATISTICS_FIELD(timeouts.traverse),
491                 STATISTICS_FIELD(locks.num_calls),
492                 STATISTICS_FIELD(locks.num_current),
493                 STATISTICS_FIELD(locks.num_pending),
494                 STATISTICS_FIELD(locks.num_failed),
495                 STATISTICS_FIELD(total_calls),
496                 STATISTICS_FIELD(pending_calls),
497                 STATISTICS_FIELD(childwrite_calls),
498                 STATISTICS_FIELD(pending_childwrite_calls),
499                 STATISTICS_FIELD(memory_used),
500                 STATISTICS_FIELD(max_hop_count),
501                 STATISTICS_FIELD(total_ro_delegations),
502                 STATISTICS_FIELD(total_ro_revokes),
503         };
504         
505         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
506         seconds = tmp%60;
507         tmp    /= 60;
508         minutes = tmp%60;
509         tmp    /= 60;
510         hours   = tmp%24;
511         tmp    /= 24;
512         days    = tmp;
513
514         if (options.machinereadable){
515                 if (show_header) {
516                         printm("CTDB version:");
517                         printm("Current time of statistics:");
518                         printm("Statistics collected since:");
519                         for (i=0;i<ARRAY_SIZE(fields);i++) {
520                                 printm("%s:", fields[i].name);
521                         }
522                         printm("num_reclock_ctdbd_latency:");
523                         printm("min_reclock_ctdbd_latency:");
524                         printm("avg_reclock_ctdbd_latency:");
525                         printm("max_reclock_ctdbd_latency:");
526
527                         printm("num_reclock_recd_latency:");
528                         printm("min_reclock_recd_latency:");
529                         printm("avg_reclock_recd_latency:");
530                         printm("max_reclock_recd_latency:");
531
532                         printm("num_call_latency:");
533                         printm("min_call_latency:");
534                         printm("avg_call_latency:");
535                         printm("max_call_latency:");
536
537                         printm("num_lockwait_latency:");
538                         printm("min_lockwait_latency:");
539                         printm("avg_lockwait_latency:");
540                         printm("max_lockwait_latency:");
541
542                         printm("num_childwrite_latency:");
543                         printm("min_childwrite_latency:");
544                         printm("avg_childwrite_latency:");
545                         printm("max_childwrite_latency:");
546                         printm("\n");
547                 }
548                 printm("%d:", CTDB_PROTOCOL);
549                 printm("%d:", (int)s->statistics_current_time.tv_sec);
550                 printm("%d:", (int)s->statistics_start_time.tv_sec);
551                 for (i=0;i<ARRAY_SIZE(fields);i++) {
552                         printm("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
553                 }
554                 printm("%d:", s->reclock.ctdbd.num);
555                 printm("%.6f:", s->reclock.ctdbd.min);
556                 printm("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
557                 printm("%.6f:", s->reclock.ctdbd.max);
558
559                 printm("%d:", s->reclock.recd.num);
560                 printm("%.6f:", s->reclock.recd.min);
561                 printm("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
562                 printm("%.6f:", s->reclock.recd.max);
563
564                 printm("%d:", s->call_latency.num);
565                 printm("%.6f:", s->call_latency.min);
566                 printm("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
567                 printm("%.6f:", s->call_latency.max);
568
569                 printm("%d:", s->childwrite_latency.num);
570                 printm("%.6f:", s->childwrite_latency.min);
571                 printm("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
572                 printm("%.6f:", s->childwrite_latency.max);
573                 printm("\n");
574         } else {
575                 printf("CTDB version %u\n", CTDB_PROTOCOL);
576                 printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
577                 printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
578
579                 for (i=0;i<ARRAY_SIZE(fields);i++) {
580                         if (strchr(fields[i].name, '.')) {
581                                 preflen = strcspn(fields[i].name, ".")+1;
582                                 if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
583                                         prefix = fields[i].name;
584                                         printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
585                                 }
586                         } else {
587                                 preflen = 0;
588                         }
589                         printf(" %*s%-22s%*s%10u\n", 
590                                preflen?4:0, "",
591                                fields[i].name+preflen, 
592                                preflen?0:4, "",
593                                *(uint32_t *)(fields[i].offset+(uint8_t *)s));
594                 }
595                 printf(" hop_count_buckets:");
596                 for (i=0;i<MAX_COUNT_BUCKETS;i++) {
597                         printf(" %d", s->hop_count_bucket[i]);
598                 }
599                 printf("\n");
600                 printf(" lock_buckets:");
601                 for (i=0; i<MAX_COUNT_BUCKETS; i++) {
602                         printf(" %d", s->locks.buckets[i]);
603                 }
604                 printf("\n");
605                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "locks_latency      MIN/AVG/MAX", s->locks.latency.min, s->locks.latency.num?s->locks.latency.total/s->locks.latency.num:0.0, s->locks.latency.max, s->locks.latency.num);
606
607                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd      MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
608
609                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
610
611                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
612                 printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
613         }
614
615         talloc_free(tmp_ctx);
616 }
617
618 /*
619   display remote ctdb statistics combined from all nodes
620  */
621 static int control_statistics_all(struct ctdb_context *ctdb)
622 {
623         int ret, i;
624         struct ctdb_statistics statistics;
625         uint32_t *nodes;
626         uint32_t num_nodes;
627
628         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
629         CTDB_NO_MEMORY(ctdb, nodes);
630         
631         ZERO_STRUCT(statistics);
632
633         for (i=0;i<num_nodes;i++) {
634                 struct ctdb_statistics s1;
635                 int j;
636                 uint32_t *v1 = (uint32_t *)&s1;
637                 uint32_t *v2 = (uint32_t *)&statistics;
638                 uint32_t num_ints = 
639                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
640                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
641                 if (ret != 0) {
642                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
643                         return ret;
644                 }
645                 for (j=0;j<num_ints;j++) {
646                         v2[j] += v1[j];
647                 }
648                 statistics.max_hop_count = 
649                         MAX(statistics.max_hop_count, s1.max_hop_count);
650                 statistics.call_latency.max = 
651                         MAX(statistics.call_latency.max, s1.call_latency.max);
652         }
653         talloc_free(nodes);
654         printf("Gathered statistics for %u nodes\n", num_nodes);
655         show_statistics(&statistics, 1);
656         return 0;
657 }
658
659 /*
660   display remote ctdb statistics
661  */
662 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
663 {
664         int ret;
665         struct ctdb_statistics statistics;
666
667         if (options.pnn == CTDB_BROADCAST_ALL) {
668                 return control_statistics_all(ctdb);
669         }
670
671         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
672         if (ret != 0) {
673                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
674                 return ret;
675         }
676         show_statistics(&statistics, 1);
677         return 0;
678 }
679
680
681 /*
682   reset remote ctdb statistics
683  */
684 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
685 {
686         int ret;
687
688         ret = ctdb_statistics_reset(ctdb, options.pnn);
689         if (ret != 0) {
690                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
691                 return ret;
692         }
693         return 0;
694 }
695
696
697 /*
698   display remote ctdb rolling statistics
699  */
700 static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
701 {
702         int ret;
703         struct ctdb_statistics_list_old *stats;
704         int i, num_records = -1;
705
706         assert_single_node_only();
707
708         if (argc ==1) {
709                 num_records = atoi(argv[0]) - 1;
710         }
711
712         ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
713         if (ret != 0) {
714                 DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
715                 return ret;
716         }
717         for (i=0;i<stats->num;i++) {
718                 if (stats->stats[i].statistics_start_time.tv_sec == 0) {
719                         continue;
720                 }
721                 show_statistics(&stats->stats[i], i==0);
722                 if (i == num_records) {
723                         break;
724                 }
725         }
726         return 0;
727 }
728
729
730 /*
731   display remote ctdb db statistics
732  */
733 static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
734 {
735         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
736         struct ctdb_db_statistics_old *dbstat;
737         int i;
738         uint32_t db_id;
739         int num_hot_keys;
740         int ret;
741
742         if (argc < 1) {
743                 usage();
744         }
745
746         if (!db_exists(ctdb, argv[0], &db_id, NULL, NULL)) {
747                 return -1;
748         }
749
750         ret = ctdb_ctrl_dbstatistics(ctdb, options.pnn, db_id, tmp_ctx, &dbstat);
751         if (ret != 0) {
752                 DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
753                 talloc_free(tmp_ctx);
754                 return -1;
755         }
756
757         printf("DB Statistics: %s\n", argv[0]);
758         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_delegations", 4, "",
759                 dbstat->db_ro_delegations);
760         printf(" %*s%-22s%*s%10u\n", 0, "", "ro_revokes", 4, "",
761                 dbstat->db_ro_delegations);
762         printf(" %s\n", "locks");
763         printf(" %*s%-22s%*s%10u\n", 4, "", "total", 0, "",
764                 dbstat->locks.num_calls);
765         printf(" %*s%-22s%*s%10u\n", 4, "", "failed", 0, "",
766                 dbstat->locks.num_failed);
767         printf(" %*s%-22s%*s%10u\n", 4, "", "current", 0, "",
768                 dbstat->locks.num_current);
769         printf(" %*s%-22s%*s%10u\n", 4, "", "pending", 0, "",
770                 dbstat->locks.num_pending);
771         printf(" %s", "hop_count_buckets:");
772         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
773                 printf(" %d", dbstat->hop_count_bucket[i]);
774         }
775         printf("\n");
776         printf(" %s", "lock_buckets:");
777         for (i=0; i<MAX_COUNT_BUCKETS; i++) {
778                 printf(" %d", dbstat->locks.buckets[i]);
779         }
780         printf("\n");
781         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
782                 "locks_latency      MIN/AVG/MAX",
783                 dbstat->locks.latency.min,
784                 (dbstat->locks.latency.num ?
785                  dbstat->locks.latency.total /dbstat->locks.latency.num :
786                  0.0),
787                 dbstat->locks.latency.max,
788                 dbstat->locks.latency.num);
789         printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n",
790                 "vacuum_latency     MIN/AVG/MAX",
791                 dbstat->vacuum.latency.min,
792                 (dbstat->vacuum.latency.num ?
793                  dbstat->vacuum.latency.total /dbstat->vacuum.latency.num :
794                  0.0),
795                 dbstat->vacuum.latency.max,
796                 dbstat->vacuum.latency.num);
797         num_hot_keys = 0;
798         for (i=0; i<dbstat->num_hot_keys; i++) {
799                 if (dbstat->hot_keys[i].count > 0) {
800                         num_hot_keys++;
801                 }
802         }
803         dbstat->num_hot_keys = num_hot_keys;
804
805         printf(" Num Hot Keys:     %d\n", dbstat->num_hot_keys);
806         for (i = 0; i < dbstat->num_hot_keys; i++) {
807                 int j;
808                 printf("     Count:%d Key:", dbstat->hot_keys[i].count);
809                 for (j = 0; j < dbstat->hot_keys[i].key.dsize; j++) {
810                         printf("%02x", dbstat->hot_keys[i].key.dptr[j]&0xff);
811                 }
812                 printf("\n");
813         }
814
815         talloc_free(tmp_ctx);
816         return 0;
817 }
818
819 /*
820   display uptime of remote node
821  */
822 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
823 {
824         int ret;
825         struct ctdb_uptime *uptime = NULL;
826         int tmp, days, hours, minutes, seconds;
827
828         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
829         if (ret != 0) {
830                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
831                 return ret;
832         }
833
834         if (options.machinereadable){
835                 printm(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
836                 printm(":%u:%u:%u:%lf\n",
837                         (unsigned int)uptime->current_time.tv_sec,
838                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
839                         (unsigned int)uptime->last_recovery_finished.tv_sec,
840                         timeval_delta(&uptime->last_recovery_finished,
841                                       &uptime->last_recovery_started)
842                 );
843                 return 0;
844         }
845
846         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
847
848         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
849         seconds = tmp%60;
850         tmp    /= 60;
851         minutes = tmp%60;
852         tmp    /= 60;
853         hours   = tmp%24;
854         tmp    /= 24;
855         days    = tmp;
856         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
857
858         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
859         seconds = tmp%60;
860         tmp    /= 60;
861         minutes = tmp%60;
862         tmp    /= 60;
863         hours   = tmp%24;
864         tmp    /= 24;
865         days    = tmp;
866         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
867         
868         printf("Duration of last recovery/failover: %lf seconds\n",
869                 timeval_delta(&uptime->last_recovery_finished,
870                               &uptime->last_recovery_started));
871
872         return 0;
873 }
874
875 /*
876   show the PNN of the current node
877  */
878 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
879 {
880         uint32_t mypnn;
881
882         mypnn = getpnn(ctdb);
883
884         printf("PNN:%d\n", mypnn);
885         return 0;
886 }
887
888
889 static struct ctdb_node_map_old *read_nodes_file(TALLOC_CTX *mem_ctx)
890 {
891         const char *nodes_list;
892
893         /* read the nodes file */
894         nodes_list = getenv("CTDB_NODES");
895         if (nodes_list == NULL) {
896                 nodes_list = talloc_asprintf(mem_ctx, "%s/nodes",
897                                              getenv("CTDB_BASE"));
898                 if (nodes_list == NULL) {
899                         DEBUG(DEBUG_ALERT,(__location__ " Out of memory\n"));
900                         exit(1);
901                 }
902         }
903
904         return ctdb_read_nodes_file(mem_ctx, nodes_list);
905 }
906
907 /*
908   show the PNN of the current node
909   discover the pnn by loading the nodes file and try to bind to all
910   addresses one at a time until the ip address is found.
911  */
912 static int find_node_xpnn(void)
913 {
914         TALLOC_CTX *mem_ctx = talloc_new(NULL);
915         struct ctdb_node_map_old *node_map;
916         int i, pnn;
917
918         node_map = read_nodes_file(mem_ctx);
919         if (node_map == NULL) {
920                 talloc_free(mem_ctx);
921                 return -1;
922         }
923
924         for (i = 0; i < node_map->num; i++) {
925                 if (node_map->nodes[i].flags & NODE_FLAGS_DELETED) {
926                         continue;
927                 }
928                 if (ctdb_sys_have_ip(&node_map->nodes[i].addr)) {
929                         pnn = node_map->nodes[i].pnn;
930                         talloc_free(mem_ctx);
931                         return pnn;
932                 }
933         }
934
935         printf("Failed to detect which PNN this node is\n");
936         talloc_free(mem_ctx);
937         return -1;
938 }
939
940 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
941 {
942         uint32_t pnn;
943
944         assert_single_node_only();
945
946         pnn = find_node_xpnn();
947         if (pnn == -1) {
948                 return -1;
949         }
950
951         printf("PNN:%d\n", pnn);
952         return 0;
953 }
954
955 /* Helpers for ctdb status
956  */
957 static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_flags *node)
958 {
959         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
960         int j;
961         bool ret = false;
962
963         if (node->flags == 0) {
964                 struct ctdb_iface_list_old *ifaces;
965
966                 if (ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), node->pnn,
967                                          tmp_ctx, &ifaces) == 0) {
968                         for (j=0; j < ifaces->num; j++) {
969                                 if (ifaces->ifaces[j].link_state != 0) {
970                                         continue;
971                                 }
972                                 ret = true;
973                                 break;
974                         }
975                 }
976         }
977         talloc_free(tmp_ctx);
978
979         return ret;
980 }
981
982 static void control_status_header_machine(void)
983 {
984         printm(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
985                ":Inactive:PartiallyOnline:ThisNode:\n");
986 }
987
988 static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn,
989                                     struct ctdb_node_and_flags *node)
990 {
991         printm(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn,
992                ctdb_addr_to_str(&node->addr),
993                !!(node->flags&NODE_FLAGS_DISCONNECTED),
994                !!(node->flags&NODE_FLAGS_BANNED),
995                !!(node->flags&NODE_FLAGS_PERMANENTLY_DISABLED),
996                !!(node->flags&NODE_FLAGS_UNHEALTHY),
997                !!(node->flags&NODE_FLAGS_STOPPED),
998                !!(node->flags&NODE_FLAGS_INACTIVE),
999                is_partially_online(ctdb, node) ? 1 : 0,
1000                (node->pnn == mypnn)?'Y':'N');
1001
1002         return node->flags;
1003 }
1004
1005 static int control_status_1_human(struct ctdb_context *ctdb, int mypnn,
1006                                   struct ctdb_node_and_flags *node)
1007 {
1008        printf("pnn:%d %-16s %s%s\n", node->pnn,
1009               ctdb_addr_to_str(&node->addr),
1010               is_partially_online(ctdb, node) ? "PARTIALLYONLINE" : pretty_print_flags(node->flags),
1011               node->pnn == mypnn?" (THIS NODE)":"");
1012
1013        return node->flags;
1014 }
1015
1016 /*
1017   display remote ctdb status
1018  */
1019 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
1020 {
1021         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1022         int i;
1023         struct ctdb_vnn_map *vnnmap=NULL;
1024         struct ctdb_node_map_old *nodemap=NULL;
1025         uint32_t recmode, recmaster, mypnn;
1026         int num_deleted_nodes = 0;
1027         int ret;
1028
1029         mypnn = getpnn(ctdb);
1030
1031         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1032         if (ret != 0) {
1033                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1034                 talloc_free(tmp_ctx);
1035                 return -1;
1036         }
1037
1038         if (options.machinereadable) {
1039                 control_status_header_machine();
1040                 for (i=0;i<nodemap->num;i++) {
1041                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1042                                 continue;
1043                         }
1044                         (void) control_status_1_machine(ctdb, mypnn,
1045                                                         &nodemap->nodes[i]);
1046                 }
1047                 talloc_free(tmp_ctx);
1048                 return 0;
1049         }
1050
1051         for (i=0; i<nodemap->num; i++) {
1052                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1053                         num_deleted_nodes++;
1054                 }
1055         }
1056         if (num_deleted_nodes == 0) {
1057                 printf("Number of nodes:%d\n", nodemap->num);
1058         } else {
1059                 printf("Number of nodes:%d (including %d deleted nodes)\n",
1060                        nodemap->num, num_deleted_nodes);
1061         }
1062         for(i=0;i<nodemap->num;i++){
1063                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1064                         continue;
1065                 }
1066                 (void) control_status_1_human(ctdb, mypnn, &nodemap->nodes[i]);
1067         }
1068
1069         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
1070         if (ret != 0) {
1071                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
1072                 talloc_free(tmp_ctx);
1073                 return -1;
1074         }
1075         if (vnnmap->generation == INVALID_GENERATION) {
1076                 printf("Generation:INVALID\n");
1077         } else {
1078                 printf("Generation:%d\n",vnnmap->generation);
1079         }
1080         printf("Size:%d\n",vnnmap->size);
1081         for(i=0;i<vnnmap->size;i++){
1082                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
1083         }
1084
1085         ret = ctdb_ctrl_getrecmode(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmode);
1086         if (ret != 0) {
1087                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
1088                 talloc_free(tmp_ctx);
1089                 return -1;
1090         }
1091         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
1092
1093         ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, TIMELIMIT(), options.pnn, &recmaster);
1094         if (ret != 0) {
1095                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1096                 talloc_free(tmp_ctx);
1097                 return -1;
1098         }
1099         printf("Recovery master:%d\n",recmaster);
1100
1101         talloc_free(tmp_ctx);
1102         return 0;
1103 }
1104
1105 static int control_nodestatus(struct ctdb_context *ctdb, int argc, const char **argv)
1106 {
1107         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1108         int i, ret;
1109         struct ctdb_node_map_old *nodemap=NULL;
1110         uint32_t * nodes;
1111         uint32_t pnn_mode, mypnn;
1112
1113         if (argc > 1) {
1114                 usage();
1115         }
1116
1117         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1118                               options.pnn, true, &nodes, &pnn_mode)) {
1119                 return -1;
1120         }
1121
1122         if (options.machinereadable) {
1123                 control_status_header_machine();
1124         } else if (pnn_mode == CTDB_BROADCAST_ALL) {
1125                 printf("Number of nodes:%d\n", (int) talloc_array_length(nodes));
1126         }
1127
1128         mypnn = getpnn(ctdb);
1129
1130         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1131         if (ret != 0) {
1132                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1133                 talloc_free(tmp_ctx);
1134                 return -1;
1135         }
1136
1137         ret = 0;
1138
1139         for (i = 0; i < talloc_array_length(nodes); i++) {
1140                 if (options.machinereadable) {
1141                         ret |= control_status_1_machine(ctdb, mypnn,
1142                                                         &nodemap->nodes[nodes[i]]);
1143                 } else {
1144                         ret |= control_status_1_human(ctdb, mypnn,
1145                                                       &nodemap->nodes[nodes[i]]);
1146                 }
1147         }
1148
1149         talloc_free(tmp_ctx);
1150         return ret;
1151 }
1152
1153 /* talloc off the existing nodemap... */
1154 static struct ctdb_node_map_old *talloc_nodemap(struct ctdb_node_map_old *nodemap)
1155 {
1156         return talloc_zero_size(nodemap,
1157                                 offsetof(struct ctdb_node_map_old, nodes) +
1158                                 nodemap->num * sizeof(struct ctdb_node_and_flags));
1159 }
1160
1161 static struct ctdb_node_map_old *
1162 filter_nodemap_by_capabilities(struct ctdb_context *ctdb,
1163                                struct ctdb_node_map_old *nodemap,
1164                                uint32_t required_capabilities,
1165                                bool first_only)
1166 {
1167         int i;
1168         uint32_t capabilities;
1169         struct ctdb_node_map_old *ret;
1170
1171         ret = talloc_nodemap(nodemap);
1172         CTDB_NO_MEMORY_NULL(ctdb, ret);
1173
1174         ret->num = 0;
1175
1176         for (i = 0; i < nodemap->num; i++) {
1177                 int res;
1178
1179                 /* Disconnected nodes have no capabilities! */
1180                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1181                         continue;
1182                 }
1183
1184                 res = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(),
1185                                                 nodemap->nodes[i].pnn,
1186                                                 &capabilities);
1187                 if (res != 0) {
1188                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n",
1189                                           nodemap->nodes[i].pnn));
1190                         talloc_free(ret);
1191                         return NULL;
1192                 }
1193                 if (!(capabilities & required_capabilities)) {
1194                         continue;
1195                 }
1196
1197                 ret->nodes[ret->num] = nodemap->nodes[i];
1198                 ret->num++;
1199                 if (first_only) {
1200                         break;
1201                 }
1202         }
1203
1204         return ret;
1205 }
1206
1207 static struct ctdb_node_map_old *
1208 filter_nodemap_by_flags(struct ctdb_context *ctdb,
1209                         struct ctdb_node_map_old *nodemap,
1210                         uint32_t flags_mask)
1211 {
1212         int i;
1213         struct ctdb_node_map_old *ret;
1214
1215         ret = talloc_nodemap(nodemap);
1216         CTDB_NO_MEMORY_NULL(ctdb, ret);
1217
1218         ret->num = 0;
1219
1220         for (i = 0; i < nodemap->num; i++) {
1221                 if (nodemap->nodes[i].flags & flags_mask) {
1222                         continue;
1223                 }
1224
1225                 ret->nodes[ret->num] = nodemap->nodes[i];
1226                 ret->num++;
1227         }
1228
1229         return ret;
1230 }
1231
1232 /*
1233   display the list of nodes belonging to this natgw configuration
1234  */
1235 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
1236 {
1237         static char prog[PATH_MAX+1] = "";
1238
1239         if (argc != 0) {
1240                 usage();
1241         }
1242
1243         if (!ctdb_set_helper("NAT gateway helper", prog, sizeof(prog),
1244                              "CTDB_NATGW_HELPER", CTDB_HELPER_BINDIR,
1245                              "ctdb_natgw")) {
1246                 DEBUG(DEBUG_ERR, ("Unable to set NAT gateway helper\n"));
1247                 exit(1);
1248         }
1249
1250         execl(prog, prog, "natgwlist", NULL);
1251
1252         DEBUG(DEBUG_ERR,
1253               ("Unable to run NAT gateway helper %s\n", strerror(errno)));
1254         exit(1);
1255 }
1256
1257 /*
1258   display the status of the scripts for monitoring (or other events)
1259  */
1260 static int control_one_scriptstatus(struct ctdb_context *ctdb,
1261                                     enum ctdb_event type)
1262 {
1263         struct ctdb_script_list_old *script_status;
1264         int ret, i;
1265
1266         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
1267         if (ret != 0) {
1268                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
1269                 return ret;
1270         }
1271
1272         if (script_status == NULL) {
1273                 if (!options.machinereadable) {
1274                         printf("%s cycle never run\n",
1275                                ctdb_eventscript_call_names[type]);
1276                 }
1277                 return 0;
1278         }
1279
1280         if (!options.machinereadable) {
1281                 int num_run = 0;
1282                 for (i=0; i<script_status->num_scripts; i++) {
1283                         if (script_status->scripts[i].status != -ENOEXEC) {
1284                                 num_run++;
1285                         }
1286                 }
1287                 printf("%d scripts were executed last %s cycle\n",
1288                        num_run,
1289                        ctdb_eventscript_call_names[type]);
1290         }
1291         for (i=0; i<script_status->num_scripts; i++) {
1292                 const char *status = NULL;
1293
1294                 /* The ETIME status is ignored for certain events.
1295                  * In that case the status is 0, but endtime is not set.
1296                  */
1297                 if (script_status->scripts[i].status == 0 &&
1298                     timeval_is_zero(&script_status->scripts[i].finished)) {
1299                         script_status->scripts[i].status = -ETIME;
1300                 }
1301
1302                 switch (script_status->scripts[i].status) {
1303                 case -ETIME:
1304                         status = "TIMEDOUT";
1305                         break;
1306                 case -ENOEXEC:
1307                         status = "DISABLED";
1308                         break;
1309                 case 0:
1310                         status = "OK";
1311                         break;
1312                 default:
1313                         if (script_status->scripts[i].status > 0)
1314                                 status = "ERROR";
1315                         break;
1316                 }
1317                 if (options.machinereadable) {
1318                         printm(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
1319                                ctdb_eventscript_call_names[type],
1320                                script_status->scripts[i].name,
1321                                script_status->scripts[i].status,
1322                                status,
1323                                (long)script_status->scripts[i].start.tv_sec,
1324                                (long)script_status->scripts[i].start.tv_usec,
1325                                (long)script_status->scripts[i].finished.tv_sec,
1326                                (long)script_status->scripts[i].finished.tv_usec,
1327                                script_status->scripts[i].output);
1328                         continue;
1329                 }
1330                 if (status)
1331                         printf("%-20s Status:%s    ",
1332                                script_status->scripts[i].name, status);
1333                 else
1334                         /* Some other error, eg from stat. */
1335                         printf("%-20s Status:CANNOT RUN (%s)",
1336                                script_status->scripts[i].name,
1337                                strerror(-script_status->scripts[i].status));
1338
1339                 if (script_status->scripts[i].status >= 0) {
1340                         printf("Duration:%.3lf ",
1341                         timeval_delta(&script_status->scripts[i].finished,
1342                               &script_status->scripts[i].start));
1343                 }
1344                 if (script_status->scripts[i].status != -ENOEXEC) {
1345                         printf("%s",
1346                                ctime(&script_status->scripts[i].start.tv_sec));
1347                         if (script_status->scripts[i].status != 0) {
1348                                 printf("   OUTPUT:%s\n",
1349                                        script_status->scripts[i].output);
1350                         }
1351                 } else {
1352                         printf("\n");
1353                 }
1354         }
1355         return 0;
1356 }
1357
1358
1359 static int control_scriptstatus(struct ctdb_context *ctdb,
1360                                 int argc, const char **argv)
1361 {
1362         int ret;
1363         enum ctdb_event type, min, max;
1364         const char *arg;
1365
1366         if (argc > 1) {
1367                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
1368                 return -1;
1369         }
1370
1371         if (argc == 0)
1372                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
1373         else
1374                 arg = argv[0];
1375
1376         for (type = 0; type < CTDB_EVENT_MAX; type++) {
1377                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
1378                         min = type;
1379                         max = type+1;
1380                         break;
1381                 }
1382         }
1383         if (type == CTDB_EVENT_MAX) {
1384                 if (strcmp(arg, "all") == 0) {
1385                         min = 0;
1386                         max = CTDB_EVENT_MAX;
1387                 } else {
1388                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
1389                         return -1;
1390                 }
1391         }
1392
1393         if (options.machinereadable) {
1394                 printm(":Type:Name:Code:Status:Start:End:Error Output...:\n");
1395         }
1396
1397         for (type = min; type < max; type++) {
1398                 ret = control_one_scriptstatus(ctdb, type);
1399                 if (ret != 0) {
1400                         return ret;
1401                 }
1402         }
1403
1404         return 0;
1405 }
1406
1407 /*
1408   enable an eventscript
1409  */
1410 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1411 {
1412         int ret;
1413
1414         if (argc < 1) {
1415                 usage();
1416         }
1417
1418         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1419         if (ret != 0) {
1420           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
1421                 return ret;
1422         }
1423
1424         return 0;
1425 }
1426
1427 /*
1428   disable an eventscript
1429  */
1430 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
1431 {
1432         int ret;
1433
1434         if (argc < 1) {
1435                 usage();
1436         }
1437
1438         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
1439         if (ret != 0) {
1440           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
1441                 return ret;
1442         }
1443
1444         return 0;
1445 }
1446
1447 /*
1448   display the pnn of the recovery master
1449  */
1450 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
1451 {
1452         uint32_t recmaster;
1453         int ret;
1454
1455         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
1456         if (ret != 0) {
1457                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
1458                 return -1;
1459         }
1460         printf("%d\n",recmaster);
1461
1462         return 0;
1463 }
1464
1465 /*
1466   add a tickle to a public address
1467  */
1468 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1469 {
1470         struct ctdb_connection t;
1471         TDB_DATA data;
1472         int ret;
1473
1474         assert_single_node_only();
1475
1476         if (argc < 2) {
1477                 usage();
1478         }
1479
1480         if (parse_ip_port(argv[0], &t.src) == 0) {
1481                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1482                 return -1;
1483         }
1484         if (parse_ip_port(argv[1], &t.dst) == 0) {
1485                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1486                 return -1;
1487         }
1488
1489         data.dptr = (uint8_t *)&t;
1490         data.dsize = sizeof(t);
1491
1492         /* tell all nodes about this tcp connection */
1493         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1494                            0, data, ctdb, NULL, NULL, NULL, NULL);
1495         if (ret != 0) {
1496                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1497                 return -1;
1498         }
1499         
1500         return 0;
1501 }
1502
1503
1504 /*
1505   delete a tickle from a node
1506  */
1507 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1508 {
1509         struct ctdb_connection t;
1510         TDB_DATA data;
1511         int ret;
1512
1513         assert_single_node_only();
1514
1515         if (argc < 2) {
1516                 usage();
1517         }
1518
1519         if (parse_ip_port(argv[0], &t.src) == 0) {
1520                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1521                 return -1;
1522         }
1523         if (parse_ip_port(argv[1], &t.dst) == 0) {
1524                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1525                 return -1;
1526         }
1527
1528         data.dptr = (uint8_t *)&t;
1529         data.dsize = sizeof(t);
1530
1531         /* tell all nodes about this tcp connection */
1532         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1533                            0, data, ctdb, NULL, NULL, NULL, NULL);
1534         if (ret != 0) {
1535                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1536                 return -1;
1537         }
1538         
1539         return 0;
1540 }
1541
1542
1543 /*
1544   get a list of all tickles for this pnn
1545  */
1546 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1547 {
1548         struct ctdb_tickle_list_old *list;
1549         ctdb_sock_addr addr;
1550         int i, ret;
1551         unsigned port = 0;
1552
1553         assert_single_node_only();
1554
1555         if (argc < 1) {
1556                 usage();
1557         }
1558
1559         if (argc == 2) {
1560                 port = atoi(argv[1]);
1561         }
1562
1563         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1564                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1565                 return -1;
1566         }
1567
1568         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1569         if (ret == -1) {
1570                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1571                 return -1;
1572         }
1573
1574         if (options.machinereadable){
1575                 printm(":source ip:port:destination ip:port:\n");
1576                 for (i=0;i<list->num;i++) {
1577                         if (port && port != ntohs(list->connections[i].dst.ip.sin_port)) {
1578                                 continue;
1579                         }
1580                         printm(":%s:%u", ctdb_addr_to_str(&list->connections[i].src), ntohs(list->connections[i].src.ip.sin_port));
1581                         printm(":%s:%u:\n", ctdb_addr_to_str(&list->connections[i].dst), ntohs(list->connections[i].dst.ip.sin_port));
1582                 }
1583         } else {
1584                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1585                 printf("Num tickles:%u\n", list->num);
1586                 for (i=0;i<list->num;i++) {
1587                         if (port && port != ntohs(list->connections[i].dst.ip.sin_port)) {
1588                                 continue;
1589                         }
1590                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->connections[i].src), ntohs(list->connections[i].src.ip.sin_port));
1591                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->connections[i].dst), ntohs(list->connections[i].dst.ip.sin_port));
1592                 }
1593         }
1594
1595         talloc_free(list);
1596         
1597         return 0;
1598 }
1599
1600
1601 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1602 {
1603         struct ctdb_public_ip_list_old *ips;
1604         struct ctdb_public_ip ip;
1605         int i, ret;
1606         uint32_t *nodes;
1607         uint32_t disable_time;
1608         TDB_DATA data;
1609         struct ctdb_node_map_old *nodemap=NULL;
1610         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1611
1612         disable_time = 30;
1613         data.dptr  = (uint8_t*)&disable_time;
1614         data.dsize = sizeof(disable_time);
1615         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1616         if (ret != 0) {
1617                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1618                 return -1;
1619         }
1620
1621
1622
1623         /* read the public ip list from the node */
1624         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1625         if (ret != 0) {
1626                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1627                 talloc_free(tmp_ctx);
1628                 return -1;
1629         }
1630
1631         for (i=0;i<ips->num;i++) {
1632                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1633                         break;
1634                 }
1635         }
1636         if (i==ips->num) {
1637                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1638                         pnn, ctdb_addr_to_str(addr)));
1639                 talloc_free(tmp_ctx);
1640                 return -1;
1641         }
1642
1643         ip.pnn  = pnn;
1644         ip.addr = *addr;
1645
1646         data.dptr  = (uint8_t *)&ip;
1647         data.dsize = sizeof(ip);
1648
1649         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1650         if (ret != 0) {
1651                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1652                 talloc_free(tmp_ctx);
1653                 return ret;
1654         }
1655
1656         nodes = list_of_nodes(ctdb, nodemap, tmp_ctx, NODE_FLAGS_INACTIVE, pnn);
1657         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1658                                         nodes, 0,
1659                                         LONGTIMELIMIT(),
1660                                         false, data,
1661                                         NULL, NULL,
1662                                         NULL);
1663         if (ret != 0) {
1664                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1665                 talloc_free(tmp_ctx);
1666                 return -1;
1667         }
1668
1669         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1670         if (ret != 0) {
1671                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1672                 talloc_free(tmp_ctx);
1673                 return -1;
1674         }
1675
1676         /* update the recovery daemon so it now knows to expect the new
1677            node assignment for this ip.
1678         */
1679         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1680         if (ret != 0) {
1681                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1682                 return -1;
1683         }
1684
1685         talloc_free(tmp_ctx);
1686         return 0;
1687 }
1688
1689
1690 /* 
1691  * scans all other nodes and returns a pnn for another node that can host this 
1692  * ip address or -1
1693  */
1694 static int
1695 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1696 {
1697         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1698         struct ctdb_public_ip_list_old *ips;
1699         struct ctdb_node_map_old *nodemap=NULL;
1700         int i, j, ret;
1701         int pnn;
1702
1703         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1704         if (ret != 0) {
1705                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1706                 talloc_free(tmp_ctx);
1707                 return ret;
1708         }
1709
1710         for(i=0;i<nodemap->num;i++){
1711                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1712                         continue;
1713                 }
1714                 if (nodemap->nodes[i].pnn == options.pnn) {
1715                         continue;
1716                 }
1717
1718                 /* read the public ip list from this node */
1719                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1720                 if (ret != 0) {
1721                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1722                         return -1;
1723                 }
1724
1725                 for (j=0;j<ips->num;j++) {
1726                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1727                                 pnn = nodemap->nodes[i].pnn;
1728                                 talloc_free(tmp_ctx);
1729                                 return pnn;
1730                         }
1731                 }
1732                 talloc_free(ips);
1733         }
1734
1735         talloc_free(tmp_ctx);
1736         return -1;
1737 }
1738
1739 /* If pnn is -1 then try to find a node to move IP to... */
1740 static bool try_moveip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1741 {
1742         bool pnn_specified = (pnn == -1 ? false : true);
1743         int retries = 0;
1744
1745         while (retries < 5) {
1746                 if (!pnn_specified) {
1747                         pnn = find_other_host_for_public_ip(ctdb, addr);
1748                         if (pnn == -1) {
1749                                 return false;
1750                         }
1751                         DEBUG(DEBUG_NOTICE,
1752                               ("Trying to move public IP to node %u\n", pnn));
1753                 }
1754
1755                 if (move_ip(ctdb, addr, pnn) == 0) {
1756                         return true;
1757                 }
1758
1759                 sleep(3);
1760                 retries++;
1761         }
1762
1763         return false;
1764 }
1765
1766
1767 /*
1768   move/failover an ip address to a specific node
1769  */
1770 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1771 {
1772         uint32_t pnn;
1773         ctdb_sock_addr addr;
1774
1775         assert_single_node_only();
1776
1777         if (argc < 2) {
1778                 usage();
1779                 return -1;
1780         }
1781
1782         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1783                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1784                 return -1;
1785         }
1786
1787
1788         if (sscanf(argv[1], "%u", &pnn) != 1) {
1789                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1790                 return -1;
1791         }
1792
1793         if (!try_moveip(ctdb, &addr, pnn)) {
1794                 DEBUG(DEBUG_ERR,("Failed to move IP to node %d.\n", pnn));
1795                 return -1;
1796         }
1797
1798         return 0;
1799 }
1800
1801 static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
1802 {
1803         TDB_DATA data;
1804
1805         data.dptr  = (uint8_t *)&pnn;
1806         data.dsize = sizeof(uint32_t);
1807         if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
1808                 DEBUG(DEBUG_ERR,
1809                       ("Failed to send message to force node %u to be a rebalancing target\n",
1810                        pnn));
1811                 return -1;
1812         }
1813
1814         return 0;
1815 }
1816
1817
1818 /*
1819   rebalance a node by setting it to allow failback and triggering a
1820   takeover run
1821  */
1822 static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
1823 {
1824         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1825         uint32_t *nodes;
1826         uint32_t pnn_mode;
1827         int i, ret;
1828
1829         assert_single_node_only();
1830
1831         if (argc > 1) {
1832                 usage();
1833         }
1834
1835         /* Determine the nodes where IPs need to be reloaded */
1836         if (!parse_nodestring(ctdb, tmp_ctx, argc == 1 ? argv[0] : NULL,
1837                               options.pnn, true, &nodes, &pnn_mode)) {
1838                 ret = -1;
1839                 goto done;
1840         }
1841
1842         for (i = 0; i < talloc_array_length(nodes); i++) {
1843                 if (!rebalance_node(ctdb, nodes[i])) {
1844                         ret = -1;
1845                 }
1846         }
1847
1848 done:
1849         talloc_free(tmp_ctx);
1850         return ret;
1851 }
1852
1853 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1854 {
1855         struct ctdb_public_ip ip;
1856         int ret;
1857         uint32_t *nodes;
1858         uint32_t disable_time;
1859         TDB_DATA data;
1860         struct ctdb_node_map_old *nodemap=NULL;
1861         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1862
1863         disable_time = 30;
1864         data.dptr  = (uint8_t*)&disable_time;
1865         data.dsize = sizeof(disable_time);
1866         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1867         if (ret != 0) {
1868                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1869                 return -1;
1870         }
1871
1872         ip.pnn  = -1;
1873         ip.addr = *addr;
1874
1875         data.dptr  = (uint8_t *)&ip;
1876         data.dsize = sizeof(ip);
1877
1878         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1879         if (ret != 0) {
1880                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1881                 talloc_free(tmp_ctx);
1882                 return ret;
1883         }
1884
1885         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
1886         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1887                                         nodes, 0,
1888                                         LONGTIMELIMIT(),
1889                                         false, data,
1890                                         NULL, NULL,
1891                                         NULL);
1892         if (ret != 0) {
1893                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1894                 talloc_free(tmp_ctx);
1895                 return -1;
1896         }
1897
1898         talloc_free(tmp_ctx);
1899         return 0;
1900 }
1901
1902 /*
1903   release an ip form all nodes and have it re-assigned by recd
1904  */
1905 static int control_rebalanceip(struct ctdb_context *ctdb, int argc, const char **argv)
1906 {
1907         ctdb_sock_addr addr;
1908
1909         assert_single_node_only();
1910
1911         if (argc < 1) {
1912                 usage();
1913                 return -1;
1914         }
1915
1916         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1917                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1918                 return -1;
1919         }
1920
1921         if (rebalance_ip(ctdb, &addr) != 0) {
1922                 DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
1923                 return -1;
1924         }
1925
1926         return 0;
1927 }
1928
1929 static int getips_store_callback(void *param, void *data)
1930 {
1931         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
1932         struct ctdb_public_ip_list_old *ips = param;
1933         int i;
1934
1935         i = ips->num++;
1936         ips->ips[i].pnn  = node_ip->pnn;
1937         ips->ips[i].addr = node_ip->addr;
1938         return 0;
1939 }
1940
1941 static int getips_count_callback(void *param, void *data)
1942 {
1943         uint32_t *count = param;
1944
1945         (*count)++;
1946         return 0;
1947 }
1948
1949 #define IP_KEYLEN       4
1950 static uint32_t *ip_key(ctdb_sock_addr *ip)
1951 {
1952         static uint32_t key[IP_KEYLEN];
1953
1954         bzero(key, sizeof(key));
1955
1956         switch (ip->sa.sa_family) {
1957         case AF_INET:
1958                 key[0]  = ip->ip.sin_addr.s_addr;
1959                 break;
1960         case AF_INET6: {
1961                 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
1962                 key[0]  = s6_a32[3];
1963                 key[1]  = s6_a32[2];
1964                 key[2]  = s6_a32[1];
1965                 key[3]  = s6_a32[0];
1966                 break;
1967         }
1968         default:
1969                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
1970                 return key;
1971         }
1972
1973         return key;
1974 }
1975
1976 static void *add_ip_callback(void *parm, void *data)
1977 {
1978         return parm;
1979 }
1980
1981 static int
1982 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_public_ip_list_old **ips)
1983 {
1984         struct ctdb_public_ip_list_old *tmp_ips;
1985         struct ctdb_node_map_old *nodemap=NULL;
1986         trbt_tree_t *ip_tree;
1987         int i, j, len, ret;
1988         uint32_t count;
1989
1990         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1991         if (ret != 0) {
1992                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1993                 return ret;
1994         }
1995
1996         ip_tree = trbt_create(tmp_ctx, 0);
1997
1998         for(i=0;i<nodemap->num;i++){
1999                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
2000                         continue;
2001                 }
2002                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2003                         continue;
2004                 }
2005
2006                 /* read the public ip list from this node */
2007                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
2008                 if (ret != 0) {
2009                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
2010                         return -1;
2011                 }
2012         
2013                 for (j=0; j<tmp_ips->num;j++) {
2014                         struct ctdb_public_ip *node_ip;
2015
2016                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
2017                         node_ip->pnn  = tmp_ips->ips[j].pnn;
2018                         node_ip->addr = tmp_ips->ips[j].addr;
2019
2020                         trbt_insertarray32_callback(ip_tree,
2021                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
2022                                 add_ip_callback,
2023                                 node_ip);
2024                 }
2025                 talloc_free(tmp_ips);
2026         }
2027
2028         /* traverse */
2029         count = 0;
2030         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
2031
2032         len = offsetof(struct ctdb_public_ip_list_old, ips) +
2033                 count*sizeof(struct ctdb_public_ip);
2034         tmp_ips = talloc_zero_size(tmp_ctx, len);
2035         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
2036
2037         *ips = tmp_ips;
2038
2039         return 0;
2040 }
2041
2042
2043 static void ctdb_every_second(struct tevent_context *ev,
2044                               struct tevent_timer *te,
2045                               struct timeval t, void *p)
2046 {
2047         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2048
2049         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2050                          ctdb_every_second, ctdb);
2051 }
2052
2053 struct srvid_reply_handler_data {
2054         bool done;
2055         bool wait_for_all;
2056         uint32_t *nodes;
2057         const char *srvid_str;
2058 };
2059
2060 static void srvid_broadcast_reply_handler(uint64_t srvid, TDB_DATA data,
2061                                          void *private_data)
2062 {
2063         struct srvid_reply_handler_data *d =
2064                 (struct srvid_reply_handler_data *)private_data;
2065         int i;
2066         int32_t ret;
2067
2068         if (data.dsize != sizeof(ret)) {
2069                 DEBUG(DEBUG_ERR, (__location__ " Wrong reply size\n"));
2070                 return;
2071         }
2072
2073         /* ret will be a PNN (i.e. >=0) on success, or negative on error */
2074         ret = *(int32_t *)data.dptr;
2075         if (ret < 0) {
2076                 DEBUG(DEBUG_ERR,
2077                       ("%s failed with result %d\n", d->srvid_str, ret));
2078                 return;
2079         }
2080
2081         if (!d->wait_for_all) {
2082                 d->done = true;
2083                 return;
2084         }
2085
2086         /* Wait for all replies */
2087         d->done = true;
2088         for (i = 0; i < talloc_array_length(d->nodes); i++) {
2089                 if (d->nodes[i] == ret) {
2090                         DEBUG(DEBUG_DEBUG,
2091                               ("%s reply received from node %u\n",
2092                                d->srvid_str, ret));
2093                         d->nodes[i] = -1;
2094                 }
2095                 if (d->nodes[i] != -1) {
2096                         /* Found a node that hasn't yet replied */
2097                         d->done = false;
2098                 }
2099         }
2100 }
2101
2102 /* Broadcast the given SRVID to all connected nodes.  Wait for 1 reply
2103  * or replies from all connected nodes.  arg is the data argument to
2104  * pass in the srvid_request structure - pass 0 if this isn't needed.
2105  */
2106 static int srvid_broadcast(struct ctdb_context *ctdb,
2107                            uint64_t srvid, uint32_t *arg,
2108                            const char *srvid_str, bool wait_for_all)
2109 {
2110         int ret;
2111         TDB_DATA data;
2112         uint32_t pnn;
2113         uint64_t reply_srvid;
2114         struct ctdb_srvid_message request;
2115         struct ctdb_disable_message request_data;
2116         struct srvid_reply_handler_data reply_data;
2117         struct timeval tv;
2118
2119         ZERO_STRUCT(request);
2120
2121         /* Time ticks to enable timeouts to be processed */
2122         tevent_add_timer(ctdb->ev, ctdb, timeval_current_ofs(1, 0),
2123                          ctdb_every_second, ctdb);
2124
2125         pnn = ctdb_get_pnn(ctdb);
2126         reply_srvid = getpid();
2127
2128         if (arg == NULL) {
2129                 request.pnn = pnn;
2130                 request.srvid = reply_srvid;
2131
2132                 data.dptr = (uint8_t *)&request;
2133                 data.dsize = sizeof(request);
2134         } else {
2135                 request_data.pnn = pnn;
2136                 request_data.srvid = reply_srvid;
2137                 request_data.timeout = *arg;
2138
2139                 data.dptr = (uint8_t *)&request_data;
2140                 data.dsize = sizeof(request_data);
2141         }
2142
2143         /* Register message port for reply from recovery master */
2144         ctdb_client_set_message_handler(ctdb, reply_srvid,
2145                                         srvid_broadcast_reply_handler,
2146                                         &reply_data);
2147
2148         reply_data.wait_for_all = wait_for_all;
2149         reply_data.nodes = NULL;
2150         reply_data.srvid_str = srvid_str;
2151
2152 again:
2153         reply_data.done = false;
2154
2155         if (wait_for_all) {
2156                 struct ctdb_node_map_old *nodemap;
2157
2158                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(),
2159                                            CTDB_CURRENT_NODE, ctdb, &nodemap);
2160                 if (ret != 0) {
2161                         DEBUG(DEBUG_ERR,
2162                               ("Unable to get nodemap from current node, try again\n"));
2163                         sleep(1);
2164                         goto again;
2165                 }
2166
2167                 if (reply_data.nodes != NULL) {
2168                         talloc_free(reply_data.nodes);
2169                 }
2170                 reply_data.nodes = list_of_connected_nodes(ctdb, nodemap,
2171                                                            NULL, true);
2172
2173                 talloc_free(nodemap);
2174         }
2175
2176         /* Send to all connected nodes. Only recmaster replies */
2177         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2178                                        srvid, data);
2179         if (ret != 0) {
2180                 /* This can only happen if the socket is closed and
2181                  * there's no way to recover from that, so don't try
2182                  * again.
2183                  */
2184                 DEBUG(DEBUG_ERR,
2185                       ("Failed to send %s request to connected nodes\n",
2186                        srvid_str));
2187                 return -1;
2188         }
2189
2190         tv = timeval_current();
2191         /* This loop terminates the reply is received */
2192         while (timeval_elapsed(&tv) < 5.0 && !reply_data.done) {
2193                 tevent_loop_once(ctdb->ev);
2194         }
2195
2196         if (!reply_data.done) {
2197                 DEBUG(DEBUG_NOTICE,
2198                       ("Still waiting for confirmation of %s\n", srvid_str));
2199                 sleep(1);
2200                 goto again;
2201         }
2202
2203         ctdb_client_remove_message_handler(ctdb, reply_srvid, &reply_data);
2204
2205         talloc_free(reply_data.nodes);
2206
2207         return 0;
2208 }
2209
2210 static int ipreallocate(struct ctdb_context *ctdb)
2211 {
2212         return srvid_broadcast(ctdb, CTDB_SRVID_TAKEOVER_RUN, NULL,
2213                                "IP reallocation", false);
2214 }
2215
2216
2217 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2218 {
2219         return ipreallocate(ctdb);
2220 }
2221
2222 /*
2223   add a public ip address to a node
2224  */
2225 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
2226 {
2227         int i, ret;
2228         int len, retries = 0;
2229         unsigned mask;
2230         ctdb_sock_addr addr;
2231         struct ctdb_addr_info_old *pub;
2232         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2233         struct ctdb_public_ip_list_old *ips;
2234
2235
2236         if (argc != 2) {
2237                 talloc_free(tmp_ctx);
2238                 usage();
2239         }
2240
2241         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
2242                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
2243                 talloc_free(tmp_ctx);
2244                 return -1;
2245         }
2246
2247         /* read the public ip list from the node */
2248         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2249         if (ret != 0) {
2250                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", options.pnn));
2251                 talloc_free(tmp_ctx);
2252                 return -1;
2253         }
2254         for (i=0;i<ips->num;i++) {
2255                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2256                         DEBUG(DEBUG_ERR,("Can not add ip to node. Node already hosts this ip\n"));
2257                         return 0;
2258                 }
2259         }
2260
2261
2262
2263         /* Dont timeout. This command waits for an ip reallocation
2264            which sometimes can take wuite a while if there has
2265            been a recent recovery
2266         */
2267         alarm(0);
2268
2269         len = offsetof(struct ctdb_addr_info_old, iface) + strlen(argv[1]) + 1;
2270         pub = talloc_size(tmp_ctx, len); 
2271         CTDB_NO_MEMORY(ctdb, pub);
2272
2273         pub->addr  = addr;
2274         pub->mask  = mask;
2275         pub->len   = strlen(argv[1])+1;
2276         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
2277
2278         do {
2279                 ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
2280                 if (ret != 0) {
2281                         DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
2282                         sleep(3);
2283                         retries++;
2284                 }
2285         } while (retries < 5 && ret != 0);
2286         if (ret != 0) {
2287                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
2288                 talloc_free(tmp_ctx);
2289                 return ret;
2290         }
2291
2292         if (rebalance_node(ctdb, options.pnn) != 0) {
2293                 DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
2294                 return ret;
2295         }
2296
2297         talloc_free(tmp_ctx);
2298         return 0;
2299 }
2300
2301 /*
2302   add a public ip address to a node
2303  */
2304 static int control_ipiface(struct ctdb_context *ctdb, int argc, const char **argv)
2305 {
2306         ctdb_sock_addr addr;
2307         char *iface = NULL;
2308
2309         if (argc != 1) {
2310                 usage();
2311         }
2312
2313         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2314                 printf("Badly formed ip : %s\n", argv[0]);
2315                 return -1;
2316         }
2317
2318         iface = ctdb_sys_find_ifname(&addr);
2319         if (iface == NULL) {
2320                 printf("Failed to get interface name for ip: %s", argv[0]);
2321                 return -1;
2322         }
2323
2324         printf("IP on interface %s\n", iface);
2325
2326         free(iface);
2327
2328         return 0;
2329 }
2330
2331 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
2332
2333 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
2334 {
2335         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2336         struct ctdb_node_map_old *nodemap=NULL;
2337         struct ctdb_public_ip_list_old *ips;
2338         int ret, i, j;
2339
2340         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
2341         if (ret != 0) {
2342                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
2343                 return ret;
2344         }
2345
2346         /* remove it from the nodes that are not hosting the ip currently */
2347         for(i=0;i<nodemap->num;i++){
2348                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2349                         continue;
2350                 }
2351                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2352                 if (ret != 0) {
2353                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2354                         continue;
2355                 }
2356
2357                 for (j=0;j<ips->num;j++) {
2358                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2359                                 break;
2360                         }
2361                 }
2362                 if (j==ips->num) {
2363                         continue;
2364                 }
2365
2366                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
2367                         continue;
2368                 }
2369
2370                 options.pnn = nodemap->nodes[i].pnn;
2371                 control_delip(ctdb, argc, argv);
2372         }
2373
2374
2375         /* remove it from every node (also the one hosting it) */
2376         for(i=0;i<nodemap->num;i++){
2377                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2378                         continue;
2379                 }
2380                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
2381                 if (ret != 0) {
2382                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
2383                         continue;
2384                 }
2385
2386                 for (j=0;j<ips->num;j++) {
2387                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
2388                                 break;
2389                         }
2390                 }
2391                 if (j==ips->num) {
2392                         continue;
2393                 }
2394
2395                 options.pnn = nodemap->nodes[i].pnn;
2396                 control_delip(ctdb, argc, argv);
2397         }
2398
2399         talloc_free(tmp_ctx);
2400         return 0;
2401 }
2402         
2403 /*
2404   delete a public ip address from a node
2405  */
2406 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
2407 {
2408         int i, ret;
2409         ctdb_sock_addr addr;
2410         struct ctdb_addr_info_old pub;
2411         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2412         struct ctdb_public_ip_list_old *ips;
2413
2414         if (argc != 1) {
2415                 talloc_free(tmp_ctx);
2416                 usage();
2417         }
2418
2419         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2420                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2421                 return -1;
2422         }
2423
2424         if (options.pnn == CTDB_BROADCAST_ALL) {
2425                 return control_delip_all(ctdb, argc, argv, &addr);
2426         }
2427
2428         pub.addr  = addr;
2429         pub.mask  = 0;
2430         pub.len   = 0;
2431
2432         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2433         if (ret != 0) {
2434                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
2435                 talloc_free(tmp_ctx);
2436                 return ret;
2437         }
2438         
2439         for (i=0;i<ips->num;i++) {
2440                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
2441                         break;
2442                 }
2443         }
2444
2445         if (i==ips->num) {
2446                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
2447                         ctdb_addr_to_str(&addr)));
2448                 talloc_free(tmp_ctx);
2449                 return -1;
2450         }
2451
2452         /* This is an optimisation.  If this node is hosting the IP
2453          * then try to move it somewhere else without invoking a full
2454          * takeover run.  We don't care if this doesn't work!
2455          */
2456         if (ips->ips[i].pnn == options.pnn) {
2457                 (void) try_moveip(ctdb, &addr, -1);
2458         }
2459
2460         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
2461         if (ret != 0) {
2462                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
2463                 talloc_free(tmp_ctx);
2464                 return ret;
2465         }
2466
2467         talloc_free(tmp_ctx);
2468         return 0;
2469 }
2470
2471 static int kill_tcp_from_file(struct ctdb_context *ctdb,
2472                               int argc, const char **argv)
2473 {
2474         struct ctdb_connection *killtcp;
2475         int max_entries, current, i;
2476         struct timeval timeout;
2477         char line[128], src[128], dst[128];
2478         int linenum;
2479         TDB_DATA data;
2480         struct client_async_data *async_data;
2481         struct ctdb_client_control_state *state;
2482
2483         if (argc != 0) {
2484                 usage();
2485         }
2486
2487         linenum = 1;
2488         killtcp = NULL;
2489         max_entries = 0;
2490         current = 0;
2491         while (!feof(stdin)) {
2492                 if (fgets(line, sizeof(line), stdin) == NULL) {
2493                         continue;
2494                 }
2495
2496                 /* Silently skip empty lines */
2497                 if (line[0] == '\n') {
2498                         continue;
2499                 }
2500
2501                 if (sscanf(line, "%s %s\n", src, dst) != 2) {
2502                         DEBUG(DEBUG_ERR, ("Bad line [%d]: '%s'\n",
2503                                           linenum, line));
2504                         talloc_free(killtcp);
2505                         return -1;
2506                 }
2507
2508                 if (current >= max_entries) {
2509                         max_entries += 1024;
2510                         killtcp = talloc_realloc(ctdb, killtcp,
2511                                                  struct ctdb_connection,
2512                                                  max_entries);
2513                         CTDB_NO_MEMORY(ctdb, killtcp);
2514                 }
2515
2516                 if (!parse_ip_port(src, &killtcp[current].src)) {
2517                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2518                                           linenum, src));
2519                         talloc_free(killtcp);
2520                         return -1;
2521                 }
2522
2523                 if (!parse_ip_port(dst, &killtcp[current].dst)) {
2524                         DEBUG(DEBUG_ERR, ("Bad IP:port on line [%d]: '%s'\n",
2525                                           linenum, dst));
2526                         talloc_free(killtcp);
2527                         return -1;
2528                 }
2529
2530                 current++;
2531         }
2532
2533         async_data = talloc_zero(ctdb, struct client_async_data);
2534         if (async_data == NULL) {
2535                 talloc_free(killtcp);
2536                 return -1;
2537         }
2538
2539         for (i = 0; i < current; i++) {
2540
2541                 data.dsize = sizeof(struct ctdb_connection);
2542                 data.dptr  = (unsigned char *)&killtcp[i];
2543
2544                 timeout = TIMELIMIT();
2545                 state = ctdb_control_send(ctdb, options.pnn, 0,
2546                                           CTDB_CONTROL_KILL_TCP, 0, data,
2547                                           async_data, &timeout, NULL);
2548
2549                 if (state == NULL) {
2550                         DEBUG(DEBUG_ERR,
2551                               ("Failed to call async killtcp control to node %u\n",
2552                                options.pnn));
2553                         talloc_free(killtcp);
2554                         return -1;
2555                 }
2556                 
2557                 ctdb_client_async_add(async_data, state);
2558         }
2559
2560         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2561                 DEBUG(DEBUG_ERR,("killtcp failed\n"));
2562                 talloc_free(killtcp);
2563                 return -1;
2564         }
2565
2566         talloc_free(killtcp);
2567         return 0;
2568 }
2569
2570
2571 /*
2572   kill a tcp connection
2573  */
2574 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2575 {
2576         int ret;
2577         struct ctdb_connection killtcp;
2578
2579         assert_single_node_only();
2580
2581         if (argc == 0) {
2582                 return kill_tcp_from_file(ctdb, argc, argv);
2583         }
2584
2585         if (argc < 2) {
2586                 usage();
2587         }
2588
2589         if (!parse_ip_port(argv[0], &killtcp.src)) {
2590                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2591                 return -1;
2592         }
2593
2594         if (!parse_ip_port(argv[1], &killtcp.dst)) {
2595                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2596                 return -1;
2597         }
2598
2599         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
2600         if (ret != 0) {
2601                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
2602                 return ret;
2603         }
2604
2605         return 0;
2606 }
2607
2608
2609 /*
2610   send a gratious arp
2611  */
2612 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
2613 {
2614         int ret;
2615         ctdb_sock_addr addr;
2616
2617         assert_single_node_only();
2618
2619         if (argc < 2) {
2620                 usage();
2621         }
2622
2623         if (!parse_ip(argv[0], NULL, 0, &addr)) {
2624                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
2625                 return -1;
2626         }
2627
2628         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
2629         if (ret != 0) {
2630                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
2631                 return ret;
2632         }
2633
2634         return 0;
2635 }
2636
2637 /*
2638   register a server id
2639  */
2640 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2641 {
2642         int ret;
2643         struct ctdb_client_id server_id;
2644
2645         if (argc < 3) {
2646                 usage();
2647         }
2648
2649         server_id.pnn       = strtoul(argv[0], NULL, 0);
2650         server_id.type      = strtoul(argv[1], NULL, 0);
2651         server_id.server_id = strtoul(argv[2], NULL, 0);
2652
2653         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
2654         if (ret != 0) {
2655                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
2656                 return ret;
2657         }
2658         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
2659         sleep(999);
2660         return -1;
2661 }
2662
2663 /*
2664   unregister a server id
2665  */
2666 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2667 {
2668         int ret;
2669         struct ctdb_client_id server_id;
2670
2671         if (argc < 3) {
2672                 usage();
2673         }
2674
2675         server_id.pnn       = strtoul(argv[0], NULL, 0);
2676         server_id.type      = strtoul(argv[1], NULL, 0);
2677         server_id.server_id = strtoul(argv[2], NULL, 0);
2678
2679         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
2680         if (ret != 0) {
2681                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
2682                 return ret;
2683         }
2684         return -1;
2685 }
2686
2687 /*
2688   check if a server id exists
2689  */
2690 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
2691 {
2692         uint32_t status = 0;
2693         int ret;
2694         struct ctdb_client_id server_id;
2695
2696         if (argc < 3) {
2697                 usage();
2698         }
2699
2700         server_id.pnn       = strtoul(argv[0], NULL, 0);
2701         server_id.type      = strtoul(argv[1], NULL, 0);
2702         server_id.server_id = strtoul(argv[2], NULL, 0);
2703
2704         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
2705         if (ret != 0) {
2706                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
2707                 return ret;
2708         }
2709
2710         if (status) {
2711                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
2712         } else {
2713                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
2714         }
2715         return 0;
2716 }
2717
2718 /*
2719   get a list of all server ids that are registered on a node
2720  */
2721 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
2722 {
2723         int i, ret;
2724         struct ctdb_client_id_list_old *server_ids;
2725
2726         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
2727         if (ret != 0) {
2728                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
2729                 return ret;
2730         }
2731
2732         for (i=0; i<server_ids->num; i++) {
2733                 printf("Server id %d:%d:%d\n", 
2734                         server_ids->server_ids[i].pnn, 
2735                         server_ids->server_ids[i].type, 
2736                         server_ids->server_ids[i].server_id); 
2737         }
2738
2739         return -1;
2740 }
2741
2742 /*
2743   check if a server id exists
2744  */
2745 static int check_srvids(struct ctdb_context *ctdb, int argc, const char **argv)
2746 {
2747         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
2748         uint64_t *ids;
2749         uint8_t *result;
2750         int i;
2751
2752         if (argc < 1) {
2753                 talloc_free(tmp_ctx);
2754                 usage();
2755         }
2756
2757         ids    = talloc_array(tmp_ctx, uint64_t, argc);
2758         result = talloc_array(tmp_ctx, uint8_t, argc);
2759
2760         for (i = 0; i < argc; i++) {
2761                 ids[i] = strtoull(argv[i], NULL, 0);
2762         }
2763
2764         if (!ctdb_client_check_message_handlers(ctdb, ids, argc, result)) {
2765                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n",
2766                                   options.pnn));
2767                 talloc_free(tmp_ctx);
2768                 return -1;
2769         }
2770
2771         for (i=0; i < argc; i++) {
2772                 printf("Server id %d:%llu %s\n", options.pnn, (long long)ids[i],
2773                        result[i] ? "exists" : "does not exist");
2774         }
2775
2776         talloc_free(tmp_ctx);
2777         return 0;
2778 }
2779
2780 /*
2781   send a tcp tickle ack
2782  */
2783 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
2784 {
2785         int ret;
2786         ctdb_sock_addr  src, dst;
2787
2788         if (argc < 2) {
2789                 usage();
2790         }
2791
2792         if (!parse_ip_port(argv[0], &src)) {
2793                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
2794                 return -1;
2795         }
2796
2797         if (!parse_ip_port(argv[1], &dst)) {
2798                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
2799                 return -1;
2800         }
2801
2802         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
2803         if (ret==0) {
2804                 return 0;
2805         }
2806         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
2807
2808         return -1;
2809 }
2810
2811
2812 /*
2813   display public ip status
2814  */
2815 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
2816 {
2817         int i, ret;
2818         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2819         struct ctdb_public_ip_list_old *ips;
2820
2821         if (argc == 1 && strcmp(argv[0], "all") == 0) {
2822                 options.pnn = CTDB_BROADCAST_ALL;
2823         }
2824
2825         if (options.pnn == CTDB_BROADCAST_ALL) {
2826                 /* read the list of public ips from all nodes */
2827                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
2828         } else {
2829                 /* read the public ip list from this node */
2830                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
2831         }
2832         if (ret != 0) {
2833                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
2834                 talloc_free(tmp_ctx);
2835                 return ret;
2836         }
2837
2838         if (options.machinereadable){
2839                 printm(":Public IP:Node:");
2840                 if (options.verbose){
2841                         printm("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
2842                 }
2843                 printm("\n");
2844         } else {
2845                 if (options.pnn == CTDB_BROADCAST_ALL) {
2846                         printf("Public IPs on ALL nodes\n");
2847                 } else {
2848                         printf("Public IPs on node %u\n", options.pnn);
2849                 }
2850         }
2851
2852         for (i=1;i<=ips->num;i++) {
2853                 struct ctdb_public_ip_info_old *info = NULL;
2854                 int32_t pnn;
2855                 char *aciface = NULL;
2856                 char *avifaces = NULL;
2857                 char *cifaces = NULL;
2858
2859                 if (options.pnn == CTDB_BROADCAST_ALL) {
2860                         pnn = ips->ips[ips->num-i].pnn;
2861                 } else {
2862                         pnn = options.pnn;
2863                 }
2864
2865                 if (pnn != -1) {
2866                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
2867                                                    &ips->ips[ips->num-i].addr, &info);
2868                 } else {
2869                         ret = -1;
2870                 }
2871
2872                 if (ret == 0) {
2873                         int j;
2874                         for (j=0; j < info->num; j++) {
2875                                 if (cifaces == NULL) {
2876                                         cifaces = talloc_strdup(info,
2877                                                                 info->ifaces[j].name);
2878                                 } else {
2879                                         cifaces = talloc_asprintf_append(cifaces,
2880                                                                          ",%s",
2881                                                                          info->ifaces[j].name);
2882                                 }
2883
2884                                 if (info->active_idx == j) {
2885                                         aciface = info->ifaces[j].name;
2886                                 }
2887
2888                                 if (info->ifaces[j].link_state == 0) {
2889                                         continue;
2890                                 }
2891
2892                                 if (avifaces == NULL) {
2893                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
2894                                 } else {
2895                                         avifaces = talloc_asprintf_append(avifaces,
2896                                                                           ",%s",
2897                                                                           info->ifaces[j].name);
2898                                 }
2899                         }
2900                 }
2901
2902                 if (options.machinereadable){
2903                         printm(":%s:%d:",
2904                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2905                                 ips->ips[ips->num-i].pnn);
2906                         if (options.verbose){
2907                                 printm("%s:%s:%s:",
2908                                         aciface?aciface:"",
2909                                         avifaces?avifaces:"",
2910                                         cifaces?cifaces:"");
2911                         }
2912                         printf("\n");
2913                 } else {
2914                         if (options.verbose) {
2915                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
2916                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2917                                         ips->ips[ips->num-i].pnn,
2918                                         aciface?aciface:"",
2919                                         avifaces?avifaces:"",
2920                                         cifaces?cifaces:"");
2921                         } else {
2922                                 printf("%s %d\n",
2923                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
2924                                         ips->ips[ips->num-i].pnn);
2925                         }
2926                 }
2927                 talloc_free(info);
2928         }
2929
2930         talloc_free(tmp_ctx);
2931         return 0;
2932 }
2933
2934 /*
2935   public ip info
2936  */
2937 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
2938 {
2939         int i, ret;
2940         ctdb_sock_addr addr;
2941         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2942         struct ctdb_public_ip_info_old *info;
2943
2944         if (argc != 1) {
2945                 talloc_free(tmp_ctx);
2946                 usage();
2947         }
2948
2949         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
2950                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
2951                 return -1;
2952         }
2953
2954         /* read the public ip info from this node */
2955         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
2956                                            tmp_ctx, &addr, &info);
2957         if (ret != 0) {
2958                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
2959                                   argv[0], options.pnn));
2960                 talloc_free(tmp_ctx);
2961                 return ret;
2962         }
2963
2964         printf("Public IP[%s] info on node %u\n",
2965                ctdb_addr_to_str(&info->ip.addr),
2966                options.pnn);
2967
2968         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
2969                ctdb_addr_to_str(&info->ip.addr),
2970                info->ip.pnn, info->num);
2971
2972         for (i=0; i<info->num; i++) {
2973                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
2974
2975                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
2976                        i+1, info->ifaces[i].name,
2977                        info->ifaces[i].link_state?"up":"down",
2978                        (unsigned int)info->ifaces[i].references,
2979                        (i==info->active_idx)?" (active)":"");
2980         }
2981
2982         talloc_free(tmp_ctx);
2983         return 0;
2984 }
2985
2986 /*
2987   display interfaces status
2988  */
2989 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
2990 {
2991         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2992         int i;
2993         struct ctdb_iface_list_old *ifaces;
2994         int ret;
2995
2996         /* read the public ip list from this node */
2997         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ifaces);
2998         if (ret != 0) {
2999                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
3000                                   options.pnn));
3001                 talloc_free(tmp_ctx);
3002                 return -1;
3003         }
3004
3005         if (options.machinereadable){
3006                 printm(":Name:LinkStatus:References:\n");
3007         } else {
3008                 printf("Interfaces on node %u\n", options.pnn);
3009         }
3010
3011         for (i=0; i<ifaces->num; i++) {
3012                 if (options.machinereadable){
3013                         printm(":%s:%s:%u:\n",
3014                                ifaces->ifaces[i].name,
3015                                ifaces->ifaces[i].link_state?"1":"0",
3016                                (unsigned int)ifaces->ifaces[i].references);
3017                 } else {
3018                         printf("name:%s link:%s references:%u\n",
3019                                ifaces->ifaces[i].name,
3020                                ifaces->ifaces[i].link_state?"up":"down",
3021                                (unsigned int)ifaces->ifaces[i].references);
3022                 }
3023         }
3024
3025         talloc_free(tmp_ctx);
3026         return 0;
3027 }
3028
3029
3030 /*
3031   set link status of an interface
3032  */
3033 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
3034 {
3035         int ret;
3036         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3037         struct ctdb_iface info;
3038
3039         ZERO_STRUCT(info);
3040
3041         if (argc != 2) {
3042                 usage();
3043         }
3044
3045         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
3046                 DEBUG(DEBUG_ERR, ("interfaces name '%s' too long\n",
3047                                   argv[0]));
3048                 talloc_free(tmp_ctx);
3049                 return -1;
3050         }
3051         strcpy(info.name, argv[0]);
3052
3053         if (strcmp(argv[1], "up") == 0) {
3054                 info.link_state = 1;
3055         } else if (strcmp(argv[1], "down") == 0) {
3056                 info.link_state = 0;
3057         } else {
3058                 DEBUG(DEBUG_ERR, ("link state invalid '%s' should be 'up' or 'down'\n",
3059                                   argv[1]));
3060                 talloc_free(tmp_ctx);
3061                 return -1;
3062         }
3063
3064         /* read the public ip list from this node */
3065         ret = ctdb_ctrl_set_iface_link(ctdb, TIMELIMIT(), options.pnn,
3066                                    tmp_ctx, &info);
3067         if (ret != 0) {
3068                 DEBUG(DEBUG_ERR, ("Unable to set link state for interfaces %s node %u\n",
3069                                   argv[0], options.pnn));
3070                 talloc_free(tmp_ctx);
3071                 return ret;
3072         }
3073
3074         talloc_free(tmp_ctx);
3075         return 0;
3076 }
3077
3078 /*
3079   display pid of a ctdb daemon
3080  */
3081 static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
3082 {
3083         uint32_t pid;
3084         int ret;
3085
3086         ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.pnn, &pid);
3087         if (ret != 0) {
3088                 DEBUG(DEBUG_ERR, ("Unable to get daemon pid from node %u\n", options.pnn));
3089                 return ret;
3090         }
3091         printf("Pid:%d\n", pid);
3092
3093         return 0;
3094 }
3095
3096 typedef bool update_flags_handler_t(struct ctdb_context *ctdb, void *data);
3097
3098 static int update_flags_and_ipreallocate(struct ctdb_context *ctdb,
3099                                               void *data,
3100                                               update_flags_handler_t handler,
3101                                               uint32_t flag,
3102                                               const char *desc,
3103                                               bool set_flag)
3104 {
3105         struct ctdb_node_map_old *nodemap = NULL;
3106         bool flag_is_set;
3107         int ret;
3108
3109         /* Check if the node is already in the desired state */
3110         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
3111         if (ret != 0) {
3112                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
3113                 exit(10);
3114         }
3115         flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3116         if (set_flag == flag_is_set) {
3117                 DEBUG(DEBUG_NOTICE, ("Node %d is %s %s\n", options.pnn,
3118                                      (set_flag ? "already" : "not"), desc));
3119                 return 0;
3120         }
3121
3122         do {
3123                 if (!handler(ctdb, data)) {
3124                         DEBUG(DEBUG_WARNING,
3125                               ("Failed to send control to set state %s on node %u, try again\n",
3126                                desc, options.pnn));
3127                 }
3128
3129                 sleep(1);
3130
3131                 /* Read the nodemap and verify the change took effect.
3132                  * Even if the above control/hanlder timed out then it
3133                  * could still have worked!
3134                  */
3135                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE,
3136                                          ctdb, &nodemap);
3137                 if (ret != 0) {
3138                         DEBUG(DEBUG_WARNING,
3139                               ("Unable to get nodemap from local node, try again\n"));
3140                 }
3141                 flag_is_set = nodemap->nodes[options.pnn].flags & flag;
3142         } while (nodemap == NULL || (set_flag != flag_is_set));
3143
3144         return ipreallocate(ctdb);
3145 }
3146
3147 /* Administratively disable a node */
3148 static bool update_flags_disabled(struct ctdb_context *ctdb, void *data)
3149 {
3150         int ret;
3151
3152         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3153                                  NODE_FLAGS_PERMANENTLY_DISABLED, 0);
3154         return ret == 0;
3155 }
3156
3157 static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
3158 {
3159         return update_flags_and_ipreallocate(ctdb, NULL,
3160                                                   update_flags_disabled,
3161                                                   NODE_FLAGS_PERMANENTLY_DISABLED,
3162                                                   "disabled",
3163                                                   true /* set_flag*/);
3164 }
3165
3166 /* Administratively re-enable a node */
3167 static bool update_flags_not_disabled(struct ctdb_context *ctdb, void *data)
3168 {
3169         int ret;
3170
3171         ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn,
3172                                  0, NODE_FLAGS_PERMANENTLY_DISABLED);
3173         return ret == 0;
3174 }
3175
3176 static int control_enable(struct ctdb_context *ctdb,  int argc, const char **argv)
3177 {
3178         return update_flags_and_ipreallocate(ctdb, NULL,
3179                                                   update_flags_not_disabled,
3180                                                   NODE_FLAGS_PERMANENTLY_DISABLED,