4 Copyright (C) Ronnie Sahlberg 2007
5 Copyright (C) Andrew Tridgell 2007
6 Copyright (C) Martin Schwenke 2011
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, see <http://www.gnu.org/licenses/>.
23 #include "lib/util/dlinklist.h"
24 #include "system/network.h"
25 #include "system/filesys.h"
26 #include "system/wait.h"
27 #include "../include/ctdb_private.h"
28 #include "../common/rb_tree.h"
31 #define TAKEOVER_TIMEOUT() timeval_current_ofs(ctdb->tunable.takeover_timeout,0)
33 #define CTDB_ARP_INTERVAL 1
34 #define CTDB_ARP_REPEAT 3
36 /* Flags used in IP allocation algorithms. */
43 struct ctdb_iface *prev, *next;
49 static const char *ctdb_vnn_iface_string(const struct ctdb_vnn *vnn)
52 return vnn->iface->name;
58 static int ctdb_add_local_iface(struct ctdb_context *ctdb, const char *iface)
62 /* Verify that we dont have an entry for this ip yet */
63 for (i=ctdb->ifaces;i;i=i->next) {
64 if (strcmp(i->name, iface) == 0) {
69 /* create a new structure for this interface */
70 i = talloc_zero(ctdb, struct ctdb_iface);
71 CTDB_NO_MEMORY_FATAL(ctdb, i);
72 i->name = talloc_strdup(i, iface);
73 CTDB_NO_MEMORY(ctdb, i->name);
75 * If link_up defaults to true then IPs can be allocated to a
76 * node during the first recovery. However, then an interface
77 * could have its link marked down during the startup event,
78 * causing the IP to move almost immediately. If link_up
79 * defaults to false then, during normal operation, IPs added
80 * to a new interface can't be assigned until a monitor cycle
81 * has occurred and marked the new interfaces up. This makes
82 * IP allocation unpredictable. The following is a neat
83 * compromise: early in startup link_up defaults to false, so
84 * IPs can't be assigned, and after startup IPs can be
85 * assigned immediately.
87 i->link_up = (ctdb->runstate == CTDB_RUNSTATE_RUNNING);
89 DLIST_ADD(ctdb->ifaces, i);
94 static bool vnn_has_interface_with_name(struct ctdb_vnn *vnn,
99 for (n = 0; vnn->ifaces[n] != NULL; n++) {
100 if (strcmp(name, vnn->ifaces[n]) == 0) {
108 /* If any interfaces now have no possible IPs then delete them. This
109 * implementation is naive (i.e. simple) rather than clever
110 * (i.e. complex). Given that this is run on delip and that operation
111 * is rare, this doesn't need to be efficient - it needs to be
112 * foolproof. One alternative is reference counting, where the logic
113 * is distributed and can, therefore, be broken in multiple places.
114 * Another alternative is to build a red-black tree of interfaces that
115 * can have addresses (by walking ctdb->vnn and ctdb->single_ip_vnn
116 * once) and then walking ctdb->ifaces once and deleting those not in
117 * the tree. Let's go to one of those if the naive implementation
118 * causes problems... :-)
120 static void ctdb_remove_orphaned_ifaces(struct ctdb_context *ctdb,
121 struct ctdb_vnn *vnn,
124 struct ctdb_iface *i;
126 /* For each interface, check if there's an IP using it. */
127 for(i=ctdb->ifaces; i; i=i->next) {
131 /* Only consider interfaces named in the given VNN. */
132 if (!vnn_has_interface_with_name(vnn, i->name)) {
136 /* Is the "single IP" on this interface? */
137 if ((ctdb->single_ip_vnn != NULL) &&
138 (ctdb->single_ip_vnn->ifaces[0] != NULL) &&
139 (strcmp(i->name, ctdb->single_ip_vnn->ifaces[0]) == 0)) {
140 /* Found, next interface please... */
143 /* Search for a vnn with this interface. */
145 for (tv=ctdb->vnn; tv; tv=tv->next) {
146 if (vnn_has_interface_with_name(tv, i->name)) {
153 /* None of the VNNs are using this interface. */
154 DLIST_REMOVE(ctdb->ifaces, i);
155 /* Caller will free mem_ctx when convenient. */
156 talloc_steal(mem_ctx, i);
162 static struct ctdb_iface *ctdb_find_iface(struct ctdb_context *ctdb,
165 struct ctdb_iface *i;
167 for (i=ctdb->ifaces;i;i=i->next) {
168 if (strcmp(i->name, iface) == 0) {
176 static struct ctdb_iface *ctdb_vnn_best_iface(struct ctdb_context *ctdb,
177 struct ctdb_vnn *vnn)
180 struct ctdb_iface *cur = NULL;
181 struct ctdb_iface *best = NULL;
183 for (i=0; vnn->ifaces[i]; i++) {
185 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
199 if (cur->references < best->references) {
208 static int32_t ctdb_vnn_assign_iface(struct ctdb_context *ctdb,
209 struct ctdb_vnn *vnn)
211 struct ctdb_iface *best = NULL;
214 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
215 "still assigned to iface '%s'\n",
216 ctdb_addr_to_str(&vnn->public_address),
217 ctdb_vnn_iface_string(vnn)));
221 best = ctdb_vnn_best_iface(ctdb, vnn);
223 DEBUG(DEBUG_ERR, (__location__ " public address '%s' "
224 "cannot assign to iface any iface\n",
225 ctdb_addr_to_str(&vnn->public_address)));
231 vnn->pnn = ctdb->pnn;
233 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
234 "now assigned to iface '%s' refs[%d]\n",
235 ctdb_addr_to_str(&vnn->public_address),
236 ctdb_vnn_iface_string(vnn),
241 static void ctdb_vnn_unassign_iface(struct ctdb_context *ctdb,
242 struct ctdb_vnn *vnn)
244 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
245 "now unassigned (old iface '%s' refs[%d])\n",
246 ctdb_addr_to_str(&vnn->public_address),
247 ctdb_vnn_iface_string(vnn),
248 vnn->iface?vnn->iface->references:0));
250 vnn->iface->references--;
253 if (vnn->pnn == ctdb->pnn) {
258 static bool ctdb_vnn_available(struct ctdb_context *ctdb,
259 struct ctdb_vnn *vnn)
263 if (vnn->iface && vnn->iface->link_up) {
267 for (i=0; vnn->ifaces[i]; i++) {
268 struct ctdb_iface *cur;
270 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
283 struct ctdb_takeover_arp {
284 struct ctdb_context *ctdb;
287 struct ctdb_tcp_array *tcparray;
288 struct ctdb_vnn *vnn;
293 lists of tcp endpoints
295 struct ctdb_tcp_list {
296 struct ctdb_tcp_list *prev, *next;
297 struct ctdb_tcp_connection connection;
301 list of clients to kill on IP release
303 struct ctdb_client_ip {
304 struct ctdb_client_ip *prev, *next;
305 struct ctdb_context *ctdb;
312 send a gratuitous arp
314 static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *te,
315 struct timeval t, void *private_data)
317 struct ctdb_takeover_arp *arp = talloc_get_type(private_data,
318 struct ctdb_takeover_arp);
320 struct ctdb_tcp_array *tcparray;
321 const char *iface = ctdb_vnn_iface_string(arp->vnn);
323 ret = ctdb_sys_send_arp(&arp->addr, iface);
325 DEBUG(DEBUG_CRIT,(__location__ " sending of arp failed on iface '%s' (%s)\n",
326 iface, strerror(errno)));
329 tcparray = arp->tcparray;
331 for (i=0;i<tcparray->num;i++) {
332 struct ctdb_tcp_connection *tcon;
334 tcon = &tcparray->connections[i];
335 DEBUG(DEBUG_INFO,("sending tcp tickle ack for %u->%s:%u\n",
336 (unsigned)ntohs(tcon->dst_addr.ip.sin_port),
337 ctdb_addr_to_str(&tcon->src_addr),
338 (unsigned)ntohs(tcon->src_addr.ip.sin_port)));
339 ret = ctdb_sys_send_tcp(
344 DEBUG(DEBUG_CRIT,(__location__ " Failed to send tcp tickle ack for %s\n",
345 ctdb_addr_to_str(&tcon->src_addr)));
352 if (arp->count == CTDB_ARP_REPEAT) {
357 event_add_timed(arp->ctdb->ev, arp->vnn->takeover_ctx,
358 timeval_current_ofs(CTDB_ARP_INTERVAL, 100000),
359 ctdb_control_send_arp, arp);
362 static int32_t ctdb_announce_vnn_iface(struct ctdb_context *ctdb,
363 struct ctdb_vnn *vnn)
365 struct ctdb_takeover_arp *arp;
366 struct ctdb_tcp_array *tcparray;
368 if (!vnn->takeover_ctx) {
369 vnn->takeover_ctx = talloc_new(vnn);
370 if (!vnn->takeover_ctx) {
375 arp = talloc_zero(vnn->takeover_ctx, struct ctdb_takeover_arp);
381 arp->addr = vnn->public_address;
384 tcparray = vnn->tcp_array;
386 /* add all of the known tcp connections for this IP to the
387 list of tcp connections to send tickle acks for */
388 arp->tcparray = talloc_steal(arp, tcparray);
390 vnn->tcp_array = NULL;
391 vnn->tcp_update_needed = true;
394 event_add_timed(arp->ctdb->ev, vnn->takeover_ctx,
395 timeval_zero(), ctdb_control_send_arp, arp);
400 struct takeover_callback_state {
401 struct ctdb_req_control *c;
402 ctdb_sock_addr *addr;
403 struct ctdb_vnn *vnn;
406 struct ctdb_do_takeip_state {
407 struct ctdb_req_control *c;
408 struct ctdb_vnn *vnn;
412 called when takeip event finishes
414 static void ctdb_do_takeip_callback(struct ctdb_context *ctdb, int status,
417 struct ctdb_do_takeip_state *state =
418 talloc_get_type(private_data, struct ctdb_do_takeip_state);
423 struct ctdb_node *node = ctdb->nodes[ctdb->pnn];
425 if (status == -ETIME) {
428 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
429 ctdb_addr_to_str(&state->vnn->public_address),
430 ctdb_vnn_iface_string(state->vnn)));
431 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
433 node->flags |= NODE_FLAGS_UNHEALTHY;
438 if (ctdb->do_checkpublicip) {
440 ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
442 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
449 data.dptr = (uint8_t *)ctdb_addr_to_str(&state->vnn->public_address);
450 data.dsize = strlen((char *)data.dptr) + 1;
451 DEBUG(DEBUG_INFO,(__location__ " sending TAKE_IP for '%s'\n", data.dptr));
453 ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_TAKE_IP, data);
456 /* the control succeeded */
457 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
462 static int ctdb_takeip_destructor(struct ctdb_do_takeip_state *state)
464 state->vnn->update_in_flight = false;
469 take over an ip address
471 static int32_t ctdb_do_takeip(struct ctdb_context *ctdb,
472 struct ctdb_req_control *c,
473 struct ctdb_vnn *vnn)
476 struct ctdb_do_takeip_state *state;
478 if (vnn->update_in_flight) {
479 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u rejected "
480 "update for this IP already in flight\n",
481 ctdb_addr_to_str(&vnn->public_address),
482 vnn->public_netmask_bits));
486 ret = ctdb_vnn_assign_iface(ctdb, vnn);
488 DEBUG(DEBUG_ERR,("Takeover of IP %s/%u failed to "
489 "assign a usable interface\n",
490 ctdb_addr_to_str(&vnn->public_address),
491 vnn->public_netmask_bits));
495 state = talloc(vnn, struct ctdb_do_takeip_state);
496 CTDB_NO_MEMORY(ctdb, state);
498 state->c = talloc_steal(ctdb, c);
501 vnn->update_in_flight = true;
502 talloc_set_destructor(state, ctdb_takeip_destructor);
504 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u on interface %s\n",
505 ctdb_addr_to_str(&vnn->public_address),
506 vnn->public_netmask_bits,
507 ctdb_vnn_iface_string(vnn)));
509 ret = ctdb_event_script_callback(ctdb,
511 ctdb_do_takeip_callback,
515 ctdb_vnn_iface_string(vnn),
516 ctdb_addr_to_str(&vnn->public_address),
517 vnn->public_netmask_bits);
520 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
521 ctdb_addr_to_str(&vnn->public_address),
522 ctdb_vnn_iface_string(vnn)));
530 struct ctdb_do_updateip_state {
531 struct ctdb_req_control *c;
532 struct ctdb_iface *old;
533 struct ctdb_vnn *vnn;
537 called when updateip event finishes
539 static void ctdb_do_updateip_callback(struct ctdb_context *ctdb, int status,
542 struct ctdb_do_updateip_state *state =
543 talloc_get_type(private_data, struct ctdb_do_updateip_state);
547 if (status == -ETIME) {
550 DEBUG(DEBUG_ERR,(__location__ " Failed to move IP %s from interface %s to %s\n",
551 ctdb_addr_to_str(&state->vnn->public_address),
553 ctdb_vnn_iface_string(state->vnn)));
556 * All we can do is reset the old interface
557 * and let the next run fix it
559 ctdb_vnn_unassign_iface(ctdb, state->vnn);
560 state->vnn->iface = state->old;
561 state->vnn->iface->references++;
563 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
568 if (ctdb->do_checkpublicip) {
570 ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
572 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
579 /* the control succeeded */
580 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
585 static int ctdb_updateip_destructor(struct ctdb_do_updateip_state *state)
587 state->vnn->update_in_flight = false;
592 update (move) an ip address
594 static int32_t ctdb_do_updateip(struct ctdb_context *ctdb,
595 struct ctdb_req_control *c,
596 struct ctdb_vnn *vnn)
599 struct ctdb_do_updateip_state *state;
600 struct ctdb_iface *old = vnn->iface;
601 const char *new_name;
603 if (vnn->update_in_flight) {
604 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u rejected "
605 "update for this IP already in flight\n",
606 ctdb_addr_to_str(&vnn->public_address),
607 vnn->public_netmask_bits));
611 ctdb_vnn_unassign_iface(ctdb, vnn);
612 ret = ctdb_vnn_assign_iface(ctdb, vnn);
614 DEBUG(DEBUG_ERR,("update of IP %s/%u failed to "
615 "assin a usable interface (old iface '%s')\n",
616 ctdb_addr_to_str(&vnn->public_address),
617 vnn->public_netmask_bits,
622 new_name = ctdb_vnn_iface_string(vnn);
623 if (old->name != NULL && new_name != NULL && !strcmp(old->name, new_name)) {
624 /* A benign update from one interface onto itself.
625 * no need to run the eventscripts in this case, just return
628 ctdb_request_control_reply(ctdb, c, NULL, 0, NULL);
632 state = talloc(vnn, struct ctdb_do_updateip_state);
633 CTDB_NO_MEMORY(ctdb, state);
635 state->c = talloc_steal(ctdb, c);
639 vnn->update_in_flight = true;
640 talloc_set_destructor(state, ctdb_updateip_destructor);
642 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u from "
643 "interface %s to %s\n",
644 ctdb_addr_to_str(&vnn->public_address),
645 vnn->public_netmask_bits,
649 ret = ctdb_event_script_callback(ctdb,
651 ctdb_do_updateip_callback,
653 CTDB_EVENT_UPDATE_IP,
657 ctdb_addr_to_str(&vnn->public_address),
658 vnn->public_netmask_bits);
660 DEBUG(DEBUG_ERR,(__location__ " Failed update IP %s from interface %s to %s\n",
661 ctdb_addr_to_str(&vnn->public_address),
662 old->name, new_name));
671 Find the vnn of the node that has a public ip address
672 returns -1 if the address is not known as a public address
674 static struct ctdb_vnn *find_public_ip_vnn(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
676 struct ctdb_vnn *vnn;
678 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
679 if (ctdb_same_ip(&vnn->public_address, addr)) {
688 take over an ip address
690 int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb,
691 struct ctdb_req_control *c,
696 struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
697 struct ctdb_vnn *vnn;
698 bool have_ip = false;
699 bool do_updateip = false;
700 bool do_takeip = false;
701 struct ctdb_iface *best_iface = NULL;
703 if (pip->pnn != ctdb->pnn) {
704 DEBUG(DEBUG_ERR,(__location__" takeoverip called for an ip '%s' "
705 "with pnn %d, but we're node %d\n",
706 ctdb_addr_to_str(&pip->addr),
707 pip->pnn, ctdb->pnn));
711 /* update out vnn list */
712 vnn = find_public_ip_vnn(ctdb, &pip->addr);
714 DEBUG(DEBUG_INFO,("takeoverip called for an ip '%s' that is not a public address\n",
715 ctdb_addr_to_str(&pip->addr)));
719 if (ctdb->do_checkpublicip) {
720 have_ip = ctdb_sys_have_ip(&pip->addr);
722 best_iface = ctdb_vnn_best_iface(ctdb, vnn);
723 if (best_iface == NULL) {
724 DEBUG(DEBUG_ERR,("takeoverip of IP %s/%u failed to find"
725 "a usable interface (old %s, have_ip %d)\n",
726 ctdb_addr_to_str(&vnn->public_address),
727 vnn->public_netmask_bits,
728 ctdb_vnn_iface_string(vnn),
733 if (vnn->iface == NULL && vnn->pnn == -1 && have_ip && best_iface != NULL) {
734 DEBUG(DEBUG_ERR,("Taking over newly created ip\n"));
739 if (vnn->iface == NULL && have_ip) {
740 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
741 "but we have no interface assigned, has someone manually configured it? Ignore for now.\n",
742 ctdb_addr_to_str(&vnn->public_address)));
746 if (vnn->pnn != ctdb->pnn && have_ip && vnn->pnn != -1) {
747 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
748 "and we have it on iface[%s], but it was assigned to node %d"
749 "and we are node %d, banning ourself\n",
750 ctdb_addr_to_str(&vnn->public_address),
751 ctdb_vnn_iface_string(vnn), vnn->pnn, ctdb->pnn));
756 if (vnn->pnn == -1 && have_ip) {
757 vnn->pnn = ctdb->pnn;
758 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
759 "and we already have it on iface[%s], update local daemon\n",
760 ctdb_addr_to_str(&vnn->public_address),
761 ctdb_vnn_iface_string(vnn)));
766 if (vnn->iface != best_iface) {
767 if (!vnn->iface->link_up) {
769 } else if (vnn->iface->references > (best_iface->references + 1)) {
770 /* only move when the rebalance gains something */
778 ctdb_vnn_unassign_iface(ctdb, vnn);
785 ret = ctdb_do_takeip(ctdb, c, vnn);
789 } else if (do_updateip) {
790 ret = ctdb_do_updateip(ctdb, c, vnn);
796 * The interface is up and the kernel known the ip
799 DEBUG(DEBUG_INFO,("Redundant takeover of IP %s/%u on interface %s (ip already held)\n",
800 ctdb_addr_to_str(&pip->addr),
801 vnn->public_netmask_bits,
802 ctdb_vnn_iface_string(vnn)));
806 /* tell ctdb_control.c that we will be replying asynchronously */
813 takeover an ip address old v4 style
815 int32_t ctdb_control_takeover_ipv4(struct ctdb_context *ctdb,
816 struct ctdb_req_control *c,
822 data.dsize = sizeof(struct ctdb_public_ip);
823 data.dptr = (uint8_t *)talloc_zero(c, struct ctdb_public_ip);
824 CTDB_NO_MEMORY(ctdb, data.dptr);
826 memcpy(data.dptr, indata.dptr, indata.dsize);
827 return ctdb_control_takeover_ip(ctdb, c, data, async_reply);
831 kill any clients that are registered with a IP that is being released
833 static void release_kill_clients(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
835 struct ctdb_client_ip *ip;
837 DEBUG(DEBUG_INFO,("release_kill_clients for ip %s\n",
838 ctdb_addr_to_str(addr)));
840 for (ip=ctdb->client_ip_list; ip; ip=ip->next) {
841 ctdb_sock_addr tmp_addr;
844 DEBUG(DEBUG_INFO,("checking for client %u with IP %s\n",
846 ctdb_addr_to_str(&ip->addr)));
848 if (ctdb_same_ip(&tmp_addr, addr)) {
849 struct ctdb_client *client = ctdb_reqid_find(ctdb,
852 DEBUG(DEBUG_INFO,("matched client %u with IP %s and pid %u\n",
854 ctdb_addr_to_str(&ip->addr),
857 if (client->pid != 0) {
858 DEBUG(DEBUG_INFO,(__location__ " Killing client pid %u for IP %s on client_id %u\n",
859 (unsigned)client->pid,
860 ctdb_addr_to_str(addr),
862 kill(client->pid, SIGKILL);
869 called when releaseip event finishes
871 static void release_ip_callback(struct ctdb_context *ctdb, int status,
874 struct takeover_callback_state *state =
875 talloc_get_type(private_data, struct takeover_callback_state);
878 if (status == -ETIME) {
882 if (ctdb->do_checkpublicip && ctdb_sys_have_ip(state->addr)) {
883 DEBUG(DEBUG_ERR, ("IP %s still hosted during release IP callback, failing\n",
884 ctdb_addr_to_str(state->addr)));
885 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
890 /* send a message to all clients of this node telling them
891 that the cluster has been reconfigured and they should
892 release any sockets on this IP */
893 data.dptr = (uint8_t *)talloc_strdup(state, ctdb_addr_to_str(state->addr));
894 CTDB_NO_MEMORY_VOID(ctdb, data.dptr);
895 data.dsize = strlen((char *)data.dptr)+1;
897 DEBUG(DEBUG_INFO,(__location__ " sending RELEASE_IP for '%s'\n", data.dptr));
899 ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_RELEASE_IP, data);
901 /* kill clients that have registered with this IP */
902 release_kill_clients(ctdb, state->addr);
904 ctdb_vnn_unassign_iface(ctdb, state->vnn);
906 /* the control succeeded */
907 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
911 static int ctdb_releaseip_destructor(struct takeover_callback_state *state)
913 state->vnn->update_in_flight = false;
918 release an ip address
920 int32_t ctdb_control_release_ip(struct ctdb_context *ctdb,
921 struct ctdb_req_control *c,
926 struct takeover_callback_state *state;
927 struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
928 struct ctdb_vnn *vnn;
931 /* update our vnn list */
932 vnn = find_public_ip_vnn(ctdb, &pip->addr);
934 DEBUG(DEBUG_INFO,("releaseip called for an ip '%s' that is not a public address\n",
935 ctdb_addr_to_str(&pip->addr)));
940 /* stop any previous arps */
941 talloc_free(vnn->takeover_ctx);
942 vnn->takeover_ctx = NULL;
944 /* Some ctdb tool commands (e.g. moveip, rebalanceip) send
945 * lazy multicast to drop an IP from any node that isn't the
946 * intended new node. The following causes makes ctdbd ignore
947 * a release for any address it doesn't host.
949 if (ctdb->do_checkpublicip) {
950 if (!ctdb_sys_have_ip(&pip->addr)) {
951 DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u on interface %s (ip not held)\n",
952 ctdb_addr_to_str(&pip->addr),
953 vnn->public_netmask_bits,
954 ctdb_vnn_iface_string(vnn)));
955 ctdb_vnn_unassign_iface(ctdb, vnn);
959 if (vnn->iface == NULL) {
960 DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u (ip not held)\n",
961 ctdb_addr_to_str(&pip->addr),
962 vnn->public_netmask_bits));
967 /* There is a potential race between take_ip and us because we
968 * update the VNN via a callback that run when the
969 * eventscripts have been run. Avoid the race by allowing one
970 * update to be in flight at a time.
972 if (vnn->update_in_flight) {
973 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u rejected "
974 "update for this IP already in flight\n",
975 ctdb_addr_to_str(&vnn->public_address),
976 vnn->public_netmask_bits));
980 if (ctdb->do_checkpublicip) {
981 iface = ctdb_sys_find_ifname(&pip->addr);
983 DEBUG(DEBUG_ERR, ("Could not find which interface the ip address is hosted on. can not release it\n"));
986 if (vnn->iface == NULL) {
988 ("Public IP %s is hosted on interface %s but we have no VNN\n",
989 ctdb_addr_to_str(&pip->addr),
991 } else if (strcmp(iface, ctdb_vnn_iface_string(vnn)) != 0) {
993 ("Public IP %s is hosted on inteterface %s but VNN says %s\n",
994 ctdb_addr_to_str(&pip->addr),
996 ctdb_vnn_iface_string(vnn)));
997 /* Should we fix vnn->iface? If we do, what
998 * happens to reference counts?
1002 iface = strdup(ctdb_vnn_iface_string(vnn));
1005 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s node:%d\n",
1006 ctdb_addr_to_str(&pip->addr),
1007 vnn->public_netmask_bits,
1011 state = talloc(ctdb, struct takeover_callback_state);
1012 CTDB_NO_MEMORY(ctdb, state);
1014 state->c = talloc_steal(state, c);
1015 state->addr = talloc(state, ctdb_sock_addr);
1016 CTDB_NO_MEMORY(ctdb, state->addr);
1017 *state->addr = pip->addr;
1020 vnn->update_in_flight = true;
1021 talloc_set_destructor(state, ctdb_releaseip_destructor);
1023 ret = ctdb_event_script_callback(ctdb,
1024 state, release_ip_callback, state,
1025 CTDB_EVENT_RELEASE_IP,
1028 ctdb_addr_to_str(&pip->addr),
1029 vnn->public_netmask_bits);
1032 DEBUG(DEBUG_ERR,(__location__ " Failed to release IP %s on interface %s\n",
1033 ctdb_addr_to_str(&pip->addr),
1034 ctdb_vnn_iface_string(vnn)));
1039 /* tell the control that we will be reply asynchronously */
1040 *async_reply = true;
1045 release an ip address old v4 style
1047 int32_t ctdb_control_release_ipv4(struct ctdb_context *ctdb,
1048 struct ctdb_req_control *c,
1054 data.dsize = sizeof(struct ctdb_public_ip);
1055 data.dptr = (uint8_t *)talloc_zero(c, struct ctdb_public_ip);
1056 CTDB_NO_MEMORY(ctdb, data.dptr);
1058 memcpy(data.dptr, indata.dptr, indata.dsize);
1059 return ctdb_control_release_ip(ctdb, c, data, async_reply);
1063 static int ctdb_add_public_address(struct ctdb_context *ctdb,
1064 ctdb_sock_addr *addr,
1065 unsigned mask, const char *ifaces,
1068 struct ctdb_vnn *vnn;
1075 tmp = strdup(ifaces);
1076 for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1077 if (!ctdb_sys_check_iface_exists(iface)) {
1078 DEBUG(DEBUG_CRIT,("Interface %s does not exist. Can not add public-address : %s\n", iface, ctdb_addr_to_str(addr)));
1085 /* Verify that we dont have an entry for this ip yet */
1086 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1087 if (ctdb_same_sockaddr(addr, &vnn->public_address)) {
1088 DEBUG(DEBUG_CRIT,("Same ip '%s' specified multiple times in the public address list \n",
1089 ctdb_addr_to_str(addr)));
1094 /* create a new vnn structure for this ip address */
1095 vnn = talloc_zero(ctdb, struct ctdb_vnn);
1096 CTDB_NO_MEMORY_FATAL(ctdb, vnn);
1097 vnn->ifaces = talloc_array(vnn, const char *, num + 2);
1098 tmp = talloc_strdup(vnn, ifaces);
1099 CTDB_NO_MEMORY_FATAL(ctdb, tmp);
1100 for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1101 vnn->ifaces = talloc_realloc(vnn, vnn->ifaces, const char *, num + 2);
1102 CTDB_NO_MEMORY_FATAL(ctdb, vnn->ifaces);
1103 vnn->ifaces[num] = talloc_strdup(vnn, iface);
1104 CTDB_NO_MEMORY_FATAL(ctdb, vnn->ifaces[num]);
1108 vnn->ifaces[num] = NULL;
1109 vnn->public_address = *addr;
1110 vnn->public_netmask_bits = mask;
1112 if (check_address) {
1113 if (ctdb_sys_have_ip(addr)) {
1114 DEBUG(DEBUG_ERR,("We are already hosting public address '%s'. setting PNN to ourself:%d\n", ctdb_addr_to_str(addr), ctdb->pnn));
1115 vnn->pnn = ctdb->pnn;
1119 for (i=0; vnn->ifaces[i]; i++) {
1120 ret = ctdb_add_local_iface(ctdb, vnn->ifaces[i]);
1122 DEBUG(DEBUG_CRIT, (__location__ " failed to add iface[%s] "
1123 "for public_address[%s]\n",
1124 vnn->ifaces[i], ctdb_addr_to_str(addr)));
1130 DLIST_ADD(ctdb->vnn, vnn);
1135 static void ctdb_check_interfaces_event(struct event_context *ev, struct timed_event *te,
1136 struct timeval t, void *private_data)
1138 struct ctdb_context *ctdb = talloc_get_type(private_data,
1139 struct ctdb_context);
1140 struct ctdb_vnn *vnn;
1142 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1145 for (i=0; vnn->ifaces[i] != NULL; i++) {
1146 if (!ctdb_sys_check_iface_exists(vnn->ifaces[i])) {
1147 DEBUG(DEBUG_CRIT,("Interface %s does not exist but is used by public ip %s\n",
1149 ctdb_addr_to_str(&vnn->public_address)));
1154 event_add_timed(ctdb->ev, ctdb->check_public_ifaces_ctx,
1155 timeval_current_ofs(30, 0),
1156 ctdb_check_interfaces_event, ctdb);
1160 int ctdb_start_monitoring_interfaces(struct ctdb_context *ctdb)
1162 if (ctdb->check_public_ifaces_ctx != NULL) {
1163 talloc_free(ctdb->check_public_ifaces_ctx);
1164 ctdb->check_public_ifaces_ctx = NULL;
1167 ctdb->check_public_ifaces_ctx = talloc_new(ctdb);
1168 if (ctdb->check_public_ifaces_ctx == NULL) {
1169 ctdb_fatal(ctdb, "failed to allocate context for checking interfaces");
1172 event_add_timed(ctdb->ev, ctdb->check_public_ifaces_ctx,
1173 timeval_current_ofs(30, 0),
1174 ctdb_check_interfaces_event, ctdb);
1181 setup the public address lists from a file
1183 int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses)
1189 lines = file_lines_load(ctdb->public_addresses_file, &nlines, ctdb);
1190 if (lines == NULL) {
1191 ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", ctdb->public_addresses_file);
1194 while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
1198 for (i=0;i<nlines;i++) {
1200 ctdb_sock_addr addr;
1201 const char *addrstr;
1206 while ((*line == ' ') || (*line == '\t')) {
1212 if (strcmp(line, "") == 0) {
1215 tok = strtok(line, " \t");
1217 tok = strtok(NULL, " \t");
1219 if (NULL == ctdb->default_public_interface) {
1220 DEBUG(DEBUG_CRIT,("No default public interface and no interface specified at line %u of public address list\n",
1225 ifaces = ctdb->default_public_interface;
1230 if (!addrstr || !parse_ip_mask(addrstr, ifaces, &addr, &mask)) {
1231 DEBUG(DEBUG_CRIT,("Badly formed line %u in public address list\n", i+1));
1235 if (ctdb_add_public_address(ctdb, &addr, mask, ifaces, check_addresses)) {
1236 DEBUG(DEBUG_CRIT,("Failed to add line %u to the public address list\n", i+1));
1247 int ctdb_set_single_public_ip(struct ctdb_context *ctdb,
1251 struct ctdb_vnn *svnn;
1252 struct ctdb_iface *cur = NULL;
1256 svnn = talloc_zero(ctdb, struct ctdb_vnn);
1257 CTDB_NO_MEMORY(ctdb, svnn);
1259 svnn->ifaces = talloc_array(svnn, const char *, 2);
1260 CTDB_NO_MEMORY(ctdb, svnn->ifaces);
1261 svnn->ifaces[0] = talloc_strdup(svnn->ifaces, iface);
1262 CTDB_NO_MEMORY(ctdb, svnn->ifaces[0]);
1263 svnn->ifaces[1] = NULL;
1265 ok = parse_ip(ip, iface, 0, &svnn->public_address);
1271 ret = ctdb_add_local_iface(ctdb, svnn->ifaces[0]);
1273 DEBUG(DEBUG_CRIT, (__location__ " failed to add iface[%s] "
1274 "for single_ip[%s]\n",
1276 ctdb_addr_to_str(&svnn->public_address)));
1281 /* assume the single public ip interface is initially "good" */
1282 cur = ctdb_find_iface(ctdb, iface);
1284 DEBUG(DEBUG_CRIT,("Can not find public interface %s used by --single-public-ip", iface));
1287 cur->link_up = true;
1289 ret = ctdb_vnn_assign_iface(ctdb, svnn);
1295 ctdb->single_ip_vnn = svnn;
1299 struct ctdb_public_ip_list {
1300 struct ctdb_public_ip_list *next;
1302 ctdb_sock_addr addr;
1305 /* Given a physical node, return the number of
1306 public addresses that is currently assigned to this node.
1308 static int node_ip_coverage(struct ctdb_context *ctdb,
1310 struct ctdb_public_ip_list *ips)
1314 for (;ips;ips=ips->next) {
1315 if (ips->pnn == pnn) {
1323 /* Can the given node host the given IP: is the public IP known to the
1324 * node and is NOIPHOST unset?
1326 static bool can_node_host_ip(struct ctdb_context *ctdb, int32_t pnn,
1327 struct ctdb_ipflags ipflags,
1328 struct ctdb_public_ip_list *ip)
1330 struct ctdb_all_public_ips *public_ips;
1333 if (ipflags.noiphost) {
1337 public_ips = ctdb->nodes[pnn]->available_public_ips;
1339 if (public_ips == NULL) {
1343 for (i=0; i<public_ips->num; i++) {
1344 if (ctdb_same_ip(&ip->addr, &public_ips->ips[i].addr)) {
1345 /* yes, this node can serve this public ip */
1353 static bool can_node_takeover_ip(struct ctdb_context *ctdb, int32_t pnn,
1354 struct ctdb_ipflags ipflags,
1355 struct ctdb_public_ip_list *ip)
1357 if (ipflags.noiptakeover) {
1361 return can_node_host_ip(ctdb, pnn, ipflags, ip);
1364 /* search the node lists list for a node to takeover this ip.
1365 pick the node that currently are serving the least number of ips
1366 so that the ips get spread out evenly.
1368 static int find_takeover_node(struct ctdb_context *ctdb,
1369 struct ctdb_ipflags *ipflags,
1370 struct ctdb_public_ip_list *ip,
1371 struct ctdb_public_ip_list *all_ips)
1373 int pnn, min=0, num;
1376 numnodes = talloc_array_length(ipflags);
1378 for (i=0; i<numnodes; i++) {
1379 /* verify that this node can serve this ip */
1380 if (!can_node_takeover_ip(ctdb, i, ipflags[i], ip)) {
1381 /* no it couldnt so skip to the next node */
1385 num = node_ip_coverage(ctdb, i, all_ips);
1386 /* was this the first node we checked ? */
1398 DEBUG(DEBUG_WARNING,(__location__ " Could not find node to take over public address '%s'\n",
1399 ctdb_addr_to_str(&ip->addr)));
1409 static uint32_t *ip_key(ctdb_sock_addr *ip)
1411 static uint32_t key[IP_KEYLEN];
1413 bzero(key, sizeof(key));
1415 switch (ip->sa.sa_family) {
1417 key[3] = htonl(ip->ip.sin_addr.s_addr);
1420 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
1421 key[0] = htonl(s6_a32[0]);
1422 key[1] = htonl(s6_a32[1]);
1423 key[2] = htonl(s6_a32[2]);
1424 key[3] = htonl(s6_a32[3]);
1428 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
1435 static void *add_ip_callback(void *parm, void *data)
1437 struct ctdb_public_ip_list *this_ip = parm;
1438 struct ctdb_public_ip_list *prev_ip = data;
1440 if (prev_ip == NULL) {
1443 if (this_ip->pnn == -1) {
1444 this_ip->pnn = prev_ip->pnn;
1450 static int getips_count_callback(void *param, void *data)
1452 struct ctdb_public_ip_list **ip_list = (struct ctdb_public_ip_list **)param;
1453 struct ctdb_public_ip_list *new_ip = (struct ctdb_public_ip_list *)data;
1455 new_ip->next = *ip_list;
1460 static struct ctdb_public_ip_list *
1461 create_merged_ip_list(struct ctdb_context *ctdb)
1464 struct ctdb_public_ip_list *ip_list;
1465 struct ctdb_all_public_ips *public_ips;
1467 if (ctdb->ip_tree != NULL) {
1468 talloc_free(ctdb->ip_tree);
1469 ctdb->ip_tree = NULL;
1471 ctdb->ip_tree = trbt_create(ctdb, 0);
1473 for (i=0;i<ctdb->num_nodes;i++) {
1474 public_ips = ctdb->nodes[i]->known_public_ips;
1476 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
1480 /* there were no public ips for this node */
1481 if (public_ips == NULL) {
1485 for (j=0;j<public_ips->num;j++) {
1486 struct ctdb_public_ip_list *tmp_ip;
1488 tmp_ip = talloc_zero(ctdb->ip_tree, struct ctdb_public_ip_list);
1489 CTDB_NO_MEMORY_NULL(ctdb, tmp_ip);
1490 /* Do not use information about IP addresses hosted
1491 * on other nodes, it may not be accurate */
1492 if (public_ips->ips[j].pnn == ctdb->nodes[i]->pnn) {
1493 tmp_ip->pnn = public_ips->ips[j].pnn;
1497 tmp_ip->addr = public_ips->ips[j].addr;
1498 tmp_ip->next = NULL;
1500 trbt_insertarray32_callback(ctdb->ip_tree,
1501 IP_KEYLEN, ip_key(&public_ips->ips[j].addr),
1508 trbt_traversearray32(ctdb->ip_tree, IP_KEYLEN, getips_count_callback, &ip_list);
1514 * This is the length of the longtest common prefix between the IPs.
1515 * It is calculated by XOR-ing the 2 IPs together and counting the
1516 * number of leading zeroes. The implementation means that all
1517 * addresses end up being 128 bits long.
1519 * FIXME? Should we consider IPv4 and IPv6 separately given that the
1520 * 12 bytes of 0 prefix padding will hurt the algorithm if there are
1521 * lots of nodes and IP addresses?
1523 static uint32_t ip_distance(ctdb_sock_addr *ip1, ctdb_sock_addr *ip2)
1525 uint32_t ip1_k[IP_KEYLEN];
1530 uint32_t distance = 0;
1532 memcpy(ip1_k, ip_key(ip1), sizeof(ip1_k));
1534 for (i=0; i<IP_KEYLEN; i++) {
1535 x = ip1_k[i] ^ t[i];
1539 /* Count number of leading zeroes.
1540 * FIXME? This could be optimised...
1542 while ((x & (1 << 31)) == 0) {
1552 /* Calculate the IP distance for the given IP relative to IPs on the
1553 given node. The ips argument is generally the all_ips variable
1554 used in the main part of the algorithm.
1556 static uint32_t ip_distance_2_sum(ctdb_sock_addr *ip,
1557 struct ctdb_public_ip_list *ips,
1560 struct ctdb_public_ip_list *t;
1565 for (t=ips; t != NULL; t=t->next) {
1566 if (t->pnn != pnn) {
1570 /* Optimisation: We never calculate the distance
1571 * between an address and itself. This allows us to
1572 * calculate the effect of removing an address from a
1573 * node by simply calculating the distance between
1574 * that address and all of the exitsing addresses.
1575 * Moreover, we assume that we're only ever dealing
1576 * with addresses from all_ips so we can identify an
1577 * address via a pointer rather than doing a more
1578 * expensive address comparison. */
1579 if (&(t->addr) == ip) {
1583 d = ip_distance(ip, &(t->addr));
1584 sum += d * d; /* Cheaper than pulling in math.h :-) */
1590 /* Return the LCP2 imbalance metric for addresses currently assigned
1593 static uint32_t lcp2_imbalance(struct ctdb_public_ip_list * all_ips, int pnn)
1595 struct ctdb_public_ip_list *t;
1597 uint32_t imbalance = 0;
1599 for (t=all_ips; t!=NULL; t=t->next) {
1600 if (t->pnn != pnn) {
1603 /* Pass the rest of the IPs rather than the whole
1606 imbalance += ip_distance_2_sum(&(t->addr), t->next, pnn);
1612 /* Allocate any unassigned IPs just by looping through the IPs and
1613 * finding the best node for each.
1615 static void basic_allocate_unassigned(struct ctdb_context *ctdb,
1616 struct ctdb_ipflags *ipflags,
1617 struct ctdb_public_ip_list *all_ips)
1619 struct ctdb_public_ip_list *tmp_ip;
1621 /* loop over all ip's and find a physical node to cover for
1624 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1625 if (tmp_ip->pnn == -1) {
1626 if (find_takeover_node(ctdb, ipflags, tmp_ip, all_ips)) {
1627 DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
1628 ctdb_addr_to_str(&tmp_ip->addr)));
1634 /* Basic non-deterministic rebalancing algorithm.
1636 static void basic_failback(struct ctdb_context *ctdb,
1637 struct ctdb_ipflags *ipflags,
1638 struct ctdb_public_ip_list *all_ips,
1642 int maxnode, maxnum, minnode, minnum, num, retries;
1643 struct ctdb_public_ip_list *tmp_ip;
1645 numnodes = talloc_array_length(ipflags);
1652 /* for each ip address, loop over all nodes that can serve
1653 this ip and make sure that the difference between the node
1654 serving the most and the node serving the least ip's are
1657 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1658 if (tmp_ip->pnn == -1) {
1662 /* Get the highest and lowest number of ips's served by any
1663 valid node which can serve this ip.
1667 for (i=0; i<numnodes; i++) {
1668 /* only check nodes that can actually serve this ip */
1669 if (!can_node_takeover_ip(ctdb, i, ipflags[i], tmp_ip)) {
1670 /* no it couldnt so skip to the next node */
1674 num = node_ip_coverage(ctdb, i, all_ips);
1675 if (maxnode == -1) {
1684 if (minnode == -1) {
1694 if (maxnode == -1) {
1695 DEBUG(DEBUG_WARNING,(__location__ " Could not find maxnode. May not be able to serve ip '%s'\n",
1696 ctdb_addr_to_str(&tmp_ip->addr)));
1701 /* if the spread between the smallest and largest coverage by
1702 a node is >=2 we steal one of the ips from the node with
1703 most coverage to even things out a bit.
1704 try to do this a limited number of times since we dont
1705 want to spend too much time balancing the ip coverage.
1707 if ( (maxnum > minnum+1)
1708 && (retries < (num_ips + 5)) ){
1709 struct ctdb_public_ip_list *tmp;
1711 /* Reassign one of maxnode's VNNs */
1712 for (tmp=all_ips;tmp;tmp=tmp->next) {
1713 if (tmp->pnn == maxnode) {
1714 (void)find_takeover_node(ctdb, ipflags, tmp, all_ips);
1723 static void lcp2_init(struct ctdb_context *tmp_ctx,
1724 struct ctdb_ipflags *ipflags,
1725 struct ctdb_public_ip_list *all_ips,
1726 uint32_t *force_rebalance_nodes,
1727 uint32_t **lcp2_imbalances,
1728 bool **rebalance_candidates)
1731 struct ctdb_public_ip_list *tmp_ip;
1733 numnodes = talloc_array_length(ipflags);
1735 *rebalance_candidates = talloc_array(tmp_ctx, bool, numnodes);
1736 CTDB_NO_MEMORY_FATAL(tmp_ctx, *rebalance_candidates);
1737 *lcp2_imbalances = talloc_array(tmp_ctx, uint32_t, numnodes);
1738 CTDB_NO_MEMORY_FATAL(tmp_ctx, *lcp2_imbalances);
1740 for (i=0; i<numnodes; i++) {
1741 (*lcp2_imbalances)[i] = lcp2_imbalance(all_ips, i);
1742 /* First step: assume all nodes are candidates */
1743 (*rebalance_candidates)[i] = true;
1746 /* 2nd step: if a node has IPs assigned then it must have been
1747 * healthy before, so we remove it from consideration. This
1748 * is overkill but is all we have because we don't maintain
1749 * state between takeover runs. An alternative would be to
1750 * keep state and invalidate it every time the recovery master
1753 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1754 if (tmp_ip->pnn != -1) {
1755 (*rebalance_candidates)[tmp_ip->pnn] = false;
1759 /* 3rd step: if a node is forced to re-balance then
1760 we allow failback onto the node */
1761 if (force_rebalance_nodes == NULL) {
1764 for (i = 0; i < talloc_array_length(force_rebalance_nodes); i++) {
1765 uint32_t pnn = force_rebalance_nodes[i];
1766 if (pnn >= numnodes) {
1768 (__location__ "unknown node %u\n", pnn));
1773 ("Forcing rebalancing of IPs to node %u\n", pnn));
1774 (*rebalance_candidates)[pnn] = true;
1778 /* Allocate any unassigned addresses using the LCP2 algorithm to find
1779 * the IP/node combination that will cost the least.
1781 static void lcp2_allocate_unassigned(struct ctdb_context *ctdb,
1782 struct ctdb_ipflags *ipflags,
1783 struct ctdb_public_ip_list *all_ips,
1784 uint32_t *lcp2_imbalances)
1786 struct ctdb_public_ip_list *tmp_ip;
1787 int dstnode, numnodes;
1790 uint32_t mindsum, dstdsum, dstimbl, minimbl;
1791 struct ctdb_public_ip_list *minip;
1793 bool should_loop = true;
1794 bool have_unassigned = true;
1796 numnodes = talloc_array_length(ipflags);
1798 while (have_unassigned && should_loop) {
1799 should_loop = false;
1801 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1802 DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES (UNASSIGNED)\n"));
1808 /* loop over each unassigned ip. */
1809 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1810 if (tmp_ip->pnn != -1) {
1814 for (dstnode=0; dstnode<numnodes; dstnode++) {
1815 /* only check nodes that can actually takeover this ip */
1816 if (!can_node_takeover_ip(ctdb, dstnode,
1819 /* no it couldnt so skip to the next node */
1823 dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
1824 dstimbl = lcp2_imbalances[dstnode] + dstdsum;
1825 DEBUG(DEBUG_DEBUG,(" %s -> %d [+%d]\n",
1826 ctdb_addr_to_str(&(tmp_ip->addr)),
1828 dstimbl - lcp2_imbalances[dstnode]));
1831 if ((minnode == -1) || (dstdsum < mindsum)) {
1841 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1843 /* If we found one then assign it to the given node. */
1844 if (minnode != -1) {
1845 minip->pnn = minnode;
1846 lcp2_imbalances[minnode] = minimbl;
1847 DEBUG(DEBUG_INFO,(" %s -> %d [+%d]\n",
1848 ctdb_addr_to_str(&(minip->addr)),
1853 /* There might be a better way but at least this is clear. */
1854 have_unassigned = false;
1855 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1856 if (tmp_ip->pnn == -1) {
1857 have_unassigned = true;
1862 /* We know if we have an unassigned addresses so we might as
1865 if (have_unassigned) {
1866 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1867 if (tmp_ip->pnn == -1) {
1868 DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
1869 ctdb_addr_to_str(&tmp_ip->addr)));
1875 /* LCP2 algorithm for rebalancing the cluster. Given a candidate node
1876 * to move IPs from, determines the best IP/destination node
1877 * combination to move from the source node.
1879 static bool lcp2_failback_candidate(struct ctdb_context *ctdb,
1880 struct ctdb_ipflags *ipflags,
1881 struct ctdb_public_ip_list *all_ips,
1883 uint32_t *lcp2_imbalances,
1884 bool *rebalance_candidates)
1886 int dstnode, mindstnode, numnodes;
1887 uint32_t srcimbl, srcdsum, dstimbl, dstdsum;
1888 uint32_t minsrcimbl, mindstimbl;
1889 struct ctdb_public_ip_list *minip;
1890 struct ctdb_public_ip_list *tmp_ip;
1892 /* Find an IP and destination node that best reduces imbalance. */
1899 numnodes = talloc_array_length(ipflags);
1901 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1902 DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES FROM %d [%d]\n",
1903 srcnode, lcp2_imbalances[srcnode]));
1905 for (tmp_ip=all_ips; tmp_ip; tmp_ip=tmp_ip->next) {
1906 /* Only consider addresses on srcnode. */
1907 if (tmp_ip->pnn != srcnode) {
1911 /* What is this IP address costing the source node? */
1912 srcdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, srcnode);
1913 srcimbl = lcp2_imbalances[srcnode] - srcdsum;
1915 /* Consider this IP address would cost each potential
1916 * destination node. Destination nodes are limited to
1917 * those that are newly healthy, since we don't want
1918 * to do gratuitous failover of IPs just to make minor
1919 * balance improvements.
1921 for (dstnode=0; dstnode<numnodes; dstnode++) {
1922 if (!rebalance_candidates[dstnode]) {
1926 /* only check nodes that can actually takeover this ip */
1927 if (!can_node_takeover_ip(ctdb, dstnode,
1928 ipflags[dstnode], tmp_ip)) {
1929 /* no it couldnt so skip to the next node */
1933 dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
1934 dstimbl = lcp2_imbalances[dstnode] + dstdsum;
1935 DEBUG(DEBUG_DEBUG,(" %d [%d] -> %s -> %d [+%d]\n",
1937 ctdb_addr_to_str(&(tmp_ip->addr)),
1940 if ((dstimbl < lcp2_imbalances[srcnode]) &&
1941 (dstdsum < srcdsum) && \
1942 ((mindstnode == -1) || \
1943 ((srcimbl + dstimbl) < (minsrcimbl + mindstimbl)))) {
1946 minsrcimbl = srcimbl;
1947 mindstnode = dstnode;
1948 mindstimbl = dstimbl;
1952 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1954 if (mindstnode != -1) {
1955 /* We found a move that makes things better... */
1956 DEBUG(DEBUG_INFO,("%d [%d] -> %s -> %d [+%d]\n",
1957 srcnode, minsrcimbl - lcp2_imbalances[srcnode],
1958 ctdb_addr_to_str(&(minip->addr)),
1959 mindstnode, mindstimbl - lcp2_imbalances[mindstnode]));
1962 lcp2_imbalances[srcnode] = minsrcimbl;
1963 lcp2_imbalances[mindstnode] = mindstimbl;
1964 minip->pnn = mindstnode;
1973 struct lcp2_imbalance_pnn {
1978 static int lcp2_cmp_imbalance_pnn(const void * a, const void * b)
1980 const struct lcp2_imbalance_pnn * lipa = (const struct lcp2_imbalance_pnn *) a;
1981 const struct lcp2_imbalance_pnn * lipb = (const struct lcp2_imbalance_pnn *) b;
1983 if (lipa->imbalance > lipb->imbalance) {
1985 } else if (lipa->imbalance == lipb->imbalance) {
1992 /* LCP2 algorithm for rebalancing the cluster. This finds the source
1993 * node with the highest LCP2 imbalance, and then determines the best
1994 * IP/destination node combination to move from the source node.
1996 static void lcp2_failback(struct ctdb_context *ctdb,
1997 struct ctdb_ipflags *ipflags,
1998 struct ctdb_public_ip_list *all_ips,
1999 uint32_t *lcp2_imbalances,
2000 bool *rebalance_candidates)
2003 struct lcp2_imbalance_pnn * lips;
2006 numnodes = talloc_array_length(ipflags);
2009 /* Put the imbalances and nodes into an array, sort them and
2010 * iterate through candidates. Usually the 1st one will be
2011 * used, so this doesn't cost much...
2013 DEBUG(DEBUG_DEBUG,("+++++++++++++++++++++++++++++++++++++++++\n"));
2014 DEBUG(DEBUG_DEBUG,("Selecting most imbalanced node from:\n"));
2015 lips = talloc_array(ctdb, struct lcp2_imbalance_pnn, numnodes);
2016 for (i=0; i<numnodes; i++) {
2017 lips[i].imbalance = lcp2_imbalances[i];
2019 DEBUG(DEBUG_DEBUG,(" %d [%d]\n", i, lcp2_imbalances[i]));
2021 qsort(lips, numnodes, sizeof(struct lcp2_imbalance_pnn),
2022 lcp2_cmp_imbalance_pnn);
2025 for (i=0; i<numnodes; i++) {
2026 /* This means that all nodes had 0 or 1 addresses, so
2027 * can't be imbalanced.
2029 if (lips[i].imbalance == 0) {
2033 if (lcp2_failback_candidate(ctdb,
2038 rebalance_candidates)) {
2050 static void unassign_unsuitable_ips(struct ctdb_context *ctdb,
2051 struct ctdb_ipflags *ipflags,
2052 struct ctdb_public_ip_list *all_ips)
2054 struct ctdb_public_ip_list *tmp_ip;
2056 /* verify that the assigned nodes can serve that public ip
2057 and set it to -1 if not
2059 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2060 if (tmp_ip->pnn == -1) {
2063 if (!can_node_host_ip(ctdb, tmp_ip->pnn,
2064 ipflags[tmp_ip->pnn], tmp_ip) != 0) {
2065 /* this node can not serve this ip. */
2066 DEBUG(DEBUG_DEBUG,("Unassign IP: %s from %d\n",
2067 ctdb_addr_to_str(&(tmp_ip->addr)),
2074 static void ip_alloc_deterministic_ips(struct ctdb_context *ctdb,
2075 struct ctdb_ipflags *ipflags,
2076 struct ctdb_public_ip_list *all_ips)
2078 struct ctdb_public_ip_list *tmp_ip;
2081 numnodes = talloc_array_length(ipflags);
2083 DEBUG(DEBUG_NOTICE,("Deterministic IPs enabled. Resetting all ip allocations\n"));
2084 /* Allocate IPs to nodes in a modulo fashion so that IPs will
2085 * always be allocated the same way for a specific set of
2086 * available/unavailable nodes.
2089 for (i=0,tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next,i++) {
2090 tmp_ip->pnn = i % numnodes;
2093 /* IP failback doesn't make sense with deterministic
2094 * IPs, since the modulo step above implicitly fails
2095 * back IPs to their "home" node.
2097 if (1 == ctdb->tunable.no_ip_failback) {
2098 DEBUG(DEBUG_WARNING, ("WARNING: 'NoIPFailback' set but ignored - incompatible with 'DeterministicIPs\n"));
2101 unassign_unsuitable_ips(ctdb, ipflags, all_ips);
2103 basic_allocate_unassigned(ctdb, ipflags, all_ips);
2105 /* No failback here! */
2108 static void ip_alloc_nondeterministic_ips(struct ctdb_context *ctdb,
2109 struct ctdb_ipflags *ipflags,
2110 struct ctdb_public_ip_list *all_ips)
2112 /* This should be pushed down into basic_failback. */
2113 struct ctdb_public_ip_list *tmp_ip;
2115 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2119 unassign_unsuitable_ips(ctdb, ipflags, all_ips);
2121 basic_allocate_unassigned(ctdb, ipflags, all_ips);
2123 /* If we don't want IPs to fail back then don't rebalance IPs. */
2124 if (1 == ctdb->tunable.no_ip_failback) {
2128 /* Now, try to make sure the ip adresses are evenly distributed
2131 basic_failback(ctdb, ipflags, all_ips, num_ips);
2134 static void ip_alloc_lcp2(struct ctdb_context *ctdb,
2135 struct ctdb_ipflags *ipflags,
2136 struct ctdb_public_ip_list *all_ips,
2137 uint32_t *force_rebalance_nodes)
2139 uint32_t *lcp2_imbalances;
2140 bool *rebalance_candidates;
2141 int numnodes, num_rebalance_candidates, i;
2143 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2145 unassign_unsuitable_ips(ctdb, ipflags, all_ips);
2147 lcp2_init(tmp_ctx, ipflags, all_ips,force_rebalance_nodes,
2148 &lcp2_imbalances, &rebalance_candidates);
2150 lcp2_allocate_unassigned(ctdb, ipflags, all_ips, lcp2_imbalances);
2152 /* If we don't want IPs to fail back then don't rebalance IPs. */
2153 if (1 == ctdb->tunable.no_ip_failback) {
2157 /* It is only worth continuing if we have suitable target
2158 * nodes to transfer IPs to. This check is much cheaper than
2161 numnodes = talloc_array_length(ipflags);
2162 num_rebalance_candidates = 0;
2163 for (i=0; i<numnodes; i++) {
2164 if (rebalance_candidates[i]) {
2165 num_rebalance_candidates++;
2168 if (num_rebalance_candidates == 0) {
2172 /* Now, try to make sure the ip adresses are evenly distributed
2175 lcp2_failback(ctdb, ipflags, all_ips,
2176 lcp2_imbalances, rebalance_candidates);
2179 talloc_free(tmp_ctx);
2182 static bool all_nodes_are_disabled(struct ctdb_node_map *nodemap)
2186 /* Count how many completely healthy nodes we have */
2188 for (i=0;i<nodemap->num;i++) {
2189 if (!(nodemap->nodes[i].flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED))) {
2194 return num_healthy == 0;
2197 /* The calculation part of the IP allocation algorithm. */
2198 static void ctdb_takeover_run_core(struct ctdb_context *ctdb,
2199 struct ctdb_ipflags *ipflags,
2200 struct ctdb_public_ip_list **all_ips_p,
2201 uint32_t *force_rebalance_nodes)
2203 /* since nodes only know about those public addresses that
2204 can be served by that particular node, no single node has
2205 a full list of all public addresses that exist in the cluster.
2206 Walk over all node structures and create a merged list of
2207 all public addresses that exist in the cluster.
2209 keep the tree of ips around as ctdb->ip_tree
2211 *all_ips_p = create_merged_ip_list(ctdb);
2213 if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
2214 ip_alloc_lcp2(ctdb, ipflags, *all_ips_p, force_rebalance_nodes);
2215 } else if (1 == ctdb->tunable.deterministic_public_ips) {
2216 ip_alloc_deterministic_ips(ctdb, ipflags, *all_ips_p);
2218 ip_alloc_nondeterministic_ips(ctdb, ipflags, *all_ips_p);
2221 /* at this point ->pnn is the node which will own each IP
2222 or -1 if there is no node that can cover this ip
2228 struct get_tunable_callback_data {
2229 const char *tunable;
2234 static void get_tunable_callback(struct ctdb_context *ctdb, uint32_t pnn,
2235 int32_t res, TDB_DATA outdata,
2238 struct get_tunable_callback_data *cd =
2239 (struct get_tunable_callback_data *)callback;
2243 /* Already handled in fail callback */
2247 if (outdata.dsize != sizeof(uint32_t)) {
2248 DEBUG(DEBUG_ERR,("Wrong size of returned data when reading \"%s\" tunable from node %d. Expected %d bytes but received %d bytes\n",
2249 cd->tunable, pnn, (int)sizeof(uint32_t),
2250 (int)outdata.dsize));
2255 size = talloc_array_length(cd->out);
2257 DEBUG(DEBUG_ERR,("Got %s reply from node %d but nodemap only has %d entries\n",
2258 cd->tunable, pnn, size));
2263 cd->out[pnn] = *(uint32_t *)outdata.dptr;
2266 static void get_tunable_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
2267 int32_t res, TDB_DATA outdata,
2270 struct get_tunable_callback_data *cd =
2271 (struct get_tunable_callback_data *)callback;
2276 ("Timed out getting tunable \"%s\" from node %d\n",
2282 DEBUG(DEBUG_WARNING,
2283 ("Tunable \"%s\" not implemented on node %d\n",
2288 ("Unexpected error getting tunable \"%s\" from node %d\n",
2294 static uint32_t *get_tunable_from_nodes(struct ctdb_context *ctdb,
2295 TALLOC_CTX *tmp_ctx,
2296 struct ctdb_node_map *nodemap,
2297 const char *tunable,
2298 uint32_t default_value)
2301 struct ctdb_control_get_tunable *t;
2304 struct get_tunable_callback_data callback_data;
2307 tvals = talloc_array(tmp_ctx, uint32_t, nodemap->num);
2308 CTDB_NO_MEMORY_NULL(ctdb, tvals);
2309 for (i=0; i<nodemap->num; i++) {
2310 tvals[i] = default_value;
2313 callback_data.out = tvals;
2314 callback_data.tunable = tunable;
2315 callback_data.fatal = false;
2317 data.dsize = offsetof(struct ctdb_control_get_tunable, name) + strlen(tunable) + 1;
2318 data.dptr = talloc_size(tmp_ctx, data.dsize);
2319 t = (struct ctdb_control_get_tunable *)data.dptr;
2320 t->length = strlen(tunable)+1;
2321 memcpy(t->name, tunable, t->length);
2322 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2323 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_TUNABLE,
2324 nodes, 0, TAKEOVER_TIMEOUT(),
2326 get_tunable_callback,
2327 get_tunable_fail_callback,
2328 &callback_data) != 0) {
2329 if (callback_data.fatal) {
2335 talloc_free(data.dptr);
2340 struct get_runstate_callback_data {
2341 enum ctdb_runstate *out;
2345 static void get_runstate_callback(struct ctdb_context *ctdb, uint32_t pnn,
2346 int32_t res, TDB_DATA outdata,
2347 void *callback_data)
2349 struct get_runstate_callback_data *cd =
2350 (struct get_runstate_callback_data *)callback_data;
2354 /* Already handled in fail callback */
2358 if (outdata.dsize != sizeof(uint32_t)) {
2359 DEBUG(DEBUG_ERR,("Wrong size of returned data when getting runstate from node %d. Expected %d bytes but received %d bytes\n",
2360 pnn, (int)sizeof(uint32_t),
2361 (int)outdata.dsize));
2366 size = talloc_array_length(cd->out);
2368 DEBUG(DEBUG_ERR,("Got reply from node %d but nodemap only has %d entries\n",
2373 cd->out[pnn] = (enum ctdb_runstate)*(uint32_t *)outdata.dptr;
2376 static void get_runstate_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
2377 int32_t res, TDB_DATA outdata,
2380 struct get_runstate_callback_data *cd =
2381 (struct get_runstate_callback_data *)callback;
2386 ("Timed out getting runstate from node %d\n", pnn));
2390 DEBUG(DEBUG_WARNING,
2391 ("Error getting runstate from node %d - assuming runstates not supported\n",
2396 static enum ctdb_runstate * get_runstate_from_nodes(struct ctdb_context *ctdb,
2397 TALLOC_CTX *tmp_ctx,
2398 struct ctdb_node_map *nodemap,
2399 enum ctdb_runstate default_value)
2402 enum ctdb_runstate *rs;
2403 struct get_runstate_callback_data callback_data;
2406 rs = talloc_array(tmp_ctx, enum ctdb_runstate, nodemap->num);
2407 CTDB_NO_MEMORY_NULL(ctdb, rs);
2408 for (i=0; i<nodemap->num; i++) {
2409 rs[i] = default_value;
2412 callback_data.out = rs;
2413 callback_data.fatal = false;
2415 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2416 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_RUNSTATE,
2417 nodes, 0, TAKEOVER_TIMEOUT(),
2419 get_runstate_callback,
2420 get_runstate_fail_callback,
2421 &callback_data) != 0) {
2422 if (callback_data.fatal) {
2432 /* Set internal flags for IP allocation:
2434 * Set NOIPTAKOVER ip flags from per-node NoIPTakeover tunable
2435 * Set NOIPHOST ip flag for each INACTIVE node
2436 * if all nodes are disabled:
2437 * Set NOIPHOST ip flags from per-node NoIPHostOnAllDisabled tunable
2439 * Set NOIPHOST ip flags for disabled nodes
2441 static struct ctdb_ipflags *
2442 set_ipflags_internal(struct ctdb_context *ctdb,
2443 TALLOC_CTX *tmp_ctx,
2444 struct ctdb_node_map *nodemap,
2445 uint32_t *tval_noiptakeover,
2446 uint32_t *tval_noiphostonalldisabled,
2447 enum ctdb_runstate *runstate)
2450 struct ctdb_ipflags *ipflags;
2452 /* Clear IP flags - implicit due to talloc_zero */
2453 ipflags = talloc_zero_array(tmp_ctx, struct ctdb_ipflags, nodemap->num);
2454 CTDB_NO_MEMORY_NULL(ctdb, ipflags);
2456 for (i=0;i<nodemap->num;i++) {
2457 /* Can not take IPs on node with NoIPTakeover set */
2458 if (tval_noiptakeover[i] != 0) {
2459 ipflags[i].noiptakeover = true;
2462 /* Can not host IPs on node not in RUNNING state */
2463 if (runstate[i] != CTDB_RUNSTATE_RUNNING) {
2464 ipflags[i].noiphost = true;
2467 /* Can not host IPs on INACTIVE node */
2468 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2469 ipflags[i].noiphost = true;
2473 if (all_nodes_are_disabled(nodemap)) {
2474 /* If all nodes are disabled, can not host IPs on node
2475 * with NoIPHostOnAllDisabled set
2477 for (i=0;i<nodemap->num;i++) {
2478 if (tval_noiphostonalldisabled[i] != 0) {
2479 ipflags[i].noiphost = true;
2483 /* If some nodes are not disabled, then can not host
2484 * IPs on DISABLED node
2486 for (i=0;i<nodemap->num;i++) {
2487 if (nodemap->nodes[i].flags & NODE_FLAGS_DISABLED) {
2488 ipflags[i].noiphost = true;
2496 static struct ctdb_ipflags *set_ipflags(struct ctdb_context *ctdb,
2497 TALLOC_CTX *tmp_ctx,
2498 struct ctdb_node_map *nodemap)
2500 uint32_t *tval_noiptakeover;
2501 uint32_t *tval_noiphostonalldisabled;
2502 struct ctdb_ipflags *ipflags;
2503 enum ctdb_runstate *runstate;
2506 tval_noiptakeover = get_tunable_from_nodes(ctdb, tmp_ctx, nodemap,
2508 if (tval_noiptakeover == NULL) {
2512 tval_noiphostonalldisabled =
2513 get_tunable_from_nodes(ctdb, tmp_ctx, nodemap,
2514 "NoIPHostOnAllDisabled", 0);
2515 if (tval_noiphostonalldisabled == NULL) {
2516 /* Caller frees tmp_ctx */
2520 /* Any nodes where CTDB_CONTROL_GET_RUNSTATE is not supported
2521 * will default to CTDB_RUNSTATE_RUNNING. This ensures
2522 * reasonable behaviour on a mixed cluster during upgrade.
2524 runstate = get_runstate_from_nodes(ctdb, tmp_ctx, nodemap,
2525 CTDB_RUNSTATE_RUNNING);
2526 if (runstate == NULL) {
2527 /* Caller frees tmp_ctx */
2531 ipflags = set_ipflags_internal(ctdb, tmp_ctx, nodemap,
2533 tval_noiphostonalldisabled,
2536 talloc_free(tval_noiptakeover);
2537 talloc_free(tval_noiphostonalldisabled);
2538 talloc_free(runstate);
2543 struct iprealloc_callback_data {
2546 client_async_callback fail_callback;
2547 void *fail_callback_data;
2548 struct ctdb_node_map *nodemap;
2551 static void iprealloc_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
2552 int32_t res, TDB_DATA outdata,
2556 struct iprealloc_callback_data *cd =
2557 (struct iprealloc_callback_data *)callback;
2559 numnodes = talloc_array_length(cd->retry_nodes);
2560 if (pnn > numnodes) {
2562 ("ipreallocated failure from node %d, "
2563 "but only %d nodes in nodemap\n",
2568 /* Can't run the "ipreallocated" event on a INACTIVE node */
2569 if (cd->nodemap->nodes[pnn].flags & NODE_FLAGS_INACTIVE) {
2570 DEBUG(DEBUG_WARNING,
2571 ("ipreallocated failed on inactive node %d, ignoring\n",
2578 /* If the control timed out then that's a real error,
2579 * so call the real fail callback
2581 if (cd->fail_callback) {
2582 cd->fail_callback(ctdb, pnn, res, outdata,
2583 cd->fail_callback_data);
2585 DEBUG(DEBUG_WARNING,
2586 ("iprealloc timed out but no callback registered\n"));
2590 /* If not a timeout then either the ipreallocated
2591 * eventscript (or some setup) failed. This might
2592 * have failed because the IPREALLOCATED control isn't
2593 * implemented - right now there is no way of knowing
2594 * because the error codes are all folded down to -1.
2595 * Consider retrying using EVENTSCRIPT control...
2597 DEBUG(DEBUG_WARNING,
2598 ("ipreallocated failure from node %d, flagging retry\n",
2600 cd->retry_nodes[pnn] = true;
2605 struct takeover_callback_data {
2607 client_async_callback fail_callback;
2608 void *fail_callback_data;
2609 struct ctdb_node_map *nodemap;
2612 static void takeover_run_fail_callback(struct ctdb_context *ctdb,
2613 uint32_t node_pnn, int32_t res,
2614 TDB_DATA outdata, void *callback_data)
2616 struct takeover_callback_data *cd =
2617 talloc_get_type_abort(callback_data,
2618 struct takeover_callback_data);
2621 for (i = 0; i < cd->nodemap->num; i++) {
2622 if (node_pnn == cd->nodemap->nodes[i].pnn) {
2627 if (i == cd->nodemap->num) {
2628 DEBUG(DEBUG_ERR, (__location__ " invalid PNN %u\n", node_pnn));
2632 if (!cd->node_failed[i]) {
2633 cd->node_failed[i] = true;
2634 cd->fail_callback(ctdb, node_pnn, res, outdata,
2635 cd->fail_callback_data);
2640 make any IP alias changes for public addresses that are necessary
2642 int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
2643 uint32_t *force_rebalance_nodes,
2644 client_async_callback fail_callback, void *callback_data)
2647 struct ctdb_public_ip ip;
2648 struct ctdb_public_ipv4 ipv4;
2650 struct ctdb_public_ip_list *all_ips, *tmp_ip;
2652 struct timeval timeout;
2653 struct client_async_data *async_data;
2654 struct ctdb_client_control_state *state;
2655 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2656 struct ctdb_ipflags *ipflags;
2657 struct takeover_callback_data *takeover_data;
2658 struct iprealloc_callback_data iprealloc_data;
2662 * ip failover is completely disabled, just send out the
2663 * ipreallocated event.
2665 if (ctdb->tunable.disable_ip_failover != 0) {
2669 ipflags = set_ipflags(ctdb, tmp_ctx, nodemap);
2670 if (ipflags == NULL) {
2671 DEBUG(DEBUG_ERR,("Failed to set IP flags - aborting takeover run\n"));
2672 talloc_free(tmp_ctx);
2678 /* Do the IP reassignment calculations */
2679 ctdb_takeover_run_core(ctdb, ipflags, &all_ips, force_rebalance_nodes);
2681 /* Now tell all nodes to release any public IPs should not
2682 * host. This will be a NOOP on nodes that don't currently
2683 * hold the given IP.
2685 takeover_data = talloc_zero(tmp_ctx, struct takeover_callback_data);
2686 CTDB_NO_MEMORY_FATAL(ctdb, takeover_data);
2688 takeover_data->node_failed = talloc_zero_array(tmp_ctx,
2689 bool, nodemap->num);
2690 CTDB_NO_MEMORY_FATAL(ctdb, takeover_data->node_failed);
2691 takeover_data->fail_callback = fail_callback;
2692 takeover_data->fail_callback_data = callback_data;
2693 takeover_data->nodemap = nodemap;
2695 async_data = talloc_zero(tmp_ctx, struct client_async_data);
2696 CTDB_NO_MEMORY_FATAL(ctdb, async_data);
2698 async_data->fail_callback = takeover_run_fail_callback;
2699 async_data->callback_data = takeover_data;
2701 for (i=0;i<nodemap->num;i++) {
2702 /* don't talk to unconnected nodes, but do talk to banned nodes */
2703 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2707 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2708 if (tmp_ip->pnn == nodemap->nodes[i].pnn) {
2709 /* This node should be serving this
2710 vnn so dont tell it to release the ip
2714 if (tmp_ip->addr.sa.sa_family == AF_INET) {
2715 ipv4.pnn = tmp_ip->pnn;
2716 ipv4.sin = tmp_ip->addr.ip;
2718 timeout = TAKEOVER_TIMEOUT();
2719 data.dsize = sizeof(ipv4);
2720 data.dptr = (uint8_t *)&ipv4;
2721 state = ctdb_control_send(ctdb, nodemap->nodes[i].pnn,
2722 0, CTDB_CONTROL_RELEASE_IPv4, 0,
2726 ip.pnn = tmp_ip->pnn;
2727 ip.addr = tmp_ip->addr;
2729 timeout = TAKEOVER_TIMEOUT();
2730 data.dsize = sizeof(ip);
2731 data.dptr = (uint8_t *)&ip;
2732 state = ctdb_control_send(ctdb, nodemap->nodes[i].pnn,
2733 0, CTDB_CONTROL_RELEASE_IP, 0,
2738 if (state == NULL) {
2739 DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_RELEASE_IP to node %u\n", nodemap->nodes[i].pnn));
2740 talloc_free(tmp_ctx);
2744 ctdb_client_async_add(async_data, state);
2747 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2748 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_RELEASE_IP failed\n"));
2749 talloc_free(tmp_ctx);
2752 talloc_free(async_data);
2755 /* tell all nodes to get their own IPs */
2756 async_data = talloc_zero(tmp_ctx, struct client_async_data);
2757 CTDB_NO_MEMORY_FATAL(ctdb, async_data);
2759 async_data->fail_callback = fail_callback;
2760 async_data->callback_data = callback_data;
2762 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2763 if (tmp_ip->pnn == -1) {
2764 /* this IP won't be taken over */
2768 if (tmp_ip->addr.sa.sa_family == AF_INET) {
2769 ipv4.pnn = tmp_ip->pnn;
2770 ipv4.sin = tmp_ip->addr.ip;
2772 timeout = TAKEOVER_TIMEOUT();
2773 data.dsize = sizeof(ipv4);
2774 data.dptr = (uint8_t *)&ipv4;
2775 state = ctdb_control_send(ctdb, tmp_ip->pnn,
2776 0, CTDB_CONTROL_TAKEOVER_IPv4, 0,
2780 ip.pnn = tmp_ip->pnn;
2781 ip.addr = tmp_ip->addr;
2783 timeout = TAKEOVER_TIMEOUT();
2784 data.dsize = sizeof(ip);
2785 data.dptr = (uint8_t *)&ip;
2786 state = ctdb_control_send(ctdb, tmp_ip->pnn,
2787 0, CTDB_CONTROL_TAKEOVER_IP, 0,
2791 if (state == NULL) {
2792 DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_TAKEOVER_IP to node %u\n", tmp_ip->pnn));
2793 talloc_free(tmp_ctx);
2797 ctdb_client_async_add(async_data, state);
2799 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2800 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_TAKEOVER_IP failed\n"));
2801 talloc_free(tmp_ctx);
2807 * Tell all nodes to run eventscripts to process the
2808 * "ipreallocated" event. This can do a lot of things,
2809 * including restarting services to reconfigure them if public
2810 * IPs have moved. Once upon a time this event only used to
2813 retry_data = talloc_zero_array(tmp_ctx, bool, nodemap->num);
2814 CTDB_NO_MEMORY_FATAL(ctdb, retry_data);
2815 iprealloc_data.retry_nodes = retry_data;
2816 iprealloc_data.retry_count = 0;
2817 iprealloc_data.fail_callback = fail_callback;
2818 iprealloc_data.fail_callback_data = callback_data;
2819 iprealloc_data.nodemap = nodemap;
2821 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2822 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_IPREALLOCATED,
2823 nodes, 0, TAKEOVER_TIMEOUT(),
2825 NULL, iprealloc_fail_callback,
2828 /* If the control failed then we should retry to any
2829 * nodes flagged by iprealloc_fail_callback using the
2830 * EVENTSCRIPT control. This is a best-effort at
2831 * backward compatiblity when running a mixed cluster
2832 * where some nodes have not yet been upgraded to
2833 * support the IPREALLOCATED control.
2835 DEBUG(DEBUG_WARNING,
2836 ("Retry ipreallocated to some nodes using eventscript control\n"));
2838 nodes = talloc_array(tmp_ctx, uint32_t,
2839 iprealloc_data.retry_count);
2840 CTDB_NO_MEMORY_FATAL(ctdb, nodes);
2843 for (i=0; i<nodemap->num; i++) {
2844 if (iprealloc_data.retry_nodes[i]) {
2850 data.dptr = discard_const("ipreallocated");
2851 data.dsize = strlen((char *)data.dptr) + 1;
2852 ret = ctdb_client_async_control(ctdb,
2853 CTDB_CONTROL_RUN_EVENTSCRIPTS,
2854 nodes, 0, TAKEOVER_TIMEOUT(),
2856 NULL, fail_callback,
2859 DEBUG(DEBUG_ERR, (__location__ " failed to send control to run eventscripts with \"ipreallocated\"\n"));
2863 talloc_free(tmp_ctx);
2869 destroy a ctdb_client_ip structure
2871 static int ctdb_client_ip_destructor(struct ctdb_client_ip *ip)
2873 DEBUG(DEBUG_DEBUG,("destroying client tcp for %s:%u (client_id %u)\n",
2874 ctdb_addr_to_str(&ip->addr),
2875 ntohs(ip->addr.ip.sin_port),
2878 DLIST_REMOVE(ip->ctdb->client_ip_list, ip);
2883 called by a client to inform us of a TCP connection that it is managing
2884 that should tickled with an ACK when IP takeover is done
2885 we handle both the old ipv4 style of packets as well as the new ipv4/6
2888 int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
2891 struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
2892 struct ctdb_control_tcp *old_addr = NULL;
2893 struct ctdb_control_tcp_addr new_addr;
2894 struct ctdb_control_tcp_addr *tcp_sock = NULL;
2895 struct ctdb_tcp_list *tcp;
2896 struct ctdb_tcp_connection t;
2899 struct ctdb_client_ip *ip;
2900 struct ctdb_vnn *vnn;
2901 ctdb_sock_addr addr;
2903 switch (indata.dsize) {
2904 case sizeof(struct ctdb_control_tcp):
2905 old_addr = (struct ctdb_control_tcp *)indata.dptr;
2906 ZERO_STRUCT(new_addr);
2907 tcp_sock = &new_addr;
2908 tcp_sock->src.ip = old_addr->src;
2909 tcp_sock->dest.ip = old_addr->dest;
2911 case sizeof(struct ctdb_control_tcp_addr):
2912 tcp_sock = (struct ctdb_control_tcp_addr *)indata.dptr;
2915 DEBUG(DEBUG_ERR,(__location__ " Invalid data structure passed "
2916 "to ctdb_control_tcp_client. size was %d but "
2917 "only allowed sizes are %lu and %lu\n",
2919 (long unsigned)sizeof(struct ctdb_control_tcp),
2920 (long unsigned)sizeof(struct ctdb_control_tcp_addr)));
2924 addr = tcp_sock->src;
2925 ctdb_canonicalize_ip(&addr, &tcp_sock->src);
2926 addr = tcp_sock->dest;
2927 ctdb_canonicalize_ip(&addr, &tcp_sock->dest);
2930 memcpy(&addr, &tcp_sock->dest, sizeof(addr));
2931 vnn = find_public_ip_vnn(ctdb, &addr);
2933 switch (addr.sa.sa_family) {
2935 if (ntohl(addr.ip.sin_addr.s_addr) != INADDR_LOOPBACK) {
2936 DEBUG(DEBUG_ERR,("Could not add client IP %s. This is not a public address.\n",
2937 ctdb_addr_to_str(&addr)));
2941 DEBUG(DEBUG_ERR,("Could not add client IP %s. This is not a public ipv6 address.\n",
2942 ctdb_addr_to_str(&addr)));
2945 DEBUG(DEBUG_ERR,(__location__ " Unknown family type %d\n", addr.sa.sa_family));
2951 if (vnn->pnn != ctdb->pnn) {
2952 DEBUG(DEBUG_ERR,("Attempt to register tcp client for IP %s we don't hold - failing (client_id %u pid %u)\n",
2953 ctdb_addr_to_str(&addr),
2954 client_id, client->pid));
2955 /* failing this call will tell smbd to die */
2959 ip = talloc(client, struct ctdb_client_ip);
2960 CTDB_NO_MEMORY(ctdb, ip);
2964 ip->client_id = client_id;
2965 talloc_set_destructor(ip, ctdb_client_ip_destructor);
2966 DLIST_ADD(ctdb->client_ip_list, ip);
2968 tcp = talloc(client, struct ctdb_tcp_list);
2969 CTDB_NO_MEMORY(ctdb, tcp);
2971 tcp->connection.src_addr = tcp_sock->src;
2972 tcp->connection.dst_addr = tcp_sock->dest;
2974 DLIST_ADD(client->tcp_list, tcp);
2976 t.src_addr = tcp_sock->src;
2977 t.dst_addr = tcp_sock->dest;
2979 data.dptr = (uint8_t *)&t;
2980 data.dsize = sizeof(t);
2982 switch (addr.sa.sa_family) {
2984 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
2985 (unsigned)ntohs(tcp_sock->dest.ip.sin_port),
2986 ctdb_addr_to_str(&tcp_sock->src),
2987 (unsigned)ntohs(tcp_sock->src.ip.sin_port), client_id, client->pid));
2990 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
2991 (unsigned)ntohs(tcp_sock->dest.ip6.sin6_port),
2992 ctdb_addr_to_str(&tcp_sock->src),
2993 (unsigned)ntohs(tcp_sock->src.ip6.sin6_port), client_id, client->pid));
2996 DEBUG(DEBUG_ERR,(__location__ " Unknown family %d\n", addr.sa.sa_family));
3000 /* tell all nodes about this tcp connection */
3001 ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
3002 CTDB_CONTROL_TCP_ADD,
3003 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
3005 DEBUG(DEBUG_ERR,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
3013 find a tcp address on a list
3015 static struct ctdb_tcp_connection *ctdb_tcp_find(struct ctdb_tcp_array *array,
3016 struct ctdb_tcp_connection *tcp)
3020 if (array == NULL) {
3024 for (i=0;i<array->num;i++) {
3025 if (ctdb_same_sockaddr(&array->connections[i].src_addr, &tcp->src_addr) &&
3026 ctdb_same_sockaddr(&array->connections[i].dst_addr, &tcp->dst_addr)) {
3027 return &array->connections[i];
3036 called by a daemon to inform us of a TCP connection that one of its
3037 clients managing that should tickled with an ACK when IP takeover is
3040 int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata, bool tcp_update_needed)
3042 struct ctdb_tcp_connection *p = (struct ctdb_tcp_connection *)indata.dptr;
3043 struct ctdb_tcp_array *tcparray;
3044 struct ctdb_tcp_connection tcp;
3045 struct ctdb_vnn *vnn;
3047 vnn = find_public_ip_vnn(ctdb, &p->dst_addr);
3049 DEBUG(DEBUG_INFO,(__location__ " got TCP_ADD control for an address which is not a public address '%s'\n",
3050 ctdb_addr_to_str(&p->dst_addr)));
3056 tcparray = vnn->tcp_array;
3058 /* If this is the first tickle */
3059 if (tcparray == NULL) {
3060 tcparray = talloc(vnn, struct ctdb_tcp_array);
3061 CTDB_NO_MEMORY(ctdb, tcparray);
3062 vnn->tcp_array = tcparray;
3065 tcparray->connections = talloc_size(tcparray, sizeof(struct ctdb_tcp_connection));
3066 CTDB_NO_MEMORY(ctdb, tcparray->connections);
3068 tcparray->connections[tcparray->num].src_addr = p->src_addr;
3069 tcparray->connections[tcparray->num].dst_addr = p->dst_addr;
3072 if (tcp_update_needed) {
3073 vnn->tcp_update_needed = true;
3079 /* Do we already have this tickle ?*/
3080 tcp.src_addr = p->src_addr;
3081 tcp.dst_addr = p->dst_addr;
3082 if (ctdb_tcp_find(tcparray, &tcp) != NULL) {
3083 DEBUG(DEBUG_DEBUG,("Already had tickle info for %s:%u for vnn:%u\n",
3084 ctdb_addr_to_str(&tcp.dst_addr),
3085 ntohs(tcp.dst_addr.ip.sin_port),
3090 /* A new tickle, we must add it to the array */
3091 tcparray->connections = talloc_realloc(tcparray, tcparray->connections,
3092 struct ctdb_tcp_connection,
3094 CTDB_NO_MEMORY(ctdb, tcparray->connections);
3096 tcparray->connections[tcparray->num].src_addr = p->src_addr;
3097 tcparray->connections[tcparray->num].dst_addr = p->dst_addr;
3100 DEBUG(DEBUG_INFO,("Added tickle info for %s:%u from vnn %u\n",
3101 ctdb_addr_to_str(&tcp.dst_addr),
3102 ntohs(tcp.dst_addr.ip.sin_port),
3105 if (tcp_update_needed) {
3106 vnn->tcp_update_needed = true;
3114 called by a daemon to inform us of a TCP connection that one of its
3115 clients managing that should tickled with an ACK when IP takeover is
3118 static void ctdb_remove_tcp_connection(struct ctdb_context *ctdb, struct ctdb_tcp_connection *conn)
3120 struct ctdb_tcp_connection *tcpp;
3121 struct ctdb_vnn *vnn = find_public_ip_vnn(ctdb, &conn->dst_addr);
3124 DEBUG(DEBUG_ERR,(__location__ " unable to find public address %s\n",
3125 ctdb_addr_to_str(&conn->dst_addr)));
3129 /* if the array is empty we cant remove it
3130 and we dont need to do anything
3132 if (vnn->tcp_array == NULL) {
3133 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist (array is empty) %s:%u\n",
3134 ctdb_addr_to_str(&conn->dst_addr),
3135 ntohs(conn->dst_addr.ip.sin_port)));
3140 /* See if we know this connection
3141 if we dont know this connection then we dont need to do anything
3143 tcpp = ctdb_tcp_find(vnn->tcp_array, conn);
3145 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist %s:%u\n",
3146 ctdb_addr_to_str(&conn->dst_addr),
3147 ntohs(conn->dst_addr.ip.sin_port)));
3152 /* We need to remove this entry from the array.
3153 Instead of allocating a new array and copying data to it
3154 we cheat and just copy the last entry in the existing array
3155 to the entry that is to be removed and just shring the
3158 *tcpp = vnn->tcp_array->connections[vnn->tcp_array->num - 1];
3159 vnn->tcp_array->num--;
3161 /* If we deleted the last entry we also need to remove the entire array
3163 if (vnn->tcp_array->num == 0) {
3164 talloc_free(vnn->tcp_array);
3165 vnn->tcp_array = NULL;
3168 vnn->tcp_update_needed = true;
3170 DEBUG(DEBUG_INFO,("Removed tickle info for %s:%u\n",
3171 ctdb_addr_to_str(&conn->src_addr),
3172 ntohs(conn->src_addr.ip.sin_port)));
3177 called by a daemon to inform us of a TCP connection that one of its
3178 clients used are no longer needed in the tickle database
3180 int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
3182 struct ctdb_tcp_connection *conn = (struct ctdb_tcp_connection *)indata.dptr;
3184 ctdb_remove_tcp_connection(ctdb, conn);
3191 Called when another daemon starts - caises all tickles for all
3192 public addresses we are serving to be sent to the new node on the
3193 next check. This actually causes the next scheduled call to
3194 tdb_update_tcp_tickles() to update all nodes. This is simple and
3195 doesn't require careful error handling.
3197 int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t pnn)
3199 struct ctdb_vnn *vnn;
3201 for (vnn = ctdb->vnn; vnn != NULL; vnn = vnn->next) {
3202 vnn->tcp_update_needed = true;
3210 called when a client structure goes away - hook to remove
3211 elements from the tcp_list in all daemons
3213 void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
3215 while (client->tcp_list) {
3216 struct ctdb_tcp_list *tcp = client->tcp_list;
3217 DLIST_REMOVE(client->tcp_list, tcp);
3218 ctdb_remove_tcp_connection(client->ctdb, &tcp->connection);
3224 release all IPs on shutdown
3226 void ctdb_release_all_ips(struct ctdb_context *ctdb)
3228 struct ctdb_vnn *vnn;
3231 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3232 if (!ctdb_sys_have_ip(&vnn->public_address)) {
3233 ctdb_vnn_unassign_iface(ctdb, vnn);
3240 DEBUG(DEBUG_INFO,("Release of IP %s/%u on interface %s node:-1\n",
3241 ctdb_addr_to_str(&vnn->public_address),
3242 vnn->public_netmask_bits,
3243 ctdb_vnn_iface_string(vnn)));
3245 ctdb_event_script_args(ctdb, CTDB_EVENT_RELEASE_IP, "%s %s %u",
3246 ctdb_vnn_iface_string(vnn),
3247 ctdb_addr_to_str(&vnn->public_address),
3248 vnn->public_netmask_bits);
3249 release_kill_clients(ctdb, &vnn->public_address);
3250 ctdb_vnn_unassign_iface(ctdb, vnn);
3254 DEBUG(DEBUG_NOTICE,(__location__ " Released %d public IPs\n", count));
3259 get list of public IPs
3261 int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb,
3262 struct ctdb_req_control *c, TDB_DATA *outdata)
3265 struct ctdb_all_public_ips *ips;
3266 struct ctdb_vnn *vnn;
3267 bool only_available = false;
3269 if (c->flags & CTDB_PUBLIC_IP_FLAGS_ONLY_AVAILABLE) {
3270 only_available = true;
3273 /* count how many public ip structures we have */
3275 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3279 len = offsetof(struct ctdb_all_public_ips, ips) +
3280 num*sizeof(struct ctdb_public_ip);
3281 ips = talloc_zero_size(outdata, len);
3282 CTDB_NO_MEMORY(ctdb, ips);
3285 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3286 if (only_available && !ctdb_vnn_available(ctdb, vnn)) {
3289 ips->ips[i].pnn = vnn->pnn;
3290 ips->ips[i].addr = vnn->public_address;
3294 len = offsetof(struct ctdb_all_public_ips, ips) +
3295 i*sizeof(struct ctdb_public_ip);
3297 outdata->dsize = len;
3298 outdata->dptr = (uint8_t *)ips;
3305 get list of public IPs, old ipv4 style. only returns ipv4 addresses
3307 int32_t ctdb_control_get_public_ipsv4(struct ctdb_context *ctdb,
3308 struct ctdb_req_control *c, TDB_DATA *outdata)
3311 struct ctdb_all_public_ipsv4 *ips;
3312 struct ctdb_vnn *vnn;
3314 /* count how many public ip structures we have */
3316 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3317 if (vnn->public_address.sa.sa_family != AF_INET) {
3323 len = offsetof(struct ctdb_all_public_ipsv4, ips) +
3324 num*sizeof(struct ctdb_public_ipv4);
3325 ips = talloc_zero_size(outdata, len);
3326 CTDB_NO_MEMORY(ctdb, ips);
3328 outdata->dsize = len;
3329 outdata->dptr = (uint8_t *)ips;
3333 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3334 if (vnn->public_address.sa.sa_family != AF_INET) {
3337 ips->ips[i].pnn = vnn->pnn;
3338 ips->ips[i].sin = vnn->public_address.ip;
3345 int32_t ctdb_control_get_public_ip_info(struct ctdb_context *ctdb,
3346 struct ctdb_req_control *c,
3351 ctdb_sock_addr *addr;
3352 struct ctdb_control_public_ip_info *info;
3353 struct ctdb_vnn *vnn;
3355 addr = (ctdb_sock_addr *)indata.dptr;
3357 vnn = find_public_ip_vnn(ctdb, addr);
3359 /* if it is not a public ip it could be our 'single ip' */
3360 if (ctdb->single_ip_vnn) {
3361 if (ctdb_same_ip(&ctdb->single_ip_vnn->public_address, addr)) {
3362 vnn = ctdb->single_ip_vnn;
3367 DEBUG(DEBUG_ERR,(__location__ " Could not get public ip info, "
3368 "'%s'not a public address\n",
3369 ctdb_addr_to_str(addr)));
3373 /* count how many public ip structures we have */
3375 for (;vnn->ifaces[num];) {
3379 len = offsetof(struct ctdb_control_public_ip_info, ifaces) +
3380 num*sizeof(struct ctdb_control_iface_info);
3381 info = talloc_zero_size(outdata, len);
3382 CTDB_NO_MEMORY(ctdb, info);
3384 info->ip.addr = vnn->public_address;
3385 info->ip.pnn = vnn->pnn;
3386 info->active_idx = 0xFFFFFFFF;
3388 for (i=0; vnn->ifaces[i]; i++) {
3389 struct ctdb_iface *cur;
3391 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
3393 DEBUG(DEBUG_CRIT, (__location__ " internal error iface[%s] unknown\n",
3397 if (vnn->iface == cur) {
3398 info->active_idx = i;
3400 strncpy(info->ifaces[i].name, cur->name, sizeof(info->ifaces[i].name)-1);
3401 info->ifaces[i].link_state = cur->link_up;
3402 info->ifaces[i].references = cur->references;
3405 len = offsetof(struct ctdb_control_public_ip_info, ifaces) +
3406 i*sizeof(struct ctdb_control_iface_info);
3408 outdata->dsize = len;
3409 outdata->dptr = (uint8_t *)info;
3414 int32_t ctdb_control_get_ifaces(struct ctdb_context *ctdb,
3415 struct ctdb_req_control *c,
3419 struct ctdb_control_get_ifaces *ifaces;
3420 struct ctdb_iface *cur;
3422 /* count how many public ip structures we have */
3424 for (cur=ctdb->ifaces;cur;cur=cur->next) {
3428 len = offsetof(struct ctdb_control_get_ifaces, ifaces) +
3429 num*sizeof(struct ctdb_control_iface_info);
3430 ifaces = talloc_zero_size(outdata, len);
3431 CTDB_NO_MEMORY(ctdb, ifaces);
3434 for (cur=ctdb->ifaces;cur;cur=cur->next) {
3435 strcpy(ifaces->ifaces[i].name, cur->name);
3436 ifaces->ifaces[i].link_state = cur->link_up;
3437 ifaces->ifaces[i].references = cur->references;
3441 len = offsetof(struct ctdb_control_get_ifaces, ifaces) +
3442 i*sizeof(struct ctdb_control_iface_info);
3444 outdata->dsize = len;
3445 outdata->dptr = (uint8_t *)ifaces;
3450 int32_t ctdb_control_set_iface_link(struct ctdb_context *ctdb,
3451 struct ctdb_req_control *c,
3454 struct ctdb_control_iface_info *info;
3455 struct ctdb_iface *iface;
3456 bool link_up = false;
3458 info = (struct ctdb_control_iface_info *)indata.dptr;
3460 if (info->name[CTDB_IFACE_SIZE] != '\0') {
3461 int len = strnlen(info->name, CTDB_IFACE_SIZE);
3462 DEBUG(DEBUG_ERR, (__location__ " name[%*.*s] not terminated\n",
3463 len, len, info->name));
3467 switch (info->link_state) {
3475 DEBUG(DEBUG_ERR, (__location__ " link_state[%u] invalid\n",
3476 (unsigned int)info->link_state));
3480 if (info->references != 0) {
3481 DEBUG(DEBUG_ERR, (__location__ " references[%u] should be 0\n",
3482 (unsigned int)info->references));
3486 iface = ctdb_find_iface(ctdb, info->name);
3487 if (iface == NULL) {
3491 if (link_up == iface->link_up) {
3495 DEBUG(iface->link_up?DEBUG_ERR:DEBUG_NOTICE,
3496 ("iface[%s] has changed it's link status %s => %s\n",
3498 iface->link_up?"up":"down",
3499 link_up?"up":"down"));
3501 iface->link_up = link_up;
3507 structure containing the listening socket and the list of tcp connections
3508 that the ctdb daemon is to kill
3510 struct ctdb_kill_tcp {
3511 struct ctdb_vnn *vnn;
3512 struct ctdb_context *ctdb;
3514 struct fd_event *fde;
3515 trbt_tree_t *connections;
3520 a tcp connection that is to be killed
3522 struct ctdb_killtcp_con {
3523 ctdb_sock_addr src_addr;
3524 ctdb_sock_addr dst_addr;
3526 struct ctdb_kill_tcp *killtcp;
3529 /* this function is used to create a key to represent this socketpair
3530 in the killtcp tree.
3531 this key is used to insert and lookup matching socketpairs that are
3532 to be tickled and RST
3534 #define KILLTCP_KEYLEN 10
3535 static uint32_t *killtcp_key(ctdb_sock_addr *src, ctdb_sock_addr *dst)
3537 static uint32_t key[KILLTCP_KEYLEN];
3539 bzero(key, sizeof(key));
3541 if (src->sa.sa_family != dst->sa.sa_family) {
3542 DEBUG(DEBUG_ERR, (__location__ " ERROR, different families passed :%u vs %u\n", src->sa.sa_family, dst->sa.sa_family));
3546 switch (src->sa.sa_family) {
3548 key[0] = dst->ip.sin_addr.s_addr;
3549 key[1] = src->ip.sin_addr.s_addr;
3550 key[2] = dst->ip.sin_port;
3551 key[3] = src->ip.sin_port;
3554 uint32_t *dst6_addr32 =
3555 (uint32_t *)&(dst->ip6.sin6_addr.s6_addr);
3556 uint32_t *src6_addr32 =
3557 (uint32_t *)&(src->ip6.sin6_addr.s6_addr);
3558 key[0] = dst6_addr32[3];
3559 key[1] = src6_addr32[3];
3560 key[2] = dst6_addr32[2];
3561 key[3] = src6_addr32[2];
3562 key[4] = dst6_addr32[1];
3563 key[5] = src6_addr32[1];
3564 key[6] = dst6_addr32[0];
3565 key[7] = src6_addr32[0];
3566 key[8] = dst->ip6.sin6_port;
3567 key[9] = src->ip6.sin6_port;
3571 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", src->sa.sa_family));
3579 called when we get a read event on the raw socket
3581 static void capture_tcp_handler(struct event_context *ev, struct fd_event *fde,
3582 uint16_t flags, void *private_data)
3584 struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
3585 struct ctdb_killtcp_con *con;
3586 ctdb_sock_addr src, dst;
3587 uint32_t ack_seq, seq;
3589 if (!(flags & EVENT_FD_READ)) {
3593 if (ctdb_sys_read_tcp_packet(killtcp->capture_fd,
3594 killtcp->private_data,
3596 &ack_seq, &seq) != 0) {
3597 /* probably a non-tcp ACK packet */
3601 /* check if we have this guy in our list of connections
3604 con = trbt_lookuparray32(killtcp->connections,
3605 KILLTCP_KEYLEN, killtcp_key(&src, &dst));
3607 /* no this was some other packet we can just ignore */
3611 /* This one has been tickled !
3612 now reset him and remove him from the list.
3614 DEBUG(DEBUG_INFO, ("sending a tcp reset to kill connection :%d -> %s:%d\n",
3615 ntohs(con->dst_addr.ip.sin_port),
3616 ctdb_addr_to_str(&con->src_addr),
3617 ntohs(con->src_addr.ip.sin_port)));
3619 ctdb_sys_send_tcp(&con->dst_addr, &con->src_addr, ack_seq, seq, 1);
3624 /* when traversing the list of all tcp connections to send tickle acks to
3625 (so that we can capture the ack coming back and kill the connection
3627 this callback is called for each connection we are currently trying to kill
3629 static int tickle_connection_traverse(void *param, void *data)
3631 struct ctdb_killtcp_con *con = talloc_get_type(data, struct ctdb_killtcp_con);
3633 /* have tried too many times, just give up */
3634 if (con->count >= 5) {
3635 /* can't delete in traverse: reparent to delete_cons */
3636 talloc_steal(param, con);
3640 /* othervise, try tickling it again */
3643 (ctdb_sock_addr *)&con->dst_addr,
3644 (ctdb_sock_addr *)&con->src_addr,
3651 called every second until all sentenced connections have been reset
3653 static void ctdb_tickle_sentenced_connections(struct event_context *ev, struct timed_event *te,
3654 struct timeval t, void *private_data)
3656 struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
3657 void *delete_cons = talloc_new(NULL);
3659 /* loop over all connections sending tickle ACKs */
3660 trbt_traversearray32(killtcp->connections, KILLTCP_KEYLEN, tickle_connection_traverse, delete_cons);
3662 /* now we've finished traverse, it's safe to do deletion. */
3663 talloc_free(delete_cons);
3665 /* If there are no more connections to kill we can remove the
3666 entire killtcp structure
3668 if ( (killtcp->connections == NULL) ||
3669 (killtcp->connections->root == NULL) ) {
3670 talloc_free(killtcp);
3674 /* try tickling them again in a seconds time
3676 event_add_timed(killtcp->ctdb->ev, killtcp, timeval_current_ofs(1, 0),
3677 ctdb_tickle_sentenced_connections, killtcp);
3681 destroy the killtcp structure
3683 static int ctdb_killtcp_destructor(struct ctdb_kill_tcp *killtcp)
3685 struct ctdb_vnn *tmpvnn;
3687 /* verify that this vnn is still active */
3688 for (tmpvnn = killtcp->ctdb->vnn; tmpvnn; tmpvnn = tmpvnn->next) {
3689 if (tmpvnn == killtcp->vnn) {
3694 if (tmpvnn == NULL) {
3698 if (killtcp->vnn->killtcp != killtcp) {
3702 killtcp->vnn->killtcp = NULL;
3708 /* nothing fancy here, just unconditionally replace any existing
3709 connection structure with the new one.
3711 dont even free the old one if it did exist, that one is talloc_stolen
3712 by the same node in the tree anyway and will be deleted when the new data
3715 static void *add_killtcp_callback(void *parm, void *data)
3721 add a tcp socket to the list of connections we want to RST
3723 static int ctdb_killtcp_add_connection(struct ctdb_context *ctdb,
3727 ctdb_sock_addr src, dst;
3728 struct ctdb_kill_tcp *killtcp;
3729 struct ctdb_killtcp_con *con;
3730 struct ctdb_vnn *vnn;
3732 ctdb_canonicalize_ip(s, &src);
3733 ctdb_canonicalize_ip(d, &dst);
3735 vnn = find_public_ip_vnn(ctdb, &dst);
3737 vnn = find_public_ip_vnn(ctdb, &src);
3740 /* if it is not a public ip it could be our 'single ip' */
3741 if (ctdb->single_ip_vnn) {
3742 if (ctdb_same_ip(&ctdb->single_ip_vnn->public_address, &dst)) {
3743 vnn = ctdb->single_ip_vnn;
3748 DEBUG(DEBUG_ERR,(__location__ " Could not killtcp, not a public address\n"));
3752 killtcp = vnn->killtcp;
3754 /* If this is the first connection to kill we must allocate
3757 if (killtcp == NULL) {
3758 killtcp = talloc_zero(vnn, struct ctdb_kill_tcp);
3759 CTDB_NO_MEMORY(ctdb, killtcp);
3762 killtcp->ctdb = ctdb;
3763 killtcp->capture_fd = -1;
3764 killtcp->connections = trbt_create(killtcp, 0);
3766 vnn->killtcp = killtcp;
3767 talloc_set_destructor(killtcp, ctdb_killtcp_destructor);
3772 /* create a structure that describes this connection we want to
3773 RST and store it in killtcp->connections
3775 con = talloc(killtcp, struct ctdb_killtcp_con);
3776 CTDB_NO_MEMORY(ctdb, con);
3777 con->src_addr = src;
3778 con->dst_addr = dst;
3780 con->killtcp = killtcp;
3783 trbt_insertarray32_callback(killtcp->connections,
3784 KILLTCP_KEYLEN, killtcp_key(&con->dst_addr, &con->src_addr),
3785 add_killtcp_callback, con);
3788 If we dont have a socket to listen on yet we must create it
3790 if (killtcp->capture_fd == -1) {
3791 const char *iface = ctdb_vnn_iface_string(vnn);
3792 killtcp->capture_fd = ctdb_sys_open_capture_socket(iface, &killtcp->private_data);
3793 if (killtcp->capture_fd == -1) {
3794 DEBUG(DEBUG_CRIT,(__location__ " Failed to open capturing "
3795 "socket on iface '%s' for killtcp (%s)\n",
3796 iface, strerror(errno)));
3802 if (killtcp->fde == NULL) {
3803 killtcp->fde = event_add_fd(ctdb->ev, killtcp, killtcp->capture_fd,
3805 capture_tcp_handler, killtcp);
3806 tevent_fd_set_auto_close(killtcp->fde);
3808 /* We also need to set up some events to tickle all these connections
3809 until they are all reset
3811 event_add_timed(ctdb->ev, killtcp, timeval_current_ofs(1, 0),
3812 ctdb_tickle_sentenced_connections, killtcp);
3815 /* tickle him once now */
3824 talloc_free(vnn->killtcp);
3825 vnn->killtcp = NULL;
3830 kill a TCP connection.
3832 int32_t ctdb_control_kill_tcp(struct ctdb_context *ctdb, TDB_DATA indata)
3834 struct ctdb_control_killtcp *killtcp = (struct ctdb_control_killtcp *)indata.dptr;
3836 return ctdb_killtcp_add_connection(ctdb, &killtcp->src_addr, &killtcp->dst_addr);
3840 called by a daemon to inform us of the entire list of TCP tickles for
3841 a particular public address.
3842 this control should only be sent by the node that is currently serving
3843 that public address.
3845 int32_t ctdb_control_set_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata)
3847 struct ctdb_control_tcp_tickle_list *list = (struct ctdb_control_tcp_tickle_list *)indata.dptr;
3848 struct ctdb_tcp_array *tcparray;
3849 struct ctdb_vnn *vnn;
3851 /* We must at least have tickles.num or else we cant verify the size
3852 of the received data blob
3854 if (indata.dsize < offsetof(struct ctdb_control_tcp_tickle_list,
3855 tickles.connections)) {
3856 DEBUG(DEBUG_ERR,("Bad indata in ctdb_control_set_tcp_tickle_list. Not enough data for the tickle.num field\n"));
3860 /* verify that the size of data matches what we expect */
3861 if (indata.dsize < offsetof(struct ctdb_control_tcp_tickle_list,
3862 tickles.connections)
3863 + sizeof(struct ctdb_tcp_connection)
3864 * list->tickles.num) {
3865 DEBUG(DEBUG_ERR,("Bad indata in ctdb_control_set_tcp_tickle_list\n"));
3869 vnn = find_public_ip_vnn(ctdb, &list->addr);
3871 DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
3872 ctdb_addr_to_str(&list->addr)));
3877 /* remove any old ticklelist we might have */
3878 talloc_free(vnn->tcp_array);
3879 vnn->tcp_array = NULL;
3881 tcparray = talloc(vnn, struct ctdb_tcp_array);
3882 CTDB_NO_MEMORY(ctdb, tcparray);
3884 tcparray->num = list->tickles.num;
3886 tcparray->connections = talloc_array(tcparray, struct ctdb_tcp_connection, tcparray->num);
3887 CTDB_NO_MEMORY(ctdb, tcparray->connections);
3889 memcpy(tcparray->connections, &list->tickles.connections[0],
3890 sizeof(struct ctdb_tcp_connection)*tcparray->num);
3892 /* We now have a new fresh tickle list array for this vnn */
3893 vnn->tcp_array = tcparray;
3899 called to return the full list of tickles for the puclic address associated
3900 with the provided vnn
3902 int32_t ctdb_control_get_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
3904 ctdb_sock_addr *addr = (ctdb_sock_addr *)indata.dptr;
3905 struct ctdb_control_tcp_tickle_list *list;
3906 struct ctdb_tcp_array *tcparray;
3908 struct ctdb_vnn *vnn;
3910 vnn = find_public_ip_vnn(ctdb, addr);
3912 DEBUG(DEBUG_ERR,(__location__ " Could not get tcp tickle list, '%s' is not a public address\n",
3913 ctdb_addr_to_str(addr)));
3918 tcparray = vnn->tcp_array;
3920 num = tcparray->num;
3925 outdata->dsize = offsetof(struct ctdb_control_tcp_tickle_list,
3926 tickles.connections)
3927 + sizeof(struct ctdb_tcp_connection) * num;
3929 outdata->dptr = talloc_size(outdata, outdata->dsize);
3930 CTDB_NO_MEMORY(ctdb, outdata->dptr);
3931 list = (struct ctdb_control_tcp_tickle_list *)outdata->dptr;
3934 list->tickles.num = num;
3936 memcpy(&list->tickles.connections[0], tcparray->connections,
3937 sizeof(struct ctdb_tcp_connection) * num);
3945 set the list of all tcp tickles for a public address
3947 static int ctdb_ctrl_set_tcp_tickles(struct ctdb_context *ctdb,
3948 struct timeval timeout, uint32_t destnode,
3949 ctdb_sock_addr *addr,
3950 struct ctdb_tcp_array *tcparray)
3954 struct ctdb_control_tcp_tickle_list *list;
3957 num = tcparray->num;
3962 data.dsize = offsetof(struct ctdb_control_tcp_tickle_list,
3963 tickles.connections) +
3964 sizeof(struct ctdb_tcp_connection) * num;
3965 data.dptr = talloc_size(ctdb, data.dsize);
3966 CTDB_NO_MEMORY(ctdb, data.dptr);
3968 list = (struct ctdb_control_tcp_tickle_list *)data.dptr;
3970 list->tickles.num = num;
3972 memcpy(&list->tickles.connections[0], tcparray->connections, sizeof(struct ctdb_tcp_connection) * num);
3975 ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
3976 CTDB_CONTROL_SET_TCP_TICKLE_LIST,
3977 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
3979 DEBUG(DEBUG_ERR,(__location__ " ctdb_control for set tcp tickles failed\n"));
3983 talloc_free(data.dptr);
3990 perform tickle updates if required
3992 static void ctdb_update_tcp_tickles(struct event_context *ev,
3993 struct timed_event *te,
3994 struct timeval t, void *private_data)
3996 struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
3998 struct ctdb_vnn *vnn;
4000 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
4001 /* we only send out updates for public addresses that
4004 if (ctdb->pnn != vnn->pnn) {
4007 /* We only send out the updates if we need to */
4008 if (!vnn->tcp_update_needed) {
4011 ret = ctdb_ctrl_set_tcp_tickles(ctdb,
4013 CTDB_BROADCAST_CONNECTED,
4014 &vnn->public_address,
4017 DEBUG(DEBUG_ERR,("Failed to send the tickle update for public address %s\n",
4018 ctdb_addr_to_str(&vnn->public_address)));
4020 vnn->tcp_update_needed = false;
4024 event_add_timed(ctdb->ev, ctdb->tickle_update_context,
4025 timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
4026 ctdb_update_tcp_tickles, ctdb);
4031 start periodic update of tcp tickles
4033 void ctdb_start_tcp_tickle_update(struct ctdb_context *ctdb)
4035 ctdb->tickle_update_context = talloc_new(ctdb);
4037 event_add_timed(ctdb->ev, ctdb->tickle_update_context,
4038 timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
4039 ctdb_update_tcp_tickles, ctdb);
4045 struct control_gratious_arp {
4046 struct ctdb_context *ctdb;
4047 ctdb_sock_addr addr;
4053 send a control_gratuitous arp
4055 static void send_gratious_arp(struct event_context *ev, struct timed_event *te,
4056 struct timeval t, void *private_data)
4059 struct control_gratious_arp *arp = talloc_get_type(private_data,
4060 struct control_gratious_arp);
4062 ret = ctdb_sys_send_arp(&arp->addr, arp->iface);
4064 DEBUG(DEBUG_ERR,(__location__ " sending of gratious arp on iface '%s' failed (%s)\n",
4065 arp->iface, strerror(errno)));
4070 if (arp->count == CTDB_ARP_REPEAT) {
4075 event_add_timed(arp->ctdb->ev, arp,
4076 timeval_current_ofs(CTDB_ARP_INTERVAL, 0),
4077 send_gratious_arp, arp);
4084 int32_t ctdb_control_send_gratious_arp(struct ctdb_context *ctdb, TDB_DATA indata)
4086 struct ctdb_control_gratious_arp *gratious_arp = (struct ctdb_control_gratious_arp *)indata.dptr;
4087 struct control_gratious_arp *arp;
4089 /* verify the size of indata */
4090 if (indata.dsize < offsetof(struct ctdb_control_gratious_arp, iface)) {
4091 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_gratious_arp structure. Got %u require %u bytes\n",
4092 (unsigned)indata.dsize,
4093 (unsigned)offsetof(struct ctdb_control_gratious_arp, iface)));
4097 ( offsetof(struct ctdb_control_gratious_arp, iface)
4098 + gratious_arp->len ) ){
4100 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
4101 "but should be %u bytes\n",
4102 (unsigned)indata.dsize,
4103 (unsigned)(offsetof(struct ctdb_control_gratious_arp, iface)+gratious_arp->len)));
4108 arp = talloc(ctdb, struct control_gratious_arp);
4109 CTDB_NO_MEMORY(ctdb, arp);
4112 arp->addr = gratious_arp->addr;
4113 arp->iface = talloc_strdup(arp, gratious_arp->iface);
4114 CTDB_NO_MEMORY(ctdb, arp->iface);
4117 event_add_timed(arp->ctdb->ev, arp,
4118 timeval_zero(), send_gratious_arp, arp);
4123 int32_t ctdb_control_add_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
4125 struct ctdb_control_ip_iface *pub = (struct ctdb_control_ip_iface *)indata.dptr;
4128 /* verify the size of indata */
4129 if (indata.dsize < offsetof(struct ctdb_control_ip_iface, iface)) {
4130 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_ip_iface structure\n"));
4134 ( offsetof(struct ctdb_control_ip_iface, iface)
4137 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
4138 "but should be %u bytes\n",
4139 (unsigned)indata.dsize,
4140 (unsigned)(offsetof(struct ctdb_control_ip_iface, iface)+pub->len)));
4144 DEBUG(DEBUG_NOTICE,("Add IP %s\n", ctdb_addr_to_str(&pub->addr)));
4146 ret = ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0], true);
4149 DEBUG(DEBUG_ERR,(__location__ " Failed to add public address\n"));
4157 called when releaseip event finishes for del_public_address
4159 static void delete_ip_callback(struct ctdb_context *ctdb, int status,
4162 talloc_free(private_data);
4165 int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
4167 struct ctdb_control_ip_iface *pub = (struct ctdb_control_ip_iface *)indata.dptr;
4168 struct ctdb_vnn *vnn;
4171 /* verify the size of indata */
4172 if (indata.dsize < offsetof(struct ctdb_control_ip_iface, iface)) {
4173 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_ip_iface structure\n"));
4177 ( offsetof(struct ctdb_control_ip_iface, iface)
4180 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
4181 "but should be %u bytes\n",
4182 (unsigned)indata.dsize,
4183 (unsigned)(offsetof(struct ctdb_control_ip_iface, iface)+pub->len)));
4187 DEBUG(DEBUG_NOTICE,("Delete IP %s\n", ctdb_addr_to_str(&pub->addr)));
4189 /* walk over all public addresses until we find a match */
4190 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
4191 if (ctdb_same_ip(&vnn->public_address, &pub->addr)) {
4192 TALLOC_CTX *mem_ctx = talloc_new(ctdb);
4194 DLIST_REMOVE(ctdb->vnn, vnn);
4195 talloc_steal(mem_ctx, vnn);
4196 ctdb_remove_orphaned_ifaces(ctdb, vnn, mem_ctx);
4197 if (vnn->pnn != ctdb->pnn) {
4198 if (vnn->iface != NULL) {
4199 ctdb_vnn_unassign_iface(ctdb, vnn);
4201 talloc_free(mem_ctx);
4206 ret = ctdb_event_script_callback(ctdb,
4207 mem_ctx, delete_ip_callback, mem_ctx,
4208 CTDB_EVENT_RELEASE_IP,
4210 ctdb_vnn_iface_string(vnn),
4211 ctdb_addr_to_str(&vnn->public_address),
4212 vnn->public_netmask_bits);
4213 if (vnn->iface != NULL) {
4214 ctdb_vnn_unassign_iface(ctdb, vnn);
4227 struct ipreallocated_callback_state {
4228 struct ctdb_req_control *c;
4231 static void ctdb_ipreallocated_callback(struct ctdb_context *ctdb,
4232 int status, void *p)
4234 struct ipreallocated_callback_state *state =
4235 talloc_get_type(p, struct ipreallocated_callback_state);
4239 (" \"ipreallocated\" event script failed (status %d)\n",
4241 if (status == -ETIME) {
4242 ctdb_ban_self(ctdb);
4246 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
4250 /* A control to run the ipreallocated event */
4251 int32_t ctdb_control_ipreallocated(struct ctdb_context *ctdb,
4252 struct ctdb_req_control *c,
4256 struct ipreallocated_callback_state *state;
4258 state = talloc(ctdb, struct ipreallocated_callback_state);
4259 CTDB_NO_MEMORY(ctdb, state);
4261 DEBUG(DEBUG_INFO,(__location__ " Running \"ipreallocated\" event\n"));
4263 ret = ctdb_event_script_callback(ctdb, state,
4264 ctdb_ipreallocated_callback, state,
4265 CTDB_EVENT_IPREALLOCATED,
4269 DEBUG(DEBUG_ERR,("Failed to run \"ipreallocated\" event \n"));
4274 /* tell the control that we will be reply asynchronously */
4275 state->c = talloc_steal(state, c);
4276 *async_reply = true;
4282 /* This function is called from the recovery daemon to verify that a remote
4283 node has the expected ip allocation.
4284 This is verified against ctdb->ip_tree
4286 int verify_remote_ip_allocation(struct ctdb_context *ctdb,
4287 struct ctdb_all_public_ips *ips,
4290 struct ctdb_public_ip_list *tmp_ip;
4293 if (ctdb->ip_tree == NULL) {
4294 /* dont know the expected allocation yet, assume remote node
4303 for (i=0; i<ips->num; i++) {
4304 tmp_ip = trbt_lookuparray32(ctdb->ip_tree, IP_KEYLEN, ip_key(&ips->ips[i].addr));
4305 if (tmp_ip == NULL) {
4306 DEBUG(DEBUG_ERR,("Node %u has new or unknown public IP %s\n", pnn, ctdb_addr_to_str(&ips->ips[i].addr)));
4310 if (tmp_ip->pnn == -1 || ips->ips[i].pnn == -1) {
4314 if (tmp_ip->pnn != ips->ips[i].pnn) {
4316 ("Inconsistent IP allocation - node %u thinks %s is held by node %u while it is assigned to node %u\n",
4318 ctdb_addr_to_str(&ips->ips[i].addr),
4319 ips->ips[i].pnn, tmp_ip->pnn));
4327 int update_ip_assignment_tree(struct ctdb_context *ctdb, struct ctdb_public_ip *ip)
4329 struct ctdb_public_ip_list *tmp_ip;
4331 if (ctdb->ip_tree == NULL) {
4332 DEBUG(DEBUG_ERR,("No ctdb->ip_tree yet. Failed to update ip assignment\n"));
4336 tmp_ip = trbt_lookuparray32(ctdb->ip_tree, IP_KEYLEN, ip_key(&ip->addr));
4337 if (tmp_ip == NULL) {
4338 DEBUG(DEBUG_ERR,(__location__ " Could not find record for address %s, update ip\n", ctdb_addr_to_str(&ip->addr)));
4342 DEBUG(DEBUG_NOTICE,("Updated ip assignment tree for ip : %s from node %u to node %u\n", ctdb_addr_to_str(&ip->addr), tmp_ip->pnn, ip->pnn));
4343 tmp_ip->pnn = ip->pnn;
4349 struct ctdb_reloadips_handle {
4350 struct ctdb_context *ctdb;
4351 struct ctdb_req_control *c;
4355 struct fd_event *fde;
4358 static int ctdb_reloadips_destructor(struct ctdb_reloadips_handle *h)
4360 if (h == h->ctdb->reload_ips) {
4361 h->ctdb->reload_ips = NULL;
4364 ctdb_request_control_reply(h->ctdb, h->c, NULL, h->status, NULL);
4367 ctdb_kill(h->ctdb, h->child, SIGKILL);
4371 static void ctdb_reloadips_timeout_event(struct event_context *ev,
4372 struct timed_event *te,
4373 struct timeval t, void *private_data)
4375 struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
4380 static void ctdb_reloadips_child_handler(struct event_context *ev, struct fd_event *fde,
4381 uint16_t flags, void *private_data)
4383 struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
4388 ret = read(h->fd[0], &res, 1);
4389 if (ret < 1 || res != 0) {
4390 DEBUG(DEBUG_ERR, (__location__ " Reloadips child process returned error\n"));
4398 static int ctdb_reloadips_child(struct ctdb_context *ctdb)
4400 TALLOC_CTX *mem_ctx = talloc_new(NULL);
4401 struct ctdb_all_public_ips *ips;
4402 struct ctdb_vnn *vnn;
4403 struct client_async_data *async_data;
4404 struct timeval timeout;
4406 struct ctdb_client_control_state *state;
4410 CTDB_NO_MEMORY(ctdb, mem_ctx);
4412 /* Read IPs from local node */
4413 ret = ctdb_ctrl_get_public_ips(ctdb, TAKEOVER_TIMEOUT(),
4414 CTDB_CURRENT_NODE, mem_ctx, &ips);
4417 ("Unable to fetch public IPs from local node\n"));
4418 talloc_free(mem_ctx);
4422 /* Read IPs file - this is safe since this is a child process */
4424 if (ctdb_set_public_addresses(ctdb, false) != 0) {
4425 DEBUG(DEBUG_ERR,("Failed to re-read public addresses file\n"));
4426 talloc_free(mem_ctx);
4430 async_data = talloc_zero(mem_ctx, struct client_async_data);
4431 CTDB_NO_MEMORY(ctdb, async_data);
4433 /* Compare IPs between node and file for IPs to be deleted */
4434 for (i = 0; i < ips->num; i++) {
4436 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
4437 if (ctdb_same_ip(&vnn->public_address,
4438 &ips->ips[i].addr)) {
4439 /* IP is still in file */
4445 /* Delete IP ips->ips[i] */
4446 struct ctdb_control_ip_iface *pub;
4449 ("IP %s no longer configured, deleting it\n",
4450 ctdb_addr_to_str(&ips->ips[i].addr)));
4452 pub = talloc_zero(mem_ctx,
4453 struct ctdb_control_ip_iface);
4454 CTDB_NO_MEMORY(ctdb, pub);
4456 pub->addr = ips->ips[i].addr;
4460 timeout = TAKEOVER_TIMEOUT();
4462 data.dsize = offsetof(struct ctdb_control_ip_iface,
4464 data.dptr = (uint8_t *)pub;
4466 state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
4467 CTDB_CONTROL_DEL_PUBLIC_IP,
4468 0, data, async_data,
4470 if (state == NULL) {
4473 " failed sending CTDB_CONTROL_DEL_PUBLIC_IP\n"));
4477 ctdb_client_async_add(async_data, state);
4481 /* Compare IPs between node and file for IPs to be added */
4483 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
4484 for (i = 0; i < ips->num; i++) {
4485 if (ctdb_same_ip(&vnn->public_address,
4486 &ips->ips[i].addr)) {
4487 /* IP already on node */
4491 if (i == ips->num) {
4492 /* Add IP ips->ips[i] */
4493 struct ctdb_control_ip_iface *pub;
4494 const char *ifaces = NULL;
4499 ("New IP %s configured, adding it\n",
4500 ctdb_addr_to_str(&vnn->public_address)));
4502 uint32_t pnn = ctdb_get_pnn(ctdb);
4504 data.dsize = sizeof(pnn);
4505 data.dptr = (uint8_t *)&pnn;
4507 ret = ctdb_client_send_message(
4509 CTDB_BROADCAST_CONNECTED,
4510 CTDB_SRVID_REBALANCE_NODE,
4513 DEBUG(DEBUG_WARNING,
4514 ("Failed to send message to force node reallocation - IPs may be unbalanced\n"));
4520 ifaces = vnn->ifaces[0];
4522 while (vnn->ifaces[iface] != NULL) {
4523 ifaces = talloc_asprintf(vnn, "%s,%s", ifaces,
4524 vnn->ifaces[iface]);
4528 len = strlen(ifaces) + 1;
4529 pub = talloc_zero_size(mem_ctx,
4530 offsetof(struct ctdb_control_ip_iface, iface) + len);
4531 CTDB_NO_MEMORY(ctdb, pub);
4533 pub->addr = vnn->public_address;
4534 pub->mask = vnn->public_netmask_bits;
4536 memcpy(&pub->iface[0], ifaces, pub->len);
4538 timeout = TAKEOVER_TIMEOUT();
4540 data.dsize = offsetof(struct ctdb_control_ip_iface,
4542 data.dptr = (uint8_t *)pub;
4544 state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
4545 CTDB_CONTROL_ADD_PUBLIC_IP,
4546 0, data, async_data,
4548 if (state == NULL) {
4551 " failed sending CTDB_CONTROL_ADD_PUBLIC_IP\n"));
4555 ctdb_client_async_add(async_data, state);
4559 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
4560 DEBUG(DEBUG_ERR,(__location__ " Add/delete IPs failed\n"));
4564 talloc_free(mem_ctx);
4568 talloc_free(mem_ctx);
4572 /* This control is sent to force the node to re-read the public addresses file
4573 and drop any addresses we should nnot longer host, and add new addresses
4574 that we are now able to host
4576 int32_t ctdb_control_reload_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
4578 struct ctdb_reloadips_handle *h;
4579 pid_t parent = getpid();
4581 if (ctdb->reload_ips != NULL) {
4582 talloc_free(ctdb->reload_ips);
4583 ctdb->reload_ips = NULL;
4586 h = talloc(ctdb, struct ctdb_reloadips_handle);
4587 CTDB_NO_MEMORY(ctdb, h);
4592 if (pipe(h->fd) == -1) {
4593 DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
4598 h->child = ctdb_fork(ctdb);
4599 if (h->child == (pid_t)-1) {
4600 DEBUG(DEBUG_ERR, ("Failed to fork a child for reloadips\n"));
4608 if (h->child == 0) {
4609 signed char res = 0;
4612 debug_extra = talloc_asprintf(NULL, "reloadips:");
4614 ctdb_set_process_name("ctdb_reloadips");
4615 if (switch_from_server_to_client(ctdb, "reloadips-child") != 0) {
4616 DEBUG(DEBUG_CRIT,("ERROR: Failed to switch reloadips child into client mode\n"));
4619 res = ctdb_reloadips_child(ctdb);
4621 DEBUG(DEBUG_ERR,("Failed to reload ips on local node\n"));
4625 write(h->fd[1], &res, 1);
4626 /* make sure we die when our parent dies */
4627 while (ctdb_kill(ctdb, parent, 0) == 0 || errno != ESRCH) {
4633 h->c = talloc_steal(h, c);
4636 set_close_on_exec(h->fd[0]);
4638 talloc_set_destructor(h, ctdb_reloadips_destructor);
4641 h->fde = event_add_fd(ctdb->ev, h, h->fd[0],
4642 EVENT_FD_READ, ctdb_reloadips_child_handler,
4644 tevent_fd_set_auto_close(h->fde);
4646 event_add_timed(ctdb->ev, h,
4647 timeval_current_ofs(120, 0),
4648 ctdb_reloadips_timeout_event, h);
4650 /* we reply later */
4651 *async_reply = true;