4 Copyright (C) Andrew Tridgell 2006
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #include "lib/events/events.h"
22 #include "lib/tdb/include/tdb.h"
23 #include "system/network.h"
24 #include "system/filesys.h"
25 #include "../include/ctdb_private.h"
29 stop any connecting (established or pending) to a node
31 void ctdb_tcp_stop_connection(struct ctdb_node *node)
33 struct ctdb_tcp_node *tnode = talloc_get_type(
34 node->private_data, struct ctdb_tcp_node);
36 ctdb_queue_set_fd(tnode->out_queue, -1);
37 talloc_free(tnode->connect_te);
38 talloc_free(tnode->connect_fde);
39 tnode->connect_fde = NULL;
40 tnode->connect_te = NULL;
41 if (tnode->fd != -1) {
49 called when a complete packet has come in - should not happen on this socket
51 void ctdb_tcp_tnode_cb(uint8_t *data, size_t cnt, void *private_data)
53 struct ctdb_node *node = talloc_get_type(private_data, struct ctdb_node);
54 struct ctdb_tcp_node *tnode = talloc_get_type(
55 node->private_data, struct ctdb_tcp_node);
58 node->ctdb->upcalls->node_dead(node);
61 ctdb_tcp_stop_connection(node);
62 tnode->connect_te = event_add_timed(node->ctdb->ev, tnode, timeval_zero(),
63 ctdb_tcp_node_connect, node);
67 called when socket becomes writeable on connect
69 static void ctdb_node_connect_write(struct event_context *ev, struct fd_event *fde,
70 uint16_t flags, void *private_data)
72 struct ctdb_node *node = talloc_get_type(private_data,
74 struct ctdb_tcp_node *tnode = talloc_get_type(node->private_data,
75 struct ctdb_tcp_node);
76 struct ctdb_context *ctdb = node->ctdb;
78 socklen_t len = sizeof(error);
81 talloc_free(tnode->connect_te);
82 tnode->connect_te = NULL;
84 if (getsockopt(tnode->fd, SOL_SOCKET, SO_ERROR, &error, &len) != 0 ||
86 ctdb_tcp_stop_connection(node);
87 tnode->connect_te = event_add_timed(ctdb->ev, tnode,
88 timeval_current_ofs(1, 0),
89 ctdb_tcp_node_connect, node);
93 talloc_free(tnode->connect_fde);
94 tnode->connect_fde = NULL;
96 setsockopt(tnode->fd,IPPROTO_TCP,TCP_NODELAY,(char *)&one,sizeof(one));
97 setsockopt(tnode->fd,SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(one));
99 ctdb_queue_set_fd(tnode->out_queue, tnode->fd);
101 /* the queue subsystem now owns this fd */
104 /* tell the ctdb layer we are connected */
105 node->ctdb->upcalls->node_connected(node);
109 static int ctdb_tcp_get_address(struct ctdb_context *ctdb,
110 const char *address, struct in_addr *addr)
112 if (inet_pton(AF_INET, address, addr) <= 0) {
113 struct hostent *he = gethostbyname(address);
114 if (he == NULL || he->h_length > sizeof(*addr)) {
115 ctdb_set_error(ctdb, "invalid nework address '%s'\n",
119 memcpy(addr, he->h_addr, he->h_length);
125 called when we should try and establish a tcp connection to a node
127 void ctdb_tcp_node_connect(struct event_context *ev, struct timed_event *te,
128 struct timeval t, void *private_data)
130 struct ctdb_node *node = talloc_get_type(private_data,
132 struct ctdb_tcp_node *tnode = talloc_get_type(node->private_data,
133 struct ctdb_tcp_node);
134 struct ctdb_context *ctdb = node->ctdb;
135 struct sockaddr_in sock_in;
136 struct sockaddr_in sock_out;
138 ctdb_tcp_stop_connection(node);
140 tnode->fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
142 set_nonblocking(tnode->fd);
143 set_close_on_exec(tnode->fd);
145 ZERO_STRUCT(sock_out);
146 #ifdef HAVE_SOCK_SIN_LEN
147 sock_out.sin_len = sizeof(sock_out);
149 if (ctdb_tcp_get_address(ctdb, node->address.address, &sock_out.sin_addr) != 0) {
152 sock_out.sin_port = htons(node->address.port);
153 sock_out.sin_family = PF_INET;
156 /* Bind our side of the socketpair to the same address we use to listen
157 * on incoming CTDB traffic.
158 * We must specify this address to make sure that the address we expose to
159 * the remote side is actually routable in case CTDB traffic will run on
160 * a dedicated non-routeable network.
162 ZERO_STRUCT(sock_in);
163 #ifdef HAVE_SOCK_SIN_LEN
164 sock_in.sin_len = sizeof(sock_in);
166 if (ctdb_tcp_get_address(ctdb, ctdb->address.address, &sock_in.sin_addr) != 0) {
169 sock_in.sin_port = htons(0); /* INPORT_ANY is not always available */
170 sock_in.sin_family = PF_INET;
171 bind(tnode->fd, (struct sockaddr *)&sock_in, sizeof(sock_in));
173 if (connect(tnode->fd, (struct sockaddr *)&sock_out, sizeof(sock_out)) != 0 &&
174 errno != EINPROGRESS) {
175 ctdb_tcp_stop_connection(node);
176 tnode->connect_te = event_add_timed(ctdb->ev, tnode,
177 timeval_current_ofs(1, 0),
178 ctdb_tcp_node_connect, node);
182 /* non-blocking connect - wait for write event */
183 tnode->connect_fde = event_add_fd(node->ctdb->ev, tnode, tnode->fd,
184 EVENT_FD_WRITE|EVENT_FD_READ,
185 ctdb_node_connect_write, node);
187 /* don't give it long to connect - retry in one second. This ensures
188 that we find a node is up quickly (tcp normally backs off a syn reply
189 delay by quite a lot) */
190 tnode->connect_te = event_add_timed(ctdb->ev, tnode, timeval_current_ofs(1, 0),
191 ctdb_tcp_node_connect, node);
195 called when we get contacted by another node
196 currently makes no attempt to check if the connection is really from a ctdb
199 static void ctdb_listen_event(struct event_context *ev, struct fd_event *fde,
200 uint16_t flags, void *private_data)
202 struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
203 struct ctdb_tcp *ctcp = talloc_get_type(ctdb->private_data, struct ctdb_tcp);
204 struct sockaddr_in addr;
207 struct ctdb_incoming *in;
209 const char *incoming_node;
211 memset(&addr, 0, sizeof(addr));
213 fd = accept(ctcp->listen_fd, (struct sockaddr *)&addr, &len);
214 if (fd == -1) return;
216 incoming_node = inet_ntoa(addr.sin_addr);
217 nodeid = ctdb_ip_to_nodeid(ctdb, incoming_node);
220 DEBUG(DEBUG_ERR, ("Refused connection from unknown node %s\n", incoming_node));
225 in = talloc_zero(ctcp, struct ctdb_incoming);
229 set_nonblocking(in->fd);
230 set_close_on_exec(in->fd);
232 setsockopt(in->fd,SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(one));
234 in->queue = ctdb_queue_setup(ctdb, in, in->fd, CTDB_TCP_ALIGNMENT,
235 ctdb_tcp_read_cb, in);
240 automatically find which address to listen on
242 static int ctdb_tcp_listen_automatic(struct ctdb_context *ctdb)
244 struct ctdb_tcp *ctcp = talloc_get_type(ctdb->private_data,
246 struct sockaddr_in sock;
248 const char *lock_path = "/tmp/.ctdb_socket_lock";
251 /* in order to ensure that we don't get two nodes with the
252 same adddress, we must make the bind() and listen() calls
253 atomic. The SO_REUSEADDR setsockopt only prevents double
254 binds if the first socket is in LISTEN state */
255 lock_fd = open(lock_path, O_RDWR|O_CREAT, 0666);
257 DEBUG(DEBUG_CRIT,("Unable to open %s\n", lock_path));
261 lock.l_type = F_WRLCK;
262 lock.l_whence = SEEK_SET;
267 if (fcntl(lock_fd, F_SETLKW, &lock) != 0) {
268 DEBUG(DEBUG_CRIT,("Unable to lock %s\n", lock_path));
273 for (i=0;i<ctdb->num_nodes;i++) {
274 /* if node_ip is specified we will only try to bind to that
277 if (ctdb->node_ip != NULL) {
278 if (strcmp(ctdb->node_ip, ctdb->nodes[i]->address.address)) {
284 #ifdef HAVE_SOCK_SIN_LEN
285 sock.sin_len = sizeof(sock);
287 sock.sin_port = htons(ctdb->nodes[i]->address.port);
288 sock.sin_family = PF_INET;
289 if (ctdb_tcp_get_address(ctdb,
290 ctdb->nodes[i]->address.address,
291 &sock.sin_addr) != 0) {
295 if (bind(ctcp->listen_fd, (struct sockaddr * )&sock,
296 sizeof(sock)) == 0) {
301 if (i == ctdb->num_nodes) {
302 DEBUG(DEBUG_CRIT,("Unable to bind to any of the node addresses - giving up\n"));
305 ctdb->address.address = talloc_strdup(ctdb, ctdb->nodes[i]->address.address);
306 ctdb->address.port = ctdb->nodes[i]->address.port;
307 ctdb->name = talloc_asprintf(ctdb, "%s:%u",
308 ctdb->address.address,
310 ctdb->pnn = ctdb->nodes[i]->pnn;
311 ctdb->nodes[i]->flags &= ~NODE_FLAGS_DISCONNECTED;
312 DEBUG(DEBUG_INFO,("ctdb chose network address %s:%u pnn %u\n",
313 ctdb->address.address,
316 /* do we start out in DISABLED mode? */
317 if (ctdb->start_as_disabled != 0) {
318 DEBUG(DEBUG_INFO, ("This node is configured to start in DISABLED state\n"));
319 ctdb->nodes[i]->flags |= NODE_FLAGS_DISABLED;
322 if (listen(ctcp->listen_fd, 10) == -1) {
326 event_add_fd(ctdb->ev, ctcp, ctcp->listen_fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
327 ctdb_listen_event, ctdb);
334 close(ctcp->listen_fd);
335 ctcp->listen_fd = -1;
341 listen on our own address
343 int ctdb_tcp_listen(struct ctdb_context *ctdb)
345 struct ctdb_tcp *ctcp = talloc_get_type(ctdb->private_data,
347 struct sockaddr_in sock;
350 ctcp->listen_fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
351 if (ctcp->listen_fd == -1) {
352 ctdb_set_error(ctdb, "socket failed\n");
356 set_close_on_exec(ctcp->listen_fd);
358 setsockopt(ctcp->listen_fd,SOL_SOCKET,SO_REUSEADDR,(char *)&one,sizeof(one));
360 /* we can either auto-bind to the first available address, or we can
361 use a specified address */
362 if (!ctdb->address.address) {
363 return ctdb_tcp_listen_automatic(ctdb);
367 #ifdef HAVE_SOCK_SIN_LEN
368 sock.sin_len = sizeof(sock);
370 sock.sin_port = htons(ctdb->address.port);
371 sock.sin_family = PF_INET;
373 if (ctdb_tcp_get_address(ctdb, ctdb->address.address,
374 &sock.sin_addr) != 0) {
378 if (bind(ctcp->listen_fd, (struct sockaddr * )&sock, sizeof(sock)) != 0) {
382 if (listen(ctcp->listen_fd, 10) == -1) {
386 event_add_fd(ctdb->ev, ctcp, ctcp->listen_fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
387 ctdb_listen_event, ctdb);
392 close(ctcp->listen_fd);
393 ctcp->listen_fd = -1;