4 Copyright (C) Ronnie sahlberg 2010
5 Copyright (C) Rusty Russell 2010
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
29 #include <netinet/in.h>
30 #include <ctdb_protocol.h>
33 * ctdb - a library for accessing tdbs controlled by ctdbd
35 * ctdbd (clustered tdb daemon) is a daemon designed to syncronize TDB
36 * databases across a cluster. Using this library, you can communicate with
37 * the daemon to access the databases, pass messages across the cluster, and
38 * control the daemon itself.
40 * The general API is event-driven and asynchronous: you call the
41 * *_send functions, supplying callbacks, then when the ctdbd file
42 * descriptor is usable, call ctdb_service() to perform read from it
43 * and call your callbacks, which use the *_recv functions to unpack
44 * the replies from ctdbd.
46 * There is also a synchronous wrapper for each function for trivial
47 * programs; these can be found in the section marked "Synchronous API".
51 * ctdb_log_fn_t - logging function for ctdbd
52 * @log_priv: private (typesafe) arg via ctdb_connect
53 * @severity: syslog-style severity
54 * @format: printf-style format string.
55 * @ap: arguments for formatting.
57 * The severity passed to log() are as per syslog(3). In particular,
58 * LOG_DEBUG is used for tracing, LOG_WARNING is used for unusual
59 * conditions which don't necessarily return an error through the API,
60 * LOG_ERR is used for errors such as lost communication with ctdbd or
61 * out-of-memory, LOG_ALERT is used for library usage bugs, LOG_CRIT is
62 * used for libctdb internal consistency checks.
64 * The log() function can be typesafe: the @log_priv arg to
65 * ctdb_donnect and signature of log() should match.
67 typedef void (*ctdb_log_fn_t)(void *log_priv,
68 int severity, const char *format, va_list ap);
71 * ctdb_connect - connect to ctdb using the specified domain socket.
72 * @addr: the socket address, or NULL for default
73 * @log: the logging function
74 * @log_priv: the private argument to the logging function.
76 * Returns a ctdb context if successful or NULL. Use ctdb_disconnect() to
77 * release the returned ctdb_connection when finished.
80 * ctdb_log_fn_t, ctdb_log_file()
82 struct ctdb_connection *ctdb_connect(const char *addr,
83 ctdb_log_fn_t log_fn, void *log_priv);
86 * ctdb_log_file - example logging function
88 * Logs everything at priority LOG_WARNING or above to the file given (via
89 * the log_priv argument, usually stderr).
91 void ctdb_log_file(FILE *, int, const char *, va_list);
94 * ctdb_log_level - level at which to call logging function
96 * This variable globally controls filtering on the logging function.
97 * It is initialized to LOG_WARNING, meaning that strange but nonfatal
98 * events, as well as errors and API misuses are reported.
100 * Set it to LOG_DEBUG to receive all messages.
102 extern int ctdb_log_level;
105 * ctdb_disconnect - close down a connection to ctdbd.
106 * @ctdb: the ctdb connectio returned from ctdb_connect.
108 * The @ctdb arg will be freed by this call, and must not be used again.
110 void ctdb_disconnect(struct ctdb_connection *ctdb);
119 * ctdb_get_fd - get the filedescriptor to select/poll on
120 * @ctdb: the ctdb_connection from ctdb_connect.
122 * By using poll or select on this file descriptor, you will know when to call
126 * ctdb_which_events(), ctdb_service()
128 int ctdb_get_fd(struct ctdb_connection *ctdb);
131 * ctdb_which_events - determine which events ctdb_service wants to see
132 * @ctdb: the ctdb_connection from ctdb_connect.
134 * This returns POLLIN, possibly or'd with POLLOUT if there are writes
135 * pending. You can set this straight into poll.events.
140 int ctdb_which_events(struct ctdb_connection *ctdb);
143 * ctdb_service - service any I/O and callbacks from ctdbd communication
144 * @ctdb: the ctdb_connection from ctdb_connect.
145 * @revents: which events are available.
147 * This is the core of the library: it read and writes to the ctdbd
148 * socket. It may call callbacks registered with the various _send
151 * revents is a bitset: POLLIN and/or POLLOUT may be set to indicate
152 * it is worth attempting to read/write the (nonblocking)
153 * filedescriptor respectively.
155 * Note that the synchronous functions call this internally.
156 * Returns false on catastrophic failure.
158 bool ctdb_service(struct ctdb_connection *ctdb, int revents);
161 * struct ctdb_request - handle for an outstanding request
163 * This opaque structure returned from various *_send functions gives
164 * you a handle by which you can cancel a request. You can't do
165 * anything else with it until the request is completed and it is
166 * handed to your callback function.
171 * ctdb_request_free - free a completed request
173 * This frees a request: you should only call it once it has been
174 * handed to your callback. For incomplete requests, see ctdb_cancel().
176 void ctdb_request_free(struct ctdb_connection *ctdb, struct ctdb_request *req);
179 * ctdb_callback_t - callback for completed requests.
181 * This would normally unpack the request using ctdb_*_recv(). You
182 * must free the request using ctdb_request_free().
184 * Note that due to macro magic, actual your callback can be typesafe:
185 * instead of taking a void *, it can take a type which matches the
186 * actual private parameter.
188 typedef void (*ctdb_callback_t)(struct ctdb_connection *ctdb,
189 struct ctdb_request *req, void *private_data);
192 * struct ctdb_db - connection to a particular open TDB
194 * This represents a particular open database: you receive it from
195 * ctdb_attachdb or ctdb_attachdb_recv to manipulate a database.
197 * You have to free the handle with ctdb_detachdb() when finished with it.
202 * ctdb_attachdb_send - open a clustered TDB
203 * @ctdb: the ctdb_connection from ctdb_connect.
204 * @name: the filename of the database (no /).
205 * @persistent: whether the database is persistent across ctdbd's life
206 * @tdb_flags: the flags to pass to tdb_open.
207 * @callback: the callback when we're attached or failed (typesafe)
208 * @cbdata: the argument to callback()
210 * This function connects to a TDB controlled by ctdbd. It can create
211 * a new TDB if it does not exist, depending on tdb_flags. Returns
212 * the pending request, or NULL on error.
214 struct ctdb_request *
215 ctdb_attachdb_send(struct ctdb_connection *ctdb,
216 const char *name, bool persistent, uint32_t tdb_flags,
217 ctdb_callback_t callback, void *cbdata);
220 * ctdb_attachdb_recv - read an ctdb_attach reply from ctdbd
221 * @ctdb: the ctdb_connection from ctdb_connect.
222 * @req: the completed request.
224 * This returns NULL if something went wrong, or otherwise the open database.
226 struct ctdb_db *ctdb_attachdb_recv(struct ctdb_connection *ctdb,
227 struct ctdb_request *req);
231 * struct ctdb_lock - a record lock on a clustered TDB database
233 * This locks a subset of the database across the entire cluster; it
234 * is the fundamental sychronization element for ctdb. You cannot have
235 * more than one lock at once.
237 * You MUST NOT block during holding this lock and MUST release it
238 * quickly by performing ctdb_release_lock(lock).
239 * Do NOT make any system calls that may block while holding the lock.
241 * Try to release the lock as quickly as possible.
246 * ctdb_rrl_callback_t - callback for ctdb_readrecordlock_async
248 * This is not the standard ctdb_callback_t, because there is often no
249 * request required to access a database record (ie. if it is local already).
250 * So the callback is handed the lock directly: it might be NULL if there
251 * was an error obtaining the lock.
254 * ctdb_readrecordlock_async(), ctdb_readrecordlock()
256 typedef void (*ctdb_rrl_callback_t)(struct ctdb_db *ctdb_db,
257 struct ctdb_lock *lock,
262 * ctdb_readrecordlock_async - read and lock a record
263 * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
264 * @key: the key of the record to lock.
265 * @callback: the callback once the record is locked (typesafe).
266 * @cbdata: the argument to callback()
268 * This returns true on success. Commonly, we can obtain the record
269 * immediately and so the callback will be invoked. Otherwise a request
270 * will be queued to ctdbd for the record.
272 * If failure is immediate, false is returned. Otherwise, the callback
273 * may receive a NULL lock arg to indicate asynchronous failure.
275 bool ctdb_readrecordlock_async(struct ctdb_db *ctdb_db, TDB_DATA key,
276 ctdb_rrl_callback_t callback, void *cbdata);
279 * ctdb_writerecord - write a locked record in a TDB
280 * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
281 * @lock: the lock from ctdb_readrecordlock/ctdb_readrecordlock_recv
282 * @data: the new data to place in the record.
284 bool ctdb_writerecord(struct ctdb_db *ctdb_db,
285 struct ctdb_lock *lock, TDB_DATA data);
288 * ctdb_release_lock - release a record lock on a TDB
289 * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
290 * @lock: the lock from ctdb_readrecordlock/ctdb_readrecordlock_async
292 void ctdb_release_lock(struct ctdb_db *ctdb_db, struct ctdb_lock *lock);
295 * ctdb_message_fn_t - messaging callback for ctdb messages
297 * ctdbd provides a simple messaging API; you can register for a particular
298 * 64-bit id on which you want to send messages, and send to other ids.
301 * ctdb_set_message_handler_send()
303 typedef void (*ctdb_message_fn_t)(struct ctdb_connection *,
304 uint64_t srvid, TDB_DATA data, void *);
307 * ctdb_set_message_handler_send - register for messages to a srvid
308 * @ctdb: the ctdb_connection from ctdb_connect.
309 * @srvid: the 64 bit identifier for our messages.
310 * @handler: the callback when we receive such a message (typesafe)
311 * @handler_data: the argument to handler()
312 * @callback: the callback when ctdb replies to our message (typesafe)
313 * @cbdata: the argument to callback()
315 * Note: our callback will always be called before handler.
318 * ctdb_set_message_handler_recv(), ctdb_remove_message_handler_send()
320 struct ctdb_request *
321 ctdb_set_message_handler_send(struct ctdb_connection *ctdb, uint64_t srvid,
322 ctdb_message_fn_t handler,
324 ctdb_callback_t callback,
328 * ctdb_set_message_handler_recv - read a set_message_handler result
329 * @ctdb: the ctdb_connection from ctdb_connect.
330 * @req: the completed request
332 * If this returns true, the registered handler may be called from the next
333 * ctdb_service(). If this returns false, the registration failed.
335 bool ctdb_set_message_handler_recv(struct ctdb_connection *ctdb,
336 struct ctdb_request *handle);
339 * ctdb_remove_message_handler_send - unregister for messages to a srvid
340 * @ctdb: the ctdb_connection from ctdb_connect.
341 * @srvid: the 64 bit identifier for our messages.
342 * @handler: the callback when we receive such a message (typesafe)
343 * @handler_data: the argument to handler()
344 * @callback: the callback when ctdb replies to our message (typesafe)
345 * @cbdata: the argument to callback()
347 * This undoes a successful ctdb_set_message_handler or
348 * ctdb_set_message_handler_recv.
350 struct ctdb_request *
351 ctdb_remove_message_handler_send(struct ctdb_connection *ctdb, uint64_t srvid,
352 ctdb_message_fn_t handler, void *handler_data,
353 ctdb_callback_t callback, void *cbdata);
356 * ctdb_remove_message_handler_recv - read a remove_message_handler result
357 * @ctdb: the ctdb_connection from ctdb_connect.
358 * @req: the completed request
360 * After this returns true, the registered handler will no longer be called.
361 * If this returns false, the de-registration failed.
363 bool ctdb_remove_message_handler_recv(struct ctdb_connection *ctdb,
364 struct ctdb_request *req);
368 * ctdb_send_message - send a message via ctdbd
369 * @ctdb: the ctdb_connection from ctdb_connect.
370 * @pnn: the physical node number to send to
371 * @srvid: the 64 bit identifier for this message type.
372 * @data: the data to send
374 * This allows arbitrary messages to be sent across the cluster to those
375 * listening (via ctdb_set_message_handler et al).
377 * This queues a message to be sent: you will need to call
378 * ctdb_service() to actually send the message. There is no callback
379 * because there is no acknowledgement.
382 * ctdb_getpnn_send(), ctdb_getpnn()
384 bool ctdb_send_message(struct ctdb_connection *ctdb, uint32_t pnn, uint64_t srvid, TDB_DATA data);
387 * ctdb_getpnn_send - read the pnn number of a node.
388 * @ctdb: the ctdb_connection from ctdb_connect.
389 * @destnode: the destination node (see below)
390 * @callback: the callback when ctdb replies to our message (typesafe)
391 * @cbdata: the argument to callback()
393 * There are several special values for destnode, detailed in
394 * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
397 struct ctdb_request *
398 ctdb_getpnn_send(struct ctdb_connection *ctdb,
400 ctdb_callback_t callback,
403 * ctdb_getpnn_recv - read an ctdb_getpnn reply from ctdbd
404 * @ctdb: the ctdb_connection from ctdb_connect.
405 * @req: the completed request.
406 * @pnn: a pointer to the pnn to fill in
408 * This returns false if something went wrong, or otherwise fills in pnn.
410 bool ctdb_getpnn_recv(struct ctdb_connection *ctdb,
411 struct ctdb_request *req, uint32_t *pnn);
415 * ctdb_getnodemap_send - read the nodemap number from a node.
416 * @ctdb: the ctdb_connection from ctdb_connect.
417 * @destnode: the destination node (see below)
418 * @callback: the callback when ctdb replies to our message (typesafe)
419 * @cbdata: the argument to callback()
421 * There are several special values for destnode, detailed in
422 * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
425 struct ctdb_request *
426 ctdb_getnodemap_send(struct ctdb_connection *ctdb,
428 ctdb_callback_t callback,
431 * ctdb_getnodemap_recv - read an ctdb_getnodemap reply from ctdbd
432 * @ctdb: the ctdb_connection from ctdb_connect.
433 * @req: the completed request.
434 * @nodemap: a pointer to the returned nodemap structure
436 * This returns false if something went wrong.
437 * If the command failed, it guarantees to set nodemap to NULL.
438 * A non-NULL value for nodemap means the command was successful.
440 * A non-NULL value of the nodemap must be release released/freed
441 * by ctdb_free_nodemap().
443 bool ctdb_getnodemap_recv(struct ctdb_connection *ctdb,
444 struct ctdb_request *req, struct ctdb_node_map **nodemap);
447 * ctdb_getrecmaster_send - read the recovery master of a node
448 * @ctdb: the ctdb_connection from ctdb_connect.
449 * @destnode: the destination node (see below)
450 * @callback: the callback when ctdb replies to our message (typesafe)
451 * @cbdata: the argument to callback()
453 * There are several special values for destnode, detailed in
454 * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
457 struct ctdb_request *
458 ctdb_getrecmaster_send(struct ctdb_connection *ctdb,
460 ctdb_callback_t callback, void *cbdata);
463 * ctdb_getrecmaster_recv - read an ctdb_getrecmaster reply from ctdbd
464 * @ctdb: the ctdb_connection from ctdb_connect.
465 * @req: the completed request.
466 * @recmaster: a pointer to the recmaster to fill in
468 * This returns false if something went wrong, or otherwise fills in
471 bool ctdb_getrecmaster_recv(struct ctdb_connection *ctdb,
472 struct ctdb_request *handle,
473 uint32_t *recmaster);
476 * ctdb_cancel - cancel an uncompleted request
477 * @ctdb: the ctdb_connection from ctdb_connect.
478 * @req: the uncompleted request.
480 * This cancels a request, returning true. You may not cancel a
481 * request which has already been completed (ie. once its callback has
482 * been called); you should simply use ctdb_request_free() in that case.
484 void ctdb_cancel(struct ctdb_connection *ctdb, struct ctdb_request *req);
493 * ctdb_attachdb - open a clustered TDB (synchronous)
494 * @ctdb: the ctdb_connection from ctdb_connect.
495 * @name: the filename of the database (no /).
496 * @persistent: whether the database is persistent across ctdbd's life
497 * @tdb_flags: the flags to pass to tdb_open.
499 * Do a ctdb_attachdb_send and wait for it to complete.
500 * Returns NULL on failure.
502 struct ctdb_db *ctdb_attachdb(struct ctdb_connection *ctdb,
503 const char *name, bool persistent,
507 * ctdb_detachdb - close a clustered TDB.
508 * @ctdb: the ctdb_connection from ctdb_connect.
509 * @db: the database from ctdb_attachdb/ctdb_attachdb_send
511 * Closes a clustered tdb.
513 void ctdb_detachdb(struct ctdb_connection *ctdb, struct ctdb_db *db);
516 * ctdb_readrecordlock - read and lock a record (synchronous)
517 * @ctdb: the ctdb_connection from ctdb_connect.
518 * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
519 * @key: the key of the record to lock.
520 * @req: a pointer to the request, if one is needed.
522 * Do a ctdb_readrecordlock_send and wait for it to complete.
523 * Returns NULL on failure.
525 struct ctdb_lock *ctdb_readrecordlock(struct ctdb_connection *ctdb,
526 struct ctdb_db *ctdb_db, TDB_DATA key,
531 * ctdb_set_message_handler - register for messages to a srvid (synchronous)
532 * @ctdb: the ctdb_connection from ctdb_connect.
533 * @srvid: the 64 bit identifier for our messages.
534 * @handler: the callback when we receive such a message (typesafe)
535 * @cbdata: the argument to handler()
537 * If this returns true, the message handler can be called from any
538 * ctdb_service() (which is also called indirectly by other
539 * synchronous functions). If this returns false, the registration
542 bool ctdb_set_message_handler(struct ctdb_connection *ctdb, uint64_t srvid,
543 ctdb_message_fn_t handler, void *cbdata);
547 * ctdb_remove_message_handler - deregister for messages (synchronous)
548 * @ctdb: the ctdb_connection from ctdb_connect.
549 * @srvid: the 64 bit identifier for our messages.
550 * @handler: the callback when we receive such a message (typesafe)
551 * @handler_data: the argument to handler()
553 * If this returns true, the message handler will no longer be called.
554 * If this returns false, the deregistration failed.
556 bool ctdb_remove_message_handler(struct ctdb_connection *ctdb, uint64_t srvid,
557 ctdb_message_fn_t handler, void *handler_data);
560 * ctdb_getpnn - read the pnn number of a node (synchronous)
561 * @ctdb: the ctdb_connection from ctdb_connect.
562 * @destnode: the destination node (see below)
563 * @pnn: a pointer to the pnn to fill in
565 * There are several special values for destnode, detailed in
566 * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
569 * Returns true and fills in *pnn on success.
571 bool ctdb_getpnn(struct ctdb_connection *ctdb,
576 * ctdb_getrecmaster - read the recovery master of a node (synchronous)
577 * @ctdb: the ctdb_connection from ctdb_connect.
578 * @destnode: the destination node (see below)
579 * @recmaster: a pointer to the recmaster to fill in
581 * There are several special values for destnode, detailed in
582 * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
585 * Returns true and fills in *recmaster on success.
587 bool ctdb_getrecmaster(struct ctdb_connection *ctdb,
589 uint32_t *recmaster);
593 * ctdb_getnodemap - read the nodemap from a node (synchronous)
594 * @ctdb: the ctdb_connection from ctdb_connect.
595 * @destnode: the destination node (see below)
596 * @nodemap: a pointer to the nodemap to fill in
598 * There are several special values for destnode, detailed in
599 * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
602 * Returns true and fills in *nodemap on success.
603 * A non-NULL nodemap must be freed by calling ctdb_free_nodemap.
605 bool ctdb_getnodemap(struct ctdb_connection *ctdb,
606 uint32_t destnode, struct ctdb_node_map **nodemap);
609 * This function is used to release/free the nodemap structure returned
610 * by ctdb_getnodemap() and ctdb_getnodemap_recv()
612 void ctdb_free_nodemap(struct ctdb_node_map *nodemap);
616 /* These ugly macro wrappers make the callbacks typesafe. */
617 #include <ctdb_typesafe_cb.h>
618 #define ctdb_sendcb(cb, cbdata) \
619 typesafe_cb_preargs(void, (cb), (cbdata), \
620 struct ctdb_connection *, struct ctdb_request *)
622 #define ctdb_msgcb(cb, cbdata) \
623 typesafe_cb_preargs(void, (cb), (cbdata), \
624 struct ctdb_connection *, uint64_t, TDB_DATA)
626 #define ctdb_connect(addr, log, logpriv) \
627 ctdb_connect((addr), \
628 typesafe_cb_postargs(void, (log), (logpriv), \
629 int, const char *, va_list), \
632 #define ctdb_set_message_handler(ctdb, srvid, handler, hdata) \
633 ctdb_set_message_handler((ctdb), (srvid), \
634 ctdb_msgcb((handler), (hdata)), (hdata))
636 #define ctdb_remove_message_handler(ctdb, srvid, handler, hdata) \
637 ctdb_remove_message_handler((ctdb), (srvid), \
638 ctdb_msgcb((handler), (hdata)), (hdata))
640 #define ctdb_attachdb_send(ctdb, name, persistent, tdb_flags, cb, cbdata) \
641 ctdb_attachdb_send((ctdb), (name), (persistent), (tdb_flags), \
642 ctdb_sendcb((cb), (cbdata)), (cbdata))
644 #define ctdb_readrecordlock_async(_ctdb_db, key, cb, cbdata) \
645 ctdb_readrecordlock_async((_ctdb_db), (key), \
646 typesafe_cb_preargs(void, (cb), (cbdata), \
647 struct ctdb_db *, struct ctdb_lock *, \
650 #define ctdb_set_message_handler_send(ctdb, srvid, handler, hdata, cb, cbdata) \
651 ctdb_set_message_handler_send((ctdb), (srvid), \
652 ctdb_msgcb((handler), (hdata)), (hdata), \
653 ctdb_sendcb((cb), (cbdata)), (cbdata))
655 #define ctdb_remove_message_handler_send(ctdb, srvid, handler, hdata, cb, cbdata) \
656 ctdb_remove_message_handler_send((ctdb), (srvid), \
657 ctdb_msgcb((handler), (hdata)), (hdata), \
658 ctdb_sendcb((cb), (cbdata)), (cbdata))
660 #define ctdb_getpnn_send(ctdb, destnode, cb, cbdata) \
661 ctdb_getpnn_send((ctdb), (destnode), \
662 ctdb_sendcb((cb), (cbdata)), (cbdata))
664 #define ctdb_getrecmaster_send(ctdb, destnode, cb, cbdata) \
665 ctdb_getrecmaster_send((ctdb), (destnode), \
666 ctdb_sendcb((cb), (cbdata)), (cbdata))
668 #define ctdb_getnodemap_send(ctdb, destnode, cb, cbdata) \
669 ctdb_getnodemap_send((ctdb), (destnode), \
670 ctdb_sendcb((cb), (cbdata)), (cbdata))