4 Copyright (C) Andrew Tridgell 2007
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #include "system/filesys.h"
22 #include "system/wait.h"
23 #include "system/dir.h"
24 #include "system/locale.h"
25 #include "../include/ctdb_private.h"
26 #include "lib/events/events.h"
27 #include "../common/rb_tree.h"
31 const char *script_running;
35 ctdbd sends us a SIGTERM when we should time out the current script
37 static void sigterm(int sig)
39 DEBUG(DEBUG_ERR,("Timed out running script '%s' after %.1f seconds\n",
40 child_state.script_running, timeval_elapsed(&child_state.start)));
41 /* all the child processes will be running in the same process group */
42 kill(-getpgrp(), SIGKILL);
46 struct ctdb_event_script_state {
47 struct ctdb_context *ctdb;
49 void (*callback)(struct ctdb_context *, int, void *);
56 struct ctdb_monitor_script_status {
57 struct ctdb_monitor_script_status *next;
60 struct timeval finished;
66 struct ctdb_monitoring_status {
68 struct timeval finished;
70 struct ctdb_monitor_script_status *scripts;
74 /* called from ctdb_logging when we have received output on STDERR from
75 * one of the eventscripts
77 int ctdb_log_event_script_output(struct ctdb_context *ctdb, char *str, uint16_t len)
79 struct ctdb_monitoring_status *monitoring_status =
80 talloc_get_type(ctdb->script_monitoring_ctx,
81 struct ctdb_monitoring_status);
82 struct ctdb_monitor_script_status *script;
84 if (monitoring_status == NULL) {
88 script = monitoring_status->scripts;
93 if (script->output == NULL) {
94 script->output = talloc_asprintf(script, "%*.*s", len, len, str);
96 script->output = talloc_asprintf_append(script->output, "%*.*s", len, len, str);
102 /* called from the event script child process when we are starting a new
105 int32_t ctdb_control_event_script_init(struct ctdb_context *ctdb)
107 struct ctdb_monitoring_status *monitoring_status;
109 DEBUG(DEBUG_INFO, ("event script init called\n"));
110 if (ctdb->script_monitoring_ctx != NULL) {
111 talloc_free(ctdb->script_monitoring_ctx);
112 ctdb->script_monitoring_ctx = NULL;
115 monitoring_status = talloc_zero(ctdb, struct ctdb_monitoring_status);
116 if (monitoring_status == NULL) {
117 DEBUG(DEBUG_ERR, (__location__ " ERROR: Failed to talloc script_monitoring context\n"));
121 ctdb->script_monitoring_ctx = monitoring_status;
122 monitoring_status->start = timeval_current();
128 /* called from the event script child process when we are star running
131 int32_t ctdb_control_event_script_start(struct ctdb_context *ctdb, TDB_DATA indata)
133 const char *name = (const char *)indata.dptr;
134 struct ctdb_monitoring_status *monitoring_status =
135 talloc_get_type(ctdb->script_monitoring_ctx,
136 struct ctdb_monitoring_status);
137 struct ctdb_monitor_script_status *script;
139 DEBUG(DEBUG_INFO, ("event script start called : %s\n", name));
141 if (monitoring_status == NULL) {
142 DEBUG(DEBUG_ERR,(__location__ " script_status is NULL when starting to run script %s\n", name));
146 script = talloc_zero(monitoring_status, struct ctdb_monitor_script_status);
147 if (script == NULL) {
148 DEBUG(DEBUG_ERR,(__location__ " Failed to talloc ctdb_monitor_script_status for script %s\n", name));
152 script->next = monitoring_status->scripts;
153 script->name = talloc_strdup(script, name);
154 script->start = timeval_current();
155 monitoring_status->scripts = script;
160 /* called from the event script child process when we have finished running
163 int32_t ctdb_control_event_script_stop(struct ctdb_context *ctdb, TDB_DATA indata)
165 int32_t res = *((int32_t *)indata.dptr);
166 struct ctdb_monitoring_status *monitoring_status =
167 talloc_get_type(ctdb->script_monitoring_ctx,
168 struct ctdb_monitoring_status);
169 struct ctdb_monitor_script_status *script;
171 DEBUG(DEBUG_INFO, ("event script stop called : %d\n", (int)res));
173 if (monitoring_status == NULL) {
174 DEBUG(DEBUG_ERR,(__location__ " script_status is NULL when script finished.\n"));
178 script = monitoring_status->scripts;
179 if (script == NULL) {
180 DEBUG(DEBUG_ERR,(__location__ " script is NULL when the script had finished\n"));
184 script->finished = timeval_current();
185 script->status = res;
190 /* called from the event script child process when we have completed a
193 int32_t ctdb_control_event_script_finished(struct ctdb_context *ctdb)
195 struct ctdb_monitoring_status *monitoring_status =
196 talloc_get_type(ctdb->script_monitoring_ctx,
197 struct ctdb_monitoring_status);
199 DEBUG(DEBUG_INFO, ("event script finished called\n"));
201 if (monitoring_status == NULL) {
202 DEBUG(DEBUG_ERR,(__location__ " script_status is NULL when monitoring event finished\n"));
206 monitoring_status->finished = timeval_current();
207 monitoring_status->status = MONITOR_SCRIPT_OK;
208 if (ctdb->last_monitoring_ctx) {
209 talloc_free(ctdb->last_monitoring_ctx);
211 ctdb->last_monitoring_ctx = ctdb->script_monitoring_ctx;
212 ctdb->script_monitoring_ctx = NULL;
217 static struct ctdb_monitoring_wire *marshall_monitoring_scripts(TALLOC_CTX *mem_ctx, struct ctdb_monitoring_wire *monitoring_scripts, struct ctdb_monitor_script_status *script)
219 struct ctdb_monitoring_script_wire script_wire;
222 if (script == NULL) {
223 return monitoring_scripts;
225 monitoring_scripts = marshall_monitoring_scripts(mem_ctx, monitoring_scripts, script->next);
226 if (monitoring_scripts == NULL) {
230 bzero(&script_wire, sizeof(struct ctdb_monitoring_script_wire));
231 strncpy(script_wire.name, script->name, MAX_SCRIPT_NAME);
232 script_wire.start = script->start;
233 script_wire.finished = script->finished;
234 script_wire.status = script->status;
235 script_wire.timedout = script->timedout;
236 if (script->output != NULL) {
237 strncpy(script_wire.output, script->output, MAX_SCRIPT_OUTPUT);
240 size = talloc_get_size(monitoring_scripts);
241 monitoring_scripts = talloc_realloc_size(mem_ctx, monitoring_scripts, size + sizeof(struct ctdb_monitoring_script_wire));
242 if (monitoring_scripts == NULL) {
243 DEBUG(DEBUG_ERR,(__location__ " Failed to talloc_resize monitoring_scripts blob\n"));
247 memcpy(&monitoring_scripts->scripts[monitoring_scripts->num_scripts], &script_wire, sizeof(script_wire));
248 monitoring_scripts->num_scripts++;
250 return monitoring_scripts;
253 int32_t ctdb_control_get_event_script_status(struct ctdb_context *ctdb, TDB_DATA *outdata)
255 struct ctdb_monitoring_status *monitoring_status =
256 talloc_get_type(ctdb->last_monitoring_ctx,
257 struct ctdb_monitoring_status);
258 struct ctdb_monitoring_wire *monitoring_scripts;
260 if (monitoring_status == NULL) {
261 DEBUG(DEBUG_ERR,(__location__ " last_monitor_ctx is NULL when reading status\n"));
265 monitoring_scripts = talloc_size(outdata, offsetof(struct ctdb_monitoring_wire, scripts));
266 if (monitoring_scripts == NULL) {
267 DEBUG(DEBUG_ERR,(__location__ " failed to talloc monitoring_scripts structure\n"));
271 monitoring_scripts->num_scripts = 0;
272 monitoring_scripts = marshall_monitoring_scripts(outdata, monitoring_scripts, monitoring_status->scripts);
273 if (monitoring_scripts == NULL) {
274 DEBUG(DEBUG_ERR,(__location__ " Monitoring scritps is NULL. can not return data to client\n"));
278 outdata->dsize = talloc_get_size(monitoring_scripts);
279 outdata->dptr = (uint8_t *)monitoring_scripts;
285 run the event script - varargs version
286 this function is called and run in the context of a forked child
287 which allows it to do blocking calls such as system()
289 static int ctdb_event_script_v(struct ctdb_context *ctdb, const char *options)
294 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
302 if (!strcmp(options, "monitor")) {
306 if (is_monitor == 1) {
307 /* This is running in the forked child process. At this stage
308 * we want to switch from being a ctdb daemon into being a
309 * client and connect to the real local daemon.
311 if (switch_from_server_to_client(ctdb) != 0) {
312 DEBUG(DEBUG_CRIT, (__location__ "ERROR: failed to switch eventscript child into client mode. shutting down.\n"));
316 if (ctdb_ctrl_event_script_init(ctdb) != 0) {
317 DEBUG(DEBUG_ERR,(__location__ " Failed to init event script monitoring\n"));
318 talloc_free(tmp_ctx);
323 if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
324 /* we guarantee that only some specifically allowed event scripts are run
326 const char *allowed_scripts[] = {"startrecovery", "shutdown", "releaseip" };
328 for (i=0;i<ARRAY_SIZE(allowed_scripts);i++) {
329 if (strncmp(options, allowed_scripts[i], strlen(allowed_scripts[i])) == 0) break;
331 if (i == ARRAY_SIZE(allowed_scripts)) {
332 DEBUG(DEBUG_ERR,("Refusing to run event scripts with option '%s' while in recovery\n",
334 talloc_free(tmp_ctx);
339 if (setpgid(0,0) != 0) {
340 DEBUG(DEBUG_ERR,("Failed to create process group for event scripts - %s\n",
342 talloc_free(tmp_ctx);
346 signal(SIGTERM, sigterm);
348 child_state.start = timeval_current();
349 child_state.script_running = "startup";
352 the service specific event scripts
354 if (stat(ctdb->event_script_dir, &st) != 0 &&
356 DEBUG(DEBUG_CRIT,("No event script directory found at '%s'\n", ctdb->event_script_dir));
357 talloc_free(tmp_ctx);
361 /* create a tree to store all the script names in */
362 tree = trbt_create(tmp_ctx, 0);
364 /* scan all directory entries and insert all valid scripts into the
367 dir = opendir(ctdb->event_script_dir);
369 DEBUG(DEBUG_CRIT,("Failed to open event script directory '%s'\n", ctdb->event_script_dir));
370 talloc_free(tmp_ctx);
375 while ((de=readdir(dir)) != NULL) {
380 namlen = strlen(de->d_name);
386 if (de->d_name[namlen-1] == '~') {
387 /* skip files emacs left behind */
391 if (de->d_name[2] != '.') {
395 if (sscanf(de->d_name, "%02u.", &num) != 1) {
399 /* Make sure the event script is executable */
400 str = talloc_asprintf(tree, "%s/%s", ctdb->event_script_dir, de->d_name);
401 if (stat(str, &st) != 0) {
402 DEBUG(DEBUG_ERR,("Could not stat event script %s. Ignoring this event script\n", str));
405 if (!(st.st_mode & S_IXUSR)) {
406 DEBUG(DEBUG_ERR,("Event script %s is not executable. Ignoring this event script\n", str));
411 /* store the event script in the tree */
412 trbt_insert32(tree, (num<<16)|count++, talloc_strdup(tree, de->d_name));
416 /* fetch the scripts from the tree one by one and execute
419 while ((script=trbt_findfirstarray32(tree, 1)) != NULL) {
420 cmdstr = talloc_asprintf(tmp_ctx, "%s/%s %s",
421 ctdb->event_script_dir,
423 CTDB_NO_MEMORY(ctdb, cmdstr);
425 DEBUG(DEBUG_INFO,("Executing event script %s\n",cmdstr));
427 child_state.start = timeval_current();
428 child_state.script_running = cmdstr;
430 if (is_monitor == 1) {
431 if (ctdb_ctrl_event_script_start(ctdb, script) != 0) {
432 DEBUG(DEBUG_ERR,(__location__ " Failed to start event script monitoring\n"));
433 talloc_free(tmp_ctx);
438 ret = system(cmdstr);
439 /* if the system() call was successful, translate ret into the
440 return code from the command
443 ret = WEXITSTATUS(ret);
445 if (is_monitor == 1) {
446 if (ctdb_ctrl_event_script_stop(ctdb, ret) != 0) {
447 DEBUG(DEBUG_ERR,(__location__ " Failed to stop event script monitoring\n"));
448 talloc_free(tmp_ctx);
453 /* return an error if the script failed */
455 DEBUG(DEBUG_ERR,("Event script %s failed with error %d\n", cmdstr, ret));
456 if (is_monitor == 1) {
457 if (ctdb_ctrl_event_script_finished(ctdb) != 0) {
458 DEBUG(DEBUG_ERR,(__location__ " Failed to finish event script monitoring\n"));
459 talloc_free(tmp_ctx);
464 talloc_free(tmp_ctx);
468 /* remove this script from the tree */
472 child_state.start = timeval_current();
473 child_state.script_running = "finished";
475 if (is_monitor == 1) {
476 if (ctdb_ctrl_event_script_finished(ctdb) != 0) {
477 DEBUG(DEBUG_ERR,(__location__ " Failed to finish event script monitoring\n"));
478 talloc_free(tmp_ctx);
483 talloc_free(tmp_ctx);
487 /* called when child is finished */
488 static void ctdb_event_script_handler(struct event_context *ev, struct fd_event *fde,
489 uint16_t flags, void *p)
491 struct ctdb_event_script_state *state =
492 talloc_get_type(p, struct ctdb_event_script_state);
493 void (*callback)(struct ctdb_context *, int, void *) = state->callback;
494 void *private_data = state->private_data;
495 struct ctdb_context *ctdb = state->ctdb;
498 read(state->fd[0], &rt, sizeof(rt));
500 talloc_set_destructor(state, NULL);
502 callback(ctdb, rt, private_data);
504 ctdb->event_script_timeouts = 0;
507 static void ctdb_ban_self(struct ctdb_context *ctdb, uint32_t ban_period)
510 struct ctdb_ban_info b;
514 b.ban_time = ban_period;
516 data.dptr = (uint8_t *)&b;
517 data.dsize = sizeof(b);
519 ret = ctdb_daemon_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
520 CTDB_SRVID_BAN_NODE, data);
522 DEBUG(DEBUG_ERR,(__location__ " Failed to send ban message\n"));
527 /* called when child times out */
528 static void ctdb_event_script_timeout(struct event_context *ev, struct timed_event *te,
529 struct timeval t, void *p)
531 struct ctdb_event_script_state *state = talloc_get_type(p, struct ctdb_event_script_state);
532 void (*callback)(struct ctdb_context *, int, void *) = state->callback;
533 void *private_data = state->private_data;
534 struct ctdb_context *ctdb = state->ctdb;
536 struct ctdb_monitoring_status *monitoring_status =
537 talloc_get_type(ctdb->script_monitoring_ctx,
538 struct ctdb_monitoring_status);
540 DEBUG(DEBUG_ERR,("Event script timed out : %s count : %u\n", state->options, ctdb->event_script_timeouts));
542 options = talloc_strdup(ctdb, state->options);
543 CTDB_NO_MEMORY_VOID(ctdb, options);
546 if (!strcmp(options, "monitor")) {
547 /* if it is a monitor event, we allow it to "hang" a few times
548 before we declare it a failure and ban ourself (and make
551 DEBUG(DEBUG_ERR, (__location__ " eventscript for monitor event timedout.\n"));
553 ctdb->event_script_timeouts++;
554 if (ctdb->event_script_timeouts > ctdb->tunable.script_ban_count) {
555 ctdb->event_script_timeouts = 0;
556 DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Banning self for %d seconds\n", ctdb->tunable.script_ban_count, ctdb->tunable.recovery_ban_period));
557 ctdb_ban_self(ctdb, ctdb->tunable.recovery_ban_period);
558 callback(ctdb, -1, private_data);
560 callback(ctdb, 0, private_data);
562 } else if (!strcmp(options, "startup")) {
563 DEBUG(DEBUG_ERR, (__location__ " eventscript for startup event timedout.\n"));
564 callback(ctdb, -1, private_data);
566 /* if it is not a monitor event we ban ourself immediately */
567 DEBUG(DEBUG_ERR, (__location__ " eventscript for NON-monitor/NON-startup event timedout. Immediately banning ourself for %d seconds\n", ctdb->tunable.recovery_ban_period));
568 ctdb_ban_self(ctdb, ctdb->tunable.recovery_ban_period);
569 callback(ctdb, -1, private_data);
572 if (monitoring_status != NULL) {
573 struct ctdb_monitor_script_status *script;
575 script = monitoring_status->scripts;
576 if (script != NULL) {
577 script->timedout = 1;
579 monitoring_status->status = MONITOR_SCRIPT_TIMEOUT;
580 if (ctdb->last_monitoring_ctx) {
581 talloc_free(ctdb->last_monitoring_ctx);
582 ctdb->last_monitoring_ctx = ctdb->script_monitoring_ctx;
583 ctdb->script_monitoring_ctx = NULL;
587 talloc_free(options);
591 destroy a running event script
593 static int event_script_destructor(struct ctdb_event_script_state *state)
595 DEBUG(DEBUG_ERR,(__location__ " Sending SIGTERM to child pid:%d\n", state->child));
596 kill(state->child, SIGTERM);
601 run the event script in the background, calling the callback when
604 static int ctdb_event_script_callback_v(struct ctdb_context *ctdb,
605 struct timeval timeout,
607 void (*callback)(struct ctdb_context *, int, void *),
609 const char *fmt, va_list ap)
611 struct ctdb_event_script_state *state;
614 state = talloc(mem_ctx, struct ctdb_event_script_state);
615 CTDB_NO_MEMORY(ctdb, state);
618 state->callback = callback;
619 state->private_data = private_data;
620 state->options = talloc_vasprintf(state, fmt, ap);
621 CTDB_NO_MEMORY(ctdb, state->options);
623 ret = pipe(state->fd);
629 state->child = fork();
631 if (state->child == (pid_t)-1) {
638 if (state->child == 0) {
642 set_close_on_exec(state->fd[1]);
644 rt = ctdb_event_script_v(ctdb, state->options);
645 while ((ret = write(state->fd[1], &rt, sizeof(rt))) != sizeof(rt)) {
651 talloc_set_destructor(state, event_script_destructor);
655 event_add_fd(ctdb->ev, state, state->fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
656 ctdb_event_script_handler, state);
658 if (!timeval_is_zero(&timeout)) {
659 event_add_timed(ctdb->ev, state, timeout, ctdb_event_script_timeout, state);
661 DEBUG(DEBUG_ERR, (__location__ " eventscript %s called with no timeout\n", state->options));
669 run the event script in the background, calling the callback when
672 int ctdb_event_script_callback(struct ctdb_context *ctdb,
673 struct timeval timeout,
675 void (*callback)(struct ctdb_context *, int, void *),
677 const char *fmt, ...)
683 ret = ctdb_event_script_callback_v(ctdb, timeout, mem_ctx, callback, private_data, fmt, ap);
690 struct callback_status {
696 called when ctdb_event_script() finishes
698 static void event_script_callback(struct ctdb_context *ctdb, int status, void *private_data)
700 struct callback_status *s = (struct callback_status *)private_data;
706 run the event script, waiting for it to complete. Used when the caller doesn't want to
707 continue till the event script has finished.
709 int ctdb_event_script(struct ctdb_context *ctdb, const char *fmt, ...)
713 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
714 struct callback_status status;
717 ret = ctdb_event_script_callback_v(ctdb, timeval_zero(), tmp_ctx, event_script_callback, &status, fmt, ap);
721 talloc_free(tmp_ctx);
728 while (status.done == false && event_loop_once(ctdb->ev) == 0) /* noop */;
730 talloc_free(tmp_ctx);
732 return status.status;
736 struct eventscript_callback_state {
737 struct ctdb_req_control *c;
741 called when takeip event finishes
743 static void run_eventscripts_callback(struct ctdb_context *ctdb, int status,
746 struct eventscript_callback_state *state =
747 talloc_get_type(private_data, struct eventscript_callback_state);
749 ctdb_enable_monitoring(ctdb);
752 DEBUG(DEBUG_ERR,(__location__ " Failed to forcibly run eventscripts\n"));
753 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
758 /* the control succeeded */
759 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
765 A control to force running of the eventscripts from the ctdb client tool
767 int32_t ctdb_run_eventscripts(struct ctdb_context *ctdb,
768 struct ctdb_req_control *c,
769 TDB_DATA indata, bool *async_reply)
772 struct eventscript_callback_state *state;
774 /* kill off any previous invokations of forced eventscripts */
775 if (ctdb->eventscripts_ctx) {
776 talloc_free(ctdb->eventscripts_ctx);
778 ctdb->eventscripts_ctx = talloc_new(ctdb);
779 CTDB_NO_MEMORY(ctdb, ctdb->eventscripts_ctx);
781 state = talloc(ctdb->eventscripts_ctx, struct eventscript_callback_state);
782 CTDB_NO_MEMORY(ctdb, state);
784 state->c = talloc_steal(state, c);
786 DEBUG(DEBUG_NOTICE,("Forced running of eventscripts with arguments %s\n", indata.dptr));
788 if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
789 DEBUG(DEBUG_ERR, (__location__ " Aborted running eventscript \"%s\" while in RECOVERY mode\n", indata.dptr));
793 ctdb_disable_monitoring(ctdb);
795 ret = ctdb_event_script_callback(ctdb,
796 timeval_current_ofs(ctdb->tunable.script_timeout, 0),
797 state, run_eventscripts_callback, state,
798 (const char *)indata.dptr);
801 ctdb_enable_monitoring(ctdb);
802 DEBUG(DEBUG_ERR,(__location__ " Failed to run eventscripts with arguments %s\n", indata.dptr));
807 /* tell ctdb_control.c that we will be replying asynchronously */