#include "system/dir.h"
#include "system/locale.h"
#include "../include/ctdb_private.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
#include "../common/rb_tree.h"
-static struct {
- struct timeval start;
- const char *script_running;
-} child_state;
-
-static const char *call_names[] = {
- "startup",
- "startrecovery",
- "recovered",
- "takeip",
- "releaseip",
- "stopped",
- "monitor",
- "status",
- "shutdown",
- "reload"
-};
-
static void ctdb_event_script_timeout(struct event_context *ev, struct timed_event *te, struct timeval t, void *p);
/*
- ctdbd sends us a SIGTERM when we should time out the current script
+ ctdbd sends us a SIGTERM when we should die.
*/
static void sigterm(int sig)
{
- char tbuf[100], buf[200];
- time_t t;
-
- DEBUG(DEBUG_ERR,("Timed out running script '%s' after %.1f seconds pid :%d\n",
- child_state.script_running, timeval_elapsed(&child_state.start), getpid()));
-
- t = time(NULL);
-
- strftime(tbuf, sizeof(tbuf)-1, "%Y%m%d%H%M%S", localtime(&t));
- sprintf(buf, "pstree -p >/tmp/ctdb.event.%s.%d", tbuf, getpid());
- system(buf);
-
- DEBUG(DEBUG_ERR,("Logged timedout eventscript : %s\n", buf));
-
/* all the child processes will be running in the same process group */
kill(-getpgrp(), SIGKILL);
_exit(1);
pid_t child;
/* Warning: this can free us! */
void (*callback)(struct ctdb_context *, int, void *);
- int cb_status;
int fd[2];
void *private_data;
+ bool from_user;
enum ctdb_eventscript_call call;
const char *options;
struct timeval timeout;
-};
-
-
-struct ctdb_monitor_script_status {
- struct ctdb_monitor_script_status *next;
- const char *name;
- struct timeval start;
- struct timeval finished;
- int32_t status;
- char *output;
-};
-struct ctdb_monitor_script_status_ctx {
- struct ctdb_monitor_script_status *scripts;
+ unsigned int current;
+ struct ctdb_scripts_wire *scripts;
};
-/* called from ctdb_logging when we have received output on STDERR from
- * one of the eventscripts
- */
-int ctdb_log_event_script_output(struct ctdb_context *ctdb, char *str, uint16_t len)
-{
- struct ctdb_monitor_script_status *script;
-
- if (ctdb->current_monitor_status_ctx == NULL) {
- return -1;
- }
-
- script = ctdb->current_monitor_status_ctx->scripts;
- if (script == NULL) {
- return -1;
- }
-
- if (script->output == NULL) {
- script->output = talloc_asprintf(script, "%*.*s", len, len, str);
- } else {
- script->output = talloc_asprintf_append(script->output, "%*.*s", len, len, str);
- }
-
- return 0;
-}
-
-/* called from the event script child process when we are starting a new
- * monitor event
- */
-int32_t ctdb_control_event_script_init(struct ctdb_context *ctdb)
-{
- DEBUG(DEBUG_INFO, ("event script init called\n"));
-
- if (ctdb->current_monitor_status_ctx == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " current_monitor_status_ctx is NULL when initing script\n"));
- return -1;
- }
-
- return 0;
-}
-
-
-/* called from the event script child process when we are star running
- * an eventscript
- */
-int32_t ctdb_control_event_script_start(struct ctdb_context *ctdb, TDB_DATA indata)
+static struct ctdb_script_wire *get_current_script(struct ctdb_event_script_state *state)
{
- const char *name = (const char *)indata.dptr;
- struct ctdb_monitor_script_status *script;
-
- DEBUG(DEBUG_INFO, ("event script start called : %s\n", name));
-
- if (ctdb->current_monitor_status_ctx == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " current_monitor_status_ctx is NULL when starting script\n"));
- return -1;
- }
-
- script = talloc_zero(ctdb->current_monitor_status_ctx, struct ctdb_monitor_script_status);
- if (script == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to talloc ctdb_monitor_script_status for script %s\n", name));
- return -1;
- }
-
- script->next = ctdb->current_monitor_status_ctx->scripts;
- script->name = talloc_strdup(script, name);
- CTDB_NO_MEMORY(ctdb, script->name);
- script->start = timeval_current();
- ctdb->current_monitor_status_ctx->scripts = script;
-
- return 0;
+ return &state->scripts->scripts[state->current];
}
-/* called from the event script child process when we have finished running
- * an eventscript
+/* called from ctdb_logging when we have received output on STDERR from
+ * one of the eventscripts
*/
-int32_t ctdb_control_event_script_stop(struct ctdb_context *ctdb, TDB_DATA indata)
+static void log_event_script_output(const char *str, uint16_t len, void *p)
{
- int32_t res = *((int32_t *)indata.dptr);
- struct ctdb_monitor_script_status *script;
-
- if (ctdb->current_monitor_status_ctx == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " current_monitor_status_ctx is NULL when script finished\n"));
- return -1;
- }
-
- script = ctdb->current_monitor_status_ctx->scripts;
- if (script == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " script is NULL when the script had finished\n"));
- return -1;
- }
+ struct ctdb_event_script_state *state
+ = talloc_get_type(p, struct ctdb_event_script_state);
+ struct ctdb_script_wire *current;
+ unsigned int slen, min;
- script->finished = timeval_current();
- script->status = res;
-
- DEBUG(DEBUG_INFO, ("event script stop called for script:%s duration:%.1f status:%d\n", script->name, timeval_elapsed(&script->start), (int)res));
-
- return 0;
-}
-
-static struct ctdb_monitoring_wire *marshall_monitoring_scripts(TALLOC_CTX *mem_ctx, struct ctdb_monitoring_wire *monitoring_scripts, struct ctdb_monitor_script_status *script)
-{
- struct ctdb_monitoring_script_wire script_wire;
- size_t size;
-
- if (script == NULL) {
- return monitoring_scripts;
- }
- monitoring_scripts = marshall_monitoring_scripts(mem_ctx, monitoring_scripts, script->next);
- if (monitoring_scripts == NULL) {
- return NULL;
+ /* We may have been aborted to run something else. Discard */
+ if (state->scripts == NULL) {
+ return;
}
- bzero(&script_wire, sizeof(struct ctdb_monitoring_script_wire));
- strncpy(script_wire.name, script->name, MAX_SCRIPT_NAME);
- script_wire.start = script->start;
- script_wire.finished = script->finished;
- script_wire.status = script->status;
- if (script->output != NULL) {
- strncpy(script_wire.output, script->output, MAX_SCRIPT_OUTPUT);
- }
+ current = get_current_script(state);
- size = talloc_get_size(monitoring_scripts);
- monitoring_scripts = talloc_realloc_size(mem_ctx, monitoring_scripts, size + sizeof(struct ctdb_monitoring_script_wire));
- if (monitoring_scripts == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to talloc_resize monitoring_scripts blob\n"));
- return NULL;
- }
+ /* Append, but don't overfill buffer. It starts zero-filled. */
+ slen = strlen(current->output);
+ min = MIN(len, sizeof(current->output) - slen - 1);
- memcpy(&monitoring_scripts->scripts[monitoring_scripts->num_scripts], &script_wire, sizeof(script_wire));
- monitoring_scripts->num_scripts++;
-
- return monitoring_scripts;
+ memcpy(current->output + slen, str, min);
}
-/* called from the event script child process when we have completed a
- * monitor event
- */
-int32_t ctdb_control_event_script_finished(struct ctdb_context *ctdb)
+int32_t ctdb_control_get_event_script_status(struct ctdb_context *ctdb,
+ uint32_t call_type,
+ TDB_DATA *outdata)
{
- DEBUG(DEBUG_INFO, ("event script finished called\n"));
-
- if (ctdb->current_monitor_status_ctx == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " script_status is NULL when monitoring event finished\n"));
+ if (call_type >= CTDB_EVENT_MAX) {
return -1;
}
- talloc_free(ctdb->last_status);
- ctdb->last_status = talloc_size(ctdb, offsetof(struct ctdb_monitoring_wire, scripts));
- if (ctdb->last_status == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " failed to talloc last_status\n"));
- return -1;
- }
-
- ctdb->last_status->num_scripts = 0;
- ctdb->last_status = marshall_monitoring_scripts(ctdb, ctdb->last_status, ctdb->current_monitor_status_ctx->scripts);
- talloc_free(ctdb->current_monitor_status_ctx);
- ctdb->current_monitor_status_ctx = NULL;
-
- return 0;
-}
-
-int32_t ctdb_control_get_event_script_status(struct ctdb_context *ctdb, TDB_DATA *outdata)
-{
- struct ctdb_monitoring_wire *monitoring_scripts = ctdb->last_status;
-
- if (monitoring_scripts == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " last_monitor_status_ctx is NULL when reading status\n"));
- return -1;
+ if (ctdb->last_status[call_type] == NULL) {
+ /* If it's never been run, return nothing so they can tell. */
+ outdata->dsize = 0;
+ } else {
+ outdata->dsize = talloc_get_size(ctdb->last_status[call_type]);
+ outdata->dptr = (uint8_t *)ctdb->last_status[call_type];
}
-
- outdata->dsize = talloc_get_size(monitoring_scripts);
- outdata->dptr = (uint8_t *)monitoring_scripts;
-
return 0;
}
int error;
};
-struct ctdb_script_list {
- struct ctdb_script_list *next;
- const char *name;
- int error;
-};
-
/* Return true if OK, otherwise set errno. */
static bool check_executable(const char *dir, const char *name)
{
}
if (!(st.st_mode & S_IXUSR)) {
- DEBUG(DEBUG_INFO,("Event script %s is not executable. Ignoring this event script\n", full));
+ DEBUG(DEBUG_DEBUG,("Event script %s is not executable. Ignoring this event script\n", full));
errno = ENOEXEC;
talloc_free(full);
return false;
return true;
}
-static struct ctdb_script_list *ctdb_get_script_list(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx)
+static struct ctdb_scripts_wire *ctdb_get_script_list(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx)
{
DIR *dir;
struct dirent *de;
struct stat st;
trbt_tree_t *tree;
- struct ctdb_script_list *head, *tail, *new_item;
+ struct ctdb_scripts_wire *scripts;
TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
struct ctdb_script_tree_item *tree_item;
int count;
continue;
}
+ if (strlen(de->d_name) > MAX_SCRIPT_NAME) {
+ DEBUG(DEBUG_ERR,("Script name %s too long! %u chars max",
+ de->d_name, MAX_SCRIPT_NAME));
+ continue;
+ }
+
tree_item = talloc(tree, struct ctdb_script_tree_item);
if (tree_item == NULL) {
DEBUG(DEBUG_ERR, (__location__ " Failed to allocate new tree item\n"));
}
closedir(dir);
+ /* Overallocates by one, but that's OK */
+ scripts = talloc_zero_size(tmp_ctx,
+ sizeof(*scripts)
+ + sizeof(scripts->scripts[0]) * count);
+ if (scripts == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " Failed to allocate scripts\n"));
+ talloc_free(tmp_ctx);
+ return NULL;
+ }
+ scripts->num_scripts = count;
- head = NULL;
- tail = NULL;
+ for (count = 0; count < scripts->num_scripts; count++) {
+ tree_item = trbt_findfirstarray32(tree, 1);
- /* fetch the scripts from the tree one by one and add them to the linked
- list
- */
- while ((tree_item=trbt_findfirstarray32(tree, 1)) != NULL) {
+ strcpy(scripts->scripts[count].name, tree_item->name);
+ scripts->scripts[count].status = -tree_item->error;
- new_item = talloc(tmp_ctx, struct ctdb_script_list);
- if (new_item == NULL) {
- DEBUG(DEBUG_ERR, (__location__ " Failed to allocate new list item\n"));
- talloc_free(tmp_ctx);
- return NULL;
- }
+ /* remove this script from the tree */
+ talloc_free(tree_item);
+ }
- new_item->next = NULL;
- new_item->name = talloc_steal(new_item, tree_item->name);
- new_item->error = tree_item->error;
+ talloc_steal(mem_ctx, scripts);
+ talloc_free(tmp_ctx);
+ return scripts;
+}
- if (head == NULL) {
- head = new_item;
- tail = new_item;
- } else {
- tail->next = new_item;
- tail = new_item;
- }
+static int child_setup(struct ctdb_context *ctdb)
+{
+ if (setpgid(0,0) != 0) {
+ int ret = -errno;
+ DEBUG(DEBUG_ERR,("Failed to create process group for event scripts - %s\n",
+ strerror(errno)));
+ return ret;
+ }
- talloc_steal(mem_ctx, new_item);
+ signal(SIGTERM, sigterm);
+ return 0;
+}
- /* remove this script from the tree */
- talloc_free(tree_item);
- }
+static char *child_command_string(struct ctdb_context *ctdb,
+ TALLOC_CTX *ctx,
+ bool from_user,
+ const char *scriptname,
+ enum ctdb_eventscript_call call,
+ const char *options)
+{
+ const char *str = from_user ? "CTDB_CALLED_BY_USER=1 " : "";
- talloc_free(tmp_ctx);
- return head;
+ /* Allow a setting where we run the actual monitor event
+ from an external source and replace it with
+ a "status" event that just picks up the actual
+ status of the event asynchronously.
+ */
+ if ((ctdb->tunable.use_status_events_for_monitoring != 0)
+ && (call == CTDB_EVENT_MONITOR)
+ && !from_user) {
+ return talloc_asprintf(ctx, "%s%s/%s %s",
+ str,
+ ctdb->event_script_dir,
+ scriptname, "status");
+ } else {
+ return talloc_asprintf(ctx, "%s%s/%s %s %s",
+ str,
+ ctdb->event_script_dir,
+ scriptname,
+ ctdb_eventscript_call_names[call],
+ options);
+ }
}
+static int child_run_one(struct ctdb_context *ctdb,
+ const char *scriptname, const char *cmdstr)
+{
+ int ret;
+
+ ret = system(cmdstr);
+ /* if the system() call was successful, translate ret into the
+ return code from the command
+ */
+ if (ret != -1) {
+ ret = WEXITSTATUS(ret);
+ } else {
+ ret = -errno;
+ }
+ /* 127 could mean it does not exist, 126 non-executable. */
+ if (ret == 127 || ret == 126) {
+ /* Re-check it... */
+ if (!check_executable(ctdb->event_script_dir, scriptname)) {
+ DEBUG(DEBUG_ERR,("Script %s returned status %u. Someone just deleted it?\n",
+ cmdstr, ret));
+ ret = -errno;
+ }
+ }
+ return ret;
+}
/*
- Actually run the event script
+ Actually run one event script
this function is called and run in the context of a forked child
which allows it to do blocking calls such as system()
*/
-static int ctdb_run_event_script(struct ctdb_context *ctdb,
- bool from_user,
- enum ctdb_eventscript_call call,
- const char *options)
+static int child_run_script(struct ctdb_context *ctdb,
+ bool from_user,
+ enum ctdb_eventscript_call call,
+ const char *options,
+ struct ctdb_script_wire *current)
{
char *cmdstr;
- int ret = 0;
+ int ret;
TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
- struct ctdb_script_list *scripts, *current;
-
- if (!from_user && call == CTDB_EVENT_MONITOR) {
- /* This is running in the forked child process. At this stage
- * we want to switch from being a ctdb daemon into being a
- * client and connect to the real local daemon.
- */
- if (switch_from_server_to_client(ctdb) != 0) {
- DEBUG(DEBUG_CRIT, (__location__ "ERROR: failed to switch eventscript child into client mode. shutting down.\n"));
- _exit(1);
- }
- if (ctdb_ctrl_event_script_init(ctdb) != 0) {
- ret = -EIO;
- goto out;
- }
- }
+ ret = child_setup(ctdb);
+ if (ret != 0)
+ goto out;
- if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
- /* we guarantee that only some specifically allowed event scripts are run
- while in recovery */
- const enum ctdb_eventscript_call allowed_calls[] = {
- CTDB_EVENT_START_RECOVERY, CTDB_EVENT_SHUTDOWN, CTDB_EVENT_RELEASE_IP, CTDB_EVENT_STOPPED };
- int i;
- for (i=0;i<ARRAY_SIZE(allowed_calls);i++) {
- if (call == allowed_calls[i]) break;
- }
- if (i == ARRAY_SIZE(allowed_calls)) {
- DEBUG(DEBUG_ERR,("Refusing to run event scripts call '%s' while in recovery\n",
- call_names[call]));
- ret = -EBUSY;
- goto out;
- }
- }
+ cmdstr = child_command_string(ctdb, tmp_ctx, from_user,
+ current->name, call, options);
+ CTDB_NO_MEMORY(ctdb, cmdstr);
- if (setpgid(0,0) != 0) {
- ret = -errno;
- DEBUG(DEBUG_ERR,("Failed to create process group for event scripts - %s\n",
- strerror(errno)));
+ DEBUG(DEBUG_DEBUG,("Executing event script %s\n",cmdstr));
+
+ if (current->status) {
+ ret = current->status;
goto out;
}
- signal(SIGTERM, sigterm);
+ ret = child_run_one(ctdb, current->name, cmdstr);
+out:
+ talloc_free(tmp_ctx);
+ return ret;
+}
- child_state.start = timeval_current();
- child_state.script_running = "startup";
+static void ctdb_event_script_handler(struct event_context *ev, struct fd_event *fde,
+ uint16_t flags, void *p);
- scripts = ctdb_get_script_list(ctdb, tmp_ctx);
+static int fork_child_for_script(struct ctdb_context *ctdb,
+ struct ctdb_event_script_state *state)
+{
+ int r;
+ struct tevent_fd *fde;
+ struct ctdb_script_wire *current = get_current_script(state);
- /* fetch the scripts from the tree one by one and execute
- them
- */
- for (current=scripts; current; current=current->next) {
- const char *str = from_user ? "CTDB_CALLED_BY_USER=1 " : "";
-
- /* Allow a setting where we run the actual monitor event
- from an external source and replace it with
- a "status" event that just picks up the actual
- status of the event asynchronously.
- */
- if ((ctdb->tunable.use_status_events_for_monitoring != 0)
- && (call == CTDB_EVENT_MONITOR)
- && !from_user) {
- cmdstr = talloc_asprintf(tmp_ctx, "%s%s/%s %s",
- str,
- ctdb->event_script_dir,
- current->name, "status");
- } else {
- cmdstr = talloc_asprintf(tmp_ctx, "%s%s/%s %s %s",
- str,
- ctdb->event_script_dir,
- current->name, call_names[call], options);
- }
- CTDB_NO_MEMORY(ctdb, cmdstr);
+ current->start = timeval_current();
- DEBUG(DEBUG_INFO,("Executing event script %s\n",cmdstr));
+ r = pipe(state->fd);
+ if (r != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " pipe failed for child eventscript process\n"));
+ return -errno;
+ }
- child_state.start = timeval_current();
- child_state.script_running = cmdstr;
+ if (!ctdb_fork_with_logging(state, ctdb, log_event_script_output,
+ state, &state->child)) {
+ r = -errno;
+ close(state->fd[0]);
+ close(state->fd[1]);
+ return r;
+ }
- if (!from_user && call == CTDB_EVENT_MONITOR) {
- if (ctdb_ctrl_event_script_start(ctdb, current->name) != 0) {
- ret = -EIO;
- goto out;
- }
+ /* If we are the child, do the work. */
+ if (state->child == 0) {
+ int rt;
- if (current->error) {
- if (ctdb_ctrl_event_script_stop(ctdb, -current->error) != 0) {
- ret = -EIO;
- goto out;
- }
- }
- }
+ debug_extra = talloc_asprintf(NULL, "eventscript-%s-%s:",
+ current->name,
+ ctdb_eventscript_call_names[state->call]);
+ close(state->fd[0]);
+ set_close_on_exec(state->fd[1]);
- if (current->error) {
- continue;
- }
+ rt = child_run_script(ctdb, state->from_user, state->call, state->options, current);
+ /* We must be able to write PIPEBUF bytes at least; if this
+ somehow fails, the read above will be short. */
+ write(state->fd[1], &rt, sizeof(rt));
+ close(state->fd[1]);
+ _exit(rt);
+ }
- ret = system(cmdstr);
- /* if the system() call was successful, translate ret into the
- return code from the command
- */
- if (ret != -1) {
- ret = WEXITSTATUS(ret);
- } else {
- ret = -errno;
- }
+ close(state->fd[1]);
+ set_close_on_exec(state->fd[0]);
- /* 127 could mean it does not exist, 126 non-executable. */
- if (ret == 127 || ret == 126) {
- /* Re-check it... */
- if (!check_executable(ctdb->event_script_dir,
- current->name)) {
- DEBUG(DEBUG_ERR,("Script %s returned status %u. Someone just deleted it?\n",
- cmdstr, ret));
- ret = -errno;
- }
- }
-
- if (!from_user && call == CTDB_EVENT_MONITOR) {
- if (ctdb_ctrl_event_script_stop(ctdb, ret) != 0) {
- ret = -EIO;
- goto out;
- }
- }
+ DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to child eventscript process\n", state->fd[0]));
- /* now we've reported the per-script error, don't exit the loop
- * just because it vanished or was disabled. */
- if (ret == -ENOENT || ret == -ENOEXEC) {
- ret = 0;
- }
+ /* Set ourselves up to be called when that's done. */
+ fde = event_add_fd(ctdb->ev, state, state->fd[0], EVENT_FD_READ,
+ ctdb_event_script_handler, state);
+ tevent_fd_set_auto_close(fde);
- /* return an error if the script failed */
- if (ret != 0) {
- break;
- }
- }
+ return 0;
+}
- child_state.start = timeval_current();
- child_state.script_running = "finished";
-
- if (!from_user && call == CTDB_EVENT_MONITOR) {
- if (ctdb_ctrl_event_script_finished(ctdb) != 0) {
- ret = -EIO;
+/*
+ Summarize status of this run of scripts.
+ */
+static int script_status(struct ctdb_scripts_wire *scripts)
+{
+ unsigned int i;
+
+ for (i = 0; i < scripts->num_scripts; i++) {
+ switch (scripts->scripts[i].status) {
+ case -ENOENT:
+ case -ENOEXEC:
+ /* Disabled or missing; that's OK. */
+ break;
+ case 0:
+ /* No problem. */
+ break;
+ default:
+ return scripts->scripts[i].status;
}
}
-out:
- talloc_free(tmp_ctx);
- return ret;
+ /* All OK! */
+ return 0;
}
/* called when child is finished */
{
struct ctdb_event_script_state *state =
talloc_get_type(p, struct ctdb_event_script_state);
+ struct ctdb_script_wire *current = get_current_script(state);
struct ctdb_context *ctdb = state->ctdb;
- int r;
+ int r, status;
- r = read(state->fd[0], &state->cb_status, sizeof(state->cb_status));
+ r = read(state->fd[0], ¤t->status, sizeof(current->status));
if (r < 0) {
- state->cb_status = -errno;
- } else if (r != sizeof(state->cb_status)) {
- state->cb_status = -EIO;
+ current->status = -errno;
+ } else if (r != sizeof(current->status)) {
+ current->status = -EIO;
}
- DEBUG(DEBUG_INFO,(__location__ " Eventscript %s %s finished with state %d\n",
- call_names[state->call], state->options, state->cb_status));
+ current->finished = timeval_current();
+ /* valgrind gets overloaded if we run next script as it's still doing
+ * post-execution analysis, so kill finished child here. */
+ if (ctdb->valgrinding) {
+ kill(state->child, SIGKILL);
+ }
state->child = 0;
- ctdb->event_script_timeouts = 0;
- talloc_free(state);
+
+ status = script_status(state->scripts);
+
+ /* Aborted or finished all scripts? We're done. */
+ if (status != 0 || state->current+1 == state->scripts->num_scripts) {
+ DEBUG(DEBUG_INFO,(__location__ " Eventscript %s %s finished with state %d\n",
+ ctdb_eventscript_call_names[state->call], state->options, status));
+
+ ctdb->event_script_timeouts = 0;
+ talloc_free(state);
+ return;
+ }
+
+ /* Forget about that old fd. */
+ talloc_free(fde);
+
+ /* Next script! */
+ state->current++;
+ current++;
+ current->status = fork_child_for_script(ctdb, state);
+ if (current->status != 0) {
+ /* This calls the callback. */
+ talloc_free(state);
+ }
}
-static void ctdb_ban_self(struct ctdb_context *ctdb, uint32_t ban_period)
+static void debug_timeout(struct ctdb_event_script_state *state)
{
- TDB_DATA data;
- struct ctdb_ban_time bantime;
+ struct ctdb_script_wire *current = get_current_script(state);
+ char *cmd;
+ pid_t pid;
+ time_t t;
+ char tbuf[100], buf[200];
- bantime.pnn = ctdb->pnn;
- bantime.time = ban_period;
+ cmd = child_command_string(state->ctdb, state,
+ state->from_user, current->name,
+ state->call, state->options);
+ CTDB_NO_MEMORY_VOID(state->ctdb, cmd);
- data.dsize = sizeof(bantime);
- data.dptr = (uint8_t *)&bantime;
+ DEBUG(DEBUG_ERR,("Timed out running script '%s' after %.1f seconds pid :%d\n",
+ cmd, timeval_elapsed(¤t->start), state->child));
+ talloc_free(cmd);
- ctdb_control_set_ban_state(ctdb, data);
+ t = time(NULL);
+ strftime(tbuf, sizeof(tbuf)-1, "%Y%m%d%H%M%S", localtime(&t));
+ sprintf(buf, "{ pstree -p; cat /proc/locks; ls -li /var/ctdb/ /var/ctdb/persistent; }"
+ " >/tmp/ctdb.event.%s.%d", tbuf, getpid());
+
+ pid = ctdb_fork(state->ctdb);
+ if (pid == 0) {
+ system(buf);
+ /* Now we can kill the child */
+ kill(state->child, SIGTERM);
+ exit(0);
+ }
+ if (pid == -1) {
+ DEBUG(DEBUG_ERR,("Fork for debug script failed : %s\n",
+ strerror(errno)));
+ } else {
+ DEBUG(DEBUG_ERR,("Logged timedout eventscript : %s\n", buf));
+ /* Don't kill child until timeout done. */
+ state->child = 0;
+ }
}
-
/* called when child times out */
static void ctdb_event_script_timeout(struct event_context *ev, struct timed_event *te,
struct timeval t, void *p)
{
struct ctdb_event_script_state *state = talloc_get_type(p, struct ctdb_event_script_state);
struct ctdb_context *ctdb = state->ctdb;
+ struct ctdb_script_wire *current = get_current_script(state);
- DEBUG(DEBUG_ERR,("Event script timed out : %s %s count : %u pid : %d\n",
- call_names[state->call], state->options, ctdb->event_script_timeouts, state->child));
-
- if (kill(state->child, 0) != 0) {
- DEBUG(DEBUG_ERR,("Event script child process already dead, errno %s(%d)\n", strerror(errno), errno));
- state->child = 0;
- state->cb_status = -ETIME;
- talloc_free(state);
- return;
- }
-
- if (state->call == CTDB_EVENT_MONITOR) {
- /* if it is a monitor event, we allow it to "hang" a few times
- before we declare it a failure and ban ourself (and make
- ourself unhealthy)
- */
- DEBUG(DEBUG_ERR, (__location__ " eventscript for monitor event timedout.\n"));
-
- ctdb->event_script_timeouts++;
-
- if (ctdb->event_script_timeouts > ctdb->tunable.script_ban_count) {
- DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Making node unhealthy\n", ctdb->tunable.script_ban_count));
- state->cb_status = -ETIME;
- } else {
- state->cb_status = 0;
- }
- } else if (state->call == CTDB_EVENT_STARTUP) {
- DEBUG(DEBUG_ERR, (__location__ " eventscript for startup event timedout.\n"));
- state->cb_status = -ETIME;
- } else {
- /* if it is not a monitor or a startup event we ban ourself
- immediately
- */
- DEBUG(DEBUG_ERR, (__location__ " eventscript for NON-monitor/NON-startup event timedout. Immediately banning ourself for %d seconds\n", ctdb->tunable.recovery_ban_period));
-
- ctdb_ban_self(ctdb, ctdb->tunable.recovery_ban_period);
-
- state->cb_status = -ETIME;
- }
-
- if (state->call == CTDB_EVENT_MONITOR || state->call == CTDB_EVENT_STATUS) {
- struct ctdb_monitor_script_status *script;
-
- if (ctdb->current_monitor_status_ctx == NULL) {
- talloc_free(state);
- return;
- }
-
- script = ctdb->current_monitor_status_ctx->scripts;
- if (script != NULL) {
- script->status = state->cb_status;
- }
+ DEBUG(DEBUG_ERR,("Event script timed out : %s %s %s count : %u pid : %d\n",
+ current->name, ctdb_eventscript_call_names[state->call], state->options, ctdb->event_script_timeouts, state->child));
- ctdb_control_event_script_finished(ctdb);
+ /* ignore timeouts for these events */
+ switch (state->call) {
+ case CTDB_EVENT_START_RECOVERY:
+ case CTDB_EVENT_RECOVERED:
+ case CTDB_EVENT_TAKE_IP:
+ case CTDB_EVENT_RELEASE_IP:
+ case CTDB_EVENT_STOPPED:
+ case CTDB_EVENT_MONITOR:
+ case CTDB_EVENT_STATUS:
+ state->scripts->scripts[state->current].status = 0;
+ DEBUG(DEBUG_ERR,("Ignoring hung script for %s call %d\n", state->options, state->call));
+ break;
+ default:
+ state->scripts->scripts[state->current].status = -ETIME;
+ debug_timeout(state);
}
talloc_free(state);
*/
static int event_script_destructor(struct ctdb_event_script_state *state)
{
+ int status;
+
if (state->child) {
DEBUG(DEBUG_ERR,(__location__ " Sending SIGTERM to child pid:%d\n", state->child));
}
}
+ /* If we were the current monitor, we no longer are. */
+ if (state->ctdb->current_monitor == state) {
+ state->ctdb->current_monitor = NULL;
+ }
+
+ /* Save our scripts as the last executed status, if we have them.
+ * See ctdb_event_script_callback_v where we abort monitor event. */
+ if (state->scripts) {
+ talloc_free(state->ctdb->last_status[state->call]);
+ state->ctdb->last_status[state->call] = state->scripts;
+ if (state->current < state->ctdb->last_status[state->call]->num_scripts) {
+ state->ctdb->last_status[state->call]->num_scripts = state->current+1;
+ }
+ }
+
+ /* Use last status as result, or "OK" if none. */
+ if (state->ctdb->last_status[state->call]) {
+ status = script_status(state->ctdb->last_status[state->call]);
+ } else {
+ status = 0;
+ }
+
/* This is allowed to free us; talloc will prevent double free anyway,
* but beware if you call this outside the destructor! */
if (state->callback) {
- state->callback(state->ctdb, state->cb_status, state->private_data);
+ state->callback(state->ctdb, status, state->private_data);
}
return 0;
{
switch (call) {
/* These all take no arguments. */
+ case CTDB_EVENT_INIT:
+ case CTDB_EVENT_SETUP:
case CTDB_EVENT_STARTUP:
case CTDB_EVENT_START_RECOVERY:
case CTDB_EVENT_RECOVERED:
case CTDB_EVENT_STATUS:
case CTDB_EVENT_SHUTDOWN:
case CTDB_EVENT_RELOAD:
+ case CTDB_EVENT_IPREALLOCATED:
return count_words(options) == 0;
case CTDB_EVENT_TAKE_IP: /* interface, IP address, netmask bits. */
case CTDB_EVENT_RELEASE_IP:
return count_words(options) == 3;
+ case CTDB_EVENT_UPDATE_IP: /* old interface, new interface, IP address, netmask bits. */
+ return count_words(options) == 4;
+
default:
DEBUG(DEBUG_ERR,(__location__ "Unknown ctdb_eventscript_call %u\n", call));
return false;
enum ctdb_eventscript_call call,
const char *fmt, va_list ap)
{
- TALLOC_CTX *mem_ctx;
struct ctdb_event_script_state *state;
- int ret;
-
- if (!from_user && (call == CTDB_EVENT_MONITOR || call == CTDB_EVENT_STATUS)) {
- /* if this was a "monitor" or a status event, we recycle the
- context to start a new monitor event
- */
- if (ctdb->monitor_event_script_ctx != NULL) {
- talloc_free(ctdb->monitor_event_script_ctx);
- ctdb->monitor_event_script_ctx = NULL;
- }
- ctdb->monitor_event_script_ctx = talloc_new(ctdb);
- mem_ctx = ctdb->monitor_event_script_ctx;
-
- if (ctdb->current_monitor_status_ctx != NULL) {
- talloc_free(ctdb->current_monitor_status_ctx);
- ctdb->current_monitor_status_ctx = NULL;
- }
-
- ctdb->current_monitor_status_ctx = talloc(ctdb, struct ctdb_monitor_script_status_ctx);
- CTDB_NO_MEMORY(ctdb, ctdb->current_monitor_status_ctx);
- ctdb->current_monitor_status_ctx->scripts = NULL;
- } else {
- /* any other script will first terminate any monitor event */
- if (ctdb->monitor_event_script_ctx != NULL) {
- talloc_free(ctdb->monitor_event_script_ctx);
- ctdb->monitor_event_script_ctx = NULL;
- }
- /* and then use a context common for all non-monitor events */
- if (ctdb->other_event_script_ctx == NULL) {
- ctdb->other_event_script_ctx = talloc_new(ctdb);
- }
- mem_ctx = ctdb->other_event_script_ctx;
- }
- state = talloc(mem_ctx, struct ctdb_event_script_state);
+ state = talloc(ctdb->event_script_ctx, struct ctdb_event_script_state);
CTDB_NO_MEMORY(ctdb, state);
state->ctdb = ctdb;
state->callback = callback;
state->private_data = private_data;
+ state->from_user = from_user;
state->call = call;
state->options = talloc_vasprintf(state, fmt, ap);
state->timeout = timeval_set(ctdb->tunable.script_timeout, 0);
+ state->scripts = NULL;
if (state->options == NULL) {
DEBUG(DEBUG_ERR, (__location__ " could not allocate state->options\n"));
talloc_free(state);
}
if (!check_options(state->call, state->options)) {
DEBUG(DEBUG_ERR, ("Bad eventscript options '%s' for %s\n",
- call_names[state->call], state->options));
+ ctdb_eventscript_call_names[state->call], state->options));
talloc_free(state);
return -1;
}
- DEBUG(DEBUG_INFO,(__location__ " Starting eventscript %s %s\n",
- call_names[state->call], state->options));
-
- ret = pipe(state->fd);
- if (ret != 0) {
- talloc_free(state);
- return -1;
+ if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
+ /* we guarantee that only some specifically allowed event scripts are run
+ while in recovery */
+ const enum ctdb_eventscript_call allowed_calls[] = {
+ CTDB_EVENT_INIT,
+ CTDB_EVENT_SETUP,
+ CTDB_EVENT_START_RECOVERY,
+ CTDB_EVENT_SHUTDOWN,
+ CTDB_EVENT_RELEASE_IP,
+ CTDB_EVENT_STOPPED
+ };
+ int i;
+ for (i=0;i<ARRAY_SIZE(allowed_calls);i++) {
+ if (call == allowed_calls[i]) break;
+ }
+ if (i == ARRAY_SIZE(allowed_calls)) {
+ DEBUG(DEBUG_ERR,("Refusing to run event scripts call '%s' while in recovery\n",
+ ctdb_eventscript_call_names[call]));
+ talloc_free(state);
+ return -1;
+ }
}
- state->child = fork();
+ /* Kill off any running monitor events to run this event. */
+ if (ctdb->current_monitor) {
+ /* Discard script status so we don't save to last_status */
+ talloc_free(ctdb->current_monitor->scripts);
+ ctdb->current_monitor->scripts = NULL;
+ talloc_free(ctdb->current_monitor);
+ ctdb->current_monitor = NULL;
+ }
- if (state->child == (pid_t)-1) {
- close(state->fd[0]);
- close(state->fd[1]);
+ DEBUG(DEBUG_INFO,(__location__ " Starting eventscript %s %s\n",
+ ctdb_eventscript_call_names[state->call],
+ state->options));
+
+ /* This is not a child of state, since we save it in destructor. */
+ state->scripts = ctdb_get_script_list(ctdb, ctdb);
+ if (state->scripts == NULL) {
talloc_free(state);
return -1;
}
+ state->current = 0;
+ state->child = 0;
- if (state->child == 0) {
- int rt;
-
- close(state->fd[0]);
- set_close_on_exec(state->fd[1]);
-
- rt = ctdb_run_event_script(ctdb, from_user, state->call, state->options);
- /* We must be able to write PIPEBUF bytes at least; if this
- somehow fails, the read above will be short. */
- write(state->fd[1], &rt, sizeof(rt));
- close(state->fd[1]);
- _exit(rt);
+ if (!from_user && (call == CTDB_EVENT_MONITOR || call == CTDB_EVENT_STATUS)) {
+ ctdb->current_monitor = state;
}
- close(state->fd[1]);
- set_close_on_exec(state->fd[0]);
talloc_set_destructor(state, event_script_destructor);
- DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to child eventscript process\n", state->fd[0]));
+ /* Nothing to do? */
+ if (state->scripts->num_scripts == 0) {
+ talloc_free(state);
+ return 0;
+ }
- event_add_fd(ctdb->ev, state, state->fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
- ctdb_event_script_handler, state);
+ state->scripts->scripts[0].status = fork_child_for_script(ctdb, state);
+ if (state->scripts->scripts[0].status != 0) {
+ /* Callback is called from destructor, with fail result. */
+ talloc_free(state);
+ return 0;
+ }
if (!timeval_is_zero(&state->timeout)) {
event_add_timed(ctdb->ev, state, timeval_current_ofs(state->timeout.tv_sec, state->timeout.tv_usec), ctdb_event_script_timeout, state);
} else {
DEBUG(DEBUG_ERR, (__location__ " eventscript %s %s called with no timeout\n",
- call_names[state->call], state->options));
+ ctdb_eventscript_call_names[state->call],
+ state->options));
}
return 0;
while (status.done == false && event_loop_once(ctdb->ev) == 0) /* noop */;
+ if (status.status == -ETIME) {
+ DEBUG(DEBUG_ERR, (__location__ " eventscript for '%s' timedout."
+ " Immediately banning ourself for %d seconds\n",
+ ctdb_eventscript_call_names[call],
+ ctdb->tunable.recovery_ban_period));
+ ctdb_ban_self(ctdb);
+ }
+
return status.status;
}
p += strspn(p, " \t");
/* See if we match any. */
- for (*call = 0; *call < ARRAY_SIZE(call_names); (*call)++) {
- len = strlen(call_names[*call]);
- if (strncmp(p, call_names[*call], len) == 0) {
+ for (*call = 0; *call < CTDB_EVENT_MAX; (*call)++) {
+ len = strlen(ctdb_eventscript_call_names[*call]);
+ if (strncmp(p, ctdb_eventscript_call_names[*call], len) == 0) {
/* If end of string or whitespace, we're done. */
if (strcspn(p + len, " \t") == 0) {
return p + len;
return -1;
}
- state = talloc(ctdb->other_event_script_ctx, struct eventscript_callback_state);
+ state = talloc(ctdb->event_script_ctx, struct eventscript_callback_state);
CTDB_NO_MEMORY(ctdb, state);
state->c = talloc_steal(state, c);