*/
#include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
#include "system/filesys.h"
#include "system/wait.h"
#include "../include/ctdb_private.h"
+struct ctdb_monitor_state {
+ uint32_t monitoring_mode;
+ TALLOC_CTX *monitor_context;
+ uint32_t next_interval;
+};
+
+static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data);
+
/*
- see if any nodes are dead
- */
-static void ctdb_check_for_dead_nodes(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *private_data)
+ setup the notification script
+*/
+int ctdb_set_notification_script(struct ctdb_context *ctdb, const char *script)
{
- struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
- int i;
+ ctdb->notification_script = talloc_strdup(ctdb, script);
+ CTDB_NO_MEMORY(ctdb, ctdb->notification_script);
+ return 0;
+}
- if (ctdb->monitoring_mode == CTDB_MONITORING_DISABLED) {
- event_add_timed(ctdb->ev, ctdb->monitor_context,
- timeval_current_ofs(ctdb->tunable.keepalive_interval, 0),
- ctdb_check_for_dead_nodes, ctdb);
- return;
+static int ctdb_run_notification_script_child(struct ctdb_context *ctdb, const char *event)
+{
+ struct stat st;
+ int ret;
+ char *cmd;
+
+ if (stat(ctdb->notification_script, &st) != 0) {
+ DEBUG(DEBUG_ERR,("Could not stat notification script %s. Can not send notifications.\n", ctdb->notification_script));
+ return -1;
+ }
+ if (!(st.st_mode & S_IXUSR)) {
+ DEBUG(DEBUG_ERR,("Notification script %s is not executable.\n", ctdb->notification_script));
+ return -1;
}
- /* send a keepalive to all other nodes, unless */
- for (i=0;i<ctdb->num_nodes;i++) {
- struct ctdb_node *node = ctdb->nodes[i];
- if (node->pnn == ctdb->pnn) {
- continue;
- }
-
- if (node->flags & NODE_FLAGS_DISCONNECTED) {
- /* it might have come alive again */
- if (node->rx_cnt != 0) {
- ctdb_node_connected(node);
- }
- continue;
- }
+ cmd = talloc_asprintf(ctdb, "%s %s\n", ctdb->notification_script, event);
+ CTDB_NO_MEMORY(ctdb, cmd);
+
+ ret = system(cmd);
+ /* if the system() call was successful, translate ret into the
+ return code from the command
+ */
+ if (ret != -1) {
+ ret = WEXITSTATUS(ret);
+ }
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Notification script \"%s\" failed with error %d\n", cmd, ret));
+ }
+ return ret;
+}
- if (node->rx_cnt == 0) {
- node->dead_count++;
- } else {
- node->dead_count = 0;
- }
+void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *event)
+{
+ pid_t child;
- node->rx_cnt = 0;
+ if (ctdb->notification_script == NULL) {
+ return;
+ }
- if (node->dead_count >= ctdb->tunable.keepalive_limit) {
- DEBUG(0,("dead count reached for node %u\n", node->pnn));
- ctdb_node_dead(node);
- ctdb_send_keepalive(ctdb, node->pnn);
- /* maybe tell the transport layer to kill the
- sockets as well?
- */
- continue;
- }
-
- if (node->tx_cnt == 0) {
- DEBUG(5,("sending keepalive to %u\n", node->pnn));
- ctdb_send_keepalive(ctdb, node->pnn);
- }
+ child = fork();
+ if (child == (pid_t)-1) {
+ DEBUG(DEBUG_ERR,("Failed to fork() a notification child process\n"));
+ return;
+ }
+ if (child == 0) {
+ int ret;
- node->tx_cnt = 0;
+ debug_extra = talloc_asprintf(NULL, "notification-%s:", event);
+ ret = ctdb_run_notification_script_child(ctdb, event);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Notification script failed\n"));
+ }
+ _exit(0);
}
-
- event_add_timed(ctdb->ev, ctdb->monitor_context,
- timeval_current_ofs(ctdb->tunable.keepalive_interval, 0),
- ctdb_check_for_dead_nodes, ctdb);
-}
-static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *private_data);
+ return;
+}
/*
called when a health monitoring event script finishes
TDB_DATA data;
struct ctdb_node_flag_change c;
uint32_t next_interval;
+ int ret;
+ TDB_DATA rddata;
+ struct takeover_run_reply rd;
+ const char *state_str = NULL;
c.pnn = ctdb->pnn;
c.old_flags = node->flags;
+ rd.pnn = ctdb->pnn;
+ rd.srvid = CTDB_SRVID_TAKEOVER_RUN_RESPONSE;
+
+ rddata.dptr = (uint8_t *)&rd;
+ rddata.dsize = sizeof(rd);
+
+ if (status == -ETIME) {
+ ctdb->event_script_timeouts++;
+
+ if (ctdb->event_script_timeouts >= ctdb->tunable.script_timeout_count) {
+ DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Making node unhealthy\n", ctdb->tunable.script_timeout_count));
+ } else {
+ /* We pretend this is OK. */
+ goto after_change_status;
+ }
+ }
+
if (status != 0 && !(node->flags & NODE_FLAGS_UNHEALTHY)) {
- DEBUG(0,("monitor event failed - disabling node\n"));
+ DEBUG(DEBUG_NOTICE,("monitor event failed - disabling node\n"));
node->flags |= NODE_FLAGS_UNHEALTHY;
+ ctdb->monitor->next_interval = 5;
+
+ ctdb_run_notification_script(ctdb, "unhealthy");
} else if (status == 0 && (node->flags & NODE_FLAGS_UNHEALTHY)) {
- DEBUG(0,("monitor event OK - node re-enabled\n"));
+ DEBUG(DEBUG_NOTICE,("monitor event OK - node re-enabled\n"));
node->flags &= ~NODE_FLAGS_UNHEALTHY;
+ ctdb->monitor->next_interval = 5;
+
+ ctdb_run_notification_script(ctdb, "healthy");
}
- if (node->flags & NODE_FLAGS_UNHEALTHY) {
- next_interval = ctdb->tunable.monitor_retry;
- } else {
- next_interval = ctdb->tunable.monitor_interval;
+after_change_status:
+ next_interval = ctdb->monitor->next_interval;
+
+ ctdb->monitor->next_interval *= 2;
+ if (ctdb->monitor->next_interval > ctdb->tunable.monitor_interval) {
+ ctdb->monitor->next_interval = ctdb->tunable.monitor_interval;
}
- event_add_timed(ctdb->ev, ctdb->monitor_context,
- timeval_current_ofs(next_interval, 0),
- ctdb_check_health, ctdb);
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(next_interval, 0),
+ ctdb_check_health, ctdb);
if (c.old_flags == node->flags) {
return;
data.dptr = (uint8_t *)&c;
data.dsize = sizeof(c);
- /* tell the other nodes that something has changed */
- ctdb_daemon_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
- CTDB_SRVID_NODE_FLAGS_CHANGED, data);
+ /* ask the recovery daemon to push these changes out to all nodes */
+ ctdb_daemon_send_message(ctdb, ctdb->pnn,
+ CTDB_SRVID_PUSH_NODE_FLAGS, data);
+
+ if (c.new_flags & NODE_FLAGS_UNHEALTHY) {
+ state_str = "UNHEALTHY";
+ } else {
+ state_str = "HEALTHY";
+ }
+
+ /* ask the recmaster to reallocate all addresses */
+ DEBUG(DEBUG_ERR,("Node became %s. Ask recovery master %u to perform ip reallocation\n",
+ state_str, ctdb->recovery_master));
+ ret = ctdb_daemon_send_message(ctdb, ctdb->recovery_master, CTDB_SRVID_TAKEOVER_RUN, rddata);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to send ip takeover run request message to %u\n", ctdb->recovery_master));
+ }
+}
+
+
+/*
+ called when the startup event script finishes
+ */
+static void ctdb_startup_callback(struct ctdb_context *ctdb, int status, void *p)
+{
+ if (status != 0) {
+ DEBUG(DEBUG_ERR,("startup event failed\n"));
+ } else if (status == 0) {
+ DEBUG(DEBUG_NOTICE,("startup event OK - enabling monitoring\n"));
+ ctdb->done_startup = true;
+ ctdb->monitor->next_interval = 2;
+ ctdb_run_notification_script(ctdb, "startup");
+ }
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(ctdb->monitor->next_interval, 0),
+ ctdb_check_health, ctdb);
+}
+
+
+/*
+ wait until we have finished initial recoveries before we start the
+ monitoring events
+ */
+static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+ int ret;
+
+ DEBUG(DEBUG_NOTICE,("CTDB_WAIT_UNTIL_RECOVERED\n"));
+
+ if (ctdb->vnn_map->generation == INVALID_GENERATION) {
+ ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
+ DEBUG(DEBUG_NOTICE,(__location__ " generation is INVALID. Wait one more second\n"));
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+ return;
+ }
+
+ if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
+ ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
+ DEBUG(DEBUG_NOTICE,(__location__ " in recovery. Wait one more second\n"));
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+ return;
+ }
+
+
+ if (!fast_start && timeval_elapsed(&ctdb->last_recovery_finished) < (ctdb->tunable.rerecovery_timeout + 3)) {
+ ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
+ DEBUG(DEBUG_NOTICE,(__location__ " wait for pending recoveries to end. Wait one more second.\n"));
+
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+ return;
+ }
+
+ if (ctdb->vnn_map->generation == ctdb->db_persistent_startup_generation) {
+ DEBUG(DEBUG_INFO,(__location__ " skip ctdb_recheck_persistent_health() "
+ "until the next recovery\n"));
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+ return;
+ }
+
+ ctdb->db_persistent_startup_generation = ctdb->vnn_map->generation;
+ ret = ctdb_recheck_persistent_health(ctdb);
+ if (ret != 0) {
+ ctdb->db_persistent_check_errors++;
+ if (ctdb->db_persistent_check_errors < ctdb->max_persistent_check_errors) {
+ DEBUG(ctdb->db_persistent_check_errors==1?DEBUG_ERR:DEBUG_WARNING,
+ (__location__ "ctdb_recheck_persistent_health() "
+ "failed (%llu of %llu times) - retry later\n",
+ (unsigned long long)ctdb->db_persistent_check_errors,
+ (unsigned long long)ctdb->max_persistent_check_errors));
+ event_add_timed(ctdb->ev,
+ ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+ return;
+ }
+ DEBUG(DEBUG_ALERT,(__location__
+ "ctdb_recheck_persistent_health() failed (%llu times) - prepare shutdown\n",
+ (unsigned long long)ctdb->db_persistent_check_errors));
+ ctdb_stop_recoverd(ctdb);
+ ctdb_stop_keepalive(ctdb);
+ ctdb_stop_monitoring(ctdb);
+ ctdb_release_all_ips(ctdb);
+ if (ctdb->methods != NULL) {
+ ctdb->methods->shutdown(ctdb);
+ }
+ ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
+ DEBUG(DEBUG_ALERT,("ctdb_recheck_persistent_health() failed - Stopping CTDB daemon\n"));
+ exit(11);
+ }
+ ctdb->db_persistent_check_errors = 0;
+ DEBUG(DEBUG_NOTICE,(__location__
+ "ctdb_start_monitoring: ctdb_recheck_persistent_health() OK\n"));
+
+ DEBUG(DEBUG_NOTICE,(__location__ " Recoveries finished. Running the \"startup\" event.\n"));
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current(),
+ ctdb_check_health, ctdb);
}
struct timeval t, void *private_data)
{
struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
- int ret;
+ int ret = 0;
- if (ctdb->monitoring_mode == CTDB_MONITORING_DISABLED) {
- event_add_timed(ctdb->ev, ctdb->monitor_context,
- timeval_current_ofs(ctdb->tunable.monitor_interval, 0),
+ if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL ||
+ (ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED && ctdb->done_startup)) {
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(ctdb->monitor->next_interval, 0),
ctdb_check_health, ctdb);
return;
}
- ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
- ctdb->monitor_context, ctdb_health_callback, ctdb, "monitor");
+ if (!ctdb->done_startup) {
+ ret = ctdb_event_script_callback(ctdb,
+ ctdb->monitor->monitor_context, ctdb_startup_callback,
+ ctdb, false,
+ CTDB_EVENT_STARTUP, "%s", "");
+ } else {
+ int i;
+ int skip_monitoring = 0;
+
+ if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
+ skip_monitoring = 1;
+ DEBUG(DEBUG_ERR,("Skip monitoring during recovery\n"));
+ }
+ for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+ if (ctdb->freeze_handles[i] != NULL) {
+ DEBUG(DEBUG_ERR,("Skip monitoring since databases are frozen\n"));
+ skip_monitoring = 1;
+ break;
+ }
+ }
+ if (skip_monitoring != 0) {
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(ctdb->monitor->next_interval, 0),
+ ctdb_check_health, ctdb);
+ return;
+ } else {
+ ret = ctdb_event_script_callback(ctdb,
+ ctdb->monitor->monitor_context, ctdb_health_callback,
+ ctdb, false,
+ CTDB_EVENT_MONITOR, "%s", "");
+ }
+ }
+
if (ret != 0) {
- DEBUG(0,("Unable to launch monitor event script\n"));
- event_add_timed(ctdb->ev, ctdb->monitor_context,
- timeval_current_ofs(ctdb->tunable.monitor_retry, 0),
- ctdb_check_health, ctdb);
- }
+ DEBUG(DEBUG_ERR,("Unable to launch monitor event script\n"));
+ ctdb->monitor->next_interval = 5;
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(5, 0),
+ ctdb_check_health, ctdb);
+ }
}
-/* stop any monitoring */
+/*
+ (Temporaily) Disabling monitoring will stop the monitor event scripts
+ from running but node health checks will still occur
+*/
+void ctdb_disable_monitoring(struct ctdb_context *ctdb)
+{
+ ctdb->monitor->monitoring_mode = CTDB_MONITORING_DISABLED;
+ DEBUG(DEBUG_INFO,("Monitoring has been disabled\n"));
+}
+
+/*
+ Re-enable running monitor events after they have been disabled
+ */
+void ctdb_enable_monitoring(struct ctdb_context *ctdb)
+{
+ ctdb->monitor->monitoring_mode = CTDB_MONITORING_ACTIVE;
+ ctdb->monitor->next_interval = 5;
+ DEBUG(DEBUG_INFO,("Monitoring has been enabled\n"));
+}
+
+/* stop any monitoring
+ this should only be done when shutting down the daemon
+*/
void ctdb_stop_monitoring(struct ctdb_context *ctdb)
{
- talloc_free(ctdb->monitor_context);
- ctdb->monitor_context = talloc_new(ctdb);
- CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor_context);
+ talloc_free(ctdb->monitor->monitor_context);
+ ctdb->monitor->monitor_context = NULL;
+
+ ctdb->monitor->monitoring_mode = CTDB_MONITORING_DISABLED;
+ ctdb->monitor->next_interval = 5;
+ DEBUG(DEBUG_NOTICE,("Monitoring has been stopped\n"));
}
/*
*/
void ctdb_start_monitoring(struct ctdb_context *ctdb)
{
- struct timed_event *te;
+ if (ctdb->monitor != NULL) {
+ return;
+ }
- ctdb_stop_monitoring(ctdb);
+ ctdb->monitor = talloc(ctdb, struct ctdb_monitor_state);
+ CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor);
- te = event_add_timed(ctdb->ev, ctdb->monitor_context,
- timeval_current_ofs(ctdb->tunable.keepalive_interval, 0),
- ctdb_check_for_dead_nodes, ctdb);
- CTDB_NO_MEMORY_FATAL(ctdb, te);
+ ctdb->monitor->next_interval = 5;
- te = event_add_timed(ctdb->ev, ctdb->monitor_context,
- timeval_current_ofs(ctdb->tunable.monitor_retry, 0),
- ctdb_check_health, ctdb);
- CTDB_NO_MEMORY_FATAL(ctdb, te);
+ ctdb->monitor->monitor_context = talloc_new(ctdb->monitor);
+ CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor->monitor_context);
+
+ event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+
+ ctdb->monitor->monitoring_mode = CTDB_MONITORING_ACTIVE;
+ DEBUG(DEBUG_NOTICE,("Monitoring has been started\n"));
}
*/
int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
{
- struct ctdb_node_modflags *m = (struct ctdb_node_modflags *)indata.dptr;
- TDB_DATA data;
- struct ctdb_node_flag_change c;
- struct ctdb_node *node = ctdb->nodes[ctdb->pnn];
- uint32_t old_flags = node->flags;
+ struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)indata.dptr;
+ struct ctdb_node *node;
+ uint32_t old_flags;
- node->flags |= m->set;
- node->flags &= ~m->clear;
+ if (c->pnn >= ctdb->num_nodes) {
+ DEBUG(DEBUG_ERR,(__location__ " Node %d is invalid, num_nodes :%d\n", c->pnn, ctdb->num_nodes));
+ return -1;
+ }
- if (node->flags == old_flags) {
- /* no change */
- return 0;
+ node = ctdb->nodes[c->pnn];
+ old_flags = node->flags;
+ if (c->pnn != ctdb->pnn) {
+ c->old_flags = node->flags;
+ }
+ node->flags = c->new_flags & ~NODE_FLAGS_DISCONNECTED;
+ node->flags |= (c->old_flags & NODE_FLAGS_DISCONNECTED);
+
+ /* we dont let other nodes modify our STOPPED status */
+ if (c->pnn == ctdb->pnn) {
+ node->flags &= ~NODE_FLAGS_STOPPED;
+ if (old_flags & NODE_FLAGS_STOPPED) {
+ node->flags |= NODE_FLAGS_STOPPED;
+ }
}
- DEBUG(0, ("Control modflags on node %u - flags now 0x%x\n", ctdb->pnn, node->flags));
+ /* we dont let other nodes modify our BANNED status */
+ if (c->pnn == ctdb->pnn) {
+ node->flags &= ~NODE_FLAGS_BANNED;
+ if (old_flags & NODE_FLAGS_BANNED) {
+ node->flags |= NODE_FLAGS_BANNED;
+ }
+ }
- /* if we have been banned, go into recovery mode */
- c.pnn = ctdb->pnn;
- c.old_flags = old_flags;
- c.new_flags = node->flags;
+ if (node->flags == c->old_flags) {
+ DEBUG(DEBUG_INFO, ("Control modflags on node %u - Unchanged - flags 0x%x\n", c->pnn, node->flags));
+ return 0;
+ }
- data.dptr = (uint8_t *)&c;
- data.dsize = sizeof(c);
+ DEBUG(DEBUG_INFO, ("Control modflags on node %u - flags now 0x%x\n", c->pnn, node->flags));
- /* tell the other nodes that something has changed */
- ctdb_daemon_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
- CTDB_SRVID_NODE_FLAGS_CHANGED, data);
-
- if ((node->flags & NODE_FLAGS_BANNED) && !(old_flags & NODE_FLAGS_BANNED)) {
- /* make sure we are frozen */
- DEBUG(0,("This node has been banned - forcing freeze and recovery\n"));
- /* Reset the generation id to 1 to make us ignore any
- REQ/REPLY CALL/DMASTER someone sends to us.
- We are now banned so we shouldnt service database calls
- anymore.
- */
- ctdb->vnn_map->generation = INVALID_GENERATION;
-
- ctdb_start_freeze(ctdb);
- ctdb_release_all_ips(ctdb);
+ if (node->flags == 0 && !ctdb->done_startup) {
+ DEBUG(DEBUG_ERR, (__location__ " Node %u became healthy - force recovery for startup\n",
+ c->pnn));
ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
}
+
+ /* tell the recovery daemon something has changed */
+ ctdb_daemon_send_message(ctdb, ctdb->pnn,
+ CTDB_SRVID_SET_NODE_FLAGS, indata);
+
+ /* if we have become banned, we should go into recovery mode */
+ if ((node->flags & NODE_FLAGS_BANNED) && !(c->old_flags & NODE_FLAGS_BANNED) && (node->pnn == ctdb->pnn)) {
+ return ctdb_local_node_got_banned(ctdb);
+ }
return 0;
}
+
+/*
+ return the monitoring mode
+ */
+int32_t ctdb_monitoring_mode(struct ctdb_context *ctdb)
+{
+ if (ctdb->monitor == NULL) {
+ return CTDB_MONITORING_DISABLED;
+ }
+ return ctdb->monitor->monitoring_mode;
+}
+
git.samba.org / metze / ctdb / wip.git / blobdiff