along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
-#include "includes.h"
-#include "lib/events/events.h"
+#include "replace.h"
#include "system/filesys.h"
+#include "system/network.h"
#include "system/wait.h"
-#include "../include/ctdb_private.h"
+
+#include <talloc.h>
+#include <tevent.h>
+
+#include "lib/util/debug.h"
+#include "lib/util/samba_util.h"
+
+#include "ctdb_private.h"
+
+#include "common/system.h"
+#include "common/common.h"
+#include "common/logging.h"
struct ctdb_monitor_state {
uint32_t monitoring_mode;
TALLOC_CTX *monitor_context;
uint32_t next_interval;
+ uint32_t event_script_timeouts;
};
-static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
+static void ctdb_check_health(struct tevent_context *ev,
+ struct tevent_timer *te,
struct timeval t, void *private_data);
/*
return ret;
}
-static void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *event)
+void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *event)
{
pid_t child;
return;
}
- child = fork();
+ child = ctdb_fork(ctdb);
if (child == (pid_t)-1) {
DEBUG(DEBUG_ERR,("Failed to fork() a notification child process\n"));
return;
if (child == 0) {
int ret;
+ ctdb_set_process_name("ctdb_notification");
+ debug_extra = talloc_asprintf(NULL, "notification-%s:", event);
ret = ctdb_run_notification_script_child(ctdb, event);
if (ret != 0) {
DEBUG(DEBUG_ERR,(__location__ " Notification script failed\n"));
uint32_t next_interval;
int ret;
TDB_DATA rddata;
- struct takeover_run_reply rd;
+ struct ctdb_srvid_message rd;
+ const char *state_str = NULL;
c.pnn = ctdb->pnn;
c.old_flags = node->flags;
rddata.dptr = (uint8_t *)&rd;
rddata.dsize = sizeof(rd);
- if (status == -ETIME) {
- ctdb->event_script_timeouts++;
+ if (status == -ECANCELED) {
+ DEBUG(DEBUG_ERR,("Monitoring event was cancelled\n"));
+ goto after_change_status;
+ }
- if (ctdb->event_script_timeouts >= ctdb->tunable.script_timeout_count) {
- DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Making node unhealthy\n", ctdb->tunable.script_timeout_count));
+ if (status == -ETIME) {
+ ctdb->monitor->event_script_timeouts++;
+
+ if (ctdb->monitor->event_script_timeouts >=
+ ctdb->tunable.monitor_timeout_count) {
+ DEBUG(DEBUG_ERR,
+ ("Maximum monitor timeout count %u reached."
+ " Making node unhealthy\n",
+ ctdb->tunable.monitor_timeout_count));
} else {
/* We pretend this is OK. */
goto after_change_status;
}
+ } else {
+ ctdb->monitor->event_script_timeouts = 0;
}
if (status != 0 && !(node->flags & NODE_FLAGS_UNHEALTHY)) {
ctdb->monitor->next_interval = 5;
ctdb_run_notification_script(ctdb, "unhealthy");
-
- /* ask the recmaster to reallocate all addresses */
- DEBUG(DEBUG_ERR,("Node became UNHEALTHY. Ask recovery master %u to perform ip reallocation\n", ctdb->recovery_master));
- ret = ctdb_daemon_send_message(ctdb, ctdb->recovery_master, CTDB_SRVID_TAKEOVER_RUN, rddata);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to send ip takeover run request message to %u\n", ctdb->recovery_master));
- }
-
} else if (status == 0 && (node->flags & NODE_FLAGS_UNHEALTHY)) {
DEBUG(DEBUG_NOTICE,("monitor event OK - node re-enabled\n"));
node->flags &= ~NODE_FLAGS_UNHEALTHY;
ctdb->monitor->next_interval = 5;
ctdb_run_notification_script(ctdb, "healthy");
-
- /* ask the recmaster to reallocate all addresses */
- DEBUG(DEBUG_ERR,("Node became HEALTHY. Ask recovery master %u to perform ip reallocation\n", ctdb->recovery_master));
- ret = ctdb_daemon_send_message(ctdb, ctdb->recovery_master, CTDB_SRVID_TAKEOVER_RUN, rddata);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to send ip takeover run request message to %u\n", ctdb->recovery_master));
- }
-
}
after_change_status:
ctdb->monitor->next_interval = ctdb->tunable.monitor_interval;
}
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(next_interval, 0),
- ctdb_check_health, ctdb);
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(next_interval, 0),
+ ctdb_check_health, ctdb);
if (c.old_flags == node->flags) {
return;
ctdb_daemon_send_message(ctdb, ctdb->pnn,
CTDB_SRVID_PUSH_NODE_FLAGS, data);
+ if (c.new_flags & NODE_FLAGS_UNHEALTHY) {
+ state_str = "UNHEALTHY";
+ } else {
+ state_str = "HEALTHY";
+ }
+
+ /* ask the recmaster to reallocate all addresses */
+ DEBUG(DEBUG_ERR,
+ ("Node became %s. Ask recovery master to reallocate IPs\n",
+ state_str));
+ ret = ctdb_daemon_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_TAKEOVER_RUN, rddata);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,
+ (__location__
+ " Failed to send IP takeover run request\n"));
+ }
}
+static void ctdb_run_startup(struct tevent_context *ev,
+ struct tevent_timer *te,
+ struct timeval t, void *private_data);
/*
called when the startup event script finishes
*/
{
if (status != 0) {
DEBUG(DEBUG_ERR,("startup event failed\n"));
- } else if (status == 0) {
- DEBUG(DEBUG_NOTICE,("startup event OK - enabling monitoring\n"));
- ctdb->done_startup = true;
- ctdb->monitor->next_interval = 5;
- ctdb_run_notification_script(ctdb, "startup");
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(5, 0),
+ ctdb_run_startup, ctdb);
+ return;
}
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(ctdb->monitor->next_interval, 0),
- ctdb_check_health, ctdb);
+ DEBUG(DEBUG_NOTICE,("startup event OK - enabling monitoring\n"));
+ ctdb_set_runstate(ctdb, CTDB_RUNSTATE_RUNNING);
+ ctdb->monitor->next_interval = 2;
+ ctdb_run_notification_script(ctdb, "startup");
+
+ ctdb->monitor->monitoring_mode = CTDB_MONITORING_ACTIVE;
+
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(ctdb->monitor->next_interval, 0),
+ ctdb_check_health, ctdb);
}
+static void ctdb_run_startup(struct tevent_context *ev,
+ struct tevent_timer *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_context *ctdb = talloc_get_type(private_data,
+ struct ctdb_context);
+ int ret;
+
+ /* This is necessary to avoid the "startup" event colliding
+ * with the "ipreallocated" event from the takeover run
+ * following the first recovery. We might as well serialise
+ * these things if we can.
+ */
+ if (ctdb->runstate < CTDB_RUNSTATE_STARTUP) {
+ DEBUG(DEBUG_NOTICE,
+ ("Not yet in startup runstate. Wait one more second\n"));
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_run_startup, ctdb);
+ return;
+ }
+
+ /* release any IPs we hold from previous runs of the daemon */
+ ctdb_release_all_ips(ctdb);
+
+ DEBUG(DEBUG_NOTICE,("Running the \"startup\" event.\n"));
+ ret = ctdb_event_script_callback(ctdb,
+ ctdb->monitor->monitor_context,
+ ctdb_startup_callback,
+ ctdb, CTDB_EVENT_STARTUP, "%s", "");
+
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Unable to launch startup event script\n"));
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(5, 0),
+ ctdb_run_startup, ctdb);
+ }
+}
/*
wait until we have finished initial recoveries before we start the
monitoring events
*/
-static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *private_data)
+static void ctdb_wait_until_recovered(struct tevent_context *ev,
+ struct tevent_timer *te,
+ struct timeval t, void *private_data)
{
struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+ int ret;
+ static int count = 0;
- DEBUG(DEBUG_NOTICE,("CTDB_WAIT_UNTIL_RECOVERED\n"));
+ count++;
+
+ if (count < 60 || count%600 == 0) {
+ DEBUG(DEBUG_NOTICE,("CTDB_WAIT_UNTIL_RECOVERED\n"));
+ if (ctdb->nodes[ctdb->pnn]->flags & NODE_FLAGS_STOPPED) {
+ DEBUG(DEBUG_NOTICE,("Node is STOPPED. Node will NOT recover.\n"));
+ }
+ }
if (ctdb->vnn_map->generation == INVALID_GENERATION) {
- DEBUG(DEBUG_NOTICE,(__location__ " generation is INVALID. Wait one more second\n"));
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(1, 0),
- ctdb_wait_until_recovered, ctdb);
+ ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
return;
}
if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
+ ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
DEBUG(DEBUG_NOTICE,(__location__ " in recovery. Wait one more second\n"));
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(1, 0),
- ctdb_wait_until_recovered, ctdb);
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
return;
}
- if (timeval_elapsed(&ctdb->last_recovery_finished) < (ctdb->tunable.rerecovery_timeout + 3)) {
+ if (!fast_start && timeval_elapsed(&ctdb->last_recovery_finished) < (ctdb->tunable.rerecovery_timeout + 3)) {
+ ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
DEBUG(DEBUG_NOTICE,(__location__ " wait for pending recoveries to end. Wait one more second.\n"));
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(1, 0),
- ctdb_wait_until_recovered, ctdb);
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
return;
}
+ if (ctdb->vnn_map->generation == ctdb->db_persistent_startup_generation) {
+ DEBUG(DEBUG_INFO,(__location__ " skip ctdb_recheck_persistent_health() "
+ "until the next recovery\n"));
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+ return;
+ }
- DEBUG(DEBUG_NOTICE,(__location__ " Recoveries finished. Running the \"startup\" event.\n"));
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(1, 0),
- ctdb_check_health, ctdb);
+ ctdb->db_persistent_startup_generation = ctdb->vnn_map->generation;
+ ret = ctdb_recheck_persistent_health(ctdb);
+ if (ret != 0) {
+ ctdb->db_persistent_check_errors++;
+ if (ctdb->db_persistent_check_errors < ctdb->max_persistent_check_errors) {
+ DEBUG(ctdb->db_persistent_check_errors==1?DEBUG_ERR:DEBUG_WARNING,
+ (__location__ "ctdb_recheck_persistent_health() "
+ "failed (%llu of %llu times) - retry later\n",
+ (unsigned long long)ctdb->db_persistent_check_errors,
+ (unsigned long long)ctdb->max_persistent_check_errors));
+ tevent_add_timer(ctdb->ev,
+ ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
+ return;
+ }
+ DEBUG(DEBUG_ALERT,(__location__
+ "ctdb_recheck_persistent_health() failed (%llu times) - prepare shutdown\n",
+ (unsigned long long)ctdb->db_persistent_check_errors));
+ ctdb_shutdown_sequence(ctdb, 11);
+ /* In case above returns due to duplicate shutdown */
+ return;
+ }
+ ctdb->db_persistent_check_errors = 0;
+
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current(), ctdb_run_startup, ctdb);
}
/*
see if the event scripts think we are healthy
*/
-static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
+static void ctdb_check_health(struct tevent_context *ev,
+ struct tevent_timer *te,
struct timeval t, void *private_data)
{
struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+ bool skip_monitoring = false;
int ret = 0;
if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL ||
- (ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED && ctdb->done_startup)) {
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(ctdb->monitor->next_interval, 0),
- ctdb_check_health, ctdb);
- return;
- }
-
- if (!ctdb->done_startup) {
- ret = ctdb_event_script_callback(ctdb,
- ctdb->monitor->monitor_context, ctdb_startup_callback,
- ctdb, false,
- CTDB_EVENT_STARTUP, "%s", "");
+ ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED) {
+ skip_monitoring = true;
} else {
- int i;
- int skip_monitoring = 0;
-
- if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
- skip_monitoring = 1;
- DEBUG(DEBUG_ERR,("Skip monitoring during recovery\n"));
- }
- for (i=1; i<=NUM_DB_PRIORITIES; i++) {
- if (ctdb->freeze_handles[i] != NULL) {
- DEBUG(DEBUG_ERR,("Skip monitoring since databases are frozen\n"));
- skip_monitoring = 1;
- break;
- }
- }
- if (skip_monitoring != 0) {
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(ctdb->monitor->next_interval, 0),
- ctdb_check_health, ctdb);
- return;
- } else {
- ret = ctdb_event_script_callback(ctdb,
- ctdb->monitor->monitor_context, ctdb_health_callback,
- ctdb, false,
- CTDB_EVENT_MONITOR, "%s", "");
+ if (ctdb_db_all_frozen(ctdb)) {
+ DEBUG(DEBUG_ERR,
+ ("Skip monitoring since databases are frozen\n"));
+ skip_monitoring = true;
}
}
+ if (skip_monitoring) {
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(ctdb->monitor->next_interval, 0),
+ ctdb_check_health, ctdb);
+ return;
+ }
+
+ ret = ctdb_event_script_callback(ctdb,
+ ctdb->monitor->monitor_context,
+ ctdb_health_callback,
+ ctdb, CTDB_EVENT_MONITOR, "%s", "");
if (ret != 0) {
DEBUG(DEBUG_ERR,("Unable to launch monitor event script\n"));
ctdb->monitor->next_interval = 5;
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(5, 0),
- ctdb_check_health, ctdb);
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(5, 0),
+ ctdb_check_health, ctdb);
}
}
/*
start watching for nodes that might be dead
*/
-void ctdb_start_monitoring(struct ctdb_context *ctdb)
+void ctdb_wait_for_first_recovery(struct ctdb_context *ctdb)
{
- if (ctdb->monitor != NULL) {
- return;
- }
+ ctdb_set_runstate(ctdb, CTDB_RUNSTATE_FIRST_RECOVERY);
ctdb->monitor = talloc(ctdb, struct ctdb_monitor_state);
CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor);
- ctdb->monitor->next_interval = 5;
-
ctdb->monitor->monitor_context = talloc_new(ctdb->monitor);
CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor->monitor_context);
- event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(1, 0),
- ctdb_wait_until_recovered, ctdb);
-
- ctdb->monitor->monitoring_mode = CTDB_MONITORING_ACTIVE;
- DEBUG(DEBUG_NOTICE,("Monitoring has been started\n"));
+ tevent_add_timer(ctdb->ev, ctdb->monitor->monitor_context,
+ timeval_current_ofs(1, 0),
+ ctdb_wait_until_recovered, ctdb);
}
struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)indata.dptr;
struct ctdb_node *node;
uint32_t old_flags;
- int i;
if (c->pnn >= ctdb->num_nodes) {
DEBUG(DEBUG_ERR,(__location__ " Node %d is invalid, num_nodes :%d\n", c->pnn, ctdb->num_nodes));
node->flags = c->new_flags & ~NODE_FLAGS_DISCONNECTED;
node->flags |= (c->old_flags & NODE_FLAGS_DISCONNECTED);
- /* we dont let other nodes modify our STOPPED status */
+ /* we don't let other nodes modify our STOPPED status */
if (c->pnn == ctdb->pnn) {
node->flags &= ~NODE_FLAGS_STOPPED;
if (old_flags & NODE_FLAGS_STOPPED) {
}
}
- /* we dont let other nodes modify our BANNED status */
+ /* we don't let other nodes modify our BANNED status */
if (c->pnn == ctdb->pnn) {
node->flags &= ~NODE_FLAGS_BANNED;
if (old_flags & NODE_FLAGS_BANNED) {
DEBUG(DEBUG_INFO, ("Control modflags on node %u - flags now 0x%x\n", c->pnn, node->flags));
+ if (node->flags == 0 && ctdb->runstate <= CTDB_RUNSTATE_STARTUP) {
+ DEBUG(DEBUG_ERR, (__location__ " Node %u became healthy - force recovery for startup\n",
+ c->pnn));
+ ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
+ }
/* tell the recovery daemon something has changed */
+ c->new_flags = node->flags;
ctdb_daemon_send_message(ctdb, ctdb->pnn,
CTDB_SRVID_SET_NODE_FLAGS, indata);
/* if we have become banned, we should go into recovery mode */
if ((node->flags & NODE_FLAGS_BANNED) && !(c->old_flags & NODE_FLAGS_BANNED) && (node->pnn == ctdb->pnn)) {
- /* make sure we are frozen */
- DEBUG(DEBUG_NOTICE,("This node has been banned - forcing freeze and recovery\n"));
- /* Reset the generation id to 1 to make us ignore any
- REQ/REPLY CALL/DMASTER someone sends to us.
- We are now banned so we shouldnt service database calls
- anymore.
- */
- ctdb->vnn_map->generation = INVALID_GENERATION;
-
- for (i=1; i<=NUM_DB_PRIORITIES; i++) {
- if (ctdb_start_freeze(ctdb, i) != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to freeze db priority %u\n", i));
- }
- }
- ctdb_release_all_ips(ctdb);
- ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
+ ctdb_local_node_got_banned(ctdb);
}
return 0;
return ctdb->monitor->monitoring_mode;
}
+/*
+ * Check if monitoring has been stopped
+ */
+bool ctdb_stopped_monitoring(struct ctdb_context *ctdb)
+{
+ return (ctdb->monitor->monitor_context == NULL ? true : false);
+}
git.samba.org / obnox / samba / samba-obnox.git / blobdiff