From: Rusty Russell <rusty@rustcorp.com.au>
Date: Wed, 16 Dec 2009 10:27:20 +0000 (+1030)
Subject: ctdb: use mlockall, cautiously
X-Git-Tag: ctdb-1.0.109~2^2
X-Git-Url: http://git.samba.org/?a=commitdiff_plain;h=82f778e85440bc713d3f87c08ddc955d3cfce926;p=sahlberg%2Fctdb.git

ctdb: use mlockall, cautiously

We don't want ctdb stalling due to paging; this can be far worse than
scheduling delays.  But if we simply do mlockall(MCL_FUTURE), it
increases the risk that mmap (ie. tdb open) or malloc will fail,
causing us to abort.

This patch is a compromise: we mlock all current pages (including
10k of future stack for expansion) and then relock when a client
asks us to open a TDB.  We warn, but don't exit, if it fails.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---

diff --git a/common/ctdb_util.c b/common/ctdb_util.c
index fe2ada45..0b34f2ad 100644
--- a/common/ctdb_util.c
+++ b/common/ctdb_util.c
@@ -23,6 +23,7 @@
 #include "system/network.h"
 #include "system/filesys.h"
 #include "system/wait.h"
+#include "system/shmem.h"
 #include "../include/ctdb_private.h"
 
 int LogLevel = DEBUG_NOTICE;
@@ -629,6 +630,28 @@ int32_t get_debug_by_desc(const char *desc)
 	return DEBUG_ERR;
 }
 
+/* we don't lock future pages here; it would increase the chance that
+ * we'd fail to mmap later on. */
+void ctdb_lockdown_memory(struct ctdb_context *ctdb)
+{
+#ifdef HAVE_MLOCKALL
+	/* Extra stack, please! */
+	char dummy[10000];
+	memset(dummy, 0, sizeof(dummy));
+
+	if (ctdb->valgrinding) {
+		return;
+	}
+
+	/* Avoid compiler optimizing out dummy. */
+	mlock(dummy, sizeof(dummy));
+	if (mlockall(MCL_CURRENT) != 0) {
+		DEBUG(DEBUG_WARNING,("Failed to lock memory: %s'\n",
+				     strerror(errno)));
+	}
+#endif
+}
+
 const char *ctdb_eventscript_call_names[] = {
 	"startup",
 	"startrecovery",
diff --git a/configure.ac b/configure.ac
index aaec420c..d4098a6a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -71,6 +71,7 @@ m4_include(ib/config.m4)
 
 AC_CHECK_HEADERS(sched.h)
 AC_CHECK_FUNCS(sched_setscheduler)
+AC_CHECK_FUNCS(mlockall)
 
 AC_CACHE_CHECK([for sin_len in sock],ctdb_cv_HAVE_SOCK_SIN_LEN,[
 AC_TRY_COMPILE([#include <sys/types.h>
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 9fc2be74..de74bb88 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -1442,7 +1442,7 @@ void ctdb_block_signal(int signum);
 void ctdb_unblock_signal(int signum);
 int32_t ctdb_monitoring_mode(struct ctdb_context *ctdb);
 int ctdb_set_child_logging(struct ctdb_context *ctdb);
-
+void ctdb_lockdown_memory(struct ctdb_context *ctdb);
 
 typedef void (*client_async_callback)(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data);
 
diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c
index 97051625..cc076fbc 100644
--- a/server/ctdb_daemon.c
+++ b/server/ctdb_daemon.c
@@ -805,6 +805,7 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
 		}
 	}
 
+	ctdb_lockdown_memory(ctdb);
 	  
 	/* go into a wait loop to allow other nodes to complete */
 	event_loop_wait(ctdb->ev);
diff --git a/server/ctdb_ltdb_server.c b/server/ctdb_ltdb_server.c
index 6c204821..c24d4214 100644
--- a/server/ctdb_ltdb_server.c
+++ b/server/ctdb_ltdb_server.c
@@ -361,6 +361,9 @@ int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata,
 	outdata->dptr  = (uint8_t *)&db->db_id;
 	outdata->dsize = sizeof(db->db_id);
 
+	/* Try to ensure it's locked in mem */
+	ctdb_lockdown_memory(ctdb);
+
 	/* tell all the other nodes about this database */
 	ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, 0,
 				 persistent?CTDB_CONTROL_DB_ATTACH_PERSISTENT: