make it possible to tweak the max number of concurrent migrations

diff --git a/doc/remote-cache.1.xml b/doc/remote-cache.1.xml
index 2e3c72e125fa61d4c6578f5067dabc5ade88b3a0..f02a009b69e5569cb663b51dffd9531abd9c6d4a 100644 (file)
--- a/doc/remote-cache.1.xml
+++ b/doc/remote-cache.1.xml
@@ -189,6 +189,20 @@
     </refsect2>
 
 
+    <refsect2><title>--file-migrate-max-concurrent</title>
+    <para>
+    This parameter defaults to 1000.
+    </para>
+    <para>
+    Maximum number of concurrent migrations of files we allow before we start
+    failing the migration.
+
+    Note that failing a migration is not a severe issue, it just means that the
+    file will be migrate a little while later instead next time it is accessed.
+    </para>
+    </refsect2>
+
+
 
     <refsect2><title>--file-check-parent-mtime</title>
     <para>

diff --git a/migrate/remote-cache.c b/migrate/remote-cache.c
index 33533ee2eee42ea8151348b9f7bececade8d6a65..9b6a3dea3ad03829e5a6775aa92c2e3531eebc6e 100644 (file)
--- a/migrate/remote-cache.c
+++ b/migrate/remote-cache.c
@@ -22,7 +22,6 @@
 /* we use a local lock file to limit how many files we are migrating
    across simultaneously
 */
-#define FILE_MIGRATE_MAX_CONCURRENT    1000
 #define FILE_MIGRATE_LIMITER_DIR "/var/lib/remote-cache"
 #define FILE_MIGRATE_LIMITER     "/var/lib/remote-cache/lockfile"
 
@@ -116,6 +115,12 @@ static int read_write = 0;
 */
 static int direct_io = 0;
 
+/* maximum number of concurrent migrations
+   do nor spawn more than this number of concurrent rsync processes to pull
+   files into the cache
+*/
+static int file_migrate_max_concurrent = 1000;
+
 static int sigchild_initialized = 0;
 static int debug_initialized = 0;
 static int log_fd = -1;
@@ -1096,10 +1101,10 @@ void child_migrate_file(TALLOC_CTX *mem_ctx, char *old_path, char *new_path)
 
        llock.l_type = F_WRLCK;
        llock.l_whence = SEEK_SET;
-       llock.l_start = random()%FILE_MIGRATE_MAX_CONCURRENT;
+       llock.l_start = (random()&0x00ffffff)%file_migrate_max_concurrent;
        llock.l_len = 1;
        llock.l_pid = getpid();
-       if (fcntl(lfd, F_SETLK, &llock) != 0 || fail < 2) {
+       if (fcntl(lfd, F_SETLK, &llock) != 0) {
                DEBUG(DEBUG_ERR, (__location__ " Failed to get migration lock. Too many concurrent migrations. File might be migrated on next read instead.\n"));
                perror("error ");
                close(lfd);
@@ -2329,7 +2334,8 @@ int main(int argc, const char *argv[])
                { "debug", 'd', POPT_ARG_INT, &debug_level, 0, "debug level", "integer" },
                { "file-check-parent-mtime", 0, POPT_ARG_NONE, &file_check_parent_mtime, 0, "check mtime of remote file instead of parent directory mtime", NULL },
                { "read-write", 0, POPT_ARG_NONE, &read_write, 0, "read/write cache", NULL },
-               { "direct_io", 0, POPT_ARG_NONE, &direct_io, 0, "Use direct_io for better write performance (not kNFSd compatible)", NULL },
+               { "direct-io", 0, POPT_ARG_NONE, &direct_io, 0, "Use direct_io for better write performance (not kNFSd compatible)", NULL },
+               { "file-migrate-max-concurrent", 0, POPT_ARG_INT, &file_migrate_max_concurrent, 0, "maximum number of concurrent rsync migrations", "integer" },
 
                POPT_TABLEEND
        };