Make idev, hlink and file_struct + strings use allocation
authorJ.W. Schultz <jw@samba.org>
Tue, 10 Feb 2004 03:23:37 +0000 (03:23 +0000)
committerJ.W. Schultz <jw@samba.org>
Tue, 10 Feb 2004 03:23:37 +0000 (03:23 +0000)
pools.

Makefile.in
NEWS
backup.c
batch.c
flist.c
hlink.c
receiver.c
rsync.h

index d60309f9a5052324a0879258d3434f53804f8ac5..03c7983cff9a0813b8b25a327f771a70b80c1e91 100644 (file)
@@ -27,7 +27,7 @@ VERSION=@VERSION@
 
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o \
-       lib/permstring.o @LIBOBJS@
+       lib/permstring.o lib/pool_alloc.o @LIBOBJS@
 ZLIBOBJ=zlib/deflate.o zlib/infblock.o zlib/infcodes.o zlib/inffast.o \
        zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \
        zlib/zutil.o zlib/adler32.o
diff --git a/NEWS b/NEWS
index 61a30f91645f6de82a71bfbc059f1fd8c683795d..82f734b5e055c39b50748129295dec317f721d54 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -100,6 +100,10 @@ Changes since 2.6.0:
 
     * Less memory is used in the file list (a per-file savings).
 
+    * Changed hardlink info and file_struct + strings to use
+      allocation pools.  This reduces memory use for large
+      filesets and permits freeing memory to the OS.  (J.W. Schultz) 
+
     * The 2 pipes used between the receiver and generator processes
       (which are forked on the same machine) were reduced to 1 pipe
       and the protocol improved so that (1) it is now impossible to
index c1b80834f30c72d0ccef530c872468146a2d1b45..248ae90a5143df1f5de096b7a9c347a279615359 100644 (file)
--- a/backup.c
+++ b/backup.c
@@ -189,6 +189,7 @@ static int keep_backup(char *fname)
                        backup_dir[--backup_dir_len] = '\0';
                if (verbose > 0)
                        rprintf(FINFO, "backup_dir is %s\n", backup_dir);
+
                initialised = 1;
        }
 
@@ -199,7 +200,7 @@ static int keep_backup(char *fname)
        if (do_stat(fname, &st)) return 1;
 #endif
 
-       file = make_file(fname, NO_EXCLUDES);
+       file = make_file(fname, NULL, NO_EXCLUDES);
 
        /* the file could have disappeared */
        if (!file) return 1;
@@ -282,7 +283,7 @@ static int keep_backup(char *fname)
                }
        }
        set_perms(keep_name, file, NULL, 0);
-       free_file(file, FREE_STRUCT);
+       free(file);
 
        if (verbose > 1)
                rprintf(FINFO, "keep_backup %s -> %s\n", fname, keep_name);
diff --git a/batch.c b/batch.c
index d90c87b9773a68375d8004d80344b833b31cb86d..dee68f071358ae18751f6034a1b2b505d2692b4e 100644 (file)
--- a/batch.c
+++ b/batch.c
@@ -136,9 +136,7 @@ struct file_list *create_flist_from_batch(void)
                exit_cleanup(1);
        }
 
-       batch_flist = new(struct file_list);
-       if (!batch_flist)
-               out_of_memory("create_flist_from_batch");
+       batch_flist = flist_new(WITH_HLINK, "create_flist_from_batch");
 
        save_read = stats.total_read;
        save_pv = protocol_version;
@@ -150,9 +148,9 @@ struct file_list *create_flist_from_batch(void)
        for (i = 0; (flags = read_byte(f)) != 0; i++) {
                if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
                        flags |= read_byte(f) << 8;
-               receive_file_entry(&batch_flist->files[i], flags, f);
+               receive_file_entry(&batch_flist->files[i], flags, batch_flist, f);
        }
-       receive_file_entry(NULL, 0, 0); /* Signal that we're done. */
+       receive_file_entry(NULL, 0, NULL, 0); /* Signal that we're done. */
 
        protocol_version = save_pv;
        stats.total_read = save_read;
diff --git a/flist.c b/flist.c
index f21adb7f87c1704b362866e62787f11dbdccb90d..19931a5f2b35cf255333df7522a75600d81a15f9 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -71,18 +71,17 @@ extern struct exclude_struct **local_exclude_list;
 int io_error;
 
 static char empty_sum[MD4_SUM_LENGTH];
-static unsigned int min_file_struct_len;
+static unsigned int file_struct_len;
 
 static void clean_flist(struct file_list *flist, int strip_root, int no_dups);
 static void output_flist(struct file_list *flist);
 
-
 void init_flist(void)
 {
        struct file_struct f;
 
        /* Figure out how big the file_struct is without trailing padding */
-       min_file_struct_len = ((char*)&f.flags - (char*)&f) + sizeof f.flags;
+       file_struct_len = ((char*)&f.flags - (char*)&f) + sizeof f.flags;
 }
 
 
@@ -507,7 +506,8 @@ void send_file_entry(struct file_struct *file, int f, unsigned short base_flags)
 
 
 
-void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
+void receive_file_entry(struct file_struct **fptr, unsigned short flags,
+    struct file_list *flist, int f)
 {
        static time_t modtime;
        static mode_t mode;
@@ -520,7 +520,6 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
        char thisname[MAXPATHLEN];
        unsigned int l1 = 0, l2 = 0;
        int alloc_len, basename_len, dirname_len, linkname_len, sum_len;
-       int file_struct_len, idev_len;
        OFF_T file_length;
        char *basename, *dirname, *bp;
        struct file_struct *file;
@@ -614,24 +613,14 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
 #endif
                linkname_len = 0;
 
-#if SUPPORT_HARD_LINKS
-       if (preserve_hard_links && protocol_version < 28 && S_ISREG(mode))
-               flags |= XMIT_HAS_IDEV_DATA;
-       if (flags & XMIT_HAS_IDEV_DATA)
-               idev_len = sizeof (struct idev);
-       else
-#endif
-               idev_len = 0;
-
        sum_len = always_checksum && S_ISREG(mode) ? MD4_SUM_LENGTH : 0;
-       file_struct_len = idev_len? sizeof file[0] : min_file_struct_len;
 
        alloc_len = file_struct_len + dirname_len + basename_len
-                 + linkname_len + sum_len + idev_len;
-       if (!(bp = new_array(char, alloc_len)))
-               out_of_memory("receive_file_entry");
+                 + linkname_len + sum_len;
+       bp = pool_alloc(flist->file_pool, alloc_len, "receive_file_entry");
+
        file = *fptr = (struct file_struct *)bp;
-       memset(bp, 0, min_file_struct_len);
+       memset(bp, 0, file_struct_len);
        bp += file_struct_len;
 
        file->flags = flags & XMIT_TOP_DIR ? FLAG_TOP_DIR : 0;
@@ -641,13 +630,6 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
        file->uid = uid;
        file->gid = gid;
 
-#if SUPPORT_HARD_LINKS
-       if (idev_len) {
-               file->link_u.idev = (struct idev *)bp;
-               bp += idev_len;
-       }
-#endif
-
        if (dirname_len) {
                file->dirname = lastdir = bp;
                lastdir_len = dirname_len - 1;
@@ -675,16 +657,24 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
 #endif
 
 #if SUPPORT_HARD_LINKS
-       if (idev_len) {
+       if (preserve_hard_links && protocol_version < 28 && S_ISREG(mode))
+               flags |= XMIT_HAS_IDEV_DATA;
+       if (flags & XMIT_HAS_IDEV_DATA && flist->hlink_pool) {
+               INO64_T inode;
+               file->link_u.idev = pool_talloc(flist->hlink_pool,
+                   struct idev, 1, "inode_table");
                if (protocol_version < 26) {
                        dev = read_int(f);
-                       file->F_INODE = read_int(f);
+                       inode = read_int(f);
                } else {
                        if (!(flags & XMIT_SAME_DEV))
                                dev = read_longint(f);
-                       file->F_INODE = read_longint(f);
+                       inode = read_longint(f);
+               }
+               if (flist->hlink_pool) {
+                       file->F_INODE = inode;
+                       file->F_DEV = dev;
                }
-               file->F_DEV = dev;
        }
 #endif
 
@@ -728,7 +718,8 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
  * statting directories if we're not recursing, but this is not a very
  * important case.  Some systems may not have d_type.
  **/
-struct file_struct *make_file(char *fname, int exclude_level)
+struct file_struct *make_file(char *fname,
+    struct file_list *flist, int exclude_level)
 {
        static char *lastdir;
        static int lastdir_len = -1;
@@ -738,10 +729,10 @@ struct file_struct *make_file(char *fname, int exclude_level)
        char thisname[MAXPATHLEN];
        char linkname[MAXPATHLEN];
        int alloc_len, basename_len, dirname_len, linkname_len, sum_len;
-       int file_struct_len, idev_len;
        char *basename, *dirname, *bp;
        unsigned short flags = 0;
 
+
        if (strlcpy(thisname, fname, sizeof thisname)
            >= sizeof thisname - flist_dir_len) {
                rprintf(FINFO, "skipping overly long name: %s\n", fname);
@@ -820,32 +811,20 @@ struct file_struct *make_file(char *fname, int exclude_level)
        linkname_len = 0;
 #endif
 
-#if SUPPORT_HARD_LINKS
-       if (preserve_hard_links) {
-               if (protocol_version < 28) {
-                       if (S_ISREG(st.st_mode))
-                               idev_len = sizeof (struct idev);
-                       else
-                               idev_len = 0;
-               } else {
-                       if (!S_ISDIR(st.st_mode) && st.st_nlink > 1)
-                               idev_len = sizeof (struct idev);
-                       else
-                               idev_len = 0;
-               }
-       } else
-#endif
-               idev_len = 0;
-
        sum_len = always_checksum && S_ISREG(st.st_mode) ? MD4_SUM_LENGTH : 0;
-       file_struct_len = idev_len? sizeof file[0] : min_file_struct_len;
 
        alloc_len = file_struct_len + dirname_len + basename_len
-                 + linkname_len + sum_len + idev_len;
-       if (!(bp = new_array(char, alloc_len)))
-               out_of_memory("receive_file_entry");
+           + linkname_len + sum_len;
+       if (flist) {
+               bp = pool_alloc(flist->file_pool, alloc_len,
+                   "receive_file_entry");
+       } else {
+               if (!(bp = new_array(char, alloc_len)))
+                       out_of_memory("receive_file_entry");
+       }
+
        file = (struct file_struct *)bp;
-       memset(bp, 0, min_file_struct_len);
+       memset(bp, 0, file_struct_len);
        bp += file_struct_len;
 
        file->flags = flags;
@@ -856,9 +835,20 @@ struct file_struct *make_file(char *fname, int exclude_level)
        file->gid = st.st_gid;
 
 #if SUPPORT_HARD_LINKS
-       if (idev_len) {
-               file->link_u.idev = (struct idev *)bp;
-               bp += idev_len;
+       if (flist && flist->hlink_pool) {
+               if (protocol_version < 28) {
+                       if (S_ISREG(st.st_mode))
+                               file->link_u.idev = pool_talloc(
+                                   flist->hlink_pool, struct idev, 1,
+                                   "inode_table");
+               } else {
+                       if (!S_ISDIR(st.st_mode) && st.st_nlink > 1)
+                               file->link_u.idev = pool_talloc(
+                                   flist->hlink_pool, struct idev, 1,
+                                   "inode_table");
+               }
+       }
+       if (file->link_u.idev) {
                file->F_DEV = st.st_dev;
                file->F_INODE = st.st_ino;
        }
@@ -913,9 +903,8 @@ void send_file_name(int f, struct file_list *flist, char *fname,
        extern int delete_excluded;
 
        /* f is set to -1 when calculating deletion file list */
-       file = make_file(fname,
-                        f == -1 && delete_excluded? SERVER_EXCLUDES
-                                                  : ALL_EXCLUDES);
+       file = make_file(fname, flist,
+           f == -1 && delete_excluded? SERVER_EXCLUDES : ALL_EXCLUDES);
 
        if (!file)
                return;
@@ -1034,7 +1023,8 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
        start_write = stats.total_written;
 
-       flist = flist_new();
+       flist = flist_new(f == -1 ? WITHOUT_HLINK : WITH_HLINK,
+           "send_file_list");
 
        if (f != -1) {
                io_start_buffering_out(f);
@@ -1185,6 +1175,12 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
                        finish_filelist_progress(flist);
        }
 
+       if (flist->hlink_pool)
+       {
+               pool_destroy(flist->hlink_pool);
+               flist->hlink_pool = NULL;
+       }
+
        clean_flist(flist, 0, 0);
 
        if (f != -1) {
@@ -1224,9 +1220,7 @@ struct file_list *recv_file_list(int f)
 
        start_read = stats.total_read;
 
-       flist = new(struct file_list);
-       if (!flist)
-               goto oom;
+       flist = flist_new(WITH_HLINK, "recv_file_list");
 
        flist->count = 0;
        flist->malloced = 1000;
@@ -1242,7 +1236,7 @@ struct file_list *recv_file_list(int f)
 
                if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
                        flags |= read_byte(f) << 8;
-               receive_file_entry(&flist->files[i], flags, f);
+               receive_file_entry(&flist->files[i], flags, flist, f);
 
                if (S_ISREG(flist->files[i]->mode))
                        stats.total_size += flist->files[i]->length;
@@ -1256,7 +1250,7 @@ struct file_list *recv_file_list(int f)
                                f_name(flist->files[i]));
                }
        }
-       receive_file_entry(NULL, 0, 0); /* Signal that we're done. */
+       receive_file_entry(NULL, 0, NULL, 0); /* Signal that we're done. */
 
        if (verbose > 2)
                rprintf(FINFO, "received %d names\n", flist->count);
@@ -1345,34 +1339,42 @@ int flist_find(struct file_list *flist, struct file_struct *f)
        return -1;
 }
 
-
 /*
- * Free up any resources a file_struct has allocated, and optionally free
- * it up as well.
+ * Free up any resources a file_struct has allocated
+ * and clear the file.
  */
-void free_file(struct file_struct *file, int free_the_struct)
+void clear_file(int i, struct file_list *flist)
 {
-       if (free_the_struct)
-               free(file);
-       else
-               memset(file, 0, min_file_struct_len);
+       if (flist->hlink_pool && flist->files[i]->link_u.idev)
+               pool_free(flist->hlink_pool, 0, flist->files[i]->link_u.idev);
+       memset(flist->files[i], 0, file_struct_len);
 }
 
 
 /*
  * allocate a new file list
  */
-struct file_list *flist_new(void)
+struct file_list *flist_new(int with_hlink, char *msg)
 {
        struct file_list *flist;
 
        flist = new(struct file_list);
        if (!flist)
-               out_of_memory("send_file_list");
+               out_of_memory(msg);
 
-       flist->count = 0;
-       flist->malloced = 0;
-       flist->files = NULL;
+       memset(flist, 0, sizeof (struct file_list));
+
+       if (!(flist->file_pool = pool_create(FILE_EXTENT, 0,
+           out_of_memory, POOL_INTERN)))
+               out_of_memory(msg);
+
+#if SUPPORT_HARD_LINKS
+       if (with_hlink && preserve_hard_links) {
+               if (!(flist->hlink_pool = pool_create(HLINK_EXTENT, 
+                   sizeof (struct idev), out_of_memory, POOL_INTERN)))
+                       out_of_memory(msg);
+       }
+#endif
 
        return flist;
 }
@@ -1382,9 +1384,8 @@ struct file_list *flist_new(void)
  */
 void flist_free(struct file_list *flist)
 {
-       int i;
-       for (i = 1; i < flist->count; i++)
-               free_file(flist->files[i], FREE_STRUCT);
+       pool_destroy(flist->file_pool);
+       pool_destroy(flist->hlink_pool);
        free(flist->files);
        free(flist);
 }
@@ -1424,7 +1425,8 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
                         * else deletions will mysteriously fail with -R). */
                        if (flist->files[i]->flags & FLAG_TOP_DIR)
                                flist->files[prev_i]->flags |= FLAG_TOP_DIR;
-                       free_file(flist->files[i], CLEAR_STRUCT);
+
+                       clear_file(i, flist);
                } else
                        prev_i = i;
        }
diff --git a/hlink.c b/hlink.c
index 2dbc0d99cd8e0e96c1ba3cca7054c9637b00c7aa..f42df53d4fde68f29c9de0391f23e87e06b6e68d 100644 (file)
--- a/hlink.c
+++ b/hlink.c
@@ -46,26 +46,41 @@ int hlink_count;
 
 /* Analyze the data in the hlink_list[], remove items that aren't multiply
  * linked, and replace the dev+inode data with the hlindex+next linked list. */
-static void link_idev_data(void)
+static void link_idev_data(struct file_list *flist)
 {
        struct file_struct *head;
        int from, to, start;
 
+       alloc_pool_t hlink_pool;
+       alloc_pool_t idev_pool = flist->hlink_pool;
+
+       hlink_pool = pool_create(128 * 1024, sizeof (struct hlink),
+           out_of_memory, POOL_INTERN);
+
        for (from = to = 0; from < hlink_count; from++) {
                start = from;
                head = hlink_list[start];
                while (from < hlink_count-1
                    && LINKED(hlink_list[from], hlink_list[from+1])) {
+                       pool_free(idev_pool, 0, hlink_list[from]->link_u.idev);
+                       hlink_list[from]->link_u.links = pool_talloc(hlink_pool,
+                           struct hlink, 1, "hlink_list");
+
                        hlink_list[from]->F_HLINDEX = to;
                        hlink_list[from]->F_NEXT = hlink_list[from+1];
                        from++;
                }
                if (from > start) {
+                       pool_free(idev_pool, 0, hlink_list[from]->link_u.idev);
+                       hlink_list[from]->link_u.links = pool_talloc(hlink_pool,
+                           struct hlink, 1, "hlink_list");
+
                        hlink_list[from]->F_HLINDEX = to;
                        hlink_list[from]->F_NEXT = head;
                        hlink_list[from]->flags |= FLAG_HLINK_EOL;
                        hlink_list[to++] = head;
                } else {
+                       pool_free(idev_pool, 0, head->link_u.idev);
                        head->link_u.idev = NULL;
                }
        }
@@ -73,12 +88,16 @@ static void link_idev_data(void)
        if (!to) {
                free(hlink_list);
                hlink_list = NULL;
+               pool_destroy(hlink_pool);
+               hlink_pool = NULL;
        } else {
                hlink_count = to;
                if (!(hlink_list = realloc_array(hlink_list,
                    struct file_struct *, hlink_count)))
                        out_of_memory("init_hard_links");
        }
+       flist->hlink_pool = hlink_pool;
+       pool_destroy(idev_pool);
 }
 #endif
 
@@ -109,7 +128,7 @@ void init_hard_links(struct file_list *flist)
                free(hlink_list);
                hlink_list = NULL;
        } else
-               link_idev_data();
+               link_idev_data(flist);
 #endif
 }
 
index da89819a79cceb891fb3c2f4b4952ae5e17853e9..ca6bf23f43ddab15bd497947d9f40a719400ff97 100644 (file)
@@ -305,6 +305,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name)
                rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
        }
 
+       if (flist->hlink_pool)
+       {
+               pool_destroy(flist->hlink_pool);
+               flist->hlink_pool = NULL;
+       }
+
        while (1) {
                cleanup_disable();
 
diff --git a/rsync.h b/rsync.h
index aa22cfe94143724eead81ce4d5d69c069e44f675..37a5d531e736df37e1ef40efa388d715097d7d03 100644 (file)
--- a/rsync.h
+++ b/rsync.h
 #define FULL_FLUSH     1
 #define NORMAL_FLUSH   0
 
-#define CLEAR_STRUCT   0
-#define FREE_STRUCT    1
 
 /* Log-message categories.  FLOG is only used on the daemon side to
  * output messages to the log file. */
@@ -254,6 +252,7 @@ enum msgcode {
 
 #include <assert.h>
 
+#include "lib/pool_alloc.h"
 
 #define BOOL int
 
@@ -434,10 +433,27 @@ struct file_struct {
  */
 #define FLIST_START    (32 * 1024)
 #define FLIST_LINEAR   (FLIST_START * 512)
+/*
+ * Extent size for allocation pools A minimum size of 128KB
+ * is needed to mmap them so that freeing will release the
+ * space to the OS.
+ *
+ * Larger sizes reduce leftover fragments and speed free calls
+ * (when they happen) Smaller sizes increase the chance of
+ * freed allocations freeing whole extents.
+ */
+
+#define FILE_EXTENT    (256 * 1024)
+#define HLINK_EXTENT   (128 * 1024)
+
+#define WITH_HLINK     1
+#define WITHOUT_HLINK  0
 
 struct file_list {
        int count;
        int malloced;
+       alloc_pool_t file_pool;
+       alloc_pool_t hlink_pool;
        struct file_struct **files;
 };