A resumed partial-dir file is transferred in-place.

Fixed bug #13071.

diff --git a/NEWS b/NEWS
index 5e9d356f6eef492de295cbc77ef90f723e696807..d2043baeef1053ca81a898b44373bd6d06d9223a 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -28,6 +28,8 @@ Changes since 3.1.3:
 
     - Some rrsync fixes and enhancements to handle the latest options.
 
+    - Fixed a crash in the --iconv code.
+
   ENHANCEMENTS:
 
     - Improved the --atimes patch and promoted it to be in the release.
@@ -49,11 +51,13 @@ Changes since 3.1.3:
       rsync commands into/from untrusted directories (such as backups and
       restores).
 
+    - When resuming the transfer of a file in the --partial-dir, rsync will now
+      update that partial file in-place instead of creating yet another tmp
+      file copy.  This requires both sender & receiver to be at least v3.2.0.
+
     - Added support for RSYNC_SHELL & RSYNC_NO_XFER_EXEC environment variables
       that affect the pre-xfer exec and post-xfer exec rsync daemon options.
 
-    - Fixed a crash in the --iconv code.
-
     - Fixed a problem with the --link-dest|--copy-dest code when --xattrs was
       specified along with multiple alternate-destination directories (it could
       possibly choose a bad file match while trying to find a better xattr

diff --git a/compat.c b/compat.c
index 40de14bb5c6716c045cca25e18c7c28d6f1391e7..13a3b9f9832c7838f0237dffb7ebe17115f6c58e 100644 (file)
--- a/compat.c
+++ b/compat.c
@@ -28,6 +28,7 @@ int compat_flags = 0;
 int use_safe_inc_flist = 0;
 int want_xattr_optim = 0;
 int proper_seed_order = 0;
+int inplace_partial = 0;
 
 extern int am_server;
 extern int am_sender;
@@ -81,6 +82,7 @@ int filesfrom_convert = 0;
 #define CF_SAFE_FLIST   (1<<3)
 #define CF_AVOID_XATTR_OPTIM (1<<4)
 #define CF_CHKSUM_SEED_FIX (1<<5)
+#define CF_INPLACE_PARTIAL_DIR (1<<6)
 
 static const char *client_info;
 
@@ -283,6 +285,8 @@ void setup_protocol(int f_out,int f_in)
                                compat_flags |= CF_AVOID_XATTR_OPTIM;
                        if (local_server || strchr(client_info, 'C') != NULL)
                                compat_flags |= CF_CHKSUM_SEED_FIX;
+                       if (local_server || strchr(client_info, 'I') != NULL)
+                               compat_flags |= CF_INPLACE_PARTIAL_DIR;
                        write_byte(f_out, compat_flags);
                } else
                        compat_flags = read_byte(f_in);
@@ -313,6 +317,8 @@ void setup_protocol(int f_out,int f_in)
                }
                use_safe_inc_flist = (compat_flags & CF_SAFE_FLIST) || protocol_version >= 31;
                need_messages_from_generator = 1;
+               if (compat_flags & CF_INPLACE_PARTIAL_DIR)
+                       inplace_partial = 1;
 #ifdef CAN_SET_SYMLINK_TIMES
        } else if (!am_sender) {
                receiver_symlink_times = 1;

diff --git a/options.c b/options.c
index 2319b14d161129fe303e1cbf06b3a86b4e7a0985..b2cbda53ba065b6114107afd0a26e2b32d8b92ff 100644 (file)
--- a/options.c
+++ b/options.c
@@ -2577,6 +2577,7 @@ void server_options(char **args, int *argc_p)
                eFlags[x++] = 'f'; /* flist I/O-error safety support */
                eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */
                eFlags[x++] = 'C'; /* support checksum seed order fix */
+               eFlags[x++] = 'I'; /* support inplace_partial behavior */
 #undef eFlags
        }
 

diff --git a/receiver.c b/receiver.c
index afc8932cbcbaf5d8db30e53ea1963459750ba49d..e95f1786d75a510677c37884acf9a0bb425043bd 100644 (file)
--- a/receiver.c
+++ b/receiver.c
@@ -52,6 +52,7 @@ extern int keep_partial;
 extern int checksum_seed;
 extern int whole_file;
 extern int inplace;
+extern int inplace_partial;
 extern int allowed_lull;
 extern int delay_updates;
 extern int xfersum_type;
@@ -69,7 +70,7 @@ extern OFF_T preallocated_len;
 static struct bitbag *delayed_bits = NULL;
 static int phase = 0, redoing = 0;
 static flist_ndx_list batch_redo_list;
-/* We're either updating the basis file or an identical copy: */
+/* This is non-0 when we are updating the basis file or an identical copy: */
 static int updating_basis_or_equiv;
 
 #define TMPNAME_SUFFIX ".XXXXXX"
@@ -233,7 +234,7 @@ int open_tmpfile(char *fnametmp, const char *fname, struct file_struct *file)
 }
 
 static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
-                       const char *fname, int fd, struct file_struct *file)
+                       const char *fname, int fd, struct file_struct *file, int inplace_sizing)
 {
        static char file_sum1[MAX_DIGEST_LEN];
        struct map_struct *mapbuf;
@@ -248,14 +249,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
        char *map = NULL;
 
 #ifdef SUPPORT_PREALLOCATION
-       if (preallocate_files && fd != -1 && total_size > 0 && (!inplace || total_size > size_r)) {
+       if (preallocate_files && fd != -1 && total_size > 0 && (!inplace_sizing || total_size > size_r)) {
                /* Try to preallocate enough space for file's eventual length.  Can
                 * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */
                if ((preallocated_len = do_fallocate(fd, 0, total_size)) < 0)
                        rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname));
        } else
 #endif
-       if (inplace) {
+       if (inplace_sizing) {
 #ifdef HAVE_FTRUNCATE
                /* The most compatible way to create a sparse file is to start with no length. */
                if (sparse_files > 0 && whole_file && fd >= 0 && do_ftruncate(fd, 0) == 0)
@@ -382,9 +383,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
        /* inplace: New data could be shorter than old data.
         * preallocate_files: total_size could have been an overestimate.
         *     Cut off any extra preallocated zeros from dest file. */
-       if ((inplace || preallocated_len > offset) && fd != -1 && !IS_DEVICE(file->mode) && do_ftruncate(fd, offset) < 0) {
-               rsyserr(FERROR_XFER, errno, "ftruncate failed on %s",
-                       full_fname(fname));
+       if ((inplace_sizing || preallocated_len > offset) && fd != -1 && !IS_DEVICE(file->mode)) {
+               if (do_ftruncate(fd, offset) < 0)
+                       rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", full_fname(fname));
        }
 #endif
 
@@ -407,7 +408,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 
 static void discard_receive_data(int f_in, struct file_struct *file)
 {
-       receive_data(f_in, NULL, -1, 0, NULL, -1, file);
+       receive_data(f_in, NULL, -1, 0, NULL, -1, file, 0);
 }
 
 static void handle_delayed_updates(char *local_name)
@@ -518,7 +519,7 @@ int recv_files(int f_in, int f_out, char *local_name)
        int iflags, xlen;
        char *fname, fbuf[MAXPATHLEN];
        char xname[MAXPATHLEN];
-       char fnametmp[MAXPATHLEN];
+       char *fnametmp, fnametmpbuf[MAXPATHLEN];
        char *fnamecmp, *partialptr;
        char fnamecmpbuf[MAXPATHLEN];
        uchar fnamecmp_type;
@@ -530,7 +531,7 @@ int recv_files(int f_in, int f_out, char *local_name)
 #ifdef SUPPORT_ACLS
        const char *parent_dirname = "";
 #endif
-       int ndx, recv_ok;
+       int ndx, recv_ok, one_inplace;
 
        if (DEBUG_GTE(RECV, 1))
                rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used);
@@ -752,6 +753,7 @@ int recv_files(int f_in, int f_out, char *local_name)
                if (fd1 == -1 && protocol_version < 29) {
                        if (fnamecmp != fname) {
                                fnamecmp = fname;
+                               fnamecmp_type = FNAMECMP_FNAME;
                                fd1 = do_open(fnamecmp, O_RDONLY, 0);
                        }
 
@@ -760,12 +762,14 @@ int recv_files(int f_in, int f_out, char *local_name)
                                pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
                                         basis_dir[0], fname);
                                fnamecmp = fnamecmpbuf;
+                               fnamecmp_type = FNAMECMP_BASIS_DIR_LOW;
                                fd1 = do_open(fnamecmp, O_RDONLY, 0);
                        }
                }
 
-               updating_basis_or_equiv = inplace
-                   && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP);
+               one_inplace = inplace_partial && fnamecmp_type == FNAMECMP_PARTIAL_DIR;
+               updating_basis_or_equiv = one_inplace
+                   || (inplace && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP));
 
                if (fd1 == -1) {
                        st.st_mode = 0;
@@ -831,14 +835,16 @@ int recv_files(int f_in, int f_out, char *local_name)
                }
 
                /* We now check to see if we are writing the file "inplace" */
-               if (inplace)  {
-                       fd2 = do_open(fname, O_WRONLY|O_CREAT, 0600);
+               if (inplace || one_inplace)  {
+                       fnametmp = one_inplace ? partialptr : fname;
+                       fd2 = do_open(fnametmp, O_WRONLY|O_CREAT, 0600);
                        if (fd2 == -1) {
                                rsyserr(FERROR_XFER, errno, "open %s failed",
-                                       full_fname(fname));
+                                       full_fname(fnametmp));
                        } else if (updating_basis_or_equiv)
                                cleanup_set(NULL, NULL, file, fd1, fd2);
                } else {
+                       fnametmp = fnametmpbuf;
                        fd2 = open_tmpfile(fnametmp, fname, file);
                        if (fd2 != -1)
                                cleanup_set(fnametmp, partialptr, file, fd1, fd2);
@@ -860,7 +866,7 @@ int recv_files(int f_in, int f_out, char *local_name)
                        rprintf(FINFO, "%s\n", fname);
 
                /* recv file data */
-               recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, file);
+               recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, file, inplace || one_inplace);
 
                log_item(log_code, file, iflags, NULL);
                if (want_progress_now)
@@ -881,10 +887,11 @@ int recv_files(int f_in, int f_out, char *local_name)
                                             partialptr, file, recv_ok, 1))
                                recv_ok = -1;
                        else if (fnamecmp == partialptr) {
-                               do_unlink(partialptr);
+                               if (!one_inplace)
+                                       do_unlink(partialptr);
                                handle_partial_dir(partialptr, PDIR_DELETE);
                        }
-               } else if (keep_partial && partialptr) {
+               } else if (keep_partial && partialptr && !one_inplace) {
                        if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
                                rprintf(FERROR,
                                    "Unable to create partial-dir for %s -- discarding %s.\n",
@@ -900,7 +907,7 @@ int recv_files(int f_in, int f_out, char *local_name)
                                recv_ok = 2;
                        } else
                                partialptr = NULL;
-               } else
+               } else if (!one_inplace)
                        do_unlink(fnametmp);
 
                cleanup_disable();

diff --git a/rsync.yo b/rsync.yo
index 59d842650949a20b974a57b31a0e9b973a7f69b1..b1e4ec3da2705ca7263ec5bdda5ce0d48936d859 100644 (file)
--- a/rsync.yo
+++ b/rsync.yo
@@ -2459,7 +2459,9 @@ Rsync will create the em(DIR) if it is missing (just the last dir -- not
 the whole path).  This makes it easy to use a relative path (such as
 "bf(--partial-dir=.rsync-partial)") to have rsync create the
 partial-directory in the destination file's directory when needed, and then
-remove it again when the partial file is deleted.
+remove it again when the partial file is deleted.  Note that the directory
+is only removed if it is a relative pathname, as it is expected that an
+absolute path is to a directory that is reserved for partial-dir work.
 
 If the partial-dir value is not an absolute path, rsync will add an exclude
 rule at the end of all your existing excludes.  This will prevent the
@@ -2493,6 +2495,11 @@ option does not look for this environment value are (1) when bf(--inplace) was
 specified (since bf(--inplace) conflicts with bf(--partial-dir)), and (2) when
 bf(--delay-updates) was specified (see below).
 
+When a modern rsync resumes the transfer of a file in the partial-dir, that
+partial file is now updated in-place instead of creating yet another tmp-file
+copy (so it maxes out at dest + tmp instead of dest + partial + tmp).  This
+requires both ends of the transfer to be at least version 3.2.0.
+
 For the purposes of the daemon-config's "refuse options" setting,
 bf(--partial-dir) does em(not) imply bf(--partial).  This is so that a
 refusal of the bf(--partial) option can be used to disallow the overwriting

diff --git a/sender.c b/sender.c
index 4f34cded30fd0ed84b3a5ab51abae2deae4d9bac..51feec72065e976e742fa39c7c0071b472338a7b 100644 (file)
--- a/sender.c
+++ b/sender.c
@@ -42,6 +42,7 @@ extern int remove_source_files;
 extern int updating_basis_file;
 extern int make_backups;
 extern int inplace;
+extern int inplace_partial;
 extern int batch_fd;
 extern int write_batch;
 extern int file_old_total;
@@ -316,8 +317,8 @@ void send_files(int f_in, int f_out)
                                stats.created_files++;
                }
 
-               updating_basis_file = inplace && (protocol_version >= 29
-                       ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0);
+               updating_basis_file = (inplace_partial && fnamecmp_type == FNAMECMP_PARTIAL_DIR)
+                   || (inplace && (protocol_version >= 29 ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0));
 
                if (!am_server)
                        set_current_file_index(file, ndx);

diff --git a/util.c b/util.c
index 03917c3161244142bf6ee880b5da2b843baf0f0d..edb3ae03b310af2d27b410a79a5049f200a80535 100644 (file)
--- a/util.c
+++ b/util.c
@@ -498,6 +498,11 @@ int robust_rename(const char *from, const char *to, const char *partialptr,
 {
        int tries = 4;
 
+       /* A resumed in-place partial-dir transfer might call us with from and
+        * to pointing to the same buf if the transfer failed yet again. */
+       if (from == to)
+               return 0;
+
        while (tries--) {
                if (do_rename(from, to) == 0)
                        return 0;