AC_DEFINE(HAVE_FALLOCATE, 1, [Define to 1 if you have the fallocate function and it compiles and links without error])
fi
+AC_MSG_CHECKING([for FALLOC_FL_PUNCH_HOLE])
+AC_PREPROC_IFELSE([AC_LANG_SOURCE([[
+ #define _GNU_SOURCE 1
+ #include <linux/falloc.h>
+ #ifndef FALLOC_FL_PUNCH_HOLE
+ #error FALLOC_FL_PUNCH_HOLE is missing
+ #endif
+ ]])], [
+ AC_MSG_RESULT([yes])
+ AC_DEFINE([HAVE_FALLOC_FL_PUNCH_HOLE], [1], [Define if FALLOC_FL_PUNCH_HOLE is available.])
+ ], [
+ AC_MSG_RESULT([no])
+ ]
+)
+
+AC_MSG_CHECKING([for FALLOC_FL_ZERO_RANGE])
+AC_PREPROC_IFELSE([AC_LANG_SOURCE([[
+ #define _GNU_SOURCE 1
+ #include <linux/falloc.h>
+ #ifndef FALLOC_FL_ZERO_RANGE
+ #error FALLOC_FL_ZERO_RANGE is missing
+ #endif
+ ]])], [
+ AC_MSG_RESULT([yes])
+ AC_DEFINE([HAVE_FALLOC_FL_ZERO_RANGE], [1], [Define if FALLOC_FL_ZERO_RANGE is available.])
+ ], [
+ AC_MSG_RESULT([no])
+ ]
+)
+
AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/syscall.h>
#include <sys/types.h>]], [[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);]])],[rsync_cv_have_sys_fallocate=yes],[rsync_cv_have_sys_fallocate=no])])
extern int sparse_files;
+OFF_T preallocated_len = 0;
+
static OFF_T sparse_seek = 0;
+static OFF_T sparse_past_write = 0;
int sparse_end(int f, OFF_T size)
{
return ret;
}
-
-static int write_sparse(int f, char *buf, int len)
+/* Note that the offset is just the caller letting us know where
+ * the current file position is in the file. The use_seek arg tells
+ * us that we should seek over matching data instead of writing it. */
+static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int len)
{
int l1 = 0, l2 = 0;
int ret;
if (l1 == len)
return len;
- if (sparse_seek)
- do_lseek(f, sparse_seek, SEEK_CUR);
+ if (sparse_seek) {
+ if (sparse_past_write >= preallocated_len) {
+ if (do_lseek(f, sparse_seek, SEEK_CUR) < 0)
+ return -1;
+ } else if (do_punch_hole(f, sparse_past_write, sparse_seek) < 0) {
+ sparse_seek = 0;
+ return -1;
+ }
+ }
sparse_seek = l2;
+ sparse_past_write = offset + len - l2;
+
+ if (use_seek) {
+ /* The in-place data already matches. */
+ if (do_lseek(f, len - (l1+l2), SEEK_CUR) < 0)
+ return -1;
+ return len;
+ }
while ((ret = write(f, buf + l1, len - (l1+l2))) <= 0) {
if (ret < 0 && errno == EINTR)
return len;
}
-
static char *wf_writeBuf;
static size_t wf_writeBufSize;
static size_t wf_writeBufCnt;
return ret;
}
-
-/*
- * write_file does not allow incomplete writes. It loops internally
- * until len bytes are written or errno is set.
- */
-int write_file(int f, char *buf, int len)
+/* write_file does not allow incomplete writes. It loops internally
+ * until len bytes are written or errno is set. Note that use_seek and
+ * offset are only used in sparse processing (see write_sparse()). */
+int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len)
{
int ret = 0;
int r1;
if (sparse_files > 0) {
int len1 = MIN(len, SPARSE_WRITE_SIZE);
- r1 = write_sparse(f, buf, len1);
+ r1 = write_sparse(f, use_seek, offset, buf, len1);
+ offset += r1;
} else {
if (!wf_writeBuf) {
wf_writeBufSize = WRITE_SIZE * 8;
return ret;
}
+/* An in-place update found identical data at an identical location. We either
+ * just seek past it, or (for an in-place sparse update), we give the data to
+ * the sparse processor with the use_seek flag set. */
+int skip_matched(int fd, OFF_T offset, const char *buf, int len)
+{
+ OFF_T pos;
+
+ if (sparse_files > 0) {
+ if (write_file(fd, 1, offset, buf, len) != len)
+ return -1;
+ return 0;
+ }
+
+ if (flush_write_file(fd) < 0)
+ return -1;
+
+ if ((pos = do_lseek(fd, len, SEEK_CUR)) != offset + len) {
+ rsyserr(FERROR_XFER, errno, "lseek returned %s, not %s",
+ big_num(pos), big_num(offset));
+ return -1;
+ }
+
+ return 0;
+}
/* This provides functionality somewhat similar to mmap() but using read().
* It gives sliding window access to a file. mmap() is not used because of
return map->p + align_fudge;
}
-
int unmap_file(struct map_struct *map)
{
int ret;
#ifdef SUPPORT_XATTRS
rprintf(F," --fake-super store/recover privileged attrs using xattrs\n");
#endif
- rprintf(F," -S, --sparse handle sparse files efficiently\n");
+ rprintf(F," -S, --sparse turn sequences of nulls into sparse blocks\n");
#ifdef SUPPORT_PREALLOCATION
rprintf(F," --preallocate allocate dest files before writing them\n");
#else
bwlimit_writemax = 512;
}
- if (sparse_files && inplace) {
- /* Note: we don't check for this below, because --append is
- * OK with --sparse (as long as redos are handled right). */
- snprintf(err_buf, sizeof err_buf,
- "--sparse cannot be used with --inplace\n");
- return 0;
- }
-
if (append_mode) {
if (whole_file > 0) {
snprintf(err_buf, sizeof err_buf,
extern int preallocate_files;
extern int keep_partial;
extern int checksum_seed;
+extern int whole_file;
extern int inplace;
extern int allowed_lull;
extern int delay_updates;
extern char sender_file_sum[MAX_DIGEST_LEN];
extern struct file_list *cur_flist, *first_flist, *dir_flist;
extern filter_rule_list daemon_filter_list;
+#ifdef SUPPORT_PREALLOCATION
+extern OFF_T preallocated_len;
+#endif
static struct bitbag *delayed_bits = NULL;
static int phase = 0, redoing = 0;
char *data;
int32 i;
char *map = NULL;
-#ifdef SUPPORT_PREALLOCATION
-#ifdef PREALLOCATE_NEEDS_TRUNCATE
- OFF_T preallocated_len = 0;
-#endif
+#ifdef SUPPORT_PREALLOCATION
if (preallocate_files && fd != -1 && total_size > 0 && (!inplace || total_size > size_r)) {
/* Try to preallocate enough space for file's eventual length. Can
* reduce fragmentation on filesystems like ext4, xfs, and NTFS. */
- if (do_fallocate(fd, 0, total_size) == 0) {
-#ifdef PREALLOCATE_NEEDS_TRUNCATE
- preallocated_len = total_size;
-#endif
- } else
+ if ((preallocated_len = do_fallocate(fd, 0, total_size)) < 0)
rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname));
- }
+ } else
+#endif
+ if (inplace) {
+#ifdef HAVE_FTRUNCATE
+ /* The most compatible way to create a sparse file is to start with no length. */
+ if (sparse_files > 0 && whole_file && fd >= 0 && do_ftruncate(fd, 0) == 0)
+ preallocated_len = 0;
+ else
#endif
+ preallocated_len = size_r;
+ } else
+ preallocated_len = 0;
read_sum_head(f_in, &sum);
sum_update(data, i);
- if (fd != -1 && write_file(fd,data,i) != i)
+ if (fd != -1 && write_file(fd, 0, offset, data, i) != i)
goto report_write_error;
offset += i;
continue;
if (updating_basis_or_equiv) {
if (offset == offset2 && fd != -1) {
- OFF_T pos;
- if (flush_write_file(fd) < 0)
+ if (skip_matched(fd, offset, map, len) < 0)
goto report_write_error;
offset += len;
- if ((pos = do_lseek(fd, len, SEEK_CUR)) != offset) {
- rsyserr(FERROR_XFER, errno,
- "lseek of %s returned %s, not %s",
- full_fname(fname),
- big_num(pos), big_num(offset));
- exit_cleanup(RERR_FILEIO);
- }
continue;
}
}
- if (fd != -1 && map && write_file(fd, map, len) != (int)len)
+ if (fd != -1 && map && write_file(fd, 0, offset, map, len) != (int)len)
goto report_write_error;
offset += len;
}
- if (flush_write_file(fd) < 0)
- goto report_write_error;
+ if (fd != -1 && offset > 0) {
+ if (sparse_files > 0) {
+ if (sparse_end(fd, offset) != 0)
+ goto report_write_error;
+ } else if (flush_write_file(fd) < 0) {
+ report_write_error:
+ rsyserr(FERROR_XFER, errno, "write failed on %s", full_fname(fname));
+ exit_cleanup(RERR_FILEIO);
+ }
+ }
#ifdef HAVE_FTRUNCATE
/* inplace: New data could be shorter than old data.
* preallocate_files: total_size could have been an overestimate.
* Cut off any extra preallocated zeros from dest file. */
- if ((inplace
-#ifdef PREALLOCATE_NEEDS_TRUNCATE
- || preallocated_len > offset
-#endif
- ) && fd != -1 && do_ftruncate(fd, offset) < 0) {
+ if ((inplace || preallocated_len > offset) && fd != -1 && do_ftruncate(fd, offset) < 0) {
rsyserr(FERROR_XFER, errno, "ftruncate failed on %s",
full_fname(fname));
}
if (INFO_GTE(PROGRESS, 1))
end_progress(total_size);
- if (fd != -1 && offset > 0 && sparse_end(fd, offset) != 0) {
- report_write_error:
- rsyserr(FERROR_XFER, errno, "write failed on %s",
- full_fname(fname));
- exit_cleanup(RERR_FILEIO);
- }
-
checksum_len = sum_end(file_sum1);
if (mapbuf)
-J, --omit-link-times omit symlinks from --times
--super receiver attempts super-user activities
--fake-super store/recover privileged attrs using xattrs
- -S, --sparse handle sparse files efficiently
+ -S, --sparse turn sequences of nulls into sparse blocks
--preallocate allocate dest files before writing
-n, --dry-run perform a trial run with no changes made
-W, --whole-file copy files whole (w/o delta-xfer algorithm)
does not interfere with the updating of a file's non-content attributes
(e.g. permissions, ownership, etc.) when the file does not need to be
transferred, nor does it affect the updating of any non-regular files.
-Implies bf(--inplace),
-but does not conflict with bf(--sparse) (since it is always extending a
-file's length).
+Implies bf(--inplace).
The use of bf(--append) can be dangerous if you aren't 100% sure that the files
that are longer have only grown by the appending of data onto the end. You
See also the "fake super" setting in the daemon's rsyncd.conf file.
dit(bf(-S, --sparse)) Try to handle sparse files efficiently so they take
-up less space on the destination. Conflicts with bf(--inplace) because it's
-not possible to overwrite data in a sparse fashion.
+up less space on the destination. If combined with bf(--inplace) the
+file created might not end up with sparse blocks with some combinations
+of kernel version and/or filesystem type. If bf(--whole-file) is in
+effect (e.g. for a local copy) then it will always work because rsync
+truncates the file prior to writing out the updated version.
+
+Note that versions of rsync older than 3.1.3 will reject the combination of
+bf(--sparse) and bf(--inplace).
dit(bf(--preallocate)) This tells the receiver to allocate each destination
-file to its eventual size before writing data to the file. Rsync will only use
-the real filesystem-level preallocation support provided by Linux's
+file to its eventual size before writing data to the file. Rsync will only
+use the real filesystem-level preallocation support provided by Linux's
bf(fallocate)(2) system call or Cygwin's bf(posix_fallocate)(3), not the slow
-glibc implementation that writes a zero byte into each block.
+glibc implementation that writes a null byte into each block.
Without this option, larger files may not be entirely contiguous on the
filesystem, but with this option rsync will probably copy more slowly. If the
destination is not an extent-supporting filesystem (such as ext4, xfs, NTFS,
etc.), this option may have no positive effect at all.
+If combined with bf(--sparse), the file will only have sparse blocks (as
+opposed to allocated sequences of null bytes) if the kernel version and
+filesystem type support creating holes in the allocated data.
+
dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't
make any changes (and produces mostly the same output as a real run). It
is most commonly used in combination with the bf(-v, --verbose) and/or
extern int am_sender;
extern int read_only;
extern int list_only;
+extern int inplace;
+extern int preallocate_files;
extern int preserve_perms;
extern int preserve_executability;
#endif
#ifdef SUPPORT_PREALLOCATION
-int do_fallocate(int fd, OFF_T offset, OFF_T length)
-{
#ifdef FALLOC_FL_KEEP_SIZE
#define DO_FALLOC_OPTIONS FALLOC_FL_KEEP_SIZE
#else
#define DO_FALLOC_OPTIONS 0
#endif
+
+OFF_T do_fallocate(int fd, OFF_T offset, OFF_T length)
+{
+ int opts = inplace || preallocate_files ? 0 : DO_FALLOC_OPTIONS;
+ int ret;
RETURN_ERROR_IF(dry_run, 0);
RETURN_ERROR_IF_RO_OR_LO;
+ if (length & 1) /* make the length not match the desired length */
+ length++;
+ else
+ length--;
#if defined HAVE_FALLOCATE
- return fallocate(fd, DO_FALLOC_OPTIONS, offset, length);
+ ret = fallocate(fd, opts, offset, length);
#elif defined HAVE_SYS_FALLOCATE
- return syscall(SYS_fallocate, fd, DO_FALLOC_OPTIONS, (loff_t)offset, (loff_t)length);
+ ret = syscall(SYS_fallocate, fd, opts, (loff_t)offset, (loff_t)length);
#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE
- return posix_fallocate(fd, offset, length);
+ ret = posix_fallocate(fd, offset, length);
#else
#error Coding error in SUPPORT_PREALLOCATION logic.
#endif
+ if (ret < 0)
+ return ret;
+ if (opts == 0) {
+ STRUCT_STAT st;
+ if (do_fstat(fd, &st) < 0)
+ return length;
+ return st.st_blocks * 512;
+ }
+ return 0;
}
#endif
+/* Punch a hole at pos for len bytes. The current file position must be at pos and will be
+ * changed to be at pos + len. */
+int do_punch_hole(int fd, UNUSED(OFF_T pos), int len)
+{
+#ifdef HAVE_FALLOCATE
+# ifdef HAVE_FALLOC_FL_PUNCH_HOLE
+ if (fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, pos, len) == 0) {
+ if (do_lseek(fd, len, SEEK_CUR) != pos + len)
+ return -1;
+ return 0;
+ }
+# endif
+# ifdef HAVE_FALLOC_FL_ZERO_RANGE
+ if (fallocate(fd, FALLOC_FL_ZERO_RANGE, pos, len) == 0) {
+ if (do_lseek(fd, len, SEEK_CUR) != pos + len)
+ return -1;
+ return 0;
+ }
+# endif
+#endif
+ {
+ char zeros[4096];
+ memset(zeros, 0, sizeof zeros);
+ while (len > 0) {
+ int chunk = len > (int)sizeof zeros ? (int)sizeof zeros : len;
+ int wrote = write(fd, zeros, chunk);
+ if (wrote <= 0) {
+ if (wrote < 0 && errno == EINTR)
+ continue;
+ return -1;
+ }
+ len -= wrote;
+ }
+ }
+ return 0;
+}
+
int do_open_nofollow(const char *pathname, int flags)
{
#ifndef O_NOFOLLOW
#include "rsync.h"
+int inplace = 0;
int modify_window = 0;
int preallocate_files = 0;
int protect_args = 0;
int nsec_times = 0;
int preserve_perms = 0;
int preserve_executability = 0;
+int preallocate_files = 0;
+int inplace = 0;
#ifdef SUPPORT_XATTRS
int list_only = 0;
int preserve_perms = 0;
int preserve_executability = 0;
+int preallocate_files = 0;
+int inplace = 0;
int
main(int argc, char **argv)
int ifd;
char buf[1024 * 8];
int len; /* Number of bytes read into `buf'. */
-#ifdef PREALLOCATE_NEEDS_TRUNCATE
- OFF_T preallocated_len = 0, offset = 0;
-#endif
+ OFF_T prealloc_len = 0, offset = 0;
if ((ifd = do_open(source, O_RDONLY, 0)) < 0) {
int save_errno = errno;
if (do_fstat(ifd, &srcst) < 0)
rsyserr(FWARNING, errno, "fstat %s", full_fname(source));
else if (srcst.st_size > 0) {
- if (do_fallocate(ofd, 0, srcst.st_size) == 0) {
-#ifdef PREALLOCATE_NEEDS_TRUNCATE
- preallocated_len = srcst.st_size;
-#endif
- } else
+ prealloc_len = do_fallocate(ofd, 0, srcst.st_size);
+ if (prealloc_len < 0)
rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest));
}
}
errno = save_errno;
return -1;
}
-#ifdef PREALLOCATE_NEEDS_TRUNCATE
offset += len;
-#endif
}
if (len < 0) {
full_fname(source));
}
-#ifdef PREALLOCATE_NEEDS_TRUNCATE
/* Source file might have shrunk since we fstatted it.
* Cut off any extra preallocated zeros from dest file. */
- if (offset < preallocated_len && do_ftruncate(ofd, offset) < 0) {
+ if (offset < prealloc_len && do_ftruncate(ofd, offset) < 0) {
/* If we fail to truncate, the dest file may be wrong, so we
* must trigger the "partial transfer" error. */
rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest));
}
-#endif
if (close(ofd) < 0) {
int save_errno = errno;