1 This builds on the checksum-reading patch and adds the ability to
2 create and/or update the .rsyncsums files using extended mode args to
3 the --sumfiles=MODE option and the "checksum files = MODE" daemon
6 CAUTION: This patch is only lightly tested. If you're interested
7 in using it, please help out.
9 To use this patch, run these commands for a successful build:
11 patch -p1 <patches/checksum-reading.diff
12 patch -p1 <patches/checksum-updating.diff
13 ./configure (optional if already run)
18 - Fix the code that removes .rsyncsums files when a dir becomes empty.
20 diff --git a/flist.c b/flist.c
21 index 8a42d80..d65c475 100644
32 @@ -99,6 +100,9 @@ extern iconv_t ic_send, ic_recv;
34 #define PTR_SIZE (sizeof (struct file_struct *))
36 +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
37 +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
41 dev_t filesystem_dev; /* used to implement -x */
42 @@ -137,8 +141,13 @@ static char tmp_sum[MAX_DIGEST_LEN];
43 static char empty_sum[MAX_DIGEST_LEN];
44 static int flist_count_offset; /* for --delete --progress */
46 +#define REGULAR_SKIPPED(flist) ((flist)->to_redo)
48 static struct csum_cache {
49 struct file_list *flist;
50 + const char *dirname;
51 + int checksum_matches;
52 + int checksum_updates;
55 static void flist_sort_and_clean(struct file_list *flist, int flags);
56 @@ -356,7 +365,79 @@ static void flist_done_allocating(struct file_list *flist)
57 flist->pool_boundary = ptr;
60 -void reset_checksum_cache()
61 +static void checksum_filename(int slot, const char *dirname, char *fbuf)
63 + if (dirname && *dirname) {
66 + len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN);
67 + if (len >= MAXPATHLEN)
71 + if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len)
74 + strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN);
77 +static void write_checksums(int slot, struct file_list *flist, int whole_dir)
81 + char fbuf[MAXPATHLEN];
82 + int new_entries = csum_cache[slot].checksum_updates != 0;
83 + int counts_match = flist->used == csum_cache[slot].checksum_matches;
84 + int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0;
85 + const char *dirname = csum_cache[slot].dirname;
87 + flist_sort_and_clean(flist, 0);
89 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
92 + checksum_filename(slot, dirname, fbuf);
94 + if (flist->high - flist->low < 0 && no_skipped) {
99 + if (!new_entries && (counts_match || !whole_dir))
102 + if (!(out_fp = fopen(fbuf, "w")))
105 + for (i = flist->low; i <= flist->high; i++) {
106 + struct file_struct *file = flist->sorted[i];
107 + const char *cp = F_SUM(file);
108 + const char *end = cp + checksum_len;
109 + const char *alt_sum = file->basename + strlen(file->basename) + 1;
110 + if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
112 + if (protocol_version >= 30)
113 + fprintf(out_fp, "%s ", alt_sum);
114 + if (file->flags & FLAG_SUM_MISSING) {
116 + fputs("==", out_fp);
117 + } while (++cp != end);
120 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
121 + } while (++cp != end);
123 + if (protocol_version < 30)
124 + fprintf(out_fp, " %s", alt_sum);
125 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
126 + (double)F_LENGTH(file), (double)file->modtime,
127 + (long)F_CTIME(file), (long)F_INODE(file), file->basename);
133 +void reset_checksum_cache(int whole_dir)
135 int slot, slots = am_sender ? 1 : basis_dir_cnt + 1;
137 @@ -370,6 +451,9 @@ void reset_checksum_cache()
138 struct file_list *flist = csum_cache[slot].flist;
141 + if (checksum_files & CSF_UPDATE && flist->next)
142 + write_checksums(slot, flist, whole_dir);
144 /* Reset the pool memory and empty the file-list array. */
145 pool_free_old(flist->file_pool,
146 pool_boundary(flist->file_pool, 0));
147 @@ -380,6 +464,10 @@ void reset_checksum_cache()
152 + csum_cache[slot].checksum_matches = 0;
153 + csum_cache[slot].checksum_updates = 0;
154 + REGULAR_SKIPPED(flist) = 0;
158 @@ -387,7 +475,7 @@ void reset_checksum_cache()
159 static int add_checksum(struct file_list *flist, const char *dirname,
160 const char *basename, int basename_len, OFF_T file_length,
161 time_t mtime, uint32 ctime, uint32 inode,
163 + const char *sum, const char *alt_sum, int flags)
165 struct file_struct *file;
166 int alloc_len, extra_len;
167 @@ -404,7 +492,7 @@ static int add_checksum(struct file_list *flist, const char *dirname,
168 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
169 extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
171 - alloc_len = FILE_STRUCT_LEN + extra_len + basename_len;
172 + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + checksum_len*2 + 1;
173 bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum");
175 memset(bp, 0, extra_len + FILE_STRUCT_LEN);
176 @@ -413,7 +501,14 @@ static int add_checksum(struct file_list *flist, const char *dirname,
177 bp += FILE_STRUCT_LEN;
179 memcpy(bp, basename, basename_len);
181 + strlcpy(bp+basename_len, alt_sum, checksum_len*2 + 1);
183 + memset(bp+basename_len, '=', checksum_len*2);
184 + bp[basename_len+checksum_len*2] = '\0';
187 + file->flags = flags;
188 file->mode = S_IFREG;
189 file->modtime = mtime;
190 file->len32 = (uint32)file_length;
191 @@ -442,10 +537,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
192 char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
200 + const char *alt_sum = NULL;
201 int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0;
203 if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN))
204 @@ -466,7 +562,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
205 while (fgets(line, sizeof line, fp)) {
207 if (protocol_version >= 30) {
208 - char *alt_sum = cp;
211 while (*++cp == '=') {}
213 @@ -477,7 +573,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
218 + for (i = 0; i < checksum_len*2; i++, cp++) {
224 + memset(sum, 0, checksum_len);
225 + flags = FLAG_SUM_MISSING;
227 for (i = 0; i < checksum_len*2; i++, cp++) {
229 @@ -495,13 +598,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
237 while (*++cp == ' ') {}
239 if (protocol_version < 30) {
240 - char *alt_sum = cp;
243 while (*++cp == '=') {}
245 @@ -551,24 +655,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
248 strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
249 + if (is_excluded(fbuf, 0, ALL_FILTERS)) {
250 + flags |= FLAG_SUM_KEEP;
251 + csum_cache[slot].checksum_matches++;
254 add_checksum(flist, dirname, cp, len, file_length,
257 + sum, alt_sum, flags);
261 flist_sort_and_clean(flist, CLEAN_KEEP_LAST);
264 +void set_cached_checksum(struct file_list *file_flist, struct file_struct *file)
269 + char fbuf[MAXPATHLEN];
270 + const char *fn = f_name(file, NULL);
271 + struct file_list *flist = csum_cache[0].flist;
273 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
276 + if (stat(fn, &st) < 0)
279 + checksum_filename(0, file->dirname, fbuf);
281 + if (file_flist != flist->next) {
282 + const char *cp = F_SUM(file);
283 + const char *end = cp + checksum_len;
285 + if (!(out_fp = fopen(fbuf, "a")))
288 + if (protocol_version >= 30) {
289 + for (j = 0; j < checksum_len; j++)
290 + fputs("==", out_fp);
291 + fputc(' ', out_fp);
294 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
295 + } while (++cp != end);
296 + if (protocol_version < 30) {
297 + fputc(' ', out_fp);
298 + for (j = 0; j < checksum_len; j++)
299 + fputs("==", out_fp);
301 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
302 + (double)st.st_size, (double)st.st_mtime,
303 + (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino,
310 + if ((j = flist_find(flist, file)) >= 0) {
311 + struct file_struct *fp = flist->sorted[j];
313 + if (F_LENGTH(fp) != st.st_size) {
314 + fp->len32 = (uint32)st.st_size;
315 + if (st.st_size > 0xFFFFFFFFu) {
316 + OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32);
317 + fp->flags |= FLAG_LENGTH64;
319 + fp->flags &= FLAG_LENGTH64;
322 + if (fp->modtime != st.st_mtime) {
323 + fp->modtime = st.st_mtime;
326 + if (F_CTIME(fp) != (uint32)st.st_ctime) {
327 + F_CTIME(fp) = (uint32)st.st_ctime;
330 + if (F_INODE(fp) != (uint32)st.st_ino) {
331 + F_INODE(fp) = (uint32)st.st_ino;
334 + memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN);
335 + csum_cache[0].checksum_updates += inc;
336 + fp->flags &= ~FLAG_SUM_MISSING;
337 + fp->flags |= FLAG_SUM_KEEP;
341 + csum_cache[0].checksum_updates +=
342 + add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1,
343 + st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime,
344 + st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP);
347 void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
348 - STRUCT_STAT *stp, char *sum_buf)
349 + int basename_len, STRUCT_STAT *stp, char *sum_buf)
351 struct file_list *flist = csum_cache[slot].flist;
355 flist->next = cur_flist; /* next points from checksum flist to file flist */
356 + csum_cache[slot].dirname = file->dirname;
357 read_checksums(slot, flist, file->dirname);
360 @@ -580,12 +772,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
361 && (checksum_files & CSF_LAX
362 || (F_CTIME(fp) == (uint32)stp->st_ctime
363 && F_INODE(fp) == (uint32)stp->st_ino))) {
364 - memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
365 + if (fp->flags & FLAG_SUM_MISSING) {
366 + fp->flags &= ~FLAG_SUM_MISSING;
367 + csum_cache[slot].checksum_updates++;
368 + file_checksum(fname, stp->st_size, sum_buf);
369 + memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN);
371 + csum_cache[slot].checksum_matches++;
372 + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
374 + fp->flags |= FLAG_SUM_KEEP;
380 file_checksum(fname, stp->st_size, sum_buf);
382 + if (checksum_files & CSF_UPDATE) {
383 + if (basename_len < 0)
384 + basename_len = strlen(file->basename) + 1;
385 + csum_cache[slot].checksum_updates +=
386 + add_checksum(flist, file->dirname, file->basename, basename_len,
387 + stp->st_size, stp->st_mtime, (uint32)stp->st_ctime,
388 + (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP);
392 /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
393 @@ -1452,6 +1663,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
394 if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
395 if (ignore_perishable)
396 non_perishable_cnt++;
397 + if (S_ISREG(st.st_mode))
398 + REGULAR_SKIPPED(flist)++;
402 @@ -1498,13 +1711,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
405 if (checksum_files && am_sender && flist)
406 - reset_checksum_cache();
407 + reset_checksum_cache(0);
411 if (checksum_files && am_sender && flist && lastdir_len == -2) {
413 - reset_checksum_cache();
414 + reset_checksum_cache(0);
417 basename_len = strlen(basename) + 1; /* count the '\0' */
418 @@ -1599,7 +1812,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
420 if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
421 if (flist && checksum_files)
422 - get_cached_checksum(0, thisname, file, &st, tmp_sum);
423 + get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum);
425 file_checksum(thisname, st.st_size, tmp_sum);
426 if (sender_keeps_checksum)
427 @@ -1971,6 +2184,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len,
431 + if (checksum_files & CSF_UPDATE && am_sender && f >= 0)
432 + reset_checksum_cache(1);
434 if (f >= 0 && recurse && !divert_dirs) {
435 int i, end = flist->used - 1;
436 /* send_if_directory() bumps flist->used, so use "end". */
437 @@ -2589,6 +2805,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
441 + if (checksum_files & CSF_UPDATE && flist_eof)
442 + reset_checksum_cache(0); /* writes any last updates */
447 diff --git a/generator.c b/generator.c
448 index 48a5062..8717ab7 100644
451 @@ -111,6 +111,7 @@ static int dir_tweaking;
452 static int symlink_timeset_failed_flags;
453 static int need_retouch_dir_times;
454 static int need_retouch_dir_perms;
455 +static int started_whole_dir, upcoming_whole_dir;
456 static const char *solo_file = NULL;
459 @@ -533,7 +534,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot
460 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
461 char sum[MAX_DIGEST_LEN];
462 if (checksum_files && slot >= 0)
463 - get_cached_checksum(slot, fn, file, st, sum);
464 + get_cached_checksum(slot, fn, file, -1, st, sum);
466 file_checksum(fn, st->st_size, sum);
467 return memcmp(sum, F_SUM(file), checksum_len) == 0;
468 @@ -1177,7 +1178,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
469 fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1);
471 if (checksum_files) {
472 - reset_checksum_cache();
473 + reset_checksum_cache(started_whole_dir);
474 + started_whole_dir = upcoming_whole_dir;
476 need_new_dirscan = 0;
478 @@ -1335,6 +1337,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
480 change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
482 + upcoming_whole_dir = file->flags & FLAG_CONTENT_DIR && f_out != -1 ? 1 : 0;
486 @@ -1627,6 +1630,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
487 handle_partial_dir(partialptr, PDIR_DELETE);
489 set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
490 + if (checksum_files & CSF_UPDATE)
491 + set_cached_checksum(cur_flist, file);
493 itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL);
494 #ifdef SUPPORT_HARD_LINKS
495 @@ -2068,6 +2073,7 @@ void generate_files(int f_out, const char *local_name)
497 change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
499 + upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0;
501 for (i = cur_flist->low; i <= cur_flist->high; i++) {
502 struct file_struct *file = cur_flist->sorted[i];
503 @@ -2162,6 +2168,9 @@ void generate_files(int f_out, const char *local_name)
507 + if (checksum_files)
508 + reset_checksum_cache(started_whole_dir);
510 info_levels[INFO_FLIST] = save_info_flist;
511 info_levels[INFO_PROGRESS] = save_info_progress;
513 diff --git a/io.c b/io.c
514 index 6a89c8f..50e73b1 100644
517 @@ -49,6 +49,7 @@ extern int list_only;
518 extern int read_batch;
519 extern int protect_args;
520 extern int checksum_seed;
521 +extern int checksum_files;
522 extern int protocol_version;
523 extern int remove_source_files;
524 extern int preserve_hard_links;
525 @@ -161,6 +162,9 @@ static void got_flist_entry_status(enum festatus status, const char *buf)
526 flist_ndx_push(&hlink_list, ndx);
527 flist->in_progress++;
529 + } else if (checksum_files & CSF_UPDATE) {
530 + struct file_struct *file = flist->files[ndx - flist->ndx_start];
531 + set_cached_checksum(flist, file);
535 diff --git a/loadparm.c b/loadparm.c
536 index 899d2b5..3123c93 100644
539 @@ -311,6 +311,10 @@ static struct enum_list enum_csum_modes[] = {
540 { CSF_IGNORE_FILES, "none" },
541 { CSF_LAX_MODE, "lax" },
542 { CSF_STRICT_MODE, "strict" },
543 + { CSF_LAX_MODE|CSF_UPDATE, "+lax" },
544 + { CSF_STRICT_MODE|CSF_UPDATE, "+strict" },
545 + { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" },
546 + { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" },
550 diff --git a/options.c b/options.c
551 index 2e110f3..26d5561 100644
554 @@ -1635,7 +1635,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
557 arg = poptGetOptArg(pc);
558 - checksum_files = 0;
561 + checksum_files = CSF_UPDATE;
564 + checksum_files |= CSF_AFFECT_DRYRUN;
567 + checksum_files = 0;
568 if (strcmp(arg, "lax") == 0)
569 checksum_files |= CSF_LAX_MODE;
570 else if (strcmp(arg, "strict") == 0)
571 diff --git a/receiver.c b/receiver.c
572 index 4325e30..2cea8fe 100644
575 @@ -47,6 +47,7 @@ extern int sparse_files;
576 extern int keep_partial;
577 extern int checksum_len;
578 extern int checksum_seed;
579 +extern int checksum_files;
581 extern int delay_updates;
582 extern mode_t orig_umask;
583 @@ -344,7 +345,7 @@ static void handle_delayed_updates(char *local_name)
584 "rename failed for %s (from %s)",
585 full_fname(fname), partialptr);
587 - if (remove_source_files
588 + if (remove_source_files || checksum_files & CSF_UPDATE
589 || (preserve_hard_links && F_IS_HLINKED(file)))
590 send_msg_int(MSG_SUCCESS, ndx);
591 handle_partial_dir(partialptr, PDIR_DELETE);
592 @@ -794,7 +795,7 @@ int recv_files(int f_in, char *local_name)
596 - if (remove_source_files || inc_recurse
597 + if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE
598 || (preserve_hard_links && F_IS_HLINKED(file)))
599 send_msg_int(MSG_SUCCESS, ndx);
601 diff --git a/rsync.h b/rsync.h
602 index ba8f3db..89c47bd 100644
605 @@ -908,6 +908,8 @@ typedef struct {
607 #define CSF_ENABLE (1<<1)
608 #define CSF_LAX (1<<2)
609 +#define CSF_UPDATE (1<<3)
610 +#define CSF_AFFECT_DRYRUN (1<<4)
612 #define CSF_IGNORE_FILES 0
613 #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX)
614 diff --git a/rsync.yo b/rsync.yo
615 index 7aa62cf..a119ed7 100644
618 @@ -596,9 +596,13 @@ computed just as it would be if bf(--sumfiles) was not specified.
620 The MODE value is either "lax", for relaxed checking (which compares size
621 and mtime), "strict" (which also compares ctime and inode), or "none" to
622 -ignore any .rsyncsums files ("none" is the default). Rsync does not create
623 -or update these files, but there is a perl script in the support directory
624 -named "rsyncsums" that can be used for that.
625 +ignore any .rsyncsums files ("none" is the default).
626 +If you want rsync to create and/or update these files, specify a prefixed
627 +plus ("+lax" or "+strict").
628 +Adding a second prefixed '+' causes the checksum-file updates to happen
629 +even when the transfer is in bf(--dry-run) mode ("++lax" or "++strict").
630 +There is also a perl script in the support directory named "rsyncsums"
631 +that can be used to update the .rsyncsums files.
633 This option has no effect unless bf(--checksum, -c) was also specified. It
634 also only affects the current side of the transfer, so if you want the
635 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
636 index 0fc98fd..3024842 100644
639 @@ -295,13 +295,15 @@ The default is tt(/var/run/rsyncd.lock).
640 dit(bf(checksum files)) This parameter tells rsync to make use of any cached
641 checksum information it finds in per-directory .rsyncsums files when the
642 current transfer is using the bf(--checksum) option. The value can be set
643 -to either "lax", "strict", or "none" -- see the client's bf(--sumfiles)
644 -option for what these choices do.
645 +to either "lax", "strict", "+lax", "+strict", "++lax", "++strict", or
646 +"none". See the client's bf(--sumfiles) option for what these choices do.
648 Note also that the client's command-line option, bf(--sumfiles), has no
649 effect on a daemon. A daemon will only access checksum files if this
650 -config option tells it to. See also the bf(exclude) directive for a way
651 -to hide the .rsyncsums files from the user.
652 +config option tells it to. You can configure updating of the .rsyncsums
653 +files even if the module itself is configured to be read-only. See also
654 +the bf(exclude) directive for a way to hide the .rsyncsums files from the
657 dit(bf(read only)) This parameter determines whether clients
658 will be able to upload files or not. If "read only" is true then any