- Made flist_find() and receive_file_entry() static functions.
[rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2
3    rsync -- fast file replication program
4
5    Copyright (C) 1996-2000 by Andrew Tridgell
6    Copyright (C) Paul Mackerras 1996
7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
8
9    This program is free software; you can redistribute it and/or modify
10    it under the terms of the GNU General Public License as published by
11    the Free Software Foundation; either version 2 of the License, or
12    (at your option) any later version.
13
14    This program is distributed in the hope that it will be useful,
15    but WITHOUT ANY WARRANTY; without even the implied warranty of
16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17    GNU General Public License for more details.
18
19    You should have received a copy of the GNU General Public License
20    along with this program; if not, write to the Free Software
21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 */
23
24 #include "rsync.h"
25
26 extern int verbose;
27 extern int dry_run;
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
31 extern int am_root;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int preserve_times;
38 extern int omit_dir_times;
39 extern int delete_during;
40 extern int update_only;
41 extern int opt_ignore_existing;
42 extern int inplace;
43 extern int make_backups;
44 extern int csum_length;
45 extern int ignore_times;
46 extern int size_only;
47 extern OFF_T max_size;
48 extern int io_timeout;
49 extern int protocol_version;
50 extern int always_checksum;
51 extern char *partial_dir;
52 extern char *basis_dir[];
53 extern int compare_dest;
54 extern int link_dest;
55 extern int whole_file;
56 extern int local_server;
57 extern int list_only;
58 extern int read_batch;
59 extern int only_existing;
60 extern int orig_umask;
61 extern int safe_symlinks;
62 extern long block_size; /* "long" because popt can't set an int32. */
63
64 extern struct filter_list_struct server_filter_list;
65
66 static int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
67 {
68         if (preserve_perms
69          && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
70                 return 0;
71
72         if (am_root && preserve_uid && st->st_uid != file->uid)
73                 return 0;
74
75         if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
76                 return 0;
77
78         return 1;
79 }
80
81 /* Perform our quick-check heuristic for determining if a file is unchanged. */
82 static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
83 {
84         if (st->st_size != file->length)
85                 return 0;
86
87         /* if always checksum is set then we use the checksum instead
88            of the file time to determine whether to sync */
89         if (always_checksum && S_ISREG(st->st_mode)) {
90                 char sum[MD4_SUM_LENGTH];
91                 file_checksum(fn, sum, st->st_size);
92                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
93                                                         : MD4_SUM_LENGTH) == 0;
94         }
95
96         if (size_only)
97                 return 1;
98
99         if (ignore_times)
100                 return 0;
101
102         return cmp_modtime(st->st_mtime, file->modtime) == 0;
103 }
104
105
106 /*
107  * set (initialize) the size entries in the per-file sum_struct
108  * calculating dynamic block and checksum sizes.
109  *
110  * This is only called from generate_and_send_sums() but is a separate
111  * function to encapsulate the logic.
112  *
113  * The block size is a rounded square root of file length.
114  *
115  * The checksum size is determined according to:
116  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
117  * provided by Donovan Baarda which gives a probability of rsync
118  * algorithm corrupting data and falling back using the whole md4
119  * checksums.
120  *
121  * This might be made one of several selectable heuristics.
122  */
123 static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
124 {
125         int32 blength;
126         int s2length;
127
128         if (block_size)
129                 blength = block_size;
130         else if (len <= BLOCK_SIZE * BLOCK_SIZE)
131                 blength = BLOCK_SIZE;
132         else {
133                 int32 c;
134                 int64 l;
135                 int cnt;
136                 for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
137                 if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
138                         blength = MAX_BLOCK_SIZE;
139                 else {
140                     blength = 0;
141                     do {
142                             blength |= c;
143                             if (len < (int64)blength * blength)
144                                     blength &= ~c;
145                             c >>= 1;
146                     } while (c >= 8);   /* round to multiple of 8 */
147                     blength = MAX(blength, BLOCK_SIZE);
148                 }
149         }
150
151         if (protocol_version < 27) {
152                 s2length = csum_length;
153         } else if (csum_length == SUM_LENGTH) {
154                 s2length = SUM_LENGTH;
155         } else {
156                 int32 c;
157                 int64 l;
158                 int b = BLOCKSUM_BIAS;
159                 for (l = len; l >>= 1; b += 2) {}
160                 for (c = blength; c >>= 1 && b; b--) {}
161                 /* add a bit, subtract rollsum, round up. */
162                 s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
163                 s2length = MAX(s2length, csum_length);
164                 s2length = MIN(s2length, SUM_LENGTH);
165         }
166
167         sum->flength    = len;
168         sum->blength    = blength;
169         sum->s2length   = s2length;
170         sum->count      = (len + (blength - 1)) / blength;
171         sum->remainder  = (len % blength);
172
173         if (sum->count && verbose > 2) {
174                 rprintf(FINFO,
175                         "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
176                         (double)sum->count, (long)sum->remainder, (long)sum->blength,
177                         sum->s2length, (double)sum->flength);
178         }
179 }
180
181
182 /*
183  * Generate and send a stream of signatures/checksums that describe a buffer
184  *
185  * Generate approximately one checksum every block_len bytes.
186  */
187 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
188 {
189         size_t i;
190         struct map_struct *mapbuf;
191         struct sum_struct sum;
192         OFF_T offset = 0;
193
194         sum_sizes_sqroot(&sum, len);
195
196         if (len > 0)
197                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
198         else
199                 mapbuf = NULL;
200
201         write_sum_head(f_out, &sum);
202
203         for (i = 0; i < sum.count; i++) {
204                 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
205                 char *map = map_ptr(mapbuf, offset, n1);
206                 uint32 sum1 = get_checksum1(map, n1);
207                 char sum2[SUM_LENGTH];
208
209                 if (f_copy >= 0)
210                         full_write(f_copy, map, n1);
211
212                 get_checksum2(map, n1, sum2);
213
214                 if (verbose > 3) {
215                         rprintf(FINFO,
216                                 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
217                                 (double)i, (double)offset, (long)n1,
218                                 (unsigned long)sum1);
219                 }
220                 write_int(f_out, sum1);
221                 write_buf(f_out, sum2, sum.s2length);
222                 len -= n1;
223                 offset += n1;
224         }
225
226         if (mapbuf)
227                 unmap_file(mapbuf);
228 }
229
230
231 /*
232  * Acts on file number @p i from @p flist, whose name is @p fname.
233  *
234  * First fixes up permissions, then generates checksums for the file.
235  *
236  * @note This comment was added later by mbp who was trying to work it
237  * out.  It might be wrong.
238  */
239 static void recv_generator(char *fname, struct file_list *flist,
240                            struct file_struct *file, int i,
241                            int f_out, int f_out_name)
242 {
243         int fd = -1, f_copy = -1;
244         STRUCT_STAT st, partial_st;
245         struct file_struct *back_file = NULL;
246         int statret, stat_errno;
247         char *fnamecmp, *partialptr, *backupptr = NULL;
248         char fnamecmpbuf[MAXPATHLEN];
249         uchar fnamecmp_type;
250
251         if (list_only)
252                 return;
253
254         if (verbose > 2)
255                 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
256
257         if (server_filter_list.head
258             && check_filter(&server_filter_list, fname,
259                             S_ISDIR(file->mode)) < 0) {
260                 if (verbose) {
261                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
262                                 safe_fname(fname));
263                 }
264                 return;
265         }
266
267         if (dry_run > 1) {
268                 statret = -1;
269                 stat_errno = ENOENT;
270         } else {
271                 statret = link_stat(fname, &st,
272                                     keep_dirlinks && S_ISDIR(file->mode));
273                 stat_errno = errno;
274         }
275
276         if (only_existing && statret == -1 && stat_errno == ENOENT) {
277                 /* we only want to update existing files */
278                 if (verbose > 1) {
279                         rprintf(FINFO, "not creating new file \"%s\"\n",
280                                 safe_fname(fname));
281                 }
282                 return;
283         }
284
285         if (statret == 0 && !preserve_perms
286             && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
287                 /* if the file exists already and we aren't perserving
288                  * permissions then act as though the remote end sent
289                  * us the file permissions we already have */
290                 file->mode = (file->mode & ~CHMOD_BITS)
291                            | (st.st_mode & CHMOD_BITS);
292         }
293
294         if (S_ISDIR(file->mode)) {
295                 /* The file to be received is a directory, so we need
296                  * to prepare appropriately.  If there is already a
297                  * file of that name and it is *not* a directory, then
298                  * we need to delete it.  If it doesn't exist, then
299                  * (perhaps recursively) create it. */
300                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
301                         delete_file(fname, DEL_TERSE);
302                         statret = -1;
303                 }
304                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
305                         if (!relative_paths || errno != ENOENT
306                             || create_directory_path(fname, orig_umask) < 0
307                             || do_mkdir(fname, file->mode) < 0) {
308                                 rsyserr(FERROR, errno,
309                                         "recv_generator: mkdir %s failed",
310                                         full_fname(fname));
311                         }
312                 }
313                 /* f_out is set to -1 when doing final directory-permission
314                  * and modification-time repair. */
315                 if (set_perms(fname, file, statret ? NULL : &st, 0)
316                     && verbose && f_out != -1)
317                         rprintf(FINFO, "%s/\n", safe_fname(fname));
318                 if (delete_during && f_out != -1 && csum_length != SUM_LENGTH
319                     && (file->flags & FLAG_DEL_START))
320                         delete_in_dir(flist, fname);
321                 return;
322         } else if (max_size && file->length > max_size) {
323                 if (verbose > 1)
324                         rprintf(FINFO, "%s is over max-size\n", fname);
325                 return;
326         }
327
328         if (preserve_links && S_ISLNK(file->mode)) {
329 #if SUPPORT_LINKS
330                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
331                         if (verbose) {
332                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
333                                         full_fname(fname), file->u.link);
334                         }
335                         return;
336                 }
337                 if (statret == 0) {
338                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
339                         char lnk[MAXPATHLEN];
340                         int len;
341
342                         if (!dflag
343                             && (len = readlink(fname, lnk, MAXPATHLEN-1)) > 0) {
344                                 lnk[len] = 0;
345                                 /* A link already pointing to the
346                                  * right place -- no further action
347                                  * required. */
348                                 if (strcmp(lnk, file->u.link) == 0) {
349                                         set_perms(fname, file, &st,
350                                                   PERMS_REPORT);
351                                         return;
352                                 }
353                         }
354                         /* Not the right symlink (or not a symlink), so
355                          * delete it. */
356                         delete_file(fname, dflag | DEL_TERSE);
357                 }
358                 if (do_symlink(file->u.link,fname) != 0) {
359                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
360                                 full_fname(fname), safe_fname(file->u.link));
361                 } else {
362                         set_perms(fname,file,NULL,0);
363                         if (verbose) {
364                                 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
365                                         safe_fname(file->u.link));
366                         }
367                 }
368 #endif
369                 return;
370         }
371
372         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
373                 if (statret != 0 ||
374                     st.st_mode != file->mode ||
375                     st.st_rdev != file->u.rdev) {
376                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
377                         delete_file(fname, dflag | DEL_TERSE);
378                         if (verbose > 2) {
379                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
380                                         safe_fname(fname),
381                                         (int)file->mode, (int)file->u.rdev);
382                         }
383                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
384                                 rsyserr(FERROR, errno, "mknod %s failed",
385                                         full_fname(fname));
386                         } else {
387                                 set_perms(fname,file,NULL,0);
388                                 if (verbose) {
389                                         rprintf(FINFO, "%s\n",
390                                                 safe_fname(fname));
391                                 }
392                         }
393                 } else {
394                         set_perms(fname, file, &st, PERMS_REPORT);
395                 }
396                 return;
397         }
398
399         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
400                 return;
401
402         if (!S_ISREG(file->mode)) {
403                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
404                         safe_fname(fname));
405                 return;
406         }
407
408         fnamecmp = fname;
409         fnamecmp_type = FNAMECMP_FNAME;
410
411         if (statret == -1 && basis_dir[0] != NULL) {
412                 int fallback_match = -1;
413                 int match_level = 0;
414                 int i = 0;
415                 do {
416                         pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
417                                  basis_dir[i], fname);
418                         if (link_stat(fnamecmpbuf, &st, 0) == 0
419                             && S_ISREG(st.st_mode)) {
420                                 statret = 0;
421                                 if (link_dest) {
422                                         if (!match_level) {
423                                                 fallback_match = i;
424                                                 match_level = 1;
425                                         } else if (match_level == 2
426                                             && !unchanged_attrs(file, &st))
427                                                 continue;
428                                         if (!unchanged_file(fnamecmpbuf, file, &st))
429                                                 continue;
430                                         fallback_match = i;
431                                         match_level = 2;
432                                         if (!unchanged_attrs(file, &st))
433                                                 continue;
434                                 }
435                                 match_level = 3;
436                                 break;
437                         }
438                 } while (basis_dir[++i] != NULL);
439                 if (statret == 0) {
440                         if (match_level < 3) {
441                                 i = fallback_match;
442                                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
443                                          basis_dir[i], fname);
444                         }
445 #if HAVE_LINK
446                         if (link_dest && match_level == 3 && !dry_run) {
447                                 if (do_link(fnamecmpbuf, fname) < 0) {
448                                         if (verbose) {
449                                                 rsyserr(FINFO, errno,
450                                                         "link %s => %s",
451                                                         fnamecmpbuf,
452                                                         safe_fname(fname));
453                                         }
454                                         fnamecmp = fnamecmpbuf;
455                                         fnamecmp_type = i;
456                                 }
457                         } else
458 #endif
459                         {
460                                 fnamecmp = fnamecmpbuf;
461                                 fnamecmp_type = i;
462                         }
463                 }
464         }
465
466         if (statret == 0 && !S_ISREG(st.st_mode)) {
467                 int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
468                 if (delete_file(fname, dflag | DEL_TERSE) != 0)
469                         return;
470                 statret = -1;
471                 stat_errno = ENOENT;
472         }
473
474         if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
475             && link_stat(partialptr, &partial_st, 0) == 0
476             && S_ISREG(partial_st.st_mode)) {
477                 if (statret == -1)
478                         goto prepare_to_open;
479         } else
480                 partialptr = NULL;
481
482         if (statret == -1) {
483                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
484                         return;
485                 if (stat_errno == ENOENT)
486                         goto notify_others;
487                 if (verbose > 1) {
488                         rsyserr(FERROR, stat_errno,
489                                 "recv_generator: failed to stat %s",
490                                 full_fname(fname));
491                 }
492                 return;
493         }
494
495         if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
496                 if (verbose > 1)
497                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
498                 return;
499         }
500
501         if (update_only && fnamecmp_type == FNAMECMP_FNAME
502             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
503                 if (verbose > 1)
504                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
505                 return;
506         }
507
508         if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
509                 ;
510         else if (unchanged_file(fnamecmp, file, &st)) {
511                 if (fnamecmp_type == FNAMECMP_FNAME)
512                         set_perms(fname, file, &st, PERMS_REPORT);
513                 return;
514         }
515
516 prepare_to_open:
517         if (partialptr) {
518                 st = partial_st;
519                 fnamecmp = partialptr;
520                 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
521                 statret = 0;
522         }
523
524         if (dry_run || whole_file > 0) {
525                 statret = -1;
526                 goto notify_others;
527         }
528         if (read_batch)
529                 goto notify_others;
530
531         /* open the file */
532         fd = do_open(fnamecmp, O_RDONLY, 0);
533
534         if (fd == -1) {
535                 rsyserr(FERROR, errno, "failed to open %s, continuing",
536                         full_fname(fnamecmp));
537             pretend_missing:
538                 /* pretend the file didn't exist */
539                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
540                         return;
541                 statret = -1;
542                 goto notify_others;
543         }
544
545         if (inplace && make_backups) {
546                 if (!(backupptr = get_backup_name(fname))) {
547                         close(fd);
548                         return;
549                 }
550                 if (!(back_file = make_file(fname, NULL, NO_FILTERS))) {
551                         close(fd);
552                         goto pretend_missing;
553                 }
554                 if (robust_unlink(backupptr) && errno != ENOENT) {
555                         rsyserr(FERROR, errno, "unlink %s",
556                                 full_fname(backupptr));
557                         free(back_file);
558                         close(fd);
559                         return;
560                 }
561                 if ((f_copy = do_open(backupptr,
562                     O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
563                         rsyserr(FERROR, errno, "open %s",
564                                 full_fname(backupptr));
565                         free(back_file);
566                         close(fd);
567                         return;
568                 }
569                 fnamecmp_type = FNAMECMP_BACKUP;
570         }
571
572         if (verbose > 3) {
573                 rprintf(FINFO, "gen mapped %s of size %.0f\n",
574                         safe_fname(fnamecmp), (double)st.st_size);
575         }
576
577         if (verbose > 2)
578                 rprintf(FINFO, "generating and sending sums for %d\n", i);
579
580 notify_others:
581         write_int(f_out, i);
582         if (protocol_version >= 29 && inplace && !read_batch)
583                 write_byte(f_out, fnamecmp_type);
584         if (f_out_name >= 0)
585                 write_byte(f_out_name, fnamecmp_type);
586
587         if (dry_run || read_batch)
588                 return;
589
590         if (statret == 0) {
591                 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
592
593                 if (f_copy >= 0) {
594                         close(f_copy);
595                         set_perms(backupptr, back_file, NULL, 0);
596                         if (verbose > 1) {
597                                 rprintf(FINFO, "backed up %s to %s\n",
598                                         fname, backupptr);
599                         }
600                         free(back_file);
601                 }
602
603                 close(fd);
604         } else
605                 write_sum_head(f_out, NULL);
606 }
607
608
609 void generate_files(int f_out, struct file_list *flist, char *local_name,
610                     int f_out_name)
611 {
612         int i;
613         int phase = 0;
614         char fbuf[MAXPATHLEN];
615         int need_retouch_dir_times = preserve_times && !omit_dir_times;
616         int need_retouch_dir_perms = 0;
617
618         if (verbose > 2) {
619                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
620                         (long)getpid(), flist->count);
621         }
622
623         if (verbose >= 2) {
624                 rprintf(FINFO,
625                         whole_file > 0
626                         ? "delta-transmission disabled for local transfer or --whole-file\n"
627                         : "delta transmission enabled\n");
628         }
629
630         /* we expect to just sit around now, so don't exit on a
631            timeout. If we really get a timeout then the other process should
632            exit */
633         io_timeout = 0;
634
635         for (i = 0; i < flist->count; i++) {
636                 struct file_struct *file = flist->files[i];
637                 struct file_struct copy;
638
639                 if (!file->basename)
640                         continue;
641                 /* we need to ensure that any directories we create have writeable
642                    permissions initially so that we can create the files within
643                    them. This is then fixed after the files are transferred */
644                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
645                         copy = *file;
646                         copy.mode |= S_IWUSR; /* user write */
647                         file = &copy;
648                         need_retouch_dir_perms = 1;
649                 }
650
651                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
652                                flist, file, i, f_out, f_out_name);
653         }
654         if (delete_during)
655                 delete_in_dir(NULL, NULL);
656
657         phase++;
658         csum_length = SUM_LENGTH;
659         ignore_times = 1;
660
661         if (verbose > 2)
662                 rprintf(FINFO,"generate_files phase=%d\n",phase);
663
664         write_int(f_out, -1);
665
666         /* files can cycle through the system more than once
667          * to catch initial checksum errors */
668         while ((i = get_redo_num()) != -1) {
669                 struct file_struct *file = flist->files[i];
670                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
671                                flist, file, i, f_out, f_out_name);
672         }
673
674         phase++;
675         if (verbose > 2)
676                 rprintf(FINFO,"generate_files phase=%d\n",phase);
677
678         write_int(f_out, -1);
679
680         /* Read post-redo-phase MSG_DONE and any prior messages. */
681         get_redo_num();
682
683         if (preserve_hard_links)
684                 do_hard_links();
685
686         if (need_retouch_dir_perms || need_retouch_dir_times) {
687                 /* Now we need to fix any directory permissions that were
688                  * modified during the transfer and/or re-set any tweaked
689                  * modified-time values. */
690                 for (i = 0; i < flist->count; i++) {
691                         struct file_struct *file = flist->files[i];
692                         if (!file->basename || !S_ISDIR(file->mode))
693                                 continue;
694                         if (!need_retouch_dir_times && file->mode & S_IWUSR)
695                                 continue;
696                         recv_generator(local_name ? local_name : f_name(file),
697                                        flist, file, i, -1, -1);
698                 }
699         }
700
701         if (verbose > 2)
702                 rprintf(FINFO,"generate_files finished\n");
703 }