added --modify-window option from David Bolen <db3l@fitlinxx.com>
[rsync.git] / generator.c
1 /* 
2    Copyright (C) Andrew Tridgell 1996
3    Copyright (C) Paul Mackerras 1996
4    
5    This program is free software; you can redistribute it and/or modify
6    it under the terms of the GNU General Public License as published by
7    the Free Software Foundation; either version 2 of the License, or
8    (at your option) any later version.
9    
10    This program is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13    GNU General Public License for more details.
14    
15    You should have received a copy of the GNU General Public License
16    along with this program; if not, write to the Free Software
17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20 #include "rsync.h"
21
22 extern int verbose;
23 extern int dry_run;
24 extern int relative_paths;
25 extern int preserve_links;
26 extern int am_root;
27 extern int preserve_devices;
28 extern int preserve_hard_links;
29 extern int update_only;
30 extern int whole_file;
31 extern int block_size;
32 extern int csum_length;
33 extern int ignore_times;
34 extern int size_only;
35 extern int io_timeout;
36 extern int remote_version;
37 extern int always_checksum;
38 extern int modify_window;
39 extern char *compare_dest;
40
41
42 /* choose whether to skip a particular file */
43 static int skip_file(char *fname,
44                      struct file_struct *file, STRUCT_STAT *st)
45 {
46         if (st->st_size != file->length) {
47                 return 0;
48         }
49         
50         /* if always checksum is set then we use the checksum instead 
51            of the file time to determine whether to sync */
52         if (always_checksum && S_ISREG(st->st_mode)) {
53                 char sum[MD4_SUM_LENGTH];
54                 char fnamecmpdest[MAXPATHLEN];
55
56                 if (compare_dest != NULL) {
57                         if (access(fname, 0) != 0) {
58                                 slprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
59                                                     compare_dest,fname);
60                                 fname = fnamecmpdest;
61                         }
62                 }
63                 file_checksum(fname,sum,st->st_size);
64                 if (remote_version < 21) {
65                         return (memcmp(sum,file->sum,2) == 0);
66                 } else {
67                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
68                 }
69         }
70
71         if (size_only) {
72                 return 1;
73         }
74
75         if (ignore_times) {
76                 return 0;
77         }
78
79         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
80 }
81
82
83 /* use a larger block size for really big files */
84 static int adapt_block_size(struct file_struct *file, int bsize)
85 {
86         int ret;
87
88         if (bsize != BLOCK_SIZE) return bsize;
89
90         ret = file->length / (10000); /* rough heuristic */
91         ret = ret & ~15; /* multiple of 16 */
92         if (ret < bsize) ret = bsize;
93         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
94         return ret;
95 }
96
97
98 /*
99   send a sums struct down a fd
100   */
101 static void send_sums(struct sum_struct *s,int f_out)
102 {
103         int i;
104         
105         /* tell the other guy how many we are going to be doing and how many
106            bytes there are in the last chunk */
107         write_int(f_out,s?s->count:0);
108         write_int(f_out,s?s->n:block_size);
109         write_int(f_out,s?s->remainder:0);
110
111         if (!s) return;
112
113         for (i=0;i<s->count;i++) {
114                 write_int(f_out,s->sums[i].sum1);
115                 write_buf(f_out,s->sums[i].sum2,csum_length);
116         }
117 }
118
119
120 /*
121   generate a stream of signatures/checksums that describe a buffer
122
123   generate approximately one checksum every n bytes
124   */
125 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
126 {
127         int i;
128         struct sum_struct *s;
129         int count;
130         int block_len = n;
131         int remainder = (len%block_len);
132         OFF_T offset = 0;
133
134         count = (len+(block_len-1))/block_len;
135
136         s = (struct sum_struct *)malloc(sizeof(*s));
137         if (!s) out_of_memory("generate_sums");
138
139         s->count = count;
140         s->remainder = remainder;
141         s->n = n;
142         s->flength = len;
143
144         if (count==0) {
145                 s->sums = NULL;
146                 return s;
147         }
148
149         if (verbose > 3)
150                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
151                         s->count,s->remainder,s->n,(double)s->flength);
152
153         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
154         if (!s->sums) out_of_memory("generate_sums");
155   
156         for (i=0;i<count;i++) {
157                 int n1 = MIN(len,n);
158                 char *map = map_ptr(buf,offset,n1);
159
160                 s->sums[i].sum1 = get_checksum1(map,n1);
161                 get_checksum2(map,n1,s->sums[i].sum2);
162
163                 s->sums[i].offset = offset;
164                 s->sums[i].len = n1;
165                 s->sums[i].i = i;
166
167                 if (verbose > 3)
168                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
169                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
170
171                 len -= n1;
172                 offset += n1;
173         }
174
175         return s;
176 }
177
178
179 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
180 {  
181         int fd;
182         STRUCT_STAT st;
183         struct map_struct *buf;
184         struct sum_struct *s;
185         int statret;
186         struct file_struct *file = flist->files[i];
187         char *fnamecmp;
188         char fnamecmpbuf[MAXPATHLEN];
189         extern char *compare_dest;
190         extern int list_only;
191         extern int preserve_perms;
192         extern int only_existing;
193
194         if (list_only) return;
195
196         if (verbose > 2)
197                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
198
199         statret = link_stat(fname,&st);
200
201         if (only_existing && statret == -1 && errno == ENOENT) {
202                 /* we only want to update existing files */
203                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
204                 return;
205         }
206
207         if (statret == 0 && 
208             !preserve_perms && 
209             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
210                 /* if the file exists already and we aren't perserving
211                    presmissions then act as though the remote end sent
212                    us the file permissions we already have */
213                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
214         }
215
216         if (S_ISDIR(file->mode)) {
217                 if (dry_run) return;
218                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
219                         if (robust_unlink(fname) != 0) {
220                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
221                                 return;
222                         }
223                         statret = -1;
224                 }
225                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
226                         if (!(relative_paths && errno==ENOENT && 
227                               create_directory_path(fname)==0 && 
228                               do_mkdir(fname,file->mode)==0)) {
229                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
230                                         fname,strerror(errno));
231                         }
232                 }
233                 if (set_perms(fname,file,NULL,0) && verbose) 
234                         rprintf(FINFO,"%s/\n",fname);
235                 return;
236         }
237
238         if (preserve_links && S_ISLNK(file->mode)) {
239 #if SUPPORT_LINKS
240                 char lnk[MAXPATHLEN];
241                 int l;
242                 extern int safe_symlinks;
243
244                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
245                         if (verbose) {
246                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
247                                         fname,file->link);
248                         }
249                         return;
250                 }
251                 if (statret == 0) {
252                         l = readlink(fname,lnk,MAXPATHLEN-1);
253                         if (l > 0) {
254                                 lnk[l] = 0;
255                                 if (strcmp(lnk,file->link) == 0) {
256                                         set_perms(fname,file,&st,1);
257                                         return;
258                                 }
259                         }
260                         delete_file(fname);
261                 }
262                 if (do_symlink(file->link,fname) != 0) {
263                         rprintf(FERROR,"symlink %s -> %s : %s\n",
264                                 fname,file->link,strerror(errno));
265                 } else {
266                         set_perms(fname,file,NULL,0);
267                         if (verbose) {
268                                 rprintf(FINFO,"%s -> %s\n",
269                                         fname,file->link);
270                         }
271                 }
272 #endif
273                 return;
274         }
275
276 #ifdef HAVE_MKNOD
277         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
278                 if (statret != 0 || 
279                     st.st_mode != file->mode ||
280                     st.st_rdev != file->rdev) { 
281                         delete_file(fname);
282                         if (verbose > 2)
283                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
284                                         fname,(int)file->mode,(int)file->rdev);
285                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
286                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
287                         } else {
288                                 set_perms(fname,file,NULL,0);
289                                 if (verbose)
290                                         rprintf(FINFO,"%s\n",fname);
291                         }
292                 } else {
293                         set_perms(fname,file,&st,1);
294                 }
295                 return;
296         }
297 #endif
298
299         if (preserve_hard_links && check_hard_link(file)) {
300                 if (verbose > 1)
301                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
302                 return;
303         }
304
305         if (!S_ISREG(file->mode)) {
306                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
307                 return;
308         }
309
310         fnamecmp = fname;
311
312         if ((statret == -1) && (compare_dest != NULL)) {
313                 /* try the file at compare_dest instead */
314                 int saveerrno = errno;
315                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
316                 statret = link_stat(fnamecmpbuf,&st);
317                 if (!S_ISREG(st.st_mode))
318                         statret = -1;
319                 if (statret == -1)
320                         errno = saveerrno;
321                 else
322                         fnamecmp = fnamecmpbuf;
323         }
324
325         if (statret == -1) {
326                 if (errno == ENOENT) {
327                         write_int(f_out,i);
328                         if (!dry_run) send_sums(NULL,f_out);
329                 } else {
330                         if (verbose > 1)
331                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
332                 }
333                 return;
334         }
335
336         if (!S_ISREG(st.st_mode)) {
337                 if (delete_file(fname) != 0) {
338                         return;
339                 }
340
341                 /* now pretend the file didn't exist */
342                 write_int(f_out,i);
343                 if (!dry_run) send_sums(NULL,f_out);    
344                 return;
345         }
346
347         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
348                 if (verbose > 1)
349                         rprintf(FINFO,"%s is newer\n",fname);
350                 return;
351         }
352
353         if (skip_file(fname, file, &st)) {
354                 if (fnamecmp == fname)
355                         set_perms(fname,file,&st,1);
356                 return;
357         }
358
359         if (dry_run) {
360                 write_int(f_out,i);
361                 return;
362         }
363
364         if (whole_file) {
365                 write_int(f_out,i);
366                 send_sums(NULL,f_out);    
367                 return;
368         }
369
370         /* open the file */  
371         fd = do_open(fnamecmp, O_RDONLY, 0);
372
373         if (fd == -1) {
374                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
375                 /* pretend the file didn't exist */
376                 write_int(f_out,i);
377                 send_sums(NULL,f_out);
378                 return;
379         }
380
381         if (st.st_size > 0) {
382                 buf = map_file(fd,st.st_size);
383         } else {
384                 buf = NULL;
385         }
386
387         if (verbose > 3)
388                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
389
390         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
391
392         if (verbose > 2)
393                 rprintf(FINFO,"sending sums for %d\n",i);
394
395         write_int(f_out,i);
396         send_sums(s,f_out);
397
398         close(fd);
399         if (buf) unmap_file(buf);
400
401         free_sums(s);
402 }
403
404
405
406 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
407 {
408         int i;
409         int phase=0;
410
411         if (verbose > 2)
412                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
413                         (int)getpid(),flist->count);
414
415         for (i = 0; i < flist->count; i++) {
416                 struct file_struct *file = flist->files[i];
417                 mode_t saved_mode = file->mode;
418                 if (!file->basename) continue;
419
420                 /* we need to ensure that any directories we create have writeable
421                    permissions initially so that we can create the files within
422                    them. This is then fixed after the files are transferred */
423                 if (!am_root && S_ISDIR(file->mode)) {
424                         file->mode |= S_IWUSR; /* user write */
425                 }
426
427                 recv_generator(local_name?local_name:f_name(file),
428                                flist,i,f);
429
430                 file->mode = saved_mode;
431         }
432
433         phase++;
434         csum_length = SUM_LENGTH;
435         ignore_times=1;
436
437         if (verbose > 2)
438                 rprintf(FINFO,"generate_files phase=%d\n",phase);
439
440         write_int(f,-1);
441
442         /* we expect to just sit around now, so don't exit on a
443            timeout. If we really get a timeout then the other process should
444            exit */
445         io_timeout = 0;
446
447         if (remote_version >= 13) {
448                 /* in newer versions of the protocol the files can cycle through
449                    the system more than once to catch initial checksum errors */
450                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
451                         struct file_struct *file = flist->files[i];
452                         recv_generator(local_name?local_name:f_name(file),
453                                        flist,i,f);    
454                 }
455
456                 phase++;
457                 if (verbose > 2)
458                         rprintf(FINFO,"generate_files phase=%d\n",phase);
459
460                 write_int(f,-1);
461         }
462 }