93faa54dca747cecad2571b87896ba5d087ffa2a
[rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2    
3    Copyright (C) 1996-2000 by Andrew Tridgell 
4    Copyright (C) Paul Mackerras 1996
5    
6    This program is free software; you can redistribute it and/or modify
7    it under the terms of the GNU General Public License as published by
8    the Free Software Foundation; either version 2 of the License, or
9    (at your option) any later version.
10    
11    This program is distributed in the hope that it will be useful,
12    but WITHOUT ANY WARRANTY; without even the implied warranty of
13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14    GNU General Public License for more details.
15    
16    You should have received a copy of the GNU General Public License
17    along with this program; if not, write to the Free Software
18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21 #include "rsync.h"
22
23 extern int verbose;
24 extern int dry_run;
25 extern int relative_paths;
26 extern int preserve_links;
27 extern int am_root;
28 extern int preserve_devices;
29 extern int preserve_hard_links;
30 extern int update_only;
31 extern int whole_file;
32 extern int block_size;
33 extern int csum_length;
34 extern int ignore_times;
35 extern int size_only;
36 extern int io_timeout;
37 extern int remote_version;
38 extern int always_checksum;
39 extern int modify_window;
40 extern char *compare_dest;
41
42
43 /* choose whether to skip a particular file */
44 static int skip_file(char *fname,
45                      struct file_struct *file, STRUCT_STAT *st)
46 {
47         if (st->st_size != file->length) {
48                 return 0;
49         }
50         
51         /* if always checksum is set then we use the checksum instead 
52            of the file time to determine whether to sync */
53         if (always_checksum && S_ISREG(st->st_mode)) {
54                 char sum[MD4_SUM_LENGTH];
55                 char fnamecmpdest[MAXPATHLEN];
56
57                 if (compare_dest != NULL) {
58                         if (access(fname, 0) != 0) {
59                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
60                                                     compare_dest,fname);
61                                 fname = fnamecmpdest;
62                         }
63                 }
64                 file_checksum(fname,sum,st->st_size);
65                 if (remote_version < 21) {
66                         return (memcmp(sum,file->sum,2) == 0);
67                 } else {
68                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
69                 }
70         }
71
72         if (size_only) {
73                 return 1;
74         }
75
76         if (ignore_times) {
77                 return 0;
78         }
79
80         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
81 }
82
83
84 /* use a larger block size for really big files */
85 static int adapt_block_size(struct file_struct *file, int bsize)
86 {
87         int ret;
88
89         if (bsize != BLOCK_SIZE) return bsize;
90
91         ret = file->length / (10000); /* rough heuristic */
92         ret = ret & ~15; /* multiple of 16 */
93         if (ret < bsize) ret = bsize;
94         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
95         return ret;
96 }
97
98
99 /*
100   send a sums struct down a fd
101   */
102 static void send_sums(struct sum_struct *s,int f_out)
103 {
104         int i;
105         
106         /* tell the other guy how many we are going to be doing and how many
107            bytes there are in the last chunk */
108         write_int(f_out,s?s->count:0);
109         write_int(f_out,s?s->n:block_size);
110         write_int(f_out,s?s->remainder:0);
111
112         if (!s) return;
113
114         for (i=0;i<s->count;i++) {
115                 write_int(f_out,s->sums[i].sum1);
116                 write_buf(f_out,s->sums[i].sum2,csum_length);
117         }
118 }
119
120
121 /*
122   generate a stream of signatures/checksums that describe a buffer
123
124   generate approximately one checksum every n bytes
125   */
126 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
127 {
128         int i;
129         struct sum_struct *s;
130         int count;
131         int block_len = n;
132         int remainder = (len%block_len);
133         OFF_T offset = 0;
134
135         count = (len+(block_len-1))/block_len;
136
137         s = (struct sum_struct *)malloc(sizeof(*s));
138         if (!s) out_of_memory("generate_sums");
139
140         s->count = count;
141         s->remainder = remainder;
142         s->n = n;
143         s->flength = len;
144
145         if (count==0) {
146                 s->sums = NULL;
147                 return s;
148         }
149
150         if (verbose > 3)
151                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
152                         s->count,s->remainder,s->n,(double)s->flength);
153
154         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
155         if (!s->sums) out_of_memory("generate_sums");
156   
157         for (i=0;i<count;i++) {
158                 int n1 = MIN(len,n);
159                 char *map = map_ptr(buf,offset,n1);
160
161                 s->sums[i].sum1 = get_checksum1(map,n1);
162                 get_checksum2(map,n1,s->sums[i].sum2);
163
164                 s->sums[i].offset = offset;
165                 s->sums[i].len = n1;
166                 s->sums[i].i = i;
167
168                 if (verbose > 3)
169                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
170                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
171
172                 len -= n1;
173                 offset += n1;
174         }
175
176         return s;
177 }
178
179
180
181 /*
182  * Acts on file number I from FLIST, whose name is fname.
183  *
184  * First fixes up permissions, then generates checksums for the file.
185  *
186  * (This comment was added later by mbp who was trying to work it out;
187  * it might be wrong.)
188  */ 
189 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
190 {  
191         int fd;
192         STRUCT_STAT st;
193         struct map_struct *buf;
194         struct sum_struct *s;
195         int statret;
196         struct file_struct *file = flist->files[i];
197         char *fnamecmp;
198         char fnamecmpbuf[MAXPATHLEN];
199         extern char *compare_dest;
200         extern int list_only;
201         extern int preserve_perms;
202         extern int only_existing;
203
204         if (list_only) return;
205
206         if (verbose > 2)
207                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
208
209         statret = link_stat(fname,&st);
210
211         if (only_existing && statret == -1 && errno == ENOENT) {
212                 /* we only want to update existing files */
213                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
214                 return;
215         }
216
217         if (statret == 0 && 
218             !preserve_perms && 
219             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
220                 /* if the file exists already and we aren't perserving
221                    presmissions then act as though the remote end sent
222                    us the file permissions we already have */
223                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
224         }
225
226         if (S_ISDIR(file->mode)) {
227                 /* The file to be received is a directory, so we need
228                  * to prepare appropriately.  If there is already a
229                  * file of that name and it is *not* a directory, then
230                  * we need to delete it.  If it doesn't exist, then
231                  * recursively create it. */
232           
233                 if (dry_run) return;
234                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
235                         if (robust_unlink(fname) != 0) {
236                                 rprintf(FERROR,"recv_generator: unlink %s: %s\n",
237                                         fname,strerror(errno));
238                                 return;
239                         }
240                         statret = -1;
241                 }
242                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
243                         if (!(relative_paths && errno==ENOENT && 
244                               create_directory_path(fname)==0 && 
245                               do_mkdir(fname,file->mode)==0)) {
246                                 rprintf(FERROR,"recv_generator: mkdir %s: %s (2)\n",
247                                         fname,strerror(errno));
248                         }
249                 }
250                 /* f_out is set to -1 when doing final directory 
251                    permission and modification time repair */
252                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1)) 
253                         rprintf(FINFO,"%s/\n",fname);
254                 return;
255         }
256
257         if (preserve_links && S_ISLNK(file->mode)) {
258 #if SUPPORT_LINKS
259                 char lnk[MAXPATHLEN];
260                 int l;
261                 extern int safe_symlinks;
262
263                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
264                         if (verbose) {
265                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
266                                         fname,file->link);
267                         }
268                         return;
269                 }
270                 if (statret == 0) {
271                         l = readlink(fname,lnk,MAXPATHLEN-1);
272                         if (l > 0) {
273                                 lnk[l] = 0;
274                                 if (strcmp(lnk,file->link) == 0) {
275                                         set_perms(fname,file,&st,1);
276                                         return;
277                                 }
278                         }
279                         delete_file(fname);
280                 }
281                 if (do_symlink(file->link,fname) != 0) {
282                         rprintf(FERROR,"symlink %s -> %s : %s\n",
283                                 fname,file->link,strerror(errno));
284                 } else {
285                         set_perms(fname,file,NULL,0);
286                         if (verbose) {
287                                 rprintf(FINFO,"%s -> %s\n",
288                                         fname,file->link);
289                         }
290                 }
291 #endif
292                 return;
293         }
294
295 #ifdef HAVE_MKNOD
296         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
297                 if (statret != 0 || 
298                     st.st_mode != file->mode ||
299                     st.st_rdev != file->rdev) { 
300                         delete_file(fname);
301                         if (verbose > 2)
302                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
303                                         fname,(int)file->mode,(int)file->rdev);
304                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
305                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
306                         } else {
307                                 set_perms(fname,file,NULL,0);
308                                 if (verbose)
309                                         rprintf(FINFO,"%s\n",fname);
310                         }
311                 } else {
312                         set_perms(fname,file,&st,1);
313                 }
314                 return;
315         }
316 #endif
317
318         if (preserve_hard_links && check_hard_link(file)) {
319                 if (verbose > 1)
320                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
321                 return;
322         }
323
324         if (!S_ISREG(file->mode)) {
325                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
326                 return;
327         }
328
329         fnamecmp = fname;
330
331         if ((statret == -1) && (compare_dest != NULL)) {
332                 /* try the file at compare_dest instead */
333                 int saveerrno = errno;
334                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
335                 statret = link_stat(fnamecmpbuf,&st);
336                 if (!S_ISREG(st.st_mode))
337                         statret = -1;
338                 if (statret == -1)
339                         errno = saveerrno;
340                 else
341                         fnamecmp = fnamecmpbuf;
342         }
343
344         if (statret == -1) {
345                 if (errno == ENOENT) {
346                         write_int(f_out,i);
347                         if (!dry_run) send_sums(NULL,f_out);
348                 } else {
349                         if (verbose > 1)
350                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
351                 }
352                 return;
353         }
354
355         if (!S_ISREG(st.st_mode)) {
356                 if (delete_file(fname) != 0) {
357                         return;
358                 }
359
360                 /* now pretend the file didn't exist */
361                 write_int(f_out,i);
362                 if (!dry_run) send_sums(NULL,f_out);    
363                 return;
364         }
365
366         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
367                 if (verbose > 1)
368                         rprintf(FINFO,"%s is newer\n",fname);
369                 return;
370         }
371
372         if (skip_file(fname, file, &st)) {
373                 if (fnamecmp == fname)
374                         set_perms(fname,file,&st,1);
375                 return;
376         }
377
378         if (dry_run) {
379                 write_int(f_out,i);
380                 return;
381         }
382
383         if (whole_file) {
384                 write_int(f_out,i);
385                 send_sums(NULL,f_out);    
386                 return;
387         }
388
389         /* open the file */  
390         fd = do_open(fnamecmp, O_RDONLY, 0);
391
392         if (fd == -1) {
393                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
394                 /* pretend the file didn't exist */
395                 write_int(f_out,i);
396                 send_sums(NULL,f_out);
397                 return;
398         }
399
400         if (st.st_size > 0) {
401                 buf = map_file(fd,st.st_size);
402         } else {
403                 buf = NULL;
404         }
405
406         if (verbose > 3)
407                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
408
409         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
410
411         if (verbose > 2)
412                 rprintf(FINFO,"sending sums for %d\n",i);
413
414         write_int(f_out,i);
415         send_sums(s,f_out);
416
417         close(fd);
418         if (buf) unmap_file(buf);
419
420         free_sums(s);
421 }
422
423
424
425 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
426 {
427         int i;
428         int phase=0;
429
430         if (verbose > 2)
431                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
432                         (int)getpid(),flist->count);
433
434         /* we expect to just sit around now, so don't exit on a
435            timeout. If we really get a timeout then the other process should
436            exit */
437         io_timeout = 0;
438
439         for (i = 0; i < flist->count; i++) {
440                 struct file_struct *file = flist->files[i];
441                 mode_t saved_mode = file->mode;
442                 if (!file->basename) continue;
443
444                 /* we need to ensure that any directories we create have writeable
445                    permissions initially so that we can create the files within
446                    them. This is then fixed after the files are transferred */
447                 if (!am_root && S_ISDIR(file->mode)) {
448                         file->mode |= S_IWUSR; /* user write */
449                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
450                          * handling of permissions is strange? */
451                 }
452
453                 recv_generator(local_name?local_name:f_name(file),
454                                flist,i,f);
455
456                 file->mode = saved_mode;
457         }
458
459         phase++;
460         csum_length = SUM_LENGTH;
461         ignore_times=1;
462
463         if (verbose > 2)
464                 rprintf(FINFO,"generate_files phase=%d\n",phase);
465
466         write_int(f,-1);
467
468         if (remote_version >= 13) {
469                 /* in newer versions of the protocol the files can cycle through
470                    the system more than once to catch initial checksum errors */
471                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
472                         struct file_struct *file = flist->files[i];
473                         recv_generator(local_name?local_name:f_name(file),
474                                        flist,i,f);    
475                 }
476
477                 phase++;
478                 if (verbose > 2)
479                         rprintf(FINFO,"generate_files phase=%d\n",phase);
480
481                 write_int(f,-1);
482         }
483 }