78d867c6e6b9c755c50854fa85d05262143e97dd
[rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2    
3    Copyright (C) 1996-2000 by Andrew Tridgell 
4    Copyright (C) Paul Mackerras 1996
5    
6    This program is free software; you can redistribute it and/or modify
7    it under the terms of the GNU General Public License as published by
8    the Free Software Foundation; either version 2 of the License, or
9    (at your option) any later version.
10    
11    This program is distributed in the hope that it will be useful,
12    but WITHOUT ANY WARRANTY; without even the implied warranty of
13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14    GNU General Public License for more details.
15    
16    You should have received a copy of the GNU General Public License
17    along with this program; if not, write to the Free Software
18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21 #include "rsync.h"
22
23 extern int verbose;
24 extern int dry_run;
25 extern int relative_paths;
26 extern int preserve_links;
27 extern int am_root;
28 extern int preserve_devices;
29 extern int preserve_hard_links;
30 extern int update_only;
31 extern int whole_file;
32 extern int block_size;
33 extern int csum_length;
34 extern int ignore_times;
35 extern int size_only;
36 extern int io_timeout;
37 extern int remote_version;
38 extern int always_checksum;
39 extern int modify_window;
40 extern char *compare_dest;
41
42
43 /* choose whether to skip a particular file */
44 static int skip_file(char *fname,
45                      struct file_struct *file, STRUCT_STAT *st)
46 {
47         if (st->st_size != file->length) {
48                 return 0;
49         }
50         
51         /* if always checksum is set then we use the checksum instead 
52            of the file time to determine whether to sync */
53         if (always_checksum && S_ISREG(st->st_mode)) {
54                 char sum[MD4_SUM_LENGTH];
55                 char fnamecmpdest[MAXPATHLEN];
56
57                 if (compare_dest != NULL) {
58                         if (access(fname, 0) != 0) {
59                                 slprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
60                                                     compare_dest,fname);
61                                 fname = fnamecmpdest;
62                         }
63                 }
64                 file_checksum(fname,sum,st->st_size);
65                 if (remote_version < 21) {
66                         return (memcmp(sum,file->sum,2) == 0);
67                 } else {
68                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
69                 }
70         }
71
72         if (size_only) {
73                 return 1;
74         }
75
76         if (ignore_times) {
77                 return 0;
78         }
79
80         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
81 }
82
83
84 /* use a larger block size for really big files */
85 static int adapt_block_size(struct file_struct *file, int bsize)
86 {
87         int ret;
88
89         if (bsize != BLOCK_SIZE) return bsize;
90
91         ret = file->length / (10000); /* rough heuristic */
92         ret = ret & ~15; /* multiple of 16 */
93         if (ret < bsize) ret = bsize;
94         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
95         return ret;
96 }
97
98
99 /*
100   send a sums struct down a fd
101   */
102 static void send_sums(struct sum_struct *s,int f_out)
103 {
104         int i;
105         
106         /* tell the other guy how many we are going to be doing and how many
107            bytes there are in the last chunk */
108         write_int(f_out,s?s->count:0);
109         write_int(f_out,s?s->n:block_size);
110         write_int(f_out,s?s->remainder:0);
111
112         if (!s) return;
113
114         for (i=0;i<s->count;i++) {
115                 write_int(f_out,s->sums[i].sum1);
116                 write_buf(f_out,s->sums[i].sum2,csum_length);
117         }
118 }
119
120
121 /*
122   generate a stream of signatures/checksums that describe a buffer
123
124   generate approximately one checksum every n bytes
125   */
126 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
127 {
128         int i;
129         struct sum_struct *s;
130         int count;
131         int block_len = n;
132         int remainder = (len%block_len);
133         OFF_T offset = 0;
134
135         count = (len+(block_len-1))/block_len;
136
137         s = (struct sum_struct *)malloc(sizeof(*s));
138         if (!s) out_of_memory("generate_sums");
139
140         s->count = count;
141         s->remainder = remainder;
142         s->n = n;
143         s->flength = len;
144
145         if (count==0) {
146                 s->sums = NULL;
147                 return s;
148         }
149
150         if (verbose > 3)
151                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
152                         s->count,s->remainder,s->n,(double)s->flength);
153
154         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
155         if (!s->sums) out_of_memory("generate_sums");
156   
157         for (i=0;i<count;i++) {
158                 int n1 = MIN(len,n);
159                 char *map = map_ptr(buf,offset,n1);
160
161                 s->sums[i].sum1 = get_checksum1(map,n1);
162                 get_checksum2(map,n1,s->sums[i].sum2);
163
164                 s->sums[i].offset = offset;
165                 s->sums[i].len = n1;
166                 s->sums[i].i = i;
167
168                 if (verbose > 3)
169                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
170                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
171
172                 len -= n1;
173                 offset += n1;
174         }
175
176         return s;
177 }
178
179
180
181 /*
182  * Acts on file number I from FLIST, whose name is fname.
183  *
184  * First fixes up permissions, then generates checksums for the file.
185  *
186  * (This comment was added later by mbp who was trying to work it out;
187  * it might be wrong.)
188  */ 
189 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
190 {  
191         int fd;
192         STRUCT_STAT st;
193         struct map_struct *buf;
194         struct sum_struct *s;
195         int statret;
196         struct file_struct *file = flist->files[i];
197         char *fnamecmp;
198         char fnamecmpbuf[MAXPATHLEN];
199         extern char *compare_dest;
200         extern int list_only;
201         extern int preserve_perms;
202         extern int only_existing;
203
204         if (list_only) return;
205
206         if (verbose > 2)
207                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
208
209         statret = link_stat(fname,&st);
210
211         if (only_existing && statret == -1 && errno == ENOENT) {
212                 /* we only want to update existing files */
213                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
214                 return;
215         }
216
217         if (statret == 0 && 
218             !preserve_perms && 
219             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
220                 /* if the file exists already and we aren't perserving
221                    presmissions then act as though the remote end sent
222                    us the file permissions we already have */
223                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
224         }
225
226         if (S_ISDIR(file->mode)) {
227                 if (dry_run) return;
228                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
229                         if (robust_unlink(fname) != 0) {
230                                 rprintf(FERROR,"recv_generator: unlink %s: %s\n",fname,strerror(errno));
231                                 return;
232                         }
233                         statret = -1;
234                 }
235                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
236                         if (!(relative_paths && errno==ENOENT && 
237                               create_directory_path(fname)==0 && 
238                               do_mkdir(fname,file->mode)==0)) {
239                                 rprintf(FERROR,"recv_generator: mkdir %s: %s (2)\n",
240                                         fname,strerror(errno));
241                         }
242                 }
243                 if (set_perms(fname,file,NULL,0) && verbose) 
244                         rprintf(FINFO,"%s/\n",fname);
245                 return;
246         }
247
248         if (preserve_links && S_ISLNK(file->mode)) {
249 #if SUPPORT_LINKS
250                 char lnk[MAXPATHLEN];
251                 int l;
252                 extern int safe_symlinks;
253
254                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
255                         if (verbose) {
256                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
257                                         fname,file->link);
258                         }
259                         return;
260                 }
261                 if (statret == 0) {
262                         l = readlink(fname,lnk,MAXPATHLEN-1);
263                         if (l > 0) {
264                                 lnk[l] = 0;
265                                 if (strcmp(lnk,file->link) == 0) {
266                                         set_perms(fname,file,&st,1);
267                                         return;
268                                 }
269                         }
270                         delete_file(fname);
271                 }
272                 if (do_symlink(file->link,fname) != 0) {
273                         rprintf(FERROR,"symlink %s -> %s : %s\n",
274                                 fname,file->link,strerror(errno));
275                 } else {
276                         set_perms(fname,file,NULL,0);
277                         if (verbose) {
278                                 rprintf(FINFO,"%s -> %s\n",
279                                         fname,file->link);
280                         }
281                 }
282 #endif
283                 return;
284         }
285
286 #ifdef HAVE_MKNOD
287         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
288                 if (statret != 0 || 
289                     st.st_mode != file->mode ||
290                     st.st_rdev != file->rdev) { 
291                         delete_file(fname);
292                         if (verbose > 2)
293                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
294                                         fname,(int)file->mode,(int)file->rdev);
295                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
296                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
297                         } else {
298                                 set_perms(fname,file,NULL,0);
299                                 if (verbose)
300                                         rprintf(FINFO,"%s\n",fname);
301                         }
302                 } else {
303                         set_perms(fname,file,&st,1);
304                 }
305                 return;
306         }
307 #endif
308
309         if (preserve_hard_links && check_hard_link(file)) {
310                 if (verbose > 1)
311                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
312                 return;
313         }
314
315         if (!S_ISREG(file->mode)) {
316                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
317                 return;
318         }
319
320         fnamecmp = fname;
321
322         if ((statret == -1) && (compare_dest != NULL)) {
323                 /* try the file at compare_dest instead */
324                 int saveerrno = errno;
325                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
326                 statret = link_stat(fnamecmpbuf,&st);
327                 if (!S_ISREG(st.st_mode))
328                         statret = -1;
329                 if (statret == -1)
330                         errno = saveerrno;
331                 else
332                         fnamecmp = fnamecmpbuf;
333         }
334
335         if (statret == -1) {
336                 if (errno == ENOENT) {
337                         write_int(f_out,i);
338                         if (!dry_run) send_sums(NULL,f_out);
339                 } else {
340                         if (verbose > 1)
341                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
342                 }
343                 return;
344         }
345
346         if (!S_ISREG(st.st_mode)) {
347                 if (delete_file(fname) != 0) {
348                         return;
349                 }
350
351                 /* now pretend the file didn't exist */
352                 write_int(f_out,i);
353                 if (!dry_run) send_sums(NULL,f_out);    
354                 return;
355         }
356
357         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
358                 if (verbose > 1)
359                         rprintf(FINFO,"%s is newer\n",fname);
360                 return;
361         }
362
363         if (skip_file(fname, file, &st)) {
364                 if (fnamecmp == fname)
365                         set_perms(fname,file,&st,1);
366                 return;
367         }
368
369         if (dry_run) {
370                 write_int(f_out,i);
371                 return;
372         }
373
374         if (whole_file) {
375                 write_int(f_out,i);
376                 send_sums(NULL,f_out);    
377                 return;
378         }
379
380         /* open the file */  
381         fd = do_open(fnamecmp, O_RDONLY, 0);
382
383         if (fd == -1) {
384                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
385                 /* pretend the file didn't exist */
386                 write_int(f_out,i);
387                 send_sums(NULL,f_out);
388                 return;
389         }
390
391         if (st.st_size > 0) {
392                 buf = map_file(fd,st.st_size);
393         } else {
394                 buf = NULL;
395         }
396
397         if (verbose > 3)
398                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
399
400         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
401
402         if (verbose > 2)
403                 rprintf(FINFO,"sending sums for %d\n",i);
404
405         write_int(f_out,i);
406         send_sums(s,f_out);
407
408         close(fd);
409         if (buf) unmap_file(buf);
410
411         free_sums(s);
412 }
413
414
415
416 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
417 {
418         int i;
419         int phase=0;
420
421         if (verbose > 2)
422                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
423                         (int)getpid(),flist->count);
424
425         for (i = 0; i < flist->count; i++) {
426                 struct file_struct *file = flist->files[i];
427                 mode_t saved_mode = file->mode;
428                 if (!file->basename) continue;
429
430                 /* we need to ensure that any directories we create have writeable
431                    permissions initially so that we can create the files within
432                    them. This is then fixed after the files are transferred */
433                 if (!am_root && S_ISDIR(file->mode)) {
434                         file->mode |= S_IWUSR; /* user write */
435                 }
436
437                 recv_generator(local_name?local_name:f_name(file),
438                                flist,i,f);
439
440                 file->mode = saved_mode;
441         }
442
443         phase++;
444         csum_length = SUM_LENGTH;
445         ignore_times=1;
446
447         if (verbose > 2)
448                 rprintf(FINFO,"generate_files phase=%d\n",phase);
449
450         write_int(f,-1);
451
452         /* we expect to just sit around now, so don't exit on a
453            timeout. If we really get a timeout then the other process should
454            exit */
455         io_timeout = 0;
456
457         if (remote_version >= 13) {
458                 /* in newer versions of the protocol the files can cycle through
459                    the system more than once to catch initial checksum errors */
460                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
461                         struct file_struct *file = flist->files[i];
462                         recv_generator(local_name?local_name:f_name(file),
463                                        flist,i,f);    
464                 }
465
466                 phase++;
467                 if (verbose > 2)
468                         rprintf(FINFO,"generate_files phase=%d\n",phase);
469
470                 write_int(f,-1);
471         }
472 }