rsync.c was getting a bit unwieldy so I split the code into 3 modules,
[rsync.git] / generator.c
1 /* 
2    Copyright (C) Andrew Tridgell 1996
3    Copyright (C) Paul Mackerras 1996
4    
5    This program is free software; you can redistribute it and/or modify
6    it under the terms of the GNU General Public License as published by
7    the Free Software Foundation; either version 2 of the License, or
8    (at your option) any later version.
9    
10    This program is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13    GNU General Public License for more details.
14    
15    You should have received a copy of the GNU General Public License
16    along with this program; if not, write to the Free Software
17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20 #include "rsync.h"
21
22 extern int verbose;
23 extern int dry_run;
24 extern int relative_paths;
25 extern int preserve_links;
26 extern int am_root;
27 extern int preserve_devices;
28 extern int preserve_hard_links;
29 extern int update_only;
30 extern int whole_file;
31 extern int block_size;
32 extern int csum_length;
33 extern int ignore_times;
34 extern int io_timeout;
35 extern int remote_version;
36 extern int always_checksum;
37
38
39 /* choose whether to skip a particular file */
40 static int skip_file(char *fname,
41                      struct file_struct *file, STRUCT_STAT *st)
42 {
43         if (st->st_size != file->length) {
44                 return 0;
45         }
46         
47         /* if always checksum is set then we use the checksum instead 
48            of the file time to determine whether to sync */
49         if (always_checksum && S_ISREG(st->st_mode)) {
50                 char sum[MD4_SUM_LENGTH];
51                 file_checksum(fname,sum,st->st_size);
52                 return (memcmp(sum,file->sum,csum_length) == 0);
53         }
54
55         if (ignore_times) {
56                 return 0;
57         }
58
59         return (st->st_mtime == file->modtime);
60 }
61
62
63 /* use a larger block size for really big files */
64 static int adapt_block_size(struct file_struct *file, int bsize)
65 {
66         int ret;
67
68         if (bsize != BLOCK_SIZE) return bsize;
69
70         ret = file->length / (10000); /* rough heuristic */
71         ret = ret & ~15; /* multiple of 16 */
72         if (ret < bsize) ret = bsize;
73         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
74         return ret;
75 }
76
77
78 /*
79   send a sums struct down a fd
80   */
81 static void send_sums(struct sum_struct *s,int f_out)
82 {
83         int i;
84
85   /* tell the other guy how many we are going to be doing and how many
86      bytes there are in the last chunk */
87         write_int(f_out,s?s->count:0);
88         write_int(f_out,s?s->n:block_size);
89         write_int(f_out,s?s->remainder:0);
90         if (s)
91                 for (i=0;i<s->count;i++) {
92                         write_int(f_out,s->sums[i].sum1);
93                         write_buf(f_out,s->sums[i].sum2,csum_length);
94                 }
95 }
96
97
98 /*
99   generate a stream of signatures/checksums that describe a buffer
100
101   generate approximately one checksum every n bytes
102   */
103 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
104 {
105         int i;
106         struct sum_struct *s;
107         int count;
108         int block_len = n;
109         int remainder = (len%block_len);
110         OFF_T offset = 0;
111
112         count = (len+(block_len-1))/block_len;
113
114         s = (struct sum_struct *)malloc(sizeof(*s));
115         if (!s) out_of_memory("generate_sums");
116
117         s->count = count;
118         s->remainder = remainder;
119         s->n = n;
120         s->flength = len;
121
122         if (count==0) {
123                 s->sums = NULL;
124                 return s;
125         }
126
127         if (verbose > 3)
128                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
129                         s->count,s->remainder,s->n,(int)s->flength);
130
131         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
132         if (!s->sums) out_of_memory("generate_sums");
133   
134         for (i=0;i<count;i++) {
135                 int n1 = MIN(len,n);
136                 char *map = map_ptr(buf,offset,n1);
137
138                 s->sums[i].sum1 = get_checksum1(map,n1);
139                 get_checksum2(map,n1,s->sums[i].sum2);
140
141                 s->sums[i].offset = offset;
142                 s->sums[i].len = n1;
143                 s->sums[i].i = i;
144
145                 if (verbose > 3)
146                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
147                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
148
149                 len -= n1;
150                 offset += n1;
151         }
152
153         return s;
154 }
155
156
157 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
158 {  
159         int fd;
160         STRUCT_STAT st;
161         struct map_struct *buf;
162         struct sum_struct *s;
163         int statret;
164         struct file_struct *file = flist->files[i];
165
166         if (verbose > 2)
167                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
168
169         statret = link_stat(fname,&st);
170
171         if (S_ISDIR(file->mode)) {
172                 if (dry_run) return;
173                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
174                         if (do_unlink(fname) != 0) {
175                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
176                                 return;
177                         }
178                         statret = -1;
179                 }
180                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
181                         if (!(relative_paths && errno==ENOENT && 
182                               create_directory_path(fname)==0 && 
183                               do_mkdir(fname,file->mode)==0)) {
184                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
185                                         fname,strerror(errno));
186                         }
187                 }
188                 if (set_perms(fname,file,NULL,0) && verbose) 
189                         rprintf(FINFO,"%s/\n",fname);
190                 return;
191         }
192
193         if (preserve_links && S_ISLNK(file->mode)) {
194 #if SUPPORT_LINKS
195                 char lnk[MAXPATHLEN];
196                 int l;
197                 extern int safe_symlinks;
198
199                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
200                         if (verbose) {
201                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
202                                         fname,file->link);
203                         }
204                         return;
205                 }
206                 if (statret == 0) {
207                         l = readlink(fname,lnk,MAXPATHLEN-1);
208                         if (l > 0) {
209                                 lnk[l] = 0;
210                                 if (strcmp(lnk,file->link) == 0) {
211                                         set_perms(fname,file,&st,1);
212                                         return;
213                                 }
214                         }
215                 }
216                 delete_file(fname);
217                 if (do_symlink(file->link,fname) != 0) {
218                         rprintf(FERROR,"link %s -> %s : %s\n",
219                                 fname,file->link,strerror(errno));
220                 } else {
221                         set_perms(fname,file,NULL,0);
222                         if (verbose) {
223                                 rprintf(FINFO,"%s -> %s\n",
224                                         fname,file->link);
225                         }
226                 }
227 #endif
228                 return;
229         }
230
231 #ifdef HAVE_MKNOD
232         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
233                 if (statret != 0 || 
234                     st.st_mode != file->mode ||
235                     st.st_rdev != file->rdev) { 
236                         delete_file(fname);
237                         if (verbose > 2)
238                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
239                                         fname,(int)file->mode,(int)file->rdev);
240                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
241                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
242                         } else {
243                                 set_perms(fname,file,NULL,0);
244                                 if (verbose)
245                                         rprintf(FINFO,"%s\n",fname);
246                         }
247                 } else {
248                         set_perms(fname,file,&st,1);
249                 }
250                 return;
251         }
252 #endif
253
254         if (preserve_hard_links && check_hard_link(file)) {
255                 if (verbose > 1)
256                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
257                 return;
258         }
259
260         if (!S_ISREG(file->mode)) {
261                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
262                 return;
263         }
264
265         if (statret == -1) {
266                 if (errno == ENOENT) {
267                         write_int(f_out,i);
268                         if (!dry_run) send_sums(NULL,f_out);
269                 } else {
270                         if (verbose > 1)
271                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
272                 }
273                 return;
274         }
275
276         if (!S_ISREG(st.st_mode)) {
277                 if (delete_file(fname) != 0) {
278                         return;
279                 }
280
281                 /* now pretend the file didn't exist */
282                 write_int(f_out,i);
283                 if (!dry_run) send_sums(NULL,f_out);    
284                 return;
285         }
286
287         if (update_only && st.st_mtime > file->modtime) {
288                 if (verbose > 1)
289                         rprintf(FINFO,"%s is newer\n",fname);
290                 return;
291         }
292
293         if (skip_file(fname, file, &st)) {
294                 set_perms(fname,file,&st,1);
295                 return;
296         }
297
298         if (dry_run) {
299                 write_int(f_out,i);
300                 return;
301         }
302
303         if (whole_file) {
304                 write_int(f_out,i);
305                 send_sums(NULL,f_out);    
306                 return;
307         }
308
309         /* open the file */  
310         fd = open(fname,O_RDONLY);
311
312         if (fd == -1) {
313                 rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
314                 rprintf(FERROR,"skipping %s\n",fname);
315                 return;
316         }
317
318         if (st.st_size > 0) {
319                 buf = map_file(fd,st.st_size);
320         } else {
321                 buf = NULL;
322         }
323
324         if (verbose > 3)
325                 rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
326
327         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
328
329         if (verbose > 2)
330                 rprintf(FINFO,"sending sums for %d\n",i);
331
332         write_int(f_out,i);
333         send_sums(s,f_out);
334
335         close(fd);
336         if (buf) unmap_file(buf);
337
338         free_sums(s);
339 }
340
341
342
343 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
344 {
345         int i;
346         int phase=0;
347
348         if (verbose > 2)
349                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
350                         (int)getpid(),flist->count);
351
352         for (i = 0; i < flist->count; i++) {
353                 struct file_struct *file = flist->files[i];
354                 mode_t saved_mode = file->mode;
355                 if (!file->basename) continue;
356
357                 /* we need to ensure that any directories we create have writeable
358                    permissions initially so that we can create the files within
359                    them. This is then fixed after the files are transferred */
360                 if (!am_root && S_ISDIR(file->mode)) {
361                         file->mode |= S_IWUSR; /* user write */
362                 }
363
364                 recv_generator(local_name?local_name:f_name(file),
365                                flist,i,f);
366
367                 file->mode = saved_mode;
368         }
369
370         phase++;
371         csum_length = SUM_LENGTH;
372         ignore_times=1;
373
374         if (verbose > 2)
375                 rprintf(FINFO,"generate_files phase=%d\n",phase);
376
377         write_int(f,-1);
378
379         /* we expect to just sit around now, so don't exit on a
380            timeout. If we really get a timeout then the other process should
381            exit */
382         io_timeout = 0;
383
384         if (remote_version >= 13) {
385                 /* in newer versions of the protocol the files can cycle through
386                    the system more than once to catch initial checksum errors */
387                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
388                         struct file_struct *file = flist->files[i];
389                         recv_generator(local_name?local_name:f_name(file),
390                                        flist,i,f);    
391                 }
392
393                 phase++;
394                 if (verbose > 2)
395                         rprintf(FINFO,"generate_files phase=%d\n",phase);
396
397                 write_int(f,-1);
398         }
399 }