generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int io_timeout;
  35 extern int remote_version;
  36 extern int always_checksum;
  37
  38
  39 /* choose whether to skip a particular file */
  40 static int skip_file(char *fname,
  41                      struct file_struct *file, STRUCT_STAT *st)
  42 {
  43         if (st->st_size != file->length) {
  44                 return 0;
  45         }
  46
  47         /* if always checksum is set then we use the checksum instead
  48            of the file time to determine whether to sync */
  49         if (always_checksum && S_ISREG(st->st_mode)) {
  50                 char sum[MD4_SUM_LENGTH];
  51                 file_checksum(fname,sum,st->st_size);
  52                 return (memcmp(sum,file->sum,csum_length) == 0);
  53         }
  54
  55         if (ignore_times) {
  56                 return 0;
  57         }
  58
  59         return (st->st_mtime == file->modtime);
  60 }
  61
  62
  63 /* use a larger block size for really big files */
  64 static int adapt_block_size(struct file_struct *file, int bsize)
  65 {
  66         int ret;
  67
  68         if (bsize != BLOCK_SIZE) return bsize;
  69
  70         ret = file->length / (10000); /* rough heuristic */
  71         ret = ret & ~15; /* multiple of 16 */
  72         if (ret < bsize) ret = bsize;
  73         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  74         return ret;
  75 }
  76
  77
  78 /*
  79   send a sums struct down a fd
  80   */
  81 static void send_sums(struct sum_struct *s,int f_out)
  82 {
  83         int i;
  84
  85   /* tell the other guy how many we are going to be doing and how many
  86      bytes there are in the last chunk */
  87         write_int(f_out,s?s->count:0);
  88         write_int(f_out,s?s->n:block_size);
  89         write_int(f_out,s?s->remainder:0);
  90         if (s)
  91                 for (i=0;i<s->count;i++) {
  92                         write_int(f_out,s->sums[i].sum1);
  93                         write_buf(f_out,s->sums[i].sum2,csum_length);
  94                 }
  95 }
  96
  97
  98 /*
  99   generate a stream of signatures/checksums that describe a buffer
 100
 101   generate approximately one checksum every n bytes
 102   */
 103 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 104 {
 105         int i;
 106         struct sum_struct *s;
 107         int count;
 108         int block_len = n;
 109         int remainder = (len%block_len);
 110         OFF_T offset = 0;
 111
 112         count = (len+(block_len-1))/block_len;
 113
 114         s = (struct sum_struct *)malloc(sizeof(*s));
 115         if (!s) out_of_memory("generate_sums");
 116
 117         s->count = count;
 118         s->remainder = remainder;
 119         s->n = n;
 120         s->flength = len;
 121
 122         if (count==0) {
 123                 s->sums = NULL;
 124                 return s;
 125         }
 126
 127         if (verbose > 3)
 128                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
 129                         s->count,s->remainder,s->n,(int)s->flength);
 130
 131         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 132         if (!s->sums) out_of_memory("generate_sums");
 133
 134         for (i=0;i<count;i++) {
 135                 int n1 = MIN(len,n);
 136                 char *map = map_ptr(buf,offset,n1);
 137
 138                 s->sums[i].sum1 = get_checksum1(map,n1);
 139                 get_checksum2(map,n1,s->sums[i].sum2);
 140
 141                 s->sums[i].offset = offset;
 142                 s->sums[i].len = n1;
 143                 s->sums[i].i = i;
 144
 145                 if (verbose > 3)
 146                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
 147                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 148
 149                 len -= n1;
 150                 offset += n1;
 151         }
 152
 153         return s;
 154 }
 155
 156
 157 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 158 {
 159         int fd;
 160         STRUCT_STAT st;
 161         struct map_struct *buf;
 162         struct sum_struct *s;
 163         int statret;
 164         struct file_struct *file = flist->files[i];
 165         char *fnamecmp;
 166         char fnamecmpbuf[MAXPATHLEN];
 167         extern char *compare_dest;
 168         extern int list_only;
 169
 170         if (list_only) return;
 171
 172         if (verbose > 2)
 173                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 174
 175         statret = link_stat(fname,&st);
 176
 177         if (S_ISDIR(file->mode)) {
 178                 if (dry_run) return;
 179                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 180                         if (do_unlink(fname) != 0) {
 181                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 182                                 return;
 183                         }
 184                         statret = -1;
 185                 }
 186                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 187                         if (!(relative_paths && errno==ENOENT &&
 188                               create_directory_path(fname)==0 &&
 189                               do_mkdir(fname,file->mode)==0)) {
 190                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 191                                         fname,strerror(errno));
 192                         }
 193                 }
 194                 if (set_perms(fname,file,NULL,0) && verbose)
 195                         rprintf(FINFO,"%s/\n",fname);
 196                 return;
 197         }
 198
 199         if (preserve_links && S_ISLNK(file->mode)) {
 200 #if SUPPORT_LINKS
 201                 char lnk[MAXPATHLEN];
 202                 int l;
 203                 extern int safe_symlinks;
 204
 205                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 206                         if (verbose) {
 207                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 208                                         fname,file->link);
 209                         }
 210                         return;
 211                 }
 212                 if (statret == 0) {
 213                         l = readlink(fname,lnk,MAXPATHLEN-1);
 214                         if (l > 0) {
 215                                 lnk[l] = 0;
 216                                 if (strcmp(lnk,file->link) == 0) {
 217                                         set_perms(fname,file,&st,1);
 218                                         return;
 219                                 }
 220                         }
 221                 }
 222                 delete_file(fname);
 223                 if (do_symlink(file->link,fname) != 0) {
 224                         rprintf(FERROR,"link %s -> %s : %s\n",
 225                                 fname,file->link,strerror(errno));
 226                 } else {
 227                         set_perms(fname,file,NULL,0);
 228                         if (verbose) {
 229                                 rprintf(FINFO,"%s -> %s\n",
 230                                         fname,file->link);
 231                         }
 232                 }
 233 #endif
 234                 return;
 235         }
 236
 237 #ifdef HAVE_MKNOD
 238         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 239                 if (statret != 0 ||
 240                     st.st_mode != file->mode ||
 241                     st.st_rdev != file->rdev) {
 242                         delete_file(fname);
 243                         if (verbose > 2)
 244                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 245                                         fname,(int)file->mode,(int)file->rdev);
 246                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 247                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 248                         } else {
 249                                 set_perms(fname,file,NULL,0);
 250                                 if (verbose)
 251                                         rprintf(FINFO,"%s\n",fname);
 252                         }
 253                 } else {
 254                         set_perms(fname,file,&st,1);
 255                 }
 256                 return;
 257         }
 258 #endif
 259
 260         if (preserve_hard_links && check_hard_link(file)) {
 261                 if (verbose > 1)
 262                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 263                 return;
 264         }
 265
 266         if (!S_ISREG(file->mode)) {
 267                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 268                 return;
 269         }
 270
 271         fnamecmp = fname;
 272
 273         if ((statret == -1) && (compare_dest != NULL)) {
 274                 /* try the file at compare_dest instead */
 275                 int saveerrno = errno;
 276                 slprintf(fnamecmpbuf,MAXPATHLEN-1,"%s/%s",compare_dest,fname);
 277                 statret = link_stat(fnamecmpbuf,&st);
 278                 if (!S_ISREG(st.st_mode))
 279                         statret = -1;
 280                 if (statret == -1)
 281                         errno = saveerrno;
 282                 else
 283                         fnamecmp = fnamecmpbuf;
 284         }
 285
 286         if (statret == -1) {
 287                 if (errno == ENOENT) {
 288                         write_int(f_out,i);
 289                         if (!dry_run) send_sums(NULL,f_out);
 290                 } else {
 291                         if (verbose > 1)
 292                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 293                 }
 294                 return;
 295         }
 296
 297         if (!S_ISREG(st.st_mode)) {
 298                 if (delete_file(fname) != 0) {
 299                         return;
 300                 }
 301
 302                 /* now pretend the file didn't exist */
 303                 write_int(f_out,i);
 304                 if (!dry_run) send_sums(NULL,f_out);
 305                 return;
 306         }
 307
 308         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 309                 if (verbose > 1)
 310                         rprintf(FINFO,"%s is newer\n",fname);
 311                 return;
 312         }
 313
 314         if (skip_file(fname, file, &st)) {
 315                 set_perms(fname,file,&st,1);
 316                 return;
 317         }
 318
 319         if (dry_run) {
 320                 write_int(f_out,i);
 321                 return;
 322         }
 323
 324         if (whole_file) {
 325                 write_int(f_out,i);
 326                 send_sums(NULL,f_out);
 327                 return;
 328         }
 329
 330         /* open the file */
 331         fd = open(fnamecmp,O_RDONLY);
 332
 333         if (fd == -1) {
 334                 rprintf(FERROR,"failed to open %s : %s\n",fnamecmp,strerror(errno));
 335                 rprintf(FERROR,"skipping %s\n",fname);
 336                 return;
 337         }
 338
 339         if (st.st_size > 0) {
 340                 buf = map_file(fd,st.st_size);
 341         } else {
 342                 buf = NULL;
 343         }
 344
 345         if (verbose > 3)
 346                 rprintf(FINFO,"gen mapped %s of size %d\n",fnamecmp,(int)st.st_size);
 347
 348         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 349
 350         if (verbose > 2)
 351                 rprintf(FINFO,"sending sums for %d\n",i);
 352
 353         write_int(f_out,i);
 354         send_sums(s,f_out);
 355
 356         close(fd);
 357         if (buf) unmap_file(buf);
 358
 359         free_sums(s);
 360 }
 361
 362
 363
 364 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 365 {
 366         int i;
 367         int phase=0;
 368
 369         if (verbose > 2)
 370                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 371                         (int)getpid(),flist->count);
 372
 373         for (i = 0; i < flist->count; i++) {
 374                 struct file_struct *file = flist->files[i];
 375                 mode_t saved_mode = file->mode;
 376                 if (!file->basename) continue;
 377
 378                 /* we need to ensure that any directories we create have writeable
 379                    permissions initially so that we can create the files within
 380                    them. This is then fixed after the files are transferred */
 381                 if (!am_root && S_ISDIR(file->mode)) {
 382                         file->mode |= S_IWUSR; /* user write */
 383                 }
 384
 385                 recv_generator(local_name?local_name:f_name(file),
 386                                flist,i,f);
 387
 388                 file->mode = saved_mode;
 389         }
 390
 391         phase++;
 392         csum_length = SUM_LENGTH;
 393         ignore_times=1;
 394
 395         if (verbose > 2)
 396                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 397
 398         write_int(f,-1);
 399
 400         /* we expect to just sit around now, so don't exit on a
 401            timeout. If we really get a timeout then the other process should
 402            exit */
 403         io_timeout = 0;
 404
 405         if (remote_version >= 13) {
 406                 /* in newer versions of the protocol the files can cycle through
 407                    the system more than once to catch initial checksum errors */
 408                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 409                         struct file_struct *file = flist->files[i];
 410                         recv_generator(local_name?local_name:f_name(file),
 411                                        flist,i,f);
 412                 }
 413
 414                 phase++;
 415                 if (verbose > 2)
 416                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 417
 418                 write_int(f,-1);
 419         }
 420 }