generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    Copyright (C) 1996-2000 by Andrew Tridgell
   4    Copyright (C) Paul Mackerras 1996
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, write to the Free Software
  18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21 #include "rsync.h"
  22
  23 extern int verbose;
  24 extern int dry_run;
  25 extern int relative_paths;
  26 extern int preserve_links;
  27 extern int am_root;
  28 extern int preserve_devices;
  29 extern int preserve_hard_links;
  30 extern int update_only;
  31 extern int whole_file;
  32 extern int block_size;
  33 extern int csum_length;
  34 extern int ignore_times;
  35 extern int size_only;
  36 extern int io_timeout;
  37 extern int remote_version;
  38 extern int always_checksum;
  39 extern int modify_window;
  40 extern char *compare_dest;
  41
  42
  43 /* choose whether to skip a particular file */
  44 static int skip_file(char *fname,
  45                      struct file_struct *file, STRUCT_STAT *st)
  46 {
  47         if (st->st_size != file->length) {
  48                 return 0;
  49         }
  50
  51         /* if always checksum is set then we use the checksum instead
  52            of the file time to determine whether to sync */
  53         if (always_checksum && S_ISREG(st->st_mode)) {
  54                 char sum[MD4_SUM_LENGTH];
  55                 char fnamecmpdest[MAXPATHLEN];
  56
  57                 if (compare_dest != NULL) {
  58                         if (access(fname, 0) != 0) {
  59                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  60                                                     compare_dest,fname);
  61                                 fname = fnamecmpdest;
  62                         }
  63                 }
  64                 file_checksum(fname,sum,st->st_size);
  65                 if (remote_version < 21) {
  66                         return (memcmp(sum,file->sum,2) == 0);
  67                 } else {
  68                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  69                 }
  70         }
  71
  72         if (size_only) {
  73                 return 1;
  74         }
  75
  76         if (ignore_times) {
  77                 return 0;
  78         }
  79
  80         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  81 }
  82
  83
  84 /* use a larger block size for really big files */
  85 static int adapt_block_size(struct file_struct *file, int bsize)
  86 {
  87         int ret;
  88
  89         if (bsize != BLOCK_SIZE) return bsize;
  90
  91         ret = file->length / (10000); /* rough heuristic */
  92         ret = ret & ~15; /* multiple of 16 */
  93         if (ret < bsize) ret = bsize;
  94         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  95         return ret;
  96 }
  97
  98
  99 /*
 100   send a sums struct down a fd
 101   */
 102 static void send_sums(struct sum_struct *s,int f_out)
 103 {
 104         int i;
 105
 106         /* tell the other guy how many we are going to be doing and how many
 107            bytes there are in the last chunk */
 108         write_int(f_out,s?s->count:0);
 109         write_int(f_out,s?s->n:block_size);
 110         write_int(f_out,s?s->remainder:0);
 111
 112         if (!s) return;
 113
 114         for (i=0;i<s->count;i++) {
 115                 write_int(f_out,s->sums[i].sum1);
 116                 write_buf(f_out,s->sums[i].sum2,csum_length);
 117         }
 118 }
 119
 120
 121 /*
 122   generate a stream of signatures/checksums that describe a buffer
 123
 124   generate approximately one checksum every n bytes
 125   */
 126 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 127 {
 128         int i;
 129         struct sum_struct *s;
 130         int count;
 131         int block_len = n;
 132         int remainder = (len%block_len);
 133         OFF_T offset = 0;
 134
 135         count = (len+(block_len-1))/block_len;
 136
 137         s = (struct sum_struct *)malloc(sizeof(*s));
 138         if (!s) out_of_memory("generate_sums");
 139
 140         s->count = count;
 141         s->remainder = remainder;
 142         s->n = n;
 143         s->flength = len;
 144
 145         if (count==0) {
 146                 s->sums = NULL;
 147                 return s;
 148         }
 149
 150         if (verbose > 3)
 151                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 152                         s->count,s->remainder,s->n,(double)s->flength);
 153
 154         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 155         if (!s->sums) out_of_memory("generate_sums");
 156
 157         for (i=0;i<count;i++) {
 158                 int n1 = MIN(len,n);
 159                 char *map = map_ptr(buf,offset,n1);
 160
 161                 s->sums[i].sum1 = get_checksum1(map,n1);
 162                 get_checksum2(map,n1,s->sums[i].sum2);
 163
 164                 s->sums[i].offset = offset;
 165                 s->sums[i].len = n1;
 166                 s->sums[i].i = i;
 167
 168                 if (verbose > 3)
 169                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 170                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 171
 172                 len -= n1;
 173                 offset += n1;
 174         }
 175
 176         return s;
 177 }
 178
 179
 180
 181 /*
 182  * Acts on file number I from FLIST, whose name is fname.
 183  *
 184  * First fixes up permissions, then generates checksums for the file.
 185  *
 186  * (This comment was added later by mbp who was trying to work it out;
 187  * it might be wrong.)
 188  */
 189 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 190 {
 191         int fd;
 192         STRUCT_STAT st;
 193         struct map_struct *buf;
 194         struct sum_struct *s;
 195         int statret;
 196         struct file_struct *file = flist->files[i];
 197         char *fnamecmp;
 198         char fnamecmpbuf[MAXPATHLEN];
 199         extern char *compare_dest;
 200         extern int list_only;
 201         extern int preserve_perms;
 202         extern int only_existing;
 203
 204         if (list_only) return;
 205
 206         if (verbose > 2)
 207                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 208
 209         statret = link_stat(fname,&st);
 210
 211         if (only_existing && statret == -1 && errno == ENOENT) {
 212                 /* we only want to update existing files */
 213                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
 214                 return;
 215         }
 216
 217         if (statret == 0 &&
 218             !preserve_perms &&
 219             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 220                 /* if the file exists already and we aren't perserving
 221                    presmissions then act as though the remote end sent
 222                    us the file permissions we already have */
 223                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 224         }
 225
 226         if (S_ISDIR(file->mode)) {
 227                 /* The file to be received is a directory, so we need
 228                  * to prepare appropriately.  If there is already a
 229                  * file of that name and it is *not* a directory, then
 230                  * we need to delete it.  If it doesn't exist, then
 231                  * recursively create it. */
 232
 233                 if (dry_run) return;
 234                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 235                         if (robust_unlink(fname) != 0) {
 236                                 rprintf(FERROR,"recv_generator: unlink %s: %s\n",
 237                                         fname,strerror(errno));
 238                                 return;
 239                         }
 240                         statret = -1;
 241                 }
 242                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 243                         if (!(relative_paths && errno==ENOENT &&
 244                               create_directory_path(fname)==0 &&
 245                               do_mkdir(fname,file->mode)==0)) {
 246                                 rprintf(FERROR,"recv_generator: mkdir %s: %s (2)\n",
 247                                         fname,strerror(errno));
 248                         }
 249                 }
 250                 /* f_out is set to -1 when doing final directory
 251                    permission and modification time repair */
 252                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 253                         rprintf(FINFO,"%s/\n",fname);
 254                 return;
 255         }
 256
 257         if (preserve_links && S_ISLNK(file->mode)) {
 258 #if SUPPORT_LINKS
 259                 char lnk[MAXPATHLEN];
 260                 int l;
 261                 extern int safe_symlinks;
 262
 263                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 264                         if (verbose) {
 265                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 266                                         fname,file->link);
 267                         }
 268                         return;
 269                 }
 270                 if (statret == 0) {
 271                         l = readlink(fname,lnk,MAXPATHLEN-1);
 272                         if (l > 0) {
 273                                 lnk[l] = 0;
 274                                 if (strcmp(lnk,file->link) == 0) {
 275                                         set_perms(fname,file,&st,1);
 276                                         return;
 277                                 }
 278                         }
 279                         delete_file(fname);
 280                 }
 281                 if (do_symlink(file->link,fname) != 0) {
 282                         rprintf(FERROR,"symlink %s -> %s : %s\n",
 283                                 fname,file->link,strerror(errno));
 284                 } else {
 285                         set_perms(fname,file,NULL,0);
 286                         if (verbose) {
 287                                 rprintf(FINFO,"%s -> %s\n",
 288                                         fname,file->link);
 289                         }
 290                 }
 291 #endif
 292                 return;
 293         }
 294
 295 #ifdef HAVE_MKNOD
 296         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 297                 if (statret != 0 ||
 298                     st.st_mode != file->mode ||
 299                     st.st_rdev != file->rdev) {
 300                         delete_file(fname);
 301                         if (verbose > 2)
 302                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 303                                         fname,(int)file->mode,(int)file->rdev);
 304                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 305                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 306                         } else {
 307                                 set_perms(fname,file,NULL,0);
 308                                 if (verbose)
 309                                         rprintf(FINFO,"%s\n",fname);
 310                         }
 311                 } else {
 312                         set_perms(fname,file,&st,1);
 313                 }
 314                 return;
 315         }
 316 #endif
 317
 318         if (preserve_hard_links && check_hard_link(file)) {
 319                 if (verbose > 1)
 320                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 321                 return;
 322         }
 323
 324         if (!S_ISREG(file->mode)) {
 325                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 326                 return;
 327         }
 328
 329         fnamecmp = fname;
 330
 331         if ((statret == -1) && (compare_dest != NULL)) {
 332                 /* try the file at compare_dest instead */
 333                 int saveerrno = errno;
 334                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 335                 statret = link_stat(fnamecmpbuf,&st);
 336                 if (!S_ISREG(st.st_mode))
 337                         statret = -1;
 338                 if (statret == -1)
 339                         errno = saveerrno;
 340                 else
 341                         fnamecmp = fnamecmpbuf;
 342         }
 343
 344         if (statret == -1) {
 345                 if (errno == ENOENT) {
 346                         write_int(f_out,i);
 347                         if (!dry_run) send_sums(NULL,f_out);
 348                 } else {
 349                         if (verbose > 1)
 350                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 351                 }
 352                 return;
 353         }
 354
 355         if (!S_ISREG(st.st_mode)) {
 356                 if (delete_file(fname) != 0) {
 357                         return;
 358                 }
 359
 360                 /* now pretend the file didn't exist */
 361                 write_int(f_out,i);
 362                 if (!dry_run) send_sums(NULL,f_out);
 363                 return;
 364         }
 365
 366         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 367                 if (verbose > 1)
 368                         rprintf(FINFO,"%s is newer\n",fname);
 369                 return;
 370         }
 371
 372         if (skip_file(fname, file, &st)) {
 373                 if (fnamecmp == fname)
 374                         set_perms(fname,file,&st,1);
 375                 return;
 376         }
 377
 378         if (dry_run) {
 379                 write_int(f_out,i);
 380                 return;
 381         }
 382
 383         if (whole_file) {
 384                 write_int(f_out,i);
 385                 send_sums(NULL,f_out);
 386                 return;
 387         }
 388
 389         /* open the file */
 390         fd = do_open(fnamecmp, O_RDONLY, 0);
 391
 392         if (fd == -1) {
 393                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
 394                 /* pretend the file didn't exist */
 395                 write_int(f_out,i);
 396                 send_sums(NULL,f_out);
 397                 return;
 398         }
 399
 400         if (st.st_size > 0) {
 401                 buf = map_file(fd,st.st_size);
 402         } else {
 403                 buf = NULL;
 404         }
 405
 406         if (verbose > 3)
 407                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 408
 409         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 410
 411         if (verbose > 2)
 412                 rprintf(FINFO,"sending sums for %d\n",i);
 413
 414         write_int(f_out,i);
 415         send_sums(s,f_out);
 416
 417         close(fd);
 418         if (buf) unmap_file(buf);
 419
 420         free_sums(s);
 421 }
 422
 423
 424
 425 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 426 {
 427         int i;
 428         int phase=0;
 429
 430         if (verbose > 2)
 431                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 432                         (int)getpid(),flist->count);
 433
 434         /* we expect to just sit around now, so don't exit on a
 435            timeout. If we really get a timeout then the other process should
 436            exit */
 437         io_timeout = 0;
 438
 439         for (i = 0; i < flist->count; i++) {
 440                 struct file_struct *file = flist->files[i];
 441                 mode_t saved_mode = file->mode;
 442                 if (!file->basename) continue;
 443
 444                 /* we need to ensure that any directories we create have writeable
 445                    permissions initially so that we can create the files within
 446                    them. This is then fixed after the files are transferred */
 447                 if (!am_root && S_ISDIR(file->mode)) {
 448                         file->mode |= S_IWUSR; /* user write */
 449                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 450                          * handling of permissions is strange? */
 451                 }
 452
 453                 recv_generator(local_name?local_name:f_name(file),
 454                                flist,i,f);
 455
 456                 file->mode = saved_mode;
 457         }
 458
 459         phase++;
 460         csum_length = SUM_LENGTH;
 461         ignore_times=1;
 462
 463         if (verbose > 2)
 464                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 465
 466         write_int(f,-1);
 467
 468         if (remote_version >= 13) {
 469                 /* in newer versions of the protocol the files can cycle through
 470                    the system more than once to catch initial checksum errors */
 471                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 472                         struct file_struct *file = flist->files[i];
 473                         recv_generator(local_name?local_name:f_name(file),
 474                                        flist,i,f);
 475                 }
 476
 477                 phase++;
 478                 if (verbose > 2)
 479                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 480
 481                 write_int(f,-1);
 482         }
 483 }