generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int update_only;
  34 extern int whole_file;
  35 extern int block_size;
  36 extern int csum_length;
  37 extern int ignore_times;
  38 extern int size_only;
  39 extern int io_timeout;
  40 extern int remote_version;
  41 extern int always_checksum;
  42 extern int modify_window;
  43 extern char *compare_dest;
  44
  45
  46 /* choose whether to skip a particular file */
  47 static int skip_file(char *fname,
  48                      struct file_struct *file, STRUCT_STAT *st)
  49 {
  50         if (st->st_size != file->length) {
  51                 return 0;
  52         }
  53
  54         /* if always checksum is set then we use the checksum instead
  55            of the file time to determine whether to sync */
  56         if (always_checksum && S_ISREG(st->st_mode)) {
  57                 char sum[MD4_SUM_LENGTH];
  58                 char fnamecmpdest[MAXPATHLEN];
  59
  60                 if (compare_dest != NULL) {
  61                         if (access(fname, 0) != 0) {
  62                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  63                                                     compare_dest,fname);
  64                                 fname = fnamecmpdest;
  65                         }
  66                 }
  67                 file_checksum(fname,sum,st->st_size);
  68                 if (remote_version < 21) {
  69                         return (memcmp(sum,file->sum,2) == 0);
  70                 } else {
  71                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  72                 }
  73         }
  74
  75         if (size_only) {
  76                 return 1;
  77         }
  78
  79         if (ignore_times) {
  80                 return 0;
  81         }
  82
  83         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  84 }
  85
  86
  87 /* use a larger block size for really big files */
  88 static int adapt_block_size(struct file_struct *file, int bsize)
  89 {
  90         int ret;
  91
  92         if (bsize != BLOCK_SIZE) return bsize;
  93
  94         ret = file->length / (10000); /* rough heuristic */
  95         ret = ret & ~15; /* multiple of 16 */
  96         if (ret < bsize) ret = bsize;
  97         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  98         return ret;
  99 }
 100
 101
 102 /*
 103   send a sums struct down a fd
 104   */
 105 static void send_sums(struct sum_struct *s, int f_out)
 106 {
 107         if (s) {
 108                 size_t i;
 109
 110                 /* tell the other guy how many we are going to be
 111                    doing and how many bytes there are in the last
 112                    chunk */
 113                 write_int(f_out, s->count);
 114                 write_int(f_out, s->n);
 115                 write_int(f_out, s->remainder);
 116
 117                 for (i = 0; i < s->count; i++) {
 118                         write_int(f_out, s->sums[i].sum1);
 119                         write_buf(f_out, s->sums[i].sum2, csum_length);
 120                 }
 121         } else {
 122                 /* we don't have checksums */
 123                 write_int(f_out, 0);
 124                 write_int(f_out, block_size);
 125                 write_int(f_out, 0);
 126         }
 127 }
 128
 129 /*
 130   generate a stream of signatures/checksums that describe a buffer
 131
 132   generate approximately one checksum every n bytes
 133   */
 134 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 135 {
 136         int i;
 137         struct sum_struct *s;
 138         int count;
 139         int block_len = n;
 140         int remainder = (len%block_len);
 141         OFF_T offset = 0;
 142
 143         count = (len+(block_len-1))/block_len;
 144
 145         s = (struct sum_struct *)malloc(sizeof(*s));
 146         if (!s) out_of_memory("generate_sums");
 147
 148         s->count = count;
 149         s->remainder = remainder;
 150         s->n = n;
 151         s->flength = len;
 152
 153         if (count==0) {
 154                 s->sums = NULL;
 155                 return s;
 156         }
 157
 158         if (verbose > 3)
 159                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 160                         s->count,s->remainder,s->n,(double)s->flength);
 161
 162         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 163         if (!s->sums) out_of_memory("generate_sums");
 164
 165         for (i=0;i<count;i++) {
 166                 int n1 = MIN(len,n);
 167                 char *map = map_ptr(buf,offset,n1);
 168
 169                 s->sums[i].sum1 = get_checksum1(map,n1);
 170                 get_checksum2(map,n1,s->sums[i].sum2);
 171
 172                 s->sums[i].offset = offset;
 173                 s->sums[i].len = n1;
 174                 s->sums[i].i = i;
 175
 176                 if (verbose > 3)
 177                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 178                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 179
 180                 len -= n1;
 181                 offset += n1;
 182         }
 183
 184         return s;
 185 }
 186
 187
 188
 189 /*
 190  * Acts on file number I from FLIST, whose name is fname.
 191  *
 192  * First fixes up permissions, then generates checksums for the file.
 193  *
 194  * (This comment was added later by mbp who was trying to work it out;
 195  * it might be wrong.)
 196  */
 197 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 198 {
 199         int fd;
 200         STRUCT_STAT st;
 201         struct map_struct *buf;
 202         struct sum_struct *s;
 203         int statret;
 204         struct file_struct *file = flist->files[i];
 205         char *fnamecmp;
 206         char fnamecmpbuf[MAXPATHLEN];
 207         extern char *compare_dest;
 208         extern int list_only;
 209         extern int preserve_perms;
 210         extern int only_existing;
 211
 212         if (list_only) return;
 213
 214         if (verbose > 2)
 215                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 216
 217         statret = link_stat(fname,&st);
 218
 219         if (only_existing && statret == -1 && errno == ENOENT) {
 220                 /* we only want to update existing files */
 221                 if (verbose > 1) rprintf(FINFO, RSYNC_NAME
 222                                          ": not creating new file \"%s\"\n",fname);
 223                 return;
 224         }
 225
 226         if (statret == 0 &&
 227             !preserve_perms &&
 228             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 229                 /* if the file exists already and we aren't perserving
 230                    presmissions then act as though the remote end sent
 231                    us the file permissions we already have */
 232                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 233         }
 234
 235         if (S_ISDIR(file->mode)) {
 236                 /* The file to be received is a directory, so we need
 237                  * to prepare appropriately.  If there is already a
 238                  * file of that name and it is *not* a directory, then
 239                  * we need to delete it.  If it doesn't exist, then
 240                  * recursively create it. */
 241
 242                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 243                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 244                         if (robust_unlink(fname) != 0) {
 245                                 rprintf(FERROR, RSYNC_NAME
 246                                         ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
 247                                         fname,strerror(errno));
 248                                 return;
 249                         }
 250                         statret = -1;
 251                 }
 252                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 253                         if (!(relative_paths && errno==ENOENT &&
 254                               create_directory_path(fname)==0 &&
 255                               do_mkdir(fname,file->mode)==0)) {
 256                                 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
 257                                         fname,strerror(errno));
 258                         }
 259                 }
 260                 /* f_out is set to -1 when doing final directory
 261                    permission and modification time repair */
 262                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 263                         rprintf(FINFO,"%s/\n",fname);
 264                 return;
 265         }
 266
 267         if (preserve_links && S_ISLNK(file->mode)) {
 268 #if SUPPORT_LINKS
 269                 char lnk[MAXPATHLEN];
 270                 int l;
 271                 extern int safe_symlinks;
 272
 273                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 274                         if (verbose) {
 275                                 rprintf(FINFO,RSYNC_NAME ": ignoring unsafe symlink \"%s\" -> \"%s\"\n",
 276                                         fname,file->link);
 277                         }
 278                         return;
 279                 }
 280                 if (statret == 0) {
 281                         l = readlink(fname,lnk,MAXPATHLEN-1);
 282                         if (l > 0) {
 283                                 lnk[l] = 0;
 284                                 /* A link already pointing to the
 285                                  * right place -- no further action
 286                                  * required. */
 287                                 if (strcmp(lnk,file->link) == 0) {
 288                                         set_perms(fname,file,&st,1);
 289                                         return;
 290                                 }
 291                         }
 292                         /* Not a symlink, so delete whatever's
 293                          * already there and put a new symlink
 294                          * in place. */
 295                         delete_file(fname);
 296                 }
 297                 if (do_symlink(file->link,fname) != 0) {
 298                         rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
 299                                 fname,file->link,strerror(errno));
 300                 } else {
 301                         set_perms(fname,file,NULL,0);
 302                         if (verbose) {
 303                                 rprintf(FINFO,RSYNC_NAME": %s -> %s\n",
 304                                         fname,file->link);
 305                         }
 306                 }
 307 #endif
 308                 return;
 309         }
 310
 311 #ifdef HAVE_MKNOD
 312         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 313                 if (statret != 0 ||
 314                     st.st_mode != file->mode ||
 315                     st.st_rdev != file->rdev) {
 316                         delete_file(fname);
 317                         if (verbose > 2)
 318                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 319                                         fname,(int)file->mode,(int)file->rdev);
 320                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 321                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 322                         } else {
 323                                 set_perms(fname,file,NULL,0);
 324                                 if (verbose)
 325                                         rprintf(FINFO,"%s\n",fname);
 326                         }
 327                 } else {
 328                         set_perms(fname,file,&st,1);
 329                 }
 330                 return;
 331         }
 332 #endif
 333
 334         if (preserve_hard_links && check_hard_link(file)) {
 335                 if (verbose > 1)
 336                         rprintf(FINFO, RSYNC_NAME
 337                                 ": \"%s\" is a hard link\n",f_name(file));
 338                 return;
 339         }
 340
 341         if (!S_ISREG(file->mode)) {
 342                 rprintf(FINFO, RSYNC_NAME
 343                         ": skipping non-regular file \"%s\"\n",fname);
 344                 return;
 345         }
 346
 347         fnamecmp = fname;
 348
 349         if ((statret == -1) && (compare_dest != NULL)) {
 350                 /* try the file at compare_dest instead */
 351                 int saveerrno = errno;
 352                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 353                 statret = link_stat(fnamecmpbuf,&st);
 354                 if (!S_ISREG(st.st_mode))
 355                         statret = -1;
 356                 if (statret == -1)
 357                         errno = saveerrno;
 358                 else
 359                         fnamecmp = fnamecmpbuf;
 360         }
 361
 362         if (statret == -1) {
 363                 if (errno == ENOENT) {
 364                         write_int(f_out,i);
 365                         if (!dry_run) send_sums(NULL,f_out);
 366                 } else {
 367                         if (verbose > 1)
 368                                 rprintf(FERROR, RSYNC_NAME
 369                                         ": recv_generator failed to open \"%s\": %s\n",
 370                                         fname, strerror(errno));
 371                 }
 372                 return;
 373         }
 374
 375         if (!S_ISREG(st.st_mode)) {
 376                 if (delete_file(fname) != 0) {
 377                         return;
 378                 }
 379
 380                 /* now pretend the file didn't exist */
 381                 write_int(f_out,i);
 382                 if (!dry_run) send_sums(NULL,f_out);
 383                 return;
 384         }
 385
 386         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 387                 if (verbose > 1)
 388                         rprintf(FINFO,"%s is newer\n",fname);
 389                 return;
 390         }
 391
 392         if (skip_file(fname, file, &st)) {
 393                 if (fnamecmp == fname)
 394                         set_perms(fname,file,&st,1);
 395                 return;
 396         }
 397
 398         if (dry_run) {
 399                 write_int(f_out,i);
 400                 return;
 401         }
 402
 403         if (whole_file) {
 404                 write_int(f_out,i);
 405                 send_sums(NULL,f_out);
 406                 return;
 407         }
 408
 409         /* open the file */
 410         fd = do_open(fnamecmp, O_RDONLY, 0);
 411
 412         if (fd == -1) {
 413                 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
 414                 /* pretend the file didn't exist */
 415                 write_int(f_out,i);
 416                 send_sums(NULL,f_out);
 417                 return;
 418         }
 419
 420         if (st.st_size > 0) {
 421                 buf = map_file(fd,st.st_size);
 422         } else {
 423                 buf = NULL;
 424         }
 425
 426         if (verbose > 3)
 427                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 428
 429         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 430
 431         if (verbose > 2)
 432                 rprintf(FINFO,"sending sums for %d\n",i);
 433
 434         write_int(f_out,i);
 435         send_sums(s,f_out);
 436
 437         close(fd);
 438         if (buf) unmap_file(buf);
 439
 440         free_sums(s);
 441 }
 442
 443
 444
 445 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 446 {
 447         int i;
 448         int phase=0;
 449
 450         if (verbose > 2)
 451                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 452                         (int)getpid(),flist->count);
 453
 454         /* we expect to just sit around now, so don't exit on a
 455            timeout. If we really get a timeout then the other process should
 456            exit */
 457         io_timeout = 0;
 458
 459         for (i = 0; i < flist->count; i++) {
 460                 struct file_struct *file = flist->files[i];
 461                 mode_t saved_mode = file->mode;
 462                 if (!file->basename) continue;
 463
 464                 /* we need to ensure that any directories we create have writeable
 465                    permissions initially so that we can create the files within
 466                    them. This is then fixed after the files are transferred */
 467                 if (!am_root && S_ISDIR(file->mode)) {
 468                         file->mode |= S_IWUSR; /* user write */
 469                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 470                          * handling of permissions is strange? */
 471                 }
 472
 473                 recv_generator(local_name?local_name:f_name(file),
 474                                flist,i,f);
 475
 476                 file->mode = saved_mode;
 477         }
 478
 479         phase++;
 480         csum_length = SUM_LENGTH;
 481         ignore_times=1;
 482
 483         if (verbose > 2)
 484                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 485
 486         write_int(f,-1);
 487
 488         if (remote_version >= 13) {
 489                 /* in newer versions of the protocol the files can cycle through
 490                    the system more than once to catch initial checksum errors */
 491                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 492                         struct file_struct *file = flist->files[i];
 493                         recv_generator(local_name?local_name:f_name(file),
 494                                        flist,i,f);
 495                 }
 496
 497                 phase++;
 498                 if (verbose > 2)
 499                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 500
 501                 write_int(f,-1);
 502         }
 503 }