generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int update_only;
  34 extern int opt_ignore_existing;
  35 extern int whole_file;
  36 extern int block_size;
  37 extern int csum_length;
  38 extern int ignore_times;
  39 extern int size_only;
  40 extern int io_timeout;
  41 extern int remote_version;
  42 extern int always_checksum;
  43 extern int modify_window;
  44 extern char *compare_dest;
  45
  46
  47 /* choose whether to skip a particular file */
  48 static int skip_file(char *fname,
  49                      struct file_struct *file, STRUCT_STAT *st)
  50 {
  51         if (st->st_size != file->length) {
  52                 return 0;
  53         }
  54
  55         /* if always checksum is set then we use the checksum instead
  56            of the file time to determine whether to sync */
  57         if (always_checksum && S_ISREG(st->st_mode)) {
  58                 char sum[MD4_SUM_LENGTH];
  59                 char fnamecmpdest[MAXPATHLEN];
  60
  61                 if (compare_dest != NULL) {
  62                         if (access(fname, 0) != 0) {
  63                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  64                                                     compare_dest,fname);
  65                                 fname = fnamecmpdest;
  66                         }
  67                 }
  68                 file_checksum(fname,sum,st->st_size);
  69                 if (remote_version < 21) {
  70                         return (memcmp(sum,file->sum,2) == 0);
  71                 } else {
  72                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  73                 }
  74         }
  75
  76         if (size_only) {
  77                 return 1;
  78         }
  79
  80         if (ignore_times) {
  81                 return 0;
  82         }
  83
  84         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  85 }
  86
  87
  88 /* use a larger block size for really big files */
  89 static int adapt_block_size(struct file_struct *file, int bsize)
  90 {
  91         int ret;
  92
  93         if (bsize != BLOCK_SIZE) return bsize;
  94
  95         ret = file->length / (10000); /* rough heuristic */
  96         ret = ret & ~15; /* multiple of 16 */
  97         if (ret < bsize) ret = bsize;
  98         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  99         return ret;
 100 }
 101
 102
 103 /*
 104   send a sums struct down a fd
 105   */
 106 static void send_sums(struct sum_struct *s, int f_out)
 107 {
 108         if (s) {
 109                 size_t i;
 110
 111                 /* tell the other guy how many we are going to be
 112                    doing and how many bytes there are in the last
 113                    chunk */
 114                 write_int(f_out, s->count);
 115                 write_int(f_out, s->n);
 116                 write_int(f_out, s->remainder);
 117
 118                 for (i = 0; i < s->count; i++) {
 119                         write_int(f_out, s->sums[i].sum1);
 120                         write_buf(f_out, s->sums[i].sum2, csum_length);
 121                 }
 122         } else {
 123                 /* we don't have checksums */
 124                 write_int(f_out, 0);
 125                 write_int(f_out, block_size);
 126                 write_int(f_out, 0);
 127         }
 128 }
 129
 130 /*
 131   generate a stream of signatures/checksums that describe a buffer
 132
 133   generate approximately one checksum every n bytes
 134   */
 135 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 136 {
 137         int i;
 138         struct sum_struct *s;
 139         int count;
 140         int block_len = n;
 141         int remainder = (len%block_len);
 142         OFF_T offset = 0;
 143
 144         count = (len+(block_len-1))/block_len;
 145
 146         s = (struct sum_struct *)malloc(sizeof(*s));
 147         if (!s) out_of_memory("generate_sums");
 148
 149         s->count = count;
 150         s->remainder = remainder;
 151         s->n = n;
 152         s->flength = len;
 153
 154         if (count==0) {
 155                 s->sums = NULL;
 156                 return s;
 157         }
 158
 159         if (verbose > 3)
 160                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 161                         s->count,s->remainder,s->n,(double)s->flength);
 162
 163         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 164         if (!s->sums) out_of_memory("generate_sums");
 165
 166         for (i=0;i<count;i++) {
 167                 int n1 = MIN(len,n);
 168                 char *map = map_ptr(buf,offset,n1);
 169
 170                 s->sums[i].sum1 = get_checksum1(map,n1);
 171                 get_checksum2(map,n1,s->sums[i].sum2);
 172
 173                 s->sums[i].offset = offset;
 174                 s->sums[i].len = n1;
 175                 s->sums[i].i = i;
 176
 177                 if (verbose > 3)
 178                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 179                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 180
 181                 len -= n1;
 182                 offset += n1;
 183         }
 184
 185         return s;
 186 }
 187
 188
 189
 190 /*
 191  * Acts on file number I from FLIST, whose name is fname.
 192  *
 193  * First fixes up permissions, then generates checksums for the file.
 194  *
 195  * (This comment was added later by mbp who was trying to work it out;
 196  * it might be wrong.)
 197  */
 198 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 199 {
 200         int fd;
 201         STRUCT_STAT st;
 202         struct map_struct *buf;
 203         struct sum_struct *s;
 204         int statret;
 205         struct file_struct *file = flist->files[i];
 206         char *fnamecmp;
 207         char fnamecmpbuf[MAXPATHLEN];
 208         extern char *compare_dest;
 209         extern int list_only;
 210         extern int preserve_perms;
 211         extern int only_existing;
 212
 213         if (list_only) return;
 214
 215         if (verbose > 2)
 216                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 217
 218         statret = link_stat(fname,&st);
 219
 220         if (only_existing && statret == -1 && errno == ENOENT) {
 221                 /* we only want to update existing files */
 222                 if (verbose > 1) rprintf(FINFO, "not creating new file \"%s\"\n",fname);
 223                 return;
 224         }
 225
 226         if (statret == 0 &&
 227             !preserve_perms &&
 228             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 229                 /* if the file exists already and we aren't perserving
 230                    presmissions then act as though the remote end sent
 231                    us the file permissions we already have */
 232                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 233         }
 234
 235         if (S_ISDIR(file->mode)) {
 236                 /* The file to be received is a directory, so we need
 237                  * to prepare appropriately.  If there is already a
 238                  * file of that name and it is *not* a directory, then
 239                  * we need to delete it.  If it doesn't exist, then
 240                  * recursively create it. */
 241
 242                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 243                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 244                         if (robust_unlink(fname) != 0) {
 245                                 rprintf(FERROR, RSYNC_NAME
 246                                         ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
 247                                         fname,strerror(errno));
 248                                 return;
 249                         }
 250                         statret = -1;
 251                 }
 252                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 253                         if (!(relative_paths && errno==ENOENT &&
 254                               create_directory_path(fname)==0 &&
 255                               do_mkdir(fname,file->mode)==0)) {
 256                                 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
 257                                         fname,strerror(errno));
 258                         }
 259                 }
 260                 /* f_out is set to -1 when doing final directory
 261                    permission and modification time repair */
 262                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 263                         rprintf(FINFO,"%s/\n",fname);
 264                 return;
 265         }
 266
 267         if (preserve_links && S_ISLNK(file->mode)) {
 268 #if SUPPORT_LINKS
 269                 char lnk[MAXPATHLEN];
 270                 int l;
 271                 extern int safe_symlinks;
 272
 273                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 274                         if (verbose) {
 275                                 rprintf(FINFO,"ignoring unsafe symlink \"%s\" -> \"%s\"\n",
 276                                         fname,file->link);
 277                         }
 278                         return;
 279                 }
 280                 if (statret == 0) {
 281                         l = readlink(fname,lnk,MAXPATHLEN-1);
 282                         if (l > 0) {
 283                                 lnk[l] = 0;
 284                                 /* A link already pointing to the
 285                                  * right place -- no further action
 286                                  * required. */
 287                                 if (strcmp(lnk,file->link) == 0) {
 288                                         set_perms(fname,file,&st,1);
 289                                         return;
 290                                 }
 291                         }
 292                         /* Not a symlink, so delete whatever's
 293                          * already there and put a new symlink
 294                          * in place. */
 295                         delete_file(fname);
 296                 }
 297                 if (do_symlink(file->link,fname) != 0) {
 298                         rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
 299                                 fname,file->link,strerror(errno));
 300                 } else {
 301                         set_perms(fname,file,NULL,0);
 302                         if (verbose) {
 303                                 rprintf(FINFO,"%s -> %s\n", fname,file->link);
 304                         }
 305                 }
 306 #endif
 307                 return;
 308         }
 309
 310 #ifdef HAVE_MKNOD
 311         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 312                 if (statret != 0 ||
 313                     st.st_mode != file->mode ||
 314                     st.st_rdev != file->rdev) {
 315                         delete_file(fname);
 316                         if (verbose > 2)
 317                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 318                                         fname,(int)file->mode,(int)file->rdev);
 319                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 320                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 321                         } else {
 322                                 set_perms(fname,file,NULL,0);
 323                                 if (verbose)
 324                                         rprintf(FINFO,"%s\n",fname);
 325                         }
 326                 } else {
 327                         set_perms(fname,file,&st,1);
 328                 }
 329                 return;
 330         }
 331 #endif
 332
 333         if (preserve_hard_links && check_hard_link(file)) {
 334                 if (verbose > 1)
 335                         rprintf(FINFO, "\"%s\" is a hard link\n",f_name(file));
 336                 return;
 337         }
 338
 339         if (!S_ISREG(file->mode)) {
 340                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 341                 return;
 342         }
 343
 344         fnamecmp = fname;
 345
 346         if ((statret == -1) && (compare_dest != NULL)) {
 347                 /* try the file at compare_dest instead */
 348                 int saveerrno = errno;
 349                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 350                 statret = link_stat(fnamecmpbuf,&st);
 351                 if (!S_ISREG(st.st_mode))
 352                         statret = -1;
 353                 if (statret == -1)
 354                         errno = saveerrno;
 355                 else
 356                         fnamecmp = fnamecmpbuf;
 357         }
 358
 359         if (statret == -1) {
 360                 if (errno == ENOENT) {
 361                         write_int(f_out,i);
 362                         if (!dry_run) send_sums(NULL,f_out);
 363                 } else {
 364                         if (verbose > 1)
 365                                 rprintf(FERROR, RSYNC_NAME
 366                                         ": recv_generator failed to open \"%s\": %s\n",
 367                                         fname, strerror(errno));
 368                 }
 369                 return;
 370         }
 371
 372         if (!S_ISREG(st.st_mode)) {
 373                 if (delete_file(fname) != 0) {
 374                         return;
 375                 }
 376
 377                 /* now pretend the file didn't exist */
 378                 write_int(f_out,i);
 379                 if (!dry_run) send_sums(NULL,f_out);
 380                 return;
 381         }
 382
 383         if (opt_ignore_existing && fnamecmp == fname) {
 384                 if (verbose > 1)
 385                         rprintf(FINFO,"%s exists\n",fname);
 386                 return;
 387         }
 388
 389         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 390                 if (verbose > 1)
 391                         rprintf(FINFO,"%s is newer\n",fname);
 392                 return;
 393         }
 394
 395         if (skip_file(fname, file, &st)) {
 396                 if (fnamecmp == fname)
 397                         set_perms(fname,file,&st,1);
 398                 return;
 399         }
 400
 401         if (dry_run) {
 402                 write_int(f_out,i);
 403                 return;
 404         }
 405
 406         if (whole_file) {
 407                 write_int(f_out,i);
 408                 send_sums(NULL,f_out);
 409                 return;
 410         }
 411
 412         /* open the file */
 413         fd = do_open(fnamecmp, O_RDONLY, 0);
 414
 415         if (fd == -1) {
 416                 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
 417                 /* pretend the file didn't exist */
 418                 write_int(f_out,i);
 419                 send_sums(NULL,f_out);
 420                 return;
 421         }
 422
 423         if (st.st_size > 0) {
 424                 buf = map_file(fd,st.st_size);
 425         } else {
 426                 buf = NULL;
 427         }
 428
 429         if (verbose > 3)
 430                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 431
 432         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 433
 434         if (verbose > 2)
 435                 rprintf(FINFO,"sending sums for %d\n",i);
 436
 437         write_int(f_out,i);
 438         send_sums(s,f_out);
 439
 440         close(fd);
 441         if (buf) unmap_file(buf);
 442
 443         free_sums(s);
 444 }
 445
 446
 447
 448 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 449 {
 450         int i;
 451         int phase=0;
 452
 453         if (verbose > 2)
 454                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 455                         (int)getpid(),flist->count);
 456
 457         /* we expect to just sit around now, so don't exit on a
 458            timeout. If we really get a timeout then the other process should
 459            exit */
 460         io_timeout = 0;
 461
 462         for (i = 0; i < flist->count; i++) {
 463                 struct file_struct *file = flist->files[i];
 464                 mode_t saved_mode = file->mode;
 465                 if (!file->basename) continue;
 466
 467                 /* we need to ensure that any directories we create have writeable
 468                    permissions initially so that we can create the files within
 469                    them. This is then fixed after the files are transferred */
 470                 if (!am_root && S_ISDIR(file->mode)) {
 471                         file->mode |= S_IWUSR; /* user write */
 472                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 473                          * handling of permissions is strange? */
 474                 }
 475
 476                 recv_generator(local_name?local_name:f_name(file),
 477                                flist,i,f);
 478
 479                 file->mode = saved_mode;
 480         }
 481
 482         phase++;
 483         csum_length = SUM_LENGTH;
 484         ignore_times=1;
 485
 486         if (verbose > 2)
 487                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 488
 489         write_int(f,-1);
 490
 491         if (remote_version >= 13) {
 492                 /* in newer versions of the protocol the files can cycle through
 493                    the system more than once to catch initial checksum errors */
 494                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 495                         struct file_struct *file = flist->files[i];
 496                         recv_generator(local_name?local_name:f_name(file),
 497                                        flist,i,f);
 498                 }
 499
 500                 phase++;
 501                 if (verbose > 2)
 502                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 503
 504                 write_int(f,-1);
 505         }
 506 }