generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int update_only;
  34 extern int opt_ignore_existing;
  35 extern int whole_file;
  36 extern int block_size;
  37 extern int csum_length;
  38 extern int ignore_times;
  39 extern int size_only;
  40 extern int io_timeout;
  41 extern int remote_version;
  42 extern int always_checksum;
  43 extern int modify_window;
  44 extern char *compare_dest;
  45
  46
  47 /* choose whether to skip a particular file */
  48 static int skip_file(char *fname,
  49                      struct file_struct *file, STRUCT_STAT *st)
  50 {
  51         if (st->st_size != file->length) {
  52                 return 0;
  53         }
  54
  55         /* if always checksum is set then we use the checksum instead
  56            of the file time to determine whether to sync */
  57         if (always_checksum && S_ISREG(st->st_mode)) {
  58                 char sum[MD4_SUM_LENGTH];
  59                 char fnamecmpdest[MAXPATHLEN];
  60
  61                 if (compare_dest != NULL) {
  62                         if (access(fname, 0) != 0) {
  63                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  64                                                     compare_dest,fname);
  65                                 fname = fnamecmpdest;
  66                         }
  67                 }
  68                 file_checksum(fname,sum,st->st_size);
  69                 if (remote_version < 21) {
  70                         return (memcmp(sum,file->sum,2) == 0);
  71                 } else {
  72                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  73                 }
  74         }
  75
  76         if (size_only) {
  77                 return 1;
  78         }
  79
  80         if (ignore_times) {
  81                 return 0;
  82         }
  83
  84         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  85 }
  86
  87
  88 /* use a larger block size for really big files */
  89 static int adapt_block_size(struct file_struct *file, int bsize)
  90 {
  91         int ret;
  92
  93         if (bsize != BLOCK_SIZE) return bsize;
  94
  95         ret = file->length / (10000); /* rough heuristic */
  96         ret = ret & ~15; /* multiple of 16 */
  97         if (ret < bsize) ret = bsize;
  98         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  99         return ret;
 100 }
 101
 102
 103 /*
 104   send a sums struct down a fd
 105   */
 106 static void send_sums(struct sum_struct *s, int f_out)
 107 {
 108         if (s) {
 109                 size_t i;
 110
 111                 /* tell the other guy how many we are going to be
 112                    doing and how many bytes there are in the last
 113                    chunk */
 114                 write_int(f_out, s->count);
 115                 write_int(f_out, s->n);
 116                 write_int(f_out, s->remainder);
 117
 118                 for (i = 0; i < s->count; i++) {
 119                         write_int(f_out, s->sums[i].sum1);
 120                         write_buf(f_out, s->sums[i].sum2, csum_length);
 121                 }
 122         } else {
 123                 /* we don't have checksums */
 124                 write_int(f_out, 0);
 125                 write_int(f_out, block_size);
 126                 write_int(f_out, 0);
 127         }
 128 }
 129
 130 /*
 131   generate a stream of signatures/checksums that describe a buffer
 132
 133   generate approximately one checksum every n bytes
 134   */
 135 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 136 {
 137         int i;
 138         struct sum_struct *s;
 139         int count;
 140         int block_len = n;
 141         int remainder = (len%block_len);
 142         OFF_T offset = 0;
 143
 144         count = (len+(block_len-1))/block_len;
 145
 146         s = (struct sum_struct *)malloc(sizeof(*s));
 147         if (!s) out_of_memory("generate_sums");
 148
 149         s->count = count;
 150         s->remainder = remainder;
 151         s->n = n;
 152         s->flength = len;
 153
 154         if (count==0) {
 155                 s->sums = NULL;
 156                 return s;
 157         }
 158
 159         if (verbose > 3)
 160                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 161                         s->count,s->remainder,s->n,(double)s->flength);
 162
 163         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 164         if (!s->sums) out_of_memory("generate_sums");
 165
 166         for (i=0;i<count;i++) {
 167                 int n1 = MIN(len,n);
 168                 char *map = map_ptr(buf,offset,n1);
 169
 170                 s->sums[i].sum1 = get_checksum1(map,n1);
 171                 get_checksum2(map,n1,s->sums[i].sum2);
 172
 173                 s->sums[i].offset = offset;
 174                 s->sums[i].len = n1;
 175                 s->sums[i].i = i;
 176
 177                 if (verbose > 3)
 178                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 179                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 180
 181                 len -= n1;
 182                 offset += n1;
 183         }
 184
 185         return s;
 186 }
 187
 188
 189
 190 /*
 191  * Acts on file number I from FLIST, whose name is fname.
 192  *
 193  * First fixes up permissions, then generates checksums for the file.
 194  *
 195  * (This comment was added later by mbp who was trying to work it out;
 196  * it might be wrong.)
 197  */
 198 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 199 {
 200         int fd;
 201         STRUCT_STAT st;
 202         struct map_struct *buf;
 203         struct sum_struct *s;
 204         int statret;
 205         struct file_struct *file = flist->files[i];
 206         char *fnamecmp;
 207         char fnamecmpbuf[MAXPATHLEN];
 208         extern char *compare_dest;
 209         extern int list_only;
 210         extern int preserve_perms;
 211         extern int only_existing;
 212
 213         if (list_only) return;
 214
 215         if (verbose > 2)
 216                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 217
 218         statret = link_stat(fname,&st);
 219
 220         if (only_existing && statret == -1 && errno == ENOENT) {
 221                 /* we only want to update existing files */
 222                 if (verbose > 1) rprintf(FINFO, RSYNC_NAME
 223                                          ": not creating new file \"%s\"\n",fname);
 224                 return;
 225         }
 226
 227         if (statret == 0 &&
 228             !preserve_perms &&
 229             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 230                 /* if the file exists already and we aren't perserving
 231                    presmissions then act as though the remote end sent
 232                    us the file permissions we already have */
 233                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 234         }
 235
 236         if (S_ISDIR(file->mode)) {
 237                 /* The file to be received is a directory, so we need
 238                  * to prepare appropriately.  If there is already a
 239                  * file of that name and it is *not* a directory, then
 240                  * we need to delete it.  If it doesn't exist, then
 241                  * recursively create it. */
 242
 243                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 244                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 245                         if (robust_unlink(fname) != 0) {
 246                                 rprintf(FERROR, RSYNC_NAME
 247                                         ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
 248                                         fname,strerror(errno));
 249                                 return;
 250                         }
 251                         statret = -1;
 252                 }
 253                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 254                         if (!(relative_paths && errno==ENOENT &&
 255                               create_directory_path(fname)==0 &&
 256                               do_mkdir(fname,file->mode)==0)) {
 257                                 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
 258                                         fname,strerror(errno));
 259                         }
 260                 }
 261                 /* f_out is set to -1 when doing final directory
 262                    permission and modification time repair */
 263                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 264                         rprintf(FINFO,"%s/\n",fname);
 265                 return;
 266         }
 267
 268         if (preserve_links && S_ISLNK(file->mode)) {
 269 #if SUPPORT_LINKS
 270                 char lnk[MAXPATHLEN];
 271                 int l;
 272                 extern int safe_symlinks;
 273
 274                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 275                         if (verbose) {
 276                                 rprintf(FINFO,RSYNC_NAME ": ignoring unsafe symlink \"%s\" -> \"%s\"\n",
 277                                         fname,file->link);
 278                         }
 279                         return;
 280                 }
 281                 if (statret == 0) {
 282                         l = readlink(fname,lnk,MAXPATHLEN-1);
 283                         if (l > 0) {
 284                                 lnk[l] = 0;
 285                                 /* A link already pointing to the
 286                                  * right place -- no further action
 287                                  * required. */
 288                                 if (strcmp(lnk,file->link) == 0) {
 289                                         set_perms(fname,file,&st,1);
 290                                         return;
 291                                 }
 292                         }
 293                         /* Not a symlink, so delete whatever's
 294                          * already there and put a new symlink
 295                          * in place. */
 296                         delete_file(fname);
 297                 }
 298                 if (do_symlink(file->link,fname) != 0) {
 299                         rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
 300                                 fname,file->link,strerror(errno));
 301                 } else {
 302                         set_perms(fname,file,NULL,0);
 303                         if (verbose) {
 304                                 rprintf(FINFO,RSYNC_NAME": %s -> %s\n",
 305                                         fname,file->link);
 306                         }
 307                 }
 308 #endif
 309                 return;
 310         }
 311
 312 #ifdef HAVE_MKNOD
 313         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 314                 if (statret != 0 ||
 315                     st.st_mode != file->mode ||
 316                     st.st_rdev != file->rdev) {
 317                         delete_file(fname);
 318                         if (verbose > 2)
 319                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 320                                         fname,(int)file->mode,(int)file->rdev);
 321                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 322                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 323                         } else {
 324                                 set_perms(fname,file,NULL,0);
 325                                 if (verbose)
 326                                         rprintf(FINFO,"%s\n",fname);
 327                         }
 328                 } else {
 329                         set_perms(fname,file,&st,1);
 330                 }
 331                 return;
 332         }
 333 #endif
 334
 335         if (preserve_hard_links && check_hard_link(file)) {
 336                 if (verbose > 1)
 337                         rprintf(FINFO, RSYNC_NAME
 338                                 ": \"%s\" is a hard link\n",f_name(file));
 339                 return;
 340         }
 341
 342         if (!S_ISREG(file->mode)) {
 343                 rprintf(FINFO, RSYNC_NAME
 344                         ": skipping non-regular file \"%s\"\n",fname);
 345                 return;
 346         }
 347
 348         fnamecmp = fname;
 349
 350         if ((statret == -1) && (compare_dest != NULL)) {
 351                 /* try the file at compare_dest instead */
 352                 int saveerrno = errno;
 353                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 354                 statret = link_stat(fnamecmpbuf,&st);
 355                 if (!S_ISREG(st.st_mode))
 356                         statret = -1;
 357                 if (statret == -1)
 358                         errno = saveerrno;
 359                 else
 360                         fnamecmp = fnamecmpbuf;
 361         }
 362
 363         if (statret == -1) {
 364                 if (errno == ENOENT) {
 365                         write_int(f_out,i);
 366                         if (!dry_run) send_sums(NULL,f_out);
 367                 } else {
 368                         if (verbose > 1)
 369                                 rprintf(FERROR, RSYNC_NAME
 370                                         ": recv_generator failed to open \"%s\": %s\n",
 371                                         fname, strerror(errno));
 372                 }
 373                 return;
 374         }
 375
 376         if (!S_ISREG(st.st_mode)) {
 377                 if (delete_file(fname) != 0) {
 378                         return;
 379                 }
 380
 381                 /* now pretend the file didn't exist */
 382                 write_int(f_out,i);
 383                 if (!dry_run) send_sums(NULL,f_out);
 384                 return;
 385         }
 386
 387         if (opt_ignore_existing && fnamecmp == fname) {
 388                 if (verbose > 1)
 389                         rprintf(FINFO,"%s exists\n",fname);
 390                 return;
 391         }
 392
 393         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 394                 if (verbose > 1)
 395                         rprintf(FINFO,"%s is newer\n",fname);
 396                 return;
 397         }
 398
 399         if (skip_file(fname, file, &st)) {
 400                 if (fnamecmp == fname)
 401                         set_perms(fname,file,&st,1);
 402                 return;
 403         }
 404
 405         if (dry_run) {
 406                 write_int(f_out,i);
 407                 return;
 408         }
 409
 410         if (whole_file) {
 411                 write_int(f_out,i);
 412                 send_sums(NULL,f_out);
 413                 return;
 414         }
 415
 416         /* open the file */
 417         fd = do_open(fnamecmp, O_RDONLY, 0);
 418
 419         if (fd == -1) {
 420                 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
 421                 /* pretend the file didn't exist */
 422                 write_int(f_out,i);
 423                 send_sums(NULL,f_out);
 424                 return;
 425         }
 426
 427         if (st.st_size > 0) {
 428                 buf = map_file(fd,st.st_size);
 429         } else {
 430                 buf = NULL;
 431         }
 432
 433         if (verbose > 3)
 434                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 435
 436         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 437
 438         if (verbose > 2)
 439                 rprintf(FINFO,"sending sums for %d\n",i);
 440
 441         write_int(f_out,i);
 442         send_sums(s,f_out);
 443
 444         close(fd);
 445         if (buf) unmap_file(buf);
 446
 447         free_sums(s);
 448 }
 449
 450
 451
 452 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 453 {
 454         int i;
 455         int phase=0;
 456
 457         if (verbose > 2)
 458                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 459                         (int)getpid(),flist->count);
 460
 461         /* we expect to just sit around now, so don't exit on a
 462            timeout. If we really get a timeout then the other process should
 463            exit */
 464         io_timeout = 0;
 465
 466         for (i = 0; i < flist->count; i++) {
 467                 struct file_struct *file = flist->files[i];
 468                 mode_t saved_mode = file->mode;
 469                 if (!file->basename) continue;
 470
 471                 /* we need to ensure that any directories we create have writeable
 472                    permissions initially so that we can create the files within
 473                    them. This is then fixed after the files are transferred */
 474                 if (!am_root && S_ISDIR(file->mode)) {
 475                         file->mode |= S_IWUSR; /* user write */
 476                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 477                          * handling of permissions is strange? */
 478                 }
 479
 480                 recv_generator(local_name?local_name:f_name(file),
 481                                flist,i,f);
 482
 483                 file->mode = saved_mode;
 484         }
 485
 486         phase++;
 487         csum_length = SUM_LENGTH;
 488         ignore_times=1;
 489
 490         if (verbose > 2)
 491                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 492
 493         write_int(f,-1);
 494
 495         if (remote_version >= 13) {
 496                 /* in newer versions of the protocol the files can cycle through
 497                    the system more than once to catch initial checksum errors */
 498                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 499                         struct file_struct *file = flist->files[i];
 500                         recv_generator(local_name?local_name:f_name(file),
 501                                        flist,i,f);
 502                 }
 503
 504                 phase++;
 505                 if (verbose > 2)
 506                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 507
 508                 write_int(f,-1);
 509         }
 510 }