generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    Copyright (C) 1996-2000 by Andrew Tridgell
   4    Copyright (C) Paul Mackerras 1996
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, write to the Free Software
  18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21 #include "rsync.h"
  22
  23 extern int verbose;
  24 extern int dry_run;
  25 extern int relative_paths;
  26 extern int preserve_links;
  27 extern int am_root;
  28 extern int preserve_devices;
  29 extern int preserve_hard_links;
  30 extern int update_only;
  31 extern int whole_file;
  32 extern int block_size;
  33 extern int csum_length;
  34 extern int ignore_times;
  35 extern int size_only;
  36 extern int io_timeout;
  37 extern int remote_version;
  38 extern int always_checksum;
  39 extern int modify_window;
  40 extern char *compare_dest;
  41
  42
  43 /* choose whether to skip a particular file */
  44 static int skip_file(char *fname,
  45                      struct file_struct *file, STRUCT_STAT *st)
  46 {
  47         if (st->st_size != file->length) {
  48                 return 0;
  49         }
  50
  51         /* if always checksum is set then we use the checksum instead
  52            of the file time to determine whether to sync */
  53         if (always_checksum && S_ISREG(st->st_mode)) {
  54                 char sum[MD4_SUM_LENGTH];
  55                 char fnamecmpdest[MAXPATHLEN];
  56
  57                 if (compare_dest != NULL) {
  58                         if (access(fname, 0) != 0) {
  59                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  60                                                     compare_dest,fname);
  61                                 fname = fnamecmpdest;
  62                         }
  63                 }
  64                 file_checksum(fname,sum,st->st_size);
  65                 if (remote_version < 21) {
  66                         return (memcmp(sum,file->sum,2) == 0);
  67                 } else {
  68                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  69                 }
  70         }
  71
  72         if (size_only) {
  73                 return 1;
  74         }
  75
  76         if (ignore_times) {
  77                 return 0;
  78         }
  79
  80         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  81 }
  82
  83
  84 /* use a larger block size for really big files */
  85 static int adapt_block_size(struct file_struct *file, int bsize)
  86 {
  87         int ret;
  88
  89         if (bsize != BLOCK_SIZE) return bsize;
  90
  91         ret = file->length / (10000); /* rough heuristic */
  92         ret = ret & ~15; /* multiple of 16 */
  93         if (ret < bsize) ret = bsize;
  94         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  95         return ret;
  96 }
  97
  98
  99 /*
 100   send a sums struct down a fd
 101   */
 102 static void send_sums(struct sum_struct *s,int f_out)
 103 {
 104         int i;
 105
 106         /* tell the other guy how many we are going to be doing and how many
 107            bytes there are in the last chunk */
 108         write_int(f_out,s?s->count:0);
 109         write_int(f_out,s?s->n:block_size);
 110         write_int(f_out,s?s->remainder:0);
 111
 112         if (!s) return;
 113
 114         for (i=0;i<s->count;i++) {
 115                 write_int(f_out,s->sums[i].sum1);
 116                 write_buf(f_out,s->sums[i].sum2,csum_length);
 117         }
 118 }
 119
 120
 121 /*
 122   generate a stream of signatures/checksums that describe a buffer
 123
 124   generate approximately one checksum every n bytes
 125   */
 126 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 127 {
 128         int i;
 129         struct sum_struct *s;
 130         int count;
 131         int block_len = n;
 132         int remainder = (len%block_len);
 133         OFF_T offset = 0;
 134
 135         count = (len+(block_len-1))/block_len;
 136
 137         s = (struct sum_struct *)malloc(sizeof(*s));
 138         if (!s) out_of_memory("generate_sums");
 139
 140         s->count = count;
 141         s->remainder = remainder;
 142         s->n = n;
 143         s->flength = len;
 144
 145         if (count==0) {
 146                 s->sums = NULL;
 147                 return s;
 148         }
 149
 150         if (verbose > 3)
 151                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 152                         s->count,s->remainder,s->n,(double)s->flength);
 153
 154         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 155         if (!s->sums) out_of_memory("generate_sums");
 156
 157         for (i=0;i<count;i++) {
 158                 int n1 = MIN(len,n);
 159                 char *map = map_ptr(buf,offset,n1);
 160
 161                 s->sums[i].sum1 = get_checksum1(map,n1);
 162                 get_checksum2(map,n1,s->sums[i].sum2);
 163
 164                 s->sums[i].offset = offset;
 165                 s->sums[i].len = n1;
 166                 s->sums[i].i = i;
 167
 168                 if (verbose > 3)
 169                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 170                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 171
 172                 len -= n1;
 173                 offset += n1;
 174         }
 175
 176         return s;
 177 }
 178
 179
 180
 181 /*
 182  * Acts on file number I from FLIST, whose name is fname.
 183  *
 184  * First fixes up permissions, then generates checksums for the file.
 185  *
 186  * (This comment was added later by mbp who was trying to work it out;
 187  * it might be wrong.)
 188  */
 189 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 190 {
 191         int fd;
 192         STRUCT_STAT st;
 193         struct map_struct *buf;
 194         struct sum_struct *s;
 195         int statret;
 196         struct file_struct *file = flist->files[i];
 197         char *fnamecmp;
 198         char fnamecmpbuf[MAXPATHLEN];
 199         extern char *compare_dest;
 200         extern int list_only;
 201         extern int preserve_perms;
 202         extern int only_existing;
 203
 204         if (list_only) return;
 205
 206         if (verbose > 2)
 207                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 208
 209         statret = link_stat(fname,&st);
 210
 211         if (only_existing && statret == -1 && errno == ENOENT) {
 212                 /* we only want to update existing files */
 213                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
 214                 return;
 215         }
 216
 217         if (statret == 0 &&
 218             !preserve_perms &&
 219             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 220                 /* if the file exists already and we aren't perserving
 221                    presmissions then act as though the remote end sent
 222                    us the file permissions we already have */
 223                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 224         }
 225
 226         if (S_ISDIR(file->mode)) {
 227                 /* The file to be received is a directory, so we need
 228                  * to prepare appropriately.  If there is already a
 229                  * file of that name and it is *not* a directory, then
 230                  * we need to delete it.  If it doesn't exist, then
 231                  * recursively create it. */
 232
 233                 if (dry_run) return;
 234                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 235                         if (robust_unlink(fname) != 0) {
 236                                 rprintf(FERROR,"recv_generator: unlink %s: %s\n",
 237                                         fname,strerror(errno));
 238                                 return;
 239                         }
 240                         statret = -1;
 241                 }
 242                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 243                         if (!(relative_paths && errno==ENOENT &&
 244                               create_directory_path(fname)==0 &&
 245                               do_mkdir(fname,file->mode)==0)) {
 246                                 rprintf(FERROR,"recv_generator: mkdir %s: %s (2)\n",
 247                                         fname,strerror(errno));
 248                         }
 249                 }
 250                 if (set_perms(fname,file,NULL,0) && verbose)
 251                         rprintf(FINFO,"%s/\n",fname);
 252                 return;
 253         }
 254
 255         if (preserve_links && S_ISLNK(file->mode)) {
 256 #if SUPPORT_LINKS
 257                 char lnk[MAXPATHLEN];
 258                 int l;
 259                 extern int safe_symlinks;
 260
 261                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 262                         if (verbose) {
 263                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 264                                         fname,file->link);
 265                         }
 266                         return;
 267                 }
 268                 if (statret == 0) {
 269                         l = readlink(fname,lnk,MAXPATHLEN-1);
 270                         if (l > 0) {
 271                                 lnk[l] = 0;
 272                                 if (strcmp(lnk,file->link) == 0) {
 273                                         set_perms(fname,file,&st,1);
 274                                         return;
 275                                 }
 276                         }
 277                         delete_file(fname);
 278                 }
 279                 if (do_symlink(file->link,fname) != 0) {
 280                         rprintf(FERROR,"symlink %s -> %s : %s\n",
 281                                 fname,file->link,strerror(errno));
 282                 } else {
 283                         set_perms(fname,file,NULL,0);
 284                         if (verbose) {
 285                                 rprintf(FINFO,"%s -> %s\n",
 286                                         fname,file->link);
 287                         }
 288                 }
 289 #endif
 290                 return;
 291         }
 292
 293 #ifdef HAVE_MKNOD
 294         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 295                 if (statret != 0 ||
 296                     st.st_mode != file->mode ||
 297                     st.st_rdev != file->rdev) {
 298                         delete_file(fname);
 299                         if (verbose > 2)
 300                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 301                                         fname,(int)file->mode,(int)file->rdev);
 302                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 303                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 304                         } else {
 305                                 set_perms(fname,file,NULL,0);
 306                                 if (verbose)
 307                                         rprintf(FINFO,"%s\n",fname);
 308                         }
 309                 } else {
 310                         set_perms(fname,file,&st,1);
 311                 }
 312                 return;
 313         }
 314 #endif
 315
 316         if (preserve_hard_links && check_hard_link(file)) {
 317                 if (verbose > 1)
 318                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 319                 return;
 320         }
 321
 322         if (!S_ISREG(file->mode)) {
 323                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 324                 return;
 325         }
 326
 327         fnamecmp = fname;
 328
 329         if ((statret == -1) && (compare_dest != NULL)) {
 330                 /* try the file at compare_dest instead */
 331                 int saveerrno = errno;
 332                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 333                 statret = link_stat(fnamecmpbuf,&st);
 334                 if (!S_ISREG(st.st_mode))
 335                         statret = -1;
 336                 if (statret == -1)
 337                         errno = saveerrno;
 338                 else
 339                         fnamecmp = fnamecmpbuf;
 340         }
 341
 342         if (statret == -1) {
 343                 if (errno == ENOENT) {
 344                         write_int(f_out,i);
 345                         if (!dry_run) send_sums(NULL,f_out);
 346                 } else {
 347                         if (verbose > 1)
 348                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 349                 }
 350                 return;
 351         }
 352
 353         if (!S_ISREG(st.st_mode)) {
 354                 if (delete_file(fname) != 0) {
 355                         return;
 356                 }
 357
 358                 /* now pretend the file didn't exist */
 359                 write_int(f_out,i);
 360                 if (!dry_run) send_sums(NULL,f_out);
 361                 return;
 362         }
 363
 364         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 365                 if (verbose > 1)
 366                         rprintf(FINFO,"%s is newer\n",fname);
 367                 return;
 368         }
 369
 370         if (skip_file(fname, file, &st)) {
 371                 if (fnamecmp == fname)
 372                         set_perms(fname,file,&st,1);
 373                 return;
 374         }
 375
 376         if (dry_run) {
 377                 write_int(f_out,i);
 378                 return;
 379         }
 380
 381         if (whole_file) {
 382                 write_int(f_out,i);
 383                 send_sums(NULL,f_out);
 384                 return;
 385         }
 386
 387         /* open the file */
 388         fd = do_open(fnamecmp, O_RDONLY, 0);
 389
 390         if (fd == -1) {
 391                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
 392                 /* pretend the file didn't exist */
 393                 write_int(f_out,i);
 394                 send_sums(NULL,f_out);
 395                 return;
 396         }
 397
 398         if (st.st_size > 0) {
 399                 buf = map_file(fd,st.st_size);
 400         } else {
 401                 buf = NULL;
 402         }
 403
 404         if (verbose > 3)
 405                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 406
 407         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 408
 409         if (verbose > 2)
 410                 rprintf(FINFO,"sending sums for %d\n",i);
 411
 412         write_int(f_out,i);
 413         send_sums(s,f_out);
 414
 415         close(fd);
 416         if (buf) unmap_file(buf);
 417
 418         free_sums(s);
 419 }
 420
 421
 422
 423 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 424 {
 425         int i;
 426         int phase=0;
 427
 428         if (verbose > 2)
 429                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 430                         (int)getpid(),flist->count);
 431
 432         /* we expect to just sit around now, so don't exit on a
 433            timeout. If we really get a timeout then the other process should
 434            exit */
 435         io_timeout = 0;
 436
 437         for (i = 0; i < flist->count; i++) {
 438                 struct file_struct *file = flist->files[i];
 439                 mode_t saved_mode = file->mode;
 440                 if (!file->basename) continue;
 441
 442                 /* we need to ensure that any directories we create have writeable
 443                    permissions initially so that we can create the files within
 444                    them. This is then fixed after the files are transferred */
 445                 if (!am_root && S_ISDIR(file->mode)) {
 446                         file->mode |= S_IWUSR; /* user write */
 447                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 448                          * handling of permissions is strange? */
 449                 }
 450
 451                 recv_generator(local_name?local_name:f_name(file),
 452                                flist,i,f);
 453
 454                 file->mode = saved_mode;
 455         }
 456
 457         phase++;
 458         csum_length = SUM_LENGTH;
 459         ignore_times=1;
 460
 461         if (verbose > 2)
 462                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 463
 464         write_int(f,-1);
 465
 466         if (remote_version >= 13) {
 467                 /* in newer versions of the protocol the files can cycle through
 468                    the system more than once to catch initial checksum errors */
 469                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 470                         struct file_struct *file = flist->files[i];
 471                         recv_generator(local_name?local_name:f_name(file),
 472                                        flist,i,f);
 473                 }
 474
 475                 phase++;
 476                 if (verbose > 2)
 477                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 478
 479                 write_int(f,-1);
 480         }
 481 }