generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    Copyright (C) 1996-2000 by Andrew Tridgell
   4    Copyright (C) Paul Mackerras 1996
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, write to the Free Software
  18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21 #include "rsync.h"
  22
  23 extern int verbose;
  24 extern int dry_run;
  25 extern int relative_paths;
  26 extern int preserve_links;
  27 extern int am_root;
  28 extern int preserve_devices;
  29 extern int preserve_hard_links;
  30 extern int update_only;
  31 extern int whole_file;
  32 extern int block_size;
  33 extern int csum_length;
  34 extern int ignore_times;
  35 extern int size_only;
  36 extern int io_timeout;
  37 extern int remote_version;
  38 extern int always_checksum;
  39 extern int modify_window;
  40 extern char *compare_dest;
  41
  42
  43 /* choose whether to skip a particular file */
  44 static int skip_file(char *fname,
  45                      struct file_struct *file, STRUCT_STAT *st)
  46 {
  47         if (st->st_size != file->length) {
  48                 return 0;
  49         }
  50
  51         /* if always checksum is set then we use the checksum instead
  52            of the file time to determine whether to sync */
  53         if (always_checksum && S_ISREG(st->st_mode)) {
  54                 char sum[MD4_SUM_LENGTH];
  55                 char fnamecmpdest[MAXPATHLEN];
  56
  57                 if (compare_dest != NULL) {
  58                         if (access(fname, 0) != 0) {
  59                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  60                                                     compare_dest,fname);
  61                                 fname = fnamecmpdest;
  62                         }
  63                 }
  64                 file_checksum(fname,sum,st->st_size);
  65                 if (remote_version < 21) {
  66                         return (memcmp(sum,file->sum,2) == 0);
  67                 } else {
  68                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  69                 }
  70         }
  71
  72         if (size_only) {
  73                 return 1;
  74         }
  75
  76         if (ignore_times) {
  77                 return 0;
  78         }
  79
  80         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  81 }
  82
  83
  84 /* use a larger block size for really big files */
  85 static int adapt_block_size(struct file_struct *file, int bsize)
  86 {
  87         int ret;
  88
  89         if (bsize != BLOCK_SIZE) return bsize;
  90
  91         ret = file->length / (10000); /* rough heuristic */
  92         ret = ret & ~15; /* multiple of 16 */
  93         if (ret < bsize) ret = bsize;
  94         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  95         return ret;
  96 }
  97
  98
  99 /*
 100   send a sums struct down a fd
 101   */
 102 static void send_sums(struct sum_struct *s,int f_out)
 103 {
 104         int i;
 105
 106         /* tell the other guy how many we are going to be doing and how many
 107            bytes there are in the last chunk */
 108         write_int(f_out,s?s->count:0);
 109         write_int(f_out,s?s->n:block_size);
 110         write_int(f_out,s?s->remainder:0);
 111
 112         if (!s) return;
 113
 114         for (i=0;i<s->count;i++) {
 115                 write_int(f_out,s->sums[i].sum1);
 116                 write_buf(f_out,s->sums[i].sum2,csum_length);
 117         }
 118 }
 119
 120
 121 /*
 122   generate a stream of signatures/checksums that describe a buffer
 123
 124   generate approximately one checksum every n bytes
 125   */
 126 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 127 {
 128         int i;
 129         struct sum_struct *s;
 130         int count;
 131         int block_len = n;
 132         int remainder = (len%block_len);
 133         OFF_T offset = 0;
 134
 135         count = (len+(block_len-1))/block_len;
 136
 137         s = (struct sum_struct *)malloc(sizeof(*s));
 138         if (!s) out_of_memory("generate_sums");
 139
 140         s->count = count;
 141         s->remainder = remainder;
 142         s->n = n;
 143         s->flength = len;
 144
 145         if (count==0) {
 146                 s->sums = NULL;
 147                 return s;
 148         }
 149
 150         if (verbose > 3)
 151                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 152                         s->count,s->remainder,s->n,(double)s->flength);
 153
 154         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 155         if (!s->sums) out_of_memory("generate_sums");
 156
 157         for (i=0;i<count;i++) {
 158                 int n1 = MIN(len,n);
 159                 char *map = map_ptr(buf,offset,n1);
 160
 161                 s->sums[i].sum1 = get_checksum1(map,n1);
 162                 get_checksum2(map,n1,s->sums[i].sum2);
 163
 164                 s->sums[i].offset = offset;
 165                 s->sums[i].len = n1;
 166                 s->sums[i].i = i;
 167
 168                 if (verbose > 3)
 169                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 170                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 171
 172                 len -= n1;
 173                 offset += n1;
 174         }
 175
 176         return s;
 177 }
 178
 179
 180
 181 /*
 182  * Acts on file number I from FLIST, whose name is fname.
 183  *
 184  * First fixes up permissions, then generates checksums for the file.
 185  *
 186  * (This comment was added later by mbp who was trying to work it out;
 187  * it might be wrong.)
 188  */
 189 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 190 {
 191         int fd;
 192         STRUCT_STAT st;
 193         struct map_struct *buf;
 194         struct sum_struct *s;
 195         int statret;
 196         struct file_struct *file = flist->files[i];
 197         char *fnamecmp;
 198         char fnamecmpbuf[MAXPATHLEN];
 199         extern char *compare_dest;
 200         extern int list_only;
 201         extern int preserve_perms;
 202         extern int only_existing;
 203
 204         if (list_only) return;
 205
 206         if (verbose > 2)
 207                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 208
 209         statret = link_stat(fname,&st);
 210
 211         if (only_existing && statret == -1 && errno == ENOENT) {
 212                 /* we only want to update existing files */
 213                 if (verbose > 1) rprintf(FINFO, RSYNC_NAME
 214                                          ": not creating new file \"%s\"\n",fname);
 215                 return;
 216         }
 217
 218         if (statret == 0 &&
 219             !preserve_perms &&
 220             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 221                 /* if the file exists already and we aren't perserving
 222                    presmissions then act as though the remote end sent
 223                    us the file permissions we already have */
 224                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 225         }
 226
 227         if (S_ISDIR(file->mode)) {
 228                 /* The file to be received is a directory, so we need
 229                  * to prepare appropriately.  If there is already a
 230                  * file of that name and it is *not* a directory, then
 231                  * we need to delete it.  If it doesn't exist, then
 232                  * recursively create it. */
 233
 234                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 235                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 236                         if (robust_unlink(fname) != 0) {
 237                                 rprintf(FERROR, RSYNC_NAME
 238                                         ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
 239                                         fname,strerror(errno));
 240                                 return;
 241                         }
 242                         statret = -1;
 243                 }
 244                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 245                         if (!(relative_paths && errno==ENOENT &&
 246                               create_directory_path(fname)==0 &&
 247                               do_mkdir(fname,file->mode)==0)) {
 248                                 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
 249                                         fname,strerror(errno));
 250                         }
 251                 }
 252                 /* f_out is set to -1 when doing final directory
 253                    permission and modification time repair */
 254                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 255                         rprintf(FINFO,"%s/\n",fname);
 256                 return;
 257         }
 258
 259         if (preserve_links && S_ISLNK(file->mode)) {
 260 #if SUPPORT_LINKS
 261                 char lnk[MAXPATHLEN];
 262                 int l;
 263                 extern int safe_symlinks;
 264
 265                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 266                         if (verbose) {
 267                                 rprintf(FINFO,RSYNC_NAME ": ignoring unsafe symlink \"%s\" -> \"%s\"\n",
 268                                         fname,file->link);
 269                         }
 270                         return;
 271                 }
 272                 if (statret == 0) {
 273                         l = readlink(fname,lnk,MAXPATHLEN-1);
 274                         if (l > 0) {
 275                                 lnk[l] = 0;
 276                                 /* A link already pointing to the
 277                                  * right place -- no further action
 278                                  * required. */
 279                                 if (strcmp(lnk,file->link) == 0) {
 280                                         set_perms(fname,file,&st,1);
 281                                         return;
 282                                 }
 283                         }
 284                         /* Not a symlink, so delete whatever's
 285                          * already there and put a new symlink
 286                          * in place. */
 287                         delete_file(fname);
 288                 }
 289                 if (do_symlink(file->link,fname) != 0) {
 290                         rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
 291                                 fname,file->link,strerror(errno));
 292                 } else {
 293                         set_perms(fname,file,NULL,0);
 294                         if (verbose) {
 295                                 rprintf(FINFO,RSYNC_NAME": %s -> %s\n",
 296                                         fname,file->link);
 297                         }
 298                 }
 299 #endif
 300                 return;
 301         }
 302
 303 #ifdef HAVE_MKNOD
 304         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 305                 if (statret != 0 ||
 306                     st.st_mode != file->mode ||
 307                     st.st_rdev != file->rdev) {
 308                         delete_file(fname);
 309                         if (verbose > 2)
 310                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 311                                         fname,(int)file->mode,(int)file->rdev);
 312                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 313                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 314                         } else {
 315                                 set_perms(fname,file,NULL,0);
 316                                 if (verbose)
 317                                         rprintf(FINFO,"%s\n",fname);
 318                         }
 319                 } else {
 320                         set_perms(fname,file,&st,1);
 321                 }
 322                 return;
 323         }
 324 #endif
 325
 326         if (preserve_hard_links && check_hard_link(file)) {
 327                 if (verbose > 1)
 328                         rprintf(FINFO, RSYNC_NAME
 329                                 ": \"%s\" is a hard link\n",f_name(file));
 330                 return;
 331         }
 332
 333         if (!S_ISREG(file->mode)) {
 334                 rprintf(FINFO, RSYNC_NAME
 335                         ": skipping non-regular file \"%s\"\n",fname);
 336                 return;
 337         }
 338
 339         fnamecmp = fname;
 340
 341         if ((statret == -1) && (compare_dest != NULL)) {
 342                 /* try the file at compare_dest instead */
 343                 int saveerrno = errno;
 344                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 345                 statret = link_stat(fnamecmpbuf,&st);
 346                 if (!S_ISREG(st.st_mode))
 347                         statret = -1;
 348                 if (statret == -1)
 349                         errno = saveerrno;
 350                 else
 351                         fnamecmp = fnamecmpbuf;
 352         }
 353
 354         if (statret == -1) {
 355                 if (errno == ENOENT) {
 356                         write_int(f_out,i);
 357                         if (!dry_run) send_sums(NULL,f_out);
 358                 } else {
 359                         if (verbose > 1)
 360                                 rprintf(FERROR, RSYNC_NAME
 361                                         ": recv_generator failed to open \"%s\": %s\n",
 362                                         fname, strerror(errno));
 363                 }
 364                 return;
 365         }
 366
 367         if (!S_ISREG(st.st_mode)) {
 368                 if (delete_file(fname) != 0) {
 369                         return;
 370                 }
 371
 372                 /* now pretend the file didn't exist */
 373                 write_int(f_out,i);
 374                 if (!dry_run) send_sums(NULL,f_out);
 375                 return;
 376         }
 377
 378         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 379                 if (verbose > 1)
 380                         rprintf(FINFO,"%s is newer\n",fname);
 381                 return;
 382         }
 383
 384         if (skip_file(fname, file, &st)) {
 385                 if (fnamecmp == fname)
 386                         set_perms(fname,file,&st,1);
 387                 return;
 388         }
 389
 390         if (dry_run) {
 391                 write_int(f_out,i);
 392                 return;
 393         }
 394
 395         if (whole_file) {
 396                 write_int(f_out,i);
 397                 send_sums(NULL,f_out);
 398                 return;
 399         }
 400
 401         /* open the file */
 402         fd = do_open(fnamecmp, O_RDONLY, 0);
 403
 404         if (fd == -1) {
 405                 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
 406                 /* pretend the file didn't exist */
 407                 write_int(f_out,i);
 408                 send_sums(NULL,f_out);
 409                 return;
 410         }
 411
 412         if (st.st_size > 0) {
 413                 buf = map_file(fd,st.st_size);
 414         } else {
 415                 buf = NULL;
 416         }
 417
 418         if (verbose > 3)
 419                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 420
 421         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 422
 423         if (verbose > 2)
 424                 rprintf(FINFO,"sending sums for %d\n",i);
 425
 426         write_int(f_out,i);
 427         send_sums(s,f_out);
 428
 429         close(fd);
 430         if (buf) unmap_file(buf);
 431
 432         free_sums(s);
 433 }
 434
 435
 436
 437 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 438 {
 439         int i;
 440         int phase=0;
 441
 442         if (verbose > 2)
 443                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 444                         (int)getpid(),flist->count);
 445
 446         /* we expect to just sit around now, so don't exit on a
 447            timeout. If we really get a timeout then the other process should
 448            exit */
 449         io_timeout = 0;
 450
 451         for (i = 0; i < flist->count; i++) {
 452                 struct file_struct *file = flist->files[i];
 453                 mode_t saved_mode = file->mode;
 454                 if (!file->basename) continue;
 455
 456                 /* we need to ensure that any directories we create have writeable
 457                    permissions initially so that we can create the files within
 458                    them. This is then fixed after the files are transferred */
 459                 if (!am_root && S_ISDIR(file->mode)) {
 460                         file->mode |= S_IWUSR; /* user write */
 461                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 462                          * handling of permissions is strange? */
 463                 }
 464
 465                 recv_generator(local_name?local_name:f_name(file),
 466                                flist,i,f);
 467
 468                 file->mode = saved_mode;
 469         }
 470
 471         phase++;
 472         csum_length = SUM_LENGTH;
 473         ignore_times=1;
 474
 475         if (verbose > 2)
 476                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 477
 478         write_int(f,-1);
 479
 480         if (remote_version >= 13) {
 481                 /* in newer versions of the protocol the files can cycle through
 482                    the system more than once to catch initial checksum errors */
 483                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 484                         struct file_struct *file = flist->files[i];
 485                         recv_generator(local_name?local_name:f_name(file),
 486                                        flist,i,f);
 487                 }
 488
 489                 phase++;
 490                 if (verbose > 2)
 491                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 492
 493                 write_int(f,-1);
 494         }
 495 }