generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    Copyright (C) 1996-2000 by Andrew Tridgell
   4    Copyright (C) Paul Mackerras 1996
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, write to the Free Software
  18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21 #include "rsync.h"
  22
  23 extern int verbose;
  24 extern int dry_run;
  25 extern int relative_paths;
  26 extern int preserve_links;
  27 extern int am_root;
  28 extern int preserve_devices;
  29 extern int preserve_hard_links;
  30 extern int update_only;
  31 extern int whole_file;
  32 extern int block_size;
  33 extern int csum_length;
  34 extern int ignore_times;
  35 extern int size_only;
  36 extern int io_timeout;
  37 extern int remote_version;
  38 extern int always_checksum;
  39 extern int modify_window;
  40 extern char *compare_dest;
  41
  42
  43 /* choose whether to skip a particular file */
  44 static int skip_file(char *fname,
  45                      struct file_struct *file, STRUCT_STAT *st)
  46 {
  47         if (st->st_size != file->length) {
  48                 return 0;
  49         }
  50
  51         /* if always checksum is set then we use the checksum instead
  52            of the file time to determine whether to sync */
  53         if (always_checksum && S_ISREG(st->st_mode)) {
  54                 char sum[MD4_SUM_LENGTH];
  55                 char fnamecmpdest[MAXPATHLEN];
  56
  57                 if (compare_dest != NULL) {
  58                         if (access(fname, 0) != 0) {
  59                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  60                                                     compare_dest,fname);
  61                                 fname = fnamecmpdest;
  62                         }
  63                 }
  64                 file_checksum(fname,sum,st->st_size);
  65                 if (remote_version < 21) {
  66                         return (memcmp(sum,file->sum,2) == 0);
  67                 } else {
  68                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  69                 }
  70         }
  71
  72         if (size_only) {
  73                 return 1;
  74         }
  75
  76         if (ignore_times) {
  77                 return 0;
  78         }
  79
  80         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  81 }
  82
  83
  84 /* use a larger block size for really big files */
  85 static int adapt_block_size(struct file_struct *file, int bsize)
  86 {
  87         int ret;
  88
  89         if (bsize != BLOCK_SIZE) return bsize;
  90
  91         ret = file->length / (10000); /* rough heuristic */
  92         ret = ret & ~15; /* multiple of 16 */
  93         if (ret < bsize) ret = bsize;
  94         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  95         return ret;
  96 }
  97
  98
  99 /*
 100   send a sums struct down a fd
 101   */
 102 static void send_sums(struct sum_struct *s,int f_out)
 103 {
 104         int i;
 105
 106         /* tell the other guy how many we are going to be doing and how many
 107            bytes there are in the last chunk */
 108         write_int(f_out,s?s->count:0);
 109         write_int(f_out,s?s->n:block_size);
 110         write_int(f_out,s?s->remainder:0);
 111
 112         if (!s) return;
 113
 114         for (i=0;i<s->count;i++) {
 115                 write_int(f_out,s->sums[i].sum1);
 116                 write_buf(f_out,s->sums[i].sum2,csum_length);
 117         }
 118 }
 119
 120
 121 /*
 122   generate a stream of signatures/checksums that describe a buffer
 123
 124   generate approximately one checksum every n bytes
 125   */
 126 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 127 {
 128         int i;
 129         struct sum_struct *s;
 130         int count;
 131         int block_len = n;
 132         int remainder = (len%block_len);
 133         OFF_T offset = 0;
 134
 135         count = (len+(block_len-1))/block_len;
 136
 137         s = (struct sum_struct *)malloc(sizeof(*s));
 138         if (!s) out_of_memory("generate_sums");
 139
 140         s->count = count;
 141         s->remainder = remainder;
 142         s->n = n;
 143         s->flength = len;
 144
 145         if (count==0) {
 146                 s->sums = NULL;
 147                 return s;
 148         }
 149
 150         if (verbose > 3)
 151                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 152                         s->count,s->remainder,s->n,(double)s->flength);
 153
 154         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 155         if (!s->sums) out_of_memory("generate_sums");
 156
 157         for (i=0;i<count;i++) {
 158                 int n1 = MIN(len,n);
 159                 char *map = map_ptr(buf,offset,n1);
 160
 161                 s->sums[i].sum1 = get_checksum1(map,n1);
 162                 get_checksum2(map,n1,s->sums[i].sum2);
 163
 164                 s->sums[i].offset = offset;
 165                 s->sums[i].len = n1;
 166                 s->sums[i].i = i;
 167
 168                 if (verbose > 3)
 169                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 170                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 171
 172                 len -= n1;
 173                 offset += n1;
 174         }
 175
 176         return s;
 177 }
 178
 179
 180
 181 /*
 182  * Acts on file number I from FLIST, whose name is fname.
 183  *
 184  * First fixes up permissions, then generates checksums for the file.
 185  *
 186  * (This comment was added later by mbp who was trying to work it out;
 187  * it might be wrong.)
 188  */
 189 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 190 {
 191         int fd;
 192         STRUCT_STAT st;
 193         struct map_struct *buf;
 194         struct sum_struct *s;
 195         int statret;
 196         struct file_struct *file = flist->files[i];
 197         char *fnamecmp;
 198         char fnamecmpbuf[MAXPATHLEN];
 199         extern char *compare_dest;
 200         extern int list_only;
 201         extern int preserve_perms;
 202         extern int only_existing;
 203
 204         if (list_only) return;
 205
 206         if (verbose > 2)
 207                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 208
 209         statret = link_stat(fname,&st);
 210 read
 211         if (only_existing && statret == -1 && errno == ENOENT) {
 212                 /* we only want to update existing files */
 213                 if (verbose > 1) rprintf(FINFO, RSYNC_NAME
 214                                          ": not creating new file \"%s\"\n",fname);
 215                 return;
 216         }
 217
 218         if (statret == 0 &&
 219             !preserve_perms &&
 220             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 221                 /* if the file exists already and we aren't perserving
 222                    presmissions then act as though the remote end sent
 223                    us the file permissions we already have */
 224                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 225         }
 226
 227         if (S_ISDIR(file->mode)) {
 228                 /* The file to be received is a directory, so we need
 229                  * to prepare appropriately.  If there is already a
 230                  * file of that name and it is *not* a directory, then
 231                  * we need to delete it.  If it doesn't exist, then
 232                  * recursively create it. */
 233
 234                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 235                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 236                         if (robust_unlink(fname) != 0) {
 237                                 rprintf(FERROR, RSYNC_NAME
 238                                         ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
 239                                         fname,strerror(errno));
 240                                 return;
 241                         }
 242                         statret = -1;
 243                 }
 244                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 245                         if (!(relative_paths && errno==ENOENT &&
 246                               create_directory_path(fname)==0 &&
 247                               do_mkdir(fname,file->mode)==0)) {
 248                                 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
 249                                         fname,strerror(errno));
 250                         }
 251                 }
 252                 /* f_out is set to -1 when doing final directory
 253                    permission and modification time repair */
 254                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 255                         rprintf(FINFO,"%s/\n",fname);
 256                 return;
 257         }
 258
 259         if (preserve_links && S_ISLNK(file->mode)) {
 260 #if SUPPORT_LINKS
 261                 char lnk[MAXPATHLEN];
 262                 int l;
 263                 extern int safe_symlinks;
 264
 265                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 266                         if (verbose) {
 267                                 rprintf(FINFO,RSYNC_NAME ": ignoring unsafe symlink \"%s\" -> \"%s\"\n",
 268                                         fname,file->link);
 269                         }
 270                         return;
 271                 }
 272                 if (statret == 0) {
 273                         l = readlink(fname,lnk,MAXPATHLEN-1);
 274                         if (l > 0) {
 275                                 lnk[l] = 0;
 276                                 /* A link already pointing to the
 277                                  * right place -- no further action
 278                                  * required. */
 279                                 if (strcmp(lnk,file->link) == 0) {
 280                                         set_perms(fname,file,&st,1);
 281                                         return;
 282                                 }
 283                         }
 284                         /* Not a symlink, so delete whatever's
 285                          * already there and put a new symlink
 286                          * in place. */
 287                         delete_file(fname);
 288                 }
 289                 if (do_symlink(file->link,fname) != 0) {
 290                         rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
 291                                 fname,file->link,strerror(errno));
 292                 } else {
 293                         set_perms(fname,file,NULL,0);
 294                         if (verbose) {
 295                                 rprintf(FINFO,RSYNC_NAME": %s -> %s\n",
 296                                         fname,file->link);
 297                         }
 298                 }
 299 #endif
 300                 return;
 301         }
 302
 303 #ifdef HAVE_MKNOD
 304         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 305                 if (statret != 0 ||
 306                     st.st_mode != file->mode ||
 307                     st.st_rdev != file->rdev) {
 308                         delete_file(fname);
 309                         if (verbose > 2)
 310                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 311                                         fname,(int)file->mode,(int)file->rdev);
 312                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 313                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 314                         } else {
 315                                 set_perms(fname,file,NULL,0);
 316                                 if (verbose)
 317                                         rprintf(FINFO,"%s\n",fname);
 318                         }
 319                 } else {
 320                         set_perms(fname,file,&st,1);
 321                 }
 322                 return;
 323         }
 324 #endif
 325
 326         if (preserve_hard_links && check_hard_link(file)) {
 327                 if (verbose > 1)
 328                         rprintf(FINFO, RSYNC_NAME
 329                                 ": \"%s\" is a hard link\n",f_name(file));
 330                 return;
 331         }
 332
 333         if (!S_ISREG(file->mode)) {
 334                 rprintf(FINFO, RSYNC_NAME
 335                         ": skipping non-regular file \"%s\"\n",fname);
 336                 return;
 337         }
 338
 339         fnamecmp = fname;
 340
 341         if ((statret == -1) && (compare_dest != NULL)) {
 342                 /* try the file at compare_dest instead */
 343                 int saveerrno = errno;
 344                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 345                 statret = link_stat(fnamecmpbuf,&st);
 346                 if (!S_ISREG(st.st_mode))
 347                         statret = -1;
 348                 if (statret == -1)
 349                         errno = saveerrno;
 350                 else
 351                         fnamecmp = fnamecmpbuf;
 352         }
 353
 354         if (statret == -1) {
 355                 if (errno == ENOENT) {
 356                         write_int(f_out,i);
 357                         if (!dry_run) send_sums(NULL,f_out);
 358                 } else {
 359                         if (verbose > 1)
 360                                 rprintf(FERROR,RSYNC_NAME": recv_generator failed to open \%s\": %s\n",fname,
 361                                         strerror(errno));
 362                 }
 363                 return;
 364         }
 365
 366         if (!S_ISREG(st.st_mode)) {
 367                 if (delete_file(fname) != 0) {
 368                         return;
 369                 }
 370
 371                 /* now pretend the file didn't exist */
 372                 write_int(f_out,i);
 373                 if (!dry_run) send_sums(NULL,f_out);
 374                 return;
 375         }
 376
 377         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 378                 if (verbose > 1)
 379                         rprintf(FINFO,"%s is newer\n",fname);
 380                 return;
 381         }
 382
 383         if (skip_file(fname, file, &st)) {
 384                 if (fnamecmp == fname)
 385                         set_perms(fname,file,&st,1);
 386                 return;
 387         }
 388
 389         if (dry_run) {
 390                 write_int(f_out,i);
 391                 return;
 392         }
 393
 394         if (whole_file) {
 395                 write_int(f_out,i);
 396                 send_sums(NULL,f_out);
 397                 return;
 398         }
 399
 400         /* open the file */
 401         fd = do_open(fnamecmp, O_RDONLY, 0);
 402
 403         if (fd == -1) {
 404                 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
 405                 /* pretend the file didn't exist */
 406                 write_int(f_out,i);
 407                 send_sums(NULL,f_out);
 408                 return;
 409         }
 410
 411         if (st.st_size > 0) {
 412                 buf = map_file(fd,st.st_size);
 413         } else {
 414                 buf = NULL;
 415         }
 416
 417         if (verbose > 3)
 418                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 419
 420         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 421
 422         if (verbose > 2)
 423                 rprintf(FINFO,"sending sums for %d\n",i);
 424
 425         write_int(f_out,i);
 426         send_sums(s,f_out);
 427
 428         close(fd);
 429         if (buf) unmap_file(buf);
 430
 431         free_sums(s);
 432 }
 433
 434
 435
 436 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 437 {
 438         int i;
 439         int phase=0;
 440
 441         if (verbose > 2)
 442                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 443                         (int)getpid(),flist->count);
 444
 445         /* we expect to just sit around now, so don't exit on a
 446            timeout. If we really get a timeout then the other process should
 447            exit */
 448         io_timeout = 0;
 449
 450         for (i = 0; i < flist->count; i++) {
 451                 struct file_struct *file = flist->files[i];
 452                 mode_t saved_mode = file->mode;
 453                 if (!file->basename) continue;
 454
 455                 /* we need to ensure that any directories we create have writeable
 456                    permissions initially so that we can create the files within
 457                    them. This is then fixed after the files are transferred */
 458                 if (!am_root && S_ISDIR(file->mode)) {
 459                         file->mode |= S_IWUSR; /* user write */
 460                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 461                          * handling of permissions is strange? */
 462                 }
 463
 464                 recv_generator(local_name?local_name:f_name(file),
 465                                flist,i,f);
 466
 467                 file->mode = saved_mode;
 468         }
 469
 470         phase++;
 471         csum_length = SUM_LENGTH;
 472         ignore_times=1;
 473
 474         if (verbose > 2)
 475                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 476
 477         write_int(f,-1);
 478
 479         if (remote_version >= 13) {
 480                 /* in newer versions of the protocol the files can cycle through
 481                    the system more than once to catch initial checksum errors */
 482                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 483                         struct file_struct *file = flist->files[i];
 484                         recv_generator(local_name?local_name:f_name(file),
 485                                        flist,i,f);
 486                 }
 487
 488                 phase++;
 489                 if (verbose > 2)
 490                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 491
 492                 write_int(f,-1);
 493         }
 494 }