generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int keep_dirlinks;
  30 extern int preserve_links;
  31 extern int am_root;
  32 extern int preserve_devices;
  33 extern int preserve_hard_links;
  34 extern int preserve_perms;
  35 extern int preserve_uid;
  36 extern int preserve_gid;
  37 extern int update_only;
  38 extern int opt_ignore_existing;
  39 extern int csum_length;
  40 extern int ignore_times;
  41 extern int size_only;
  42 extern int io_timeout;
  43 extern int protocol_version;
  44 extern int always_checksum;
  45 extern char *compare_dest;
  46 extern int link_dest;
  47 extern int whole_file;
  48 extern int local_server;
  49 extern int list_only;
  50 extern int only_existing;
  51 extern int orig_umask;
  52 extern int safe_symlinks;
  53 extern unsigned int block_size;
  54
  55 extern struct exclude_list_struct server_exclude_list;
  56
  57
  58 /* choose whether to skip a particular file */
  59 static int skip_file(char *fname, struct file_struct *file, STRUCT_STAT *st)
  60 {
  61         if (st->st_size != file->length)
  62                 return 0;
  63         if (link_dest) {
  64                 if (preserve_perms
  65                     && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
  66                         return 0;
  67
  68                 if (am_root && preserve_uid && st->st_uid != file->uid)
  69                         return 0;
  70
  71                 if (preserve_gid && file->gid != GID_NONE
  72                     && st->st_gid != file->gid)
  73                         return 0;
  74         }
  75
  76         /* if always checksum is set then we use the checksum instead
  77            of the file time to determine whether to sync */
  78         if (always_checksum && S_ISREG(st->st_mode)) {
  79                 char sum[MD4_SUM_LENGTH];
  80                 char fnamecmpdest[MAXPATHLEN];
  81
  82                 if (compare_dest != NULL) {
  83                         if (access(fname, 0) != 0) {
  84                                 pathjoin(fnamecmpdest, sizeof fnamecmpdest,
  85                                          compare_dest, fname);
  86                                 fname = fnamecmpdest;
  87                         }
  88                 }
  89                 file_checksum(fname,sum,st->st_size);
  90                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
  91                                                         : MD4_SUM_LENGTH) == 0;
  92         }
  93
  94         if (size_only)
  95                 return 1;
  96
  97         if (ignore_times)
  98                 return 0;
  99
 100         return cmp_modtime(st->st_mtime, file->modtime) == 0;
 101 }
 102
 103
 104 /*
 105  * NULL sum_struct means we have no checksums
 106  */
 107 void write_sum_head(int f, struct sum_struct *sum)
 108 {
 109         static struct sum_struct null_sum;
 110
 111         if (sum == NULL)
 112                 sum = &null_sum;
 113
 114         write_int(f, sum->count);
 115         write_int(f, sum->blength);
 116         if (protocol_version >= 27)
 117                 write_int(f, sum->s2length);
 118         write_int(f, sum->remainder);
 119 }
 120
 121 /*
 122  * set (initialize) the size entries in the per-file sum_struct
 123  * calculating dynamic block and checksum sizes.
 124  *
 125  * This is only called from generate_and_send_sums() but is a separate
 126  * function to encapsulate the logic.
 127  *
 128  * The block size is a rounded square root of file length.
 129  *
 130  * The checksum size is determined according to:
 131  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 132  * provided by Donovan Baarda which gives a probability of rsync
 133  * algorithm corrupting data and falling back using the whole md4
 134  * checksums.
 135  *
 136  * This might be made one of several selectable heuristics.
 137  */
 138
 139 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
 140 {
 141         unsigned int blength;
 142         int s2length;
 143         uint32 c;
 144         uint64 l;
 145
 146         if (block_size) {
 147                 blength = block_size;
 148         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 149                 blength = BLOCK_SIZE;
 150         } else {
 151                 l = len;
 152                 c = 1;
 153                 while (l >>= 2) {
 154                         c <<= 1;
 155                 }
 156                 blength = 0;
 157                 do {
 158                         blength |= c;
 159                         if (len < (uint64)blength * blength)
 160                                 blength &= ~c;
 161                         c >>= 1;
 162                 } while (c >= 8);       /* round to multiple of 8 */
 163                 blength = MAX(blength, BLOCK_SIZE);
 164         }
 165
 166         if (protocol_version < 27) {
 167                 s2length = csum_length;
 168         } else if (csum_length == SUM_LENGTH) {
 169                 s2length = SUM_LENGTH;
 170         } else {
 171                 int b = BLOCKSUM_BIAS;
 172                 l = len;
 173                 while (l >>= 1) {
 174                         b += 2;
 175                 }
 176                 c = blength;
 177                 while (c >>= 1 && b) {
 178                         b--;
 179                 }
 180                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 181                                                   * subtract rollsum,
 182                                                   * round up
 183                                                   *    --optimize in compiler--
 184                                                   */
 185                 s2length = MAX(s2length, csum_length);
 186                 s2length = MIN(s2length, SUM_LENGTH);
 187         }
 188
 189         sum->flength    = len;
 190         sum->blength    = blength;
 191         sum->s2length   = s2length;
 192         sum->count      = (len + (blength - 1)) / blength;
 193         sum->remainder  = (len % blength);
 194
 195         if (sum->count && verbose > 2) {
 196                 rprintf(FINFO, "count=%.0f rem=%u blength=%u s2length=%d flength=%.0f\n",
 197                         (double)sum->count, sum->remainder, sum->blength,
 198                         sum->s2length, (double)sum->flength);
 199         }
 200 }
 201
 202
 203 /*
 204  * Generate and send a stream of signatures/checksums that describe a buffer
 205  *
 206  * Generate approximately one checksum every block_len bytes.
 207  */
 208 static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out)
 209 {
 210         size_t i;
 211         struct sum_struct sum;
 212         OFF_T offset = 0;
 213
 214         sum_sizes_sqroot(&sum, len);
 215
 216         write_sum_head(f_out, &sum);
 217
 218         for (i = 0; i < sum.count; i++) {
 219                 unsigned int n1 = MIN(len, sum.blength);
 220                 char *map = map_ptr(buf, offset, n1);
 221                 uint32 sum1 = get_checksum1(map, n1);
 222                 char sum2[SUM_LENGTH];
 223
 224                 get_checksum2(map, n1, sum2);
 225
 226                 if (verbose > 3) {
 227                         rprintf(FINFO,
 228                                 "chunk[%.0f] offset=%.0f len=%u sum1=%08lx\n",
 229                                 (double)i, (double)offset, n1,
 230                                 (unsigned long)sum1);
 231                 }
 232                 write_int(f_out, sum1);
 233                 write_buf(f_out, sum2, sum.s2length);
 234                 len -= n1;
 235                 offset += n1;
 236         }
 237 }
 238
 239
 240
 241 /*
 242  * Acts on file number @p i from @p flist, whose name is @p fname.
 243  *
 244  * First fixes up permissions, then generates checksums for the file.
 245  *
 246  * @note This comment was added later by mbp who was trying to work it
 247  * out.  It might be wrong.
 248  */
 249 static void recv_generator(char *fname, struct file_struct *file, int i,
 250                            int f_out)
 251 {
 252         int fd;
 253         STRUCT_STAT st;
 254         struct map_struct *mapbuf;
 255         int statret;
 256         char *fnamecmp;
 257         char fnamecmpbuf[MAXPATHLEN];
 258
 259         if (list_only)
 260                 return;
 261
 262         if (verbose > 2)
 263                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 264
 265         if (server_exclude_list.head
 266             && check_exclude(&server_exclude_list, fname,
 267                              S_ISDIR(file->mode)) < 0) {
 268                 if (verbose) {
 269                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
 270                                 fname);
 271                 }
 272                 return;
 273         }
 274
 275         statret = link_stat(fname, &st, keep_dirlinks && S_ISDIR(file->mode));
 276
 277         if (only_existing && statret == -1 && errno == ENOENT) {
 278                 /* we only want to update existing files */
 279                 if (verbose > 1)
 280                         rprintf(FINFO, "not creating new file \"%s\"\n", fname);
 281                 return;
 282         }
 283
 284         if (statret == 0 &&
 285             !preserve_perms &&
 286             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 287                 /* if the file exists already and we aren't perserving
 288                  * permissions then act as though the remote end sent
 289                  * us the file permissions we already have */
 290                 file->mode = (file->mode & ~CHMOD_BITS)
 291                            | (st.st_mode & CHMOD_BITS);
 292         }
 293
 294         if (S_ISDIR(file->mode)) {
 295                 /* The file to be received is a directory, so we need
 296                  * to prepare appropriately.  If there is already a
 297                  * file of that name and it is *not* a directory, then
 298                  * we need to delete it.  If it doesn't exist, then
 299                  * recursively create it. */
 300
 301                 if (dry_run)
 302                         return; /* TODO: causes inaccuracies -- fix */
 303                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 304                         if (robust_unlink(fname) != 0) {
 305                                 rsyserr(FERROR, errno,
 306                                         "recv_generator: unlink %s to make room for directory",
 307                                         full_fname(fname));
 308                                 return;
 309                         }
 310                         statret = -1;
 311                 }
 312                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 313                         if (!(relative_paths && errno == ENOENT
 314                             && create_directory_path(fname, orig_umask) == 0
 315                             && do_mkdir(fname, file->mode) == 0)) {
 316                                 rsyserr(FERROR, errno,
 317                                         "recv_generator: mkdir %s failed",
 318                                         full_fname(fname));
 319                         }
 320                 }
 321                 /* f_out is set to -1 when doing final directory-permission
 322                  * and modification-time repair. */
 323                 if (set_perms(fname, file, statret ? NULL : &st, 0)
 324                     && verbose && f_out != -1)
 325                         rprintf(FINFO,"%s/\n",fname);
 326                 return;
 327         }
 328
 329         if (preserve_links && S_ISLNK(file->mode)) {
 330 #if SUPPORT_LINKS
 331                 char lnk[MAXPATHLEN];
 332                 int l;
 333
 334                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
 335                         if (verbose) {
 336                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
 337                                         full_fname(fname), file->u.link);
 338                         }
 339                         return;
 340                 }
 341                 if (statret == 0) {
 342                         l = readlink(fname,lnk,MAXPATHLEN-1);
 343                         if (l > 0) {
 344                                 lnk[l] = 0;
 345                                 /* A link already pointing to the
 346                                  * right place -- no further action
 347                                  * required. */
 348                                 if (strcmp(lnk,file->u.link) == 0) {
 349                                         set_perms(fname, file, &st,
 350                                                   PERMS_REPORT);
 351                                         return;
 352                                 }
 353                         }
 354                         /* Not a symlink, so delete whatever's
 355                          * already there and put a new symlink
 356                          * in place. */
 357                         delete_file(fname);
 358                 }
 359                 if (do_symlink(file->u.link,fname) != 0) {
 360                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
 361                                 full_fname(fname), file->u.link);
 362                 } else {
 363                         set_perms(fname,file,NULL,0);
 364                         if (verbose) {
 365                                 rprintf(FINFO,"%s -> %s\n", fname,file->u.link);
 366                         }
 367                 }
 368 #endif
 369                 return;
 370         }
 371
 372 #ifdef HAVE_MKNOD
 373         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 374                 if (statret != 0 ||
 375                     st.st_mode != file->mode ||
 376                     st.st_rdev != file->u.rdev) {
 377                         delete_file(fname);
 378                         if (verbose > 2) {
 379                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 380                                         fname,(int)file->mode,(int)file->u.rdev);
 381                         }
 382                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
 383                                 rsyserr(FERROR, errno, "mknod %s failed",
 384                                         full_fname(fname));
 385                         } else {
 386                                 set_perms(fname,file,NULL,0);
 387                                 if (verbose)
 388                                         rprintf(FINFO,"%s\n",fname);
 389                         }
 390                 } else {
 391                         set_perms(fname, file, &st, PERMS_REPORT);
 392                 }
 393                 return;
 394         }
 395 #endif
 396
 397         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
 398                 return;
 399
 400         if (!S_ISREG(file->mode)) {
 401                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 402                 return;
 403         }
 404
 405         fnamecmp = fname;
 406
 407         if (statret == -1 && compare_dest != NULL) {
 408                 /* try the file at compare_dest instead */
 409                 int saveerrno = errno;
 410                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
 411                 statret = link_stat(fnamecmpbuf, &st, 0);
 412                 if (!S_ISREG(st.st_mode))
 413                         statret = -1;
 414                 if (statret == -1)
 415                         errno = saveerrno;
 416 #if HAVE_LINK
 417                 else if (link_dest && !dry_run) {
 418                         if (do_link(fnamecmpbuf, fname) != 0) {
 419                                 if (verbose > 0) {
 420                                         rsyserr(FINFO, errno, "link %s => %s",
 421                                                 fnamecmpbuf, fname);
 422                                 }
 423                         }
 424                         fnamecmp = fnamecmpbuf;
 425                 }
 426 #endif
 427                 else
 428                         fnamecmp = fnamecmpbuf;
 429         }
 430
 431         if (statret == -1) {
 432                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 433                         return;
 434                 if (errno == ENOENT) {
 435                         write_int(f_out,i);
 436                         if (!dry_run)
 437                                 write_sum_head(f_out, NULL);
 438                 } else if (verbose > 1) {
 439                         rsyserr(FERROR, errno,
 440                                 "recv_generator: failed to open %s",
 441                                 full_fname(fname));
 442                 }
 443                 return;
 444         }
 445
 446         if (!S_ISREG(st.st_mode)) {
 447                 if (delete_file(fname) != 0) {
 448                         return;
 449                 }
 450
 451                 /* now pretend the file didn't exist */
 452                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 453                         return;
 454                 write_int(f_out,i);
 455                 if (!dry_run)
 456                         write_sum_head(f_out, NULL);
 457                 return;
 458         }
 459
 460         if (opt_ignore_existing && fnamecmp == fname) {
 461                 if (verbose > 1)
 462                         rprintf(FINFO,"%s exists\n",fname);
 463                 return;
 464         }
 465
 466         if (update_only && fnamecmp == fname
 467             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 468                 if (verbose > 1)
 469                         rprintf(FINFO,"%s is newer\n",fname);
 470                 return;
 471         }
 472
 473         if (skip_file(fname, file, &st)) {
 474                 if (fnamecmp == fname)
 475                         set_perms(fname, file, &st, PERMS_REPORT);
 476                 return;
 477         }
 478
 479         if (dry_run) {
 480                 write_int(f_out,i);
 481                 return;
 482         }
 483
 484         if (whole_file > 0) {
 485                 write_int(f_out,i);
 486                 write_sum_head(f_out, NULL);
 487                 return;
 488         }
 489
 490         /* open the file */
 491         fd = do_open(fnamecmp, O_RDONLY, 0);
 492
 493         if (fd == -1) {
 494                 rsyserr(FERROR, errno, "failed to open %s, continuing",
 495                         full_fname(fnamecmp));
 496                 /* pretend the file didn't exist */
 497                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 498                         return;
 499                 write_int(f_out,i);
 500                 write_sum_head(f_out, NULL);
 501                 return;
 502         }
 503
 504         if (st.st_size > 0)
 505                 mapbuf = map_file(fd,st.st_size);
 506         else
 507                 mapbuf = NULL;
 508
 509         if (verbose > 3) {
 510                 rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
 511                         (double)st.st_size);
 512         }
 513
 514         if (verbose > 2)
 515                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 516
 517         write_int(f_out,i);
 518         generate_and_send_sums(mapbuf, st.st_size, f_out);
 519
 520         close(fd);
 521         if (mapbuf)
 522                 unmap_file(mapbuf);
 523 }
 524
 525
 526 void generate_files(int f_out, struct file_list *flist, char *local_name)
 527 {
 528         int i;
 529         int phase = 0;
 530         char fbuf[MAXPATHLEN];
 531
 532         if (verbose > 2) {
 533                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
 534                         (long)getpid(), flist->count);
 535         }
 536
 537         if (verbose >= 2) {
 538                 rprintf(FINFO,
 539                         whole_file > 0
 540                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 541                         : "delta transmission enabled\n");
 542         }
 543
 544         /* we expect to just sit around now, so don't exit on a
 545            timeout. If we really get a timeout then the other process should
 546            exit */
 547         io_timeout = 0;
 548
 549         for (i = 0; i < flist->count; i++) {
 550                 struct file_struct *file = flist->files[i];
 551                 struct file_struct copy;
 552
 553                 if (!file->basename)
 554                         continue;
 555                 /* we need to ensure that any directories we create have writeable
 556                    permissions initially so that we can create the files within
 557                    them. This is then fixed after the files are transferred */
 558                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
 559                         copy = *file;
 560                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 561                          * handling of permissions is strange? */
 562                         copy.mode |= S_IWUSR; /* user write */
 563                         file = &copy;
 564                 }
 565
 566                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 567                                file, i, f_out);
 568         }
 569
 570         phase++;
 571         csum_length = SUM_LENGTH;
 572         ignore_times = 1;
 573
 574         if (verbose > 2)
 575                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 576
 577         write_int(f_out, -1);
 578
 579         /* files can cycle through the system more than once
 580          * to catch initial checksum errors */
 581         while ((i = get_redo_num()) != -1) {
 582                 struct file_struct *file = flist->files[i];
 583                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 584                                file, i, f_out);
 585         }
 586
 587         phase++;
 588         if (verbose > 2)
 589                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 590
 591         write_int(f_out, -1);
 592
 593         if (preserve_hard_links)
 594                 do_hard_links();
 595
 596         /* now we need to fix any directory permissions that were
 597          * modified during the transfer */
 598         for (i = 0; i < flist->count; i++) {
 599                 struct file_struct *file = flist->files[i];
 600                 if (!file->basename || !S_ISDIR(file->mode))
 601                         continue;
 602                 recv_generator(local_name ? local_name : f_name(file),
 603                                file, i, -1);
 604         }
 605
 606         if (verbose > 2)
 607                 rprintf(FINFO,"generate_files finished\n");
 608 }