generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int size_only;
  35 extern int io_timeout;
  36 extern int remote_version;
  37 extern int always_checksum;
  38
  39
  40 /* choose whether to skip a particular file */
  41 static int skip_file(char *fname,
  42                      struct file_struct *file, STRUCT_STAT *st)
  43 {
  44         if (st->st_size != file->length) {
  45                 return 0;
  46         }
  47
  48         /* if always checksum is set then we use the checksum instead
  49            of the file time to determine whether to sync */
  50         if (always_checksum && S_ISREG(st->st_mode)) {
  51                 char sum[MD4_SUM_LENGTH];
  52                 file_checksum(fname,sum,st->st_size);
  53                 if (remote_version < 21) {
  54                         return (memcmp(sum,file->sum,2) == 0);
  55                 } else {
  56                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  57                 }
  58         }
  59
  60         if (size_only) {
  61                 return 1;
  62         }
  63
  64         if (ignore_times) {
  65                 return 0;
  66         }
  67
  68         return (st->st_mtime == file->modtime);
  69 }
  70
  71
  72 /* use a larger block size for really big files */
  73 static int adapt_block_size(struct file_struct *file, int bsize)
  74 {
  75         int ret;
  76
  77         if (bsize != BLOCK_SIZE) return bsize;
  78
  79         ret = file->length / (10000); /* rough heuristic */
  80         ret = ret & ~15; /* multiple of 16 */
  81         if (ret < bsize) ret = bsize;
  82         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  83         return ret;
  84 }
  85
  86
  87 /*
  88   send a sums struct down a fd
  89   */
  90 static void send_sums(struct sum_struct *s,int f_out)
  91 {
  92         int i;
  93
  94         /* tell the other guy how many we are going to be doing and how many
  95            bytes there are in the last chunk */
  96         write_int(f_out,s?s->count:0);
  97         write_int(f_out,s?s->n:block_size);
  98         write_int(f_out,s?s->remainder:0);
  99
 100         if (!s) return;
 101
 102         for (i=0;i<s->count;i++) {
 103                 write_int(f_out,s->sums[i].sum1);
 104                 write_buf(f_out,s->sums[i].sum2,csum_length);
 105         }
 106 }
 107
 108
 109 /*
 110   generate a stream of signatures/checksums that describe a buffer
 111
 112   generate approximately one checksum every n bytes
 113   */
 114 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 115 {
 116         int i;
 117         struct sum_struct *s;
 118         int count;
 119         int block_len = n;
 120         int remainder = (len%block_len);
 121         OFF_T offset = 0;
 122
 123         count = (len+(block_len-1))/block_len;
 124
 125         s = (struct sum_struct *)malloc(sizeof(*s));
 126         if (!s) out_of_memory("generate_sums");
 127
 128         s->count = count;
 129         s->remainder = remainder;
 130         s->n = n;
 131         s->flength = len;
 132
 133         if (count==0) {
 134                 s->sums = NULL;
 135                 return s;
 136         }
 137
 138         if (verbose > 3)
 139                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
 140                         s->count,s->remainder,s->n,(int)s->flength);
 141
 142         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 143         if (!s->sums) out_of_memory("generate_sums");
 144
 145         for (i=0;i<count;i++) {
 146                 int n1 = MIN(len,n);
 147                 char *map = map_ptr(buf,offset,n1);
 148
 149                 s->sums[i].sum1 = get_checksum1(map,n1);
 150                 get_checksum2(map,n1,s->sums[i].sum2);
 151
 152                 s->sums[i].offset = offset;
 153                 s->sums[i].len = n1;
 154                 s->sums[i].i = i;
 155
 156                 if (verbose > 3)
 157                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
 158                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 159
 160                 len -= n1;
 161                 offset += n1;
 162         }
 163
 164         return s;
 165 }
 166
 167
 168 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 169 {
 170         int fd;
 171         STRUCT_STAT st;
 172         struct map_struct *buf;
 173         struct sum_struct *s;
 174         int statret;
 175         struct file_struct *file = flist->files[i];
 176         char *fnamecmp;
 177         char fnamecmpbuf[MAXPATHLEN];
 178         extern char *compare_dest;
 179         extern int list_only;
 180         extern int preserve_perms;
 181
 182         if (list_only) return;
 183
 184         if (verbose > 2)
 185                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 186
 187         statret = link_stat(fname,&st);
 188
 189         if (statret == 0 &&
 190             !preserve_perms &&
 191             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 192                 /* if the file exists already and we aren't perserving
 193                    presmissions then act as though the remote end sent
 194                    us the file permissions we already have */
 195                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 196         }
 197
 198         if (S_ISDIR(file->mode)) {
 199                 if (dry_run) return;
 200                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 201                         if (robust_unlink(fname) != 0) {
 202                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 203                                 return;
 204                         }
 205                         statret = -1;
 206                 }
 207                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 208                         if (!(relative_paths && errno==ENOENT &&
 209                               create_directory_path(fname)==0 &&
 210                               do_mkdir(fname,file->mode)==0)) {
 211                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 212                                         fname,strerror(errno));
 213                         }
 214                 }
 215                 if (set_perms(fname,file,NULL,0) && verbose)
 216                         rprintf(FINFO,"%s/\n",fname);
 217                 return;
 218         }
 219
 220         if (preserve_links && S_ISLNK(file->mode)) {
 221 #if SUPPORT_LINKS
 222                 char lnk[MAXPATHLEN];
 223                 int l;
 224                 extern int safe_symlinks;
 225
 226                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 227                         if (verbose) {
 228                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 229                                         fname,file->link);
 230                         }
 231                         return;
 232                 }
 233                 if (statret == 0) {
 234                         l = readlink(fname,lnk,MAXPATHLEN-1);
 235                         if (l > 0) {
 236                                 lnk[l] = 0;
 237                                 if (strcmp(lnk,file->link) == 0) {
 238                                         set_perms(fname,file,&st,1);
 239                                         return;
 240                                 }
 241                         }
 242                 }
 243                 delete_file(fname);
 244                 if (do_symlink(file->link,fname) != 0) {
 245                         rprintf(FERROR,"link %s -> %s : %s\n",
 246                                 fname,file->link,strerror(errno));
 247                 } else {
 248                         set_perms(fname,file,NULL,0);
 249                         if (verbose) {
 250                                 rprintf(FINFO,"%s -> %s\n",
 251                                         fname,file->link);
 252                         }
 253                 }
 254 #endif
 255                 return;
 256         }
 257
 258 #ifdef HAVE_MKNOD
 259         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 260                 if (statret != 0 ||
 261                     st.st_mode != file->mode ||
 262                     st.st_rdev != file->rdev) {
 263                         delete_file(fname);
 264                         if (verbose > 2)
 265                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 266                                         fname,(int)file->mode,(int)file->rdev);
 267                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 268                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 269                         } else {
 270                                 set_perms(fname,file,NULL,0);
 271                                 if (verbose)
 272                                         rprintf(FINFO,"%s\n",fname);
 273                         }
 274                 } else {
 275                         set_perms(fname,file,&st,1);
 276                 }
 277                 return;
 278         }
 279 #endif
 280
 281         if (preserve_hard_links && check_hard_link(file)) {
 282                 if (verbose > 1)
 283                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 284                 return;
 285         }
 286
 287         if (!S_ISREG(file->mode)) {
 288                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 289                 return;
 290         }
 291
 292         fnamecmp = fname;
 293
 294         if ((statret == -1) && (compare_dest != NULL)) {
 295                 /* try the file at compare_dest instead */
 296                 int saveerrno = errno;
 297                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 298                 statret = link_stat(fnamecmpbuf,&st);
 299                 if (!S_ISREG(st.st_mode))
 300                         statret = -1;
 301                 if (statret == -1)
 302                         errno = saveerrno;
 303                 else
 304                         fnamecmp = fnamecmpbuf;
 305         }
 306
 307         if (statret == -1) {
 308                 if (errno == ENOENT) {
 309                         write_int(f_out,i);
 310                         if (!dry_run) send_sums(NULL,f_out);
 311                 } else {
 312                         if (verbose > 1)
 313                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 314                 }
 315                 return;
 316         }
 317
 318         if (!S_ISREG(st.st_mode)) {
 319                 if (delete_file(fname) != 0) {
 320                         return;
 321                 }
 322
 323                 /* now pretend the file didn't exist */
 324                 write_int(f_out,i);
 325                 if (!dry_run) send_sums(NULL,f_out);
 326                 return;
 327         }
 328
 329         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 330                 if (verbose > 1)
 331                         rprintf(FINFO,"%s is newer\n",fname);
 332                 return;
 333         }
 334
 335         if (skip_file(fname, file, &st)) {
 336                 if (fnamecmp == fname)
 337                         set_perms(fname,file,&st,1);
 338                 return;
 339         }
 340
 341         if (dry_run) {
 342                 write_int(f_out,i);
 343                 return;
 344         }
 345
 346         if (whole_file) {
 347                 write_int(f_out,i);
 348                 send_sums(NULL,f_out);
 349                 return;
 350         }
 351
 352         /* open the file */
 353         fd = do_open(fnamecmp, O_RDONLY, 0);
 354
 355         if (fd == -1) {
 356                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
 357                 /* pretend the file didn't exist */
 358                 write_int(f_out,i);
 359                 send_sums(NULL,f_out);
 360                 return;
 361         }
 362
 363         if (st.st_size > 0) {
 364                 buf = map_file(fd,st.st_size);
 365         } else {
 366                 buf = NULL;
 367         }
 368
 369         if (verbose > 3)
 370                 rprintf(FINFO,"gen mapped %s of size %d\n",fnamecmp,(int)st.st_size);
 371
 372         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 373
 374         if (verbose > 2)
 375                 rprintf(FINFO,"sending sums for %d\n",i);
 376
 377         write_int(f_out,i);
 378         send_sums(s,f_out);
 379
 380         close(fd);
 381         if (buf) unmap_file(buf);
 382
 383         free_sums(s);
 384 }
 385
 386
 387
 388 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 389 {
 390         int i;
 391         int phase=0;
 392
 393         if (verbose > 2)
 394                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 395                         (int)getpid(),flist->count);
 396
 397         for (i = 0; i < flist->count; i++) {
 398                 struct file_struct *file = flist->files[i];
 399                 mode_t saved_mode = file->mode;
 400                 if (!file->basename) continue;
 401
 402                 /* we need to ensure that any directories we create have writeable
 403                    permissions initially so that we can create the files within
 404                    them. This is then fixed after the files are transferred */
 405                 if (!am_root && S_ISDIR(file->mode)) {
 406                         file->mode |= S_IWUSR; /* user write */
 407                 }
 408
 409                 recv_generator(local_name?local_name:f_name(file),
 410                                flist,i,f);
 411
 412                 file->mode = saved_mode;
 413         }
 414
 415         phase++;
 416         csum_length = SUM_LENGTH;
 417         ignore_times=1;
 418
 419         if (verbose > 2)
 420                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 421
 422         write_int(f,-1);
 423
 424         /* we expect to just sit around now, so don't exit on a
 425            timeout. If we really get a timeout then the other process should
 426            exit */
 427         io_timeout = 0;
 428
 429         if (remote_version >= 13) {
 430                 /* in newer versions of the protocol the files can cycle through
 431                    the system more than once to catch initial checksum errors */
 432                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 433                         struct file_struct *file = flist->files[i];
 434                         recv_generator(local_name?local_name:f_name(file),
 435                                        flist,i,f);
 436                 }
 437
 438                 phase++;
 439                 if (verbose > 2)
 440                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 441
 442                 write_int(f,-1);
 443         }
 444 }