generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int size_only;
  35 extern int io_timeout;
  36 extern int remote_version;
  37 extern int always_checksum;
  38
  39
  40 /* choose whether to skip a particular file */
  41 static int skip_file(char *fname,
  42                      struct file_struct *file, STRUCT_STAT *st)
  43 {
  44         if (st->st_size != file->length) {
  45                 return 0;
  46         }
  47
  48         /* if always checksum is set then we use the checksum instead
  49            of the file time to determine whether to sync */
  50         if (always_checksum && S_ISREG(st->st_mode)) {
  51                 char sum[MD4_SUM_LENGTH];
  52                 file_checksum(fname,sum,st->st_size);
  53                 return (memcmp(sum,file->sum,csum_length) == 0);
  54         }
  55
  56         if (size_only) {
  57                 return 1;
  58         }
  59
  60         if (ignore_times) {
  61                 return 0;
  62         }
  63
  64         return (st->st_mtime == file->modtime);
  65 }
  66
  67
  68 /* use a larger block size for really big files */
  69 static int adapt_block_size(struct file_struct *file, int bsize)
  70 {
  71         int ret;
  72
  73         if (bsize != BLOCK_SIZE) return bsize;
  74
  75         ret = file->length / (10000); /* rough heuristic */
  76         ret = ret & ~15; /* multiple of 16 */
  77         if (ret < bsize) ret = bsize;
  78         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  79         return ret;
  80 }
  81
  82
  83 /*
  84   send a sums struct down a fd
  85   */
  86 static void send_sums(struct sum_struct *s,int f_out)
  87 {
  88         int i;
  89
  90   /* tell the other guy how many we are going to be doing and how many
  91      bytes there are in the last chunk */
  92         write_int(f_out,s?s->count:0);
  93         write_int(f_out,s?s->n:block_size);
  94         write_int(f_out,s?s->remainder:0);
  95         if (s)
  96                 for (i=0;i<s->count;i++) {
  97                         write_int(f_out,s->sums[i].sum1);
  98                         write_buf(f_out,s->sums[i].sum2,csum_length);
  99                 }
 100 }
 101
 102
 103 /*
 104   generate a stream of signatures/checksums that describe a buffer
 105
 106   generate approximately one checksum every n bytes
 107   */
 108 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 109 {
 110         int i;
 111         struct sum_struct *s;
 112         int count;
 113         int block_len = n;
 114         int remainder = (len%block_len);
 115         OFF_T offset = 0;
 116
 117         count = (len+(block_len-1))/block_len;
 118
 119         s = (struct sum_struct *)malloc(sizeof(*s));
 120         if (!s) out_of_memory("generate_sums");
 121
 122         s->count = count;
 123         s->remainder = remainder;
 124         s->n = n;
 125         s->flength = len;
 126
 127         if (count==0) {
 128                 s->sums = NULL;
 129                 return s;
 130         }
 131
 132         if (verbose > 3)
 133                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
 134                         s->count,s->remainder,s->n,(int)s->flength);
 135
 136         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 137         if (!s->sums) out_of_memory("generate_sums");
 138
 139         for (i=0;i<count;i++) {
 140                 int n1 = MIN(len,n);
 141                 char *map = map_ptr(buf,offset,n1);
 142
 143                 s->sums[i].sum1 = get_checksum1(map,n1);
 144                 get_checksum2(map,n1,s->sums[i].sum2);
 145
 146                 s->sums[i].offset = offset;
 147                 s->sums[i].len = n1;
 148                 s->sums[i].i = i;
 149
 150                 if (verbose > 3)
 151                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
 152                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 153
 154                 len -= n1;
 155                 offset += n1;
 156         }
 157
 158         return s;
 159 }
 160
 161
 162 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 163 {
 164         int fd;
 165         STRUCT_STAT st;
 166         struct map_struct *buf;
 167         struct sum_struct *s;
 168         int statret;
 169         struct file_struct *file = flist->files[i];
 170         char *fnamecmp;
 171         char fnamecmpbuf[MAXPATHLEN];
 172         extern char *compare_dest;
 173         extern int list_only;
 174
 175         if (list_only) return;
 176
 177         if (verbose > 2)
 178                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 179
 180         statret = link_stat(fname,&st);
 181
 182         if (S_ISDIR(file->mode)) {
 183                 if (dry_run) return;
 184                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 185                         if (robust_unlink(fname) != 0) {
 186                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 187                                 return;
 188                         }
 189                         statret = -1;
 190                 }
 191                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 192                         if (!(relative_paths && errno==ENOENT &&
 193                               create_directory_path(fname)==0 &&
 194                               do_mkdir(fname,file->mode)==0)) {
 195                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 196                                         fname,strerror(errno));
 197                         }
 198                 }
 199                 if (set_perms(fname,file,NULL,0) && verbose)
 200                         rprintf(FINFO,"%s/\n",fname);
 201                 return;
 202         }
 203
 204         if (preserve_links && S_ISLNK(file->mode)) {
 205 #if SUPPORT_LINKS
 206                 char lnk[MAXPATHLEN];
 207                 int l;
 208                 extern int safe_symlinks;
 209
 210                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 211                         if (verbose) {
 212                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 213                                         fname,file->link);
 214                         }
 215                         return;
 216                 }
 217                 if (statret == 0) {
 218                         l = readlink(fname,lnk,MAXPATHLEN-1);
 219                         if (l > 0) {
 220                                 lnk[l] = 0;
 221                                 if (strcmp(lnk,file->link) == 0) {
 222                                         set_perms(fname,file,&st,1);
 223                                         return;
 224                                 }
 225                         }
 226                 }
 227                 delete_file(fname);
 228                 if (do_symlink(file->link,fname) != 0) {
 229                         rprintf(FERROR,"link %s -> %s : %s\n",
 230                                 fname,file->link,strerror(errno));
 231                 } else {
 232                         set_perms(fname,file,NULL,0);
 233                         if (verbose) {
 234                                 rprintf(FINFO,"%s -> %s\n",
 235                                         fname,file->link);
 236                         }
 237                 }
 238 #endif
 239                 return;
 240         }
 241
 242 #ifdef HAVE_MKNOD
 243         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 244                 if (statret != 0 ||
 245                     st.st_mode != file->mode ||
 246                     st.st_rdev != file->rdev) {
 247                         delete_file(fname);
 248                         if (verbose > 2)
 249                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 250                                         fname,(int)file->mode,(int)file->rdev);
 251                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 252                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 253                         } else {
 254                                 set_perms(fname,file,NULL,0);
 255                                 if (verbose)
 256                                         rprintf(FINFO,"%s\n",fname);
 257                         }
 258                 } else {
 259                         set_perms(fname,file,&st,1);
 260                 }
 261                 return;
 262         }
 263 #endif
 264
 265         if (preserve_hard_links && check_hard_link(file)) {
 266                 if (verbose > 1)
 267                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 268                 return;
 269         }
 270
 271         if (!S_ISREG(file->mode)) {
 272                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 273                 return;
 274         }
 275
 276         fnamecmp = fname;
 277
 278         if ((statret == -1) && (compare_dest != NULL)) {
 279                 /* try the file at compare_dest instead */
 280                 int saveerrno = errno;
 281                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 282                 statret = link_stat(fnamecmpbuf,&st);
 283                 if (!S_ISREG(st.st_mode))
 284                         statret = -1;
 285                 if (statret == -1)
 286                         errno = saveerrno;
 287                 else
 288                         fnamecmp = fnamecmpbuf;
 289         }
 290
 291         if (statret == -1) {
 292                 if (errno == ENOENT) {
 293                         write_int(f_out,i);
 294                         if (!dry_run) send_sums(NULL,f_out);
 295                 } else {
 296                         if (verbose > 1)
 297                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 298                 }
 299                 return;
 300         }
 301
 302         if (!S_ISREG(st.st_mode)) {
 303                 if (delete_file(fname) != 0) {
 304                         return;
 305                 }
 306
 307                 /* now pretend the file didn't exist */
 308                 write_int(f_out,i);
 309                 if (!dry_run) send_sums(NULL,f_out);
 310                 return;
 311         }
 312
 313         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 314                 if (verbose > 1)
 315                         rprintf(FINFO,"%s is newer\n",fname);
 316                 return;
 317         }
 318
 319         if (skip_file(fname, file, &st)) {
 320                 set_perms(fname,file,&st,1);
 321                 return;
 322         }
 323
 324         if (dry_run) {
 325                 write_int(f_out,i);
 326                 return;
 327         }
 328
 329         if (whole_file) {
 330                 write_int(f_out,i);
 331                 send_sums(NULL,f_out);
 332                 return;
 333         }
 334
 335         /* open the file */
 336         fd = open(fnamecmp,O_RDONLY);
 337
 338         if (fd == -1) {
 339                 rprintf(FERROR,"failed to open %s : %s\n",fnamecmp,strerror(errno));
 340                 rprintf(FERROR,"skipping %s\n",fname);
 341                 return;
 342         }
 343
 344         if (st.st_size > 0) {
 345                 buf = map_file(fd,st.st_size);
 346         } else {
 347                 buf = NULL;
 348         }
 349
 350         if (verbose > 3)
 351                 rprintf(FINFO,"gen mapped %s of size %d\n",fnamecmp,(int)st.st_size);
 352
 353         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 354
 355         if (verbose > 2)
 356                 rprintf(FINFO,"sending sums for %d\n",i);
 357
 358         write_int(f_out,i);
 359         send_sums(s,f_out);
 360
 361         close(fd);
 362         if (buf) unmap_file(buf);
 363
 364         free_sums(s);
 365 }
 366
 367
 368
 369 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 370 {
 371         int i;
 372         int phase=0;
 373
 374         if (verbose > 2)
 375                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 376                         (int)getpid(),flist->count);
 377
 378         for (i = 0; i < flist->count; i++) {
 379                 struct file_struct *file = flist->files[i];
 380                 mode_t saved_mode = file->mode;
 381                 if (!file->basename) continue;
 382
 383                 /* we need to ensure that any directories we create have writeable
 384                    permissions initially so that we can create the files within
 385                    them. This is then fixed after the files are transferred */
 386                 if (!am_root && S_ISDIR(file->mode)) {
 387                         file->mode |= S_IWUSR; /* user write */
 388                 }
 389
 390                 recv_generator(local_name?local_name:f_name(file),
 391                                flist,i,f);
 392
 393                 file->mode = saved_mode;
 394         }
 395
 396         phase++;
 397         csum_length = SUM_LENGTH;
 398         ignore_times=1;
 399
 400         if (verbose > 2)
 401                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 402
 403         write_int(f,-1);
 404
 405         /* we expect to just sit around now, so don't exit on a
 406            timeout. If we really get a timeout then the other process should
 407            exit */
 408         io_timeout = 0;
 409
 410         if (remote_version >= 13) {
 411                 /* in newer versions of the protocol the files can cycle through
 412                    the system more than once to catch initial checksum errors */
 413                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 414                         struct file_struct *file = flist->files[i];
 415                         recv_generator(local_name?local_name:f_name(file),
 416                                        flist,i,f);
 417                 }
 418
 419                 phase++;
 420                 if (verbose > 2)
 421                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 422
 423                 write_int(f,-1);
 424         }
 425 }