generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int size_only;
  35 extern int io_timeout;
  36 extern int remote_version;
  37 extern int always_checksum;
  38
  39
  40 /* choose whether to skip a particular file */
  41 static int skip_file(char *fname,
  42                      struct file_struct *file, STRUCT_STAT *st)
  43 {
  44         if (st->st_size != file->length) {
  45                 return 0;
  46         }
  47
  48         /* if always checksum is set then we use the checksum instead
  49            of the file time to determine whether to sync */
  50         if (always_checksum && S_ISREG(st->st_mode)) {
  51                 char sum[MD4_SUM_LENGTH];
  52                 file_checksum(fname,sum,st->st_size);
  53                 if (remote_version < 21) {
  54                         return (memcmp(sum,file->sum,2) == 0);
  55                 } else {
  56                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  57                 }
  58         }
  59
  60         if (size_only) {
  61                 return 1;
  62         }
  63
  64         if (ignore_times) {
  65                 return 0;
  66         }
  67
  68         return (st->st_mtime == file->modtime);
  69 }
  70
  71
  72 /* use a larger block size for really big files */
  73 static int adapt_block_size(struct file_struct *file, int bsize)
  74 {
  75         int ret;
  76
  77         if (bsize != BLOCK_SIZE) return bsize;
  78
  79         ret = file->length / (10000); /* rough heuristic */
  80         ret = ret & ~15; /* multiple of 16 */
  81         if (ret < bsize) ret = bsize;
  82         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  83         return ret;
  84 }
  85
  86
  87 /*
  88   send a sums struct down a fd
  89   */
  90 static void send_sums(struct sum_struct *s,int f_out)
  91 {
  92         int i;
  93
  94         /* tell the other guy how many we are going to be doing and how many
  95            bytes there are in the last chunk */
  96         write_int(f_out,s?s->count:0);
  97         write_int(f_out,s?s->n:block_size);
  98         write_int(f_out,s?s->remainder:0);
  99
 100         if (!s) return;
 101
 102         for (i=0;i<s->count;i++) {
 103                 write_int(f_out,s->sums[i].sum1);
 104                 write_buf(f_out,s->sums[i].sum2,csum_length);
 105         }
 106 }
 107
 108
 109 /*
 110   generate a stream of signatures/checksums that describe a buffer
 111
 112   generate approximately one checksum every n bytes
 113   */
 114 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 115 {
 116         int i;
 117         struct sum_struct *s;
 118         int count;
 119         int block_len = n;
 120         int remainder = (len%block_len);
 121         OFF_T offset = 0;
 122
 123         count = (len+(block_len-1))/block_len;
 124
 125         s = (struct sum_struct *)malloc(sizeof(*s));
 126         if (!s) out_of_memory("generate_sums");
 127
 128         s->count = count;
 129         s->remainder = remainder;
 130         s->n = n;
 131         s->flength = len;
 132
 133         if (count==0) {
 134                 s->sums = NULL;
 135                 return s;
 136         }
 137
 138         if (verbose > 3)
 139                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
 140                         s->count,s->remainder,s->n,(int)s->flength);
 141
 142         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 143         if (!s->sums) out_of_memory("generate_sums");
 144
 145         for (i=0;i<count;i++) {
 146                 int n1 = MIN(len,n);
 147                 char *map = map_ptr(buf,offset,n1);
 148
 149                 s->sums[i].sum1 = get_checksum1(map,n1);
 150                 get_checksum2(map,n1,s->sums[i].sum2);
 151
 152                 s->sums[i].offset = offset;
 153                 s->sums[i].len = n1;
 154                 s->sums[i].i = i;
 155
 156                 if (verbose > 3)
 157                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
 158                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 159
 160                 len -= n1;
 161                 offset += n1;
 162         }
 163
 164         return s;
 165 }
 166
 167
 168 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 169 {
 170         int fd;
 171         STRUCT_STAT st;
 172         struct map_struct *buf;
 173         struct sum_struct *s;
 174         int statret;
 175         struct file_struct *file = flist->files[i];
 176         char *fnamecmp;
 177         char fnamecmpbuf[MAXPATHLEN];
 178         extern char *compare_dest;
 179         extern int list_only;
 180
 181         if (list_only) return;
 182
 183         if (verbose > 2)
 184                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 185
 186         statret = link_stat(fname,&st);
 187
 188         if (S_ISDIR(file->mode)) {
 189                 if (dry_run) return;
 190                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 191                         if (robust_unlink(fname) != 0) {
 192                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 193                                 return;
 194                         }
 195                         statret = -1;
 196                 }
 197                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 198                         if (!(relative_paths && errno==ENOENT &&
 199                               create_directory_path(fname)==0 &&
 200                               do_mkdir(fname,file->mode)==0)) {
 201                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 202                                         fname,strerror(errno));
 203                         }
 204                 }
 205                 if (set_perms(fname,file,NULL,0) && verbose)
 206                         rprintf(FINFO,"%s/\n",fname);
 207                 return;
 208         }
 209
 210         if (preserve_links && S_ISLNK(file->mode)) {
 211 #if SUPPORT_LINKS
 212                 char lnk[MAXPATHLEN];
 213                 int l;
 214                 extern int safe_symlinks;
 215
 216                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 217                         if (verbose) {
 218                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 219                                         fname,file->link);
 220                         }
 221                         return;
 222                 }
 223                 if (statret == 0) {
 224                         l = readlink(fname,lnk,MAXPATHLEN-1);
 225                         if (l > 0) {
 226                                 lnk[l] = 0;
 227                                 if (strcmp(lnk,file->link) == 0) {
 228                                         set_perms(fname,file,&st,1);
 229                                         return;
 230                                 }
 231                         }
 232                 }
 233                 delete_file(fname);
 234                 if (do_symlink(file->link,fname) != 0) {
 235                         rprintf(FERROR,"link %s -> %s : %s\n",
 236                                 fname,file->link,strerror(errno));
 237                 } else {
 238                         set_perms(fname,file,NULL,0);
 239                         if (verbose) {
 240                                 rprintf(FINFO,"%s -> %s\n",
 241                                         fname,file->link);
 242                         }
 243                 }
 244 #endif
 245                 return;
 246         }
 247
 248 #ifdef HAVE_MKNOD
 249         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 250                 if (statret != 0 ||
 251                     st.st_mode != file->mode ||
 252                     st.st_rdev != file->rdev) {
 253                         delete_file(fname);
 254                         if (verbose > 2)
 255                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 256                                         fname,(int)file->mode,(int)file->rdev);
 257                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 258                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 259                         } else {
 260                                 set_perms(fname,file,NULL,0);
 261                                 if (verbose)
 262                                         rprintf(FINFO,"%s\n",fname);
 263                         }
 264                 } else {
 265                         set_perms(fname,file,&st,1);
 266                 }
 267                 return;
 268         }
 269 #endif
 270
 271         if (preserve_hard_links && check_hard_link(file)) {
 272                 if (verbose > 1)
 273                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 274                 return;
 275         }
 276
 277         if (!S_ISREG(file->mode)) {
 278                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 279                 return;
 280         }
 281
 282         fnamecmp = fname;
 283
 284         if ((statret == -1) && (compare_dest != NULL)) {
 285                 /* try the file at compare_dest instead */
 286                 int saveerrno = errno;
 287                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 288                 statret = link_stat(fnamecmpbuf,&st);
 289                 if (!S_ISREG(st.st_mode))
 290                         statret = -1;
 291                 if (statret == -1)
 292                         errno = saveerrno;
 293                 else
 294                         fnamecmp = fnamecmpbuf;
 295         }
 296
 297         if (statret == -1) {
 298                 if (errno == ENOENT) {
 299                         write_int(f_out,i);
 300                         if (!dry_run) send_sums(NULL,f_out);
 301                 } else {
 302                         if (verbose > 1)
 303                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 304                 }
 305                 return;
 306         }
 307
 308         if (!S_ISREG(st.st_mode)) {
 309                 if (delete_file(fname) != 0) {
 310                         return;
 311                 }
 312
 313                 /* now pretend the file didn't exist */
 314                 write_int(f_out,i);
 315                 if (!dry_run) send_sums(NULL,f_out);
 316                 return;
 317         }
 318
 319         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 320                 if (verbose > 1)
 321                         rprintf(FINFO,"%s is newer\n",fname);
 322                 return;
 323         }
 324
 325         if (skip_file(fname, file, &st)) {
 326                 if (fnamecmp == fname)
 327                         set_perms(fname,file,&st,1);
 328                 return;
 329         }
 330
 331         if (dry_run) {
 332                 write_int(f_out,i);
 333                 return;
 334         }
 335
 336         if (whole_file) {
 337                 write_int(f_out,i);
 338                 send_sums(NULL,f_out);
 339                 return;
 340         }
 341
 342         /* open the file */
 343         fd = open(fnamecmp,O_RDONLY);
 344
 345         if (fd == -1) {
 346                 rprintf(FERROR,"failed to open %s : %s\n",fnamecmp,strerror(errno));
 347                 rprintf(FERROR,"skipping %s\n",fname);
 348                 return;
 349         }
 350
 351         if (st.st_size > 0) {
 352                 buf = map_file(fd,st.st_size);
 353         } else {
 354                 buf = NULL;
 355         }
 356
 357         if (verbose > 3)
 358                 rprintf(FINFO,"gen mapped %s of size %d\n",fnamecmp,(int)st.st_size);
 359
 360         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 361
 362         if (verbose > 2)
 363                 rprintf(FINFO,"sending sums for %d\n",i);
 364
 365         write_int(f_out,i);
 366         send_sums(s,f_out);
 367
 368         close(fd);
 369         if (buf) unmap_file(buf);
 370
 371         free_sums(s);
 372 }
 373
 374
 375
 376 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 377 {
 378         int i;
 379         int phase=0;
 380
 381         if (verbose > 2)
 382                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 383                         (int)getpid(),flist->count);
 384
 385         for (i = 0; i < flist->count; i++) {
 386                 struct file_struct *file = flist->files[i];
 387                 mode_t saved_mode = file->mode;
 388                 if (!file->basename) continue;
 389
 390                 /* we need to ensure that any directories we create have writeable
 391                    permissions initially so that we can create the files within
 392                    them. This is then fixed after the files are transferred */
 393                 if (!am_root && S_ISDIR(file->mode)) {
 394                         file->mode |= S_IWUSR; /* user write */
 395                 }
 396
 397                 recv_generator(local_name?local_name:f_name(file),
 398                                flist,i,f);
 399
 400                 file->mode = saved_mode;
 401         }
 402
 403         phase++;
 404         csum_length = SUM_LENGTH;
 405         ignore_times=1;
 406
 407         if (verbose > 2)
 408                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 409
 410         write_int(f,-1);
 411
 412         /* we expect to just sit around now, so don't exit on a
 413            timeout. If we really get a timeout then the other process should
 414            exit */
 415         io_timeout = 0;
 416
 417         if (remote_version >= 13) {
 418                 /* in newer versions of the protocol the files can cycle through
 419                    the system more than once to catch initial checksum errors */
 420                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 421                         struct file_struct *file = flist->files[i];
 422                         recv_generator(local_name?local_name:f_name(file),
 423                                        flist,i,f);
 424                 }
 425
 426                 phase++;
 427                 if (verbose > 2)
 428                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 429
 430                 write_int(f,-1);
 431         }
 432 }