g2r-basis-filename.diff

   1 Added a pipe from the generator to the receiver that communicates
   2 what basis file we used to generate the file data (if it was not
   3 the default name).  This optimizes away the basis-file search in
   4 the receiver and makes future options that do more basis-file
   5 searching more efficient (such as the --fuzzy option and the
   6 support for multiple --compare-dest options).
   7
   8 Also fixes a potential synchronization problem between the generator
   9 and the receiver in read-batch mode.  Should consider making the
  10 sending of the index value the default for this pipe (it's currently
  11 only sent in batch mode due to the sender not listening to the
  12 generator to determine what files get updated).
  13
  14 You must run "make proto" before compiling.
  15
  16 --- orig/generator.c    2004-07-20 21:36:07
  17 +++ generator.c 2004-07-17 15:50:09
  18 @@ -258,9 +258,9 @@ static void generate_and_send_sums(int f
  19   * out.  It might be wrong.
  20   */
  21  static void recv_generator(char *fname, struct file_struct *file, int i,
  22 -                          int f_out)
  23 +                          int f_out, int f_nameout)
  24  {
  25 -       int fd;
  26 +       int fd = -1;
  27         STRUCT_STAT st;
  28         int statret;
  29         char *fnamecmp;
  30 @@ -404,9 +404,6 @@ static void recv_generator(char *fname,
  31         }
  32  #endif
  33
  34 -       if (read_batch)
  35 -               return;
  36 -
  37         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
  38                 return;
  39
  40 @@ -424,8 +421,10 @@ static void recv_generator(char *fname,
  41                 statret = link_stat(fnamecmpbuf, &st, 0);
  42                 if (!S_ISREG(st.st_mode))
  43                         statret = -1;
  44 -               if (statret == -1)
  45 +               if (statret == -1) {
  46                         errno = saveerrno;
  47 +                       *fnamecmpbuf = '\0';
  48 +               }
  49  #if HAVE_LINK
  50                 else if (link_dest && !dry_run) {
  51                         if (do_link(fnamecmpbuf, fname) != 0) {
  52 @@ -433,22 +432,22 @@ static void recv_generator(char *fname,
  53                                         rsyserr(FINFO, errno, "link %s => %s",
  54                                                 fnamecmpbuf, fname);
  55                                 }
  56 -                       }
  57 -                       fnamecmp = fnamecmpbuf;
  58 +                               fnamecmp = fnamecmpbuf;
  59 +                       } else
  60 +                               *fnamecmpbuf = '\0';
  61                 }
  62  #endif
  63                 else
  64                         fnamecmp = fnamecmpbuf;
  65 -       }
  66 +       } else
  67 +               *fnamecmpbuf = '\0';
  68
  69         if (statret == -1) {
  70                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  71                         return;
  72 -               if (errno == ENOENT) {
  73 -                       write_int(f_out,i);
  74 -                       if (!dry_run)
  75 -                               write_sum_head(f_out, NULL);
  76 -               } else if (verbose > 1) {
  77 +               if (errno == ENOENT)
  78 +                       goto notify_others;
  79 +               if (verbose > 1) {
  80                         rsyserr(FERROR, errno,
  81                                 "recv_generator: failed to open %s",
  82                                 full_fname(fname));
  83 @@ -457,26 +456,23 @@ static void recv_generator(char *fname,
  84         }
  85
  86         if (!S_ISREG(st.st_mode)) {
  87 -               if (delete_file(fname) != 0) {
  88 +               if (delete_file(fname) != 0)
  89                         return;
  90 -               }
  91
  92                 /* now pretend the file didn't exist */
  93                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  94                         return;
  95 -               write_int(f_out,i);
  96 -               if (!dry_run)
  97 -                       write_sum_head(f_out, NULL);
  98 -               return;
  99 +               statret = -1;
 100 +               goto notify_others;
 101         }
 102
 103 -       if (opt_ignore_existing && fnamecmp == fname) {
 104 +       if (opt_ignore_existing && !*fnamecmpbuf) {
 105                 if (verbose > 1)
 106                         rprintf(FINFO,"%s exists\n",fname);
 107                 return;
 108         }
 109
 110 -       if (update_only && fnamecmp == fname
 111 +       if (update_only && !*fnamecmpbuf
 112             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 113                 if (verbose > 1)
 114                         rprintf(FINFO,"%s is newer\n",fname);
 115 @@ -484,21 +480,17 @@ static void recv_generator(char *fname,
 116         }
 117
 118         if (skip_file(fname, file, &st)) {
 119 -               if (fnamecmp == fname)
 120 +               if (!*fnamecmpbuf)
 121                         set_perms(fname, file, &st, PERMS_REPORT);
 122                 return;
 123         }
 124
 125 -       if (dry_run) {
 126 -               write_int(f_out,i);
 127 -               return;
 128 -       }
 129 -
 130 -       if (whole_file > 0) {
 131 -               write_int(f_out,i);
 132 -               write_sum_head(f_out, NULL);
 133 -               return;
 134 +       if (dry_run || whole_file) {
 135 +               statret = -1;
 136 +               goto notify_others;
 137         }
 138 +       if (read_batch)
 139 +               goto notify_others;
 140
 141         /* open the file */
 142         fd = do_open(fnamecmp, O_RDONLY, 0);
 143 @@ -509,9 +501,8 @@ static void recv_generator(char *fname,
 144                 /* pretend the file didn't exist */
 145                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 146                         return;
 147 -               write_int(f_out,i);
 148 -               write_sum_head(f_out, NULL);
 149 -               return;
 150 +               statret = -1;
 151 +               goto notify_others;
 152         }
 153
 154         if (verbose > 3) {
 155 @@ -522,14 +513,41 @@ static void recv_generator(char *fname,
 156         if (verbose > 2)
 157                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 158
 159 -       write_int(f_out,i);
 160 -       generate_and_send_sums(fd, st.st_size, f_out);
 161 +notify_others:
 162 +       if (f_nameout >= 0) {
 163 +               uchar lenbuf[3], *lb = lenbuf;
 164 +               int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
 165 +               if (read_batch)
 166 +                       write_int(f_nameout, i);
 167 +               if (len > 0x7F) {
 168 +#if MAXPATHLEN > 0x7FFF
 169 +                       *lb++ = len / 0x10000 + 0x80;
 170 +                       *lb++ = len / 0x100;
 171 +#else
 172 +                       *lb++ = len / 0x100 + 0x80;
 173 +#endif
 174 +               }
 175 +               *lb = len;
 176 +               write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
 177 +               if (len)
 178 +                       write_buf(f_nameout, fnamecmpbuf, len);
 179 +       }
 180
 181 -       close(fd);
 182 +       if (read_batch)
 183 +               return;
 184 +
 185 +       write_int(f_out, i);
 186 +       if (statret == 0) {
 187 +               generate_and_send_sums(fd, st.st_size, f_out);
 188 +
 189 +               close(fd);
 190 +       } else if (!dry_run)
 191 +               write_sum_head(f_out, NULL);
 192  }
 193
 194
 195 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 196 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 197 +                   int f_nameout)
 198  {
 199         int i;
 200         int phase = 0;
 201 @@ -570,7 +588,7 @@ void generate_files(int f_out, struct fi
 202                 }
 203
 204                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 205 -                              file, i, f_out);
 206 +                              file, i, f_out, f_nameout);
 207         }
 208
 209         phase++;
 210 @@ -581,13 +599,15 @@ void generate_files(int f_out, struct fi
 211                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 212
 213         write_int(f_out, -1);
 214 +       if (read_batch)
 215 +               write_int(f_nameout, flist->count);
 216
 217         /* files can cycle through the system more than once
 218          * to catch initial checksum errors */
 219         while ((i = get_redo_num()) != -1) {
 220                 struct file_struct *file = flist->files[i];
 221                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 222 -                              file, i, f_out);
 223 +                              file, i, f_out, f_nameout);
 224         }
 225
 226         phase++;
 227 @@ -595,6 +615,8 @@ void generate_files(int f_out, struct fi
 228                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 229
 230         write_int(f_out, -1);
 231 +       if (read_batch)
 232 +               write_int(f_nameout, flist->count);
 233
 234         if (preserve_hard_links)
 235                 do_hard_links();
 236 @@ -606,7 +628,7 @@ void generate_files(int f_out, struct fi
 237                 if (!file->basename || !S_ISDIR(file->mode))
 238                         continue;
 239                 recv_generator(local_name ? local_name : f_name(file),
 240 -                              file, i, -1);
 241 +                              file, i, -1, -1);
 242         }
 243
 244         if (verbose > 2)
 245 --- orig/main.c 2004-07-19 17:14:44
 246 +++ main.c      2004-07-17 15:58:11
 247 @@ -57,6 +57,7 @@ extern int filesfrom_fd;
 248  extern pid_t cleanup_child_pid;
 249  extern char *files_from;
 250  extern char *remote_filesfrom_file;
 251 +extern char *compare_dest;
 252  extern char *rsync_path;
 253  extern char *shell_cmd;
 254  extern char *batch_name;
 255 @@ -444,20 +445,21 @@ static int do_recv(int f_in,int f_out,st
 256  {
 257         int pid;
 258         int status = 0;
 259 -       int error_pipe[2];
 260 +       int error_pipe[2], name_pipe[2];
 261 +       int need_name_pipe = compare_dest || read_batch;
 262
 263         if (preserve_hard_links)
 264                 init_hard_links(flist);
 265
 266         if (!delete_after) {
 267                 /* I moved this here from recv_files() to prevent a race condition */
 268 -               if (recurse && delete_mode && !local_name && flist->count>0) {
 269 +               if (recurse && delete_mode && !local_name && flist->count > 0)
 270                         delete_files(flist);
 271 -               }
 272         }
 273
 274 -       if (fd_pair(error_pipe) < 0) {
 275 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 276 +       if (fd_pair(error_pipe) < 0
 277 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 278 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 279                 exit_cleanup(RERR_SOCKETIO);
 280         }
 281
 282 @@ -465,6 +467,11 @@ static int do_recv(int f_in,int f_out,st
 283
 284         if ((pid = do_fork()) == 0) {
 285                 close(error_pipe[0]);
 286 +               if (need_name_pipe) {
 287 +                       close(name_pipe[1]);
 288 +                       set_blocking(name_pipe[0]);
 289 +               } else
 290 +                       name_pipe[0] = -1;
 291                 if (f_in != f_out)
 292                         close(f_out);
 293
 294 @@ -474,7 +481,7 @@ static int do_recv(int f_in,int f_out,st
 295                 /* set place to send errors */
 296                 set_msg_fd_out(error_pipe[1]);
 297
 298 -               recv_files(f_in,flist,local_name);
 299 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 300                 io_flush(FULL_FLUSH);
 301                 report(f_in);
 302
 303 @@ -492,6 +499,11 @@ static int do_recv(int f_in,int f_out,st
 304                 stop_write_batch();
 305
 306         close(error_pipe[1]);
 307 +       if (need_name_pipe) {
 308 +               close(name_pipe[0]);
 309 +               set_nonblocking(name_pipe[1]);
 310 +       } else
 311 +               name_pipe[1] = -1;
 312         if (f_in != f_out)
 313                 close(f_in);
 314
 315 @@ -499,7 +511,7 @@ static int do_recv(int f_in,int f_out,st
 316
 317         set_msg_fd_in(error_pipe[0]);
 318
 319 -       generate_files(f_out, flist, local_name);
 320 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 321
 322         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 323         report(-1);
 324 --- orig/receiver.c     2004-07-20 21:36:07
 325 +++ receiver.c  2004-07-19 16:44:39
 326 @@ -28,6 +28,7 @@ extern int max_delete;
 327  extern int csum_length;
 328  extern struct stats stats;
 329  extern int dry_run;
 330 +extern int read_batch;
 331  extern int am_server;
 332  extern int relative_paths;
 333  extern int keep_dirlinks;
 334 @@ -318,6 +319,30 @@ static int receive_data(int f_in, char *
 335  }
 336
 337
 338 +static char *read_gen_name(int fd, char *buf, char *realname)
 339 +{
 340 +       int len = read_byte(fd);
 341 +       if (len & 0x80) {
 342 +#if MAXPATHLEN > 32767
 343 +               uchar lenbuf[2];
 344 +               read_buf(fd, (char *)lenbuf, 2);
 345 +               len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
 346 +#else
 347 +               len = (len & ~0x80) * 0x100 + read_byte(fd);
 348 +#endif
 349 +       }
 350 +       if (len) {
 351 +               if (len >= MAXPATHLEN) {
 352 +                       rprintf(FERROR, "bogus data on generator name pipe\n");
 353 +                       exit_cleanup(RERR_PROTOCOL);
 354 +               }
 355 +               read_sbuf(fd, buf, len);
 356 +               return buf;
 357 +       }
 358 +       return realname;
 359 +}
 360 +
 361 +
 362  static void discard_receive_data(int f_in, OFF_T length)
 363  {
 364         receive_data(f_in, NULL, -1, 0, NULL, -1, length);
 365 @@ -328,8 +353,10 @@ static void discard_receive_data(int f_i
 366   * main routine for receiver process.
 367   *
 368   * Receiver process runs on the same host as the generator process. */
 369 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 370 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 371 +              int f_name_in)
 372  {
 373 +       int next_gen_i = -1;
 374         int fd1,fd2;
 375         STRUCT_STAT st;
 376         char *fname, fbuf[MAXPATHLEN];
 377 @@ -355,6 +382,17 @@ int recv_files(int f_in, struct file_lis
 378
 379                 i = read_int(f_in);
 380                 if (i == -1) {
 381 +                       if (read_batch) {
 382 +                               if (next_gen_i < 0)
 383 +                                       next_gen_i = read_int(f_name_in);
 384 +                               while (next_gen_i < flist->count) {
 385 +                                       read_gen_name(f_name_in, fnamecmpbuf,
 386 +                                                     NULL);
 387 +                                       next_gen_i = read_int(f_name_in);
 388 +                               }
 389 +                               next_gen_i = -1;
 390 +                       }
 391 +
 392                         if (phase)
 393                                 break;
 394
 395 @@ -397,7 +435,26 @@ int recv_files(int f_in, struct file_lis
 396                 if (verbose > 2)
 397                         rprintf(FINFO,"recv_files(%s)\n",fname);
 398
 399 -               fnamecmp = fname;
 400 +               if (read_batch) {
 401 +                       if (next_gen_i < 0)
 402 +                               next_gen_i = read_int(f_name_in);
 403 +                       while (i > next_gen_i) {
 404 +                               read_gen_name(f_name_in, fnamecmpbuf, NULL);
 405 +                               next_gen_i = read_int(f_name_in);
 406 +                       }
 407 +                       if (i < next_gen_i) {
 408 +                               rprintf(FINFO, "skipping update for \"%s\"\n",
 409 +                                       fname);
 410 +                               discard_receive_data(f_in, file->length);
 411 +                               continue;
 412 +                       }
 413 +                       next_gen_i = -1;
 414 +               }
 415 +
 416 +               if (f_name_in >= 0)
 417 +                       fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname);
 418 +               else
 419 +                       fnamecmp = fname;
 420
 421                 if (server_exclude_list.head
 422                     && check_exclude(&server_exclude_list, fname,
 423 @@ -414,14 +471,6 @@ int recv_files(int f_in, struct file_lis
 424                 /* open the file */
 425                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 426
 427 -               if (fd1 == -1 && compare_dest != NULL) {
 428 -                       /* try the file at compare_dest instead */
 429 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 430 -                                compare_dest, fname);
 431 -                       fnamecmp = fnamecmpbuf;
 432 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 433 -               }
 434 -
 435                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 436                         rsyserr(FERROR, errno, "fstat %s failed",
 437                                 full_fname(fnamecmp));