append.diff

   1 This patch adds the --append option, which works like a "resume" mode in
   2 an ftp client, appending new data onto the end of the files it updates.
   3
   4 --- orig/generator.c    2005-03-29 01:05:12
   5 +++ generator.c 2005-03-03 02:59:17
   6 @@ -52,6 +52,7 @@ extern int delay_updates;
   7  extern int update_only;
   8  extern int opt_ignore_existing;
   9  extern int inplace;
  10 +extern int append_mode;
  11  extern int make_backups;
  12  extern int csum_length;
  13  extern int ignore_times;
  14 @@ -465,35 +466,42 @@ static void generate_and_send_sums(int f
  15         OFF_T offset = 0;
  16
  17         sum_sizes_sqroot(&sum, len);
  18 +       write_sum_head(f_out, &sum);
  19 +
  20 +       if (append_mode > 0 && f_copy < 0)
  21 +               return;
  22
  23         if (len > 0)
  24                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
  25         else
  26                 mapbuf = NULL;
  27
  28 -       write_sum_head(f_out, &sum);
  29 -
  30         for (i = 0; i < sum.count; i++) {
  31                 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
  32                 char *map = map_ptr(mapbuf, offset, n1);
  33 -               uint32 sum1 = get_checksum1(map, n1);
  34                 char sum2[SUM_LENGTH];
  35 +               uint32 sum1;
  36 +
  37 +               len -= n1;
  38 +               offset += n1;
  39
  40 -               if (f_copy >= 0)
  41 +               if (f_copy >= 0) {
  42                         full_write(f_copy, map, n1);
  43 +                       if (append_mode > 0)
  44 +                               continue;
  45 +               }
  46
  47 +               sum1 = get_checksum1(map, n1);
  48                 get_checksum2(map, n1, sum2);
  49
  50                 if (verbose > 3) {
  51                         rprintf(FINFO,
  52                                 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
  53 -                               (double)i, (double)offset, (long)n1,
  54 +                               (double)i, (double)offset - n1, (long)n1,
  55                                 (unsigned long)sum1);
  56                 }
  57                 write_int(f_out, sum1);
  58                 write_buf(f_out, sum2, sum.s2length);
  59 -               len -= n1;
  60 -               offset += n1;
  61         }
  62
  63         if (mapbuf)
  64 @@ -972,6 +980,9 @@ static void recv_generator(char *fname,
  65                 return;
  66         }
  67
  68 +       if (append_mode && st.st_size > file->length)
  69 +               return;
  70 +
  71         if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
  72                 ;
  73         else if (fnamecmp_type == FNAMECMP_FUZZY)
  74 @@ -1142,7 +1153,7 @@ void generate_files(int f_out, struct fi
  75         if (delete_before && !local_name && flist->count > 0)
  76                 do_delete_pass(flist);
  77
  78 -       if (whole_file < 0)
  79 +       if (append_mode || whole_file < 0)
  80                 whole_file = 0;
  81         if (verbose >= 2) {
  82                 rprintf(FINFO, "delta-transmission %s\n",
  83 @@ -1193,6 +1204,8 @@ void generate_files(int f_out, struct fi
  84         only_existing = max_size = opt_ignore_existing = 0;
  85         update_only = always_checksum = size_only = 0;
  86         ignore_times = 1;
  87 +       if (append_mode)  /* resend w/o append mode */
  88 +               append_mode = -1; /* ... but only longer files */
  89         make_backups = 0; /* avoid a duplicate backup for inplace processing */
  90
  91         /* We expect to just sit around now, so don't exit on a timeout.
  92 --- orig/match.c        2005-03-05 17:51:23
  93 +++ match.c     2005-02-11 20:26:31
  94 @@ -23,6 +23,7 @@ extern int verbose;
  95  extern int am_server;
  96  extern int do_progress;
  97  extern int checksum_seed;
  98 +extern int append_mode;
  99
 100  int updating_basis_file;
 101
 102 @@ -330,6 +331,21 @@ void match_sums(int f, struct sum_struct
 103
 104         sum_init(checksum_seed);
 105
 106 +       if (append_mode) {
 107 +               OFF_T j = 0;
 108 +               for (j = CHUNK_SIZE; j < s->flength; j += CHUNK_SIZE) {
 109 +                       sum_update(map_ptr(buf, last_match, CHUNK_SIZE),
 110 +                                  CHUNK_SIZE);
 111 +                       last_match = j;
 112 +               }
 113 +               if (last_match < s->flength) {
 114 +                       int32 len = s->flength - last_match;
 115 +                       sum_update(map_ptr(buf, last_match, len), len);
 116 +                       last_match = s->flength;
 117 +               }
 118 +               s->count = 0;
 119 +       }
 120 +
 121         if (len > 0 && s->count > 0) {
 122                 build_hash_table(s);
 123
 124 @@ -343,7 +359,7 @@ void match_sums(int f, struct sum_struct
 125         } else {
 126                 OFF_T j;
 127                 /* by doing this in pieces we avoid too many seeks */
 128 -               for (j = CHUNK_SIZE; j < len; j += CHUNK_SIZE)
 129 +               for (j = last_match + CHUNK_SIZE; j < len; j += CHUNK_SIZE)
 130                         matched(f, s, buf, j, -2);
 131                 matched(f, s, buf, len, -1);
 132         }
 133 --- orig/options.c      2005-03-28 20:56:55
 134 +++ options.c   2005-03-11 11:21:34
 135 @@ -39,6 +39,7 @@ int make_backups = 0;
 136   **/
 137  int whole_file = -1;
 138
 139 +int append_mode = 0;
 140  int archive_mode = 0;
 141  int keep_dirlinks = 0;
 142  int copy_links = 0;
 143 @@ -165,6 +166,7 @@ static int F_option_cnt = 0;
 144  static int modify_window_set;
 145  static int refused_delete, refused_archive_part;
 146  static int refused_partial, refused_progress, refused_delete_before;
 147 +static int refused_inplace;
 148  static char *max_size_arg;
 149  static char partialdir_for_delayupdate[] = ".~tmp~";
 150
 151 @@ -273,6 +275,7 @@ void usage(enum logcode F)
 152    rprintf(F,"     --suffix=SUFFIX         set backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX);
 153    rprintf(F," -u, --update                skip files that are newer on the receiver\n");
 154    rprintf(F,"     --inplace               update destination files in-place (SEE MAN PAGE)\n");
 155 +  rprintf(F,"     --append                append data onto shorter files\n");
 156    rprintf(F," -d, --dirs                  transfer directories without recursing\n");
 157    rprintf(F," -l, --links                 copy symlinks as symlinks\n");
 158    rprintf(F," -L, --copy-links            transform symlink into referent file/dir\n");
 159 @@ -401,6 +404,7 @@ static struct poptOption long_options[]
 160    {"links",           'l', POPT_ARG_NONE,   &preserve_links, 0, 0, 0 },
 161    {"copy-links",      'L', POPT_ARG_NONE,   &copy_links, 0, 0, 0 },
 162    {"keep-dirlinks",   'K', POPT_ARG_NONE,   &keep_dirlinks, 0, 0, 0 },
 163 +  {"append",           0,  POPT_ARG_VAL,    &append_mode, 1, 0, 0 },
 164    {"whole-file",      'W', POPT_ARG_VAL,    &whole_file, 1, 0, 0 },
 165    {"no-whole-file",    0,  POPT_ARG_VAL,    &whole_file, 0, 0, 0 },
 166    {"copy-unsafe-links",0,  POPT_ARG_NONE,   &copy_unsafe_links, 0, 0, 0 },
 167 @@ -575,6 +579,8 @@ static void set_refuse_options(char *bp)
 168                                                 refused_partial = op->val;
 169                                         else if (wildmatch("progress", op->longName))
 170                                                 refused_progress = op->val;
 171 +                                       else if (wildmatch("inplace", op->longName))
 172 +                                               refused_inplace = op->val;
 173                                         break;
 174                                 }
 175                                 if (!is_wild)
 176 @@ -1109,6 +1115,14 @@ int parse_arguments(int *argc, const cha
 177                         bwlimit_writemax = 512;
 178         }
 179
 180 +       if (append_mode) {
 181 +               if (refused_inplace) {
 182 +                       create_refuse_error(refused_inplace);
 183 +                       return 0;
 184 +               }
 185 +               inplace = 1;
 186 +       }
 187 +
 188         if (delay_updates && !partial_dir)
 189                 partial_dir = partialdir_for_delayupdate;
 190
 191 @@ -1419,7 +1433,9 @@ void server_options(char **args,int *arg
 192         if (opt_ignore_existing && am_sender)
 193                 args[ac++] = "--ignore-existing";
 194
 195 -       if (inplace)
 196 +       if (append_mode)
 197 +               args[ac++] = "--append";
 198 +       else if (inplace)
 199                 args[ac++] = "--inplace";
 200
 201         if (tmpdir) {
 202 --- orig/receiver.c     2005-03-24 16:41:46
 203 +++ receiver.c  2005-02-11 20:26:32
 204 @@ -44,6 +44,7 @@ extern int remove_sent_files;
 205  extern int module_id;
 206  extern int ignore_errors;
 207  extern int orig_umask;
 208 +extern int append_mode;
 209  extern int keep_partial;
 210  extern int checksum_seed;
 211  extern int inplace;
 212 @@ -210,6 +211,28 @@ static int receive_data(int f_in, char *
 213
 214         sum_init(checksum_seed);
 215
 216 +       if (append_mode) {
 217 +               OFF_T j;
 218 +               sum.flength = (OFF_T)sum.count * sum.blength;
 219 +               if (sum.remainder)
 220 +                       sum.flength -= sum.blength - sum.remainder;
 221 +               for (j = CHUNK_SIZE; j < sum.flength; j += CHUNK_SIZE) {
 222 +                       sum_update(map_ptr(mapbuf, offset, CHUNK_SIZE),
 223 +                                  CHUNK_SIZE);
 224 +                       offset = j;
 225 +               }
 226 +               if (offset < sum.flength) {
 227 +                       int32 len = sum.flength - offset;
 228 +                       sum_update(map_ptr(mapbuf, offset, len), len);
 229 +                       offset = sum.flength;
 230 +               }
 231 +               if (fd != -1 && do_lseek(fd, offset, SEEK_SET) != offset) {
 232 +                       rsyserr(FERROR, errno, "lseek failed on %s",
 233 +                               full_fname(fname));
 234 +                       exit_cleanup(RERR_FILEIO);
 235 +               }
 236 +       }
 237 +
 238         while ((i = recv_token(f_in, &data)) != 0) {
 239                 if (do_progress)
 240                         show_progress(offset, total_size);
 241 @@ -399,6 +422,7 @@ int recv_files(int f_in, struct file_lis
 242                         send_msg(MSG_DONE, "", 0);
 243                         if (keep_partial && !partial_dir)
 244                                 make_backups = 0; /* prevents double backup */
 245 +                       append_mode = 0;
 246                         continue;
 247                 }
 248
 249 --- orig/rsync.yo       2005-03-28 20:56:55
 250 +++ rsync.yo    2005-02-11 20:36:38
 251 @@ -308,6 +308,7 @@ to the detailed description below for a
 252       --suffix=SUFFIX         backup suffix (default ~ w/o --backup-dir)
 253   -u, --update                skip files that are newer on the receiver
 254       --inplace               update destination files in-place
 255 +     --append                append data onto shorter files
 256   -d, --dirs                  transfer directories without recursing
 257   -l, --links                 copy symlinks as symlinks
 258   -L, --copy-links            transform symlink into referent file/dir
 259 @@ -565,6 +566,14 @@ should not use this option to update fil
 260  rsync will be unable to update a file in-place that is not writable by the
 261  receiving user.
 262
 263 +dit(bf(--append)) This causes rsync to update a file by appending data onto
 264 +the end of the file, which presumes that the data that already exists on
 265 +the receiving side is identical with the start of the file on the sending
 266 +side.  If that is not true, the file will fail the checksum check, and a
 267 +normal bf(--inplace) update will correct the mismatch.  Any file on the
 268 +receiving side that is longer than a file on the sending side is skipped.
 269 +Implies bf(--inplace).
 270 +
 271  dit(bf(-d, --dirs)) Tell the sending side to include any directories that
 272  are encountered.  Unlike bf(--recursive), a directory's contents are not copied
 273  unless the directory was specified on the command-line as either "." or a
 274 --- orig/sender.c       2005-03-28 20:56:55
 275 +++ sender.c    2005-03-16 02:25:18
 276 @@ -28,6 +28,7 @@ extern int log_format_has_i;
 277  extern int daemon_log_format_has_i;
 278  extern int csum_length;
 279  extern int io_error;
 280 +extern int append_mode;
 281  extern int protocol_version;
 282  extern int remove_sent_files;
 283  extern int updating_basis_file;
 284 @@ -68,6 +69,13 @@ static struct sum_struct *receive_sums(i
 285                         (double)s->count, (long)s->blength, (long)s->remainder);
 286         }
 287
 288 +       if (append_mode) {
 289 +               s->flength = (OFF_T)s->count * s->blength;
 290 +               if (s->remainder)
 291 +                       s->flength -= s->blength - s->remainder;
 292 +               return s;
 293 +       }
 294 +
 295         if (s->count == 0)
 296                 return(s);
 297
 298 @@ -227,6 +235,7 @@ void send_files(struct file_list *flist,
 299                         /* For inplace: redo phase turns off the backup
 300                          * flag so that we do a regular inplace send. */
 301                         make_backups = 0;
 302 +                       append_mode = 0;
 303                         continue;
 304                 }
 305