Merge across rsync+ patch; add a little documentation to the manpage. More documenta...
authorMartin Pool <mbp@samba.org>
Tue, 14 Aug 2001 02:04:47 +0000 (02:04 +0000)
committerMartin Pool <mbp@samba.org>
Tue, 14 Aug 2001 02:04:47 +0000 (02:04 +0000)
13 files changed:
Makefile.in
NEWS
batch.c [new file with mode: 0644]
compat.c
flist.c
main.c
match.c
options.c
rsync.c
rsync.yo
sender.c
token.c
util.c

index 3bf7d4921b89dc3007009e7316fc36f8bfa02d65..56e1b0427bfca6c012228aa4af97e162f935c3ff 100644 (file)
@@ -26,7 +26,7 @@ ZLIBOBJ=zlib/deflate.o zlib/infblock.o zlib/infcodes.o zlib/inffast.o \
        zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \
        zlib/zutil.o zlib/adler32.o 
 OBJS1=rsync.o generator.o receiver.o cleanup.o sender.o exclude.o util.o main.o checksum.o match.o syscall.o log.o backup.o
-OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o fileio.o
+OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o fileio.o batch.o
 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
 popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
        popt/popthelp.o popt/poptparse.o
diff --git a/NEWS b/NEWS
index a5ce2542bbc826a40266af1c705bbff7ffd283a4..c9a4c71dd4f7285a59efc06231c8d7d38a99436c 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -10,6 +10,10 @@ rsync 2.4.7 (sometime in 2001, maybe :)
 
     * Shell wildcards are allowed in "auth users" lines.
 
+    * Merged UNC rsync+ patch to support creation of standalone patch
+      sets.  By Bert J. Dempsey and Debra Weiss, updated by Jos
+      Backus.  <http://www.ils.unc.edu/i2dsi/unc_rsync+.html>
+
   ENHANCEMENTS
 
     * Include/exclude cluestick: with -vv, print out whether files are
diff --git a/batch.c b/batch.c
new file mode 100644 (file)
index 0000000..200ff8b
--- /dev/null
+++ b/batch.c
@@ -0,0 +1,572 @@
+/* 
+   Weiss 1/1999
+   Batch utilities 
+
+*/ 
+
+#include "rsync.h"
+#include <time.h>
+
+char rsync_flist_file[27] = "rsync_flist.";    
+char rsync_csums_file[27] = "rsync_csums.";     
+char rsync_delta_file[27] = "rsync_delta.";    
+char rsync_argvs_file[27] = "rsync_argvs.";    
+
+char batch_file_ext[15];
+int fdb;
+int fdb_delta;
+int fdb_open;   
+int fdb_close;  
+
+struct file_list *batch_flist;
+
+void create_batch_file_ext()
+{
+    struct tm *timeptr;
+    time_t elapsed_seconds;
+    
+    /* Save run date and time to use for batch file extensions */
+    time(&elapsed_seconds);
+    timeptr = localtime(&elapsed_seconds);
+
+    sprintf(batch_file_ext, "%4d%02d%02d%02d%02d%02d",
+             timeptr->tm_year+1900, timeptr->tm_mon+1, timeptr->tm_mday,
+             timeptr->tm_hour, timeptr->tm_min, timeptr->tm_sec);
+}
+
+void set_batch_file_ext(char *ext)
+{
+    strcpy(batch_file_ext, ext);  
+}
+
+void write_batch_flist_file(char *buff, int bytes_to_write)
+{
+
+    if (fdb_open) {
+       /* Set up file extension */
+       strcat(rsync_flist_file, batch_file_ext); 
+
+       /* Open batch flist file for writing; create it if it doesn't exist */
+       fdb = do_open(rsync_flist_file, O_WRONLY|O_CREAT|O_TRUNC,
+                                              S_IREAD|S_IWRITE);
+       if (fdb == -1) {
+          rprintf(FERROR, "Batch file %s open error: %s\n",
+          rsync_flist_file, strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+       }
+       fdb_open = 0;
+    }
+
+    /* Write buffer to batch flist file */
+
+    if ( write(fdb, buff, bytes_to_write) == -1 ) { 
+            rprintf(FERROR, "Batch file %s write error: %s\n",
+          rsync_flist_file, strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+    }
+
+
+    if (fdb_close) {
+       close(fdb);
+    }
+}
+
+void write_batch_flist_info(int flist_count, struct file_struct **fptr)
+{
+    int i;
+    int bytes_to_write;   
+
+    /* Write flist info to batch file */
+
+    bytes_to_write = sizeof(unsigned) +
+                     sizeof(time_t) +
+                     sizeof(OFF_T) +
+                     sizeof(mode_t) +
+                     sizeof(INO_T) +
+                     (2 * sizeof(dev_t)) +
+                     sizeof(uid_t) +
+                     sizeof(gid_t);   
+
+    fdb_open = 1;
+    fdb_close = 0;
+
+    for (i=0; i<flist_count; i++) {
+       write_batch_flist_file( (char *) fptr[i], bytes_to_write);
+       write_char_bufs(fptr[i]->basename);
+       write_char_bufs(fptr[i]->dirname);
+       write_char_bufs(fptr[i]->basedir);
+       write_char_bufs(fptr[i]->link);
+       if (i==flist_count - 1) {
+          fdb_close = 1;
+       }
+       write_char_bufs(fptr[i]->sum);
+    }  
+
+}
+
+void write_char_bufs(char *buf)
+{
+   /* Write the size of the string which will follow  */ 
+
+   char b[4];
+   if (buf != NULL)
+      SIVAL(b,0,strlen(buf));  
+   else {
+      SIVAL(b,0,0);
+   }
+
+   write_batch_flist_file(b, sizeof(int));  
+
+   /*  Write the string if there is one */
+
+   if (buf != NULL) {
+      write_batch_flist_file(buf, strlen(buf)); 
+   }
+}
+
+void write_batch_argvs_file(int orig_argc, int argc, char **argv)
+{
+    int fdb;
+    int i;
+    char buff[256];
+
+    strcat(rsync_argvs_file, batch_file_ext);
+    
+
+    /* Open batch argvs file for writing; create it if it doesn't exist */
+    fdb = do_open(rsync_argvs_file, O_WRONLY|O_CREAT|O_TRUNC,
+                                           S_IREAD|S_IWRITE|S_IEXEC);
+    if (fdb == -1) {
+       rprintf(FERROR, "Batch file %s open error: %s\n", rsync_argvs_file,
+         strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+    }
+    buff[0] = '\0';
+    /* Write argvs info to batch file */
+   
+    for (i=argc - orig_argc;i<argc;i++) {
+       if ( !strcmp(argv[i],"-F") ){  /* safer to change it here than script*/ 
+          strncat(buff,"-f ",3);  /* chg to -f + ext to get ready for remote */
+          strncat(buff,batch_file_ext,strlen(batch_file_ext));
+       }
+       else {
+          strncat(buff,argv[i],strlen(argv[i]));
+       }
+    
+       if (i < (argc - 1)) {
+          strncat(buff," ",1);
+       }
+    }
+    if (!write(fdb, buff, strlen(buff))) { 
+          rprintf(FERROR, "Batch file %s write error: %s\n",
+            rsync_argvs_file, strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+    }
+    close(fdb);
+}
+
+struct file_list *create_flist_from_batch()
+{
+   unsigned char flags;
+
+   fdb_open = 1;
+   fdb_close = 0;  
+
+   batch_flist = (struct file_list *)malloc(sizeof(batch_flist[0]));
+   if (!batch_flist) {
+     out_of_memory("create_flist_from_batch"); 
+   }
+   batch_flist->count=0;
+   batch_flist->malloced=1000;
+   batch_flist->files = (struct file_struct **)malloc(sizeof(batch_flist->files[0])* batch_flist->malloced);
+   if (!batch_flist->files) {
+     out_of_memory("create_flist_from_batch");   /* dw -- will exit */
+   }
+
+   for ( flags=read_batch_flags() ; flags; flags=read_batch_flags() ) {
+
+     int i = batch_flist->count;
+     if (i >= batch_flist->malloced) {
+         if (batch_flist->malloced < 1000)
+                 batch_flist->malloced += 1000;
+         else
+                 batch_flist->malloced *= 2;
+         batch_flist->files =(struct file_struct **)realloc(batch_flist->files,
+                                              sizeof(batch_flist->files[0])*
+                                              batch_flist->malloced);
+         if (!batch_flist->files)
+                 out_of_memory("create_flist_from_batch");
+    }
+    read_batch_flist_info(&batch_flist->files[i]);
+    batch_flist->files[i]->flags = flags;   
+
+    batch_flist->count++;
+  }
+
+  return batch_flist;
+
+}
+
+int read_batch_flist_file(char *buff, int len)
+{
+    int bytes_read;
+      
+    if (fdb_open) {
+
+       /*  Set up file extension  */
+       strcat(rsync_flist_file, batch_file_ext);
+
+       /* Open batch flist file for reading */
+       fdb = do_open(rsync_flist_file, O_RDONLY, 0);
+       if (fdb == -1) {
+          rprintf(FERROR, "Batch file %s open error: %s\n", rsync_flist_file,
+            strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+       }
+       fdb_open = 0;
+    }
+
+    /* Read flist batch file */
+
+    bytes_read = read(fdb, buff, len); 
+
+    if (bytes_read == -1) { 
+       rprintf(FERROR, "Batch file %s read error: %s\n",
+          rsync_flist_file, strerror(errno));
+       close(fdb);
+       exit_cleanup(1);
+    }
+    if (bytes_read == 0) {    /* EOF */
+       close(fdb);
+    }
+    return bytes_read;
+}
+
+unsigned char read_batch_flags()
+{
+    int flags; 
+
+    if (read_batch_flist_file((char *)&flags, 4) ) { 
+       return 1;
+    }
+    else {
+       return 0; 
+    }
+}
+
+void read_batch_flist_info(struct file_struct **fptr)
+{
+    int int_str_len;   
+    char char_str_len[4];   
+    char buff[256];   
+    struct file_struct *file;
+
+    file = (struct file_struct *)malloc(sizeof(*file));
+    if (!file) out_of_memory("read_batch_flist_info");
+    memset((char *)file, 0, sizeof(*file));
+
+    (*fptr) = file;
+
+    read_batch_flist_file((char *)&file->modtime, sizeof(time_t)); 
+    read_batch_flist_file((char *)&file->length, sizeof(OFF_T)); 
+    read_batch_flist_file((char *)&file->mode, sizeof(mode_t)); 
+    read_batch_flist_file((char *)&file->inode, sizeof(INO_T)); 
+    read_batch_flist_file((char *)&file->dev, sizeof(dev_t)); 
+    read_batch_flist_file((char *)&file->rdev, sizeof(dev_t)); 
+    read_batch_flist_file((char *)&file->uid, sizeof(uid_t)); 
+    read_batch_flist_file((char *)&file->gid, sizeof(gid_t)); 
+    read_batch_flist_file(char_str_len, sizeof(char_str_len)); 
+    int_str_len = IVAL(char_str_len,0);
+    if (int_str_len > 0) {
+       read_batch_flist_file(buff, int_str_len); 
+       buff[int_str_len] = '\0';
+       file->basename = strdup(buff); 
+    }
+    else {
+       file->basename = NULL;
+    }
+
+    read_batch_flist_file(char_str_len, sizeof(char_str_len)); 
+    int_str_len = IVAL(char_str_len,0);
+    if (int_str_len > 0) {
+       read_batch_flist_file(buff, int_str_len); 
+       buff[int_str_len] = '\0';
+       file[0].dirname = strdup(buff); 
+    }
+    else {
+       file[0].dirname = NULL;
+    }
+
+    read_batch_flist_file(char_str_len, sizeof(char_str_len)); 
+    int_str_len = IVAL(char_str_len,0);
+    if (int_str_len > 0) {
+       read_batch_flist_file(buff, int_str_len); 
+       buff[int_str_len] = '\0';
+       file[0].basedir = strdup(buff); 
+    }
+    else {
+       file[0].basedir = NULL;
+    }
+
+    read_batch_flist_file(char_str_len, sizeof(char_str_len)); 
+    int_str_len = IVAL(char_str_len,0);
+    if (int_str_len > 0) {
+       read_batch_flist_file(buff, int_str_len); 
+       buff[int_str_len] = '\0';
+       file[0].link = strdup(buff); 
+    }
+    else {
+       file[0].link = NULL;
+    }
+
+    read_batch_flist_file(char_str_len, sizeof(char_str_len)); 
+    int_str_len = IVAL(char_str_len,0);
+    if (int_str_len > 0) {
+       read_batch_flist_file(buff, int_str_len); 
+       buff[int_str_len] = '\0';
+       file[0].sum = strdup(buff); 
+    }
+    else {
+       file[0].sum = NULL;
+    }
+}
+
+void write_batch_csums_file(char *buff, int bytes_to_write)
+{
+
+    static int fdb_open = 1;
+
+    if (fdb_open) {
+       /* Set up file extension */
+       strcat(rsync_csums_file, batch_file_ext); 
+
+       /* Open batch csums file for writing; create it if it doesn't exist */
+       fdb = do_open(rsync_csums_file, O_WRONLY|O_CREAT|O_TRUNC,
+                                              S_IREAD|S_IWRITE);
+       if (fdb == -1) {
+          rprintf(FERROR, "Batch file %s open error: %s\n",
+          rsync_csums_file, strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+       }
+       fdb_open = 0; 
+    }
+
+    /* Write buffer to batch csums file */
+
+    if ( write(fdb, buff, bytes_to_write) == -1 ) { 
+            rprintf(FERROR, "Batch file %s write error: %s\n",
+          rsync_csums_file, strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+    }
+}
+
+void close_batch_csums_file() 
+{
+    close(fdb);
+
+}
+
+void write_batch_csum_info(int *flist_entry, int flist_count, struct sum_struct *s)
+{
+    int i;
+    int int_zero = 0;
+    extern int block_size;
+    extern int csum_length;
+
+    fdb_open = 1;
+
+    /* Write csum info to batch file */
+
+    write_batch_csums_file ( (char *) flist_entry, sizeof(int) );
+    write_batch_csums_file ( (char *) (s?&s->count:&int_zero), sizeof(int) );
+    if (s) {
+       for (i=0; i < s->count; i++) {
+          write_batch_csums_file( (char *) &s->sums[i].sum1, sizeof(uint32));
+          if ( (*flist_entry == flist_count - 1) && (i == s->count - 1) ) {
+             fdb_close = 1;
+          }
+          write_batch_csums_file( s->sums[i].sum2, csum_length);
+       } 
+    }  
+}
+
+int read_batch_csums_file(char *buff, int len)
+{
+    static int fdb_open = 1; 
+    int bytes_read;
+      
+    if (fdb_open) {
+
+       /*  Set up file extension  */
+       strcat(rsync_csums_file, batch_file_ext);
+
+       /* Open batch flist file for reading */
+       fdb = do_open(rsync_csums_file, O_RDONLY, 0);
+       if (fdb == -1) {
+          rprintf(FERROR, "Batch file %s open error: %s\n", rsync_csums_file,
+            strerror(errno));
+          close(fdb);
+          exit_cleanup(1);
+       }
+       fdb_open = 0;
+    }
+
+    /* Read csums batch file */
+
+    bytes_read = read(fdb, buff, len); 
+
+    if (bytes_read == -1) { 
+       rprintf(FERROR, "Batch file %s read error: %s\n",
+          rsync_csums_file, strerror(errno));
+       close(fdb);
+       exit_cleanup(1);
+    }
+    return bytes_read;
+}
+
+
+void read_batch_csum_info(int flist_entry, struct sum_struct *s, int *checksums_match)
+{
+    int i;
+    int file_flist_entry;
+    int file_chunk_ct;
+    uint32 file_sum1;
+    char file_sum2[SUM_LENGTH]; 
+    extern int csum_length;
+
+    read_batch_csums_file((char *)&file_flist_entry, sizeof(int)); 
+    if (file_flist_entry != flist_entry) {
+       rprintf(FINFO,"file_list_entry NE flist_entry\n");
+       rprintf(FINFO,"file_flist_entry = %d  flist_entry = %d\n", file_flist_entry, flist_entry);
+       close(fdb);
+       exit_cleanup(1);
+    
+    }
+    else {
+       read_batch_csums_file((char *)&file_chunk_ct, sizeof(int)); 
+       *checksums_match = 1;
+       for (i = 0;i < file_chunk_ct;i++) {
+          
+          read_batch_csums_file((char *)&file_sum1, sizeof(uint32)); 
+          read_batch_csums_file(file_sum2, csum_length); 
+
+          if ( (s->sums[i].sum1 != file_sum1) ||  
+               ( memcmp(s->sums[i].sum2,file_sum2, csum_length)!=0) ) {
+                   *checksums_match = 0;
+          }
+       }  /*  end for  */
+    }
+
+}
+
+void write_batch_delta_file(char *buff, int bytes_to_write)
+{
+    static int fdb_delta_open = 1;
+
+    if (fdb_delta_open) {
+       /* Set up file extension */
+       strcat(rsync_delta_file, batch_file_ext); 
+
+       /* Open batch delta file for writing; create it if it doesn't exist */
+       fdb_delta = do_open(rsync_delta_file, O_WRONLY|O_CREAT|O_TRUNC,
+                                              S_IREAD|S_IWRITE);
+       if (fdb_delta == -1) {
+          rprintf(FERROR, "Batch file %s open error: %s\n",
+          rsync_delta_file, strerror(errno));
+          close(fdb_delta);
+          exit_cleanup(1);
+       }
+       fdb_delta_open = 0; 
+    }
+
+    /* Write buffer to batch delta file */
+
+    if ( write(fdb_delta, buff, bytes_to_write) == -1 ) { 
+            rprintf(FERROR, "Batch file %s write error: %s\n",
+          rsync_delta_file, strerror(errno));
+          close(fdb_delta);
+          exit_cleanup(1);
+    }
+}
+void close_batch_delta_file()
+{
+   close(fdb_delta);
+
+}
+
+int read_batch_delta_file(char *buff, int len)
+{
+    static int fdb_delta_open = 1;
+    int bytes_read;
+      
+    if (fdb_delta_open) {
+
+       /*  Set up file extension  */
+       strcat(rsync_delta_file, batch_file_ext);
+
+       /* Open batch flist file for reading */
+       fdb_delta = do_open(rsync_delta_file, O_RDONLY, 0);
+       if (fdb_delta == -1) {
+          rprintf(FERROR, "Batch file %s open error: %s\n", rsync_delta_file,
+            strerror(errno));
+          close(fdb_delta);
+          exit_cleanup(1);
+       }
+       fdb_delta_open = 0;
+    }
+
+    /* Read delta batch file */
+
+    bytes_read = read(fdb_delta, buff, len); 
+
+    if (bytes_read == -1) { 
+       rprintf(FERROR, "Batch file %s read error: %s\n",
+          rsync_delta_file, strerror(errno));
+       close(fdb_delta);
+       exit_cleanup(1);
+    }
+    return bytes_read;
+}
+
+
+void show_flist(int index, struct file_struct **fptr)
+{
+   /*  for debugging    show_flist(flist->count, flist->files **/ 
+
+    int i;
+    for (i=0;i<index;i++) {
+       rprintf(FINFO,"flist->flags=%x\n",fptr[i]->flags);  
+       rprintf(FINFO,"flist->modtime=%x\n",fptr[i]->modtime);  
+       rprintf(FINFO,"flist->length=%x\n",fptr[i]->length);  
+       rprintf(FINFO,"flist->mode=%x\n",fptr[i]->mode);  
+       rprintf(FINFO,"flist->basename=%s\n",fptr[i]->basename);
+       if (fptr[i]->dirname) 
+       rprintf(FINFO,"flist->dirname=%s\n",fptr[i]->dirname);
+       if (fptr[i]->basedir) 
+       rprintf(FINFO,"flist->basedir=%s\n",fptr[i]->basedir);
+    }
+}
+
+void show_argvs(int argc, char *argv[])
+{
+   /*  for debugging  **/ 
+
+    int i;
+    rprintf(FINFO,"BATCH.C:show_argvs,argc=%d\n",argc);
+    for (i=0;i<argc;i++) {
+   /*    if (argv[i])   */
+       rprintf(FINFO,"i=%d,argv[i]=%s\n",i,argv[i]);  
+
+    }
+}
+    
index 305c827dea14564a533f1c252fb77ed7c7bd6049..72e0f99f647353c96b1b16684ad24532ef371bbf 100644 (file)
--- a/compat.c
+++ b/compat.c
@@ -36,6 +36,9 @@ extern int checksum_seed;
 extern int remote_version;
 extern int verbose;
 
+extern int read_batch;  /* dw */
+extern int write_batch;  /* dw */
+
 void setup_protocol(int f_out,int f_in)
 {
        if (remote_version == 0) {
@@ -57,6 +60,9 @@ void setup_protocol(int f_out,int f_in)
        
        if (remote_version >= 12) {
                if (am_server) {
+                   if (read_batch || write_batch) /* dw */
+                       checksum_seed = 32761;
+                   else
                        checksum_seed = time(NULL);
                        write_int(f_out,checksum_seed);
                } else {
diff --git a/flist.c b/flist.c
index bb6149e72fd89ddf7c9822c65c1693b8dc7f58a2..edd6024fb353646070e98eabc143cbd18d52f7ab 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -47,6 +47,9 @@ extern int remote_version;
 extern int io_error;
 extern int sanitize_paths;
 
+extern int read_batch;
+extern int write_batch;
+
 static char topsrcname[MAXPATHLEN];
 
 static struct exclude_struct **local_exclude_list;
@@ -613,6 +616,9 @@ void send_file_name(int f,struct file_list *flist,char *fname,
                  out_of_memory("send_file_name");
   }
 
+  if (write_batch) /*  dw  */
+    file->flags = FLAG_DELETE;
+
   if (strcmp(file->basename,"")) {
     flist->files[flist->count++] = file;
     send_file_entry(file,f,base_flags);
@@ -841,6 +847,8 @@ struct file_list *send_file_list(int f,int argc,char *argv[])
                io_end_buffering(f);
                stats.flist_size = stats.total_written - start_write;
                stats.num_files = flist->count;
+               if (write_batch) /*  dw  */
+                   write_batch_flist_info(flist->count, flist->files);
        }
 
        if (verbose > 2)
@@ -918,7 +926,7 @@ struct file_list *recv_file_list(int f)
   }
 
   /* if protocol version is >= 17 then recv the io_error flag */
-  if (f != -1 && remote_version >= 17) {
+  if (f != -1 && remote_version >= 17  && !read_batch) {  /* dw-added readbatch */
          extern int module_id;
          extern int ignore_errors;
          if (lp_ignore_errors(module_id) || ignore_errors) {
diff --git a/main.c b/main.c
index bc51571fb5a510c180afb9c3d1dc332fa285f714..ec393527e5cd12c2e5ac606a2ed9f3670b695f35 100644 (file)
--- a/main.c
+++ b/main.c
@@ -135,8 +135,9 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int
        extern int local_server;
        extern char *rsync_path;
        extern int blocking_io;
+       extern int read_batch;
 
-       if (!local_server) {
+       if (!read_batch && !local_server) { /* dw -- added read_batch */
                if (!cmd)
                        cmd = getenv(RSYNC_RSH_ENV);
                if (!cmd)
@@ -187,6 +188,8 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int
        }
 
        if (local_server) {
+               if (read_batch)
+                   create_flist_from_batch();
                ret = local_child(argc, args, f_in, f_out);
        } else {
                ret = piped_child(args,f_in,f_out);
@@ -399,6 +402,9 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[])
        extern int am_daemon;
        extern int module_id;
        extern int am_sender;
+       extern int read_batch;   /* dw */
+       extern int write_batch;  /* dw */
+       extern struct file_list *batch_flist;  /* dw */
 
        if (verbose > 2)
                rprintf(FINFO,"server_recv(%d) starting pid=%d\n",argc,(int)getpid());
@@ -424,7 +430,10 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[])
        if (delete_mode && !delete_excluded)
                recv_exclude_list(f_in);
 
-       flist = recv_file_list(f_in);
+       if (read_batch) /*  dw  */
+           flist = batch_flist;
+       else
+           flist = recv_file_list(f_in);
        if (!flist) {
                rprintf(FERROR,"server_recv: recv_file_list error\n");
                exit_cleanup(RERR_FILESELECT);
@@ -448,6 +457,7 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
        extern int cvs_exclude;
        extern int am_sender;
        extern int remote_version;
+       extern int read_batch; /* dw */
 
        setup_protocol(f_out, f_in);
 
@@ -458,9 +468,11 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
                io_start_multiplex_out(f_out);
 
        if (am_sender) {
-               recv_exclude_list(f_in);
-               if (cvs_exclude)
+               if (!read_batch) { /* dw */
+                   recv_exclude_list(f_in);
+                   if (cvs_exclude)
                        add_cvs_excludes();
+               }
                do_server_sender(f_in, f_out, argc, argv);
        } else {
                do_server_recv(f_in, f_out, argc, argv);
@@ -481,8 +493,13 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
        extern int am_sender;
        extern int remote_version;
        extern pid_t cleanup_child_pid;
+       extern int write_batch; /* dw */
+       extern int read_batch; /* dw */
+       extern struct file_list *batch_flist; /*  dw */
 
        cleanup_child_pid = pid;
+       if (read_batch)
+           flist = batch_flist;  /* dw */
 
        set_nonblocking(f_in);
        set_nonblocking(f_out);
@@ -500,7 +517,8 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
                        add_cvs_excludes();
                if (delete_mode && !delete_excluded) 
                        send_exclude_list(f_out);
-               flist = send_file_list(f_out,argc,argv);
+               if (!read_batch) /*  dw -- don't write to pipe */
+                   flist = send_file_list(f_out,argc,argv);
                if (verbose > 3) 
                        rprintf(FINFO,"file list sent\n");
 
@@ -524,7 +542,8 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
                list_only = 1;
        }
        
-       send_exclude_list(f_out);
+       if (!write_batch) /* dw */
+           send_exclude_list(f_out);
        
        flist = recv_file_list(f_in);
        if (!flist || flist->count == 0) {
@@ -584,6 +603,7 @@ static int start_client(int argc, char *argv[])
        extern int rsync_port;
        extern int whole_file;
        char *argv0 = strdup(argv[0]);
+       extern int read_batch;
 
        if (strncasecmp(URL_PREFIX, argv0, strlen(URL_PREFIX)) == 0) {
                char *host, *path;
@@ -604,7 +624,8 @@ static int start_client(int argc, char *argv[])
                return start_socket_client(host, path, argc-1, argv+1);
        }
 
-       p = find_colon(argv0);
+       if (!read_batch) { /* dw */
+           p = find_colon(argv[0]);
 
        if (p) {
                if (p[1] == ':') {
@@ -651,7 +672,12 @@ static int start_client(int argc, char *argv[])
                }
                argc--;
        }
-       
+       } else {
+           am_sender = 1;  /*  dw */
+           local_server = 1;  /* dw */
+           shell_path = argv[argc-1];  /* dw */
+       }
+
        if (shell_machine) {
                p = strchr(shell_machine,'@');
                if (p) {
@@ -714,6 +740,13 @@ int main(int argc,char *argv[])
        extern int am_daemon;
        extern int am_server;
        int ret;
+       extern int read_batch;   /*  dw */
+       extern int write_batch;  /*  dw */
+       extern char *batch_ext;   /*  dw */
+       int i;          /*   dw */
+       int orig_argc;  /* dw */
+
+       orig_argc = argc;   /* dw */
 
        signal(SIGUSR1, sigusr1_handler);
        signal(SIGUSR2, sigusr2_handler);
@@ -751,6 +784,15 @@ int main(int argc,char *argv[])
           that implement getcwd that way "pwd" can't be found after chroot. */
        push_dir(NULL,0);
 
+       if (write_batch) { /* dw */
+           create_batch_file_ext();
+           write_batch_argvs_file(orig_argc, argc, argv);
+       }
+
+       if (read_batch) { /* dw */
+           set_batch_file_ext(batch_ext);
+       }
+
        if (am_daemon) {
                return daemon_main();
        }
diff --git a/match.c b/match.c
index c0ae38c88c1f72caab0ee72f8d9b2caeeb265e29..7b0f6017eb161afb0247fdb48c2d299be0fa56e6 100644 (file)
--- a/match.c
+++ b/match.c
@@ -260,6 +260,7 @@ static void hash_search(int f,struct sum_struct *s,
 void match_sums(int f,struct sum_struct *s,struct map_struct *buf,OFF_T len)
 {
        char file_sum[MD4_SUM_LENGTH];
+       extern int write_batch;  /*  dw */
 
        last_match = 0;
        false_alarms = 0;
@@ -295,6 +296,8 @@ void match_sums(int f,struct sum_struct *s,struct map_struct *buf,OFF_T len)
                if (verbose > 2)
                        rprintf(FINFO,"sending file_sum\n");
                write_buf(f,file_sum,MD4_SUM_LENGTH);
+               if (write_batch) /* dw */
+                   write_batch_delta_file(file_sum, MD4_SUM_LENGTH);
        }
 
        if (targets) {
index 35a6dc64f95053823792b05477b311b7d1a72e5a..bfce8f681fbf9f9e9b53067176650105aaf59ac2 100644 (file)
--- a/options.c
+++ b/options.c
@@ -74,6 +74,9 @@ int modify_window=0;
 #endif
 int blocking_io=0;
 
+int read_batch=0;  /* dw */
+int write_batch=0; /* dw */
+
 char *backup_suffix = BACKUP_SUFFIX;
 char *tmpdir = NULL;
 char *compare_dest = NULL;
@@ -90,6 +93,8 @@ int quiet = 0;
 int always_checksum = 0;
 int list_only = 0;
 
+char *batch_ext = NULL;
+
 static int modify_window_set;
 
 
@@ -206,6 +211,8 @@ void usage(enum logcode F)
   rprintf(F,"     --log-format=FORMAT     log file transfers using specified format\n");  
   rprintf(F,"     --password-file=FILE    get password from FILE\n");
   rprintf(F,"     --bwlimit=KBPS          limit I/O bandwidth, KBytes per second\n");
+  rprintf(F," -f  --read-batch=FILE       read batch file\n");
+  rprintf(F," -F  --write-batch           write batch file\n");
   rprintf(F," -h, --help                  show this help screen\n");
 
   rprintf(F,"\n");
@@ -290,10 +297,11 @@ static struct poptOption long_options[] = {
   {"address",          0,  POPT_ARG_STRING, 0,               OPT_ADDRESS},
   {"backup-dir",       0,  POPT_ARG_STRING, &backup_dir},
   {"hard-links",      'H', POPT_ARG_NONE,   &preserve_hard_links},
+  {"read-batch",      'f', POPT_ARG_STRING, &batch_ext, 'f'},
+  {"write-batch",     'F', POPT_ARG_NONE,   &write_batch, 0},
   {0,0,0,0}
 };
 
-
 static char err_buf[100];
 
 
@@ -468,6 +476,11 @@ int parse_arguments(int *argc, const char ***argv, int frommain)
                        }
                        break;
 
+               case 'f':
+                       /* The filename is stored for us by popt */
+                       read_batch = 1;
+                       break;
+
                default:
                         /* FIXME: If --daemon is specified, then errors for later
                          * parameters seem to disappear. */
@@ -501,6 +514,7 @@ void server_options(char **args,int *argc)
        static char mdelete[30];
        static char mwindow[30];
        static char bw[50];
+       static char fext[20]; /* dw */
 
        int i, x;
 
@@ -555,6 +569,8 @@ void server_options(char **args,int *argc)
                argstr[x++] = 'S';
        if (do_compression)
                argstr[x++] = 'z';
+       if (write_batch)
+           argstr[x++] = 'F'; /* dw */
 
        /* this is a complete hack - blame Rusty 
 
@@ -576,6 +592,11 @@ void server_options(char **args,int *argc)
                snprintf(mdelete,sizeof(mdelete),"--max-delete=%d",max_delete);
                args[ac++] = mdelete;
        }    
+       
+       if (batch_ext != NULL) {
+               sprintf(fext,"-f%s",batch_ext);
+               args[ac++] = fext;
+       }
 
        if (io_timeout) {
                snprintf(iotime,sizeof(iotime),"--timeout=%d",io_timeout);
diff --git a/rsync.c b/rsync.c
index 3e50f38db5bcd8af9b99ff7102da3fc7adf87ead..890d6a890b4a17d63f89e36e3ae91ba045d7a652 100644 (file)
--- a/rsync.c
+++ b/rsync.c
@@ -226,6 +226,7 @@ int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st,
 
 void sig_int(void)
 {
+       rprintf(FINFO,"\nrsync.c:sig_int() called.\n");
        exit_cleanup(RERR_SIGNAL);
 }
 
index 3717b9e552003242e5927b53284f4bbf3cf0dcaf..d4209ab47bd61cc6ba37cbca8535a3122ad13487 100644 (file)
--- a/rsync.yo
+++ b/rsync.yo
@@ -276,7 +276,11 @@ verb(
      --log-format=FORMAT     log file transfers using specified format
      --password-file=FILE    get password from FILE
      --bwlimit=KBPS          limit I/O bandwidth, KBytes per second
+ -f, --read-batch=FILE       read batch file
+ -F, --write-batch           write batch file
  -h, --help                  show this help screen
+
+
 )
 
 manpageoptions()
@@ -675,6 +679,11 @@ transfer was too fast, it will wait before sending the next data block. The
 result is an average transfer rate equalling the specified limit. A value
 of zero specifies no limit.
 
+dit(bf(--read-batch)) Apply a previously generated change batch.
+
+dit(bf(--write-batch)) Generate a set of files that can be transferred
+as a batch update.
+
 enddit()
 
 manpagesection(EXCLUDE PATTERNS)
@@ -765,6 +774,29 @@ itemize(
   it would be excluded by the "*")
 )
 
+manpagesection(BATCH MODE)
+
+The following call generates 4 files that encapsulate the information
+for synchronizing the contents of bf(target_dir) with the updates found in
+bf(src_dir)
+
+quote(
+$ rsync -F [other rsync options here] \nl()
+           /somewhere/src_dir /somewhere/target_dir
+)
+
+The generated files are labeled with a common timestamp:
+
+itemize(
+it() bf(rsync_argvs.<timestamp>) command-line arguments
+it() bf(rsync_flist.<timestamp>) rsync internal file metadata
+it() bf(rsync_csums.<timestamp>) rsync checksums
+it() bf(rsync_delta.<timestamp>) data blocks for file update & change
+)
+
+See bf(http://www.ils.unc.edu/i2dsi/unc_rsync+.html) for papers and technical
+reports.
+
 manpagesection(DIAGNOSTICS)
 
 rsync occasionally produces error messages that may seem a little
index d2ab1b3417928500f9ff8e505875a4c559b3bd5f..39bb2797f85e55ad5fe3822f77b8896d28008f06 100644 (file)
--- a/sender.c
+++ b/sender.c
@@ -93,6 +93,14 @@ void send_files(struct file_list *flist,int f_out,int f_in)
        int phase = 0;
        extern struct stats stats;              
        struct stats initial_stats;
+       extern int write_batch;   /* dw */
+       int negative_one;  /* dw */
+       extern int read_batch;    /* dw */
+       int checksums_match;   /* dw */
+       int buff_len;  /* dw */
+       char buff[CHUNK_SIZE];    /* dw */
+       int j;   /* dw */
+       int done;   /* dw */
 
        if (verbose > 2)
                rprintf(FINFO,"send_files starting\n");
@@ -152,12 +160,15 @@ void send_files(struct file_list *flist,int f_out,int f_in)
                initial_stats = stats;
 
                s = receive_sums(f_in);
+               if (write_batch) /* dw */
+                   write_batch_csum_info(&i,flist->count,s);
                if (!s) {
                        io_error = 1;
                        rprintf(FERROR,"receive_sums failed\n");
                        return;
                }
          
+               if (!read_batch) {
                fd = do_open(fname, O_RDONLY, 0);
                if (fd == -1) {
                        io_error = 1;
@@ -185,28 +196,78 @@ void send_files(struct file_list *flist,int f_out,int f_in)
                if (verbose > 2)
                        rprintf(FINFO,"send_files mapped %s of size %.0f\n",
                                fname,(double)st.st_size);
+               }
 
-               write_int(f_out,i);
+               if (!read_batch) { /* dw */
+                   write_int(f_out,i);
          
-               write_int(f_out,s->count);
-               write_int(f_out,s->n);
-               write_int(f_out,s->remainder);
+                   if (write_batch)
+                       write_batch_delta_file((char *)&i,sizeof(i));
+
+                   write_int(f_out,s->count);
+                   write_int(f_out,s->n);
+                   write_int(f_out,s->remainder);
+               }
          
                if (verbose > 2)
-                       rprintf(FINFO,"calling match_sums %s\n",fname);
+                       if (!read_batch)
+                           rprintf(FINFO,"calling match_sums %s\n",fname);
          
                if (!am_server) {
                        log_transfer(file, fname+offset);
                }
 
                set_compression(fname);
-         
-               match_sums(f_out,s,buf,st.st_size);
 
-               log_send(file, &initial_stats);
-                 
-               if (buf) unmap_file(buf);
-               close(fd);
+                if (read_batch) { /* dw */
+                   /* read checksums originally computed on sender side */
+                   read_batch_csum_info(i, s, &checksums_match);
+                   if (checksums_match) {
+                       read_batch_delta_file( (char *) &j, sizeof(int) );
+                       if (j != i) {    /* if flist index entries don't match*/ 
+                          rprintf(FINFO,"index mismatch in send_files\n");
+                          rprintf(FINFO,"read index = %d flist ndx = %d\n",j,i);
+                          close_batch_delta_file();
+                          close_batch_csums_file();
+                          exit_cleanup(1);
+                       }
+                       else {
+                         write_int(f_out,j);
+                         write_int(f_out,s->count);
+                         write_int(f_out,s->n);
+                         write_int(f_out,s->remainder);
+                         done=0;
+                         while (!done) {
+                            read_batch_delta_file( (char *) &buff_len, sizeof(int) );
+                            write_int(f_out,buff_len);
+                            if (buff_len == 0) {
+                               done = 1;
+                            }
+                            else {
+                               if (buff_len > 0) {
+                                  read_batch_delta_file(buff, buff_len);
+                                  write_buf(f_out,buff,buff_len);
+                               }
+                            }
+                         }  /* end while  */
+                         read_batch_delta_file( buff, MD4_SUM_LENGTH);
+                         write_buf(f_out, buff, MD4_SUM_LENGTH);
+
+                       }  /* j=i */
+                   } else {  /* not checksum match */
+                      rprintf(FINFO,"readbatch & checksums don't match\n");
+                      rprintf(FINFO,"filename=%s is being skipped\n");
+                      continue;
+                   }
+                } else  {
+                   match_sums(f_out,s,buf,st.st_size);
+                   log_send(file, &initial_stats);
+                }
+
+               if (!read_batch) { /* dw */
+                   if (buf) unmap_file(buf);
+                   close(fd);
+               }
          
                free_sums(s);
          
@@ -220,6 +281,11 @@ void send_files(struct file_list *flist,int f_out,int f_in)
        match_report();
 
        write_int(f_out,-1);
+       if (write_batch || read_batch) { /* dw */
+           close_batch_csums_file();
+           close_batch_delta_file();
+       }
+
 }
 
 
diff --git a/token.c b/token.c
index 2967b44c1550fdad682839ecd88460bb4245d926..174f121a791a043ecaa43100b5eceec57c828852 100644 (file)
--- a/token.c
+++ b/token.c
@@ -90,18 +90,29 @@ static int simple_recv_token(int f,char **data)
 static void simple_send_token(int f,int token,
                              struct map_struct *buf,OFF_T offset,int n)
 {
+       extern int write_batch; /* dw */
+       int hold_int; /* dw */
+
        if (n > 0) {
                int l = 0;
                while (l < n) {
                        int n1 = MIN(CHUNK_SIZE,n-l);
                        write_int(f,n1);
                        write_buf(f,map_ptr(buf,offset+l,n1),n1);
+                       if (write_batch) {
+                           write_batch_delta_file( (char *) &n1, sizeof(int) );
+                           write_batch_delta_file(map_ptr(buf,offset+l,n1),n1);
+                       }
                        l += n1;
                }
        }
        /* a -2 token means to send data only and no token */
        if (token != -2) {
                write_int(f,-(token+1));
+               if (write_batch) {
+                   hold_int = -(token+1);
+                   write_batch_delta_file( (char *) &hold_int, sizeof(int) );
+               }
        }
 }
 
@@ -134,6 +145,8 @@ send_deflated_token(int f, int token,
 {
        int n, r;
        static int init_done, flush_pending;
+       extern int write_batch;  /* dw */
+       char temp_byte;   /* dw */
 
        if (last_token == -1) {
                /* initialization */
@@ -166,13 +179,27 @@ send_deflated_token(int f, int token,
                n = last_token - run_start;
                if (r >= 0 && r <= 63) {
                        write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
+                       if (write_batch) { /* dw */
+                           temp_byte = (char)( (n==0? TOKEN_REL: TOKENRUN_REL) + r);
+                           write_batch_delta_file(&temp_byte,sizeof(char));
+                       }
                } else {
                        write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
                        write_int(f, run_start);
+                       if (write_batch) { /* dw */
+                           temp_byte = (char)(n==0? TOKEN_LONG: TOKENRUN_LONG);
+                           write_batch_delta_file(&temp_byte,sizeof(temp_byte));
+                           write_batch_delta_file((char *)&run_start,sizeof(run_start));
+                       }
                }
                if (n != 0) {
                        write_byte(f, n);
                        write_byte(f, n >> 8);
+                       if (write_batch) { /* dw */
+                           write_batch_delta_file((char *)&n,sizeof(char));
+                           temp_byte = (char) n >> 8;
+                           write_batch_delta_file(&temp_byte,sizeof(temp_byte));
+                       }
                }
                last_run_end = last_token;
                run_start = token;
@@ -231,6 +258,8 @@ send_deflated_token(int f, int token,
                                        obuf[0] = DEFLATED_DATA + (n >> 8);
                                        obuf[1] = n;
                                        write_buf(f, obuf, n+2);
+                                       if (write_batch) /* dw */
+                                           write_batch_delta_file(obuf,n+2);
                                }
                        }
                } while (nb != 0 || tx_strm.avail_out == 0);
@@ -240,6 +269,10 @@ send_deflated_token(int f, int token,
        if (token == -1) {
                /* end of file - clean up */
                write_byte(f, END_FLAG);
+               if (write_batch) { /* dw */
+                   temp_byte = END_FLAG;
+                   write_batch_delta_file((char *)&temp_byte,sizeof(temp_byte));
+               }
 
        } else if (token != -2) {
                /* add the data in the current block to the compressor's
diff --git a/util.c b/util.c
index 50fdaf8ecb6c631a02e1731800429e84965ae015..ada404653e3b615135f4c8e5a5a01fd6cfaa61ed 100644 (file)
--- a/util.c
+++ b/util.c
@@ -154,6 +154,7 @@ pid_t local_child(int argc, char **argv,int *f_in,int *f_out)
        pid_t pid;
        int to_child_pipe[2];
        int from_child_pipe[2];
+       extern int read_batch;  /* dw */
 
        if (fd_pair(to_child_pipe) < 0 ||
            fd_pair(from_child_pipe) < 0) {
@@ -172,7 +173,10 @@ pid_t local_child(int argc, char **argv,int *f_in,int *f_out)
                extern int am_sender;
                extern int am_server;
 
-               am_sender = !am_sender;
+               if (read_batch)
+                   am_sender = 0;
+               else
+                   am_sender = !am_sender;
                am_server = 1;          
 
                if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||