Checksum negotiation & more bits for compat_flags

- Add checksum negotiation to the protocol so that we can easily add new
  checksum algorithms and each will be used when both sides support it.
- Increase the size of the compat_flags value in the protocol from a
  byte to an int.

diff --git a/checksum.c b/checksum.c
index 8698543dc6426422dccaaf2abffef9572163eadf..980d262bffeb80f3e39ac7b3abc5f5c9d575ee49 100644 (file)
--- a/checksum.c
+++ b/checksum.c
@@ -21,6 +21,10 @@
 
 #include "rsync.h"
 
+extern int am_server;
+extern int local_server;
+extern int whole_file;
+extern int read_batch;
 extern int checksum_seed;
 extern int protocol_version;
 extern int proper_seed_order;
@@ -33,27 +37,21 @@ extern char *checksum_choice;
 #define CSUM_MD4 4
 #define CSUM_MD5 5
 
+const char *default_checksum_list =
+       "md5 md4";
+
+#define MAX_CHECKSUM_LIST 1024
+
 int xfersum_type = 0; /* used for the file transfer checksums */
 int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */
+const char *negotiated_csum_name = NULL;
 
-/* Returns 1 if --whole-file must be enabled. */
-int parse_checksum_choice(void)
-{
-       char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL;
-       if (cp) {
-               xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice);
-               checksum_type = parse_csum_name(cp+1, -1);
-       } else
-               xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1);
-       return xfersum_type == CSUM_NONE;
-}
-
-int parse_csum_name(const char *name, int len)
+static int parse_csum_name(const char *name, int len, int allow_auto)
 {
        if (len < 0 && name)
                len = strlen(name);
 
-       if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) {
+       if (!name || (allow_auto && len == 4 && strncasecmp(name, "auto", 4) == 0)) {
                if (protocol_version >= 30)
                        return CSUM_MD5;
                if (protocol_version >= 27)
@@ -69,7 +67,65 @@ int parse_csum_name(const char *name, int len)
        if (len == 4 && strncasecmp(name, "none", 4) == 0)
                return CSUM_NONE;
 
-       rprintf(FERROR, "unknown checksum name: %s\n", name);
+       if (allow_auto) {
+               rprintf(FERROR, "unknown checksum name: %s\n", name);
+               exit_cleanup(RERR_UNSUPPORTED);
+       }
+
+       return -1;
+}
+
+void parse_checksum_choice(void)
+{
+       if (!negotiated_csum_name) {
+               char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL;
+               if (cp) {
+                       xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice, 1);
+                       checksum_type = parse_csum_name(cp+1, -1, 1);
+               } else
+                       xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1, 1);
+       }
+       if (xfersum_type == CSUM_NONE)
+               whole_file = 1;
+}
+
+void negotiate_checksum(int f_in, int f_out, const char *csum_list)
+{
+       char *tok, sumbuf[MAX_CHECKSUM_LIST];
+       int sum_type, len;
+
+       if (!am_server || local_server) {
+               if (!csum_list || !*csum_list)
+                       csum_list = default_checksum_list;
+               len = strlen(csum_list);
+               if (len >= (int)sizeof sumbuf) {
+                       rprintf(FERROR, "The checksum list is too long.\n");
+                       exit_cleanup(RERR_UNSUPPORTED);
+               }
+               if (!local_server)
+                       write_vstring(f_out, csum_list, len);
+       }
+
+       if (local_server && !read_batch)
+               memcpy(sumbuf, csum_list, len+1);
+       else
+               len = read_vstring(f_in, sumbuf, sizeof sumbuf);
+
+       if (len > 0) {
+               for (tok = strtok(sumbuf, " \t"); tok; tok = strtok(NULL, " \t")) {
+                       len = strlen(tok);
+                       sum_type = parse_csum_name(tok, len, 0);
+                       if (sum_type >= CSUM_NONE) {
+                               xfersum_type = checksum_type = sum_type;
+                               if (am_server && !local_server)
+                                       write_vstring(f_out, tok, len);
+                               negotiated_csum_name = strdup(tok);
+                               return;
+                       }
+               }
+       }
+
+       rprintf(FERROR, "Failed to negotiate a common checksum\n");
        exit_cleanup(RERR_UNSUPPORTED);
 }
 
@@ -260,7 +316,7 @@ void sum_init(int csum_type, int seed)
        char s[4];
 
        if (csum_type < 0)
-               csum_type = parse_csum_name(NULL, 0);
+               csum_type = parse_csum_name(NULL, 0, 1);
        cursum_type = csum_type;
 
        switch (csum_type) {

diff --git a/compat.c b/compat.c
index bd313fa9fe715d1960c20d116b32848f7fc352ba..b29b9637ffdb131ba955c69fac09b6d12f67f2cc 100644 (file)
--- a/compat.c
+++ b/compat.c
@@ -41,6 +41,7 @@ extern int preallocate_files;
 extern int append_mode;
 extern int fuzzy_basis;
 extern int read_batch;
+extern int write_batch;
 extern int delay_updates;
 extern int checksum_seed;
 extern int basis_dir_cnt;
@@ -60,12 +61,14 @@ extern char *partial_dir;
 extern char *dest_option;
 extern char *files_from;
 extern char *filesfrom_host;
+extern char *checksum_choice;
 extern filter_rule_list filter_list;
 extern int need_unsorted_flist;
 #ifdef ICONV_OPTION
 extern iconv_t ic_send, ic_recv;
 extern char *iconv_opt;
 #endif
+extern const char *negotiated_csum_name;
 
 /* These index values are for the file-list's extra-attribute array. */
 int pathname_ndx, depth_ndx, atimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
@@ -141,6 +144,8 @@ void set_allow_inc_recurse(void)
 
 void setup_protocol(int f_out,int f_in)
 {
+       int csum_exchange = 0;
+
        assert(file_extra_cnt == 0);
        assert(EXTRA64_CNT == 2 || EXTRA64_CNT == 1);
 
@@ -289,11 +294,23 @@ void setup_protocol(int f_out,int f_in)
                                compat_flags |= CF_CHKSUM_SEED_FIX;
                        if (local_server || strchr(client_info, 'I') != NULL)
                                compat_flags |= CF_INPLACE_PARTIAL_DIR;
-                       if (local_server || strchr(client_info, 'V') != NULL)
-                               compat_flags |= CF_VARINT_FLIST_FLAGS;
-                       write_byte(f_out, compat_flags);
-               } else
-                       compat_flags = read_byte(f_in);
+                       if (local_server || strchr(client_info, 'v') != NULL) {
+                               if (!write_batch || protocol_version >= 30) {
+                                       csum_exchange = 1;
+                                       compat_flags |= CF_VARINT_FLIST_FLAGS;
+                               }
+                       }
+                       if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */
+                               if (!write_batch)
+                                       compat_flags |= CF_VARINT_FLIST_FLAGS;
+                               write_byte(f_out, compat_flags);
+                       } else
+                               write_varint(f_out, compat_flags);
+               } else { /* read_varint() is compatible with the older write_byte() when the 0x80 bit isn't on. */
+                       compat_flags = read_varint(f_in);
+                       if  (compat_flags & CF_VARINT_FLIST_FLAGS)
+                               csum_exchange = 1;
+               }
                /* The inc_recurse var MUST be set to 0 or 1. */
                inc_recurse = compat_flags & CF_INC_RECURSE ? 1 : 0;
                want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM);
@@ -358,5 +375,22 @@ void setup_protocol(int f_out,int f_in)
                checksum_seed = read_int(f_in);
        }
 
+       if (!checksum_choice) {
+               const char *rcl = getenv("RSYNC_CHECKSUM_LIST");
+               if (csum_exchange)
+                       negotiate_checksum(f_in, f_out, rcl);
+               else if (!am_server && rcl && *rcl && strstr(rcl, "FAIL")) {
+                       rprintf(FERROR, "Remote rsync is too old for checksum negotation\n");
+                       exit_cleanup(RERR_UNSUPPORTED);
+               }
+       }
+
        init_flist();
 }
+
+void maybe_write_checksum(int batch_fd)
+{
+       assert(negotiated_csum_name != NULL);
+       if (compat_flags & CF_VARINT_FLIST_FLAGS)
+               write_vstring(batch_fd, negotiated_csum_name, strlen(negotiated_csum_name));
+}

diff --git a/io.c b/io.c
index c6d2023c73425b6128af3d062391c27e7074604f..446a5f343c3300a8f6c87d90023d2a79bf4bd2bc 100644 (file)
--- a/io.c
+++ b/io.c
@@ -2368,8 +2368,9 @@ void start_write_batch(int fd)
         * is involved. */
        write_int(batch_fd, protocol_version);
        if (protocol_version >= 30)
-               write_byte(batch_fd, compat_flags);
+               write_varint(batch_fd, compat_flags);
        write_int(batch_fd, checksum_seed);
+       maybe_write_checksum(batch_fd);
 
        if (am_sender)
                write_batch_monitor_out = fd;

diff --git a/options.c b/options.c
index ca3b97e1cce08ebf3883d56fdde2c2df411d05cd..c1e957b8f1d24bf79e28eaad265e22d1015520c3 100644 (file)
--- a/options.c
+++ b/options.c
@@ -1928,12 +1928,11 @@ int parse_arguments(int *argc_p, const char ***argv_p)
                }
        }
 
-       if (checksum_choice && strcmp(checksum_choice, "auto") != 0 && strcmp(checksum_choice, "auto,auto") != 0) {
+       if (checksum_choice && strcasecmp(checksum_choice, "auto") != 0 && strcasecmp(checksum_choice, "auto,auto") != 0) {
                /* Call this early to verify the args and figure out if we need to force
                 * --whole-file. Note that the parse function will get called again later,
                 * just in case an "auto" choice needs to know the protocol_version. */
-               if (parse_checksum_choice())
-                       whole_file = 1;
+               parse_checksum_choice();
        } else
                checksum_choice = NULL;
 
@@ -2642,7 +2641,8 @@ void server_options(char **args, int *argc_p)
                eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */
                eFlags[x++] = 'C'; /* support checksum seed order fix */
                eFlags[x++] = 'I'; /* support inplace_partial behavior */
-               eFlags[x++] = 'V'; /* use varint for flist flags */
+               eFlags[x++] = 'v'; /* use varint for flist & compat flags; negotiate checksum */
+               /* NOTE: Avoid using 'V' -- it was the high bit of a write_byte() that became write_varint(). */
 #undef eFlags
        }
 

diff --git a/rsync.yo b/rsync.yo
index ed7749687b5c58cf241011aeca1ce22378ac6ba0..a1917cf69348d4c41f365337cb66abf5e2a94b87 100644 (file)
--- a/rsync.yo
+++ b/rsync.yo
@@ -657,8 +657,9 @@ checksum that is generated as the file is transferred, but that
 automatic after-the-transfer verification has nothing to do with this
 option's before-the-transfer "Does this file need to be updated?" check.
 
-For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
-MD5.  For older protocols, the checksum used is MD4.
+The checksum used is auto-negotiated between the client and the server, but
+can be overridden using either the bf(--checksum-choice) option or an
+environment variable that is discussed in that option's section.
 
 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
 way of saying you want recursion and want to preserve almost
@@ -1371,16 +1372,38 @@ batch-writing option is in effect.
 
 dit(bf(--checksum-choice=STR)) This option overrides the checksum algorithms.
 If one algorithm name is specified, it is used for both the transfer checksums
-and (assuming bf(--checksum) is specified) the pre-transfer checksumming. If two
+and (assuming bf(--checksum) is specified) the pre-transfer checksums. If two
 comma-separated names are supplied, the first name affects the transfer
-checksums, and the second name affects the pre-transfer checksumming.
-
-The algorithm choices are "auto", "md4", "md5", and "none".  If "none" is
-specified for the first name, the bf(--whole-file) option is forced on and no
-checksum verification is performed on the transferred data.  If "none" is
-specified for the second name, the bf(--checksum) option cannot be used. The
-"auto" option is the default, where rsync bases its algorithm choice on the
-protocol version (for backward compatibility with older rsync versions).
+checksums, and the second name affects the pre-transfer checksums (bf(-c)).
+
+The algorithm choices are "auto", "MD5", "MD4", and "none".
+
+If "none" is specified for the first (or only) name, the bf(--whole-file) option
+is forced on and no checksum verification is performed on the transferred data.
+If "none" is specified for the second (or only) name, the bf(--checksum) option
+cannot be used.
+
+The "auto" option is the default, where rsync bases its algorithm choice on a
+negotation between the client and the server as follows:
+
+If both the client and the server are at least version 3.2.0, they will
+exchange a list of checksum names and choose the first one in the list that
+they have in common.
+This typically means that they will choose MD5.
+If one side of the transfer is not new enough to support this checksum
+negotation, then a value is chosen based on the protocol version (which
+chooses between MD5 and various flavors of MD4 based on protocol age).
+
+You can also override the checksum using the RSYNC_CHECKSUM_LIST environment
+variable by setting it to a space-separated list of checksum names that you
+consider acceptable. If no common checksum is found, the client exits with an
+error. This method does not allow you to specify the transfer checksum
+separately from the pre-transfer checksum, and it ignores "auto" and all
+unknown checksum names.  If the remote rsync is not new enough to handle a
+checksum negotiation list, the list is silently ignored unless it contains the
+string "FAIL" in it.
+
+The use of the bf(--checksum-choice) option overrides this environment list.
 
 dit(bf(-x, --one-file-system)) This tells rsync to avoid crossing a
 filesystem boundary when recursing.  This does not limit the user's ability