More compress changes

- Add the zlibx (external-code compatible) compression name.
- Re-enable zlib support with the external library so it can be
  tried as a fallback if zlibx isn't available.
- Add --compress-choice=STR (aka -zz=STR) option.
- Make --cc=STR an alias for --checksum-choice=STR.
- Hook up the new compression negotiation logic.

diff --git a/compat.c b/compat.c
index 410e6f696966cdd15dbda04b5915bf561d2d4fb2..3a5ad46d3bf4b95af08fdad4e8f5221b2e310efc 100644 (file)
--- a/compat.c
+++ b/compat.c
@@ -89,10 +89,17 @@ int filesfrom_convert = 0;
 
 #define CPRES_NONE 0
 #define CPRES_ZLIB 1
+#define CPRES_ZLIBX 2
 
 struct name_num_obj valid_compressions = {
        "compress", NULL, NULL, 0, 0, {
+#ifndef EXTERNAL_ZLIB
                { CPRES_ZLIB, "zlib", NULL },
+#endif
+               { CPRES_ZLIBX, "zlibx", NULL },
+#ifdef EXTERNAL_ZLIB
+               { CPRES_ZLIB, "zlib", NULL },
+#endif
                { CPRES_NONE, "none", NULL },
                { 0, NULL, NULL }
        }
@@ -160,6 +167,37 @@ void set_allow_inc_recurse(void)
                allow_inc_recurse = 0;
 }
 
+void parse_compress_choice(int final_call)
+{
+       int num;
+
+       if (valid_compressions.negotiated_name)
+               num = valid_compressions.negotiated_num;
+       else if (compress_choice) {
+               struct name_num_item *nni = get_nni_by_name(&valid_compressions, compress_choice, -1);
+               if (!nni) {
+                       rprintf(FERROR, "unknown compress name: %s\n", compress_choice);
+                       exit_cleanup(RERR_UNSUPPORTED);
+               }
+               num = nni->num;
+       } else
+               num = CPRES_NONE;
+
+       if (num == CPRES_NONE) {
+               do_compression = 0;
+               compress_choice = NULL;
+       } else if (num > 0)
+               do_compression = num != CPRES_ZLIB ? 2 : 1;
+
+       if (final_call && DEBUG_GTE(NSTR, am_server ? 2 : 1)) {
+               const char *c_s = am_server ? "Server" : "Client";
+               if (valid_compressions.negotiated_name)
+                       rprintf(FINFO, "%s negotiated compress: %s\n", c_s, valid_compressions.negotiated_name);
+               else
+                       rprintf(FINFO, "%s compress: %s\n", c_s, do_compression ? compress_choice : "none");
+       }
+}
+
 struct name_num_item *get_nni_by_name(struct name_num_obj *nno, const char *name, int len)
 {
        struct name_num_item *nni;

diff --git a/flist.c b/flist.c
index 940071fd43604c4244949ac08faf872530a764ee..a1b3e4bb8d224e7060958efcc77f393909605688 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -143,6 +143,7 @@ void init_flist(void)
                        (int)FILE_STRUCT_LEN, (int)EXTRA_LEN);
        }
        parse_checksum_choice(1); /* Sets checksum_type && xfersum_type */
+       parse_compress_choice(1); /* Sets do_compression */
        flist_csum_len = csum_len_for_type(checksum_type, 1);
 
        show_filelist_progress = INFO_GTE(FLIST, 1) && xfer_dirs && !am_server && !inc_recurse;

diff --git a/options.c b/options.c
index 62fe0fb528dbce1d35b5749a65da902fea78836e..469b9fb6ea8806a9b899dbef3c33d88c3f368341 100644 (file)
--- a/options.c
+++ b/options.c
@@ -840,6 +840,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
       OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
       OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
       OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT,
+      OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS,
       OPT_SERVER, OPT_REFUSED_BASE = 9000};
 
 static struct poptOption long_options[] = {
@@ -981,11 +982,12 @@ static struct poptOption long_options[] = {
   {"cvs-exclude",     'C', POPT_ARG_NONE,   &cvs_exclude, 0, 0, 0 },
   {"whole-file",      'W', POPT_ARG_VAL,    &whole_file, 1, 0, 0 },
   {"no-whole-file",    0,  POPT_ARG_VAL,    &whole_file, 0, 0, 0 },
-  {"checksum-choice",  0,  POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
   {"no-W",             0,  POPT_ARG_VAL,    &whole_file, 0, 0, 0 },
   {"checksum",        'c', POPT_ARG_VAL,    &always_checksum, 1, 0, 0 },
   {"no-checksum",      0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
   {"no-c",             0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
+  {"checksum-choice",  0,  POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
+  {"cc",               0,  POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
   {"block-size",      'B', POPT_ARG_LONG,   &block_size, 0, 0, 0 },
   {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
   {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
@@ -994,10 +996,12 @@ static struct poptOption long_options[] = {
   {"no-fuzzy",         0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
   {"no-y",             0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
   {"compress",        'z', POPT_ARG_NONE,   0, 'z', 0, 0 },
-  {"old-compress",     0,  POPT_ARG_VAL,    &do_compression, 1, 0, 0 },
-  {"new-compress",     0,  POPT_ARG_VAL,    &do_compression, 2, 0, 0 },
-  {"no-compress",      0,  POPT_ARG_VAL,    &do_compression, 0, 0, 0 },
-  {"no-z",             0,  POPT_ARG_VAL,    &do_compression, 0, 0, 0 },
+  {"old-compress",     0,  POPT_ARG_NONE,   0, OPT_OLD_COMPRESS, 0, 0 },
+  {"new-compress",     0,  POPT_ARG_NONE,   0, OPT_NEW_COMPRESS, 0, 0 },
+  {"no-compress",      0,  POPT_ARG_NONE,   0, OPT_NO_COMPRESS, 0, 0 },
+  {"no-z",             0,  POPT_ARG_NONE,   0, OPT_NO_COMPRESS, 0, 0 },
+  {"compress-choice",  0,  POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
+  {"zz",               0,  POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
   {"skip-compress",    0,  POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
   {"compress-level",   0,  POPT_ARG_INT,    &def_compress_level, 0, 0, 0 },
   {0,                 'P', POPT_ARG_NONE,   0, 'P', 0, 0 },
@@ -1668,6 +1672,19 @@ int parse_arguments(int *argc_p, const char ***argv_p)
                        do_compression++;
                        break;
 
+               case OPT_OLD_COMPRESS:
+                       compress_choice = "zlib";
+                       break;
+
+               case OPT_NEW_COMPRESS:
+                       compress_choice = "zlibx";
+                       break;
+
+               case OPT_NO_COMPRESS:
+                       do_compression = 0;
+                       compress_choice = NULL;
+                       break;
+
                case 'M':
                        arg = poptGetOptArg(pc);
                        if (*arg != '-') {
@@ -1948,6 +1965,13 @@ int parse_arguments(int *argc_p, const char ***argv_p)
                exit_cleanup(0);
        }
 
+       if (!compress_choice && do_compression > 1)
+               compress_choice = "zlibx";
+       if (compress_choice && strcasecmp(compress_choice, "auto") != 0)
+               parse_compress_choice(0); /* Can twiddle do_compression and possibly NULL-out compress_choice  */
+       else
+               compress_choice = NULL;
+
        if (do_compression || def_compress_level != NOT_SPECIFIED) {
                if (def_compress_level == NOT_SPECIFIED)
                        def_compress_level = Z_DEFAULT_COMPRESSION;
@@ -1955,24 +1979,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
                        snprintf(err_buf, sizeof err_buf, "--compress-level value is invalid: %d\n",
                                 def_compress_level);
                        return 0;
-               } else if (def_compress_level == Z_NO_COMPRESSION)
+               } else if (def_compress_level == Z_NO_COMPRESSION) {
                        do_compression = 0;
-               else if (!do_compression)
+                       compress_choice = NULL;
+               } else if (!do_compression)
                        do_compression = 1;
                if (do_compression && refused_compress) {
                        create_refuse_error(refused_compress);
                        return 0;
                }
-#ifdef EXTERNAL_ZLIB
-               if (do_compression == 1) {
-                       snprintf(err_buf, sizeof err_buf,
-                               "This rsync lacks old-style --compress due to its external zlib.  Try -zz.\n");
-                       if (am_server)
-                               return 0;
-                       fprintf(stderr, "%s" "Continuing without compression.\n\n", err_buf);
-                       do_compression = 0;
-               }
-#endif
        }
 
 #ifdef HAVE_SETVBUF
@@ -2750,6 +2765,16 @@ void server_options(char **args, int *argc_p)
                args[ac++] = arg;
        }
 
+       if ((!compress_choice && do_compression > 1) || (compress_choice && strcasecmp(compress_choice, "zlibx") == 0))
+               args[ac++] = "--new-compress";
+       else if (compress_choice && strcasecmp(compress_choice, "zlib") == 0)
+               args[ac++] = "--old-compress";
+       else if (compress_choice) {
+               if (asprintf(&arg, "--compress-choice=%s", compress_choice) < 0)
+                       goto oom;
+               args[ac++] = arg;
+       }
+
        if (am_sender) {
                if (max_delete > 0) {
                        if (asprintf(&arg, "--max-delete=%d", max_delete) < 0)
@@ -2930,9 +2955,6 @@ void server_options(char **args, int *argc_p)
                exit_cleanup(RERR_MALLOC);
        }
 
-       if (do_compression > 1)
-               args[ac++] = "--new-compress";
-
        if (remote_option_cnt) {
                int j;
                if (ac + remote_option_cnt > MAX_SERVER_ARGS) {

diff --git a/rsync.yo b/rsync.yo
index ff8bca0458e57a4a4034abe26d90e025358d4c09..0b6c8d650057feea3a8591332a3ff7376c3e80af 100644 (file)
--- a/rsync.yo
+++ b/rsync.yo
@@ -1370,7 +1370,7 @@ destination machines is higher than the bandwidth to disk (especially when the
 the source and destination are specified as local paths, but only if no
 batch-writing option is in effect.
 
-dit(bf(--checksum-choice=STR)) This option overrides the checksum algorithms.
+dit(bf(--checksum-choice=STR, --cc=STR)) This option overrides the checksum algorithms.
 If one algorithm name is specified, it is used for both the transfer checksums
 and (assuming bf(--checksum) is specified) the pre-transfer checksums. If two
 comma-separated names are supplied, the first name affects the transfer
@@ -1402,7 +1402,7 @@ error. This method does not allow you to specify the transfer checksum
 separately from the pre-transfer checksum, and it ignores "auto" and all
 unknown checksum names.  If the remote rsync is not new enough to handle a
 checksum negotiation list, the list is silently ignored unless it contains the
-string "FAIL" in it.
+string "FAIL".
 
 The use of the bf(--checksum-choice) option overrides this environment list.
 
@@ -2056,27 +2056,54 @@ dit(bf(-z, --compress)) With this option, rsync compresses the file data
 as it is sent to the destination machine, which reduces the amount of data
 being transmitted -- something that is useful over a slow connection.
 
-Note that this option typically achieves better compression ratios than can
-be achieved by using a compressing remote shell or a compressing transport
+The "zlib" compression method typically achieves better compression ratios than
+can be achieved by using a compressing remote shell or a compressing transport
 because it takes advantage of the implicit information in the matching data
 blocks that are not explicitly sent over the connection.  This matching-data
-compression comes at a cost of CPU, though, and can be disabled by repeating
-the bf(-z) option, but only if both sides are at least version 3.1.1.
+compression comes at a cost of CPU, though, and can be disabled by using the
+"zlibx" compresson method instead.  This can be selected by repeating the
+bf(-z) option or specifying bf(--compress-choice=zlibx), but it only works if
+both sides of the transfer are at least version 3.1.1.
 
 Note that if your version of rsync was compiled with an external zlib (instead
-of the zlib that comes packaged with rsync) then it will not support the
-old-style compression, only the new-style (repeated-option) compression.  In
-the future this new-style compression will likely become the default.
-
-The client rsync requests new-style compression on the server via the
-bf(--new-compress) option, so if you see that option rejected it means that
-the server is not new enough to support bf(-zz).  Rsync also accepts the
-bf(--old-compress) option for a future time when new-style compression
-becomes the default.
+of the zlib that comes packaged with rsync) then it will give preference to
+using the "zlibx" algorithm over the "zlib" algorithm since the external zlib
+code doesn't seem to handle the extra compression properly.  You can try
+forcing the regular algorithm via bf(--zz=zlib) and be on the lookout for
+transfer failures.  If all else fails, disable compression altogether.
+
+Note that if you see an error about an option named bf(--old-compress) or
+bf(--new-compress), this is rsync trying to send the bf(--compress-choice=zlib)
+or bf(--compress-choice=zlibx) option in a backward-compatible manner that more
+rsync versions understand. This error indicates that the older rsync version
+will not allow you to force the compression type.
 
 See the bf(--skip-compress) option for the default list of file suffixes
 that will not be compressed.
 
+dit(bf(--compress-choice=STR, --zz=STR)) This option can be used to override the
+automatic selection of the compression algorithm that is the default when
+bf(--compress) is used.
+
+Currently the STR can be "zlib", "zlibx", or "none".
+
+The "zlib" algorithm is given preference over "zlibx" if your rsync was
+compiled with the internal zlib code, otherwise that preference is reversed.
+These 2 algorithms are the stame except that "zlibx" does not try to include
+matched data that was not transferred in the compression computations.
+
+If "none" is specified, that is equivalent to using bf(--no-compress).
+
+This option implies bf(--compress) unless "none" was specified.
+
+You can also override the compression negotation using the RSYNC_COMPRESS_LIST
+environment variable by setting it to a space-separated list of compression
+names that you consider acceptable. If no common compress choice is found, the
+client exits with an error. It ignores "auto" and all unknown compression
+names.  If the remote rsync is not new enough to handle a compression
+negotiation list, the list is silently ignored unless it contains the string
+"FAIL".
+
 dit(bf(--compress-level=NUM)) Explicitly set the compression level to use
 (see bf(--compress)) instead of letting it default.  If NUM is non-zero,
 the bf(--compress) option is implied.

diff --git a/token.c b/token.c
index a5ed6ff0fd9600b4d5bac8e3da327e4df3ece736..1fdfa425588bafd71e04895065ac33459f11e948 100644 (file)
--- a/token.c
+++ b/token.c
@@ -29,6 +29,10 @@ extern int module_id;
 extern int def_compress_level;
 extern char *skip_compress;
 
+#ifndef Z_INSERT_ONLY
+#define Z_INSERT_ONLY Z_SYNC_FLUSH
+#endif
+
 static int compression_level, per_file_default_level;
 
 struct suffix_tree {
@@ -405,7 +409,6 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
        } else if (token != -2 && do_compression == 1) {
                /* Add the data in the current block to the compressor's
                 * history and hash table. */
-#ifndef EXTERNAL_ZLIB
                do {
                        /* Break up long sections in the same way that
                         * see_deflate_token() does. */
@@ -424,11 +427,6 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
                                exit_cleanup(RERR_STREAMIO);
                        }
                } while (toklen > 0);
-#else
-               toklen++;
-               rprintf(FERROR, "Impossible error in external-zlib code (1).\n");
-               exit_cleanup(RERR_STREAMIO);
-#endif
        }
 }
 
@@ -579,7 +577,6 @@ static int32 recv_deflated_token(int f, char **data)
  */
 static void see_deflate_token(char *buf, int32 len)
 {
-#ifndef EXTERNAL_ZLIB
        int r;
        int32 blklen;
        unsigned char hdr[5];
@@ -617,11 +614,6 @@ static void see_deflate_token(char *buf, int32 len)
                        exit_cleanup(RERR_STREAMIO);
                }
        } while (len || rx_strm.avail_out == 0);
-#else
-       buf++; len++;
-       rprintf(FERROR, "Impossible error in external-zlib code (2).\n");
-       exit_cleanup(RERR_STREAMIO);
-#endif
 }
 
 /**