preparing for release of 2.5.7
[rsync.git] / token.c
1 /* 
2    Copyright (C) Andrew Tridgell 1996
3    Copyright (C) Paul Mackerras 1996
4    
5    This program is free software; you can redistribute it and/or modify
6    it under the terms of the GNU General Public License as published by
7    the Free Software Foundation; either version 2 of the License, or
8    (at your option) any later version.
9    
10    This program is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13    GNU General Public License for more details.
14    
15    You should have received a copy of the GNU General Public License
16    along with this program; if not, write to the Free Software
17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20 #include "rsync.h"
21 #include "zlib/zlib.h"
22
23 extern int do_compression;
24 static int compression_level = Z_DEFAULT_COMPRESSION;
25
26 /* determine the compression level based on a wildcard filename list */
27 void set_compression(char *fname)
28 {
29         extern int module_id;
30         char *dont;
31         char *tok;
32
33         if (!do_compression) return;
34
35         compression_level = Z_DEFAULT_COMPRESSION;
36         dont = lp_dont_compress(module_id);
37
38         if (!dont || !*dont) return;
39
40         if ((dont[0] == '*') && (!dont[1])) {
41                 /* an optimization to skip the rest of this routine */
42                 compression_level = 0;
43                 return;
44         }
45
46         dont = strdup(dont);
47         fname = strdup(fname);
48         if (!dont || !fname) return;
49
50         strlower(dont);
51         strlower(fname);
52
53         for (tok=strtok(dont," ");tok;tok=strtok(NULL," ")) {
54                 if (fnmatch(tok, fname, 0) == 0) {
55                         compression_level = 0;
56                         break;
57                 }
58         }
59         free(dont);
60         free(fname);
61 }
62
63 /* non-compressing recv token */
64 static int simple_recv_token(int f,char **data)
65 {
66         static int residue;
67         static char *buf;
68         int n;
69
70         if (!buf) {
71                 buf = new_array(char, CHUNK_SIZE);
72                 if (!buf) out_of_memory("simple_recv_token");
73         }
74
75         if (residue == 0) {
76                 int i = read_int(f);
77                 if (i <= 0) return i;
78                 residue = i;
79         }
80
81         *data = buf;
82         n = MIN(CHUNK_SIZE,residue);
83         residue -= n;
84         read_buf(f,buf,n);
85         return n;
86 }
87
88
89 /* non-compressing send token */
90 static void simple_send_token(int f,int token,
91                               struct map_struct *buf,OFF_T offset,int n)
92 {
93         extern int write_batch; /* dw */
94         int hold_int; /* dw */
95
96         if (n > 0) {
97                 int l = 0;
98                 while (l < n) {
99                         int n1 = MIN(CHUNK_SIZE,n-l);
100                         write_int(f,n1);
101                         write_buf(f,map_ptr(buf,offset+l,n1),n1);
102                         if (write_batch) {
103                             write_batch_delta_file( (char *) &n1, sizeof(int) );
104                             write_batch_delta_file(map_ptr(buf,offset+l,n1),n1);
105                         }
106                         l += n1;
107                 }
108         }
109         /* a -2 token means to send data only and no token */
110         if (token != -2) {
111                 write_int(f,-(token+1));
112                 if (write_batch) {
113                     hold_int = -(token+1);
114                     write_batch_delta_file( (char *) &hold_int, sizeof(int) );
115                 }
116         }
117 }
118
119
120 /* Flag bytes in compressed stream are encoded as follows: */
121 #define END_FLAG        0       /* that's all folks */
122 #define TOKEN_LONG      0x20    /* followed by 32-bit token number */
123 #define TOKENRUN_LONG   0x21    /* ditto with 16-bit run count */
124 #define DEFLATED_DATA   0x40    /* + 6-bit high len, then low len byte */
125 #define TOKEN_REL       0x80    /* + 6-bit relative token number */
126 #define TOKENRUN_REL    0xc0    /* ditto with 16-bit run count */
127
128 #define MAX_DATA_COUNT  16383   /* fit 14 bit count into 2 bytes with flags */
129
130 /* For coding runs of tokens */
131 static int last_token = -1;
132 static int run_start;
133 static int last_run_end;
134
135 /* Deflation state */
136 static z_stream tx_strm;
137
138 /* Output buffer */
139 static char *obuf;
140
141 /* Send a deflated token */
142 static void
143 send_deflated_token(int f, int token,
144                     struct map_struct *buf, OFF_T offset, int nb, int toklen)
145 {
146         int n, r;
147         static int init_done, flush_pending;
148         extern int write_batch;  /* dw */
149         char temp_byte;   /* dw */
150
151         if (last_token == -1) {
152                 /* initialization */
153                 if (!init_done) {
154                         tx_strm.next_in = NULL;
155                         tx_strm.zalloc = NULL;
156                         tx_strm.zfree = NULL;
157                         if (deflateInit2(&tx_strm, compression_level,
158                                          Z_DEFLATED, -15, 8,
159                                          Z_DEFAULT_STRATEGY) != Z_OK) {
160                                 rprintf(FERROR, "compression init failed\n");
161                                 exit_cleanup(RERR_STREAMIO);
162                         }
163                         if ((obuf = new_array(char, MAX_DATA_COUNT+2)) == NULL)
164                                 out_of_memory("send_deflated_token");
165                         init_done = 1;
166                 } else
167                         deflateReset(&tx_strm);
168                 last_run_end = 0;
169                 run_start = token;
170                 flush_pending = 0;
171
172         } else if (last_token == -2) {
173                 run_start = token;
174
175         } else if (nb != 0 || token != last_token + 1
176                    || token >= run_start + 65536) {
177                 /* output previous run */
178                 r = run_start - last_run_end;
179                 n = last_token - run_start;
180                 if (r >= 0 && r <= 63) {
181                         write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
182                         if (write_batch) { /* dw */
183                             temp_byte = (char)( (n==0? TOKEN_REL: TOKENRUN_REL) + r);
184                             write_batch_delta_file(&temp_byte,sizeof(char));
185                         }
186                 } else {
187                         write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
188                         write_int(f, run_start);
189                         if (write_batch) { /* dw */
190                             temp_byte = (char)(n==0? TOKEN_LONG: TOKENRUN_LONG);
191                             write_batch_delta_file(&temp_byte,sizeof(temp_byte));
192                             write_batch_delta_file((char *)&run_start,sizeof(run_start));
193                         }
194                 }
195                 if (n != 0) {
196                         write_byte(f, n);
197                         write_byte(f, n >> 8);
198                         if (write_batch) { /* dw */
199                             write_batch_delta_file((char *)&n,sizeof(char));
200                             temp_byte = (char) n >> 8;
201                             write_batch_delta_file(&temp_byte,sizeof(temp_byte));
202                         }
203                 }
204                 last_run_end = last_token;
205                 run_start = token;
206         }
207
208         last_token = token;
209
210         if (nb != 0 || flush_pending) {
211                 /* deflate the data starting at offset */
212                 int flush = Z_NO_FLUSH;
213                 tx_strm.avail_in = 0;
214                 tx_strm.avail_out = 0;
215                 do {
216                         if (tx_strm.avail_in == 0 && nb != 0) {
217                                 /* give it some more input */
218                                 n = MIN(nb, CHUNK_SIZE);
219                                 tx_strm.next_in = (Bytef *)
220                                         map_ptr(buf, offset, n);
221                                 tx_strm.avail_in = n;
222                                 nb -= n;
223                                 offset += n;
224                         }
225                         if (tx_strm.avail_out == 0) {
226                                 tx_strm.next_out = (Bytef *)(obuf + 2);
227                                 tx_strm.avail_out = MAX_DATA_COUNT;
228                                 if (flush != Z_NO_FLUSH) {
229                                         /*
230                                          * We left the last 4 bytes in the
231                                          * buffer, in case they are the
232                                          * last 4.  Move them to the front.
233                                          */
234                                         memcpy(tx_strm.next_out,
235                                                obuf+MAX_DATA_COUNT-2, 4);
236                                         tx_strm.next_out += 4;
237                                         tx_strm.avail_out -= 4;
238                                 }
239                         }
240                         if (nb == 0 && token != -2)
241                                 flush = Z_SYNC_FLUSH;
242                         r = deflate(&tx_strm, flush);
243                         if (r != Z_OK) {
244                                 rprintf(FERROR, "deflate returned %d\n", r);
245                                 exit_cleanup(RERR_STREAMIO);
246                         }
247                         if (nb == 0 || tx_strm.avail_out == 0) {
248                                 n = MAX_DATA_COUNT - tx_strm.avail_out;
249                                 if (flush != Z_NO_FLUSH) {
250                                         /*
251                                          * We have to trim off the last 4
252                                          * bytes of output when flushing
253                                          * (they are just 0, 0, ff, ff).
254                                          */
255                                         n -= 4;
256                                 }
257                                 if (n > 0) {
258                                         obuf[0] = DEFLATED_DATA + (n >> 8);
259                                         obuf[1] = n;
260                                         write_buf(f, obuf, n+2);
261                                         if (write_batch) /* dw */
262                                             write_batch_delta_file(obuf,n+2);
263                                 }
264                         }
265                 } while (nb != 0 || tx_strm.avail_out == 0);
266                 flush_pending = token == -2;
267         }
268
269         if (token == -1) {
270                 /* end of file - clean up */
271                 write_byte(f, END_FLAG);
272                 if (write_batch) { /* dw */
273                     temp_byte = END_FLAG;
274                     write_batch_delta_file((char *)&temp_byte,sizeof(temp_byte));
275                 }
276
277         } else if (token != -2) {
278                 /* add the data in the current block to the compressor's
279                    history and hash table */
280                 tx_strm.next_in = (Bytef *) map_ptr(buf, offset, toklen);
281                 tx_strm.avail_in = toklen;
282                 tx_strm.next_out = (Bytef *) obuf;
283                 tx_strm.avail_out = MAX_DATA_COUNT;
284                 r = deflate(&tx_strm, Z_INSERT_ONLY);
285                 if (r != Z_OK || tx_strm.avail_in != 0) {
286                         rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
287                                 r, tx_strm.avail_in);
288                         exit_cleanup(RERR_STREAMIO);
289                 }
290         }
291 }
292
293
294 /* tells us what the receiver is in the middle of doing */
295 static enum { r_init, r_idle, r_running, r_inflating, r_inflated } recv_state;
296
297 /* for inflating stuff */
298 static z_stream rx_strm;
299 static char *cbuf;
300 static char *dbuf;
301
302 /* for decoding runs of tokens */
303 static int rx_token;
304 static int rx_run;
305
306 /* Receive a deflated token and inflate it */
307 static int
308 recv_deflated_token(int f, char **data)
309 {
310         int n, r, flag;
311         static int init_done;
312         static int saved_flag;
313
314         for (;;) {
315                 switch (recv_state) {
316                 case r_init:
317                         if (!init_done) {
318                                 rx_strm.next_out = NULL;
319                                 rx_strm.zalloc = NULL;
320                                 rx_strm.zfree = NULL;
321                                 if (inflateInit2(&rx_strm, -15) != Z_OK) {
322                                         rprintf(FERROR, "inflate init failed\n");
323                                         exit_cleanup(RERR_STREAMIO);
324                                 }
325                                 if (!(cbuf = new_array(char, MAX_DATA_COUNT))
326                                     || !(dbuf = new_array(char, CHUNK_SIZE)))
327                                         out_of_memory("recv_deflated_token");
328                                 init_done = 1;
329                         } else {
330                                 inflateReset(&rx_strm);
331                         }
332                         recv_state = r_idle;
333                         rx_token = 0;
334                         break;
335
336                 case r_idle:
337                 case r_inflated:
338                         if (saved_flag) {
339                                 flag = saved_flag & 0xff;
340                                 saved_flag = 0;
341                         } else
342                                 flag = read_byte(f);
343                         if ((flag & 0xC0) == DEFLATED_DATA) {
344                                 n = ((flag & 0x3f) << 8) + read_byte(f);
345                                 read_buf(f, cbuf, n);
346                                 rx_strm.next_in = (Bytef *)cbuf;
347                                 rx_strm.avail_in = n;
348                                 recv_state = r_inflating;
349                                 break;
350                         }
351                         if (recv_state == r_inflated) {
352                                 /* check previous inflated stuff ended correctly */
353                                 rx_strm.avail_in = 0;
354                                 rx_strm.next_out = (Bytef *)dbuf;
355                                 rx_strm.avail_out = CHUNK_SIZE;
356                                 r = inflate(&rx_strm, Z_SYNC_FLUSH);
357                                 n = CHUNK_SIZE - rx_strm.avail_out;
358                                 /*
359                                  * Z_BUF_ERROR just means no progress was
360                                  * made, i.e. the decompressor didn't have
361                                  * any pending output for us.
362                                  */
363                                 if (r != Z_OK && r != Z_BUF_ERROR) {
364                                         rprintf(FERROR, "inflate flush returned %d (%d bytes)\n",
365                                                 r, n);
366                                         exit_cleanup(RERR_STREAMIO);
367                                 }
368                                 if (n != 0 && r != Z_BUF_ERROR) {
369                                         /* have to return some more data and
370                                            save the flag for later. */
371                                         saved_flag = flag + 0x10000;
372                                         *data = dbuf;
373                                         return n;
374                                 }
375                                 /*
376                                  * At this point the decompressor should
377                                  * be expecting to see the 0, 0, ff, ff bytes.
378                                  */
379                                 if (!inflateSyncPoint(&rx_strm)) {
380                                         rprintf(FERROR, "decompressor lost sync!\n");
381                                         exit_cleanup(RERR_STREAMIO);
382                                 }
383                                 rx_strm.avail_in = 4;
384                                 rx_strm.next_in = (Bytef *)cbuf;
385                                 cbuf[0] = cbuf[1] = 0;
386                                 cbuf[2] = cbuf[3] = 0xff;
387                                 inflate(&rx_strm, Z_SYNC_FLUSH);
388                                 recv_state = r_idle;
389                         }
390                         if (flag == END_FLAG) {
391                                 /* that's all folks */
392                                 recv_state = r_init;
393                                 return 0;
394                         }
395
396                         /* here we have a token of some kind */
397                         if (flag & TOKEN_REL) {
398                                 rx_token += flag & 0x3f;
399                                 flag >>= 6;
400                         } else
401                                 rx_token = read_int(f);
402                         if (flag & 1) {
403                                 rx_run = read_byte(f);
404                                 rx_run += read_byte(f) << 8;
405                                 recv_state = r_running;
406                         }
407                         return -1 - rx_token;
408
409                 case r_inflating:
410                         rx_strm.next_out = (Bytef *)dbuf;
411                         rx_strm.avail_out = CHUNK_SIZE;
412                         r = inflate(&rx_strm, Z_NO_FLUSH);
413                         n = CHUNK_SIZE - rx_strm.avail_out;
414                         if (r != Z_OK) {
415                                 rprintf(FERROR, "inflate returned %d (%d bytes)\n", r, n);
416                                 exit_cleanup(RERR_STREAMIO);
417                         }
418                         if (rx_strm.avail_in == 0)
419                                 recv_state = r_inflated;
420                         if (n != 0) {
421                                 *data = dbuf;
422                                 return n;
423                         }
424                         break;
425
426                 case r_running:
427                         ++rx_token;
428                         if (--rx_run == 0)
429                                 recv_state = r_idle;
430                         return -1 - rx_token;
431                 }
432         }
433 }
434
435 /*
436  * put the data corresponding to a token that we've just returned
437  * from recv_deflated_token into the decompressor's history buffer.
438  */
439 static void see_deflate_token(char *buf, int len)
440 {
441         int r, blklen;
442         unsigned char hdr[5];
443
444         rx_strm.avail_in = 0;
445         blklen = 0;
446         hdr[0] = 0;
447         do {
448                 if (rx_strm.avail_in == 0 && len != 0) {
449                         if (blklen == 0) {
450                                 /* Give it a fake stored-block header. */
451                                 rx_strm.next_in = (Bytef *)hdr;
452                                 rx_strm.avail_in = 5;
453                                 blklen = len;
454                                 if (blklen > 0xffff)
455                                         blklen = 0xffff;
456                                 hdr[1] = blklen;
457                                 hdr[2] = blklen >> 8;
458                                 hdr[3] = ~hdr[1];
459                                 hdr[4] = ~hdr[2];
460                         } else {
461                                 rx_strm.next_in = (Bytef *)buf;
462                                 rx_strm.avail_in = blklen;
463                                 len -= blklen;
464                                 blklen = 0;
465                         }
466                 }
467                 rx_strm.next_out = (Bytef *)dbuf;
468                 rx_strm.avail_out = CHUNK_SIZE;
469                 r = inflate(&rx_strm, Z_SYNC_FLUSH);
470                 if (r != Z_OK) {
471                         rprintf(FERROR, "inflate (token) returned %d\n", r);
472                         exit_cleanup(RERR_STREAMIO);
473                 }
474         } while (len || rx_strm.avail_out == 0);
475 }
476
477 /**
478  * Transmit a verbatim buffer of length @p n followed by a token.
479  * If token == -1 then we have reached EOF 
480  * If n == 0 then don't send a buffer
481  */
482 void send_token(int f,int token,struct map_struct *buf,OFF_T offset,
483                 int n,int toklen)
484 {
485         if (!do_compression) {
486                 simple_send_token(f,token,buf,offset,n);
487         } else {
488                 send_deflated_token(f, token, buf, offset, n, toklen);
489         }
490 }
491
492
493 /*
494  * receive a token or buffer from the other end. If the reurn value is >0 then
495  * it is a data buffer of that length, and *data will point at the data.
496  * if the return value is -i then it represents token i-1
497  * if the return value is 0 then the end has been reached
498  */
499 int recv_token(int f,char **data)
500 {
501         int tok;
502
503         if (!do_compression) {
504                 tok = simple_recv_token(f,data);
505         } else {
506                 tok = recv_deflated_token(f, data);
507         }
508         return tok;
509 }
510
511 /*
512  * look at the data corresponding to a token, if necessary
513  */
514 void see_token(char *data, int toklen)
515 {
516         if (do_compression)
517                 see_deflate_token(data, toklen);
518 }