Use do_ftruncate() for the added call too.
[rsync-patches.git] / link-by-hash.diff
1 Jason M. Felice wrote:
2
3 This patch adds the --link-by-hash=DIR option, which hard links received
4 files in a link farm arranged by MD4 file hash.  The result is that the system
5 will only store one copy of the unique contents of each file, regardless of
6 the file's name.
7
8 To use this patch, run these commands for a successful build:
9
10     patch -p1 <patches/link-by-hash.diff
11     ./prepare-source
12     ./configure
13     make
14
15 based-on: d64bda1c1e79dc385f194d74f7957ce7cd118654
16 diff --git a/Makefile.in b/Makefile.in
17 --- a/Makefile.in
18 +++ b/Makefile.in
19 @@ -37,7 +37,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
20         util.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
21  OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
22         fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
23 -OBJS3=progress.o pipe.o
24 +OBJS3=progress.o pipe.o hashlink.o
25  DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
26  popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
27         popt/popthelp.o popt/poptparse.o
28 diff --git a/flist.c b/flist.c
29 --- a/flist.c
30 +++ b/flist.c
31 @@ -74,6 +74,7 @@ extern int sender_keeps_checksum;
32  extern int unsort_ndx;
33  extern struct stats stats;
34  extern char *filesfrom_host;
35 +extern char *link_by_hash_dir;
36  extern char *usermap, *groupmap;
37  
38  extern char curr_dir[MAXPATHLEN];
39 @@ -912,7 +913,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
40                 extra_len += EXTRA_LEN;
41  #endif
42  
43 -       if (always_checksum && S_ISREG(mode))
44 +       if ((always_checksum || link_by_hash_dir) && S_ISREG(mode))
45                 extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
46  
47  #if SIZEOF_INT64 >= 8
48 diff --git a/hashlink.c b/hashlink.c
49 new file mode 100644
50 --- /dev/null
51 +++ b/hashlink.c
52 @@ -0,0 +1,339 @@
53 +/*
54 +   Copyright (C) Cronosys, LLC 2004
55 +
56 +   This program is free software; you can redistribute it and/or modify
57 +   it under the terms of the GNU General Public License as published by
58 +   the Free Software Foundation; either version 2 of the License, or
59 +   (at your option) any later version.
60 +
61 +   This program is distributed in the hope that it will be useful,
62 +   but WITHOUT ANY WARRANTY; without even the implied warranty of
63 +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
64 +   GNU General Public License for more details.
65 +
66 +   You should have received a copy of the GNU General Public License
67 +   along with this program; if not, write to the Free Software
68 +   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
69 +*/
70 +
71 +/* This file contains code used by the --link-by-hash option. */
72 +
73 +#include "rsync.h"
74 +
75 +extern char *link_by_hash_dir;
76 +
77 +#ifdef HAVE_LINK
78 +
79 +char *make_hash_name(struct file_struct *file)
80 +{
81 +       char hash[33], *dst;
82 +       uchar c, *src = (uchar*)F_SUM(file);
83 +       int i;
84 +
85 +       for (dst = hash, i = 0; i < 4; i++, src++) {
86 +               c = *src >> 4;
87 +               *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
88 +               c = *src & 0x0f;
89 +               *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
90 +       }
91 +       *dst++ = '/';
92 +       for (i = 0; i < 12; i++, src++) {
93 +               c = *src >> 4;
94 +               *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
95 +               c = *src & 0x0f;
96 +               *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0');
97 +       }
98 +       *dst = 0;
99 +
100 +       if (asprintf(&dst,"%s/%s",link_by_hash_dir,hash) < 0)
101 +               out_of_memory("make_hash_name");
102 +       return dst;
103 +}
104 +
105 +
106 +void kill_hashfile(struct hashfile_struct *hashfile)
107 +{
108 +       if (!hashfile)
109 +               return;
110 +       free(hashfile->name);
111 +       close(hashfile->fd);
112 +       free(hashfile);
113 +}
114 +
115 +
116 +void kill_hashfiles(struct hashfile_struct *hashfiles)
117 +{
118 +       struct hashfile_struct *iter, *next;
119 +       if ((iter = hashfiles) != NULL) {
120 +               do {
121 +                       next = iter->next;
122 +                       kill_hashfile(iter);
123 +                       iter = next;
124 +               } while (iter != hashfiles);
125 +       }
126 +}
127 +
128 +
129 +struct hashfile_struct *find_hashfiles(char *hashname, int64 size, long *fnbr)
130 +{
131 +       DIR *d;
132 +       struct dirent *di;
133 +       struct hashfile_struct *hashfiles = NULL, *hashfile;
134 +       STRUCT_STAT st;
135 +       long this_fnbr;
136 +
137 +       *fnbr = 0;
138 +
139 +       /* Build a list of potential candidates and open
140 +        * them. */
141 +       if ((d = opendir(hashname)) == NULL) {
142 +               rsyserr(FERROR, errno, "opendir failed: \"%s\"", hashname);
143 +               free(hashname);
144 +               return NULL;
145 +       }
146 +       while ((di = readdir(d)) != NULL) {
147 +               if (!strcmp(di->d_name,".") || !strcmp(di->d_name,"..")) {
148 +                       continue;
149 +               }
150 +
151 +               /* We need to have the largest fnbr in case we need to store
152 +                * a new file. */
153 +               this_fnbr = atol(di->d_name);
154 +               if (this_fnbr > *fnbr)
155 +                       *fnbr = this_fnbr;
156 +
157 +               hashfile = new_array(struct hashfile_struct, 1);
158 +               if (asprintf(&hashfile->name,"%s/%s",hashname, di->d_name) < 0)
159 +                       out_of_memory("find_hashfiles");
160 +               if (do_stat(hashfile->name,&st) == -1) {
161 +                       rsyserr(FERROR, errno, "stat failed: %s", hashfile->name);
162 +                       kill_hashfile(hashfile);
163 +                       continue;
164 +               }
165 +               if (st.st_size != size) {
166 +                       kill_hashfile(hashfile);
167 +                       continue;
168 +               }
169 +               hashfile->nlink = st.st_nlink;
170 +               hashfile->fd = open(hashfile->name,O_RDONLY|O_BINARY);
171 +               if (hashfile->fd == -1) {
172 +                       rsyserr(FERROR, errno, "open failed: %s", hashfile->name);
173 +                       kill_hashfile(hashfile);
174 +                       continue;
175 +               }
176 +               if (hashfiles == NULL)
177 +                       hashfiles = hashfile->next = hashfile->prev = hashfile;
178 +               else {
179 +                       hashfile->next = hashfiles;
180 +                       hashfile->prev = hashfiles->prev;
181 +                       hashfile->next->prev = hashfile;
182 +                       hashfile->prev->next = hashfile;
183 +               }
184 +       }
185 +       closedir(d);
186 +
187 +       return hashfiles;
188 +}
189 +
190 +
191 +struct hashfile_struct *compare_hashfiles(int fd,struct hashfile_struct *files)
192 +{
193 +       int amt, hamt;
194 +       char buffer[BUFSIZ], cmpbuffer[BUFSIZ];
195 +       struct hashfile_struct *iter, *next, *best;
196 +       uint32 nlink;
197 +
198 +       if (!files)
199 +               return NULL;
200 +
201 +       iter = files; /* in case files are 0 bytes */
202 +       while ((amt = read(fd, buffer, BUFSIZ)) > 0) {
203 +               iter = files;
204 +               do {
205 +                       /* Icky bit to resync when we steal the first node. */
206 +                       if (!files)
207 +                               files = iter;
208 +
209 +                       next = iter->next;
210 +
211 +                       hamt = read(iter->fd, cmpbuffer, BUFSIZ);
212 +                       if (amt != hamt || memcmp(buffer, cmpbuffer, amt)) {
213 +                               if (iter == files) {
214 +                                       files = files->prev;
215 +                               }
216 +                               if (iter->next == iter) {
217 +                                       files = next = NULL;
218 +                               } else {
219 +                                       next = iter->next;
220 +                                       if (iter == files) {
221 +                                               /* So we know to resync */
222 +                                               files = NULL;
223 +                                       }
224 +                               }
225 +                               iter->next->prev = iter->prev;
226 +                               iter->prev->next = iter->next;
227 +                               kill_hashfile(iter);
228 +                       }
229 +
230 +                       iter = next;
231 +               } while (iter != files);
232 +
233 +               if (iter == NULL && files == NULL) {
234 +                       /* There are no matches. */
235 +                       return NULL;
236 +               }
237 +       }
238 +
239 +       if (amt == -1) {
240 +               rsyserr(FERROR, errno, "read failed in compare_hashfiles()");
241 +               kill_hashfiles(files);
242 +               return NULL;
243 +       }
244 +
245 +       /* If we only have one file left, use it. */
246 +       if (files == files->next) {
247 +               return files;
248 +       }
249 +
250 +       /* All files which remain in the list are identical and should have
251 +        * the same size.  We pick the one with the lowest link count (we
252 +        * may have rolled over because we hit the maximum link count for
253 +        * the filesystem). */
254 +       best = iter = files;
255 +       nlink = iter->nlink;
256 +       do {
257 +               if (iter->nlink < nlink) {
258 +                       nlink = iter->nlink;
259 +                       best = iter;
260 +               }
261 +               iter = iter->next;
262 +       } while (iter != files);
263 +
264 +       best->next->prev = best->prev;
265 +       best->prev->next = best->next;
266 +       if (files == best)
267 +               files = files->next;
268 +       kill_hashfiles(files);
269 +       return best;
270 +}
271 +
272 +
273 +int link_by_hash(const char *fnametmp, const char *fname, struct file_struct *file)
274 +{
275 +       STRUCT_STAT st;
276 +       char *hashname = make_hash_name(file);
277 +       int first = 0, rc;
278 +       char *linkname;
279 +       long last_fnbr;
280 +
281 +       if (F_LENGTH(file) == 0)
282 +               return robust_rename(fnametmp, fname, NULL, 0644);
283 +
284 +       if (do_stat(hashname, &st) == -1) {
285 +               char *dirname;
286 +
287 +               /* Directory does not exist. */
288 +               dirname = strdup(hashname);
289 +               *strrchr(dirname,'/') = 0;
290 +               if (do_mkdir(dirname, 0755) == -1 && errno != EEXIST) {
291 +                       rsyserr(FERROR, errno, "mkdir failed: %s", dirname);
292 +                       free(hashname);
293 +                       free(dirname);
294 +                       return robust_rename(fnametmp, fname, NULL, 0644);
295 +               }
296 +               free(dirname);
297 +
298 +               if (do_mkdir(hashname, 0755) == -1 && errno != EEXIST) {
299 +                       rsyserr(FERROR, errno, "mkdir failed: %s", hashname);
300 +                       free(hashname);
301 +                       return robust_rename(fnametmp, fname, NULL, 0644);
302 +               }
303 +
304 +               first = 1;
305 +               if (asprintf(&linkname,"%s/0",hashname) < 0)
306 +                       out_of_memory("link_by_hash");
307 +               rprintf(FINFO, "(1) linkname = %s\n", linkname);
308 +       } else {
309 +               struct hashfile_struct *hashfiles, *hashfile;
310 +
311 +               if (do_stat(fnametmp,&st) == -1) {
312 +                       rsyserr(FERROR, errno, "stat failed: %s", fname);
313 +                       return -1;
314 +               }
315 +               hashfiles = find_hashfiles(hashname, st.st_size, &last_fnbr);
316 +
317 +               if (hashfiles == NULL) {
318 +                       first = 1;
319 +                       if (asprintf(&linkname,"%s/0",hashname) < 0)
320 +                               out_of_memory("link_by_hash");
321 +                       rprintf(FINFO, "(2) linkname = %s\n", linkname);
322 +               } else {
323 +                       int fd;
324 +                       /* Search for one identical to us. */
325 +                       if ((fd = open(fnametmp,O_RDONLY|O_BINARY)) == -1) {
326 +                               rsyserr(FERROR, errno, "open failed: %s", fnametmp);
327 +                               kill_hashfiles(hashfiles);
328 +                               return -1;
329 +                       }
330 +                       hashfile = compare_hashfiles(fd, hashfiles);
331 +                       hashfiles = NULL;
332 +                       close(fd);
333 +
334 +                       if (hashfile) {
335 +                               first = 0;
336 +                               linkname = strdup(hashfile->name);
337 +                               rprintf(FINFO, "(3) linkname = %s\n", linkname);
338 +                               kill_hashfile(hashfile);
339 +                       } else {
340 +                               first = 1;
341 +                               if (asprintf(&linkname, "%s/%ld", hashname, last_fnbr + 1) < 0)
342 +                                       out_of_memory("link_by_hash");
343 +                               rprintf(FINFO, "(4) linkname = %s\n", linkname);
344 +                       }
345 +               }
346 +       }
347 +
348 +       if (!first) {
349 +               rprintf(FINFO, "link-by-hash (existing): \"%s\" -> %s\n",
350 +                               linkname, full_fname(fname));
351 +               robust_unlink(fname);
352 +               rc = do_link(linkname, fname);
353 +               if (rc == -1) {
354 +                       if (errno == EMLINK) {
355 +                               first = 1;
356 +                               free(linkname);
357 +                               if (asprintf(&linkname,"%s/%ld",hashname, last_fnbr + 1) < 0)
358 +                                       out_of_memory("link_by_hash");
359 +                               rprintf(FINFO, "(5) linkname = %s\n", linkname);
360 +                               rprintf(FINFO,"link-by-hash: max link count exceeded, starting new file \"%s\".\n", linkname);
361 +                       } else {
362 +                               rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
363 +                                       linkname, full_fname(fname));
364 +                               rc = robust_rename(fnametmp, fname, NULL, 0644);
365 +                       }
366 +               } else {
367 +                       do_unlink(fnametmp);
368 +               }
369 +       }
370 +
371 +       if (first) {
372 +               rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n",
373 +                               full_fname(fname),linkname);
374 +
375 +               rc = robust_rename(fnametmp, fname, NULL, 0644);
376 +               if (rc != 0) {
377 +                       rsyserr(FERROR, errno, "rename \"%s\" -> \"%s\"",
378 +                               full_fname(fnametmp), full_fname(fname));
379 +               }
380 +               rc = do_link(fname,linkname);
381 +               if (rc != 0) {
382 +                       rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
383 +                               full_fname(fname), linkname);
384 +               }
385 +       }
386 +
387 +       free(linkname);
388 +       free(hashname);
389 +       return rc;
390 +}
391 +#endif
392 diff --git a/options.c b/options.c
393 --- a/options.c
394 +++ b/options.c
395 @@ -158,6 +158,7 @@ char *backup_suffix = NULL;
396  char *tmpdir = NULL;
397  char *partial_dir = NULL;
398  char *basis_dir[MAX_BASIS_DIRS+1];
399 +char *link_by_hash_dir = NULL;
400  char *config_file = NULL;
401  char *shell_cmd = NULL;
402  char *logfile_name = NULL;
403 @@ -748,6 +749,7 @@ void usage(enum logcode F)
404    rprintf(F,"     --compare-dest=DIR      also compare destination files relative to DIR\n");
405    rprintf(F,"     --copy-dest=DIR         ... and include copies of unchanged files\n");
406    rprintf(F,"     --link-dest=DIR         hardlink to files in DIR when unchanged\n");
407 +  rprintf(F,"     --link-by-hash=DIR      create hardlinks by hash into DIR\n");
408    rprintf(F," -z, --compress              compress file data during the transfer\n");
409    rprintf(F,"     --compress-level=NUM    explicitly set compression level\n");
410    rprintf(F,"     --skip-compress=LIST    skip compressing files with a suffix in LIST\n");
411 @@ -800,7 +802,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
412        OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST, OPT_HELP,
413        OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
414        OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
415 -      OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
416 +      OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_LINK_BY_HASH,
417        OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT,
418        OPT_SERVER, OPT_REFUSED_BASE = 9000};
419  
420 @@ -943,6 +945,7 @@ static struct poptOption long_options[] = {
421    {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
422    {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
423    {"link-dest",        0,  POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 },
424 +  {"link-by-hash",     0,  POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0},
425    {"fuzzy",           'y', POPT_ARG_VAL,    &fuzzy_basis, 1, 0, 0 },
426    {"no-fuzzy",         0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
427    {"no-y",             0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
428 @@ -1769,6 +1772,21 @@ int parse_arguments(int *argc_p, const char ***argv_p)
429                         return 0;
430  #endif
431  
432 +                case OPT_LINK_BY_HASH:
433 +#ifdef HAVE_LINK
434 +                       arg = poptGetOptArg(pc);
435 +                       if (sanitize_paths)
436 +                               arg = sanitize_path(NULL, arg, NULL, 0, SP_DEFAULT);
437 +                       link_by_hash_dir = (char *)arg;
438 +                       break;
439 +#else
440 +                       snprintf(err_buf, sizeof err_buf,
441 +                                "hard links are not supported on this %s\n",
442 +                                am_server ? "server" : "client");
443 +                       rprintf(FERROR, "ERROR: %s", err_buf);
444 +                       return 0;
445 +#endif
446 +
447                 default:
448                         /* A large opt value means that set_refuse_options()
449                          * turned this option off. */
450 @@ -2649,6 +2667,11 @@ void server_options(char **args, int *argc_p)
451         } else if (inplace)
452                 args[ac++] = "--inplace";
453  
454 +       if (link_by_hash_dir && am_sender) {
455 +               args[ac++] = "--link-by-hash";
456 +               args[ac++] = link_by_hash_dir;
457 +       }
458 +
459         if (files_from && (!am_sender || filesfrom_host)) {
460                 if (filesfrom_host) {
461                         args[ac++] = "--files-from";
462 diff --git a/receiver.c b/receiver.c
463 --- a/receiver.c
464 +++ b/receiver.c
465 @@ -197,11 +197,13 @@ int open_tmpfile(char *fnametmp, const char *fname, struct file_struct *file)
466  }
467  
468  static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
469 -                       const char *fname, int fd, OFF_T total_size)
470 +                       const char *fname, int fd, OFF_T total_size,
471 +                       const char *md4)
472  {
473         static char file_sum1[MAX_DIGEST_LEN];
474         struct map_struct *mapbuf;
475         struct sum_struct sum;
476 +       md_context mdfour_data;
477         int32 len;
478         OFF_T offset = 0;
479         OFF_T offset2;
480 @@ -221,6 +223,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
481         } else
482                 mapbuf = NULL;
483  
484 +       if (md4)
485 +               mdfour_begin(&mdfour_data);
486 +
487         sum_init(checksum_seed);
488  
489         if (append_mode > 0) {
490 @@ -268,6 +273,8 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
491                         cleanup_got_literal = 1;
492  
493                         sum_update(data, i);
494 +                       if (md4)
495 +                               mdfour_update(&mdfour_data, (uchar*)data, i);
496  
497                         if (fd != -1 && write_file(fd,data,i) != i)
498                                 goto report_write_error;
499 @@ -294,6 +301,8 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
500  
501                         see_token(map, len);
502                         sum_update(map, len);
503 +                       if (md4)
504 +                               mdfour_update(&mdfour_data, (uchar*)map, len);
505                 }
506  
507                 if (updating_basis_or_equiv) {
508 @@ -341,6 +350,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
509         if (sum_end(file_sum1) != checksum_len)
510                 overflow_exit("checksum_len"); /* Impossible... */
511  
512 +       if (md4)
513 +               mdfour_result(&mdfour_data, (uchar*)md4);
514 +
515         if (mapbuf)
516                 unmap_file(mapbuf);
517  
518 @@ -355,7 +367,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
519  
520  static void discard_receive_data(int f_in, OFF_T length)
521  {
522 -       receive_data(f_in, NULL, -1, 0, NULL, -1, length);
523 +       receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL);
524  }
525  
526  static void handle_delayed_updates(char *local_name)
527 @@ -783,7 +795,7 @@ int recv_files(int f_in, int f_out, char *local_name)
528  
529                 /* recv file data */
530                 recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size,
531 -                                      fname, fd2, F_LENGTH(file));
532 +                                      fname, fd2, F_LENGTH(file), F_SUM(file));
533  
534                 log_item(log_code, file, iflags, NULL);
535  
536 diff --git a/rsync.c b/rsync.c
537 --- a/rsync.c
538 +++ b/rsync.c
539 @@ -49,6 +49,7 @@ extern int flist_eof;
540  extern int file_old_total;
541  extern int keep_dirlinks;
542  extern int make_backups;
543 +extern char *link_by_hash_dir;
544  extern struct file_list *cur_flist, *first_flist, *dir_flist;
545  extern struct chmod_mode_struct *daemon_chmod_modes;
546  #ifdef ICONV_OPTION
547 @@ -646,7 +647,12 @@ int finish_transfer(const char *fname, const char *fnametmp,
548         /* move tmp file over real file */
549         if (DEBUG_GTE(RECV, 1))
550                 rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname);
551 -       ret = robust_rename(fnametmp, fname, temp_copy_name, file->mode);
552 +#ifdef HAVE_LINK
553 +       if (link_by_hash_dir)
554 +               ret = link_by_hash(fnametmp, fname, file);
555 +       else
556 +#endif
557 +               ret = robust_rename(fnametmp, fname, temp_copy_name, file->mode);
558         if (ret < 0) {
559                 rsyserr(FERROR_XFER, errno, "%s %s -> \"%s\"",
560                         ret == -2 ? "copy" : "rename",
561 diff --git a/rsync.h b/rsync.h
562 --- a/rsync.h
563 +++ b/rsync.h
564 @@ -871,6 +871,14 @@ struct stats {
565         int xferred_files;
566  };
567  
568 +struct hashfile_struct {
569 +       struct hashfile_struct *next;
570 +       struct hashfile_struct *prev;
571 +       char *name;
572 +       int fd;
573 +       uint32 nlink;
574 +};
575 +
576  struct chmod_mode_struct;
577  
578  struct flist_ndx_item {
579 diff --git a/rsync.yo b/rsync.yo
580 --- a/rsync.yo
581 +++ b/rsync.yo
582 @@ -401,6 +401,7 @@ to the detailed description below for a complete description.  verb(
583       --compare-dest=DIR      also compare received files relative to DIR
584       --copy-dest=DIR         ... and include copies of unchanged files
585       --link-dest=DIR         hardlink to files in DIR when unchanged
586 +     --link-by-hash=DIR      create hardlinks by hash into DIR
587   -z, --compress              compress file data during the transfer
588       --compress-level=NUM    explicitly set compression level
589       --skip-compress=LIST    skip compressing files with suffix in LIST