Moved some of the delete code from receiver.c to here and implemented
[rsync.git] / flist.c
1 /*
2    Copyright (C) Andrew Tridgell 1996
3    Copyright (C) Paul Mackerras 1996
4    Copyright (C) 2001, 2002 by Martin Pool <mbp@samba.org>
5
6    This program is free software; you can redistribute it and/or modify
7    it under the terms of the GNU General Public License as published by
8    the Free Software Foundation; either version 2 of the License, or
9    (at your option) any later version.
10
11    This program is distributed in the hope that it will be useful,
12    but WITHOUT ANY WARRANTY; without even the implied warranty of
13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14    GNU General Public License for more details.
15
16    You should have received a copy of the GNU General Public License
17    along with this program; if not, write to the Free Software
18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21 /** @file flist.c
22  * Generate and receive file lists
23  *
24  * @sa http://lists.samba.org/pipermail/rsync/2000-June/002351.html
25  *
26  **/
27
28 #include "rsync.h"
29
30 extern struct stats stats;
31
32 extern int verbose;
33 extern int do_progress;
34 extern int am_root;
35 extern int am_server;
36 extern int am_daemon;
37 extern int am_sender;
38 extern int delete_during;
39 extern int always_checksum;
40 extern int module_id;
41 extern int ignore_errors;
42 extern int numeric_ids;
43
44 extern int cvs_exclude;
45
46 extern int recurse;
47 extern int xfer_dirs;
48 extern char curr_dir[MAXPATHLEN];
49 extern char *backup_dir;
50 extern char *backup_suffix;
51 extern int filesfrom_fd;
52
53 extern int one_file_system;
54 extern int keep_dirlinks;
55 extern int preserve_links;
56 extern int preserve_hard_links;
57 extern int preserve_perms;
58 extern int preserve_devices;
59 extern int preserve_uid;
60 extern int preserve_gid;
61 extern int relative_paths;
62 extern int implied_dirs;
63 extern int make_backups;
64 extern int backup_suffix_len;
65 extern int copy_links;
66 extern int copy_unsafe_links;
67 extern int protocol_version;
68 extern int sanitize_paths;
69 extern int delete_excluded;
70 extern int max_delete;
71 extern int orig_umask;
72 extern int list_only;
73
74 extern struct exclude_list_struct exclude_list;
75 extern struct exclude_list_struct server_exclude_list;
76 extern struct exclude_list_struct local_exclude_list;
77
78 int io_error;
79
80 static char empty_sum[MD4_SUM_LENGTH];
81 static unsigned int file_struct_len;
82 static struct file_list *received_flist;
83
84 static void clean_flist(struct file_list *flist, int strip_root, int no_dups);
85 static void output_flist(struct file_list *flist, const char *whose_list);
86
87 void init_flist(void)
88 {
89         struct file_struct f;
90
91         /* Figure out how big the file_struct is without trailing padding */
92         file_struct_len = offsetof(struct file_struct, flags) + sizeof f.flags;
93 }
94
95
96 static int show_filelist_p(void)
97 {
98         return verbose && xfer_dirs && !am_server;
99 }
100
101 static void start_filelist_progress(char *kind)
102 {
103         rprintf(FINFO, "%s ... ", kind);
104         if (verbose > 1 || do_progress)
105                 rprintf(FINFO, "\n");
106         rflush(FINFO);
107 }
108
109
110 static void emit_filelist_progress(const struct file_list *flist)
111 {
112         rprintf(FINFO, " %d files...\r", flist->count);
113 }
114
115
116 static void maybe_emit_filelist_progress(const struct file_list *flist)
117 {
118         if (do_progress && show_filelist_p() && (flist->count % 100) == 0)
119                 emit_filelist_progress(flist);
120 }
121
122
123 static void finish_filelist_progress(const struct file_list *flist)
124 {
125         if (do_progress) {
126                 /* This overwrites the progress line */
127                 rprintf(FINFO, "%d file%sto consider\n",
128                         flist->count, flist->count == 1 ? " " : "s ");
129         } else
130                 rprintf(FINFO, "done\n");
131 }
132
133 void show_flist_stats(void)
134 {
135         /* Nothing yet */
136 }
137
138
139 static void list_file_entry(struct file_struct *f)
140 {
141         char perms[11];
142
143         if (!f->basename) {
144                 /* this can happen if duplicate names were removed */
145                 return;
146         }
147
148         permstring(perms, f->mode);
149
150 #if SUPPORT_LINKS
151         if (preserve_links && S_ISLNK(f->mode)) {
152                 rprintf(FINFO, "%s %11.0f %s %s -> %s\n",
153                         perms,
154                         (double)f->length, timestring(f->modtime),
155                         f_name(f), f->u.link);
156         } else
157 #endif
158         {
159                 rprintf(FINFO, "%s %11.0f %s %s\n",
160                         perms,
161                         (double)f->length, timestring(f->modtime),
162                         f_name(f));
163         }
164 }
165
166
167 /**
168  * Stat either a symlink or its referent, depending on the settings of
169  * copy_links, copy_unsafe_links, etc.
170  *
171  * @retval -1 on error
172  *
173  * @retval 0 for success
174  *
175  * @post If @p path is a symlink, then @p linkbuf (of size @c
176  * MAXPATHLEN) contains the symlink target.
177  *
178  * @post @p buffer contains information about the link or the
179  * referrent as appropriate, if they exist.
180  **/
181 static int readlink_stat(const char *path, STRUCT_STAT *buffer, char *linkbuf)
182 {
183 #if SUPPORT_LINKS
184         if (copy_links)
185                 return do_stat(path, buffer);
186         if (link_stat(path, buffer, 0) < 0)
187                 return -1;
188         if (S_ISLNK(buffer->st_mode)) {
189                 int l = readlink((char *)path, linkbuf, MAXPATHLEN - 1);
190                 if (l == -1)
191                         return -1;
192                 linkbuf[l] = 0;
193                 if (copy_unsafe_links && unsafe_symlink(linkbuf, path)) {
194                         if (verbose > 1) {
195                                 rprintf(FINFO,"copying unsafe symlink \"%s\" -> \"%s\"\n",
196                                         path, linkbuf);
197                         }
198                         return do_stat(path, buffer);
199                 }
200         }
201         return 0;
202 #else
203         return do_stat(path, buffer);
204 #endif
205 }
206
207 int link_stat(const char *path, STRUCT_STAT *buffer, int follow_dirlinks)
208 {
209 #if SUPPORT_LINKS
210         if (copy_links)
211                 return do_stat(path, buffer);
212         if (do_lstat(path, buffer) < 0)
213                 return -1;
214         if (follow_dirlinks && S_ISLNK(buffer->st_mode)) {
215                 STRUCT_STAT st;
216                 if (do_stat(path, &st) == 0 && S_ISDIR(st.st_mode))
217                         *buffer = st;
218         }
219         return 0;
220 #else
221         return do_stat(path, buffer);
222 #endif
223 }
224
225 /*
226  * This function is used to check if a file should be included/excluded
227  * from the list of files based on its name and type etc.  The value of
228  * exclude_level is set to either SERVER_EXCLUDES or ALL_EXCLUDES.
229  */
230 static int check_exclude_file(char *fname, int is_dir, int exclude_level)
231 {
232         int rc;
233
234 #if 0 /* This currently never happens, so avoid a useless compare. */
235         if (exclude_level == NO_EXCLUDES)
236                 return 0;
237 #endif
238         if (fname) {
239                 /* never exclude '.', even if somebody does --exclude '*' */
240                 if (fname[0] == '.' && !fname[1])
241                         return 0;
242                 /* Handle the -R version of the '.' dir. */
243                 if (fname[0] == '/') {
244                         int len = strlen(fname);
245                         if (fname[len-1] == '.' && fname[len-2] == '/')
246                                 return 0;
247                 }
248         }
249         if (server_exclude_list.head
250             && check_exclude(&server_exclude_list, fname, is_dir) < 0)
251                 return 1;
252         if (exclude_level != ALL_EXCLUDES)
253                 return 0;
254         if (exclude_list.head
255             && (rc = check_exclude(&exclude_list, fname, is_dir)) != 0)
256                 return rc < 0;
257         if (local_exclude_list.head
258             && check_exclude(&local_exclude_list, fname, is_dir) < 0)
259                 return 1;
260         return 0;
261 }
262
263 /* used by the one_file_system code */
264 static dev_t filesystem_dev;
265
266 static void set_filesystem(char *fname)
267 {
268         STRUCT_STAT st;
269         if (do_stat(fname, &st) != 0)
270                 return;
271         filesystem_dev = st.st_dev;
272 }
273
274
275 static int to_wire_mode(mode_t mode)
276 {
277 #if SUPPORT_LINKS
278         if (S_ISLNK(mode) && (_S_IFLNK != 0120000))
279                 return (mode & ~(_S_IFMT)) | 0120000;
280 #endif
281         return (int)mode;
282 }
283
284 static mode_t from_wire_mode(int mode)
285 {
286         if ((mode & (_S_IFMT)) == 0120000 && (_S_IFLNK != 0120000))
287                 return (mode & ~(_S_IFMT)) | _S_IFLNK;
288         return (mode_t)mode;
289 }
290
291
292 static void send_directory(int f, struct file_list *flist, char *dir);
293
294 static char *flist_dir;
295 static int flist_dir_len;
296
297
298 /**
299  * Make sure @p flist is big enough to hold at least @p flist->count
300  * entries.
301  **/
302 void flist_expand(struct file_list *flist)
303 {
304         struct file_struct **new_ptr;
305
306         if (flist->count < flist->malloced)
307                 return;
308
309         if (flist->malloced < FLIST_START)
310                 flist->malloced = FLIST_START;
311         else if (flist->malloced >= FLIST_LINEAR)
312                 flist->malloced += FLIST_LINEAR;
313         else
314                 flist->malloced *= 2;
315
316         /*
317          * In case count jumped or we are starting the list
318          * with a known size just set it.
319          */
320         if (flist->malloced < flist->count)
321                 flist->malloced = flist->count;
322
323         new_ptr = realloc_array(flist->files, struct file_struct *,
324                                 flist->malloced);
325
326         if (verbose >= 2 && flist->malloced != FLIST_START) {
327                 rprintf(FINFO, "[%s] expand file_list to %.0f bytes, did%s move\n",
328                     who_am_i(),
329                     (double)sizeof flist->files[0] * flist->malloced,
330                     (new_ptr == flist->files) ? " not" : "");
331         }
332
333         flist->files = new_ptr;
334
335         if (!flist->files)
336                 out_of_memory("flist_expand");
337 }
338
339 void send_file_entry(struct file_struct *file, int f, unsigned short base_flags)
340 {
341         unsigned short flags;
342         static time_t modtime;
343         static mode_t mode;
344         static int64 dev;
345         static dev_t rdev;
346         static uint32 rdev_major;
347         static uid_t uid;
348         static gid_t gid;
349         static char lastname[MAXPATHLEN];
350         char fname[MAXPATHLEN];
351         int l1, l2;
352
353         if (f == -1)
354                 return;
355
356         if (!file) {
357                 write_byte(f, 0);
358                 modtime = 0, mode = 0;
359                 dev = 0, rdev = makedev(0, 0);
360                 rdev_major = 0;
361                 uid = 0, gid = 0;
362                 *lastname = '\0';
363                 return;
364         }
365
366         io_write_phase = "send_file_entry";
367
368         f_name_to(file, fname);
369
370         flags = base_flags;
371
372         if (file->mode == mode)
373                 flags |= XMIT_SAME_MODE;
374         else
375                 mode = file->mode;
376         if (preserve_devices) {
377                 if (protocol_version < 28) {
378                         if (IS_DEVICE(mode)) {
379                                 if (file->u.rdev == rdev)
380                                         flags |= XMIT_SAME_RDEV_pre28;
381                                 else
382                                         rdev = file->u.rdev;
383                         } else
384                                 rdev = makedev(0, 0);
385                 } else if (IS_DEVICE(mode)) {
386                         rdev = file->u.rdev;
387                         if ((uint32)major(rdev) == rdev_major)
388                                 flags |= XMIT_SAME_RDEV_MAJOR;
389                         else
390                                 rdev_major = major(rdev);
391                         if ((uint32)minor(rdev) <= 0xFFu)
392                                 flags |= XMIT_RDEV_MINOR_IS_SMALL;
393                 }
394         }
395         if (file->uid == uid)
396                 flags |= XMIT_SAME_UID;
397         else
398                 uid = file->uid;
399         if (file->gid == gid)
400                 flags |= XMIT_SAME_GID;
401         else
402                 gid = file->gid;
403         if (file->modtime == modtime)
404                 flags |= XMIT_SAME_TIME;
405         else
406                 modtime = file->modtime;
407
408 #if SUPPORT_HARD_LINKS
409         if (file->link_u.idev) {
410                 if (file->F_DEV == dev) {
411                         if (protocol_version >= 28)
412                                 flags |= XMIT_SAME_DEV;
413                 } else
414                         dev = file->F_DEV;
415                 flags |= XMIT_HAS_IDEV_DATA;
416         }
417 #endif
418
419         for (l1 = 0;
420             lastname[l1] && (fname[l1] == lastname[l1]) && (l1 < 255);
421             l1++) {}
422         l2 = strlen(fname+l1);
423
424         if (l1 > 0)
425                 flags |= XMIT_SAME_NAME;
426         if (l2 > 255)
427                 flags |= XMIT_LONG_NAME;
428
429         /* We must make sure we don't send a zero flag byte or the
430          * other end will terminate the flist transfer.  Note that
431          * the use of XMIT_DEL_START on a non-dir has no meaning, so
432          * it's harmless way to add a bit to the first flag byte. */
433         if (protocol_version >= 28) {
434                 if (!flags && !S_ISDIR(mode))
435                         flags |= XMIT_DEL_START;
436                 if ((flags & 0xFF00) || !flags) {
437                         flags |= XMIT_EXTENDED_FLAGS;
438                         write_byte(f, flags);
439                         write_byte(f, flags >> 8);
440                 } else
441                         write_byte(f, flags);
442         } else {
443                 if (!(flags & 0xFF) && !S_ISDIR(mode))
444                         flags |= XMIT_DEL_START;
445                 if (!(flags & 0xFF))
446                         flags |= XMIT_LONG_NAME;
447                 write_byte(f, flags);
448         }
449         if (flags & XMIT_SAME_NAME)
450                 write_byte(f, l1);
451         if (flags & XMIT_LONG_NAME)
452                 write_int(f, l2);
453         else
454                 write_byte(f, l2);
455         write_buf(f, fname + l1, l2);
456
457         write_longint(f, file->length);
458         if (!(flags & XMIT_SAME_TIME))
459                 write_int(f, modtime);
460         if (!(flags & XMIT_SAME_MODE))
461                 write_int(f, to_wire_mode(mode));
462         if (preserve_uid && !(flags & XMIT_SAME_UID)) {
463                 if (!numeric_ids)
464                         add_uid(uid);
465                 write_int(f, uid);
466         }
467         if (preserve_gid && !(flags & XMIT_SAME_GID)) {
468                 if (!numeric_ids)
469                         add_gid(gid);
470                 write_int(f, gid);
471         }
472         if (preserve_devices && IS_DEVICE(mode)) {
473                 if (protocol_version < 28) {
474                         if (!(flags & XMIT_SAME_RDEV_pre28))
475                                 write_int(f, (int)rdev);
476                 } else {
477                         if (!(flags & XMIT_SAME_RDEV_MAJOR))
478                                 write_int(f, major(rdev));
479                         if (flags & XMIT_RDEV_MINOR_IS_SMALL)
480                                 write_byte(f, minor(rdev));
481                         else
482                                 write_int(f, minor(rdev));
483                 }
484         }
485
486 #if SUPPORT_LINKS
487         if (preserve_links && S_ISLNK(mode)) {
488                 int len = strlen(file->u.link);
489                 write_int(f, len);
490                 write_buf(f, file->u.link, len);
491         }
492 #endif
493
494 #if SUPPORT_HARD_LINKS
495         if (flags & XMIT_HAS_IDEV_DATA) {
496                 if (protocol_version < 26) {
497                         /* 32-bit dev_t and ino_t */
498                         write_int(f, dev);
499                         write_int(f, file->F_INODE);
500                 } else {
501                         /* 64-bit dev_t and ino_t */
502                         if (!(flags & XMIT_SAME_DEV))
503                                 write_longint(f, dev);
504                         write_longint(f, file->F_INODE);
505                 }
506         }
507 #endif
508
509         if (always_checksum) {
510                 char *sum;
511                 if (S_ISREG(mode))
512                         sum = file->u.sum;
513                 else if (protocol_version < 28) {
514                         /* Prior to 28, we sent a useless set of nulls. */
515                         sum = empty_sum;
516                 } else
517                         sum = NULL;
518                 if (sum) {
519                         write_buf(f, sum,
520                             protocol_version < 21 ? 2 : MD4_SUM_LENGTH);
521                 }
522         }
523
524         strlcpy(lastname, fname, MAXPATHLEN);
525
526         io_write_phase = "unknown";
527 }
528
529
530
531 void receive_file_entry(struct file_struct **fptr, unsigned short flags,
532                         struct file_list *flist, int f)
533 {
534         static time_t modtime;
535         static mode_t mode;
536         static int64 dev;
537         static dev_t rdev;
538         static uint32 rdev_major;
539         static uid_t uid;
540         static gid_t gid;
541         static char lastname[MAXPATHLEN], *lastdir;
542         static int lastdir_depth, lastdir_len = -1;
543         static unsigned int del_heir_name_len = -1;
544         static int in_del_hier = 0;
545         char thisname[MAXPATHLEN];
546         unsigned int l1 = 0, l2 = 0;
547         int alloc_len, basename_len, dirname_len, linkname_len, sum_len;
548         OFF_T file_length;
549         char *basename, *dirname, *bp;
550         struct file_struct *file;
551
552         if (!fptr) {
553                 modtime = 0, mode = 0;
554                 dev = 0, rdev = makedev(0, 0);
555                 rdev_major = 0;
556                 uid = 0, gid = 0;
557                 *lastname = '\0';
558                 del_heir_name_len = lastdir_len = -1;
559                 in_del_hier = 0;
560                 return;
561         }
562
563         if (flags & XMIT_SAME_NAME)
564                 l1 = read_byte(f);
565
566         if (flags & XMIT_LONG_NAME)
567                 l2 = read_int(f);
568         else
569                 l2 = read_byte(f);
570
571         if (l2 >= MAXPATHLEN - l1) {
572                 rprintf(FERROR,
573                         "overflow: flags=0x%x l1=%d l2=%d lastname=%s\n",
574                         flags, l1, l2, lastname);
575                 overflow("receive_file_entry");
576         }
577
578         strlcpy(thisname, lastname, l1 + 1);
579         read_sbuf(f, &thisname[l1], l2);
580         thisname[l1 + l2] = 0;
581
582         strlcpy(lastname, thisname, MAXPATHLEN);
583
584         clean_fname(thisname, 0);
585
586         if (sanitize_paths)
587                 sanitize_path(thisname, thisname, "", 0);
588
589         if ((basename = strrchr(thisname, '/')) != NULL) {
590                 dirname_len = ++basename - thisname; /* counts future '\0' */
591                 if (lastdir_len == dirname_len - 1
592                     && strncmp(thisname, lastdir, lastdir_len) == 0) {
593                         dirname = lastdir;
594                         dirname_len = 0; /* indicates no copy is needed */
595                 } else
596                         dirname = thisname;
597         } else {
598                 basename = thisname;
599                 dirname = NULL;
600                 dirname_len = 0;
601         }
602         basename_len = strlen(basename) + 1; /* count the '\0' */
603
604         file_length = read_longint(f);
605         if (!(flags & XMIT_SAME_TIME))
606                 modtime = (time_t)read_int(f);
607         if (!(flags & XMIT_SAME_MODE))
608                 mode = from_wire_mode(read_int(f));
609
610         if (preserve_uid && !(flags & XMIT_SAME_UID))
611                 uid = (uid_t)read_int(f);
612         if (preserve_gid && !(flags & XMIT_SAME_GID))
613                 gid = (gid_t)read_int(f);
614
615         if (preserve_devices) {
616                 if (protocol_version < 28) {
617                         if (IS_DEVICE(mode)) {
618                                 if (!(flags & XMIT_SAME_RDEV_pre28))
619                                         rdev = (dev_t)read_int(f);
620                         } else
621                                 rdev = makedev(0, 0);
622                 } else if (IS_DEVICE(mode)) {
623                         uint32 rdev_minor;
624                         if (!(flags & XMIT_SAME_RDEV_MAJOR))
625                                 rdev_major = read_int(f);
626                         if (flags & XMIT_RDEV_MINOR_IS_SMALL)
627                                 rdev_minor = read_byte(f);
628                         else
629                                 rdev_minor = read_int(f);
630                         rdev = makedev(rdev_major, rdev_minor);
631                 }
632         }
633
634 #if SUPPORT_LINKS
635         if (preserve_links && S_ISLNK(mode)) {
636                 linkname_len = read_int(f) + 1; /* count the '\0' */
637                 if (linkname_len <= 0 || linkname_len > MAXPATHLEN) {
638                         rprintf(FERROR, "overflow: linkname_len=%d\n",
639                                 linkname_len - 1);
640                         overflow("receive_file_entry");
641                 }
642         }
643         else
644 #endif
645                 linkname_len = 0;
646
647         sum_len = always_checksum && S_ISREG(mode) ? MD4_SUM_LENGTH : 0;
648
649         alloc_len = file_struct_len + dirname_len + basename_len
650                   + linkname_len + sum_len;
651         bp = pool_alloc(flist->file_pool, alloc_len, "receive_file_entry");
652
653         file = *fptr = (struct file_struct *)bp;
654         memset(bp, 0, file_struct_len);
655         bp += file_struct_len;
656
657         file->flags = 0;
658         file->modtime = modtime;
659         file->length = file_length;
660         file->mode = mode;
661         file->uid = uid;
662         file->gid = gid;
663
664         if (S_ISDIR(mode)) {
665                 if (flags & XMIT_DEL_START) {
666                         in_del_hier = 1;
667                         del_heir_name_len = l1 + l2;
668                         file->flags |= FLAG_DEL_START;
669                 } else if (delete_during && in_del_hier) {
670                         if (!relative_paths || (l1 >= del_heir_name_len
671                             && thisname[del_heir_name_len] == '/'))
672                                 file->flags |= FLAG_DEL_START;
673                         else
674                                 in_del_hier = 0;
675                 }
676         }
677
678         if (dirname_len) {
679                 file->dirname = lastdir = bp;
680                 lastdir_len = dirname_len - 1;
681                 memcpy(bp, dirname, dirname_len - 1);
682                 bp += dirname_len;
683                 bp[-1] = '\0';
684                 if (sanitize_paths)
685                         lastdir_depth = count_dir_elements(lastdir);
686         } else if (dirname)
687                 file->dirname = dirname;
688
689         file->basename = bp;
690         memcpy(bp, basename, basename_len);
691         bp += basename_len;
692
693         if (preserve_devices && IS_DEVICE(mode))
694                 file->u.rdev = rdev;
695
696 #if SUPPORT_LINKS
697         if (linkname_len) {
698                 file->u.link = bp;
699                 read_sbuf(f, bp, linkname_len - 1);
700                 if (sanitize_paths)
701                         sanitize_path(bp, bp, "", lastdir_depth);
702                 bp += linkname_len;
703         }
704 #endif
705
706 #if SUPPORT_HARD_LINKS
707         if (preserve_hard_links && protocol_version < 28 && S_ISREG(mode))
708                 flags |= XMIT_HAS_IDEV_DATA;
709         if (flags & XMIT_HAS_IDEV_DATA) {
710                 int64 inode;
711                 if (protocol_version < 26) {
712                         dev = read_int(f);
713                         inode = read_int(f);
714                 } else {
715                         if (!(flags & XMIT_SAME_DEV))
716                                 dev = read_longint(f);
717                         inode = read_longint(f);
718                 }
719                 if (flist->hlink_pool) {
720                         file->link_u.idev = pool_talloc(flist->hlink_pool,
721                             struct idev, 1, "inode_table");
722                         file->F_INODE = inode;
723                         file->F_DEV = dev;
724                 }
725         }
726 #endif
727
728         if (always_checksum) {
729                 char *sum;
730                 if (sum_len) {
731                         file->u.sum = sum = bp;
732                         /*bp += sum_len;*/
733                 } else if (protocol_version < 28) {
734                         /* Prior to 28, we get a useless set of nulls. */
735                         sum = empty_sum;
736                 } else
737                         sum = NULL;
738                 if (sum) {
739                         read_buf(f, sum,
740                             protocol_version < 21 ? 2 : MD4_SUM_LENGTH);
741                 }
742         }
743
744         if (!preserve_perms) {
745                 /* set an appropriate set of permissions based on original
746                  * permissions and umask. This emulates what GNU cp does */
747                 file->mode &= ~orig_umask;
748         }
749 }
750
751
752 /**
753  * Create a file_struct for a named file by reading its stat()
754  * information and performing extensive checks against global
755  * options.
756  *
757  * @return the new file, or NULL if there was an error or this file
758  * should be excluded.
759  *
760  * @todo There is a small optimization opportunity here to avoid
761  * stat()ing the file in some circumstances, which has a certain cost.
762  * We are called immediately after doing readdir(), and so we may
763  * already know the d_type of the file.  We could for example avoid
764  * statting directories if we're not recursing, but this is not a very
765  * important case.  Some systems may not have d_type.
766  **/
767 struct file_struct *make_file(char *fname, struct file_list *flist,
768                               int exclude_level)
769 {
770         static char *lastdir;
771         static int lastdir_len = -1;
772         struct file_struct *file;
773         STRUCT_STAT st;
774         char sum[SUM_LENGTH];
775         char thisname[MAXPATHLEN];
776         char linkname[MAXPATHLEN];
777         int alloc_len, basename_len, dirname_len, linkname_len, sum_len;
778         char *basename, *dirname, *bp;
779         unsigned short flags = 0;
780
781         if (!flist || !flist->count)    /* Ignore lastdir when invalid. */
782                 lastdir_len = -1;
783
784         if (strlcpy(thisname, fname, sizeof thisname)
785             >= sizeof thisname - flist_dir_len) {
786                 rprintf(FINFO, "skipping overly long name: %s\n", fname);
787                 return NULL;
788         }
789         clean_fname(thisname, 0);
790         if (sanitize_paths)
791                 sanitize_path(thisname, thisname, "", 0);
792
793         memset(sum, 0, SUM_LENGTH);
794
795         if (readlink_stat(thisname, &st, linkname) != 0) {
796                 int save_errno = errno;
797                 /* See if file is excluded before reporting an error. */
798                 if (exclude_level != NO_EXCLUDES
799                     && check_exclude_file(thisname, 0, exclude_level))
800                         return NULL;
801                 if (save_errno == ENOENT) {
802 #if SUPPORT_LINKS
803                         /* Avoid "vanished" error if symlink points nowhere. */
804                         if (copy_links && do_lstat(thisname, &st) == 0
805                             && S_ISLNK(st.st_mode)) {
806                                 io_error |= IOERR_GENERAL;
807                                 rprintf(FERROR, "symlink has no referent: %s\n",
808                                         full_fname(thisname));
809                         } else
810 #endif
811                         {
812                                 enum logcode c = am_daemon && protocol_version < 28
813                                     ? FERROR : FINFO;
814                                 io_error |= IOERR_VANISHED;
815                                 rprintf(c, "file has vanished: %s\n",
816                                         full_fname(thisname));
817                         }
818                 } else {
819                         io_error |= IOERR_GENERAL;
820                         rsyserr(FERROR, save_errno, "readlink %s failed",
821                                 full_fname(thisname));
822                 }
823                 return NULL;
824         }
825
826         /* backup.c calls us with exclude_level set to NO_EXCLUDES. */
827         if (exclude_level == NO_EXCLUDES)
828                 goto skip_excludes;
829
830         if (S_ISDIR(st.st_mode) && !xfer_dirs) {
831                 rprintf(FINFO, "skipping directory %s\n", thisname);
832                 return NULL;
833         }
834
835         /* We only care about directories because we need to avoid recursing
836          * into a mount-point directory, not to avoid copying a symlinked
837          * file if -L (or similar) was specified. */
838         if (one_file_system && st.st_dev != filesystem_dev
839             && S_ISDIR(st.st_mode))
840                 flags |= FLAG_MOUNT_POINT;
841
842         if (check_exclude_file(thisname, S_ISDIR(st.st_mode) != 0, exclude_level))
843                 return NULL;
844
845         if (lp_ignore_nonreadable(module_id)) {
846 #if SUPPORT_LINKS
847                 if (!S_ISLNK(st.st_mode))
848 #endif
849                         if (access(thisname, R_OK) != 0)
850                                 return NULL;
851         }
852
853 skip_excludes:
854
855         if (verbose > 2) {
856                 rprintf(FINFO, "[%s] make_file(%s,*,%d)\n",
857                         who_am_i(), thisname, exclude_level);
858         }
859
860         if ((basename = strrchr(thisname, '/')) != NULL) {
861                 dirname_len = ++basename - thisname; /* counts future '\0' */
862                 if (lastdir_len == dirname_len - 1
863                     && strncmp(thisname, lastdir, lastdir_len) == 0) {
864                         dirname = lastdir;
865                         dirname_len = 0; /* indicates no copy is needed */
866                 } else
867                         dirname = thisname;
868         } else {
869                 basename = thisname;
870                 dirname = NULL;
871                 dirname_len = 0;
872         }
873         basename_len = strlen(basename) + 1; /* count the '\0' */
874
875 #if SUPPORT_LINKS
876         linkname_len = S_ISLNK(st.st_mode) ? strlen(linkname) + 1 : 0;
877 #else
878         linkname_len = 0;
879 #endif
880
881         sum_len = always_checksum && S_ISREG(st.st_mode) ? MD4_SUM_LENGTH : 0;
882
883         alloc_len = file_struct_len + dirname_len + basename_len
884             + linkname_len + sum_len;
885         if (flist) {
886                 bp = pool_alloc(flist->file_pool, alloc_len,
887                     "receive_file_entry");
888         } else {
889                 if (!(bp = new_array(char, alloc_len)))
890                         out_of_memory("receive_file_entry");
891         }
892
893         file = (struct file_struct *)bp;
894         memset(bp, 0, file_struct_len);
895         bp += file_struct_len;
896
897         file->flags = flags;
898         file->modtime = st.st_mtime;
899         file->length = st.st_size;
900         file->mode = st.st_mode;
901         file->uid = st.st_uid;
902         file->gid = st.st_gid;
903
904 #if SUPPORT_HARD_LINKS
905         if (flist && flist->hlink_pool) {
906                 if (protocol_version < 28) {
907                         if (S_ISREG(st.st_mode))
908                                 file->link_u.idev = pool_talloc(
909                                     flist->hlink_pool, struct idev, 1,
910                                     "inode_table");
911                 } else {
912                         if (!S_ISDIR(st.st_mode) && st.st_nlink > 1)
913                                 file->link_u.idev = pool_talloc(
914                                     flist->hlink_pool, struct idev, 1,
915                                     "inode_table");
916                 }
917         }
918         if (file->link_u.idev) {
919                 file->F_DEV = st.st_dev;
920                 file->F_INODE = st.st_ino;
921         }
922 #endif
923
924         if (dirname_len) {
925                 file->dirname = lastdir = bp;
926                 lastdir_len = dirname_len - 1;
927                 memcpy(bp, dirname, dirname_len - 1);
928                 bp += dirname_len;
929                 bp[-1] = '\0';
930         } else if (dirname)
931                 file->dirname = dirname;
932
933         file->basename = bp;
934         memcpy(bp, basename, basename_len);
935         bp += basename_len;
936
937 #ifdef HAVE_STRUCT_STAT_ST_RDEV
938         if (preserve_devices && IS_DEVICE(st.st_mode))
939                 file->u.rdev = st.st_rdev;
940 #endif
941
942 #if SUPPORT_LINKS
943         if (linkname_len) {
944                 file->u.link = bp;
945                 memcpy(bp, linkname, linkname_len);
946                 bp += linkname_len;
947         }
948 #endif
949
950         if (sum_len) {
951                 file->u.sum = bp;
952                 file_checksum(thisname, bp, st.st_size);
953                 /*bp += sum_len;*/
954         }
955
956         file->basedir = flist_dir;
957
958         /* This code is only used by the receiver when it is building
959          * a list of files for a delete pass. */
960         if (keep_dirlinks && linkname_len && flist) {
961                 STRUCT_STAT st2;
962                 int i = flist_find(received_flist, file);
963                 if (i >= 0 && S_ISDIR(received_flist->files[i]->mode)
964                     && do_stat(thisname, &st2) == 0 && S_ISDIR(st2.st_mode)) {
965                         file->modtime = st2.st_mtime;
966                         file->length = st2.st_size;
967                         file->mode = st2.st_mode;
968                         file->uid = st2.st_uid;
969                         file->gid = st2.st_gid;
970                         file->u.link = NULL;
971                         if (file->link_u.idev) {
972                                 pool_free(flist->hlink_pool, 0, file->link_u.idev);
973                                 file->link_u.idev = NULL;
974                         }
975                 }
976         }
977
978         if (!S_ISDIR(st.st_mode))
979                 stats.total_size += st.st_size;
980
981         return file;
982 }
983
984
985 void send_file_name(int f, struct file_list *flist, char *fname,
986                     int recursive, unsigned short base_flags)
987 {
988         struct file_struct *file;
989         char fbuf[MAXPATHLEN];
990
991         /* f is set to -1 when calculating deletion file list */
992         file = make_file(fname, flist,
993             f == -1 && delete_excluded? SERVER_EXCLUDES : ALL_EXCLUDES);
994
995         if (!file)
996                 return;
997
998         maybe_emit_filelist_progress(flist);
999
1000         flist_expand(flist);
1001
1002         if (file->basename[0]) {
1003                 flist->files[flist->count++] = file;
1004                 send_file_entry(file, f, base_flags);
1005         }
1006
1007         if (recursive && S_ISDIR(file->mode)
1008             && !(file->flags & FLAG_MOUNT_POINT)) {
1009                 struct exclude_list_struct last_list = local_exclude_list;
1010                 local_exclude_list.head = local_exclude_list.tail = NULL;
1011                 send_directory(f, flist, f_name_to(file, fbuf));
1012                 if (verbose > 2) {
1013                         rprintf(FINFO, "[%s] popping %sexclude list\n",
1014                                 who_am_i(), local_exclude_list.debug_type);
1015                 }
1016                 clear_exclude_list(&local_exclude_list);
1017                 local_exclude_list = last_list;
1018         }
1019 }
1020
1021
1022 /* Note that the "recurse" value either contains -1, for infinite recursion,
1023  * or a number >= 0 indicating how many levels of recursion we will allow. */
1024 static void send_directory(int f, struct file_list *flist, char *dir)
1025 {
1026         DIR *d;
1027         struct dirent *di;
1028         char fname[MAXPATHLEN];
1029         unsigned int offset;
1030         char *p;
1031
1032         d = opendir(dir);
1033         if (!d) {
1034                 io_error |= IOERR_GENERAL;
1035                 rsyserr(FERROR, errno, "opendir %s failed", full_fname(dir));
1036                 return;
1037         }
1038
1039         offset = strlcpy(fname, dir, MAXPATHLEN);
1040         p = fname + offset;
1041         if (offset >= MAXPATHLEN || p[-1] != '/') {
1042                 if (offset >= MAXPATHLEN - 1) {
1043                         io_error |= IOERR_GENERAL;
1044                         rprintf(FERROR, "skipping long-named directory: %s\n",
1045                                 full_fname(fname));
1046                         closedir(d);
1047                         return;
1048                 }
1049                 *p++ = '/';
1050                 offset++;
1051         }
1052
1053         if (cvs_exclude) {
1054                 if (strlcpy(p, ".cvsignore", MAXPATHLEN - offset)
1055                     < MAXPATHLEN - offset) {
1056                         add_exclude_file(&local_exclude_list, fname,
1057                                          XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
1058                 } else {
1059                         io_error |= IOERR_GENERAL;
1060                         rprintf(FINFO,
1061                                 "cannot cvs-exclude in long-named directory %s\n",
1062                                 full_fname(fname));
1063                 }
1064         }
1065
1066         for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) {
1067                 char *dname = d_name(di);
1068                 if (dname[0] == '.' && (dname[1] == '\0'
1069                     || (dname[1] == '.' && dname[2] == '\0')))
1070                         continue;
1071                 if (strlcpy(p, dname, MAXPATHLEN - offset) < MAXPATHLEN - offset) {
1072                         int do_subdirs = recurse >= 1 ? recurse-- : recurse;
1073                         send_file_name(f, flist, fname, do_subdirs, 0);
1074                 } else {
1075                         io_error |= IOERR_GENERAL;
1076                         rprintf(FINFO,
1077                                 "cannot send long-named file %s\n",
1078                                 full_fname(fname));
1079                 }
1080         }
1081         if (errno) {
1082                 io_error |= IOERR_GENERAL;
1083                 rsyserr(FERROR, errno, "readdir(%s)", dir);
1084         }
1085
1086         closedir(d);
1087 }
1088
1089
1090 /* This function is normally called by the sender, but the receiving side
1091  * also uses it to construct one or more file lists if one of the --delete
1092  * options have been specified.  The delete_in_dir() function sets f to -1
1093  * so that we just construct the file list in memory without sending it
1094  * over the wire.  It also has the side-effect of ignoring user-excludes if
1095  * delete_excluded is set (so that the delete list includes user-excluded
1096  * files) and it avoids some per-arg init code for limited recursion (since
1097  * delete_in_dir() sets recurse before calling this function). */
1098 struct file_list *send_file_list(int f, int argc, char *argv[])
1099 {
1100         int l;
1101         STRUCT_STAT st;
1102         char *p, *dir, olddir[sizeof curr_dir];
1103         char lastpath[MAXPATHLEN] = "";
1104         struct file_list *flist;
1105         int64 start_write;
1106         int use_ff_fd = 0;
1107
1108         if (show_filelist_p() && f != -1)
1109                 start_filelist_progress("building file list");
1110
1111         start_write = stats.total_written;
1112
1113         flist = flist_new(f == -1 ? WITHOUT_HLINK : WITH_HLINK,
1114                           "send_file_list");
1115
1116         if (f != -1) {
1117                 io_start_buffering_out();
1118                 if (filesfrom_fd >= 0) {
1119                         if (argv[0] && !push_dir(argv[0])) {
1120                                 rsyserr(FERROR, errno, "push_dir %s failed",
1121                                         full_fname(argv[0]));
1122                                 exit_cleanup(RERR_FILESELECT);
1123                         }
1124                         use_ff_fd = 1;
1125                 }
1126         }
1127
1128         while (1) {
1129                 char fname2[MAXPATHLEN];
1130                 char *fname = fname2;
1131                 int do_subdirs;
1132
1133                 if (use_ff_fd) {
1134                         if (read_filesfrom_line(filesfrom_fd, fname) == 0)
1135                                 break;
1136                         sanitize_path(fname, fname, "", 0);
1137                 } else {
1138                         if (argc-- == 0)
1139                                 break;
1140                         strlcpy(fname, *argv++, MAXPATHLEN);
1141                         if (sanitize_paths)
1142                                 sanitize_path(fname, fname, "", 0);
1143                 }
1144
1145                 l = strlen(fname);
1146                 if (!l || fname[l - 1] == '/') {
1147                         if (l == 2 && fname[0] == '.') {
1148                                 /* Turn "./" into just "." rather than "./." */
1149                                 fname[1] = '\0';
1150                         } else if (l < MAXPATHLEN) {
1151                                 fname[l++] = '.';
1152                                 fname[l] = '\0';
1153                         }
1154                 }
1155                 if (f == -1)
1156                         ; /* recurse is pre-set */
1157                 else if (fname[l-1] == '.' && (l == 1 || fname[l-2] == '/')) {
1158                         if (!recurse && xfer_dirs)
1159                                 recurse = 1; /* allow one level */
1160                 } else if (recurse > 0)
1161                         recurse = 0;
1162
1163                 if (link_stat(fname, &st, keep_dirlinks) != 0) {
1164                         if (f != -1) {
1165                                 io_error |= IOERR_GENERAL;
1166                                 rsyserr(FERROR, errno, "link_stat %s failed",
1167                                         full_fname(fname));
1168                         }
1169                         continue;
1170                 }
1171
1172                 if (S_ISDIR(st.st_mode) && !xfer_dirs) {
1173                         rprintf(FINFO, "skipping directory %s\n", fname);
1174                         continue;
1175                 }
1176
1177                 dir = NULL;
1178                 olddir[0] = '\0';
1179
1180                 if (!relative_paths) {
1181                         p = strrchr(fname, '/');
1182                         if (p) {
1183                                 *p = 0;
1184                                 if (p == fname)
1185                                         dir = "/";
1186                                 else
1187                                         dir = fname;
1188                                 fname = p + 1;
1189                         }
1190                 } else if (f != -1 && implied_dirs && (p=strrchr(fname,'/')) && p != fname) {
1191                         /* this ensures we send the intermediate directories,
1192                            thus getting their permissions right */
1193                         char *lp = lastpath, *fn = fname, *slash = fname;
1194                         *p = 0;
1195                         /* Skip any initial directories in our path that we
1196                          * have in common with lastpath. */
1197                         while (*fn && *lp == *fn) {
1198                                 if (*fn == '/')
1199                                         slash = fn;
1200                                 lp++, fn++;
1201                         }
1202                         *p = '/';
1203                         if (fn != p || (*lp && *lp != '/')) {
1204                                 int save_copy_links = copy_links;
1205                                 int save_xfer_dirs = xfer_dirs;
1206                                 copy_links = copy_unsafe_links;
1207                                 xfer_dirs = 1;
1208                                 while ((slash = strchr(slash+1, '/')) != 0) {
1209                                         *slash = 0;
1210                                         send_file_name(f, flist, fname, 0, 0);
1211                                         *slash = '/';
1212                                 }
1213                                 copy_links = save_copy_links;
1214                                 xfer_dirs = save_xfer_dirs;
1215                                 *p = 0;
1216                                 strlcpy(lastpath, fname, sizeof lastpath);
1217                                 *p = '/';
1218                         }
1219                 }
1220
1221                 if (!*fname)
1222                         fname = ".";
1223
1224                 if (dir && *dir) {
1225                         static char *lastdir;
1226                         static int lastdir_len;
1227
1228                         strcpy(olddir, curr_dir); /* can't overflow */
1229
1230                         if (!push_dir(dir)) {
1231                                 io_error |= IOERR_GENERAL;
1232                                 rsyserr(FERROR, errno, "push_dir %s failed",
1233                                         full_fname(dir));
1234                                 continue;
1235                         }
1236
1237                         if (lastdir && strcmp(lastdir, dir) == 0) {
1238                                 flist_dir = lastdir;
1239                                 flist_dir_len = lastdir_len;
1240                         } else {
1241                                 flist_dir = lastdir = strdup(dir);
1242                                 flist_dir_len = lastdir_len = strlen(dir);
1243                         }
1244                 }
1245
1246                 if (one_file_system)
1247                         set_filesystem(fname);
1248
1249                 do_subdirs = recurse >= 1 ? recurse-- : recurse;
1250                 send_file_name(f, flist, fname, do_subdirs, XMIT_DEL_START);
1251
1252                 if (olddir[0]) {
1253                         flist_dir = NULL;
1254                         flist_dir_len = 0;
1255                         if (!pop_dir(olddir)) {
1256                                 rsyserr(FERROR, errno, "pop_dir %s failed",
1257                                         full_fname(dir));
1258                                 exit_cleanup(RERR_FILESELECT);
1259                         }
1260                 }
1261         }
1262
1263         if (f != -1) {
1264                 send_file_entry(NULL, f, 0);
1265
1266                 if (show_filelist_p())
1267                         finish_filelist_progress(flist);
1268         }
1269
1270         if (flist->hlink_pool) {
1271                 pool_destroy(flist->hlink_pool);
1272                 flist->hlink_pool = NULL;
1273         }
1274
1275         clean_flist(flist, 0, 0);
1276
1277         if (f != -1) {
1278                 /* Now send the uid/gid list. This was introduced in
1279                  * protocol version 15 */
1280                 send_uid_list(f);
1281
1282                 /* send the io_error flag */
1283                 write_int(f, lp_ignore_errors(module_id) ? 0 : io_error);
1284
1285                 io_end_buffering();
1286                 stats.flist_size = stats.total_written - start_write;
1287                 stats.num_files = flist->count;
1288         }
1289
1290         if (verbose > 3)
1291                 output_flist(flist, f < 0 ? "delete" : who_am_i());
1292
1293         if (verbose > 2)
1294                 rprintf(FINFO, "send_file_list done\n");
1295
1296         return flist;
1297 }
1298
1299
1300 struct file_list *recv_file_list(int f)
1301 {
1302         struct file_list *flist;
1303         unsigned short flags;
1304         int64 start_read;
1305
1306         if (show_filelist_p())
1307                 start_filelist_progress("receiving file list");
1308
1309         start_read = stats.total_read;
1310
1311         flist = flist_new(WITH_HLINK, "recv_file_list");
1312         received_flist = flist;
1313
1314         flist->count = 0;
1315         flist->malloced = 1000;
1316         flist->files = new_array(struct file_struct *, flist->malloced);
1317         if (!flist->files)
1318                 goto oom;
1319
1320
1321         while ((flags = read_byte(f)) != 0) {
1322                 int i = flist->count;
1323
1324                 flist_expand(flist);
1325
1326                 if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
1327                         flags |= read_byte(f) << 8;
1328                 receive_file_entry(&flist->files[i], flags, flist, f);
1329
1330                 if (S_ISREG(flist->files[i]->mode))
1331                         stats.total_size += flist->files[i]->length;
1332
1333                 flist->count++;
1334
1335                 maybe_emit_filelist_progress(flist);
1336
1337                 if (verbose > 2) {
1338                         rprintf(FINFO, "recv_file_name(%s)\n",
1339                                 f_name(flist->files[i]));
1340                 }
1341         }
1342         receive_file_entry(NULL, 0, NULL, 0); /* Signal that we're done. */
1343
1344         if (verbose > 2)
1345                 rprintf(FINFO, "received %d names\n", flist->count);
1346
1347         if (show_filelist_p())
1348                 finish_filelist_progress(flist);
1349
1350         clean_flist(flist, relative_paths, 1);
1351
1352         if (f != -1) {
1353                 /* Now send the uid/gid list. This was introduced in
1354                  * protocol version 15 */
1355                 recv_uid_list(f, flist);
1356
1357                 /* Recv the io_error flag */
1358                 if (lp_ignore_errors(module_id) || ignore_errors)
1359                         read_int(f);
1360                 else
1361                         io_error |= read_int(f);
1362         }
1363
1364         if (verbose > 3)
1365                 output_flist(flist, who_am_i());
1366
1367         if (list_only) {
1368                 int i;
1369                 for (i = 0; i < flist->count; i++)
1370                         list_file_entry(flist->files[i]);
1371         }
1372
1373         if (verbose > 2)
1374                 rprintf(FINFO, "recv_file_list done\n");
1375
1376         stats.flist_size = stats.total_read - start_read;
1377         stats.num_files = flist->count;
1378
1379         return flist;
1380
1381 oom:
1382         out_of_memory("recv_file_list");
1383         return NULL;            /* not reached */
1384 }
1385
1386
1387 int file_compare(struct file_struct **file1, struct file_struct **file2)
1388 {
1389         struct file_struct *f1 = *file1;
1390         struct file_struct *f2 = *file2;
1391
1392         if (!f1->basename && !f2->basename)
1393                 return 0;
1394         if (!f1->basename)
1395                 return -1;
1396         if (!f2->basename)
1397                 return 1;
1398         if (f1->dirname == f2->dirname)
1399                 return u_strcmp(f1->basename, f2->basename);
1400         return f_name_cmp(f1, f2);
1401 }
1402
1403
1404 int flist_find(struct file_list *flist, struct file_struct *f)
1405 {
1406         int low = 0, high = flist->count - 1;
1407
1408         while (high >= 0 && !flist->files[high]->basename) high--;
1409
1410         if (high < 0)
1411                 return -1;
1412
1413         while (low != high) {
1414                 int mid = (low + high) / 2;
1415                 int ret = file_compare(&flist->files[flist_up(flist, mid)],&f);
1416                 if (ret == 0)
1417                         return flist_up(flist, mid);
1418                 if (ret > 0)
1419                         high = mid;
1420                 else
1421                         low = mid + 1;
1422         }
1423
1424         if (file_compare(&flist->files[flist_up(flist, low)], &f) == 0)
1425                 return flist_up(flist, low);
1426         return -1;
1427 }
1428
1429 /*
1430  * Free up any resources a file_struct has allocated
1431  * and clear the file.
1432  */
1433 void clear_file(int i, struct file_list *flist)
1434 {
1435         if (flist->hlink_pool && flist->files[i]->link_u.idev)
1436                 pool_free(flist->hlink_pool, 0, flist->files[i]->link_u.idev);
1437         memset(flist->files[i], 0, file_struct_len);
1438 }
1439
1440
1441 /*
1442  * allocate a new file list
1443  */
1444 struct file_list *flist_new(int with_hlink, char *msg)
1445 {
1446         struct file_list *flist;
1447
1448         flist = new(struct file_list);
1449         if (!flist)
1450                 out_of_memory(msg);
1451
1452         memset(flist, 0, sizeof (struct file_list));
1453
1454         if (!(flist->file_pool = pool_create(FILE_EXTENT, 0,
1455             out_of_memory, POOL_INTERN)))
1456                 out_of_memory(msg);
1457
1458 #if SUPPORT_HARD_LINKS
1459         if (with_hlink && preserve_hard_links) {
1460                 if (!(flist->hlink_pool = pool_create(HLINK_EXTENT,
1461                     sizeof (struct idev), out_of_memory, POOL_INTERN)))
1462                         out_of_memory(msg);
1463         }
1464 #endif
1465
1466         return flist;
1467 }
1468
1469 /*
1470  * free up all elements in a flist
1471  */
1472 void flist_free(struct file_list *flist)
1473 {
1474         pool_destroy(flist->file_pool);
1475         pool_destroy(flist->hlink_pool);
1476         free(flist->files);
1477         free(flist);
1478 }
1479
1480
1481 /*
1482  * This routine ensures we don't have any duplicate names in our file list.
1483  * duplicate names can cause corruption because of the pipelining
1484  */
1485 static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
1486 {
1487         int i, prev_i = 0;
1488
1489         if (!flist || flist->count == 0)
1490                 return;
1491
1492         qsort(flist->files, flist->count,
1493             sizeof flist->files[0], (int (*)())file_compare);
1494
1495         for (i = no_dups? 0 : flist->count; i < flist->count; i++) {
1496                 if (flist->files[i]->basename) {
1497                         prev_i = i;
1498                         break;
1499                 }
1500         }
1501         while (++i < flist->count) {
1502                 if (!flist->files[i]->basename)
1503                         continue;
1504                 if (f_name_cmp(flist->files[i], flist->files[prev_i]) == 0) {
1505                         if (verbose > 1 && !am_server) {
1506                                 rprintf(FINFO,
1507                                         "removing duplicate name %s from file list %d\n",
1508                                         f_name(flist->files[i]), i);
1509                         }
1510                         /* Make sure that if we unduplicate '.', that we don't
1511                          * lose track of a user-specified starting point (or
1512                          * else deletions will mysteriously fail with -R). */
1513                         if (flist->files[i]->flags & FLAG_DEL_START)
1514                                 flist->files[prev_i]->flags |= FLAG_DEL_START;
1515
1516                         clear_file(i, flist);
1517                 } else
1518                         prev_i = i;
1519         }
1520
1521         if (strip_root) {
1522                 /* we need to strip off the root directory in the case
1523                    of relative paths, but this must be done _after_
1524                    the sorting phase */
1525                 for (i = 0; i < flist->count; i++) {
1526                         if (flist->files[i]->dirname &&
1527                             flist->files[i]->dirname[0] == '/') {
1528                                 memmove(&flist->files[i]->dirname[0],
1529                                         &flist->files[i]->dirname[1],
1530                                         strlen(flist->files[i]->dirname));
1531                         }
1532
1533                         if (flist->files[i]->dirname &&
1534                             !flist->files[i]->dirname[0]) {
1535                                 flist->files[i]->dirname = NULL;
1536                         }
1537                 }
1538         }
1539 }
1540
1541 static void output_flist(struct file_list *flist, const char *whose_list)
1542 {
1543         char uidbuf[16], gidbuf[16];
1544         struct file_struct *file;
1545         int i;
1546
1547         for (i = 0; i < flist->count; i++) {
1548                 file = flist->files[i];
1549                 if ((am_root || am_sender) && preserve_uid)
1550                         sprintf(uidbuf, " uid=%ld", (long)file->uid);
1551                 else
1552                         *uidbuf = '\0';
1553                 if (preserve_gid && file->gid != GID_NONE)
1554                         sprintf(gidbuf, " gid=%ld", (long)file->gid);
1555                 else
1556                         *gidbuf = '\0';
1557                 rprintf(FINFO, "[%s] i=%d %s %s %s mode=0%o len=%.0f%s%s\n",
1558                         whose_list, i, NS(file->basedir), NS(file->dirname),
1559                         NS(file->basename), (int)file->mode,
1560                         (double)file->length, uidbuf, gidbuf);
1561         }
1562 }
1563
1564
1565 enum fnc_state { fnc_DIR, fnc_SLASH, fnc_BASE };
1566
1567 /* Compare the names of two file_struct entities, just like strcmp()
1568  * would do if it were operating on the joined strings.  We assume
1569  * that there are no 0-length strings.
1570  */
1571 int f_name_cmp(struct file_struct *f1, struct file_struct *f2)
1572 {
1573         int dif;
1574         const uchar *c1, *c2;
1575         enum fnc_state state1, state2;
1576
1577         if (!f1 || !f1->basename) {
1578                 if (!f2 || !f2->basename)
1579                         return 0;
1580                 return -1;
1581         }
1582         if (!f2 || !f2->basename)
1583                 return 1;
1584
1585         if (!(c1 = (uchar*)f1->dirname)) {
1586                 state1 = fnc_BASE;
1587                 c1 = (uchar*)f1->basename;
1588         } else if (!*c1) {
1589                 state1 = fnc_SLASH;
1590                 c1 = (uchar*)"/";
1591         } else
1592                 state1 = fnc_DIR;
1593         if (!(c2 = (uchar*)f2->dirname)) {
1594                 state2 = fnc_BASE;
1595                 c2 = (uchar*)f2->basename;
1596         } else if (!*c2) {
1597                 state2 = fnc_SLASH;
1598                 c2 = (uchar*)"/";
1599         } else
1600                 state2 = fnc_DIR;
1601
1602         while (1) {
1603                 if ((dif = (int)*c1 - (int)*c2) != 0)
1604                         break;
1605                 if (!*++c1) {
1606                         switch (state1) {
1607                         case fnc_DIR:
1608                                 state1 = fnc_SLASH;
1609                                 c1 = (uchar*)"/";
1610                                 break;
1611                         case fnc_SLASH:
1612                                 state1 = fnc_BASE;
1613                                 c1 = (uchar*)f1->basename;
1614                                 break;
1615                         case fnc_BASE:
1616                                 break;
1617                         }
1618                 }
1619                 if (!*++c2) {
1620                         switch (state2) {
1621                         case fnc_DIR:
1622                                 state2 = fnc_SLASH;
1623                                 c2 = (uchar*)"/";
1624                                 break;
1625                         case fnc_SLASH:
1626                                 state2 = fnc_BASE;
1627                                 c2 = (uchar*)f2->basename;
1628                                 break;
1629                         case fnc_BASE:
1630                                 if (!*c1)
1631                                         return 0;
1632                                 break;
1633                         }
1634                 }
1635         }
1636
1637         return dif;
1638 }
1639
1640
1641 /* Return a copy of the full filename of a flist entry, using the indicated
1642  * buffer.  No size-checking is done because we checked the size when creating
1643  * the file_struct entry.
1644  */
1645 char *f_name_to(struct file_struct *f, char *fbuf)
1646 {
1647         if (!f || !f->basename)
1648                 return NULL;
1649
1650         if (f->dirname) {
1651                 int len = strlen(f->dirname);
1652                 memcpy(fbuf, f->dirname, len);
1653                 fbuf[len] = '/';
1654                 strcpy(fbuf + len + 1, f->basename);
1655         } else
1656                 strcpy(fbuf, f->basename);
1657         return fbuf;
1658 }
1659
1660
1661 /* Like f_name_to(), but we rotate through 5 static buffers of our own.
1662  */
1663 char *f_name(struct file_struct *f)
1664 {
1665         static char names[5][MAXPATHLEN];
1666         static unsigned int n;
1667
1668         n = (n + 1) % (sizeof names / sizeof names[0]);
1669
1670         return f_name_to(f, names[n]);
1671 }
1672
1673 static int is_backup_file(char *fn)
1674 {
1675         int k = strlen(fn) - backup_suffix_len;
1676         return k > 0 && strcmp(fn+k, backup_suffix) == 0;
1677 }
1678
1679 void delete_in_dir(struct file_list *flist, char *fname)
1680 {
1681         static int deletion_count = 0;
1682         struct file_list *del_flist;
1683         int save_recurse = recurse;
1684         int save_xfer_dirs = xfer_dirs;
1685         int save_implied_dirs = implied_dirs;
1686         int save_relative_paths = relative_paths;
1687         char *argv[1];
1688         int i, j, mode;
1689
1690         if (max_delete && deletion_count >= max_delete)
1691                 return;
1692
1693         if (io_error && !(lp_ignore_errors(module_id) || ignore_errors)) {
1694                 rprintf(FINFO, "IO error encountered - skipping file deletion\n");
1695                 max_delete = -1; /* avoid duplicating the above warning */
1696                 return;
1697         }
1698
1699         if (delete_during) {
1700                 recurse = 1; /* allow one level only */
1701                 xfer_dirs = 1;
1702                 implied_dirs = 0;
1703                 relative_paths = 1;
1704         }
1705
1706         argv[0] = fname;
1707         del_flist = send_file_list(-1, 1, argv);
1708
1709         relative_paths = save_relative_paths;
1710         implied_dirs = save_implied_dirs;
1711         xfer_dirs = save_xfer_dirs;
1712         recurse = save_recurse;
1713
1714         if (!del_flist)
1715                 return;
1716
1717         if (verbose > 1)
1718                 rprintf(FINFO, "deleting in %s\n", safe_fname(fname));
1719
1720         for (i = del_flist->count-1; i >= 0; i--) {
1721                 if (max_delete && deletion_count >= max_delete)
1722                         break;
1723                 if (!del_flist->files[i]->basename)
1724                         continue;
1725                 mode = del_flist->files[i]->mode;
1726                 if ((j = flist_find(flist, del_flist->files[i])) < 0
1727                     || (delete_during && S_ISDIR(mode)
1728                      && !S_ISDIR(flist->files[j]->mode))) {
1729                         char *f = f_name(del_flist->files[i]);
1730                         if (make_backups && (backup_dir || !is_backup_file(f))
1731                           && !S_ISDIR(mode)) {
1732                                 make_backup(f);
1733                                 if (verbose) {
1734                                         rprintf(FINFO, "deleting %s\n",
1735                                                 safe_fname(f));
1736                                 }
1737                         } else {
1738                                 delete_file(f, S_ISDIR(mode)
1739                                                 ? DEL_DIR | DEL_RECURSE : 0);
1740                         }
1741                         deletion_count++;
1742                 }
1743         }
1744         flist_free(del_flist);
1745 }