1 This patch adds the ability to merge rules into your excludes/includes
2 using a ". FILE" idiom. If you specify a name without slashes, that
3 filename will be looked for in every subdirectory that rsync visits,
4 and its rules will affect the current directory and its subdirectories.
8 rsync -av --exclude='. .excl' from/ to
10 The above will look for a file named ".excl" in every directory of the
11 hierarchy that rsync visits, and it will exclude (by default) names
12 based on the rules found therein. If one of the .excl files contains
20 Then the file ".excl2" will also be read in the current dir, and all
21 subdirs of the current dir. The file ".excl3" would just be read in
22 for the current dir because its name contained a slash.
26 --- exclude.c 27 Apr 2004 01:36:06 -0000 1.75
27 +++ exclude.c 27 Apr 2004 01:44:51 -0000
28 @@ -30,32 +30,65 @@ extern int verbose;
33 +extern int sanitize_paths;
35 extern char curr_dir[];
37 -struct exclude_list_struct exclude_list = { 0, 0, "" };
38 -struct exclude_list_struct local_exclude_list = { 0, 0, "per-dir .cvsignore " };
39 -struct exclude_list_struct server_exclude_list = { 0, 0, "server " };
40 +struct exclude_list_struct exclude_list = { 0, 0, 0, 0, "" };
41 +struct exclude_list_struct server_exclude_list = { 0, 0, 0, 0, "server " };
42 char *exclude_path_prefix = NULL;
44 +struct exclude_list_root {
45 + struct exclude_list_struct *head;
49 +static char dirbuf[MAXPATHLEN];
50 +static unsigned int dirbuf_offset = 0;
52 +static void clear_exclude_list(struct exclude_list_struct *listp,
53 + struct exclude_struct *extra)
55 + listp->head = listp->extra = extra;
59 /** Build an exclude structure given a exclude pattern */
60 -static void make_exclude(struct exclude_list_struct *listp, const char *pattern,
61 - int pat_len, int include)
62 +static void make_exclude(struct exclude_list_struct *listp, const char *pat,
63 + unsigned int pat_len, int mflags)
65 struct exclude_struct *ret;
68 + unsigned int ex_len;
70 + if (mflags & MATCHFLG_MERGE_FILE) {
71 + struct exclude_struct *ex;
72 + /* If the local include file was already mentioned, don't
74 + for (ex = listp->head; ex; ex = ex->next) {
75 + if ((ex->match_flags & MATCHFLG_MERGE_FILE)
76 + && strlen(ex->pattern) == pat_len
77 + && strncmp(ex->pattern, pat, pat_len) == 0)
80 + if ((pat_len == 10 || (pat_len > 10 && pat[pat_len-11] == '/'))
81 + && strncmp(pat+pat_len-10, ".cvsignore", 10) == 0) {
82 + mflags |= MATCHFLG_CVSIGNORE;
83 + mflags &= ~MATCHFLG_INCLUDE;
85 + mflags &= ~MATCHFLG_CVSIGNORE;
88 ret = new(struct exclude_struct);
90 out_of_memory("make_exclude");
92 memset(ret, 0, sizeof ret[0]);
93 - ret->include = include;
95 if (exclude_path_prefix)
96 - ret->match_flags |= MATCHFLG_ABS_PATH;
97 - if (exclude_path_prefix && *pattern == '/')
98 + mflags |= MATCHFLG_ABS_PATH;
99 + if (exclude_path_prefix && *pat == '/')
100 ex_len = strlen(exclude_path_prefix);
103 @@ -64,33 +97,52 @@ static void make_exclude(struct exclude_
104 out_of_memory("make_exclude");
106 memcpy(ret->pattern, exclude_path_prefix, ex_len);
107 - strlcpy(ret->pattern + ex_len, pattern, pat_len + 1);
108 + strlcpy(ret->pattern + ex_len, pat, pat_len + 1);
111 if (strpbrk(ret->pattern, "*[?")) {
112 - ret->match_flags |= MATCHFLG_WILD;
113 + mflags |= MATCHFLG_WILD;
114 if ((cp = strstr(ret->pattern, "**")) != NULL) {
115 - ret->match_flags |= MATCHFLG_WILD2;
116 + mflags |= MATCHFLG_WILD2;
117 /* If the pattern starts with **, note that. */
118 if (cp == ret->pattern)
119 - ret->match_flags |= MATCHFLG_WILD2_PREFIX;
120 + mflags |= MATCHFLG_WILD2_PREFIX;
124 if (pat_len > 1 && ret->pattern[pat_len-1] == '/') {
125 ret->pattern[pat_len-1] = 0;
126 - ret->directory = 1;
127 + mflags |= MATCHFLG_DIRECTORY;
130 for (cp = ret->pattern; (cp = strchr(cp, '/')) != NULL; cp++)
133 + ret->next = listp->extra;
136 listp->head = listp->tail = ret;
138 listp->tail->next = ret;
142 + if (mflags & MATCHFLG_MERGE_FILE) {
143 + struct exclude_list_struct *lp;
144 + int ndx = local_lists.cnt++;
145 + local_lists.head = realloc_array(local_lists.head,
146 + struct exclude_list_struct, local_lists.cnt);
147 + if (!local_lists.head)
148 + out_of_memory("make_exclude");
149 + lp = &local_lists.head[ndx];
150 + clear_exclude_list(lp, NULL);
151 + if (asprintf(&lp->debug_type, "per-dir %s ", ret->pattern) < 0)
152 + out_of_memory("make_exclude");
154 + ret->slash_cnt = ndx;
157 + ret->match_flags = mflags;
160 static void free_exclude(struct exclude_struct *ex)
161 @@ -99,13 +151,15 @@ static void free_exclude(struct exclude_
165 -void free_exclude_list(struct exclude_list_struct *listp)
166 +static void free_exclude_list(struct exclude_list_struct *listp)
168 struct exclude_struct *ent, *next;
171 - rprintf(FINFO, "[%s] clearing %sexclude list\n",
172 - who_am_i(), listp->debug_type);
173 + if (listp->extra) {
175 + listp->tail->next = NULL;
177 + listp->head = NULL;
180 for (ent = listp->head; ent; ent = next) {
181 @@ -113,7 +167,78 @@ void free_exclude_list(struct exclude_li
185 - listp->head = listp->tail = NULL;
186 + clear_exclude_list(listp, NULL);
189 +void *push_local_excludes(char *fname, unsigned int offset)
192 + struct exclude_list_root *push = new_array(struct exclude_list_root, 1);
195 + out_of_memory("push_local_excludes");
197 + push->cnt = local_lists.cnt;
198 + push->head = new_array(struct exclude_list_struct, local_lists.cnt);
200 + out_of_memory("push_local_excludes");
202 + memcpy(push->head, local_lists.head,
203 + sizeof (struct exclude_list_struct) * local_lists.cnt);
205 + /* Make it easy to construct the full path for a merge that has
206 + * a relative path by saving it off. */
207 + memcpy(dirbuf, fname, offset);
208 + dirbuf_offset = offset;
210 + for (i = 0; i < local_lists.cnt; i++) {
211 + struct exclude_list_struct *listp = &local_lists.head[i];
212 + struct exclude_struct *extra;
213 + char *file = listp->parent->pattern;
217 + rprintf(FINFO, "[%s] pushing %sexclude list\n",
218 + who_am_i(), listp->debug_type);
220 + if (listp->parent->match_flags & MATCHFLG_CVSIGNORE) {
221 + flags = XFLG_WORD_SPLIT | XFLG_WORDS_ONLY;
224 + flags = listp->parent->match_flags & MATCHFLG_INCLUDE
225 + ? XFLG_DEF_INCLUDE : 0;
226 + extra = listp->head; /* Subdirs inherit our rules. */
228 + clear_exclude_list(listp, extra);
229 + if (strlcpy(fname + offset, file, MAXPATHLEN - offset)
230 + < MAXPATHLEN - offset)
231 + add_exclude_file(listp, fname, flags);
233 + io_error |= IOERR_GENERAL;
235 + "cannot add local excludes in long-named directory %s\n",
236 + full_fname(fname));
240 + return (void*)push;
243 +void pop_local_excludes(void *mem)
247 + for (i = 0; i < local_lists.cnt; i++) {
248 + struct exclude_list_struct *listp = &local_lists.head[i];
250 + rprintf(FINFO, "[%s] popping %sexclude list\n",
251 + who_am_i(), listp->debug_type);
253 + free_exclude_list(listp);
255 + free(local_lists.head);
256 + local_lists = *(struct exclude_list_root*)mem;
260 static int check_one_exclude(char *name, struct exclude_struct *ex,
261 @@ -139,7 +264,8 @@ static int check_one_exclude(char *name,
263 if (!name[0]) return 0;
265 - if (ex->directory && !name_is_dir) return 0;
266 + if ((ex->match_flags & MATCHFLG_DIRECTORY) && !name_is_dir)
269 if (*pattern == '/') {
271 @@ -206,9 +332,11 @@ static void report_exclude_result(char c
274 rprintf(FINFO, "[%s] %scluding %s %s because of %spattern %s%s\n",
275 - who_am_i(), ent->include ? "in" : "ex",
277 + ent->match_flags & MATCHFLG_INCLUDE ? "in" : "ex",
278 name_is_dir ? "directory" : "file", name, type,
279 - ent->pattern, ent->directory ? "/" : "");
281 + ent->match_flags & MATCHFLG_DIRECTORY ? "/" : "");
285 @@ -222,10 +350,18 @@ int check_exclude(struct exclude_list_st
286 struct exclude_struct *ent;
288 for (ent = listp->head; ent; ent = ent->next) {
289 + if (ent->match_flags & MATCHFLG_MERGE_FILE) {
290 + struct exclude_list_struct *lp
291 + = &local_lists.head[ent->slash_cnt];
292 + int rc = check_exclude(lp, name, name_is_dir);
297 if (check_one_exclude(name, ent, name_is_dir)) {
298 report_exclude_result(name, ent, name_is_dir,
300 - return ent->include ? 1 : -1;
301 + return (ent->match_flags & MATCHFLG_INCLUDE) ? 1 : -1;
305 @@ -241,11 +377,11 @@ int check_exclude(struct exclude_list_st
306 * *incl_ptr value will be 1 for an include, 0 for an exclude, and -1 for
307 * the list-clearing "!" token.
309 -static const char *get_exclude_tok(const char *p, int *len_ptr, int *incl_ptr,
310 +static const char *get_exclude_tok(const char *p, int *len_ptr, int *flag_ptr,
313 const unsigned char *s = (const unsigned char *)p;
315 + int len, mflags = 0;
317 if (xflags & XFLG_WORD_SPLIT) {
318 /* Skip over any initial whitespace. */
319 @@ -255,13 +391,19 @@ static const char *get_exclude_tok(const
323 - /* Is this a '+' or '-' followed by a space (not whitespace)? */
324 + /* Is this a +/-/. followed by a space (not whitespace)? */
325 if (!(xflags & XFLG_WORDS_ONLY)
326 - && (*s == '-' || *s == '+') && s[1] == ' ') {
327 - *incl_ptr = *s == '+';
328 + && (*s == '-' || *s == '+' || *s == '.') && s[1] == ' ') {
330 + mflags |= MATCHFLG_INCLUDE;
331 + else if (*s == '.') {
332 + mflags |= MATCHFLG_MERGE_FILE;
333 + if (xflags & XFLG_DEF_INCLUDE)
334 + mflags |= MATCHFLG_INCLUDE;
338 - *incl_ptr = xflags & XFLG_DEF_INCLUDE;
339 + } else if (xflags & XFLG_DEF_INCLUDE)
340 + mflags |= MATCHFLG_INCLUDE;
342 if (xflags & XFLG_WORD_SPLIT) {
343 const unsigned char *cp = s;
344 @@ -273,9 +415,10 @@ static const char *get_exclude_tok(const
347 if (*p == '!' && len == 1 && !(xflags & XFLG_WORDS_ONLY))
349 + mflags |= MATCHFLG_CLEAR_LIST;
352 + *flag_ptr = mflags;
353 return (const char *)s;
356 @@ -283,7 +426,7 @@ static const char *get_exclude_tok(const
357 void add_exclude(struct exclude_list_struct *listp, const char *pattern,
361 + int pat_len, mflags;
365 @@ -292,22 +435,48 @@ void add_exclude(struct exclude_list_str
369 - cp = get_exclude_tok(cp + pat_len, &pat_len, &incl, xflags);
370 + cp = get_exclude_tok(cp + pat_len, &pat_len, &mflags, xflags);
373 - /* If we got the special "!" token, clear the list. */
375 - free_exclude_list(listp);
377 - make_exclude(listp, cp, pat_len, incl);
379 + if (mflags & MATCHFLG_CLEAR_LIST) {
381 - rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s)\n",
382 - who_am_i(), pat_len, cp,
384 - incl ? "include" : "exclude");
385 + rprintf(FINFO, "[%s] clearing %sexclude list\n",
386 + who_am_i(), listp->debug_type);
388 + free_exclude_list(listp);
391 + if (mflags & MATCHFLG_MERGE_FILE) {
392 + char name[MAXPATHLEN];
393 + if ((unsigned) pat_len >= sizeof name)
394 + continue; /* XXX complain? */
395 + strlcpy(name, cp, pat_len+1);
396 + if (strchr(name, '/') != NULL) {
397 + if (sanitize_paths)
398 + sanitize_path(name, curr_dir);
402 + if (strlcpy(dirbuf + dirbuf_offset,
403 + name, MAXPATHLEN - dirbuf_offset)
404 + >= MAXPATHLEN - dirbuf_offset)
405 + continue; /* XXX complain? */
408 + add_exclude_file(listp, cp,
409 + xflags | XFLG_FATAL_ERRORS);
414 + make_exclude(listp, cp, pat_len, mflags);
417 + rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s%sclude)\n",
418 + who_am_i(), pat_len, cp, listp->debug_type,
419 + mflags & MATCHFLG_MERGE_FILE ? "FILE " : "",
420 + mflags & MATCHFLG_INCLUDE ? "in" : "ex");
425 @@ -383,15 +552,19 @@ void send_exclude_list(int f)
426 l = strlcpy(p, ent->pattern, sizeof p);
427 if (l == 0 || l >= MAXPATHLEN)
429 - if (ent->directory) {
430 + if (ent->match_flags & MATCHFLG_DIRECTORY) {
435 - if (ent->include) {
436 + if (ent->match_flags & MATCHFLG_INCLUDE) {
438 write_buf(f, "+ ", 2);
439 - } else if ((*p == '-' || *p == '+') && p[1] == ' ') {
440 + } else if (ent->match_flags & MATCHFLG_MERGE_FILE) {
441 + write_int(f, l + 2);
442 + write_buf(f, ". ", 2);
443 + } else if ((*p == '-' || *p == '+' || *p == '.')
446 write_buf(f, "- ", 2);
448 @@ -432,6 +605,7 @@ void add_cvs_excludes(void)
449 char fname[MAXPATHLEN];
452 + add_exclude(&exclude_list, ". .cvsignore", 0);
453 add_exclude(&exclude_list, default_cvsignore,
454 XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
456 --- flist.c 27 Apr 2004 01:36:10 -0000 1.217
457 +++ flist.c 27 Apr 2004 01:44:52 -0000
458 @@ -39,8 +39,6 @@ extern int module_id;
459 extern int ignore_errors;
460 extern int numeric_ids;
462 -extern int cvs_exclude;
465 extern char curr_dir[MAXPATHLEN];
466 extern char *files_from;
467 @@ -66,7 +64,6 @@ extern int write_batch;
469 extern struct exclude_list_struct exclude_list;
470 extern struct exclude_list_struct server_exclude_list;
471 -extern struct exclude_list_struct local_exclude_list;
475 @@ -211,8 +208,6 @@ int link_stat(const char *path, STRUCT_S
477 static int check_exclude_file(char *fname, int is_dir, int exclude_level)
481 #if 0 /* This currently never happens, so avoid a useless compare. */
482 if (exclude_level == NO_EXCLUDES)
484 @@ -234,10 +229,7 @@ static int check_exclude_file(char *fnam
485 if (exclude_level != ALL_EXCLUDES)
487 if (exclude_list.head
488 - && (rc = check_exclude(&exclude_list, fname, is_dir)) != 0)
490 - if (local_exclude_list.head
491 - && check_exclude(&local_exclude_list, fname, is_dir) < 0)
492 + && check_exclude(&exclude_list, fname, is_dir) < 0)
496 @@ -946,11 +938,7 @@ void send_file_name(int f, struct file_l
498 if (recursive && S_ISDIR(file->mode)
499 && !(file->flags & FLAG_MOUNT_POINT)) {
500 - struct exclude_list_struct last_list = local_exclude_list;
501 - local_exclude_list.head = local_exclude_list.tail = NULL;
502 send_directory(f, flist, f_name_to(file, fbuf));
503 - free_exclude_list(&local_exclude_list);
504 - local_exclude_list = last_list;
508 @@ -961,6 +949,7 @@ static void send_directory(int f, struct
510 char fname[MAXPATHLEN];
512 + void *save_excludes;
516 @@ -985,18 +974,7 @@ static void send_directory(int f, struct
521 - if (strlcpy(p, ".cvsignore", MAXPATHLEN - offset)
522 - < MAXPATHLEN - offset) {
523 - add_exclude_file(&local_exclude_list, fname,
524 - XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
526 - io_error |= IOERR_GENERAL;
528 - "cannot cvs-exclude in long-named directory %s\n",
529 - full_fname(fname));
532 + save_excludes = push_local_excludes(fname, offset);
534 for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) {
535 char *dname = d_name(di);
536 @@ -1017,6 +995,8 @@ static void send_directory(int f, struct
537 rprintf(FERROR, "readdir(%s): (%d) %s\n",
538 dir, errno, strerror(errno));
541 + pop_local_excludes(save_excludes);
545 --- proto.h 22 Apr 2004 09:58:09 -0000 1.189
546 +++ proto.h 27 Apr 2004 01:44:52 -0000
547 @@ -51,7 +51,8 @@ int start_daemon(int f_in, int f_out);
548 int daemon_main(void);
549 void setup_protocol(int f_out,int f_in);
550 int claim_connection(char *fname,int max_connections);
551 -void free_exclude_list(struct exclude_list_struct *listp);
552 +void *push_local_excludes(char *fname, unsigned int offset);
553 +void pop_local_excludes(void *mem);
554 int check_exclude(struct exclude_list_struct *listp, char *name, int name_is_dir);
555 void add_exclude(struct exclude_list_struct *listp, const char *pattern,
557 --- rsync.h 27 Apr 2004 01:36:16 -0000 1.199
558 +++ rsync.h 27 Apr 2004 01:44:52 -0000
559 @@ -490,18 +490,21 @@ struct map_struct {
560 #define MATCHFLG_WILD2 (1<<1) /* pattern has '**' */
561 #define MATCHFLG_WILD2_PREFIX (1<<2) /* pattern starts with '**' */
562 #define MATCHFLG_ABS_PATH (1<<3) /* path-match on absolute path */
563 +#define MATCHFLG_INCLUDE (1<<4) /* this is an include, not an exclude */
564 +#define MATCHFLG_CLEAR_LIST (1<<5) /* this item is the "!" token */
565 +#define MATCHFLG_DIRECTORY (1<<6) /* this matches only directories */
566 +#define MATCHFLG_MERGE_FILE (1<<7) /* specifies a file to merge */
567 +#define MATCHFLG_CVSIGNORE (1<<8) /* parse this as a .cvsignore file */
568 struct exclude_struct {
569 struct exclude_struct *next;
577 struct exclude_list_struct {
578 - struct exclude_struct *head;
579 - struct exclude_struct *tail;
580 + struct exclude_struct *head, *tail;
581 + struct exclude_struct *extra, *parent;
585 --- rsync.yo 24 Apr 2004 06:16:04 -0000 1.164
586 +++ rsync.yo 27 Apr 2004 01:44:53 -0000
587 @@ -1064,6 +1064,72 @@ itemize(
588 it would be excluded by the "*")
591 +manpagesection(MERGING EXCLUDE FILES)
593 +You can merge whole files into an exclude file using a rule that starts
594 +with a ". " (a dot followed by a space) and has a filename in place of the
595 +pattern. There are two types of merge rules, single-instance and
599 + it() If the filename has no slashes in it, it is a per-directory merge;
600 + rsync scans every directory that is traversed and merges the named file's
601 + contents (when it exists), putting the contents of each subdirectory's
602 + file at the start of this per-directory sub-list (so subdirectories
603 + inherit the contents of their parent directories by default, but each
604 + subdirectory's rules have precedence over the parent's rules).
606 + it() If a filename has a slash in it, it is a single-instance merge; the
607 + named file's contents will be merged into the current exclude file,
608 + replacing the merge rule. Thus, you should use the name ./foo instead of
609 + foo if you don't want to scan for "foo" in all the subdirectories of the
613 +Note also that you can eliminate all the inherited rules for the current
614 +per-directory ruleset by putting the list-clearing token (!) in the file.
615 +This clears only the rules of the current per-directory sub-list (up
616 +through the token) and only for the current directory and its
619 +Here's an example. Specify the file that holds this set of rules via a
620 +normal --exclude-from option:
623 + . /home/user/.global_excludes
630 +This will merge the contents of the /home/user/.global_excludes file at the
631 +start of the list and also turns the ".excl" filename into a per-directory
632 +exclude file whose local contents will be merged into the list in place of
635 +Additionally, you can affect where the --cvs-exclude (-C) option's
636 +inclusion of a per-directory .cvsignore file gets placed into your rules by
637 +adding an explicit a merge rule for ".cvsignore". For instance, specifying
641 + rsync -avC --exclude='. .cvsignore' --exclude-from=foo a/ b
644 +will merge all the per-directory .cvsignore rules at the start of your list
645 +rather than at the end. This allows their dir-specific rules to supersede
646 +your rules instead of being subservient to them. (The global rules taken
647 +from the $HOME/.cvsignore file and from $CVSIGNORE are not affected by
650 +Note also that the parsing of any merge-file named ".cvsignore" is always
651 +done in a CVS-compatible manner (even if -C wasn't specified) -- i.e. the
652 +rules are always exclude rules (even when specified by an include option),
653 +they are split on whitespace, no special prefixes or list-clearing tokens
654 +are honored, and (for per-directory files) subdirectories don't inherit the
655 +parent directory's rules.
657 manpagesection(BATCH MODE)
659 bf(Note:) Batch mode should be considered experimental in this version