- Do not allow a negotiated checksum or compression choice of "none" unless
the user authorized it via an environment variable or command-line option.
- - Improved the man page a bit more.
+ - Added the `--max-alloc=SIZE` option to be able to override the memory
+ allocator's sanity-check limit. It defaults to 1G (as before) but the error
+ message when exceeding it specifically mentions the new option so that you
+ can differentiate an out-of-memory error from a failure of this limit. It
+ also allows you to specify the value via the RSYNC_MAX_ALLOC environment
+ variable.
+
+ - The memory allocation functions now automatically check for a failure and
+ die when out of memory. This eliminated some caller-side check-and-die
+ code and added some missing sanity-checking of allocations.
- Preparing for an upcoming xxHash release that provides new XXH3 & XXH128
hashing routines (disabled until their code is finalized).
+ - Improved the man page a bit more.
+
------------------------------------------------------------------------------
<a name="3.2.1"></a>
*/
#include "rsync.h"
+#include "ifuncs.h"
static int allow_forward_dns;
if (strcmp(addr, inet_ntoa(*(struct in_addr*)(hp->h_addr_list[i]))) == 0) {
/* If reverse lookups are off, we'll use the conf-specified
* hostname in preference to UNDETERMINED. */
- if (host == undetermined_hostname) {
- if (!(*host_ptr = strdup(tok)))
- *host_ptr = undetermined_hostname;
- }
+ if (host == undetermined_hostname)
+ *host_ptr = strdup(tok);
return 1;
}
}
char *tok;
char *list2 = strdup(list);
- if (!list2)
- out_of_memory("access_match");
-
strlower(list2);
for (tok = strtok(list2, " ,\t"); tok; tok = strtok(NULL, " ,\t")) {
{
rsync_acl *racl = new(rsync_acl);
- if (!racl)
- out_of_memory("create_racl");
*racl = empty_rsync_acl;
return racl;
qsort(temp_ida_list.items, temp_ida_list.count, sizeof (id_access), id_access_sorter);
}
#endif
- if (!(racl->names.idas = new_array(id_access, temp_ida_list.count)))
- out_of_memory("unpack_smb_acl");
+ racl->names.idas = new_array(id_access, temp_ida_list.count);
memcpy(racl->names.idas, temp_ida_list.items, temp_ida_list.count * sizeof (id_access));
} else
racl->names.idas = NULL;
if (cnt) {
char *bp = buf + 4*4;
- id_access *ida;
- if (!(ida = racl->names.idas = new_array(id_access, cnt)))
- out_of_memory("get_rsync_acl");
+ id_access *ida = racl->names.idas = new_array(id_access, cnt);
racl->names.count = cnt;
for ( ; cnt--; ida++, bp += 4+4) {
ida->id = IVAL(bp, 0);
uchar computed_mask_bits = 0;
int i, count = read_varint(f);
- if (count) {
- if (!(ent->idas = new_array(id_access, count)))
- out_of_memory("recv_ida_entries");
- } else
- ent->idas = NULL;
-
+ ent->idas = count ? new_array(id_access, count) : NULL;
ent->count = count;
for (i = 0; i < count; i++) {
#include "rsync.h"
#include "itypes.h"
+#include "ifuncs.h"
extern int read_only;
extern char *password_file;
}
*pass++ = '\0';
- if (!(users = strdup(users)))
- out_of_memory("auth_server");
+ users = strdup(users);
for (tok = strtok(users, " ,\t"); tok; tok = strtok(NULL, " ,\t")) {
char *opts;
else {
gid_t *gid_array = gid_list.items;
auth_uid_groups_cnt = gid_list.count;
- if ((auth_uid_groups = new_array(char *, auth_uid_groups_cnt)) == NULL)
- out_of_memory("auth_server");
+ auth_uid_groups = new_array(char *, auth_uid_groups_cnt);
for (j = 0; j < auth_uid_groups_cnt; j++)
auth_uid_groups[j] = gid_to_group(gid_array[j]);
}
free(buf1);
buf1 = new_array(char, len+4);
len1 = len;
- if (!buf1)
- out_of_memory("get_checksum2");
}
memcpy(buf1, buf, len);
else
modlen = p - *argv;
- if (!(modname = new_array(char, modlen+1+1))) /* room for '/' & '\0' */
- out_of_memory("start_inband_exchange");
+ modname = new_array(char, modlen+1+1); /* room for '/' & '\0' */
strlcpy(modname, *argv, modlen + 1);
modname[modlen] = '/';
modname[modlen+1] = '\0';
io_printf(f_out, "@ERROR: invalid early_input length\n");
return -1;
}
- if (!(early_input = new_array(char, early_input_len)))
- out_of_memory("exchange_protocols");
+ early_input = new_array(char, early_input_len);
read_buf(f_in, early_input, early_input_len);
if (!read_line_old(f_in, line, sizeof line, 0))
}
if (!nno->saw) {
- if (!(nno->saw = new_array0(uchar, nno->saw_len)))
- out_of_memory("init_nno_saw");
+ nno->saw = new_array0(uchar, nno->saw_len);
/* We'll take this opportunity to make sure that the main_name values are set right. */
for (cnt = 1, nni = nno->list; nni->name; nni++, cnt++) {
#include "rsync.h"
#include "default-cvsignore.h"
+#include "ifuncs.h"
extern int am_server;
extern int am_sender;
} else
suf_len = 0;
- if (!(rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1)))
- out_of_memory("add_rule");
+ rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1);
if (pre_len) {
memcpy(rule->pattern, dirbuf + module_dirlen, pre_len);
for (cp = rule->pattern; cp < rule->pattern + pre_len; cp++) {
}
}
- if (!(lp = new_array0(filter_rule_list, 1)))
- out_of_memory("add_rule");
+ lp = new_array0(filter_rule_list, 1);
if (asprintf(&lp->debug_type, " [per-dir %s]", cp) < 0)
out_of_memory("add_rule");
rule->u.mergelist = lp;
if (mergelist_cnt == mergelist_size) {
mergelist_size += 5;
- mergelist_parents = realloc_array(mergelist_parents,
- filter_rule *,
- mergelist_size);
- if (!mergelist_parents)
- out_of_memory("add_rule");
+ mergelist_parents = realloc_array(mergelist_parents, filter_rule *, mergelist_size);
}
if (DEBUG_GTE(FILTER, 2)) {
rprintf(FINFO, "[%s] activating mergelist #%d%s\n",
push = (struct local_filter_state *)new_array(char,
sizeof (struct local_filter_state)
+ (mergelist_cnt-1) * sizeof (filter_rule_list));
- if (!push)
- out_of_memory("push_local_filters");
push->mergelist_cnt = mergelist_cnt;
for (i = 0; i < mergelist_cnt; i++) {
if (!*s)
return NULL;
- if (!(rule = new0(filter_rule)))
- out_of_memory("parse_rule_tok");
+ rule = new0(filter_rule);
/* Inherit from the template. Don't inherit FILTRULES_SIDES; we check
* that later. */
const char *name;
filter_rule *excl_self;
- if (!(excl_self = new0(filter_rule)))
- out_of_memory("parse_filter_str");
+ excl_self = new0(filter_rule);
/* Find the beginning of the basename and add an exclude for it. */
for (name = pat + pat_len; name > pat && name[-1] != '/'; name--) {}
add_rule(listp, name, (pat + pat_len) - name, excl_self, 0);
wf_writeBufSize = WRITE_SIZE * 8;
wf_writeBufCnt = 0;
wf_writeBuf = new_array(char, wf_writeBufSize);
- if (!wf_writeBuf)
- out_of_memory("write_file");
}
r1 = (int)MIN((size_t)len, wf_writeBufSize - wf_writeBufCnt);
if (r1) {
{
struct map_struct *map;
- if (!(map = new0(struct map_struct)))
- out_of_memory("map_file");
+ map = new0(struct map_struct);
if (blk_size && (read_size % blk_size))
read_size += blk_size - (read_size % blk_size);
/* make sure we have allocated enough memory for the window */
if (window_size > map->p_size) {
map->p = realloc_array(map->p, char, window_size);
- if (!map->p)
- out_of_memory("map_ptr");
map->p_size = window_size;
}
if (flist->malloced < flist->used + extra)
flist->malloced = flist->used + extra;
- new_ptr = realloc_array(flist->files, struct file_struct *,
- flist->malloced);
+ new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
+ linkname_len;
if (pool)
bp = pool_alloc(pool, alloc_len, "make_file");
- else {
- if (!(bp = new_array(char, alloc_len)))
- out_of_memory("make_file");
- }
+ else
+ bp = new_array(char, alloc_len);
memset(bp, 0, extra_len + FILE_STRUCT_LEN);
bp += extra_len;
if (use_qsort)
qsort(fp, num, PTR_SIZE, file_compare);
else {
- struct file_struct **tmp = new_array(struct file_struct *,
- (num+1) / 2);
+ struct file_struct **tmp = new_array(struct file_struct *, (num+1) / 2);
fsort_tmp(fp, num, tmp);
free(tmp);
}
len = strlen(limit+1);
memcpy(&relname_list, F_DIR_RELNAMES_P(lastpath_struct), sizeof relname_list);
if (!relname_list) {
- if (!(relname_list = new0(item_list)))
- out_of_memory("send_implied_dirs");
+ relname_list = new0(item_list);
memcpy(F_DIR_RELNAMES_P(lastpath_struct), &relname_list, sizeof relname_list);
}
rnpp = EXPAND_ITEM_LIST(relname_list, relnamecache *, 32);
- if (!(*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len)))
- out_of_memory("send_implied_dirs");
+ *rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len);
(*rnpp)->name_type = name_type;
strlcpy((*rnpp)->fname, limit+1, len + 1);
}
if (need_unsorted_flist) {
- if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
- out_of_memory("send_extra_file_list");
+ flist->sorted = new_array(struct file_struct *, flist->used);
memcpy(flist->sorted, flist->files,
flist->used * sizeof (struct file_struct*));
} else
* recursion mode, the sender marks duplicate dirs so that it can
* send them together in a single file-list. */
if (need_unsorted_flist) {
- if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
- out_of_memory("send_file_list");
+ flist->sorted = new_array(struct file_struct *, flist->used);
memcpy(flist->sorted, flist->files,
flist->used * sizeof (struct file_struct*));
} else
* order and for calling flist_find()). We keep the "files"
* list unsorted for our exchange of index numbers with the
* other side (since their names may not sort the same). */
- if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
- out_of_memory("recv_file_list");
+ flist->sorted = new_array(struct file_struct *, flist->used);
memcpy(flist->sorted, flist->files,
flist->used * sizeof (struct file_struct*));
if (inc_recurse && dir_flist->used > dstart) {
{
struct file_list *flist;
- if (!(flist = new0(struct file_list)))
- out_of_memory(msg);
+ flist = new0(struct file_list);
if (flags & FLIST_TEMP) {
if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0,
if (delete_during == 2) {
deldelay_size = BIGPATHBUFLEN * 4;
deldelay_buf = new_array(char, deldelay_size);
- if (!deldelay_buf)
- out_of_memory("delete-delay");
}
info_levels[INFO_FLIST] = info_levels[INFO_PROGRESS] = 0;
#include "rsync.h"
-int
-main(UNUSED(int argc), UNUSED(char *argv[]))
+ int main(UNUSED(int argc), UNUSED(char *argv[]))
{
int n, i;
gid_t *list;
size *= 2;
}
- if (!(tbl = new(struct hashtable))
- || !(tbl->nodes = new_array0(char, size * node_size)))
- out_of_memory("hashtable_create");
+ tbl = new(struct hashtable);
+ tbl->nodes = new_array0(char, size * node_size);
tbl->size = size;
tbl->entries = 0;
tbl->node_size = node_size;
int size = tbl->size * 2;
int i;
- if (!(tbl->nodes = new_array0(char, size * tbl->node_size)))
- out_of_memory("hashtable_node");
+ tbl->nodes = new_array0(char, size * tbl->node_size);
tbl->size = size;
tbl->entries = 0;
if (inc_recurse) {
node = hashtable_find(prior_hlinks, gnum, data_when_new);
if (node->data == data_when_new) {
- if (!(node->data = new_array0(char, 5)))
- out_of_memory("match_gnums");
+ node->data = new_array0(char, 5);
assert(gnum >= hlink_flist->ndx_start);
file->flags |= FLAG_HLINK_FIRST;
prev = -1;
int i, ndx_count = 0;
int32 *ndx_list;
- if (!(ndx_list = new_array(int32, flist->used)))
- out_of_memory("match_hard_links");
+ ndx_list = new_array(int32, flist->used);
for (i = 0; i < flist->used; i++) {
if (F_IS_HLINKED(flist->sorted[i]))
exit_cleanup(RERR_MESSAGEIO);
}
free(node->data);
- if (!(node->data = strdup(our_name)))
- out_of_memory("finish_hard_link");
+ node->data = strdup(our_name);
}
}
static inline void
alloc_xbuf(xbuf *xb, size_t sz)
{
- if (!(xb->buf = new_array(char, sz)))
- out_of_memory("alloc_xbuf");
+ xb->buf = new_array(char, sz);
xb->size = sz;
xb->len = xb->pos = 0;
}
realloc_xbuf(xbuf *xb, size_t sz)
{
char *bf = realloc_array(xb->buf, char, sz);
- if (!bf)
- out_of_memory("realloc_xbuf");
xb->buf = bf;
xb->size = sz;
}
}
#endif
}
+
+static inline char *my_strdup(const char *str, const char *file, int line)
+{
+ int len = strlen(str)+1;
+ char *buf = _my_alloc(do_malloc, len, 1, file, line);
+ memcpy(buf, str, len);
+ return buf;
+}
rl_flags |= (protect_args && ic_recv != (iconv_t)-1 ? RL_CONVERT : 0);
#endif
- if (!(argv = new_array(char *, maxargs)))
- out_of_memory("read_args");
+ argv = new_array(char *, maxargs);
if (mod_name && !protect_args)
argv[argc++] = "rsyncd";
if (argc == maxargs-1) {
maxargs += MAX_ARGS;
- if (!(argv = realloc_array(argv, char *, maxargs)))
- out_of_memory("read_args");
+ argv = realloc_array(argv, char *, maxargs);
}
if (dot_pos) {
int len = strlen(buf);
if (request_len)
request_p[0][request_len++] = ' ';
- if (!(*request_p = realloc_array(*request_p, char, request_len + len + 1)))
- out_of_memory("read_args");
+ *request_p = realloc_array(*request_p, char, request_len + len + 1);
memcpy(*request_p + request_len, buf, len + 1);
request_len += len;
}
else
glob_expand(buf, &argv, &argc, &maxargs);
} else {
- if (!(p = strdup(buf)))
- out_of_memory("read_args");
+ p = strdup(buf);
argv[argc++] = p;
if (*p == '.' && p[1] == '\0')
dot_pos = argc;
#include "rsync.h"
#include "itypes.h"
+#include "ifuncs.h"
#include "default-dont-compress.h"
extern item_list dparam_list;
return str;
bufsize = strlen(str) + 2048;
- if ((buf = new_array(char, bufsize+1)) == NULL) /* +1 for trailing '\0' */
- out_of_memory("expand_vars");
+ buf = new_array(char, bufsize+1); /* +1 for trailing '\0' */
for (t = buf, f = str; bufsize && *f; ) {
if (*f == '%' && *++f != '%') {
* the start, so any lost memory is inconsequential. */
static inline void string_set(char **s, const char *v)
{
- if (!v)
- *s = NULL;
- else if (!(*s = strdup(v)))
- out_of_memory("string_set");
+ *s = v ? strdup(v) : NULL;
}
/* Copy local_vars into a new section. No need to strdup since we don't free. */
#include "rsync.h"
#include "inums.h"
+#include "ifuncs.h"
#include "io.h"
#if defined CONFIG_LOCALE && defined HAVE_LOCALE_H
#include <locale.h>
if (!cmd)
cmd = RSYNC_RSH;
cmd = need_to_free = strdup(cmd);
- if (!cmd)
- goto oom;
for (t = f = cmd; *f; f++) {
if (*f == ' ')
free(need_to_free);
return pid;
-
- oom:
- out_of_memory("do_cmd");
- return 0; /* not reached */
}
/* The receiving side operates in one of two modes:
if (dry_run > 1 && *bdir != '/') {
int len = curr_dir_len + 1 + bd_len + 1;
char *new = new_array(char, len);
- if (!new)
- out_of_memory("check_alt_basis_dirs");
if (slash && strncmp(bdir, "../", 3) == 0) {
/* We want to remove only one leading "../" prefix for
* the directory we couldn't create in dry-run mode:
return MAX(exit_code, exit_code2);
}
-static int copy_argv(char *argv[])
+static void dup_argv(char *argv[])
{
int i;
- for (i = 0; argv[i]; i++) {
- if (!(argv[i] = strdup(argv[i]))) {
- rprintf (FERROR, "out of memory at %s(%d)\n",
- __FILE__, __LINE__);
- return RERR_MALLOC;
- }
- }
-
- return 0;
+ for (i = 0; argv[i]; i++)
+ argv[i] = strdup(argv[i]);
}
/* Don't clobber argv[] so that ps(1) can still show the right
* command line. */
- if ((ret = copy_argv(argv)) != 0)
- return ret;
+ dup_argv(argv);
if (!read_batch) { /* for read_batch, NO source is specified */
char *path = check_for_hostspec(argv[0], &shell_machine, &rsync_port);
if (hash_table)
free(hash_table);
hash_table = new_array(int32, tablesize);
- if (!hash_table)
- out_of_memory("build_hash_table");
alloc_size = tablesize;
}
#include "rsync.h"
#include "itypes.h"
+#include "ifuncs.h"
#include "latest-year.h"
#include <popt.h>
int alt_dest_type = 0;
int basis_dir_cnt = 0;
+#define DEFAULT_MAX_ALLOC (1024L * 1024 * 1024)
+size_t max_alloc = DEFAULT_MAX_ALLOC;
+char *max_alloc_arg;
+
static int version_opt_cnt = 0;
static int remote_option_alloc = 0;
int remote_option_cnt = 0;
return NULL;
len++;
- if (!(buf = new_array(char, len)))
- out_of_memory("make_output_option");
+ buf = new_array(char, len);
pos = 0;
if (skipped || max < 5)
{"ignore-existing", 0, POPT_ARG_NONE, &ignore_existing, 0, 0, 0 },
{"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 },
{"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 },
+ {"max-alloc", 0, POPT_ARG_STRING, &max_alloc_arg, 0, 0, 0 },
{"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 },
{"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 },
{"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 },
return i;
}
-
-static OFF_T parse_size_arg(char **size_arg, char def_suf)
+/* If the size_arg is an invalid string or the value is < min_value, an error
+ * is put into err_buf & the return is -1. Note that this parser does NOT
+ * support negative numbers, so a min_value < 0 doesn't make any sense. */
+static ssize_t parse_size_arg(char *size_arg, char def_suf, const char *opt_name, ssize_t min_value)
{
- int reps, mult, make_compatible = 0;
- const char *arg;
- OFF_T size = 1;
+ int reps, mult;
+ const char *arg, *err = "invalid";
+ ssize_t size = 1;
- for (arg = *size_arg; isDigit(arg); arg++) {}
+ for (arg = size_arg; isDigit(arg); arg++) {}
if (*arg == '.')
for (arg++; isDigit(arg); arg++) {}
switch (*arg && *arg != '+' && *arg != '-' ? *arg++ : def_suf) {
case 'g': case 'G':
reps = 3;
break;
+ case 't': case 'T':
+ reps = 4;
+ break;
+ case 'p': case 'P':
+ reps = 5;
+ break;
default:
- return -1;
+ goto failure;
}
if (*arg == 'b' || *arg == 'B')
- mult = 1000, make_compatible = 1, arg++;
+ mult = 1000, arg++;
else if (!*arg || *arg == '+' || *arg == '-')
mult = 1024;
else if (strncasecmp(arg, "ib", 2) == 0)
mult = 1024, arg += 2;
else
- return -1;
+ goto failure;
while (reps--)
size *= mult;
- size *= atof(*size_arg);
+ size *= atof(size_arg);
if ((*arg == '+' || *arg == '-') && arg[1] == '1')
- size += atoi(arg), make_compatible = 1, arg += 2;
+ size += atoi(arg), arg += 2;
if (*arg)
- return -1;
- if (size > 0 && make_compatible && def_suf == 'b') {
- /* We convert this manually because we may need %lld precision,
- * and that's not a portable sprintf() escape. */
- char buf[128], *s = buf + sizeof buf - 1;
- OFF_T num = size;
- *s = '\0';
- while (num) {
- *--s = (char)(num % 10) + '0';
- num /= 10;
- }
- if (!(*size_arg = strdup(s)))
- out_of_memory("parse_size_arg");
+ goto failure;
+ if (size < min_value) {
+ err = size < 0 ? "too big" : "too small";
+ goto failure;
}
return size;
-}
+failure:
+ snprintf(err_buf, sizeof err_buf, "--%s value is %s: %s\n", opt_name, err, size_arg);
+ return -1;
+}
static void create_refuse_error(int which)
{
if (daemon_filter_list.head) {
int rej;
char *cp = strdup(arg);
- if (!cp)
- out_of_memory("parse_arguments");
if (!*cp)
rej = 1;
else {
remote_option_alloc += 16;
remote_options = realloc_array(remote_options,
const char *, remote_option_alloc);
- if (!remote_options)
- out_of_memory("parse_arguments");
if (!remote_option_cnt)
remote_options[0] = "ARG0";
}
break;
case OPT_MAX_SIZE:
- if ((max_size = parse_size_arg(&max_size_arg, 'b')) < 0) {
- snprintf(err_buf, sizeof err_buf,
- "--max-size value is invalid: %s\n",
- max_size_arg);
+ if ((max_size = parse_size_arg(max_size_arg, 'b', "max-size", 0)) < 0)
return 0;
- }
+ max_size_arg = num_to_byte_string(max_size);
break;
case OPT_MIN_SIZE:
- if ((min_size = parse_size_arg(&min_size_arg, 'b')) < 0) {
- snprintf(err_buf, sizeof err_buf,
- "--min-size value is invalid: %s\n",
- min_size_arg);
+ if ((min_size = parse_size_arg(min_size_arg, 'b', "min-size", 0)) < 0)
return 0;
- }
+ min_size_arg = num_to_byte_string(min_size);
break;
- case OPT_BWLIMIT:
- {
- OFF_T limit = parse_size_arg(&bwlimit_arg, 'K');
- if (limit < 0) {
- snprintf(err_buf, sizeof err_buf,
- "--bwlimit value is invalid: %s\n", bwlimit_arg);
- return 0;
- }
- bwlimit = (limit + 512) / 1024;
- if (limit && !bwlimit) {
- snprintf(err_buf, sizeof err_buf,
- "--bwlimit value is too small: %s\n", bwlimit_arg);
- return 0;
- }
- }
+ case OPT_BWLIMIT: {
+ ssize_t size = parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512);
+ if (size < 0)
+ return 0;
+ bwlimit_arg = num_to_byte_string(size);
+ bwlimit = (size + 512) / 1024;
break;
+ }
case OPT_APPEND:
if (am_server)
exit_cleanup(0);
}
+ if (!max_alloc_arg) {
+ max_alloc_arg = getenv("RSYNC_MAX_ALLOC");
+ if (max_alloc_arg && !*max_alloc_arg)
+ max_alloc_arg = NULL;
+ }
+ if (max_alloc_arg) {
+ ssize_t size = parse_size_arg(max_alloc_arg, 'B', "max-alloc", 1024*1024);
+ if (size < 0)
+ return 0;
+ max_alloc = size;
+ }
+
if (protect_args < 0) {
if (am_server)
protect_args = 0;
}
}
+ if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) {
+ args[ac++] = "--max-alloc";
+ args[ac++] = max_alloc_arg;
+ }
+
/* --delete-missing-args needs the cooperation of both sides, but
* the sender can handle --ignore-missing-args by itself. */
if (missing_args == 2)
{
bSize += BUFR_INC;
bufr = realloc_array( bufr, char, bSize );
- if( NULL == bufr )
- {
- rprintf(FLOG, "%s Memory re-allocation failure.", func);
- return( False );
- }
}
/* Handle a single character. */
{
bSize += BUFR_INC;
bufr = realloc_array( bufr, char, bSize );
- if( NULL == bufr )
- {
- rprintf(FLOG, "%s Memory re-allocation failure.", func) ;
- return( False );
- }
}
switch( c )
{
bSize += BUFR_INC;
bufr = realloc_array( bufr, char, bSize );
- if( NULL == bufr )
- {
- rprintf(FLOG, "%s Memory re-allocation failure.", func) ;
- return( False );
- }
}
switch( c )
{ /* allocate one, then parse, */
bSize = BUFR_INC; /* then free. */
bufr = new_array( char, bSize );
- if( NULL == bufr )
- {
- rprintf(FLOG, "%s memory allocation failure.\n", func);
- fclose(InFile);
- return( False );
- }
result = Parse( InFile, sfunc, pfunc );
free( bufr );
bufr = NULL;
--max-delete=NUM don't delete more than NUM files
--max-size=SIZE don't transfer any file larger than SIZE
--min-size=SIZE don't transfer any file smaller than SIZE
+--max-alloc=SIZE change a limit relating to memory alloc
--partial keep partially transferred files
--partial-dir=DIR put a partially transferred file into DIR
--delay-updates put all updated files into place at end
helps to avoid overpopulating the protocol data with extra message data.
The option does not affect the remote side of a transfer without using
- `--remote-option` -- e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`.
+ `--remote-option`, e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`.
Also keep in mind that connecting to a normal (non-remote-shell) daemon
does not have a stderr channel to send messages back to the client side, so
data that goes into the file-lists, and thus it doesn't affect deletions.
It just limits the files that the receiver requests to be transferred.
- The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or
- "MiB") is a mebibyte (1024\*1024), and "G" (or "GiB") is a gibibyte
- (1024\*1024\*1024). If you want the multiplier to be 1000 instead of 1024,
- use "KB", "MB", or "GB". (Note: lower-case is also accepted for all
- values.) Finally, if the suffix ends in either "+1" or "-1", the value will
- be offset by one byte in the indicated direction.
+ The suffix letters are (in upper/lower-case): `B`, `K`, `G`, `T`, and `P`
+ for bytes, kilobytes/kibibytes, megabytes/mebibytes, gigabytes/gibibytes,
+ terabytes/tebibytes, and petabytes/pebibytes. If you use a single-char
+ suffix or add-on "ib" to it (e.g. "G" or "GiB") then you get units that are
+ multiples of 1024. If you use a two-letter suffix that ends with a "B"
+ (e.g. "kb") then you get units that are multiples of 1000.
+
+ Finally, if the value ends with either "+1" or "-1", it will be offset by
+ one byte in the indicated direction. The largest possible value is
+ `8192P-1`.
Examples: `--max-size=1.5mb-1` is 1499999 bytes, and `--max-size=2g+1` is
2147483649 bytes.
Note that rsync versions prior to 3.1.0 did not allow `--min-size=0`.
+0. `--max-alloc=SIZE`
+
+ By default rsync limits an individual malloc/realloc to about 1GB in size.
+ For most people this limit works just fine and prevents a code issue
+ causing rsync to request massive amounts of memory. However, if you have
+ many millions of files in a transfer, a huge amount of server memory, and
+ you don't want to split up your transfer into multiple parts, you can
+ increase the per-allocation limit to something larger and rsync will
+ consume more memory.
+
+ Keep in mind that this is not a limit on the total size of allocated
+ memory. It is a sanity-check value for individual allocations.
+
+ See the `--max-size` option for a description of how SIZE can be specified.
+ The default suffix if none is given is bytes.
+
+ You can set a default value using the environment variable RSYNC_MAX_ALLOC
+ using the same SIZE values as supported by this option. If the remote
+ rsync doesn't understand the `--max-alloc` option, you can override the
+ setting by specifying `--max-alloc=1g` (because rsync will not send the
+ option to the remote side when the value is the default).
+
0. `--block-size=SIZE`, `-B`
This forces the block size used in rsync's delta-transfer algorithm to a
/* handler for null strings in printf format */
#define NS(s) ((s)?(s):"<NULL>")
+extern char *do_malloc;
+
/* Convenient wrappers for malloc and realloc. Use them. */
-#define new(type) ((type*)malloc(sizeof (type)))
-#define new0(type) ((type*)calloc(1, sizeof (type)))
-#define new_array(type, num) ((type*)_new_array((num), sizeof (type), 0))
-#define new_array0(type, num) ((type*)_new_array((num), sizeof (type), 1))
-#define realloc_array(ptr, type, num) ((type*)_realloc_array((ptr), (num), sizeof (type)))
+#define new(type) ((type*)_my_alloc(do_malloc, 1, sizeof (type), __FILE__, __LINE__))
+#define new0(type) ((type*)_my_alloc(NULL, 1, sizeof (type), __FILE__, __LINE__))
+#define realloc_buf(ptr, num) _my_alloc((ptr), (num), 1, __FILE__, __LINE__)
+
+#define new_array(type, num) ((type*)_my_alloc(do_malloc, (num), sizeof (type), __FILE__, __LINE__))
+#define new_array0(type, num) ((type*)_my_alloc(NULL, (num), sizeof (type), __FILE__, __LINE__))
+#define realloc_array(ptr, type, num) ((type*)_my_alloc((ptr), (num), sizeof (type), __FILE__, __LINE__))
+
+#define strdup(s) my_strdup(s, __FILE__, __LINE__)
/* use magic gcc attributes to catch format errors */
void rprintf(enum logcode , const char *, ...)
**/
static struct sum_struct *receive_sums(int f)
{
- struct sum_struct *s;
- int32 i;
+ struct sum_struct *s = new(struct sum_struct);
int lull_mod = protocol_version >= 31 ? 0 : allowed_lull * 5;
OFF_T offset = 0;
-
- if (!(s = new(struct sum_struct)))
- out_of_memory("receive_sums");
+ int32 i;
read_sum_head(f, s);
if (s->count == 0)
return(s);
- if (!(s->sums = new_array(struct sum_buf, s->count)))
- out_of_memory("receive_sums");
+ s->sums = new_array(struct sum_buf, s->count);
for (i = 0; i < s->count; i++) {
s->sums[i].sum1 = read_int(f);
#include "rsync.h"
#include "itypes.h"
+#include "ifuncs.h"
#ifdef HAVE_NETINET_IN_SYSTM_H
#include <netinet/in_systm.h>
#endif
for (res = res0, addr_cnt = 0; res; res = res->ai_next, addr_cnt++) {}
errnos = new_array0(int, addr_cnt);
- if (!errnos)
- out_of_memory("open_socket_out");
s = -1;
/* Try to connect to all addresses for this machine until we get
len += hlen;
}
f = prog;
- if (!(prog = new_array(char, len)))
- out_of_memory("open_socket_out_wrapped");
+ prog = new_array(char, len);
for (t = prog; *f; f++) {
if (*f == '%') {
switch (*++f) {
socks = new_array(int, maxs + 1);
errmsgs = new_array(char *, maxs);
- if (!socks || !errmsgs)
- out_of_memory("open_socket_in");
/* We may not be able to create the socket, if for example the
* machine knows about IPv6 in the C library, but not in the
options = strdup(options);
- if (!options)
- out_of_memory("set_socket_options");
-
for (tok = strtok(options, " \t,"); tok; tok = strtok(NULL," \t,")) {
int ret=0,i;
int value = 1;
int preserve_perms = 0;
int preserve_executability = 0;
int open_noatime = 0;
+size_t max_alloc = 1024*1024*1024;
char *partial_dir;
char *module_dir;
filter_rule_list daemon_filter_list;
if (node->letter > ltr)
break;
}
- if (!(newnode = new(struct suffix_tree)))
- out_of_memory("add_suffix");
+ newnode = new(struct suffix_tree);
newnode->sibling = node;
newnode->child = NULL;
newnode->letter = ltr;
char *buf, *t;
const char *f = str;
- if (!(buf = new_array(char, strlen(f) + 1)))
- out_of_memory("add_nocompress_suffixes");
+ buf = new_array(char, strlen(f) + 1);
while (*f) {
if (*f == '/') {
else
f = lp_dont_compress(module_id);
- if (!(match_list = t = new_array(char, strlen(f) + 2)))
- out_of_memory("set_compression");
+ match_list = t = new_array(char, strlen(f) + 2);
per_file_default_level = do_compression_level;
static char *buf;
int32 n;
- if (!buf) {
+ if (!buf)
buf = new_array(char, CHUNK_SIZE);
- if (!buf)
- out_of_memory("simple_recv_token");
- }
if (residue == 0) {
int32 i = read_int(f);
rprintf(FERROR, "compression init failed\n");
exit_cleanup(RERR_PROTOCOL);
}
- if ((obuf = new_array(char, OBUF_SIZE)) == NULL)
- out_of_memory("send_deflated_token");
+ obuf = new_array(char, OBUF_SIZE);
init_done = 1;
} else
deflateReset(&tx_strm);
rprintf(FERROR, "inflate init failed\n");
exit_cleanup(RERR_PROTOCOL);
}
- if (!(cbuf = new_array(char, MAX_DATA_COUNT))
- || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE))))
- out_of_memory("recv_deflated_token");
+ cbuf = new_array(char, MAX_DATA_COUNT);
+ dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE));
init_done = 1;
} else {
inflateReset(&rx_strm);
}
obuf = new_array(char, OBUF_SIZE);
- if (!obuf)
- out_of_memory("send_deflated_token");
ZSTD_CCtx_setParameter(zstd_cctx, ZSTD_c_compressionLevel, do_compression_level);
zstd_out_buff.dst = obuf + 2;
out_buffer_size = ZSTD_DStreamOutSize() * 2;
cbuf = new_array(char, MAX_DATA_COUNT);
dbuf = new_array(char, out_buffer_size);
- if (!cbuf || !dbuf)
- out_of_memory("recv_zstd_token");
zstd_in_buff.src = cbuf;
zstd_out_buff.dst = dbuf;
if (last_token == -1) {
if (!init_done) {
- if ((obuf = new_array(char, size)) == NULL)
- out_of_memory("send_compressed_token");
+ obuf = new_array(char, size);
init_done = 1;
}
last_run_end = 0;
switch (recv_state) {
case r_init:
if (!init_done) {
- if (!(cbuf = new_array(char, MAX_DATA_COUNT))
- || !(dbuf = new_array(char, size)))
- out_of_memory("recv_compressed_token");
+ cbuf = new_array(char, MAX_DATA_COUNT);
+ dbuf = new_array(char, size);
init_done = 1;
}
recv_state = r_idle;
id_t id2, uint16 flags)
{
struct idlist *node = new(struct idlist);
- if (!node)
- out_of_memory("add_to_list");
node->next = *root;
node->u = noiu;
node->id = id;
if ((ngroups = getgroups(0, NULL)) < 0)
ngroups = 0;
gidset = new_array(GETGROUPS_T, ngroups+1);
- if (!gidset)
- out_of_memory("is_in_group");
if (ngroups > 0)
ngroups = getgroups(ngroups, gidset);
/* The default gid might not be in the list on some systems. */
if (DEBUG_GTE(OWN, 2)) {
int pos;
char *gidbuf = new_array(char, ngroups*21+32);
- if (!gidbuf)
- out_of_memory("is_in_group");
pos = snprintf(gidbuf, 32, "process has %d gid%s: ", ngroups, ngroups == 1? "" : "s");
for (n = 0; n < ngroups; n++) {
pos += snprintf(gidbuf+pos, 21, " %d", (int)gidset[n]);
struct idlist *node;
int len = read_byte(f);
char *name = new_array(char, len+1);
- if (!name)
- out_of_memory("recv_user_name");
read_sbuf(f, name, len);
if (numeric_ids < 0) {
free(name);
struct idlist *node;
int len = read_byte(f);
char *name = new_array(char, len+1);
- if (!name)
- out_of_memory("recv_group_name");
read_sbuf(f, name, len);
if (numeric_ids < 0) {
free(name);
}
#define ENSURE_MEMSPACE(buf, type, sz, req) \
- if ((req) > sz && !(buf = realloc_array(buf, type, sz = MAX(sz * 2, req)))) \
- out_of_memory("glob_expand")
+ do { if ((req) > sz) buf = realloc_array(buf, type, sz = MAX(sz * 2, req)); } while(0)
static inline void call_glob_match(const char *name, int len, int from_glob,
char *arg, int abpos, int fbpos);
glob_match(arg, abpos, fbpos);
} else {
ENSURE_MEMSPACE(glob.argv, char *, glob.maxargs, glob.argc + 1);
- if (!(glob.argv[glob.argc++] = strdup(glob.arg_buf)))
- out_of_memory("glob_match");
+ glob.argv[glob.argc++] = strdup(glob.arg_buf);
}
}
s = sanitize_path(NULL, arg, "", 0, SP_KEEP_DOT_DIRS);
else {
s = strdup(arg);
- if (!s)
- out_of_memory("glob_expand");
clean_fname(s, CFN_KEEP_DOT_DIRS | CFN_KEEP_TRAILING_SLASH | CFN_COLLAPSE_DOT_DOT_DIRS);
}
return;
}
- if (!(arg = strdup(arg)))
- out_of_memory("glob_expand_module");
+ arg = strdup(arg);
if (asprintf(&base," %s/", base1) < 0)
out_of_memory("glob_expand_module");
depth = 0;
p++;
}
- if (dest) {
- if (rlen + plen + 1 >= MAXPATHLEN)
- return NULL;
- } else if (!(dest = new_array(char, MAX(rlen + plen + 1, 2))))
- out_of_memory("sanitize_path");
+ if (!dest)
+ dest = new_array(char, MAX(rlen + plen + 1, 2));
+ else if (rlen + plen + 1 >= MAXPATHLEN)
+ return NULL;
if (rlen) { /* only true if p previously started with a slash */
memcpy(dest, rootdir, rlen);
if (rlen > 1) /* a rootdir of len 1 is "/", so this avoids a 2nd slash */
return NULL;
curr_dir[curr_dir_len] = '/';
memcpy(curr_dir + curr_dir_len + 1, path, len + 1);
- if (!(path = strdup(curr_dir)))
- out_of_memory("normalize_path");
+ path = strdup(curr_dir);
curr_dir[curr_dir_len] = '\0';
- } else if (force_newbuf) {
- if (!(path = strdup(path)))
- out_of_memory("normalize_path");
- }
+ } else if (force_newbuf)
+ path = strdup(path);
len = clean_fname(path, CFN_COLLAPSE_DOT_DOT_DIRS | CFN_DROP_TRAILING_DOT_DIR);
struct bitbag *bb = new(struct bitbag);
bb->slot_cnt = (max_ndx + BB_PER_SLOT_BITS - 1) / BB_PER_SLOT_BITS;
- if (!(bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*))))
- out_of_memory("bitbag_create");
+ bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*));
return bb;
}
{
struct flist_ndx_item *item;
- if (!(item = new(struct flist_ndx_item)))
- out_of_memory("flist_ndx_push");
+ item = new(struct flist_ndx_item);
item->next = NULL;
item->ndx = ndx;
if (lp->tail)
new_size = 1;
if (new_size <= lp->malloced)
overflow_exit("expand_item_list");
- /* Using _realloc_array() lets us pass the size, not a type. */
- new_ptr = _realloc_array(lp->items, new_size, item_size);
+ new_ptr = realloc_buf(lp->items, new_size * item_size);
if (DEBUG_GTE(FLIST, 3)) {
rprintf(FINFO, "[%s] expand %s to %s bytes, did%s move\n",
who_am_i(), desc, big_num(new_size * item_size),
#include "itypes.h"
#include "inums.h"
+extern size_t max_alloc;
+
+char *do_malloc = "42";
+
/**
* Sleep for a specified number of milliseconds.
*
return True;
}
-#define MALLOC_MAX 0x40000000
-
-void *_new_array(size_t num, size_t size, int use_calloc)
+/* We convert a num manually because need %lld precision, and that's not a portable sprintf() escape. */
+char *num_to_byte_string(ssize_t num)
{
- if (num >= MALLOC_MAX/size)
- return NULL;
- return use_calloc ? calloc(num, size) : malloc(num * size);
+ char buf[128], *s = buf + sizeof buf - 1;
+
+ *s = '\0';
+ while (num) {
+ *--s = (char)(num % 10) + '0';
+ num /= 10;
+ }
+ return strdup(s);
}
-void *_realloc_array(void *ptr, size_t num, size_t size)
+void *_my_alloc(void *ptr, size_t num, size_t size, const char *file, int line)
{
- if (num >= MALLOC_MAX/size)
- return NULL;
+ if (num >= max_alloc/size) {
+ if (!file)
+ return NULL;
+ rprintf(FERROR, "[%s] exceeded --max-alloc=%s setting (file=%s, line=%d)\n",
+ who_am_i(), num_to_byte_string(max_alloc), file, line);
+ exit_cleanup(RERR_MALLOC);
+ }
if (!ptr)
- return malloc(size * num);
- return realloc(ptr, size * num);
+ ptr = calloc(num, size);
+ else if (ptr == do_malloc)
+ ptr = malloc(num * size);
+ else
+ ptr = realloc(ptr, num * size);
+ if (!ptr && file) {
+ rprintf(FERROR, "[%s] out of memory (file=%s, line=%d)\n", who_am_i(), file, line);
+ exit_cleanup(RERR_MALLOC);
+ }
+ return ptr;
}
const char *sum_as_hex(int csum_type, const char *sum, int flist_csum)
if (!namebuf) {
namebuf_len = 1024;
namebuf = new_array(char, namebuf_len);
- if (!namebuf)
- out_of_memory("get_xattr_names");
}
while (1) {
free(namebuf);
namebuf_len = list_len + 1024;
namebuf = new_array(char, namebuf_len);
- if (!namebuf)
- out_of_memory("get_xattr_names");
}
return list_len;
extra_len = 1; /* request non-zero amount of memory */
if (datum_len + extra_len < datum_len)
overflow_exit("get_xattr_data");
- if (!(ptr = new_array(char, datum_len + extra_len)))
- out_of_memory("get_xattr_data");
+ ptr = new_array(char, datum_len + extra_len);
if (datum_len) {
size_t len = sys_lgetxattr(fname, name, ptr, datum_len);
sum_update(ptr, datum_len);
free(ptr);
- if (!(ptr = new_array(char, name_offset + name_len)))
- out_of_memory("rsync_xal_get");
+ ptr = new_array(char, name_offset + name_len);
*ptr = XSTATE_ABBREV;
sum_end(ptr + 1);
} else
out_of_memory("rsync_xal_h hashtable_create()");
new_ref = new0(rsync_xa_list_ref);
- if (new_ref == NULL)
- out_of_memory("new0(rsync_xa_list_ref)");
new_ref->ndx = ndx;
node = hashtable_find(rsync_xal_h, new_list->key, new_ref);
if (rxa->name_len + rxa->datum_len < rxa->name_len)
overflow_exit("recv_xattr_request");
rxa->datum = new_array(char, rxa->datum_len + rxa->name_len);
- if (!rxa->datum)
- out_of_memory("recv_xattr_request");
name = rxa->datum + rxa->datum_len;
memcpy(name, rxa->name, rxa->name_len);
rxa->name = name;
|| (dget_len + extra_len + name_len < dget_len + extra_len))
overflow_exit("receive_xattr");
ptr = new_array(char, dget_len + extra_len + name_len);
- if (!ptr)
- out_of_memory("receive_xattr");
name = ptr + dget_len + extra_len;
read_buf(f, name, name_len);
if (name_len < 1 || name[name_len-1] != '\0') {
git.samba.org / rsync.git / commitdiff