2 Unix SMB/CIFS implementation.
4 trivial database library
6 Copyright (C) Andrew Tridgell 1999-2005
7 Copyright (C) Paul `Rusty' Russell 2000
8 Copyright (C) Jeremy Allison 2000-2003
10 ** NOTE! The following LGPL license applies to the tdb
11 ** library. This does NOT imply that all of Samba is released
14 This library is free software; you can redistribute it and/or
15 modify it under the terms of the GNU Lesser General Public
16 License as published by the Free Software Foundation; either
17 version 3 of the License, or (at your option) any later version.
19 This library is distributed in the hope that it will be useful,
20 but WITHOUT ANY WARRANTY; without even the implied warranty of
21 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22 Lesser General Public License for more details.
24 You should have received a copy of the GNU Lesser General Public
25 License along with this library; if not, see <http://www.gnu.org/licenses/>.
29 #include "tdb_private.h"
32 * We prepend the mutex area, so fixup offsets. See mutex.c for details.
33 * tdb->hdr_ofs is 0 or header.mutex_size.
35 * Note: that we only have the 4GB limit of tdb_off_t for
36 * tdb->map_size. The file size on disk can be 4GB + tdb->hdr_ofs!
39 static bool tdb_adjust_offset(struct tdb_context *tdb, off_t *off)
41 off_t tmp = tdb->hdr_ofs + *off;
43 if ((tmp < tdb->hdr_ofs) || (tmp < *off)) {
52 static ssize_t tdb_pwrite(struct tdb_context *tdb, const void *buf,
53 size_t count, off_t offset)
57 if (!tdb_adjust_offset(tdb, &offset)) {
62 ret = pwrite(tdb->fd, buf, count, offset);
63 } while ((ret == -1) && (errno == EINTR));
68 static ssize_t tdb_pread(struct tdb_context *tdb, void *buf,
69 size_t count, off_t offset)
73 if (!tdb_adjust_offset(tdb, &offset)) {
78 ret = pread(tdb->fd, buf, count, offset);
79 } while ((ret == -1) && (errno == EINTR));
84 static int tdb_ftruncate(struct tdb_context *tdb, off_t length)
88 if (!tdb_adjust_offset(tdb, &length)) {
93 ret = ftruncate(tdb->fd, length);
94 } while ((ret == -1) && (errno == EINTR));
99 static int tdb_fstat(struct tdb_context *tdb, struct stat *buf)
103 ret = fstat(tdb->fd, buf);
108 if (buf->st_size < tdb->hdr_ofs) {
112 buf->st_size -= tdb->hdr_ofs;
117 /* check for an out of bounds access - if it is out of bounds then
118 see if the database has been expanded by someone else and expand
121 static int tdb_oob(struct tdb_context *tdb, tdb_off_t off, tdb_len_t len,
125 if (len + off < len) {
127 /* Ensure ecode is set for log fn. */
128 tdb->ecode = TDB_ERR_IO;
129 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob off %u len %u wrap\n",
135 if (off + len <= tdb->map_size)
137 if (tdb->flags & TDB_INTERNAL) {
139 /* Ensure ecode is set for log fn. */
140 tdb->ecode = TDB_ERR_IO;
141 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob len %u beyond internal malloc size %u\n",
142 (int)(off + len), (int)tdb->map_size));
147 if (tdb_fstat(tdb, &st) == -1) {
148 tdb->ecode = TDB_ERR_IO;
152 /* Beware >4G files! */
153 if ((tdb_off_t)st.st_size != st.st_size) {
154 /* Ensure ecode is set for log fn. */
155 tdb->ecode = TDB_ERR_IO;
156 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_oob len %llu too large!\n",
157 (long long)st.st_size));
161 /* Unmap, update size, remap. We do this unconditionally, to handle
162 * the unusual case where the db is truncated.
164 * This can happen to a child using tdb_reopen_all(true) on a
165 * TDB_CLEAR_IF_FIRST tdb whose parent crashes: the next
166 * opener will truncate the database. */
167 if (tdb_munmap(tdb) == -1) {
168 tdb->ecode = TDB_ERR_IO;
171 tdb->map_size = st.st_size;
172 if (tdb_mmap(tdb) != 0) {
176 if (st.st_size < (size_t)off + len) {
178 /* Ensure ecode is set for log fn. */
179 tdb->ecode = TDB_ERR_IO;
180 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob len %u beyond eof at %u\n",
181 (int)(off + len), (int)st.st_size));
188 /* write a lump of data at a specified offset */
189 static int tdb_write(struct tdb_context *tdb, tdb_off_t off,
190 const void *buf, tdb_len_t len)
196 if (tdb->read_only || tdb->traverse_read) {
197 tdb->ecode = TDB_ERR_RDONLY;
201 if (tdb->methods->tdb_oob(tdb, off, len, 0) != 0)
205 memcpy(off + (char *)tdb->map_ptr, buf, len);
207 #ifdef HAVE_INCOHERENT_MMAP
208 tdb->ecode = TDB_ERR_IO;
213 written = tdb_pwrite(tdb, buf, len, off);
215 if ((written != (ssize_t)len) && (written != -1)) {
217 tdb->ecode = TDB_ERR_IO;
218 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_write: wrote only "
219 "%zi of %u bytes at %u, trying once more\n",
221 written = tdb_pwrite(tdb, (const char *)buf+written,
222 len-written, off+written);
225 /* Ensure ecode is set for log fn. */
226 tdb->ecode = TDB_ERR_IO;
227 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_write failed at %u "
228 "len=%u (%s)\n", off, len, strerror(errno)));
230 } else if (written != (ssize_t)len) {
231 tdb->ecode = TDB_ERR_IO;
232 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_write: failed to "
233 "write %u bytes at %u in two attempts\n",
242 /* Endian conversion: we only ever deal with 4 byte quantities */
243 void *tdb_convert(void *buf, uint32_t size)
245 uint32_t i, *p = (uint32_t *)buf;
246 for (i = 0; i < size / 4; i++)
247 p[i] = TDB_BYTEREV(p[i]);
252 /* read a lump of data at a specified offset, maybe convert */
253 static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf,
254 tdb_len_t len, int cv)
256 if (tdb->methods->tdb_oob(tdb, off, len, 0) != 0) {
261 memcpy(buf, off + (char *)tdb->map_ptr, len);
263 #ifdef HAVE_INCOHERENT_MMAP
264 tdb->ecode = TDB_ERR_IO;
269 ret = tdb_pread(tdb, buf, len, off);
270 if (ret != (ssize_t)len) {
271 /* Ensure ecode is set for log fn. */
272 tdb->ecode = TDB_ERR_IO;
273 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %u "
274 "len=%u ret=%zi (%s) map_size=%u\n",
275 off, len, ret, strerror(errno),
282 tdb_convert(buf, len);
290 do an unlocked scan of the hash table heads to find the next non-zero head. The value
291 will then be confirmed with the lock held
293 static void tdb_next_hash_chain(struct tdb_context *tdb, uint32_t *chain)
297 for (;h < tdb->hash_size;h++) {
298 if (0 != *(uint32_t *)(TDB_HASH_TOP(h) + (unsigned char *)tdb->map_ptr)) {
304 for (;h < tdb->hash_size;h++) {
305 if (tdb_ofs_read(tdb, TDB_HASH_TOP(h), &off) != 0 || off != 0) {
314 int tdb_munmap(struct tdb_context *tdb)
316 if (tdb->flags & TDB_INTERNAL)
323 ret = munmap(tdb->map_ptr, tdb->map_size);
332 /* If mmap isn't coherent, *everyone* must always mmap. */
333 static bool should_mmap(const struct tdb_context *tdb)
335 #ifdef HAVE_INCOHERENT_MMAP
338 return !(tdb->flags & TDB_NOMMAP);
342 int tdb_mmap(struct tdb_context *tdb)
344 if (tdb->flags & TDB_INTERNAL)
348 if (should_mmap(tdb)) {
349 tdb->map_ptr = mmap(NULL, tdb->map_size,
350 PROT_READ|(tdb->read_only? 0:PROT_WRITE),
351 MAP_SHARED|MAP_FILE, tdb->fd,
355 * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!!
358 if (tdb->map_ptr == MAP_FAILED) {
360 TDB_LOG((tdb, TDB_DEBUG_WARNING, "tdb_mmap failed for size %u (%s)\n",
361 tdb->map_size, strerror(errno)));
362 #ifdef HAVE_INCOHERENT_MMAP
363 tdb->ecode = TDB_ERR_IO;
376 /* expand a file. we prefer to use ftruncate, as that is what posix
377 says to use for mmap expansion */
378 static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t addition)
383 if (tdb->read_only || tdb->traverse_read) {
384 tdb->ecode = TDB_ERR_RDONLY;
388 if (!tdb_add_off_t(size, addition, &new_size)) {
389 tdb->ecode = TDB_ERR_OOM;
390 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write "
391 "overflow detected current size[%u] addition[%u]!\n",
392 (unsigned)size, (unsigned)addition));
397 if (tdb_ftruncate(tdb, new_size) == -1) {
399 ssize_t written = tdb_pwrite(tdb, &b, 1, new_size - 1);
401 /* try once more, potentially revealing errno */
402 written = tdb_pwrite(tdb, &b, 1, new_size - 1);
405 /* again - give up, guessing errno */
409 tdb->ecode = TDB_ERR_OOM;
410 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %u failed (%s)\n",
411 (unsigned)new_size, strerror(errno)));
416 /* now fill the file with something. This ensures that the
417 file isn't sparse, which would be very bad if we ran out of
418 disk. This must be done with write, not via mmap */
419 memset(buf, TDB_PAD_BYTE, sizeof(buf));
421 size_t n = addition>sizeof(buf)?sizeof(buf):addition;
422 ssize_t written = tdb_pwrite(tdb, buf, n, size);
424 /* prevent infinite loops: try _once_ more */
425 written = tdb_pwrite(tdb, buf, n, size);
428 /* give up, trying to provide a useful errno */
429 tdb->ecode = TDB_ERR_OOM;
430 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write "
431 "returned 0 twice: giving up!\n"));
436 tdb->ecode = TDB_ERR_OOM;
437 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of "
438 "%u bytes failed (%s)\n", (int)n,
443 TDB_LOG((tdb, TDB_DEBUG_WARNING, "expand_file: wrote "
444 "only %zu of %zi bytes - retrying\n", written,
458 * We're holding the freelist lock or are inside a
459 * transaction. Cutting the file is safe, the space we
460 * tried to allocate can't have been used anywhere in
464 ret = tdb_ftruncate(tdb, size);
466 TDB_LOG((tdb, TDB_DEBUG_WARNING, "expand_file: "
467 "retruncate to %ju failed\n",
477 /* You need 'size', this tells you how much you should expand by. */
478 tdb_off_t tdb_expand_adjust(tdb_off_t map_size, tdb_off_t size, int page_size)
480 tdb_off_t new_size, top_size, increment;
481 tdb_off_t max_size = UINT32_MAX - map_size;
483 if (size > max_size) {
485 * We can't round up anymore, just give back
486 * what we're asked for.
488 * The caller has to take care of the ENOSPC handling.
493 /* limit size in order to avoid using up huge amounts of memory for
494 * in memory tdbs if an oddball huge record creeps in */
495 if (size > 100 * 1024) {
496 increment = size * 2;
498 increment = size * 100;
500 if (increment < size) {
504 if (!tdb_add_off_t(map_size, increment, &top_size)) {
508 /* always make room for at least top_size more records, and at
509 least 25% more space. if the DB is smaller than 100MiB,
510 otherwise grow it by 10% only. */
511 if (map_size > 100 * 1024 * 1024) {
512 new_size = map_size * 1.10;
514 new_size = map_size * 1.25;
516 if (new_size < map_size) {
520 /* Round the database up to a multiple of the page size */
521 new_size = MAX(top_size, new_size);
523 if (new_size + page_size < new_size) {
524 /* There's a "+" in TDB_ALIGN that might overflow... */
528 return TDB_ALIGN(new_size, page_size) - map_size;
532 * Somewhere in between we went over 4GB. Make one big jump to
533 * exactly 4GB database size.
538 /* expand the database at least size bytes by expanding the underlying
539 file and doing the mmap again if necessary */
540 int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
542 struct tdb_record rec;
546 if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
547 TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));
551 /* must know about any previous expansions by another process */
552 tdb->methods->tdb_oob(tdb, tdb->map_size, 1, 1);
555 * Note: that we don't care about tdb->hdr_ofs != 0 here
557 * The 4GB limitation is just related to tdb->map_size
558 * and the offset calculation in the records.
560 * The file on disk can be up to 4GB + tdb->hdr_ofs
562 size = tdb_expand_adjust(tdb->map_size, size, tdb->page_size);
564 if (!tdb_add_off_t(tdb->map_size, size, &new_size)) {
565 tdb->ecode = TDB_ERR_OOM;
566 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_expand "
567 "overflow detected current map_size[%u] size[%u]!\n",
568 (unsigned)tdb->map_size, (unsigned)size));
572 /* form a new freelist record */
573 offset = tdb->map_size;
574 memset(&rec,'\0',sizeof(rec));
575 rec.rec_len = size - sizeof(rec);
577 if (tdb->flags & TDB_INTERNAL) {
580 new_map_ptr = (char *)realloc(tdb->map_ptr, new_size);
582 tdb->ecode = TDB_ERR_OOM;
585 tdb->map_ptr = new_map_ptr;
586 tdb->map_size = new_size;
591 * expand the file itself
593 ret = tdb->methods->tdb_expand_file(tdb, tdb->map_size, size);
598 /* Explicitly remap: if we're in a transaction, this won't
599 * happen automatically! */
601 tdb->map_size = new_size;
602 if (tdb_mmap(tdb) != 0) {
607 /* link it into the free list */
608 if (tdb_free(tdb, offset, &rec) == -1)
611 tdb_unlock(tdb, -1, F_WRLCK);
614 tdb_unlock(tdb, -1, F_WRLCK);
618 /* read/write a tdb_off_t */
619 int tdb_ofs_read(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d)
621 return tdb->methods->tdb_read(tdb, offset, (char*)d, sizeof(*d), DOCONV());
624 int tdb_ofs_write(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d)
627 return tdb->methods->tdb_write(tdb, offset, CONVERT(off), sizeof(*d));
631 /* read a lump of data, allocating the space for it */
632 unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len_t len)
636 /* some systems don't like zero length malloc */
638 if (!(buf = (unsigned char *)malloc(len ? len : 1))) {
639 /* Ensure ecode is set for log fn. */
640 tdb->ecode = TDB_ERR_OOM;
641 TDB_LOG((tdb, TDB_DEBUG_ERROR,"tdb_alloc_read malloc failed len=%u (%s)\n",
642 len, strerror(errno)));
645 if (tdb->methods->tdb_read(tdb, offset, buf, len, 0) == -1) {
652 /* Give a piece of tdb data to a parser */
654 int tdb_parse_data(struct tdb_context *tdb, TDB_DATA key,
655 tdb_off_t offset, tdb_len_t len,
656 int (*parser)(TDB_DATA key, TDB_DATA data,
665 if ((tdb->transaction == NULL) && (tdb->map_ptr != NULL)) {
667 * Optimize by avoiding the malloc/memcpy/free, point the
668 * parser directly at the mmap area.
670 if (tdb->methods->tdb_oob(tdb, offset, len, 0) != 0) {
673 data.dptr = offset + (unsigned char *)tdb->map_ptr;
674 return parser(key, data, private_data);
677 if (!(data.dptr = tdb_alloc_read(tdb, offset, len))) {
681 result = parser(key, data, private_data);
686 /* read/write a record */
687 int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct tdb_record *rec)
689 if (tdb->methods->tdb_read(tdb, offset, rec, sizeof(*rec),DOCONV()) == -1)
691 if (TDB_BAD_MAGIC(rec)) {
692 /* Ensure ecode is set for log fn. */
693 tdb->ecode = TDB_ERR_CORRUPT;
694 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_rec_read bad magic 0x%x at offset=%u\n", rec->magic, offset));
697 return tdb->methods->tdb_oob(tdb, rec->next, sizeof(*rec), 0);
700 int tdb_rec_write(struct tdb_context *tdb, tdb_off_t offset, struct tdb_record *rec)
702 struct tdb_record r = *rec;
703 return tdb->methods->tdb_write(tdb, offset, CONVERT(r), sizeof(r));
706 static const struct tdb_methods io_methods = {
715 initialise the default methods table
717 void tdb_io_init(struct tdb_context *tdb)
719 tdb->methods = &io_methods;