2 Unix SMB/CIFS implementation.
4 trivial database library
6 Copyright (C) Andrew Tridgell 1999-2005
7 Copyright (C) Paul `Rusty' Russell 2000
8 Copyright (C) Jeremy Allison 2000-2003
10 ** NOTE! The following LGPL license applies to the tdb
11 ** library. This does NOT imply that all of Samba is released
14 This library is free software; you can redistribute it and/or
15 modify it under the terms of the GNU Lesser General Public
16 License as published by the Free Software Foundation; either
17 version 3 of the License, or (at your option) any later version.
19 This library is distributed in the hope that it will be useful,
20 but WITHOUT ANY WARRANTY; without even the implied warranty of
21 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22 Lesser General Public License for more details.
24 You should have received a copy of the GNU Lesser General Public
25 License along with this library; if not, see <http://www.gnu.org/licenses/>.
29 #include "tdb_private.h"
32 * tdb->hdr_ofs is 0 for now.
34 * Note: that we only have the 4GB limit of tdb_off_t for
35 * tdb->map_size. The file size on disk can be 4GB + tdb->hdr_ofs!
38 static bool tdb_adjust_offset(struct tdb_context *tdb, off_t *off)
40 off_t tmp = tdb->hdr_ofs + *off;
42 if ((tmp < tdb->hdr_ofs) || (tmp < *off)) {
51 static ssize_t tdb_pwrite(struct tdb_context *tdb, const void *buf,
52 size_t count, off_t offset)
54 if (!tdb_adjust_offset(tdb, &offset)) {
57 return pwrite(tdb->fd, buf, count, offset);
60 static ssize_t tdb_pread(struct tdb_context *tdb, void *buf,
61 size_t count, off_t offset)
63 if (!tdb_adjust_offset(tdb, &offset)) {
66 return pread(tdb->fd, buf, count, offset);
69 static int tdb_ftruncate(struct tdb_context *tdb, off_t length)
71 if (!tdb_adjust_offset(tdb, &length)) {
74 return ftruncate(tdb->fd, length);
77 static int tdb_fstat(struct tdb_context *tdb, struct stat *buf)
81 ret = fstat(tdb->fd, buf);
86 if (buf->st_size < tdb->hdr_ofs) {
90 buf->st_size -= tdb->hdr_ofs;
95 /* check for an out of bounds access - if it is out of bounds then
96 see if the database has been expanded by someone else and expand
99 static int tdb_oob(struct tdb_context *tdb, tdb_off_t off, tdb_len_t len,
103 if (len + off < len) {
105 /* Ensure ecode is set for log fn. */
106 tdb->ecode = TDB_ERR_IO;
107 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob off %u len %u wrap\n",
113 if (off + len <= tdb->map_size)
115 if (tdb->flags & TDB_INTERNAL) {
117 /* Ensure ecode is set for log fn. */
118 tdb->ecode = TDB_ERR_IO;
119 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob len %u beyond internal malloc size %u\n",
120 (int)(off + len), (int)tdb->map_size));
125 if (tdb_fstat(tdb, &st) == -1) {
126 tdb->ecode = TDB_ERR_IO;
130 /* Beware >4G files! */
131 if ((tdb_off_t)st.st_size != st.st_size) {
132 /* Ensure ecode is set for log fn. */
133 tdb->ecode = TDB_ERR_IO;
134 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_oob len %llu too large!\n",
135 (long long)st.st_size));
139 /* Unmap, update size, remap. We do this unconditionally, to handle
140 * the unusual case where the db is truncated.
142 * This can happen to a child using tdb_reopen_all(true) on a
143 * TDB_CLEAR_IF_FIRST tdb whose parent crashes: the next
144 * opener will truncate the database. */
145 if (tdb_munmap(tdb) == -1) {
146 tdb->ecode = TDB_ERR_IO;
149 tdb->map_size = st.st_size;
150 if (tdb_mmap(tdb) != 0) {
154 if (st.st_size < (size_t)off + len) {
156 /* Ensure ecode is set for log fn. */
157 tdb->ecode = TDB_ERR_IO;
158 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob len %u beyond eof at %u\n",
159 (int)(off + len), (int)st.st_size));
166 /* write a lump of data at a specified offset */
167 static int tdb_write(struct tdb_context *tdb, tdb_off_t off,
168 const void *buf, tdb_len_t len)
174 if (tdb->read_only || tdb->traverse_read) {
175 tdb->ecode = TDB_ERR_RDONLY;
179 if (tdb->methods->tdb_oob(tdb, off, len, 0) != 0)
183 memcpy(off + (char *)tdb->map_ptr, buf, len);
185 #ifdef HAVE_INCOHERENT_MMAP
186 tdb->ecode = TDB_ERR_IO;
191 written = tdb_pwrite(tdb, buf, len, off);
193 if ((written != (ssize_t)len) && (written != -1)) {
195 tdb->ecode = TDB_ERR_IO;
196 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_write: wrote only "
197 "%zi of %u bytes at %u, trying once more\n",
199 written = tdb_pwrite(tdb, (const char *)buf+written,
200 len-written, off+written);
203 /* Ensure ecode is set for log fn. */
204 tdb->ecode = TDB_ERR_IO;
205 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_write failed at %u "
206 "len=%u (%s)\n", off, len, strerror(errno)));
208 } else if (written != (ssize_t)len) {
209 tdb->ecode = TDB_ERR_IO;
210 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_write: failed to "
211 "write %u bytes at %u in two attempts\n",
220 /* Endian conversion: we only ever deal with 4 byte quantities */
221 void *tdb_convert(void *buf, uint32_t size)
223 uint32_t i, *p = (uint32_t *)buf;
224 for (i = 0; i < size / 4; i++)
225 p[i] = TDB_BYTEREV(p[i]);
230 /* read a lump of data at a specified offset, maybe convert */
231 static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf,
232 tdb_len_t len, int cv)
234 if (tdb->methods->tdb_oob(tdb, off, len, 0) != 0) {
239 memcpy(buf, off + (char *)tdb->map_ptr, len);
241 #ifdef HAVE_INCOHERENT_MMAP
242 tdb->ecode = TDB_ERR_IO;
247 ret = tdb_pread(tdb, buf, len, off);
248 if (ret != (ssize_t)len) {
249 /* Ensure ecode is set for log fn. */
250 tdb->ecode = TDB_ERR_IO;
251 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %u "
252 "len=%u ret=%zi (%s) map_size=%u\n",
253 off, len, ret, strerror(errno),
260 tdb_convert(buf, len);
268 do an unlocked scan of the hash table heads to find the next non-zero head. The value
269 will then be confirmed with the lock held
271 static void tdb_next_hash_chain(struct tdb_context *tdb, uint32_t *chain)
275 for (;h < tdb->hash_size;h++) {
276 if (0 != *(uint32_t *)(TDB_HASH_TOP(h) + (unsigned char *)tdb->map_ptr)) {
282 for (;h < tdb->hash_size;h++) {
283 if (tdb_ofs_read(tdb, TDB_HASH_TOP(h), &off) != 0 || off != 0) {
292 int tdb_munmap(struct tdb_context *tdb)
294 if (tdb->flags & TDB_INTERNAL)
301 ret = munmap(tdb->map_ptr, tdb->map_size);
310 /* If mmap isn't coherent, *everyone* must always mmap. */
311 static bool should_mmap(const struct tdb_context *tdb)
313 #ifdef HAVE_INCOHERENT_MMAP
316 return !(tdb->flags & TDB_NOMMAP);
320 int tdb_mmap(struct tdb_context *tdb)
322 if (tdb->flags & TDB_INTERNAL)
326 if (should_mmap(tdb)) {
327 tdb->map_ptr = mmap(NULL, tdb->map_size,
328 PROT_READ|(tdb->read_only? 0:PROT_WRITE),
329 MAP_SHARED|MAP_FILE, tdb->fd,
333 * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!!
336 if (tdb->map_ptr == MAP_FAILED) {
338 TDB_LOG((tdb, TDB_DEBUG_WARNING, "tdb_mmap failed for size %u (%s)\n",
339 tdb->map_size, strerror(errno)));
340 #ifdef HAVE_INCOHERENT_MMAP
341 tdb->ecode = TDB_ERR_IO;
354 /* expand a file. we prefer to use ftruncate, as that is what posix
355 says to use for mmap expansion */
356 static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t addition)
361 if (tdb->read_only || tdb->traverse_read) {
362 tdb->ecode = TDB_ERR_RDONLY;
366 if (!tdb_add_off_t(size, addition, &new_size)) {
367 tdb->ecode = TDB_ERR_OOM;
368 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write "
369 "overflow detected current size[%u] addition[%u]!\n",
370 (unsigned)size, (unsigned)addition));
375 if (tdb_ftruncate(tdb, new_size) == -1) {
377 ssize_t written = tdb_pwrite(tdb, &b, 1, new_size - 1);
379 /* try once more, potentially revealing errno */
380 written = tdb_pwrite(tdb, &b, 1, new_size - 1);
383 /* again - give up, guessing errno */
387 tdb->ecode = TDB_ERR_OOM;
388 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %u failed (%s)\n",
389 (unsigned)new_size, strerror(errno)));
394 /* now fill the file with something. This ensures that the
395 file isn't sparse, which would be very bad if we ran out of
396 disk. This must be done with write, not via mmap */
397 memset(buf, TDB_PAD_BYTE, sizeof(buf));
399 size_t n = addition>sizeof(buf)?sizeof(buf):addition;
400 ssize_t written = tdb_pwrite(tdb, buf, n, size);
402 /* prevent infinite loops: try _once_ more */
403 written = tdb_pwrite(tdb, buf, n, size);
406 /* give up, trying to provide a useful errno */
407 tdb->ecode = TDB_ERR_OOM;
408 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write "
409 "returned 0 twice: giving up!\n"));
414 tdb->ecode = TDB_ERR_OOM;
415 TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of "
416 "%u bytes failed (%s)\n", (int)n,
421 TDB_LOG((tdb, TDB_DEBUG_WARNING, "expand_file: wrote "
422 "only %zu of %zi bytes - retrying\n", written,
432 /* You need 'size', this tells you how much you should expand by. */
433 tdb_off_t tdb_expand_adjust(tdb_off_t map_size, tdb_off_t size, int page_size)
435 tdb_off_t new_size, top_size, increment;
436 tdb_off_t max_size = UINT32_MAX - map_size;
438 if (size > max_size) {
440 * We can't round up anymore, just give back
441 * what we're asked for.
443 * The caller has to take care of the ENOSPC handling.
448 /* limit size in order to avoid using up huge amounts of memory for
449 * in memory tdbs if an oddball huge record creeps in */
450 if (size > 100 * 1024) {
451 increment = size * 2;
453 increment = size * 100;
455 if (increment < size) {
459 if (!tdb_add_off_t(map_size, increment, &top_size)) {
463 /* always make room for at least top_size more records, and at
464 least 25% more space. if the DB is smaller than 100MiB,
465 otherwise grow it by 10% only. */
466 if (map_size > 100 * 1024 * 1024) {
467 new_size = map_size * 1.10;
469 new_size = map_size * 1.25;
471 if (new_size < map_size) {
475 /* Round the database up to a multiple of the page size */
476 new_size = MAX(top_size, new_size);
478 if (new_size + page_size < new_size) {
479 /* There's a "+" in TDB_ALIGN that might overflow... */
483 return TDB_ALIGN(new_size, page_size) - map_size;
487 * Somewhere in between we went over 4GB. Make one big jump to
488 * exactly 4GB database size.
493 /* expand the database at least size bytes by expanding the underlying
494 file and doing the mmap again if necessary */
495 int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
497 struct tdb_record rec;
501 if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
502 TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));
506 /* must know about any previous expansions by another process */
507 tdb->methods->tdb_oob(tdb, tdb->map_size, 1, 1);
510 * Note: that we don't care about tdb->hdr_ofs != 0 here
512 * The 4GB limitation is just related to tdb->map_size
513 * and the offset calculation in the records.
515 * The file on disk can be up to 4GB + tdb->hdr_ofs
517 size = tdb_expand_adjust(tdb->map_size, size, tdb->page_size);
519 if (!tdb_add_off_t(tdb->map_size, size, &new_size)) {
520 tdb->ecode = TDB_ERR_OOM;
521 TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_expand "
522 "overflow detected current map_size[%u] size[%u]!\n",
523 (unsigned)tdb->map_size, (unsigned)size));
527 /* form a new freelist record */
528 offset = tdb->map_size;
529 memset(&rec,'\0',sizeof(rec));
530 rec.rec_len = size - sizeof(rec);
532 if (tdb->flags & TDB_INTERNAL) {
535 new_map_ptr = (char *)realloc(tdb->map_ptr, new_size);
537 tdb->ecode = TDB_ERR_OOM;
540 tdb->map_ptr = new_map_ptr;
541 tdb->map_size = new_size;
546 * expand the file itself
548 ret = tdb->methods->tdb_expand_file(tdb, tdb->map_size, size);
553 /* Explicitly remap: if we're in a transaction, this won't
554 * happen automatically! */
556 tdb->map_size = new_size;
557 if (tdb_mmap(tdb) != 0) {
562 /* link it into the free list */
563 if (tdb_free(tdb, offset, &rec) == -1)
566 tdb_unlock(tdb, -1, F_WRLCK);
569 tdb_unlock(tdb, -1, F_WRLCK);
573 /* read/write a tdb_off_t */
574 int tdb_ofs_read(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d)
576 return tdb->methods->tdb_read(tdb, offset, (char*)d, sizeof(*d), DOCONV());
579 int tdb_ofs_write(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d)
582 return tdb->methods->tdb_write(tdb, offset, CONVERT(off), sizeof(*d));
586 /* read a lump of data, allocating the space for it */
587 unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len_t len)
591 /* some systems don't like zero length malloc */
593 if (!(buf = (unsigned char *)malloc(len ? len : 1))) {
594 /* Ensure ecode is set for log fn. */
595 tdb->ecode = TDB_ERR_OOM;
596 TDB_LOG((tdb, TDB_DEBUG_ERROR,"tdb_alloc_read malloc failed len=%u (%s)\n",
597 len, strerror(errno)));
600 if (tdb->methods->tdb_read(tdb, offset, buf, len, 0) == -1) {
607 /* Give a piece of tdb data to a parser */
609 int tdb_parse_data(struct tdb_context *tdb, TDB_DATA key,
610 tdb_off_t offset, tdb_len_t len,
611 int (*parser)(TDB_DATA key, TDB_DATA data,
620 if ((tdb->transaction == NULL) && (tdb->map_ptr != NULL)) {
622 * Optimize by avoiding the malloc/memcpy/free, point the
623 * parser directly at the mmap area.
625 if (tdb->methods->tdb_oob(tdb, offset, len, 0) != 0) {
628 data.dptr = offset + (unsigned char *)tdb->map_ptr;
629 return parser(key, data, private_data);
632 if (!(data.dptr = tdb_alloc_read(tdb, offset, len))) {
636 result = parser(key, data, private_data);
641 /* read/write a record */
642 int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct tdb_record *rec)
644 if (tdb->methods->tdb_read(tdb, offset, rec, sizeof(*rec),DOCONV()) == -1)
646 if (TDB_BAD_MAGIC(rec)) {
647 /* Ensure ecode is set for log fn. */
648 tdb->ecode = TDB_ERR_CORRUPT;
649 TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_rec_read bad magic 0x%x at offset=%u\n", rec->magic, offset));
652 return tdb->methods->tdb_oob(tdb, rec->next, sizeof(*rec), 0);
655 int tdb_rec_write(struct tdb_context *tdb, tdb_off_t offset, struct tdb_record *rec)
657 struct tdb_record r = *rec;
658 return tdb->methods->tdb_write(tdb, offset, CONVERT(r), sizeof(r));
661 static const struct tdb_methods io_methods = {
670 initialise the default methods table
672 void tdb_io_init(struct tdb_context *tdb)
674 tdb->methods = &io_methods;