try to make the tailer code much more robust. When a record

can't be merged don't fail the operation, instead just add
it to the free list anyway

added logging to tdb

diff --git a/source/tdb/tdb.c b/source/tdb/tdb.c
index 2c2d78f719483f8aec8bdede2d00e1ae00fae8dd..eb5ce0aa952c8c9ef3a9910ea210b92c542ab5df 100644 (file)
--- a/source/tdb/tdb.c
+++ b/source/tdb/tdb.c
@@ -56,6 +56,7 @@
 #define TDB_DEAD(r) ((r)->magic == TDB_DEAD_MAGIC)
 #define TDB_BAD_MAGIC(r) ((r)->magic != TDB_MAGIC && !TDB_DEAD(r))
 #define TDB_HASH_TOP(hash) (FREELIST_TOP + (BUCKET(hash)+1)*sizeof(tdb_off))
+#define TDB_LOG(x) (tdb->log_fn?tdb->log_fn x : 0)
 
 /* lock offsets */
 #define GLOBAL_LOCK 0
@@ -299,6 +300,63 @@ static int update_tailer(TDB_CONTEXT *tdb, tdb_off offset,
 }
 
 #ifdef TDB_DEBUG
+static tdb_off tdb_dump_record(TDB_CONTEXT *tdb, tdb_off offset)
+{
+       struct list_struct rec;
+       tdb_off tailer_ofs, tailer;
+
+       if (tdb_read(tdb, offset, (char *)&rec, sizeof(rec), DOCONV()) == -1) {
+               printf("ERROR: failed to read record at %u\n", offset);
+               return 0;
+       }
+
+       printf(" rec: offset=%u next=%d rec_len=%d key_len=%d data_len=%d full_hash=0x%x magic=0x%x\n",
+              offset, rec.next, rec.rec_len, rec.key_len, rec.data_len, rec.full_hash, rec.magic);
+
+       tailer_ofs = offset + sizeof(rec) + rec.rec_len - sizeof(tdb_off);
+       if (ofs_read(tdb, tailer_ofs, &tailer) == -1) {
+               printf("ERROR: failed to read tailer at %u\n", tailer_ofs);
+               return rec.next;
+       }
+
+       if (tailer != rec.rec_len + sizeof(rec)) {
+               printf("ERROR: tailer does not match record! tailer=%u totalsize=%u\n", tailer, rec.rec_len + sizeof(rec));
+       }
+       return rec.next;
+}
+
+static void tdb_dump_chain(TDB_CONTEXT *tdb, int i)
+{
+       tdb_off rec_ptr, top;
+
+       top = TDB_HASH_TOP(i);
+
+       tdb_lock(tdb, i, F_WRLCK);
+
+       if (ofs_read(tdb, top, &rec_ptr) == -1) {
+               tdb_unlock(tdb, i, F_WRLCK);
+               return;
+       }
+
+       if (rec_ptr) printf("hash=%d\n", i);
+
+       while (rec_ptr) {
+               rec_ptr = tdb_dump_record(tdb, rec_ptr);
+       }
+       tdb_unlock(tdb, i, F_WRLCK);
+}
+
+void tdb_dump_all(TDB_CONTEXT *tdb)
+{
+       tdb_off off;
+       int i;
+       for (i=0;i<tdb->header.hash_size;i++) {
+               tdb_dump_chain(tdb, i);
+       }
+       printf("freelist:\n");
+       tdb_dump_chain(tdb, -1);
+}
+
 void tdb_printfreelist(TDB_CONTEXT *tdb)
 {
        long total_free = 0;
@@ -365,45 +423,66 @@ static int tdb_free(TDB_CONTEXT *tdb, tdb_off offset, struct list_struct *rec)
        /* Allocation and tailer lock */
        if (tdb_lock(tdb, -1, F_WRLCK) != 0) return -1;
 
+       /* set an initial tailer, so if we fail we don't leave a bogus record */
+       update_tailer(tdb, offset, rec);
+
        /* Look right first (I'm an Australian, dammit) */
        right = offset + sizeof(*rec) + rec->rec_len;
        if (tdb_oob(tdb, right + sizeof(*rec)) == 0) {
                struct list_struct r;
 
-               if (tdb_read(tdb, right, &r, sizeof(r), DOCONV()) == -1)
-                       goto fail;
+               if (tdb_read(tdb, right, &r, sizeof(r), DOCONV()) == -1) {
+                       TDB_LOG((tdb, 0, "tdb_free: right read failed at %u\n", right));
+                       goto left;
+               }
 
                /* If it's free, expand to include it. */
                if (r.magic == TDB_FREE_MAGIC) {
-                       if (remove_from_freelist(tdb, right, r.next) == -1)
-                               goto fail;
+                       if (remove_from_freelist(tdb, right, r.next) == -1) {
+                               TDB_LOG((tdb, 0, "tdb_free: right free failed at %u\n", right));
+                               goto left;
+                       }
                        rec->rec_len += sizeof(r) + r.rec_len;
                }
        }
 
+left:
        /* Look left */
-       left = offset - 4;
+       left = offset - sizeof(tdb_off);
        if (left > TDB_HASH_TOP(tdb->header.hash_size-1)) {
                struct list_struct l;
                tdb_off leftsize;
 
                /* Read in tailer and jump back to header */
-               if (ofs_read(tdb, left, &leftsize) == -1) goto fail;
+               if (ofs_read(tdb, left, &leftsize) == -1) {
+                       TDB_LOG((tdb, 0, "tdb_free: left offset read failed at %u\n", left));
+                       goto update;
+               }
                left = offset - leftsize;
 
                /* Now read in record */
-               if (tdb_read(tdb, left, &l, sizeof(l), DOCONV()) == -1)
-                       goto fail;
+               if (tdb_read(tdb, left, &l, sizeof(l), DOCONV()) == -1) {
+                       TDB_LOG((tdb, 0, "tdb_free: left read failed at %u (%u)\n", left, leftsize));
+                       goto update;
+               }
 
                /* If it's free, expand to include it. */
                if (l.magic == TDB_FREE_MAGIC) {
-                       if (remove_from_freelist(tdb, left, l.next) == -1)
-                               goto fail;
-                       offset = left;
-                       rec->rec_len += leftsize;
+                       if (remove_from_freelist(tdb, left, l.next) == -1) {
+                               TDB_LOG((tdb, 0, "tdb_free: left free failed at %u\n", left));
+                               goto update;
+                       } else {
+                               offset = left;
+                               rec->rec_len += leftsize;
+                       }
                }
        }
-       if (update_tailer(tdb, offset, rec) == -1) goto fail;
+
+update:
+       if (update_tailer(tdb, offset, rec) == -1) {
+               TDB_LOG((tdb, 0, "tdb_free: update_tailer failed at %u\n", offset));
+               goto fail;
+       }
 
        /* Now, prepend to free list */
        rec->magic = TDB_FREE_MAGIC;
@@ -1313,3 +1392,10 @@ void tdb_chainunlock(TDB_CONTEXT *tdb, TDB_DATA key)
 {
        tdb_unlock(tdb, BUCKET(tdb_hash(&key)), F_WRLCK);
 }
+
+
+/* register a loging function */
+void tdb_logging_function(TDB_CONTEXT *tdb, void (*fn)(TDB_CONTEXT *, int , const char *, ...))
+{
+       tdb->log_fn = fn;
+}

diff --git a/source/tdb/tdb.h b/source/tdb/tdb.h
index f09c6453b5310c34d71efe5958de3097b4b61471..cc37825e7811fa539487062f5b5b919f781f12df 100644 (file)
--- a/source/tdb/tdb.h
+++ b/source/tdb/tdb.h
@@ -94,12 +94,14 @@ typedef struct tdb_context {
        struct tdb_context *next; /* all tdbs to avoid multiple opens */
        dev_t device;   /* uniquely identifies this tdb */
        ino_t inode;    /* uniquely identifies this tdb */
+       void (*log_fn)(struct tdb_context *tdb, int level, const char *, ...); /* logging function */
 } TDB_CONTEXT;
 
 typedef int (*tdb_traverse_func)(TDB_CONTEXT *, TDB_DATA, TDB_DATA, void *);
 
 TDB_CONTEXT *tdb_open(char *name, int hash_size, int tdb_flags,
                      int open_flags, mode_t mode);
+void tdb_logging_function(TDB_CONTEXT *tdb, void (*fn)(TDB_CONTEXT *, int , const char *, ...));
 enum TDB_ERROR tdb_error(TDB_CONTEXT *tdb);
 const char *tdb_errorstr(TDB_CONTEXT *tdb);
 TDB_DATA tdb_fetch(TDB_CONTEXT *tdb, TDB_DATA key);

diff --git a/source/tdb/tdbtest.c b/source/tdb/tdbtest.c
index 9e636eef83abbadc007c6bb2667027983f559b9c..2cc7e887297ff1a6ef8ea14ed389b9481074b29f 100644 (file)
--- a/source/tdb/tdbtest.c
+++ b/source/tdb/tdbtest.c
@@ -4,6 +4,7 @@
 #include <unistd.h>
 #include <string.h>
 #include <fcntl.h>
+#include <stdarg.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <sys/time.h>
@@ -40,6 +41,14 @@ static void fatal(char *why)
        exit(1);
 }
 
+static void tdb_log(TDB_CONTEXT *tdb, int level, const char *format, ...)
+{
+       va_list ap;
+    
+       va_start(ap, format);
+       vfprintf(stdout, format, ap);
+       va_end(ap);
+}
 
 static void compare_db(void)
 {
@@ -230,6 +239,7 @@ int main(int argc, char *argv[])
                fatal("db open failed");
        }
 
+       tdb_logging_function(db, tdb_log);
        
 #if 1
        srand(seed);

diff --git a/source/tdb/tdbtorture.c b/source/tdb/tdbtorture.c
index 90dcc38aba1e0ea73735a67a12caad8a15272211..4020fe69cb79ffd59d3e4d1dbef65232997bc7a0 100644 (file)
--- a/source/tdb/tdbtorture.c
+++ b/source/tdb/tdbtorture.c
@@ -4,6 +4,7 @@
 #include <unistd.h>
 #include <string.h>
 #include <fcntl.h>
+#include <stdarg.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <sys/time.h>
@@ -22,6 +23,15 @@
 
 static TDB_CONTEXT *db;
 
+static void tdb_log(TDB_CONTEXT *tdb, int level, const char *format, ...)
+{
+       va_list ap;
+    
+       va_start(ap, format);
+       vfprintf(stdout, format, ap);
+       va_end(ap);
+}
+
 static void fatal(char *why)
 {
        perror(why);
@@ -86,7 +96,7 @@ static int traverse_fn(TDB_CONTEXT *db, TDB_DATA key, TDB_DATA dbuf,
 #endif
 
 #ifndef NLOOPS
-#define NLOOPS 50000
+#define NLOOPS 5000000
 #endif
 
 int main(int argc, char *argv[])
@@ -103,6 +113,7 @@ int main(int argc, char *argv[])
        if (!db) {
                fatal("db open failed");
        }
+       tdb_logging_function(db, tdb_log);
 
        srand(seed + getpid());
        for (i=0;i<loops;i++) addrec_db();