lib/ntdb/ntdb.c

   1  /*
   2    Trivial Database 2: fetch, store and misc routines.
   3    Copyright (C) Rusty Russell 2010
   4
   5    This library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 3 of the License, or (at your option) any later version.
   9
  10    This library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with this library; if not, see <http://www.gnu.org/licenses/>.
  17 */
  18 #include "private.h"
  19 #ifndef HAVE_LIBREPLACE
  20 #include <ccan/asprintf/asprintf.h>
  21 #include <stdarg.h>
  22 #endif
  23
  24 static enum NTDB_ERROR update_rec_hdr(struct ntdb_context *ntdb,
  25                                      ntdb_off_t off,
  26                                      ntdb_len_t keylen,
  27                                      ntdb_len_t datalen,
  28                                      struct ntdb_used_record *rec)
  29 {
  30         uint64_t dataroom = rec_data_length(rec) + rec_extra_padding(rec);
  31         enum NTDB_ERROR ecode;
  32
  33         ecode = set_header(ntdb, rec, NTDB_USED_MAGIC, keylen, datalen,
  34                            keylen + dataroom);
  35         if (ecode == NTDB_SUCCESS) {
  36                 ecode = ntdb_write_convert(ntdb, off, rec, sizeof(*rec));
  37         }
  38         return ecode;
  39 }
  40
  41 static enum NTDB_ERROR replace_data(struct ntdb_context *ntdb,
  42                                    struct hash_info *h,
  43                                    NTDB_DATA key, NTDB_DATA dbuf,
  44                                    ntdb_off_t old_off, ntdb_len_t old_room,
  45                                    bool growing)
  46 {
  47         ntdb_off_t new_off;
  48         enum NTDB_ERROR ecode;
  49
  50         /* Allocate a new record. */
  51         new_off = alloc(ntdb, key.dsize, dbuf.dsize, NTDB_USED_MAGIC, growing);
  52         if (NTDB_OFF_IS_ERR(new_off)) {
  53                 return NTDB_OFF_TO_ERR(new_off);
  54         }
  55
  56         /* We didn't like the existing one: remove it. */
  57         if (old_off) {
  58                 ntdb->stats.frees++;
  59                 ecode = add_free_record(ntdb, old_off,
  60                                         sizeof(struct ntdb_used_record)
  61                                         + key.dsize + old_room,
  62                                         NTDB_LOCK_WAIT, true);
  63                 if (ecode == NTDB_SUCCESS)
  64                         ecode = replace_in_hash(ntdb, h, new_off);
  65         } else {
  66                 ecode = add_to_hash(ntdb, h, new_off);
  67         }
  68         if (ecode != NTDB_SUCCESS) {
  69                 return ecode;
  70         }
  71
  72         new_off += sizeof(struct ntdb_used_record);
  73         ecode = ntdb->io->twrite(ntdb, new_off, key.dptr, key.dsize);
  74         if (ecode != NTDB_SUCCESS) {
  75                 return ecode;
  76         }
  77
  78         new_off += key.dsize;
  79         ecode = ntdb->io->twrite(ntdb, new_off, dbuf.dptr, dbuf.dsize);
  80         if (ecode != NTDB_SUCCESS) {
  81                 return ecode;
  82         }
  83
  84         if (ntdb->flags & NTDB_SEQNUM)
  85                 ntdb_inc_seqnum(ntdb);
  86
  87         return NTDB_SUCCESS;
  88 }
  89
  90 static enum NTDB_ERROR update_data(struct ntdb_context *ntdb,
  91                                   ntdb_off_t off,
  92                                   NTDB_DATA dbuf,
  93                                   ntdb_len_t extra)
  94 {
  95         enum NTDB_ERROR ecode;
  96
  97         ecode = ntdb->io->twrite(ntdb, off, dbuf.dptr, dbuf.dsize);
  98         if (ecode == NTDB_SUCCESS && extra) {
  99                 /* Put a zero in; future versions may append other data. */
 100                 ecode = ntdb->io->twrite(ntdb, off + dbuf.dsize, "", 1);
 101         }
 102         if (ntdb->flags & NTDB_SEQNUM)
 103                 ntdb_inc_seqnum(ntdb);
 104
 105         return ecode;
 106 }
 107
 108 _PUBLIC_ enum NTDB_ERROR ntdb_store(struct ntdb_context *ntdb,
 109                          NTDB_DATA key, NTDB_DATA dbuf, int flag)
 110 {
 111         struct hash_info h;
 112         ntdb_off_t off;
 113         ntdb_len_t old_room = 0;
 114         struct ntdb_used_record rec;
 115         enum NTDB_ERROR ecode;
 116
 117         off = find_and_lock(ntdb, key, F_WRLCK, &h, &rec, NULL);
 118         if (NTDB_OFF_IS_ERR(off)) {
 119                 return NTDB_OFF_TO_ERR(off);
 120         }
 121
 122         /* Now we have lock on this hash bucket. */
 123         if (flag == NTDB_INSERT) {
 124                 if (off) {
 125                         ecode = NTDB_ERR_EXISTS;
 126                         goto out;
 127                 }
 128         } else {
 129                 if (off) {
 130                         old_room = rec_data_length(&rec)
 131                                 + rec_extra_padding(&rec);
 132                         if (old_room >= dbuf.dsize) {
 133                                 /* Can modify in-place.  Easy! */
 134                                 ecode = update_rec_hdr(ntdb, off,
 135                                                        key.dsize, dbuf.dsize,
 136                                                        &rec);
 137                                 if (ecode != NTDB_SUCCESS) {
 138                                         goto out;
 139                                 }
 140                                 ecode = update_data(ntdb,
 141                                                     off + sizeof(rec)
 142                                                     + key.dsize, dbuf,
 143                                                     old_room - dbuf.dsize);
 144                                 if (ecode != NTDB_SUCCESS) {
 145                                         goto out;
 146                                 }
 147                                 ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 148                                 return NTDB_SUCCESS;
 149                         }
 150                 } else {
 151                         if (flag == NTDB_MODIFY) {
 152                                 /* if the record doesn't exist and we
 153                                    are in NTDB_MODIFY mode then we should fail
 154                                    the store */
 155                                 ecode = NTDB_ERR_NOEXIST;
 156                                 goto out;
 157                         }
 158                 }
 159         }
 160
 161         /* If we didn't use the old record, this implies we're growing. */
 162         ecode = replace_data(ntdb, &h, key, dbuf, off, old_room, off);
 163 out:
 164         ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 165         return ecode;
 166 }
 167
 168 _PUBLIC_ enum NTDB_ERROR ntdb_append(struct ntdb_context *ntdb,
 169                           NTDB_DATA key, NTDB_DATA dbuf)
 170 {
 171         struct hash_info h;
 172         ntdb_off_t off;
 173         struct ntdb_used_record rec;
 174         ntdb_len_t old_room = 0, old_dlen;
 175         unsigned char *newdata;
 176         NTDB_DATA new_dbuf;
 177         enum NTDB_ERROR ecode;
 178
 179         off = find_and_lock(ntdb, key, F_WRLCK, &h, &rec, NULL);
 180         if (NTDB_OFF_IS_ERR(off)) {
 181                 return NTDB_OFF_TO_ERR(off);
 182         }
 183
 184         if (off) {
 185                 old_dlen = rec_data_length(&rec);
 186                 old_room = old_dlen + rec_extra_padding(&rec);
 187
 188                 /* Fast path: can append in place. */
 189                 if (rec_extra_padding(&rec) >= dbuf.dsize) {
 190                         ecode = update_rec_hdr(ntdb, off, key.dsize,
 191                                                old_dlen + dbuf.dsize, &rec);
 192                         if (ecode != NTDB_SUCCESS) {
 193                                 goto out;
 194                         }
 195
 196                         off += sizeof(rec) + key.dsize + old_dlen;
 197                         ecode = update_data(ntdb, off, dbuf,
 198                                             rec_extra_padding(&rec));
 199                         goto out;
 200                 }
 201
 202                 /* Slow path. */
 203                 newdata = ntdb->alloc_fn(ntdb, key.dsize + old_dlen + dbuf.dsize,
 204                                      ntdb->alloc_data);
 205                 if (!newdata) {
 206                         ecode = ntdb_logerr(ntdb, NTDB_ERR_OOM, NTDB_LOG_ERROR,
 207                                            "ntdb_append:"
 208                                            " failed to allocate %zu bytes",
 209                                            (size_t)(key.dsize + old_dlen
 210                                                     + dbuf.dsize));
 211                         goto out;
 212                 }
 213                 ecode = ntdb->io->tread(ntdb, off + sizeof(rec) + key.dsize,
 214                                        newdata, old_dlen);
 215                 if (ecode != NTDB_SUCCESS) {
 216                         goto out_free_newdata;
 217                 }
 218                 memcpy(newdata + old_dlen, dbuf.dptr, dbuf.dsize);
 219                 new_dbuf.dptr = newdata;
 220                 new_dbuf.dsize = old_dlen + dbuf.dsize;
 221         } else {
 222                 newdata = NULL;
 223                 new_dbuf = dbuf;
 224         }
 225
 226         /* If they're using ntdb_append(), it implies they're growing record. */
 227         ecode = replace_data(ntdb, &h, key, new_dbuf, off, old_room, true);
 228
 229 out_free_newdata:
 230         ntdb->free_fn(newdata, ntdb->alloc_data);
 231 out:
 232         ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 233         return ecode;
 234 }
 235
 236 _PUBLIC_ enum NTDB_ERROR ntdb_fetch(struct ntdb_context *ntdb, NTDB_DATA key,
 237                          NTDB_DATA *data)
 238 {
 239         ntdb_off_t off;
 240         struct ntdb_used_record rec;
 241         struct hash_info h;
 242         enum NTDB_ERROR ecode;
 243         const char *keyp;
 244
 245         off = find_and_lock(ntdb, key, F_RDLCK, &h, &rec, &keyp);
 246         if (NTDB_OFF_IS_ERR(off)) {
 247                 return NTDB_OFF_TO_ERR(off);
 248         }
 249
 250         if (!off) {
 251                 ecode = NTDB_ERR_NOEXIST;
 252         } else {
 253                 data->dsize = rec_data_length(&rec);
 254                 data->dptr = ntdb->alloc_fn(ntdb, data->dsize, ntdb->alloc_data);
 255                 if (unlikely(!data->dptr)) {
 256                         ecode = NTDB_ERR_OOM;
 257                 } else {
 258                         memcpy(data->dptr, keyp + key.dsize, data->dsize);
 259                         ecode = NTDB_SUCCESS;
 260                 }
 261                 ntdb_access_release(ntdb, keyp);
 262         }
 263
 264         ntdb_unlock_hash(ntdb, h.h, F_RDLCK);
 265         return ecode;
 266 }
 267
 268 _PUBLIC_ bool ntdb_exists(struct ntdb_context *ntdb, NTDB_DATA key)
 269 {
 270         ntdb_off_t off;
 271         struct ntdb_used_record rec;
 272         struct hash_info h;
 273
 274         off = find_and_lock(ntdb, key, F_RDLCK, &h, &rec, NULL);
 275         if (NTDB_OFF_IS_ERR(off)) {
 276                 return false;
 277         }
 278         ntdb_unlock_hash(ntdb, h.h, F_RDLCK);
 279
 280         return off ? true : false;
 281 }
 282
 283 _PUBLIC_ enum NTDB_ERROR ntdb_delete(struct ntdb_context *ntdb, NTDB_DATA key)
 284 {
 285         ntdb_off_t off;
 286         struct ntdb_used_record rec;
 287         struct hash_info h;
 288         enum NTDB_ERROR ecode;
 289
 290         off = find_and_lock(ntdb, key, F_WRLCK, &h, &rec, NULL);
 291         if (NTDB_OFF_IS_ERR(off)) {
 292                 return NTDB_OFF_TO_ERR(off);
 293         }
 294
 295         if (!off) {
 296                 ecode = NTDB_ERR_NOEXIST;
 297                 goto unlock;
 298         }
 299
 300         ecode = delete_from_hash(ntdb, &h);
 301         if (ecode != NTDB_SUCCESS) {
 302                 goto unlock;
 303         }
 304
 305         /* Free the deleted entry. */
 306         ntdb->stats.frees++;
 307         ecode = add_free_record(ntdb, off,
 308                                 sizeof(struct ntdb_used_record)
 309                                 + rec_key_length(&rec)
 310                                 + rec_data_length(&rec)
 311                                 + rec_extra_padding(&rec),
 312                                 NTDB_LOCK_WAIT, true);
 313
 314         if (ntdb->flags & NTDB_SEQNUM)
 315                 ntdb_inc_seqnum(ntdb);
 316
 317 unlock:
 318         ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 319         return ecode;
 320 }
 321
 322 _PUBLIC_ unsigned int ntdb_get_flags(struct ntdb_context *ntdb)
 323 {
 324         return ntdb->flags;
 325 }
 326
 327 static bool inside_transaction(const struct ntdb_context *ntdb)
 328 {
 329         return ntdb->transaction != NULL;
 330 }
 331
 332 static bool readonly_changable(struct ntdb_context *ntdb, const char *caller)
 333 {
 334         if (inside_transaction(ntdb)) {
 335                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 336                             "%s: can't change"
 337                             " NTDB_RDONLY inside transaction",
 338                             caller);
 339                 return false;
 340         }
 341         return true;
 342 }
 343
 344 _PUBLIC_ void ntdb_add_flag(struct ntdb_context *ntdb, unsigned flag)
 345 {
 346         if (ntdb->flags & NTDB_INTERNAL) {
 347                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 348                             "ntdb_add_flag: internal db");
 349                 return;
 350         }
 351         switch (flag) {
 352         case NTDB_NOLOCK:
 353                 ntdb->flags |= NTDB_NOLOCK;
 354                 break;
 355         case NTDB_NOMMAP:
 356                 ntdb->flags |= NTDB_NOMMAP;
 357 #ifndef HAVE_INCOHERENT_MMAP
 358                 ntdb_munmap(ntdb->file);
 359 #endif
 360                 break;
 361         case NTDB_NOSYNC:
 362                 ntdb->flags |= NTDB_NOSYNC;
 363                 break;
 364         case NTDB_SEQNUM:
 365                 ntdb->flags |= NTDB_SEQNUM;
 366                 break;
 367         case NTDB_ALLOW_NESTING:
 368                 ntdb->flags |= NTDB_ALLOW_NESTING;
 369                 break;
 370         case NTDB_RDONLY:
 371                 if (readonly_changable(ntdb, "ntdb_add_flag"))
 372                         ntdb->flags |= NTDB_RDONLY;
 373                 break;
 374         default:
 375                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 376                             "ntdb_add_flag: Unknown flag %u", flag);
 377         }
 378 }
 379
 380 _PUBLIC_ void ntdb_remove_flag(struct ntdb_context *ntdb, unsigned flag)
 381 {
 382         if (ntdb->flags & NTDB_INTERNAL) {
 383                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 384                             "ntdb_remove_flag: internal db");
 385                 return;
 386         }
 387         switch (flag) {
 388         case NTDB_NOLOCK:
 389                 ntdb->flags &= ~NTDB_NOLOCK;
 390                 break;
 391         case NTDB_NOMMAP:
 392                 ntdb->flags &= ~NTDB_NOMMAP;
 393 #ifndef HAVE_INCOHERENT_MMAP
 394                 /* If mmap incoherent, we were mmaping anyway. */
 395                 ntdb_mmap(ntdb);
 396 #endif
 397                 break;
 398         case NTDB_NOSYNC:
 399                 ntdb->flags &= ~NTDB_NOSYNC;
 400                 break;
 401         case NTDB_SEQNUM:
 402                 ntdb->flags &= ~NTDB_SEQNUM;
 403                 break;
 404         case NTDB_ALLOW_NESTING:
 405                 ntdb->flags &= ~NTDB_ALLOW_NESTING;
 406                 break;
 407         case NTDB_RDONLY:
 408                 if ((ntdb->open_flags & O_ACCMODE) == O_RDONLY) {
 409                         ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 410                                     "ntdb_remove_flag: can't"
 411                                     " remove NTDB_RDONLY on ntdb"
 412                                     " opened with O_RDONLY");
 413                         break;
 414                 }
 415                 if (readonly_changable(ntdb, "ntdb_remove_flag"))
 416                         ntdb->flags &= ~NTDB_RDONLY;
 417                 break;
 418         default:
 419                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 420                             "ntdb_remove_flag: Unknown flag %u",
 421                             flag);
 422         }
 423 }
 424
 425 _PUBLIC_ const char *ntdb_errorstr(enum NTDB_ERROR ecode)
 426 {
 427         /* Gcc warns if you miss a case in the switch, so use that. */
 428         switch (NTDB_ERR_TO_OFF(ecode)) {
 429         case NTDB_ERR_TO_OFF(NTDB_SUCCESS): return "Success";
 430         case NTDB_ERR_TO_OFF(NTDB_ERR_CORRUPT): return "Corrupt database";
 431         case NTDB_ERR_TO_OFF(NTDB_ERR_IO): return "IO Error";
 432         case NTDB_ERR_TO_OFF(NTDB_ERR_LOCK): return "Locking error";
 433         case NTDB_ERR_TO_OFF(NTDB_ERR_OOM): return "Out of memory";
 434         case NTDB_ERR_TO_OFF(NTDB_ERR_EXISTS): return "Record exists";
 435         case NTDB_ERR_TO_OFF(NTDB_ERR_EINVAL): return "Invalid parameter";
 436         case NTDB_ERR_TO_OFF(NTDB_ERR_NOEXIST): return "Record does not exist";
 437         case NTDB_ERR_TO_OFF(NTDB_ERR_RDONLY): return "write not permitted";
 438         }
 439         return "Invalid error code";
 440 }
 441
 442 enum NTDB_ERROR COLD ntdb_logerr(struct ntdb_context *ntdb,
 443                                enum NTDB_ERROR ecode,
 444                                enum ntdb_log_level level,
 445                                const char *fmt, ...)
 446 {
 447         char *message;
 448         va_list ap;
 449         size_t len;
 450         /* ntdb_open paths care about errno, so save it. */
 451         int saved_errno = errno;
 452
 453         if (!ntdb->log_fn)
 454                 return ecode;
 455
 456         va_start(ap, fmt);
 457         len = vsnprintf(NULL, 0, fmt, ap);
 458         va_end(ap);
 459
 460         message = ntdb->alloc_fn(ntdb, len + 1, ntdb->alloc_data);
 461         if (!message) {
 462                 ntdb->log_fn(ntdb, NTDB_LOG_ERROR, NTDB_ERR_OOM,
 463                             "out of memory formatting message:", ntdb->log_data);
 464                 ntdb->log_fn(ntdb, level, ecode, fmt, ntdb->log_data);
 465         } else {
 466                 va_start(ap, fmt);
 467                 vsnprintf(message, len+1, fmt, ap);
 468                 va_end(ap);
 469                 ntdb->log_fn(ntdb, level, ecode, message, ntdb->log_data);
 470                 ntdb->free_fn(message, ntdb->alloc_data);
 471         }
 472         errno = saved_errno;
 473         return ecode;
 474 }
 475
 476 _PUBLIC_ enum NTDB_ERROR ntdb_parse_record_(struct ntdb_context *ntdb,
 477                                  NTDB_DATA key,
 478                                  enum NTDB_ERROR (*parse)(NTDB_DATA k,
 479                                                          NTDB_DATA d,
 480                                                          void *data),
 481                                  void *data)
 482 {
 483         ntdb_off_t off;
 484         struct ntdb_used_record rec;
 485         struct hash_info h;
 486         enum NTDB_ERROR ecode;
 487         const char *keyp;
 488
 489         off = find_and_lock(ntdb, key, F_RDLCK, &h, &rec, &keyp);
 490         if (NTDB_OFF_IS_ERR(off)) {
 491                 return NTDB_OFF_TO_ERR(off);
 492         }
 493
 494         if (!off) {
 495                 ecode = NTDB_ERR_NOEXIST;
 496         } else {
 497                 NTDB_DATA d = ntdb_mkdata(keyp + key.dsize,
 498                                           rec_data_length(&rec));
 499
 500                 ecode = parse(key, d, data);
 501                 ntdb_access_release(ntdb, keyp);
 502         }
 503
 504         ntdb_unlock_hash(ntdb, h.h, F_RDLCK);
 505         return ecode;
 506 }
 507
 508 _PUBLIC_ const char *ntdb_name(const struct ntdb_context *ntdb)
 509 {
 510         return ntdb->name;
 511 }
 512
 513 _PUBLIC_ int64_t ntdb_get_seqnum(struct ntdb_context *ntdb)
 514 {
 515         return ntdb_read_off(ntdb, offsetof(struct ntdb_header, seqnum));
 516 }
 517
 518
 519 _PUBLIC_ int ntdb_fd(const struct ntdb_context *ntdb)
 520 {
 521         return ntdb->file->fd;
 522 }
 523
 524 struct traverse_state {
 525         enum NTDB_ERROR error;
 526         struct ntdb_context *dest_db;
 527 };
 528
 529 /*
 530   traverse function for repacking
 531  */
 532 static int repack_traverse(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA data,
 533                            struct traverse_state *state)
 534 {
 535         state->error = ntdb_store(state->dest_db, key, data, NTDB_INSERT);
 536         if (state->error != NTDB_SUCCESS) {
 537                 return -1;
 538         }
 539         return 0;
 540 }
 541
 542 _PUBLIC_ enum NTDB_ERROR ntdb_repack(struct ntdb_context *ntdb)
 543 {
 544         struct ntdb_context *tmp_db;
 545         struct traverse_state state;
 546
 547         state.error = ntdb_transaction_start(ntdb);
 548         if (state.error != NTDB_SUCCESS) {
 549                 return state.error;
 550         }
 551
 552         tmp_db = ntdb_open("tmpdb", NTDB_INTERNAL, O_RDWR|O_CREAT, 0, NULL);
 553         if (tmp_db == NULL) {
 554                 state.error = ntdb_logerr(ntdb, NTDB_ERR_OOM, NTDB_LOG_ERROR,
 555                                          __location__
 556                                          " Failed to create tmp_db");
 557                 ntdb_transaction_cancel(ntdb);
 558                 return state.error;
 559         }
 560
 561         state.dest_db = tmp_db;
 562         if (ntdb_traverse(ntdb, repack_traverse, &state) < 0) {
 563                 goto fail;
 564         }
 565
 566         state.error = ntdb_wipe_all(ntdb);
 567         if (state.error != NTDB_SUCCESS) {
 568                 goto fail;
 569         }
 570
 571         state.dest_db = ntdb;
 572         if (ntdb_traverse(tmp_db, repack_traverse, &state) < 0) {
 573                 goto fail;
 574         }
 575
 576         ntdb_close(tmp_db);
 577         return ntdb_transaction_commit(ntdb);
 578
 579 fail:
 580         ntdb_transaction_cancel(ntdb);
 581         ntdb_close(tmp_db);
 582         return state.error;
 583 }