server/ctdb_freeze.c

   1 /*
   2    ctdb freeze handling
   3
   4    Copyright (C) Andrew Tridgell  2007
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19 #include "includes.h"
  20 #include "lib/events/events.h"
  21 #include "lib/tdb/include/tdb.h"
  22 #include "system/network.h"
  23 #include "system/filesys.h"
  24 #include "system/wait.h"
  25 #include "../include/ctdb_private.h"
  26 #include "lib/util/dlinklist.h"
  27 #include "db_wrap.h"
  28
  29
  30 /*
  31   lock all databases
  32  */
  33 static int ctdb_lock_all_databases(struct ctdb_context *ctdb, uint32_t priority)
  34 {
  35         struct ctdb_db_context *ctdb_db;
  36         /* REMOVE later */
  37         /* This double loop is for backward compatibility and deadlock
  38            avoidance for old samba versions that not yet support
  39            the set prio call.
  40            This code shall be removed later
  41         */
  42         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
  43                 if (ctdb_db->priority != priority) {
  44                         continue;
  45                 }
  46                 if (strstr(ctdb_db->db_name, "notify") != NULL) {
  47                         continue;
  48                 }
  49                 DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
  50                 if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
  51                         return -1;
  52                 }
  53         }
  54         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
  55                 if (ctdb_db->priority != priority) {
  56                         continue;
  57                 }
  58                 if (strstr(ctdb_db->db_name, "notify") == NULL) {
  59                         continue;
  60                 }
  61                 DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
  62                 if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
  63                         return -1;
  64                 }
  65         }
  66         return 0;
  67 }
  68
  69 /*
  70   a list of control requests waiting for a freeze lock child to get
  71   the database locks
  72  */
  73 struct ctdb_freeze_waiter {
  74         struct ctdb_freeze_waiter *next, *prev;
  75         struct ctdb_context *ctdb;
  76         struct ctdb_req_control *c;
  77         uint32_t priority;
  78         int32_t status;
  79 };
  80
  81 /* a handle to a freeze lock child process */
  82 struct ctdb_freeze_handle {
  83         struct ctdb_context *ctdb;
  84         uint32_t priority;
  85         pid_t child;
  86         int fd;
  87         struct ctdb_freeze_waiter *waiters;
  88 };
  89
  90 /*
  91   destroy a freeze handle
  92  */
  93 static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
  94 {
  95         struct ctdb_context *ctdb = h->ctdb;
  96         struct ctdb_db_context *ctdb_db;
  97
  98         DEBUG(DEBUG_ERR,("Release freeze handler for prio %u\n", h->priority));
  99
 100         /* cancel any pending transactions */
 101         if (ctdb->freeze_transaction_started) {
 102                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 103                         if (ctdb_db->priority != h->priority) {
 104                                 continue;
 105                         }
 106                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 107                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 108                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 109                                          ctdb_db->db_name));
 110                         }
 111                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 112                 }
 113                 ctdb->freeze_transaction_started = false;
 114         }
 115
 116         ctdb->freeze_mode[h->priority]    = CTDB_FREEZE_NONE;
 117         ctdb->freeze_handles[h->priority] = NULL;
 118
 119         kill(h->child, SIGKILL);
 120         return 0;
 121 }
 122
 123 /*
 124   called when the child writes its status to us
 125  */
 126 static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *fde,
 127                                        uint16_t flags, void *private_data)
 128 {
 129         struct ctdb_freeze_handle *h = talloc_get_type(private_data, struct ctdb_freeze_handle);
 130         int32_t status;
 131         struct ctdb_freeze_waiter *w;
 132
 133         if (h->ctdb->freeze_mode[h->priority] == CTDB_FREEZE_FROZEN) {
 134                 DEBUG(DEBUG_INFO,("freeze child died - unfreezing\n"));
 135                 talloc_free(h);
 136                 return;
 137         }
 138
 139         if (read(h->fd, &status, sizeof(status)) != sizeof(status)) {
 140                 DEBUG(DEBUG_ERR,("read error from freeze lock child\n"));
 141                 status = -1;
 142         }
 143
 144         if (status == -1) {
 145                 DEBUG(DEBUG_ERR,("Failed to get locks in ctdb_freeze_child\n"));
 146                 /* we didn't get the locks - destroy the handle */
 147                 talloc_free(h);
 148                 return;
 149         }
 150
 151         h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
 152
 153         /* notify the waiters */
 154         if (h != h->ctdb->freeze_handles[h->priority]) {
 155                 DEBUG(DEBUG_ERR,("lockwait finished but h is not linked\n"));
 156         }
 157         while ((w = h->waiters)) {
 158                 w->status = status;
 159                 DLIST_REMOVE(h->waiters, w);
 160                 talloc_free(w);
 161         }
 162 }
 163
 164 /*
 165   create a child which gets locks on all the open databases, then calls the callback telling the parent
 166   that it is done
 167  */
 168 static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, uint32_t priority)
 169 {
 170         struct ctdb_freeze_handle *h;
 171         int fd[2];
 172         struct fd_event *fde;
 173
 174         h = talloc_zero(ctdb, struct ctdb_freeze_handle);
 175         CTDB_NO_MEMORY_NULL(ctdb, h);
 176
 177         h->ctdb     = ctdb;
 178         h->priority = priority;
 179
 180         /* use socketpair() instead of pipe() so we have bi-directional fds */
 181         if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != 0) {
 182                 DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
 183                 talloc_free(h);
 184                 return NULL;
 185         }
 186
 187         h->child = fork();
 188         if (h->child == -1) {
 189                 DEBUG(DEBUG_ERR,("Failed to fork child for ctdb_freeze_lock\n"));
 190                 talloc_free(h);
 191                 return NULL;
 192         }
 193
 194         if (h->child == 0) {
 195                 int ret;
 196                 int count = 0;
 197                 /* in the child */
 198                 close(fd[0]);
 199                 ret = ctdb_lock_all_databases(ctdb, priority);
 200                 if (ret != 0) {
 201                         _exit(0);
 202                 }
 203
 204                 alarm(30);
 205
 206                 while (count++ < 30) {
 207                         ret = write(fd[1], &ret, sizeof(ret));
 208                         if (ret == sizeof(ret)) {
 209                                 break;
 210                         }
 211                         DEBUG(DEBUG_ERR, (__location__ " Failed to write to socket from freeze child. ret:%d errno:%u\n", ret, errno));
 212                         sleep (1);
 213                 }
 214                 if (count >= 30) {
 215                         DEBUG(DEBUG_ERR, (__location__ " Failed to write to socket from freeze child. Aborting freeze child\n"));
 216                         _exit(0);
 217                 }
 218
 219                 /* the read here means we will die if the parent exits */
 220                 read(fd[1], &ret, sizeof(ret));
 221                 _exit(0);
 222         }
 223
 224         talloc_set_destructor(h, ctdb_freeze_handle_destructor);
 225
 226         close(fd[1]);
 227
 228         h->fd = fd[0];
 229
 230         fde = event_add_fd(ctdb->ev, h, h->fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
 231                            ctdb_freeze_lock_handler, h);
 232         if (fde == NULL) {
 233                 DEBUG(DEBUG_ERR,("Failed to setup fd event for ctdb_freeze_lock\n"));
 234                 close(fd[0]);
 235                 talloc_free(h);
 236                 return NULL;
 237         }
 238
 239         return h;
 240 }
 241
 242 /*
 243   destroy a waiter for a freeze mode change
 244  */
 245 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 246 {
 247         ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
 248         return 0;
 249 }
 250
 251 /*
 252   start the freeze process for a certain priority
 253  */
 254 int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 255 {
 256         if (priority == 0) {
 257                 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
 258                 priority = 1;
 259         }
 260
 261         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 262                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 263                 return -1;
 264         }
 265
 266         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 267                 /* we're already frozen */
 268                 return 0;
 269         }
 270
 271         /* if there isn't a freeze lock child then create one */
 272         if (ctdb->freeze_handles[priority] == NULL) {
 273                 ctdb->freeze_handles[priority] = ctdb_freeze_lock(ctdb, priority);
 274                 CTDB_NO_MEMORY(ctdb, ctdb->freeze_handles[priority]);
 275                 ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
 276         }
 277
 278         return 0;
 279 }
 280
 281 /*
 282   freeze the databases
 283  */
 284 int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
 285 {
 286         struct ctdb_freeze_waiter *w;
 287         uint32_t priority;
 288
 289         priority = (uint32_t)c->srvid;
 290
 291         DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
 292
 293         if (priority == 0) {
 294                 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
 295                 priority = 1;
 296         }
 297
 298         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 299                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 300                 return -1;
 301         }
 302
 303         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 304                 /* we're already frozen */
 305                 return 0;
 306         }
 307
 308         if (ctdb_start_freeze(ctdb, priority) != 0) {
 309                 DEBUG(DEBUG_ERR,(__location__ " Failed to start freezing databases with priority %u\n", priority));
 310                 return -1;
 311         }
 312
 313         /* add ourselves to list of waiters */
 314         if (ctdb->freeze_handles[priority] == NULL) {
 315                 DEBUG(DEBUG_ERR,("No freeze lock handle when adding a waiter\n"));
 316                 return -1;
 317         }
 318
 319         w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
 320         CTDB_NO_MEMORY(ctdb, w);
 321         w->ctdb     = ctdb;
 322         w->c        = talloc_steal(w, c);
 323         w->priority = priority;
 324         w->status   = -1;
 325         talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
 326         DLIST_ADD(ctdb->freeze_handles[priority]->waiters, w);
 327
 328         /* we won't reply till later */
 329         *async_reply = True;
 330         return 0;
 331 }
 332
 333
 334 /*
 335   block until we are frozen, used during daemon startup
 336  */
 337 bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
 338 {
 339         int i;
 340
 341         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
 342                 if (ctdb_start_freeze(ctdb, i)) {
 343                         DEBUG(DEBUG_ERR,(__location__ " Failed to freeze databases of prio %u\n", i));
 344                         continue;
 345                 }
 346
 347                 /* block until frozen */
 348                 while (ctdb->freeze_mode[i] == CTDB_FREEZE_PENDING) {
 349                         event_loop_once(ctdb->ev);
 350                 }
 351         }
 352
 353         return 0;
 354 }
 355
 356
 357 static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
 358 {
 359         DEBUG(DEBUG_ERR,("Thawing priority %u\n", priority));
 360
 361         /* cancel any pending transactions */
 362         if (ctdb->freeze_transaction_started) {
 363                 struct ctdb_db_context *ctdb_db;
 364
 365                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 366                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 367                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 368                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 369                                          ctdb_db->db_name));
 370                         }
 371                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 372                 }
 373         }
 374         ctdb->freeze_transaction_started = false;
 375
 376 #if 0
 377         /* this hack can be used to get a copy of the databases at the end of a recovery */
 378         system("mkdir -p /var/ctdb.saved; /usr/bin/rsync --delete -a /var/ctdb/ /var/ctdb.saved/$$ 2>&1 > /dev/null");
 379 #endif
 380
 381 #if 0
 382         /* and this one for local testing */
 383         system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
 384 #endif
 385
 386         if (ctdb->freeze_handles[priority] != NULL) {
 387                 talloc_free(ctdb->freeze_handles[priority]);
 388                 ctdb->freeze_handles[priority] = NULL;
 389         }
 390 }
 391
 392 /*
 393   thaw the databases
 394  */
 395 int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority)
 396 {
 397
 398         if (priority > NUM_DB_PRIORITIES) {
 399                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 400                 return -1;
 401         }
 402
 403         if (priority == 0) {
 404                 int i;
 405                 for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 406                         thaw_priority(ctdb, i);
 407                 }
 408         } else {
 409                 thaw_priority(ctdb, priority);
 410         }
 411
 412         ctdb_call_resend_all(ctdb);
 413         return 0;
 414 }
 415
 416
 417 /*
 418   start a transaction on all databases - used for recovery
 419  */
 420 int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 421 {
 422         struct ctdb_db_context *ctdb_db;
 423         int i;
 424
 425         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 426                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 427                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 428                         return -1;
 429                 }
 430         }
 431
 432         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 433                 int ret;
 434
 435                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 436
 437                 if (ctdb->freeze_transaction_started) {
 438                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 439                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 440                                          ctdb_db->db_name));
 441                                 /* not a fatal error */
 442                         }
 443                 }
 444
 445                 ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
 446
 447                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 448
 449                 if (ret != 0) {
 450                         DEBUG(DEBUG_ERR,(__location__ " Failed to start transaction for db '%s'\n",
 451                                  ctdb_db->db_name));
 452                         return -1;
 453                 }
 454         }
 455
 456         ctdb->freeze_transaction_started = true;
 457         ctdb->freeze_transaction_id = id;
 458
 459         return 0;
 460 }
 461
 462 /*
 463   cancel a transaction for all databases - used for recovery
 464  */
 465 int32_t ctdb_control_transaction_cancel(struct ctdb_context *ctdb)
 466 {
 467         struct ctdb_db_context *ctdb_db;
 468
 469         DEBUG(DEBUG_ERR,(__location__ " recovery transaction cancelled called\n"));
 470
 471         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 472                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 473
 474                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 475                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",  ctdb_db->db_name));
 476                         /* not a fatal error */
 477                 }
 478
 479                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 480         }
 481
 482         ctdb->freeze_transaction_started = false;
 483
 484         return 0;
 485 }
 486
 487 /*
 488   commit transactions on all databases
 489  */
 490 int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 491 {
 492         struct ctdb_db_context *ctdb_db;
 493         int i;
 494
 495         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 496                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 497                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 498                         return -1;
 499                 }
 500         }
 501
 502         if (!ctdb->freeze_transaction_started) {
 503                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 504                 return -1;
 505         }
 506
 507         if (id != ctdb->freeze_transaction_id) {
 508                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", id));
 509                 return -1;
 510         }
 511
 512         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 513                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 514                 if (tdb_transaction_commit(ctdb_db->ltdb->tdb) != 0) {
 515                         DEBUG(DEBUG_ERR,(__location__ " Failed to commit transaction for db '%s'. Cancel all transactions and resetting transaction_started to false.\n",
 516                                  ctdb_db->db_name));
 517
 518                         /* cancel any pending transactions */
 519                         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 520                                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 521                                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 522                                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 523                                                  ctdb_db->db_name));
 524                                 }
 525                                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 526                         }
 527                         ctdb->freeze_transaction_started = false;
 528
 529                         return -1;
 530                 }
 531                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 532         }
 533
 534         ctdb->freeze_transaction_started = false;
 535         ctdb->freeze_transaction_id = 0;
 536
 537         return 0;
 538 }
 539
 540 /*
 541   wipe a database - only possible when in a frozen transaction
 542  */
 543 int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata)
 544 {
 545         struct ctdb_control_wipe_database w = *(struct ctdb_control_wipe_database *)indata.dptr;
 546         struct ctdb_db_context *ctdb_db;
 547
 548         ctdb_db = find_ctdb_db(ctdb, w.db_id);
 549         if (!ctdb_db) {
 550                 DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
 551                 return -1;
 552         }
 553
 554         if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
 555                 DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 556                 return -1;
 557         }
 558
 559         if (!ctdb->freeze_transaction_started) {
 560                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 561                 return -1;
 562         }
 563
 564         if (w.transaction_id != ctdb->freeze_transaction_id) {
 565                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
 566                 return -1;
 567         }
 568
 569         if (tdb_wipe_all(ctdb_db->ltdb->tdb) != 0) {
 570                 DEBUG(DEBUG_ERR,(__location__ " Failed to wipe database for db '%s'\n",
 571                          ctdb_db->db_name));
 572                 return -1;
 573         }
 574
 575         return 0;
 576 }