ctdb/server/ctdb_freeze.c

   1 /*
   2    ctdb freeze handling
   3
   4    Copyright (C) Andrew Tridgell  2007
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19 #include "includes.h"
  20 #include "tdb.h"
  21 #include "system/network.h"
  22 #include "system/filesys.h"
  23 #include "system/wait.h"
  24 #include "../include/ctdb_private.h"
  25 #include "lib/util/dlinklist.h"
  26 #include "db_wrap.h"
  27 #include "../common/rb_tree.h"
  28
  29 /*
  30   a list of control requests waiting for a freeze lock child to get
  31   the database locks
  32  */
  33 struct ctdb_freeze_waiter {
  34         struct ctdb_freeze_waiter *next, *prev;
  35         struct ctdb_context *ctdb;
  36         struct ctdb_req_control *c;
  37         uint32_t priority;
  38         int32_t status;
  39 };
  40
  41 /* a handle to a freeze lock child process */
  42 struct ctdb_freeze_handle {
  43         struct ctdb_context *ctdb;
  44         uint32_t priority;
  45         struct lock_request *lreq;
  46         struct ctdb_freeze_waiter *waiters;
  47 };
  48
  49 /*
  50   destroy a freeze handle
  51  */
  52 static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
  53 {
  54         struct ctdb_context *ctdb = h->ctdb;
  55         struct ctdb_db_context *ctdb_db;
  56
  57         DEBUG(DEBUG_ERR,("Release freeze handler for prio %u\n", h->priority));
  58
  59         /* cancel any pending transactions */
  60         if (ctdb->freeze_transaction_started) {
  61                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
  62                         if (ctdb_db->priority != h->priority) {
  63                                 continue;
  64                         }
  65                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
  66                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
  67                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
  68                                          ctdb_db->db_name));
  69                         }
  70                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
  71                 }
  72                 ctdb->freeze_transaction_started = false;
  73         }
  74
  75         ctdb->freeze_mode[h->priority]    = CTDB_FREEZE_NONE;
  76         ctdb->freeze_handles[h->priority] = NULL;
  77
  78         ctdb_lock_free_request_context(h->lreq);
  79         return 0;
  80 }
  81
  82 /*
  83   called when the child writes its status to us
  84  */
  85 static void ctdb_freeze_lock_handler(void *private_data, bool locked)
  86 {
  87         struct ctdb_freeze_handle *h = talloc_get_type_abort(private_data,
  88                                                              struct ctdb_freeze_handle);
  89         struct ctdb_freeze_waiter *w;
  90
  91         if (h->ctdb->freeze_mode[h->priority] == CTDB_FREEZE_FROZEN) {
  92                 DEBUG(DEBUG_INFO,("freeze child died - unfreezing\n"));
  93                 talloc_free(h);
  94                 return;
  95         }
  96
  97         if (!locked) {
  98                 DEBUG(DEBUG_ERR,("Failed to get locks in ctdb_freeze_child\n"));
  99                 /* we didn't get the locks - destroy the handle */
 100                 talloc_free(h);
 101                 return;
 102         }
 103
 104         h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
 105
 106         /* notify the waiters */
 107         if (h != h->ctdb->freeze_handles[h->priority]) {
 108                 DEBUG(DEBUG_ERR,("lockwait finished but h is not linked\n"));
 109         }
 110         while ((w = h->waiters)) {
 111                 w->status = 0;
 112                 DLIST_REMOVE(h->waiters, w);
 113                 talloc_free(w);
 114         }
 115 }
 116
 117 /*
 118   destroy a waiter for a freeze mode change
 119  */
 120 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 121 {
 122         ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
 123         return 0;
 124 }
 125
 126 /*
 127   start the freeze process for a certain priority
 128  */
 129 void ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 130 {
 131         struct ctdb_freeze_handle *h;
 132
 133         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 134                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 135                 ctdb_fatal(ctdb, "Internal error");
 136         }
 137
 138         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 139                 /* we're already frozen */
 140                 return;
 141         }
 142
 143         DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
 144
 145         /* Stop any vacuuming going on: we don't want to wait. */
 146         ctdb_stop_vacuuming(ctdb);
 147
 148         /* if there isn't a freeze lock child then create one */
 149         if (ctdb->freeze_handles[priority] == NULL) {
 150                 h = talloc_zero(ctdb, struct ctdb_freeze_handle);
 151                 CTDB_NO_MEMORY_FATAL(ctdb, h);
 152                 h->ctdb = ctdb;
 153                 h->priority = priority;
 154                 talloc_set_destructor(h, ctdb_freeze_handle_destructor);
 155
 156                 h->lreq = ctdb_lock_alldb_prio(ctdb, priority, false, ctdb_freeze_lock_handler, h);
 157                 CTDB_NO_MEMORY_FATAL(ctdb, h->lreq);
 158                 ctdb->freeze_handles[priority] = h;
 159                 ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
 160         }
 161 }
 162
 163 /*
 164   freeze the databases
 165  */
 166 int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
 167 {
 168         struct ctdb_freeze_waiter *w;
 169         uint32_t priority;
 170
 171         priority = (uint32_t)c->srvid;
 172
 173         if (priority == 0) {
 174                 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
 175                 priority = 1;
 176         }
 177
 178         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 179                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 180                 return -1;
 181         }
 182
 183         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 184                 DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
 185                 /* we're already frozen */
 186                 return 0;
 187         }
 188
 189         ctdb_start_freeze(ctdb, priority);
 190
 191         /* add ourselves to list of waiters */
 192         if (ctdb->freeze_handles[priority] == NULL) {
 193                 DEBUG(DEBUG_ERR,("No freeze lock handle when adding a waiter\n"));
 194                 return -1;
 195         }
 196
 197         w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
 198         CTDB_NO_MEMORY(ctdb, w);
 199         w->ctdb     = ctdb;
 200         w->c        = talloc_steal(w, c);
 201         w->priority = priority;
 202         w->status   = -1;
 203         talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
 204         DLIST_ADD(ctdb->freeze_handles[priority]->waiters, w);
 205
 206         /* we won't reply till later */
 207         *async_reply = true;
 208         return 0;
 209 }
 210
 211
 212 /*
 213   block until we are frozen, used during daemon startup
 214  */
 215 bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
 216 {
 217         int i;
 218
 219         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
 220                 ctdb_start_freeze(ctdb, i);
 221
 222                 /* block until frozen */
 223                 while (ctdb->freeze_mode[i] == CTDB_FREEZE_PENDING) {
 224                         event_loop_once(ctdb->ev);
 225                 }
 226         }
 227
 228         return true;
 229 }
 230
 231
 232 static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
 233 {
 234         DEBUG(DEBUG_ERR,("Thawing priority %u\n", priority));
 235
 236         /* cancel any pending transactions */
 237         if (ctdb->freeze_transaction_started) {
 238                 struct ctdb_db_context *ctdb_db;
 239
 240                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 241                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 242                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 243                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 244                                          ctdb_db->db_name));
 245                         }
 246                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 247                 }
 248         }
 249         ctdb->freeze_transaction_started = false;
 250
 251 #if 0
 252         /* this hack can be used to get a copy of the databases at the end of a recovery */
 253         system("mkdir -p /var/ctdb.saved; /usr/bin/rsync --delete -a /var/ctdb/ /var/ctdb.saved/$$ 2>&1 > /dev/null");
 254 #endif
 255
 256 #if 0
 257         /* and this one for local testing */
 258         system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
 259 #endif
 260
 261         if (ctdb->freeze_handles[priority] != NULL) {
 262                 talloc_free(ctdb->freeze_handles[priority]);
 263                 ctdb->freeze_handles[priority] = NULL;
 264         }
 265 }
 266
 267 /*
 268   thaw the databases
 269  */
 270 int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority)
 271 {
 272
 273         if (priority > NUM_DB_PRIORITIES) {
 274                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 275                 return -1;
 276         }
 277
 278         if (priority == 0) {
 279                 int i;
 280                 for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 281                         thaw_priority(ctdb, i);
 282                 }
 283         } else {
 284                 thaw_priority(ctdb, priority);
 285         }
 286
 287         ctdb_call_resend_all(ctdb);
 288         return 0;
 289 }
 290
 291
 292 /*
 293   start a transaction on all databases - used for recovery
 294  */
 295 int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 296 {
 297         struct ctdb_db_context *ctdb_db;
 298         int i;
 299
 300         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 301                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 302                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 303                         return -1;
 304                 }
 305         }
 306
 307         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 308                 int ret;
 309
 310                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 311
 312                 if (ctdb->freeze_transaction_started) {
 313                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 314                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 315                                          ctdb_db->db_name));
 316                                 /* not a fatal error */
 317                         }
 318                 }
 319
 320                 ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
 321
 322                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 323
 324                 if (ret != 0) {
 325                         DEBUG(DEBUG_ERR,(__location__ " Failed to start transaction for db '%s'\n",
 326                                  ctdb_db->db_name));
 327                         return -1;
 328                 }
 329         }
 330
 331         ctdb->freeze_transaction_started = true;
 332         ctdb->freeze_transaction_id = id;
 333
 334         return 0;
 335 }
 336
 337 /*
 338   cancel a transaction for all databases - used for recovery
 339  */
 340 int32_t ctdb_control_transaction_cancel(struct ctdb_context *ctdb)
 341 {
 342         struct ctdb_db_context *ctdb_db;
 343
 344         DEBUG(DEBUG_ERR,(__location__ " recovery transaction cancelled called\n"));
 345
 346         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 347                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 348
 349                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 350                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",  ctdb_db->db_name));
 351                         /* not a fatal error */
 352                 }
 353
 354                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 355         }
 356
 357         ctdb->freeze_transaction_started = false;
 358
 359         return 0;
 360 }
 361
 362 /*
 363   commit transactions on all databases
 364  */
 365 int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 366 {
 367         struct ctdb_db_context *ctdb_db;
 368         int i;
 369         int healthy_nodes = 0;
 370
 371         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 372                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 373                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 374                         return -1;
 375                 }
 376         }
 377
 378         if (!ctdb->freeze_transaction_started) {
 379                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 380                 return -1;
 381         }
 382
 383         if (id != ctdb->freeze_transaction_id) {
 384                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", id));
 385                 return -1;
 386         }
 387
 388         DEBUG(DEBUG_DEBUG,(__location__ " num_nodes[%d]\n", ctdb->num_nodes));
 389         for (i=0; i < ctdb->num_nodes; i++) {
 390                 DEBUG(DEBUG_DEBUG,(__location__ " node[%d].flags[0x%X]\n",
 391                                    i, ctdb->nodes[i]->flags));
 392                 if (ctdb->nodes[i]->flags == 0) {
 393                         healthy_nodes++;
 394                 }
 395         }
 396         DEBUG(DEBUG_INFO,(__location__ " healthy_nodes[%d]\n", healthy_nodes));
 397
 398         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 399                 int ret;
 400
 401                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 402                 ret = tdb_transaction_commit(ctdb_db->ltdb->tdb);
 403                 if (ret != 0) {
 404                         DEBUG(DEBUG_ERR,(__location__ " Failed to commit transaction for db '%s'. Cancel all transactions and resetting transaction_started to false.\n",
 405                                  ctdb_db->db_name));
 406                         goto fail;
 407                 }
 408                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 409
 410                 ret = ctdb_update_persistent_health(ctdb, ctdb_db, NULL, healthy_nodes);
 411                 if (ret != 0) {
 412                         DEBUG(DEBUG_CRIT,(__location__ " Failed to update persistent health for db '%s'. "
 413                                          "Cancel all remaining transactions and resetting transaction_started to false.\n",
 414                                          ctdb_db->db_name));
 415                         goto fail;
 416                 }
 417         }
 418
 419         ctdb->freeze_transaction_started = false;
 420         ctdb->freeze_transaction_id = 0;
 421
 422         return 0;
 423
 424 fail:
 425         /* cancel any pending transactions */
 426         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 427                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 428                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 429                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 430                                  ctdb_db->db_name));
 431                 }
 432                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 433         }
 434         ctdb->freeze_transaction_started = false;
 435
 436         return -1;
 437 }
 438
 439 /*
 440   wipe a database - only possible when in a frozen transaction
 441  */
 442 int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata)
 443 {
 444         struct ctdb_control_wipe_database w = *(struct ctdb_control_wipe_database *)indata.dptr;
 445         struct ctdb_db_context *ctdb_db;
 446
 447         ctdb_db = find_ctdb_db(ctdb, w.db_id);
 448         if (!ctdb_db) {
 449                 DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
 450                 return -1;
 451         }
 452
 453         if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
 454                 DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 455                 return -1;
 456         }
 457
 458         if (!ctdb->freeze_transaction_started) {
 459                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 460                 return -1;
 461         }
 462
 463         if (w.transaction_id != ctdb->freeze_transaction_id) {
 464                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
 465                 return -1;
 466         }
 467
 468         if (tdb_wipe_all(ctdb_db->ltdb->tdb) != 0) {
 469                 DEBUG(DEBUG_ERR,(__location__ " Failed to wipe database for db '%s'\n",
 470                          ctdb_db->db_name));
 471                 return -1;
 472         }
 473
 474         if (!ctdb_db->persistent) {
 475                 talloc_free(ctdb_db->delete_queue);
 476                 ctdb_db->delete_queue = trbt_create(ctdb_db, 0);
 477                 if (ctdb_db->delete_queue == NULL) {
 478                         DEBUG(DEBUG_ERR, (__location__ " Failed to re-create "
 479                                           "the vacuum tree.\n"));
 480                         return -1;
 481                 }
 482         }
 483
 484         return 0;
 485 }