release/src/router/mysql/server-tools/instance-manager/guardian.cc

   1 /* Copyright (c) 2004-2007 MySQL AB
   2
   3    This program is free software; you can redistribute it and/or modify
   4    it under the terms of the GNU General Public License as published by
   5    the Free Software Foundation; version 2 of the License.
   6
   7    This program is distributed in the hope that it will be useful,
   8    but WITHOUT ANY WARRANTY; without even the implied warranty of
   9    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10    GNU General Public License for more details.
  11
  12    You should have received a copy of the GNU General Public License
  13    along with this program; if not, write to the Free Software
  14    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA */
  15
  16
  17 #if defined(__GNUC__) && defined(USE_PRAGMA_IMPLEMENTATION)
  18 #pragma implementation
  19 #endif
  20
  21 #include "guardian.h"
  22 #include <string.h>
  23 #include <sys/types.h>
  24 #include <signal.h>
  25
  26 #include "instance.h"
  27 #include "instance_map.h"
  28 #include "log.h"
  29 #include "mysql_manager_error.h"
  30 #include "options.h"
  31
  32
  33 /*************************************************************************
  34  {{{ Constructor & destructor.
  35 *************************************************************************/
  36
  37 /**
  38   Guardian constructor.
  39
  40   SYNOPSIS
  41     Guardian()
  42     thread_registry_arg
  43     instance_map_arg
  44
  45   DESCRIPTION
  46     Nominal contructor intended for assigning references and initialize
  47     trivial objects. Real initialization is made by init() method.
  48 */
  49
  50 Guardian::Guardian(Thread_registry *thread_registry_arg,
  51                    Instance_map *instance_map_arg)
  52   :shutdown_requested(FALSE),
  53   stopped(FALSE),
  54   thread_registry(thread_registry_arg),
  55   instance_map(instance_map_arg)
  56 {
  57   pthread_mutex_init(&LOCK_guardian, 0);
  58   pthread_cond_init(&COND_guardian, 0);
  59 }
  60
  61
  62 Guardian::~Guardian()
  63 {
  64   /*
  65     NOTE: it's necessary to synchronize here, because Guiardian thread can be
  66     still alive an hold the mutex (because it is detached and we have no
  67     control over it).
  68   */
  69
  70   lock();
  71   unlock();
  72
  73   pthread_mutex_destroy(&LOCK_guardian);
  74   pthread_cond_destroy(&COND_guardian);
  75 }
  76
  77 /*************************************************************************
  78   }}}
  79 *************************************************************************/
  80
  81
  82 /**
  83   Send request to stop Guardian.
  84
  85   SYNOPSIS
  86     request_shutdown()
  87 */
  88
  89 void Guardian::request_shutdown()
  90 {
  91   stop_instances();
  92
  93   lock();
  94   shutdown_requested= TRUE;
  95   unlock();
  96
  97   ping();
  98 }
  99
 100
 101 /**
 102   Process an instance.
 103
 104   SYNOPSIS
 105     process_instance()
 106     instance  a pointer to the instance for processing
 107
 108   MT-NOTE:
 109     - the given instance must be locked before calling this operation;
 110     - Guardian must be locked before calling this operation.
 111 */
 112
 113 void Guardian::process_instance(Instance *instance)
 114 {
 115   int restart_retry= 100;
 116   time_t current_time= time(NULL);
 117
 118   if (instance->get_state() == Instance::STOPPING)
 119   {
 120     /* This brach is executed during shutdown. */
 121
 122     /* This returns TRUE if and only if an instance was stopped for sure. */
 123     if (instance->is_crashed())
 124     {
 125       log_info("Guardian: '%s' stopped.",
 126                (const char *) instance->get_name()->str);
 127
 128       instance->set_state(Instance::STOPPED);
 129     }
 130     else if ((uint) (current_time - instance->last_checked) >=
 131              instance->options.get_shutdown_delay())
 132     {
 133       log_info("Guardian: '%s' hasn't stopped within %d secs.",
 134                (const char *) instance->get_name()->str,
 135                (int) instance->options.get_shutdown_delay());
 136
 137       instance->kill_mysqld(SIGKILL);
 138
 139       log_info("Guardian: pretend that '%s' is killed.",
 140                (const char *) instance->get_name()->str);
 141
 142       instance->set_state(Instance::STOPPED);
 143     }
 144     else
 145     {
 146       log_info("Guardian: waiting for '%s' to stop (%d secs left).",
 147                (const char *) instance->get_name()->str,
 148                (int) (instance->options.get_shutdown_delay() -
 149                       current_time + instance->last_checked));
 150     }
 151
 152     return;
 153   }
 154
 155   if (instance->is_mysqld_running())
 156   {
 157     /* The instance can be contacted  on it's port */
 158
 159     /* If STARTING also check that pidfile has been created */
 160     if (instance->get_state() == Instance::STARTING &&
 161         instance->options.load_pid() == 0)
 162     {
 163       /* Pid file not created yet, don't go to STARTED state yet  */
 164     }
 165     else if (instance->get_state() != Instance::STARTED)
 166     {
 167       /* clear status fields */
 168       log_info("Guardian: '%s' is running, set state to STARTED.",
 169                (const char *) instance->options.instance_name.str);
 170       instance->reset_stat();
 171       instance->set_state(Instance::STARTED);
 172     }
 173   }
 174   else
 175   {
 176     switch (instance->get_state()) {
 177     case Instance::NOT_STARTED:
 178       log_info("Guardian: starting '%s'...",
 179                (const char *) instance->options.instance_name.str);
 180
 181       /* NOTE: set state to STARTING _before_ start() is called. */
 182       instance->set_state(Instance::STARTING);
 183       instance->last_checked= current_time;
 184
 185       instance->start_mysqld();
 186
 187       return;
 188
 189     case Instance::STARTED:     /* fallthrough */
 190     case Instance::STARTING:    /* let the instance start or crash */
 191       if (!instance->is_crashed())
 192         return;
 193
 194       instance->crash_moment= current_time;
 195       instance->last_checked= current_time;
 196       instance->set_state(Instance::JUST_CRASHED);
 197       /* fallthrough -- restart an instance immediately */
 198
 199     case Instance::JUST_CRASHED:
 200       if (current_time - instance->crash_moment <= 2)
 201       {
 202         if (instance->is_crashed())
 203         {
 204           instance->start_mysqld();
 205           log_info("Guardian: starting '%s'...",
 206                    (const char *) instance->options.instance_name.str);
 207         }
 208       }
 209       else
 210         instance->set_state(Instance::CRASHED);
 211
 212       return;
 213
 214     case Instance::CRASHED:    /* just regular restarts */
 215       if ((ulong) (current_time - instance->last_checked) <=
 216           (ulong) Options::Main::monitoring_interval)
 217         return;
 218
 219       if (instance->restart_counter < restart_retry)
 220       {
 221         if (instance->is_crashed())
 222         {
 223           instance->start_mysqld();
 224           instance->last_checked= current_time;
 225
 226           log_info("Guardian: restarting '%s'...",
 227                    (const char *) instance->options.instance_name.str);
 228         }
 229       }
 230       else
 231       {
 232         log_info("Guardian: can not start '%s'. "
 233                  "Abandoning attempts to (re)start it",
 234                  (const char *) instance->options.instance_name.str);
 235
 236         instance->set_state(Instance::CRASHED_AND_ABANDONED);
 237       }
 238
 239       return;
 240
 241     case Instance::CRASHED_AND_ABANDONED:
 242       return; /* do nothing */
 243
 244     default:
 245       DBUG_ASSERT(0);
 246     }
 247   }
 248 }
 249
 250
 251 /**
 252   Main function of Guardian thread.
 253
 254   SYNOPSIS
 255     run()
 256
 257   DESCRIPTION
 258     Check for all guarded instances and restart them if needed.
 259 */
 260
 261 void Guardian::run()
 262 {
 263   struct timespec timeout;
 264
 265   log_info("Guardian: started.");
 266
 267   thread_registry->register_thread(&thread_info);
 268
 269   /* Loop, until all instances were shut down at the end. */
 270
 271   while (true)
 272   {
 273     Instance_map::Iterator instances_it(instance_map);
 274     Instance *instance;
 275     bool all_instances_stopped= TRUE;
 276
 277     instance_map->lock();
 278
 279     while ((instance= instances_it.next()))
 280     {
 281       instance->lock();
 282
 283       if (!instance->is_guarded() ||
 284           instance->get_state() == Instance::STOPPED)
 285       {
 286         instance->unlock();
 287         continue;
 288       }
 289
 290       process_instance(instance);
 291
 292       if (instance->get_state() != Instance::STOPPED)
 293         all_instances_stopped= FALSE;
 294
 295       instance->unlock();
 296     }
 297
 298     instance_map->unlock();
 299
 300     lock();
 301
 302     if (shutdown_requested && all_instances_stopped)
 303     {
 304       log_info("Guardian: all guarded mysqlds stopped.");
 305
 306       stopped= TRUE;
 307       unlock();
 308       break;
 309     }
 310
 311     set_timespec(timeout, Options::Main::monitoring_interval);
 312
 313     thread_registry->cond_timedwait(&thread_info, &COND_guardian,
 314                                     &LOCK_guardian, &timeout);
 315     unlock();
 316   }
 317
 318   log_info("Guardian: stopped.");
 319
 320   /* Now, when the Guardian is stopped we can stop the IM. */
 321
 322   thread_registry->unregister_thread(&thread_info);
 323   thread_registry->request_shutdown();
 324
 325   log_info("Guardian: finished.");
 326 }
 327
 328
 329 /**
 330   Return the value of stopped flag.
 331 */
 332
 333 bool Guardian::is_stopped()
 334 {
 335   int var;
 336
 337   lock();
 338   var= stopped;
 339   unlock();
 340
 341   return var;
 342 }
 343
 344
 345 /**
 346   Wake up Guardian thread.
 347
 348   MT-NOTE: though usually the mutex associated with condition variable should
 349   be acquired before signalling the variable, here this is not needed.
 350   Signalling under locked mutex is used to avoid lost signals. In the current
 351   logic however locking mutex does not guarantee that the signal will not be
 352   lost.
 353 */
 354
 355 void Guardian::ping()
 356 {
 357   pthread_cond_signal(&COND_guardian);
 358 }
 359
 360
 361 /**
 362   Prepare list of instances.
 363
 364   SYNOPSIS
 365     init()
 366
 367   MT-NOTE: Instance Map must be locked before calling the operation.
 368 */
 369
 370 void Guardian::init()
 371 {
 372   Instance *instance;
 373   Instance_map::Iterator iterator(instance_map);
 374
 375   while ((instance= iterator.next()))
 376   {
 377     instance->lock();
 378
 379     instance->reset_stat();
 380     instance->set_state(Instance::NOT_STARTED);
 381
 382     instance->unlock();
 383   }
 384 }
 385
 386
 387 /**
 388   An internal method which is called at shutdown to unregister instances and
 389   attempt to stop them if requested.
 390
 391   SYNOPSIS
 392     stop_instances()
 393
 394   DESCRIPTION
 395     Loops through the guarded_instances list and prepares them for shutdown.
 396     For each instance we issue a stop command and change the state
 397     accordingly.
 398
 399   NOTE
 400     Guardian object should be locked by the caller.
 401
 402 */
 403
 404 void Guardian::stop_instances()
 405 {
 406   static const int NUM_STOP_ATTEMPTS = 100;
 407
 408   Instance_map::Iterator instances_it(instance_map);
 409   Instance *instance;
 410
 411   instance_map->lock();
 412
 413   while ((instance= instances_it.next()))
 414   {
 415     instance->lock();
 416
 417     if (!instance->is_guarded() ||
 418         instance->get_state() == Instance::STOPPED)
 419     {
 420       instance->unlock();
 421       continue;
 422     }
 423
 424     /*
 425       If instance is running or was running (and now probably hanging),
 426       request stop.
 427     */
 428
 429     if (instance->is_mysqld_running() ||
 430         instance->get_state() == Instance::STARTED)
 431     {
 432       instance->set_state(Instance::STOPPING);
 433       instance->last_checked= time(NULL);
 434     }
 435     else
 436     {
 437       /* Otherwise mark it as STOPPED. */
 438       instance->set_state(Instance::STOPPED);
 439     }
 440
 441     /* Request mysqld to stop. */
 442
 443     bool instance_stopped= FALSE;
 444
 445     for (int cur_attempt= 0; cur_attempt < NUM_STOP_ATTEMPTS; ++cur_attempt)
 446     {
 447       if (!instance->kill_mysqld(SIGTERM))
 448       {
 449         instance_stopped= TRUE;
 450         break;
 451       }
 452
 453       if (!instance->is_active())
 454       {
 455         instance_stopped= TRUE;
 456         break;
 457       }
 458
 459       /* Sleep for 0.3 sec and check again. */
 460
 461       my_sleep(300000);
 462     }
 463
 464     /*
 465       Abort if we failed to stop mysqld instance. That should not happen,
 466       but if it happened, we don't know what to do and prefer to have clear
 467       failure with coredump.
 468     */
 469
 470     DBUG_ASSERT(instance_stopped);
 471
 472     instance->unlock();
 473   }
 474
 475   instance_map->unlock();
 476 }
 477
 478
 479 /**
 480   Lock Guardian.
 481 */
 482
 483 void Guardian::lock()
 484 {
 485   pthread_mutex_lock(&LOCK_guardian);
 486 }
 487
 488
 489 /**
 490   Unlock Guardian.
 491 */
 492
 493 void Guardian::unlock()
 494 {
 495   pthread_mutex_unlock(&LOCK_guardian);
 496 }