release/src/router/mysql/storage/ndb/src/kernel/vm/WatchDog.cpp

   1 /* Copyright (c) 2003-2005, 2007 MySQL AB
   2
   3    This program is free software; you can redistribute it and/or modify
   4    it under the terms of the GNU General Public License as published by
   5    the Free Software Foundation; version 2 of the License.
   6
   7    This program is distributed in the hope that it will be useful,
   8    but WITHOUT ANY WARRANTY; without even the implied warranty of
   9    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10    GNU General Public License for more details.
  11
  12    You should have received a copy of the GNU General Public License
  13    along with this program; if not, write to the Free Software
  14    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA */
  15
  16
  17 #include <ndb_global.h>
  18 #include <my_pthread.h>
  19 #include <sys/times.h>
  20
  21 #include "WatchDog.hpp"
  22 #include "GlobalData.hpp"
  23 #include <NdbOut.hpp>
  24 #include <NdbSleep.h>
  25 #include <ErrorHandlingMacros.hpp>
  26 #include <EventLogger.hpp>
  27
  28 #include <NdbTick.h>
  29
  30 extern EventLogger g_eventLogger;
  31
  32 extern "C"
  33 void*
  34 runWatchDog(void* w){
  35   ((WatchDog*)w)->run();
  36   return NULL;
  37 }
  38
  39 WatchDog::WatchDog(Uint32 interval) :
  40   theIPValue(globalData.getWatchDogPtr())
  41 {
  42   setCheckInterval(interval);
  43   theStop = false;
  44   theThreadPtr = 0;
  45 }
  46
  47 WatchDog::~WatchDog(){
  48   doStop();
  49 }
  50
  51 Uint32
  52 WatchDog::setCheckInterval(Uint32 interval){
  53   // An interval of less than 70ms is not acceptable
  54   return theInterval = (interval < 70 ? 70 : interval);
  55 }
  56
  57 void
  58 WatchDog::doStart(){
  59   theStop = false;
  60   theThreadPtr = NdbThread_Create(runWatchDog,
  61                                   (void**)this,
  62                                   32768,
  63                                   "ndb_watchdog",
  64                                   NDB_THREAD_PRIO_HIGH);
  65 }
  66
  67 void
  68 WatchDog::doStop(){
  69   void *status;
  70   theStop = true;
  71   if(theThreadPtr){
  72     NdbThread_WaitFor(theThreadPtr, &status);
  73     NdbThread_Destroy(&theThreadPtr);
  74   }
  75 }
  76
  77 const char *get_action(Uint32 IPValue)
  78 {
  79   const char *action;
  80   switch (IPValue) {
  81   case 1:
  82     action = "Job Handling";
  83     break;
  84   case 2:
  85     action = "Scanning Timers";
  86     break;
  87   case 3:
  88     action = "External I/O";
  89     break;
  90   case 4:
  91     action = "Print Job Buffers at crash";
  92     break;
  93   case 5:
  94     action = "Checking connections";
  95     break;
  96   case 6:
  97     action = "Performing Send";
  98     break;
  99   case 7:
 100     action = "Polling for Receive";
 101     break;
 102   case 8:
 103     action = "Performing Receive";
 104     break;
 105   case 9:
 106     action = "Allocating memory";
 107     break;
 108   default:
 109     action = "Unknown place";
 110     break;
 111   }//switch
 112   return action;
 113 }
 114
 115 void
 116 WatchDog::run()
 117 {
 118   unsigned int anIPValue, sleep_time;
 119   unsigned int oldIPValue = 0;
 120   unsigned int theIntervalCheck = theInterval;
 121   struct MicroSecondTimer start_time, last_time, now;
 122   NdbTick_getMicroTimer(&start_time);
 123   last_time = start_time;
 124
 125   // WatchDog for the single threaded NDB
 126   while (!theStop)
 127   {
 128     sleep_time= 100;
 129
 130     NdbSleep_MilliSleep(sleep_time);
 131     if(theStop)
 132       break;
 133
 134     NdbTick_getMicroTimer(&now);
 135     if (NdbTick_getMicrosPassed(last_time, now)/1000 > sleep_time*2)
 136     {
 137       struct tms my_tms;
 138       times(&my_tms);
 139       g_eventLogger.info("Watchdog: User time: %llu  System time: %llu",
 140                          (Uint64)my_tms.tms_utime,
 141                          (Uint64)my_tms.tms_stime);
 142       g_eventLogger.warning("Watchdog: Warning overslept %u ms, expected %u ms.",
 143                             NdbTick_getMicrosPassed(last_time, now)/1000,
 144                             sleep_time);
 145     }
 146     last_time = now;
 147
 148     // Verify that the IP thread is not stuck in a loop
 149     anIPValue = *theIPValue;
 150     if (anIPValue != 0)
 151     {
 152       oldIPValue = anIPValue;
 153       globalData.incrementWatchDogCounter(0);
 154       NdbTick_getMicroTimer(&start_time);
 155       theIntervalCheck = theInterval;
 156     }
 157     else
 158     {
 159       int warn = 1;
 160       Uint32 elapsed = NdbTick_getMicrosPassed(start_time, now)/1000;
 161       /*
 162         oldIPValue == 9 indicates malloc going on, this can take some time
 163         so only warn if we pass the watchdog interval
 164       */
 165       if (oldIPValue == 9)
 166         if (elapsed < theIntervalCheck)
 167           warn = 0;
 168         else
 169           theIntervalCheck += theInterval;
 170
 171       if (warn)
 172       {
 173         const char *last_stuck_action = get_action(oldIPValue);
 174         g_eventLogger.warning("Ndb kernel is stuck in: %s", last_stuck_action);
 175         {
 176           struct tms my_tms;
 177           times(&my_tms);
 178           g_eventLogger.info("Watchdog: User time: %llu  System time: %llu",
 179                              (Uint64)my_tms.tms_utime,
 180                              (Uint64)my_tms.tms_stime);
 181         }
 182         if (elapsed > 3 * theInterval)
 183         {
 184           shutdownSystem(last_stuck_action);
 185         }
 186       }
 187     }
 188   }
 189   return;
 190 }
 191
 192 void
 193 WatchDog::shutdownSystem(const char *last_stuck_action){
 194
 195   ErrorReporter::handleError(NDBD_EXIT_WATCHDOG_TERMINATE,
 196                              last_stuck_action,
 197                              __FILE__,
 198                              NST_Watchdog);
 199 }