src/ck-ldb/LBDatabase.h

   1 /**
   2  * \addtogroup CkLdb
   3 */
   4 /*@{*/
   5
   6 #ifndef LBDATABASE_H
   7 #define LBDATABASE_H
   8
   9 #include "lbdb.h"
  10 #include "LBDBManager.h"
  11 #include "lbdb++.h"
  12
  13 #define LB_FORMAT_VERSION     3
  14
  15 class MetaBalancer;
  16 extern int _lb_version;
  17
  18 // command line options
  19 class CkLBArgs
  20 {
  21 private:
  22   double _autoLbPeriod;         // in seconds
  23   double _lb_alpha;             // per message send overhead
  24   double _lb_beta;              // per byte send overhead
  25   int _lb_debug;                // 1 or greater
  26   int _lb_printsumamry;         // print summary
  27   int _lb_loop;                 // use multiple load balancers in loop
  28   int _lb_ignoreBgLoad;
  29   int _lb_migObjOnly;           // only consider migratable objs
  30   int _lb_syncResume;
  31   int _lb_samePeSpeed;          // ignore cpu speed
  32   int _lb_testPeSpeed;          // test cpu speed
  33   int _lb_useCpuTime;           // use cpu instead of wallclock time
  34   int _lb_statson;              // stats collection
  35   int _lb_traceComm;            // stats collection for comm
  36   int _lb_central_pe;           // processor number for centralized startegy
  37   int _lb_percentMovesAllowed; //Specifies restriction on num of chares to be moved(as a percentage of total number of chares). Used by RefineKLB
  38   int _lb_teamSize;             // specifies the team size for TeamLB
  39   int _lb_maxDistPhases;  // Specifies the max number of LB phases in DistributedLB
  40   double _lb_targetRatio; // Specifies the target load ratio for LBs that aim for a particular load ratio
  41   int _lb_metaLbOn;
  42   char* _lb_metaLbModelDir;
  43
  44  public:
  45   CkLBArgs() {
  46 #if CMK_BIGSIM_CHARM
  47     _autoLbPeriod = 0.02;       // bigsim needs it to be faster (lb may hang)
  48 #else
  49     _autoLbPeriod = 0.5;        // 0.5 second default
  50 #endif
  51     _lb_debug = _lb_ignoreBgLoad = _lb_syncResume = _lb_useCpuTime = 0;
  52     _lb_printsumamry = _lb_migObjOnly = 0;
  53     _lb_statson = _lb_traceComm = 1;
  54     _lb_percentMovesAllowed=100;
  55     _lb_loop = 0;
  56     _lb_central_pe = 0;
  57     _lb_teamSize = 1;
  58     _lb_maxDistPhases = 10;
  59     _lb_targetRatio = 1.05;
  60     _lb_metaLbOn = 0;
  61     _lb_metaLbModelDir = nullptr;
  62   }
  63   inline double & lbperiod() { return _autoLbPeriod; }
  64   inline int & debug() { return _lb_debug; }
  65   inline int & teamSize() {return _lb_teamSize; }
  66   inline int & printSummary() { return _lb_printsumamry; }
  67   inline int & lbversion() { return _lb_version; }
  68   inline int & loop() { return _lb_loop; }
  69   inline int & ignoreBgLoad() { return _lb_ignoreBgLoad; }
  70   inline int & migObjOnly() { return _lb_migObjOnly; }
  71   inline int & syncResume() { return _lb_syncResume; }
  72   inline int & samePeSpeed() { return _lb_samePeSpeed; }
  73   inline int & testPeSpeed() { return _lb_testPeSpeed; }
  74   inline int & useCpuTime() { return _lb_useCpuTime; }
  75   inline int & statsOn() { return _lb_statson; }
  76   inline int & traceComm() { return _lb_traceComm; }
  77   inline int & central_pe() { return _lb_central_pe; }
  78   inline double & alpha() { return _lb_alpha; }
  79   inline double & beta() { return _lb_beta; }
  80   inline int & percentMovesAllowed() { return _lb_percentMovesAllowed;}
  81   inline int & maxDistPhases() { return _lb_maxDistPhases; }
  82   inline double & targetRatio() { return _lb_targetRatio; }
  83   inline int & metaLbOn() {return _lb_metaLbOn;}
  84   inline char*& metaLbModelDir() { return _lb_metaLbModelDir; }
  85 };
  86
  87 extern CkLBArgs _lb_args;
  88
  89 extern int _lb_predict;
  90 extern int _lb_predict_delay;
  91 extern int _lb_predict_window;
  92 extern bool _lb_psizer_on;
  93 #ifndef PREDICT_DEBUG
  94 #define PREDICT_DEBUG  0   // 0 = No debug, 1 = Debug info on
  95 #endif
  96 #define PredictorPrintf  if (PREDICT_DEBUG) CmiPrintf
  97
  98 // used in constructor of all load balancers
  99 class CkLBOptions
 100 {
 101 private:
 102   int seqno;            // for centralized lb, the seqno
 103 public:
 104   CkLBOptions(): seqno(-1) {}
 105   CkLBOptions(int s): seqno(s) {}
 106   int getSeqNo() const { return seqno; }
 107 };
 108 PUPbytes(CkLBOptions)
 109
 110 #include "LBDatabase.decl.h"
 111
 112 extern CkGroupID _lbdb;
 113
 114 class LBDB;
 115
 116 CkpvExtern(int, numLoadBalancers);
 117 CkpvExtern(bool, hasNullLB);
 118 CkpvExtern(bool, lbdatabaseInited);
 119
 120 // LB options, mostly controled by user parameter
 121 extern char * _lbtopo;
 122
 123 typedef void (*LBCreateFn)();
 124 typedef BaseLB * (*LBAllocFn)();
 125 void LBDefaultCreate(LBCreateFn f);
 126
 127 void LBRegisterBalancer(const char *, LBCreateFn, LBAllocFn, const char *, int shown=1);
 128
 129 void _LBDBInit();
 130
 131 // main chare
 132 class LBDBInit : public Chare {
 133   public:
 134     LBDBInit(CkArgMsg*);
 135     LBDBInit(CkMigrateMessage *m):Chare(m) {}
 136 };
 137
 138 // class which implement a virtual function for the FuturePredictor
 139 class LBPredictorFunction {
 140 public:
 141   virtual ~LBPredictorFunction() {}
 142   int num_params;
 143
 144   virtual void initialize_params(double *x) {double normall=1.0/pow((double)2,(double)31); for (int i=0; i<num_params; ++i) x[i]=rand()*normall;}
 145
 146   virtual double predict(double x, double *params) =0;
 147   virtual void print(double *params) {PredictorPrintf("LB: unknown model\n");};
 148   virtual void function(double x, double *param, double &y, double *dyda) =0;
 149 };
 150
 151 // a default implementation for a FuturePredictor function
 152 class DefaultFunction : public LBPredictorFunction {
 153  public:
 154   // constructor
 155   DefaultFunction() {num_params=6;};
 156
 157   // compute the prediction function for the variable x with parameters param
 158   double predict(double x, double *param) {return (param[0] + param[1]*x + param[2]*x*x + param[3]*sin(param[4]*(x+param[5])));}
 159
 160   void print(double *param) {PredictorPrintf("LB: %f + %fx + %fx^2 + %fsin%f(x+%f)\n",param[0],param[1],param[2],param[3],param[4],param[5]);}
 161
 162   // compute the prediction function and its derivatives respect to the parameters
 163   void function(double x, double *param, double &y, double *dyda) {
 164     double tmp;
 165
 166     y = predict(x, param);
 167
 168     dyda[0] = 1;
 169     dyda[1] = x;
 170     dyda[2] = x*x;
 171     tmp = param[4] * (x+param[5]);
 172     dyda[3] = sin(tmp);
 173     dyda[4] = param[3] * (x+param[5]) * cos(tmp);
 174     dyda[5] = param[3] * param[4] *cos(tmp);
 175   }
 176 };
 177
 178
 179 class LBDatabase : public IrrGroup {
 180 public:
 181   LBDatabase(void)  { init(); }
 182   LBDatabase(CkMigrateMessage *m)  { (void)m; init(); }
 183   ~LBDatabase()  { if (avail_vector) delete [] avail_vector; }
 184
 185 private:
 186   void init();
 187 public:
 188   inline static LBDatabase * Object() { return CkpvAccess(lbdatabaseInited)?(LBDatabase *)CkLocalBranch(_lbdb):NULL; }
 189 #if CMK_LBDB_ON
 190   inline LBDB *getLBDB() {return (LBDB*)(myLDHandle.handle);}
 191 #endif
 192
 193   static void initnodeFn(void);
 194
 195   void pup(PUP::er& p);
 196
 197   /*
 198    * Calls from object managers to load database
 199    */
 200   inline LDOMHandle RegisterOM(LDOMid userID, void *userptr, LDCallbacks cb) {
 201     return LDRegisterOM(myLDHandle,userID, userptr, cb);
 202   };
 203
 204   inline void UnregisterOM(LDOMHandle omHandle) {
 205     return LDUnregisterOM(myLDHandle, omHandle);
 206   };
 207
 208   inline void RegisteringObjects(LDOMHandle _om) {
 209     LDRegisteringObjects(_om);
 210   };
 211
 212   inline void DoneRegisteringObjects(LDOMHandle _om) {
 213     LDDoneRegisteringObjects(_om);
 214   };
 215
 216   void ResetAdaptive();
 217
 218   inline LDObjHandle RegisterObj(LDOMHandle h, CmiUInt8 id,
 219                           void *userptr,int migratable) {
 220     return LDRegisterObj(h,id,userptr,migratable);
 221   };
 222
 223   inline void UnregisterObj(LDObjHandle h) { LDUnregisterObj(h); };
 224
 225   inline void ObjTime(LDObjHandle h, double walltime, double cputime) {
 226     LDObjTime(h,walltime,cputime);
 227   };
 228
 229   inline void GetObjLoad(LDObjHandle &h, LBRealType &walltime, LBRealType &cputime) {
 230     LDGetObjLoad(h,&walltime,&cputime);
 231   };
 232
 233 #if CMK_LB_USER_DATA
 234   inline void *GetDBObjUserData(LDObjHandle &h, int idx)
 235   {
 236     return LDDBObjUserData(h, idx);
 237   }
 238 #endif
 239
 240   inline void QueryKnownObjLoad(LDObjHandle &h, LBRealType &walltime, LBRealType &cputime) {
 241     LDQueryKnownObjLoad(h,&walltime,&cputime);
 242   };
 243
 244   inline int RunningObject(LDObjHandle* _o) const {
 245 #if CMK_LBDB_ON
 246       LBDB *const db = (LBDB*)(myLDHandle.handle);
 247       if (db->ObjIsRunning()) {
 248         *_o = db->RunningObj();
 249         return 1;
 250       }
 251 #endif
 252       return 0;
 253       //return LDRunningObject(myLDHandle,_o);
 254   };
 255   inline const LDObjHandle *RunningObject() const {
 256 #if CMK_LBDB_ON
 257       LBDB *const db = (LBDB*)(myLDHandle.handle);
 258       if (db->ObjIsRunning()) {
 259         return &db->RunningObj();
 260       }
 261 #endif
 262       return NULL;
 263   };
 264   inline const LDObjHandle &GetObjHandle(int idx) { return LDGetObjHandle(myLDHandle, idx);}
 265   inline void ObjectStart(const LDObjHandle &_h) { LDObjectStart(_h); };
 266   inline void ObjectStop(const LDObjHandle &_h) { LDObjectStop(_h); };
 267   inline void Send(const LDOMHandle &_om, const CmiUInt8 _id, unsigned int _b, int _p, int force = 0) {
 268     LDSend(_om, _id, _b, _p, force);
 269   };
 270   inline void MulticastSend(const LDOMHandle &_om, CmiUInt8 *_ids, int _n, unsigned int _b, int _nMsgs=1) {
 271     LDMulticastSend(_om, _ids, _n, _b, _nMsgs);
 272   };
 273
 274   void EstObjLoad(const LDObjHandle &h, double cpuload);
 275   inline void NonMigratable(LDObjHandle h) { LDNonMigratable(h); };
 276   inline void Migratable(LDObjHandle h) { LDMigratable(h); };
 277   inline void setPupSize(LDObjHandle h, size_t pup_size) {LDSetPupSize(h, pup_size);};
 278   inline void UseAsyncMigrate(LDObjHandle h, bool flag) { LDAsyncMigrate(h, flag); };
 279   inline void DumpDatabase(void) { LDDumpDatabase(myLDHandle); };
 280
 281   /*
 282    * Calls from load balancer to load database
 283    */
 284   inline void NotifyMigrated(LDMigratedFn fn, void *data)
 285   {
 286     LDNotifyMigrated(myLDHandle,fn,data);
 287   };
 288
 289   inline void AddStartLBFn(LDStartLBFn fn, void *data)
 290   {
 291     LDAddStartLBFn(myLDHandle,fn,data);
 292   };
 293
 294   inline void RemoveStartLBFn(LDStartLBFn fn)
 295   {
 296     LDRemoveStartLBFn(myLDHandle,fn);
 297   };
 298
 299   inline void StartLB() { LDStartLB(myLDHandle); }
 300
 301   inline void AddMigrationDoneFn(LDMigrationDoneFn fn, void *data)
 302   {
 303     LDAddMigrationDoneFn(myLDHandle,fn,data);
 304   };
 305
 306   inline void RemoveMigrationDoneFn(LDMigrationDoneFn fn)
 307   {
 308     LDRemoveMigrationDoneFn(myLDHandle,fn);
 309   };
 310
 311   inline void MigrationDone() { LDMigrationDone(myLDHandle); }
 312
 313 public:
 314   inline void TurnManualLBOn() { LDTurnManualLBOn(myLDHandle); }
 315   inline void TurnManualLBOff() { LDTurnManualLBOff(myLDHandle); }
 316
 317   inline void PredictorOn(LBPredictorFunction *model) { LDTurnPredictorOn(myLDHandle,model); }
 318   inline void PredictorOn(LBPredictorFunction *model,int wind) { LDTurnPredictorOnWin(myLDHandle,model,wind); }
 319   inline void PredictorOff() { LDTurnPredictorOff(myLDHandle); }
 320   inline void ChangePredictor(LBPredictorFunction *model) { LDTurnPredictorOn(myLDHandle,model); }
 321
 322   inline void CollectStatsOn(void) { LDCollectStatsOn(myLDHandle); };
 323   inline void CollectStatsOff(void) { LDCollectStatsOff(myLDHandle); };
 324   inline int  CollectingStats(void) { return LDCollectingStats(myLDHandle); };
 325   inline int  CollectingCommStats(void) { return LDCollectingStats(myLDHandle) && _lb_args.traceComm(); };
 326   inline void QueryEstLoad(void) { LDQueryEstLoad(myLDHandle); };
 327
 328   inline int GetObjDataSz(void) { return LDGetObjDataSz(myLDHandle); };
 329   inline void GetObjData(LDObjData *data) { LDGetObjData(myLDHandle,data); };
 330   inline int GetCommDataSz(void) { return LDGetCommDataSz(myLDHandle); };
 331   inline void GetCommData(LDCommData *data) { LDGetCommData(myLDHandle,data); };
 332
 333   inline void GetCommInfo(int& bytes, int& msgs, int& withinbytes, int& outsidebytes, int& num_ngh, int& hops, int& hopbytes) {
 334     return LDGetCommInfo(myLDHandle, bytes, msgs, withinbytes, outsidebytes, num_ngh, hops, hopbytes);
 335   };
 336
 337   inline void BackgroundLoad(LBRealType *walltime, LBRealType *cputime) {
 338     LDBackgroundLoad(myLDHandle,walltime,cputime);
 339   }
 340
 341   inline void IdleTime(LBRealType *walltime) {
 342     LDIdleTime(myLDHandle,walltime);
 343   };
 344
 345   inline void TotalTime(LBRealType *walltime, LBRealType *cputime) {
 346     LDTotalTime(myLDHandle,walltime,cputime);
 347   }
 348
 349   inline void GetTime(LBRealType *total_walltime,LBRealType *total_cputime,
 350                    LBRealType *idletime, LBRealType *bg_walltime, LBRealType *bg_cputime) {
 351     LDGetTime(myLDHandle, total_walltime, total_cputime, idletime, bg_walltime, bg_cputime);
 352   }
 353
 354   inline void ClearLoads(void) { LDClearLoads(myLDHandle); };
 355   inline int Migrate(LDObjHandle h, int dest) { return LDMigrate(h,dest); };
 356
 357   inline void Migrated(LDObjHandle h, int waitBarrier=1) {
 358     LDMigrated(h, waitBarrier);
 359   };
 360
 361   inline LDBarrierClient AddLocalBarrierClient(LDResumeFn fn, void* data) {
 362     return LDAddLocalBarrierClient(myLDHandle,fn,data);
 363   };
 364
 365   inline void RemoveLocalBarrierClient(LDBarrierClient h) {
 366     LDRemoveLocalBarrierClient(myLDHandle, h);
 367   };
 368
 369   inline LDBarrierReceiver AddLocalBarrierReceiver(LDBarrierFn fn, void *data) {
 370     return LDAddLocalBarrierReceiver(myLDHandle,fn,data);
 371   };
 372
 373   inline void RemoveLocalBarrierReceiver(LDBarrierReceiver h) {
 374     LDRemoveLocalBarrierReceiver(myLDHandle,h);
 375   };
 376
 377   inline void AtLocalBarrier(LDBarrierClient h) {
 378     LDAtLocalBarrier(myLDHandle,h);
 379   }
 380   inline void DecreaseLocalBarrier(LDBarrierClient h, int c) {
 381     LDDecreaseLocalBarrier(myLDHandle,h,c);
 382   }
 383   inline void LocalBarrierOn(void) { LDLocalBarrierOn(myLDHandle); };
 384   inline void LocalBarrierOff(void) { LDLocalBarrierOn(myLDHandle); };
 385   void ResumeClients();
 386   inline int ProcessorSpeed() { return LDProcessorSpeed(); };
 387   inline void SetLBPeriod(double s) { LDSetLBPeriod(myLDHandle, s);}
 388   inline double GetLBPeriod() { return LDGetLBPeriod(myLDHandle);}
 389
 390   inline void MetaLBResumeWaitingChares(int lb_period) {
 391 #if CMK_LBDB_ON
 392     LDOMMetaLBResumeWaitingChares(myLDHandle, lb_period);
 393 #endif
 394   }
 395
 396   inline void MetaLBCallLBOnChares() {
 397 #if CMK_LBDB_ON
 398     LDOMMetaLBCallLBOnChares(myLDHandle);
 399 #endif
 400   }
 401
 402   void SetMigrationCost(double cost);
 403   void SetStrategyCost(double cost);
 404         void UpdateDataAfterLB(double mLoad, double mCpuLoad, double avgLoad);
 405
 406 private:
 407   int mystep;
 408   LDHandle myLDHandle;
 409   static char *avail_vector;    // processor bit vector
 410   static bool avail_vector_set;
 411   int new_ld_balancer;          // for Node 0
 412   CkVec<BaseLB *>   loadbalancers;
 413   int nloadbalancers;
 414   MetaBalancer* metabalancer;
 415
 416 public:
 417   BaseLB** getLoadBalancers() {return loadbalancers.getVec();}
 418   int getNLoadBalancers() {return nloadbalancers;}
 419
 420 public:
 421   static bool manualOn;
 422
 423 public:
 424   char *availVector() { return avail_vector; }
 425   void get_avail_vector(char * bitmap);
 426   void set_avail_vector(char * bitmap, int new_ld=-1);
 427   int & new_lbbalancer() { return new_ld_balancer; }
 428
 429   struct LastLBInfo {
 430     LBRealType *expectedLoad;
 431     LastLBInfo();
 432   };
 433   LastLBInfo lastLBInfo;
 434   inline LBRealType myExpectedLoad() { return lastLBInfo.expectedLoad[CkMyPe()]; }
 435   inline LBRealType* expectedLoad() { return lastLBInfo.expectedLoad; }
 436   inline int useMem() { return LDMemusage(myLDHandle); }
 437
 438   int getLoadbalancerTicket();
 439   void addLoadbalancer(BaseLB *lb, int seq);
 440   void nextLoadbalancer(int seq);
 441   void switchLoadbalancer(int switchFrom, int switchTo);
 442   const char *loadbalancer(int seq);
 443
 444   inline int step() { return mystep; }
 445   inline void incStep() { mystep++; }
 446 };
 447
 448 void TurnManualLBOn();
 449 void TurnManualLBOff();
 450
 451 void LBTurnPredictorOn(LBPredictorFunction *model);
 452 void LBTurnPredictorOn(LBPredictorFunction *model, int wind);
 453 void LBTurnPredictorOff();
 454 void LBChangePredictor(LBPredictorFunction *model);
 455
 456 void LBSetPeriod(double second);
 457
 458 #if CMK_LB_USER_DATA
 459 int LBRegisterObjUserData(int size);
 460 #endif
 461
 462 extern "C" void LBTurnInstrumentOn();
 463 extern "C" void LBTurnInstrumentOff();
 464 extern "C" void LBTurnCommOn();
 465 extern "C" void LBTurnCommOff();
 466 void LBClearLoads();
 467
 468 inline LBDatabase* LBDatabaseObj() { return LBDatabase::Object(); }
 469
 470 inline void CkStartLB() { LBDatabase::Object()->StartLB(); }
 471
 472 inline void get_avail_vector(char * bitmap) {
 473   LBDatabaseObj()->get_avail_vector(bitmap);
 474 }
 475
 476 inline void set_avail_vector(char * bitmap) {
 477   LBDatabaseObj()->set_avail_vector(bitmap);
 478 }
 479
 480 //  a helper class to suspend/resume load instrumentation when calling into
 481 //  runtime apis
 482
 483 class SystemLoad
 484 {
 485   const LDObjHandle *objHandle;
 486   LBDatabase *lbdb;
 487 public:
 488   SystemLoad() {
 489     lbdb = LBDatabaseObj();
 490     objHandle = lbdb->RunningObject();
 491     if (objHandle != NULL) {
 492       lbdb->ObjectStop(*objHandle);
 493     }
 494   }
 495   ~SystemLoad() {
 496     if (objHandle) lbdb->ObjectStart(*objHandle);
 497   }
 498 };
 499
 500 #define CK_RUNTIME_API          SystemLoad load_entry;
 501
 502 #endif /* LDATABASE_H */
 503
 504 /*@}*/