src/ck-core/cklocation.h

   1 /*
   2 Location manager: keeps track of an indexed set of migratable
   3 objects.  Used by the array manager to locate array elements,
   4 interact with the load balancer, and perform migrations.
   5
   6 Does not handle reductions (see ckreduction.h), broadcasts,
   7 array proxies, or the details of element creation (see ckarray.h).
   8 */
   9 #ifndef __CKLOCATION_H
  10 #define __CKLOCATION_H
  11
  12 #include <map>
  13
  14 /*********************** Array Messages ************************/
  15 class CkArrayMessage : public CkMessage {
  16 public:
  17   //These routines are implementation utilities
  18   CmiUInt8 array_element_id(void);
  19   unsigned short &array_ep(void);
  20   unsigned short &array_ep_bcast(void);
  21   unsigned char &array_hops(void);
  22   unsigned int array_getSrcPe(void);
  23   unsigned int array_ifNotThere(void);
  24   void array_setIfNotThere(unsigned int);
  25
  26   //This allows us to delete bare CkArrayMessages
  27   void operator delete(void *p){CkFreeMsg(p);}
  28 };
  29
  30 /* Utility */
  31 //#if CMK_LBDB_ON
  32 #include "LBDatabase.h"
  33 #include "MetaBalancer.h"
  34 class LBDatabase;
  35 //#endif
  36
  37 //Forward declarations
  38 class CkArray;
  39 class ArrayElement;
  40 //What to do if an entry method is invoked on
  41 // an array element that does not (yet) exist:
  42 typedef enum {
  43         CkArray_IfNotThere_buffer=0, //Wait for it to be created
  44         CkArray_IfNotThere_createhere=1, //Make it on sending Pe
  45         CkArray_IfNotThere_createhome=2 //Make it on (a) home Pe
  46 } CkArray_IfNotThere;
  47
  48 /// How to do a message delivery:
  49 typedef enum {
  50         CkDeliver_queue=0, //Deliver via the scheduler's queue
  51         CkDeliver_inline=1  //Deliver via a regular call
  52 } CkDeliver_t;
  53
  54 class CkArrayOptions;
  55 #include "CkLocation.decl.h"
  56
  57 /************************** Array Messages ****************************/
  58 /**
  59  *  This is the message type used to actually send a migrating array element.
  60  */
  61
  62 class CkArrayElementMigrateMessage : public CMessage_CkArrayElementMigrateMessage {
  63 public:
  64   CkArrayElementMigrateMessage(CkArrayIndex idx_, CmiUInt8 id_, bool ignoreArrival_, int length_,
  65                                int nManagers_, bool bounced_)
  66     : idx(idx_), id(id_), ignoreArrival(ignoreArrival_), length(length_), nManagers(nManagers_), bounced(bounced_)
  67   { }
  68
  69         CkArrayIndex idx; // Array index that is migrating
  70         CmiUInt8 id; // ID of the elements with this index in this collection
  71         bool ignoreArrival;   // if to inform LB of arrival
  72         int length;//Size in bytes of the packed data
  73         int nManagers; // Number of associated array managers
  74 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
  75         CkGroupID gid; //gid of location manager
  76 #endif
  77         bool bounced; // Fault evac related?
  78         char* packData;
  79 };
  80
  81 /******************* Map object ******************/
  82
  83 extern CkGroupID _defaultArrayMapID;
  84 extern CkGroupID _fastArrayMapID;
  85 class CkLocMgr;
  86 class CkArray;
  87
  88 /**
  89 \addtogroup CkArray
  90 */
  91 /*@{*/
  92
  93 /** The "map" is used by the array manager to map an array index to
  94  * a home processor number.
  95  */
  96 class CkArrayMap : public IrrGroup // : public CkGroupReadyCallback
  97 {
  98 public:
  99   CkArrayMap(void);
 100   CkArrayMap(CkMigrateMessage *m): IrrGroup(m) {}
 101   virtual ~CkArrayMap();
 102   virtual int registerArray(const CkArrayIndex& numElements, CkArrayID aid);
 103   virtual void unregisterArray(int idx);
 104   virtual void populateInitial(int arrayHdl,CkArrayOptions& options,void *ctorMsg,CkArray *mgr);
 105   virtual int procNum(int arrayHdl,const CkArrayIndex &element) =0;
 106   virtual int homePe(int arrayHdl,const CkArrayIndex &element)
 107              { return procNum(arrayHdl, element); }
 108 };
 109 /*@}*/
 110
 111 /**
 112 \addtogroup CkArrayImpl
 113 \brief Migratable Chare Arrays: Implementation classes.
 114 */
 115 /*@{*/
 116 static inline CkGroupID CkCreatePropMap(void)
 117 {
 118   return CProxy_PropMap::ckNew();
 119 }
 120
 121 extern void _propMapInit(void);
 122 extern void _CkMigratable_initInfoInit(void);
 123
 124 #include "cklocrec.h"
 125
 126 /*********************** CkMigratable ******************************/
 127 /** This is the superclass of all migratable parallel objects.
 128  *  Currently, that's just array elements.
 129  */
 130 #if CMK_OUT_OF_CORE
 131 #  include "conv-ooc.h"
 132 extern CooPrefetchManager CkArrayElementPrefetcher;
 133 // If this flag is set, this creation/deletion is just
 134 //   a "fake" constructor/destructor call for prefetching.
 135 CkpvExtern(int,CkSaveRestorePrefetch);
 136 #endif
 137
 138 #include "ckmigratable.h"
 139
 140
 141
 142 /********************** CkLocMgr ********************/
 143 /// A tiny class for detecting heap corruption
 144 class CkMagicNumber_impl {
 145  protected:
 146         int magic;
 147         void badMagicNumber(int expected,const char *file,int line,void *obj) const;
 148         CkMagicNumber_impl(int m);
 149 };
 150 template<class T>
 151 class CkMagicNumber : public CkMagicNumber_impl {
 152         enum {good=sizeof(T)^0x7EDC0000};
 153  public:
 154         CkMagicNumber(void) :CkMagicNumber_impl(good) {}
 155         inline void check(const char *file,int line,void *obj) const {
 156                 if (magic!=good) badMagicNumber(good,file,line,obj);
 157         }
 158 #if CMK_ERROR_CHECKING
 159 #   define CK_MAGICNUMBER_CHECK magic.check(__FILE__,__LINE__,this);
 160 #else
 161 #   define CK_MAGICNUMBER_CHECK /*empty, for speed*/
 162 #endif
 163 };
 164
 165 /**
 166  * The "data" class passed to a CkLocIterator, which refers to a bound
 167  * glob of array elements.
 168  * This is a transient class-- do not attempt to store it or send
 169  * it across processors.
 170  */
 171
 172
 173 class CkLocation {
 174         CkLocMgr *mgr;
 175         CkLocRec *rec;
 176 public:
 177         CkLocation(CkLocMgr *mgr_, CkLocRec *rec_);
 178
 179         /// Find our location manager
 180         inline CkLocMgr *getManager(void) const {return mgr;}
 181
 182         /// Find the local record that refers to this element
 183         inline CkLocRec *getLocalRecord(void) const {return rec;}
 184
 185         /// Look up and return the array index of this location.
 186         const CkArrayIndex &getIndex(void) const;
 187         CmiUInt8 getID() const;
 188
 189         void destroyAll();
 190
 191         /// Pup all the array elements at this location.
 192         void pup(PUP::er &p);
 193 };
 194
 195 /**
 196  * This interface describes the destination for an iterator over
 197  * the locations in an array.
 198  */
 199 class CkLocIterator {
 200 public:
 201         virtual ~CkLocIterator();
 202
 203         /// This location is part of the calling location manager.
 204         virtual void addLocation(CkLocation &loc) =0;
 205 };
 206
 207 enum CkElementCreation_t {
 208   CkElementCreation_migrate=2, // Create object for normal migration arrival
 209   CkElementCreation_resume=3,  // Create object after checkpoint
 210   CkElementCreation_restore=4  // Create object after checkpoint, skip listeners
 211 };
 212
 213
 214 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 215 typedef void (*CkLocFn)(CkArray *,void *,CkLocRec *,CkArrayIndex *);
 216 #endif
 217
 218
 219 /**
 220  * A group which manages the location of an indexed set of
 221  * migratable objects.  Knows about insertions, deletions,
 222  * home processors, migration, and message forwarding.
 223  */
 224 class CkLocMgr : public IrrGroup {
 225         CkMagicNumber<CkMigratable> magic; //To detect heap corruption
 226 public:
 227         CkLocMgr(CkArrayOptions opts);
 228         CkLocMgr(CkMigrateMessage *m);
 229         ~CkLocMgr();
 230
 231         inline bool isLocMgr(void) { return true; }
 232         CkGroupID &getGroupID(void) {return thisgroup;}
 233         inline CProxy_CkLocMgr &getProxy(void) {return thisProxy;}
 234         inline CProxyElement_CkLocMgr &getLocalProxy(void) {return thislocalproxy;}
 235
 236 //Interface used by external users:
 237         /// Home mapping
 238         inline int homePe(const CkArrayIndex &idx) const {return map->homePe(mapHandle,idx);}
 239         inline int homePe(const CmiUInt8 id) const {
 240           if (compressor)
 241             return homePe(compressor->decompress(id));
 242
 243           return id >> 24;
 244         }
 245         inline int procNum(const CkArrayIndex &idx) const {return map->procNum(mapHandle,idx);}
 246         inline bool isHome (const CkArrayIndex &idx) const {return (bool)(homePe(idx)==CkMyPe());}
 247   int whichPE(const CkArrayIndex &idx) const;
 248   int whichPE(const CmiUInt8 id) const;
 249         /// Return the "last-known" location (returns a processor number)
 250         int lastKnown(const CkArrayIndex &idx);
 251         int lastKnown(CmiUInt8 id);
 252
 253         inline CmiUInt8 lookupID(const CkArrayIndex &idx) const {
 254           if (compressor) {
 255             return compressor->compress(idx);
 256           } else {
 257             CkLocMgr::IdxIdMap::const_iterator itr = idx2id.find(idx);
 258             CkAssert(itr != idx2id.end());
 259             return itr->second;
 260           }
 261         }
 262
 263         inline bool lookupID(const CkArrayIndex &idx, CmiUInt8& id) const {
 264           if (compressor) {
 265             id = compressor->compress(idx);
 266             return true;
 267           } else {
 268             CkLocMgr::IdxIdMap::const_iterator itr = idx2id.find(idx);
 269             if (itr == idx2id.end()) {
 270               return false;
 271             } else {
 272               id = itr->second;
 273               return true;
 274             }
 275           }
 276         }
 277
 278         //Look up array element in hash table.  Index out-of-bounds if not found.
 279         CkLocRec *elementRec(const CkArrayIndex &idx);
 280         //Look up array element in hash table.  Return NULL if not there.
 281         CkLocRec *elementNrec(const CmiUInt8 id);
 282
 283         /// Return true if this array element lives on another processor
 284         bool isRemote(const CkArrayIndex &idx,int *onPe) const;
 285
 286 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 287         //mark the duringMigration variable .. used for parallel restart
 288         void setDuringMigration(bool _duringMigration);
 289 #endif
 290
 291         void setDuringDestruction(bool _duringDestruction);
 292
 293         /// Pass each of our locations (each separate array index) to this destination.
 294         void iterate(CkLocIterator &dest);
 295
 296         /// Insert and unpack this array element from this checkpoint (e.g., from CkLocation::pup), skip listeners
 297         void restore(const CkArrayIndex &idx, CmiUInt8 id, PUP::er &p);
 298         /// Insert and unpack this array element from this checkpoint (e.g., from CkLocation::pup)
 299 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 300         void resume(const CkArrayIndex &idx, CmiUInt8 id, PUP::er &p, bool create, int dummy=0);
 301 #else
 302         void resume(const CkArrayIndex &idx, CmiUInt8 id, PUP::er &p, bool notify=true,bool=false);
 303 #endif
 304
 305 //Interface used by array manager and proxies
 306         /// Add a new local array manager to our list.
 307         void addManager(CkArrayID aid,CkArray *mgr);
 308         void deleteManager(CkArrayID aid, CkArray *mgr);
 309
 310         /// Populate this array with initial elements
 311         void populateInitial(CkArrayOptions& options,void *initMsg,CkArray *mgr)
 312     { map->populateInitial(mapHandle,options,initMsg,mgr); }
 313
 314         /// Add a new local array element, calling element's constructor
 315         ///  Returns true if the element was successfully added; false if the element migrated away or deleted itself.
 316         bool addElement(CkArrayID aid,const CkArrayIndex &idx, CkMigratable *elt,int ctorIdx,void *ctorMsg);
 317
 318         ///Done inserting elements for now
 319         void doneInserting(void);
 320         void startInserting(void);
 321
 322         // How many elements of each associated array are local to this PE?
 323         // If this returns n, and there are k associated arrays, that
 324         // means k*n elements are living here
 325         unsigned int numLocalElements();
 326
 327         ///Deliver message to this element:
 328         //int deliverMsg(CkMessage *m, CkArrayID mgr, const CkArrayIndex &idx, CkDeliver_t type, int opts = 0);
 329         int deliverMsg(CkArrayMessage *m, CkArrayID mgr, CmiUInt8 id, const CkArrayIndex* idx, CkDeliver_t type, int opts = 0);
 330
 331         void sendMsg(CkArrayMessage *msg, CkArrayID mgr, const CkArrayIndex &idx, CkDeliver_t type, int opts);
 332
 333 //Advisories:
 334         ///This index now lives on the given processor-- update local records
 335         void inform(const CkArrayIndex &idx, CmiUInt8 id, int nowOnPe);
 336         void inform(CmiUInt8 id, int nowOnPe);
 337
 338         ///This index now lives on the given processor-- tell the home processor
 339         void informHome(const CkArrayIndex &idx,int nowOnPe);
 340
 341         ///This message took several hops to reach us-- fix it
 342         void multiHop(CkArrayMessage *m);
 343
 344 //Interface used by CkLocRec
 345
 346         //Migrate us to another processor
 347         void emigrate(CkLocRec *rec,int toPe);
 348     void informLBPeriod(CkLocRec *rec, int lb_ideal_period);
 349     void metaLBCallLB(CkLocRec *rec);
 350
 351 #if CMK_LBDB_ON
 352         LBDatabase *getLBDB(void) const { return the_lbdb; }
 353     MetaBalancer *getMetaBalancer(void) const { return the_metalb;}
 354         const LDOMHandle &getOMHandle(void) const { return myLBHandle; }
 355 #endif
 356
 357         //This index will no longer be used-- delete the associated elements
 358         void reclaim(const CkArrayIndex &idx);
 359
 360         bool demandCreateElement(CkArrayMessage *msg, const CkArrayIndex &idx, int onPe, CkDeliver_t type);
 361         void demandCreateElement(const CkArrayIndex &idx, int chareType, int onPe, CkArrayID mgr);
 362
 363 //Communication:
 364         void immigrate(CkArrayElementMigrateMessage *msg);
 365         void requestLocation(const CkArrayIndex &idx, int peToTell, bool suppressIfHere, int ifNonExistent, int chareType, CkArrayID mgr);
 366         void requestLocation(CmiUInt8 id, int peToTell, bool suppressIfHere);
 367         void updateLocation(const CkArrayIndex &idx, CmiUInt8 id, int nowOnPe);
 368         void updateLocation(CmiUInt8 id, int nowOnPe);
 369         void reclaimRemote(const CkArrayIndex &idx,int deletedOnPe);
 370         void dummyAtSync(void);
 371
 372         /// return a list of migratables in this local record
 373         void migratableList(CkLocRec *rec, CkVec<CkMigratable *> &list);
 374
 375         void flushAllRecs(void);
 376         void flushLocalRecs(void);
 377         void pup(PUP::er &p);
 378
 379
 380 private:
 381 //Internal interface:
 382         //Add given element array record at idx, replacing the existing record
 383         void insertRec(CkLocRec *rec,const CmiUInt8 &id);
 384
 385         //Remove this entry from the table (does not delete record)
 386         void removeFromTable(const CmiUInt8 id);
 387
 388         friend class CkLocation; //so it can call pupElementsFor
 389         friend class ArrayElement;
 390         friend class MemElementPacker;
 391 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 392         void pupElementsFor(PUP::er &p,CkLocRec *rec,
 393         CkElementCreation_t type, bool create=true, int dummy=0);
 394 #else
 395         void pupElementsFor(PUP::er &p,CkLocRec *rec,
 396                 CkElementCreation_t type,bool rebuild = false);
 397 #endif
 398
 399         /// Call this member function on each element of this location:
 400         typedef void (CkMigratable::* CkMigratable_voidfn_t)(void);
 401
 402         typedef void (CkMigratable::* CkMigratable_voidfn_arg_t)(void*);
 403         void callMethod(CkLocRec *rec,CkMigratable_voidfn_arg_t fn, void*);
 404
 405         void deliverUnknown(CkArrayMessage *msg, const CkArrayIndex* idx, CkDeliver_t type, int opts);
 406         typedef std::map<CmiUInt8, std::vector<CkArrayMessage*> > MsgBuffer;
 407         /// Deliver any buffered msgs to a newly created array element
 408         void deliverAnyBufferedMsgs(CmiUInt8, MsgBuffer &buffer);
 409
 410         /// Create a new local record at this array index.
 411 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 412 CkLocRec *createLocal(const CkArrayIndex &idx,
 413         bool forMigration, bool ignoreArrival,
 414         bool notifyHome,int dummy=0);
 415 #else
 416         CkLocRec *createLocal(const CkArrayIndex &idx,
 417                 bool forMigration, bool ignoreArrival,
 418                 bool notifyHome);
 419 #endif
 420
 421         std::map<CkArrayIndex, std::vector<std::pair<int, bool> > > bufferedLocationRequests;
 422
 423 public:
 424         void callMethod(CkLocRec *rec,CkMigratable_voidfn_t fn);
 425
 426 //Data Members:
 427         //Map array ID to manager and elements
 428     std::map<CkArrayID, CkArray*> managers;
 429     // Map object ID to location
 430     std::map<CmiUInt8, int> id2pe;
 431
 432     typedef std::map<CkArrayIndex, CmiUInt8> IdxIdMap;
 433
 434     // Map array element index to object ID
 435     IdxIdMap idx2id;
 436     // Next ID to assign newly constructed array elements
 437     CmiUInt8 idCounter;
 438     CmiUInt8 getNewObjectID(const CkArrayIndex &idx);
 439
 440     /// Map idx to undelivered msgs
 441     /// @todo: We should not buffer msgs for uncreated array elements forever.
 442     /// After some timeout or other policy, we should throw errors or warnings
 443     /// or at least report and discard any msgs addressed to uncreated array elements
 444     MsgBuffer bufferedMsgs;
 445     MsgBuffer bufferedRemoteMsgs;
 446     MsgBuffer bufferedShadowElemMsgs;
 447
 448     std::map<CkArrayIndex, std::vector<CkArrayMessage *> > bufferedIndexMsgs;
 449
 450         bool addElementToRec(CkLocRec *rec,CkArray *m,
 451                 CkMigratable *elt,int ctorIdx,void *ctorMsg);
 452
 453         CProxy_CkLocMgr thisProxy;
 454         CProxyElement_CkLocMgr thislocalproxy;
 455
 456         /// This flag is set while we delete an old copy of a migrator
 457         bool duringMigration;
 458         /// This flag is set while we are deleting location manager
 459         bool duringDestruction;
 460
 461 private:
 462         /// The core of the location manager: map array index to element representative
 463         CkHashtableT<CkHashtableAdaptorT<CmiUInt8>, CkLocRec *> hash;
 464         CmiImmediateLockType hashImmLock;
 465
 466         //Map object
 467         CkGroupID mapID;
 468         int mapHandle;
 469         CkArrayMap *map;
 470
 471         CkGroupID lbdbID;
 472         CkGroupID metalbID;
 473
 474         ck::ArrayIndexCompressor *compressor;
 475         CkArrayIndex bounds;
 476         void checkInBounds(const CkArrayIndex &idx);
 477
 478 #if CMK_LBDB_ON
 479         LBDatabase *the_lbdb;
 480   MetaBalancer *the_metalb;
 481         LDBarrierClient dummyBarrierHandle;
 482         static void staticDummyResumeFromSync(void* data);
 483         void dummyResumeFromSync(void);
 484         static void staticRecvAtSync(void* data);
 485         void recvAtSync(void);
 486         LDOMHandle myLBHandle;
 487         LDBarrierReceiver lbBarrierReceiver;
 488 #endif
 489 private:
 490         void initLB(CkGroupID lbdbID, CkGroupID metalbID);
 491
 492 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 493 public:
 494         void callForAllRecords(CkLocFn,CkArray *,void *);
 495         int homeElementCount;
 496 #endif
 497
 498 };
 499
 500
 501
 502
 503
 504 /// check the command line arguments to determine if we can use ConfigurableRRMap
 505 bool haveConfigurableRRMap();
 506
 507
 508
 509
 510 /*@}*/
 511
 512 #endif /*def(thisHeader)*/