src/ck-core/ckarray.h

   1 /* Generalized Chare Arrays
   2
   3 These classes implement Chare Arrays.
   4 These are dynamic (i.e. allowing insertion
   5 and deletion) collections of ordinary Chares
   6 indexed by arbitrary runs of bytes.
   7
   8 The general structure is:
   9
  10 CkArray is the "array manager" Group, or BOC--
  11 it creates, keeps track of, and cares for all the
  12 array elements on this PE (i.e.. "local" elements).
  13 It does so using a hashtable of CkArrayRec objects--
  14 there's an entry for each local, home-here, and recently
  15 communicated remote array elements.
  16
  17 CkArrayElement is the type of the array
  18 elements (a subclass of Chare).
  19
  20 CkArrayIndex is an arbitrary run of bytes,
  21 used to index into the CkArray hashtable.
  22
  23 Converted from 1-D arrays 2/27/2000 by
  24 Orion Sky Lawlor, olawlor@acm.org
  25 */
  26 #ifndef __CKARRAY_H
  27 #define __CKARRAY_H
  28
  29 #include "cklocation.h"
  30 #include "ckmulticast.h"
  31 #include "ckmemcheckpoint.h" // for CkArrayCheckPTReqMessage
  32 #include "ckarrayindex.h"
  33
  34 /***********************************************************
  35         Utility defines, includes, etc.
  36 */
  37 extern void _registerCkArray(void);
  38 CpvExtern (int ,serializer);
  39
  40 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
  41 #define _MLOG_BCAST_TREE_ 1
  42 #define _MLOG_BCAST_BFACTOR_ 8
  43 #endif
  44
  45 /** This flag is true when in the system there is anytime migration, false when
  46  *  the user code guarantees that no migration happens except during load balancing
  47  *  (in which case it can only happen between AtSync and ResumeFromSync). */
  48 extern bool _isAnytimeMigration;
  49
  50 /**
  51   Array elements are only inserted at construction
  52  */
  53 extern bool _isStaticInsertion;
  54
  55 /** This flag is true when users are sure there is at least one charm array element
  56  *  per processor. In such case, when doing reduction on the array, the children
  57  *  don't need to be notified that reduction starts
  58  */
  59 extern bool _isNotifyChildInRed;
  60
  61 /**
  62 \addtogroup CkArray
  63 \brief Migratable Chare Arrays: user-visible classes.
  64
  65 All these classes are defined in ckarray.C.
  66 */
  67 /*@{*/
  68
  69 /********************* CkArrayListener ****************/
  70 ///An arrayListener is an object that gets informed whenever
  71 /// an array element is created, migrated, or destroyed.
  72 ///This abstract superclass just ignores everything sent to it.
  73 class ArrayElement;
  74 class CkArrayListener : public PUP::able {
  75   int nInts; //Number of ints of data to store per element
  76   int dataOffset; //Int offset of our data within the element
  77  public:
  78   CkArrayListener(int nInts_);
  79   CkArrayListener(CkMigrateMessage *m);
  80   virtual void pup(PUP::er &p);
  81   PUPable_abstract(CkArrayListener)
  82
  83   ///Register this array type.  Our data is stored in the element at dataOffset
  84   virtual void ckRegister(CkArray *arrMgr,int dataOffset_);
  85
  86   ///Return the number of ints of data to store per element
  87   inline int ckGetLen(void) const {return nInts;}
  88   ///Return the offset of our data into the element
  89   inline int ckGetOffset(void) const {return dataOffset;}
  90   ///Return our data associated with this array element
  91   inline int *ckGetData(ArrayElement *el) const;
  92
  93   ///Elements may be being created
  94   virtual void ckBeginInserting(void) {}
  95   ///No more elements will be created (for now)
  96   virtual void ckEndInserting(void) {}
  97
  98 //The stamp/creating/created/died sequence happens, in order, exactly
  99 // once per array element.  Migrations don't show up here.
 100   ///Element creation message is about to be sent
 101   virtual void ckElementStamp(int *eltInfo) { (void)eltInfo; }
 102   ///Element is about to be created on this processor
 103   virtual void ckElementCreating(ArrayElement *elt) { (void)elt; }
 104   ///Element was just created on this processor
 105   /// Return false if the element migrated away or deleted itself.
 106   virtual bool ckElementCreated(ArrayElement *elt) {
 107     (void)elt;
 108     return true;
 109   }
 110
 111   ///Element is about to be destroyed
 112   virtual void ckElementDied(ArrayElement *elt) { (void)elt; }
 113
 114 //The leaving/arriving seqeunce happens once per migration.
 115   ///Element is about to leave this processor (so about to call pup)
 116   virtual void ckElementLeaving(ArrayElement *elt) { (void)elt; }
 117
 118   ///Element just arrived on this processor (so just called pup)
 119   /// Return false if the element migrated away or deleted itself.
 120   virtual bool ckElementArriving(ArrayElement *elt) {
 121     (void)elt;
 122     return true;
 123   }
 124
 125   /// used by checkpointing to reset the states
 126   virtual void flushState()  {}
 127 };
 128
 129 /*@}*/
 130
 131 //This simple arrayListener just prints each event to stdout:
 132 class CkVerboseListener : public CkArrayListener {
 133  public:
 134   CkVerboseListener(void);
 135   CkVerboseListener(CkMigrateMessage *m):CkArrayListener(m) {}
 136   PUPable_decl(CkVerboseListener);
 137
 138   virtual void ckRegister(CkArray *arrMgr,int dataOffset_);
 139   virtual void ckBeginInserting(void);
 140   virtual void ckEndInserting(void);
 141
 142   virtual void ckElementStamp(int *eltInfo);
 143   virtual void ckElementCreating(ArrayElement *elt);
 144   virtual bool ckElementCreated(ArrayElement *elt);
 145   virtual void ckElementDied(ArrayElement *elt);
 146
 147   virtual void ckElementLeaving(ArrayElement *elt);
 148   virtual bool ckElementArriving(ArrayElement *elt);
 149 };
 150
 151 /**
 152 \addtogroup CkArray
 153 */
 154 /*@{*/
 155 /*********************** CkArrayOptions *******************************/
 156 /// Arguments for array creation:
 157 class CkArrayOptions {
 158         friend class CkArray;
 159
 160         CkArrayIndex start, end, step;
 161         CkArrayIndex numInitial;///< Number of elements to create
 162         /// Limits of element counts in each dimension of this and all bound arrays
 163         CkArrayIndex bounds;
 164         CkGroupID map;///< Array location map object
 165         CkGroupID locMgr;///< Location manager to bind to
 166         CkGroupID mCastMgr;/// <ckmulticast mgr to bind to, for sections
 167         CkPupAblePtrVec<CkArrayListener> arrayListeners; //CkArrayListeners for this array
 168         CkCallback reductionClient; // Default target of reductions
 169         bool anytimeMigration; // Elements are allowed to move freely
 170         bool disableNotifyChildInRed; //Child elements are not notified when reduction starts
 171         bool staticInsertion; // Elements are only inserted at construction
 172         bool broadcastViaScheduler;     // broadcast inline or through scheduler
 173         bool sectionAutoDelegate;  //Create a mCastMgr and auto-delegate all sections
 174
 175         /// Set various safe defaults for all the constructors
 176         void init();
 177
 178         /// Helper functions to keep numInitial and start/step/end consistent
 179         void updateIndices();
 180         void updateNumInitial();
 181
 182  public:
 183  //Used by external world:
 184         CkArrayOptions(void); ///< Default: empty array
 185         CkArrayOptions(int ni1_); ///< With initial elements 1D
 186         CkArrayOptions(int ni1_, int ni2_); ///< With initial elements 2D
 187         CkArrayOptions(int ni1_, int ni2_, int ni3); ///< With initial elements 3D
 188         CkArrayOptions(short ni1_, short ni2_, short ni3, short ni4_); ///< With initial elements 4D
 189         CkArrayOptions(short ni1_, short ni2_, short ni3, short ni4_, short ni5_); ///< With initial elements 5D
 190         CkArrayOptions(short ni1_, short ni2_, short ni3, short ni4_, short ni5_, short ni6_); ///< With initial elements 6D
 191         CkArrayOptions(CkArrayIndex s, CkArrayIndex e, CkArrayIndex step); ///< Initialize the start, end, and step
 192
 193         /**
 194          * These functions return "this" so you can string them together, e.g.:
 195          *   foo(CkArrayOptions().setMap(mid).bindTo(aid));
 196          */
 197
 198         /// Set the start, end, and step for the initial elements to populate
 199         CkArrayOptions &setStart(CkArrayIndex s)
 200                 { start = s; updateNumInitial(); return *this; }
 201         CkArrayOptions &setEnd(CkArrayIndex e)
 202                 { end = e; updateNumInitial(); return *this; }
 203         CkArrayOptions &setStep(CkArrayIndex s)
 204                 { step = s; updateNumInitial(); return *this; }
 205
 206         /// Create this many initial elements 1D
 207         CkArrayOptions &setNumInitial(int ni)
 208                 {numInitial=CkArrayIndex1D(ni); updateIndices(); return *this;}
 209         /// Create this many initial elements 2D
 210         CkArrayOptions &setNumInitial(int ni1, int ni2)
 211                 {numInitial=CkArrayIndex2D(ni1, ni2); updateIndices(); return *this;}
 212         /// Create this many initial elements 3D
 213         CkArrayOptions &setNumInitial(int ni1, int ni2, int ni3)
 214                 {numInitial=CkArrayIndex3D(ni1, ni2, ni3); updateIndices(); return *this;}
 215         /// Create this many initial elements 4D
 216         CkArrayOptions &setNumInitial(short ni1, short ni2, short ni3, short ni4)
 217                 {numInitial=CkArrayIndex4D(ni1, ni2, ni3, ni4); updateIndices(); return *this;}
 218         /// Create this many initial elements 5D
 219         CkArrayOptions &setNumInitial(short ni1, short ni2, short ni3, short ni4, short ni5)
 220                 {numInitial=CkArrayIndex5D(ni1, ni2, ni3, ni4, ni5); updateIndices(); return *this;}
 221         /// Create this many initial elements 6D
 222         CkArrayOptions &setNumInitial(short ni1, short ni2, short ni3, short ni4, short ni5, short ni6)
 223                 {numInitial=CkArrayIndex6D(ni1, ni2, ni3, ni4, ni5, ni6); updateIndices(); return *this;}
 224
 225         /// Allow up to this many elements in 1D
 226         CkArrayOptions &setBounds(int ni)
 227                 {bounds=CkArrayIndex1D(ni); return *this;}
 228         /// Allow up to this many elements in 2D
 229         CkArrayOptions &setBounds(int ni1, int ni2)
 230                 {bounds=CkArrayIndex2D(ni1, ni2); return *this;}
 231         /// Allow up to this many elements in 3D
 232         CkArrayOptions &setBounds(int ni1, int ni2, int ni3)
 233                 {bounds=CkArrayIndex3D(ni1 ,ni2, ni3); return *this;}
 234         /// Allow up to this many elements in 4D
 235         CkArrayOptions &setBounds(short ni1, short ni2, short ni3, short ni4)
 236                 {bounds=CkArrayIndex4D(ni1, ni2, ni3, ni4); return *this;}
 237         /// Allow up to this many elements in 5D
 238         CkArrayOptions &setBounds(short ni1, short ni2, short ni3, short ni4, short ni5)
 239                 {bounds=CkArrayIndex5D(ni1, ni2, ni3, ni4, ni5); return *this;}
 240         /// Allow up to this many elements in 6D
 241         CkArrayOptions &setBounds(short ni1, short ni2, short ni3, short ni4, short ni5, short ni6)
 242                 {bounds=CkArrayIndex6D(ni1, ni2, ni3, ni4, ni5, ni6); return *this;}
 243
 244         /// Use this location map
 245         CkArrayOptions &setMap(const CkGroupID &m)
 246                 {map=m; return *this;}
 247
 248         /// Bind our elements to this array
 249         CkArrayOptions &bindTo(const CkArrayID &b);
 250
 251         /// Use this location manager
 252         CkArrayOptions &setLocationManager(const CkGroupID &l)
 253                 {locMgr=l; return *this;}
 254
 255         /// Use this ckmulticast manager
 256         CkArrayOptions &setMcastManager(const CkGroupID &m)
 257                 {mCastMgr=m; return *this;}
 258
 259         /// Add an array listener component to this array (keeps the new'd listener)
 260         CkArrayOptions &addListener(CkArrayListener *listener);
 261
 262         CkArrayOptions &setAnytimeMigration(bool b) { anytimeMigration = b; return *this; }
 263         CkArrayOptions &setStaticInsertion(bool b);
 264         CkArrayOptions &setBroadcastViaScheduler(bool b) { broadcastViaScheduler = b; return *this; }
 265         CkArrayOptions &setSectionAutoDelegate(bool b) { sectionAutoDelegate = b; return *this; }
 266         CkArrayOptions &setReductionClient(CkCallback cb)
 267         { reductionClient = cb; return *this; }
 268
 269   //Used by the array manager:
 270         const CkArrayIndex &getStart(void) const {return start;}
 271         const CkArrayIndex &getEnd(void) const {return end;}
 272         const CkArrayIndex &getStep(void) const {return step;}
 273         const CkArrayIndex &getNumInitial(void) const {return numInitial;}
 274         const CkArrayIndex &getBounds(void) const {return bounds;}
 275         const CkGroupID &getMap(void) const {return map;}
 276         const CkGroupID &getLocationManager(void) const {return locMgr;}
 277         const CkGroupID &getMcastManager(void) const {return mCastMgr;}
 278         bool isSectionAutoDelegated(void) const {return sectionAutoDelegate;}
 279         int getListeners(void) const {return arrayListeners.size();}
 280         CkArrayListener *getListener(int listenerNum) {
 281                 CkArrayListener *ret=arrayListeners[listenerNum];
 282                 arrayListeners[listenerNum]=NULL; //Don't throw away this listener
 283                 return ret;
 284         }
 285
 286         void pup(PUP::er &p);
 287 };
 288 PUPmarshall(CkArrayOptions)
 289
 290
 291 /*********************** Proxy Support ************************/
 292 //Needed by CBase_ArrayElement
 293 class ArrayBase { /*empty*/ };
 294 /*forward*/ class ArrayElement;
 295
 296 /**
 297  * This class is a wrapper around a CkArrayIndex and ArrayID,
 298  *  used by array element proxies.  This makes the translator's
 299  *  job simpler, and the translated code smaller.
 300  */
 301 class CProxy_ArrayBase :public CProxy {
 302 private:
 303         CkArrayID _aid;
 304 public:
 305         CProxy_ArrayBase() {
 306 #if CMK_ERROR_CHECKING
 307                 _aid.setZero();
 308 #endif
 309         }
 310         CProxy_ArrayBase(const CkArrayID &aid,CK_DELCTOR_PARAM)
 311                 :CProxy(CK_DELCTOR_ARGS), _aid(aid) { }
 312         CProxy_ArrayBase(const CkArrayID &aid)
 313                 :CProxy(), _aid(aid) { }
 314         CProxy_ArrayBase(const ArrayElement *e);
 315         CProxy_ArrayBase(const CProxy_ArrayBase &cs): CProxy(cs), _aid(cs.ckGetArrayID()) {}
 316 #if CMK_ERROR_CHECKING
 317         inline void ckCheck(void) const{  //Make sure this proxy has a value
 318           if (_aid.isZero())
 319                 CkAbort("Error! This array proxy has not been initialized!");
 320         }
 321 #else
 322         inline void ckCheck(void) const {}
 323 #endif
 324
 325         static CkArrayID ckCreateEmptyArray(CkArrayOptions opts);
 326         static void ckCreateEmptyArrayAsync(CkCallback cb, CkArrayOptions opts);
 327         static CkArrayID ckCreateArray(CkArrayMessage *m,int ctor,const CkArrayOptions &opts);
 328
 329         void ckInsertIdx(CkArrayMessage *m,int ctor,int onPe,const CkArrayIndex &idx);
 330         void ckBroadcast(CkArrayMessage *m, int ep, int opts=0) const;
 331         CkArrayID ckGetArrayID(void) const { return _aid; }
 332         CkArray *ckLocalBranch(void) const { return _aid.ckLocalBranch(); }
 333         CkLocMgr *ckLocMgr(void) const;
 334         inline operator CkArrayID () const {return ckGetArrayID();}
 335         unsigned int numLocalElements() const { return ckLocMgr()->numLocalElements(); }
 336
 337         void doneInserting(void);
 338         void beginInserting(void);
 339
 340         CK_REDUCTION_CLIENT_DECL
 341
 342         void pup(PUP::er &p);
 343 };
 344 PUPmarshall(CProxy_ArrayBase)
 345
 346 class CProxyElement_ArrayBase:public CProxy_ArrayBase {
 347 private:
 348         CkArrayIndex _idx;//<- our element's array index
 349 public:
 350         CProxyElement_ArrayBase() { }
 351         CProxyElement_ArrayBase(const CkArrayID &aid,
 352                 const CkArrayIndex &idx,CK_DELCTOR_PARAM)
 353                 :CProxy_ArrayBase(aid,CK_DELCTOR_ARGS), _idx(idx) { }
 354         CProxyElement_ArrayBase(const CkArrayID &aid, const CkArrayIndex &idx)
 355                 :CProxy_ArrayBase(aid), _idx(idx) { }
 356         CProxyElement_ArrayBase(const ArrayElement *e);
 357
 358         void ckInsert(CkArrayMessage *m,int ctor,int onPe);
 359         void ckSend(CkArrayMessage *m, int ep, int opts = 0) const;
 360 //      static void ckSendWrapper(void *me, void *m, int ep, int opts = 0);
 361       static void ckSendWrapper(CkArrayID _aid, CkArrayIndex _idx, void *m, int ep, int opts);
 362         void *ckSendSync(CkArrayMessage *m, int ep) const;
 363         const CkArrayIndex &ckGetIndex() const {return _idx;}
 364
 365         ArrayElement *ckLocal(void) const;
 366         void pup(PUP::er &p);
 367 };
 368 PUPmarshall(CProxyElement_ArrayBase)
 369
 370
 371 #define _AUTO_DELEGATE_MCASTMGR_ON_ 1
 372
 373 class CProxySection_ArrayBase:public CProxy_ArrayBase {
 374 private:
 375         int _nsid;
 376         CkSectionID *_sid;
 377 public:
 378         CProxySection_ArrayBase(): _nsid(0), _sid(NULL) {}
 379         CProxySection_ArrayBase(const CkArrayID &aid,
 380                 const CkArrayIndex *elems, const int nElems, int factor=USE_DEFAULT_BRANCH_FACTOR);
 381         CProxySection_ArrayBase(const CkArrayID &aid,
 382                 const CkArrayIndex *elems, const int nElems, CK_DELCTOR_PARAM)
 383                 :CProxy_ArrayBase(aid,CK_DELCTOR_ARGS), _nsid(1) { _sid = new CkSectionID(aid, elems, nElems); }
 384         CProxySection_ArrayBase(const CkSectionID &sid)
 385                 :CProxy_ArrayBase(sid._cookie.get_aid()), _nsid(1) { _sid = new CkSectionID(sid); }
 386         CProxySection_ArrayBase(const CkSectionID &sid, CK_DELCTOR_PARAM)
 387                 :CProxy_ArrayBase(sid._cookie.get_aid(), CK_DELCTOR_ARGS), _nsid(1) { _sid = new CkSectionID(sid); }
 388         CProxySection_ArrayBase(const CProxySection_ArrayBase &cs)
 389                 :CProxy_ArrayBase(cs), _nsid(cs._nsid) {
 390       if (_nsid == 1) _sid = new CkSectionID(*cs._sid);
 391       else if (_nsid > 1) {
 392         _sid = new CkSectionID[_nsid];
 393         for (int i=0; i<_nsid; ++i) _sid[i] = cs._sid[i];
 394       } else _sid = NULL;
 395     }
 396         CProxySection_ArrayBase(const CProxySection_ArrayBase &cs, CK_DELCTOR_PARAM)
 397                 :CProxy_ArrayBase(cs.ckGetArrayID(),CK_DELCTOR_ARGS), _nsid(cs._nsid) {
 398       if (_nsid == 1) _sid = new CkSectionID(cs.ckGetArrayID(), cs.ckGetArrayElements(), cs.ckGetNumElements());
 399       else if (_nsid > 1) {
 400         _sid = new CkSectionID[_nsid];
 401         for (int i=0; i<_nsid; ++i) _sid[i] = cs._sid[i];
 402       } else _sid = NULL;
 403     }
 404     CProxySection_ArrayBase(const int n, const CkArrayID *aid, CkArrayIndex const * const *elems, const int *nElems, int factor=USE_DEFAULT_BRANCH_FACTOR);
 405     CProxySection_ArrayBase(const int n, const CkArrayID *aid, CkArrayIndex const * const *elems, const int *nElems,CK_DELCTOR_PARAM)
 406         :CProxy_ArrayBase(aid[0],CK_DELCTOR_ARGS), _nsid(n) {
 407       if (_nsid == 1) _sid = new CkSectionID(aid[0], elems[0], nElems[0]);
 408       else if (_nsid > 1) {
 409       _sid = new CkSectionID[n];
 410       for (int i=0; i<n; ++i) _sid[i] = CkSectionID(aid[i], elems[i], nElems[i]);
 411       } else _sid = NULL;
 412     }
 413
 414     ~CProxySection_ArrayBase() {
 415       if (_nsid == 1) delete _sid;
 416       else if (_nsid > 1) delete[] _sid;
 417     }
 418
 419     CProxySection_ArrayBase &operator=(const CProxySection_ArrayBase &cs) {
 420       CProxy_ArrayBase::operator=(cs);
 421       _nsid = cs._nsid;
 422       if (_nsid == 1) _sid = new CkSectionID(*cs._sid);
 423       else if (_nsid > 1) {
 424         _sid = new CkSectionID[_nsid];
 425         for (int i=0; i<_nsid; ++i) _sid[i] = cs._sid[i];
 426       } else _sid = NULL;
 427       return *this;
 428     }
 429
 430         void ckAutoDelegate(int opts=1);
 431         using CProxy_ArrayBase::setReductionClient; //compilation error o/w
 432         void setReductionClient(CkCallback *cb);
 433         void resetSection();
 434
 435         void ckSectionDelegate(CkDelegateMgr *d, int opts=1) {
 436            ckDelegate(d);
 437            if(opts==1)
 438               d->initDelegateMgr(this);
 439         }
 440 //      void ckInsert(CkArrayMessage *m,int ctor,int onPe);
 441         void ckSend(CkArrayMessage *m, int ep, int opts = 0) ;
 442
 443 //      ArrayElement *ckLocal(void) const;
 444     inline int ckGetNumSubSections() const { return _nsid; }
 445         inline CkSectionInfo &ckGetSectionInfo() {return _sid->_cookie;}
 446         inline CkSectionID *ckGetSectionIDs() {return _sid;}
 447         inline CkSectionID &ckGetSectionID() {return _sid[0];}
 448         inline CkSectionID &ckGetSectionID(int i) {return _sid[i];}
 449         inline CkArrayID ckGetArrayIDn(int i) const {return _sid[i]._cookie.get_aid();}
 450     inline CkArrayIndex *ckGetArrayElements() const {return _sid[0]._elems;}
 451     inline CkArrayIndex *ckGetArrayElements(int i) const {return _sid[i]._elems;}
 452     inline int ckGetNumElements() const { return _sid[0]._nElems; }
 453         inline int ckGetNumElements(int i) const { return _sid[i]._nElems; }
 454         inline int ckGetBfactor() const { return _sid[0].bfactor; }
 455         void pup(PUP::er &p);
 456 };
 457 PUPmarshall(CProxySection_ArrayBase)
 458
 459 //Simple C-like API:
 460 void CkSetMsgArrayIfNotThere(void *msg);
 461 void CkSendMsgArray(int entryIndex, void *msg, CkArrayID aID, const CkArrayIndex &idx, int opts=0);
 462 void CkSendMsgArrayInline(int entryIndex, void *msg, CkArrayID aID, const CkArrayIndex &idx, int opts=0);
 463 void CkBroadcastMsgArray(int entryIndex, void *msg, CkArrayID aID, int opts=0);
 464 void CkBroadcastMsgSection(int entryIndex, void *msg, CkSectionID sID, int opts=    0);
 465 /************************ Array Element *********************/
 466 /**
 467  *An array element is a chare that lives inside the array.
 468  *Unlike regular chares, array elements can migrate from one
 469  *Pe to another.  Each element has a unique index.
 470  */
 471
 472 class ArrayElement : public CkMigratable
 473 {
 474   friend class CkArray;
 475
 476   friend class CkArrayListener;
 477   int numInitialElements; // Number of elements created by ckNew(numElements)
 478   void initBasics(void);
 479 #ifdef _PIPELINED_ALLREDUCE_
 480 AllreduceMgr * allredMgr; // for allreduce
 481 #endif
 482 public:
 483   ArrayElement(void);
 484   ArrayElement(CkMigrateMessage *m);
 485   virtual ~ArrayElement();
 486
 487 /// Pack/unpack routine (called before and after migration)
 488   void pup(PUP::er &p);
 489
 490 //Overridden functions:
 491   /// Called by the system just before and after migration to another processor:
 492   virtual void ckAboutToMigrate(void);
 493   virtual void ckJustMigrated(void);
 494
 495   virtual void ckJustRestored(void);
 496
 497   virtual void ckDestroy(void);
 498   virtual char *ckDebugChareName(void);
 499   virtual int ckDebugChareID(char*, int);
 500
 501   /// Synonym for ckMigrate
 502   inline void migrateMe(int toPe) {ckMigrate(toPe);}
 503
 504 #ifdef _PIPELINED_ALLREDUCE_
 505         void contribute2(CkArrayIndex myIndex, int dataSize,const void *data,CkReduction::reducerType type,
 506                            const CkCallback &cb,CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1);
 507         void contribute2(int dataSize,const void *data,CkReduction::reducerType type,
 508                                         CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1);
 509         void contribute2(int dataSize,const void *data,CkReduction::reducerType type,
 510                                         const CkCallback &cb,CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1);
 511         void contribute2(CkReductionMsg *msg);
 512         void contribute2(const CkCallback &cb,CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1);
 513         void contribute2(CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1);
 514 #else
 515   CK_REDUCTION_CONTRIBUTE_METHODS_DECL
 516 #endif
 517         // for _PIPELINED_ALLREDUCE_, assembler entry method
 518         inline void defrag(CkReductionMsg* msg);
 519   inline const CkArrayID &ckGetArrayID(void) const {return thisArrayID;}
 520   inline ck::ObjID ckGetID(void) const { return ck::ObjID(thisArrayID, myRec->getID()); }
 521
 522   inline int ckGetArraySize(void) const { return numInitialElements; }
 523 protected:
 524   CkArray *thisArray;//My source array
 525   CkArrayID thisArrayID;//My source array's ID
 526
 527   //More verbose form of abort
 528   virtual void CkAbort(const char *str) const;
 529
 530 private:
 531 //Array implementation methods:
 532   int listenerData[CK_ARRAYLISTENER_MAXLEN];
 533
 534 #if CMK_MEM_CHECKPOINT
 535 friend class CkMemCheckPT;
 536 friend class CkLocMgr;
 537 protected:
 538   int budPEs[2];
 539 private:
 540   void init_checkpt();
 541 #endif
 542 public:
 543         void inmem_checkpoint(CkArrayCheckPTReqMessage *m);
 544         void recvBroadcast(CkMessage *);
 545
 546 #if CMK_GRID_QUEUE_AVAILABLE
 547 public:
 548   int grid_queue_interval;
 549   int grid_queue_threshold;
 550   int msg_count;
 551   int msg_count_grid;
 552   int border_flag;
 553 #endif
 554 };
 555 inline int *CkArrayListener::ckGetData(ArrayElement *el) const
 556   {return &el->listenerData[dataOffset];}
 557
 558 /**An ArrayElementT is a utility class where you are
 559  * constrained to a "thisIndex" of some fixed-sized type T.
 560  */
 561 template <class T>
 562 class ArrayElementT : public ArrayElement
 563 {
 564 public:
 565   ArrayElementT(void): thisIndex(*(const T *)thisIndexMax.data()) {
 566 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 567         mlogData->objID.data.array.idx=thisIndexMax;
 568 #endif
 569 }
 570 #ifdef _PIPELINED_ALLREDUCE_
 571         void contribute(int dataSize,const void *data,CkReduction::reducerType type,
 572                                                 CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1)
 573         {
 574                 contribute2( dataSize,data, type, userFlag);
 575
 576         }
 577         void contribute(int dataSize,const void *data,CkReduction::reducerType type,
 578                                                 const CkCallback &cb,CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1)
 579         {
 580                 contribute2((CkArrayIndex)(thisIndex) ,dataSize, data, type,   cb, userFlag);
 581         }
 582         void contribute(CkReductionMsg *msg)
 583         {
 584                 contribute2(msg);
 585         }
 586         void contribute(const CkCallback &cb,CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1)
 587         {
 588                 contribute2(cb ,userFlag);
 589         }
 590         void contribute(CMK_REFNUM_TYPE userFlag=(CMK_REFNUM_TYPE)-1)
 591         {
 592                 contribute2(userFlag);
 593         }
 594 #endif
 595   ArrayElementT(CkMigrateMessage *msg)
 596         :ArrayElement(msg),
 597         thisIndex(*(const T *)thisIndexMax.data()) {}
 598
 599   const T thisIndex;/// Object array index
 600 };
 601
 602 typedef ArrayElementT<CkIndex1D> ArrayElement1D;
 603 typedef ArrayElementT<CkIndex2D> ArrayElement2D;
 604 typedef ArrayElementT<CkIndex3D> ArrayElement3D;
 605 typedef ArrayElementT<CkIndex4D> ArrayElement4D;
 606 typedef ArrayElementT<CkIndex5D> ArrayElement5D;
 607 typedef ArrayElementT<CkIndex6D> ArrayElement6D;
 608 typedef ArrayElementT<CkIndexMax> ArrayElementMax;
 609
 610 /*@}*/
 611
 612
 613 /*********************** Array Manager BOC *******************/
 614 /**
 615 \addtogroup CkArrayImpl
 616 */
 617 /*@{*/
 618
 619 #include "CkArray.decl.h"
 620
 621 void CkSendAsyncCreateArray(int ctor, CkCallback cb, CkArrayOptions opts, void *ctorMsg);
 622
 623 struct CkArrayCreatedMsg : public CMessage_CkArrayCreatedMsg {
 624   CkArrayID aid;
 625   CkArrayCreatedMsg(CkArrayID _aid) : aid(_aid) { }
 626 };
 627
 628 class CkArrayBroadcaster;
 629 class CkArrayReducer;
 630
 631 void _ckArrayInit(void);
 632
 633 class CkArray : public CkReductionMgr {
 634   friend class ArrayElement;
 635   friend class CProxy_ArrayBase;
 636   friend class CProxyElement_ArrayBase;
 637
 638   CkMagicNumber<ArrayElement> magic; //To detect heap corruption
 639   CkLocMgr *locMgr;
 640   CkGroupID locMgrID;
 641   CkGroupID mCastMgrID;
 642   bool sectionAutoDelegate;
 643   CProxy_CkArray thisProxy;
 644   // Separate mapping and storing the element pointers to speed iteration in broadcast
 645   std::map<CmiUInt8, unsigned int> localElems;
 646   std::vector<CkMigratable *> localElemVec;
 647 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 648     int *children;
 649     int numChildren;
 650 #endif
 651 private:
 652   bool stableLocations;
 653
 654 public:
 655 //Array Creation:
 656   CkArray(CkArrayOptions &c,CkMarshalledMessage &initMsg);
 657   CkArray(CkMigrateMessage *m);
 658   ~CkArray();
 659   CkGroupID &getGroupID(void) {return thisgroup;}
 660   CkGroupID &getmCastMgr(void) {return mCastMgrID;}
 661   bool isSectionAutoDelegated(void) {return sectionAutoDelegate;}
 662
 663 //Access & information routines
 664   inline CkLocMgr *getLocMgr(void) {return locMgr;}
 665   inline const CkArrayIndex &getNumInitial(void) const {return numInitial;}
 666   inline int homePe(const CkArrayIndex &idx) const {return locMgr->homePe(idx);}
 667   inline int procNum(const CkArrayIndex &idx) const {return locMgr->procNum(idx);}
 668
 669   /// Return the last known processor for this array index.
 670   /// Valid for any possible array index.
 671   inline int lastKnown(const CkArrayIndex &idx) const
 672           {return locMgr->lastKnown(idx);}
 673   /// Deliver message to this element (directly if local)
 674   /// doFree if is local
 675   inline void deliver(CkMessage *m, const CkArrayIndex &idx, CkDeliver_t type,int opts=0)
 676   { locMgr->sendMsg((CkArrayMessage*)m, thisgroup, idx, type, opts); }
 677   inline int deliver(CkArrayMessage *m, CkDeliver_t type)
 678   { return locMgr->deliverMsg(m, thisgroup, m->array_element_id(), NULL, type); }
 679   /// Fetch a local element via its ID (return NULL if not local)
 680   inline ArrayElement *lookup(const CmiUInt8 id) { return (ArrayElement*) getEltFromArrMgr(id); }
 681   /// Fetch a local element via its index (return NULL if not local)
 682   inline ArrayElement *lookup(const CkArrayIndex &idx) {
 683     CkLocMgr::IdxIdMap::iterator itr = locMgr->idx2id.find(idx);
 684     if (itr == locMgr->idx2id.end())
 685       return NULL;
 686     else
 687       return (ArrayElement*) getEltFromArrMgr(itr->second);
 688   }
 689
 690   virtual CkMigratable* getEltFromArrMgr(const CmiUInt8 id) {
 691     std::map<CmiUInt8, unsigned int>::iterator itr = localElems.find(id);
 692     return ( itr == localElems.end() ? NULL : localElemVec[itr->second] );
 693   }
 694   virtual void putEltInArrMgr(const CmiUInt8 id, CkMigratable* elt)
 695   {
 696     localElems[id] = localElemVec.size();
 697     localElemVec.push_back(elt);
 698   }
 699   virtual void eraseEltFromArrMgr(const CmiUInt8 id)
 700   { localElems.erase(id); }
 701
 702   void deleteElt(const CmiUInt8 id) {
 703     std::map<CmiUInt8, unsigned int>::iterator itr = localElems.find(id);
 704     if (itr != localElems.end()) {
 705       unsigned int offset = itr->second;
 706       delete localElemVec[offset];
 707       localElems.erase(itr);
 708
 709       if (offset != localElemVec.size() - 1) {
 710         CkMigratable *moved = localElemVec[localElemVec.size()-1];
 711         localElemVec[offset] = moved;
 712         localElems[moved->ckGetID()] = offset;
 713       }
 714
 715       localElemVec.pop_back();
 716     }
 717   }
 718
 719 //Creation:
 720   /// Create-after-migrate:
 721   /// Create an uninitialized element after migration
 722   ///  The element's constructor will be called immediately after.
 723   virtual CkMigratable *allocateMigrated(int elChareType, CkElementCreation_t type);
 724   void stampListenerData(CkMigratable *elt);
 725
 726   /// Prepare creation message:
 727   void prepareCtorMsg(CkMessage *m, int listenerData[CK_ARRAYLISTENER_MAXLEN]);
 728
 729   int findInitialHostPe(const CkArrayIndex &idx, int proposedPe);
 730
 731   /// Create initial array elements:
 732   virtual void insertInitial(const CkArrayIndex &idx,void *ctorMsg);
 733   virtual void doneInserting(void);
 734   virtual void beginInserting(void);
 735   void remoteDoneInserting(void);
 736   void remoteBeginInserting(void);
 737
 738   /// Create manually:
 739   bool insertElement(CkArrayMessage *, const CkArrayIndex &idx, int listenerData[CK_ARRAYLISTENER_MAXLEN]);
 740   void insertElement(CkMarshalledMessage &, const CkArrayIndex &idx, int listenerData[CK_ARRAYLISTENER_MAXLEN]);
 741
 742 /// Demand-creation:
 743   /// Demand-create an element at this index on this processor
 744   void demandCreateElement(const CkArrayIndex &idx, int ctor, CkDeliver_t type);
 745
 746 /// Broadcast communication:
 747   void sendBroadcast(CkMessage *msg);
 748   void recvBroadcast(CkMessage *msg);
 749   void sendExpeditedBroadcast(CkMessage *msg);
 750   void recvExpeditedBroadcast(CkMessage *msg) { recvBroadcast(msg); }
 751   void recvBroadcastViaTree(CkMessage *msg);
 752
 753   /// Whole array destruction, including all elements and the group itself
 754   void ckDestroy();
 755
 756   void pup(PUP::er &p);
 757   void ckJustMigrated(void){ doneInserting(); }
 758
 759   virtual bool isArrMgr(void) {return true;}
 760
 761 private:
 762   CkArrayIndex numInitial;/// Number of initial array elements
 763   bool isInserting;/// Are we currently inserting elements?
 764
 765 /// Allocate space for a new array element
 766   ArrayElement *allocate(int elChareType, CkMessage *msg, bool fromMigration, int *listenerData);
 767
 768 //Spring cleaning
 769   void springCleaning(void);
 770   static void staticSpringCleaning(void *forWhom,double curWallTime);
 771   void setupSpringCleaning();
 772   int springCleaningCcd;
 773
 774 //ArrayListeners:
 775   CkPupAblePtrVec<CkArrayListener> listeners;
 776   int listenerDataOffset;
 777  public:
 778   void addListener(CkArrayListener *l) {
 779     l->ckRegister(this,listenerDataOffset);
 780     listenerDataOffset+=l->ckGetLen();
 781     listeners.push_back(l);
 782     if (listenerDataOffset>CK_ARRAYLISTENER_MAXLEN)
 783       CkAbort("Too much array listener data!\n"
 784               "You'll have to either use fewer array listeners, or increase the compile-time\n"
 785               "constant CK_ARRAYLISTENER_MAXLEN!\n");
 786   }
 787  private:
 788
 789   CkArrayReducer *reducer; //Read-only copy of default reducer
 790   CkArrayBroadcaster *broadcaster; //Read-only copy of default broadcaster
 791 public:
 792   void flushStates();
 793
 794 #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
 795         // the mlogft only support 1D arrays, then returning the number of elements in the first dimension
 796         virtual int numberReductionMessages(){CkAssert(CkMyPe() == 0);return numInitial.data()[0];}
 797         void broadcastHomeElements(void *data,CkLocRec *rec,CkArrayIndex *index);
 798         static void staticBroadcastHomeElements(CkArray *arr,void *data,CkLocRec *rec,CkArrayIndex *index);
 799 #endif
 800
 801         static bool isIrreducible() { return true; }
 802 };
 803
 804 // Maintain old name of former parent class for backwards API compatibility
 805 // with usage in maps' populateInitial()
 806 typedef CkArray CkArrMgr;
 807
 808 /*@}*/
 809
 810 #endif