src/conv-core/memory-isomalloc.c

   1 /******************************************************************************
   2
   3 A migratable memory allocator.
   4
   5 NOTE: isomalloc is threadsafe, so the isomallocs are not wrapped in CmiMemLock.
   6
   7 *****************************************************************************/
   8
   9 #define CMK_ISOMALLOC_EXCLUDE_FORTRAN_CALLS   0
  10
  11 #if ! CMK_MEMORY_BUILD_OS
  12 /* Use Gnumalloc as meta-meta malloc fallbacks (mm_*) */
  13 #include "memory-gnu.c"
  14 #endif
  15
  16 #include "memory-isomalloc.h"
  17
  18 /*The current allocation arena */
  19 CpvStaticDeclare(CmiIsomallocBlockList *,isomalloc_blocklist);
  20 CpvStaticDeclare(CmiIsomallocBlockList *,pushed_blocklist);
  21
  22 #define ISOMALLOC_PUSH \
  23         CmiIsomallocBlockList *pushed_blocklist=CpvAccess(isomalloc_blocklist);\
  24         CpvAccess(isomalloc_blocklist)=NULL;\
  25
  26 #define ISOMALLOC_POP \
  27         CpvAccess(isomalloc_blocklist)=pushed_blocklist;\
  28
  29 /* temporarily disable/enable isomalloc. Note the following two fucntions
  30  * must be used in pair, and no suspend of thread is allowed in between
  31  * */
  32 void CmiDisableIsomalloc()
  33 {
  34         CpvAccess(pushed_blocklist)=CpvAccess(isomalloc_blocklist);
  35         CpvAccess(isomalloc_blocklist)=NULL;
  36 }
  37
  38 void CmiEnableIsomalloc()
  39 {
  40         CpvAccess(isomalloc_blocklist)=CpvAccess(pushed_blocklist);
  41 }
  42
  43 #if CMK_HAS_TLS_VARIABLES
  44 /**
  45  * make sure isomalloc is only called in pthreads that is spawned by Charm++.
  46  * It is not safe to call isomalloc in system spawned pthreads for example
  47  * mpich pthreads, or aio pthreads.
  48  * Use the following TLS variable to distinguish those pthreads.
  49  * when set to 1, the current pthreads is allowed to call isomalloc.
  50  */
  51 static __thread int isomalloc_thread = 0;
  52 #else
  53 #if BIGSIM_OUT_OF_CORE && BIGSIM_OOC_PREFETCH
  54 #error TLS support is required for bigsim out-of-core prefetch optimization
  55 #endif
  56 #endif
  57
  58 static int meta_inited = 0;
  59 extern int _sync_iso;
  60 extern int _sync_iso_warned;
  61
  62 static void meta_init(char **argv)
  63 {
  64    CmiMemoryIs_flag|=CMI_MEMORY_IS_ISOMALLOC;
  65    CpvInitialize(CmiIsomallocBlockList *,isomalloc_blocklist);
  66    CpvInitialize(CmiIsomallocBlockList *,pushed_blocklist);
  67 #if CMK_HAS_TLS_VARIABLES
  68    isomalloc_thread = 1;         /* isomalloc is allowed in this pthread */
  69 #endif
  70    if (CmiMyRank()==0) meta_inited = 1;
  71    CmiNodeAllBarrier();
  72 #if CMK_SMP
  73     if (_sync_iso == 0 && _sync_iso_warned == 0) {
  74         _sync_iso_warned = 1;
  75         printf("Warning> Using Isomalloc in SMP mode, you may need to run with '+isomalloc_sync'.\n");
  76     }
  77 #endif
  78 }
  79
  80 static void *meta_malloc(size_t size)
  81 {
  82         void *ret=NULL;
  83 #if CMK_HAS_TLS_VARIABLES
  84         int _isomalloc_thread = isomalloc_thread;
  85         if (CmiThreadIs(CMI_THREAD_IS_TLS)) _isomalloc_thread = 1;
  86 #endif
  87         if (meta_inited && CpvInitialized(isomalloc_blocklist) && CpvAccess(isomalloc_blocklist)
  88 #if CMK_HAS_TLS_VARIABLES
  89              && _isomalloc_thread
  90 #endif
  91            )
  92         { /*Isomalloc a new block and link it in*/
  93                 ISOMALLOC_PUSH /*Disable isomalloc while inside isomalloc*/
  94 #if CMK_ISOMALLOC_EXCLUDE_FORTRAN_CALLS
  95                 if (CmiIsFortranLibraryCall()==1) {
  96                   ret=mm_malloc(size);
  97                 }
  98                 else
  99 #endif
 100                 ret=CmiIsomallocBlockListMalloc(pushed_blocklist,size);
 101                 ISOMALLOC_POP
 102         }
 103         else /*Just use regular malloc*/
 104                 ret=mm_malloc(size);
 105         return ret;
 106 }
 107
 108 static void meta_free(void *mem)
 109 {
 110         if (mem != NULL && CmiIsomallocInRange(mem))
 111         { /*Unlink this slot and isofree*/
 112                 ISOMALLOC_PUSH
 113                 CmiIsomallocBlockListFree(mem);
 114                 ISOMALLOC_POP
 115         }
 116         else /*Just use regular malloc*/
 117                 mm_free(mem);
 118 }
 119
 120 static void *meta_calloc(size_t nelem, size_t size)
 121 {
 122         void *ret=meta_malloc(nelem*size);
 123         if (ret != NULL) memset(ret,0,nelem*size);
 124         return ret;
 125 }
 126
 127 static void meta_cfree(void *mem)
 128 {
 129         meta_free(mem);
 130 }
 131
 132 static void *meta_realloc(void *oldBuffer, size_t newSize)
 133 {
 134         void *newBuffer;
 135         /*Just forget it for regular malloc's:*/
 136         if (!CmiIsomallocInRange(oldBuffer))
 137                 return mm_realloc(oldBuffer,newSize);
 138
 139         newBuffer = meta_malloc(newSize);
 140         if ( newBuffer && oldBuffer ) {
 141                 /*Must preserve old buffer contents, so we need the size of the
 142                   buffer.  SILLY HACK: muck with internals of blocklist header.*/
 143                 size_t size=CmiIsomallocLength(((CmiIsomallocBlockList *)oldBuffer)-1)-
 144                         sizeof(CmiIsomallocBlockList);
 145                 if (size>newSize) size=newSize;
 146                 if (size > 0)
 147                         memcpy(newBuffer, oldBuffer, size);
 148         }
 149         if (oldBuffer)
 150                 meta_free(oldBuffer);
 151         return newBuffer;
 152 }
 153
 154 static void *meta_memalign(size_t align, size_t size)
 155 {
 156         void *ret=NULL;
 157         if (CpvInitialized(isomalloc_blocklist) && CpvAccess(isomalloc_blocklist))
 158         { /*Isomalloc a new block and link it in*/
 159                 ISOMALLOC_PUSH /*Disable isomalloc while inside isomalloc*/
 160 #if CMK_ISOMALLOC_EXCLUDE_FORTRAN_CALLS
 161                 if (CmiIsFortranLibraryCall()==1) {
 162                   ret=mm_memalign(align, size);
 163                 }
 164                 else
 165 #endif
 166                   ret=CmiIsomallocBlockListMallocAlign(pushed_blocklist,align,size);
 167                 ISOMALLOC_POP
 168         }
 169         else /*Just use regular memalign*/
 170                 ret=mm_memalign(align, size);
 171         return ret;
 172 }
 173
 174 static void *meta_valloc(size_t size)
 175 {
 176         return meta_malloc(size);
 177 }
 178
 179 #define CMK_MEMORY_HAS_NOMIGRATE
 180 /*Allocate non-migratable memory:*/
 181 void *malloc_nomigrate(size_t size) {
 182   void *result;
 183   CmiMemLock();
 184   result = mm_malloc(size);
 185   CmiMemUnlock();
 186   return result;
 187 }
 188
 189 void free_nomigrate(void *mem)
 190 {
 191   CmiMemLock();
 192   mm_free(mem);
 193   CmiMemUnlock();
 194 }
 195
 196 #define CMK_MEMORY_HAS_ISOMALLOC
 197
 198 /*Make this blockList "active"-- the recipient of incoming
 199 mallocs.  Returns the old blocklist.*/
 200 CmiIsomallocBlockList *CmiIsomallocBlockListActivate(CmiIsomallocBlockList *l)
 201 {
 202         CmiIsomallocBlockList **s=&CpvAccess(isomalloc_blocklist);
 203         CmiIsomallocBlockList *ret=*s;
 204         *s=l;
 205         return ret;
 206 }
 207
 208 CmiIsomallocBlockList *CmiIsomallocBlockListCurrent(){
 209         return CpvAccess(isomalloc_blocklist);
 210 }
 211
 212
 213
 214