boehm-gc/thread_local_alloc.c

   1 /*
   2  * Copyright (c) 2000-2005 by Hewlett-Packard Company.  All rights reserved.
   3  *
   4  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
   5  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
   6  *
   7  * Permission is hereby granted to use or copy this program
   8  * for any purpose,  provided the above notices are retained on all copies.
   9  * Permission to modify the code and to distribute modified code is granted,
  10  * provided the above notices are retained, and a notice that the code was
  11  * modified is included with the above copyright notice.
  12  */
  13
  14 #include "private/gc_priv.h"
  15
  16 #if defined(THREAD_LOCAL_ALLOC)
  17
  18 #ifndef THREADS
  19 # error "invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS"
  20 #endif
  21
  22 #include "private/thread_local_alloc.h"
  23
  24 #include <stdlib.h>
  25
  26 #if defined(USE_COMPILER_TLS)
  27   __thread
  28 #elif defined(USE_WIN32_COMPILER_TLS)
  29   __declspec(thread)
  30 #endif
  31 GC_key_t GC_thread_key;
  32
  33 static GC_bool keys_initialized;
  34
  35 #ifdef ENABLE_DISCLAIM
  36   GC_INNER ptr_t * GC_finalized_objfreelist = NULL;
  37         /* This variable is declared here to prevent linking of         */
  38         /* fnlz_mlc module unless the client uses the latter one.       */
  39 #endif
  40
  41 /* Return a single nonempty freelist fl to the global one pointed to    */
  42 /* by gfl.                                                              */
  43
  44 static void return_single_freelist(void *fl, void **gfl)
  45 {
  46     void *q, **qptr;
  47
  48     if (*gfl == 0) {
  49       *gfl = fl;
  50     } else {
  51       GC_ASSERT(GC_size(fl) == GC_size(*gfl));
  52       /* Concatenate: */
  53         qptr = &(obj_link(fl));
  54         while ((word)(q = *qptr) >= HBLKSIZE)
  55           qptr = &(obj_link(q));
  56         GC_ASSERT(0 == q);
  57         *qptr = *gfl;
  58         *gfl = fl;
  59     }
  60 }
  61
  62 /* Recover the contents of the freelist array fl into the global one gfl.*/
  63 /* We hold the allocator lock.                                          */
  64 static void return_freelists(void **fl, void **gfl)
  65 {
  66     int i;
  67
  68     for (i = 1; i < TINY_FREELISTS; ++i) {
  69         if ((word)(fl[i]) >= HBLKSIZE) {
  70           return_single_freelist(fl[i], gfl+i);
  71         }
  72         /* Clear fl[i], since the thread structure may hang around.     */
  73         /* Do it in a way that is likely to trap if we access it.       */
  74         fl[i] = (ptr_t)HBLKSIZE;
  75     }
  76     /* The 0 granule freelist really contains 1 granule objects.        */
  77 #   ifdef GC_GCJ_SUPPORT
  78       if (fl[0] == ERROR_FL) return;
  79 #   endif
  80     if ((word)(fl[0]) >= HBLKSIZE) {
  81         return_single_freelist(fl[0], gfl+1);
  82     }
  83 }
  84
  85 /* Each thread structure must be initialized.   */
  86 /* This call must be made from the new thread.  */
  87 GC_INNER void GC_init_thread_local(GC_tlfs p)
  88 {
  89     int i;
  90
  91     GC_ASSERT(I_HOLD_LOCK());
  92     if (!EXPECT(keys_initialized, TRUE)) {
  93         GC_ASSERT((word)&GC_thread_key % sizeof(word) == 0);
  94         if (0 != GC_key_create(&GC_thread_key, 0)) {
  95             ABORT("Failed to create key for local allocator");
  96         }
  97         keys_initialized = TRUE;
  98     }
  99     if (0 != GC_setspecific(GC_thread_key, p)) {
 100         ABORT("Failed to set thread specific allocation pointers");
 101     }
 102     for (i = 1; i < TINY_FREELISTS; ++i) {
 103         p -> ptrfree_freelists[i] = (void *)(word)1;
 104         p -> normal_freelists[i] = (void *)(word)1;
 105 #       ifdef GC_GCJ_SUPPORT
 106           p -> gcj_freelists[i] = (void *)(word)1;
 107 #       endif
 108 #       ifdef ENABLE_DISCLAIM
 109           p -> finalized_freelists[i] = (void *)(word)1;
 110 #       endif
 111     }
 112     /* Set up the size 0 free lists.    */
 113     /* We now handle most of them like regular free lists, to ensure    */
 114     /* That explicit deallocation works.  However, allocation of a      */
 115     /* size 0 "gcj" object is always an error.                          */
 116     p -> ptrfree_freelists[0] = (void *)(word)1;
 117     p -> normal_freelists[0] = (void *)(word)1;
 118 #   ifdef GC_GCJ_SUPPORT
 119         p -> gcj_freelists[0] = ERROR_FL;
 120 #   endif
 121 #   ifdef ENABLE_DISCLAIM
 122         p -> finalized_freelists[0] = (void *)(word)1;
 123 #   endif
 124 }
 125
 126 /* We hold the allocator lock.  */
 127 GC_INNER void GC_destroy_thread_local(GC_tlfs p)
 128 {
 129     /* We currently only do this from the thread itself or from */
 130     /* the fork handler for a child process.                    */
 131     return_freelists(p -> ptrfree_freelists, GC_aobjfreelist);
 132     return_freelists(p -> normal_freelists, GC_objfreelist);
 133 #   ifdef GC_GCJ_SUPPORT
 134         return_freelists(p -> gcj_freelists, (void **)GC_gcjobjfreelist);
 135 #   endif
 136 #   ifdef ENABLE_DISCLAIM
 137         return_freelists(p -> finalized_freelists,
 138                          (void **)GC_finalized_objfreelist);
 139 #   endif
 140 }
 141
 142 #ifdef GC_ASSERTIONS
 143   /* Defined in pthread_support.c or win32_threads.c. */
 144   GC_bool GC_is_thread_tsd_valid(void *tsd);
 145 #endif
 146
 147 GC_API void * GC_CALL GC_malloc(size_t bytes)
 148 {
 149     size_t granules = ROUNDED_UP_GRANULES(bytes);
 150     void *tsd;
 151     void *result;
 152     void **tiny_fl;
 153
 154 #   if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
 155       GC_key_t k = GC_thread_key;
 156       if (EXPECT(0 == k, FALSE)) {
 157         /* We haven't yet run GC_init_parallel.  That means     */
 158         /* we also aren't locking, so this is fairly cheap.     */
 159         return GC_core_malloc(bytes);
 160       }
 161       tsd = GC_getspecific(k);
 162 #   else
 163       tsd = GC_getspecific(GC_thread_key);
 164 #   endif
 165 #   if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
 166       if (EXPECT(0 == tsd, FALSE)) {
 167         return GC_core_malloc(bytes);
 168       }
 169 #   endif
 170     GC_ASSERT(GC_is_initialized);
 171
 172     GC_ASSERT(GC_is_thread_tsd_valid(tsd));
 173
 174     tiny_fl = ((GC_tlfs)tsd) -> normal_freelists;
 175     GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
 176                          NORMAL, GC_core_malloc(bytes), obj_link(result)=0);
 177 #   ifdef LOG_ALLOCS
 178       GC_log_printf("GC_malloc(%lu) = %p, GC: %lu\n",
 179                     (unsigned long)bytes, result, (unsigned long)GC_gc_no);
 180 #   endif
 181     return result;
 182 }
 183
 184 GC_API void * GC_CALL GC_malloc_atomic(size_t bytes)
 185 {
 186     size_t granules = ROUNDED_UP_GRANULES(bytes);
 187     void *tsd;
 188     void *result;
 189     void **tiny_fl;
 190
 191 #   if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
 192       GC_key_t k = GC_thread_key;
 193       if (EXPECT(0 == k, FALSE)) {
 194         /* We haven't yet run GC_init_parallel.  That means     */
 195         /* we also aren't locking, so this is fairly cheap.     */
 196         return GC_core_malloc_atomic(bytes);
 197       }
 198       tsd = GC_getspecific(k);
 199 #   else
 200       tsd = GC_getspecific(GC_thread_key);
 201 #   endif
 202 #   if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
 203       if (EXPECT(0 == tsd, FALSE)) {
 204         return GC_core_malloc_atomic(bytes);
 205       }
 206 #   endif
 207     GC_ASSERT(GC_is_initialized);
 208     tiny_fl = ((GC_tlfs)tsd) -> ptrfree_freelists;
 209     GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES, PTRFREE,
 210                          GC_core_malloc_atomic(bytes), (void)0 /* no init */);
 211     return result;
 212 }
 213
 214 #ifdef GC_GCJ_SUPPORT
 215
 216 # include "atomic_ops.h" /* for AO_compiler_barrier() */
 217
 218 # include "include/gc_gcj.h"
 219
 220 /* Gcj-style allocation without locks is extremely tricky.  The         */
 221 /* fundamental issue is that we may end up marking a free list, which   */
 222 /* has freelist links instead of "vtable" pointers.  That is usually    */
 223 /* OK, since the next object on the free list will be cleared, and      */
 224 /* will thus be interpreted as containing a zero descriptor.  That's    */
 225 /* fine if the object has not yet been initialized.  But there are      */
 226 /* interesting potential races.                                         */
 227 /* In the case of incremental collection, this seems hopeless, since    */
 228 /* the marker may run asynchronously, and may pick up the pointer to    */
 229 /* the next freelist entry (which it thinks is a vtable pointer), get   */
 230 /* suspended for a while, and then see an allocated object instead      */
 231 /* of the vtable.  This may be avoidable with either a handshake with   */
 232 /* the collector or, probably more easily, by moving the free list      */
 233 /* links to the second word of each object.  The latter isn't a         */
 234 /* universal win, since on architecture like Itanium, nonzero offsets   */
 235 /* are not necessarily free.  And there may be cache fill order issues. */
 236 /* For now, we punt with incremental GC.  This probably means that      */
 237 /* incremental GC should be enabled before we fork a second thread.     */
 238 /* Unlike the other thread local allocation calls, we assume that the   */
 239 /* collector has been explicitly initialized.                           */
 240 GC_API void * GC_CALL GC_gcj_malloc(size_t bytes,
 241                                     void * ptr_to_struct_containing_descr)
 242 {
 243   if (EXPECT(GC_incremental, FALSE)) {
 244     return GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr);
 245   } else {
 246     size_t granules = ROUNDED_UP_GRANULES(bytes);
 247     void *result;
 248     void **tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))
 249                                         -> gcj_freelists;
 250     GC_ASSERT(GC_gcj_malloc_initialized);
 251     GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
 252                          GC_gcj_kind,
 253                          GC_core_gcj_malloc(bytes,
 254                                             ptr_to_struct_containing_descr),
 255                          {AO_compiler_barrier();
 256                           *(void **)result = ptr_to_struct_containing_descr;});
 257         /* This forces the initialization of the "method ptr".          */
 258         /* This is necessary to ensure some very subtle properties      */
 259         /* required if a GC is run in the middle of such an allocation. */
 260         /* Here we implicitly also assume atomicity for the free list.  */
 261         /* and method pointer assignments.                              */
 262         /* We must update the freelist before we store the pointer.     */
 263         /* Otherwise a GC at this point would see a corrupted           */
 264         /* free list.                                                   */
 265         /* A real memory barrier is not needed, since the               */
 266         /* action of stopping this thread will cause prior writes       */
 267         /* to complete.                                                 */
 268         /* We assert that any concurrent marker will stop us.           */
 269         /* Thus it is impossible for a mark procedure to see the        */
 270         /* allocation of the next object, but to see this object        */
 271         /* still containing a free list pointer.  Otherwise the         */
 272         /* marker, by misinterpreting the freelist link as a vtable     */
 273         /* pointer, might find a random "mark descriptor" in the next   */
 274         /* object.                                                      */
 275     return result;
 276   }
 277 }
 278
 279 #endif /* GC_GCJ_SUPPORT */
 280
 281 /* The thread support layer must arrange to mark thread-local   */
 282 /* free lists explicitly, since the link field is often         */
 283 /* invisible to the marker.  It knows how to find all threads;  */
 284 /* we take care of an individual thread freelist structure.     */
 285 GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p)
 286 {
 287     ptr_t q;
 288     int j;
 289
 290     for (j = 0; j < TINY_FREELISTS; ++j) {
 291       q = p -> ptrfree_freelists[j];
 292       if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
 293       q = p -> normal_freelists[j];
 294       if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
 295 #     ifdef GC_GCJ_SUPPORT
 296         if (j > 0) {
 297           q = p -> gcj_freelists[j];
 298           if ((word)q > HBLKSIZE) GC_set_fl_marks(q);
 299         }
 300 #     endif /* GC_GCJ_SUPPORT */
 301 #     ifdef ENABLE_DISCLAIM
 302         q = p -> finalized_freelists[j];
 303         if ((word)q > HBLKSIZE)
 304           GC_set_fl_marks(q);
 305 #     endif
 306     }
 307 }
 308
 309 #if defined(GC_ASSERTIONS)
 310     /* Check that all thread-local free-lists in p are completely marked. */
 311     void GC_check_tls_for(GC_tlfs p)
 312     {
 313         int j;
 314
 315         for (j = 1; j < TINY_FREELISTS; ++j) {
 316           GC_check_fl_marks(&p->ptrfree_freelists[j]);
 317           GC_check_fl_marks(&p->normal_freelists[j]);
 318 #         ifdef GC_GCJ_SUPPORT
 319             GC_check_fl_marks(&p->gcj_freelists[j]);
 320 #         endif
 321 #         ifdef ENABLE_DISCLAIM
 322             GC_check_fl_marks(&p->finalized_freelists[j]);
 323 #         endif
 324         }
 325     }
 326 #endif /* GC_ASSERTIONS */
 327
 328 #endif /* THREAD_LOCAL_ALLOC */