mono/sgen/sgen-gchandles.c

   1 /**
   2  * \file
   3  * SGen GC handles.
   4  *
   5  * Copyright (C) 2015 Xamarin Inc
   6  *
   7  * Licensed under the MIT license. See LICENSE file in the project root for full license information.
   8  */
   9
  10 #include "config.h"
  11 #ifdef HAVE_SGEN_GC
  12
  13 #include "mono/sgen/sgen-gc.h"
  14 #include "mono/sgen/sgen-client.h"
  15 #include "mono/sgen/sgen-array-list.h"
  16 #include "mono/utils/mono-membar.h"
  17
  18 #ifdef HEAVY_STATISTICS
  19 static volatile guint32 stat_gc_handles_allocated = 0;
  20 static volatile guint32 stat_gc_handles_max_allocated = 0;
  21 #endif
  22
  23 /*
  24  * A table of GC handle data, implementing a simple lock-free bitmap allocator.
  25  *
  26  * Each entry in a bucket is a pointer with two tag bits: if
  27  * 'GC_HANDLE_OCCUPIED' returns true for a slot, then the slot is occupied; if
  28  * so, then 'GC_HANDLE_VALID' gives whether the entry refers to a valid (1) or
  29  * NULL (0) object reference. If the reference is valid, then the pointer is an
  30  * object pointer. If the reference is NULL, and 'GC_HANDLE_TYPE_IS_WEAK' is
  31  * true for 'type', then the pointer is a metadata pointer--this allows us to
  32  * retrieve the domain ID of an expired weak reference in Mono.
  33  */
  34
  35 typedef struct {
  36         SgenArrayList entries_array;
  37         guint8 type;
  38 } HandleData;
  39
  40 static void
  41 protocol_gchandle_update (int handle_type, gpointer link, gpointer old_value, gpointer new_value)
  42 {
  43         gboolean old = MONO_GC_HANDLE_IS_OBJECT_POINTER (old_value);
  44         gboolean new_ = MONO_GC_HANDLE_IS_OBJECT_POINTER (new_value);
  45         gboolean track = handle_type == HANDLE_WEAK_TRACK;
  46
  47         if (!MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type))
  48                 return;
  49
  50         if (!old && new_)
  51                 sgen_binary_protocol_dislink_add (link, MONO_GC_REVEAL_POINTER (new_value, TRUE), track);
  52         else if (old && !new_)
  53                 sgen_binary_protocol_dislink_remove (link, track);
  54         else if (old && new_ && old_value != new_value)
  55                 sgen_binary_protocol_dislink_update (link, MONO_GC_REVEAL_POINTER (new_value, TRUE), track);
  56 }
  57
  58 /* Returns the new value in the slot, or NULL if the CAS failed. */
  59 static inline gpointer
  60 try_set_slot (volatile gpointer *slot, GCObject *obj, gpointer old, GCHandleType type)
  61 {
  62         gpointer new_;
  63         if (obj)
  64                 new_ = MONO_GC_HANDLE_OBJECT_POINTER (obj, GC_HANDLE_TYPE_IS_WEAK (type));
  65         else
  66                 new_ = MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (type));
  67         SGEN_ASSERT (0, new_, "Why is the occupied bit not set?");
  68         if (mono_atomic_cas_ptr (slot, new_, old) == old) {
  69                 protocol_gchandle_update (type, (gpointer)slot, old, new_);
  70                 return new_;
  71         }
  72         return NULL;
  73 }
  74
  75 static inline gboolean
  76 is_slot_set (volatile gpointer *slot)
  77 {
  78         gpointer entry = *slot;
  79         if (MONO_GC_HANDLE_OCCUPIED (entry))
  80                 return TRUE;
  81         return FALSE;
  82 }
  83
  84 /* Try to claim a slot by setting its occupied bit. */
  85 static inline gboolean
  86 try_occupy_slot (volatile gpointer *slot, gpointer obj, int data)
  87 {
  88         if (is_slot_set (slot))
  89                 return FALSE;
  90         return try_set_slot (slot, (GCObject *)obj, NULL, (GCHandleType)data) != NULL;
  91 }
  92
  93 static void
  94 bucket_alloc_callback (gpointer *bucket, guint32 new_bucket_size, gboolean alloc)
  95 {
  96         if (alloc)
  97                 sgen_register_root ((char *)bucket, new_bucket_size, SGEN_DESCRIPTOR_NULL, ROOT_TYPE_PINNED, MONO_ROOT_SOURCE_GC_HANDLE, NULL, "GC Handle Bucket (SGen, Pinned)");
  98         else
  99                 sgen_deregister_root ((char *)bucket);
 100 }
 101
 102 static void
 103 bucket_alloc_report_root (gpointer *bucket, guint32 new_bucket_size, gboolean alloc)
 104 {
 105         if (alloc)
 106                 sgen_client_root_registered ((char *)bucket, new_bucket_size, MONO_ROOT_SOURCE_GC_HANDLE, NULL, "GC Handle Bucket (SGen, Normal)");
 107         else
 108                 sgen_client_root_deregistered ((char *)bucket);
 109 }
 110
 111 static HandleData gc_handles [] = {
 112         { SGEN_ARRAY_LIST_INIT (NULL, is_slot_set, try_occupy_slot, -1), (HANDLE_WEAK) },
 113         { SGEN_ARRAY_LIST_INIT (NULL, is_slot_set, try_occupy_slot, -1), (HANDLE_WEAK_TRACK) },
 114         { SGEN_ARRAY_LIST_INIT (bucket_alloc_report_root, is_slot_set, try_occupy_slot, -1), (HANDLE_NORMAL) },
 115         { SGEN_ARRAY_LIST_INIT (bucket_alloc_callback, is_slot_set, try_occupy_slot, -1), (HANDLE_PINNED) },
 116         { SGEN_ARRAY_LIST_INIT (NULL, is_slot_set, try_occupy_slot, -1), (HANDLE_WEAK_FIELDS) },
 117 };
 118
 119 static HandleData *
 120 gc_handles_for_type (GCHandleType type)
 121 {
 122         return type < HANDLE_TYPE_MAX ? &gc_handles [type] : NULL;
 123 }
 124
 125 /* This assumes that the world is stopped. */
 126 void
 127 sgen_mark_normal_gc_handles (void *addr, SgenUserMarkFunc mark_func, void *gc_data)
 128 {
 129         HandleData *handles = gc_handles_for_type (HANDLE_NORMAL);
 130         SgenArrayList *array = &handles->entries_array;
 131         volatile gpointer *slot;
 132         gpointer hidden, revealed;
 133
 134         SGEN_ARRAY_LIST_FOREACH_SLOT (array, slot) {
 135                 hidden = *slot;
 136                 revealed = MONO_GC_REVEAL_POINTER (hidden, FALSE);
 137                 if (!MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden))
 138                         continue;
 139                 mark_func ((MonoObject **)&revealed, gc_data);
 140                 g_assert (revealed);
 141                 *slot = MONO_GC_HANDLE_OBJECT_POINTER (revealed, FALSE);
 142         } SGEN_ARRAY_LIST_END_FOREACH_SLOT;
 143 }
 144
 145 void
 146 sgen_gc_handles_report_roots (SgenUserReportRootFunc report_func, void *gc_data)
 147 {
 148         HandleData *handles = gc_handles_for_type (HANDLE_NORMAL);
 149         SgenArrayList *array = &handles->entries_array;
 150         volatile gpointer *slot;
 151         gpointer hidden, revealed;
 152
 153         SGEN_ARRAY_LIST_FOREACH_SLOT (array, slot) {
 154                 hidden = *slot;
 155                 revealed = MONO_GC_REVEAL_POINTER (hidden, FALSE);
 156
 157                 if (MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden))
 158                         report_func ((void*)slot, (GCObject*)revealed, gc_data);
 159         } SGEN_ARRAY_LIST_END_FOREACH_SLOT;
 160 }
 161
 162 static guint32
 163 alloc_handle (HandleData *handles, GCObject *obj, gboolean track)
 164 {
 165         guint32 res, index;
 166         SgenArrayList *array = &handles->entries_array;
 167
 168         /*
 169          * If a GC happens shortly after a new bucket is allocated, the entire
 170          * bucket could be scanned even though it's mostly empty. To avoid this,
 171          * we track the maximum index seen so far, so that we can skip the empty
 172          * slots.
 173          *
 174          * Note that we update `next_slot` before we even try occupying the
 175          * slot.  If we did it the other way around and a GC happened in
 176          * between, the GC wouldn't know that the slot was occupied.  This is
 177          * not a huge deal since `obj` is on the stack and thus pinned anyway,
 178          * but hopefully some day it won't be anymore.
 179          */
 180         index = sgen_array_list_add (array, obj, handles->type, TRUE);
 181 #ifdef HEAVY_STATISTICS
 182         mono_atomic_inc_i32 ((volatile gint32 *)&stat_gc_handles_allocated);
 183         if (stat_gc_handles_allocated > stat_gc_handles_max_allocated)
 184                 stat_gc_handles_max_allocated = stat_gc_handles_allocated;
 185 #endif
 186         /* Ensure that a GC handle cannot be given to another thread without the slot having been set. */
 187         mono_memory_write_barrier ();
 188         res = MONO_GC_HANDLE (index, handles->type);
 189         sgen_client_gchandle_created ((GCHandleType)handles->type, obj, res);
 190         return res;
 191 }
 192
 193 static gboolean
 194 object_older_than (GCObject *object, int generation)
 195 {
 196         return generation == GENERATION_NURSERY && !sgen_ptr_in_nursery (object);
 197 }
 198
 199 /*
 200  * Maps a function over all GC handles.
 201  * This assumes that the world is stopped!
 202  */
 203 void
 204 sgen_gchandle_iterate (GCHandleType handle_type, int max_generation, SgenGCHandleIterateCallback callback, gpointer user)
 205 {
 206         HandleData *handle_data = gc_handles_for_type (handle_type);
 207         SgenArrayList *array = &handle_data->entries_array;
 208         gpointer hidden, result, occupied;
 209         volatile gpointer *slot;
 210
 211         /* If a new bucket has been allocated, but the capacity has not yet been
 212          * increased, nothing can yet have been allocated in the bucket because the
 213          * world is stopped, so we shouldn't miss any handles during iteration.
 214          */
 215         SGEN_ARRAY_LIST_FOREACH_SLOT (array, slot) {
 216                 hidden = *slot;
 217                 occupied = (gpointer) MONO_GC_HANDLE_OCCUPIED (hidden);
 218                 g_assert (hidden ? !!occupied : !occupied);
 219                 if (!occupied)
 220                         continue;
 221                 result = callback (hidden, handle_type, max_generation, user);
 222                 if (result)
 223                         SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (result), "Why did the callback return an unoccupied entry?");
 224                 else
 225                         HEAVY_STAT (mono_atomic_dec_i32 ((volatile gint32 *)&stat_gc_handles_allocated));
 226                 protocol_gchandle_update (handle_type, (gpointer)slot, hidden, result);
 227                 *slot = result;
 228         } SGEN_ARRAY_LIST_END_FOREACH_SLOT;
 229 }
 230
 231 guint32
 232 sgen_gchandle_new (GCObject *obj, gboolean pinned)
 233 {
 234         return alloc_handle (gc_handles_for_type (pinned ? HANDLE_PINNED : HANDLE_NORMAL), obj, FALSE);
 235 }
 236
 237 guint32
 238 sgen_gchandle_new_weakref (GCObject *obj, gboolean track_resurrection)
 239 {
 240         return alloc_handle (gc_handles_for_type (track_resurrection ? HANDLE_WEAK_TRACK : HANDLE_WEAK), obj, track_resurrection);
 241 }
 242
 243 static GCObject *
 244 link_get (volatile gpointer *link_addr, gboolean is_weak)
 245 {
 246         void *volatile *link_addr_volatile;
 247         void *ptr;
 248         GCObject *obj;
 249 retry:
 250         link_addr_volatile = link_addr;
 251         ptr = (void*)*link_addr_volatile;
 252         /*
 253          * At this point we have a hidden pointer.  If the GC runs
 254          * here, it will not recognize the hidden pointer as a
 255          * reference, and if the object behind it is not referenced
 256          * elsewhere, it will be freed.  Once the world is restarted
 257          * we reveal the pointer, giving us a pointer to a freed
 258          * object.  To make sure we don't return it, we load the
 259          * hidden pointer again.  If it's still the same, we can be
 260          * sure the object reference is valid.
 261          */
 262         if (ptr && MONO_GC_HANDLE_IS_OBJECT_POINTER (ptr))
 263                 obj = (GCObject *)MONO_GC_REVEAL_POINTER (ptr, is_weak);
 264         else
 265                 return NULL;
 266
 267         /* Note [dummy use]:
 268          *
 269          * If a GC happens here, obj needs to be on the stack or in a
 270          * register, so we need to prevent this from being reordered
 271          * wrt the check.
 272          */
 273         sgen_dummy_use (obj);
 274         mono_memory_barrier ();
 275
 276         if (is_weak)
 277                 sgen_client_ensure_weak_gchandles_accessible ();
 278
 279         if ((void*)*link_addr_volatile != ptr)
 280                 goto retry;
 281
 282         return obj;
 283 }
 284
 285 GCObject*
 286 sgen_gchandle_get_target (guint32 gchandle)
 287 {
 288         guint index = MONO_GC_HANDLE_SLOT (gchandle);
 289         GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
 290         HandleData *handles = gc_handles_for_type (type);
 291         /* Invalid handles are possible; accessing one should produce NULL. (#34276) */
 292         if (!handles)
 293                 return NULL;
 294         return link_get (sgen_array_list_get_slot (&handles->entries_array, index), MONO_GC_HANDLE_TYPE_IS_WEAK (type));
 295 }
 296
 297 void
 298 sgen_gchandle_set_target (guint32 gchandle, GCObject *obj)
 299 {
 300         guint32 index = MONO_GC_HANDLE_SLOT (gchandle);
 301         GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
 302         HandleData *handles = gc_handles_for_type (type);
 303         volatile gpointer *slot;
 304         gpointer entry;
 305
 306         if (!handles)
 307                 return;
 308
 309         slot = sgen_array_list_get_slot (&handles->entries_array, index);
 310
 311         do {
 312                 entry = *slot;
 313                 SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (entry), "Why are we setting the target on an unoccupied slot?");
 314         } while (!try_set_slot (slot, obj, entry, (GCHandleType)handles->type));
 315 }
 316
 317 static gpointer
 318 mono_gchandle_slot_metadata (volatile gpointer *slot, gboolean is_weak)
 319 {
 320         gpointer entry;
 321         gpointer metadata;
 322 retry:
 323         entry = *slot;
 324         if (!MONO_GC_HANDLE_OCCUPIED (entry))
 325                 return NULL;
 326         if (MONO_GC_HANDLE_IS_OBJECT_POINTER (entry)) {
 327                 GCObject *obj = (GCObject *)MONO_GC_REVEAL_POINTER (entry, is_weak);
 328                 /* See note [dummy use]. */
 329                 sgen_dummy_use (obj);
 330                 /*
 331                  * FIXME: The compiler could technically not carry a reference to obj around
 332                  * at this point and recompute it later, in which case we would still use
 333                  * it.
 334                  */
 335                 if (*slot != entry)
 336                         goto retry;
 337                 return sgen_client_metadata_for_object (obj);
 338         }
 339         metadata = MONO_GC_REVEAL_POINTER (entry, is_weak);
 340         /* See note [dummy use]. */
 341         sgen_dummy_use (metadata);
 342         if (*slot != entry)
 343                 goto retry;
 344         return metadata;
 345 }
 346
 347 gpointer
 348 sgen_gchandle_get_metadata (guint32 gchandle)
 349 {
 350         guint32 index = MONO_GC_HANDLE_SLOT (gchandle);
 351         GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
 352         HandleData *handles = gc_handles_for_type (type);
 353         volatile gpointer *slot;
 354
 355         if (!handles)
 356                 return NULL;
 357         if (index >= handles->entries_array.capacity)
 358                 return NULL;
 359
 360         slot = sgen_array_list_get_slot (&handles->entries_array, index);
 361
 362         return mono_gchandle_slot_metadata (slot, MONO_GC_HANDLE_TYPE_IS_WEAK (type));
 363 }
 364
 365 void
 366 sgen_gchandle_free (guint32 gchandle)
 367 {
 368         if (!gchandle)
 369                 return;
 370
 371         guint32 index = MONO_GC_HANDLE_SLOT (gchandle);
 372         GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
 373         HandleData *handles = gc_handles_for_type (type);
 374         volatile gpointer *slot;
 375         gpointer entry;
 376         if (!handles)
 377                 return;
 378
 379         slot = sgen_array_list_get_slot (&handles->entries_array, index);
 380         entry = *slot;
 381
 382         if (index < handles->entries_array.capacity && MONO_GC_HANDLE_OCCUPIED (entry)) {
 383                 *slot = NULL;
 384                 protocol_gchandle_update (handles->type, (gpointer)slot, entry, NULL);
 385                 HEAVY_STAT (mono_atomic_dec_i32 ((volatile gint32 *)&stat_gc_handles_allocated));
 386         } else {
 387                 /* print a warning? */
 388         }
 389         sgen_client_gchandle_destroyed ((GCHandleType)handles->type, gchandle);
 390 }
 391
 392 /*
 393  * Returns whether to remove the link from its hash.
 394  */
 395 static gpointer
 396 null_link_if_necessary (gpointer hidden, GCHandleType handle_type, int max_generation, gpointer user)
 397 {
 398         const gboolean is_weak = GC_HANDLE_TYPE_IS_WEAK (handle_type);
 399         ScanCopyContext *ctx = (ScanCopyContext *)user;
 400         GCObject *obj;
 401         GCObject *copy;
 402
 403         if (!MONO_GC_HANDLE_VALID (hidden))
 404                 return hidden;
 405
 406         obj = (GCObject *)MONO_GC_REVEAL_POINTER (hidden, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type));
 407         SGEN_ASSERT (0, obj, "Why is the hidden pointer NULL?");
 408
 409         if (object_older_than (obj, max_generation))
 410                 return hidden;
 411
 412         if (sgen_major_collector.is_object_live (obj))
 413                 return hidden;
 414
 415         /* Clear link if object is ready for finalization. This check may be redundant wrt is_object_live(). */
 416         if (sgen_gc_is_object_ready_for_finalization (obj))
 417                 return MONO_GC_HANDLE_METADATA_POINTER (sgen_client_metadata_for_object (obj), is_weak);
 418
 419         copy = obj;
 420         ctx->ops->copy_or_mark_object (&copy, ctx->queue);
 421         SGEN_ASSERT (0, copy, "Why couldn't we copy the object?");
 422         /* Update link if object was moved. */
 423         return MONO_GC_HANDLE_OBJECT_POINTER (copy, is_weak);
 424 }
 425
 426 static gpointer
 427 scan_for_weak (gpointer hidden, GCHandleType handle_type, int max_generation, gpointer user)
 428 {
 429         const gboolean is_weak = GC_HANDLE_TYPE_IS_WEAK (handle_type);
 430         ScanCopyContext *ctx = (ScanCopyContext *)user;
 431
 432         if (!MONO_GC_HANDLE_VALID (hidden))
 433                 return hidden;
 434
 435         GCObject *obj = (GCObject *)MONO_GC_REVEAL_POINTER (hidden, is_weak);
 436
 437         /* If the object is dead we free the gc handle */
 438         if (!sgen_is_object_alive_for_current_gen (obj))
 439                 return NULL;
 440
 441         /* Relocate it */
 442         ctx->ops->copy_or_mark_object (&obj, ctx->queue);
 443
 444         int nbits;
 445         gsize *weak_bitmap = sgen_client_get_weak_bitmap (SGEN_LOAD_VTABLE (obj), &nbits);
 446         for (int i = 0; i < nbits; ++i) {
 447                 if (weak_bitmap [i / (sizeof (gsize) * 8)] & ((gsize)1 << (i % (sizeof (gsize) * 8)))) {
 448                         GCObject **addr = (GCObject **)((char*)obj + (i * sizeof (gpointer)));
 449                         GCObject *field = *addr;
 450
 451                         /* if the object in the weak field is alive, we relocate it */
 452                         if (field && sgen_is_object_alive_for_current_gen (field))
 453                                 ctx->ops->copy_or_mark_object (addr, ctx->queue);
 454                         else
 455                                 *addr = NULL;
 456            }
 457         }
 458
 459         /* Update link if object was moved. */
 460         return MONO_GC_HANDLE_OBJECT_POINTER (obj, is_weak);
 461 }
 462
 463 /* LOCKING: requires that the GC lock is held */
 464 void
 465 sgen_null_link_in_range (int generation, ScanCopyContext ctx, gboolean track)
 466 {
 467         sgen_gchandle_iterate (track ? HANDLE_WEAK_TRACK : HANDLE_WEAK, generation, null_link_if_necessary, &ctx);
 468
 469         //we're always called for gen zero. !track means short ref
 470         if (generation == 0 && !track)
 471                 sgen_gchandle_iterate (HANDLE_WEAK_FIELDS, generation, scan_for_weak, &ctx);
 472 }
 473
 474 typedef struct {
 475         SgenObjectPredicateFunc predicate;
 476         gpointer data;
 477 } WeakLinkAlivePredicateClosure;
 478
 479 static gpointer
 480 null_link_if (gpointer hidden, GCHandleType handle_type, int max_generation, gpointer user)
 481 {
 482         WeakLinkAlivePredicateClosure *closure = (WeakLinkAlivePredicateClosure *)user;
 483         GCObject *obj;
 484
 485         if (!MONO_GC_HANDLE_VALID (hidden))
 486                 return hidden;
 487
 488         obj = (GCObject *)MONO_GC_REVEAL_POINTER (hidden, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type));
 489         SGEN_ASSERT (0, obj, "Why is the hidden pointer NULL?");
 490
 491         if (object_older_than (obj, max_generation))
 492                 return hidden;
 493
 494         if (closure->predicate (obj, closure->data))
 495                 return MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (handle_type));
 496
 497         return hidden;
 498 }
 499
 500 /* LOCKING: requires that the GC lock is held */
 501 void
 502 sgen_null_links_if (SgenObjectPredicateFunc predicate, void *data, int generation, gboolean track)
 503 {
 504         WeakLinkAlivePredicateClosure closure = { predicate, data };
 505         sgen_gchandle_iterate (track ? HANDLE_WEAK_TRACK : HANDLE_WEAK, generation, null_link_if, &closure);
 506 }
 507
 508 void
 509 sgen_register_obj_with_weak_fields (GCObject *obj)
 510 {
 511         //
 512         // We use a gc handle to be able to do some processing for these objects at every gc
 513         //
 514         alloc_handle (gc_handles_for_type (HANDLE_WEAK_FIELDS), obj, FALSE);
 515 }
 516
 517 void
 518 sgen_init_gchandles (void)
 519 {
 520 #ifdef HEAVY_STATISTICS
 521         mono_counters_register ("GC handles allocated", MONO_COUNTER_GC | MONO_COUNTER_UINT, (void *)&stat_gc_handles_allocated);
 522         mono_counters_register ("max GC handles allocated", MONO_COUNTER_GC | MONO_COUNTER_UINT, (void *)&stat_gc_handles_max_allocated);
 523 #endif
 524 }
 525
 526 #endif