5 * Copyright (C) 2015 Xamarin Inc
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
13 #include "mono/sgen/sgen-gc.h"
14 #include "mono/sgen/sgen-client.h"
15 #include "mono/sgen/sgen-array-list.h"
16 #include "mono/utils/mono-membar.h"
18 #ifdef HEAVY_STATISTICS
19 static volatile guint32 stat_gc_handles_allocated
= 0;
20 static volatile guint32 stat_gc_handles_max_allocated
= 0;
24 * A table of GC handle data, implementing a simple lock-free bitmap allocator.
26 * Each entry in a bucket is a pointer with two tag bits: if
27 * 'GC_HANDLE_OCCUPIED' returns true for a slot, then the slot is occupied; if
28 * so, then 'GC_HANDLE_VALID' gives whether the entry refers to a valid (1) or
29 * NULL (0) object reference. If the reference is valid, then the pointer is an
30 * object pointer. If the reference is NULL, and 'GC_HANDLE_TYPE_IS_WEAK' is
31 * true for 'type', then the pointer is a metadata pointer--this allows us to
32 * retrieve the domain ID of an expired weak reference in Mono.
36 SgenArrayList entries_array
;
41 protocol_gchandle_update (int handle_type
, gpointer link
, gpointer old_value
, gpointer new_value
)
43 gboolean old
= MONO_GC_HANDLE_IS_OBJECT_POINTER (old_value
);
44 gboolean new_
= MONO_GC_HANDLE_IS_OBJECT_POINTER (new_value
);
45 gboolean track
= handle_type
== HANDLE_WEAK_TRACK
;
47 if (!MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type
))
51 sgen_binary_protocol_dislink_add (link
, MONO_GC_REVEAL_POINTER (new_value
, TRUE
), track
);
52 else if (old
&& !new_
)
53 sgen_binary_protocol_dislink_remove (link
, track
);
54 else if (old
&& new_
&& old_value
!= new_value
)
55 sgen_binary_protocol_dislink_update (link
, MONO_GC_REVEAL_POINTER (new_value
, TRUE
), track
);
58 /* Returns the new value in the slot, or NULL if the CAS failed. */
59 static inline gpointer
60 try_set_slot (volatile gpointer
*slot
, GCObject
*obj
, gpointer old
, GCHandleType type
)
64 new_
= MONO_GC_HANDLE_OBJECT_POINTER (obj
, GC_HANDLE_TYPE_IS_WEAK (type
));
66 new_
= MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (type
));
67 SGEN_ASSERT (0, new_
, "Why is the occupied bit not set?");
68 if (mono_atomic_cas_ptr (slot
, new_
, old
) == old
) {
69 protocol_gchandle_update (type
, (gpointer
)slot
, old
, new_
);
75 static inline gboolean
76 is_slot_set (volatile gpointer
*slot
)
78 gpointer entry
= *slot
;
79 if (MONO_GC_HANDLE_OCCUPIED (entry
))
84 /* Try to claim a slot by setting its occupied bit. */
85 static inline gboolean
86 try_occupy_slot (volatile gpointer
*slot
, gpointer obj
, int data
)
88 if (is_slot_set (slot
))
90 return try_set_slot (slot
, (GCObject
*)obj
, NULL
, (GCHandleType
)data
) != NULL
;
94 bucket_alloc_callback (gpointer
*bucket
, guint32 new_bucket_size
, gboolean alloc
)
97 sgen_register_root ((char *)bucket
, new_bucket_size
, SGEN_DESCRIPTOR_NULL
, ROOT_TYPE_PINNED
, MONO_ROOT_SOURCE_GC_HANDLE
, NULL
, "GC Handle Bucket (SGen, Pinned)");
99 sgen_deregister_root ((char *)bucket
);
103 bucket_alloc_report_root (gpointer
*bucket
, guint32 new_bucket_size
, gboolean alloc
)
106 sgen_client_root_registered ((char *)bucket
, new_bucket_size
, MONO_ROOT_SOURCE_GC_HANDLE
, NULL
, "GC Handle Bucket (SGen, Normal)");
108 sgen_client_root_deregistered ((char *)bucket
);
111 static HandleData gc_handles
[] = {
112 { SGEN_ARRAY_LIST_INIT (NULL
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_WEAK
) },
113 { SGEN_ARRAY_LIST_INIT (NULL
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_WEAK_TRACK
) },
114 { SGEN_ARRAY_LIST_INIT (bucket_alloc_report_root
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_NORMAL
) },
115 { SGEN_ARRAY_LIST_INIT (bucket_alloc_callback
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_PINNED
) },
116 { SGEN_ARRAY_LIST_INIT (NULL
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_WEAK_FIELDS
) },
120 gc_handles_for_type (GCHandleType type
)
122 return type
< HANDLE_TYPE_MAX
? &gc_handles
[type
] : NULL
;
125 /* This assumes that the world is stopped. */
127 sgen_mark_normal_gc_handles (void *addr
, SgenUserMarkFunc mark_func
, void *gc_data
)
129 HandleData
*handles
= gc_handles_for_type (HANDLE_NORMAL
);
130 SgenArrayList
*array
= &handles
->entries_array
;
131 volatile gpointer
*slot
;
132 gpointer hidden
, revealed
;
134 SGEN_ARRAY_LIST_FOREACH_SLOT (array
, slot
) {
136 revealed
= MONO_GC_REVEAL_POINTER (hidden
, FALSE
);
137 if (!MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden
))
139 mark_func ((MonoObject
**)&revealed
, gc_data
);
141 *slot
= MONO_GC_HANDLE_OBJECT_POINTER (revealed
, FALSE
);
142 } SGEN_ARRAY_LIST_END_FOREACH_SLOT
;
146 sgen_gc_handles_report_roots (SgenUserReportRootFunc report_func
, void *gc_data
)
148 HandleData
*handles
= gc_handles_for_type (HANDLE_NORMAL
);
149 SgenArrayList
*array
= &handles
->entries_array
;
150 volatile gpointer
*slot
;
151 gpointer hidden
, revealed
;
153 SGEN_ARRAY_LIST_FOREACH_SLOT (array
, slot
) {
155 revealed
= MONO_GC_REVEAL_POINTER (hidden
, FALSE
);
157 if (MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden
))
158 report_func ((void*)slot
, (GCObject
*)revealed
, gc_data
);
159 } SGEN_ARRAY_LIST_END_FOREACH_SLOT
;
163 alloc_handle (HandleData
*handles
, GCObject
*obj
, gboolean track
)
166 SgenArrayList
*array
= &handles
->entries_array
;
169 * If a GC happens shortly after a new bucket is allocated, the entire
170 * bucket could be scanned even though it's mostly empty. To avoid this,
171 * we track the maximum index seen so far, so that we can skip the empty
174 * Note that we update `next_slot` before we even try occupying the
175 * slot. If we did it the other way around and a GC happened in
176 * between, the GC wouldn't know that the slot was occupied. This is
177 * not a huge deal since `obj` is on the stack and thus pinned anyway,
178 * but hopefully some day it won't be anymore.
180 index
= sgen_array_list_add (array
, obj
, handles
->type
, TRUE
);
181 #ifdef HEAVY_STATISTICS
182 mono_atomic_inc_i32 ((volatile gint32
*)&stat_gc_handles_allocated
);
183 if (stat_gc_handles_allocated
> stat_gc_handles_max_allocated
)
184 stat_gc_handles_max_allocated
= stat_gc_handles_allocated
;
186 /* Ensure that a GC handle cannot be given to another thread without the slot having been set. */
187 mono_memory_write_barrier ();
188 res
= MONO_GC_HANDLE (index
, handles
->type
);
189 sgen_client_gchandle_created ((GCHandleType
)handles
->type
, obj
, res
);
194 object_older_than (GCObject
*object
, int generation
)
196 return generation
== GENERATION_NURSERY
&& !sgen_ptr_in_nursery (object
);
200 * Maps a function over all GC handles.
201 * This assumes that the world is stopped!
204 sgen_gchandle_iterate (GCHandleType handle_type
, int max_generation
, SgenGCHandleIterateCallback callback
, gpointer user
)
206 HandleData
*handle_data
= gc_handles_for_type (handle_type
);
207 SgenArrayList
*array
= &handle_data
->entries_array
;
208 gpointer hidden
, result
, occupied
;
209 volatile gpointer
*slot
;
211 /* If a new bucket has been allocated, but the capacity has not yet been
212 * increased, nothing can yet have been allocated in the bucket because the
213 * world is stopped, so we shouldn't miss any handles during iteration.
215 SGEN_ARRAY_LIST_FOREACH_SLOT (array
, slot
) {
217 occupied
= (gpointer
) MONO_GC_HANDLE_OCCUPIED (hidden
);
218 g_assert (hidden
? !!occupied
: !occupied
);
221 result
= callback (hidden
, handle_type
, max_generation
, user
);
223 SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (result
), "Why did the callback return an unoccupied entry?");
225 HEAVY_STAT (mono_atomic_dec_i32 ((volatile gint32
*)&stat_gc_handles_allocated
));
226 protocol_gchandle_update (handle_type
, (gpointer
)slot
, hidden
, result
);
228 } SGEN_ARRAY_LIST_END_FOREACH_SLOT
;
232 sgen_gchandle_new (GCObject
*obj
, gboolean pinned
)
234 return alloc_handle (gc_handles_for_type (pinned
? HANDLE_PINNED
: HANDLE_NORMAL
), obj
, FALSE
);
238 sgen_gchandle_new_weakref (GCObject
*obj
, gboolean track_resurrection
)
240 return alloc_handle (gc_handles_for_type (track_resurrection
? HANDLE_WEAK_TRACK
: HANDLE_WEAK
), obj
, track_resurrection
);
244 link_get (volatile gpointer
*link_addr
, gboolean is_weak
)
246 void *volatile *link_addr_volatile
;
250 link_addr_volatile
= link_addr
;
251 ptr
= (void*)*link_addr_volatile
;
253 * At this point we have a hidden pointer. If the GC runs
254 * here, it will not recognize the hidden pointer as a
255 * reference, and if the object behind it is not referenced
256 * elsewhere, it will be freed. Once the world is restarted
257 * we reveal the pointer, giving us a pointer to a freed
258 * object. To make sure we don't return it, we load the
259 * hidden pointer again. If it's still the same, we can be
260 * sure the object reference is valid.
262 if (ptr
&& MONO_GC_HANDLE_IS_OBJECT_POINTER (ptr
))
263 obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (ptr
, is_weak
);
269 * If a GC happens here, obj needs to be on the stack or in a
270 * register, so we need to prevent this from being reordered
273 sgen_dummy_use (obj
);
274 mono_memory_barrier ();
277 sgen_client_ensure_weak_gchandles_accessible ();
279 if ((void*)*link_addr_volatile
!= ptr
)
286 sgen_gchandle_get_target (guint32 gchandle
)
288 guint index
= MONO_GC_HANDLE_SLOT (gchandle
);
289 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
290 HandleData
*handles
= gc_handles_for_type (type
);
291 /* Invalid handles are possible; accessing one should produce NULL. (#34276) */
294 return link_get (sgen_array_list_get_slot (&handles
->entries_array
, index
), MONO_GC_HANDLE_TYPE_IS_WEAK (type
));
298 sgen_gchandle_set_target (guint32 gchandle
, GCObject
*obj
)
300 guint32 index
= MONO_GC_HANDLE_SLOT (gchandle
);
301 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
302 HandleData
*handles
= gc_handles_for_type (type
);
303 volatile gpointer
*slot
;
309 slot
= sgen_array_list_get_slot (&handles
->entries_array
, index
);
313 SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (entry
), "Why are we setting the target on an unoccupied slot?");
314 } while (!try_set_slot (slot
, obj
, entry
, (GCHandleType
)handles
->type
));
318 mono_gchandle_slot_metadata (volatile gpointer
*slot
, gboolean is_weak
)
324 if (!MONO_GC_HANDLE_OCCUPIED (entry
))
326 if (MONO_GC_HANDLE_IS_OBJECT_POINTER (entry
)) {
327 GCObject
*obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (entry
, is_weak
);
328 /* See note [dummy use]. */
329 sgen_dummy_use (obj
);
331 * FIXME: The compiler could technically not carry a reference to obj around
332 * at this point and recompute it later, in which case we would still use
337 return sgen_client_metadata_for_object (obj
);
339 metadata
= MONO_GC_REVEAL_POINTER (entry
, is_weak
);
340 /* See note [dummy use]. */
341 sgen_dummy_use (metadata
);
348 sgen_gchandle_get_metadata (guint32 gchandle
)
350 guint32 index
= MONO_GC_HANDLE_SLOT (gchandle
);
351 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
352 HandleData
*handles
= gc_handles_for_type (type
);
353 volatile gpointer
*slot
;
357 if (index
>= handles
->entries_array
.capacity
)
360 slot
= sgen_array_list_get_slot (&handles
->entries_array
, index
);
362 return mono_gchandle_slot_metadata (slot
, MONO_GC_HANDLE_TYPE_IS_WEAK (type
));
366 sgen_gchandle_free (guint32 gchandle
)
371 guint32 index
= MONO_GC_HANDLE_SLOT (gchandle
);
372 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
373 HandleData
*handles
= gc_handles_for_type (type
);
374 volatile gpointer
*slot
;
379 slot
= sgen_array_list_get_slot (&handles
->entries_array
, index
);
382 if (index
< handles
->entries_array
.capacity
&& MONO_GC_HANDLE_OCCUPIED (entry
)) {
384 protocol_gchandle_update (handles
->type
, (gpointer
)slot
, entry
, NULL
);
385 HEAVY_STAT (mono_atomic_dec_i32 ((volatile gint32
*)&stat_gc_handles_allocated
));
387 /* print a warning? */
389 sgen_client_gchandle_destroyed ((GCHandleType
)handles
->type
, gchandle
);
393 * Returns whether to remove the link from its hash.
396 null_link_if_necessary (gpointer hidden
, GCHandleType handle_type
, int max_generation
, gpointer user
)
398 const gboolean is_weak
= GC_HANDLE_TYPE_IS_WEAK (handle_type
);
399 ScanCopyContext
*ctx
= (ScanCopyContext
*)user
;
403 if (!MONO_GC_HANDLE_VALID (hidden
))
406 obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (hidden
, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type
));
407 SGEN_ASSERT (0, obj
, "Why is the hidden pointer NULL?");
409 if (object_older_than (obj
, max_generation
))
412 if (sgen_major_collector
.is_object_live (obj
))
415 /* Clear link if object is ready for finalization. This check may be redundant wrt is_object_live(). */
416 if (sgen_gc_is_object_ready_for_finalization (obj
))
417 return MONO_GC_HANDLE_METADATA_POINTER (sgen_client_metadata_for_object (obj
), is_weak
);
420 ctx
->ops
->copy_or_mark_object (©
, ctx
->queue
);
421 SGEN_ASSERT (0, copy
, "Why couldn't we copy the object?");
422 /* Update link if object was moved. */
423 return MONO_GC_HANDLE_OBJECT_POINTER (copy
, is_weak
);
427 scan_for_weak (gpointer hidden
, GCHandleType handle_type
, int max_generation
, gpointer user
)
429 const gboolean is_weak
= GC_HANDLE_TYPE_IS_WEAK (handle_type
);
430 ScanCopyContext
*ctx
= (ScanCopyContext
*)user
;
432 if (!MONO_GC_HANDLE_VALID (hidden
))
435 GCObject
*obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (hidden
, is_weak
);
437 /* If the object is dead we free the gc handle */
438 if (!sgen_is_object_alive_for_current_gen (obj
))
442 ctx
->ops
->copy_or_mark_object (&obj
, ctx
->queue
);
445 gsize
*weak_bitmap
= sgen_client_get_weak_bitmap (SGEN_LOAD_VTABLE (obj
), &nbits
);
446 for (int i
= 0; i
< nbits
; ++i
) {
447 if (weak_bitmap
[i
/ (sizeof (gsize
) * 8)] & ((gsize
)1 << (i
% (sizeof (gsize
) * 8)))) {
448 GCObject
**addr
= (GCObject
**)((char*)obj
+ (i
* sizeof (gpointer
)));
449 GCObject
*field
= *addr
;
451 /* if the object in the weak field is alive, we relocate it */
452 if (field
&& sgen_is_object_alive_for_current_gen (field
))
453 ctx
->ops
->copy_or_mark_object (addr
, ctx
->queue
);
459 /* Update link if object was moved. */
460 return MONO_GC_HANDLE_OBJECT_POINTER (obj
, is_weak
);
463 /* LOCKING: requires that the GC lock is held */
465 sgen_null_link_in_range (int generation
, ScanCopyContext ctx
, gboolean track
)
467 sgen_gchandle_iterate (track
? HANDLE_WEAK_TRACK
: HANDLE_WEAK
, generation
, null_link_if_necessary
, &ctx
);
469 //we're always called for gen zero. !track means short ref
470 if (generation
== 0 && !track
)
471 sgen_gchandle_iterate (HANDLE_WEAK_FIELDS
, generation
, scan_for_weak
, &ctx
);
475 SgenObjectPredicateFunc predicate
;
477 } WeakLinkAlivePredicateClosure
;
480 null_link_if (gpointer hidden
, GCHandleType handle_type
, int max_generation
, gpointer user
)
482 WeakLinkAlivePredicateClosure
*closure
= (WeakLinkAlivePredicateClosure
*)user
;
485 if (!MONO_GC_HANDLE_VALID (hidden
))
488 obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (hidden
, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type
));
489 SGEN_ASSERT (0, obj
, "Why is the hidden pointer NULL?");
491 if (object_older_than (obj
, max_generation
))
494 if (closure
->predicate (obj
, closure
->data
))
495 return MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (handle_type
));
500 /* LOCKING: requires that the GC lock is held */
502 sgen_null_links_if (SgenObjectPredicateFunc predicate
, void *data
, int generation
, gboolean track
)
504 WeakLinkAlivePredicateClosure closure
= { predicate
, data
};
505 sgen_gchandle_iterate (track
? HANDLE_WEAK_TRACK
: HANDLE_WEAK
, generation
, null_link_if
, &closure
);
509 sgen_register_obj_with_weak_fields (GCObject
*obj
)
512 // We use a gc handle to be able to do some processing for these objects at every gc
514 alloc_handle (gc_handles_for_type (HANDLE_WEAK_FIELDS
), obj
, FALSE
);
518 sgen_init_gchandles (void)
520 #ifdef HEAVY_STATISTICS
521 mono_counters_register ("GC handles allocated", MONO_COUNTER_GC
| MONO_COUNTER_UINT
, (void *)&stat_gc_handles_allocated
);
522 mono_counters_register ("max GC handles allocated", MONO_COUNTER_GC
| MONO_COUNTER_UINT
, (void *)&stat_gc_handles_max_allocated
);