5 * Copyright (C) 2015 Xamarin Inc
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
13 #include "mono/sgen/sgen-gc.h"
14 #include "mono/sgen/sgen-client.h"
15 #include "mono/sgen/sgen-array-list.h"
16 #include "mono/utils/mono-membar.h"
18 #ifdef HEAVY_STATISTICS
19 static volatile guint32 stat_gc_handles_allocated
= 0;
20 static volatile guint32 stat_gc_handles_max_allocated
= 0;
25 size_t num_handles
[HANDLE_TYPE_MAX
];
28 static gboolean do_gchandle_stats
= FALSE
;
30 static SgenHashTable gchandle_class_hash_table
= SGEN_HASH_TABLE_INIT (INTERNAL_MEM_STATISTICS
, INTERNAL_MEM_STAT_GCHANDLE_CLASS
, sizeof (GCHandleClassEntry
), g_str_hash
, g_str_equal
);
33 * A table of GC handle data, implementing a simple lock-free bitmap allocator.
35 * Each entry in a bucket is a pointer with two tag bits: if
36 * 'GC_HANDLE_OCCUPIED' returns true for a slot, then the slot is occupied; if
37 * so, then 'GC_HANDLE_VALID' gives whether the entry refers to a valid (1) or
38 * NULL (0) object reference. If the reference is valid, then the pointer is an
39 * object pointer. If the reference is NULL, and 'GC_HANDLE_TYPE_IS_WEAK' is
40 * true for 'type', then the pointer is a metadata pointer--this allows us to
41 * retrieve the domain ID of an expired weak reference in Mono.
45 SgenArrayList entries_array
;
50 protocol_gchandle_update (int handle_type
, gpointer link
, gpointer old_value
, gpointer new_value
)
52 gboolean old
= MONO_GC_HANDLE_IS_OBJECT_POINTER (old_value
);
53 gboolean new_
= MONO_GC_HANDLE_IS_OBJECT_POINTER (new_value
);
54 gboolean track
= handle_type
== HANDLE_WEAK_TRACK
;
56 if (!MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type
))
60 sgen_binary_protocol_dislink_add (link
, MONO_GC_REVEAL_POINTER (new_value
, TRUE
), track
);
61 else if (old
&& !new_
)
62 sgen_binary_protocol_dislink_remove (link
, track
);
63 else if (old
&& new_
&& old_value
!= new_value
)
64 sgen_binary_protocol_dislink_update (link
, MONO_GC_REVEAL_POINTER (new_value
, TRUE
), track
);
67 /* Returns the new value in the slot, or NULL if the CAS failed. */
69 try_set_slot (volatile gpointer
*slot
, GCObject
*obj
, gpointer old
, GCHandleType type
)
73 new_
= MONO_GC_HANDLE_OBJECT_POINTER (obj
, GC_HANDLE_TYPE_IS_WEAK (type
));
75 new_
= MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (type
));
76 SGEN_ASSERT (0, new_
, "Why is the occupied bit not set?");
77 if (mono_atomic_cas_ptr (slot
, new_
, old
) == old
) {
78 protocol_gchandle_update (type
, (gpointer
)slot
, old
, new_
);
85 is_slot_set (volatile gpointer
*slot
)
87 gpointer entry
= *slot
;
88 if (MONO_GC_HANDLE_OCCUPIED (entry
))
93 /* Try to claim a slot by setting its occupied bit. */
95 try_occupy_slot (volatile gpointer
*slot
, gpointer obj
, int data
)
97 if (is_slot_set (slot
))
99 return try_set_slot (slot
, (GCObject
*)obj
, NULL
, (GCHandleType
)data
) != NULL
;
103 bucket_alloc_callback (gpointer
*bucket
, guint32 new_bucket_size
, gboolean alloc
)
106 sgen_register_root ((char *)bucket
, new_bucket_size
, SGEN_DESCRIPTOR_NULL
, ROOT_TYPE_PINNED
, MONO_ROOT_SOURCE_GC_HANDLE
, NULL
, "GC Handle Bucket (SGen, Pinned)");
108 sgen_deregister_root ((char *)bucket
);
112 bucket_alloc_report_root (gpointer
*bucket
, guint32 new_bucket_size
, gboolean alloc
)
115 sgen_client_root_registered ((char *)bucket
, new_bucket_size
, MONO_ROOT_SOURCE_GC_HANDLE
, NULL
, "GC Handle Bucket (SGen, Normal)");
117 sgen_client_root_deregistered ((char *)bucket
);
120 static HandleData gc_handles
[] = {
121 { SGEN_ARRAY_LIST_INIT (NULL
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_WEAK
) },
122 { SGEN_ARRAY_LIST_INIT (NULL
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_WEAK_TRACK
) },
123 { SGEN_ARRAY_LIST_INIT (bucket_alloc_report_root
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_NORMAL
) },
124 { SGEN_ARRAY_LIST_INIT (bucket_alloc_callback
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_PINNED
) },
125 { SGEN_ARRAY_LIST_INIT (NULL
, is_slot_set
, try_occupy_slot
, -1), (HANDLE_WEAK_FIELDS
) },
129 gc_handles_for_type (GCHandleType type
)
131 return type
< HANDLE_TYPE_MAX
? &gc_handles
[type
] : NULL
;
134 /* This assumes that the world is stopped. */
136 sgen_mark_normal_gc_handles (void *addr
, SgenUserMarkFunc mark_func
, void *gc_data
)
138 HandleData
*handles
= gc_handles_for_type (HANDLE_NORMAL
);
139 SgenArrayList
*array
= &handles
->entries_array
;
140 volatile gpointer
*slot
;
141 gpointer hidden
, revealed
;
143 SGEN_ARRAY_LIST_FOREACH_SLOT (array
, slot
) {
145 revealed
= MONO_GC_REVEAL_POINTER (hidden
, FALSE
);
146 if (!MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden
))
148 mark_func ((MonoObject
**)&revealed
, gc_data
);
150 *slot
= MONO_GC_HANDLE_OBJECT_POINTER (revealed
, FALSE
);
151 } SGEN_ARRAY_LIST_END_FOREACH_SLOT
;
155 sgen_gc_handles_report_roots (SgenUserReportRootFunc report_func
, void *gc_data
)
157 HandleData
*handles
= gc_handles_for_type (HANDLE_NORMAL
);
158 SgenArrayList
*array
= &handles
->entries_array
;
159 volatile gpointer
*slot
;
160 gpointer hidden
, revealed
;
162 SGEN_ARRAY_LIST_FOREACH_SLOT (array
, slot
) {
164 revealed
= MONO_GC_REVEAL_POINTER (hidden
, FALSE
);
166 if (MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden
))
167 report_func ((void*)slot
, (GCObject
*)revealed
, gc_data
);
168 } SGEN_ARRAY_LIST_END_FOREACH_SLOT
;
172 alloc_handle (HandleData
*handles
, GCObject
*obj
, gboolean track
)
175 SgenArrayList
*array
= &handles
->entries_array
;
178 * If a GC happens shortly after a new bucket is allocated, the entire
179 * bucket could be scanned even though it's mostly empty. To avoid this,
180 * we track the maximum index seen so far, so that we can skip the empty
183 * Note that we update `next_slot` before we even try occupying the
184 * slot. If we did it the other way around and a GC happened in
185 * between, the GC wouldn't know that the slot was occupied. This is
186 * not a huge deal since `obj` is on the stack and thus pinned anyway,
187 * but hopefully some day it won't be anymore.
189 index
= sgen_array_list_add (array
, obj
, handles
->type
, TRUE
);
190 #ifdef HEAVY_STATISTICS
191 mono_atomic_inc_i32 ((volatile gint32
*)&stat_gc_handles_allocated
);
192 if (stat_gc_handles_allocated
> stat_gc_handles_max_allocated
)
193 stat_gc_handles_max_allocated
= stat_gc_handles_allocated
;
195 /* Ensure that a GC handle cannot be given to another thread without the slot having been set. */
196 mono_memory_write_barrier ();
197 res
= MONO_GC_HANDLE (index
, handles
->type
);
198 sgen_client_gchandle_created ((GCHandleType
)handles
->type
, obj
, res
);
203 object_older_than (GCObject
*object
, int generation
)
205 return generation
== GENERATION_NURSERY
&& !sgen_ptr_in_nursery (object
);
209 * Maps a function over all GC handles.
210 * This assumes that the world is stopped!
213 sgen_gchandle_iterate (GCHandleType handle_type
, int max_generation
, SgenGCHandleIterateCallback callback
, gpointer user
)
215 HandleData
*handle_data
= gc_handles_for_type (handle_type
);
216 SgenArrayList
*array
= &handle_data
->entries_array
;
217 gpointer hidden
, result
, occupied
;
218 volatile gpointer
*slot
;
220 /* If a new bucket has been allocated, but the capacity has not yet been
221 * increased, nothing can yet have been allocated in the bucket because the
222 * world is stopped, so we shouldn't miss any handles during iteration.
224 SGEN_ARRAY_LIST_FOREACH_SLOT (array
, slot
) {
226 occupied
= (gpointer
) MONO_GC_HANDLE_OCCUPIED (hidden
);
227 g_assert (hidden
? !!occupied
: !occupied
);
230 result
= callback (hidden
, handle_type
, max_generation
, user
);
232 SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (result
), "Why did the callback return an unoccupied entry?");
234 HEAVY_STAT (mono_atomic_dec_i32 ((volatile gint32
*)&stat_gc_handles_allocated
));
235 protocol_gchandle_update (handle_type
, (gpointer
)slot
, hidden
, result
);
237 } SGEN_ARRAY_LIST_END_FOREACH_SLOT
;
241 sgen_gchandle_new (GCObject
*obj
, gboolean pinned
)
243 return alloc_handle (gc_handles_for_type (pinned
? HANDLE_PINNED
: HANDLE_NORMAL
), obj
, FALSE
);
247 sgen_gchandle_new_weakref (GCObject
*obj
, gboolean track_resurrection
)
249 return alloc_handle (gc_handles_for_type (track_resurrection
? HANDLE_WEAK_TRACK
: HANDLE_WEAK
), obj
, track_resurrection
);
253 link_get (volatile gpointer
*link_addr
, gboolean is_weak
)
255 void *volatile *link_addr_volatile
;
259 link_addr_volatile
= link_addr
;
260 ptr
= (void*)*link_addr_volatile
;
262 * At this point we have a hidden pointer. If the GC runs
263 * here, it will not recognize the hidden pointer as a
264 * reference, and if the object behind it is not referenced
265 * elsewhere, it will be freed. Once the world is restarted
266 * we reveal the pointer, giving us a pointer to a freed
267 * object. To make sure we don't return it, we load the
268 * hidden pointer again. If it's still the same, we can be
269 * sure the object reference is valid.
271 if (ptr
&& MONO_GC_HANDLE_IS_OBJECT_POINTER (ptr
))
272 obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (ptr
, is_weak
);
278 * If a GC happens here, obj needs to be on the stack or in a
279 * register, so we need to prevent this from being reordered
282 sgen_dummy_use (obj
);
283 mono_memory_barrier ();
286 sgen_client_ensure_weak_gchandles_accessible ();
288 if ((void*)*link_addr_volatile
!= ptr
)
295 sgen_gchandle_get_target (guint32 gchandle
)
297 guint index
= MONO_GC_HANDLE_SLOT (gchandle
);
298 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
299 HandleData
*handles
= gc_handles_for_type (type
);
300 /* Invalid handles are possible; accessing one should produce NULL. (#34276) */
303 return link_get (sgen_array_list_get_slot (&handles
->entries_array
, index
), MONO_GC_HANDLE_TYPE_IS_WEAK (type
));
307 sgen_gchandle_set_target (guint32 gchandle
, GCObject
*obj
)
309 guint32 index
= MONO_GC_HANDLE_SLOT (gchandle
);
310 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
311 HandleData
*handles
= gc_handles_for_type (type
);
312 volatile gpointer
*slot
;
318 slot
= sgen_array_list_get_slot (&handles
->entries_array
, index
);
322 SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (entry
), "Why are we setting the target on an unoccupied slot?");
323 } while (!try_set_slot (slot
, obj
, entry
, (GCHandleType
)handles
->type
));
327 mono_gchandle_slot_metadata (volatile gpointer
*slot
, gboolean is_weak
)
333 if (!MONO_GC_HANDLE_OCCUPIED (entry
))
335 if (MONO_GC_HANDLE_IS_OBJECT_POINTER (entry
)) {
336 GCObject
*obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (entry
, is_weak
);
337 /* See note [dummy use]. */
338 sgen_dummy_use (obj
);
340 * FIXME: The compiler could technically not carry a reference to obj around
341 * at this point and recompute it later, in which case we would still use
346 return sgen_client_metadata_for_object (obj
);
348 metadata
= MONO_GC_REVEAL_POINTER (entry
, is_weak
);
349 /* See note [dummy use]. */
350 sgen_dummy_use (metadata
);
357 sgen_gchandle_get_metadata (guint32 gchandle
)
359 guint32 index
= MONO_GC_HANDLE_SLOT (gchandle
);
360 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
361 HandleData
*handles
= gc_handles_for_type (type
);
362 volatile gpointer
*slot
;
366 if (index
>= handles
->entries_array
.capacity
)
369 slot
= sgen_array_list_get_slot (&handles
->entries_array
, index
);
371 return mono_gchandle_slot_metadata (slot
, MONO_GC_HANDLE_TYPE_IS_WEAK (type
));
375 sgen_gchandle_free (guint32 gchandle
)
380 guint32 index
= MONO_GC_HANDLE_SLOT (gchandle
);
381 GCHandleType type
= MONO_GC_HANDLE_TYPE (gchandle
);
382 HandleData
*handles
= gc_handles_for_type (type
);
383 volatile gpointer
*slot
;
388 slot
= sgen_array_list_get_slot (&handles
->entries_array
, index
);
391 if (index
< handles
->entries_array
.capacity
&& MONO_GC_HANDLE_OCCUPIED (entry
)) {
393 protocol_gchandle_update (handles
->type
, (gpointer
)slot
, entry
, NULL
);
394 HEAVY_STAT (mono_atomic_dec_i32 ((volatile gint32
*)&stat_gc_handles_allocated
));
396 /* print a warning? */
398 sgen_client_gchandle_destroyed ((GCHandleType
)handles
->type
, gchandle
);
402 * Returns whether to remove the link from its hash.
405 null_link_if_necessary (gpointer hidden
, GCHandleType handle_type
, int max_generation
, gpointer user
)
407 const gboolean is_weak
= GC_HANDLE_TYPE_IS_WEAK (handle_type
);
408 ScanCopyContext
*ctx
= (ScanCopyContext
*)user
;
412 if (!MONO_GC_HANDLE_VALID (hidden
))
415 obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (hidden
, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type
));
416 SGEN_ASSERT (0, obj
, "Why is the hidden pointer NULL?");
418 if (object_older_than (obj
, max_generation
))
421 if (sgen_major_collector
.is_object_live (obj
))
424 /* Clear link if object is ready for finalization. This check may be redundant wrt is_object_live(). */
425 if (sgen_gc_is_object_ready_for_finalization (obj
))
426 return MONO_GC_HANDLE_METADATA_POINTER (sgen_client_metadata_for_object (obj
), is_weak
);
429 ctx
->ops
->copy_or_mark_object (©
, ctx
->queue
);
430 SGEN_ASSERT (0, copy
, "Why couldn't we copy the object?");
431 /* Update link if object was moved. */
432 return MONO_GC_HANDLE_OBJECT_POINTER (copy
, is_weak
);
436 scan_for_weak (gpointer hidden
, GCHandleType handle_type
, int max_generation
, gpointer user
)
438 const gboolean is_weak
= GC_HANDLE_TYPE_IS_WEAK (handle_type
);
439 ScanCopyContext
*ctx
= (ScanCopyContext
*)user
;
441 if (!MONO_GC_HANDLE_VALID (hidden
))
444 GCObject
*obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (hidden
, is_weak
);
446 /* If the object is dead we free the gc handle */
447 if (!sgen_is_object_alive_for_current_gen (obj
))
451 ctx
->ops
->copy_or_mark_object (&obj
, ctx
->queue
);
454 gsize
*weak_bitmap
= sgen_client_get_weak_bitmap (SGEN_LOAD_VTABLE (obj
), &nbits
);
455 for (int i
= 0; i
< nbits
; ++i
) {
456 if (weak_bitmap
[i
/ (sizeof (gsize
) * 8)] & ((gsize
)1 << (i
% (sizeof (gsize
) * 8)))) {
457 GCObject
**addr
= (GCObject
**)((char*)obj
+ (i
* sizeof (gpointer
)));
458 GCObject
*field
= *addr
;
460 /* if the object in the weak field is alive, we relocate it */
461 if (field
&& sgen_is_object_alive_for_current_gen (field
))
462 ctx
->ops
->copy_or_mark_object (addr
, ctx
->queue
);
468 /* Update link if object was moved. */
469 return MONO_GC_HANDLE_OBJECT_POINTER (obj
, is_weak
);
472 /* LOCKING: requires that the GC lock is held */
474 sgen_null_link_in_range (int generation
, ScanCopyContext ctx
, gboolean track
)
476 sgen_gchandle_iterate (track
? HANDLE_WEAK_TRACK
: HANDLE_WEAK
, generation
, null_link_if_necessary
, &ctx
);
478 //we're always called for gen zero. !track means short ref
479 if (generation
== 0 && !track
)
480 sgen_gchandle_iterate (HANDLE_WEAK_FIELDS
, generation
, scan_for_weak
, &ctx
);
484 SgenObjectPredicateFunc predicate
;
486 } WeakLinkAlivePredicateClosure
;
489 null_link_if (gpointer hidden
, GCHandleType handle_type
, int max_generation
, gpointer user
)
491 WeakLinkAlivePredicateClosure
*closure
= (WeakLinkAlivePredicateClosure
*)user
;
494 if (!MONO_GC_HANDLE_VALID (hidden
))
497 obj
= (GCObject
*)MONO_GC_REVEAL_POINTER (hidden
, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type
));
498 SGEN_ASSERT (0, obj
, "Why is the hidden pointer NULL?");
500 if (object_older_than (obj
, max_generation
))
503 if (closure
->predicate (obj
, closure
->data
))
504 return MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (handle_type
));
509 /* LOCKING: requires that the GC lock is held */
511 sgen_null_links_if (SgenObjectPredicateFunc predicate
, void *data
, int generation
, gboolean track
)
513 WeakLinkAlivePredicateClosure closure
= { predicate
, data
};
514 sgen_gchandle_iterate (track
? HANDLE_WEAK_TRACK
: HANDLE_WEAK
, generation
, null_link_if
, &closure
);
518 sgen_register_obj_with_weak_fields (GCObject
*obj
)
521 // We use a gc handle to be able to do some processing for these objects at every gc
523 alloc_handle (gc_handles_for_type (HANDLE_WEAK_FIELDS
), obj
, FALSE
);
526 #ifndef DISABLE_SGEN_DEBUG_HELPERS
528 sgen_gchandle_stats_enable (void)
530 do_gchandle_stats
= TRUE
;
534 sgen_gchandle_stats_register_vtable (GCVTable vtable
, int handle_type
)
536 GCHandleClassEntry
*entry
;
538 char *name
= g_strdup_printf ("%s.%s", sgen_client_vtable_get_namespace (vtable
), sgen_client_vtable_get_name (vtable
));
539 entry
= (GCHandleClassEntry
*) sgen_hash_table_lookup (&gchandle_class_hash_table
, name
);
544 // Create the entry for this class and get the address of it
545 GCHandleClassEntry empty_entry
;
546 memset (&empty_entry
, 0, sizeof (GCHandleClassEntry
));
547 sgen_hash_table_replace (&gchandle_class_hash_table
, name
, &empty_entry
, NULL
);
548 entry
= (GCHandleClassEntry
*) sgen_hash_table_lookup (&gchandle_class_hash_table
, name
);
551 entry
->num_handles
[handle_type
]++;
555 sgen_gchandle_stats_count (void)
559 sgen_hash_table_clean (&gchandle_class_hash_table
);
561 for (i
= HANDLE_TYPE_MIN
; i
< HANDLE_TYPE_MAX
; i
++) {
562 HandleData
*handles
= gc_handles_for_type ((GCHandleType
)i
);
563 SgenArrayList
*array
= &handles
->entries_array
;
564 volatile gpointer
*slot
;
565 gpointer hidden
, revealed
;
567 SGEN_ARRAY_LIST_FOREACH_SLOT (array
, slot
) {
569 revealed
= MONO_GC_REVEAL_POINTER (hidden
, MONO_GC_HANDLE_TYPE_IS_WEAK (i
));
571 if (MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden
))
572 sgen_gchandle_stats_register_vtable (SGEN_LOAD_VTABLE (revealed
), i
);
573 } SGEN_ARRAY_LIST_END_FOREACH_SLOT
;
578 sgen_gchandle_stats_report (void)
581 GCHandleClassEntry
*gchandle_entry
;
583 if (!do_gchandle_stats
)
586 sgen_gchandle_stats_count ();
588 mono_gc_printf (sgen_gc_debug_file
, "\n%-60s %10s %10s %10s\n", "Class", "Normal", "Weak", "Pinned");
589 SGEN_HASH_TABLE_FOREACH (&gchandle_class_hash_table
, char *, name
, GCHandleClassEntry
*, gchandle_entry
) {
590 mono_gc_printf (sgen_gc_debug_file
, "%-60s", name
);
591 mono_gc_printf (sgen_gc_debug_file
, " %10ld", (long)gchandle_entry
->num_handles
[HANDLE_NORMAL
]);
592 size_t weak_handles
= gchandle_entry
->num_handles
[HANDLE_WEAK
] +
593 gchandle_entry
->num_handles
[HANDLE_WEAK_TRACK
] +
594 gchandle_entry
->num_handles
[HANDLE_WEAK_FIELDS
];
595 mono_gc_printf (sgen_gc_debug_file
, " %10ld", (long)weak_handles
);
596 mono_gc_printf (sgen_gc_debug_file
, " %10ld", (long)gchandle_entry
->num_handles
[HANDLE_PINNED
]);
597 mono_gc_printf (sgen_gc_debug_file
, "\n");
598 } SGEN_HASH_TABLE_FOREACH_END
;
603 sgen_init_gchandles (void)
605 #ifdef HEAVY_STATISTICS
606 mono_counters_register ("GC handles allocated", MONO_COUNTER_GC
| MONO_COUNTER_UINT
, (void *)&stat_gc_handles_allocated
);
607 mono_counters_register ("max GC handles allocated", MONO_COUNTER_GC
| MONO_COUNTER_UINT
, (void *)&stat_gc_handles_max_allocated
);