2 * sgen-marksweep-drain-gray-stack.h: The copy/mark and gray stack
3 * draining functions of the M&S major collector.
5 * Copyright (C) 2014 Xamarin Inc
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
11 * COPY_OR_MARK_FUNCTION_NAME must be defined to be the function name of the copy/mark
14 * SCAN_OBJECT_FUNCTION_NAME must be defined to be the function name of the object scanning
17 * DRAIN_GRAY_STACK_FUNCTION_NAME must be defined to be the function name of the gray stack
20 * Define COPY_OR_MARK_WITH_EVACUATION to support evacuation.
23 /* Returns whether the object is still in the nursery. */
24 static inline MONO_ALWAYS_INLINE gboolean
25 COPY_OR_MARK_FUNCTION_NAME (GCObject
**ptr
, GCObject
*obj
, SgenGrayQueue
*queue
)
29 #ifdef HEAVY_STATISTICS
30 ++stat_optimized_copy
;
34 if ((forwarded
= SGEN_OBJECT_IS_FORWARDED (obj
)))
35 desc
= sgen_obj_get_descriptor_safe (forwarded
);
37 desc
= sgen_obj_get_descriptor_safe (obj
);
39 sgen_descriptor_count_copied_object (desc
);
43 SGEN_ASSERT (9, obj
, "null object from pointer %p", ptr
);
44 #if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
45 SGEN_ASSERT (9, current_collection_generation
== GENERATION_OLD
, "old gen parallel allocator called from a %d collection", current_collection_generation
);
48 if (sgen_ptr_in_nursery (obj
)) {
49 #if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
51 GCObject
*forwarded
, *old_obj
;
52 mword vtable_word
= *(mword
*)obj
;
54 HEAVY_STAT (++stat_optimized_copy_nursery
);
56 #if SGEN_MAX_DEBUG_LEVEL >= 9
57 if (sgen_nursery_is_to_space (obj
))
58 SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word
) && !SGEN_VTABLE_IS_FORWARDED (vtable_word
), "To-space object can't be pinned or forwarded.");
61 if (SGEN_VTABLE_IS_PINNED (vtable_word
)) {
62 SGEN_ASSERT (9, !SGEN_VTABLE_IS_FORWARDED (vtable_word
), "Cannot be both pinned and forwarded.");
63 HEAVY_STAT (++stat_optimized_copy_nursery_pinned
);
66 if ((forwarded
= (GCObject
*)SGEN_VTABLE_IS_FORWARDED (vtable_word
))) {
67 HEAVY_STAT (++stat_optimized_copy_nursery_forwarded
);
68 SGEN_UPDATE_REFERENCE (ptr
, forwarded
);
69 return sgen_ptr_in_nursery (forwarded
);
72 /* An object in the nursery To Space has already been copied and grayed. Nothing to do. */
73 if (sgen_nursery_is_to_space (obj
))
76 #ifdef COPY_OR_MARK_WITH_EVACUATION
80 obj
= copy_object_no_checks (obj
, queue
);
81 if (G_UNLIKELY (old_obj
== obj
)) {
83 * If we fail to evacuate an object we just stop doing it for a
84 * given block size as all other will surely fail too.
86 /* FIXME: test this case somehow. */
87 if (!sgen_ptr_in_nursery (obj
)) {
89 block
= MS_BLOCK_FOR_OBJ (obj
);
90 size_index
= block
->obj_size_index
;
91 evacuate_block_obj_sizes
[size_index
] = FALSE
;
92 MS_MARK_OBJECT_AND_ENQUEUE (obj
, sgen_obj_get_descriptor (obj
), block
, queue
);
97 HEAVY_STAT (++stat_objects_copied_major
);
98 SGEN_UPDATE_REFERENCE (ptr
, obj
);
100 if (sgen_ptr_in_nursery (obj
))
104 * FIXME: See comment for copy_object_no_checks(). If
105 * we have that, we can let the allocation function
106 * give us the block info, too, and we won't have to
109 * FIXME (2): We should rework this to avoid all those nursery checks.
112 * For the split nursery allocator the object might
113 * still be in the nursery despite having being
114 * promoted, in which case we can't mark it.
116 block
= MS_BLOCK_FOR_OBJ (obj
);
117 MS_CALC_MARK_BIT (word
, bit
, obj
);
118 SGEN_ASSERT (9, !MS_MARK_BIT (block
, word
, bit
), "object %p already marked", obj
);
119 MS_SET_MARK_BIT (block
, word
, bit
);
120 binary_protocol_mark (obj
, (gpointer
)SGEN_LOAD_VTABLE (obj
), sgen_safe_object_get_size (obj
));
125 mword vtable_word
= *(mword
*)obj
;
129 HEAVY_STAT (++stat_optimized_copy_major
);
131 #ifdef COPY_OR_MARK_WITH_EVACUATION
134 if ((forwarded
= (GCObject
*)SGEN_VTABLE_IS_FORWARDED (vtable_word
))) {
135 HEAVY_STAT (++stat_optimized_copy_major_forwarded
);
136 SGEN_UPDATE_REFERENCE (ptr
, forwarded
);
137 SGEN_ASSERT (9, !sgen_ptr_in_nursery (forwarded
), "Cannot be forwarded to nursery.");
143 SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word
), "Pinned object in non-pinned block?");
145 /* We untag the vtable for concurrent M&S, in case bridge is running and it tagged it */
146 desc
= sgen_vtable_get_descriptor ((GCVTable
)SGEN_POINTER_UNTAG_VTABLE (vtable_word
));
147 type
= desc
& DESC_TYPE_MASK
;
149 if (sgen_safe_object_is_small (obj
, type
)) {
150 #ifdef HEAVY_STATISTICS
151 if (type
<= DESC_TYPE_MAX_SMALL_OBJ
)
152 ++stat_optimized_copy_major_small_fast
;
154 ++stat_optimized_copy_major_small_slow
;
157 block
= MS_BLOCK_FOR_OBJ (obj
);
159 #ifdef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
160 if (G_UNLIKELY (major_block_is_evacuating (block
))) {
162 * We don't copy within the concurrent phase. These objects will
163 * be handled below in the finishing pause, by scanning the mod-union
170 #ifdef COPY_OR_MARK_WITH_EVACUATION
171 if (major_block_is_evacuating (block
)) {
172 HEAVY_STAT (++stat_optimized_copy_major_small_evacuate
);
177 MS_MARK_OBJECT_AND_ENQUEUE (obj
, desc
, block
, queue
);
179 HEAVY_STAT (++stat_optimized_copy_major_large
);
181 if (sgen_los_object_is_pinned (obj
))
183 binary_protocol_pin (obj
, (gpointer
)SGEN_LOAD_VTABLE (obj
), sgen_safe_object_get_size (obj
));
185 sgen_los_pin_object (obj
);
186 if (SGEN_OBJECT_HAS_REFERENCES (obj
))
187 GRAY_OBJECT_ENQUEUE (queue
, obj
, desc
);
196 SCAN_OBJECT_FUNCTION_NAME (GCObject
*full_object
, SgenDescriptor desc
, SgenGrayQueue
*queue
)
198 char *start
= (char*)full_object
;
200 #ifdef HEAVY_STATISTICS
201 ++stat_optimized_major_scan
;
202 if (!sgen_gc_descr_has_references (desc
))
203 ++stat_optimized_major_scan_no_refs
;
204 sgen_descriptor_count_scanned_object (desc
);
206 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
207 add_scanned_object (start
);
210 /* Now scan the object. */
213 #if defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
214 #define HANDLE_PTR(ptr,obj) do { \
215 GCObject *__old = *(ptr); \
216 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
217 if (__old && !sgen_ptr_in_nursery (__old)) { \
218 if (G_UNLIKELY (!sgen_ptr_in_nursery (ptr) && \
219 sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \
220 major_block_is_evacuating (MS_BLOCK_FOR_OBJ (__old)))) { \
221 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
223 PREFETCH_READ (__old); \
224 COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
227 if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \
228 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
231 #elif defined(COPY_OR_MARK_CONCURRENT)
232 #define HANDLE_PTR(ptr,obj) do { \
233 GCObject *__old = *(ptr); \
234 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
235 if (__old && !sgen_ptr_in_nursery (__old)) { \
236 PREFETCH_READ (__old); \
237 COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
239 if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \
240 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
244 #define HANDLE_PTR(ptr,obj) do { \
245 GCObject *__old = *(ptr); \
246 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
248 gboolean __still_in_nursery = COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
249 if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
250 GCObject *__copy = *(ptr); \
251 sgen_add_to_global_remset ((ptr), __copy); \
257 #define SCAN_OBJECT_PROTOCOL
258 #include "sgen-scan-object.h"
261 #ifdef SCAN_VTYPE_FUNCTION_NAME
263 SCAN_VTYPE_FUNCTION_NAME (GCObject
*full_object
, char *start
, SgenDescriptor desc
, SgenGrayQueue
*queue
BINARY_PROTOCOL_ARG (size_t size
))
265 SGEN_OBJECT_LAYOUT_STATISTICS_DECLARE_BITMAP
;
267 #ifdef HEAVY_STATISTICS
268 /* FIXME: We're half scanning this object. How do we account for that? */
269 //add_scanned_object (start);
272 /* The descriptors include info about the object header as well */
273 start
-= SGEN_CLIENT_OBJECT_HEADER_SIZE
;
275 /* We use the same HANDLE_PTR from the obj scan function */
276 #define SCAN_OBJECT_NOVTABLE
277 #define SCAN_OBJECT_PROTOCOL
278 #include "sgen-scan-object.h"
280 SGEN_OBJECT_LAYOUT_STATISTICS_COMMIT_BITMAP
;
284 #ifdef SCAN_PTR_FIELD_FUNCTION_NAME
286 SCAN_PTR_FIELD_FUNCTION_NAME (GCObject
*full_object
, GCObject
**ptr
, SgenGrayQueue
*queue
)
288 HANDLE_PTR (ptr
, NULL
);
293 DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue
*queue
)
295 #if defined(COPY_OR_MARK_CONCURRENT) || defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
297 for (i
= 0; i
< 32; i
++) {
304 HEAVY_STAT (++stat_drain_loops
);
306 GRAY_OBJECT_DEQUEUE (queue
, &obj
, &desc
);
310 SCAN_OBJECT_FUNCTION_NAME (obj
, desc
, queue
);
315 #undef COPY_OR_MARK_FUNCTION_NAME
316 #undef COPY_OR_MARK_WITH_EVACUATION
317 #undef COPY_OR_MARK_CONCURRENT
318 #undef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
319 #undef SCAN_OBJECT_FUNCTION_NAME
320 #undef SCAN_VTYPE_FUNCTION_NAME
321 #undef SCAN_PTR_FIELD_FUNCTION_NAME
322 #undef DRAIN_GRAY_STACK_FUNCTION_NAME