3 * Object allocation routines + managed allocators
6 * Paolo Molaro (lupus@ximian.com)
7 * Rodrigo Kumpera (kumpera@gmail.com)
9 * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
10 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
11 * Copyright 2011 Xamarin, Inc.
12 * Copyright (C) 2012 Xamarin Inc
14 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
18 * ######################################################################
19 * ######## Object allocation
20 * ######################################################################
21 * This section of code deals with allocating memory for objects.
22 * There are several ways:
23 * *) allocate large objects
24 * *) allocate normal objects
25 * *) fast lock-free allocation
26 * *) allocation of pinned objects
34 #include "mono/sgen/sgen-gc.h"
35 #include "mono/sgen/sgen-protocol.h"
36 #include "mono/sgen/sgen-memory-governor.h"
37 #include "mono/sgen/sgen-client.h"
38 #include "mono/utils/mono-memory-model.h"
39 #include "mono/utils/mono-tls-inline.h"
41 #define ALIGN_UP SGEN_ALIGN_UP
42 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
43 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
45 #ifdef HEAVY_STATISTICS
46 static guint64 stat_objects_alloced
= 0;
47 static guint64 stat_bytes_alloced
= 0;
48 static guint64 stat_bytes_alloced_los
= 0;
51 /* The total number of bytes allocated so far in program execution by all attached threads.
52 * This is not constantly syncrhonized, but only updated on each GC. */
53 static guint64 bytes_allocated_attached
= 0;
55 /* Total bytes allocated so far in program execution by detached threads */
56 static guint64 bytes_allocated_detached
= 0;
59 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
60 * from nursery fragments.
61 * tlab_next is the pointer to the space inside the TLAB where the next object will
63 * tlab_temp_end is the pointer to the end of the temporary space reserved for
64 * the allocation: it allows us to set the scan starts at reasonable intervals.
65 * tlab_real_end points to the end of the TLAB.
68 #define TLAB_START (__thread_info__->tlab_start)
69 #define TLAB_NEXT (__thread_info__->tlab_next)
70 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
71 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
74 increment_thread_allocation_counter (size_t byte_size
)
76 mono_thread_info_current ()->total_bytes_allocated
+= byte_size
;
80 alloc_degraded (GCVTable vtable
, size_t size
, gboolean for_mature
)
83 increment_thread_allocation_counter (size
);
86 sgen_client_degraded_allocation ();
87 SGEN_ATOMIC_ADD_P (sgen_degraded_mode
, size
);
88 sgen_ensure_free_space (size
, GENERATION_OLD
);
91 if (sgen_need_major_collection (size
, &forced
))
92 sgen_perform_collection (size
, GENERATION_OLD
, "mature allocation failure", !for_mature
|| forced
, TRUE
);
96 p
= sgen_major_collector
.alloc_degraded (vtable
, size
);
99 sgen_binary_protocol_alloc_degraded (p
, vtable
, size
, sgen_client_get_provenance ());
105 zero_tlab_if_necessary (void *p
, size_t size
)
107 if (sgen_nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
|| sgen_nursery_clear_policy
== CLEAR_AT_TLAB_CREATION_DEBUG
) {
111 * This function is called for all allocations in
112 * TLABs. TLABs originate from fragments, which are
113 * initialized to be faux arrays. The remainder of
114 * the fragments are zeroed out at initialization for
115 * CLEAR_AT_GC, so here we just need to make sure that
116 * the array header is zeroed. Since we don't know
117 * whether we're called for the start of a fragment or
118 * for somewhere in between, we zero in any case, just
121 sgen_client_zero_array_fill_header (p
, size
);
126 * Provide a variant that takes just the vtable for small fixed-size objects.
127 * The aligned size is already computed and stored in vt->gc_descr.
128 * Note: every SGEN_SCAN_START_SIZE or so we are given the chance to do some special
129 * processing. We can keep track of where objects start, for example,
130 * so when we scan the thread stacks for pinned objects, we can start
131 * a search for the pinned object in SGEN_SCAN_START_SIZE chunks.
134 sgen_alloc_obj_nolock (GCVTable vtable
, size_t size
)
136 /* FIXME: handle OOM */
139 size_t real_size
= size
;
144 HEAVY_STAT (++stat_objects_alloced
);
145 if (real_size
<= SGEN_MAX_SMALL_OBJ_SIZE
)
146 HEAVY_STAT (stat_bytes_alloced
+= size
);
148 HEAVY_STAT (stat_bytes_alloced_los
+= size
);
150 size
= ALIGN_UP (size
);
152 SGEN_ASSERT (6, sgen_vtable_get_descriptor (vtable
), "VTable without descriptor");
154 if (G_UNLIKELY (sgen_has_per_allocation_action
)) {
155 static int alloc_count
;
156 int current_alloc
= mono_atomic_inc_i32 (&alloc_count
);
158 if (sgen_collect_before_allocs
) {
159 if (((current_alloc
% sgen_collect_before_allocs
) == 0) && sgen_nursery_section
) {
160 sgen_perform_collection (0, GENERATION_NURSERY
, "collect-before-alloc-triggered", TRUE
, TRUE
);
161 if (!sgen_degraded_mode
&& sgen_can_alloc_size (size
) && real_size
<= SGEN_MAX_SMALL_OBJ_SIZE
) {
163 g_assert_not_reached ();
166 } else if (sgen_verify_before_allocs
) {
167 if ((current_alloc
% sgen_verify_before_allocs
) == 0)
168 sgen_check_whole_heap_stw ();
173 * We must already have the lock here instead of after the
174 * fast path because we might be interrupted in the fast path
175 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
176 * and we'll end up allocating an object in a fragment which
177 * no longer belongs to us.
179 * The managed allocator does not do this, but it's treated
180 * specially by the world-stopping code.
183 if (real_size
> SGEN_MAX_SMALL_OBJ_SIZE
) {
184 p
= (void **)sgen_los_alloc_large_inner (vtable
, ALIGN_UP (real_size
));
186 increment_thread_allocation_counter (size
);
189 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
191 p
= (void**)TLAB_NEXT
;
192 /* FIXME: handle overflow */
193 new_next
= (char*)p
+ size
;
194 TLAB_NEXT
= new_next
;
196 if (G_LIKELY (new_next
< TLAB_TEMP_END
)) {
199 CANARIFY_ALLOC(p
,real_size
);
200 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %" G_GSIZE_FORMAT
"d", p
, vtable
, sgen_client_vtable_get_name (vtable
), size
);
201 sgen_binary_protocol_alloc (p
, vtable
, size
, sgen_client_get_provenance ());
202 g_assert (*p
== NULL
);
203 mono_atomic_store_seq (p
, vtable
);
210 /* there are two cases: the object is too big or we run out of space in the TLAB */
211 /* we also reach here when the thread does its first allocation after a minor
212 * collection, since the tlab_ variables are initialized to NULL.
213 * there can be another case (from ORP), if we cooperate with the runtime a bit:
214 * objects that need finalizers can have the high bit set in their size
215 * so the above check fails and we can readily add the object to the queue.
216 * This avoids taking again the GC lock when registering, but this is moot when
217 * doing thread-local allocation, so it may not be a good idea.
219 if (TLAB_NEXT
>= TLAB_REAL_END
) {
220 int available_in_tlab
;
222 * Run out of space in the TLAB. When this happens, some amount of space
223 * remains in the TLAB, but not enough to satisfy the current allocation
224 * request. We keep the TLAB for future allocations if the remaining
225 * space is above a treshold, and satisfy the allocation directly
226 * from the nursery. Otherwise, we attempt to get a new TLAB from the
227 * nursery and allocate into it.
230 /* when running in degraded mode, we continue allocing that way
231 * for a while, to decrease the number of useless nursery collections.
233 if (sgen_degraded_mode
&& sgen_degraded_mode
< sgen_nursery_size
)
234 return alloc_degraded (vtable
, size
, FALSE
);
236 available_in_tlab
= (int)(TLAB_REAL_END
- TLAB_NEXT
);//We'll never have tlabs > 2Gb
237 if (size
> sgen_tlab_size
|| available_in_tlab
> SGEN_MAX_NURSERY_WASTE
) {
238 /* Allocate directly from the nursery */
239 p
= (void **)sgen_nursery_alloc (size
);
242 * We couldn't allocate from the nursery, so we try
243 * collecting. Even after the collection, we might
244 * still not have enough memory to allocate the
245 * object. The reason will most likely be that we've
246 * run out of memory, but there is the theoretical
247 * possibility that other threads might have consumed
248 * the freed up memory ahead of us.
250 * What we do in this case is allocate degraded, i.e.,
251 * from the major heap.
253 * Ideally we'd like to detect the case of other
254 * threads allocating ahead of us and loop (if we
255 * always loop we will loop endlessly in the case of
258 sgen_ensure_free_space (real_size
, GENERATION_NURSERY
);
259 if (!sgen_degraded_mode
)
260 p
= (void **)sgen_nursery_alloc (size
);
263 increment_thread_allocation_counter (size
);
267 return alloc_degraded (vtable
, size
, TRUE
);
269 zero_tlab_if_necessary (p
, size
);
271 size_t alloc_size
= 0;
273 SGEN_LOG (3, "Retire TLAB: %p-%p [%ld]", TLAB_START
, TLAB_REAL_END
, (long)(TLAB_REAL_END
- TLAB_NEXT
- size
));
274 sgen_nursery_retire_region (p
, available_in_tlab
);
276 p
= (void **)sgen_nursery_alloc_range (sgen_tlab_size
, size
, &alloc_size
);
278 /* See comment above in similar case. */
279 sgen_ensure_free_space (sgen_tlab_size
, GENERATION_NURSERY
);
280 if (!sgen_degraded_mode
)
281 p
= (void **)sgen_nursery_alloc_range (sgen_tlab_size
, size
, &alloc_size
);
284 return alloc_degraded (vtable
, size
, TRUE
);
286 increment_thread_allocation_counter (TLAB_NEXT
- TLAB_START
);
288 /* Allocate a new TLAB from the current nursery fragment */
289 TLAB_START
= (char*)p
;
290 TLAB_NEXT
= TLAB_START
;
291 TLAB_REAL_END
= TLAB_START
+ alloc_size
;
292 TLAB_TEMP_END
= TLAB_START
+ MIN (SGEN_SCAN_START_SIZE
, alloc_size
);
294 zero_tlab_if_necessary (TLAB_START
, alloc_size
);
296 /* Allocate from the TLAB */
297 p
= (void **)TLAB_NEXT
;
299 sgen_set_nursery_scan_start ((char*)p
);
302 /* Reached tlab_temp_end */
304 /* record the scan start so we can find pinned objects more easily */
305 sgen_set_nursery_scan_start ((char*)p
);
306 /* we just bump tlab_temp_end as well */
307 TLAB_TEMP_END
= MIN (TLAB_REAL_END
, TLAB_NEXT
+ SGEN_SCAN_START_SIZE
);
308 SGEN_LOG (5, "Expanding local alloc: %p-%p", TLAB_NEXT
, TLAB_TEMP_END
);
310 CANARIFY_ALLOC(p
,real_size
);
314 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %" G_GSIZE_FORMAT
"d", p
, vtable
, sgen_client_vtable_get_name (vtable
), size
);
315 sgen_binary_protocol_alloc (p
, vtable
, size
, sgen_client_get_provenance ());
316 mono_atomic_store_seq (p
, vtable
);
323 sgen_try_alloc_obj_nolock (GCVTable vtable
, size_t size
)
327 size_t real_size
= size
;
332 size
= ALIGN_UP (size
);
333 SGEN_ASSERT (9, real_size
>= SGEN_CLIENT_MINIMUM_OBJECT_SIZE
, "Object too small");
335 SGEN_ASSERT (6, sgen_vtable_get_descriptor (vtable
), "VTable without descriptor");
337 if (real_size
> SGEN_MAX_SMALL_OBJ_SIZE
)
340 if (G_UNLIKELY (size
> sgen_tlab_size
)) {
341 /* Allocate directly from the nursery */
343 p
= (void **)sgen_nursery_alloc (size
);
347 increment_thread_allocation_counter (size
);
348 sgen_set_nursery_scan_start ((char*)p
);
350 /*FIXME we should use weak memory ops here. Should help specially on x86. */
351 zero_tlab_if_necessary (p
, size
);
353 int available_in_tlab
;
355 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
357 p
= (void**)TLAB_NEXT
;
358 /* FIXME: handle overflow */
359 new_next
= (char*)p
+ size
;
361 real_end
= TLAB_REAL_END
;
362 available_in_tlab
= (int)(real_end
- (char*)p
);//We'll never have tlabs > 2Gb
364 if (G_LIKELY (new_next
< real_end
)) {
365 TLAB_NEXT
= new_next
;
367 /* Second case, we overflowed temp end */
368 if (G_UNLIKELY (new_next
>= TLAB_TEMP_END
)) {
369 sgen_set_nursery_scan_start (new_next
);
370 /* we just bump tlab_temp_end as well */
371 TLAB_TEMP_END
= MIN (TLAB_REAL_END
, TLAB_NEXT
+ SGEN_SCAN_START_SIZE
);
372 SGEN_LOG (5, "Expanding local alloc: %p-%p", TLAB_NEXT
, TLAB_TEMP_END
);
374 } else if (available_in_tlab
> SGEN_MAX_NURSERY_WASTE
) {
375 /* Allocate directly from the nursery */
376 p
= (void **)sgen_nursery_alloc (size
);
380 increment_thread_allocation_counter (size
);
381 zero_tlab_if_necessary (p
, size
);
383 size_t alloc_size
= 0;
385 sgen_nursery_retire_region (p
, available_in_tlab
);
386 new_next
= (char *)sgen_nursery_alloc_range (sgen_tlab_size
, size
, &alloc_size
);
387 p
= (void**)new_next
;
391 increment_thread_allocation_counter (TLAB_NEXT
- TLAB_START
);
393 TLAB_START
= (char*)new_next
;
394 TLAB_NEXT
= new_next
+ size
;
395 TLAB_REAL_END
= new_next
+ alloc_size
;
396 TLAB_TEMP_END
= new_next
+ MIN (SGEN_SCAN_START_SIZE
, alloc_size
);
397 sgen_set_nursery_scan_start ((char*)p
);
399 zero_tlab_if_necessary (new_next
, alloc_size
);
403 HEAVY_STAT (++stat_objects_alloced
);
404 HEAVY_STAT (stat_bytes_alloced
+= size
);
406 CANARIFY_ALLOC(p
,real_size
);
407 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %" G_GSIZE_FORMAT
"d", p
, vtable
, sgen_client_vtable_get_name (vtable
), size
);
408 sgen_binary_protocol_alloc (p
, vtable
, size
, sgen_client_get_provenance ());
409 g_assert (*p
== NULL
); /* FIXME disable this in non debug builds */
411 mono_atomic_store_seq (p
, vtable
);
417 sgen_alloc_obj (GCVTable vtable
, size_t size
)
422 if (!SGEN_CAN_ALIGN_UP (size
))
425 if (G_UNLIKELY (sgen_has_per_allocation_action
)) {
426 static int alloc_count
;
427 int current_alloc
= mono_atomic_inc_i32 (&alloc_count
);
429 if (sgen_verify_before_allocs
) {
430 if ((current_alloc
% sgen_verify_before_allocs
) == 0) {
432 sgen_check_whole_heap_stw ();
436 if (sgen_collect_before_allocs
) {
437 if (((current_alloc
% sgen_collect_before_allocs
) == 0) && sgen_nursery_section
) {
439 sgen_perform_collection (0, GENERATION_NURSERY
, "collect-before-alloc-triggered", TRUE
, TRUE
);
445 ENTER_CRITICAL_REGION
;
446 res
= sgen_try_alloc_obj_nolock (vtable
, size
);
448 EXIT_CRITICAL_REGION
;
451 EXIT_CRITICAL_REGION
;
454 res
= sgen_alloc_obj_nolock (vtable
, size
);
461 * To be used for interned strings and possibly MonoThread, reflection handles.
462 * We may want to explicitly free these objects.
465 sgen_alloc_obj_pinned (GCVTable vtable
, size_t size
)
470 if (!SGEN_CAN_ALIGN_UP (size
))
472 size
= ALIGN_UP (size
);
476 if (size
> SGEN_MAX_SMALL_OBJ_SIZE
) {
477 /* large objects are always pinned anyway */
478 p
= (GCObject
*)sgen_los_alloc_large_inner (vtable
, size
);
480 SGEN_ASSERT (9, sgen_client_vtable_is_inited (vtable
), "class %s:%s is not initialized", sgen_client_vtable_get_namespace (vtable
), sgen_client_vtable_get_name (vtable
));
481 p
= sgen_major_collector
.alloc_small_pinned_obj (vtable
, size
, SGEN_VTABLE_HAS_REFERENCES (vtable
));
484 SGEN_LOG (6, "Allocated pinned object %p, vtable: %p (%s), size: %" G_GSIZE_FORMAT
"d", p
, vtable
, sgen_client_vtable_get_name (vtable
), size
);
485 increment_thread_allocation_counter (size
);
486 sgen_binary_protocol_alloc_pinned (p
, vtable
, size
, sgen_client_get_provenance ());
493 * Used to allocate thread objects during attach. Doesn't trigger collections since
494 * the thread is not yet attached.
497 sgen_alloc_obj_mature (GCVTable vtable
, size_t size
)
501 if (!SGEN_CAN_ALIGN_UP (size
))
503 size
= ALIGN_UP (size
);
506 res
= sgen_major_collector
.alloc_degraded (vtable
, size
);
510 increment_thread_allocation_counter (size
);
517 * Clear the thread local TLAB variables for all threads.
520 sgen_clear_tlabs (void)
522 guint64 total_bytes_allocated_globally
= 0;
524 FOREACH_THREAD_ALL (info
) {
525 /* A new TLAB will be allocated when the thread does its first allocation */
526 info
->total_bytes_allocated
+= info
->tlab_next
- info
->tlab_start
;
527 total_bytes_allocated_globally
+= info
->total_bytes_allocated
;
528 info
->tlab_start
= NULL
;
529 info
->tlab_next
= NULL
;
530 info
->tlab_temp_end
= NULL
;
531 info
->tlab_real_end
= NULL
;
534 sgen_set_bytes_allocated_attached (total_bytes_allocated_globally
);
537 void sgen_update_allocation_count (void)
539 guint64 total_bytes_allocated_globally
= 0;
541 FOREACH_THREAD_ALL (info
) {
542 total_bytes_allocated_globally
+= info
->tlab_next
- info
->tlab_start
;
543 total_bytes_allocated_globally
+= info
->total_bytes_allocated
;
546 sgen_set_bytes_allocated_attached (total_bytes_allocated_globally
);
550 sgen_set_bytes_allocated_attached (guint64 bytes
)
552 bytes_allocated_attached
= bytes
;
556 sgen_increment_bytes_allocated_detached (guint64 bytes
)
558 bytes_allocated_detached
+= bytes
;
562 sgen_get_total_allocated_bytes (MonoBoolean precise
)
566 sgen_stop_world (0, FALSE
);
568 sgen_update_allocation_count ();
570 sgen_restart_world (0, FALSE
);
574 return bytes_allocated_attached
+ bytes_allocated_detached
;
579 sgen_init_allocator (void)
581 #ifdef HEAVY_STATISTICS
582 mono_counters_register ("# objects allocated", MONO_COUNTER_GC
| MONO_COUNTER_ULONG
, &stat_objects_alloced
);
583 mono_counters_register ("bytes allocated", MONO_COUNTER_GC
| MONO_COUNTER_ULONG
, &stat_bytes_alloced
);
584 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC
| MONO_COUNTER_ULONG
, &stat_bytes_alloced_los
);
588 #endif /*HAVE_SGEN_GC*/