2 * sgen-alloc.c: Object allocation routines + managed allocators
5 * Paolo Molaro (lupus@ximian.com)
6 * Rodrigo Kumpera (kumpera@gmail.com)
8 * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
9 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
10 * Copyright 2011 Xamarin, Inc.
11 * Copyright (C) 2012 Xamarin Inc
13 * This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU Library General Public
15 * License 2.0 as published by the Free Software Foundation;
17 * This library is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * Library General Public License for more details.
22 * You should have received a copy of the GNU Library General Public
23 * License 2.0 along with this library; if not, write to the Free
24 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 * ######################################################################
29 * ######## Object allocation
30 * ######################################################################
31 * This section of code deals with allocating memory for objects.
32 * There are several ways:
33 * *) allocate large objects
34 * *) allocate normal objects
35 * *) fast lock-free allocation
36 * *) allocation of pinned objects
42 #include "metadata/sgen-gc.h"
43 #include "metadata/sgen-protocol.h"
44 #include "metadata/sgen-memory-governor.h"
45 #include "metadata/profiler-private.h"
46 #include "metadata/marshal.h"
47 #include "metadata/method-builder.h"
48 #include "metadata/abi-details.h"
49 #include "utils/mono-memory-model.h"
50 #include "utils/mono-counters.h"
52 #define ALIGN_UP SGEN_ALIGN_UP
53 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
54 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
55 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
57 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
61 #include "mono/cil/opcode.def"
67 static gboolean use_managed_allocator
= TRUE
;
69 #ifdef HEAVY_STATISTICS
70 static guint64 stat_objects_alloced
= 0;
71 static guint64 stat_bytes_alloced
= 0;
72 static guint64 stat_bytes_alloced_los
= 0;
77 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
78 * from nursery fragments.
79 * tlab_next is the pointer to the space inside the TLAB where the next object will
81 * tlab_temp_end is the pointer to the end of the temporary space reserved for
82 * the allocation: it allows us to set the scan starts at reasonable intervals.
83 * tlab_real_end points to the end of the TLAB.
87 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
88 * variables for next+temp_end ?
91 static __thread
char *tlab_start
;
92 static __thread
char *tlab_next
;
93 static __thread
char *tlab_temp_end
;
94 static __thread
char *tlab_real_end
;
95 /* Used by the managed allocator/wbarrier */
96 static __thread
char **tlab_next_addr
;
100 #define TLAB_START tlab_start
101 #define TLAB_NEXT tlab_next
102 #define TLAB_TEMP_END tlab_temp_end
103 #define TLAB_REAL_END tlab_real_end
105 #define TLAB_START (__thread_info__->tlab_start)
106 #define TLAB_NEXT (__thread_info__->tlab_next)
107 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
108 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
112 alloc_degraded (MonoVTable
*vtable
, size_t size
, gboolean for_mature
)
114 static int last_major_gc_warned
= -1;
115 static int num_degraded
= 0;
120 if (last_major_gc_warned
< gc_stats
.major_gc_count
) {
122 if (num_degraded
== 1 || num_degraded
== 3)
123 mono_trace (G_LOG_LEVEL_INFO
, MONO_TRACE_GC
, "Warning: Degraded allocation. Consider increasing nursery-size if the warning persists.");
124 else if (num_degraded
== 10)
125 mono_trace (G_LOG_LEVEL_INFO
, MONO_TRACE_GC
, "Warning: Repeated degraded allocation. Consider increasing nursery-size.");
126 last_major_gc_warned
= gc_stats
.major_gc_count
;
128 SGEN_ATOMIC_ADD_P (degraded_mode
, size
);
129 sgen_ensure_free_space (size
);
131 if (sgen_need_major_collection (size
))
132 sgen_perform_collection (size
, GENERATION_OLD
, "mature allocation failure", !for_mature
);
136 p
= major_collector
.alloc_degraded (vtable
, size
);
139 MONO_GC_MAJOR_OBJ_ALLOC_MATURE ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
141 binary_protocol_alloc_degraded (p
, vtable
, size
);
142 MONO_GC_MAJOR_OBJ_ALLOC_DEGRADED ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
149 zero_tlab_if_necessary (void *p
, size_t size
)
151 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
|| nursery_clear_policy
== CLEAR_AT_TLAB_CREATION_DEBUG
) {
155 * This function is called for all allocations in
156 * TLABs. TLABs originate from fragments, which are
157 * initialized to be faux arrays. The remainder of
158 * the fragments are zeroed out at initialization for
159 * CLEAR_AT_GC, so here we just need to make sure that
160 * the array header is zeroed. Since we don't know
161 * whether we're called for the start of a fragment or
162 * for somewhere in between, we zero in any case, just
166 if (size
>= sizeof (MonoArray
))
167 memset (p
, 0, sizeof (MonoArray
));
169 static guint8 zeros
[sizeof (MonoArray
)];
171 SGEN_ASSERT (0, !memcmp (p
, zeros
, size
), "TLAB segment must be zeroed out.");
177 * Provide a variant that takes just the vtable for small fixed-size objects.
178 * The aligned size is already computed and stored in vt->gc_descr.
179 * Note: every SGEN_SCAN_START_SIZE or so we are given the chance to do some special
180 * processing. We can keep track of where objects start, for example,
181 * so when we scan the thread stacks for pinned objects, we can start
182 * a search for the pinned object in SGEN_SCAN_START_SIZE chunks.
185 mono_gc_alloc_obj_nolock (MonoVTable
*vtable
, size_t size
)
187 /* FIXME: handle OOM */
190 size_t real_size
= size
;
195 HEAVY_STAT (++stat_objects_alloced
);
196 if (real_size
<= SGEN_MAX_SMALL_OBJ_SIZE
)
197 HEAVY_STAT (stat_bytes_alloced
+= size
);
199 HEAVY_STAT (stat_bytes_alloced_los
+= size
);
201 size
= ALIGN_UP (size
);
203 g_assert (vtable
->gc_descr
);
205 if (G_UNLIKELY (has_per_allocation_action
)) {
206 static int alloc_count
;
207 int current_alloc
= InterlockedIncrement (&alloc_count
);
209 if (collect_before_allocs
) {
210 if (((current_alloc
% collect_before_allocs
) == 0) && nursery_section
) {
211 sgen_perform_collection (0, GENERATION_NURSERY
, "collect-before-alloc-triggered", TRUE
);
212 if (!degraded_mode
&& sgen_can_alloc_size (size
) && real_size
<= SGEN_MAX_SMALL_OBJ_SIZE
) {
214 g_assert_not_reached ();
217 } else if (verify_before_allocs
) {
218 if ((current_alloc
% verify_before_allocs
) == 0)
219 sgen_check_whole_heap_stw ();
224 * We must already have the lock here instead of after the
225 * fast path because we might be interrupted in the fast path
226 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
227 * and we'll end up allocating an object in a fragment which
228 * no longer belongs to us.
230 * The managed allocator does not do this, but it's treated
231 * specially by the world-stopping code.
234 if (real_size
> SGEN_MAX_SMALL_OBJ_SIZE
) {
235 p
= sgen_los_alloc_large_inner (vtable
, ALIGN_UP (real_size
));
237 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
239 p
= (void**)TLAB_NEXT
;
240 /* FIXME: handle overflow */
241 new_next
= (char*)p
+ size
;
242 TLAB_NEXT
= new_next
;
244 if (G_LIKELY (new_next
< TLAB_TEMP_END
)) {
248 * FIXME: We might need a memory barrier here so the change to tlab_next is
249 * visible before the vtable store.
252 CANARIFY_ALLOC(p
,real_size
);
253 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %zd", p
, vtable
, vtable
->klass
->name
, size
);
254 binary_protocol_alloc (p
, vtable
, size
);
255 if (G_UNLIKELY (MONO_GC_NURSERY_OBJ_ALLOC_ENABLED ()))
256 MONO_GC_NURSERY_OBJ_ALLOC ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
257 g_assert (*p
== NULL
);
258 mono_atomic_store_seq (p
, vtable
);
265 /* there are two cases: the object is too big or we run out of space in the TLAB */
266 /* we also reach here when the thread does its first allocation after a minor
267 * collection, since the tlab_ variables are initialized to NULL.
268 * there can be another case (from ORP), if we cooperate with the runtime a bit:
269 * objects that need finalizers can have the high bit set in their size
270 * so the above check fails and we can readily add the object to the queue.
271 * This avoids taking again the GC lock when registering, but this is moot when
272 * doing thread-local allocation, so it may not be a good idea.
274 if (TLAB_NEXT
>= TLAB_REAL_END
) {
275 int available_in_tlab
;
277 * Run out of space in the TLAB. When this happens, some amount of space
278 * remains in the TLAB, but not enough to satisfy the current allocation
279 * request. Currently, we retire the TLAB in all cases, later we could
280 * keep it if the remaining space is above a treshold, and satisfy the
281 * allocation directly from the nursery.
284 /* when running in degraded mode, we continue allocing that way
285 * for a while, to decrease the number of useless nursery collections.
287 if (degraded_mode
&& degraded_mode
< DEFAULT_NURSERY_SIZE
)
288 return alloc_degraded (vtable
, size
, FALSE
);
290 available_in_tlab
= (int)(TLAB_REAL_END
- TLAB_NEXT
);//We'll never have tlabs > 2Gb
291 if (size
> tlab_size
|| available_in_tlab
> SGEN_MAX_NURSERY_WASTE
) {
292 /* Allocate directly from the nursery */
293 p
= sgen_nursery_alloc (size
);
296 * We couldn't allocate from the nursery, so we try
297 * collecting. Even after the collection, we might
298 * still not have enough memory to allocate the
299 * object. The reason will most likely be that we've
300 * run out of memory, but there is the theoretical
301 * possibility that other threads might have consumed
302 * the freed up memory ahead of us.
304 * What we do in this case is allocate degraded, i.e.,
305 * from the major heap.
307 * Ideally we'd like to detect the case of other
308 * threads allocating ahead of us and loop (if we
309 * always loop we will loop endlessly in the case of
312 sgen_ensure_free_space (real_size
);
314 p
= sgen_nursery_alloc (size
);
317 return alloc_degraded (vtable
, size
, FALSE
);
319 zero_tlab_if_necessary (p
, size
);
321 size_t alloc_size
= 0;
323 SGEN_LOG (3, "Retire TLAB: %p-%p [%ld]", TLAB_START
, TLAB_REAL_END
, (long)(TLAB_REAL_END
- TLAB_NEXT
- size
));
324 sgen_nursery_retire_region (p
, available_in_tlab
);
326 p
= sgen_nursery_alloc_range (tlab_size
, size
, &alloc_size
);
328 /* See comment above in similar case. */
329 sgen_ensure_free_space (tlab_size
);
331 p
= sgen_nursery_alloc_range (tlab_size
, size
, &alloc_size
);
334 return alloc_degraded (vtable
, size
, FALSE
);
336 /* Allocate a new TLAB from the current nursery fragment */
337 TLAB_START
= (char*)p
;
338 TLAB_NEXT
= TLAB_START
;
339 TLAB_REAL_END
= TLAB_START
+ alloc_size
;
340 TLAB_TEMP_END
= TLAB_START
+ MIN (SGEN_SCAN_START_SIZE
, alloc_size
);
342 zero_tlab_if_necessary (TLAB_START
, alloc_size
);
344 /* Allocate from the TLAB */
345 p
= (void*)TLAB_NEXT
;
347 sgen_set_nursery_scan_start ((char*)p
);
350 /* Reached tlab_temp_end */
352 /* record the scan start so we can find pinned objects more easily */
353 sgen_set_nursery_scan_start ((char*)p
);
354 /* we just bump tlab_temp_end as well */
355 TLAB_TEMP_END
= MIN (TLAB_REAL_END
, TLAB_NEXT
+ SGEN_SCAN_START_SIZE
);
356 SGEN_LOG (5, "Expanding local alloc: %p-%p", TLAB_NEXT
, TLAB_TEMP_END
);
358 CANARIFY_ALLOC(p
,real_size
);
362 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %zd", p
, vtable
, vtable
->klass
->name
, size
);
363 binary_protocol_alloc (p
, vtable
, size
);
364 if (G_UNLIKELY (MONO_GC_MAJOR_OBJ_ALLOC_LARGE_ENABLED ()|| MONO_GC_NURSERY_OBJ_ALLOC_ENABLED ())) {
365 if (real_size
> SGEN_MAX_SMALL_OBJ_SIZE
)
366 MONO_GC_MAJOR_OBJ_ALLOC_LARGE ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
368 MONO_GC_NURSERY_OBJ_ALLOC ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
370 mono_atomic_store_seq (p
, vtable
);
377 mono_gc_try_alloc_obj_nolock (MonoVTable
*vtable
, size_t size
)
381 size_t real_size
= size
;
386 size
= ALIGN_UP (size
);
387 SGEN_ASSERT (9, real_size
>= sizeof (MonoObject
), "Object too small");
389 g_assert (vtable
->gc_descr
);
390 if (real_size
> SGEN_MAX_SMALL_OBJ_SIZE
)
393 if (G_UNLIKELY (size
> tlab_size
)) {
394 /* Allocate directly from the nursery */
395 p
= sgen_nursery_alloc (size
);
398 sgen_set_nursery_scan_start ((char*)p
);
400 /*FIXME we should use weak memory ops here. Should help specially on x86. */
401 zero_tlab_if_necessary (p
, size
);
403 int available_in_tlab
;
405 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
407 p
= (void**)TLAB_NEXT
;
408 /* FIXME: handle overflow */
409 new_next
= (char*)p
+ size
;
411 real_end
= TLAB_REAL_END
;
412 available_in_tlab
= (int)(real_end
- (char*)p
);//We'll never have tlabs > 2Gb
414 if (G_LIKELY (new_next
< real_end
)) {
415 TLAB_NEXT
= new_next
;
417 /* Second case, we overflowed temp end */
418 if (G_UNLIKELY (new_next
>= TLAB_TEMP_END
)) {
419 sgen_set_nursery_scan_start (new_next
);
420 /* we just bump tlab_temp_end as well */
421 TLAB_TEMP_END
= MIN (TLAB_REAL_END
, TLAB_NEXT
+ SGEN_SCAN_START_SIZE
);
422 SGEN_LOG (5, "Expanding local alloc: %p-%p", TLAB_NEXT
, TLAB_TEMP_END
);
424 } else if (available_in_tlab
> SGEN_MAX_NURSERY_WASTE
) {
425 /* Allocate directly from the nursery */
426 p
= sgen_nursery_alloc (size
);
430 zero_tlab_if_necessary (p
, size
);
432 size_t alloc_size
= 0;
434 sgen_nursery_retire_region (p
, available_in_tlab
);
435 new_next
= sgen_nursery_alloc_range (tlab_size
, size
, &alloc_size
);
436 p
= (void**)new_next
;
440 TLAB_START
= (char*)new_next
;
441 TLAB_NEXT
= new_next
+ size
;
442 TLAB_REAL_END
= new_next
+ alloc_size
;
443 TLAB_TEMP_END
= new_next
+ MIN (SGEN_SCAN_START_SIZE
, alloc_size
);
444 sgen_set_nursery_scan_start ((char*)p
);
446 zero_tlab_if_necessary (new_next
, alloc_size
);
448 MONO_GC_NURSERY_TLAB_ALLOC ((mword
)new_next
, alloc_size
);
452 HEAVY_STAT (++stat_objects_alloced
);
453 HEAVY_STAT (stat_bytes_alloced
+= size
);
455 CANARIFY_ALLOC(p
,real_size
);
456 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %zd", p
, vtable
, vtable
->klass
->name
, size
);
457 binary_protocol_alloc (p
, vtable
, size
);
458 if (G_UNLIKELY (MONO_GC_NURSERY_OBJ_ALLOC_ENABLED ()))
459 MONO_GC_NURSERY_OBJ_ALLOC ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
460 g_assert (*p
== NULL
); /* FIXME disable this in non debug builds */
462 mono_atomic_store_seq (p
, vtable
);
468 mono_gc_alloc_obj (MonoVTable
*vtable
, size_t size
)
473 if (!SGEN_CAN_ALIGN_UP (size
))
476 #ifndef DISABLE_CRITICAL_REGION
478 if (G_UNLIKELY (has_per_allocation_action
)) {
479 static int alloc_count
;
480 int current_alloc
= InterlockedIncrement (&alloc_count
);
482 if (verify_before_allocs
) {
483 if ((current_alloc
% verify_before_allocs
) == 0)
484 sgen_check_whole_heap_stw ();
486 if (collect_before_allocs
) {
487 if (((current_alloc
% collect_before_allocs
) == 0) && nursery_section
) {
489 sgen_perform_collection (0, GENERATION_NURSERY
, "collect-before-alloc-triggered", TRUE
);
495 ENTER_CRITICAL_REGION
;
496 res
= mono_gc_try_alloc_obj_nolock (vtable
, size
);
498 EXIT_CRITICAL_REGION
;
501 EXIT_CRITICAL_REGION
;
504 res
= mono_gc_alloc_obj_nolock (vtable
, size
);
506 if (G_UNLIKELY (!res
))
507 return mono_gc_out_of_memory (size
);
512 mono_gc_alloc_vector (MonoVTable
*vtable
, size_t size
, uintptr_t max_length
)
517 if (!SGEN_CAN_ALIGN_UP (size
))
520 #ifndef DISABLE_CRITICAL_REGION
521 ENTER_CRITICAL_REGION
;
522 arr
= mono_gc_try_alloc_obj_nolock (vtable
, size
);
524 /*This doesn't require fencing since EXIT_CRITICAL_REGION already does it for us*/
525 arr
->max_length
= (mono_array_size_t
)max_length
;
526 EXIT_CRITICAL_REGION
;
529 EXIT_CRITICAL_REGION
;
534 arr
= mono_gc_alloc_obj_nolock (vtable
, size
);
535 if (G_UNLIKELY (!arr
)) {
537 return mono_gc_out_of_memory (size
);
540 arr
->max_length
= (mono_array_size_t
)max_length
;
548 mono_gc_alloc_array (MonoVTable
*vtable
, size_t size
, uintptr_t max_length
, uintptr_t bounds_size
)
551 MonoArrayBounds
*bounds
;
554 if (!SGEN_CAN_ALIGN_UP (size
))
557 #ifndef DISABLE_CRITICAL_REGION
558 ENTER_CRITICAL_REGION
;
559 arr
= mono_gc_try_alloc_obj_nolock (vtable
, size
);
561 /*This doesn't require fencing since EXIT_CRITICAL_REGION already does it for us*/
562 arr
->max_length
= (mono_array_size_t
)max_length
;
564 bounds
= (MonoArrayBounds
*)((char*)arr
+ size
- bounds_size
);
565 arr
->bounds
= bounds
;
566 EXIT_CRITICAL_REGION
;
569 EXIT_CRITICAL_REGION
;
574 arr
= mono_gc_alloc_obj_nolock (vtable
, size
);
575 if (G_UNLIKELY (!arr
)) {
577 return mono_gc_out_of_memory (size
);
580 arr
->max_length
= (mono_array_size_t
)max_length
;
582 bounds
= (MonoArrayBounds
*)((char*)arr
+ size
- bounds_size
);
583 arr
->bounds
= bounds
;
591 mono_gc_alloc_string (MonoVTable
*vtable
, size_t size
, gint32 len
)
596 if (!SGEN_CAN_ALIGN_UP (size
))
599 #ifndef DISABLE_CRITICAL_REGION
600 ENTER_CRITICAL_REGION
;
601 str
= mono_gc_try_alloc_obj_nolock (vtable
, size
);
603 /*This doesn't require fencing since EXIT_CRITICAL_REGION already does it for us*/
605 EXIT_CRITICAL_REGION
;
608 EXIT_CRITICAL_REGION
;
613 str
= mono_gc_alloc_obj_nolock (vtable
, size
);
614 if (G_UNLIKELY (!str
)) {
616 return mono_gc_out_of_memory (size
);
627 * To be used for interned strings and possibly MonoThread, reflection handles.
628 * We may want to explicitly free these objects.
631 mono_gc_alloc_pinned_obj (MonoVTable
*vtable
, size_t size
)
635 if (!SGEN_CAN_ALIGN_UP (size
))
637 size
= ALIGN_UP (size
);
641 if (size
> SGEN_MAX_SMALL_OBJ_SIZE
) {
642 /* large objects are always pinned anyway */
643 p
= sgen_los_alloc_large_inner (vtable
, size
);
645 SGEN_ASSERT (9, vtable
->klass
->inited
, "class %s:%s is not initialized", vtable
->klass
->name_space
, vtable
->klass
->name
);
646 p
= major_collector
.alloc_small_pinned_obj (vtable
, size
, SGEN_VTABLE_HAS_REFERENCES (vtable
));
649 SGEN_LOG (6, "Allocated pinned object %p, vtable: %p (%s), size: %zd", p
, vtable
, vtable
->klass
->name
, size
);
650 if (size
> SGEN_MAX_SMALL_OBJ_SIZE
)
651 MONO_GC_MAJOR_OBJ_ALLOC_LARGE ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
653 MONO_GC_MAJOR_OBJ_ALLOC_PINNED ((mword
)p
, size
, vtable
->klass
->name_space
, vtable
->klass
->name
);
654 binary_protocol_alloc_pinned (p
, vtable
, size
);
661 mono_gc_alloc_mature (MonoVTable
*vtable
)
664 size_t size
= vtable
->klass
->instance_size
;
666 if (!SGEN_CAN_ALIGN_UP (size
))
668 size
= ALIGN_UP (size
);
671 res
= alloc_degraded (vtable
, size
, TRUE
);
673 if (G_UNLIKELY (vtable
->klass
->has_finalize
))
674 mono_object_register_finalizer ((MonoObject
*)res
);
680 mono_gc_alloc_fixed (size_t size
, void *descr
)
682 /* FIXME: do a single allocation */
683 void *res
= calloc (1, size
);
686 if (!mono_gc_register_root (res
, size
, descr
)) {
694 mono_gc_free_fixed (void* addr
)
696 mono_gc_deregister_root (addr
);
701 sgen_init_tlab_info (SgenThreadInfo
* info
)
703 #ifndef HAVE_KW_THREAD
704 SgenThreadInfo
*__thread_info__
= info
;
707 info
->tlab_start_addr
= &TLAB_START
;
708 info
->tlab_next_addr
= &TLAB_NEXT
;
709 info
->tlab_temp_end_addr
= &TLAB_TEMP_END
;
710 info
->tlab_real_end_addr
= &TLAB_REAL_END
;
712 #ifdef HAVE_KW_THREAD
713 tlab_next_addr
= &tlab_next
;
718 * Clear the thread local TLAB variables for all threads.
721 sgen_clear_tlabs (void)
723 SgenThreadInfo
*info
;
725 FOREACH_THREAD (info
) {
726 /* A new TLAB will be allocated when the thread does its first allocation */
727 *info
->tlab_start_addr
= NULL
;
728 *info
->tlab_next_addr
= NULL
;
729 *info
->tlab_temp_end_addr
= NULL
;
730 *info
->tlab_real_end_addr
= NULL
;
734 static MonoMethod
* alloc_method_cache
[ATYPE_NUM
];
736 #ifdef MANAGED_ALLOCATION
737 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
738 * for each class. This is currently not easy to do, as it is hard to generate basic
739 * blocks + branches, but it is easy with the linear IL codebase.
741 * For this to work we'd need to solve the TLAB race, first. Now we
742 * require the allocator to be in a few known methods to make sure
743 * that they are executed atomically via the restart mechanism.
746 create_allocator (int atype
)
749 guint32 slowpath_branch
, max_size_branch
;
750 MonoMethodBuilder
*mb
;
752 MonoMethodSignature
*csig
;
753 static gboolean registered
= FALSE
;
754 int tlab_next_addr_var
, new_next_var
;
756 const char *name
= NULL
;
757 AllocatorWrapperInfo
*info
;
759 #ifdef HAVE_KW_THREAD
760 int tlab_next_addr_offset
= -1;
761 int tlab_temp_end_offset
= -1;
763 MONO_THREAD_VAR_OFFSET (tlab_next_addr
, tlab_next_addr_offset
);
764 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
766 mono_tls_key_set_offset (TLS_KEY_SGEN_TLAB_NEXT_ADDR
, tlab_next_addr_offset
);
767 mono_tls_key_set_offset (TLS_KEY_SGEN_TLAB_TEMP_END
, tlab_temp_end_offset
);
769 g_assert (tlab_next_addr_offset
!= -1);
770 g_assert (tlab_temp_end_offset
!= -1);
774 mono_register_jit_icall (mono_gc_alloc_obj
, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE
);
775 mono_register_jit_icall (mono_gc_alloc_vector
, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE
);
776 mono_register_jit_icall (mono_gc_alloc_string
, "mono_gc_alloc_string", mono_create_icall_signature ("object ptr int int32"), FALSE
);
780 if (atype
== ATYPE_SMALL
) {
783 } else if (atype
== ATYPE_NORMAL
) {
786 } else if (atype
== ATYPE_VECTOR
) {
788 name
= "AllocVector";
789 } else if (atype
== ATYPE_STRING
) {
791 name
= "AllocString";
793 g_assert_not_reached ();
796 csig
= mono_metadata_signature_alloc (mono_defaults
.corlib
, num_params
);
797 if (atype
== ATYPE_STRING
) {
798 csig
->ret
= &mono_defaults
.string_class
->byval_arg
;
799 csig
->params
[0] = &mono_defaults
.int_class
->byval_arg
;
800 csig
->params
[1] = &mono_defaults
.int32_class
->byval_arg
;
802 csig
->ret
= &mono_defaults
.object_class
->byval_arg
;
803 for (i
= 0; i
< num_params
; ++i
)
804 csig
->params
[i
] = &mono_defaults
.int_class
->byval_arg
;
807 mb
= mono_mb_new (mono_defaults
.object_class
, name
, MONO_WRAPPER_ALLOC
);
810 size_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
811 if (atype
== ATYPE_NORMAL
|| atype
== ATYPE_SMALL
) {
812 /* size = vtable->klass->instance_size; */
813 mono_mb_emit_ldarg (mb
, 0);
814 mono_mb_emit_icon (mb
, MONO_STRUCT_OFFSET (MonoVTable
, klass
));
815 mono_mb_emit_byte (mb
, CEE_ADD
);
816 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
817 mono_mb_emit_icon (mb
, MONO_STRUCT_OFFSET (MonoClass
, instance_size
));
818 mono_mb_emit_byte (mb
, CEE_ADD
);
819 /* FIXME: assert instance_size stays a 4 byte integer */
820 mono_mb_emit_byte (mb
, CEE_LDIND_U4
);
821 mono_mb_emit_byte (mb
, CEE_CONV_I
);
822 mono_mb_emit_stloc (mb
, size_var
);
823 } else if (atype
== ATYPE_VECTOR
) {
824 MonoExceptionClause
*clause
;
825 int pos
, pos_leave
, pos_error
;
826 MonoClass
*oom_exc_class
;
830 * n > MONO_ARRAY_MAX_INDEX => OutOfMemoryException
831 * n < 0 => OverflowException
833 * We can do an unsigned comparison to catch both cases, then in the error
834 * case compare signed to distinguish between them.
836 mono_mb_emit_ldarg (mb
, 1);
837 mono_mb_emit_icon (mb
, MONO_ARRAY_MAX_INDEX
);
838 mono_mb_emit_byte (mb
, CEE_CONV_U
);
839 pos
= mono_mb_emit_short_branch (mb
, CEE_BLE_UN_S
);
841 mono_mb_emit_byte (mb
, MONO_CUSTOM_PREFIX
);
842 mono_mb_emit_byte (mb
, CEE_MONO_NOT_TAKEN
);
843 mono_mb_emit_ldarg (mb
, 1);
844 mono_mb_emit_icon (mb
, 0);
845 pos_error
= mono_mb_emit_short_branch (mb
, CEE_BLT_S
);
846 mono_mb_emit_exception (mb
, "OutOfMemoryException", NULL
);
847 mono_mb_patch_short_branch (mb
, pos_error
);
848 mono_mb_emit_exception (mb
, "OverflowException", NULL
);
850 mono_mb_patch_short_branch (mb
, pos
);
852 clause
= mono_image_alloc0 (mono_defaults
.corlib
, sizeof (MonoExceptionClause
));
853 clause
->try_offset
= mono_mb_get_label (mb
);
855 /* vtable->klass->sizes.element_size */
856 mono_mb_emit_ldarg (mb
, 0);
857 mono_mb_emit_icon (mb
, MONO_STRUCT_OFFSET (MonoVTable
, klass
));
858 mono_mb_emit_byte (mb
, CEE_ADD
);
859 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
860 mono_mb_emit_icon (mb
, MONO_STRUCT_OFFSET (MonoClass
, sizes
));
861 mono_mb_emit_byte (mb
, CEE_ADD
);
862 mono_mb_emit_byte (mb
, CEE_LDIND_U4
);
863 mono_mb_emit_byte (mb
, CEE_CONV_I
);
866 mono_mb_emit_ldarg (mb
, 1);
867 mono_mb_emit_byte (mb
, CEE_MUL_OVF_UN
);
868 /* + sizeof (MonoArray) */
869 mono_mb_emit_icon (mb
, sizeof (MonoArray
));
870 mono_mb_emit_byte (mb
, CEE_ADD_OVF_UN
);
871 mono_mb_emit_stloc (mb
, size_var
);
873 pos_leave
= mono_mb_emit_branch (mb
, CEE_LEAVE
);
876 clause
->flags
= MONO_EXCEPTION_CLAUSE_NONE
;
877 clause
->try_len
= mono_mb_get_pos (mb
) - clause
->try_offset
;
878 clause
->data
.catch_class
= mono_class_from_name (mono_defaults
.corlib
,
879 "System", "OverflowException");
880 g_assert (clause
->data
.catch_class
);
881 clause
->handler_offset
= mono_mb_get_label (mb
);
883 oom_exc_class
= mono_class_from_name (mono_defaults
.corlib
,
884 "System", "OutOfMemoryException");
885 g_assert (oom_exc_class
);
886 ctor
= mono_class_get_method_from_name (oom_exc_class
, ".ctor", 0);
889 mono_mb_emit_byte (mb
, CEE_POP
);
890 mono_mb_emit_op (mb
, CEE_NEWOBJ
, ctor
);
891 mono_mb_emit_byte (mb
, CEE_THROW
);
893 clause
->handler_len
= mono_mb_get_pos (mb
) - clause
->handler_offset
;
894 mono_mb_set_clauses (mb
, 1, clause
);
895 mono_mb_patch_branch (mb
, pos_leave
);
897 } else if (atype
== ATYPE_STRING
) {
901 * a string allocator method takes the args: (vtable, len)
903 * bytes = offsetof (MonoString, chars) + ((len + 1) * 2)
907 * bytes <= INT32_MAX - (SGEN_ALLOC_ALIGN - 1)
911 * offsetof (MonoString, chars) + ((len + 1) * 2) <= INT32_MAX - (SGEN_ALLOC_ALIGN - 1)
912 * len <= (INT32_MAX - (SGEN_ALLOC_ALIGN - 1) - offsetof (MonoString, chars)) / 2 - 1
914 mono_mb_emit_ldarg (mb
, 1);
915 mono_mb_emit_icon (mb
, (INT32_MAX
- (SGEN_ALLOC_ALIGN
- 1) - MONO_STRUCT_OFFSET (MonoString
, chars
)) / 2 - 1);
916 pos
= mono_mb_emit_short_branch (mb
, MONO_CEE_BLE_UN_S
);
918 mono_mb_emit_byte (mb
, MONO_CUSTOM_PREFIX
);
919 mono_mb_emit_byte (mb
, CEE_MONO_NOT_TAKEN
);
920 mono_mb_emit_exception (mb
, "OutOfMemoryException", NULL
);
921 mono_mb_patch_short_branch (mb
, pos
);
923 mono_mb_emit_ldarg (mb
, 1);
924 mono_mb_emit_icon (mb
, 1);
925 mono_mb_emit_byte (mb
, MONO_CEE_SHL
);
926 //WE manually fold the above + 2 here
927 mono_mb_emit_icon (mb
, MONO_STRUCT_OFFSET (MonoString
, chars
) + 2);
928 mono_mb_emit_byte (mb
, CEE_ADD
);
929 mono_mb_emit_stloc (mb
, size_var
);
931 g_assert_not_reached ();
934 /* size += ALLOC_ALIGN - 1; */
935 mono_mb_emit_ldloc (mb
, size_var
);
936 mono_mb_emit_icon (mb
, ALLOC_ALIGN
- 1);
937 mono_mb_emit_byte (mb
, CEE_ADD
);
938 /* size &= ~(ALLOC_ALIGN - 1); */
939 mono_mb_emit_icon (mb
, ~(ALLOC_ALIGN
- 1));
940 mono_mb_emit_byte (mb
, CEE_AND
);
941 mono_mb_emit_stloc (mb
, size_var
);
943 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
944 if (atype
!= ATYPE_SMALL
) {
945 mono_mb_emit_ldloc (mb
, size_var
);
946 mono_mb_emit_icon (mb
, MAX_SMALL_OBJ_SIZE
);
947 max_size_branch
= mono_mb_emit_short_branch (mb
, MONO_CEE_BGT_UN_S
);
951 * We need to modify tlab_next, but the JIT only supports reading, so we read
952 * another tls var holding its address instead.
955 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
956 tlab_next_addr_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
957 EMIT_TLS_ACCESS_NEXT_ADDR (mb
);
958 mono_mb_emit_stloc (mb
, tlab_next_addr_var
);
960 /* p = (void**)tlab_next; */
961 p_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
962 mono_mb_emit_ldloc (mb
, tlab_next_addr_var
);
963 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
964 mono_mb_emit_stloc (mb
, p_var
);
966 /* new_next = (char*)p + size; */
967 new_next_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
968 mono_mb_emit_ldloc (mb
, p_var
);
969 mono_mb_emit_ldloc (mb
, size_var
);
970 mono_mb_emit_byte (mb
, CEE_CONV_I
);
971 mono_mb_emit_byte (mb
, CEE_ADD
);
972 mono_mb_emit_stloc (mb
, new_next_var
);
974 /* if (G_LIKELY (new_next < tlab_temp_end)) */
975 mono_mb_emit_ldloc (mb
, new_next_var
);
976 EMIT_TLS_ACCESS_TEMP_END (mb
);
977 slowpath_branch
= mono_mb_emit_short_branch (mb
, MONO_CEE_BLT_UN_S
);
980 if (atype
!= ATYPE_SMALL
)
981 mono_mb_patch_short_branch (mb
, max_size_branch
);
983 mono_mb_emit_byte (mb
, MONO_CUSTOM_PREFIX
);
984 mono_mb_emit_byte (mb
, CEE_MONO_NOT_TAKEN
);
986 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
987 mono_mb_emit_ldarg (mb
, 0);
988 mono_mb_emit_ldloc (mb
, size_var
);
989 if (atype
== ATYPE_NORMAL
|| atype
== ATYPE_SMALL
) {
990 mono_mb_emit_icall (mb
, mono_gc_alloc_obj
);
991 } else if (atype
== ATYPE_VECTOR
) {
992 mono_mb_emit_ldarg (mb
, 1);
993 mono_mb_emit_icall (mb
, mono_gc_alloc_vector
);
994 } else if (atype
== ATYPE_STRING
) {
995 mono_mb_emit_ldarg (mb
, 1);
996 mono_mb_emit_icall (mb
, mono_gc_alloc_string
);
998 g_assert_not_reached ();
1000 mono_mb_emit_byte (mb
, CEE_RET
);
1003 mono_mb_patch_short_branch (mb
, slowpath_branch
);
1005 /* FIXME: Memory barrier */
1007 /* tlab_next = new_next */
1008 mono_mb_emit_ldloc (mb
, tlab_next_addr_var
);
1009 mono_mb_emit_ldloc (mb
, new_next_var
);
1010 mono_mb_emit_byte (mb
, CEE_STIND_I
);
1012 /*The tlab store must be visible before the the vtable store. This could be replaced with a DDS but doing it with IL would be tricky. */
1013 mono_mb_emit_byte ((mb
), MONO_CUSTOM_PREFIX
);
1014 mono_mb_emit_op (mb
, CEE_MONO_MEMORY_BARRIER
, (gpointer
)StoreStoreBarrier
);
1017 mono_mb_emit_ldloc (mb
, p_var
);
1018 mono_mb_emit_ldarg (mb
, 0);
1019 mono_mb_emit_byte (mb
, CEE_STIND_I
);
1021 if (atype
== ATYPE_VECTOR
) {
1022 /* arr->max_length = max_length; */
1023 mono_mb_emit_ldloc (mb
, p_var
);
1024 mono_mb_emit_ldflda (mb
, MONO_STRUCT_OFFSET (MonoArray
, max_length
));
1025 mono_mb_emit_ldarg (mb
, 1);
1026 #ifdef MONO_BIG_ARRAYS
1027 mono_mb_emit_byte (mb
, CEE_STIND_I
);
1029 mono_mb_emit_byte (mb
, CEE_STIND_I4
);
1031 } else if (atype
== ATYPE_STRING
) {
1032 /* need to set length and clear the last char */
1033 /* s->length = len; */
1034 mono_mb_emit_ldloc (mb
, p_var
);
1035 mono_mb_emit_icon (mb
, MONO_STRUCT_OFFSET (MonoString
, length
));
1036 mono_mb_emit_byte (mb
, MONO_CEE_ADD
);
1037 mono_mb_emit_ldarg (mb
, 1);
1038 mono_mb_emit_byte (mb
, MONO_CEE_STIND_I4
);
1039 /* s->chars [len] = 0; */
1040 mono_mb_emit_ldloc (mb
, p_var
);
1041 mono_mb_emit_ldloc (mb
, size_var
);
1042 mono_mb_emit_icon (mb
, 2);
1043 mono_mb_emit_byte (mb
, MONO_CEE_SUB
);
1044 mono_mb_emit_byte (mb
, MONO_CEE_ADD
);
1045 mono_mb_emit_icon (mb
, 0);
1046 mono_mb_emit_byte (mb
, MONO_CEE_STIND_I2
);
1050 We must make sure both vtable and max_length are globaly visible before returning to managed land.
1052 mono_mb_emit_byte ((mb
), MONO_CUSTOM_PREFIX
);
1053 mono_mb_emit_op (mb
, CEE_MONO_MEMORY_BARRIER
, (gpointer
)StoreStoreBarrier
);
1056 mono_mb_emit_ldloc (mb
, p_var
);
1057 mono_mb_emit_byte (mb
, CEE_RET
);
1060 res
= mono_mb_create_method (mb
, csig
, 8);
1062 mono_method_get_header (res
)->init_locals
= FALSE
;
1064 info
= mono_image_alloc0 (mono_defaults
.corlib
, sizeof (AllocatorWrapperInfo
));
1065 info
->gc_name
= "sgen";
1066 info
->alloc_type
= atype
;
1067 mono_marshal_set_wrapper_info (res
, info
);
1074 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
1075 * The signature of the called method is:
1076 * object allocate (MonoVTable *vtable)
1079 mono_gc_get_managed_allocator (MonoClass
*klass
, gboolean for_box
)
1081 #ifdef MANAGED_ALLOCATION
1083 #ifdef HAVE_KW_THREAD
1084 int tlab_next_offset
= -1;
1085 int tlab_temp_end_offset
= -1;
1086 MONO_THREAD_VAR_OFFSET (tlab_next
, tlab_next_offset
);
1087 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
1089 if (tlab_next_offset
== -1 || tlab_temp_end_offset
== -1)
1092 if (collect_before_allocs
)
1094 if (!mono_runtime_has_tls_get ())
1096 if (klass
->instance_size
> tlab_size
)
1099 if (klass
->has_finalize
|| mono_class_is_marshalbyref (klass
) || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS
))
1103 if (klass
->byval_arg
.type
== MONO_TYPE_STRING
)
1104 return mono_gc_get_managed_allocator_by_type (ATYPE_STRING
);
1105 /* Generic classes have dynamic field and can go above MAX_SMALL_OBJ_SIZE. */
1106 if (ALIGN_TO (klass
->instance_size
, ALLOC_ALIGN
) < MAX_SMALL_OBJ_SIZE
&& !mono_class_is_open_constructed_type (&klass
->byval_arg
))
1107 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL
);
1109 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL
);
1116 mono_gc_get_managed_array_allocator (MonoClass
*klass
)
1118 #ifdef MANAGED_ALLOCATION
1119 #ifdef HAVE_KW_THREAD
1120 int tlab_next_offset
= -1;
1121 int tlab_temp_end_offset
= -1;
1122 MONO_THREAD_VAR_OFFSET (tlab_next
, tlab_next_offset
);
1123 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
1125 if (tlab_next_offset
== -1 || tlab_temp_end_offset
== -1)
1129 if (klass
->rank
!= 1)
1131 if (!mono_runtime_has_tls_get ())
1133 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS
)
1135 if (has_per_allocation_action
)
1137 g_assert (!mono_class_has_finalizer (klass
) && !mono_class_is_marshalbyref (klass
));
1139 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR
);
1146 sgen_set_use_managed_allocator (gboolean flag
)
1148 use_managed_allocator
= flag
;
1152 mono_gc_get_managed_allocator_by_type (int atype
)
1154 #ifdef MANAGED_ALLOCATION
1157 if (!use_managed_allocator
)
1160 if (!mono_runtime_has_tls_get ())
1163 res
= alloc_method_cache
[atype
];
1167 res
= create_allocator (atype
);
1169 if (alloc_method_cache
[atype
]) {
1170 mono_free_method (res
);
1171 res
= alloc_method_cache
[atype
];
1173 mono_memory_barrier ();
1174 alloc_method_cache
[atype
] = res
;
1185 mono_gc_get_managed_allocator_types (void)
1191 sgen_is_managed_allocator (MonoMethod
*method
)
1195 for (i
= 0; i
< ATYPE_NUM
; ++i
)
1196 if (method
== alloc_method_cache
[i
])
1202 sgen_has_managed_allocator (void)
1206 for (i
= 0; i
< ATYPE_NUM
; ++i
)
1207 if (alloc_method_cache
[i
])
1212 #ifdef HEAVY_STATISTICS
1214 sgen_alloc_init_heavy_stats (void)
1216 mono_counters_register ("# objects allocated", MONO_COUNTER_GC
| MONO_COUNTER_ULONG
, &stat_objects_alloced
);
1217 mono_counters_register ("bytes allocated", MONO_COUNTER_GC
| MONO_COUNTER_ULONG
, &stat_bytes_alloced
);
1218 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC
| MONO_COUNTER_ULONG
, &stat_bytes_alloced_los
);
1222 #endif /*HAVE_SGEN_GC*/