[Winforms] Improve X11 keyboard handling. (#18428)
[mono-project.git] / mono / sgen / sgen-alloc.c
blob84e5456eac89ab0929705c765b67892b9f8ff18f
1 /**
2 * \file
3 * Object allocation routines + managed allocators
5 * Author:
6 * Paolo Molaro (lupus@ximian.com)
7 * Rodrigo Kumpera (kumpera@gmail.com)
9 * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
10 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
11 * Copyright 2011 Xamarin, Inc.
12 * Copyright (C) 2012 Xamarin Inc
14 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
18 * ######################################################################
19 * ######## Object allocation
20 * ######################################################################
21 * This section of code deals with allocating memory for objects.
22 * There are several ways:
23 * *) allocate large objects
24 * *) allocate normal objects
25 * *) fast lock-free allocation
26 * *) allocation of pinned objects
29 #include "config.h"
30 #ifdef HAVE_SGEN_GC
32 #include <string.h>
34 #include "mono/sgen/sgen-gc.h"
35 #include "mono/sgen/sgen-protocol.h"
36 #include "mono/sgen/sgen-memory-governor.h"
37 #include "mono/sgen/sgen-client.h"
38 #include "mono/utils/mono-memory-model.h"
39 #include "mono/utils/mono-tls-inline.h"
41 #define ALIGN_UP SGEN_ALIGN_UP
42 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
43 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
45 #ifdef HEAVY_STATISTICS
46 static guint64 stat_objects_alloced = 0;
47 static guint64 stat_bytes_alloced = 0;
48 static guint64 stat_bytes_alloced_los = 0;
49 #endif
51 /* The total number of bytes allocated so far in program execution by all attached threads.
52 * This is not constantly syncrhonized, but only updated on each GC. */
53 static guint64 bytes_allocated_attached = 0;
55 /* Total bytes allocated so far in program execution by detached threads */
56 static guint64 bytes_allocated_detached = 0;
59 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
60 * from nursery fragments.
61 * tlab_next is the pointer to the space inside the TLAB where the next object will
62 * be allocated.
63 * tlab_temp_end is the pointer to the end of the temporary space reserved for
64 * the allocation: it allows us to set the scan starts at reasonable intervals.
65 * tlab_real_end points to the end of the TLAB.
68 #define TLAB_START (__thread_info__->tlab_start)
69 #define TLAB_NEXT (__thread_info__->tlab_next)
70 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
71 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
73 static void
74 increment_thread_allocation_counter (size_t byte_size)
76 mono_thread_info_current ()->total_bytes_allocated += byte_size;
79 static GCObject*
80 alloc_degraded (GCVTable vtable, size_t size, gboolean for_mature)
82 GCObject *p;
83 increment_thread_allocation_counter (size);
85 if (!for_mature) {
86 sgen_client_degraded_allocation ();
87 SGEN_ATOMIC_ADD_P (sgen_degraded_mode, size);
88 sgen_ensure_free_space (size, GENERATION_OLD);
89 } else {
90 gboolean forced;
91 if (sgen_need_major_collection (size, &forced))
92 sgen_perform_collection (size, GENERATION_OLD, "mature allocation failure", !for_mature || forced, TRUE);
96 p = sgen_major_collector.alloc_degraded (vtable, size);
98 if (!for_mature)
99 sgen_binary_protocol_alloc_degraded (p, vtable, size, sgen_client_get_provenance ());
101 return p;
104 static void
105 zero_tlab_if_necessary (void *p, size_t size)
107 if (sgen_nursery_clear_policy == CLEAR_AT_TLAB_CREATION || sgen_nursery_clear_policy == CLEAR_AT_TLAB_CREATION_DEBUG) {
108 memset (p, 0, size);
109 } else {
111 * This function is called for all allocations in
112 * TLABs. TLABs originate from fragments, which are
113 * initialized to be faux arrays. The remainder of
114 * the fragments are zeroed out at initialization for
115 * CLEAR_AT_GC, so here we just need to make sure that
116 * the array header is zeroed. Since we don't know
117 * whether we're called for the start of a fragment or
118 * for somewhere in between, we zero in any case, just
119 * to make sure.
121 sgen_client_zero_array_fill_header (p, size);
126 * Provide a variant that takes just the vtable for small fixed-size objects.
127 * The aligned size is already computed and stored in vt->gc_descr.
128 * Note: every SGEN_SCAN_START_SIZE or so we are given the chance to do some special
129 * processing. We can keep track of where objects start, for example,
130 * so when we scan the thread stacks for pinned objects, we can start
131 * a search for the pinned object in SGEN_SCAN_START_SIZE chunks.
133 GCObject*
134 sgen_alloc_obj_nolock (GCVTable vtable, size_t size)
136 /* FIXME: handle OOM */
137 void **p;
138 char *new_next;
139 size_t real_size = size;
140 TLAB_ACCESS_INIT;
142 CANARIFY_SIZE(size);
144 HEAVY_STAT (++stat_objects_alloced);
145 if (real_size <= SGEN_MAX_SMALL_OBJ_SIZE)
146 HEAVY_STAT (stat_bytes_alloced += size);
147 else
148 HEAVY_STAT (stat_bytes_alloced_los += size);
150 size = ALIGN_UP (size);
152 SGEN_ASSERT (6, sgen_vtable_get_descriptor (vtable), "VTable without descriptor");
154 if (G_UNLIKELY (sgen_has_per_allocation_action)) {
155 static int alloc_count;
156 int current_alloc = mono_atomic_inc_i32 (&alloc_count);
158 if (sgen_collect_before_allocs) {
159 if (((current_alloc % sgen_collect_before_allocs) == 0) && sgen_nursery_section) {
160 sgen_perform_collection (0, GENERATION_NURSERY, "collect-before-alloc-triggered", TRUE, TRUE);
161 if (!sgen_degraded_mode && sgen_can_alloc_size (size) && real_size <= SGEN_MAX_SMALL_OBJ_SIZE) {
162 // FIXME:
163 g_assert_not_reached ();
166 } else if (sgen_verify_before_allocs) {
167 if ((current_alloc % sgen_verify_before_allocs) == 0)
168 sgen_check_whole_heap_stw ();
173 * We must already have the lock here instead of after the
174 * fast path because we might be interrupted in the fast path
175 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
176 * and we'll end up allocating an object in a fragment which
177 * no longer belongs to us.
179 * The managed allocator does not do this, but it's treated
180 * specially by the world-stopping code.
183 if (real_size > SGEN_MAX_SMALL_OBJ_SIZE) {
184 p = (void **)sgen_los_alloc_large_inner (vtable, ALIGN_UP (real_size));
185 if (p) {
186 increment_thread_allocation_counter (size);
188 } else {
189 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
191 p = (void**)TLAB_NEXT;
192 /* FIXME: handle overflow */
193 new_next = (char*)p + size;
194 TLAB_NEXT = new_next;
196 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
197 /* Fast path */
199 CANARIFY_ALLOC(p,real_size);
200 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %zd", p, vtable, sgen_client_vtable_get_name (vtable), size);
201 sgen_binary_protocol_alloc (p , vtable, size, sgen_client_get_provenance ());
202 g_assert (*p == NULL);
203 mono_atomic_store_seq (p, vtable);
205 return (GCObject*)p;
208 /* Slow path */
210 /* there are two cases: the object is too big or we run out of space in the TLAB */
211 /* we also reach here when the thread does its first allocation after a minor
212 * collection, since the tlab_ variables are initialized to NULL.
213 * there can be another case (from ORP), if we cooperate with the runtime a bit:
214 * objects that need finalizers can have the high bit set in their size
215 * so the above check fails and we can readily add the object to the queue.
216 * This avoids taking again the GC lock when registering, but this is moot when
217 * doing thread-local allocation, so it may not be a good idea.
219 if (TLAB_NEXT >= TLAB_REAL_END) {
220 int available_in_tlab;
222 * Run out of space in the TLAB. When this happens, some amount of space
223 * remains in the TLAB, but not enough to satisfy the current allocation
224 * request. We keep the TLAB for future allocations if the remaining
225 * space is above a treshold, and satisfy the allocation directly
226 * from the nursery. Otherwise, we attempt to get a new TLAB from the
227 * nursery and allocate into it.
229 TLAB_NEXT -= size;
230 /* when running in degraded mode, we continue allocing that way
231 * for a while, to decrease the number of useless nursery collections.
233 if (sgen_degraded_mode && sgen_degraded_mode < sgen_nursery_size)
234 return alloc_degraded (vtable, size, FALSE);
236 available_in_tlab = (int)(TLAB_REAL_END - TLAB_NEXT);//We'll never have tlabs > 2Gb
237 if (size > sgen_tlab_size || available_in_tlab > SGEN_MAX_NURSERY_WASTE) {
238 /* Allocate directly from the nursery */
239 p = (void **)sgen_nursery_alloc (size);
240 if (!p) {
242 * We couldn't allocate from the nursery, so we try
243 * collecting. Even after the collection, we might
244 * still not have enough memory to allocate the
245 * object. The reason will most likely be that we've
246 * run out of memory, but there is the theoretical
247 * possibility that other threads might have consumed
248 * the freed up memory ahead of us.
250 * What we do in this case is allocate degraded, i.e.,
251 * from the major heap.
253 * Ideally we'd like to detect the case of other
254 * threads allocating ahead of us and loop (if we
255 * always loop we will loop endlessly in the case of
256 * OOM).
258 sgen_ensure_free_space (real_size, GENERATION_NURSERY);
259 if (!sgen_degraded_mode)
260 p = (void **)sgen_nursery_alloc (size);
262 if (p)
263 increment_thread_allocation_counter (size);
266 if (!p)
267 return alloc_degraded (vtable, size, TRUE);
269 zero_tlab_if_necessary (p, size);
270 } else {
271 size_t alloc_size = 0;
272 if (TLAB_START)
273 SGEN_LOG (3, "Retire TLAB: %p-%p [%ld]", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size));
274 sgen_nursery_retire_region (p, available_in_tlab);
276 p = (void **)sgen_nursery_alloc_range (sgen_tlab_size, size, &alloc_size);
277 if (!p) {
278 /* See comment above in similar case. */
279 sgen_ensure_free_space (sgen_tlab_size, GENERATION_NURSERY);
280 if (!sgen_degraded_mode)
281 p = (void **)sgen_nursery_alloc_range (sgen_tlab_size, size, &alloc_size);
283 if (!p)
284 return alloc_degraded (vtable, size, TRUE);
286 increment_thread_allocation_counter (TLAB_NEXT - TLAB_START);
288 /* Allocate a new TLAB from the current nursery fragment */
289 TLAB_START = (char*)p;
290 TLAB_NEXT = TLAB_START;
291 TLAB_REAL_END = TLAB_START + alloc_size;
292 TLAB_TEMP_END = TLAB_START + MIN (SGEN_SCAN_START_SIZE, alloc_size);
294 zero_tlab_if_necessary (TLAB_START, alloc_size);
296 /* Allocate from the TLAB */
297 p = (void **)TLAB_NEXT;
298 TLAB_NEXT += size;
299 sgen_set_nursery_scan_start ((char*)p);
301 } else {
302 /* Reached tlab_temp_end */
304 /* record the scan start so we can find pinned objects more easily */
305 sgen_set_nursery_scan_start ((char*)p);
306 /* we just bump tlab_temp_end as well */
307 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SGEN_SCAN_START_SIZE);
308 SGEN_LOG (5, "Expanding local alloc: %p-%p", TLAB_NEXT, TLAB_TEMP_END);
310 CANARIFY_ALLOC(p,real_size);
313 if (G_LIKELY (p)) {
314 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %zd", p, vtable, sgen_client_vtable_get_name (vtable), size);
315 sgen_binary_protocol_alloc (p, vtable, size, sgen_client_get_provenance ());
316 mono_atomic_store_seq (p, vtable);
319 return (GCObject*)p;
322 GCObject*
323 sgen_try_alloc_obj_nolock (GCVTable vtable, size_t size)
325 void **p;
326 char *new_next;
327 size_t real_size = size;
328 TLAB_ACCESS_INIT;
330 CANARIFY_SIZE(size);
332 size = ALIGN_UP (size);
333 SGEN_ASSERT (9, real_size >= SGEN_CLIENT_MINIMUM_OBJECT_SIZE, "Object too small");
335 SGEN_ASSERT (6, sgen_vtable_get_descriptor (vtable), "VTable without descriptor");
337 if (real_size > SGEN_MAX_SMALL_OBJ_SIZE)
338 return NULL;
340 if (G_UNLIKELY (size > sgen_tlab_size)) {
341 /* Allocate directly from the nursery */
343 p = (void **)sgen_nursery_alloc (size);
344 if (!p)
345 return NULL;
347 increment_thread_allocation_counter (size);
348 sgen_set_nursery_scan_start ((char*)p);
350 /*FIXME we should use weak memory ops here. Should help specially on x86. */
351 zero_tlab_if_necessary (p, size);
352 } else {
353 int available_in_tlab;
354 char *real_end;
355 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
357 p = (void**)TLAB_NEXT;
358 /* FIXME: handle overflow */
359 new_next = (char*)p + size;
361 real_end = TLAB_REAL_END;
362 available_in_tlab = (int)(real_end - (char*)p);//We'll never have tlabs > 2Gb
364 if (G_LIKELY (new_next < real_end)) {
365 TLAB_NEXT = new_next;
367 /* Second case, we overflowed temp end */
368 if (G_UNLIKELY (new_next >= TLAB_TEMP_END)) {
369 sgen_set_nursery_scan_start (new_next);
370 /* we just bump tlab_temp_end as well */
371 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SGEN_SCAN_START_SIZE);
372 SGEN_LOG (5, "Expanding local alloc: %p-%p", TLAB_NEXT, TLAB_TEMP_END);
374 } else if (available_in_tlab > SGEN_MAX_NURSERY_WASTE) {
375 /* Allocate directly from the nursery */
376 p = (void **)sgen_nursery_alloc (size);
377 if (!p)
378 return NULL;
380 increment_thread_allocation_counter (size);
381 zero_tlab_if_necessary (p, size);
382 } else {
383 size_t alloc_size = 0;
385 sgen_nursery_retire_region (p, available_in_tlab);
386 new_next = (char *)sgen_nursery_alloc_range (sgen_tlab_size, size, &alloc_size);
387 p = (void**)new_next;
388 if (!p)
389 return NULL;
391 increment_thread_allocation_counter (TLAB_NEXT - TLAB_START);
393 TLAB_START = (char*)new_next;
394 TLAB_NEXT = new_next + size;
395 TLAB_REAL_END = new_next + alloc_size;
396 TLAB_TEMP_END = new_next + MIN (SGEN_SCAN_START_SIZE, alloc_size);
397 sgen_set_nursery_scan_start ((char*)p);
399 zero_tlab_if_necessary (new_next, alloc_size);
403 HEAVY_STAT (++stat_objects_alloced);
404 HEAVY_STAT (stat_bytes_alloced += size);
406 CANARIFY_ALLOC(p,real_size);
407 SGEN_LOG (6, "Allocated object %p, vtable: %p (%s), size: %zd", p, vtable, sgen_client_vtable_get_name (vtable), size);
408 sgen_binary_protocol_alloc (p, vtable, size, sgen_client_get_provenance ());
409 g_assert (*p == NULL); /* FIXME disable this in non debug builds */
411 mono_atomic_store_seq (p, vtable);
413 return (GCObject*)p;
416 GCObject*
417 sgen_alloc_obj (GCVTable vtable, size_t size)
419 GCObject *res;
420 TLAB_ACCESS_INIT;
422 if (!SGEN_CAN_ALIGN_UP (size))
423 return NULL;
425 if (G_UNLIKELY (sgen_has_per_allocation_action)) {
426 static int alloc_count;
427 int current_alloc = mono_atomic_inc_i32 (&alloc_count);
429 if (sgen_verify_before_allocs) {
430 if ((current_alloc % sgen_verify_before_allocs) == 0) {
431 LOCK_GC;
432 sgen_check_whole_heap_stw ();
433 UNLOCK_GC;
436 if (sgen_collect_before_allocs) {
437 if (((current_alloc % sgen_collect_before_allocs) == 0) && sgen_nursery_section) {
438 LOCK_GC;
439 sgen_perform_collection (0, GENERATION_NURSERY, "collect-before-alloc-triggered", TRUE, TRUE);
440 UNLOCK_GC;
445 ENTER_CRITICAL_REGION;
446 res = sgen_try_alloc_obj_nolock (vtable, size);
447 if (res) {
448 EXIT_CRITICAL_REGION;
449 return res;
451 EXIT_CRITICAL_REGION;
453 LOCK_GC;
454 res = sgen_alloc_obj_nolock (vtable, size);
455 UNLOCK_GC;
457 return res;
461 * To be used for interned strings and possibly MonoThread, reflection handles.
462 * We may want to explicitly free these objects.
464 GCObject*
465 sgen_alloc_obj_pinned (GCVTable vtable, size_t size)
467 GCObject *p;
470 if (!SGEN_CAN_ALIGN_UP (size))
471 return NULL;
472 size = ALIGN_UP (size);
474 LOCK_GC;
476 if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
477 /* large objects are always pinned anyway */
478 p = (GCObject *)sgen_los_alloc_large_inner (vtable, size);
479 } else {
480 SGEN_ASSERT (9, sgen_client_vtable_is_inited (vtable), "class %s:%s is not initialized", sgen_client_vtable_get_namespace (vtable), sgen_client_vtable_get_name (vtable));
481 p = sgen_major_collector.alloc_small_pinned_obj (vtable, size, SGEN_VTABLE_HAS_REFERENCES (vtable));
483 if (G_LIKELY (p)) {
484 SGEN_LOG (6, "Allocated pinned object %p, vtable: %p (%s), size: %zd", p, vtable, sgen_client_vtable_get_name (vtable), size);
485 increment_thread_allocation_counter (size);
486 sgen_binary_protocol_alloc_pinned (p, vtable, size, sgen_client_get_provenance ());
488 UNLOCK_GC;
489 return p;
493 * Used to allocate thread objects during attach. Doesn't trigger collections since
494 * the thread is not yet attached.
496 GCObject*
497 sgen_alloc_obj_mature (GCVTable vtable, size_t size)
499 GCObject *res;
501 if (!SGEN_CAN_ALIGN_UP (size))
502 return NULL;
503 size = ALIGN_UP (size);
505 LOCK_GC;
506 res = sgen_major_collector.alloc_degraded (vtable, size);
507 UNLOCK_GC;
509 if (res) {
510 increment_thread_allocation_counter (size);
513 return res;
517 * Clear the thread local TLAB variables for all threads.
519 void
520 sgen_clear_tlabs (void)
522 guint64 total_bytes_allocated_globally = 0;
524 FOREACH_THREAD_ALL (info) {
525 /* A new TLAB will be allocated when the thread does its first allocation */
526 info->total_bytes_allocated += info->tlab_next - info->tlab_start;
527 total_bytes_allocated_globally += info->total_bytes_allocated;
528 info->tlab_start = NULL;
529 info->tlab_next = NULL;
530 info->tlab_temp_end = NULL;
531 info->tlab_real_end = NULL;
532 } FOREACH_THREAD_END
534 sgen_set_bytes_allocated_attached (total_bytes_allocated_globally);
537 void sgen_update_allocation_count (void)
539 guint64 total_bytes_allocated_globally = 0;
541 FOREACH_THREAD_ALL (info) {
542 total_bytes_allocated_globally += info->tlab_next - info->tlab_start;
543 total_bytes_allocated_globally += info->total_bytes_allocated;
544 } FOREACH_THREAD_END
546 sgen_set_bytes_allocated_attached (total_bytes_allocated_globally);
549 void
550 sgen_set_bytes_allocated_attached (guint64 bytes)
552 bytes_allocated_attached = bytes;
555 void
556 sgen_increment_bytes_allocated_detached (guint64 bytes)
558 bytes_allocated_detached += bytes;
561 guint64
562 sgen_get_total_allocated_bytes (MonoBoolean precise)
564 if (precise) {
565 LOCK_GC;
566 sgen_stop_world (0, FALSE);
568 sgen_update_allocation_count ();
570 sgen_restart_world (0, FALSE);
571 UNLOCK_GC;
574 return bytes_allocated_attached + bytes_allocated_detached;
578 void
579 sgen_init_allocator (void)
581 #ifdef HEAVY_STATISTICS
582 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_objects_alloced);
583 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_bytes_alloced);
584 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_bytes_alloced_los);
585 #endif
588 #endif /*HAVE_SGEN_GC*/