2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list and critical_fin_list are part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include <sys/types.h>
135 #include <sys/stat.h>
136 #include <sys/mman.h>
137 #include <sys/time.h>
140 #include "metadata/metadata-internals.h"
141 #include "metadata/class-internals.h"
142 #include "metadata/gc-internal.h"
143 #include "metadata/object-internals.h"
144 #include "metadata/threads.h"
145 #include "metadata/sgen-gc.h"
146 #include "metadata/sgen-archdep.h"
147 #include "metadata/mono-gc.h"
148 #include "metadata/method-builder.h"
149 #include "metadata/profiler-private.h"
150 #include "metadata/monitor.h"
151 #include "metadata/threadpool-internals.h"
152 #include "utils/mono-mmap.h"
153 #include "utils/mono-semaphore.h"
155 #ifdef HAVE_VALGRIND_MEMCHECK_H
156 #include <valgrind/memcheck.h>
159 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
163 #include "mono/cil/opcode.def"
170 * ######################################################################
171 * ######## Types and constants used by the GC.
172 * ######################################################################
174 #if SIZEOF_VOID_P == 4
175 typedef guint32 mword
;
177 typedef guint64 mword
;
180 static int gc_initialized
= 0;
181 static int gc_debug_level
= 0;
182 static FILE* gc_debug_file
;
183 /* If set, do a minor collection before every allocation */
184 static gboolean collect_before_allocs
= FALSE
;
185 /* If set, do a heap consistency check before each minor collection */
186 static gboolean consistency_check_at_minor_collection
= FALSE
;
187 /* If set, check that there are no references to the domain left at domain unload */
188 static gboolean xdomain_checks
= FALSE
;
189 /* If not null, dump the heap after each collection into this file */
190 static FILE *heap_dump_file
= NULL
;
194 mono_gc_flush_info (void)
196 fflush (gc_debug_file);
200 #define MAX_DEBUG_LEVEL 8
201 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
203 #define TV_DECLARE(name) struct timeval name
204 #define TV_GETTIME(tv) gettimeofday (&(tv), NULL)
205 #define TV_ELAPSED(start,end) (int)((((end).tv_sec - (start).tv_sec) * 1000000) + end.tv_usec - start.tv_usec)
207 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
217 /* each request from the OS ends up in a GCMemSection */
218 typedef struct _GCMemSection GCMemSection
;
219 struct _GCMemSection
{
223 /* pointer where more data could be allocated if it fits */
227 * scan starts is an array of pointers to objects equally spaced in the allocation area
228 * They let use quickly find pinned objects from pinning pointers.
231 /* in major collections indexes in the pin_queue for objects that pin this section */
234 unsigned short num_scan_start
;
238 /* large object space struct: 64+ KB */
239 /* we could make this limit much smaller to avoid memcpy copy
240 * and potentially have more room in the GC descriptor: need to measure
241 * This also means that such small OS objects will need to be
242 * allocated in a different way (using pinned chunks).
243 * We may want to put large but smaller than 64k objects in the fixed space
244 * when we move the object from one generation to another (to limit the
245 * pig in the snake effect).
246 * Note: it may be worth to have an optimized copy function, since we can
247 * assume that objects are aligned and have a multiple of 8 size.
248 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
249 * true if MONO_ZERO_LEN_ARRAY is nonzero.
251 typedef struct _LOSObject LOSObject
;
254 mword size
; /* this is the object size */
255 int dummy
; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
258 char data
[MONO_ZERO_LEN_ARRAY
];
261 /* Pinned objects are allocated in the LOS space if bigger than half a page
262 * or from freelists otherwise. We assume that pinned objects are relatively few
263 * and they have a slow dying speed (like interned strings, thread objects).
264 * As such they will be collected only at major collections.
265 * free lists are not global: when we need memory we allocate a PinnedChunk.
266 * Each pinned chunk is made of several pages, the first of wich is used
267 * internally for bookeeping (here think of a page as 4KB). The bookeeping
268 * includes the freelists vectors and info about the object size of each page
269 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
270 * a size is assigned to it, the page is divided in the proper chunks and each
271 * chunk is added to the freelist. To not waste space, the remaining space in the
272 * first page is used as objects of size 16 or 32 (need to measure which are more
274 * We use this same structure to allocate memory used internally by the GC, so
275 * we never use malloc/free if we need to alloc during collection: the world is stopped
276 * and malloc/free will deadlock.
277 * When we want to iterate over pinned objects, we just scan a page at a time
278 * linearly according to the size of objects in the page: the next pointer used to link
279 * the items in the freelist uses the same word as the vtable. Since we keep freelists
280 * for each pinned chunk, if the word points outside the pinned chunk it means
282 * We could avoid this expensive scanning in creative ways. We could have a policy
283 * of putting in the pinned space only objects we know about that have no struct fields
284 * with references and we can easily use a even expensive write barrier for them,
285 * since pointer writes on such objects should be rare.
286 * The best compromise is to just alloc interned strings and System.MonoType in them.
287 * It would be nice to allocate MonoThread in it, too: must check that we properly
288 * use write barriers so we don't have to do any expensive scanning of the whole pinned
289 * chunk list during minor collections. We can avoid it now because we alloc in it only
290 * reference-free objects.
292 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
293 #define MAX_FREELIST_SIZE 2048
294 #define PINNED_PAGE_SIZE (4096)
295 #define PINNED_CHUNK_MIN_SIZE (4096*8)
296 typedef struct _PinnedChunk PinnedChunk
;
297 struct _PinnedChunk
{
300 int *page_sizes
; /* a 0 means the page is still unused */
303 void *data
[1]; /* page sizes and free lists are stored here */
306 /* The method used to clear the nursery */
307 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
308 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
313 CLEAR_AT_TLAB_CREATION
314 } NurseryClearPolicy
;
316 static NurseryClearPolicy nursery_clear_policy
= CLEAR_AT_TLAB_CREATION
;
319 * If this is set, the nursery is aligned to an address aligned to its size, ie.
320 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
321 * speed up ptr_in_nursery () checks which are very frequent. This requires the
322 * nursery size to be a compile time constant.
324 #define ALIGN_NURSERY 1
327 * The young generation is divided into fragments. This is because
328 * we can hand one fragments to a thread for lock-less fast alloc and
329 * because the young generation ends up fragmented anyway by pinned objects.
330 * Once a collection is done, a list of fragments is created. When doing
331 * thread local alloc we use smallish nurseries so we allow new threads to
332 * allocate memory from gen0 without triggering a collection. Threads that
333 * are found to allocate lots of memory are given bigger fragments. This
334 * should make the finalizer thread use little nursery memory after a while.
335 * We should start assigning threads very small fragments: if there are many
336 * threads the nursery will be full of reserved space that the threads may not
337 * use at all, slowing down allocation speed.
338 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
339 * Allocation Buffers (TLABs).
341 typedef struct _Fragment Fragment
;
345 char *fragment_start
;
346 char *fragment_limit
; /* the current soft limit for allocation */
350 /* the runtime can register areas of memory as roots: we keep two lists of roots,
351 * a pinned root set for conservatively scanned roots and a normal one for
352 * precisely scanned roots (currently implemented as a single list).
354 typedef struct _RootRecord RootRecord
;
362 /* for use with write barriers */
363 typedef struct _RememberedSet RememberedSet
;
364 struct _RememberedSet
{
368 mword data
[MONO_ZERO_LEN_ARRAY
];
371 /* we have 4 possible values in the low 2 bits */
373 REMSET_LOCATION
, /* just a pointer to the exact location */
374 REMSET_RANGE
, /* range of pointer fields */
375 REMSET_OBJECT
, /* mark all the object for scanning */
376 REMSET_OTHER
, /* all others */
377 REMSET_TYPE_MASK
= 0x3
380 /* Subtypes of REMSET_OTHER */
382 REMSET_VTYPE
, /* a valuetype array described by a gc descriptor and a count */
383 REMSET_ROOT_LOCATION
, /* a location inside a root */
386 #ifdef HAVE_KW_THREAD
387 static __thread RememberedSet
*remembered_set MONO_TLS_FAST
;
389 static pthread_key_t remembered_set_key
;
390 static RememberedSet
*global_remset
;
391 static RememberedSet
*freed_thread_remsets
;
392 //static int store_to_global_remset = 0;
394 /* FIXME: later choose a size that takes into account the RememberedSet struct
395 * and doesn't waste any alloc paddin space.
397 #define DEFAULT_REMSET_SIZE 1024
398 static RememberedSet
* alloc_remset (int size
, gpointer id
);
400 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
401 * no cast from a pointer to an integer
408 /* these bits are set in the object vtable: we could merge them since an object can be
409 * either pinned or forwarded but not both.
410 * We store them in the vtable slot because the bits are used in the sync block for
411 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
412 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
413 * would be an invalid combination for the monitor and hash code).
414 * The values are already shifted.
415 * The forwarding address is stored in the sync block.
417 #define FORWARDED_BIT 1
419 #define VTABLE_BITS_MASK 0x3
421 /* returns NULL if not forwarded, or the forwarded address */
422 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
423 /* set the forwarded address fw_addr for object obj */
424 #define forward_object(obj,fw_addr) do { \
425 ((mword*)(obj))[0] |= FORWARDED_BIT; \
426 ((mword*)(obj))[1] = (mword)(fw_addr); \
429 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
430 #define pin_object(obj) do { \
431 ((mword*)(obj))[0] |= PINNED_BIT; \
433 #define unpin_object(obj) do { \
434 ((mword*)(obj))[0] &= ~PINNED_BIT; \
438 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
440 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
444 * Since we set bits in the vtable, use the macro to load it from the pointer to
445 * an object that is potentially pinned.
447 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
450 safe_name (void* obj
)
452 MonoVTable
*vt
= (MonoVTable
*)LOAD_VTABLE (obj
);
453 return vt
->klass
->name
;
457 safe_object_get_size (MonoObject
* o
)
459 MonoClass
*klass
= ((MonoVTable
*)LOAD_VTABLE (o
))->klass
;
460 if (klass
== mono_defaults
.string_class
) {
461 return sizeof (MonoString
) + 2 * mono_string_length ((MonoString
*) o
) + 2;
462 } else if (klass
->rank
) {
463 MonoArray
*array
= (MonoArray
*)o
;
464 size_t size
= sizeof (MonoArray
) + mono_array_element_size (klass
) * mono_array_length (array
);
465 if (G_UNLIKELY (array
->bounds
)) {
466 size
+= sizeof (mono_array_size_t
) - 1;
467 size
&= ~(sizeof (mono_array_size_t
) - 1);
468 size
+= sizeof (MonoArrayBounds
) * klass
->rank
;
472 /* from a created object: the class must be inited already */
473 return klass
->instance_size
;
477 static inline gboolean
478 is_maybe_half_constructed (MonoObject
*o
)
482 klass
= ((MonoVTable
*)LOAD_VTABLE (o
))->klass
;
483 if ((klass
== mono_defaults
.string_class
&& mono_string_length ((MonoString
*)o
) == 0) ||
484 (klass
->rank
&& mono_array_length ((MonoArray
*)o
) == 0))
491 * ######################################################################
492 * ######## Global data.
493 * ######################################################################
495 static LOCK_DECLARE (gc_mutex
);
496 static int gc_disabled
= 0;
497 static int num_minor_gcs
= 0;
498 static int num_major_gcs
= 0;
500 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
501 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
502 #define DEFAULT_NURSERY_SIZE (1024*512*2)
503 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
504 #define DEFAULT_NURSERY_BITS 20
505 #define DEFAULT_MAX_SECTION (DEFAULT_NURSERY_SIZE * 16)
506 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
507 /* to quickly find the head of an object pinned by a conservative address
508 * we keep track of the objects allocated for each SCAN_START_SIZE memory
509 * chunk in the nursery or other memory sections. Larger values have less
510 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
512 #define SCAN_START_SIZE (4096*2)
513 /* the minimum size of a fragment that we consider useful for allocation */
514 #define FRAGMENT_MIN_SIZE (512)
515 /* This is a fixed value used for pinned chunks, not the system pagesize */
516 #define FREELIST_PAGESIZE 4096
518 static mword pagesize
= 4096;
519 static mword nursery_size
= DEFAULT_NURSERY_SIZE
;
520 static mword next_section_size
= DEFAULT_NURSERY_SIZE
* 4;
521 static mword max_section_size
= DEFAULT_MAX_SECTION
;
522 static int section_size_used
= 0;
523 static int degraded_mode
= 0;
525 static LOSObject
*los_object_list
= NULL
;
526 static mword los_memory_usage
= 0;
527 static mword los_num_objects
= 0;
528 static mword next_los_collection
= 2*1024*1024; /* 2 MB, need to tune */
529 static mword total_alloc
= 0;
530 /* use this to tune when to do a major/minor collection */
531 static mword memory_pressure
= 0;
533 static GCMemSection
*section_list
= NULL
;
534 static GCMemSection
*nursery_section
= NULL
;
535 static mword lowest_heap_address
= ~(mword
)0;
536 static mword highest_heap_address
= 0;
538 typedef struct _FinalizeEntry FinalizeEntry
;
539 struct _FinalizeEntry
{
544 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable
;
545 struct _FinalizeEntryHashTable
{
546 FinalizeEntry
**table
;
551 typedef struct _DisappearingLink DisappearingLink
;
552 struct _DisappearingLink
{
553 DisappearingLink
*next
;
557 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable
;
558 struct _DisappearingLinkHashTable
{
559 DisappearingLink
**table
;
564 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
566 typedef struct _LargeInternalMemHeader LargeInternalMemHeader
;
567 struct _LargeInternalMemHeader
{
580 * The link pointer is hidden by negating each bit. We use the lowest
581 * bit of the link (before negation) to store whether it needs
582 * resurrection tracking.
584 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
585 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
587 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
588 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
591 * The finalizable hash has the object as the key, the
592 * disappearing_link hash, has the link address as key.
594 static FinalizeEntryHashTable minor_finalizable_hash
;
595 static FinalizeEntryHashTable major_finalizable_hash
;
596 /* objects that are ready to be finalized */
597 static FinalizeEntry
*fin_ready_list
= NULL
;
598 static FinalizeEntry
*critical_fin_list
= NULL
;
600 static DisappearingLinkHashTable minor_disappearing_link_hash
;
601 static DisappearingLinkHashTable major_disappearing_link_hash
;
603 static int num_ready_finalizers
= 0;
604 static int no_finalize
= 0;
606 /* keep each size a multiple of ALLOC_ALIGN */
607 /* on 64 bit systems 8 is likely completely unused. */
608 static const int freelist_sizes
[] = {
609 8, 16, 24, 32, 40, 48, 64, 80,
610 96, 128, 160, 192, 224, 256, 320, 384,
611 448, 512, 584, 680, 816, 1024, 1360, 2048};
612 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
614 static char* max_pinned_chunk_addr
= NULL
;
615 static char* min_pinned_chunk_addr
= (char*)-1;
616 /* pinned_chunk_list is used for allocations of objects that are never moved */
617 static PinnedChunk
*pinned_chunk_list
= NULL
;
618 /* internal_chunk_list is used for allocating structures needed by the GC */
619 static PinnedChunk
*internal_chunk_list
= NULL
;
622 obj_is_from_pinned_alloc (char *p
)
624 PinnedChunk
*chunk
= pinned_chunk_list
;
625 for (; chunk
; chunk
= chunk
->next
) {
626 if (p
>= (char*)chunk
->start_data
&& p
< ((char*)chunk
+ chunk
->num_pages
* FREELIST_PAGESIZE
))
632 static int slot_for_size (size_t size
);
635 free_pinned_object (PinnedChunk
*chunk
, char *obj
, size_t size
)
637 void **p
= (void**)obj
;
638 int slot
= slot_for_size (size
);
640 g_assert (obj
>= (char*)chunk
->start_data
&& obj
< ((char*)chunk
+ chunk
->num_pages
* FREELIST_PAGESIZE
));
641 *p
= chunk
->free_list
[slot
];
642 chunk
->free_list
[slot
] = p
;
646 ROOT_TYPE_NORMAL
= 0, /* "normal" roots */
647 ROOT_TYPE_PINNED
= 1, /* roots without a GC descriptor */
648 ROOT_TYPE_WBARRIER
= 2, /* roots with a write barrier */
652 /* registered roots: the key to the hash is the root start address */
654 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
656 static RootRecord
**roots_hash
[ROOT_TYPE_NUM
] = { NULL
, NULL
};
657 static int roots_hash_size
[ROOT_TYPE_NUM
] = { 0, 0, 0 };
658 static mword roots_size
= 0; /* amount of memory in the root set */
659 static int num_roots_entries
[ROOT_TYPE_NUM
] = { 0, 0, 0 };
662 * The current allocation cursors
663 * We allocate objects in the nursery.
664 * The nursery is the area between nursery_start and nursery_real_end.
665 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
666 * from nursery fragments.
667 * tlab_next is the pointer to the space inside the TLAB where the next object will
669 * tlab_temp_end is the pointer to the end of the temporary space reserved for
670 * the allocation: it allows us to set the scan starts at reasonable intervals.
671 * tlab_real_end points to the end of the TLAB.
672 * nursery_frag_real_end points to the end of the currently used nursery fragment.
673 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
674 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
675 * At the next allocation, the area of the nursery where objects can be present is
676 * between MIN(nursery_first_pinned_start, first_fragment_start) and
677 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
679 static char *nursery_start
= NULL
;
681 /* eventually share with MonoThread? */
682 typedef struct _SgenThreadInfo SgenThreadInfo
;
684 struct _SgenThreadInfo
{
685 SgenThreadInfo
*next
;
687 unsigned int stop_count
; /* to catch duplicate signals */
692 void *stack_start_limit
;
693 char **tlab_next_addr
;
694 char **tlab_start_addr
;
695 char **tlab_temp_end_addr
;
696 char **tlab_real_end_addr
;
697 RememberedSet
*remset
;
698 gpointer runtime_data
;
699 gpointer stopped_ip
; /* only valid if the thread is stopped */
700 MonoDomain
*stopped_domain
; /* ditto */
701 gpointer
*stopped_regs
; /* ditto */
702 #ifndef HAVE_KW_THREAD
710 #ifdef HAVE_KW_THREAD
711 #define TLAB_ACCESS_INIT
712 #define TLAB_START tlab_start
713 #define TLAB_NEXT tlab_next
714 #define TLAB_TEMP_END tlab_temp_end
715 #define TLAB_REAL_END tlab_real_end
716 #define REMEMBERED_SET remembered_set
718 static pthread_key_t thread_info_key
;
719 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
720 #define TLAB_START (__thread_info__->tlab_start)
721 #define TLAB_NEXT (__thread_info__->tlab_next)
722 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
723 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
724 #define REMEMBERED_SET (__thread_info__->remset)
728 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
729 * variables for next+temp_end ?
731 #ifdef HAVE_KW_THREAD
732 static __thread
char *tlab_start
;
733 static __thread
char *tlab_next
;
734 static __thread
char *tlab_temp_end
;
735 static __thread
char *tlab_real_end
;
736 /* Used by the managed allocator */
737 static __thread
char **tlab_next_addr
;
738 static __thread
char *stack_end
;
740 static char *nursery_next
= NULL
;
741 static char *nursery_frag_real_end
= NULL
;
742 static char *nursery_real_end
= NULL
;
743 //static char *nursery_first_pinned_start = NULL;
744 static char *nursery_last_pinned_end
= NULL
;
746 /* The size of a TLAB */
747 /* The bigger the value, the less often we have to go to the slow path to allocate a new
748 * one, but the more space is wasted by threads not allocating much memory.
750 * FIXME: Make this self-tuning for each thread.
752 static guint32 tlab_size
= (1024 * 4);
754 /* fragments that are free and ready to be used for allocation */
755 static Fragment
*nursery_fragments
= NULL
;
756 /* freeelist of fragment structures */
757 static Fragment
*fragment_freelist
= NULL
;
760 * used when moving the objects
761 * When the nursery is collected, objects are copied to to_space.
762 * The area between gray_first and gray_objects is used as a stack
763 * of objects that need their fields checked for more references
765 * We should optimize somehow this mechanism to avoid rescanning
766 * ptr-free objects. The order is also probably not optimal: need to
767 * test cache misses and other graph traversal orders.
769 static char *to_space
= NULL
;
770 static char *gray_first
= NULL
;
771 static char *gray_objects
= NULL
;
772 static char *to_space_end
= NULL
;
773 static GCMemSection
*to_space_section
= NULL
;
775 /* objects bigger then this go into the large object space */
776 #define MAX_SMALL_OBJ_SIZE 0xffff
778 /* Functions supplied by the runtime to be called by the GC */
779 static MonoGCCallbacks gc_callbacks
;
782 * ######################################################################
783 * ######## Macros and function declarations.
784 * ######################################################################
787 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
788 if ((mword)(low) < lowest_heap_address) \
789 lowest_heap_address = (mword)(low); \
790 if ((mword)(high) > highest_heap_address) \
791 highest_heap_address = (mword)(high); \
795 align_pointer (void *ptr
)
797 mword p
= (mword
)ptr
;
798 p
+= sizeof (gpointer
) - 1;
799 p
&= ~ (sizeof (gpointer
) - 1);
803 /* forward declarations */
804 static void* get_internal_mem (size_t size
);
805 static void free_internal_mem (void *addr
);
806 static void* get_os_memory (size_t size
, int activate
);
807 static void free_os_memory (void *addr
, size_t size
);
808 static G_GNUC_UNUSED
void report_internal_mem_usage (void);
810 static int stop_world (void);
811 static int restart_world (void);
812 static void scan_thread_data (void *start_nursery
, void *end_nursery
, gboolean precise
);
813 static void scan_from_remsets (void *start_nursery
, void *end_nursery
);
814 static void find_pinning_ref_from_thread (char *obj
, size_t size
);
815 static void update_current_thread_stack (void *start
);
816 static GCMemSection
* alloc_section (size_t size
);
817 static void finalize_in_range (char *start
, char *end
, int generation
);
818 static void add_or_remove_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
, int generation
);
819 static void null_link_in_range (char *start
, char *end
, int generation
);
820 static void null_links_for_domain (MonoDomain
*domain
, int generation
);
821 static gboolean
search_fragment_for_size (size_t size
);
822 static void mark_pinned_from_addresses (PinnedChunk
*chunk
, void **start
, void **end
);
823 static void clear_remsets (void);
824 static void clear_tlabs (void);
825 static char *find_tlab_next_from_address (char *addr
);
826 typedef void (*ScanPinnedObjectCallbackFunc
) (PinnedChunk
*, char*, size_t, void*);
827 static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback
, void *callback_data
);
828 static void sweep_pinned_objects (void);
829 static void scan_from_pinned_objects (char *addr_start
, char *addr_end
);
830 static void free_large_object (LOSObject
*obj
);
831 static void free_mem_section (GCMemSection
*section
);
833 static void mono_gc_register_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
);
835 void describe_ptr (char *ptr
);
836 void check_consistency (void);
837 char* check_object (char *start
);
839 void mono_gc_scan_for_specific_ref (MonoObject
*key
);
842 * ######################################################################
843 * ######## GC descriptors
844 * ######################################################################
845 * Used to quickly get the info the GC needs about an object: size and
846 * where the references are held.
848 /* objects are aligned to 8 bytes boundaries
849 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
850 * The low 3 bits define the type of the descriptor. The other bits
851 * depend on the type.
852 * As a general rule the 13 remaining low bits define the size, either
853 * of the whole object or of the elements in the arrays. While for objects
854 * the size is already in bytes, for arrays we need to shift, because
855 * array elements might be smaller than 8 bytes. In case of arrays, we
856 * use two bits to describe what the additional high bits represents,
857 * so the default behaviour can handle element sizes less than 2048 bytes.
858 * The high 16 bits, if 0 it means the object is pointer-free.
859 * This design should make it easy and fast to skip over ptr-free data.
860 * The first 4 types should cover >95% of the objects.
861 * Note that since the size of objects is limited to 64K, larger objects
862 * will be allocated in the large object heap.
863 * If we want 4-bytes alignment, we need to put vector and small bitmap
867 DESC_TYPE_RUN_LENGTH
, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
868 DESC_TYPE_SMALL_BITMAP
, /* 16 bits aligned byte size | 16-48 bit bitmap */
869 DESC_TYPE_STRING
, /* nothing */
870 DESC_TYPE_COMPLEX
, /* index for bitmap into complex_descriptors */
871 DESC_TYPE_VECTOR
, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
872 DESC_TYPE_ARRAY
, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
873 DESC_TYPE_LARGE_BITMAP
, /* | 29-61 bitmap bits */
874 DESC_TYPE_COMPLEX_ARR
, /* index for bitmap into complex_descriptors */
875 /* subtypes for arrays and vectors */
876 DESC_TYPE_V_PTRFREE
= 0,/* there are no refs: keep first so it has a zero value */
877 DESC_TYPE_V_REFS
, /* all the array elements are refs */
878 DESC_TYPE_V_RUN_LEN
, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
879 DESC_TYPE_V_BITMAP
/* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
882 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
883 #define LOW_TYPE_BITS 3
884 #define SMALL_BITMAP_SHIFT 16
885 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
886 #define VECTOR_INFO_SHIFT 14
887 #define VECTOR_ELSIZE_SHIFT 3
888 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
889 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
890 #define SMALL_SIZE_MASK 0xfff8
891 #define MAX_ELEMENT_SIZE 0x3ff
892 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
893 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
894 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
895 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
896 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
898 #define ALLOC_ALIGN 8
901 /* Root bitmap descriptors are simpler: the lower three bits describe the type
902 * and we either have 30/62 bitmap bits or nibble-based run-length,
903 * or a complex descriptor, or a user defined marker function.
906 ROOT_DESC_CONSERVATIVE
, /* 0, so matches NULL value */
911 ROOT_DESC_TYPE_MASK
= 0x7,
912 ROOT_DESC_TYPE_SHIFT
= 3,
915 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
917 #define MAX_USER_DESCRIPTORS 16
919 static gsize
* complex_descriptors
= NULL
;
920 static int complex_descriptors_size
= 0;
921 static int complex_descriptors_next
= 0;
922 static MonoGCMarkFunc user_descriptors
[MAX_USER_DESCRIPTORS
];
923 static int user_descriptors_next
= 0;
926 alloc_complex_descriptor (gsize
*bitmap
, int numbits
)
928 int nwords
= numbits
/GC_BITS_PER_WORD
+ 2;
933 res
= complex_descriptors_next
;
934 /* linear search, so we don't have duplicates with domain load/unload
935 * this should not be performance critical or we'd have bigger issues
936 * (the number and size of complex descriptors should be small).
938 for (i
= 0; i
< complex_descriptors_next
; ) {
939 if (complex_descriptors
[i
] == nwords
) {
941 for (j
= 0; j
< nwords
- 1; ++j
) {
942 if (complex_descriptors
[i
+ 1 + j
] != bitmap
[j
]) {
952 i
+= complex_descriptors
[i
];
954 if (complex_descriptors_next
+ nwords
> complex_descriptors_size
) {
955 int new_size
= complex_descriptors_size
* 2 + nwords
;
956 complex_descriptors
= g_realloc (complex_descriptors
, new_size
* sizeof (gsize
));
957 complex_descriptors_size
= new_size
;
959 DEBUG (6, fprintf (gc_debug_file
, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res
, nwords
, complex_descriptors_size
));
960 complex_descriptors_next
+= nwords
;
961 complex_descriptors
[res
] = nwords
;
962 for (i
= 0; i
< nwords
- 1; ++i
) {
963 complex_descriptors
[res
+ 1 + i
] = bitmap
[i
];
964 DEBUG (6, fprintf (gc_debug_file
, "\tvalue: %p\n", (void*)complex_descriptors
[res
+ 1 + i
]));
971 * Descriptor builders.
974 mono_gc_make_descr_for_string (gsize
*bitmap
, int numbits
)
976 return (void*) DESC_TYPE_STRING
;
980 mono_gc_make_descr_for_object (gsize
*bitmap
, int numbits
, size_t obj_size
)
982 int first_set
= -1, num_set
= 0, last_set
= -1, i
;
984 size_t stored_size
= obj_size
;
985 stored_size
+= ALLOC_ALIGN
- 1;
986 stored_size
&= ~(ALLOC_ALIGN
- 1);
987 for (i
= 0; i
< numbits
; ++i
) {
988 if (bitmap
[i
/ GC_BITS_PER_WORD
] & ((gsize
)1 << (i
% GC_BITS_PER_WORD
))) {
995 if (stored_size
<= MAX_SMALL_OBJ_SIZE
) {
996 /* check run-length encoding first: one byte offset, one byte number of pointers
997 * on 64 bit archs, we can have 3 runs, just one on 32.
998 * It may be better to use nibbles.
1000 if (first_set
< 0) {
1001 desc
= DESC_TYPE_RUN_LENGTH
| stored_size
;
1002 DEBUG (6, fprintf (gc_debug_file
, "Ptrfree descriptor %p, size: %zd\n", (void*)desc
, stored_size
));
1003 return (void*) desc
;
1004 } else if (first_set
< 256 && num_set
< 256 && (first_set
+ num_set
== last_set
+ 1)) {
1005 desc
= DESC_TYPE_RUN_LENGTH
| stored_size
| (first_set
<< 16) | (num_set
<< 24);
1006 DEBUG (6, fprintf (gc_debug_file
, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc
, stored_size
, first_set
, num_set
));
1007 return (void*) desc
;
1009 /* we know the 2-word header is ptr-free */
1010 if (last_set
< SMALL_BITMAP_SIZE
+ OBJECT_HEADER_WORDS
) {
1011 desc
= DESC_TYPE_SMALL_BITMAP
| stored_size
| ((*bitmap
>> OBJECT_HEADER_WORDS
) << SMALL_BITMAP_SHIFT
);
1012 DEBUG (6, fprintf (gc_debug_file
, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc
, stored_size
, last_set
));
1013 return (void*) desc
;
1016 /* we know the 2-word header is ptr-free */
1017 if (last_set
< LARGE_BITMAP_SIZE
+ OBJECT_HEADER_WORDS
) {
1018 desc
= DESC_TYPE_LARGE_BITMAP
| ((*bitmap
>> OBJECT_HEADER_WORDS
) << LOW_TYPE_BITS
);
1019 DEBUG (6, fprintf (gc_debug_file
, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc
, stored_size
, last_set
));
1020 return (void*) desc
;
1022 /* it's a complex object ... */
1023 desc
= DESC_TYPE_COMPLEX
| (alloc_complex_descriptor (bitmap
, last_set
+ 1) << LOW_TYPE_BITS
);
1024 return (void*) desc
;
1027 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1029 mono_gc_make_descr_for_array (int vector
, gsize
*elem_bitmap
, int numbits
, size_t elem_size
)
1031 int first_set
= -1, num_set
= 0, last_set
= -1, i
;
1032 mword desc
= vector
? DESC_TYPE_VECTOR
: DESC_TYPE_ARRAY
;
1033 for (i
= 0; i
< numbits
; ++i
) {
1034 if (elem_bitmap
[i
/ GC_BITS_PER_WORD
] & ((gsize
)1 << (i
% GC_BITS_PER_WORD
))) {
1041 if (elem_size
<= MAX_ELEMENT_SIZE
) {
1042 desc
|= elem_size
<< VECTOR_ELSIZE_SHIFT
;
1044 return (void*)(desc
| VECTOR_SUBTYPE_PTRFREE
);
1046 /* Note: we also handle structs with just ref fields */
1047 if (num_set
* sizeof (gpointer
) == elem_size
) {
1048 return (void*)(desc
| VECTOR_SUBTYPE_REFS
| ((gssize
)(-1) << 16));
1050 /* FIXME: try run-len first */
1051 /* Note: we can't skip the object header here, because it's not present */
1052 if (last_set
<= SMALL_BITMAP_SIZE
) {
1053 return (void*)(desc
| VECTOR_SUBTYPE_BITMAP
| (*elem_bitmap
<< 16));
1056 /* it's am array of complex structs ... */
1057 desc
= DESC_TYPE_COMPLEX_ARR
;
1058 desc
|= alloc_complex_descriptor (elem_bitmap
, last_set
+ 1) << LOW_TYPE_BITS
;
1059 return (void*) desc
;
1062 /* Return the bitmap encoded by a descriptor */
1064 mono_gc_get_bitmap_for_descr (void *descr
, int *numbits
)
1066 mword d
= (mword
)descr
;
1070 case DESC_TYPE_RUN_LENGTH
: {
1071 int first_set
= (d
>> 16) & 0xff;
1072 int num_set
= (d
>> 16) & 0xff;
1075 bitmap
= g_new0 (gsize
, (first_set
+ num_set
+ 7) / 8);
1077 for (i
= first_set
; i
< first_set
+ num_set
; ++i
)
1078 bitmap
[i
/ GC_BITS_PER_WORD
] |= ((gsize
)1 << (i
% GC_BITS_PER_WORD
));
1080 *numbits
= first_set
+ num_set
;
1084 case DESC_TYPE_SMALL_BITMAP
:
1085 bitmap
= g_new0 (gsize
, 1);
1087 bitmap
[0] = (d
>> SMALL_BITMAP_SHIFT
) << OBJECT_HEADER_WORDS
;
1089 *numbits
= GC_BITS_PER_WORD
;
1093 g_assert_not_reached ();
1097 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1098 #define STRING_SIZE(size,str) do { \
1099 (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
1100 (size) += (ALLOC_ALIGN - 1); \
1101 (size) &= ~(ALLOC_ALIGN - 1); \
1104 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1105 (size) = (desc) & 0xfff8; \
1108 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1109 (size) = (desc) & 0xfff8; \
1112 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1113 #define PREFETCH(addr)
1115 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1116 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1117 if ((desc) & 0xffff0000) { \
1118 /* there are pointers */ \
1119 void **_objptr_end; \
1120 void **_objptr = (void**)(obj); \
1121 _objptr += ((desc) >> 16) & 0xff; \
1122 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1123 while (_objptr < _objptr_end) { \
1124 HANDLE_PTR (_objptr, (obj)); \
1130 /* a bitmap desc means that there are pointer references or we'd have
1131 * choosen run-length, instead: add an assert to check.
1133 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1134 /* there are pointers */ \
1135 void **_objptr = (void**)(obj); \
1136 gsize _bmap = (desc) >> 16; \
1137 _objptr += OBJECT_HEADER_WORDS; \
1139 if ((_bmap & 1)) { \
1140 HANDLE_PTR (_objptr, (obj)); \
1147 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1148 /* there are pointers */ \
1149 void **_objptr = (void**)(obj); \
1150 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1151 _objptr += OBJECT_HEADER_WORDS; \
1153 if ((_bmap & 1)) { \
1154 HANDLE_PTR (_objptr, (obj)); \
1161 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1162 /* there are pointers */ \
1163 void **_objptr = (void**)(obj); \
1164 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1165 int bwords = (*bitmap_data) - 1; \
1166 void **start_run = _objptr; \
1169 MonoObject *myobj = (MonoObject*)obj; \
1170 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1172 while (bwords-- > 0) { \
1173 gsize _bmap = *bitmap_data++; \
1174 _objptr = start_run; \
1175 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1177 if ((_bmap & 1)) { \
1178 HANDLE_PTR (_objptr, (obj)); \
1183 start_run += GC_BITS_PER_WORD; \
1187 /* this one is untested */
1188 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1189 /* there are pointers */ \
1190 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1191 int mbwords = (*mbitmap_data++) - 1; \
1192 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1193 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1194 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1196 MonoObject *myobj = (MonoObject*)start; \
1197 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1199 while (e_start < e_end) { \
1200 void **_objptr = (void**)e_start; \
1201 gsize *bitmap_data = mbitmap_data; \
1202 unsigned int bwords = mbwords; \
1203 while (bwords-- > 0) { \
1204 gsize _bmap = *bitmap_data++; \
1205 void **start_run = _objptr; \
1206 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1208 if ((_bmap & 1)) { \
1209 HANDLE_PTR (_objptr, (obj)); \
1214 _objptr = start_run + GC_BITS_PER_WORD; \
1216 e_start += el_size; \
1220 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1221 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1222 if ((vt)->desc & 0xffffc000) { \
1223 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1224 /* there are pointers */ \
1225 int etype = (vt)->desc & 0xc000; \
1226 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1227 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1228 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1229 /* Note: this code can handle also arrays of struct with only references in them */ \
1230 while (p < end_refs) { \
1231 HANDLE_PTR (p, (obj)); \
1234 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1235 int offset = ((vt)->desc >> 16) & 0xff; \
1236 int num_refs = ((vt)->desc >> 24) & 0xff; \
1237 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1238 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1239 while (e_start < e_end) { \
1240 void **p = (void**)e_start; \
1243 for (i = 0; i < num_refs; ++i) { \
1244 HANDLE_PTR (p + i, (obj)); \
1246 e_start += el_size; \
1248 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1249 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1250 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1251 while (e_start < e_end) { \
1252 void **p = (void**)e_start; \
1253 gsize _bmap = (vt)->desc >> 16; \
1254 /* Note: there is no object header here to skip */ \
1256 if ((_bmap & 1)) { \
1257 HANDLE_PTR (p, (obj)); \
1262 e_start += el_size; \
1268 static mword new_obj_references
= 0;
1269 static mword obj_references_checked
= 0;
1272 #define HANDLE_PTR(ptr,obj) do { \
1273 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1274 new_obj_references++; \
1275 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1277 obj_references_checked++; \
1282 * ######################################################################
1283 * ######## Detecting and removing garbage.
1284 * ######################################################################
1285 * This section of code deals with detecting the objects no longer in use
1286 * and reclaiming the memory.
1289 #define COUNT_OBJECT_TYPES do { \
1290 switch (desc & 0x7) { \
1291 case DESC_TYPE_STRING: type_str++; break; \
1292 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1293 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1294 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1295 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1296 case DESC_TYPE_COMPLEX: type_complex++; break; \
1297 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1298 default: g_assert_not_reached (); \
1302 static void __attribute__((noinline
))
1303 scan_area (char *start
, char *end
)
1306 int type_str
= 0, type_rlen
= 0, type_bitmap
= 0, type_vector
= 0, type_lbit
= 0, type_complex
= 0;
1307 new_obj_references
= 0;
1308 obj_references_checked
= 0;
1309 while (start
< end
) {
1310 if (!*(void**)start
) {
1311 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
1314 vt
= (GCVTable
*)LOAD_VTABLE (start
);
1315 DEBUG (8, fprintf (gc_debug_file
, "Scanning object %p, vtable: %p (%s)\n", start
, vt
, vt
->klass
->name
));
1317 MonoObject
*obj
= (MonoObject
*)start
;
1318 g_print ("found at %p (0x%zx): %s.%s\n", start
, vt
->desc
, obj
->vtable
->klass
->name_space
, obj
->vtable
->klass
->name
);
1321 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1322 #include "sgen-scan-object.h"
1324 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1325 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1326 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1330 is_xdomain_ref_allowed (gpointer
*ptr
, char *obj
, MonoDomain
*domain
)
1332 MonoObject
*o
= (MonoObject
*)(obj
);
1333 MonoObject
*ref
= (MonoObject
*)*(ptr
);
1334 int offset
= (char*)(ptr
) - (char*)o
;
1336 if (o
->vtable
->klass
== mono_defaults
.thread_class
&& offset
== G_STRUCT_OFFSET (MonoThread
, internal_thread
))
1338 if (o
->vtable
->klass
== mono_defaults
.internal_thread_class
&& offset
== G_STRUCT_OFFSET (MonoInternalThread
, current_appcontext
))
1340 if (mono_class_has_parent (o
->vtable
->klass
, mono_defaults
.real_proxy_class
) &&
1341 offset
== G_STRUCT_OFFSET (MonoRealProxy
, unwrapped_server
))
1343 /* Thread.cached_culture_info */
1344 if (!strcmp (ref
->vtable
->klass
->name_space
, "System.Globalization") &&
1345 !strcmp (ref
->vtable
->klass
->name
, "CultureInfo") &&
1346 !strcmp(o
->vtable
->klass
->name_space
, "System") &&
1347 !strcmp(o
->vtable
->klass
->name
, "Object[]"))
1350 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1351 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1352 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1353 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1354 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1355 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1356 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1357 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1358 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1360 if (!strcmp (ref
->vtable
->klass
->name_space
, "System") &&
1361 !strcmp (ref
->vtable
->klass
->name
, "Byte[]") &&
1362 !strcmp (o
->vtable
->klass
->name_space
, "System.IO") &&
1363 !strcmp (o
->vtable
->klass
->name
, "MemoryStream"))
1365 /* append_job() in threadpool.c */
1366 if (!strcmp (ref
->vtable
->klass
->name_space
, "System.Runtime.Remoting.Messaging") &&
1367 !strcmp (ref
->vtable
->klass
->name
, "AsyncResult") &&
1368 !strcmp (o
->vtable
->klass
->name_space
, "System") &&
1369 !strcmp (o
->vtable
->klass
->name
, "Object[]") &&
1370 mono_thread_pool_is_queue_array ((MonoArray
*) o
))
1376 check_reference_for_xdomain (gpointer
*ptr
, char *obj
, MonoDomain
*domain
)
1378 MonoObject
*o
= (MonoObject
*)(obj
);
1379 MonoObject
*ref
= (MonoObject
*)*(ptr
);
1380 int offset
= (char*)(ptr
) - (char*)o
;
1382 MonoClassField
*field
;
1385 if (!ref
|| ref
->vtable
->domain
== domain
)
1387 if (is_xdomain_ref_allowed (ptr
, obj
, domain
))
1391 for (class = o
->vtable
->klass
; class; class = class->parent
) {
1394 for (i
= 0; i
< class->field
.count
; ++i
) {
1395 if (class->fields
[i
].offset
== offset
) {
1396 field
= &class->fields
[i
];
1404 if (ref
->vtable
->klass
== mono_defaults
.string_class
)
1405 str
= mono_string_to_utf8 ((MonoString
*)ref
);
1408 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1409 o
, o
->vtable
->klass
->name_space
, o
->vtable
->klass
->name
,
1410 offset
, field
? field
->name
: "",
1411 ref
, ref
->vtable
->klass
->name_space
, ref
->vtable
->klass
->name
, str
? str
: "");
1412 mono_gc_scan_for_specific_ref (o
);
1418 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1421 scan_object_for_xdomain_refs (char *start
)
1423 MonoDomain
*domain
= ((MonoObject
*)start
)->vtable
->domain
;
1425 #include "sgen-scan-object.h"
1431 scan_area_for_xdomain_refs (char *start
, char *end
)
1433 while (start
< end
) {
1434 if (!*(void**)start
) {
1435 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
1439 start
= scan_object_for_xdomain_refs (start
);
1444 #define HANDLE_PTR(ptr,obj) do { \
1445 if ((MonoObject*)*(ptr) == key) { \
1446 g_print ("found ref to %p in object %p (%s) at offset %d\n", \
1447 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1452 scan_object_for_specific_ref (char *start
, MonoObject
*key
)
1454 #include "sgen-scan-object.h"
1460 scan_area_for_specific_ref (char *start
, char *end
, MonoObject
*key
)
1462 while (start
< end
) {
1463 if (!*(void**)start
) {
1464 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
1468 start
= scan_object_for_specific_ref (start
, key
);
1473 scan_pinned_object_for_specific_ref_callback (PinnedChunk
*chunk
, char *obj
, size_t size
, MonoObject
*key
)
1475 scan_object_for_specific_ref (obj
, key
);
1479 check_root_obj_specific_ref (RootRecord
*root
, MonoObject
*key
, MonoObject
*obj
)
1483 g_print ("found ref to %p in root record %p\n", key
, root
);
1486 static MonoObject
*check_key
= NULL
;
1487 static RootRecord
*check_root
= NULL
;
1490 check_root_obj_specific_ref_from_marker (void *obj
)
1492 check_root_obj_specific_ref (check_root
, check_key
, obj
);
1497 scan_roots_for_specific_ref (MonoObject
*key
, int root_type
)
1502 for (i
= 0; i
< roots_hash_size
[root_type
]; ++i
) {
1503 for (root
= roots_hash
[root_type
][i
]; root
; root
= root
->next
) {
1504 void **start_root
= (void**)root
->start_root
;
1505 mword desc
= root
->root_desc
;
1509 switch (desc
& ROOT_DESC_TYPE_MASK
) {
1510 case ROOT_DESC_BITMAP
:
1511 desc
>>= ROOT_DESC_TYPE_SHIFT
;
1514 check_root_obj_specific_ref (root
, key
, *start_root
);
1519 case ROOT_DESC_COMPLEX
: {
1520 gsize
*bitmap_data
= complex_descriptors
+ (desc
>> ROOT_DESC_TYPE_SHIFT
);
1521 int bwords
= (*bitmap_data
) - 1;
1522 void **start_run
= start_root
;
1524 while (bwords
-- > 0) {
1525 gsize bmap
= *bitmap_data
++;
1526 void **objptr
= start_run
;
1529 check_root_obj_specific_ref (root
, key
, *objptr
);
1533 start_run
+= GC_BITS_PER_WORD
;
1537 case ROOT_DESC_USER
: {
1538 MonoGCMarkFunc marker
= user_descriptors
[desc
>> ROOT_DESC_TYPE_SHIFT
];
1539 marker (start_root
, check_root_obj_specific_ref_from_marker
);
1542 case ROOT_DESC_RUN_LEN
:
1543 g_assert_not_reached ();
1545 g_assert_not_reached ();
1554 mono_gc_scan_for_specific_ref (MonoObject
*key
)
1556 GCMemSection
*section
;
1561 for (section
= section_list
; section
; section
= section
->next
)
1562 scan_area_for_specific_ref (section
->data
, section
->end_data
, key
);
1564 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
)
1565 scan_object_for_specific_ref (bigobj
->data
, key
);
1567 scan_pinned_objects ((ScanPinnedObjectCallbackFunc
)scan_pinned_object_for_specific_ref_callback
, key
);
1569 scan_roots_for_specific_ref (key
, ROOT_TYPE_NORMAL
);
1570 scan_roots_for_specific_ref (key
, ROOT_TYPE_WBARRIER
);
1572 for (i
= 0; i
< roots_hash_size
[ROOT_TYPE_PINNED
]; ++i
) {
1573 for (root
= roots_hash
[ROOT_TYPE_PINNED
][i
]; root
; root
= root
->next
) {
1574 void **ptr
= (void**)root
->start_root
;
1576 while (ptr
< (void**)root
->end_root
) {
1577 check_root_obj_specific_ref (root
, *ptr
, key
);
1585 need_remove_object_for_domain (char *start
, MonoDomain
*domain
)
1587 if (mono_object_domain (start
) == domain
) {
1588 DEBUG (1, fprintf (gc_debug_file
, "Need to cleanup object %p, (%s)\n", start
, safe_name (start
)));
1595 process_object_for_domain_clearing (char *start
, MonoDomain
*domain
)
1597 GCVTable
*vt
= (GCVTable
*)LOAD_VTABLE (start
);
1598 if (vt
->klass
== mono_defaults
.internal_thread_class
)
1599 g_assert (mono_object_domain (start
) == mono_get_root_domain ());
1600 /* The object could be a proxy for an object in the domain
1602 if (mono_class_has_parent (vt
->klass
, mono_defaults
.real_proxy_class
)) {
1603 MonoObject
*server
= ((MonoRealProxy
*)start
)->unwrapped_server
;
1605 /* The server could already have been zeroed out, so
1606 we need to check for that, too. */
1607 if (server
&& (!LOAD_VTABLE (server
) || mono_object_domain (server
) == domain
)) {
1608 DEBUG (1, fprintf (gc_debug_file
, "Cleaning up remote pointer in %p to object %p (%s)\n",
1609 start
, server
, LOAD_VTABLE (server
) ? safe_name (server
) : "null"));
1610 ((MonoRealProxy
*)start
)->unwrapped_server
= NULL
;
1615 static void __attribute__((noinline
))
1616 scan_area_for_domain (MonoDomain
*domain
, char *start
, char *end
)
1621 while (start
< end
) {
1622 if (!*(void**)start
) {
1623 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
1626 vt
= (GCVTable
*)LOAD_VTABLE (start
);
1627 process_object_for_domain_clearing (start
, domain
);
1628 remove
= need_remove_object_for_domain (start
, domain
);
1629 if (remove
&& ((MonoObject
*)start
)->synchronisation
) {
1630 void **dislink
= mono_monitor_get_object_monitor_weak_link ((MonoObject
*)start
);
1632 mono_gc_register_disappearing_link (NULL
, dislink
, FALSE
);
1635 #define SCAN_OBJECT_NOSCAN
1636 #define SCAN_OBJECT_ACTION do { \
1637 if (remove) memset (start, 0, skip_size); \
1639 #include "sgen-scan-object.h"
1643 static MonoDomain
*check_domain
= NULL
;
1646 check_obj_not_in_domain (void *o
)
1648 g_assert (((MonoObject
*)o
)->vtable
->domain
!= check_domain
);
1653 scan_for_registered_roots_in_domain (MonoDomain
*domain
, int root_type
)
1657 check_domain
= domain
;
1658 for (i
= 0; i
< roots_hash_size
[root_type
]; ++i
) {
1659 for (root
= roots_hash
[root_type
][i
]; root
; root
= root
->next
) {
1660 void **start_root
= (void**)root
->start_root
;
1661 mword desc
= root
->root_desc
;
1663 /* The MonoDomain struct is allowed to hold
1664 references to objects in its own domain. */
1665 if (start_root
== (void**)domain
)
1668 switch (desc
& ROOT_DESC_TYPE_MASK
) {
1669 case ROOT_DESC_BITMAP
:
1670 desc
>>= ROOT_DESC_TYPE_SHIFT
;
1672 if ((desc
& 1) && *start_root
)
1673 check_obj_not_in_domain (*start_root
);
1678 case ROOT_DESC_COMPLEX
: {
1679 gsize
*bitmap_data
= complex_descriptors
+ (desc
>> ROOT_DESC_TYPE_SHIFT
);
1680 int bwords
= (*bitmap_data
) - 1;
1681 void **start_run
= start_root
;
1683 while (bwords
-- > 0) {
1684 gsize bmap
= *bitmap_data
++;
1685 void **objptr
= start_run
;
1687 if ((bmap
& 1) && *objptr
)
1688 check_obj_not_in_domain (*objptr
);
1692 start_run
+= GC_BITS_PER_WORD
;
1696 case ROOT_DESC_USER
: {
1697 MonoGCMarkFunc marker
= user_descriptors
[desc
>> ROOT_DESC_TYPE_SHIFT
];
1698 marker (start_root
, check_obj_not_in_domain
);
1701 case ROOT_DESC_RUN_LEN
:
1702 g_assert_not_reached ();
1704 g_assert_not_reached ();
1708 check_domain
= NULL
;
1712 clear_domain_process_pinned_object_callback (PinnedChunk
*chunk
, char *obj
, size_t size
, MonoDomain
*domain
)
1714 process_object_for_domain_clearing (obj
, domain
);
1718 clear_domain_free_pinned_object_callback (PinnedChunk
*chunk
, char *obj
, size_t size
, MonoDomain
*domain
)
1720 if (need_remove_object_for_domain (obj
, domain
))
1721 free_pinned_object (chunk
, obj
, size
);
1725 scan_pinned_object_for_xdomain_refs_callback (PinnedChunk
*chunk
, char *obj
, size_t size
, gpointer dummy
)
1727 scan_object_for_xdomain_refs (obj
);
1731 check_for_xdomain_refs (void)
1733 GCMemSection
*section
;
1736 for (section
= section_list
; section
; section
= section
->next
)
1737 scan_area_for_xdomain_refs (section
->data
, section
->end_data
);
1739 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
)
1740 scan_object_for_xdomain_refs (bigobj
->data
);
1742 scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback
, NULL
);
1746 * When appdomains are unloaded we can easily remove objects that have finalizers,
1747 * but all the others could still be present in random places on the heap.
1748 * We need a sweep to get rid of them even though it's going to be costly
1750 * The reason we need to remove them is because we access the vtable and class
1751 * structures to know the object size and the reference bitmap: once the domain is
1752 * unloaded the point to random memory.
1755 mono_gc_clear_domain (MonoDomain
* domain
)
1757 GCMemSection
*section
;
1758 LOSObject
*bigobj
, *prev
;
1763 /* Clear all remaining nursery fragments */
1764 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
1765 g_assert (nursery_next
<= nursery_frag_real_end
);
1766 memset (nursery_next
, 0, nursery_frag_real_end
- nursery_next
);
1767 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
1768 memset (frag
->fragment_start
, 0, frag
->fragment_end
- frag
->fragment_start
);
1772 if (xdomain_checks
&& domain
!= mono_get_root_domain ()) {
1773 scan_for_registered_roots_in_domain (domain
, ROOT_TYPE_NORMAL
);
1774 scan_for_registered_roots_in_domain (domain
, ROOT_TYPE_WBARRIER
);
1775 check_for_xdomain_refs ();
1778 for (section
= section_list
; section
; section
= section
->next
) {
1779 scan_area_for_domain (domain
, section
->data
, section
->end_data
);
1782 /* We need two passes over pinned and large objects because
1783 freeing such an object gives its memory back to the OS (in
1784 the case of large objects) or obliterates its vtable
1785 (pinned objects), but we might need to dereference a
1786 pointer from an object to another object if the first
1787 object is a proxy. */
1788 scan_pinned_objects ((ScanPinnedObjectCallbackFunc
)clear_domain_process_pinned_object_callback
, domain
);
1789 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
)
1790 process_object_for_domain_clearing (bigobj
->data
, domain
);
1793 for (bigobj
= los_object_list
; bigobj
;) {
1794 if (need_remove_object_for_domain (bigobj
->data
, domain
)) {
1795 LOSObject
*to_free
= bigobj
;
1797 prev
->next
= bigobj
->next
;
1799 los_object_list
= bigobj
->next
;
1800 bigobj
= bigobj
->next
;
1801 DEBUG (1, fprintf (gc_debug_file
, "Freeing large object %p (%s)\n",
1802 bigobj
->data
, safe_name (bigobj
->data
)));
1803 free_large_object (to_free
);
1807 bigobj
= bigobj
->next
;
1809 scan_pinned_objects ((ScanPinnedObjectCallbackFunc
)clear_domain_free_pinned_object_callback
, domain
);
1811 for (i
= GENERATION_NURSERY
; i
< GENERATION_MAX
; ++i
)
1812 null_links_for_domain (domain
, i
);
1818 * add_to_global_remset:
1820 * The global remset contains locations which point into newspace after
1821 * a minor collection. This can happen if the objects they point to are pinned.
1824 add_to_global_remset (gpointer ptr
, gboolean root
)
1828 DEBUG (8, fprintf (gc_debug_file
, "Adding global remset for %p\n", ptr
));
1831 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1832 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1834 if (global_remset
->store_next
+ 3 < global_remset
->end_set
) {
1836 *(global_remset
->store_next
++) = (mword
)ptr
| REMSET_OTHER
;
1837 *(global_remset
->store_next
++) = (mword
)REMSET_ROOT_LOCATION
;
1839 *(global_remset
->store_next
++) = (mword
)ptr
;
1843 rs
= alloc_remset (global_remset
->end_set
- global_remset
->data
, NULL
);
1844 rs
->next
= global_remset
;
1847 *(global_remset
->store_next
++) = (mword
)ptr
| REMSET_OTHER
;
1848 *(global_remset
->store_next
++) = (mword
)REMSET_ROOT_LOCATION
;
1850 *(global_remset
->store_next
++) = (mword
)ptr
;
1854 int global_rs_size
= 0;
1856 for (rs
= global_remset
; rs
; rs
= rs
->next
) {
1857 global_rs_size
+= rs
->store_next
- rs
->data
;
1859 DEBUG (4, fprintf (gc_debug_file
, "Global remset now has size %d\n", global_rs_size
));
1864 * This is how the copying happens from the nursery to the old generation.
1865 * We assume that at this time all the pinned objects have been identified and
1867 * We run scan_object() for each pinned object so that each referenced
1868 * objects if possible are copied. The new gray objects created can have
1869 * scan_object() run on them right away, too.
1870 * Then we run copy_object() for the precisely tracked roots. At this point
1871 * all the roots are either gray or black. We run scan_object() on the gray
1872 * objects until no more gray objects are created.
1873 * At the end of the process we walk again the pinned list and we unmark
1874 * the pinned flag. As we go we also create the list of free space for use
1875 * in the next allocation runs.
1877 * We need to remember objects from the old generation that point to the new one
1878 * (or just addresses?).
1880 * copy_object could be made into a macro once debugged (use inline for now).
1883 static char* __attribute__((noinline
))
1884 copy_object (char *obj
, char *from_space_start
, char *from_space_end
)
1886 static void *copy_labels
[] = { &&LAB_0
, &&LAB_1
, &&LAB_2
, &&LAB_3
, &&LAB_4
, &&LAB_5
, &&LAB_6
, &&LAB_7
, &&LAB_8
};
1889 * FIXME: The second set of checks is only needed if we are called for tospace
1892 if (obj
>= from_space_start
&& obj
< from_space_end
&& (obj
< to_space
|| obj
>= to_space_end
)) {
1896 DEBUG (9, fprintf (gc_debug_file
, "Precise copy of %p", obj
));
1897 if ((forwarded
= object_is_forwarded (obj
))) {
1898 g_assert (((MonoVTable
*)LOAD_VTABLE(obj
))->gc_descr
);
1899 DEBUG (9, fprintf (gc_debug_file
, " (already forwarded to %p)\n", forwarded
));
1902 if (object_is_pinned (obj
)) {
1903 g_assert (((MonoVTable
*)LOAD_VTABLE(obj
))->gc_descr
);
1904 DEBUG (9, fprintf (gc_debug_file
, " (pinned, no change)\n"));
1907 objsize
= safe_object_get_size ((MonoObject
*)obj
);
1908 objsize
+= ALLOC_ALIGN
- 1;
1909 objsize
&= ~(ALLOC_ALIGN
- 1);
1910 DEBUG (9, fprintf (gc_debug_file
, " (to %p, %s size: %zd)\n", gray_objects
, ((MonoObject
*)obj
)->vtable
->klass
->name
, objsize
));
1911 /* FIXME: handle pinned allocs:
1912 * Large objects are simple, at least until we always follow the rule:
1913 * if objsize >= MAX_SMALL_OBJ_SIZE, pin the object and return it.
1914 * At the end of major collections, we walk the los list and if
1915 * the object is pinned, it is marked, otherwise it can be freed.
1917 if (G_UNLIKELY (objsize
>= MAX_SMALL_OBJ_SIZE
|| (obj
>= min_pinned_chunk_addr
&& obj
< max_pinned_chunk_addr
&& obj_is_from_pinned_alloc (obj
)))) {
1918 DEBUG (9, fprintf (gc_debug_file
, "Marked LOS/Pinned %p (%s), size: %zd\n", obj
, safe_name (obj
), objsize
));
1922 /* ok, the object is not pinned, we can move it */
1923 /* use a optimized memcpy here */
1924 if (objsize
<= sizeof (gpointer
) * 8) {
1925 mword
*dest
= (mword
*)gray_objects
;
1926 goto *copy_labels
[objsize
/ sizeof (gpointer
)];
1928 (dest
) [7] = ((mword
*)obj
) [7];
1930 (dest
) [6] = ((mword
*)obj
) [6];
1932 (dest
) [5] = ((mword
*)obj
) [5];
1934 (dest
) [4] = ((mword
*)obj
) [4];
1936 (dest
) [3] = ((mword
*)obj
) [3];
1938 (dest
) [2] = ((mword
*)obj
) [2];
1940 (dest
) [1] = ((mword
*)obj
) [1];
1942 (dest
) [0] = ((mword
*)obj
) [0];
1950 char* edi
= gray_objects
;
1951 __asm__
__volatile__(
1953 : "=&c" (ecx
), "=&D" (edi
), "=&S" (esi
)
1954 : "0" (objsize
/4), "1" (edi
),"2" (esi
)
1959 memcpy (gray_objects
, obj
, objsize
);
1962 /* adjust array->bounds */
1963 vt
= ((MonoObject
*)obj
)->vtable
;
1964 g_assert (vt
->gc_descr
);
1965 if (G_UNLIKELY (vt
->rank
&& ((MonoArray
*)obj
)->bounds
)) {
1966 MonoArray
*array
= (MonoArray
*)gray_objects
;
1967 array
->bounds
= (MonoArrayBounds
*)((char*)gray_objects
+ ((char*)((MonoArray
*)obj
)->bounds
- (char*)obj
));
1968 DEBUG (9, fprintf (gc_debug_file
, "Array instance %p: size: %zd, rank: %d, length: %d\n", array
, objsize
, vt
->rank
, mono_array_length (array
)));
1970 /* set the forwarding pointer */
1971 forward_object (obj
, gray_objects
);
1973 to_space_section
->scan_starts
[((char*)obj
- (char*)to_space_section
->data
)/SCAN_START_SIZE
] = obj
;
1974 gray_objects
+= objsize
;
1975 DEBUG (8, g_assert (gray_objects
<= to_space_end
));
1982 #define HANDLE_PTR(ptr,obj) do { \
1983 void *__old = *(ptr); \
1985 *(ptr) = copy_object (__old, from_start, from_end); \
1986 DEBUG (9, if (__old != *(ptr)) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
1987 if (G_UNLIKELY (*(ptr) >= (void*)from_start && *(ptr) < (void*)from_end) && !ptr_in_nursery (ptr)) \
1988 add_to_global_remset ((ptr), FALSE); \
1993 * Scan the object pointed to by @start for references to
1994 * other objects between @from_start and @from_end and copy
1995 * them to the gray_objects area.
1996 * Returns a pointer to the end of the object.
1999 scan_object (char *start
, char* from_start
, char* from_end
)
2001 #include "sgen-scan-object.h"
2009 * Scan objects in the gray stack until the stack is empty. This should be called
2010 * frequently after each object is copied, to achieve better locality and cache
2014 drain_gray_stack (char *start_addr
, char *end_addr
)
2016 char *gray_start
= gray_first
;
2018 while (gray_start
< gray_objects
) {
2019 DEBUG (9, fprintf (gc_debug_file
, "Precise gray object scan %p (%s)\n", gray_start
, safe_name (gray_start
)));
2020 gray_start
= scan_object (gray_start
, start_addr
, end_addr
);
2023 gray_first
= gray_start
;
2029 * Scan the valuetype pointed to by START, described by DESC for references to
2030 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2031 * Returns a pointer to the end of the object.
2034 scan_vtype (char *start
, mword desc
, char* from_start
, char* from_end
)
2038 /* The descriptors include info about the MonoObject header as well */
2039 start
-= sizeof (MonoObject
);
2041 switch (desc
& 0x7) {
2042 case DESC_TYPE_RUN_LENGTH
:
2043 OBJ_RUN_LEN_FOREACH_PTR (desc
,start
);
2044 OBJ_RUN_LEN_SIZE (skip_size
, desc
, start
);
2045 g_assert (skip_size
);
2046 return start
+ skip_size
;
2047 case DESC_TYPE_SMALL_BITMAP
:
2048 OBJ_BITMAP_FOREACH_PTR (desc
,start
);
2049 OBJ_BITMAP_SIZE (skip_size
, desc
, start
);
2050 return start
+ skip_size
;
2051 case DESC_TYPE_LARGE_BITMAP
:
2052 case DESC_TYPE_COMPLEX
:
2054 g_assert_not_reached ();
2057 // The other descriptors can't happen with vtypes
2058 g_assert_not_reached ();
2065 * Addresses from start to end are already sorted. This function finds the object header
2066 * for each address and pins the object. The addresses must be inside the passed section.
2067 * Return the number of pinned objects.
2070 pin_objects_from_addresses (GCMemSection
*section
, void **start
, void **end
, void *start_nursery
, void *end_nursery
)
2075 void *last_obj
= NULL
;
2076 size_t last_obj_size
= 0;
2079 void **definitely_pinned
= start
;
2080 while (start
< end
) {
2082 /* the range check should be reduntant */
2083 if (addr
!= last
&& addr
>= start_nursery
&& addr
< end_nursery
) {
2084 DEBUG (5, fprintf (gc_debug_file
, "Considering pinning addr %p\n", addr
));
2085 /* multiple pointers to the same object */
2086 if (addr
>= last_obj
&& (char*)addr
< (char*)last_obj
+ last_obj_size
) {
2090 idx
= ((char*)addr
- (char*)section
->data
) / SCAN_START_SIZE
;
2091 search_start
= (void*)section
->scan_starts
[idx
];
2092 if (!search_start
|| search_start
> addr
) {
2095 search_start
= section
->scan_starts
[idx
];
2096 if (search_start
&& search_start
<= addr
)
2099 if (!search_start
|| search_start
> addr
)
2100 search_start
= start_nursery
;
2102 if (search_start
< last_obj
)
2103 search_start
= (char*)last_obj
+ last_obj_size
;
2104 /* now addr should be in an object a short distance from search_start
2105 * Note that search_start must point to zeroed mem or point to an object.
2108 if (!*(void**)search_start
) {
2109 mword p
= (mword
)search_start
;
2110 p
+= sizeof (gpointer
);
2111 p
+= ALLOC_ALIGN
- 1;
2112 p
&= ~(ALLOC_ALIGN
- 1);
2113 search_start
= (void*)p
;
2116 last_obj
= search_start
;
2117 last_obj_size
= safe_object_get_size ((MonoObject
*)search_start
);
2118 last_obj_size
+= ALLOC_ALIGN
- 1;
2119 last_obj_size
&= ~(ALLOC_ALIGN
- 1);
2120 DEBUG (8, fprintf (gc_debug_file
, "Pinned try match %p (%s), size %zd\n", last_obj
, safe_name (last_obj
), last_obj_size
));
2121 if (addr
>= search_start
&& (char*)addr
< (char*)last_obj
+ last_obj_size
) {
2122 DEBUG (4, fprintf (gc_debug_file
, "Pinned object %p, vtable %p (%s), count %d\n", search_start
, *(void**)search_start
, safe_name (search_start
), count
));
2123 pin_object (search_start
);
2124 definitely_pinned
[count
] = search_start
;
2128 /* skip to the next object */
2129 search_start
= (void*)((char*)search_start
+ last_obj_size
);
2130 } while (search_start
<= addr
);
2131 /* we either pinned the correct object or we ignored the addr because
2132 * it points to unused zeroed memory.
2138 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2142 static void** pin_queue
;
2143 static int pin_queue_size
= 0;
2144 static int next_pin_slot
= 0;
2149 gap
= (gap
* 10) / 13;
2150 if (gap
== 9 || gap
== 10)
2159 compare_addr (const void *a
, const void *b
)
2161 return *(const void **)a
- *(const void **)b
;
2165 /* sort the addresses in array in increasing order */
2167 sort_addresses (void **array
, int size
)
2170 * qsort is slower as predicted.
2171 * qsort (array, size, sizeof (gpointer), compare_addr);
2178 gap
= new_gap (gap
);
2181 for (i
= 0; i
< end
; i
++) {
2183 if (array
[i
] > array
[j
]) {
2184 void* val
= array
[i
];
2185 array
[i
] = array
[j
];
2190 if (gap
== 1 && !swapped
)
2195 static G_GNUC_UNUSED
void
2196 print_nursery_gaps (void* start_nursery
, void *end_nursery
)
2199 gpointer first
= start_nursery
;
2201 for (i
= 0; i
< next_pin_slot
; ++i
) {
2202 next
= pin_queue
[i
];
2203 fprintf (gc_debug_file
, "Nursery range: %p-%p, size: %zd\n", first
, next
, (char*)next
-(char*)first
);
2207 fprintf (gc_debug_file
, "Nursery range: %p-%p, size: %zd\n", first
, next
, (char*)next
-(char*)first
);
2210 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2212 optimize_pin_queue (int start_slot
)
2214 void **start
, **cur
, **end
;
2215 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2216 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2217 DEBUG (5, fprintf (gc_debug_file
, "Sorting pin queue, size: %d\n", next_pin_slot
));
2218 if ((next_pin_slot
- start_slot
) > 1)
2219 sort_addresses (pin_queue
+ start_slot
, next_pin_slot
- start_slot
);
2220 start
= cur
= pin_queue
+ start_slot
;
2221 end
= pin_queue
+ next_pin_slot
;
2224 while (*start
== *cur
&& cur
< end
)
2228 next_pin_slot
= start
- pin_queue
;
2229 DEBUG (5, fprintf (gc_debug_file
, "Pin queue reduced to size: %d\n", next_pin_slot
));
2230 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2235 realloc_pin_queue (void)
2237 int new_size
= pin_queue_size
? pin_queue_size
+ pin_queue_size
/2: 1024;
2238 void **new_pin
= get_internal_mem (sizeof (void*) * new_size
);
2239 memcpy (new_pin
, pin_queue
, sizeof (void*) * next_pin_slot
);
2240 free_internal_mem (pin_queue
);
2241 pin_queue
= new_pin
;
2242 pin_queue_size
= new_size
;
2243 DEBUG (4, fprintf (gc_debug_file
, "Reallocated pin queue to size: %d\n", new_size
));
2247 * Scan the memory between start and end and queue values which could be pointers
2248 * to the area between start_nursery and end_nursery for later consideration.
2249 * Typically used for thread stacks.
2252 conservatively_pin_objects_from (void **start
, void **end
, void *start_nursery
, void *end_nursery
)
2255 while (start
< end
) {
2256 if (*start
>= start_nursery
&& *start
< end_nursery
) {
2258 * *start can point to the middle of an object
2259 * note: should we handle pointing at the end of an object?
2260 * pinning in C# code disallows pointing at the end of an object
2261 * but there is some small chance that an optimizing C compiler
2262 * may keep the only reference to an object by pointing
2263 * at the end of it. We ignore this small chance for now.
2264 * Pointers to the end of an object are indistinguishable
2265 * from pointers to the start of the next object in memory
2266 * so if we allow that we'd need to pin two objects...
2267 * We queue the pointer in an array, the
2268 * array will then be sorted and uniqued. This way
2269 * we can coalesce several pinning pointers and it should
2270 * be faster since we'd do a memory scan with increasing
2271 * addresses. Note: we can align the address to the allocation
2272 * alignment, so the unique process is more effective.
2274 mword addr
= (mword
)*start
;
2275 addr
&= ~(ALLOC_ALIGN
- 1);
2276 if (next_pin_slot
>= pin_queue_size
)
2277 realloc_pin_queue ();
2278 pin_queue
[next_pin_slot
++] = (void*)addr
;
2279 DEBUG (6, if (count
) fprintf (gc_debug_file
, "Pinning address %p\n", (void*)addr
));
2284 DEBUG (7, if (count
) fprintf (gc_debug_file
, "found %d potential pinned heap pointers\n", count
));
2286 #ifdef HAVE_VALGRIND_MEMCHECK_H
2288 * The pinning addresses might come from undefined memory, this is normal. Since they
2289 * are used in lots of functions, we make the memory defined here instead of having
2290 * to add a supression for those functions.
2292 VALGRIND_MAKE_MEM_DEFINED (pin_queue
, next_pin_slot
* sizeof (pin_queue
[0]));
2297 * If generation is 0, just mark objects in the nursery, the others we don't care,
2298 * since they are not going to move anyway.
2299 * There are different areas that are scanned for pinned pointers:
2300 * *) the thread stacks (when jit support is ready only the unmanaged frames)
2301 * *) the pinned handle table
2302 * *) the pinned roots
2304 * Note: when we'll use a write barrier for old to new gen references, we need to
2305 * keep track of old gen objects that point to pinned new gen objects because in that
2306 * case the referenced object will be moved maybe at the next collection, but there
2307 * is no write in the old generation area where the pinned object is referenced
2308 * and we may not consider it as reachable.
2310 static G_GNUC_UNUSED
void
2311 mark_pinned_objects (int generation
)
2316 * Debugging function: find in the conservative roots where @obj is being pinned.
2318 static G_GNUC_UNUSED
void
2319 find_pinning_reference (char *obj
, size_t size
)
2323 char *endobj
= obj
+ size
;
2324 for (i
= 0; i
< roots_hash_size
[0]; ++i
) {
2325 for (root
= roots_hash
[0][i
]; root
; root
= root
->next
) {
2326 /* if desc is non-null it has precise info */
2327 if (!root
->root_desc
) {
2328 char ** start
= (char**)root
->start_root
;
2329 while (start
< (char**)root
->end_root
) {
2330 if (*start
>= obj
&& *start
< endobj
) {
2331 DEBUG (0, fprintf (gc_debug_file
, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj
, root
->start_root
, root
->end_root
, start
, root
));
2338 find_pinning_ref_from_thread (obj
, size
);
2342 * The first thing we do in a collection is to identify pinned objects.
2343 * This function considers all the areas of memory that need to be
2344 * conservatively scanned.
2347 pin_from_roots (void *start_nursery
, void *end_nursery
)
2351 DEBUG (2, fprintf (gc_debug_file
, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size
, num_roots_entries
[ROOT_TYPE_NORMAL
], num_roots_entries
[ROOT_TYPE_PINNED
]));
2352 /* objects pinned from the API are inside these roots */
2353 for (i
= 0; i
< roots_hash_size
[ROOT_TYPE_PINNED
]; ++i
) {
2354 for (root
= roots_hash
[ROOT_TYPE_PINNED
][i
]; root
; root
= root
->next
) {
2355 DEBUG (6, fprintf (gc_debug_file
, "Pinned roots %p-%p\n", root
->start_root
, root
->end_root
));
2356 conservatively_pin_objects_from ((void**)root
->start_root
, (void**)root
->end_root
, start_nursery
, end_nursery
);
2359 /* now deal with the thread stacks
2360 * in the future we should be able to conservatively scan only:
2361 * *) the cpu registers
2362 * *) the unmanaged stack frames
2363 * *) the _last_ managed stack frame
2364 * *) pointers slots in managed frames
2366 scan_thread_data (start_nursery
, end_nursery
, FALSE
);
2369 /* Copy function called from user defined mark functions */
2370 static char *user_copy_n_start
;
2371 static char *user_copy_n_end
;
2374 user_copy (void *addr
)
2377 return copy_object (addr
, user_copy_n_start
, user_copy_n_end
);
2383 * The memory area from start_root to end_root contains pointers to objects.
2384 * Their position is precisely described by @desc (this means that the pointer
2385 * can be either NULL or the pointer to the start of an object).
2386 * This functions copies them to to_space updates them.
2389 precisely_scan_objects_from (void** start_root
, void** end_root
, char* n_start
, char *n_end
, mword desc
)
2391 switch (desc
& ROOT_DESC_TYPE_MASK
) {
2392 case ROOT_DESC_BITMAP
:
2393 desc
>>= ROOT_DESC_TYPE_SHIFT
;
2395 if ((desc
& 1) && *start_root
) {
2396 *start_root
= copy_object (*start_root
, n_start
, n_end
);
2397 DEBUG (9, fprintf (gc_debug_file
, "Overwrote root at %p with %p\n", start_root
, *start_root
));
2398 drain_gray_stack (n_start
, n_end
);
2404 case ROOT_DESC_COMPLEX
: {
2405 gsize
*bitmap_data
= complex_descriptors
+ (desc
>> ROOT_DESC_TYPE_SHIFT
);
2406 int bwords
= (*bitmap_data
) - 1;
2407 void **start_run
= start_root
;
2409 while (bwords
-- > 0) {
2410 gsize bmap
= *bitmap_data
++;
2411 void **objptr
= start_run
;
2413 if ((bmap
& 1) && *objptr
) {
2414 *objptr
= copy_object (*objptr
, n_start
, n_end
);
2415 DEBUG (9, fprintf (gc_debug_file
, "Overwrote root at %p with %p\n", objptr
, *objptr
));
2416 drain_gray_stack (n_start
, n_end
);
2421 start_run
+= GC_BITS_PER_WORD
;
2425 case ROOT_DESC_USER
: {
2426 MonoGCMarkFunc marker
= user_descriptors
[desc
>> ROOT_DESC_TYPE_SHIFT
];
2428 user_copy_n_start
= n_start
;
2429 user_copy_n_end
= n_end
;
2430 marker (start_root
, user_copy
);
2433 case ROOT_DESC_RUN_LEN
:
2434 g_assert_not_reached ();
2436 g_assert_not_reached ();
2441 alloc_fragment (void)
2443 Fragment
*frag
= fragment_freelist
;
2445 fragment_freelist
= frag
->next
;
2449 frag
= get_internal_mem (sizeof (Fragment
));
2455 * Allocate and setup the data structures needed to be able to allocate objects
2456 * in the nursery. The nursery is stored in nursery_section.
2459 alloc_nursery (void)
2461 GCMemSection
*section
;
2467 if (nursery_section
)
2469 DEBUG (2, fprintf (gc_debug_file
, "Allocating nursery size: %zd\n", nursery_size
));
2470 /* later we will alloc a larger area for the nursery but only activate
2471 * what we need. The rest will be used as expansion if we have too many pinned
2472 * objects in the existing nursery.
2474 /* FIXME: handle OOM */
2475 section
= get_internal_mem (sizeof (GCMemSection
));
2477 #ifdef ALIGN_NURSERY
2478 /* Allocate twice the memory to be able to put the nursery at an aligned address */
2479 g_assert (nursery_size
== DEFAULT_NURSERY_SIZE
);
2481 alloc_size
= nursery_size
* 2;
2482 data
= get_os_memory (alloc_size
, TRUE
);
2483 nursery_start
= (void*)(((mword
)data
+ (1 << DEFAULT_NURSERY_BITS
) - 1) & ~((1 << DEFAULT_NURSERY_BITS
) - 1));
2484 g_assert ((char*)nursery_start
+ nursery_size
<= ((char*)data
+ alloc_size
));
2485 /* FIXME: Use the remaining size for something else, if it is big enough */
2487 alloc_size
= nursery_size
;
2488 data
= get_os_memory (alloc_size
, TRUE
);
2489 nursery_start
= data
;
2491 nursery_real_end
= nursery_start
+ nursery_size
;
2492 UPDATE_HEAP_BOUNDARIES (nursery_start
, nursery_real_end
);
2493 nursery_next
= nursery_start
;
2494 total_alloc
+= alloc_size
;
2495 DEBUG (4, fprintf (gc_debug_file
, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data
, data
+ alloc_size
, nursery_size
, total_alloc
));
2496 section
->data
= section
->next_data
= data
;
2497 section
->size
= alloc_size
;
2498 section
->end_data
= nursery_real_end
;
2499 scan_starts
= alloc_size
/ SCAN_START_SIZE
;
2500 section
->scan_starts
= get_internal_mem (sizeof (char*) * scan_starts
);
2501 section
->num_scan_start
= scan_starts
;
2502 section
->role
= MEMORY_ROLE_GEN0
;
2504 /* add to the section list */
2505 section
->next
= section_list
;
2506 section_list
= section
;
2508 nursery_section
= section
;
2510 /* Setup the single first large fragment */
2511 frag
= alloc_fragment ();
2512 frag
->fragment_start
= nursery_start
;
2513 frag
->fragment_limit
= nursery_start
;
2514 frag
->fragment_end
= nursery_real_end
;
2515 nursery_frag_real_end
= nursery_real_end
;
2516 /* FIXME: frag here is lost */
2520 scan_finalizer_entries (FinalizeEntry
*list
, char *start
, char *end
) {
2523 for (fin
= list
; fin
; fin
= fin
->next
) {
2526 DEBUG (5, fprintf (gc_debug_file
, "Scan of fin ready object: %p (%s)\n", fin
->object
, safe_name (fin
->object
)));
2527 fin
->object
= copy_object (fin
->object
, start
, end
);
2532 * Update roots in the old generation. Since we currently don't have the
2533 * info from the write barriers, we just scan all the objects.
2535 static G_GNUC_UNUSED
void
2536 scan_old_generation (char *start
, char* end
)
2538 GCMemSection
*section
;
2539 LOSObject
*big_object
;
2542 for (section
= section_list
; section
; section
= section
->next
) {
2543 if (section
== nursery_section
)
2545 DEBUG (2, fprintf (gc_debug_file
, "Scan of old section: %p-%p, size: %d\n", section
->data
, section
->next_data
, (int)(section
->next_data
- section
->data
)));
2546 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2548 while (p
< section
->next_data
) {
2553 DEBUG (8, fprintf (gc_debug_file
, "Precise old object scan of %p (%s)\n", p
, safe_name (p
)));
2554 p
= scan_object (p
, start
, end
);
2557 /* scan the old object space, too */
2558 for (big_object
= los_object_list
; big_object
; big_object
= big_object
->next
) {
2559 DEBUG (5, fprintf (gc_debug_file
, "Scan of big object: %p (%s), size: %zd\n", big_object
->data
, safe_name (big_object
->data
), big_object
->size
));
2560 scan_object (big_object
->data
, start
, end
);
2562 /* scan the list of objects ready for finalization */
2563 scan_finalizer_entries (fin_ready_list
, start
, end
);
2564 scan_finalizer_entries (critical_fin_list
, start
, end
);
2567 static mword fragment_total
= 0;
2569 * We found a fragment of free memory in the nursery: memzero it and if
2570 * it is big enough, add it to the list of fragments that can be used for
2574 add_nursery_frag (size_t frag_size
, char* frag_start
, char* frag_end
)
2577 DEBUG (4, fprintf (gc_debug_file
, "Found empty fragment: %p-%p, size: %zd\n", frag_start
, frag_end
, frag_size
));
2578 /* memsetting just the first chunk start is bound to provide better cache locality */
2579 if (nursery_clear_policy
== CLEAR_AT_GC
)
2580 memset (frag_start
, 0, frag_size
);
2581 /* Not worth dealing with smaller fragments: need to tune */
2582 if (frag_size
>= FRAGMENT_MIN_SIZE
) {
2583 fragment
= alloc_fragment ();
2584 fragment
->fragment_start
= frag_start
;
2585 fragment
->fragment_limit
= frag_start
;
2586 fragment
->fragment_end
= frag_end
;
2587 fragment
->next
= nursery_fragments
;
2588 nursery_fragments
= fragment
;
2589 fragment_total
+= frag_size
;
2591 /* Clear unused fragments, pinning depends on this */
2592 memset (frag_start
, 0, frag_size
);
2597 scan_needed_big_objects (char *start_addr
, char *end_addr
)
2599 LOSObject
*big_object
;
2601 for (big_object
= los_object_list
; big_object
; big_object
= big_object
->next
) {
2602 if (!big_object
->scanned
&& object_is_pinned (big_object
->data
)) {
2603 DEBUG (5, fprintf (gc_debug_file
, "Scan of big object: %p (%s), size: %zd\n", big_object
->data
, safe_name (big_object
->data
), big_object
->size
));
2604 scan_object (big_object
->data
, start_addr
, end_addr
);
2605 big_object
->scanned
= TRUE
;
2613 generation_name (int generation
)
2615 switch (generation
) {
2616 case GENERATION_NURSERY
: return "nursery";
2617 case GENERATION_OLD
: return "old";
2618 default: g_assert_not_reached ();
2622 static DisappearingLinkHashTable
*
2623 get_dislink_hash_table (int generation
)
2625 switch (generation
) {
2626 case GENERATION_NURSERY
: return &minor_disappearing_link_hash
;
2627 case GENERATION_OLD
: return &major_disappearing_link_hash
;
2628 default: g_assert_not_reached ();
2632 static FinalizeEntryHashTable
*
2633 get_finalize_entry_hash_table (int generation
)
2635 switch (generation
) {
2636 case GENERATION_NURSERY
: return &minor_finalizable_hash
;
2637 case GENERATION_OLD
: return &major_finalizable_hash
;
2638 default: g_assert_not_reached ();
2643 finish_gray_stack (char *start_addr
, char *end_addr
, int generation
)
2647 int fin_ready
, bigo_scanned_num
;
2651 * We copied all the reachable objects. Now it's the time to copy
2652 * the objects that were not referenced by the roots, but by the copied objects.
2653 * we built a stack of objects pointed to by gray_start: they are
2654 * additional roots and we may add more items as we go.
2655 * We loop until gray_start == gray_objects which means no more objects have
2656 * been added. Note this is iterative: no recursion is involved.
2657 * We need to walk the LO list as well in search of marked big objects
2658 * (use a flag since this is needed only on major collections). We need to loop
2659 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2660 * To achieve better cache locality and cache usage, we drain the gray stack
2661 * frequently, after each object is copied, and just finish the work here.
2663 gray_start
= gray_first
;
2664 while (gray_start
< gray_objects
) {
2665 DEBUG (9, fprintf (gc_debug_file
, "Precise gray object scan %p (%s)\n", gray_start
, safe_name (gray_start
)));
2666 gray_start
= scan_object (gray_start
, start_addr
, end_addr
);
2669 //scan_old_generation (start_addr, end_addr);
2670 DEBUG (2, fprintf (gc_debug_file
, "%s generation done\n", generation_name (generation
)));
2671 /* walk the finalization queue and move also the objects that need to be
2672 * finalized: use the finalized objects as new roots so the objects they depend
2673 * on are also not reclaimed. As with the roots above, only objects in the nursery
2674 * are marked/copied.
2675 * We need a loop here, since objects ready for finalizers may reference other objects
2676 * that are fin-ready. Speedup with a flag?
2679 fin_ready
= num_ready_finalizers
;
2680 finalize_in_range (start_addr
, end_addr
, generation
);
2681 if (generation
== GENERATION_OLD
)
2682 finalize_in_range (nursery_start
, nursery_real_end
, GENERATION_NURSERY
);
2683 bigo_scanned_num
= scan_needed_big_objects (start_addr
, end_addr
);
2685 /* drain the new stack that might have been created */
2686 DEBUG (6, fprintf (gc_debug_file
, "Precise scan of gray area post fin: %p-%p, size: %d\n", gray_start
, gray_objects
, (int)(gray_objects
- gray_start
)));
2687 while (gray_start
< gray_objects
) {
2688 DEBUG (9, fprintf (gc_debug_file
, "Precise gray object scan %p (%s)\n", gray_start
, safe_name (gray_start
)));
2689 gray_start
= scan_object (gray_start
, start_addr
, end_addr
);
2691 } while (fin_ready
!= num_ready_finalizers
|| bigo_scanned_num
);
2693 DEBUG (2, fprintf (gc_debug_file
, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation
), TV_ELAPSED (atv
, btv
)));
2696 * handle disappearing links
2697 * Note we do this after checking the finalization queue because if an object
2698 * survives (at least long enough to be finalized) we don't clear the link.
2699 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2700 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2703 g_assert (gray_start
== gray_objects
);
2705 null_link_in_range (start_addr
, end_addr
, generation
);
2706 if (generation
== GENERATION_OLD
)
2707 null_link_in_range (start_addr
, end_addr
, GENERATION_NURSERY
);
2708 if (gray_start
== gray_objects
)
2710 while (gray_start
< gray_objects
) {
2711 DEBUG (9, fprintf (gc_debug_file
, "Precise gray object scan %p (%s)\n", gray_start
, safe_name (gray_start
)));
2712 gray_start
= scan_object (gray_start
, start_addr
, end_addr
);
2716 g_assert (gray_start
== gray_objects
);
2717 DEBUG (2, fprintf (gc_debug_file
, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation
), (int)(gray_objects
- to_space
), to_space
, gray_objects
));
2718 to_space
= gray_start
;
2719 to_space_section
->next_data
= to_space
;
2722 static int last_num_pinned
= 0;
2725 build_nursery_fragments (int start_pin
, int end_pin
)
2727 char *frag_start
, *frag_end
;
2731 /* FIXME: handle non-NULL fragment_freelist */
2732 fragment_freelist
= nursery_fragments
;
2733 nursery_fragments
= NULL
;
2734 frag_start
= nursery_start
;
2736 /* clear scan starts */
2737 memset (nursery_section
->scan_starts
, 0, nursery_section
->num_scan_start
* sizeof (gpointer
));
2738 for (i
= start_pin
; i
< end_pin
; ++i
) {
2739 frag_end
= pin_queue
[i
];
2740 /* remove the pin bit from pinned objects */
2741 unpin_object (frag_end
);
2742 nursery_section
->scan_starts
[((char*)frag_end
- (char*)nursery_section
->data
)/SCAN_START_SIZE
] = frag_end
;
2743 frag_size
= frag_end
- frag_start
;
2745 add_nursery_frag (frag_size
, frag_start
, frag_end
);
2746 frag_size
= safe_object_get_size ((MonoObject
*)pin_queue
[i
]);
2747 frag_size
+= ALLOC_ALIGN
- 1;
2748 frag_size
&= ~(ALLOC_ALIGN
- 1);
2749 frag_start
= (char*)pin_queue
[i
] + frag_size
;
2751 * pin_queue [i] might point to a half-constructed string or vector whose
2752 * length field is not set. In that case, frag_start points inside the
2753 * (zero initialized) object. Find the end of the object by scanning forward.
2756 if (is_maybe_half_constructed (pin_queue
[i
])) {
2759 /* This is also hit for zero length arrays/strings */
2761 /* Find the end of the TLAB which contained this allocation */
2762 tlab_end
= find_tlab_next_from_address (pin_queue
[i
]);
2765 while ((frag_start
< tlab_end
) && *(mword
*)frag_start
== 0)
2766 frag_start
+= sizeof (mword
);
2769 * FIXME: The object is either not allocated in a TLAB, or it isn't a
2770 * half constructed object.
2775 nursery_last_pinned_end
= frag_start
;
2776 frag_end
= nursery_real_end
;
2777 frag_size
= frag_end
- frag_start
;
2779 add_nursery_frag (frag_size
, frag_start
, frag_end
);
2780 if (!nursery_fragments
) {
2781 DEBUG (1, fprintf (gc_debug_file
, "Nursery fully pinned (%d)\n", end_pin
- start_pin
));
2782 for (i
= start_pin
; i
< end_pin
; ++i
) {
2783 DEBUG (3, fprintf (gc_debug_file
, "Bastard pinning obj %p (%s), size: %d\n", pin_queue
[i
], safe_name (pin_queue
[i
]), safe_object_get_size (pin_queue
[i
])));
2788 nursery_next
= nursery_frag_real_end
= NULL
;
2790 /* Clear TLABs for all threads */
2795 shorten_section (GCMemSection
*section
, mword size
)
2797 size
+= pagesize
- 1;
2798 size
&= ~(pagesize
- 1);
2800 if (size
== section
->size
)
2803 free_os_memory (section
->data
+ size
, section
->size
- size
);
2804 total_alloc
-= section
->size
- size
;
2806 DEBUG (2, fprintf (gc_debug_file
, "Shortening section %p from %d bytes to %d bytes\n",
2807 section
, section
->size
, size
));
2808 section
->size
= size
;
2809 section
->end_data
= section
->data
+ size
;
2812 /* FIXME: later reduce code duplication here with the above
2813 * We don't keep track of section fragments for non-nursery sections yet, so
2817 build_section_fragments (GCMemSection
*section
)
2820 char *frag_start
, *frag_end
;
2823 /* clear scan starts */
2824 memset (section
->scan_starts
, 0, section
->num_scan_start
* sizeof (gpointer
));
2825 frag_start
= section
->data
;
2826 section
->next_data
= section
->data
;
2827 for (i
= section
->pin_queue_start
; i
< section
->pin_queue_end
; ++i
) {
2828 frag_end
= pin_queue
[i
];
2829 /* remove the pin bit from pinned objects */
2830 unpin_object (frag_end
);
2831 if (frag_end
>= section
->data
+ section
->size
) {
2832 frag_end
= section
->data
+ section
->size
;
2834 section
->scan_starts
[((char*)frag_end
- (char*)section
->data
)/SCAN_START_SIZE
] = frag_end
;
2836 frag_size
= frag_end
- frag_start
;
2838 memset (frag_start
, 0, frag_size
);
2839 frag_size
= safe_object_get_size ((MonoObject
*)pin_queue
[i
]);
2840 frag_size
+= ALLOC_ALIGN
- 1;
2841 frag_size
&= ~(ALLOC_ALIGN
- 1);
2842 frag_start
= (char*)pin_queue
[i
] + frag_size
;
2843 section
->next_data
= MAX (section
->next_data
, frag_start
);
2845 shorten_section (section
, frag_start
- section
->data
);
2846 frag_end
= section
->end_data
;
2847 frag_size
= frag_end
- frag_start
;
2849 memset (frag_start
, 0, frag_size
);
2853 scan_from_registered_roots (char *addr_start
, char *addr_end
, int root_type
)
2857 for (i
= 0; i
< roots_hash_size
[root_type
]; ++i
) {
2858 for (root
= roots_hash
[root_type
][i
]; root
; root
= root
->next
) {
2859 DEBUG (6, fprintf (gc_debug_file
, "Precise root scan %p-%p (desc: %p)\n", root
->start_root
, root
->end_root
, (void*)root
->root_desc
));
2860 precisely_scan_objects_from ((void**)root
->start_root
, (void**)root
->end_root
, addr_start
, addr_end
, root
->root_desc
);
2866 dump_occupied (char *start
, char *end
, char *section_start
)
2868 fprintf (heap_dump_file
, "<occupied offset=\"%d\" size=\"%d\"/>\n", start
- section_start
, end
- start
);
2872 dump_section (GCMemSection
*section
, const char *type
)
2874 char *start
= section
->data
;
2875 char *end
= section
->data
+ section
->size
;
2876 char *occ_start
= NULL
;
2879 char *old_start
= NULL
; /* just for debugging */
2881 fprintf (heap_dump_file
, "<section type=\"%s\" size=\"%d\">\n", type
, section
->size
);
2883 while (start
< end
) {
2887 if (!*(void**)start
) {
2889 dump_occupied (occ_start
, start
, section
->data
);
2892 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
2895 g_assert (start
< section
->next_data
);
2900 vt
= (GCVTable
*)LOAD_VTABLE (start
);
2903 size
= safe_object_get_size ((MonoObject
*) start
);
2904 size
+= ALLOC_ALIGN
- 1;
2905 size
&= ~(ALLOC_ALIGN
- 1);
2908 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2909 start - section->data,
2910 vt->klass->name_space, vt->klass->name,
2918 dump_occupied (occ_start
, start
, section
->data
);
2920 fprintf (heap_dump_file
, "</section>\n");
2924 dump_heap (const char *type
, int num
, const char *reason
)
2926 GCMemSection
*section
;
2929 fprintf (heap_dump_file
, "<collection type=\"%s\" num=\"%d\"", type
, num
);
2931 fprintf (heap_dump_file
, " reason=\"%s\"", reason
);
2932 fprintf (heap_dump_file
, ">\n");
2934 dump_section (nursery_section
, "nursery");
2936 for (section
= section_list
; section
; section
= section
->next
) {
2937 if (section
!= nursery_section
)
2938 dump_section (section
, "old");
2941 fprintf (heap_dump_file
, "<los>\n");
2942 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
) {
2943 MonoObject
*obj
= (MonoObject
*) bigobj
->data
;
2944 MonoClass
*class = mono_object_class (obj
);
2946 fprintf (heap_dump_file
, "<object class=\"%s.%s\" size=\"%d\"/>\n",
2947 class->name_space
, class->name
,
2948 safe_object_get_size (obj
));
2950 fprintf (heap_dump_file
, "</los>\n");
2952 fprintf (heap_dump_file
, "</collection>\n");
2956 * Collect objects in the nursery. Returns whether to trigger a major
2960 collect_nursery (size_t requested_size
)
2962 GCMemSection
*section
;
2963 size_t max_garbage_amount
;
2965 char *orig_nursery_next
;
2967 gboolean invoke_major_gc
= FALSE
;
2968 TV_DECLARE (all_atv
);
2969 TV_DECLARE (all_btv
);
2974 orig_nursery_next
= nursery_next
;
2975 nursery_next
= MAX (nursery_next
, nursery_last_pinned_end
);
2976 /* FIXME: optimize later to use the higher address where an object can be present */
2977 nursery_next
= MAX (nursery_next
, nursery_real_end
);
2979 if (consistency_check_at_minor_collection
)
2980 check_consistency ();
2982 DEBUG (1, fprintf (gc_debug_file
, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs
, nursery_start
, nursery_next
, (int)(nursery_next
- nursery_start
)));
2983 max_garbage_amount
= nursery_next
- nursery_start
;
2985 /* Clear all remaining nursery fragments, pinning depends on this */
2986 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
2987 g_assert (orig_nursery_next
<= nursery_frag_real_end
);
2988 memset (orig_nursery_next
, 0, nursery_frag_real_end
- orig_nursery_next
);
2989 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
2990 memset (frag
->fragment_start
, 0, frag
->fragment_end
- frag
->fragment_start
);
2995 check_for_xdomain_refs ();
2998 * not enough room in the old generation to store all the possible data from
2999 * the nursery in a single continuous space.
3000 * We reset to_space if we allocated objects in degraded mode.
3002 if (to_space_section
)
3003 to_space
= gray_objects
= gray_first
= to_space_section
->next_data
;
3004 if ((to_space_end
- to_space
) < max_garbage_amount
) {
3005 section
= alloc_section (nursery_section
->size
* 4);
3006 g_assert (nursery_section
->size
>= max_garbage_amount
);
3007 to_space
= gray_objects
= gray_first
= section
->next_data
;
3008 to_space_end
= section
->end_data
;
3009 to_space_section
= section
;
3010 invoke_major_gc
= TRUE
;
3012 DEBUG (2, fprintf (gc_debug_file
, "To space setup: %p-%p in section %p\n", to_space
, to_space_end
, to_space_section
));
3013 nursery_section
->next_data
= nursery_next
;
3016 mono_stats
.minor_gc_count
++;
3017 /* world must be stopped already */
3018 TV_GETTIME (all_atv
);
3020 /* pin from pinned handles */
3021 pin_from_roots (nursery_start
, nursery_next
);
3022 /* identify pinned objects */
3023 optimize_pin_queue (0);
3024 next_pin_slot
= pin_objects_from_addresses (nursery_section
, pin_queue
, pin_queue
+ next_pin_slot
, nursery_start
, nursery_next
);
3026 DEBUG (2, fprintf (gc_debug_file
, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot
, TV_ELAPSED (atv
, btv
)));
3027 DEBUG (4, fprintf (gc_debug_file
, "Start scan with %d pinned objects\n", next_pin_slot
));
3030 * walk all the roots and copy the young objects to the old generation,
3031 * starting from to_space
3034 scan_from_remsets (nursery_start
, nursery_next
);
3035 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3037 DEBUG (2, fprintf (gc_debug_file
, "Old generation scan: %d usecs\n", TV_ELAPSED (btv
, atv
)));
3039 /* the pinned objects are roots */
3040 for (i
= 0; i
< next_pin_slot
; ++i
) {
3041 DEBUG (6, fprintf (gc_debug_file
, "Precise object scan %d of pinned %p (%s)\n", i
, pin_queue
[i
], safe_name (pin_queue
[i
])));
3042 scan_object (pin_queue
[i
], nursery_start
, nursery_next
);
3044 /* registered roots, this includes static fields */
3045 scan_from_registered_roots (nursery_start
, nursery_next
, ROOT_TYPE_NORMAL
);
3046 scan_from_registered_roots (nursery_start
, nursery_next
, ROOT_TYPE_WBARRIER
);
3047 scan_thread_data (nursery_start
, nursery_next
, TRUE
);
3048 /* alloc_pinned objects */
3049 scan_from_pinned_objects (nursery_start
, nursery_next
);
3051 DEBUG (2, fprintf (gc_debug_file
, "Root scan: %d usecs\n", TV_ELAPSED (atv
, btv
)));
3053 finish_gray_stack (nursery_start
, nursery_next
, GENERATION_NURSERY
);
3055 /* walk the pin_queue, build up the fragment list of free memory, unmark
3056 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3059 build_nursery_fragments (0, next_pin_slot
);
3061 DEBUG (2, fprintf (gc_debug_file
, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv
, atv
), fragment_total
));
3063 TV_GETTIME (all_btv
);
3064 mono_stats
.minor_gc_time_usecs
+= TV_ELAPSED (all_atv
, all_btv
);
3067 dump_heap ("minor", num_minor_gcs
- 1, NULL
);
3069 /* prepare the pin queue for the next collection */
3070 last_num_pinned
= next_pin_slot
;
3072 if (fin_ready_list
|| critical_fin_list
) {
3073 DEBUG (4, fprintf (gc_debug_file
, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers
));
3074 mono_gc_finalize_notify ();
3077 return invoke_major_gc
;
3081 * After major collections we try to shorten the to-space so as to
3082 * avoid the next to-space to be even bigger. We are still left with
3083 * mostly empty sections which only contain pinned objects. We should
3084 * re-fill those first when we do a major collection and only allocate
3085 * a new one if they are all full. Whether or not we need to shorten
3086 * that afterwards remains to be seen.
3089 shorten_to_space (mword size
)
3091 g_assert (to_space_end
== to_space_section
->end_data
);
3093 if (to_space_section
->next_data
- to_space_section
->data
>= size
)
3096 shorten_section (to_space_section
, size
);
3098 to_space_end
= to_space_section
->end_data
;
3104 major_collection (const char *reason
)
3106 GCMemSection
*section
, *prev_section
;
3107 LOSObject
*bigobj
, *prevbo
;
3112 TV_DECLARE (all_atv
);
3113 TV_DECLARE (all_btv
);
3116 /* FIXME: only use these values for the precise scan
3117 * note that to_space pointers should be excluded anyway...
3119 char *heap_start
= NULL
;
3120 char *heap_end
= (char*)-1;
3121 size_t copy_space_required
= 0;
3124 DEBUG (1, fprintf (gc_debug_file
, "Start major collection %d\n", num_major_gcs
));
3126 mono_stats
.major_gc_count
++;
3128 /* Clear all remaining nursery fragments, pinning depends on this */
3129 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
3130 g_assert (nursery_next
<= nursery_frag_real_end
);
3131 memset (nursery_next
, 0, nursery_frag_real_end
- nursery_next
);
3132 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
3133 memset (frag
->fragment_start
, 0, frag
->fragment_end
- frag
->fragment_start
);
3138 check_for_xdomain_refs ();
3141 * FIXME: implement Mark/Compact
3142 * Until that is done, we can just apply mostly the same alg as for the nursery:
3143 * this means we need a big section to potentially copy all the other sections, so
3144 * it is not ideal specially with large heaps.
3146 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3147 collect_nursery (0);
3150 TV_GETTIME (all_atv
);
3151 /* FIXME: make sure the nursery next_data ptr is updated */
3152 nursery_section
->next_data
= nursery_real_end
;
3153 /* we should also coalesce scanning from sections close to each other
3154 * and deal with pointers outside of the sections later.
3156 /* The remsets are not useful for a major collection */
3158 /* world must be stopped already */
3160 DEBUG (6, fprintf (gc_debug_file
, "Pinning from sections\n"));
3161 for (section
= section_list
; section
; section
= section
->next
) {
3162 section
->pin_queue_start
= count
= section
->pin_queue_end
= next_pin_slot
;
3163 pin_from_roots (section
->data
, section
->next_data
);
3164 if (count
!= next_pin_slot
) {
3166 optimize_pin_queue (count
);
3167 DEBUG (6, fprintf (gc_debug_file
, "Found %d pinning addresses in section %p (%d-%d)\n", next_pin_slot
- count
, section
, count
, next_pin_slot
));
3168 reduced_to
= pin_objects_from_addresses (section
, pin_queue
+ count
, pin_queue
+ next_pin_slot
, section
->data
, section
->next_data
);
3169 section
->pin_queue_end
= next_pin_slot
= count
+ reduced_to
;
3171 copy_space_required
+= (char*)section
->next_data
- (char*)section
->data
;
3173 /* identify possible pointers to the insize of large objects */
3174 DEBUG (6, fprintf (gc_debug_file
, "Pinning from large objects\n"));
3175 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
) {
3176 count
= next_pin_slot
;
3177 pin_from_roots (bigobj
->data
, (char*)bigobj
->data
+ bigobj
->size
);
3178 /* FIXME: this is only valid until we don't optimize the pin queue midway */
3179 if (next_pin_slot
!= count
) {
3180 next_pin_slot
= count
;
3181 pin_object (bigobj
->data
);
3182 DEBUG (6, fprintf (gc_debug_file
, "Marked large object %p (%s) size: %zd from roots\n", bigobj
->data
, safe_name (bigobj
->data
), bigobj
->size
));
3185 /* look for pinned addresses for pinned-alloc objects */
3186 DEBUG (6, fprintf (gc_debug_file
, "Pinning from pinned-alloc objects\n"));
3187 for (chunk
= pinned_chunk_list
; chunk
; chunk
= chunk
->next
) {
3188 count
= next_pin_slot
;
3189 pin_from_roots (chunk
->start_data
, (char*)chunk
+ chunk
->num_pages
* FREELIST_PAGESIZE
);
3190 /* FIXME: this is only valid until we don't optimize the pin queue midway */
3191 if (next_pin_slot
!= count
) {
3192 mark_pinned_from_addresses (chunk
, pin_queue
+ count
, pin_queue
+ next_pin_slot
);
3193 next_pin_slot
= count
;
3198 DEBUG (2, fprintf (gc_debug_file
, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot
, TV_ELAPSED (atv
, btv
)));
3199 DEBUG (4, fprintf (gc_debug_file
, "Start scan with %d pinned objects\n", next_pin_slot
));
3201 /* allocate the big to space */
3202 DEBUG (4, fprintf (gc_debug_file
, "Allocate tospace for size: %zd\n", copy_space_required
));
3203 section
= alloc_section (copy_space_required
);
3204 to_space
= gray_objects
= gray_first
= section
->next_data
;
3205 to_space_end
= section
->end_data
;
3206 to_space_section
= section
;
3208 /* the old generation doesn't need to be scanned (no remembered sets or card
3209 * table needed either): the only objects that must survive are those pinned and
3210 * those referenced by the precise roots.
3211 * mark any section without pinned objects, so we can free it since we will be able to
3212 * move all the objects.
3214 /* the pinned objects are roots (big objects are included in this list, too) */
3215 for (i
= 0; i
< next_pin_slot
; ++i
) {
3216 DEBUG (6, fprintf (gc_debug_file
, "Precise object scan %d of pinned %p (%s)\n", i
, pin_queue
[i
], safe_name (pin_queue
[i
])));
3217 scan_object (pin_queue
[i
], heap_start
, heap_end
);
3219 /* registered roots, this includes static fields */
3220 scan_from_registered_roots (heap_start
, heap_end
, ROOT_TYPE_NORMAL
);
3221 scan_from_registered_roots (heap_start
, heap_end
, ROOT_TYPE_WBARRIER
);
3223 scan_thread_data (heap_start
, heap_end
, TRUE
);
3224 /* alloc_pinned objects */
3225 scan_from_pinned_objects (heap_start
, heap_end
);
3226 /* scan the list of objects ready for finalization */
3227 scan_finalizer_entries (fin_ready_list
, heap_start
, heap_end
);
3228 scan_finalizer_entries (critical_fin_list
, heap_start
, heap_end
);
3230 DEBUG (2, fprintf (gc_debug_file
, "Root scan: %d usecs\n", TV_ELAPSED (btv
, atv
)));
3232 /* we need to go over the big object list to see if any was marked and scan it
3233 * And we need to make this in a loop, considering that objects referenced by finalizable
3234 * objects could reference big objects (this happens in finish_gray_stack ())
3236 scan_needed_big_objects (heap_start
, heap_end
);
3237 /* all the objects in the heap */
3238 finish_gray_stack (heap_start
, heap_end
, GENERATION_OLD
);
3240 /* sweep the big objects list */
3242 for (bigobj
= los_object_list
; bigobj
;) {
3243 if (object_is_pinned (bigobj
->data
)) {
3244 unpin_object (bigobj
->data
);
3245 bigobj
->scanned
= FALSE
;
3248 /* not referenced anywhere, so we can free it */
3250 prevbo
->next
= bigobj
->next
;
3252 los_object_list
= bigobj
->next
;
3254 bigobj
= bigobj
->next
;
3255 free_large_object (to_free
);
3259 bigobj
= bigobj
->next
;
3261 /* unpin objects from the pinned chunks and free the unmarked ones */
3262 sweep_pinned_objects ();
3264 /* free the unused sections */
3265 prev_section
= NULL
;
3266 for (section
= section_list
; section
;) {
3267 /* to_space doesn't need handling here and the nursery is special */
3268 if (section
== to_space_section
|| section
== nursery_section
) {
3269 prev_section
= section
;
3270 section
= section
->next
;
3273 /* no pinning object, so the section is free */
3274 if (section
->pin_queue_start
== section
->pin_queue_end
) {
3275 GCMemSection
*to_free
;
3277 prev_section
->next
= section
->next
;
3279 section_list
= section
->next
;
3281 section
= section
->next
;
3282 free_mem_section (to_free
);
3285 DEBUG (6, fprintf (gc_debug_file
, "Section %p has still pinned objects (%d)\n", section
, section
->pin_queue_end
- section
->pin_queue_start
));
3286 build_section_fragments (section
);
3288 prev_section
= section
;
3289 section
= section
->next
;
3292 /* walk the pin_queue, build up the fragment list of free memory, unmark
3293 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3296 build_nursery_fragments (nursery_section
->pin_queue_start
, nursery_section
->pin_queue_end
);
3298 /* One fourth the size of the space originally allocated is a
3299 * figure arrived after a bit of trial and error. We might
3300 * need to change this to something else, maybe even adjust it
3303 shorten_to_space (copy_space_required
/ 4);
3305 TV_GETTIME (all_btv
);
3306 mono_stats
.major_gc_time_usecs
+= TV_ELAPSED (all_atv
, all_btv
);
3309 dump_heap ("major", num_major_gcs
- 1, reason
);
3311 /* prepare the pin queue for the next collection */
3313 if (fin_ready_list
|| critical_fin_list
) {
3314 DEBUG (4, fprintf (gc_debug_file
, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers
));
3315 mono_gc_finalize_notify ();
3320 * Allocate a new section of memory to be used as old generation.
3322 static GCMemSection
*
3323 alloc_section (size_t size
)
3325 GCMemSection
*section
;
3328 size_t new_size
= next_section_size
;
3330 if (size
> next_section_size
) {
3332 new_size
+= pagesize
- 1;
3333 new_size
&= ~(pagesize
- 1);
3335 section_size_used
++;
3336 if (section_size_used
> 3) {
3337 section_size_used
= 0;
3338 next_section_size
*= 2;
3339 if (next_section_size
> max_section_size
)
3340 next_section_size
= max_section_size
;
3342 section
= get_internal_mem (sizeof (GCMemSection
));
3343 data
= get_os_memory (new_size
, TRUE
);
3344 section
->data
= section
->next_data
= data
;
3345 section
->size
= new_size
;
3346 section
->end_data
= data
+ new_size
;
3347 UPDATE_HEAP_BOUNDARIES (data
, section
->end_data
);
3348 total_alloc
+= new_size
;
3349 DEBUG (2, fprintf (gc_debug_file
, "Expanding heap size: %zd (%p-%p), total: %zd\n", new_size
, data
, data
+ new_size
, total_alloc
));
3350 section
->data
= data
;
3351 section
->size
= new_size
;
3352 scan_starts
= new_size
/ SCAN_START_SIZE
;
3353 section
->scan_starts
= get_internal_mem (sizeof (char*) * scan_starts
);
3354 section
->num_scan_start
= scan_starts
;
3355 section
->role
= MEMORY_ROLE_GEN1
;
3357 /* add to the section list */
3358 section
->next
= section_list
;
3359 section_list
= section
;
3365 free_mem_section (GCMemSection
*section
)
3367 char *data
= section
->data
;
3368 size_t size
= section
->size
;
3369 DEBUG (2, fprintf (gc_debug_file
, "Freed section %p, size %zd\n", data
, size
));
3370 free_os_memory (data
, size
);
3371 free_internal_mem (section
);
3372 total_alloc
-= size
;
3376 * When deciding if it's better to collect or to expand, keep track
3377 * of how much garbage was reclaimed with the last collection: if it's too
3379 * This is called when we could not allocate a small object.
3381 static void __attribute__((noinline
))
3382 minor_collect_or_expand_inner (size_t size
)
3384 int do_minor_collection
= 1;
3386 if (!nursery_section
) {
3390 if (do_minor_collection
) {
3392 if (collect_nursery (size
))
3393 major_collection ("minor overflow");
3394 DEBUG (2, fprintf (gc_debug_file
, "Heap size: %zd, LOS size: %zd\n", total_alloc
, los_memory_usage
));
3396 /* this also sets the proper pointers for the next allocation */
3397 if (!search_fragment_for_size (size
)) {
3399 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3400 DEBUG (1, fprintf (gc_debug_file
, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size
, last_num_pinned
));
3401 for (i
= 0; i
< last_num_pinned
; ++i
) {
3402 DEBUG (3, fprintf (gc_debug_file
, "Bastard pinning obj %p (%s), size: %d\n", pin_queue
[i
], safe_name (pin_queue
[i
]), safe_object_get_size (pin_queue
[i
])));
3407 //report_internal_mem_usage ();
3411 * ######################################################################
3412 * ######## Memory allocation from the OS
3413 * ######################################################################
3414 * This section of code deals with getting memory from the OS and
3415 * allocating memory for GC-internal data structures.
3416 * Internal memory can be handled with a freelist for small objects.
3420 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3421 * This must not require any lock.
3424 get_os_memory (size_t size
, int activate
)
3427 unsigned long prot_flags
= activate
? MONO_MMAP_READ
|MONO_MMAP_WRITE
: MONO_MMAP_NONE
;
3429 prot_flags
|= MONO_MMAP_PRIVATE
| MONO_MMAP_ANON
;
3430 size
+= pagesize
- 1;
3431 size
&= ~(pagesize
- 1);
3432 ptr
= mono_valloc (0, size
, prot_flags
);
3437 * Free the memory returned by get_os_memory (), returning it to the OS.
3440 free_os_memory (void *addr
, size_t size
)
3442 munmap (addr
, size
);
3449 report_pinned_chunk (PinnedChunk
*chunk
, int seq
) {
3451 int i
, free_pages
, num_free
, free_mem
;
3453 for (i
= 0; i
< chunk
->num_pages
; ++i
) {
3454 if (!chunk
->page_sizes
[i
])
3457 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq
, chunk
, chunk
->num_pages
* FREELIST_PAGESIZE
, chunk
->num_pages
, free_pages
);
3458 free_mem
= FREELIST_PAGESIZE
* free_pages
;
3459 for (i
= 0; i
< FREELIST_NUM_SLOTS
; ++i
) {
3460 if (!chunk
->free_list
[i
])
3463 p
= chunk
->free_list
[i
];
3468 printf ("\tfree list of size %d, %d items\n", freelist_sizes
[i
], num_free
);
3469 free_mem
+= freelist_sizes
[i
] * num_free
;
3471 printf ("\tfree memory in chunk: %d\n", free_mem
);
3477 static G_GNUC_UNUSED
void
3478 report_internal_mem_usage (void) {
3481 printf ("Internal memory usage:\n");
3483 for (chunk
= internal_chunk_list
; chunk
; chunk
= chunk
->next
) {
3484 report_pinned_chunk (chunk
, i
++);
3486 printf ("Pinned memory usage:\n");
3488 for (chunk
= pinned_chunk_list
; chunk
; chunk
= chunk
->next
) {
3489 report_pinned_chunk (chunk
, i
++);
3494 * the array of pointers from @start to @end contains conservative
3495 * pointers to objects inside @chunk: mark each referenced object
3499 mark_pinned_from_addresses (PinnedChunk
*chunk
, void **start
, void **end
)
3501 for (; start
< end
; start
++) {
3502 char *addr
= *start
;
3503 int offset
= (char*)addr
- (char*)chunk
;
3504 int page
= offset
/ FREELIST_PAGESIZE
;
3505 int obj_offset
= page
== 0? offset
- ((char*)chunk
->start_data
- (char*)chunk
): offset
% FREELIST_PAGESIZE
;
3506 int slot_size
= chunk
->page_sizes
[page
];
3508 /* the page is not allocated */
3511 /* would be faster if we restrict the sizes to power of two,
3512 * but that's a waste of memory: need to measure. it could reduce
3513 * fragmentation since there are less pages needed, if for example
3514 * someone interns strings of each size we end up with one page per
3515 * interned string (still this is just ~40 KB): with more fine-grained sizes
3516 * this increases the number of used pages.
3519 obj_offset
/= slot_size
;
3520 obj_offset
*= slot_size
;
3521 addr
= (char*)chunk
->start_data
+ obj_offset
;
3523 obj_offset
/= slot_size
;
3524 obj_offset
*= slot_size
;
3525 addr
= (char*)chunk
+ page
* FREELIST_PAGESIZE
+ obj_offset
;
3528 /* if the vtable is inside the chunk it's on the freelist, so skip */
3529 if (*ptr
&& (*ptr
< (void*)chunk
->start_data
|| *ptr
> (void*)((char*)chunk
+ chunk
->num_pages
* FREELIST_PAGESIZE
))) {
3531 DEBUG (6, fprintf (gc_debug_file
, "Marked pinned object %p (%s) from roots\n", addr
, safe_name (addr
)));
3537 scan_pinned_objects (ScanPinnedObjectCallbackFunc callback
, void *callback_data
)
3544 for (chunk
= pinned_chunk_list
; chunk
; chunk
= chunk
->next
) {
3545 end_chunk
= (char*)chunk
+ chunk
->num_pages
* FREELIST_PAGESIZE
;
3546 DEBUG (6, fprintf (gc_debug_file
, "Scanning pinned chunk %p (range: %p-%p)\n", chunk
, chunk
->start_data
, end_chunk
));
3547 for (i
= 0; i
< chunk
->num_pages
; ++i
) {
3548 obj_size
= chunk
->page_sizes
[i
];
3551 p
= i
? (char*)chunk
+ i
* FREELIST_PAGESIZE
: chunk
->start_data
;
3552 endp
= i
? p
+ FREELIST_PAGESIZE
: (char*)chunk
+ FREELIST_PAGESIZE
;
3553 DEBUG (6, fprintf (gc_debug_file
, "Page %d (size: %d, range: %p-%p)\n", i
, obj_size
, p
, endp
));
3554 while (p
+ obj_size
<= endp
) {
3556 DEBUG (9, fprintf (gc_debug_file
, "Considering %p (vtable: %p)\n", ptr
, *ptr
));
3557 /* if the first word (the vtable) is outside the chunk we have an object */
3558 if (*ptr
&& (*ptr
< (void*)chunk
|| *ptr
>= end_chunk
))
3559 callback (chunk
, (char*)ptr
, obj_size
, callback_data
);
3567 sweep_pinned_objects_callback (PinnedChunk
*chunk
, char *ptr
, size_t size
, void *data
)
3569 if (object_is_pinned (ptr
)) {
3571 DEBUG (6, fprintf (gc_debug_file
, "Unmarked pinned object %p (%s)\n", ptr
, safe_name (ptr
)));
3573 DEBUG (6, fprintf (gc_debug_file
, "Freeing unmarked pinned object %p (%s)\n", ptr
, safe_name (ptr
)));
3574 free_pinned_object (chunk
, ptr
, size
);
3579 sweep_pinned_objects (void)
3581 scan_pinned_objects (sweep_pinned_objects_callback
, NULL
);
3585 scan_object_callback (PinnedChunk
*chunk
, char *ptr
, size_t size
, char **data
)
3587 DEBUG (6, fprintf (gc_debug_file
, "Precise object scan of alloc_pinned %p (%s)\n", ptr
, safe_name (ptr
)));
3588 /* FIXME: Put objects without references into separate chunks
3589 which do not need to be scanned */
3590 scan_object (ptr
, data
[0], data
[1]);
3594 scan_from_pinned_objects (char *addr_start
, char *addr_end
)
3596 char *data
[2] = { addr_start
, addr_end
};
3597 scan_pinned_objects ((ScanPinnedObjectCallbackFunc
)scan_object_callback
, data
);
3601 * Find the slot number in the freelist for memory chunks that
3602 * can contain @size objects.
3605 slot_for_size (size_t size
)
3608 /* do a binary search or lookup table later. */
3609 for (slot
= 0; slot
< FREELIST_NUM_SLOTS
; ++slot
) {
3610 if (freelist_sizes
[slot
] >= size
)
3613 g_assert_not_reached ();
3618 * Build a free list for @size memory chunks from the memory area between
3619 * start_page and end_page.
3622 build_freelist (PinnedChunk
*chunk
, int slot
, int size
, char *start_page
, char *end_page
)
3626 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3627 p
= (void**)start_page
;
3628 end
= (void**)(end_page
- size
);
3629 g_assert (!chunk
->free_list
[slot
]);
3630 chunk
->free_list
[slot
] = p
;
3631 while ((char*)p
+ size
<= (char*)end
) {
3633 *p
= (void*)((char*)p
+ size
);
3637 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3641 alloc_pinned_chunk (size_t size
)
3646 size
+= pagesize
; /* at least one page */
3647 size
+= pagesize
- 1;
3648 size
&= ~(pagesize
- 1);
3649 if (size
< PINNED_CHUNK_MIN_SIZE
* 2)
3650 size
= PINNED_CHUNK_MIN_SIZE
* 2;
3651 chunk
= get_os_memory (size
, TRUE
);
3652 UPDATE_HEAP_BOUNDARIES (chunk
, ((char*)chunk
+ size
));
3653 total_alloc
+= size
;
3655 /* setup the bookeeping fields */
3656 chunk
->num_pages
= size
/ FREELIST_PAGESIZE
;
3657 offset
= G_STRUCT_OFFSET (PinnedChunk
, data
);
3658 chunk
->page_sizes
= (void*)((char*)chunk
+ offset
);
3659 offset
+= sizeof (int) * chunk
->num_pages
;
3660 offset
+= ALLOC_ALIGN
- 1;
3661 offset
&= ~(ALLOC_ALIGN
- 1);
3662 chunk
->free_list
= (void*)((char*)chunk
+ offset
);
3663 offset
+= sizeof (void*) * FREELIST_NUM_SLOTS
;
3664 offset
+= ALLOC_ALIGN
- 1;
3665 offset
&= ~(ALLOC_ALIGN
- 1);
3666 chunk
->start_data
= (void*)((char*)chunk
+ offset
);
3668 /* allocate the first page to the freelist */
3669 chunk
->page_sizes
[0] = PINNED_FIRST_SLOT_SIZE
;
3670 build_freelist (chunk
, slot_for_size (PINNED_FIRST_SLOT_SIZE
), PINNED_FIRST_SLOT_SIZE
, chunk
->start_data
, ((char*)chunk
+ FREELIST_PAGESIZE
));
3671 DEBUG (4, fprintf (gc_debug_file
, "Allocated pinned chunk %p, size: %zd\n", chunk
, size
));
3672 min_pinned_chunk_addr
= MIN (min_pinned_chunk_addr
, (char*)chunk
->start_data
);
3673 max_pinned_chunk_addr
= MAX (max_pinned_chunk_addr
, ((char*)chunk
+ size
));
3677 /* assumes freelist for slot is empty, so try to alloc a new page */
3679 get_chunk_freelist (PinnedChunk
*chunk
, int slot
)
3683 p
= chunk
->free_list
[slot
];
3685 chunk
->free_list
[slot
] = *p
;
3688 for (i
= 0; i
< chunk
->num_pages
; ++i
) {
3690 if (chunk
->page_sizes
[i
])
3692 size
= freelist_sizes
[slot
];
3693 chunk
->page_sizes
[i
] = size
;
3694 build_freelist (chunk
, slot
, size
, (char*)chunk
+ FREELIST_PAGESIZE
* i
, (char*)chunk
+ FREELIST_PAGESIZE
* (i
+ 1));
3698 p
= chunk
->free_list
[slot
];
3700 chunk
->free_list
[slot
] = *p
;
3707 alloc_from_freelist (size_t size
)
3711 PinnedChunk
*pchunk
;
3712 slot
= slot_for_size (size
);
3713 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
3714 g_assert (size
<= freelist_sizes
[slot
]);
3715 for (pchunk
= pinned_chunk_list
; pchunk
; pchunk
= pchunk
->next
) {
3716 void **p
= pchunk
->free_list
[slot
];
3718 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
3719 pchunk
->free_list
[slot
] = *p
;
3723 for (pchunk
= pinned_chunk_list
; pchunk
; pchunk
= pchunk
->next
) {
3724 res
= get_chunk_freelist (pchunk
, slot
);
3728 pchunk
= alloc_pinned_chunk (size
);
3729 /* FIXME: handle OOM */
3730 pchunk
->next
= pinned_chunk_list
;
3731 pinned_chunk_list
= pchunk
;
3732 res
= get_chunk_freelist (pchunk
, slot
);
3736 /* used for the GC-internal data structures */
3737 /* FIXME: add support for bigger sizes by allocating more than one page
3741 get_internal_mem (size_t size
)
3745 PinnedChunk
*pchunk
;
3747 if (size
> freelist_sizes
[FREELIST_NUM_SLOTS
- 1]) {
3748 LargeInternalMemHeader
*mh
;
3750 size
+= sizeof (LargeInternalMemHeader
);
3751 mh
= get_os_memory (size
, TRUE
);
3752 mh
->magic
= LARGE_INTERNAL_MEM_HEADER_MAGIC
;
3757 slot
= slot_for_size (size
);
3758 g_assert (size
<= freelist_sizes
[slot
]);
3759 for (pchunk
= internal_chunk_list
; pchunk
; pchunk
= pchunk
->next
) {
3760 void **p
= pchunk
->free_list
[slot
];
3762 pchunk
->free_list
[slot
] = *p
;
3763 memset (p
, 0, size
);
3767 for (pchunk
= internal_chunk_list
; pchunk
; pchunk
= pchunk
->next
) {
3768 res
= get_chunk_freelist (pchunk
, slot
);
3770 memset (res
, 0, size
);
3774 pchunk
= alloc_pinned_chunk (size
);
3775 /* FIXME: handle OOM */
3776 pchunk
->next
= internal_chunk_list
;
3777 internal_chunk_list
= pchunk
;
3778 res
= get_chunk_freelist (pchunk
, slot
);
3779 memset (res
, 0, size
);
3784 free_internal_mem (void *addr
)
3786 PinnedChunk
*pchunk
;
3787 LargeInternalMemHeader
*mh
;
3790 for (pchunk
= internal_chunk_list
; pchunk
; pchunk
= pchunk
->next
) {
3791 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3792 if (addr
>= (void*)pchunk
&& (char*)addr
< (char*)pchunk
+ pchunk
->num_pages
* FREELIST_PAGESIZE
) {
3793 int offset
= (char*)addr
- (char*)pchunk
;
3794 int page
= offset
/ FREELIST_PAGESIZE
;
3795 int slot
= slot_for_size (pchunk
->page_sizes
[page
]);
3797 *p
= pchunk
->free_list
[slot
];
3798 pchunk
->free_list
[slot
] = p
;
3802 mh
= (LargeInternalMemHeader
*)((char*)addr
- G_STRUCT_OFFSET (LargeInternalMemHeader
, data
));
3803 g_assert (mh
->magic
== LARGE_INTERNAL_MEM_HEADER_MAGIC
);
3804 free_os_memory (mh
, mh
->size
);
3808 * ######################################################################
3809 * ######## Object allocation
3810 * ######################################################################
3811 * This section of code deals with allocating memory for objects.
3812 * There are several ways:
3813 * *) allocate large objects
3814 * *) allocate normal objects
3815 * *) fast lock-free allocation
3816 * *) allocation of pinned objects
3820 free_large_object (LOSObject
*obj
)
3822 size_t size
= obj
->size
;
3823 DEBUG (4, fprintf (gc_debug_file
, "Freed large object %p, size %zd\n", obj
->data
, obj
->size
));
3825 los_memory_usage
-= size
;
3826 size
+= sizeof (LOSObject
);
3827 size
+= pagesize
- 1;
3828 size
&= ~(pagesize
- 1);
3829 total_alloc
-= size
;
3831 free_os_memory (obj
, size
);
3835 * Objects with size >= 64KB are allocated in the large object space.
3836 * They are currently kept track of with a linked list.
3837 * They don't move, so there is no need to pin them during collection
3838 * and we avoid the memcpy overhead.
3840 static void* __attribute__((noinline
))
3841 alloc_large_inner (MonoVTable
*vtable
, size_t size
)
3846 int just_did_major_gc
= FALSE
;
3848 if (los_memory_usage
> next_los_collection
) {
3849 DEBUG (4, fprintf (gc_debug_file
, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size
, los_memory_usage
, next_los_collection
));
3850 just_did_major_gc
= TRUE
;
3852 major_collection ("LOS overflow");
3854 /* later increase based on a percent of the heap size */
3855 next_los_collection
= los_memory_usage
+ 5*1024*1024;
3858 alloc_size
+= sizeof (LOSObject
);
3859 alloc_size
+= pagesize
- 1;
3860 alloc_size
&= ~(pagesize
- 1);
3861 /* FIXME: handle OOM */
3862 obj
= get_os_memory (alloc_size
, TRUE
);
3864 vtslot
= (void**)obj
->data
;
3866 total_alloc
+= alloc_size
;
3867 UPDATE_HEAP_BOUNDARIES (obj
->data
, (char*)obj
->data
+ size
);
3868 obj
->next
= los_object_list
;
3869 los_object_list
= obj
;
3870 los_memory_usage
+= size
;
3872 DEBUG (4, fprintf (gc_debug_file
, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj
->data
, vtable
, vtable
->klass
->name
, size
));
3876 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3877 * an object of size @size
3878 * Return FALSE if not found (which means we need a collection)
3881 search_fragment_for_size (size_t size
)
3883 Fragment
*frag
, *prev
;
3884 DEBUG (4, fprintf (gc_debug_file
, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end
, size
));
3886 if (nursery_frag_real_end
> nursery_next
&& nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
)
3887 /* Clear the remaining space, pinning depends on this */
3888 memset (nursery_next
, 0, nursery_frag_real_end
- nursery_next
);
3891 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
3892 if (size
<= (frag
->fragment_end
- frag
->fragment_start
)) {
3893 /* remove from the list */
3895 prev
->next
= frag
->next
;
3897 nursery_fragments
= frag
->next
;
3898 nursery_next
= frag
->fragment_start
;
3899 nursery_frag_real_end
= frag
->fragment_end
;
3901 DEBUG (4, fprintf (gc_debug_file
, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next
, nursery_frag_real_end
, nursery_frag_real_end
- nursery_next
, size
));
3902 frag
->next
= fragment_freelist
;
3903 fragment_freelist
= frag
;
3912 * size is already rounded up and we hold the GC lock.
3915 alloc_degraded (MonoVTable
*vtable
, size_t size
)
3917 GCMemSection
*section
;
3919 for (section
= section_list
; section
; section
= section
->next
) {
3920 if (section
!= nursery_section
&& (section
->end_data
- section
->next_data
) >= size
) {
3921 p
= (void**)section
->next_data
;
3926 section
= alloc_section (nursery_section
->size
* 4);
3927 /* FIXME: handle OOM */
3928 p
= (void**)section
->next_data
;
3930 section
->next_data
+= size
;
3931 degraded_mode
+= size
;
3932 DEBUG (3, fprintf (gc_debug_file
, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p
, vtable
, vtable
->klass
->name
, size
, section
));
3938 * Provide a variant that takes just the vtable for small fixed-size objects.
3939 * The aligned size is already computed and stored in vt->gc_descr.
3940 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3941 * processing. We can keep track of where objects start, for example,
3942 * so when we scan the thread stacks for pinned objects, we can start
3943 * a search for the pinned object in SCAN_START_SIZE chunks.
3946 mono_gc_alloc_obj_nolock (MonoVTable
*vtable
, size_t size
)
3948 /* FIXME: handle OOM */
3954 size
+= ALLOC_ALIGN
- 1;
3955 size
&= ~(ALLOC_ALIGN
- 1);
3957 g_assert (vtable
->gc_descr
);
3959 if (G_UNLIKELY (collect_before_allocs
)) {
3960 if (nursery_section
) {
3962 collect_nursery (0);
3964 if (!degraded_mode
&& !search_fragment_for_size (size
)) {
3966 g_assert_not_reached ();
3972 * We must already have the lock here instead of after the
3973 * fast path because we might be interrupted in the fast path
3974 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
3975 * and we'll end up allocating an object in a fragment which
3976 * no longer belongs to us.
3978 * The managed allocator does not do this, but it's treated
3979 * specially by the world-stopping code.
3982 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3984 p
= (void**)TLAB_NEXT
;
3985 /* FIXME: handle overflow */
3986 new_next
= (char*)p
+ size
;
3987 TLAB_NEXT
= new_next
;
3989 if (G_LIKELY (new_next
< TLAB_TEMP_END
)) {
3993 * FIXME: We might need a memory barrier here so the change to tlab_next is
3994 * visible before the vtable store.
3997 DEBUG (6, fprintf (gc_debug_file
, "Allocated object %p, vtable: %p (%s), size: %zd\n", p
, vtable
, vtable
->klass
->name
, size
));
3998 g_assert (*p
== NULL
);
4001 g_assert (TLAB_NEXT
== new_next
);
4008 /* there are two cases: the object is too big or we run out of space in the TLAB */
4009 /* we also reach here when the thread does its first allocation after a minor
4010 * collection, since the tlab_ variables are initialized to NULL.
4011 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4012 * objects that need finalizers can have the high bit set in their size
4013 * so the above check fails and we can readily add the object to the queue.
4014 * This avoids taking again the GC lock when registering, but this is moot when
4015 * doing thread-local allocation, so it may not be a good idea.
4017 g_assert (TLAB_NEXT
== new_next
);
4018 if (size
> MAX_SMALL_OBJ_SIZE
) {
4020 p
= alloc_large_inner (vtable
, size
);
4022 if (TLAB_NEXT
>= TLAB_REAL_END
) {
4024 * Run out of space in the TLAB. When this happens, some amount of space
4025 * remains in the TLAB, but not enough to satisfy the current allocation
4026 * request. Currently, we retire the TLAB in all cases, later we could
4027 * keep it if the remaining space is above a treshold, and satisfy the
4028 * allocation directly from the nursery.
4031 /* when running in degraded mode, we continue allocing that way
4032 * for a while, to decrease the number of useless nursery collections.
4034 if (degraded_mode
&& degraded_mode
< DEFAULT_NURSERY_SIZE
) {
4035 p
= alloc_degraded (vtable
, size
);
4039 if (size
> tlab_size
) {
4040 /* Allocate directly from the nursery */
4041 if (nursery_next
+ size
>= nursery_frag_real_end
) {
4042 if (!search_fragment_for_size (size
)) {
4043 minor_collect_or_expand_inner (size
);
4044 if (degraded_mode
) {
4045 p
= alloc_degraded (vtable
, size
);
4051 p
= (void*)nursery_next
;
4052 nursery_next
+= size
;
4053 if (nursery_next
> nursery_frag_real_end
) {
4058 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
)
4059 memset (p
, 0, size
);
4062 DEBUG (3, fprintf (gc_debug_file
, "Retire TLAB: %p-%p [%ld]\n", TLAB_START
, TLAB_REAL_END
, (long)(TLAB_REAL_END
- TLAB_NEXT
- size
)));
4064 if (nursery_next
+ tlab_size
>= nursery_frag_real_end
) {
4065 res
= search_fragment_for_size (tlab_size
);
4067 minor_collect_or_expand_inner (tlab_size
);
4068 if (degraded_mode
) {
4069 p
= alloc_degraded (vtable
, size
);
4075 /* Allocate a new TLAB from the current nursery fragment */
4076 TLAB_START
= nursery_next
;
4077 nursery_next
+= tlab_size
;
4078 TLAB_NEXT
= TLAB_START
;
4079 TLAB_REAL_END
= TLAB_START
+ tlab_size
;
4080 TLAB_TEMP_END
= TLAB_START
+ MIN (SCAN_START_SIZE
, tlab_size
);
4082 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
)
4083 memset (TLAB_START
, 0, tlab_size
);
4085 /* Allocate from the TLAB */
4086 p
= (void*)TLAB_NEXT
;
4088 g_assert (TLAB_NEXT
<= TLAB_REAL_END
);
4090 nursery_section
->scan_starts
[((char*)p
- (char*)nursery_section
->data
)/SCAN_START_SIZE
] = (char*)p
;
4093 /* Reached tlab_temp_end */
4095 /* record the scan start so we can find pinned objects more easily */
4096 nursery_section
->scan_starts
[((char*)p
- (char*)nursery_section
->data
)/SCAN_START_SIZE
] = (char*)p
;
4097 /* we just bump tlab_temp_end as well */
4098 TLAB_TEMP_END
= MIN (TLAB_REAL_END
, TLAB_NEXT
+ SCAN_START_SIZE
);
4099 DEBUG (5, fprintf (gc_debug_file
, "Expanding local alloc: %p-%p\n", TLAB_NEXT
, TLAB_TEMP_END
));
4103 DEBUG (6, fprintf (gc_debug_file
, "Allocated object %p, vtable: %p (%s), size: %zd\n", p
, vtable
, vtable
->klass
->name
, size
));
4110 mono_gc_alloc_obj (MonoVTable
*vtable
, size_t size
)
4114 res
= mono_gc_alloc_obj_nolock (vtable
, size
);
4120 mono_gc_alloc_vector (MonoVTable
*vtable
, size_t size
, mono_array_size_t max_length
)
4126 arr
= mono_gc_alloc_obj_nolock (vtable
, size
);
4127 arr
->max_length
= max_length
;
4135 mono_gc_alloc_array (MonoVTable
*vtable
, size_t size
, mono_array_size_t max_length
, mono_array_size_t bounds_size
)
4138 MonoArrayBounds
*bounds
;
4142 arr
= mono_gc_alloc_obj_nolock (vtable
, size
);
4143 arr
->max_length
= max_length
;
4145 bounds
= (MonoArrayBounds
*)((char*)arr
+ size
- bounds_size
);
4146 arr
->bounds
= bounds
;
4154 * To be used for interned strings and possibly MonoThread, reflection handles.
4155 * We may want to explicitly free these objects.
4158 mono_gc_alloc_pinned_obj (MonoVTable
*vtable
, size_t size
)
4160 /* FIXME: handle OOM */
4162 size
+= ALLOC_ALIGN
- 1;
4163 size
&= ~(ALLOC_ALIGN
- 1);
4165 if (size
> MAX_FREELIST_SIZE
) {
4166 /* large objects are always pinned anyway */
4167 p
= alloc_large_inner (vtable
, size
);
4169 p
= alloc_from_freelist (size
);
4170 memset (p
, 0, size
);
4172 DEBUG (6, fprintf (gc_debug_file
, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p
, vtable
, vtable
->klass
->name
, size
));
4179 * ######################################################################
4180 * ######## Finalization support
4181 * ######################################################################
4185 * this is valid for the nursery: if the object has been forwarded it means it's
4186 * still refrenced from a root. If it is pinned it's still alive as well.
4187 * Return TRUE if @obj is ready to be finalized.
4189 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4192 is_critical_finalizer (FinalizeEntry
*entry
)
4197 if (!mono_defaults
.critical_finalizer_object
)
4200 obj
= entry
->object
;
4201 class = ((MonoVTable
*)LOAD_VTABLE (obj
))->klass
;
4203 return mono_class_has_parent (class, mono_defaults
.critical_finalizer_object
);
4207 queue_finalization_entry (FinalizeEntry
*entry
) {
4208 if (is_critical_finalizer (entry
)) {
4209 entry
->next
= critical_fin_list
;
4210 critical_fin_list
= entry
;
4212 entry
->next
= fin_ready_list
;
4213 fin_ready_list
= entry
;
4217 /* LOCKING: requires that the GC lock is held */
4219 rehash_fin_table (FinalizeEntryHashTable
*hash_table
)
4221 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4222 mword finalizable_hash_size
= hash_table
->size
;
4225 FinalizeEntry
**new_hash
;
4226 FinalizeEntry
*entry
, *next
;
4227 int new_size
= g_spaced_primes_closest (hash_table
->num_registered
);
4229 new_hash
= get_internal_mem (new_size
* sizeof (FinalizeEntry
*));
4230 for (i
= 0; i
< finalizable_hash_size
; ++i
) {
4231 for (entry
= finalizable_hash
[i
]; entry
; entry
= next
) {
4232 hash
= mono_object_hash (entry
->object
) % new_size
;
4234 entry
->next
= new_hash
[hash
];
4235 new_hash
[hash
] = entry
;
4238 free_internal_mem (finalizable_hash
);
4239 hash_table
->table
= new_hash
;
4240 hash_table
->size
= new_size
;
4243 /* LOCKING: requires that the GC lock is held */
4245 rehash_fin_table_if_necessary (FinalizeEntryHashTable
*hash_table
)
4247 if (hash_table
->num_registered
>= hash_table
->size
* 2)
4248 rehash_fin_table (hash_table
);
4251 /* LOCKING: requires that the GC lock is held */
4253 finalize_in_range (char *start
, char *end
, int generation
)
4255 FinalizeEntryHashTable
*hash_table
= get_finalize_entry_hash_table (generation
);
4256 FinalizeEntry
*entry
, *prev
;
4258 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4259 mword finalizable_hash_size
= hash_table
->size
;
4263 for (i
= 0; i
< finalizable_hash_size
; ++i
) {
4265 for (entry
= finalizable_hash
[i
]; entry
;) {
4266 if ((char*)entry
->object
>= start
&& (char*)entry
->object
< end
&& ((char*)entry
->object
< to_space
|| (char*)entry
->object
>= to_space_end
)) {
4267 gboolean is_fin_ready
= object_is_fin_ready (entry
->object
);
4268 char *copy
= copy_object (entry
->object
, start
, end
);
4271 FinalizeEntry
*next
;
4272 /* remove and put in fin_ready_list */
4274 prev
->next
= entry
->next
;
4276 finalizable_hash
[i
] = entry
->next
;
4278 num_ready_finalizers
++;
4279 hash_table
->num_registered
--;
4280 queue_finalization_entry (entry
);
4281 /* Make it survive */
4282 from
= entry
->object
;
4283 entry
->object
= copy
;
4284 DEBUG (5, fprintf (gc_debug_file
, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry
->object
, safe_name (entry
->object
), from
, num_ready_finalizers
, hash_table
->num_registered
));
4288 char *from
= entry
->object
;
4289 if (hash_table
== &minor_finalizable_hash
&& !ptr_in_nursery (copy
)) {
4290 FinalizeEntry
*next
= entry
->next
;
4291 unsigned int major_hash
;
4292 /* remove from the list */
4294 prev
->next
= entry
->next
;
4296 finalizable_hash
[i
] = entry
->next
;
4297 hash_table
->num_registered
--;
4299 entry
->object
= copy
;
4301 /* insert it into the major hash */
4302 rehash_fin_table_if_necessary (&major_finalizable_hash
);
4303 major_hash
= mono_object_hash ((MonoObject
*) copy
) %
4304 major_finalizable_hash
.size
;
4305 entry
->next
= major_finalizable_hash
.table
[major_hash
];
4306 major_finalizable_hash
.table
[major_hash
] = entry
;
4307 major_finalizable_hash
.num_registered
++;
4309 DEBUG (5, fprintf (gc_debug_file
, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy
, safe_name (copy
), from
));
4314 /* update pointer */
4315 DEBUG (5, fprintf (gc_debug_file
, "Updating object for finalization: %p (%s) (was at %p)\n", entry
->object
, safe_name (entry
->object
), from
));
4316 entry
->object
= copy
;
4321 entry
= entry
->next
;
4326 /* LOCKING: requires that the GC lock is held */
4328 null_link_in_range (char *start
, char *end
, int generation
)
4330 DisappearingLinkHashTable
*hash
= get_dislink_hash_table (generation
);
4331 DisappearingLink
**disappearing_link_hash
= hash
->table
;
4332 int disappearing_link_hash_size
= hash
->size
;
4333 DisappearingLink
*entry
, *prev
;
4335 if (!hash
->num_links
)
4337 for (i
= 0; i
< disappearing_link_hash_size
; ++i
) {
4339 for (entry
= disappearing_link_hash
[i
]; entry
;) {
4340 char *object
= DISLINK_OBJECT (entry
);
4341 if (object
>= start
&& object
< end
&& (object
< to_space
|| object
>= to_space_end
)) {
4342 gboolean track
= DISLINK_TRACK (entry
);
4343 if (!track
&& object_is_fin_ready (object
)) {
4344 void **p
= entry
->link
;
4345 DisappearingLink
*old
;
4347 /* remove from list */
4349 prev
->next
= entry
->next
;
4351 disappearing_link_hash
[i
] = entry
->next
;
4352 DEBUG (5, fprintf (gc_debug_file
, "Dislink nullified at %p to GCed object %p\n", p
, object
));
4354 free_internal_mem (entry
);
4359 char *copy
= copy_object (object
, start
, end
);
4361 /* Update pointer if it's moved. If the object
4362 * has been moved out of the nursery, we need to
4363 * remove the link from the minor hash table to
4366 * FIXME: what if an object is moved earlier?
4369 if (hash
== &minor_disappearing_link_hash
&& !ptr_in_nursery (copy
)) {
4370 void **link
= entry
->link
;
4371 DisappearingLink
*old
;
4372 /* remove from list */
4374 prev
->next
= entry
->next
;
4376 disappearing_link_hash
[i
] = entry
->next
;
4378 free_internal_mem (entry
);
4382 add_or_remove_disappearing_link ((MonoObject
*)copy
, link
,
4383 track
, GENERATION_OLD
);
4385 DEBUG (5, fprintf (gc_debug_file
, "Upgraded dislink at %p to major because object %p moved to %p\n", link
, object
, copy
));
4389 /* We set the track resurrection bit to
4390 * FALSE if the object is to be finalized
4391 * so that the object can be collected in
4392 * the next cycle (i.e. after it was
4395 *entry
->link
= HIDE_POINTER (copy
,
4396 object_is_fin_ready (object
) ? FALSE
: track
);
4397 DEBUG (5, fprintf (gc_debug_file
, "Updated dislink at %p to %p\n", entry
->link
, DISLINK_OBJECT (entry
)));
4402 entry
= entry
->next
;
4407 /* LOCKING: requires that the GC lock is held */
4409 null_links_for_domain (MonoDomain
*domain
, int generation
)
4411 DisappearingLinkHashTable
*hash
= get_dislink_hash_table (generation
);
4412 DisappearingLink
**disappearing_link_hash
= hash
->table
;
4413 int disappearing_link_hash_size
= hash
->size
;
4414 DisappearingLink
*entry
, *prev
;
4416 for (i
= 0; i
< disappearing_link_hash_size
; ++i
) {
4418 for (entry
= disappearing_link_hash
[i
]; entry
; ) {
4419 char *object
= DISLINK_OBJECT (entry
);
4420 /* FIXME: actually there should be no object
4421 left in the domain with a non-null vtable
4422 (provided we remove the Thread special
4424 if (object
&& (!((MonoObject
*)object
)->vtable
|| mono_object_domain (object
) == domain
)) {
4425 DisappearingLink
*next
= entry
->next
;
4430 disappearing_link_hash
[i
] = next
;
4432 if (*(entry
->link
)) {
4433 *(entry
->link
) = NULL
;
4434 g_warning ("Disappearing link %p not freed", entry
->link
);
4436 free_internal_mem (entry
);
4443 entry
= entry
->next
;
4448 /* LOCKING: requires that the GC lock is held */
4450 finalizers_for_domain (MonoDomain
*domain
, MonoObject
**out_array
, int out_size
,
4451 FinalizeEntryHashTable
*hash_table
)
4453 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4454 mword finalizable_hash_size
= hash_table
->size
;
4455 FinalizeEntry
*entry
, *prev
;
4458 if (no_finalize
|| !out_size
|| !out_array
)
4461 for (i
= 0; i
< finalizable_hash_size
; ++i
) {
4463 for (entry
= finalizable_hash
[i
]; entry
;) {
4464 if (mono_object_domain (entry
->object
) == domain
) {
4465 FinalizeEntry
*next
;
4466 /* remove and put in out_array */
4468 prev
->next
= entry
->next
;
4470 finalizable_hash
[i
] = entry
->next
;
4472 hash_table
->num_registered
--;
4473 out_array
[count
++] = entry
->object
;
4474 DEBUG (5, fprintf (gc_debug_file
, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry
->object
, safe_name (entry
->object
), num_ready_finalizers
, hash_table
->num_registered
));
4476 if (count
== out_size
)
4481 entry
= entry
->next
;
4488 * mono_gc_finalizers_for_domain:
4489 * @domain: the unloading appdomain
4490 * @out_array: output array
4491 * @out_size: size of output array
4493 * Store inside @out_array up to @out_size objects that belong to the unloading
4494 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4495 * until it returns 0.
4496 * The items are removed from the finalizer data structure, so the caller is supposed
4498 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4501 mono_gc_finalizers_for_domain (MonoDomain
*domain
, MonoObject
**out_array
, int out_size
)
4506 result
= finalizers_for_domain (domain
, out_array
, out_size
, &minor_finalizable_hash
);
4507 if (result
< out_size
) {
4508 result
+= finalizers_for_domain (domain
, out_array
+ result
, out_size
- result
,
4509 &major_finalizable_hash
);
4517 register_for_finalization (MonoObject
*obj
, void *user_data
, int generation
)
4519 FinalizeEntryHashTable
*hash_table
= get_finalize_entry_hash_table (generation
);
4520 FinalizeEntry
**finalizable_hash
;
4521 mword finalizable_hash_size
;
4522 FinalizeEntry
*entry
, *prev
;
4526 g_assert (user_data
== NULL
|| user_data
== mono_gc_run_finalize
);
4527 hash
= mono_object_hash (obj
);
4529 rehash_fin_table_if_necessary (hash_table
);
4530 finalizable_hash
= hash_table
->table
;
4531 finalizable_hash_size
= hash_table
->size
;
4532 hash
%= finalizable_hash_size
;
4534 for (entry
= finalizable_hash
[hash
]; entry
; entry
= entry
->next
) {
4535 if (entry
->object
== obj
) {
4537 /* remove from the list */
4539 prev
->next
= entry
->next
;
4541 finalizable_hash
[hash
] = entry
->next
;
4542 hash_table
->num_registered
--;
4543 DEBUG (5, fprintf (gc_debug_file
, "Removed finalizer %p for object: %p (%s) (%d)\n", entry
, obj
, obj
->vtable
->klass
->name
, hash_table
->num_registered
));
4544 free_internal_mem (entry
);
4552 /* request to deregister, but already out of the list */
4556 entry
= get_internal_mem (sizeof (FinalizeEntry
));
4557 entry
->object
= obj
;
4558 entry
->next
= finalizable_hash
[hash
];
4559 finalizable_hash
[hash
] = entry
;
4560 hash_table
->num_registered
++;
4561 DEBUG (5, fprintf (gc_debug_file
, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry
, obj
, obj
->vtable
->klass
->name
, hash_table
->num_registered
, generation_name (generation
)));
4566 mono_gc_register_for_finalization (MonoObject
*obj
, void *user_data
)
4568 if (ptr_in_nursery (obj
))
4569 register_for_finalization (obj
, user_data
, GENERATION_NURSERY
);
4571 register_for_finalization (obj
, user_data
, GENERATION_OLD
);
4575 rehash_dislink (DisappearingLinkHashTable
*hash_table
)
4577 DisappearingLink
**disappearing_link_hash
= hash_table
->table
;
4578 int disappearing_link_hash_size
= hash_table
->size
;
4581 DisappearingLink
**new_hash
;
4582 DisappearingLink
*entry
, *next
;
4583 int new_size
= g_spaced_primes_closest (hash_table
->num_links
);
4585 new_hash
= get_internal_mem (new_size
* sizeof (DisappearingLink
*));
4586 for (i
= 0; i
< disappearing_link_hash_size
; ++i
) {
4587 for (entry
= disappearing_link_hash
[i
]; entry
; entry
= next
) {
4588 hash
= mono_aligned_addr_hash (entry
->link
) % new_size
;
4590 entry
->next
= new_hash
[hash
];
4591 new_hash
[hash
] = entry
;
4594 free_internal_mem (disappearing_link_hash
);
4595 hash_table
->table
= new_hash
;
4596 hash_table
->size
= new_size
;
4599 /* LOCKING: assumes the GC lock is held */
4601 add_or_remove_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
, int generation
)
4603 DisappearingLinkHashTable
*hash_table
= get_dislink_hash_table (generation
);
4604 DisappearingLink
*entry
, *prev
;
4606 DisappearingLink
**disappearing_link_hash
= hash_table
->table
;
4607 int disappearing_link_hash_size
= hash_table
->size
;
4609 if (hash_table
->num_links
>= disappearing_link_hash_size
* 2) {
4610 rehash_dislink (hash_table
);
4611 disappearing_link_hash
= hash_table
->table
;
4612 disappearing_link_hash_size
= hash_table
->size
;
4614 /* FIXME: add check that link is not in the heap */
4615 hash
= mono_aligned_addr_hash (link
) % disappearing_link_hash_size
;
4616 entry
= disappearing_link_hash
[hash
];
4618 for (; entry
; entry
= entry
->next
) {
4619 /* link already added */
4620 if (link
== entry
->link
) {
4621 /* NULL obj means remove */
4624 prev
->next
= entry
->next
;
4626 disappearing_link_hash
[hash
] = entry
->next
;
4627 hash_table
->num_links
--;
4628 DEBUG (5, fprintf (gc_debug_file
, "Removed dislink %p (%d) from %s table\n", entry
, hash_table
->num_links
, generation_name (generation
)));
4629 free_internal_mem (entry
);
4632 *link
= HIDE_POINTER (obj
, track
); /* we allow the change of object */
4640 entry
= get_internal_mem (sizeof (DisappearingLink
));
4641 *link
= HIDE_POINTER (obj
, track
);
4643 entry
->next
= disappearing_link_hash
[hash
];
4644 disappearing_link_hash
[hash
] = entry
;
4645 hash_table
->num_links
++;
4646 DEBUG (5, fprintf (gc_debug_file
, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry
, obj
, obj
->vtable
->klass
->name
, link
, generation_name (generation
)));
4649 /* LOCKING: assumes the GC lock is held */
4651 mono_gc_register_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
)
4653 add_or_remove_disappearing_link (NULL
, link
, FALSE
, GENERATION_NURSERY
);
4654 add_or_remove_disappearing_link (NULL
, link
, FALSE
, GENERATION_OLD
);
4656 if (ptr_in_nursery (obj
))
4657 add_or_remove_disappearing_link (obj
, link
, track
, GENERATION_NURSERY
);
4659 add_or_remove_disappearing_link (obj
, link
, track
, GENERATION_OLD
);
4664 mono_gc_invoke_finalizers (void)
4666 FinalizeEntry
*entry
= NULL
;
4667 gboolean entry_is_critical
;
4670 /* FIXME: batch to reduce lock contention */
4671 while (fin_ready_list
|| critical_fin_list
) {
4675 FinalizeEntry
**list
= entry_is_critical
? &critical_fin_list
: &fin_ready_list
;
4677 /* We have finalized entry in the last
4678 interation, now we need to remove it from
4681 *list
= entry
->next
;
4683 FinalizeEntry
*e
= *list
;
4684 while (e
->next
!= entry
)
4686 e
->next
= entry
->next
;
4688 free_internal_mem (entry
);
4692 /* Now look for the first non-null entry. */
4693 for (entry
= fin_ready_list
; entry
&& !entry
->object
; entry
= entry
->next
)
4696 entry_is_critical
= FALSE
;
4698 entry_is_critical
= TRUE
;
4699 for (entry
= critical_fin_list
; entry
&& !entry
->object
; entry
= entry
->next
)
4704 g_assert (entry
->object
);
4705 num_ready_finalizers
--;
4706 obj
= entry
->object
;
4707 entry
->object
= NULL
;
4708 DEBUG (7, fprintf (gc_debug_file
, "Finalizing object %p (%s)\n", obj
, safe_name (obj
)));
4716 g_assert (entry
->object
== NULL
);
4718 /* the object is on the stack so it is pinned */
4719 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
4720 mono_gc_run_finalize (obj
, NULL
);
4727 mono_gc_pending_finalizers (void)
4729 return fin_ready_list
|| critical_fin_list
;
4732 /* Negative value to remove */
4734 mono_gc_add_memory_pressure (gint64 value
)
4736 /* FIXME: Use interlocked functions */
4738 memory_pressure
+= value
;
4743 * ######################################################################
4744 * ######## registered roots support
4745 * ######################################################################
4749 rehash_roots (gboolean pinned
)
4753 RootRecord
**new_hash
;
4754 RootRecord
*entry
, *next
;
4757 new_size
= g_spaced_primes_closest (num_roots_entries
[pinned
]);
4758 new_hash
= get_internal_mem (new_size
* sizeof (RootRecord
*));
4759 for (i
= 0; i
< roots_hash_size
[pinned
]; ++i
) {
4760 for (entry
= roots_hash
[pinned
][i
]; entry
; entry
= next
) {
4761 hash
= mono_aligned_addr_hash (entry
->start_root
) % new_size
;
4763 entry
->next
= new_hash
[hash
];
4764 new_hash
[hash
] = entry
;
4767 free_internal_mem (roots_hash
[pinned
]);
4768 roots_hash
[pinned
] = new_hash
;
4769 roots_hash_size
[pinned
] = new_size
;
4773 find_root (int root_type
, char *start
, guint32 addr_hash
)
4775 RootRecord
*new_root
;
4777 guint32 hash
= addr_hash
% roots_hash_size
[root_type
];
4778 for (new_root
= roots_hash
[root_type
][hash
]; new_root
; new_root
= new_root
->next
) {
4779 /* we allow changing the size and the descriptor (for thread statics etc) */
4780 if (new_root
->start_root
== start
) {
4789 * We do not coalesce roots.
4792 mono_gc_register_root_inner (char *start
, size_t size
, void *descr
, int root_type
)
4794 RootRecord
*new_root
;
4795 unsigned int hash
, addr_hash
= mono_aligned_addr_hash (start
);
4798 for (i
= 0; i
< ROOT_TYPE_NUM
; ++i
) {
4799 if (num_roots_entries
[i
] >= roots_hash_size
[i
] * 2)
4802 for (i
= 0; i
< ROOT_TYPE_NUM
; ++i
) {
4803 new_root
= find_root (i
, start
, addr_hash
);
4804 /* we allow changing the size and the descriptor (for thread statics etc) */
4806 size_t old_size
= new_root
->end_root
- new_root
->start_root
;
4807 new_root
->end_root
= new_root
->start_root
+ size
;
4808 g_assert (((new_root
->root_desc
!= 0) && (descr
!= NULL
)) ||
4809 ((new_root
->root_desc
== 0) && (descr
== NULL
)));
4810 new_root
->root_desc
= (mword
)descr
;
4812 roots_size
-= old_size
;
4817 new_root
= get_internal_mem (sizeof (RootRecord
));
4819 new_root
->start_root
= start
;
4820 new_root
->end_root
= new_root
->start_root
+ size
;
4821 new_root
->root_desc
= (mword
)descr
;
4823 hash
= addr_hash
% roots_hash_size
[root_type
];
4824 num_roots_entries
[root_type
]++;
4825 new_root
->next
= roots_hash
[root_type
] [hash
];
4826 roots_hash
[root_type
][hash
] = new_root
;
4827 DEBUG (3, fprintf (gc_debug_file
, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root
, new_root
->start_root
, new_root
->end_root
, descr
, (int)size
, (int)roots_size
));
4837 mono_gc_register_root (char *start
, size_t size
, void *descr
)
4839 return mono_gc_register_root_inner (start
, size
, descr
, descr
? ROOT_TYPE_NORMAL
: ROOT_TYPE_PINNED
);
4843 mono_gc_register_root_wbarrier (char *start
, size_t size
, void *descr
)
4845 return mono_gc_register_root_inner (start
, size
, descr
, ROOT_TYPE_WBARRIER
);
4849 mono_gc_deregister_root (char* addr
)
4851 RootRecord
*tmp
, *prev
;
4852 unsigned int hash
, addr_hash
= mono_aligned_addr_hash (addr
);
4856 for (root_type
= 0; root_type
< ROOT_TYPE_NUM
; ++root_type
) {
4857 hash
= addr_hash
% roots_hash_size
[root_type
];
4858 tmp
= roots_hash
[root_type
][hash
];
4861 if (tmp
->start_root
== (char*)addr
) {
4863 prev
->next
= tmp
->next
;
4865 roots_hash
[root_type
][hash
] = tmp
->next
;
4866 roots_size
-= (tmp
->end_root
- tmp
->start_root
);
4867 num_roots_entries
[root_type
]--;
4868 DEBUG (3, fprintf (gc_debug_file
, "Removed root %p for range: %p-%p\n", tmp
, tmp
->start_root
, tmp
->end_root
));
4869 free_internal_mem (tmp
);
4880 * ######################################################################
4881 * ######## Thread handling (stop/start code)
4882 * ######################################################################
4885 /* FIXME: handle large/small config */
4886 #define THREAD_HASH_SIZE 11
4887 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
4889 static SgenThreadInfo
* thread_table
[THREAD_HASH_SIZE
];
4891 #if USE_SIGNAL_BASED_START_STOP_WORLD
4893 static MonoSemType suspend_ack_semaphore
;
4894 static MonoSemType
*suspend_ack_semaphore_ptr
;
4895 static unsigned int global_stop_count
= 0;
4897 static int suspend_signal_num
= SIGXFSZ
;
4899 static int suspend_signal_num
= SIGPWR
;
4901 static int restart_signal_num
= SIGXCPU
;
4902 static sigset_t suspend_signal_mask
;
4903 static mword cur_thread_regs
[ARCH_NUM_REGS
] = {0};
4905 /* LOCKING: assumes the GC lock is held */
4906 static SgenThreadInfo
*
4907 thread_info_lookup (ARCH_THREAD_TYPE id
)
4909 unsigned int hash
= HASH_PTHREAD_T (id
) % THREAD_HASH_SIZE
;
4910 SgenThreadInfo
*info
;
4912 info
= thread_table
[hash
];
4913 while (info
&& !ARCH_THREAD_EQUALS (info
->id
, id
)) {
4920 update_current_thread_stack (void *start
)
4922 void *ptr
= cur_thread_regs
;
4923 SgenThreadInfo
*info
= thread_info_lookup (ARCH_GET_THREAD ());
4924 info
->stack_start
= align_pointer (&ptr
);
4925 g_assert (info
->stack_start
>= info
->stack_start_limit
&& info
->stack_start
< info
->stack_end
);
4926 ARCH_STORE_REGS (ptr
);
4927 info
->stopped_regs
= ptr
;
4928 if (gc_callbacks
.thread_suspend_func
)
4929 gc_callbacks
.thread_suspend_func (info
->runtime_data
, NULL
);
4933 signal_desc (int signum
)
4935 if (signum
== suspend_signal_num
)
4937 if (signum
== restart_signal_num
)
4943 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
4944 * have cross-domain checks in the write barrier.
4946 //#define XDOMAIN_CHECKS_IN_WBARRIER
4948 #define MANAGED_ALLOCATION
4949 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
4950 #define MANAGED_WBARRIER
4954 is_ip_in_managed_allocator (MonoDomain
*domain
, gpointer ip
);
4957 wait_for_suspend_ack (int count
)
4961 for (i
= 0; i
< count
; ++i
) {
4962 while ((result
= MONO_SEM_WAIT (suspend_ack_semaphore_ptr
)) != 0) {
4963 if (errno
!= EINTR
) {
4964 g_error ("sem_wait ()");
4970 /* LOCKING: assumes the GC lock is held */
4972 thread_handshake (int signum
)
4974 int count
, i
, result
;
4975 SgenThreadInfo
*info
;
4976 pthread_t me
= pthread_self ();
4979 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
4980 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
4981 DEBUG (4, fprintf (gc_debug_file
, "considering thread %p for signal %d (%s)\n", info
, signum
, signal_desc (signum
)));
4982 if (ARCH_THREAD_EQUALS (info
->id
, me
)) {
4983 DEBUG (4, fprintf (gc_debug_file
, "Skip (equal): %p, %p\n", (void*)me
, (void*)info
->id
));
4986 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
4988 result
= pthread_kill (info
->id
, signum
);
4990 DEBUG (4, fprintf (gc_debug_file
, "thread %p signal sent\n", info
));
4993 DEBUG (4, fprintf (gc_debug_file
, "thread %p signal failed: %d (%s)\n", (void*)info
->id
, result
, strerror (result
)));
4999 wait_for_suspend_ack (count
);
5005 restart_threads_until_none_in_managed_allocator (void)
5007 SgenThreadInfo
*info
;
5008 int i
, result
, num_threads_died
= 0;
5009 int sleep_duration
= -1;
5012 int restart_count
= 0, restarted_count
= 0;
5013 /* restart all threads that stopped in the
5015 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5016 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5019 if (!info
->stack_start
||
5020 is_ip_in_managed_allocator (info
->stopped_domain
, info
->stopped_ip
)) {
5021 result
= pthread_kill (info
->id
, restart_signal_num
);
5028 /* we set the stopped_ip to
5029 NULL for threads which
5030 we're not restarting so
5031 that we can easily identify
5033 info
->stopped_ip
= NULL
;
5034 info
->stopped_domain
= NULL
;
5038 /* if no threads were restarted, we're done */
5039 if (restart_count
== 0)
5042 /* wait for the threads to signal their restart */
5043 wait_for_suspend_ack (restart_count
);
5045 if (sleep_duration
< 0) {
5049 g_usleep (sleep_duration
);
5050 sleep_duration
+= 10;
5053 /* stop them again */
5054 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5055 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5056 if (info
->skip
|| info
->stopped_ip
== NULL
)
5058 result
= pthread_kill (info
->id
, suspend_signal_num
);
5066 /* some threads might have died */
5067 num_threads_died
+= restart_count
- restarted_count
;
5068 /* wait for the threads to signal their suspension
5070 wait_for_suspend_ack (restart_count
);
5073 return num_threads_died
;
5076 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5078 suspend_handler (int sig
, siginfo_t
*siginfo
, void *context
)
5080 SgenThreadInfo
*info
;
5083 int old_errno
= errno
;
5084 gpointer regs
[ARCH_NUM_REGS
];
5085 gpointer stack_start
;
5087 id
= pthread_self ();
5088 info
= thread_info_lookup (id
);
5089 info
->stopped_domain
= mono_domain_get ();
5090 info
->stopped_ip
= (gpointer
) ARCH_SIGCTX_IP (context
);
5091 stop_count
= global_stop_count
;
5092 /* duplicate signal */
5093 if (0 && info
->stop_count
== stop_count
) {
5097 #ifdef HAVE_KW_THREAD
5098 /* update the remset info in the thread data structure */
5099 info
->remset
= remembered_set
;
5101 stack_start
= (char*) ARCH_SIGCTX_SP (context
) - REDZONE_SIZE
;
5102 /* If stack_start is not within the limits, then don't set it
5103 in info and we will be restarted. */
5104 if (stack_start
>= info
->stack_start_limit
&& info
->stack_start
<= info
->stack_end
) {
5105 info
->stack_start
= stack_start
;
5107 ARCH_COPY_SIGCTX_REGS (regs
, context
);
5108 info
->stopped_regs
= regs
;
5110 g_assert (!info
->stack_start
);
5113 /* Notify the JIT */
5114 if (gc_callbacks
.thread_suspend_func
)
5115 gc_callbacks
.thread_suspend_func (info
->runtime_data
, context
);
5117 /* notify the waiting thread */
5118 MONO_SEM_POST (suspend_ack_semaphore_ptr
);
5119 info
->stop_count
= stop_count
;
5121 /* wait until we receive the restart signal */
5124 sigsuspend (&suspend_signal_mask
);
5125 } while (info
->signal
!= restart_signal_num
);
5127 /* notify the waiting thread */
5128 MONO_SEM_POST (suspend_ack_semaphore_ptr
);
5134 restart_handler (int sig
)
5136 SgenThreadInfo
*info
;
5137 int old_errno
= errno
;
5139 info
= thread_info_lookup (pthread_self ());
5140 info
->signal
= restart_signal_num
;
5145 static TV_DECLARE (stop_world_time
);
5146 static unsigned long max_pause_usec
= 0;
5148 /* LOCKING: assumes the GC lock is held */
5154 update_current_thread_stack (&count
);
5156 global_stop_count
++;
5157 DEBUG (3, fprintf (gc_debug_file
, "stopping world n %d from %p %p\n", global_stop_count
, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer
)ARCH_GET_THREAD ()));
5158 TV_GETTIME (stop_world_time
);
5159 count
= thread_handshake (suspend_signal_num
);
5160 count
-= restart_threads_until_none_in_managed_allocator ();
5161 g_assert (count
>= 0);
5162 DEBUG (3, fprintf (gc_debug_file
, "world stopped %d thread(s)\n", count
));
5166 /* LOCKING: assumes the GC lock is held */
5168 restart_world (void)
5171 SgenThreadInfo
*info
;
5172 TV_DECLARE (end_sw
);
5175 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5176 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5177 info
->stack_start
= NULL
;
5178 info
->stopped_regs
= NULL
;
5182 count
= thread_handshake (restart_signal_num
);
5183 TV_GETTIME (end_sw
);
5184 usec
= TV_ELAPSED (stop_world_time
, end_sw
);
5185 max_pause_usec
= MAX (usec
, max_pause_usec
);
5186 DEBUG (2, fprintf (gc_debug_file
, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count
, (int)usec
, (int)max_pause_usec
));
5190 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5193 mono_gc_set_gc_callbacks (MonoGCCallbacks
*callbacks
)
5195 gc_callbacks
= *callbacks
;
5198 /* Variables holding start/end nursery so it won't have to be passed at every call */
5199 static void *scan_area_arg_start
, *scan_area_arg_end
;
5202 mono_gc_conservatively_scan_area (void *start
, void *end
)
5204 conservatively_pin_objects_from (start
, end
, scan_area_arg_start
, scan_area_arg_end
);
5208 mono_gc_scan_object (void *obj
)
5210 return copy_object (obj
, scan_area_arg_start
, scan_area_arg_end
);
5214 * Mark from thread stacks and registers.
5217 scan_thread_data (void *start_nursery
, void *end_nursery
, gboolean precise
)
5220 SgenThreadInfo
*info
;
5222 scan_area_arg_start
= start_nursery
;
5223 scan_area_arg_end
= end_nursery
;
5225 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5226 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5228 DEBUG (2, fprintf (gc_debug_file
, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info
, info
->stack_start
, info
->stack_end
, (char*)info
->stack_end
- (char*)info
->stack_start
));
5231 DEBUG (2, fprintf (gc_debug_file
, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info
, info
->stack_start
, info
->stack_end
, (char*)info
->stack_end
- (char*)info
->stack_start
, next_pin_slot
));
5232 if (gc_callbacks
.thread_mark_func
)
5233 gc_callbacks
.thread_mark_func (info
->runtime_data
, info
->stack_start
, info
->stack_end
, precise
);
5235 conservatively_pin_objects_from (info
->stack_start
, info
->stack_end
, start_nursery
, end_nursery
);
5238 conservatively_pin_objects_from (info
->stopped_regs
, info
->stopped_regs
+ ARCH_NUM_REGS
,
5239 start_nursery
, end_nursery
);
5245 find_pinning_ref_from_thread (char *obj
, size_t size
)
5248 SgenThreadInfo
*info
;
5249 char *endobj
= obj
+ size
;
5251 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5252 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5253 char **start
= (char**)info
->stack_start
;
5256 while (start
< (char**)info
->stack_end
) {
5257 if (*start
>= obj
&& *start
< endobj
) {
5258 DEBUG (0, fprintf (gc_debug_file
, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj
, info
, (gpointer
)info
->id
, start
, info
->stack_start
, info
->stack_end
));
5263 /* FIXME: check info->stopped_regs */
5269 ptr_on_stack (void *ptr
)
5271 gpointer stack_start
= &stack_start
;
5272 SgenThreadInfo
*info
= thread_info_lookup (ARCH_GET_THREAD ());
5274 if (ptr
>= stack_start
&& ptr
< (gpointer
)info
->stack_end
)
5279 /* return TRUE if ptr points inside the managed heap */
5281 ptr_in_heap (void* ptr
)
5283 mword p
= (mword
)ptr
;
5285 GCMemSection
*section
;
5287 if (p
< lowest_heap_address
|| p
>= highest_heap_address
)
5290 if (ptr_in_nursery (ptr
))
5293 if (ptr_on_stack (ptr
))
5296 for (section
= section_list
; section
; section
= section
->next
) {
5297 if (ptr
>= (gpointer
)section
->data
&& ptr
< (gpointer
)(section
->data
+ section
->size
))
5301 if (obj_is_from_pinned_alloc (ptr
))
5304 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
) {
5305 if (ptr
>= (gpointer
)bigobj
->data
&& ptr
< (gpointer
)(bigobj
->data
+ bigobj
->size
))
5313 handle_remset (mword
*p
, void *start_nursery
, void *end_nursery
, gboolean global
)
5319 /* FIXME: exclude stack locations */
5320 switch ((*p
) & REMSET_TYPE_MASK
) {
5321 case REMSET_LOCATION
:
5323 //__builtin_prefetch (ptr);
5324 if (((void*)ptr
< start_nursery
|| (void*)ptr
>= end_nursery
) && ptr_in_heap (ptr
)) {
5325 *ptr
= copy_object (*ptr
, start_nursery
, end_nursery
);
5326 DEBUG (9, fprintf (gc_debug_file
, "Overwrote remset at %p with %p\n", ptr
, *ptr
));
5327 if (!global
&& *ptr
>= start_nursery
&& *ptr
< end_nursery
) {
5329 * If the object is pinned, each reference to it from nonpinned objects
5330 * becomes part of the global remset, which can grow very large.
5332 DEBUG (9, fprintf (gc_debug_file
, "Add to global remset because of pinning %p (%p %s)\n", ptr
, *ptr
, safe_name (*ptr
)));
5333 add_to_global_remset (ptr
, FALSE
);
5336 DEBUG (9, fprintf (gc_debug_file
, "Skipping remset at %p holding %p\n", ptr
, *ptr
));
5340 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
5341 if (((void*)ptr
>= start_nursery
&& (void*)ptr
< end_nursery
) || !ptr_in_heap (ptr
))
5344 while (count
-- > 0) {
5345 *ptr
= copy_object (*ptr
, start_nursery
, end_nursery
);
5346 DEBUG (9, fprintf (gc_debug_file
, "Overwrote remset at %p with %p (count: %d)\n", ptr
, *ptr
, (int)count
));
5347 if (!global
&& *ptr
>= start_nursery
&& *ptr
< end_nursery
)
5348 add_to_global_remset (ptr
, FALSE
);
5353 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
5354 if (((void*)ptr
>= start_nursery
&& (void*)ptr
< end_nursery
) || !ptr_in_heap (ptr
))
5356 scan_object ((char*)ptr
, start_nursery
, end_nursery
);
5358 case REMSET_OTHER
: {
5359 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
5363 if (((void*)ptr
>= start_nursery
&& (void*)ptr
< end_nursery
) || !ptr_in_heap (ptr
))
5368 ptr
= (void**) scan_vtype ((char*)ptr
, desc
, start_nursery
, end_nursery
);
5370 case REMSET_ROOT_LOCATION
:
5371 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5372 *ptr
= copy_object (*ptr
, start_nursery
, end_nursery
);
5373 DEBUG (9, fprintf (gc_debug_file
, "Overwrote root location remset at %p with %p\n", ptr
, *ptr
));
5374 if (!global
&& *ptr
>= start_nursery
&& *ptr
< end_nursery
) {
5376 * If the object is pinned, each reference to it from nonpinned objects
5377 * becomes part of the global remset, which can grow very large.
5379 DEBUG (9, fprintf (gc_debug_file
, "Add to global remset because of pinning %p (%p %s)\n", ptr
, *ptr
, safe_name (*ptr
)));
5380 add_to_global_remset (ptr
, TRUE
);
5384 g_assert_not_reached ();
5389 g_assert_not_reached ();
5395 scan_from_remsets (void *start_nursery
, void *end_nursery
)
5398 SgenThreadInfo
*info
;
5399 RememberedSet
*remset
, *next
;
5400 mword
*p
, *next_p
, *store_pos
;
5402 /* the global one */
5403 for (remset
= global_remset
; remset
; remset
= remset
->next
) {
5404 DEBUG (4, fprintf (gc_debug_file
, "Scanning global remset range: %p-%p, size: %zd\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
5405 store_pos
= remset
->data
;
5406 for (p
= remset
->data
; p
< remset
->store_next
; p
= next_p
) {
5409 next_p
= handle_remset (p
, start_nursery
, end_nursery
, TRUE
);
5412 * Clear global remsets of locations which no longer point to the
5413 * nursery. Otherwise, they could grow indefinitely between major
5416 ptr
= (p
[0] & ~REMSET_TYPE_MASK
);
5417 if ((p
[0] & REMSET_TYPE_MASK
) == REMSET_LOCATION
) {
5418 if (ptr_in_nursery (*(void**)ptr
))
5419 *store_pos
++ = p
[0];
5421 g_assert ((p
[0] & REMSET_TYPE_MASK
) == REMSET_OTHER
);
5422 g_assert (p
[1] == REMSET_ROOT_LOCATION
);
5423 if (ptr_in_nursery (*(void**)ptr
)) {
5424 *store_pos
++ = p
[0];
5425 *store_pos
++ = p
[1];
5430 /* Truncate the remset */
5431 remset
->store_next
= store_pos
;
5434 /* the per-thread ones */
5435 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5436 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5437 for (remset
= info
->remset
; remset
; remset
= next
) {
5438 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info
, remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
5439 for (p
= remset
->data
; p
< remset
->store_next
;) {
5440 p
= handle_remset (p
, start_nursery
, end_nursery
, FALSE
);
5442 remset
->store_next
= remset
->data
;
5443 next
= remset
->next
;
5444 remset
->next
= NULL
;
5445 if (remset
!= info
->remset
) {
5446 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5447 free_internal_mem (remset
);
5453 /* the freed thread ones */
5454 while (freed_thread_remsets
) {
5455 remset
= freed_thread_remsets
;
5456 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
5457 for (p
= remset
->data
; p
< remset
->store_next
;) {
5458 p
= handle_remset (p
, start_nursery
, end_nursery
, FALSE
);
5460 next
= remset
->next
;
5461 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5462 free_internal_mem (remset
);
5463 freed_thread_remsets
= next
;
5468 * Clear the info in the remembered sets: we're doing a major collection, so
5469 * the per-thread ones are not needed and the global ones will be reconstructed
5473 clear_remsets (void)
5476 SgenThreadInfo
*info
;
5477 RememberedSet
*remset
, *next
;
5479 /* the global list */
5480 for (remset
= global_remset
; remset
; remset
= next
) {
5481 remset
->store_next
= remset
->data
;
5482 next
= remset
->next
;
5483 remset
->next
= NULL
;
5484 if (remset
!= global_remset
) {
5485 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5486 free_internal_mem (remset
);
5489 /* the per-thread ones */
5490 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5491 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5492 for (remset
= info
->remset
; remset
; remset
= next
) {
5493 remset
->store_next
= remset
->data
;
5494 next
= remset
->next
;
5495 remset
->next
= NULL
;
5496 if (remset
!= info
->remset
) {
5497 DEBUG (1, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5498 free_internal_mem (remset
);
5504 /* the freed thread ones */
5505 while (freed_thread_remsets
) {
5506 next
= freed_thread_remsets
->next
;
5507 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", freed_thread_remsets
->data
));
5508 free_internal_mem (freed_thread_remsets
);
5509 freed_thread_remsets
= next
;
5514 * Clear the thread local TLAB variables for all threads.
5519 SgenThreadInfo
*info
;
5522 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5523 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5524 /* A new TLAB will be allocated when the thread does its first allocation */
5525 *info
->tlab_start_addr
= NULL
;
5526 *info
->tlab_next_addr
= NULL
;
5527 *info
->tlab_temp_end_addr
= NULL
;
5528 *info
->tlab_real_end_addr
= NULL
;
5534 * Find the tlab_next value of the TLAB which contains ADDR.
5537 find_tlab_next_from_address (char *addr
)
5539 SgenThreadInfo
*info
;
5542 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5543 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5545 * The allocator increments tlab_next before
5546 * checking whether that address is still in
5547 * the TLAB, so we have to check here.
5549 char *next_addr
= *info
->tlab_next_addr
;
5550 char *end_addr
= *info
->tlab_real_end_addr
;
5551 if (next_addr
> end_addr
)
5552 next_addr
= end_addr
;
5553 if (addr
>= *info
->tlab_start_addr
&& addr
< next_addr
)
5561 /* LOCKING: assumes the GC lock is held */
5562 static SgenThreadInfo
*
5563 gc_register_current_thread (void *addr
)
5566 SgenThreadInfo
* info
= malloc (sizeof (SgenThreadInfo
));
5567 #ifndef HAVE_KW_THREAD
5568 SgenThreadInfo
*__thread_info__
= info
;
5574 #ifndef HAVE_KW_THREAD
5575 info
->tlab_start
= info
->tlab_next
= info
->tlab_temp_end
= info
->tlab_real_end
= NULL
;
5577 g_assert (!pthread_getspecific (thread_info_key
));
5578 pthread_setspecific (thread_info_key
, info
);
5581 info
->id
= ARCH_GET_THREAD ();
5582 info
->stop_count
= -1;
5585 info
->stack_start
= NULL
;
5586 info
->tlab_start_addr
= &TLAB_START
;
5587 info
->tlab_next_addr
= &TLAB_NEXT
;
5588 info
->tlab_temp_end_addr
= &TLAB_TEMP_END
;
5589 info
->tlab_real_end_addr
= &TLAB_REAL_END
;
5590 info
->stopped_ip
= NULL
;
5591 info
->stopped_domain
= NULL
;
5592 info
->stopped_regs
= NULL
;
5594 #ifdef HAVE_KW_THREAD
5595 tlab_next_addr
= &tlab_next
;
5598 /* try to get it with attributes first */
5599 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
5603 pthread_attr_t attr
;
5604 pthread_getattr_np (pthread_self (), &attr
);
5605 pthread_attr_getstack (&attr
, &sstart
, &size
);
5606 info
->stack_start_limit
= sstart
;
5607 info
->stack_end
= (char*)sstart
+ size
;
5608 pthread_attr_destroy (&attr
);
5610 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
5611 info
->stack_end
= (char*)pthread_get_stackaddr_np (pthread_self ());
5614 /* FIXME: we assume the stack grows down */
5615 gsize stack_bottom
= (gsize
)addr
;
5616 stack_bottom
+= 4095;
5617 stack_bottom
&= ~4095;
5618 info
->stack_end
= (char*)stack_bottom
;
5622 #ifdef HAVE_KW_THREAD
5623 stack_end
= info
->stack_end
;
5626 /* hash into the table */
5627 hash
= HASH_PTHREAD_T (info
->id
) % THREAD_HASH_SIZE
;
5628 info
->next
= thread_table
[hash
];
5629 thread_table
[hash
] = info
;
5631 info
->remset
= alloc_remset (DEFAULT_REMSET_SIZE
, info
);
5632 pthread_setspecific (remembered_set_key
, info
->remset
);
5633 #ifdef HAVE_KW_THREAD
5634 remembered_set
= info
->remset
;
5636 DEBUG (3, fprintf (gc_debug_file
, "registered thread %p (%p) (hash: %d)\n", info
, (gpointer
)info
->id
, hash
));
5638 if (gc_callbacks
.thread_attach_func
)
5639 info
->runtime_data
= gc_callbacks
.thread_attach_func ();
5645 unregister_current_thread (void)
5648 SgenThreadInfo
*prev
= NULL
;
5650 RememberedSet
*rset
;
5651 ARCH_THREAD_TYPE id
= ARCH_GET_THREAD ();
5653 hash
= HASH_PTHREAD_T (id
) % THREAD_HASH_SIZE
;
5654 p
= thread_table
[hash
];
5656 DEBUG (3, fprintf (gc_debug_file
, "unregister thread %p (%p)\n", p
, (gpointer
)p
->id
));
5657 while (!ARCH_THREAD_EQUALS (p
->id
, id
)) {
5662 thread_table
[hash
] = p
->next
;
5664 prev
->next
= p
->next
;
5667 if (freed_thread_remsets
) {
5668 for (rset
= p
->remset
; rset
->next
; rset
= rset
->next
)
5670 rset
->next
= freed_thread_remsets
;
5671 freed_thread_remsets
= p
->remset
;
5673 freed_thread_remsets
= p
->remset
;
5680 unregister_thread (void *k
)
5683 unregister_current_thread ();
5688 mono_gc_register_thread (void *baseptr
)
5690 SgenThreadInfo
*info
;
5692 info
= thread_info_lookup (ARCH_GET_THREAD ());
5694 info
= gc_register_current_thread (baseptr
);
5696 return info
!= NULL
;
5699 #if USE_PTHREAD_INTERCEPT
5701 #undef pthread_create
5703 #undef pthread_detach
5706 void *(*start_routine
) (void *);
5709 MonoSemType registered
;
5710 } SgenThreadStartInfo
;
5713 gc_start_thread (void *arg
)
5715 SgenThreadStartInfo
*start_info
= arg
;
5716 SgenThreadInfo
* info
;
5717 void *t_arg
= start_info
->arg
;
5718 void *(*start_func
) (void*) = start_info
->start_routine
;
5723 info
= gc_register_current_thread (&result
);
5725 post_result
= MONO_SEM_POST (&(start_info
->registered
));
5726 g_assert (!post_result
);
5727 result
= start_func (t_arg
);
5729 * this is done by the pthread key dtor
5731 unregister_current_thread ();
5739 mono_gc_pthread_create (pthread_t
*new_thread
, const pthread_attr_t
*attr
, void *(*start_routine
)(void *), void *arg
)
5741 SgenThreadStartInfo
*start_info
;
5744 start_info
= malloc (sizeof (SgenThreadStartInfo
));
5747 result
= MONO_SEM_INIT (&(start_info
->registered
), 0);
5749 start_info
->arg
= arg
;
5750 start_info
->start_routine
= start_routine
;
5752 result
= pthread_create (new_thread
, attr
, gc_start_thread
, start_info
);
5754 while (MONO_SEM_WAIT (&(start_info
->registered
)) != 0) {
5755 /*if (EINTR != errno) ABORT("sem_wait failed"); */
5758 MONO_SEM_DESTROY (&(start_info
->registered
));
5764 mono_gc_pthread_join (pthread_t thread
, void **retval
)
5766 return pthread_join (thread
, retval
);
5770 mono_gc_pthread_detach (pthread_t thread
)
5772 return pthread_detach (thread
);
5775 #endif /* USE_PTHREAD_INTERCEPT */
5778 * ######################################################################
5779 * ######## Write barriers
5780 * ######################################################################
5783 static RememberedSet
*
5784 alloc_remset (int size
, gpointer id
) {
5785 RememberedSet
* res
= get_internal_mem (sizeof (RememberedSet
) + (size
* sizeof (gpointer
)));
5786 res
->store_next
= res
->data
;
5787 res
->end_set
= res
->data
+ size
;
5789 DEBUG (4, fprintf (gc_debug_file
, "Allocated remset size %d at %p for %p\n", size
, res
->data
, id
));
5794 * Note: the write barriers first do the needed GC work and then do the actual store:
5795 * this way the value is visible to the conservative GC scan after the write barrier
5796 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
5797 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
5798 * happens when we need to record which pointers contain references to the new generation.
5799 * The write barrier will be executed, but the pointer is still not stored.
5802 mono_gc_wbarrier_set_field (MonoObject
*obj
, gpointer field_ptr
, MonoObject
* value
)
5806 if (ptr_in_nursery (field_ptr
)) {
5807 *(void**)field_ptr
= value
;
5810 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p\n", field_ptr
));
5812 rs
= REMEMBERED_SET
;
5813 if (rs
->store_next
< rs
->end_set
) {
5814 *(rs
->store_next
++) = (mword
)field_ptr
;
5815 *(void**)field_ptr
= value
;
5819 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
5820 rs
->next
= REMEMBERED_SET
;
5821 REMEMBERED_SET
= rs
;
5822 #ifdef HAVE_KW_THREAD
5823 thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
5825 *(rs
->store_next
++) = (mword
)field_ptr
;
5826 *(void**)field_ptr
= value
;
5831 mono_gc_wbarrier_set_arrayref (MonoArray
*arr
, gpointer slot_ptr
, MonoObject
* value
)
5835 if (ptr_in_nursery (slot_ptr
)) {
5836 *(void**)slot_ptr
= value
;
5839 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p\n", slot_ptr
));
5841 rs
= REMEMBERED_SET
;
5842 if (rs
->store_next
< rs
->end_set
) {
5843 *(rs
->store_next
++) = (mword
)slot_ptr
;
5844 *(void**)slot_ptr
= value
;
5848 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
5849 rs
->next
= REMEMBERED_SET
;
5850 REMEMBERED_SET
= rs
;
5851 #ifdef HAVE_KW_THREAD
5852 thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
5854 *(rs
->store_next
++) = (mword
)slot_ptr
;
5855 *(void**)slot_ptr
= value
;
5860 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr
, gpointer src_ptr
, int count
)
5865 memmove (dest_ptr
, src_ptr
, count
* sizeof (gpointer
));
5866 if (ptr_in_nursery (dest_ptr
)) {
5870 rs
= REMEMBERED_SET
;
5871 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p, %d\n", dest_ptr
, count
));
5872 if (rs
->store_next
+ 1 < rs
->end_set
) {
5873 *(rs
->store_next
++) = (mword
)dest_ptr
| REMSET_RANGE
;
5874 *(rs
->store_next
++) = count
;
5878 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
5879 rs
->next
= REMEMBERED_SET
;
5880 REMEMBERED_SET
= rs
;
5881 #ifdef HAVE_KW_THREAD
5882 thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
5884 *(rs
->store_next
++) = (mword
)dest_ptr
| REMSET_RANGE
;
5885 *(rs
->store_next
++) = count
;
5890 find_object_for_ptr_in_area (char *ptr
, char *start
, char *end
)
5892 while (start
< end
) {
5895 if (!*(void**)start
) {
5896 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
5902 #define SCAN_OBJECT_NOSCAN
5903 #include "sgen-scan-object.h"
5905 if (ptr
>= old_start
&& ptr
< start
)
5912 static char *found_obj
;
5915 find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk
*chunk
, char *obj
, size_t size
, char *ptr
)
5917 if (ptr
>= obj
&& ptr
< obj
+ size
) {
5918 g_assert (!found_obj
);
5924 find_object_for_ptr (char *ptr
)
5926 GCMemSection
*section
;
5929 for (section
= section_list
; section
; section
= section
->next
) {
5930 if (ptr
>= section
->data
&& ptr
< section
->end_data
)
5931 return find_object_for_ptr_in_area (ptr
, section
->data
, section
->end_data
);
5934 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
) {
5935 if (ptr
>= bigobj
->data
&& ptr
< bigobj
->data
+ bigobj
->size
)
5936 return bigobj
->data
;
5940 scan_pinned_objects ((ScanPinnedObjectCallbackFunc
)find_object_for_ptr_in_pinned_chunk_callback
, ptr
);
5945 mono_gc_wbarrier_generic_nostore (gpointer ptr
)
5950 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
5951 /* FIXME: ptr_in_heap must be called with the GC lock held */
5952 if (xdomain_checks
&& *(MonoObject
**)ptr
&& ptr_in_heap (ptr
)) {
5953 char *start
= find_object_for_ptr (ptr
);
5954 MonoObject
*value
= *(MonoObject
**)ptr
;
5958 MonoObject
*obj
= (MonoObject
*)start
;
5959 if (obj
->vtable
->domain
!= value
->vtable
->domain
)
5960 g_assert (is_xdomain_ref_allowed (ptr
, start
, obj
->vtable
->domain
));
5967 if (ptr_in_nursery (ptr
) || !ptr_in_heap (ptr
)) {
5968 DEBUG (8, fprintf (gc_debug_file
, "Skipping remset at %p\n", ptr
));
5972 rs
= REMEMBERED_SET
;
5973 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p\n", ptr
));
5974 if (rs
->store_next
< rs
->end_set
) {
5975 *(rs
->store_next
++) = (mword
)ptr
;
5979 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
5980 rs
->next
= REMEMBERED_SET
;
5981 REMEMBERED_SET
= rs
;
5982 #ifdef HAVE_KW_THREAD
5983 thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
5985 *(rs
->store_next
++) = (mword
)ptr
;
5990 mono_gc_wbarrier_generic_store (gpointer ptr
, MonoObject
* value
)
5992 DEBUG (8, fprintf (gc_debug_file
, "Wbarrier store at %p to %p (%s)\n", ptr
, value
, value
? safe_name (value
) : "null"));
5993 *(void**)ptr
= value
;
5994 mono_gc_wbarrier_generic_nostore (ptr
);
5998 mono_gc_wbarrier_set_root (gpointer ptr
, MonoObject
*value
)
6002 if (ptr_in_nursery (ptr
))
6004 DEBUG (8, fprintf (gc_debug_file
, "Adding root remset at %p (%s)\n", ptr
, value
? safe_name (value
) : "null"));
6006 rs
= REMEMBERED_SET
;
6007 if (rs
->store_next
+ 2 < rs
->end_set
) {
6008 *(rs
->store_next
++) = (mword
)ptr
| REMSET_OTHER
;
6009 *(rs
->store_next
++) = (mword
)REMSET_ROOT_LOCATION
;
6010 *(void**)ptr
= value
;
6013 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6014 rs
->next
= REMEMBERED_SET
;
6015 REMEMBERED_SET
= rs
;
6016 #ifdef HAVE_KW_THREAD
6017 thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6019 *(rs
->store_next
++) = (mword
)ptr
| REMSET_OTHER
;
6020 *(rs
->store_next
++) = (mword
)REMSET_ROOT_LOCATION
;
6022 *(void**)ptr
= value
;
6026 mono_gc_wbarrier_value_copy (gpointer dest
, gpointer src
, int count
, MonoClass
*klass
)
6030 g_assert (klass
->valuetype
);
6032 memmove (dest
, src
, count
* mono_class_value_size (klass
, NULL
));
6033 rs
= REMEMBERED_SET
;
6034 if (ptr_in_nursery (dest
) || !ptr_in_heap (dest
)) {
6038 g_assert (klass
->gc_descr_inited
);
6039 DEBUG (8, fprintf (gc_debug_file
, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest
, count
, klass
->gc_descr
, klass
->name
, klass
));
6041 if (rs
->store_next
+ 3 < rs
->end_set
) {
6042 *(rs
->store_next
++) = (mword
)dest
| REMSET_OTHER
;
6043 *(rs
->store_next
++) = (mword
)REMSET_VTYPE
;
6044 *(rs
->store_next
++) = (mword
)klass
->gc_descr
;
6045 *(rs
->store_next
++) = (mword
)count
;
6049 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6050 rs
->next
= REMEMBERED_SET
;
6051 REMEMBERED_SET
= rs
;
6052 #ifdef HAVE_KW_THREAD
6053 thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6055 *(rs
->store_next
++) = (mword
)dest
| REMSET_OTHER
;
6056 *(rs
->store_next
++) = (mword
)REMSET_VTYPE
;
6057 *(rs
->store_next
++) = (mword
)klass
->gc_descr
;
6058 *(rs
->store_next
++) = (mword
)count
;
6063 * mono_gc_wbarrier_object_copy:
6065 * Write barrier to call when obj is the result of a clone or copy of an object.
6068 mono_gc_wbarrier_object_copy (MonoObject
* obj
, MonoObject
*src
)
6074 rs
= REMEMBERED_SET
;
6075 DEBUG (1, fprintf (gc_debug_file
, "Adding object remset for %p\n", obj
));
6076 size
= mono_object_class (obj
)->instance_size
;
6078 /* do not copy the sync state */
6079 memcpy ((char*)obj
+ sizeof (MonoObject
), (char*)src
+ sizeof (MonoObject
),
6080 size
- sizeof (MonoObject
));
6081 if (ptr_in_nursery (obj
) || !ptr_in_heap (obj
)) {
6085 if (rs
->store_next
< rs
->end_set
) {
6086 *(rs
->store_next
++) = (mword
)obj
| REMSET_OBJECT
;
6090 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6091 rs
->next
= REMEMBERED_SET
;
6092 REMEMBERED_SET
= rs
;
6093 #ifdef HAVE_KW_THREAD
6094 thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6096 *(rs
->store_next
++) = (mword
)obj
| REMSET_OBJECT
;
6101 * ######################################################################
6102 * ######## Collector debugging
6103 * ######################################################################
6106 const char*descriptor_types
[] = {
6118 describe_ptr (char *ptr
)
6120 GCMemSection
*section
;
6125 if (ptr_in_nursery (ptr
)) {
6126 printf ("Pointer inside nursery.\n");
6128 for (section
= section_list
; section
;) {
6129 if (ptr
>= section
->data
&& ptr
< section
->data
+ section
->size
)
6131 section
= section
->next
;
6135 printf ("Pointer inside oldspace.\n");
6136 } else if (obj_is_from_pinned_alloc (ptr
)) {
6137 printf ("Pointer is inside a pinned chunk.\n");
6139 printf ("Pointer unknown.\n");
6144 if (object_is_pinned (ptr
))
6145 printf ("Object is pinned.\n");
6147 if (object_is_forwarded (ptr
))
6148 printf ("Object is forwared.\n");
6150 // FIXME: Handle pointers to the inside of objects
6151 vtable
= (MonoVTable
*)LOAD_VTABLE (ptr
);
6153 printf ("VTable: %p\n", vtable
);
6154 if (vtable
== NULL
) {
6155 printf ("VTable is invalid (empty).\n");
6158 if (ptr_in_nursery (vtable
)) {
6159 printf ("VTable is invalid (points inside nursery).\n");
6162 printf ("Class: %s\n", vtable
->klass
->name
);
6164 desc
= ((GCVTable
*)vtable
)->desc
;
6165 printf ("Descriptor: %lx\n", (long)desc
);
6168 printf ("Descriptor type: %d (%s)\n", type
, descriptor_types
[type
]);
6172 find_in_remset_loc (mword
*p
, char *addr
, gboolean
*found
)
6178 switch ((*p
) & REMSET_TYPE_MASK
) {
6179 case REMSET_LOCATION
:
6180 if (*p
== (mword
)addr
)
6184 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
6186 if ((void**)addr
>= ptr
&& (void**)addr
< ptr
+ count
)
6190 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
6191 count
= safe_object_get_size ((MonoObject
*)ptr
);
6192 count
+= (ALLOC_ALIGN
- 1);
6193 count
&= (ALLOC_ALIGN
- 1);
6194 count
/= sizeof (mword
);
6195 if ((void**)addr
>= ptr
&& (void**)addr
< ptr
+ count
)
6198 case REMSET_OTHER
: {
6201 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
6205 switch (desc
& 0x7) {
6206 case DESC_TYPE_RUN_LENGTH
:
6207 OBJ_RUN_LEN_SIZE (skip_size
, desc
, ptr
);
6208 /* The descriptor includes the size of MonoObject */
6209 skip_size
-= sizeof (MonoObject
);
6211 if ((void**)addr
>= ptr
&& (void**)addr
< ptr
+ (skip_size
/ sizeof (gpointer
)))
6216 g_assert_not_reached ();
6220 case REMSET_ROOT_LOCATION
:
6223 g_assert_not_reached ();
6228 g_assert_not_reached ();
6234 * Return whenever ADDR occurs in the remembered sets
6237 find_in_remsets (char *addr
)
6240 SgenThreadInfo
*info
;
6241 RememberedSet
*remset
;
6243 gboolean found
= FALSE
;
6245 /* the global one */
6246 for (remset
= global_remset
; remset
; remset
= remset
->next
) {
6247 DEBUG (4, fprintf (gc_debug_file
, "Scanning global remset range: %p-%p, size: %zd\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
6248 for (p
= remset
->data
; p
< remset
->store_next
;) {
6249 p
= find_in_remset_loc (p
, addr
, &found
);
6254 /* the per-thread ones */
6255 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
6256 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
6257 for (remset
= info
->remset
; remset
; remset
= remset
->next
) {
6258 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info
, remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
6259 for (p
= remset
->data
; p
< remset
->store_next
;) {
6260 p
= find_in_remset_loc (p
, addr
, &found
);
6268 /* the freed thread ones */
6269 for (remset
= freed_thread_remsets
; remset
; remset
= remset
->next
) {
6270 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
6271 for (p
= remset
->data
; p
< remset
->store_next
;) {
6272 p
= find_in_remset_loc (p
, addr
, &found
);
6282 #define HANDLE_PTR(ptr,obj) do { \
6283 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6284 if (!find_in_remsets ((char*)(ptr))) { \
6285 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6286 g_assert_not_reached (); \
6292 * Check that each object reference inside the area which points into the nursery
6293 * can be found in the remembered sets.
6295 static void __attribute__((noinline
))
6296 check_remsets_for_area (char *start
, char *end
)
6299 int type_str
= 0, type_rlen
= 0, type_bitmap
= 0, type_vector
= 0, type_lbit
= 0, type_complex
= 0;
6300 new_obj_references
= 0;
6301 obj_references_checked
= 0;
6302 while (start
< end
) {
6303 if (!*(void**)start
) {
6304 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
6307 vt
= (GCVTable
*)LOAD_VTABLE (start
);
6308 DEBUG (8, fprintf (gc_debug_file
, "Scanning object %p, vtable: %p (%s)\n", start
, vt
, vt
->klass
->name
));
6310 MonoObject
*obj
= (MonoObject
*)start
;
6311 g_print ("found at %p (0x%lx): %s.%s\n", start
, (long)vt
->desc
, obj
->vtable
->klass
->name_space
, obj
->vtable
->klass
->name
);
6314 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
6315 #include "sgen-scan-object.h"
6320 * Perform consistency check of the heap.
6322 * Assumes the world is stopped.
6325 check_consistency (void)
6327 GCMemSection
*section
;
6329 // Need to add more checks
6330 // FIXME: Create a general heap enumeration function and use that
6332 DEBUG (1, fprintf (gc_debug_file
, "Begin heap consistency check...\n"));
6334 // Check that oldspace->newspace pointers are registered with the collector
6335 for (section
= section_list
; section
; section
= section
->next
) {
6336 if (section
->role
== MEMORY_ROLE_GEN0
)
6338 DEBUG (2, fprintf (gc_debug_file
, "Scan of old section: %p-%p, size: %d\n", section
->data
, section
->next_data
, (int)(section
->next_data
- section
->data
)));
6339 check_remsets_for_area (section
->data
, section
->next_data
);
6342 DEBUG (1, fprintf (gc_debug_file
, "Heap consistency check done.\n"));
6345 /* Check that the reference is valid */
6347 #define HANDLE_PTR(ptr,obj) do { \
6349 g_assert (safe_name (*(ptr)) != NULL); \
6356 * Perform consistency check on an object. Currently we only check that the
6357 * reference fields are valid.
6360 check_object (char *start
)
6365 #include "sgen-scan-object.h"
6371 * ######################################################################
6372 * ######## Other mono public interface functions.
6373 * ######################################################################
6377 mono_gc_collect (int generation
)
6381 if (generation
== 0) {
6382 collect_nursery (0);
6384 major_collection ("user request");
6391 mono_gc_max_generation (void)
6397 mono_gc_collection_count (int generation
)
6399 if (generation
== 0)
6400 return num_minor_gcs
;
6401 return num_major_gcs
;
6405 mono_gc_get_used_size (void)
6408 GCMemSection
*section
;
6410 tot
= los_memory_usage
;
6411 for (section
= section_list
; section
; section
= section
->next
) {
6412 /* this is approximate... */
6413 tot
+= section
->next_data
- section
->data
;
6415 /* FIXME: account for pinned objects */
6421 mono_gc_get_heap_size (void)
6427 mono_gc_disable (void)
6435 mono_gc_enable (void)
6443 mono_object_is_alive (MonoObject
* o
)
6449 mono_gc_get_generation (MonoObject
*obj
)
6451 if (ptr_in_nursery (obj
))
6457 mono_gc_enable_events (void)
6462 mono_gc_weak_link_add (void **link_addr
, MonoObject
*obj
, gboolean track
)
6465 mono_gc_register_disappearing_link (obj
, link_addr
, track
);
6470 mono_gc_weak_link_remove (void **link_addr
)
6473 mono_gc_register_disappearing_link (NULL
, link_addr
, FALSE
);
6478 mono_gc_weak_link_get (void **link_addr
)
6482 return (MonoObject
*) REVEAL_POINTER (*link_addr
);
6486 mono_gc_make_descr_from_bitmap (gsize
*bitmap
, int numbits
)
6488 if (numbits
< ((sizeof (*bitmap
) * 8) - ROOT_DESC_TYPE_SHIFT
)) {
6489 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP
, bitmap
[0]);
6491 mword
complex = alloc_complex_descriptor (bitmap
, numbits
+ 1);
6492 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX
, complex);
6497 mono_gc_make_root_descr_user (MonoGCMarkFunc marker
)
6501 g_assert (user_descriptors_next
< MAX_USER_DESCRIPTORS
);
6502 descr
= (void*)MAKE_ROOT_DESC (ROOT_DESC_USER
, (mword
)user_descriptors_next
);
6503 user_descriptors
[user_descriptors_next
++] = marker
;
6509 mono_gc_alloc_fixed (size_t size
, void *descr
)
6511 /* FIXME: do a single allocation */
6512 void *res
= calloc (1, size
);
6515 if (!mono_gc_register_root (res
, size
, descr
)) {
6523 mono_gc_free_fixed (void* addr
)
6525 mono_gc_deregister_root (addr
);
6530 mono_gc_is_gc_thread (void)
6534 result
= thread_info_lookup (ARCH_GET_THREAD ()) != NULL
;
6540 mono_gc_base_init (void)
6544 struct sigaction sinfo
;
6546 LOCK_INIT (gc_mutex
);
6548 if (gc_initialized
) {
6552 pagesize
= mono_pagesize ();
6553 gc_debug_file
= stderr
;
6554 if ((env
= getenv ("MONO_GC_DEBUG"))) {
6555 opts
= g_strsplit (env
, ",", -1);
6556 for (ptr
= opts
; ptr
&& *ptr
; ptr
++) {
6558 if (opt
[0] >= '0' && opt
[0] <= '9') {
6559 gc_debug_level
= atoi (opt
);
6564 char *rf
= g_strdup_printf ("%s.%d", opt
, getpid ());
6565 gc_debug_file
= fopen (rf
, "wb");
6567 gc_debug_file
= stderr
;
6570 } else if (!strcmp (opt
, "collect-before-allocs")) {
6571 collect_before_allocs
= TRUE
;
6572 } else if (!strcmp (opt
, "check-at-minor-collections")) {
6573 consistency_check_at_minor_collection
= TRUE
;
6574 } else if (!strcmp (opt
, "xdomain-checks")) {
6575 xdomain_checks
= TRUE
;
6576 } else if (!strcmp (opt
, "clear-at-gc")) {
6577 nursery_clear_policy
= CLEAR_AT_GC
;
6578 } else if (g_str_has_prefix (opt
, "heap-dump=")) {
6579 char *filename
= strchr (opt
, '=') + 1;
6580 nursery_clear_policy
= CLEAR_AT_GC
;
6581 heap_dump_file
= fopen (filename
, "w");
6583 fprintf (heap_dump_file
, "<sgen-dump>\n");
6585 fprintf (stderr
, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env
);
6586 fprintf (stderr
, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
6587 fprintf (stderr
, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
6594 suspend_ack_semaphore_ptr
= &suspend_ack_semaphore
;
6595 MONO_SEM_INIT (&suspend_ack_semaphore
, 0);
6597 sigfillset (&sinfo
.sa_mask
);
6598 sinfo
.sa_flags
= SA_RESTART
| SA_SIGINFO
;
6599 sinfo
.sa_sigaction
= suspend_handler
;
6600 if (sigaction (suspend_signal_num
, &sinfo
, NULL
) != 0) {
6601 g_error ("failed sigaction");
6604 sinfo
.sa_handler
= restart_handler
;
6605 if (sigaction (restart_signal_num
, &sinfo
, NULL
) != 0) {
6606 g_error ("failed sigaction");
6609 sigfillset (&suspend_signal_mask
);
6610 sigdelset (&suspend_signal_mask
, restart_signal_num
);
6612 global_remset
= alloc_remset (1024, NULL
);
6613 global_remset
->next
= NULL
;
6615 pthread_key_create (&remembered_set_key
, unregister_thread
);
6617 #ifndef HAVE_KW_THREAD
6618 pthread_key_create (&thread_info_key
, NULL
);
6621 gc_initialized
= TRUE
;
6623 mono_gc_register_thread (&sinfo
);
6627 mono_gc_get_suspend_signal (void)
6629 return suspend_signal_num
;
6638 #ifdef HAVE_KW_THREAD
6639 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
6640 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6641 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6642 mono_mb_emit_i4 ((mb), (offset)); \
6645 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
6646 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6647 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6648 mono_mb_emit_i4 ((mb), thread_info_key); \
6649 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
6650 mono_mb_emit_byte ((mb), CEE_ADD); \
6651 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
6655 #ifdef MANAGED_ALLOCATION
6656 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
6657 * for each class. This is currently not easy to do, as it is hard to generate basic
6658 * blocks + branches, but it is easy with the linear IL codebase.
6660 * For this to work we'd need to solve the TLAB race, first. Now we
6661 * require the allocator to be in a few known methods to make sure
6662 * that they are executed atomically via the restart mechanism.
6665 create_allocator (int atype
)
6667 int p_var
, size_var
;
6668 guint32 slowpath_branch
;
6669 MonoMethodBuilder
*mb
;
6671 MonoMethodSignature
*csig
;
6672 static gboolean registered
= FALSE
;
6673 int tlab_next_addr_var
, new_next_var
;
6676 #ifdef HAVE_KW_THREAD
6677 int tlab_next_addr_offset
= -1;
6678 int tlab_temp_end_offset
= -1;
6680 MONO_THREAD_VAR_OFFSET (tlab_next_addr
, tlab_next_addr_offset
);
6681 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
6683 g_assert (tlab_next_addr_offset
!= -1);
6684 g_assert (tlab_temp_end_offset
!= -1);
6688 mono_register_jit_icall (mono_gc_alloc_obj
, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE
);
6689 mono_register_jit_icall (mono_gc_alloc_vector
, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE
);
6693 if (atype
== ATYPE_NORMAL
)
6695 else if (atype
== ATYPE_VECTOR
)
6698 g_assert_not_reached ();
6700 csig
= mono_metadata_signature_alloc (mono_defaults
.corlib
, num_params
);
6701 csig
->ret
= &mono_defaults
.object_class
->byval_arg
;
6702 for (i
= 0; i
< num_params
; ++i
)
6703 csig
->params
[i
] = &mono_defaults
.int_class
->byval_arg
;
6705 mb
= mono_mb_new (mono_defaults
.object_class
, "Alloc", MONO_WRAPPER_ALLOC
);
6706 size_var
= mono_mb_add_local (mb
, &mono_defaults
.int32_class
->byval_arg
);
6707 if (atype
== ATYPE_NORMAL
) {
6708 /* size = vtable->klass->instance_size; */
6709 mono_mb_emit_ldarg (mb
, 0);
6710 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoVTable
, klass
));
6711 mono_mb_emit_byte (mb
, CEE_ADD
);
6712 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
6713 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoClass
, instance_size
));
6714 mono_mb_emit_byte (mb
, CEE_ADD
);
6715 /* FIXME: assert instance_size stays a 4 byte integer */
6716 mono_mb_emit_byte (mb
, CEE_LDIND_U4
);
6717 mono_mb_emit_stloc (mb
, size_var
);
6718 } else if (atype
== ATYPE_VECTOR
) {
6719 MonoExceptionClause
*clause
;
6721 MonoClass
*oom_exc_class
;
6724 clause
= mono_image_alloc0 (mono_defaults
.corlib
, sizeof (MonoExceptionClause
));
6725 clause
->try_offset
= mono_mb_get_label (mb
);
6727 /* vtable->klass->sizes.element_size */
6728 mono_mb_emit_ldarg (mb
, 0);
6729 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoVTable
, klass
));
6730 mono_mb_emit_byte (mb
, CEE_ADD
);
6731 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
6732 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoClass
, sizes
.element_size
));
6733 mono_mb_emit_byte (mb
, CEE_ADD
);
6734 mono_mb_emit_byte (mb
, CEE_LDIND_U4
);
6737 mono_mb_emit_ldarg (mb
, 1);
6738 mono_mb_emit_byte (mb
, CEE_MUL_OVF_UN
);
6739 /* + sizeof (MonoArray) */
6740 mono_mb_emit_icon (mb
, sizeof (MonoArray
));
6741 mono_mb_emit_byte (mb
, CEE_ADD_OVF_UN
);
6742 mono_mb_emit_stloc (mb
, size_var
);
6744 pos_leave
= mono_mb_emit_branch (mb
, CEE_LEAVE
);
6747 clause
->flags
= MONO_EXCEPTION_CLAUSE_NONE
;
6748 clause
->try_len
= mono_mb_get_pos (mb
) - clause
->try_offset
;
6749 clause
->data
.catch_class
= mono_class_from_name (mono_defaults
.corlib
,
6750 "System", "OverflowException");
6751 g_assert (clause
->data
.catch_class
);
6752 clause
->handler_offset
= mono_mb_get_label (mb
);
6754 oom_exc_class
= mono_class_from_name (mono_defaults
.corlib
,
6755 "System", "OutOfMemoryException");
6756 g_assert (oom_exc_class
);
6757 ctor
= mono_class_get_method_from_name (oom_exc_class
, ".ctor", 0);
6760 mono_mb_emit_op (mb
, CEE_NEWOBJ
, ctor
);
6761 mono_mb_emit_byte (mb
, CEE_THROW
);
6763 clause
->handler_len
= mono_mb_get_pos (mb
) - clause
->handler_offset
;
6764 mono_mb_set_clauses (mb
, 1, clause
);
6765 mono_mb_patch_branch (mb
, pos_leave
);
6768 g_assert_not_reached ();
6771 /* size += ALLOC_ALIGN - 1; */
6772 mono_mb_emit_ldloc (mb
, size_var
);
6773 mono_mb_emit_icon (mb
, ALLOC_ALIGN
- 1);
6774 mono_mb_emit_byte (mb
, CEE_ADD
);
6775 /* size &= ~(ALLOC_ALIGN - 1); */
6776 mono_mb_emit_icon (mb
, ~(ALLOC_ALIGN
- 1));
6777 mono_mb_emit_byte (mb
, CEE_AND
);
6778 mono_mb_emit_stloc (mb
, size_var
);
6781 * We need to modify tlab_next, but the JIT only supports reading, so we read
6782 * another tls var holding its address instead.
6785 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
6786 tlab_next_addr_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
6787 EMIT_TLS_ACCESS (mb
, tlab_next_addr
, tlab_next_addr_offset
);
6788 mono_mb_emit_stloc (mb
, tlab_next_addr_var
);
6790 /* p = (void**)tlab_next; */
6791 p_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
6792 mono_mb_emit_ldloc (mb
, tlab_next_addr_var
);
6793 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
6794 mono_mb_emit_stloc (mb
, p_var
);
6796 /* new_next = (char*)p + size; */
6797 new_next_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
6798 mono_mb_emit_ldloc (mb
, p_var
);
6799 mono_mb_emit_ldloc (mb
, size_var
);
6800 mono_mb_emit_byte (mb
, CEE_CONV_I
);
6801 mono_mb_emit_byte (mb
, CEE_ADD
);
6802 mono_mb_emit_stloc (mb
, new_next_var
);
6804 /* tlab_next = new_next */
6805 mono_mb_emit_ldloc (mb
, tlab_next_addr_var
);
6806 mono_mb_emit_ldloc (mb
, new_next_var
);
6807 mono_mb_emit_byte (mb
, CEE_STIND_I
);
6809 /* if (G_LIKELY (new_next < tlab_temp_end)) */
6810 mono_mb_emit_ldloc (mb
, new_next_var
);
6811 EMIT_TLS_ACCESS (mb
, tlab_temp_end
, tlab_temp_end_offset
);
6812 slowpath_branch
= mono_mb_emit_short_branch (mb
, MONO_CEE_BLT_UN_S
);
6816 mono_mb_emit_byte (mb
, MONO_CUSTOM_PREFIX
);
6817 mono_mb_emit_byte (mb
, CEE_MONO_NOT_TAKEN
);
6819 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
6820 mono_mb_emit_ldarg (mb
, 0);
6821 mono_mb_emit_ldloc (mb
, size_var
);
6822 if (atype
== ATYPE_NORMAL
) {
6823 mono_mb_emit_icall (mb
, mono_gc_alloc_obj
);
6824 } else if (atype
== ATYPE_VECTOR
) {
6825 mono_mb_emit_ldarg (mb
, 1);
6826 mono_mb_emit_icall (mb
, mono_gc_alloc_vector
);
6828 g_assert_not_reached ();
6830 mono_mb_emit_byte (mb
, CEE_RET
);
6833 mono_mb_patch_short_branch (mb
, slowpath_branch
);
6835 /* FIXME: Memory barrier */
6838 mono_mb_emit_ldloc (mb
, p_var
);
6839 mono_mb_emit_ldarg (mb
, 0);
6840 mono_mb_emit_byte (mb
, CEE_STIND_I
);
6842 if (atype
== ATYPE_VECTOR
) {
6843 /* arr->max_length = max_length; */
6844 mono_mb_emit_ldloc (mb
, p_var
);
6845 mono_mb_emit_ldflda (mb
, G_STRUCT_OFFSET (MonoArray
, max_length
));
6846 mono_mb_emit_ldarg (mb
, 1);
6847 mono_mb_emit_byte (mb
, CEE_STIND_I
);
6851 mono_mb_emit_ldloc (mb
, p_var
);
6852 mono_mb_emit_byte (mb
, CEE_RET
);
6854 res
= mono_mb_create_method (mb
, csig
, 8);
6856 mono_method_get_header (res
)->init_locals
= FALSE
;
6861 static MonoMethod
* alloc_method_cache
[ATYPE_NUM
];
6862 static MonoMethod
*write_barrier_method
;
6865 is_ip_in_managed_allocator (MonoDomain
*domain
, gpointer ip
)
6873 ji
= mono_jit_info_table_find (domain
, ip
);
6876 method
= ji
->method
;
6878 if (method
== write_barrier_method
)
6880 for (i
= 0; i
< ATYPE_NUM
; ++i
)
6881 if (method
== alloc_method_cache
[i
])
6887 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
6888 * The signature of the called method is:
6889 * object allocate (MonoVTable *vtable)
6892 mono_gc_get_managed_allocator (MonoVTable
*vtable
, gboolean for_box
)
6894 #ifdef MANAGED_ALLOCATION
6895 MonoClass
*klass
= vtable
->klass
;
6897 #ifdef HAVE_KW_THREAD
6898 int tlab_next_offset
= -1;
6899 int tlab_temp_end_offset
= -1;
6900 MONO_THREAD_VAR_OFFSET (tlab_next
, tlab_next_offset
);
6901 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
6903 if (tlab_next_offset
== -1 || tlab_temp_end_offset
== -1)
6907 if (!mono_runtime_has_tls_get ())
6909 if (klass
->instance_size
> tlab_size
)
6911 if (klass
->has_finalize
|| klass
->marshalbyref
|| (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS
))
6915 if (klass
->byval_arg
.type
== MONO_TYPE_STRING
)
6917 if (collect_before_allocs
)
6920 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL
);
6927 mono_gc_get_managed_array_allocator (MonoVTable
*vtable
, int rank
)
6929 #ifdef MANAGED_ALLOCATION
6930 MonoClass
*klass
= vtable
->klass
;
6932 #ifdef HAVE_KW_THREAD
6933 int tlab_next_offset
= -1;
6934 int tlab_temp_end_offset
= -1;
6935 MONO_THREAD_VAR_OFFSET (tlab_next
, tlab_next_offset
);
6936 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
6938 if (tlab_next_offset
== -1 || tlab_temp_end_offset
== -1)
6944 if (!mono_runtime_has_tls_get ())
6946 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS
)
6948 if (collect_before_allocs
)
6950 g_assert (!klass
->has_finalize
&& !klass
->marshalbyref
);
6952 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR
);
6959 mono_gc_get_managed_allocator_type (MonoMethod
*managed_alloc
)
6961 #ifdef MANAGED_ALLOCATION
6964 for (i
= 0; i
< ATYPE_NUM
; ++i
)
6965 if (managed_alloc
== alloc_method_cache
[i
])
6968 g_assert_not_reached ();
6973 mono_gc_get_managed_allocator_by_type (int atype
)
6975 #ifdef MANAGED_ALLOCATION
6978 if (!mono_runtime_has_tls_get ())
6981 mono_loader_lock ();
6982 res
= alloc_method_cache
[atype
];
6984 res
= alloc_method_cache
[atype
] = create_allocator (atype
);
6985 mono_loader_unlock ();
6993 mono_gc_get_managed_allocator_types (void)
7000 mono_gc_get_write_barrier (void)
7003 MonoMethodBuilder
*mb
;
7004 MonoMethodSignature
*sig
;
7005 #ifdef MANAGED_WBARRIER
7006 int label_no_wb
, label_need_wb_1
, label_need_wb_2
, label2
;
7007 int remset_var
, next_var
, dummy_var
;
7009 #ifdef HAVE_KW_THREAD
7010 int remset_offset
= -1, stack_end_offset
= -1;
7012 MONO_THREAD_VAR_OFFSET (remembered_set
, remset_offset
);
7013 MONO_THREAD_VAR_OFFSET (stack_end
, stack_end_offset
);
7014 g_assert (remset_offset
!= -1 && stack_end_offset
!= -1);
7018 // FIXME: Maybe create a separate version for ctors (the branch would be
7019 // correctly predicted more times)
7020 if (write_barrier_method
)
7021 return write_barrier_method
;
7023 /* Create the IL version of mono_gc_barrier_generic_store () */
7024 sig
= mono_metadata_signature_alloc (mono_defaults
.corlib
, 1);
7025 sig
->ret
= &mono_defaults
.void_class
->byval_arg
;
7026 sig
->params
[0] = &mono_defaults
.int_class
->byval_arg
;
7028 mb
= mono_mb_new (mono_defaults
.object_class
, "wbarrier", MONO_WRAPPER_WRITE_BARRIER
);
7030 #ifdef MANAGED_WBARRIER
7031 if (mono_runtime_has_tls_get ()) {
7032 /* ptr_in_nursery () check */
7033 #ifdef ALIGN_NURSERY
7035 * Masking out the bits might be faster, but we would have to use 64 bit
7036 * immediates, which might be slower.
7038 mono_mb_emit_ldarg (mb
, 0);
7039 mono_mb_emit_icon (mb
, DEFAULT_NURSERY_BITS
);
7040 mono_mb_emit_byte (mb
, CEE_SHR_UN
);
7041 mono_mb_emit_icon (mb
, (mword
)nursery_start
>> DEFAULT_NURSERY_BITS
);
7042 label_no_wb
= mono_mb_emit_branch (mb
, CEE_BEQ
);
7045 g_assert_not_reached ();
7048 /* Need write barrier if ptr >= stack_end */
7049 mono_mb_emit_ldarg (mb
, 0);
7050 EMIT_TLS_ACCESS (mb
, stack_end
, stack_end_offset
);
7051 label_need_wb_1
= mono_mb_emit_branch (mb
, CEE_BGE_UN
);
7053 /* Need write barrier if ptr < stack_start */
7054 dummy_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7055 mono_mb_emit_ldarg (mb
, 0);
7056 mono_mb_emit_ldloc_addr (mb
, dummy_var
);
7057 label_need_wb_2
= mono_mb_emit_branch (mb
, CEE_BLE_UN
);
7059 /* Don't need write barrier case */
7060 mono_mb_patch_branch (mb
, label_no_wb
);
7062 mono_mb_emit_byte (mb
, CEE_RET
);
7064 /* Need write barrier case */
7065 mono_mb_patch_branch (mb
, label_need_wb_1
);
7066 mono_mb_patch_branch (mb
, label_need_wb_2
);
7068 // remset_var = remembered_set;
7069 remset_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7070 EMIT_TLS_ACCESS (mb
, remset
, remset_offset
);
7071 mono_mb_emit_stloc (mb
, remset_var
);
7073 // next_var = rs->store_next
7074 next_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7075 mono_mb_emit_ldloc (mb
, remset_var
);
7076 mono_mb_emit_ldflda (mb
, G_STRUCT_OFFSET (RememberedSet
, store_next
));
7077 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7078 mono_mb_emit_stloc (mb
, next_var
);
7080 // if (rs->store_next < rs->end_set) {
7081 mono_mb_emit_ldloc (mb
, next_var
);
7082 mono_mb_emit_ldloc (mb
, remset_var
);
7083 mono_mb_emit_ldflda (mb
, G_STRUCT_OFFSET (RememberedSet
, end_set
));
7084 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7085 label2
= mono_mb_emit_branch (mb
, CEE_BGE
);
7087 /* write barrier fast path */
7088 // *(rs->store_next++) = (mword)ptr;
7089 mono_mb_emit_ldloc (mb
, next_var
);
7090 mono_mb_emit_ldarg (mb
, 0);
7091 mono_mb_emit_byte (mb
, CEE_STIND_I
);
7093 mono_mb_emit_ldloc (mb
, next_var
);
7094 mono_mb_emit_icon (mb
, sizeof (gpointer
));
7095 mono_mb_emit_byte (mb
, CEE_ADD
);
7096 mono_mb_emit_stloc (mb
, next_var
);
7098 mono_mb_emit_ldloc (mb
, remset_var
);
7099 mono_mb_emit_ldflda (mb
, G_STRUCT_OFFSET (RememberedSet
, store_next
));
7100 mono_mb_emit_ldloc (mb
, next_var
);
7101 mono_mb_emit_byte (mb
, CEE_STIND_I
);
7103 /* write barrier slow path */
7104 mono_mb_patch_branch (mb
, label2
);
7108 mono_mb_emit_ldarg (mb
, 0);
7109 mono_mb_emit_icall (mb
, mono_gc_wbarrier_generic_nostore
);
7110 mono_mb_emit_byte (mb
, CEE_RET
);
7112 res
= mono_mb_create_method (mb
, sig
, 16);
7115 mono_loader_lock ();
7116 if (write_barrier_method
) {
7117 /* Already created */
7118 mono_free_method (res
);
7120 /* double-checked locking */
7121 mono_memory_barrier ();
7122 write_barrier_method
= res
;
7124 mono_loader_unlock ();
7126 return write_barrier_method
;
7129 #endif /* HAVE_SGEN_GC */