| blob | 59e3a223d5ea3e245d539f15d3ec3b2ae3273c12 |
1 /*
2 * sgen-gc.c: Simple generational GC.
3 *
4 * Author:
5 * Paolo Molaro (lupus@ximian.com)
6 *
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
8 *
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
14 *
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
17 *
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
23 *
24 *
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
27 *
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
35 *
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
38 *
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
46 *
47 *
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
51 * 64 KB pinned space
52 * 64 KB internal space
53 * size of nursery
54 * We should provide a small memory config with half the sizes
55 *
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
58 * forwarding ptr)
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
65 *
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
68 *
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
72 * copying GC.
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
76 *
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
87 */
89 /*
90 * TODO:
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
118 (See next item)
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
148 free...)
150 *) memzero the major fragments after restarting the world and optionally a smaller
151 chunk at a time
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
156 lock contention
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
178 */
180 #ifdef HAVE_SGEN_GC
182 #include <unistd.h>
183 #include <stdio.h>
184 #include <string.h>
185 #include <semaphore.h>
186 #include <signal.h>
187 #include <errno.h>
188 #include <assert.h>
189 #ifdef __MACH__
190 #undef _XOPEN_SOURCE
191 #endif
192 #include <pthread.h>
193 #ifdef __MACH__
194 #define _XOPEN_SOURCE
195 #endif
215 #include <mono/utils/memcheck.h>
217 #if defined(__MACH__)
219 #endif
221 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
222 a = i,
226 CEE_LAST
227 };
229 #undef OPDEF
231 /*
232 * ######################################################################
233 * ######## Types and constants used by the GC.
234 * ######################################################################
235 */
240 /* If set, do a minor collection before every allocation */
242 /* If set, do a heap consistency check before each minor collection */
244 /* If set, check that there are no references to the domain left at domain unload */
246 /* If not null, dump the heap after each collection into this file */
248 /* If set, mark stacks conservatively, even if precise marking is possible */
250 /* If set, do a plausibility check on the scan_starts before and after
251 each collection */
254 /*
255 * Turning on heavy statistics will turn off the managed allocator and
256 * the managed write barrier.
257 */
258 //#define HEAVY_STATISTICS
260 #ifdef HEAVY_STATISTICS
261 #define HEAVY_STAT(x) x
262 #else
263 #define HEAVY_STAT(x)
264 #endif
266 #ifdef HEAVY_STATISTICS
305 #endif
333 /* Keep in sync with internal_mem_names in dump_heap()! */
335 INTERNAL_MEM_PIN_QUEUE,
336 INTERNAL_MEM_FRAGMENT,
337 INTERNAL_MEM_SECTION,
338 INTERNAL_MEM_SCAN_STARTS,
339 INTERNAL_MEM_FIN_TABLE,
340 INTERNAL_MEM_FINALIZE_ENTRY,
341 INTERNAL_MEM_DISLINK_TABLE,
342 INTERNAL_MEM_DISLINK,
343 INTERNAL_MEM_ROOTS_TABLE,
344 INTERNAL_MEM_ROOT_RECORD,
345 INTERNAL_MEM_STATISTICS,
346 INTERNAL_MEM_REMSET,
347 INTERNAL_MEM_GRAY_QUEUE,
348 INTERNAL_MEM_STORE_REMSET,
349 INTERNAL_MEM_MS_TABLES,
350 INTERNAL_MEM_MS_BLOCK_INFO,
351 INTERNAL_MEM_EPHEMERON_LINK,
352 INTERNAL_MEM_MAX
353 };
357 /*
358 void
359 mono_gc_flush_info (void)
360 {
361 fflush (gc_debug_file);
362 }
363 */
365 /* Define this to allow the user to change some of the constants by specifying
366 * their values in the MONO_GC_PARAMS environmental variable. See
367 * mono_gc_base_init for details. */
368 #define USER_CONFIG 1
370 #define TV_DECLARE(name) gint64 name
371 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
372 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
373 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
375 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
377 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
380 MEMORY_ROLE_GEN0,
381 MEMORY_ROLE_GEN1,
382 MEMORY_ROLE_PINNED
383 };
389 };
391 /* each request from the OS ends up in a GCMemSection */
394 Block block;
396 mword size;
397 /* pointer where more data could be allocated if it fits */
400 /*
401 * scan starts is an array of pointers to objects equally spaced in the allocation area
402 * They let use quickly find pinned objects from pinning pointers.
403 */
405 /* in major collections indexes in the pin_queue for objects that pin this section */
409 gboolean is_to_space;
410 };
412 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
414 /* Pinned objects are allocated in the LOS space if bigger than half a page
415 * or from freelists otherwise. We assume that pinned objects are relatively few
416 * and they have a slow dying speed (like interned strings, thread objects).
417 * As such they will be collected only at major collections.
418 * free lists are not global: when we need memory we allocate a PinnedChunk.
419 * Each pinned chunk is made of several pages, the first of wich is used
420 * internally for bookeeping (here think of a page as 4KB). The bookeeping
421 * includes the freelists vectors and info about the object size of each page
422 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
423 * a size is assigned to it, the page is divided in the proper chunks and each
424 * chunk is added to the freelist. To not waste space, the remaining space in the
425 * first page is used as objects of size 16 or 32 (need to measure which are more
426 * common).
427 * We use this same structure to allocate memory used internally by the GC, so
428 * we never use malloc/free if we need to alloc during collection: the world is stopped
429 * and malloc/free will deadlock.
430 * When we want to iterate over pinned objects, we just scan a page at a time
431 * linearly according to the size of objects in the page: the next pointer used to link
432 * the items in the freelist uses the same word as the vtable. Since we keep freelists
433 * for each pinned chunk, if the word points outside the pinned chunk it means
434 * it is an object.
435 * We could avoid this expensive scanning in creative ways. We could have a policy
436 * of putting in the pinned space only objects we know about that have no struct fields
437 * with references and we can easily use a even expensive write barrier for them,
438 * since pointer writes on such objects should be rare.
439 * The best compromise is to just alloc interned strings and System.MonoType in them.
440 * It would be nice to allocate MonoThread in it, too: must check that we properly
441 * use write barriers so we don't have to do any expensive scanning of the whole pinned
442 * chunk list during minor collections. We can avoid it now because we alloc in it only
443 * reference-free objects.
444 */
445 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
446 #define MAX_FREELIST_SIZE 8192
449 Block block;
455 };
457 /* The method used to clear the nursery */
458 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
459 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
460 * to find bugs.
461 */
463 CLEAR_AT_GC,
464 CLEAR_AT_TLAB_CREATION
469 /*
470 * If this is set, the nursery is aligned to an address aligned to its size, ie.
471 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
472 * speed up ptr_in_nursery () checks which are very frequent. This requires the
473 * nursery size to be a compile time constant.
474 */
475 #define ALIGN_NURSERY 1
477 /*
478 * The young generation is divided into fragments. This is because
479 * we can hand one fragments to a thread for lock-less fast alloc and
480 * because the young generation ends up fragmented anyway by pinned objects.
481 * Once a collection is done, a list of fragments is created. When doing
482 * thread local alloc we use smallish nurseries so we allow new threads to
483 * allocate memory from gen0 without triggering a collection. Threads that
484 * are found to allocate lots of memory are given bigger fragments. This
485 * should make the finalizer thread use little nursery memory after a while.
486 * We should start assigning threads very small fragments: if there are many
487 * threads the nursery will be full of reserved space that the threads may not
488 * use at all, slowing down allocation speed.
489 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
490 * Allocation Buffers (TLABs).
491 */
499 };
501 /* the runtime can register areas of memory as roots: we keep two lists of roots,
502 * a pinned root set for conservatively scanned roots and a normal one for
503 * precisely scanned roots (currently implemented as a single list).
504 */
510 mword root_desc;
511 };
513 /*
514 * We're never actually using the first element. It's always set to
515 * NULL to simplify the elimination of consecutive duplicate
516 * entries.
517 */
518 #define STORE_REMSET_BUFFER_SIZE 1024
523 /* We need one entry less because the first entry of store
524 remset buffers is always a dummy and we don't copy it. */
526 };
528 /* we have 4 possible values in the low 2 bits */
535 };
537 #ifdef HAVE_KW_THREAD
539 #endif
545 /*A two slots cache for recently inserted remsets */
548 /* FIXME: later choose a size that takes into account the RememberedSet struct
549 * and doesn't waste any alloc paddin space.
550 */
551 #define DEFAULT_REMSET_SIZE 1024
554 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
555 * no cast from a pointer to an integer
556 */
559 mword desc;
562 /* these bits are set in the object vtable: we could merge them since an object can be
563 * either pinned or forwarded but not both.
564 * We store them in the vtable slot because the bits are used in the sync block for
565 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
566 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
567 * would be an invalid combination for the monitor and hash code).
568 * The values are already shifted.
569 * The forwarding address is stored in the sync block.
570 */
571 #define FORWARDED_BIT 1
572 #define PINNED_BIT 2
573 #define VTABLE_BITS_MASK 0x3
575 /* returns NULL if not forwarded, or the forwarded address */
576 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
577 /* set the forwarded address fw_addr for object obj */
578 #define forward_object(obj,fw_addr) do { \
579 ((mword*)(obj))[0] |= FORWARDED_BIT; \
580 ((mword*)(obj))[1] = (mword)(fw_addr); \
581 } while (0)
583 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
584 #define pin_object(obj) do { \
585 ((mword*)(obj))[0] |= PINNED_BIT; \
586 } while (0)
587 #define unpin_object(obj) do { \
588 ((mword*)(obj))[0] &= ~PINNED_BIT; \
589 } while (0)
591 #ifdef ALIGN_NURSERY
592 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
593 #else
594 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
595 #endif
597 /*
598 * Since we set bits in the vtable, use the macro to load it from the pointer to
599 * an object that is potentially pinned.
600 */
601 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
605 {
608 }
612 {
623 }
626 /* from a created object: the class must be inited already */
628 }
629 }
631 /*
632 * ######################################################################
633 * ######## Global data.
634 * ######################################################################
635 */
641 #ifdef USER_CONFIG
643 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
644 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
646 #ifdef ALIGN_NURSERY
647 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
648 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
650 #endif
652 #else
654 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
655 #ifdef ALIGN_NURSERY
656 #define DEFAULT_NURSERY_BITS 22
657 #endif
659 #endif
661 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
662 /* to quickly find the head of an object pinned by a conservative address
663 * we keep track of the objects allocated for each SCAN_START_SIZE memory
664 * chunk in the nursery or other memory sections. Larger values have less
665 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
666 */
667 #define SCAN_START_SIZE (4096*2)
668 /* the minimum size of a fragment that we consider useful for allocation */
669 #define FRAGMENT_MIN_SIZE (512)
670 /* This is a fixed value used for pinned chunks, not the system pagesize */
671 #define FREELIST_PAGESIZE (16*1024)
678 /* use this to tune when to do a major/minor collection */
693 };
698 mword size;
700 };
706 };
711 mword size;
713 };
720 };
727 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
731 guint32 magic;
734 };
737 GENERATION_NURSERY,
738 GENERATION_OLD,
739 GENERATION_MAX
740 };
744 /*
745 * The link pointer is hidden by negating each bit. We use the lowest
746 * bit of the link (before negation) to store whether it needs
747 * resurrection tracking.
748 */
749 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
750 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
752 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
753 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
755 /*
756 * The finalizable hash has the object as the key, the
757 * disappearing_link hash, has the link address as key.
758 */
761 /* objects that are ready to be finalized */
773 /* keep each size a multiple of ALLOC_ALIGN */
774 /* on 64 bit systems 8 is likely completely unused. */
780 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
782 /* This is also the MAJOR_SECTION_SIZE for the copying major
783 collector */
784 #define PINNED_CHUNK_SIZE (128 * 1024)
786 /* internal_chunk_list is used for allocating structures needed by the GC */
795 ROOT_TYPE_NUM
796 };
798 /* registered roots: the key to the hash is the root start address */
799 /*
800 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
801 */
807 /*
808 * The current allocation cursors
809 * We allocate objects in the nursery.
810 * The nursery is the area between nursery_start and nursery_real_end.
811 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
812 * from nursery fragments.
813 * tlab_next is the pointer to the space inside the TLAB where the next object will
814 * be allocated.
815 * tlab_temp_end is the pointer to the end of the temporary space reserved for
816 * the allocation: it allows us to set the scan starts at reasonable intervals.
817 * tlab_real_end points to the end of the TLAB.
818 * nursery_frag_real_end points to the end of the currently used nursery fragment.
819 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
820 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
821 * At the next allocation, the area of the nursery where objects can be present is
822 * between MIN(nursery_first_pinned_start, first_fragment_start) and
823 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
824 */
827 #ifdef HAVE_KW_THREAD
828 #define TLAB_ACCESS_INIT
829 #define TLAB_START tlab_start
830 #define TLAB_NEXT tlab_next
831 #define TLAB_TEMP_END tlab_temp_end
832 #define TLAB_REAL_END tlab_real_end
833 #define REMEMBERED_SET remembered_set
834 #define STORE_REMSET_BUFFER store_remset_buffer
835 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
836 #define IN_CRITICAL_REGION thread_info->in_critical_region
837 #else
839 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
840 #define TLAB_START (__thread_info__->tlab_start)
841 #define TLAB_NEXT (__thread_info__->tlab_next)
842 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
843 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
844 #define REMEMBERED_SET (__thread_info__->remset)
845 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
846 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
847 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
848 #endif
850 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
851 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
852 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
854 /*
855 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
856 * variables for next+temp_end ?
857 */
858 #ifdef HAVE_KW_THREAD
866 /* Used by the managed allocator/wbarrier */
870 #endif
876 /* The size of a TLAB */
877 /* The bigger the value, the less often we have to go to the slow path to allocate a new
878 * one, but the more space is wasted by threads not allocating much memory.
879 * FIXME: Tune this.
880 * FIXME: Make this self-tuning for each thread.
881 */
884 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
885 #define MAX_NURSERY_TLAB_WASTE 512
887 /* fragments that are free and ready to be used for allocation */
889 /* freeelist of fragment structures */
892 /*
893 * Objects bigger then this go into the large object space. This size
894 * has a few constraints. It must fit into the major heap, which in
895 * the case of the copying collector means that it must fit into a
896 * pinned chunk. It must also play well with the GC descriptors, some
897 * of which (DESC_TYPE_RUN_LENGTH, DESC_TYPE_SMALL_BITMAP) encode the
898 * object size.
899 */
900 #define MAX_SMALL_OBJ_SIZE 8000
902 /* Functions supplied by the runtime to be called by the GC */
905 #define ALLOC_ALIGN 8
906 #define ALLOC_ALIGN_BITS 3
908 #define MOVED_OBJECTS_NUM 64
912 /*
913 * ######################################################################
914 * ######## Macros and function declarations.
915 * ######################################################################
916 */
918 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
919 if ((mword)(low) < lowest_heap_address) \
920 lowest_heap_address = (mword)(low); \
921 if ((mword)(high) > highest_heap_address) \
922 highest_heap_address = (mword)(high); \
923 } while (0)
924 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
928 {
933 }
938 /* forward declarations */
951 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type);
955 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
956 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
957 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
964 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery);
970 static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
1003 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end);
1006 //#define BINARY_PROTOCOL
1013 /*
1014 * ######################################################################
1015 * ######## GC descriptors
1016 * ######################################################################
1017 * Used to quickly get the info the GC needs about an object: size and
1018 * where the references are held.
1019 */
1020 /* objects are aligned to 8 bytes boundaries
1021 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
1022 * The low 3 bits define the type of the descriptor. The other bits
1023 * depend on the type.
1024 * As a general rule the 13 remaining low bits define the size, either
1025 * of the whole object or of the elements in the arrays. While for objects
1026 * the size is already in bytes, for arrays we need to shift, because
1027 * array elements might be smaller than 8 bytes. In case of arrays, we
1028 * use two bits to describe what the additional high bits represents,
1029 * so the default behaviour can handle element sizes less than 2048 bytes.
1030 * The high 16 bits, if 0 it means the object is pointer-free.
1031 * This design should make it easy and fast to skip over ptr-free data.
1032 * The first 4 types should cover >95% of the objects.
1033 * Note that since the size of objects is limited to 64K, larger objects
1034 * will be allocated in the large object heap.
1035 * If we want 4-bytes alignment, we need to put vector and small bitmap
1036 * inside complex.
1037 */
1039 /*
1040 * We don't use 0 so that 0 isn't a valid GC descriptor. No
1041 * deep reason for this other than to be able to identify a
1042 * non-inited descriptor for debugging.
1043 *
1044 * If an object contains no references, its GC descriptor is
1045 * always DESC_TYPE_RUN_LENGTH, without a size, no exceptions.
1046 * This is so that we can quickly check for that in
1047 * copy_object_no_checks(), without having to fetch the
1048 * object's class.
1049 */
1057 /* subtypes for arrays and vectors */
1061 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
1062 };
1064 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
1065 #define LOW_TYPE_BITS 3
1066 #define SMALL_BITMAP_SHIFT 16
1067 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
1068 #define VECTOR_INFO_SHIFT 14
1069 #define VECTOR_ELSIZE_SHIFT 3
1070 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
1071 #define MAX_ELEMENT_SIZE 0x3ff
1072 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
1073 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
1074 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
1075 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
1078 /* Root bitmap descriptors are simpler: the lower three bits describe the type
1079 * and we either have 30/62 bitmap bits or nibble-based run-length,
1080 * or a complex descriptor, or a user defined marker function.
1081 */
1084 ROOT_DESC_BITMAP,
1085 ROOT_DESC_RUN_LEN,
1086 ROOT_DESC_COMPLEX,
1087 ROOT_DESC_USER,
1090 };
1092 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1094 #define MAX_USER_DESCRIPTORS 16
1102 static int
1104 {
1110 LOCK_GC;
1112 /* linear search, so we don't have duplicates with domain load/unload
1113 * this should not be performance critical or we'd have bigger issues
1114 * (the number and size of complex descriptors should be small).
1115 */
1123 }
1124 }
1126 UNLOCK_GC;
1128 }
1129 }
1131 }
1136 }
1137 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1143 }
1144 UNLOCK_GC;
1146 }
1148 /*
1149 * Descriptor builders.
1150 */
1153 {
1155 }
1159 {
1168 num_set++;
1169 }
1170 }
1171 /*
1172 * We don't encode the size of types that don't contain
1173 * references because they might not be aligned, i.e. the
1174 * bottom two bits might be set, which would clash with the
1175 * bits we need to encode the descriptor type. Since we don't
1176 * use the encoded size to skip objects, other than for
1177 * processing remsets, in which case only the positions of
1178 * references are relevant, this is not a problem.
1179 */
1184 /* check run-length encoding first: one byte offset, one byte number of pointers
1185 * on 64 bit archs, we can have 3 runs, just one on 32.
1186 * It may be better to use nibbles.
1187 */
1190 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1194 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1196 }
1197 /* we know the 2-word header is ptr-free */
1199 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1200 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1202 }
1203 }
1204 /* we know the 2-word header is ptr-free */
1207 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1209 }
1210 /* it's a complex object ... */
1213 }
1215 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1218 {
1226 num_set++;
1227 }
1228 }
1229 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
1236 }
1237 /* Note: we also handle structs with just ref fields */
1240 }
1241 /* FIXME: try run-len first */
1242 /* Note: we can't skip the object header here, because it's not present */
1245 }
1246 }
1247 /* it's am array of complex structs ... */
1251 }
1253 /* Return the bitmap encoded by a descriptor */
1254 gsize*
1256 {
1274 }
1285 }
1286 }
1288 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1289 #define STRING_SIZE(size,str) do { \
1290 (size) = sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*)(str)) + 2; \
1291 (size) += (ALLOC_ALIGN - 1); \
1292 (size) &= ~(ALLOC_ALIGN - 1); \
1293 } while (0)
1295 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1296 (size) = ((desc) & 0xfff8) >> 1; \
1297 } while (0)
1299 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1300 (size) = ((desc) & 0xfff8) >> 1; \
1301 } while (0)
1303 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1304 #define PREFETCH(addr)
1306 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1307 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1308 if ((desc) & 0xffff0000) { \
1310 void **_objptr_end; \
1311 void **_objptr = (void**)(obj); \
1312 _objptr += ((desc) >> 16) & 0xff; \
1313 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1314 while (_objptr < _objptr_end) { \
1315 HANDLE_PTR (_objptr, (obj)); \
1316 _objptr++; \
1317 } \
1318 } \
1319 } while (0)
1321 /* a bitmap desc means that there are pointer references or we'd have
1322 * choosen run-length, instead: add an assert to check.
1323 */
1324 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1326 void **_objptr = (void**)(obj); \
1327 gsize _bmap = (desc) >> 16; \
1328 _objptr += OBJECT_HEADER_WORDS; \
1329 while (_bmap) { \
1330 if ((_bmap & 1)) { \
1331 HANDLE_PTR (_objptr, (obj)); \
1332 } \
1333 _bmap >>= 1; \
1334 ++_objptr; \
1335 } \
1336 } while (0)
1338 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1340 void **_objptr = (void**)(obj); \
1341 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1342 _objptr += OBJECT_HEADER_WORDS; \
1343 while (_bmap) { \
1344 if ((_bmap & 1)) { \
1345 HANDLE_PTR (_objptr, (obj)); \
1346 } \
1347 _bmap >>= 1; \
1348 ++_objptr; \
1349 } \
1350 } while (0)
1352 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1354 void **_objptr = (void**)(obj); \
1355 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1356 int bwords = (*bitmap_data) - 1; \
1357 void **start_run = _objptr; \
1358 bitmap_data++; \
1359 if (0) { \
1360 MonoObject *myobj = (MonoObject*)obj; \
1361 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1362 } \
1363 while (bwords-- > 0) { \
1364 gsize _bmap = *bitmap_data++; \
1365 _objptr = start_run; \
1367 while (_bmap) { \
1368 if ((_bmap & 1)) { \
1369 HANDLE_PTR (_objptr, (obj)); \
1370 } \
1371 _bmap >>= 1; \
1372 ++_objptr; \
1373 } \
1374 start_run += GC_BITS_PER_WORD; \
1375 } \
1376 } while (0)
1378 /* this one is untested */
1379 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1381 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1382 int mbwords = (*mbitmap_data++) - 1; \
1383 int el_size = mono_array_element_size (vt->klass); \
1384 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1385 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1386 if (0) \
1387 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, vt->klass->name_space, vt->klass->name); \
1388 while (e_start < e_end) { \
1389 void **_objptr = (void**)e_start; \
1390 gsize *bitmap_data = mbitmap_data; \
1391 unsigned int bwords = mbwords; \
1392 while (bwords-- > 0) { \
1393 gsize _bmap = *bitmap_data++; \
1394 void **start_run = _objptr; \
1396 while (_bmap) { \
1397 if ((_bmap & 1)) { \
1398 HANDLE_PTR (_objptr, (obj)); \
1399 } \
1400 _bmap >>= 1; \
1401 ++_objptr; \
1402 } \
1403 _objptr = start_run + GC_BITS_PER_WORD; \
1404 } \
1405 e_start += el_size; \
1406 } \
1407 } while (0)
1409 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1411 if ((vt)->desc & 0xffffc000) { \
1412 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1414 int etype = (vt)->desc & 0xc000; \
1415 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1416 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1417 void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
1419 while (p < end_refs) { \
1420 HANDLE_PTR (p, (obj)); \
1421 ++p; \
1422 } \
1423 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1424 int offset = ((vt)->desc >> 16) & 0xff; \
1425 int num_refs = ((vt)->desc >> 24) & 0xff; \
1426 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1427 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1428 while (e_start < e_end) { \
1429 void **p = (void**)e_start; \
1430 int i; \
1431 p += offset; \
1432 for (i = 0; i < num_refs; ++i) { \
1433 HANDLE_PTR (p + i, (obj)); \
1434 } \
1435 e_start += el_size; \
1436 } \
1437 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1438 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1439 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1440 while (e_start < e_end) { \
1441 void **p = (void**)e_start; \
1442 gsize _bmap = (vt)->desc >> 16; \
1444 while (_bmap) { \
1445 if ((_bmap & 1)) { \
1446 HANDLE_PTR (p, (obj)); \
1447 } \
1448 _bmap >>= 1; \
1449 ++p; \
1450 } \
1451 e_start += el_size; \
1452 } \
1453 } \
1454 } \
1455 } while (0)
1457 //#include "sgen-major-copying.c"
1460 static gboolean
1462 {
1467 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1469 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1474 /* Thread.cached_culture_info */
1480 /*
1481 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1482 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1483 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1484 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1485 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1486 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1487 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1488 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1489 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1490 */
1496 /* append_job() in threadpool.c */
1504 }
1506 static void
1508 {
1529 }
1530 }
1533 }
1537 else
1539 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1546 }
1548 #undef HANDLE_PTR
1549 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1551 static void
1553 {
1557 }
1559 #undef HANDLE_PTR
1560 #define HANDLE_PTR(ptr,obj) do { \
1561 if ((MonoObject*)*(ptr) == key) { \
1563 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1564 } \
1565 } while (0)
1567 static void
1569 {
1571 }
1573 static void
1574 scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1575 {
1581 }
1590 }
1591 }
1593 static void
1595 {
1597 }
1599 static void
1601 {
1605 }
1610 static void
1612 {
1614 }
1616 static void
1618 {
1636 start_root++;
1637 }
1643 bitmap_data++;
1652 }
1654 }
1656 }
1661 }
1666 }
1667 }
1668 }
1671 }
1673 void
1675 {
1683 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1698 }
1699 }
1700 }
1701 }
1703 /* Clear all remaining nursery fragments */
1704 static void
1706 {
1713 }
1714 }
1715 }
1717 static gboolean
1719 {
1722 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1724 }
1726 }
1728 static void
1730 {
1734 /* The object could be a proxy for an object in the domain
1735 we're deleting. */
1739 /* The server could already have been zeroed out, so
1740 we need to check for that, too. */
1745 }
1746 }
1747 }
1751 static void
1753 {
1755 }
1757 static void
1759 {
1768 /* The MonoDomain struct is allowed to hold
1769 references to objects in its own domain. */
1780 start_root++;
1781 }
1787 bitmap_data++;
1796 }
1798 }
1800 }
1805 }
1810 }
1811 }
1812 }
1814 }
1816 static void
1818 {
1821 scan_area_with_callback (nursery_section->data, nursery_section->end_data, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1823 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1827 }
1829 static gboolean
1831 {
1832 gboolean remove;
1841 }
1844 }
1846 static void
1848 {
1851 }
1853 static void
1855 {
1857 }
1859 static void
1860 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1861 {
1864 }
1866 static void
1868 {
1871 }
1873 /*
1874 * When appdomains are unloaded we can easily remove objects that have finalizers,
1875 * but all the others could still be present in random places on the heap.
1876 * We need a sweep to get rid of them even though it's going to be costly
1877 * with big heaps.
1878 * The reason we need to remove them is because we access the vtable and class
1879 * structures to know the object size and the reference bitmap: once the domain is
1880 * unloaded the point to random memory.
1881 */
1882 void
1884 {
1888 LOCK_GC;
1896 }
1901 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1902 to memory returned to the OS.*/
1908 /* We need two passes over major and large objects because
1909 freeing such objects might give their memory back to the OS
1910 (in the case of large objects) or obliterate its vtable
1911 (pinned objects with major-copying or pinned and non-pinned
1912 objects with major-mark&sweep), but we might need to
1913 dereference a pointer from an object to another object if
1914 the first object is a proxy. */
1915 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1925 else
1932 }
1935 }
1936 major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1937 major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1939 UNLOCK_GC;
1940 }
1942 static void
1944 {
1946 }
1948 /*
1949 * Tries to check if a given remset location was already added to the global remset.
1950 * It can
1951 *
1952 * A 2 entry, LRU cache of recently saw location remsets.
1953 *
1954 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1955 *
1956 * Returns TRUE is the element was added..
1957 */
1958 static gboolean
1960 {
1966 }
1971 /*Move the second to the front*/
1977 }
1982 }
1984 /*
1985 * add_to_global_remset:
1986 *
1987 * The global remset contains locations which point into newspace after
1988 * a minor collection. This can happen if the objects they point to are pinned.
1989 */
1990 static void
1992 {
2005 /*
2006 * FIXME: If an object remains pinned, we need to add it at every minor collection.
2007 * To avoid uncontrolled growth of the global remset, only add each pointer once.
2008 */
2012 }
2018 {
2023 }
2025 }
2026 }
2028 /*
2029 * FIXME: allocate before calling this function and pass the
2030 * destination address.
2031 */
2034 {
2035 static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2037 mword objsize;
2049 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %lu)\n", destination, ((MonoObject*)obj)->vtable->klass->name, (unsigned long)objsize));
2055 LAB_8:
2057 LAB_7:
2059 LAB_6:
2061 LAB_5:
2063 LAB_4:
2065 LAB_3:
2067 LAB_2:
2069 LAB_1:
2071 LAB_0:
2072 ;
2074 #if 0
2075 {
2080 "rep; movsl"
2084 );
2085 }
2086 #else
2088 #endif
2089 }
2090 /* adjust array->bounds */
2094 array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2095 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %lu, rank: %d, length: %lu\n", array, (unsigned long)objsize, vt->rank, (unsigned long)mono_array_length (array)));
2096 }
2097 /* set the forwarding pointer */
2103 }
2106 }
2111 }
2113 }
2115 /*
2116 * This is how the copying happens from the nursery to the old generation.
2117 * We assume that at this time all the pinned objects have been identified and
2118 * marked as such.
2119 * We run scan_object() for each pinned object so that each referenced
2120 * objects if possible are copied. The new gray objects created can have
2121 * scan_object() run on them right away, too.
2122 * Then we run copy_object() for the precisely tracked roots. At this point
2123 * all the roots are either gray or black. We run scan_object() on the gray
2124 * objects until no more gray objects are created.
2125 * At the end of the process we walk again the pinned list and we unmark
2126 * the pinned flag. As we go we also create the list of free space for use
2127 * in the next allocation runs.
2128 *
2129 * We need to remember objects from the old generation that point to the new one
2130 * (or just addresses?).
2131 *
2132 * copy_object could be made into a macro once debugged (use inline for now).
2133 */
2137 {
2148 }
2152 /*
2153 * Before we can copy the object we must make sure that we are
2154 * allowed to, i.e. that the object not pinned or not already
2155 * forwarded.
2156 */
2164 }
2170 }
2175 }
2177 #undef HANDLE_PTR
2178 #define HANDLE_PTR(ptr,obj) do { \
2179 void *__old = *(ptr); \
2180 void *__copy; \
2181 if (__old) { \
2182 copy_object ((ptr)); \
2183 __copy = *(ptr); \
2184 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2185 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2186 add_to_global_remset ((ptr)); \
2187 } \
2188 } while (0)
2190 /*
2191 * Scan the object pointed to by @start for references to
2192 * other objects between @from_start and @from_end and copy
2193 * them to the gray_objects area.
2194 */
2195 static void
2197 {
2201 }
2203 /*
2204 * scan_vtype:
2205 *
2206 * Scan the valuetype pointed to by START, described by DESC for references to
2207 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2208 * Returns a pointer to the end of the object.
2209 */
2212 {
2215 /* The descriptors include info about the MonoObject header as well */
2230 // FIXME:
2234 // The other descriptors can't happen with vtypes
2237 }
2239 }
2241 #undef HANDLE_PTR
2242 #define HANDLE_PTR(ptr,obj) do { \
2243 void *__old = *(ptr); \
2244 void *__copy; \
2245 if (__old) { \
2246 major_copy_or_mark_object ((ptr)); \
2247 __copy = *(ptr); \
2248 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2249 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2250 add_to_global_remset ((ptr)); \
2251 } \
2252 } while (0)
2254 static void
2256 {
2260 }
2262 /*
2263 * drain_gray_stack:
2264 *
2265 * Scan objects in the gray stack until the stack is empty. This should be called
2266 * frequently after each object is copied, to achieve better locality and cache
2267 * usage.
2268 */
2271 {
2279 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2281 }
2287 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2289 }
2290 }
2291 }
2293 /*
2294 * Addresses from start to end are already sorted. This function finds
2295 * the object header for each address and pins the object. The
2296 * addresses must be inside the passed section. The (start of the)
2297 * address array is overwritten with the addresses of the actually
2298 * pinned objects. Return the number of pinned objects.
2299 */
2300 static int
2301 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2302 {
2313 /* the range check should be reduntant */
2316 /* multiple pointers to the same object */
2318 start++;
2320 }
2330 }
2333 }
2336 /* now addr should be in an object a short distance from search_start
2337 * Note that search_start must point to zeroed mem or point to an object.
2338 */
2347 }
2352 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2354 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2355 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2361 count++;
2363 }
2364 /* skip to the next object */
2367 /* we either pinned the correct object or we ignored the addr because
2368 * it points to unused zeroed memory.
2369 */
2371 }
2372 start++;
2373 }
2374 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2376 }
2378 static void
2380 {
2389 }
2390 }
2392 /* Sort the addresses in array in increasing order.
2393 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
2394 */
2395 static void
2397 {
2414 }
2415 }
2439 }
2440 }
2441 }
2445 {
2448 gpointer next;
2451 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2453 }
2455 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2456 }
2458 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2459 static void
2461 {
2463 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2464 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2473 cur++;
2474 start++;
2475 };
2478 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2480 }
2482 /*
2483 * Scan the memory between start and end and queue values which could be pointers
2484 * to the area between start_nursery and end_nursery for later consideration.
2485 * Typically used for thread stacks.
2486 */
2487 static void
2488 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2489 {
2493 /*
2494 * *start can point to the middle of an object
2495 * note: should we handle pointing at the end of an object?
2496 * pinning in C# code disallows pointing at the end of an object
2497 * but there is some small chance that an optimizing C compiler
2498 * may keep the only reference to an object by pointing
2499 * at the end of it. We ignore this small chance for now.
2500 * Pointers to the end of an object are indistinguishable
2501 * from pointers to the start of the next object in memory
2502 * so if we allow that we'd need to pin two objects...
2503 * We queue the pointer in an array, the
2504 * array will then be sorted and uniqued. This way
2505 * we can coalesce several pinning pointers and it should
2506 * be faster since we'd do a memory scan with increasing
2507 * addresses. Note: we can align the address to the allocation
2508 * alignment, so the unique process is more effective.
2509 */
2517 count++;
2518 }
2519 start++;
2520 }
2521 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2522 }
2524 /*
2525 * Debugging function: find in the conservative roots where @obj is being pinned.
2526 */
2529 {
2535 /* if desc is non-null it has precise info */
2540 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2541 }
2542 start++;
2543 }
2544 }
2545 }
2546 }
2548 }
2550 /*
2551 * The first thing we do in a collection is to identify pinned objects.
2552 * This function considers all the areas of memory that need to be
2553 * conservatively scanned.
2554 */
2555 static void
2557 {
2560 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2561 /* objects pinned from the API are inside these roots */
2565 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2566 }
2567 }
2568 /* now deal with the thread stacks
2569 * in the future we should be able to conservatively scan only:
2570 * *) the cpu registers
2571 * *) the unmanaged stack frames
2572 * *) the _last_ managed stack frame
2573 * *) pointers slots in managed frames
2574 */
2578 }
2580 /*
2581 * The memory area from start_root to end_root contains pointers to objects.
2582 * Their position is precisely described by @desc (this means that the pointer
2583 * can be either NULL or the pointer to the start of an object).
2584 * This functions copies them to to_space updates them.
2585 */
2586 static void
2587 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2588 {
2597 }
2599 start_root++;
2600 }
2606 bitmap_data++;
2615 }
2618 }
2620 }
2622 }
2627 }
2632 }
2633 }
2637 {
2643 }
2647 }
2649 /* size must be a power of 2 */
2652 {
2653 /* Allocate twice the memory to be able to put the block on an aligned address */
2660 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2668 }
2670 /*
2671 * Allocate and setup the data structures needed to be able to allocate objects
2672 * in the nursery. The nursery is stored in nursery_section.
2673 */
2674 static void
2676 {
2685 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)nursery_size));
2686 /* later we will alloc a larger area for the nursery but only activate
2687 * what we need. The rest will be used as expansion if we have too many pinned
2688 * objects in the existing nursery.
2689 */
2690 /* FIXME: handle OOM */
2695 #ifdef ALIGN_NURSERY
2697 #else
2699 #endif
2705 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)nursery_size, (unsigned long)total_alloc));
2710 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2717 /* Setup the single first large fragment */
2723 /* FIXME: frag here is lost */
2724 }
2726 static void
2733 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2735 }
2736 }
2739 /*
2740 * We found a fragment of free memory in the nursery: memzero it and if
2741 * it is big enough, add it to the list of fragments that can be used for
2742 * allocation.
2743 */
2744 static void
2746 {
2748 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2750 /* memsetting just the first chunk start is bound to provide better cache locality */
2753 /* Not worth dealing with smaller fragments: need to tune */
2763 /* Clear unused fragments, pinning depends on this */
2764 /*TODO place an int[] here instead of the memset if size justify it*/
2766 }
2767 }
2771 {
2776 }
2777 }
2781 {
2786 }
2787 }
2791 {
2796 }
2797 }
2799 static void
2801 {
2806 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
2808 /*
2809 * We copied all the reachable objects. Now it's the time to copy
2810 * the objects that were not referenced by the roots, but by the copied objects.
2811 * we built a stack of objects pointed to by gray_start: they are
2812 * additional roots and we may add more items as we go.
2813 * We loop until gray_start == gray_objects which means no more objects have
2814 * been added. Note this is iterative: no recursion is involved.
2815 * We need to walk the LO list as well in search of marked big objects
2816 * (use a flag since this is needed only on major collections). We need to loop
2817 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2818 * To achieve better cache locality and cache usage, we drain the gray stack
2819 * frequently, after each object is copied, and just finish the work here.
2820 */
2824 /* walk the finalization queue and move also the objects that need to be
2825 * finalized: use the finalized objects as new roots so the objects they depend
2826 * on are also not reclaimed. As with the roots above, only objects in the nursery
2827 * are marked/copied.
2828 * We need a loop here, since objects ready for finalizers may reference other objects
2829 * that are fin-ready. Speedup with a flag?
2830 */
2832 /*
2833 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2834 * before processing finalizable objects to avoid finalizing reachable values.
2835 *
2836 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2837 * while they are been finalized.
2838 */
2851 /* drain the new stack that might have been created */
2856 /*
2857 * Clear ephemeron pairs with unreachable keys.
2858 * We pass the copy func so we can figure out if an array was promoted or not.
2859 */
2863 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2865 /*
2866 * handle disappearing links
2867 * Note we do this after checking the finalization queue because if an object
2868 * survives (at least long enough to be finalized) we don't clear the link.
2869 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2870 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2871 * called.
2872 */
2881 }
2884 }
2886 static void
2888 {
2894 }
2895 }
2896 }
2898 static void
2900 {
2905 }
2909 static void
2911 {
2921 }
2924 /* clear scan starts */
2928 /* remove the pin bit from pinned objects */
2930 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2938 }
2947 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2948 }
2950 }
2954 /* Clear TLABs for all threads */
2956 }
2958 static void
2959 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type)
2960 {
2965 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2966 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
2967 }
2968 }
2969 }
2971 static void
2973 {
2974 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2975 }
2977 static void
2979 {
2986 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2989 guint size;
2996 }
2999 }
3012 /*
3013 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3014 start - section->data,
3015 vt->klass->name_space, vt->klass->name,
3016 size);
3017 */
3021 }
3026 }
3028 static void
3030 {
3036 /*
3037 * Python's XML parser is too stupid to parse angle brackets
3038 * in strings, so we just ignore them;
3039 */
3045 }
3058 else
3061 }
3063 }
3065 static void
3067 {
3068 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3082 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%lld\"/>\n", pinned_chunk_bytes_alloced);
3083 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%lld\"/>\n", large_internal_bytes_alloced);
3084 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3086 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3087 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3088 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3089 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3106 }
3108 static void
3110 {
3116 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
3117 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
3118 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
3119 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
3120 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
3121 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
3122 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
3123 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
3125 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
3126 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
3127 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
3128 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
3129 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
3130 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
3131 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
3132 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
3133 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
3134 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_free_bigobjs);
3135 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_los_sweep);
3136 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
3137 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
3139 #ifdef HEAVY_STATISTICS
3140 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3141 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3142 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3143 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3144 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3145 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3146 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3147 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3149 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3150 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
3151 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3152 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
3153 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
3155 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3156 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3157 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3158 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3160 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
3161 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
3163 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
3164 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
3165 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
3167 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
3168 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
3170 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3171 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3172 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3173 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3174 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3175 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
3176 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3177 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
3179 #endif
3182 }
3184 static gboolean
3186 {
3188 return minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_alloced > minor_collection_allowance;
3189 }
3191 /*
3192 * Collect objects in the nursery. Returns whether to trigger a major
3193 * collection.
3194 */
3195 static gboolean
3197 {
3214 /* FIXME: optimize later to use the higher address where an object can be present */
3217 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3221 /* world must be stopped already */
3225 /* Pinning depends on this */
3240 num_minor_gcs++;
3245 /* pin from pinned handles */
3248 /* identify pinned objects */
3250 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3255 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
3261 /*
3262 * walk all the roots and copy the young objects to the old generation,
3263 * starting from to_space
3264 */
3267 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3276 /* registered roots, this includes static fields */
3281 /* thread data */
3291 /* walk the pin_queue, build up the fragment list of free memory, unmark
3292 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3293 * next allocations.
3294 */
3298 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total));
3311 /* prepare the pin queue for the next collection */
3315 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3317 }
3329 }
3331 static void
3333 {
3339 /* FIXME: only use these values for the precise scan
3340 * note that to_space pointers should be excluded anyway...
3341 */
3348 /*
3349 * A domain could have been freed, resulting in
3350 * los_memory_usage being less than last_los_memory_usage.
3351 */
3355 //count_ref_nonref_objs ();
3356 //consistency_check ();
3365 num_major_gcs++;
3368 /* world must be stopped already */
3372 /* Pinning depends on this */
3382 /* we should also coalesce scanning from sections close to each other
3383 * and deal with pointers outside of the sections later.
3384 */
3385 /* The remsets are not useful for a major collection */
3395 /*
3396 * pin_queue now contains all candidate pointers, sorted and
3397 * uniqued. We must do two passes now to figure out which
3398 * objects are pinned.
3399 *
3400 * The first is to find within the pin_queue the area for each
3401 * section. This requires that the pin_queue be sorted. We
3402 * also process the LOS objects and pinned chunks here.
3403 *
3404 * The second, destructive, pass is to reduce the section
3405 * areas to pointers to the actually pinned objects.
3406 */
3408 /* first pass for the sections */
3411 /* identify possible pointers to the insize of large objects */
3415 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3418 /* FIXME: only enqueue if object has references */
3421 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3422 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size));
3423 }
3424 }
3425 /* second pass for the sections */
3431 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3441 /* registered roots, this includes static fields */
3442 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL);
3443 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER);
3447 /* Threads */
3448 /* FIXME: This is the wrong place for this, because it does
3449 pinning */
3457 /* scan the list of objects ready for finalization */
3467 /* all the objects in the heap */
3472 /* sweep the big objects list */
3479 /* not referenced anywhere, so we can free it */
3482 else
3488 }
3491 }
3506 /* walk the pin_queue, build up the fragment list of free memory, unmark
3507 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3508 * next allocations.
3509 */
3521 /* prepare the pin queue for the next collection */
3524 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3526 }
3535 /*
3536 * We aim to allow the allocation of as many sections as is
3537 * necessary to reclaim save_target sections in the next
3538 * collection. We assume the collection pattern won't change.
3539 * In the last cycle, we had num_major_sections_saved for
3540 * minor_collection_sections_alloced. Assuming things won't
3541 * change, this must be the same ratio as save_target for
3542 * allowance_target, i.e.
3543 *
3544 * num_major_sections_saved save_target
3545 * --------------------------------- == ----------------
3546 * minor_collection_sections_alloced allowance_target
3547 *
3548 * hence:
3549 */
3550 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_memory_alloced) / (double)(num_major_sections_saved * MAJOR_SECTION_SIZE + los_memory_saved));
3552 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * MAJOR_SECTION_SIZE + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3563 //consistency_check ();
3564 }
3566 static void
3568 {
3572 }
3577 }
3579 /*
3580 * When deciding if it's better to collect or to expand, keep track
3581 * of how much garbage was reclaimed with the last collection: if it's too
3582 * little, expand.
3583 * This is called when we could not allocate a small object.
3584 */
3587 {
3593 }
3598 DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc, (unsigned long)los_memory_usage));
3600 /* this also sets the proper pointers for the next allocation */
3603 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3604 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3606 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3607 }
3609 }
3610 }
3611 //report_internal_mem_usage ();
3612 }
3614 /*
3615 * ######################################################################
3616 * ######## Memory allocation from the OS
3617 * ######################################################################
3618 * This section of code deals with getting memory from the OS and
3619 * allocating memory for GC-internal data structures.
3620 * Internal memory can be handled with a freelist for small objects.
3621 */
3623 /*
3624 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3625 * This must not require any lock.
3626 */
3629 {
3638 }
3640 /*
3641 * Free the memory returned by get_os_memory (), returning it to the OS.
3642 */
3643 static void
3645 {
3647 }
3649 /*
3650 * Debug reporting.
3651 */
3652 static void
3659 free_pages++;
3660 }
3661 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3669 num_free++;
3671 }
3674 }
3676 }
3678 /*
3679 * Debug reporting.
3680 */
3689 }
3692 }
3694 /*
3695 * Find the slot number in the freelist for memory chunks that
3696 * can contain @size objects.
3697 */
3698 static int
3700 {
3702 /* do a binary search or lookup table later. */
3706 }
3709 }
3711 /*
3712 * Build a free list for @size memory chunks from the memory area between
3713 * start_page and end_page.
3714 */
3715 static void
3717 {
3720 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3726 count++;
3729 }
3731 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3732 }
3736 {
3748 /* setup the bookeeping fields */
3761 /* allocate the first page to the freelist */
3763 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3766 }
3768 /* assumes freelist for slot is empty, so try to alloc a new page */
3771 {
3778 }
3785 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
3787 }
3788 /* try again */
3793 }
3795 }
3797 /* used for the GC-internal data structures */
3800 {
3816 }
3829 }
3830 }
3836 }
3837 }
3839 /* FIXME: handle OOM */
3845 }
3847 static void
3849 {
3855 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3856 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
3867 }
3868 }
3873 }
3875 /*
3876 * ######################################################################
3877 * ######## Object allocation
3878 * ######################################################################
3879 * This section of code deals with allocating memory for objects.
3880 * There are several ways:
3881 * *) allocate large objects
3882 * *) allocate normal objects
3883 * *) fast lock-free allocation
3884 * *) allocation of pinned objects
3885 */
3887 static void
3889 {
3890 /* remove from the list */
3893 else
3898 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3901 }
3903 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3904 * an object of size @size
3905 * Return FALSE if not found (which means we need a collection)
3906 */
3907 static gboolean
3909 {
3911 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3914 /* Clear the remaining space, pinning depends on this */
3922 }
3924 }
3926 }
3928 /*
3929 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3930 * This improves nursery usage.
3931 */
3932 static int
3934 {
3936 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
3939 /* Clear the remaining space, pinning depends on this */
3950 }
3955 }
3961 else
3970 }
3973 }
3977 {
3982 }
3985 }
3987 /*
3988 * Provide a variant that takes just the vtable for small fixed-size objects.
3989 * The aligned size is already computed and stored in vt->gc_descr.
3990 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3991 * processing. We can keep track of where objects start, for example,
3992 * so when we scan the thread stacks for pinned objects, we can start
3993 * a search for the pinned object in SCAN_START_SIZE chunks.
3994 */
3997 {
3998 /* FIXME: handle OOM */
4001 TLAB_ACCESS_INIT;
4006 else
4020 // FIXME:
4022 }
4023 }
4024 }
4026 /*
4027 * We must already have the lock here instead of after the
4028 * fast path because we might be interrupted in the fast path
4029 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4030 * and we'll end up allocating an object in a fragment which
4031 * no longer belongs to us.
4032 *
4033 * The managed allocator does not do this, but it's treated
4034 * specially by the world-stopping code.
4035 */
4040 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4043 /* FIXME: handle overflow */
4048 /* Fast path */
4050 /*
4051 * FIXME: We might need a memory barrier here so the change to tlab_next is
4052 * visible before the vtable store.
4053 */
4055 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4063 }
4065 /* Slow path */
4067 /* there are two cases: the object is too big or we run out of space in the TLAB */
4068 /* we also reach here when the thread does its first allocation after a minor
4069 * collection, since the tlab_ variables are initialized to NULL.
4070 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4071 * objects that need finalizers can have the high bit set in their size
4072 * so the above check fails and we can readily add the object to the queue.
4073 * This avoids taking again the GC lock when registering, but this is moot when
4074 * doing thread-local allocation, so it may not be a good idea.
4075 */
4078 /*
4079 * Run out of space in the TLAB. When this happens, some amount of space
4080 * remains in the TLAB, but not enough to satisfy the current allocation
4081 * request. Currently, we retire the TLAB in all cases, later we could
4082 * keep it if the remaining space is above a treshold, and satisfy the
4083 * allocation directly from the nursery.
4084 */
4086 /* when running in degraded mode, we continue allocing that way
4087 * for a while, to decrease the number of useless nursery collections.
4088 */
4093 }
4095 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
4097 /* Allocate directly from the nursery */
4105 }
4106 }
4107 }
4112 // no space left
4114 }
4122 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4136 }
4137 }
4138 }
4139 }
4141 /* Allocate a new TLAB from the current nursery fragment */
4151 /* Allocate from the TLAB */
4156 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4157 }
4159 /* Reached tlab_temp_end */
4161 /* record the scan start so we can find pinned objects more easily */
4162 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4163 /* we just bump tlab_temp_end as well */
4165 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4166 }
4167 }
4169 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4174 }
4178 {
4181 TLAB_ACCESS_INIT;
4188 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4191 /* FIXME: handle overflow */
4196 /* Fast path */
4198 /*
4199 * FIXME: We might need a memory barrier here so the change to tlab_next is
4200 * visible before the vtable store.
4201 */
4206 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4214 }
4215 }
4217 }
4221 {
4223 #ifndef DISABLE_CRITICAL_REGION
4224 TLAB_ACCESS_INIT;
4225 ENTER_CRITICAL_REGION;
4228 EXIT_CRITICAL_REGION;
4230 }
4231 EXIT_CRITICAL_REGION;
4232 #endif
4233 LOCK_GC;
4235 UNLOCK_GC;
4237 }
4241 {
4243 #ifndef DISABLE_CRITICAL_REGION
4244 TLAB_ACCESS_INIT;
4245 ENTER_CRITICAL_REGION;
4249 EXIT_CRITICAL_REGION;
4251 }
4252 EXIT_CRITICAL_REGION;
4253 #endif
4255 LOCK_GC;
4260 UNLOCK_GC;
4263 }
4266 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
4267 {
4271 LOCK_GC;
4279 UNLOCK_GC;
4282 }
4286 {
4288 #ifndef DISABLE_CRITICAL_REGION
4289 TLAB_ACCESS_INIT;
4290 ENTER_CRITICAL_REGION;
4294 EXIT_CRITICAL_REGION;
4296 }
4297 EXIT_CRITICAL_REGION;
4298 #endif
4300 LOCK_GC;
4305 UNLOCK_GC;
4308 }
4310 /*
4311 * To be used for interned strings and possibly MonoThread, reflection handles.
4312 * We may want to explicitly free these objects.
4313 */
4316 {
4317 /* FIXME: handle OOM */
4321 LOCK_GC;
4323 /* large objects are always pinned anyway */
4328 }
4329 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4332 UNLOCK_GC;
4334 }
4336 /*
4337 * ######################################################################
4338 * ######## Finalization support
4339 * ######################################################################
4340 */
4342 /*
4343 * this is valid for the nursery: if the object has been forwarded it means it's
4344 * still refrenced from a root. If it is pinned it's still alive as well.
4345 * Return TRUE if @obj is ready to be finalized.
4346 */
4347 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4349 static gboolean
4351 {
4362 }
4364 static void
4372 }
4373 }
4375 /* LOCKING: requires that the GC lock is held */
4376 static void
4378 {
4394 }
4395 }
4399 }
4401 /* LOCKING: requires that the GC lock is held */
4402 static void
4404 {
4407 }
4409 /* LOCKING: requires that the GC lock is held */
4410 static void
4412 {
4424 if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
4431 /* remove and put in fin_ready_list */
4434 else
4437 num_ready_finalizers++;
4440 /* Make it survive */
4443 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4451 /* remove from the list */
4454 else
4460 /* insert it into the major hash */
4468 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4473 /* update pointer */
4474 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4476 }
4477 }
4478 }
4481 }
4482 }
4483 }
4485 static int
4487 {
4488 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
4492 }
4494 /* LOCKING: requires that the GC lock is held */
4495 static void
4497 {
4508 else
4516 }
4517 }
4518 }
4520 /* LOCKING: requires that the GC lock is held */
4521 static void
4523 {
4540 else
4547 }
4553 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
4556 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
4569 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4571 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4577 }
4583 }
4587 }
4588 }
4589 }
4592 }
4593 }
4595 /* LOCKING: requires that the GC lock is held */
4596 static int
4598 {
4609 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
4613 /*It has to be alive*/
4617 }
4632 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4634 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4644 }
4645 }
4646 }
4647 }
4651 }
4653 /* LOCKING: requires that the GC lock is held */
4654 static void
4656 {
4674 /* remove from list */
4677 else
4689 /* Update pointer if it's moved. If the object
4690 * has been moved out of the nursery, we need to
4691 * remove the link from the minor hash table to
4692 * the major one.
4693 *
4694 * FIXME: what if an object is moved earlier?
4695 */
4700 /* remove from list */
4703 else
4713 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4717 /* We set the track resurrection bit to
4718 * FALSE if the object is to be finalized
4719 * so that the object can be collected in
4720 * the next cycle (i.e. after it was
4721 * finalized).
4722 */
4725 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4726 }
4727 }
4728 }
4731 }
4732 }
4733 }
4735 /* LOCKING: requires that the GC lock is held */
4736 static void
4738 {
4753 else
4761 }
4765 }
4768 }
4769 }
4770 }
4772 /* LOCKING: requires that the GC lock is held */
4773 static int
4776 {
4790 /* remove and put in out_array */
4793 else
4798 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4803 }
4806 }
4807 }
4809 }
4811 /**
4812 * mono_gc_finalizers_for_domain:
4813 * @domain: the unloading appdomain
4814 * @out_array: output array
4815 * @out_size: size of output array
4816 *
4817 * Store inside @out_array up to @out_size objects that belong to the unloading
4818 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4819 * until it returns 0.
4820 * The items are removed from the finalizer data structure, so the caller is supposed
4821 * to finalize them.
4822 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4823 */
4824 int
4826 {
4829 LOCK_GC;
4834 }
4835 UNLOCK_GC;
4838 }
4840 static void
4842 {
4845 mword finalizable_hash_size;
4852 LOCK_GC;
4861 /* remove from the list */
4864 else
4867 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4869 }
4870 UNLOCK_GC;
4872 }
4874 }
4876 /* request to deregister, but already out of the list */
4877 UNLOCK_GC;
4879 }
4885 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4886 UNLOCK_GC;
4887 }
4889 void
4891 {
4894 else
4896 }
4898 static void
4900 {
4909 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4916 }
4917 }
4921 }
4923 /* LOCKING: assumes the GC lock is held */
4924 static void
4926 {
4937 }
4938 /* FIXME: add check that link is not in the heap */
4943 /* link already added */
4945 /* NULL obj means remove */
4949 else
4952 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4957 }
4959 }
4961 }
4970 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4971 }
4973 /* LOCKING: assumes the GC lock is held */
4974 static void
4976 {
4982 else
4984 }
4985 }
4987 int
4989 {
4994 /* FIXME: batch to reduce lock contention */
4996 LOCK_GC;
5001 /* We have finalized entry in the last
5002 interation, now we need to remove it from
5003 the list. */
5011 }
5014 }
5016 /* Now look for the first non-null entry. */
5018 ;
5024 ;
5025 }
5029 num_ready_finalizers--;
5033 }
5035 UNLOCK_GC;
5041 count++;
5042 /* the object is on the stack so it is pinned */
5043 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5045 }
5048 }
5050 gboolean
5052 {
5054 }
5056 /* Negative value to remove */
5057 void
5059 {
5060 /* FIXME: Use interlocked functions */
5061 LOCK_GC;
5063 UNLOCK_GC;
5064 }
5066 /*
5067 * ######################################################################
5068 * ######## registered roots support
5069 * ######################################################################
5070 */
5072 static void
5074 {
5089 }
5090 }
5094 }
5098 {
5103 /* we allow changing the size and the descriptor (for thread statics etc) */
5106 }
5107 }
5110 }
5112 /*
5113 * We do not coalesce roots.
5114 */
5115 static int
5117 {
5121 LOCK_GC;
5125 }
5128 /* we allow changing the size and the descriptor (for thread statics etc) */
5137 UNLOCK_GC;
5139 }
5140 }
5151 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5153 UNLOCK_GC;
5155 }
5156 UNLOCK_GC;
5158 }
5160 int
5162 {
5163 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5164 }
5166 int
5168 {
5170 }
5172 void
5174 {
5179 LOCK_GC;
5188 else
5192 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5195 }
5198 }
5199 }
5200 UNLOCK_GC;
5201 }
5203 /*
5204 * ######################################################################
5205 * ######## Thread handling (stop/start code)
5206 * ######################################################################
5207 */
5209 /* FIXME: handle large/small config */
5210 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5214 #if USE_SIGNAL_BASED_START_STOP_WORLD
5223 /* LOCKING: assumes the GC lock is held */
5224 SgenThreadInfo**
5226 {
5228 }
5230 SgenThreadInfo*
5232 {
5239 }
5241 }
5243 static void
5245 {
5250 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5255 }
5257 /*
5258 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5259 * have cross-domain checks in the write barrier.
5260 */
5261 //#define XDOMAIN_CHECKS_IN_WBARRIER
5263 #ifndef BINARY_PROTOCOL
5264 #ifndef HEAVY_STATISTICS
5265 #define MANAGED_ALLOCATION
5266 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5267 #define MANAGED_WBARRIER
5268 #endif
5269 #endif
5270 #endif
5272 static gboolean
5275 void
5277 {
5284 }
5285 }
5286 }
5287 }
5289 static int
5291 {
5298 /* restart all threads that stopped in the
5299 allocator */
5307 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5309 #else
5311 #endif
5316 }
5318 /* we set the stopped_ip to
5319 NULL for threads which
5320 we're not restarting so
5321 that we can easily identify
5322 the others */
5325 }
5326 }
5327 }
5328 /* if no threads were restarted, we're done */
5332 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5333 /* mach thread_resume is synchronous so we dont need to wait for them */
5334 #else
5335 /* wait for the threads to signal their restart */
5337 #endif
5345 }
5347 /* stop them again */
5352 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5354 #else
5356 #endif
5361 }
5362 }
5363 }
5364 /* some threads might have died */
5366 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5367 /* mach thread_resume is synchronous so we dont need to wait for them */
5368 #else
5369 /* wait for the threads to signal their suspension
5370 again */
5372 #endif
5373 }
5376 }
5378 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5379 static void
5381 {
5383 pthread_t id;
5387 gpointer stack_start;
5394 /* duplicate signal */
5398 }
5399 #ifdef HAVE_KW_THREAD
5400 /* update the remset info in the thread data structure */
5402 #endif
5404 /* If stack_start is not within the limits, then don't set it
5405 in info and we will be restarted. */
5413 }
5415 /* Notify the JIT */
5419 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5420 /* notify the waiting thread */
5424 /* wait until we receive the restart signal */
5430 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5431 /* notify the waiting thread */
5435 }
5437 static void
5439 {
5445 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5448 }
5450 static void
5452 {
5453 LOCK_INTERRUPTION;
5454 }
5456 static void
5458 {
5459 UNLOCK_INTERRUPTION;
5460 }
5465 /* LOCKING: assumes the GC lock is held */
5466 static int
5468 {
5475 global_stop_count++;
5476 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5483 }
5485 /* LOCKING: assumes the GC lock is held */
5486 static int
5488 {
5494 /* notify the profiler of the leftovers */
5499 }
5500 }
5505 }
5506 }
5514 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5516 }
5520 void
5522 {
5524 }
5526 MonoGCCallbacks *
5528 {
5530 }
5532 /* Variables holding start/end nursery so it won't have to be passed at every call */
5535 void
5537 {
5538 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5539 }
5543 {
5546 else
5549 }
5551 /*
5552 * Mark from thread stacks and registers.
5553 */
5554 static void
5556 {
5566 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5568 }
5569 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5571 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5573 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5578 }
5579 }
5580 }
5582 static void
5584 {
5596 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5597 }
5598 start++;
5599 }
5601 /* FIXME: check info->stopped_regs */
5602 }
5603 }
5604 }
5606 static gboolean
5608 {
5615 }
5619 {
5621 mword count;
5622 mword desc;
5627 /* FIXME: exclude stack locations */
5631 //__builtin_prefetch (ptr);
5637 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5639 /*
5640 * If the object is pinned, each reference to it from nonpinned objects
5641 * becomes part of the global remset, which can grow very large.
5642 */
5643 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5645 }
5648 }
5657 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5661 }
5678 }
5681 }
5683 }
5685 #ifdef HEAVY_STATISTICS
5688 {
5706 }
5720 }
5721 }
5724 }
5726 static void
5728 {
5739 }
5740 }
5752 }
5753 }
5770 }
5775 }
5776 #endif
5778 static void
5780 {
5783 }
5785 static void
5787 {
5794 #ifdef HEAVY_STATISTICS
5796 #endif
5798 /* the global one */
5800 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5805 /*Ignore previously processed remset.*/
5809 }
5813 /*
5814 * Clear global remsets of locations which no longer point to the
5815 * nursery. Otherwise, they could grow indefinitely between major
5816 * collections.
5817 *
5818 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5819 */
5823 }
5824 }
5826 /* Truncate the remset */
5828 }
5830 /* the generic store ones */
5839 }
5844 }
5847 /* the per-thread ones */
5853 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5856 }
5863 }
5864 }
5866 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
5868 }
5869 }
5871 /* the freed thread ones */
5875 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5878 }
5883 }
5884 }
5886 /*
5887 * Clear the info in the remembered sets: we're doing a major collection, so
5888 * the per-thread ones are not needed and the global ones will be reconstructed
5889 * during the copy.
5890 */
5891 static void
5893 {
5898 /* the global list */
5906 }
5907 }
5908 /* the generic store ones */
5913 }
5914 /* the per-thread ones */
5924 }
5925 }
5927 }
5928 }
5930 /* the freed thread ones */
5936 }
5937 }
5939 /*
5940 * Clear the thread local TLAB variables for all threads.
5941 */
5942 static void
5944 {
5950 /* A new TLAB will be allocated when the thread does its first allocation */
5955 }
5956 }
5957 }
5959 /* LOCKING: assumes the GC lock is held */
5962 {
5965 #ifndef HAVE_KW_THREAD
5967 #endif
5973 #ifndef HAVE_KW_THREAD
5978 #else
5980 #endif
5999 #ifdef HAVE_KW_THREAD
6002 #endif
6004 /* try to get it with attributes first */
6005 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6006 {
6009 pthread_attr_t attr;
6015 }
6016 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6019 #else
6020 {
6021 /* FIXME: we assume the stack grows down */
6026 }
6027 #endif
6029 #ifdef HAVE_KW_THREAD
6031 #endif
6033 /* hash into the table */
6040 #ifdef HAVE_KW_THREAD
6042 #endif
6044 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6047 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6053 }
6055 static void
6057 {
6058 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6062 }
6064 static void
6066 {
6082 }
6087 }
6091 ;
6096 }
6097 }
6102 }
6104 static void
6106 {
6108 LOCK_GC;
6110 UNLOCK_GC;
6111 }
6113 gboolean
6115 {
6118 LOCK_GC;
6123 UNLOCK_GC;
6125 }
6127 #if USE_PTHREAD_INTERCEPT
6129 #undef pthread_create
6130 #undef pthread_join
6131 #undef pthread_detach
6137 MonoSemType registered;
6142 {
6150 LOCK_GC;
6152 UNLOCK_GC;
6157 /*
6158 * this is done by the pthread key dtor
6159 LOCK_GC;
6160 unregister_current_thread ();
6161 UNLOCK_GC;
6162 */
6165 }
6167 int
6168 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6169 {
6184 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6185 }
6186 }
6190 }
6192 int
6194 {
6196 }
6198 int
6200 {
6202 }
6206 /*
6207 * ######################################################################
6208 * ######## Write barriers
6209 * ######################################################################
6210 */
6214 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6218 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6220 }
6222 /*
6223 * Note: the write barriers first do the needed GC work and then do the actual store:
6224 * this way the value is visible to the conservative GC scan after the write barrier
6225 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6226 * the conservative scan, otherwise by the remembered set scan.
6227 */
6228 void
6230 {
6232 TLAB_ACCESS_INIT;
6237 }
6239 LOCK_GC;
6244 UNLOCK_GC;
6246 }
6250 #ifdef HAVE_KW_THREAD
6252 #endif
6255 UNLOCK_GC;
6256 }
6258 void
6260 {
6262 TLAB_ACCESS_INIT;
6267 }
6269 LOCK_GC;
6274 UNLOCK_GC;
6276 }
6280 #ifdef HAVE_KW_THREAD
6282 #endif
6285 UNLOCK_GC;
6286 }
6288 void
6290 {
6292 TLAB_ACCESS_INIT;
6294 LOCK_GC;
6297 UNLOCK_GC;
6299 }
6305 UNLOCK_GC;
6307 }
6311 #ifdef HAVE_KW_THREAD
6313 #endif
6316 UNLOCK_GC;
6317 }
6321 static void
6323 {
6327 }
6328 }
6330 /* for use in the debugger */
6334 {
6343 }
6348 }
6350 /*
6351 * Very inefficient, but this is debugging code, supposed to
6352 * be called from gdb, so we don't care.
6353 */
6355 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6357 }
6359 static void
6361 {
6363 TLAB_ACCESS_INIT;
6371 }
6373 void
6375 {
6378 TLAB_ACCESS_INIT;
6382 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6383 /* FIXME: ptr_in_heap must be called with the GC lock held */
6387 LOCK_GC;
6393 }
6394 UNLOCK_GC;
6395 }
6396 #endif
6398 LOCK_GC;
6405 UNLOCK_GC;
6407 }
6411 /* This simple optimization eliminates a sizable portion of
6412 entries. Comparing it to the last but one entry as well
6413 doesn't eliminate significantly more entries. */
6415 UNLOCK_GC;
6417 }
6428 }
6432 UNLOCK_GC;
6433 }
6435 void
6437 {
6438 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6442 }
6444 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
6445 {
6452 else
6458 }
6459 }
6462 void
6464 {
6466 TLAB_ACCESS_INIT;
6469 LOCK_GC;
6473 UNLOCK_GC;
6475 }
6477 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6483 UNLOCK_GC;
6485 }
6489 #ifdef HAVE_KW_THREAD
6491 #endif
6495 UNLOCK_GC;
6496 }
6498 /**
6499 * mono_gc_wbarrier_object_copy:
6500 *
6501 * Write barrier to call when obj is the result of a clone or copy of an object.
6502 */
6503 void
6505 {
6509 TLAB_ACCESS_INIT;
6514 LOCK_GC;
6515 /* do not copy the sync state */
6519 UNLOCK_GC;
6521 }
6524 UNLOCK_GC;
6526 }
6530 #ifdef HAVE_KW_THREAD
6532 #endif
6534 UNLOCK_GC;
6535 }
6537 /*
6538 * ######################################################################
6539 * ######## Collector debugging
6540 * ######################################################################
6541 */
6551 "complex_arr"
6552 };
6554 void
6556 {
6558 mword desc;
6571 }
6572 }
6580 // FIXME: Handle pointers to the inside of objects
6587 }
6591 }
6599 }
6603 {
6641 // FIXME:
6643 }
6645 /* The descriptor includes the size of MonoObject */
6654 }
6656 }
6658 /*
6659 * Return whenever ADDR occurs in the remembered sets
6660 */
6661 static gboolean
6663 {
6671 /* the global one */
6673 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6678 }
6679 }
6681 /* the generic store ones */
6686 }
6687 }
6689 /* the per-thread ones */
6694 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6699 }
6700 }
6704 }
6705 }
6706 }
6708 /* the freed thread ones */
6710 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6715 }
6716 }
6719 }
6723 /*
6724 * We let a missing remset slide if the target object is pinned,
6725 * because the store might have happened but the remset not yet added,
6726 * but in that case the target must be pinned. We might theoretically
6727 * miss some missing remsets this way, but it's very unlikely.
6728 */
6729 #undef HANDLE_PTR
6730 #define HANDLE_PTR(ptr,obj) do { \
6731 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6732 if (!find_in_remsets ((char*)(ptr))) { \
6733 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6734 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6735 if (!object_is_pinned (*(ptr))) \
6736 missing_remsets = TRUE; \
6737 } \
6738 } \
6739 } while (0)
6741 /*
6742 * Check that each object reference which points into the nursery can
6743 * be found in the remembered sets.
6744 */
6745 static void
6747 {
6749 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6751 #define SCAN_OBJECT_ACTION
6753 }
6755 /*
6756 * Perform consistency check of the heap.
6757 *
6758 * Assumes the world is stopped.
6759 */
6760 static void
6762 {
6765 // Need to add more checks
6771 // Check that oldspace->newspace pointers are registered with the collector
6772 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6779 #ifdef BINARY_PROTOCOL
6781 #endif
6783 }
6786 #undef HANDLE_PTR
6787 #define HANDLE_PTR(ptr,obj) do { \
6788 if (*(ptr)) \
6789 g_assert (LOAD_VTABLE (*(ptr))); \
6790 } while (0)
6792 static void
6794 {
6795 #define SCAN_OBJECT_ACTION
6797 }
6799 static void
6801 {
6804 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6808 }
6810 /* Check that the reference is valid */
6811 #undef HANDLE_PTR
6812 #define HANDLE_PTR(ptr,obj) do { \
6813 if (*(ptr)) { \
6814 g_assert (safe_name (*(ptr)) != NULL); \
6815 } \
6816 } while (0)
6818 /*
6819 * check_object:
6820 *
6821 * Perform consistency check on an object. Currently we only check that the
6822 * reference fields are valid.
6823 */
6824 void
6826 {
6831 }
6833 /*
6834 * ######################################################################
6835 * ######## Other mono public interface functions.
6836 * ######################################################################
6837 */
6839 void
6841 {
6842 LOCK_GC;
6848 }
6850 UNLOCK_GC;
6851 }
6853 int
6855 {
6857 }
6859 int
6861 {
6865 }
6867 int64_t
6869 {
6871 LOCK_GC;
6875 /* FIXME: account for pinned objects */
6876 UNLOCK_GC;
6878 }
6880 int64_t
6882 {
6884 }
6886 void
6888 {
6889 LOCK_GC;
6890 gc_disabled++;
6891 UNLOCK_GC;
6892 }
6894 void
6896 {
6897 LOCK_GC;
6898 gc_disabled--;
6899 UNLOCK_GC;
6900 }
6902 int
6904 {
6906 }
6908 gboolean
6910 {
6912 }
6914 int
6916 {
6920 }
6922 void
6924 {
6925 }
6927 void
6929 {
6930 LOCK_GC;
6932 UNLOCK_GC;
6933 }
6935 void
6937 {
6938 LOCK_GC;
6940 UNLOCK_GC;
6941 }
6943 MonoObject*
6945 {
6949 }
6951 gboolean
6953 {
6956 LOCK_GC;
6960 UNLOCK_GC;
6962 }
6969 UNLOCK_GC;
6971 }
6975 {
6981 }
6982 }
6986 {
6994 }
6998 {
6999 /* FIXME: do a single allocation */
7006 }
7008 }
7010 void
7012 {
7015 }
7019 {
7021 LOCK_INTERRUPTION;
7023 UNLOCK_INTERRUPTION;
7025 }
7027 gboolean
7029 {
7030 gboolean result;
7031 LOCK_GC;
7033 UNLOCK_GC;
7035 }
7037 #ifdef USER_CONFIG
7039 /* Tries to extract a number from the passed string, taking in to account m, k
7040 * and g suffixes */
7041 static gboolean
7043 {
7046 glong val;
7063 }
7076 }
7080 }
7082 #endif
7084 void
7086 {
7092 LOCK_GC;
7094 UNLOCK_GC;
7096 }
7100 #ifdef USER_CONFIG
7107 ;
7111 #ifdef ALIGN_NURSERY
7115 }
7119 ;
7120 #endif
7124 }
7126 fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
7128 }
7129 }
7131 #endif
7144 opt++;
7146 opt++;
7153 }
7173 #ifdef BINARY_PROTOCOL
7177 #endif
7180 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7181 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7183 }
7184 }
7186 }
7196 }
7201 }
7211 #ifndef HAVE_KW_THREAD
7213 #endif
7216 UNLOCK_GC;
7218 }
7220 int
7222 {
7224 }
7227 ATYPE_NORMAL,
7228 ATYPE_VECTOR,
7229 ATYPE_SMALL,
7230 ATYPE_NUM
7231 };
7233 #ifdef HAVE_KW_THREAD
7234 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7235 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7236 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7237 mono_mb_emit_i4 ((mb), (offset)); \
7238 } while (0)
7239 #else
7240 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7241 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7242 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7243 mono_mb_emit_i4 ((mb), thread_info_key); \
7244 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7245 mono_mb_emit_byte ((mb), CEE_ADD); \
7246 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7247 } while (0)
7248 #endif
7250 #ifdef MANAGED_ALLOCATION
7251 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7252 * for each class. This is currently not easy to do, as it is hard to generate basic
7253 * blocks + branches, but it is easy with the linear IL codebase.
7254 *
7255 * For this to work we'd need to solve the TLAB race, first. Now we
7256 * require the allocator to be in a few known methods to make sure
7257 * that they are executed atomically via the restart mechanism.
7258 */
7261 {
7273 #ifdef HAVE_KW_THREAD
7282 #endif
7285 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7286 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7288 }
7301 }
7311 /* size = vtable->klass->instance_size; */
7318 /* FIXME: assert instance_size stays a 4 byte integer */
7327 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7337 /* vtable->klass->sizes.element_size */
7346 /* * n */
7349 /* + sizeof (MonoArray) */
7356 /* catch */
7377 /* end catch */
7380 }
7382 /* size += ALLOC_ALIGN - 1; */
7386 /* size &= ~(ALLOC_ALIGN - 1); */
7391 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7396 }
7398 /*
7399 * We need to modify tlab_next, but the JIT only supports reading, so we read
7400 * another tls var holding its address instead.
7401 */
7403 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7408 /* p = (void**)tlab_next; */
7414 /* new_next = (char*)p + size; */
7422 /* tlab_next = new_next */
7427 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7432 /* Slowpath */
7439 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7449 }
7452 /* Fastpath */
7455 /* FIXME: Memory barrier */
7457 /* *p = vtable; */
7463 /* arr->max_length = max_length; */
7468 }
7470 /* return p */
7483 }
7484 #endif
7489 static gboolean
7491 {
7509 }
7511 /*
7512 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7513 * The signature of the called method is:
7514 * object allocate (MonoVTable *vtable)
7515 */
7516 MonoMethod*
7518 {
7519 #ifdef MANAGED_ALLOCATION
7522 #ifdef HAVE_KW_THREAD
7530 #endif
7536 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7547 else
7549 #else
7551 #endif
7552 }
7554 MonoMethod*
7556 {
7557 #ifdef MANAGED_ALLOCATION
7560 #ifdef HAVE_KW_THREAD
7568 #endif
7581 #else
7583 #endif
7584 }
7586 MonoMethod*
7588 {
7589 #ifdef MANAGED_ALLOCATION
7601 #else
7603 #endif
7604 }
7606 guint32
7608 {
7610 }
7613 MonoMethod*
7615 {
7619 #ifdef MANAGED_WBARRIER
7620 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7621 #ifndef ALIGN_NURSERY
7624 #endif
7627 #ifdef HAVE_KW_THREAD
7637 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7639 #endif
7640 #endif
7642 // FIXME: Maybe create a separate version for ctors (the branch would be
7643 // correctly predicted more times)
7647 /* Create the IL version of mono_gc_barrier_generic_store () */
7654 #ifdef MANAGED_WBARRIER
7656 #ifdef ALIGN_NURSERY
7657 // if (ptr_in_nursery (ptr)) return;
7658 /*
7659 * Masking out the bits might be faster, but we would have to use 64 bit
7660 * immediates, which might be slower.
7661 */
7668 // if (!ptr_in_nursery (*ptr)) return;
7675 #else
7677 // if (ptr < (nursery_start)) goto continue;
7682 // if (ptr >= nursery_real_end)) goto continue;
7687 // Otherwise return
7690 // continue:
7694 // Dereference and store in local var
7700 // if (*ptr < nursery_start) return;
7705 // if (*ptr >= nursery_end) return;
7710 #endif
7711 // if (ptr >= stack_end) goto need_wb;
7716 // if (ptr >= stack_start) return;
7722 // need_wb:
7725 // buffer = STORE_REMSET_BUFFER;
7730 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7735 // if (buffer [buffer_index] == ptr) return;
7746 // ++buffer_index;
7752 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7757 // buffer [buffer_index] = ptr;
7767 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7772 // return;
7777 #ifndef ALIGN_NURSERY
7779 #endif
7782 // slow path
7784 }
7785 #endif
7796 /* Already created */
7799 /* double-checked locking */
7802 }
7806 }
7810 {
7812 }
7814 void
7816 {
7817 }
7819 gboolean
7821 {
7823 }
7825 gboolean
7827 {
7829 }