2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
8 * Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
10 * Thread start/stop adapted from Boehm's GC:
11 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
12 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
13 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
14 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
16 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
17 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
19 * Permission is hereby granted to use or copy this program
20 * for any purpose, provided the above notices are retained on all copies.
21 * Permission to modify the code and to distribute modified code is granted,
22 * provided the above notices are retained, and a notice that the code was
23 * modified is included with the above copyright notice.
26 * Copyright 2001-2003 Ximian, Inc
27 * Copyright 2003-2010 Novell, Inc.
29 * Permission is hereby granted, free of charge, to any person obtaining
30 * a copy of this software and associated documentation files (the
31 * "Software"), to deal in the Software without restriction, including
32 * without limitation the rights to use, copy, modify, merge, publish,
33 * distribute, sublicense, and/or sell copies of the Software, and to
34 * permit persons to whom the Software is furnished to do so, subject to
35 * the following conditions:
37 * The above copyright notice and this permission notice shall be
38 * included in all copies or substantial portions of the Software.
40 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
41 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
42 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
43 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
44 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
45 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
46 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
49 * Important: allocation provides always zeroed memory, having to do
50 * a memset after allocation is deadly for performance.
51 * Memory usage at startup is currently as follows:
53 * 64 KB internal space
55 * We should provide a small memory config with half the sizes
57 * We currently try to make as few mono assumptions as possible:
58 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
60 * 2) gc descriptor is the second word in the vtable (first word in the class)
61 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
62 * 4) there is a function to get an object's size and the number of
63 * elements in an array.
64 * 5) we know the special way bounds are allocated for complex arrays
65 * 6) we know about proxies and how to treat them when domains are unloaded
67 * Always try to keep stack usage to a minimum: no recursive behaviour
68 * and no large stack allocs.
70 * General description.
71 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
72 * When the nursery is full we start a nursery collection: this is performed with a
74 * When the old generation is full we start a copying GC of the old generation as well:
75 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
76 * in the future. Maybe we'll even do both during the same collection like IMMIX.
78 * The things that complicate this description are:
79 * *) pinned objects: we can't move them so we need to keep track of them
80 * *) no precise info of the thread stacks and registers: we need to be able to
81 * quickly find the objects that may be referenced conservatively and pin them
82 * (this makes the first issues more important)
83 * *) large objects are too expensive to be dealt with using copying GC: we handle them
84 * with mark/sweep during major collections
85 * *) some objects need to not move even if they are small (interned strings, Type handles):
86 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
87 * PinnedChunks regions
93 *) we could have a function pointer in MonoClass to implement
94 customized write barriers for value types
96 *) investigate the stuff needed to advance a thread to a GC-safe
97 point (single-stepping, read from unmapped memory etc) and implement it.
98 This would enable us to inline allocations and write barriers, for example,
99 or at least parts of them, like the write barrier checks.
100 We may need this also for handling precise info on stacks, even simple things
101 as having uninitialized data on the stack and having to wait for the prolog
102 to zero it. Not an issue for the last frame that we scan conservatively.
103 We could always not trust the value in the slots anyway.
105 *) modify the jit to save info about references in stack locations:
106 this can be done just for locals as a start, so that at least
107 part of the stack is handled precisely.
109 *) test/fix endianess issues
111 *) Implement a card table as the write barrier instead of remembered
112 sets? Card tables are not easy to implement with our current
113 memory layout. We have several different kinds of major heap
114 objects: Small objects in regular blocks, small objects in pinned
115 chunks and LOS objects. If we just have a pointer we have no way
116 to tell which kind of object it points into, therefore we cannot
117 know where its card table is. The least we have to do to make
118 this happen is to get rid of write barriers for indirect stores.
121 *) Get rid of write barriers for indirect stores. We can do this by
122 telling the GC to wbarrier-register an object once we do an ldloca
123 or ldelema on it, and to unregister it once it's not used anymore
124 (it can only travel downwards on the stack). The problem with
125 unregistering is that it needs to happen eventually no matter
126 what, even if exceptions are thrown, the thread aborts, etc.
127 Rodrigo suggested that we could do only the registering part and
128 let the collector find out (pessimistically) when it's safe to
129 unregister, namely when the stack pointer of the thread that
130 registered the object is higher than it was when the registering
131 happened. This might make for a good first implementation to get
132 some data on performance.
134 *) Some sort of blacklist support? Blacklists is a concept from the
135 Boehm GC: if during a conservative scan we find pointers to an
136 area which we might use as heap, we mark that area as unusable, so
137 pointer retention by random pinning pointers is reduced.
139 *) experiment with max small object size (very small right now - 2kb,
140 because it's tied to the max freelist size)
142 *) add an option to mmap the whole heap in one chunk: it makes for many
143 simplifications in the checks (put the nursery at the top and just use a single
144 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
145 not flexible (too much of the address space may be used by default or we can't
146 increase the heap as needed) and we'd need a race-free mechanism to return memory
147 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
148 was written to, munmap is needed, but the following mmap may not find the same segment
151 *) memzero the major fragments after restarting the world and optionally a smaller
154 *) investigate having fragment zeroing threads
156 *) separate locks for finalization and other minor stuff to reduce
159 *) try a different copying order to improve memory locality
161 *) a thread abort after a store but before the write barrier will
162 prevent the write barrier from executing
164 *) specialized dynamically generated markers/copiers
166 *) Dynamically adjust TLAB size to the number of threads. If we have
167 too many threads that do allocation, we might need smaller TLABs,
168 and we might get better performance with larger TLABs if we only
169 have a handful of threads. We could sum up the space left in all
170 assigned TLABs and if that's more than some percentage of the
171 nursery size, reduce the TLAB size.
173 *) Explore placing unreachable objects on unused nursery memory.
174 Instead of memset'ng a region to zero, place an int[] covering it.
175 A good place to start is add_nursery_frag. The tricky thing here is
176 placing those objects atomically outside of a collection.
186 #include <semaphore.h>
195 #define _XOPEN_SOURCE
197 #include "metadata/metadata-internals.h"
198 #include "metadata/class-internals.h"
199 #include "metadata/gc-internal.h"
200 #include "metadata/object-internals.h"
201 #include "metadata/threads.h"
202 #include "metadata/sgen-gc.h"
203 #include "metadata/sgen-cardtable.h"
204 #include "metadata/sgen-protocol.h"
205 #include "metadata/sgen-archdep.h"
206 #include "metadata/sgen-bridge.h"
207 #include "metadata/mono-gc.h"
208 #include "metadata/method-builder.h"
209 #include "metadata/profiler-private.h"
210 #include "metadata/monitor.h"
211 #include "metadata/threadpool-internals.h"
212 #include "metadata/mempool-internals.h"
213 #include "metadata/marshal.h"
214 #include "utils/mono-mmap.h"
215 #include "utils/mono-time.h"
216 #include "utils/mono-semaphore.h"
217 #include "utils/mono-counters.h"
218 #include "utils/mono-proclib.h"
219 #include "utils/mono-logger-internal.h"
221 #include <mono/utils/memcheck.h>
223 #if defined(__MACH__)
224 #include "utils/mach-support.h"
227 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
231 #include "mono/cil/opcode.def"
237 #undef pthread_create
239 #undef pthread_detach
242 * ######################################################################
243 * ######## Types and constants used by the GC.
244 * ######################################################################
247 static int gc_initialized
= 0;
248 /* If set, do a minor collection before every X allocation */
249 static guint32 collect_before_allocs
= 0;
250 /* If set, do a heap consistency check before each minor collection */
251 static gboolean consistency_check_at_minor_collection
= FALSE
;
252 /* If set, check that there are no references to the domain left at domain unload */
253 static gboolean xdomain_checks
= FALSE
;
254 /* If not null, dump the heap after each collection into this file */
255 static FILE *heap_dump_file
= NULL
;
256 /* If set, mark stacks conservatively, even if precise marking is possible */
257 gboolean conservative_stack_mark
= FALSE
;
258 /* If set, do a plausibility check on the scan_starts before and after
260 static gboolean do_scan_starts_check
= FALSE
;
261 static gboolean disable_minor_collections
= FALSE
;
262 static gboolean disable_major_collections
= FALSE
;
264 #ifdef HEAVY_STATISTICS
265 static long long stat_objects_alloced
= 0;
266 static long long stat_bytes_alloced
= 0;
267 long long stat_objects_alloced_degraded
= 0;
268 long long stat_bytes_alloced_degraded
= 0;
269 static long long stat_bytes_alloced_los
= 0;
271 long long stat_copy_object_called_nursery
= 0;
272 long long stat_objects_copied_nursery
= 0;
273 long long stat_copy_object_called_major
= 0;
274 long long stat_objects_copied_major
= 0;
276 long long stat_scan_object_called_nursery
= 0;
277 long long stat_scan_object_called_major
= 0;
279 long long stat_nursery_copy_object_failed_from_space
= 0;
280 long long stat_nursery_copy_object_failed_forwarded
= 0;
281 long long stat_nursery_copy_object_failed_pinned
= 0;
283 static long long stat_store_remsets
= 0;
284 static long long stat_store_remsets_unique
= 0;
285 static long long stat_saved_remsets_1
= 0;
286 static long long stat_saved_remsets_2
= 0;
287 static long long stat_local_remsets_processed
= 0;
288 static long long stat_global_remsets_added
= 0;
289 static long long stat_global_remsets_readded
= 0;
290 static long long stat_global_remsets_processed
= 0;
291 static long long stat_global_remsets_discarded
= 0;
293 static long long stat_wasted_fragments_used
= 0;
294 static long long stat_wasted_fragments_bytes
= 0;
296 static int stat_wbarrier_set_field
= 0;
297 static int stat_wbarrier_set_arrayref
= 0;
298 static int stat_wbarrier_arrayref_copy
= 0;
299 static int stat_wbarrier_generic_store
= 0;
300 static int stat_wbarrier_generic_store_remset
= 0;
301 static int stat_wbarrier_set_root
= 0;
302 static int stat_wbarrier_value_copy
= 0;
303 static int stat_wbarrier_object_copy
= 0;
306 static long long stat_pinned_objects
= 0;
308 static long long time_minor_pre_collection_fragment_clear
= 0;
309 static long long time_minor_pinning
= 0;
310 static long long time_minor_scan_remsets
= 0;
311 static long long time_minor_scan_card_table
= 0;
312 static long long time_minor_scan_pinned
= 0;
313 static long long time_minor_scan_registered_roots
= 0;
314 static long long time_minor_scan_thread_data
= 0;
315 static long long time_minor_finish_gray_stack
= 0;
316 static long long time_minor_fragment_creation
= 0;
318 static long long time_major_pre_collection_fragment_clear
= 0;
319 static long long time_major_pinning
= 0;
320 static long long time_major_scan_pinned
= 0;
321 static long long time_major_scan_registered_roots
= 0;
322 static long long time_major_scan_thread_data
= 0;
323 static long long time_major_scan_alloc_pinned
= 0;
324 static long long time_major_scan_finalized
= 0;
325 static long long time_major_scan_big_objects
= 0;
326 static long long time_major_finish_gray_stack
= 0;
327 static long long time_major_free_bigobjs
= 0;
328 static long long time_major_los_sweep
= 0;
329 static long long time_major_sweep
= 0;
330 static long long time_major_fragment_creation
= 0;
332 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= SGEN_MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
334 int gc_debug_level
= 0;
339 mono_gc_flush_info (void)
341 fflush (gc_debug_file);
346 * Define this to allow the user to change the nursery size by
347 * specifying its value in the MONO_GC_PARAMS environmental
348 * variable. See mono_gc_base_init for details.
350 #define USER_CONFIG 1
352 #define TV_DECLARE SGEN_TV_DECLARE
353 #define TV_GETTIME SGEN_TV_GETTIME
354 #define TV_ELAPSED SGEN_TV_ELAPSED
355 #define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
357 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
359 /* The method used to clear the nursery */
360 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
361 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
366 CLEAR_AT_TLAB_CREATION
367 } NurseryClearPolicy
;
369 static NurseryClearPolicy nursery_clear_policy
= CLEAR_AT_TLAB_CREATION
;
372 * The young generation is divided into fragments. This is because
373 * we can hand one fragments to a thread for lock-less fast alloc and
374 * because the young generation ends up fragmented anyway by pinned objects.
375 * Once a collection is done, a list of fragments is created. When doing
376 * thread local alloc we use smallish nurseries so we allow new threads to
377 * allocate memory from gen0 without triggering a collection. Threads that
378 * are found to allocate lots of memory are given bigger fragments. This
379 * should make the finalizer thread use little nursery memory after a while.
380 * We should start assigning threads very small fragments: if there are many
381 * threads the nursery will be full of reserved space that the threads may not
382 * use at all, slowing down allocation speed.
383 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
384 * Allocation Buffers (TLABs).
386 typedef struct _Fragment Fragment
;
390 char *fragment_start
;
391 char *fragment_limit
; /* the current soft limit for allocation */
395 /* the runtime can register areas of memory as roots: we keep two lists of roots,
396 * a pinned root set for conservatively scanned roots and a normal one for
397 * precisely scanned roots (currently implemented as a single list).
399 typedef struct _RootRecord RootRecord
;
408 * We're never actually using the first element. It's always set to
409 * NULL to simplify the elimination of consecutive duplicate
412 #define STORE_REMSET_BUFFER_SIZE 1024
414 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet
;
415 struct _GenericStoreRememberedSet
{
416 GenericStoreRememberedSet
*next
;
417 /* We need one entry less because the first entry of store
418 remset buffers is always a dummy and we don't copy it. */
419 gpointer data
[STORE_REMSET_BUFFER_SIZE
- 1];
422 /* we have 4 possible values in the low 2 bits */
424 REMSET_LOCATION
, /* just a pointer to the exact location */
425 REMSET_RANGE
, /* range of pointer fields */
426 REMSET_OBJECT
, /* mark all the object for scanning */
427 REMSET_VTYPE
, /* a valuetype array described by a gc descriptor and a count */
428 REMSET_TYPE_MASK
= 0x3
431 #ifdef HAVE_KW_THREAD
432 static __thread RememberedSet
*remembered_set MONO_TLS_FAST
;
434 static pthread_key_t remembered_set_key
;
435 static RememberedSet
*global_remset
;
436 static RememberedSet
*freed_thread_remsets
;
437 static GenericStoreRememberedSet
*generic_store_remsets
= NULL
;
439 /*A two slots cache for recently inserted remsets */
440 static gpointer global_remset_cache
[2];
442 /* FIXME: later choose a size that takes into account the RememberedSet struct
443 * and doesn't waste any alloc paddin space.
445 #define DEFAULT_REMSET_SIZE 1024
446 static RememberedSet
* alloc_remset (int size
, gpointer id
);
448 #define object_is_forwarded SGEN_OBJECT_IS_FORWARDED
449 #define object_is_pinned SGEN_OBJECT_IS_PINNED
450 #define pin_object SGEN_PIN_OBJECT
451 #define unpin_object SGEN_UNPIN_OBJECT
453 #define ptr_in_nursery(p) (SGEN_PTR_IN_NURSERY ((p), DEFAULT_NURSERY_BITS, nursery_start, nursery_real_end))
455 #define LOAD_VTABLE SGEN_LOAD_VTABLE
458 safe_name (void* obj
)
460 MonoVTable
*vt
= (MonoVTable
*)LOAD_VTABLE (obj
);
461 return vt
->klass
->name
;
464 #define safe_object_get_size mono_sgen_safe_object_get_size
467 mono_sgen_safe_name (void* obj
)
469 return safe_name (obj
);
473 * ######################################################################
474 * ######## Global data.
475 * ######################################################################
477 static LOCK_DECLARE (gc_mutex
);
478 static int gc_disabled
= 0;
479 static int num_minor_gcs
= 0;
480 static int num_major_gcs
= 0;
482 static gboolean use_cardtable
;
486 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
487 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
488 static int default_nursery_size
= (1 << 22);
489 #ifdef SGEN_ALIGN_NURSERY
490 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
491 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
492 static int default_nursery_bits
= 22;
497 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
498 #ifdef SGEN_ALIGN_NURSERY
499 #define DEFAULT_NURSERY_BITS 22
504 #ifndef SGEN_ALIGN_NURSERY
505 #define DEFAULT_NURSERY_BITS -1
508 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
510 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
512 /* the minimum size of a fragment that we consider useful for allocation */
513 #define FRAGMENT_MIN_SIZE (512)
515 static mword pagesize
= 4096;
516 static mword nursery_size
;
517 static int degraded_mode
= 0;
519 static mword total_alloc
= 0;
520 /* use this to tune when to do a major/minor collection */
521 static mword memory_pressure
= 0;
522 static mword minor_collection_allowance
;
523 static int minor_collection_sections_alloced
= 0;
526 /* GC Logging stats */
527 static int last_major_num_sections
= 0;
528 static int last_los_memory_usage
= 0;
529 static gboolean major_collection_hapenned
= FALSE
;
531 static GCMemSection
*nursery_section
= NULL
;
532 static mword lowest_heap_address
= ~(mword
)0;
533 static mword highest_heap_address
= 0;
535 static LOCK_DECLARE (interruption_mutex
);
536 static LOCK_DECLARE (global_remset_mutex
);
537 static LOCK_DECLARE (pin_queue_mutex
);
539 #define LOCK_GLOBAL_REMSET pthread_mutex_lock (&global_remset_mutex)
540 #define UNLOCK_GLOBAL_REMSET pthread_mutex_unlock (&global_remset_mutex)
542 #define LOCK_PIN_QUEUE pthread_mutex_lock (&pin_queue_mutex)
543 #define UNLOCK_PIN_QUEUE pthread_mutex_unlock (&pin_queue_mutex)
545 typedef struct _FinalizeEntry FinalizeEntry
;
546 struct _FinalizeEntry
{
551 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable
;
552 struct _FinalizeEntryHashTable
{
553 FinalizeEntry
**table
;
558 typedef struct _DisappearingLink DisappearingLink
;
559 struct _DisappearingLink
{
560 DisappearingLink
*next
;
564 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable
;
565 struct _DisappearingLinkHashTable
{
566 DisappearingLink
**table
;
571 typedef struct _EphemeronLinkNode EphemeronLinkNode
;
573 struct _EphemeronLinkNode
{
574 EphemeronLinkNode
*next
;
583 int current_collection_generation
= -1;
586 * The link pointer is hidden by negating each bit. We use the lowest
587 * bit of the link (before negation) to store whether it needs
588 * resurrection tracking.
590 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
591 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
593 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
594 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
597 * The finalizable hash has the object as the key, the
598 * disappearing_link hash, has the link address as key.
600 static FinalizeEntryHashTable minor_finalizable_hash
;
601 static FinalizeEntryHashTable major_finalizable_hash
;
602 /* objects that are ready to be finalized */
603 static FinalizeEntry
*fin_ready_list
= NULL
;
604 static FinalizeEntry
*critical_fin_list
= NULL
;
606 static DisappearingLinkHashTable minor_disappearing_link_hash
;
607 static DisappearingLinkHashTable major_disappearing_link_hash
;
609 static EphemeronLinkNode
*ephemeron_list
;
611 static int num_ready_finalizers
= 0;
612 static int no_finalize
= 0;
615 ROOT_TYPE_NORMAL
= 0, /* "normal" roots */
616 ROOT_TYPE_PINNED
= 1, /* roots without a GC descriptor */
617 ROOT_TYPE_WBARRIER
= 2, /* roots with a write barrier */
621 /* registered roots: the key to the hash is the root start address */
623 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
625 static RootRecord
**roots_hash
[ROOT_TYPE_NUM
] = { NULL
, NULL
};
626 static int roots_hash_size
[ROOT_TYPE_NUM
] = { 0, 0, 0 };
627 static mword roots_size
= 0; /* amount of memory in the root set */
628 static int num_roots_entries
[ROOT_TYPE_NUM
] = { 0, 0, 0 };
630 #define GC_ROOT_NUM 32
633 void *objects
[GC_ROOT_NUM
];
634 int root_types
[GC_ROOT_NUM
];
635 uintptr_t extra_info
[GC_ROOT_NUM
];
639 notify_gc_roots (GCRootReport
*report
)
643 mono_profiler_gc_roots (report
->count
, report
->objects
, report
->root_types
, report
->extra_info
);
648 add_profile_gc_root (GCRootReport
*report
, void *object
, int rtype
, uintptr_t extra_info
)
650 if (report
->count
== GC_ROOT_NUM
)
651 notify_gc_roots (report
);
652 report
->objects
[report
->count
] = object
;
653 report
->root_types
[report
->count
] = rtype
;
654 report
->extra_info
[report
->count
++] = (uintptr_t)((MonoVTable
*)LOAD_VTABLE (object
))->klass
;
658 * The current allocation cursors
659 * We allocate objects in the nursery.
660 * The nursery is the area between nursery_start and nursery_real_end.
661 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
662 * from nursery fragments.
663 * tlab_next is the pointer to the space inside the TLAB where the next object will
665 * tlab_temp_end is the pointer to the end of the temporary space reserved for
666 * the allocation: it allows us to set the scan starts at reasonable intervals.
667 * tlab_real_end points to the end of the TLAB.
668 * nursery_frag_real_end points to the end of the currently used nursery fragment.
669 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
670 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
671 * At the next allocation, the area of the nursery where objects can be present is
672 * between MIN(nursery_first_pinned_start, first_fragment_start) and
673 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
675 static char *nursery_start
= NULL
;
677 #ifdef HAVE_KW_THREAD
678 #define TLAB_ACCESS_INIT
679 #define TLAB_START tlab_start
680 #define TLAB_NEXT tlab_next
681 #define TLAB_TEMP_END tlab_temp_end
682 #define TLAB_REAL_END tlab_real_end
683 #define REMEMBERED_SET remembered_set
684 #define STORE_REMSET_BUFFER store_remset_buffer
685 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
686 #define IN_CRITICAL_REGION thread_info->in_critical_region
688 static pthread_key_t thread_info_key
;
689 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
690 #define TLAB_START (__thread_info__->tlab_start)
691 #define TLAB_NEXT (__thread_info__->tlab_next)
692 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
693 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
694 #define REMEMBERED_SET (__thread_info__->remset)
695 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
696 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
697 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
700 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
701 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
702 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
705 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
706 * variables for next+temp_end ?
708 #ifdef HAVE_KW_THREAD
709 static __thread SgenThreadInfo
*thread_info
;
710 static __thread
char *tlab_start
;
711 static __thread
char *tlab_next
;
712 static __thread
char *tlab_temp_end
;
713 static __thread
char *tlab_real_end
;
714 static __thread gpointer
*store_remset_buffer
;
715 static __thread
long store_remset_buffer_index
;
716 /* Used by the managed allocator/wbarrier */
717 static __thread
char **tlab_next_addr
;
718 static __thread
char *stack_end
;
719 static __thread
long *store_remset_buffer_index_addr
;
721 static char *nursery_next
= NULL
;
722 static char *nursery_frag_real_end
= NULL
;
723 static char *nursery_real_end
= NULL
;
724 static char *nursery_last_pinned_end
= NULL
;
726 /* The size of a TLAB */
727 /* The bigger the value, the less often we have to go to the slow path to allocate a new
728 * one, but the more space is wasted by threads not allocating much memory.
730 * FIXME: Make this self-tuning for each thread.
732 static guint32 tlab_size
= (1024 * 4);
734 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
735 #define MAX_NURSERY_TLAB_WASTE 512
737 /* fragments that are free and ready to be used for allocation */
738 static Fragment
*nursery_fragments
= NULL
;
739 /* freeelist of fragment structures */
740 static Fragment
*fragment_freelist
= NULL
;
742 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
744 /* Functions supplied by the runtime to be called by the GC */
745 static MonoGCCallbacks gc_callbacks
;
747 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
748 #define ALLOC_ALIGN_BITS SGEN_ALLOC_ALIGN_BITS
750 #define ALIGN_UP SGEN_ALIGN_UP
752 #define MOVED_OBJECTS_NUM 64
753 static void *moved_objects
[MOVED_OBJECTS_NUM
];
754 static int moved_objects_idx
= 0;
756 /* Vtable of the objects used to fill out nursery fragments before a collection */
757 static MonoVTable
*array_fill_vtable
;
760 * ######################################################################
761 * ######## Heap size accounting
762 * ######################################################################
765 static mword max_heap_size
= ((mword
)0)- ((mword
)1);
766 static mword allocated_heap
;
768 /*Object was pinned during the current collection*/
769 static mword objects_pinned
;
772 mono_sgen_release_space (mword size
, int space
)
774 allocated_heap
-= size
;
778 available_free_space (void)
780 return max_heap_size
- MIN (allocated_heap
, max_heap_size
);
784 mono_sgen_try_alloc_space (mword size
, int space
)
786 if (available_free_space () < size
)
789 allocated_heap
+= size
;
794 init_heap_size_limits (glong max_heap
)
799 if (max_heap
< nursery_size
* 4) {
800 fprintf (stderr
, "max-heap-size must be at least 4 times larger than nursery size.\n");
803 max_heap_size
= max_heap
- nursery_size
;
807 * ######################################################################
808 * ######## Macros and function declarations.
809 * ######################################################################
813 align_pointer (void *ptr
)
815 mword p
= (mword
)ptr
;
816 p
+= sizeof (gpointer
) - 1;
817 p
&= ~ (sizeof (gpointer
) - 1);
821 typedef SgenGrayQueue GrayQueue
;
823 typedef void (*CopyOrMarkObjectFunc
) (void**, GrayQueue
*);
824 typedef char* (*ScanObjectFunc
) (char*, GrayQueue
*);
827 static inline MonoObject
*
828 finalize_entry_get_object (FinalizeEntry
*entry
)
830 return (MonoObject
*)(((mword
)entry
->object
) & ~(mword
)0x1);
833 static inline gboolean
834 finalize_entry_get_bridge_bit (FinalizeEntry
*entry
)
836 return (((mword
)entry
->object
) & 0x1);
840 finalize_entry_set_object (FinalizeEntry
*entry
, MonoObject
*object
, gboolean bridge_bit
)
842 entry
->object
= (MonoObject
*)((mword
)object
| (mword
)bridge_bit
);
845 /* forward declarations */
846 static int stop_world (int generation
);
847 static int restart_world (int generation
);
848 static void scan_thread_data (void *start_nursery
, void *end_nursery
, gboolean precise
);
849 static void scan_from_remsets (void *start_nursery
, void *end_nursery
, GrayQueue
*queue
);
850 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func
, char *addr_start
, char *addr_end
, int root_type
, GrayQueue
*queue
);
851 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func
, FinalizeEntry
*list
, GrayQueue
*queue
);
852 static void report_finalizer_roots (void);
853 static void report_registered_roots (void);
854 static void find_pinning_ref_from_thread (char *obj
, size_t size
);
855 static void update_current_thread_stack (void *start
);
856 static void collect_bridge_objects (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, int generation
, GrayQueue
*queue
);
857 static void finalize_in_range (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, int generation
, GrayQueue
*queue
);
858 static void add_or_remove_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
, int generation
);
859 static void null_link_in_range (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, int generation
, gboolean before_finalization
, GrayQueue
*queue
);
860 static void null_links_for_domain (MonoDomain
*domain
, int generation
);
861 static gboolean
search_fragment_for_size (size_t size
);
862 static int search_fragment_for_size_range (size_t desired_size
, size_t minimum_size
);
863 static void clear_nursery_fragments (char *next
);
864 static void pin_from_roots (void *start_nursery
, void *end_nursery
);
865 static int pin_objects_from_addresses (GCMemSection
*section
, void **start
, void **end
, void *start_nursery
, void *end_nursery
, GrayQueue
*queue
);
866 static void optimize_pin_queue (int start_slot
);
867 static void clear_remsets (void);
868 static void clear_tlabs (void);
869 static void sort_addresses (void **array
, int size
);
870 static void drain_gray_stack (GrayQueue
*queue
);
871 static void finish_gray_stack (char *start_addr
, char *end_addr
, int generation
, GrayQueue
*queue
);
872 static gboolean
need_major_collection (mword space_needed
);
873 static void major_collection (const char *reason
);
875 static void mono_gc_register_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
);
877 void describe_ptr (char *ptr
);
878 void check_object (char *start
);
880 static void check_consistency (void);
881 static void check_major_refs (void);
882 static void check_scan_starts (void);
883 static void check_for_xdomain_refs (void);
884 static void dump_heap (const char *type
, int num
, const char *reason
);
886 void mono_gc_scan_for_specific_ref (MonoObject
*key
);
888 static void init_stats (void);
890 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, GrayQueue
*queue
);
891 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, GrayQueue
*queue
);
892 static void null_ephemerons_for_domain (MonoDomain
*domain
);
894 SgenMajorCollector major_collector
;
896 #include "sgen-pinning.c"
897 #include "sgen-pinning-stats.c"
898 #include "sgen-gray.c"
899 #include "sgen-workers.c"
900 #include "sgen-cardtable.c"
902 /* Root bitmap descriptors are simpler: the lower three bits describe the type
903 * and we either have 30/62 bitmap bits or nibble-based run-length,
904 * or a complex descriptor, or a user defined marker function.
907 ROOT_DESC_CONSERVATIVE
, /* 0, so matches NULL value */
912 ROOT_DESC_TYPE_MASK
= 0x7,
913 ROOT_DESC_TYPE_SHIFT
= 3,
916 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
918 #define MAX_USER_DESCRIPTORS 16
920 static gsize
* complex_descriptors
= NULL
;
921 static int complex_descriptors_size
= 0;
922 static int complex_descriptors_next
= 0;
923 static MonoGCRootMarkFunc user_descriptors
[MAX_USER_DESCRIPTORS
];
924 static int user_descriptors_next
= 0;
927 alloc_complex_descriptor (gsize
*bitmap
, int numbits
)
931 numbits
= ALIGN_TO (numbits
, GC_BITS_PER_WORD
);
932 nwords
= numbits
/ GC_BITS_PER_WORD
+ 1;
935 res
= complex_descriptors_next
;
936 /* linear search, so we don't have duplicates with domain load/unload
937 * this should not be performance critical or we'd have bigger issues
938 * (the number and size of complex descriptors should be small).
940 for (i
= 0; i
< complex_descriptors_next
; ) {
941 if (complex_descriptors
[i
] == nwords
) {
943 for (j
= 0; j
< nwords
- 1; ++j
) {
944 if (complex_descriptors
[i
+ 1 + j
] != bitmap
[j
]) {
954 i
+= complex_descriptors
[i
];
956 if (complex_descriptors_next
+ nwords
> complex_descriptors_size
) {
957 int new_size
= complex_descriptors_size
* 2 + nwords
;
958 complex_descriptors
= g_realloc (complex_descriptors
, new_size
* sizeof (gsize
));
959 complex_descriptors_size
= new_size
;
961 DEBUG (6, fprintf (gc_debug_file
, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res
, nwords
, complex_descriptors_size
));
962 complex_descriptors_next
+= nwords
;
963 complex_descriptors
[res
] = nwords
;
964 for (i
= 0; i
< nwords
- 1; ++i
) {
965 complex_descriptors
[res
+ 1 + i
] = bitmap
[i
];
966 DEBUG (6, fprintf (gc_debug_file
, "\tvalue: %p\n", (void*)complex_descriptors
[res
+ 1 + i
]));
973 mono_sgen_get_complex_descriptor (GCVTable
*vt
)
975 return complex_descriptors
+ (vt
->desc
>> LOW_TYPE_BITS
);
979 * Descriptor builders.
982 mono_gc_make_descr_for_string (gsize
*bitmap
, int numbits
)
984 return (void*) DESC_TYPE_RUN_LENGTH
;
988 mono_gc_make_descr_for_object (gsize
*bitmap
, int numbits
, size_t obj_size
)
990 int first_set
= -1, num_set
= 0, last_set
= -1, i
;
992 size_t stored_size
= obj_size
;
993 for (i
= 0; i
< numbits
; ++i
) {
994 if (bitmap
[i
/ GC_BITS_PER_WORD
] & ((gsize
)1 << (i
% GC_BITS_PER_WORD
))) {
1002 * We don't encode the size of types that don't contain
1003 * references because they might not be aligned, i.e. the
1004 * bottom two bits might be set, which would clash with the
1005 * bits we need to encode the descriptor type. Since we don't
1006 * use the encoded size to skip objects, other than for
1007 * processing remsets, in which case only the positions of
1008 * references are relevant, this is not a problem.
1011 return (void*)DESC_TYPE_RUN_LENGTH
;
1012 g_assert (!(stored_size
& 0x3));
1013 if (stored_size
<= MAX_SMALL_OBJ_SIZE
) {
1014 /* check run-length encoding first: one byte offset, one byte number of pointers
1015 * on 64 bit archs, we can have 3 runs, just one on 32.
1016 * It may be better to use nibbles.
1018 if (first_set
< 0) {
1019 desc
= DESC_TYPE_RUN_LENGTH
| (stored_size
<< 1);
1020 DEBUG (6, fprintf (gc_debug_file
, "Ptrfree descriptor %p, size: %zd\n", (void*)desc
, stored_size
));
1021 return (void*) desc
;
1022 } else if (first_set
< 256 && num_set
< 256 && (first_set
+ num_set
== last_set
+ 1)) {
1023 desc
= DESC_TYPE_RUN_LENGTH
| (stored_size
<< 1) | (first_set
<< 16) | (num_set
<< 24);
1024 DEBUG (6, fprintf (gc_debug_file
, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc
, stored_size
, first_set
, num_set
));
1025 return (void*) desc
;
1027 /* we know the 2-word header is ptr-free */
1028 if (last_set
< SMALL_BITMAP_SIZE
+ OBJECT_HEADER_WORDS
) {
1029 desc
= DESC_TYPE_SMALL_BITMAP
| (stored_size
<< 1) | ((*bitmap
>> OBJECT_HEADER_WORDS
) << SMALL_BITMAP_SHIFT
);
1030 DEBUG (6, fprintf (gc_debug_file
, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc
, stored_size
, last_set
));
1031 return (void*) desc
;
1034 /* we know the 2-word header is ptr-free */
1035 if (last_set
< LARGE_BITMAP_SIZE
+ OBJECT_HEADER_WORDS
) {
1036 desc
= DESC_TYPE_LARGE_BITMAP
| ((*bitmap
>> OBJECT_HEADER_WORDS
) << LOW_TYPE_BITS
);
1037 DEBUG (6, fprintf (gc_debug_file
, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc
, stored_size
, last_set
));
1038 return (void*) desc
;
1040 /* it's a complex object ... */
1041 desc
= DESC_TYPE_COMPLEX
| (alloc_complex_descriptor (bitmap
, last_set
+ 1) << LOW_TYPE_BITS
);
1042 return (void*) desc
;
1045 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1047 mono_gc_make_descr_for_array (int vector
, gsize
*elem_bitmap
, int numbits
, size_t elem_size
)
1049 int first_set
= -1, num_set
= 0, last_set
= -1, i
;
1050 mword desc
= vector
? DESC_TYPE_VECTOR
: DESC_TYPE_ARRAY
;
1051 for (i
= 0; i
< numbits
; ++i
) {
1052 if (elem_bitmap
[i
/ GC_BITS_PER_WORD
] & ((gsize
)1 << (i
% GC_BITS_PER_WORD
))) {
1059 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
1061 return (void*)DESC_TYPE_RUN_LENGTH
;
1062 if (elem_size
<= MAX_ELEMENT_SIZE
) {
1063 desc
|= elem_size
<< VECTOR_ELSIZE_SHIFT
;
1065 return (void*)(desc
| VECTOR_SUBTYPE_PTRFREE
);
1067 /* Note: we also handle structs with just ref fields */
1068 if (num_set
* sizeof (gpointer
) == elem_size
) {
1069 return (void*)(desc
| VECTOR_SUBTYPE_REFS
| ((gssize
)(-1) << 16));
1071 /* FIXME: try run-len first */
1072 /* Note: we can't skip the object header here, because it's not present */
1073 if (last_set
<= SMALL_BITMAP_SIZE
) {
1074 return (void*)(desc
| VECTOR_SUBTYPE_BITMAP
| (*elem_bitmap
<< 16));
1077 /* it's am array of complex structs ... */
1078 desc
= DESC_TYPE_COMPLEX_ARR
;
1079 desc
|= alloc_complex_descriptor (elem_bitmap
, last_set
+ 1) << LOW_TYPE_BITS
;
1080 return (void*) desc
;
1083 /* Return the bitmap encoded by a descriptor */
1085 mono_gc_get_bitmap_for_descr (void *descr
, int *numbits
)
1087 mword d
= (mword
)descr
;
1091 case DESC_TYPE_RUN_LENGTH
: {
1092 int first_set
= (d
>> 16) & 0xff;
1093 int num_set
= (d
>> 24) & 0xff;
1096 bitmap
= g_new0 (gsize
, (first_set
+ num_set
+ 7) / 8);
1098 for (i
= first_set
; i
< first_set
+ num_set
; ++i
)
1099 bitmap
[i
/ GC_BITS_PER_WORD
] |= ((gsize
)1 << (i
% GC_BITS_PER_WORD
));
1101 *numbits
= first_set
+ num_set
;
1105 case DESC_TYPE_SMALL_BITMAP
:
1106 bitmap
= g_new0 (gsize
, 1);
1108 bitmap
[0] = (d
>> SMALL_BITMAP_SHIFT
) << OBJECT_HEADER_WORDS
;
1110 *numbits
= GC_BITS_PER_WORD
;
1114 g_assert_not_reached ();
1119 is_xdomain_ref_allowed (gpointer
*ptr
, char *obj
, MonoDomain
*domain
)
1121 MonoObject
*o
= (MonoObject
*)(obj
);
1122 MonoObject
*ref
= (MonoObject
*)*(ptr
);
1123 int offset
= (char*)(ptr
) - (char*)o
;
1125 if (o
->vtable
->klass
== mono_defaults
.thread_class
&& offset
== G_STRUCT_OFFSET (MonoThread
, internal_thread
))
1127 if (o
->vtable
->klass
== mono_defaults
.internal_thread_class
&& offset
== G_STRUCT_OFFSET (MonoInternalThread
, current_appcontext
))
1129 if (mono_class_has_parent (o
->vtable
->klass
, mono_defaults
.real_proxy_class
) &&
1130 offset
== G_STRUCT_OFFSET (MonoRealProxy
, unwrapped_server
))
1132 /* Thread.cached_culture_info */
1133 if (!strcmp (ref
->vtable
->klass
->name_space
, "System.Globalization") &&
1134 !strcmp (ref
->vtable
->klass
->name
, "CultureInfo") &&
1135 !strcmp(o
->vtable
->klass
->name_space
, "System") &&
1136 !strcmp(o
->vtable
->klass
->name
, "Object[]"))
1139 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1140 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1141 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1142 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1143 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1144 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1145 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1146 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1147 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1149 if (!strcmp (ref
->vtable
->klass
->name_space
, "System") &&
1150 !strcmp (ref
->vtable
->klass
->name
, "Byte[]") &&
1151 !strcmp (o
->vtable
->klass
->name_space
, "System.IO") &&
1152 !strcmp (o
->vtable
->klass
->name
, "MemoryStream"))
1154 /* append_job() in threadpool.c */
1155 if (!strcmp (ref
->vtable
->klass
->name_space
, "System.Runtime.Remoting.Messaging") &&
1156 !strcmp (ref
->vtable
->klass
->name
, "AsyncResult") &&
1157 !strcmp (o
->vtable
->klass
->name_space
, "System") &&
1158 !strcmp (o
->vtable
->klass
->name
, "Object[]") &&
1159 mono_thread_pool_is_queue_array ((MonoArray
*) o
))
1165 check_reference_for_xdomain (gpointer
*ptr
, char *obj
, MonoDomain
*domain
)
1167 MonoObject
*o
= (MonoObject
*)(obj
);
1168 MonoObject
*ref
= (MonoObject
*)*(ptr
);
1169 int offset
= (char*)(ptr
) - (char*)o
;
1171 MonoClassField
*field
;
1174 if (!ref
|| ref
->vtable
->domain
== domain
)
1176 if (is_xdomain_ref_allowed (ptr
, obj
, domain
))
1180 for (class = o
->vtable
->klass
; class; class = class->parent
) {
1183 for (i
= 0; i
< class->field
.count
; ++i
) {
1184 if (class->fields
[i
].offset
== offset
) {
1185 field
= &class->fields
[i
];
1193 if (ref
->vtable
->klass
== mono_defaults
.string_class
)
1194 str
= mono_string_to_utf8 ((MonoString
*)ref
);
1197 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1198 o
, o
->vtable
->klass
->name_space
, o
->vtable
->klass
->name
,
1199 offset
, field
? field
->name
: "",
1200 ref
, ref
->vtable
->klass
->name_space
, ref
->vtable
->klass
->name
, str
? str
: "");
1201 mono_gc_scan_for_specific_ref (o
);
1207 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1210 scan_object_for_xdomain_refs (char *start
, mword size
, void *data
)
1212 MonoDomain
*domain
= ((MonoObject
*)start
)->vtable
->domain
;
1214 #include "sgen-scan-object.h"
1218 #define HANDLE_PTR(ptr,obj) do { \
1219 if ((MonoObject*)*(ptr) == key) { \
1220 g_print ("found ref to %p in object %p (%s) at offset %td\n", \
1221 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1226 scan_object_for_specific_ref (char *start
, MonoObject
*key
)
1230 if ((forwarded
= SGEN_OBJECT_IS_FORWARDED (start
)))
1233 #include "sgen-scan-object.h"
1237 mono_sgen_scan_area_with_callback (char *start
, char *end
, IterateObjectCallbackFunc callback
, void *data
, gboolean allow_flags
)
1239 while (start
< end
) {
1243 if (!*(void**)start
) {
1244 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
1249 if (!(obj
= SGEN_OBJECT_IS_FORWARDED (start
)))
1255 size
= ALIGN_UP (safe_object_get_size ((MonoObject
*)obj
));
1257 callback (obj
, size
, data
);
1264 scan_object_for_specific_ref_callback (char *obj
, size_t size
, MonoObject
*key
)
1266 scan_object_for_specific_ref (obj
, key
);
1270 check_root_obj_specific_ref (RootRecord
*root
, MonoObject
*key
, MonoObject
*obj
)
1274 g_print ("found ref to %p in root record %p\n", key
, root
);
1277 static MonoObject
*check_key
= NULL
;
1278 static RootRecord
*check_root
= NULL
;
1281 check_root_obj_specific_ref_from_marker (void **obj
)
1283 check_root_obj_specific_ref (check_root
, check_key
, *obj
);
1287 scan_roots_for_specific_ref (MonoObject
*key
, int root_type
)
1292 for (i
= 0; i
< roots_hash_size
[root_type
]; ++i
) {
1293 for (root
= roots_hash
[root_type
][i
]; root
; root
= root
->next
) {
1294 void **start_root
= (void**)root
->start_root
;
1295 mword desc
= root
->root_desc
;
1299 switch (desc
& ROOT_DESC_TYPE_MASK
) {
1300 case ROOT_DESC_BITMAP
:
1301 desc
>>= ROOT_DESC_TYPE_SHIFT
;
1304 check_root_obj_specific_ref (root
, key
, *start_root
);
1309 case ROOT_DESC_COMPLEX
: {
1310 gsize
*bitmap_data
= complex_descriptors
+ (desc
>> ROOT_DESC_TYPE_SHIFT
);
1311 int bwords
= (*bitmap_data
) - 1;
1312 void **start_run
= start_root
;
1314 while (bwords
-- > 0) {
1315 gsize bmap
= *bitmap_data
++;
1316 void **objptr
= start_run
;
1319 check_root_obj_specific_ref (root
, key
, *objptr
);
1323 start_run
+= GC_BITS_PER_WORD
;
1327 case ROOT_DESC_USER
: {
1328 MonoGCRootMarkFunc marker
= user_descriptors
[desc
>> ROOT_DESC_TYPE_SHIFT
];
1329 marker (start_root
, check_root_obj_specific_ref_from_marker
);
1332 case ROOT_DESC_RUN_LEN
:
1333 g_assert_not_reached ();
1335 g_assert_not_reached ();
1344 mono_gc_scan_for_specific_ref (MonoObject
*key
)
1349 mono_sgen_scan_area_with_callback (nursery_section
->data
, nursery_section
->end_data
,
1350 (IterateObjectCallbackFunc
)scan_object_for_specific_ref_callback
, key
, TRUE
);
1352 major_collector
.iterate_objects (TRUE
, TRUE
, (IterateObjectCallbackFunc
)scan_object_for_specific_ref_callback
, key
);
1354 mono_sgen_los_iterate_objects ((IterateObjectCallbackFunc
)scan_object_for_specific_ref_callback
, key
);
1356 scan_roots_for_specific_ref (key
, ROOT_TYPE_NORMAL
);
1357 scan_roots_for_specific_ref (key
, ROOT_TYPE_WBARRIER
);
1359 for (i
= 0; i
< roots_hash_size
[ROOT_TYPE_PINNED
]; ++i
) {
1360 for (root
= roots_hash
[ROOT_TYPE_PINNED
][i
]; root
; root
= root
->next
) {
1361 void **ptr
= (void**)root
->start_root
;
1363 while (ptr
< (void**)root
->end_root
) {
1364 check_root_obj_specific_ref (root
, *ptr
, key
);
1372 clear_current_nursery_fragment (char *next
)
1374 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
1375 g_assert (next
<= nursery_frag_real_end
);
1376 DEBUG (4, fprintf (gc_debug_file
, "Clear nursery frag %p-%p\n", next
, nursery_frag_real_end
));
1377 memset (next
, 0, nursery_frag_real_end
- next
);
1381 /* Clear all remaining nursery fragments */
1383 clear_nursery_fragments (char *next
)
1386 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
1387 clear_current_nursery_fragment (next
);
1388 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
1389 DEBUG (4, fprintf (gc_debug_file
, "Clear nursery frag %p-%p\n", frag
->fragment_start
, frag
->fragment_end
));
1390 memset (frag
->fragment_start
, 0, frag
->fragment_end
- frag
->fragment_start
);
1396 need_remove_object_for_domain (char *start
, MonoDomain
*domain
)
1398 if (mono_object_domain (start
) == domain
) {
1399 DEBUG (4, fprintf (gc_debug_file
, "Need to cleanup object %p\n", start
));
1400 binary_protocol_cleanup (start
, (gpointer
)LOAD_VTABLE (start
), safe_object_get_size ((MonoObject
*)start
));
1407 process_object_for_domain_clearing (char *start
, MonoDomain
*domain
)
1409 GCVTable
*vt
= (GCVTable
*)LOAD_VTABLE (start
);
1410 if (vt
->klass
== mono_defaults
.internal_thread_class
)
1411 g_assert (mono_object_domain (start
) == mono_get_root_domain ());
1412 /* The object could be a proxy for an object in the domain
1414 if (mono_class_has_parent (vt
->klass
, mono_defaults
.real_proxy_class
)) {
1415 MonoObject
*server
= ((MonoRealProxy
*)start
)->unwrapped_server
;
1417 /* The server could already have been zeroed out, so
1418 we need to check for that, too. */
1419 if (server
&& (!LOAD_VTABLE (server
) || mono_object_domain (server
) == domain
)) {
1420 DEBUG (4, fprintf (gc_debug_file
, "Cleaning up remote pointer in %p to object %p\n",
1422 ((MonoRealProxy
*)start
)->unwrapped_server
= NULL
;
1427 static MonoDomain
*check_domain
= NULL
;
1430 check_obj_not_in_domain (void **o
)
1432 g_assert (((MonoObject
*)(*o
))->vtable
->domain
!= check_domain
);
1436 scan_for_registered_roots_in_domain (MonoDomain
*domain
, int root_type
)
1440 check_domain
= domain
;
1441 for (i
= 0; i
< roots_hash_size
[root_type
]; ++i
) {
1442 for (root
= roots_hash
[root_type
][i
]; root
; root
= root
->next
) {
1443 void **start_root
= (void**)root
->start_root
;
1444 mword desc
= root
->root_desc
;
1446 /* The MonoDomain struct is allowed to hold
1447 references to objects in its own domain. */
1448 if (start_root
== (void**)domain
)
1451 switch (desc
& ROOT_DESC_TYPE_MASK
) {
1452 case ROOT_DESC_BITMAP
:
1453 desc
>>= ROOT_DESC_TYPE_SHIFT
;
1455 if ((desc
& 1) && *start_root
)
1456 check_obj_not_in_domain (*start_root
);
1461 case ROOT_DESC_COMPLEX
: {
1462 gsize
*bitmap_data
= complex_descriptors
+ (desc
>> ROOT_DESC_TYPE_SHIFT
);
1463 int bwords
= (*bitmap_data
) - 1;
1464 void **start_run
= start_root
;
1466 while (bwords
-- > 0) {
1467 gsize bmap
= *bitmap_data
++;
1468 void **objptr
= start_run
;
1470 if ((bmap
& 1) && *objptr
)
1471 check_obj_not_in_domain (*objptr
);
1475 start_run
+= GC_BITS_PER_WORD
;
1479 case ROOT_DESC_USER
: {
1480 MonoGCRootMarkFunc marker
= user_descriptors
[desc
>> ROOT_DESC_TYPE_SHIFT
];
1481 marker (start_root
, check_obj_not_in_domain
);
1484 case ROOT_DESC_RUN_LEN
:
1485 g_assert_not_reached ();
1487 g_assert_not_reached ();
1491 check_domain
= NULL
;
1495 check_for_xdomain_refs (void)
1499 mono_sgen_scan_area_with_callback (nursery_section
->data
, nursery_section
->end_data
,
1500 (IterateObjectCallbackFunc
)scan_object_for_xdomain_refs
, NULL
, FALSE
);
1502 major_collector
.iterate_objects (TRUE
, TRUE
, (IterateObjectCallbackFunc
)scan_object_for_xdomain_refs
, NULL
);
1504 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
)
1505 scan_object_for_xdomain_refs (bigobj
->data
, bigobj
->size
, NULL
);
1509 clear_domain_process_object (char *obj
, MonoDomain
*domain
)
1513 process_object_for_domain_clearing (obj
, domain
);
1514 remove
= need_remove_object_for_domain (obj
, domain
);
1516 if (remove
&& ((MonoObject
*)obj
)->synchronisation
) {
1517 void **dislink
= mono_monitor_get_object_monitor_weak_link ((MonoObject
*)obj
);
1519 mono_gc_register_disappearing_link (NULL
, dislink
, FALSE
);
1526 clear_domain_process_minor_object_callback (char *obj
, size_t size
, MonoDomain
*domain
)
1528 if (clear_domain_process_object (obj
, domain
))
1529 memset (obj
, 0, size
);
1533 clear_domain_process_major_object_callback (char *obj
, size_t size
, MonoDomain
*domain
)
1535 clear_domain_process_object (obj
, domain
);
1539 clear_domain_free_major_non_pinned_object_callback (char *obj
, size_t size
, MonoDomain
*domain
)
1541 if (need_remove_object_for_domain (obj
, domain
))
1542 major_collector
.free_non_pinned_object (obj
, size
);
1546 clear_domain_free_major_pinned_object_callback (char *obj
, size_t size
, MonoDomain
*domain
)
1548 if (need_remove_object_for_domain (obj
, domain
))
1549 major_collector
.free_pinned_object (obj
, size
);
1553 * When appdomains are unloaded we can easily remove objects that have finalizers,
1554 * but all the others could still be present in random places on the heap.
1555 * We need a sweep to get rid of them even though it's going to be costly
1557 * The reason we need to remove them is because we access the vtable and class
1558 * structures to know the object size and the reference bitmap: once the domain is
1559 * unloaded the point to random memory.
1562 mono_gc_clear_domain (MonoDomain
* domain
)
1564 LOSObject
*bigobj
, *prev
;
1569 clear_nursery_fragments (nursery_next
);
1571 if (xdomain_checks
&& domain
!= mono_get_root_domain ()) {
1572 scan_for_registered_roots_in_domain (domain
, ROOT_TYPE_NORMAL
);
1573 scan_for_registered_roots_in_domain (domain
, ROOT_TYPE_WBARRIER
);
1574 check_for_xdomain_refs ();
1577 mono_sgen_scan_area_with_callback (nursery_section
->data
, nursery_section
->end_data
,
1578 (IterateObjectCallbackFunc
)clear_domain_process_minor_object_callback
, domain
, FALSE
);
1580 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1581 to memory returned to the OS.*/
1582 null_ephemerons_for_domain (domain
);
1584 for (i
= GENERATION_NURSERY
; i
< GENERATION_MAX
; ++i
)
1585 null_links_for_domain (domain
, i
);
1587 /* We need two passes over major and large objects because
1588 freeing such objects might give their memory back to the OS
1589 (in the case of large objects) or obliterate its vtable
1590 (pinned objects with major-copying or pinned and non-pinned
1591 objects with major-mark&sweep), but we might need to
1592 dereference a pointer from an object to another object if
1593 the first object is a proxy. */
1594 major_collector
.iterate_objects (TRUE
, TRUE
, (IterateObjectCallbackFunc
)clear_domain_process_major_object_callback
, domain
);
1595 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
)
1596 clear_domain_process_object (bigobj
->data
, domain
);
1599 for (bigobj
= los_object_list
; bigobj
;) {
1600 if (need_remove_object_for_domain (bigobj
->data
, domain
)) {
1601 LOSObject
*to_free
= bigobj
;
1603 prev
->next
= bigobj
->next
;
1605 los_object_list
= bigobj
->next
;
1606 bigobj
= bigobj
->next
;
1607 DEBUG (4, fprintf (gc_debug_file
, "Freeing large object %p\n",
1609 mono_sgen_los_free_object (to_free
);
1613 bigobj
= bigobj
->next
;
1615 major_collector
.iterate_objects (TRUE
, FALSE
, (IterateObjectCallbackFunc
)clear_domain_free_major_non_pinned_object_callback
, domain
);
1616 major_collector
.iterate_objects (FALSE
, TRUE
, (IterateObjectCallbackFunc
)clear_domain_free_major_pinned_object_callback
, domain
);
1622 global_remset_cache_clear (void)
1624 memset (global_remset_cache
, 0, sizeof (global_remset_cache
));
1628 * Tries to check if a given remset location was already added to the global remset.
1631 * A 2 entry, LRU cache of recently saw location remsets.
1633 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1635 * Returns TRUE is the element was added..
1638 global_remset_location_was_not_added (gpointer ptr
)
1641 gpointer first
= global_remset_cache
[0], second
;
1643 HEAVY_STAT (++stat_global_remsets_discarded
);
1647 second
= global_remset_cache
[1];
1649 if (second
== ptr
) {
1650 /*Move the second to the front*/
1651 global_remset_cache
[0] = second
;
1652 global_remset_cache
[1] = first
;
1654 HEAVY_STAT (++stat_global_remsets_discarded
);
1658 global_remset_cache
[0] = second
;
1659 global_remset_cache
[1] = ptr
;
1664 * mono_sgen_add_to_global_remset:
1666 * The global remset contains locations which point into newspace after
1667 * a minor collection. This can happen if the objects they point to are pinned.
1669 * LOCKING: If called from a parallel collector, the global remset
1670 * lock must be held. For serial collectors that is not necessary.
1673 mono_sgen_add_to_global_remset (gpointer ptr
)
1678 if (use_cardtable
) {
1679 sgen_card_table_mark_address ((mword
)ptr
);
1683 g_assert (!ptr_in_nursery (ptr
) && ptr_in_nursery (*(gpointer
*)ptr
));
1685 lock
= (current_collection_generation
== GENERATION_OLD
&& major_collector
.is_parallel
);
1689 if (!global_remset_location_was_not_added (ptr
))
1692 DEBUG (8, fprintf (gc_debug_file
, "Adding global remset for %p\n", ptr
));
1693 binary_protocol_global_remset (ptr
, *(gpointer
*)ptr
, (gpointer
)LOAD_VTABLE (*(gpointer
*)ptr
));
1695 HEAVY_STAT (++stat_global_remsets_added
);
1698 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1699 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1701 if (global_remset
->store_next
+ 3 < global_remset
->end_set
) {
1702 *(global_remset
->store_next
++) = (mword
)ptr
;
1705 rs
= alloc_remset (global_remset
->end_set
- global_remset
->data
, NULL
);
1706 rs
->next
= global_remset
;
1708 *(global_remset
->store_next
++) = (mword
)ptr
;
1711 int global_rs_size
= 0;
1713 for (rs
= global_remset
; rs
; rs
= rs
->next
) {
1714 global_rs_size
+= rs
->store_next
- rs
->data
;
1716 DEBUG (4, fprintf (gc_debug_file
, "Global remset now has size %d\n", global_rs_size
));
1721 UNLOCK_GLOBAL_REMSET
;
1727 * Scan objects in the gray stack until the stack is empty. This should be called
1728 * frequently after each object is copied, to achieve better locality and cache
1732 drain_gray_stack (GrayQueue
*queue
)
1736 if (current_collection_generation
== GENERATION_NURSERY
) {
1738 GRAY_OBJECT_DEQUEUE (queue
, obj
);
1741 DEBUG (9, fprintf (gc_debug_file
, "Precise gray object scan %p (%s)\n", obj
, safe_name (obj
)));
1742 major_collector
.minor_scan_object (obj
, queue
);
1745 if (major_collector
.is_parallel
&& queue
== &workers_distribute_gray_queue
)
1749 GRAY_OBJECT_DEQUEUE (queue
, obj
);
1752 DEBUG (9, fprintf (gc_debug_file
, "Precise gray object scan %p (%s)\n", obj
, safe_name (obj
)));
1753 major_collector
.major_scan_object (obj
, queue
);
1759 * Addresses from start to end are already sorted. This function finds
1760 * the object header for each address and pins the object. The
1761 * addresses must be inside the passed section. The (start of the)
1762 * address array is overwritten with the addresses of the actually
1763 * pinned objects. Return the number of pinned objects.
1766 pin_objects_from_addresses (GCMemSection
*section
, void **start
, void **end
, void *start_nursery
, void *end_nursery
, GrayQueue
*queue
)
1771 void *last_obj
= NULL
;
1772 size_t last_obj_size
= 0;
1775 void **definitely_pinned
= start
;
1779 * The code below starts the search from an entry in scan_starts, which might point into a nursery
1780 * fragment containing random data. Clearing the nursery fragments takes a lot of time, and searching
1781 * though them too, so lay arrays at each location inside a fragment where a search can start:
1782 * - scan_locations[i]
1784 * - the start of each fragment (the last_obj + last_obj case)
1785 * The third encompasses the first two, since scan_locations [i] can't point inside a nursery fragment.
1787 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
1790 g_assert (frag
->fragment_end
- frag
->fragment_start
>= sizeof (MonoArray
));
1791 o
= (MonoArray
*)frag
->fragment_start
;
1792 memset (o
, 0, sizeof (MonoArray
));
1793 g_assert (array_fill_vtable
);
1794 o
->obj
.vtable
= array_fill_vtable
;
1795 /* Mark this as not a real object */
1796 o
->obj
.synchronisation
= GINT_TO_POINTER (-1);
1797 o
->max_length
= (frag
->fragment_end
- frag
->fragment_start
) - sizeof (MonoArray
);
1798 g_assert (frag
->fragment_start
+ safe_object_get_size ((MonoObject
*)o
) == frag
->fragment_end
);
1801 while (start
< end
) {
1803 /* the range check should be reduntant */
1804 if (addr
!= last
&& addr
>= start_nursery
&& addr
< end_nursery
) {
1805 DEBUG (5, fprintf (gc_debug_file
, "Considering pinning addr %p\n", addr
));
1806 /* multiple pointers to the same object */
1807 if (addr
>= last_obj
&& (char*)addr
< (char*)last_obj
+ last_obj_size
) {
1811 idx
= ((char*)addr
- (char*)section
->data
) / SCAN_START_SIZE
;
1812 g_assert (idx
< section
->num_scan_start
);
1813 search_start
= (void*)section
->scan_starts
[idx
];
1814 if (!search_start
|| search_start
> addr
) {
1817 search_start
= section
->scan_starts
[idx
];
1818 if (search_start
&& search_start
<= addr
)
1821 if (!search_start
|| search_start
> addr
)
1822 search_start
= start_nursery
;
1824 if (search_start
< last_obj
)
1825 search_start
= (char*)last_obj
+ last_obj_size
;
1826 /* now addr should be in an object a short distance from search_start
1827 * Note that search_start must point to zeroed mem or point to an object.
1831 if (!*(void**)search_start
) {
1832 /* Consistency check */
1834 for (frag = nursery_fragments; frag; frag = frag->next) {
1835 if (search_start >= frag->fragment_start && search_start < frag->fragment_end)
1836 g_assert_not_reached ();
1840 search_start
= (void*)ALIGN_UP ((mword
)search_start
+ sizeof (gpointer
));
1843 last_obj
= search_start
;
1844 last_obj_size
= ALIGN_UP (safe_object_get_size ((MonoObject
*)search_start
));
1846 if (((MonoObject
*)last_obj
)->synchronisation
== GINT_TO_POINTER (-1)) {
1847 /* Marks the beginning of a nursery fragment, skip */
1849 DEBUG (8, fprintf (gc_debug_file
, "Pinned try match %p (%s), size %zd\n", last_obj
, safe_name (last_obj
), last_obj_size
));
1850 if (addr
>= search_start
&& (char*)addr
< (char*)last_obj
+ last_obj_size
) {
1851 DEBUG (4, fprintf (gc_debug_file
, "Pinned object %p, vtable %p (%s), count %d\n", search_start
, *(void**)search_start
, safe_name (search_start
), count
));
1852 binary_protocol_pin (search_start
, (gpointer
)LOAD_VTABLE (search_start
), safe_object_get_size (search_start
));
1853 pin_object (search_start
);
1854 GRAY_OBJECT_ENQUEUE (queue
, search_start
);
1856 mono_sgen_pin_stats_register_object (search_start
, last_obj_size
);
1857 definitely_pinned
[count
] = search_start
;
1862 /* skip to the next object */
1863 search_start
= (void*)((char*)search_start
+ last_obj_size
);
1864 } while (search_start
<= addr
);
1865 /* we either pinned the correct object or we ignored the addr because
1866 * it points to unused zeroed memory.
1872 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1873 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS
) {
1874 GCRootReport report
;
1876 for (idx
= 0; idx
< count
; ++idx
)
1877 add_profile_gc_root (&report
, definitely_pinned
[idx
], MONO_PROFILE_GC_ROOT_PINNING
, 0);
1878 notify_gc_roots (&report
);
1880 stat_pinned_objects
+= count
;
1885 mono_sgen_pin_objects_in_section (GCMemSection
*section
, GrayQueue
*queue
)
1887 int num_entries
= section
->pin_queue_num_entries
;
1889 void **start
= section
->pin_queue_start
;
1891 reduced_to
= pin_objects_from_addresses (section
, start
, start
+ num_entries
,
1892 section
->data
, section
->next_data
, queue
);
1893 section
->pin_queue_num_entries
= reduced_to
;
1895 section
->pin_queue_start
= NULL
;
1901 mono_sgen_pin_object (void *object
, GrayQueue
*queue
)
1903 if (major_collector
.is_parallel
) {
1905 /*object arrives pinned*/
1906 pin_stage_ptr (object
);
1910 SGEN_PIN_OBJECT (object
);
1911 pin_stage_ptr (object
);
1914 GRAY_OBJECT_ENQUEUE (queue
, object
);
1917 /* Sort the addresses in array in increasing order.
1918 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1921 sort_addresses (void **array
, int size
)
1926 for (i
= 1; i
< size
; ++i
) {
1929 int parent
= (child
- 1) / 2;
1931 if (array
[parent
] >= array
[child
])
1934 tmp
= array
[parent
];
1935 array
[parent
] = array
[child
];
1936 array
[child
] = tmp
;
1942 for (i
= size
- 1; i
> 0; --i
) {
1945 array
[i
] = array
[0];
1951 while (root
* 2 + 1 <= end
) {
1952 int child
= root
* 2 + 1;
1954 if (child
< end
&& array
[child
] < array
[child
+ 1])
1956 if (array
[root
] >= array
[child
])
1960 array
[root
] = array
[child
];
1961 array
[child
] = tmp
;
1968 static G_GNUC_UNUSED
void
1969 print_nursery_gaps (void* start_nursery
, void *end_nursery
)
1972 gpointer first
= start_nursery
;
1974 for (i
= 0; i
< next_pin_slot
; ++i
) {
1975 next
= pin_queue
[i
];
1976 fprintf (gc_debug_file
, "Nursery range: %p-%p, size: %td\n", first
, next
, (char*)next
-(char*)first
);
1980 fprintf (gc_debug_file
, "Nursery range: %p-%p, size: %td\n", first
, next
, (char*)next
-(char*)first
);
1983 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
1985 optimize_pin_queue (int start_slot
)
1987 void **start
, **cur
, **end
;
1988 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
1989 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
1990 DEBUG (5, fprintf (gc_debug_file
, "Sorting pin queue, size: %d\n", next_pin_slot
));
1991 if ((next_pin_slot
- start_slot
) > 1)
1992 sort_addresses (pin_queue
+ start_slot
, next_pin_slot
- start_slot
);
1993 start
= cur
= pin_queue
+ start_slot
;
1994 end
= pin_queue
+ next_pin_slot
;
1997 while (*start
== *cur
&& cur
< end
)
2001 next_pin_slot
= start
- pin_queue
;
2002 DEBUG (5, fprintf (gc_debug_file
, "Pin queue reduced to size: %d\n", next_pin_slot
));
2003 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2008 * Scan the memory between start and end and queue values which could be pointers
2009 * to the area between start_nursery and end_nursery for later consideration.
2010 * Typically used for thread stacks.
2013 conservatively_pin_objects_from (void **start
, void **end
, void *start_nursery
, void *end_nursery
, int pin_type
)
2016 while (start
< end
) {
2017 if (*start
>= start_nursery
&& *start
< end_nursery
) {
2019 * *start can point to the middle of an object
2020 * note: should we handle pointing at the end of an object?
2021 * pinning in C# code disallows pointing at the end of an object
2022 * but there is some small chance that an optimizing C compiler
2023 * may keep the only reference to an object by pointing
2024 * at the end of it. We ignore this small chance for now.
2025 * Pointers to the end of an object are indistinguishable
2026 * from pointers to the start of the next object in memory
2027 * so if we allow that we'd need to pin two objects...
2028 * We queue the pointer in an array, the
2029 * array will then be sorted and uniqued. This way
2030 * we can coalesce several pinning pointers and it should
2031 * be faster since we'd do a memory scan with increasing
2032 * addresses. Note: we can align the address to the allocation
2033 * alignment, so the unique process is more effective.
2035 mword addr
= (mword
)*start
;
2036 addr
&= ~(ALLOC_ALIGN
- 1);
2037 if (addr
>= (mword
)start_nursery
&& addr
< (mword
)end_nursery
)
2038 pin_stage_ptr ((void*)addr
);
2040 pin_stats_register_address ((char*)addr
, pin_type
);
2041 DEBUG (6, if (count
) fprintf (gc_debug_file
, "Pinning address %p from %p\n", (void*)addr
, start
));
2046 DEBUG (7, if (count
) fprintf (gc_debug_file
, "found %d potential pinned heap pointers\n", count
));
2050 * Debugging function: find in the conservative roots where @obj is being pinned.
2052 static G_GNUC_UNUSED
void
2053 find_pinning_reference (char *obj
, size_t size
)
2057 char *endobj
= obj
+ size
;
2058 for (i
= 0; i
< roots_hash_size
[0]; ++i
) {
2059 for (root
= roots_hash
[0][i
]; root
; root
= root
->next
) {
2060 /* if desc is non-null it has precise info */
2061 if (!root
->root_desc
) {
2062 char ** start
= (char**)root
->start_root
;
2063 while (start
< (char**)root
->end_root
) {
2064 if (*start
>= obj
&& *start
< endobj
) {
2065 DEBUG (0, fprintf (gc_debug_file
, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj
, root
->start_root
, root
->end_root
, start
, root
));
2072 find_pinning_ref_from_thread (obj
, size
);
2076 * The first thing we do in a collection is to identify pinned objects.
2077 * This function considers all the areas of memory that need to be
2078 * conservatively scanned.
2081 pin_from_roots (void *start_nursery
, void *end_nursery
)
2085 DEBUG (2, fprintf (gc_debug_file
, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size
, num_roots_entries
[ROOT_TYPE_NORMAL
], num_roots_entries
[ROOT_TYPE_PINNED
]));
2086 /* objects pinned from the API are inside these roots */
2087 for (i
= 0; i
< roots_hash_size
[ROOT_TYPE_PINNED
]; ++i
) {
2088 for (root
= roots_hash
[ROOT_TYPE_PINNED
][i
]; root
; root
= root
->next
) {
2089 DEBUG (6, fprintf (gc_debug_file
, "Pinned roots %p-%p\n", root
->start_root
, root
->end_root
));
2090 conservatively_pin_objects_from ((void**)root
->start_root
, (void**)root
->end_root
, start_nursery
, end_nursery
, PIN_TYPE_OTHER
);
2093 /* now deal with the thread stacks
2094 * in the future we should be able to conservatively scan only:
2095 * *) the cpu registers
2096 * *) the unmanaged stack frames
2097 * *) the _last_ managed stack frame
2098 * *) pointers slots in managed frames
2100 scan_thread_data (start_nursery
, end_nursery
, FALSE
);
2102 evacuate_pin_staging_area ();
2105 static CopyOrMarkObjectFunc user_copy_or_mark_func
;
2106 static GrayQueue
*user_copy_or_mark_queue
;
2109 single_arg_user_copy_or_mark (void **obj
)
2111 user_copy_or_mark_func (obj
, user_copy_or_mark_queue
);
2115 * The memory area from start_root to end_root contains pointers to objects.
2116 * Their position is precisely described by @desc (this means that the pointer
2117 * can be either NULL or the pointer to the start of an object).
2118 * This functions copies them to to_space updates them.
2120 * This function is not thread-safe!
2123 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func
, void** start_root
, void** end_root
, char* n_start
, char *n_end
, mword desc
, GrayQueue
*queue
)
2125 switch (desc
& ROOT_DESC_TYPE_MASK
) {
2126 case ROOT_DESC_BITMAP
:
2127 desc
>>= ROOT_DESC_TYPE_SHIFT
;
2129 if ((desc
& 1) && *start_root
) {
2130 copy_func (start_root
, queue
);
2131 DEBUG (9, fprintf (gc_debug_file
, "Overwrote root at %p with %p\n", start_root
, *start_root
));
2132 drain_gray_stack (queue
);
2138 case ROOT_DESC_COMPLEX
: {
2139 gsize
*bitmap_data
= complex_descriptors
+ (desc
>> ROOT_DESC_TYPE_SHIFT
);
2140 int bwords
= (*bitmap_data
) - 1;
2141 void **start_run
= start_root
;
2143 while (bwords
-- > 0) {
2144 gsize bmap
= *bitmap_data
++;
2145 void **objptr
= start_run
;
2147 if ((bmap
& 1) && *objptr
) {
2148 copy_func (objptr
, queue
);
2149 DEBUG (9, fprintf (gc_debug_file
, "Overwrote root at %p with %p\n", objptr
, *objptr
));
2150 drain_gray_stack (queue
);
2155 start_run
+= GC_BITS_PER_WORD
;
2159 case ROOT_DESC_USER
: {
2160 MonoGCRootMarkFunc marker
= user_descriptors
[desc
>> ROOT_DESC_TYPE_SHIFT
];
2161 user_copy_or_mark_func
= copy_func
;
2162 user_copy_or_mark_queue
= queue
;
2163 marker (start_root
, single_arg_user_copy_or_mark
);
2164 user_copy_or_mark_func
= NULL
;
2165 user_copy_or_mark_queue
= NULL
;
2168 case ROOT_DESC_RUN_LEN
:
2169 g_assert_not_reached ();
2171 g_assert_not_reached ();
2176 reset_heap_boundaries (void)
2178 lowest_heap_address
= ~(mword
)0;
2179 highest_heap_address
= 0;
2183 mono_sgen_update_heap_boundaries (mword low
, mword high
)
2188 old
= lowest_heap_address
;
2191 } while (SGEN_CAS_PTR ((gpointer
*)&lowest_heap_address
, (gpointer
)low
, (gpointer
)old
) != (gpointer
)old
);
2194 old
= highest_heap_address
;
2197 } while (SGEN_CAS_PTR ((gpointer
*)&highest_heap_address
, (gpointer
)high
, (gpointer
)old
) != (gpointer
)old
);
2201 alloc_fragment (void)
2203 Fragment
*frag
= fragment_freelist
;
2205 fragment_freelist
= frag
->next
;
2209 frag
= mono_sgen_alloc_internal (INTERNAL_MEM_FRAGMENT
);
2214 /* size must be a power of 2 */
2216 mono_sgen_alloc_os_memory_aligned (mword size
, mword alignment
, gboolean activate
)
2218 /* Allocate twice the memory to be able to put the block on an aligned address */
2219 char *mem
= mono_sgen_alloc_os_memory (size
+ alignment
, activate
);
2224 aligned
= (char*)((mword
)(mem
+ (alignment
- 1)) & ~(alignment
- 1));
2225 g_assert (aligned
>= mem
&& aligned
+ size
<= mem
+ size
+ alignment
&& !((mword
)aligned
& (alignment
- 1)));
2228 mono_sgen_free_os_memory (mem
, aligned
- mem
);
2229 if (aligned
+ size
< mem
+ size
+ alignment
)
2230 mono_sgen_free_os_memory (aligned
+ size
, (mem
+ size
+ alignment
) - (aligned
+ size
));
2236 * Allocate and setup the data structures needed to be able to allocate objects
2237 * in the nursery. The nursery is stored in nursery_section.
2240 alloc_nursery (void)
2242 GCMemSection
*section
;
2248 if (nursery_section
)
2250 DEBUG (2, fprintf (gc_debug_file
, "Allocating nursery size: %lu\n", (unsigned long)nursery_size
));
2251 /* later we will alloc a larger area for the nursery but only activate
2252 * what we need. The rest will be used as expansion if we have too many pinned
2253 * objects in the existing nursery.
2255 /* FIXME: handle OOM */
2256 section
= mono_sgen_alloc_internal (INTERNAL_MEM_SECTION
);
2258 g_assert (nursery_size
== DEFAULT_NURSERY_SIZE
);
2259 alloc_size
= nursery_size
;
2260 #ifdef SGEN_ALIGN_NURSERY
2261 data
= major_collector
.alloc_heap (alloc_size
, alloc_size
, DEFAULT_NURSERY_BITS
);
2263 data
= major_collector
.alloc_heap (alloc_size
, 0, DEFAULT_NURSERY_BITS
);
2265 nursery_start
= data
;
2266 nursery_real_end
= nursery_start
+ nursery_size
;
2267 mono_sgen_update_heap_boundaries ((mword
)nursery_start
, (mword
)nursery_real_end
);
2268 nursery_next
= nursery_start
;
2269 DEBUG (4, fprintf (gc_debug_file
, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data
, data
+ alloc_size
, (unsigned long)nursery_size
, (unsigned long)total_alloc
));
2270 section
->data
= section
->next_data
= data
;
2271 section
->size
= alloc_size
;
2272 section
->end_data
= nursery_real_end
;
2273 scan_starts
= (alloc_size
+ SCAN_START_SIZE
- 1) / SCAN_START_SIZE
;
2274 section
->scan_starts
= mono_sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts
, INTERNAL_MEM_SCAN_STARTS
);
2275 section
->num_scan_start
= scan_starts
;
2276 section
->block
.role
= MEMORY_ROLE_GEN0
;
2277 section
->block
.next
= NULL
;
2279 nursery_section
= section
;
2281 /* Setup the single first large fragment */
2282 frag
= alloc_fragment ();
2283 frag
->fragment_start
= nursery_start
;
2284 frag
->fragment_limit
= nursery_start
;
2285 frag
->fragment_end
= nursery_real_end
;
2286 nursery_frag_real_end
= nursery_real_end
;
2287 /* FIXME: frag here is lost */
2291 mono_gc_get_nursery (int *shift_bits
, size_t *size
)
2293 *size
= nursery_size
;
2294 #ifdef SGEN_ALIGN_NURSERY
2295 *shift_bits
= DEFAULT_NURSERY_BITS
;
2299 return nursery_start
;
2303 report_finalizer_roots_list (FinalizeEntry
*list
)
2305 GCRootReport report
;
2309 for (fin
= list
; fin
; fin
= fin
->next
) {
2310 MonoObject
*object
= finalize_entry_get_object (fin
);
2313 add_profile_gc_root (&report
, object
, MONO_PROFILE_GC_ROOT_FINALIZER
, 0);
2315 notify_gc_roots (&report
);
2319 report_finalizer_roots (void)
2321 report_finalizer_roots_list (fin_ready_list
);
2322 report_finalizer_roots_list (critical_fin_list
);
2325 static GCRootReport
*root_report
;
2328 single_arg_report_root (void **obj
)
2331 add_profile_gc_root (root_report
, *obj
, MONO_PROFILE_GC_ROOT_OTHER
, 0);
2335 precisely_report_roots_from (GCRootReport
*report
, void** start_root
, void** end_root
, mword desc
)
2337 switch (desc
& ROOT_DESC_TYPE_MASK
) {
2338 case ROOT_DESC_BITMAP
:
2339 desc
>>= ROOT_DESC_TYPE_SHIFT
;
2341 if ((desc
& 1) && *start_root
) {
2342 add_profile_gc_root (report
, *start_root
, MONO_PROFILE_GC_ROOT_OTHER
, 0);
2348 case ROOT_DESC_COMPLEX
: {
2349 gsize
*bitmap_data
= complex_descriptors
+ (desc
>> ROOT_DESC_TYPE_SHIFT
);
2350 int bwords
= (*bitmap_data
) - 1;
2351 void **start_run
= start_root
;
2353 while (bwords
-- > 0) {
2354 gsize bmap
= *bitmap_data
++;
2355 void **objptr
= start_run
;
2357 if ((bmap
& 1) && *objptr
) {
2358 add_profile_gc_root (report
, *objptr
, MONO_PROFILE_GC_ROOT_OTHER
, 0);
2363 start_run
+= GC_BITS_PER_WORD
;
2367 case ROOT_DESC_USER
: {
2368 MonoGCRootMarkFunc marker
= user_descriptors
[desc
>> ROOT_DESC_TYPE_SHIFT
];
2369 root_report
= report
;
2370 marker (start_root
, single_arg_report_root
);
2373 case ROOT_DESC_RUN_LEN
:
2374 g_assert_not_reached ();
2376 g_assert_not_reached ();
2381 report_registered_roots_by_type (int root_type
)
2383 GCRootReport report
;
2387 for (i
= 0; i
< roots_hash_size
[root_type
]; ++i
) {
2388 for (root
= roots_hash
[root_type
][i
]; root
; root
= root
->next
) {
2389 DEBUG (6, fprintf (gc_debug_file
, "Precise root scan %p-%p (desc: %p)\n", root
->start_root
, root
->end_root
, (void*)root
->root_desc
));
2390 precisely_report_roots_from (&report
, (void**)root
->start_root
, (void**)root
->end_root
, root
->root_desc
);
2393 notify_gc_roots (&report
);
2397 report_registered_roots (void)
2399 report_registered_roots_by_type (ROOT_TYPE_NORMAL
);
2400 report_registered_roots_by_type (ROOT_TYPE_WBARRIER
);
2404 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func
, FinalizeEntry
*list
, GrayQueue
*queue
)
2408 for (fin
= list
; fin
; fin
= fin
->next
) {
2409 void *object
= finalize_entry_get_object (fin
);
2410 gboolean bridge_bit
= finalize_entry_get_bridge_bit (fin
);
2413 DEBUG (5, fprintf (gc_debug_file
, "Scan of fin ready object: %p (%s)\n", object
, safe_name (object
)));
2414 copy_func (&object
, queue
);
2416 finalize_entry_set_object (fin
, object
, bridge_bit
);
2420 static mword fragment_total
= 0;
2422 * We found a fragment of free memory in the nursery: memzero it and if
2423 * it is big enough, add it to the list of fragments that can be used for
2427 add_nursery_frag (size_t frag_size
, char* frag_start
, char* frag_end
)
2430 DEBUG (4, fprintf (gc_debug_file
, "Found empty fragment: %p-%p, size: %zd\n", frag_start
, frag_end
, frag_size
));
2431 binary_protocol_empty (frag_start
, frag_size
);
2432 /* Not worth dealing with smaller fragments: need to tune */
2433 if (frag_size
>= FRAGMENT_MIN_SIZE
) {
2434 /* memsetting just the first chunk start is bound to provide better cache locality */
2435 if (nursery_clear_policy
== CLEAR_AT_GC
)
2436 memset (frag_start
, 0, frag_size
);
2438 fragment
= alloc_fragment ();
2439 fragment
->fragment_start
= frag_start
;
2440 fragment
->fragment_limit
= frag_start
;
2441 fragment
->fragment_end
= frag_end
;
2442 fragment
->next
= nursery_fragments
;
2443 nursery_fragments
= fragment
;
2444 fragment_total
+= frag_size
;
2446 /* Clear unused fragments, pinning depends on this */
2447 /*TODO place an int[] here instead of the memset if size justify it*/
2448 memset (frag_start
, 0, frag_size
);
2453 generation_name (int generation
)
2455 switch (generation
) {
2456 case GENERATION_NURSERY
: return "nursery";
2457 case GENERATION_OLD
: return "old";
2458 default: g_assert_not_reached ();
2462 static DisappearingLinkHashTable
*
2463 get_dislink_hash_table (int generation
)
2465 switch (generation
) {
2466 case GENERATION_NURSERY
: return &minor_disappearing_link_hash
;
2467 case GENERATION_OLD
: return &major_disappearing_link_hash
;
2468 default: g_assert_not_reached ();
2472 static FinalizeEntryHashTable
*
2473 get_finalize_entry_hash_table (int generation
)
2475 switch (generation
) {
2476 case GENERATION_NURSERY
: return &minor_finalizable_hash
;
2477 case GENERATION_OLD
: return &major_finalizable_hash
;
2478 default: g_assert_not_reached ();
2482 static MonoObject
**finalized_array
= NULL
;
2483 static int finalized_array_capacity
= 0;
2484 static int finalized_array_entries
= 0;
2487 bridge_register_finalized_object (MonoObject
*object
)
2489 if (!finalized_array
)
2492 if (finalized_array_entries
>= finalized_array_capacity
) {
2493 MonoObject
**new_array
;
2494 g_assert (finalized_array_entries
== finalized_array_capacity
);
2495 finalized_array_capacity
*= 2;
2496 new_array
= mono_sgen_alloc_internal_dynamic (sizeof (MonoObject
*) * finalized_array_capacity
, INTERNAL_MEM_BRIDGE_DATA
);
2497 memcpy (new_array
, finalized_array
, sizeof (MonoObject
*) * finalized_array_entries
);
2498 mono_sgen_free_internal_dynamic (finalized_array
, sizeof (MonoObject
*) * finalized_array_entries
, INTERNAL_MEM_BRIDGE_DATA
);
2499 finalized_array
= new_array
;
2501 finalized_array
[finalized_array_entries
++] = object
;
2505 finish_gray_stack (char *start_addr
, char *end_addr
, int generation
, GrayQueue
*queue
)
2510 int ephemeron_rounds
= 0;
2512 CopyOrMarkObjectFunc copy_func
= current_collection_generation
== GENERATION_NURSERY
? major_collector
.copy_object
: major_collector
.copy_or_mark_object
;
2515 * We copied all the reachable objects. Now it's the time to copy
2516 * the objects that were not referenced by the roots, but by the copied objects.
2517 * we built a stack of objects pointed to by gray_start: they are
2518 * additional roots and we may add more items as we go.
2519 * We loop until gray_start == gray_objects which means no more objects have
2520 * been added. Note this is iterative: no recursion is involved.
2521 * We need to walk the LO list as well in search of marked big objects
2522 * (use a flag since this is needed only on major collections). We need to loop
2523 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2524 * To achieve better cache locality and cache usage, we drain the gray stack
2525 * frequently, after each object is copied, and just finish the work here.
2527 drain_gray_stack (queue
);
2529 DEBUG (2, fprintf (gc_debug_file
, "%s generation done\n", generation_name (generation
)));
2532 We must clear weak links that don't track resurrection before processing object ready for
2533 finalization so they can be cleared before that.
2535 null_link_in_range (copy_func
, start_addr
, end_addr
, generation
, TRUE
, queue
);
2536 if (generation
== GENERATION_OLD
)
2537 null_link_in_range (copy_func
, start_addr
, end_addr
, GENERATION_NURSERY
, TRUE
, queue
);
2539 if (finalized_array
== NULL
&& mono_sgen_need_bridge_processing ()) {
2540 finalized_array_capacity
= 32;
2541 finalized_array
= mono_sgen_alloc_internal_dynamic (sizeof (MonoObject
*) * finalized_array_capacity
, INTERNAL_MEM_BRIDGE_DATA
);
2543 finalized_array_entries
= 0;
2545 /* walk the finalization queue and move also the objects that need to be
2546 * finalized: use the finalized objects as new roots so the objects they depend
2547 * on are also not reclaimed. As with the roots above, only objects in the nursery
2548 * are marked/copied.
2549 * We need a loop here, since objects ready for finalizers may reference other objects
2550 * that are fin-ready. Speedup with a flag?
2555 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2556 * before processing finalizable objects to avoid finalizing reachable values.
2558 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2559 * while they are been finalized.
2561 int done_with_ephemerons
= 0;
2563 done_with_ephemerons
= mark_ephemerons_in_range (copy_func
, start_addr
, end_addr
, queue
);
2564 drain_gray_stack (queue
);
2566 } while (!done_with_ephemerons
);
2568 if (mono_sgen_need_bridge_processing ()) {
2569 collect_bridge_objects (copy_func
, start_addr
, end_addr
, generation
, queue
);
2570 if (generation
== GENERATION_OLD
)
2571 collect_bridge_objects (copy_func
, nursery_start
, nursery_real_end
, GENERATION_NURSERY
, queue
);
2574 fin_ready
= num_ready_finalizers
;
2575 finalize_in_range (copy_func
, start_addr
, end_addr
, generation
, queue
);
2576 if (generation
== GENERATION_OLD
)
2577 finalize_in_range (copy_func
, nursery_start
, nursery_real_end
, GENERATION_NURSERY
, queue
);
2579 if (fin_ready
!= num_ready_finalizers
) {
2581 if (finalized_array
!= NULL
)
2582 mono_sgen_bridge_processing (finalized_array_entries
, finalized_array
);
2585 /* drain the new stack that might have been created */
2586 DEBUG (6, fprintf (gc_debug_file
, "Precise scan of gray area post fin\n"));
2587 drain_gray_stack (queue
);
2588 } while (fin_ready
!= num_ready_finalizers
);
2590 if (mono_sgen_need_bridge_processing ())
2591 g_assert (num_loops
<= 1);
2594 * Clear ephemeron pairs with unreachable keys.
2595 * We pass the copy func so we can figure out if an array was promoted or not.
2597 clear_unreachable_ephemerons (copy_func
, start_addr
, end_addr
, queue
);
2600 DEBUG (2, fprintf (gc_debug_file
, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation
), TV_ELAPSED (atv
, btv
), ephemeron_rounds
));
2603 * handle disappearing links
2604 * Note we do this after checking the finalization queue because if an object
2605 * survives (at least long enough to be finalized) we don't clear the link.
2606 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2607 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2610 g_assert (gray_object_queue_is_empty (queue
));
2612 null_link_in_range (copy_func
, start_addr
, end_addr
, generation
, FALSE
, queue
);
2613 if (generation
== GENERATION_OLD
)
2614 null_link_in_range (copy_func
, start_addr
, end_addr
, GENERATION_NURSERY
, FALSE
, queue
);
2615 if (gray_object_queue_is_empty (queue
))
2617 drain_gray_stack (queue
);
2620 g_assert (gray_object_queue_is_empty (queue
));
2624 mono_sgen_check_section_scan_starts (GCMemSection
*section
)
2627 for (i
= 0; i
< section
->num_scan_start
; ++i
) {
2628 if (section
->scan_starts
[i
]) {
2629 guint size
= safe_object_get_size ((MonoObject
*) section
->scan_starts
[i
]);
2630 g_assert (size
>= sizeof (MonoObject
) && size
<= MAX_SMALL_OBJ_SIZE
);
2636 check_scan_starts (void)
2638 if (!do_scan_starts_check
)
2640 mono_sgen_check_section_scan_starts (nursery_section
);
2641 major_collector
.check_scan_starts ();
2644 static int last_num_pinned
= 0;
2647 build_nursery_fragments (void **start
, int num_entries
)
2649 char *frag_start
, *frag_end
;
2653 while (nursery_fragments
) {
2654 Fragment
*next
= nursery_fragments
->next
;
2655 nursery_fragments
->next
= fragment_freelist
;
2656 fragment_freelist
= nursery_fragments
;
2657 nursery_fragments
= next
;
2659 frag_start
= nursery_start
;
2661 /* clear scan starts */
2662 memset (nursery_section
->scan_starts
, 0, nursery_section
->num_scan_start
* sizeof (gpointer
));
2663 for (i
= 0; i
< num_entries
; ++i
) {
2664 frag_end
= start
[i
];
2665 /* remove the pin bit from pinned objects */
2666 unpin_object (frag_end
);
2667 nursery_section
->scan_starts
[((char*)frag_end
- (char*)nursery_section
->data
)/SCAN_START_SIZE
] = frag_end
;
2668 frag_size
= frag_end
- frag_start
;
2670 add_nursery_frag (frag_size
, frag_start
, frag_end
);
2671 frag_size
= ALIGN_UP (safe_object_get_size ((MonoObject
*)start
[i
]));
2672 frag_start
= (char*)start
[i
] + frag_size
;
2674 nursery_last_pinned_end
= frag_start
;
2675 frag_end
= nursery_real_end
;
2676 frag_size
= frag_end
- frag_start
;
2678 add_nursery_frag (frag_size
, frag_start
, frag_end
);
2679 if (!nursery_fragments
) {
2680 DEBUG (1, fprintf (gc_debug_file
, "Nursery fully pinned (%d)\n", num_entries
));
2681 for (i
= 0; i
< num_entries
; ++i
) {
2682 DEBUG (3, fprintf (gc_debug_file
, "Bastard pinning obj %p (%s), size: %d\n", start
[i
], safe_name (start
[i
]), safe_object_get_size (start
[i
])));
2687 nursery_next
= nursery_frag_real_end
= NULL
;
2689 /* Clear TLABs for all threads */
2694 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func
, char *addr_start
, char *addr_end
, int root_type
, GrayQueue
*queue
)
2698 for (i
= 0; i
< roots_hash_size
[root_type
]; ++i
) {
2699 for (root
= roots_hash
[root_type
][i
]; root
; root
= root
->next
) {
2700 DEBUG (6, fprintf (gc_debug_file
, "Precise root scan %p-%p (desc: %p)\n", root
->start_root
, root
->end_root
, (void*)root
->root_desc
));
2701 precisely_scan_objects_from (copy_func
, (void**)root
->start_root
, (void**)root
->end_root
, addr_start
, addr_end
, root
->root_desc
, queue
);
2707 mono_sgen_dump_occupied (char *start
, char *end
, char *section_start
)
2709 fprintf (heap_dump_file
, "<occupied offset=\"%td\" size=\"%td\"/>\n", start
- section_start
, end
- start
);
2713 mono_sgen_dump_section (GCMemSection
*section
, const char *type
)
2715 char *start
= section
->data
;
2716 char *end
= section
->data
+ section
->size
;
2717 char *occ_start
= NULL
;
2719 char *old_start
= NULL
; /* just for debugging */
2721 fprintf (heap_dump_file
, "<section type=\"%s\" size=\"%lu\">\n", type
, (unsigned long)section
->size
);
2723 while (start
< end
) {
2727 if (!*(void**)start
) {
2729 mono_sgen_dump_occupied (occ_start
, start
, section
->data
);
2732 start
+= sizeof (void*); /* should be ALLOC_ALIGN, really */
2735 g_assert (start
< section
->next_data
);
2740 vt
= (GCVTable
*)LOAD_VTABLE (start
);
2743 size
= ALIGN_UP (safe_object_get_size ((MonoObject
*) start
));
2746 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2747 start - section->data,
2748 vt->klass->name_space, vt->klass->name,
2756 mono_sgen_dump_occupied (occ_start
, start
, section
->data
);
2758 fprintf (heap_dump_file
, "</section>\n");
2762 dump_object (MonoObject
*obj
, gboolean dump_location
)
2764 static char class_name
[1024];
2766 MonoClass
*class = mono_object_class (obj
);
2770 * Python's XML parser is too stupid to parse angle brackets
2771 * in strings, so we just ignore them;
2774 while (class->name
[i
] && j
< sizeof (class_name
) - 1) {
2775 if (!strchr ("<>\"", class->name
[i
]))
2776 class_name
[j
++] = class->name
[i
];
2779 g_assert (j
< sizeof (class_name
));
2782 fprintf (heap_dump_file
, "<object class=\"%s.%s\" size=\"%d\"",
2783 class->name_space
, class_name
,
2784 safe_object_get_size (obj
));
2785 if (dump_location
) {
2786 const char *location
;
2787 if (ptr_in_nursery (obj
))
2788 location
= "nursery";
2789 else if (safe_object_get_size (obj
) <= MAX_SMALL_OBJ_SIZE
)
2793 fprintf (heap_dump_file
, " location=\"%s\"", location
);
2795 fprintf (heap_dump_file
, "/>\n");
2799 dump_heap (const char *type
, int num
, const char *reason
)
2804 fprintf (heap_dump_file
, "<collection type=\"%s\" num=\"%d\"", type
, num
);
2806 fprintf (heap_dump_file
, " reason=\"%s\"", reason
);
2807 fprintf (heap_dump_file
, ">\n");
2808 fprintf (heap_dump_file
, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2809 mono_sgen_dump_internal_mem_usage (heap_dump_file
);
2810 fprintf (heap_dump_file
, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts
[PIN_TYPE_STACK
]);
2811 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2812 fprintf (heap_dump_file
, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts
[PIN_TYPE_OTHER
]);
2814 fprintf (heap_dump_file
, "<pinned-objects>\n");
2815 for (list
= pinned_objects
; list
; list
= list
->next
)
2816 dump_object (list
->obj
, TRUE
);
2817 fprintf (heap_dump_file
, "</pinned-objects>\n");
2819 mono_sgen_dump_section (nursery_section
, "nursery");
2821 major_collector
.dump_heap (heap_dump_file
);
2823 fprintf (heap_dump_file
, "<los>\n");
2824 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
)
2825 dump_object ((MonoObject
*)bigobj
->data
, FALSE
);
2826 fprintf (heap_dump_file
, "</los>\n");
2828 fprintf (heap_dump_file
, "</collection>\n");
2832 mono_sgen_register_moved_object (void *obj
, void *destination
)
2834 g_assert (mono_profiler_events
& MONO_PROFILE_GC_MOVES
);
2836 /* FIXME: handle this for parallel collector */
2837 g_assert (!major_collector
.is_parallel
);
2839 if (moved_objects_idx
== MOVED_OBJECTS_NUM
) {
2840 mono_profiler_gc_moves (moved_objects
, moved_objects_idx
);
2841 moved_objects_idx
= 0;
2843 moved_objects
[moved_objects_idx
++] = obj
;
2844 moved_objects
[moved_objects_idx
++] = destination
;
2850 static gboolean inited
= FALSE
;
2855 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_pre_collection_fragment_clear
);
2856 mono_counters_register ("Minor pinning", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_pinning
);
2857 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_scan_remsets
);
2858 mono_counters_register ("Minor scan cardtables", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_scan_card_table
);
2859 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_scan_pinned
);
2860 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_scan_registered_roots
);
2861 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_scan_thread_data
);
2862 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_finish_gray_stack
);
2863 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_minor_fragment_creation
);
2865 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_pre_collection_fragment_clear
);
2866 mono_counters_register ("Major pinning", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_pinning
);
2867 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_scan_pinned
);
2868 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_scan_registered_roots
);
2869 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_scan_thread_data
);
2870 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_scan_alloc_pinned
);
2871 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_scan_finalized
);
2872 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_scan_big_objects
);
2873 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_finish_gray_stack
);
2874 mono_counters_register ("Major free big objects", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_free_bigobjs
);
2875 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_los_sweep
);
2876 mono_counters_register ("Major sweep", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_sweep
);
2877 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &time_major_fragment_creation
);
2879 mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_pinned_objects
);
2881 #ifdef HEAVY_STATISTICS
2882 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_set_field
);
2883 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_set_arrayref
);
2884 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_arrayref_copy
);
2885 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_generic_store
);
2886 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_generic_store_remset
);
2887 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_set_root
);
2888 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_value_copy
);
2889 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &stat_wbarrier_object_copy
);
2891 mono_counters_register ("# objects allocated", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_objects_alloced
);
2892 mono_counters_register ("bytes allocated", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_bytes_alloced
);
2893 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_objects_alloced_degraded
);
2894 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_bytes_alloced_degraded
);
2895 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_bytes_alloced_los
);
2897 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_copy_object_called_nursery
);
2898 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_objects_copied_nursery
);
2899 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_copy_object_called_major
);
2900 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_objects_copied_major
);
2902 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_scan_object_called_nursery
);
2903 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_scan_object_called_major
);
2905 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_nursery_copy_object_failed_from_space
);
2906 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_nursery_copy_object_failed_forwarded
);
2907 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_nursery_copy_object_failed_pinned
);
2909 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_wasted_fragments_used
);
2910 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_wasted_fragments_bytes
);
2912 mono_counters_register ("Store remsets", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_store_remsets
);
2913 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_store_remsets_unique
);
2914 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_saved_remsets_1
);
2915 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_saved_remsets_2
);
2916 mono_counters_register ("Non-global remsets processed", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_local_remsets_processed
);
2917 mono_counters_register ("Global remsets added", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_global_remsets_added
);
2918 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_global_remsets_readded
);
2919 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_global_remsets_processed
);
2920 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC
| MONO_COUNTER_LONG
, &stat_global_remsets_discarded
);
2926 static gboolean need_calculate_minor_collection_allowance
;
2928 static int last_collection_old_num_major_sections
;
2929 static mword last_collection_los_memory_usage
= 0;
2930 static mword last_collection_old_los_memory_usage
;
2931 static mword last_collection_los_memory_alloced
;
2934 reset_minor_collection_allowance (void)
2936 need_calculate_minor_collection_allowance
= TRUE
;
2940 try_calculate_minor_collection_allowance (gboolean overwrite
)
2942 int num_major_sections
, num_major_sections_saved
, save_target
, allowance_target
;
2943 mword los_memory_saved
;
2946 g_assert (need_calculate_minor_collection_allowance
);
2948 if (!need_calculate_minor_collection_allowance
)
2951 if (!*major_collector
.have_swept
) {
2953 minor_collection_allowance
= MIN_MINOR_COLLECTION_ALLOWANCE
;
2957 num_major_sections
= major_collector
.get_num_major_sections ();
2959 num_major_sections_saved
= MAX (last_collection_old_num_major_sections
- num_major_sections
, 0);
2960 los_memory_saved
= MAX (last_collection_old_los_memory_usage
- last_collection_los_memory_usage
, 1);
2962 save_target
= ((num_major_sections
* major_collector
.section_size
) + los_memory_saved
) / 2;
2965 * We aim to allow the allocation of as many sections as is
2966 * necessary to reclaim save_target sections in the next
2967 * collection. We assume the collection pattern won't change.
2968 * In the last cycle, we had num_major_sections_saved for
2969 * minor_collection_sections_alloced. Assuming things won't
2970 * change, this must be the same ratio as save_target for
2971 * allowance_target, i.e.
2973 * num_major_sections_saved save_target
2974 * --------------------------------- == ----------------
2975 * minor_collection_sections_alloced allowance_target
2979 allowance_target
= (mword
)((double)save_target
* (double)(minor_collection_sections_alloced
* major_collector
.section_size
+ last_collection_los_memory_alloced
) / (double)(num_major_sections_saved
* major_collector
.section_size
+ los_memory_saved
));
2981 minor_collection_allowance
= MAX (MIN (allowance_target
, num_major_sections
* major_collector
.section_size
+ los_memory_usage
), MIN_MINOR_COLLECTION_ALLOWANCE
);
2983 if (major_collector
.have_computed_minor_collection_allowance
)
2984 major_collector
.have_computed_minor_collection_allowance ();
2986 need_calculate_minor_collection_allowance
= FALSE
;
2990 need_major_collection (mword space_needed
)
2992 mword los_alloced
= los_memory_usage
- MIN (last_collection_los_memory_usage
, los_memory_usage
);
2993 return (space_needed
> available_free_space ()) ||
2994 minor_collection_sections_alloced
* major_collector
.section_size
+ los_alloced
> minor_collection_allowance
;
2998 mono_sgen_need_major_collection (mword space_needed
)
3000 return need_major_collection (space_needed
);
3004 * Collect objects in the nursery. Returns whether to trigger a major
3008 collect_nursery (size_t requested_size
)
3010 gboolean needs_major
;
3011 size_t max_garbage_amount
;
3012 char *orig_nursery_next
;
3013 TV_DECLARE (all_atv
);
3014 TV_DECLARE (all_btv
);
3018 if (disable_minor_collections
)
3021 mono_perfcounters
->gc_collections0
++;
3023 current_collection_generation
= GENERATION_NURSERY
;
3025 binary_protocol_collection (GENERATION_NURSERY
);
3026 check_scan_starts ();
3030 orig_nursery_next
= nursery_next
;
3031 nursery_next
= MAX (nursery_next
, nursery_last_pinned_end
);
3032 /* FIXME: optimize later to use the higher address where an object can be present */
3033 nursery_next
= MAX (nursery_next
, nursery_real_end
);
3035 DEBUG (1, fprintf (gc_debug_file
, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs
, nursery_start
, nursery_next
, (int)(nursery_next
- nursery_start
)));
3036 max_garbage_amount
= nursery_next
- nursery_start
;
3037 g_assert (nursery_section
->size
>= max_garbage_amount
);
3039 /* world must be stopped already */
3040 TV_GETTIME (all_atv
);
3043 /* Pinning no longer depends on clearing all nursery fragments */
3044 clear_current_nursery_fragment (orig_nursery_next
);
3047 time_minor_pre_collection_fragment_clear
+= TV_ELAPSED_MS (atv
, btv
);
3050 check_for_xdomain_refs ();
3052 nursery_section
->next_data
= nursery_next
;
3054 major_collector
.start_nursery_collection ();
3056 try_calculate_minor_collection_allowance (FALSE
);
3058 gray_object_queue_init (&gray_queue
, mono_sgen_get_unmanaged_allocator ());
3061 mono_stats
.minor_gc_count
++;
3063 global_remset_cache_clear ();
3065 /* pin from pinned handles */
3067 mono_profiler_gc_event (MONO_GC_EVENT_MARK_START
, 0);
3068 pin_from_roots (nursery_start
, nursery_next
);
3069 /* identify pinned objects */
3070 optimize_pin_queue (0);
3071 next_pin_slot
= pin_objects_from_addresses (nursery_section
, pin_queue
, pin_queue
+ next_pin_slot
, nursery_start
, nursery_next
, &gray_queue
);
3072 nursery_section
->pin_queue_start
= pin_queue
;
3073 nursery_section
->pin_queue_num_entries
= next_pin_slot
;
3075 time_minor_pinning
+= TV_ELAPSED_MS (btv
, atv
);
3076 DEBUG (2, fprintf (gc_debug_file
, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot
, TV_ELAPSED (btv
, atv
)));
3077 DEBUG (4, fprintf (gc_debug_file
, "Start scan with %d pinned objects\n", next_pin_slot
));
3079 if (consistency_check_at_minor_collection
)
3080 check_consistency ();
3083 * walk all the roots and copy the young objects to the old generation,
3084 * starting from to_space
3087 scan_from_remsets (nursery_start
, nursery_next
, &gray_queue
);
3088 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3090 time_minor_scan_remsets
+= TV_ELAPSED_MS (atv
, btv
);
3091 DEBUG (2, fprintf (gc_debug_file
, "Old generation scan: %d usecs\n", TV_ELAPSED (atv
, btv
)));
3093 if (use_cardtable
) {
3095 card_tables_collect_stats (TRUE
);
3096 scan_from_card_tables (nursery_start
, nursery_next
, &gray_queue
);
3098 time_minor_scan_card_table
+= TV_ELAPSED_MS (atv
, btv
);
3101 drain_gray_stack (&gray_queue
);
3103 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS
)
3104 report_registered_roots ();
3105 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS
)
3106 report_finalizer_roots ();
3108 time_minor_scan_pinned
+= TV_ELAPSED_MS (btv
, atv
);
3109 /* registered roots, this includes static fields */
3110 scan_from_registered_roots (major_collector
.copy_object
, nursery_start
, nursery_next
, ROOT_TYPE_NORMAL
, &gray_queue
);
3111 scan_from_registered_roots (major_collector
.copy_object
, nursery_start
, nursery_next
, ROOT_TYPE_WBARRIER
, &gray_queue
);
3113 time_minor_scan_registered_roots
+= TV_ELAPSED_MS (atv
, btv
);
3115 scan_thread_data (nursery_start
, nursery_next
, TRUE
);
3117 time_minor_scan_thread_data
+= TV_ELAPSED_MS (btv
, atv
);
3120 finish_gray_stack (nursery_start
, nursery_next
, GENERATION_NURSERY
, &gray_queue
);
3122 time_minor_finish_gray_stack
+= TV_ELAPSED_MS (btv
, atv
);
3123 mono_profiler_gc_event (MONO_GC_EVENT_MARK_END
, 0);
3125 if (objects_pinned
) {
3126 evacuate_pin_staging_area ();
3127 optimize_pin_queue (0);
3128 nursery_section
->pin_queue_start
= pin_queue
;
3129 nursery_section
->pin_queue_num_entries
= next_pin_slot
;
3132 /* walk the pin_queue, build up the fragment list of free memory, unmark
3133 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3136 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START
, 0);
3137 build_nursery_fragments (pin_queue
, next_pin_slot
);
3138 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END
, 0);
3140 time_minor_fragment_creation
+= TV_ELAPSED_MS (atv
, btv
);
3141 DEBUG (2, fprintf (gc_debug_file
, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv
, btv
), (unsigned long)fragment_total
));
3143 if (consistency_check_at_minor_collection
)
3144 check_major_refs ();
3146 major_collector
.finish_nursery_collection ();
3148 TV_GETTIME (all_btv
);
3149 mono_stats
.minor_gc_time_usecs
+= TV_ELAPSED (all_atv
, all_btv
);
3152 dump_heap ("minor", num_minor_gcs
- 1, NULL
);
3154 /* prepare the pin queue for the next collection */
3155 last_num_pinned
= next_pin_slot
;
3157 if (fin_ready_list
|| critical_fin_list
) {
3158 DEBUG (4, fprintf (gc_debug_file
, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers
));
3159 mono_gc_finalize_notify ();
3163 g_assert (gray_object_queue_is_empty (&gray_queue
));
3166 card_tables_collect_stats (FALSE
);
3168 check_scan_starts ();
3170 binary_protocol_flush_buffers (FALSE
);
3172 /*objects are late pinned because of lack of memory, so a major is a good call*/
3173 needs_major
= need_major_collection (0) || objects_pinned
;
3174 current_collection_generation
= -1;
3181 major_do_collection (const char *reason
)
3183 LOSObject
*bigobj
, *prevbo
;
3184 TV_DECLARE (all_atv
);
3185 TV_DECLARE (all_btv
);
3188 /* FIXME: only use these values for the precise scan
3189 * note that to_space pointers should be excluded anyway...
3191 char *heap_start
= NULL
;
3192 char *heap_end
= (char*)-1;
3193 int old_next_pin_slot
;
3195 mono_perfcounters
->gc_collections1
++;
3197 last_collection_old_num_major_sections
= major_collector
.get_num_major_sections ();
3200 * A domain could have been freed, resulting in
3201 * los_memory_usage being less than last_collection_los_memory_usage.
3203 last_collection_los_memory_alloced
= los_memory_usage
- MIN (last_collection_los_memory_usage
, los_memory_usage
);
3204 last_collection_old_los_memory_usage
= los_memory_usage
;
3207 //count_ref_nonref_objs ();
3208 //consistency_check ();
3210 binary_protocol_collection (GENERATION_OLD
);
3211 check_scan_starts ();
3212 gray_object_queue_init (&gray_queue
, mono_sgen_get_unmanaged_allocator ());
3213 if (major_collector
.is_parallel
)
3214 gray_object_queue_init (&workers_distribute_gray_queue
, mono_sgen_get_unmanaged_allocator ());
3217 DEBUG (1, fprintf (gc_debug_file
, "Start major collection %d\n", num_major_gcs
));
3219 mono_stats
.major_gc_count
++;
3221 /* world must be stopped already */
3222 TV_GETTIME (all_atv
);
3225 /* Pinning depends on this */
3226 clear_nursery_fragments (nursery_next
);
3229 time_major_pre_collection_fragment_clear
+= TV_ELAPSED_MS (atv
, btv
);
3231 nursery_section
->next_data
= nursery_real_end
;
3232 /* we should also coalesce scanning from sections close to each other
3233 * and deal with pointers outside of the sections later.
3236 if (major_collector
.start_major_collection
)
3237 major_collector
.start_major_collection ();
3239 *major_collector
.have_swept
= FALSE
;
3240 reset_minor_collection_allowance ();
3243 check_for_xdomain_refs ();
3245 /* The remsets are not useful for a major collection */
3247 global_remset_cache_clear ();
3249 card_table_clear ();
3253 DEBUG (6, fprintf (gc_debug_file
, "Collecting pinned addresses\n"));
3254 pin_from_roots ((void*)lowest_heap_address
, (void*)highest_heap_address
);
3255 optimize_pin_queue (0);
3258 * pin_queue now contains all candidate pointers, sorted and
3259 * uniqued. We must do two passes now to figure out which
3260 * objects are pinned.
3262 * The first is to find within the pin_queue the area for each
3263 * section. This requires that the pin_queue be sorted. We
3264 * also process the LOS objects and pinned chunks here.
3266 * The second, destructive, pass is to reduce the section
3267 * areas to pointers to the actually pinned objects.
3269 DEBUG (6, fprintf (gc_debug_file
, "Pinning from sections\n"));
3270 /* first pass for the sections */
3271 mono_sgen_find_section_pin_queue_start_end (nursery_section
);
3272 major_collector
.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE
);
3273 /* identify possible pointers to the insize of large objects */
3274 DEBUG (6, fprintf (gc_debug_file
, "Pinning from large objects\n"));
3275 for (bigobj
= los_object_list
; bigobj
; bigobj
= bigobj
->next
) {
3277 if (mono_sgen_find_optimized_pin_queue_area (bigobj
->data
, (char*)bigobj
->data
+ bigobj
->size
, &dummy
)) {
3278 pin_object (bigobj
->data
);
3279 /* FIXME: only enqueue if object has references */
3280 GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE
, bigobj
->data
);
3282 mono_sgen_pin_stats_register_object ((char*) bigobj
->data
, safe_object_get_size ((MonoObject
*) bigobj
->data
));
3283 DEBUG (6, fprintf (gc_debug_file
, "Marked large object %p (%s) size: %lu from roots\n", bigobj
->data
, safe_name (bigobj
->data
), (unsigned long)bigobj
->size
));
3286 /* second pass for the sections */
3287 mono_sgen_pin_objects_in_section (nursery_section
, WORKERS_DISTRIBUTE_GRAY_QUEUE
);
3288 major_collector
.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE
);
3289 old_next_pin_slot
= next_pin_slot
;
3292 time_major_pinning
+= TV_ELAPSED_MS (atv
, btv
);
3293 DEBUG (2, fprintf (gc_debug_file
, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot
, TV_ELAPSED (atv
, btv
)));
3294 DEBUG (4, fprintf (gc_debug_file
, "Start scan with %d pinned objects\n", next_pin_slot
));
3296 major_collector
.init_to_space ();
3298 workers_start_all_workers (1);
3300 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS
)
3301 report_registered_roots ();
3303 time_major_scan_pinned
+= TV_ELAPSED_MS (btv
, atv
);
3305 /* registered roots, this includes static fields */
3306 scan_from_registered_roots (major_collector
.copy_or_mark_object
, heap_start
, heap_end
, ROOT_TYPE_NORMAL
, WORKERS_DISTRIBUTE_GRAY_QUEUE
);
3307 scan_from_registered_roots (major_collector
.copy_or_mark_object
, heap_start
, heap_end
, ROOT_TYPE_WBARRIER
, WORKERS_DISTRIBUTE_GRAY_QUEUE
);
3309 time_major_scan_registered_roots
+= TV_ELAPSED_MS (atv
, btv
);
3312 /* FIXME: This is the wrong place for this, because it does
3314 scan_thread_data (heap_start
, heap_end
, TRUE
);
3316 time_major_scan_thread_data
+= TV_ELAPSED_MS (btv
, atv
);
3319 time_major_scan_alloc_pinned
+= TV_ELAPSED_MS (atv
, btv
);
3321 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS
)
3322 report_finalizer_roots ();
3323 /* scan the list of objects ready for finalization */
3324 scan_finalizer_entries (major_collector
.copy_or_mark_object
, fin_ready_list
, WORKERS_DISTRIBUTE_GRAY_QUEUE
);
3325 scan_finalizer_entries (major_collector
.copy_or_mark_object
, critical_fin_list
, WORKERS_DISTRIBUTE_GRAY_QUEUE
);
3327 time_major_scan_finalized
+= TV_ELAPSED_MS (btv
, atv
);
3328 DEBUG (2, fprintf (gc_debug_file
, "Root scan: %d usecs\n", TV_ELAPSED (btv
, atv
)));
3331 time_major_scan_big_objects
+= TV_ELAPSED_MS (atv
, btv
);
3333 if (major_collector
.is_parallel
) {
3334 while (!gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE
)) {
3335 workers_distribute_gray_queue_sections ();
3339 workers_change_num_working (-1);
3342 if (major_collector
.is_parallel
)
3343 g_assert (gray_object_queue_is_empty (&gray_queue
));
3345 /* all the objects in the heap */
3346 finish_gray_stack (heap_start
, heap_end
, GENERATION_OLD
, &gray_queue
);
3348 time_major_finish_gray_stack
+= TV_ELAPSED_MS (btv
, atv
);
3350 if (objects_pinned
) {
3351 /*This is slow, but we just OOM'd*/
3352 mono_sgen_pin_queue_clear_discarded_entries (nursery_section
, old_next_pin_slot
);
3353 evacuate_pin_staging_area ();
3354 optimize_pin_queue (0);
3355 mono_sgen_find_section_pin_queue_start_end (nursery_section
);
3359 reset_heap_boundaries ();
3360 mono_sgen_update_heap_boundaries ((mword
)nursery_start
, (mword
)nursery_real_end
);
3362 /* sweep the big objects list */
3364 for (bigobj
= los_object_list
; bigobj
;) {
3365 if (object_is_pinned (bigobj
->data
)) {
3366 unpin_object (bigobj
->data
);
3367 mono_sgen_update_heap_boundaries ((mword
)bigobj
->data
, (mword
)bigobj
->data
+ bigobj
->size
);
3370 /* not referenced anywhere, so we can free it */
3372 prevbo
->next
= bigobj
->next
;
3374 los_object_list
= bigobj
->next
;
3376 bigobj
= bigobj
->next
;
3377 mono_sgen_los_free_object (to_free
);
3381 bigobj
= bigobj
->next
;
3385 time_major_free_bigobjs
+= TV_ELAPSED_MS (atv
, btv
);
3387 mono_sgen_los_sweep ();
3390 time_major_los_sweep
+= TV_ELAPSED_MS (btv
, atv
);
3392 major_collector
.sweep ();
3395 time_major_sweep
+= TV_ELAPSED_MS (atv
, btv
);
3397 /* walk the pin_queue, build up the fragment list of free memory, unmark
3398 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3401 build_nursery_fragments (nursery_section
->pin_queue_start
, nursery_section
->pin_queue_num_entries
);
3404 time_major_fragment_creation
+= TV_ELAPSED_MS (btv
, atv
);
3406 TV_GETTIME (all_btv
);
3407 mono_stats
.major_gc_time_usecs
+= TV_ELAPSED (all_atv
, all_btv
);
3410 dump_heap ("major", num_major_gcs
- 1, reason
);
3412 /* prepare the pin queue for the next collection */
3414 if (fin_ready_list
|| critical_fin_list
) {
3415 DEBUG (4, fprintf (gc_debug_file
, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers
));
3416 mono_gc_finalize_notify ();
3420 g_assert (gray_object_queue_is_empty (&gray_queue
));
3422 try_calculate_minor_collection_allowance (TRUE
);
3424 minor_collection_sections_alloced
= 0;
3425 last_collection_los_memory_usage
= los_memory_usage
;
3427 major_collector
.finish_major_collection ();
3429 check_scan_starts ();
3431 binary_protocol_flush_buffers (FALSE
);
3433 //consistency_check ();
3437 major_collection (const char *reason
)
3439 if (disable_major_collections
) {
3440 collect_nursery (0);
3444 major_collection_hapenned
= TRUE
;
3445 current_collection_generation
= GENERATION_OLD
;
3446 major_do_collection (reason
);
3447 current_collection_generation
= -1;
3451 sgen_collect_major_no_lock (const char *reason
)
3453 mono_profiler_gc_event (MONO_GC_EVENT_START
, 1);
3455 major_collection (reason
);
3457 mono_profiler_gc_event (MONO_GC_EVENT_END
, 1);
3461 * When deciding if it's better to collect or to expand, keep track
3462 * of how much garbage was reclaimed with the last collection: if it's too
3464 * This is called when we could not allocate a small object.
3466 static void __attribute__((noinline
))
3467 minor_collect_or_expand_inner (size_t size
)
3469 int do_minor_collection
= 1;
3471 g_assert (nursery_section
);
3472 if (do_minor_collection
) {
3473 mono_profiler_gc_event (MONO_GC_EVENT_START
, 0);
3475 if (collect_nursery (size
)) {
3476 mono_profiler_gc_event (MONO_GC_EVENT_START
, 1);
3477 major_collection ("minor overflow");
3478 /* keep events symmetric */
3479 mono_profiler_gc_event (MONO_GC_EVENT_END
, 1);
3481 DEBUG (2, fprintf (gc_debug_file
, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc
, (unsigned long)los_memory_usage
));
3483 /* this also sets the proper pointers for the next allocation */
3484 if (!search_fragment_for_size (size
)) {
3486 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3487 DEBUG (1, fprintf (gc_debug_file
, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size
, last_num_pinned
));
3488 for (i
= 0; i
< last_num_pinned
; ++i
) {
3489 DEBUG (3, fprintf (gc_debug_file
, "Bastard pinning obj %p (%s), size: %d\n", pin_queue
[i
], safe_name (pin_queue
[i
]), safe_object_get_size (pin_queue
[i
])));
3493 mono_profiler_gc_event (MONO_GC_EVENT_END
, 0);
3495 //report_internal_mem_usage ();
3499 * ######################################################################
3500 * ######## Memory allocation from the OS
3501 * ######################################################################
3502 * This section of code deals with getting memory from the OS and
3503 * allocating memory for GC-internal data structures.
3504 * Internal memory can be handled with a freelist for small objects.
3510 G_GNUC_UNUSED
static void
3511 report_internal_mem_usage (void)
3513 printf ("Internal memory usage:\n");
3514 mono_sgen_report_internal_mem_usage ();
3515 printf ("Pinned memory usage:\n");
3516 major_collector
.report_pinned_memory_usage ();
3520 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3521 * This must not require any lock.
3524 mono_sgen_alloc_os_memory (size_t size
, int activate
)
3527 unsigned long prot_flags
= activate
? MONO_MMAP_READ
|MONO_MMAP_WRITE
: MONO_MMAP_NONE
;
3529 prot_flags
|= MONO_MMAP_PRIVATE
| MONO_MMAP_ANON
;
3530 size
+= pagesize
- 1;
3531 size
&= ~(pagesize
- 1);
3532 ptr
= mono_valloc (0, size
, prot_flags
);
3534 total_alloc
+= size
;
3539 * Free the memory returned by mono_sgen_alloc_os_memory (), returning it to the OS.
3542 mono_sgen_free_os_memory (void *addr
, size_t size
)
3544 mono_vfree (addr
, size
);
3546 size
+= pagesize
- 1;
3547 size
&= ~(pagesize
- 1);
3549 total_alloc
-= size
;
3553 * ######################################################################
3554 * ######## Object allocation
3555 * ######################################################################
3556 * This section of code deals with allocating memory for objects.
3557 * There are several ways:
3558 * *) allocate large objects
3559 * *) allocate normal objects
3560 * *) fast lock-free allocation
3561 * *) allocation of pinned objects
3565 setup_fragment (Fragment
*frag
, Fragment
*prev
, size_t size
)
3567 /* remove from the list */
3569 prev
->next
= frag
->next
;
3571 nursery_fragments
= frag
->next
;
3572 nursery_next
= frag
->fragment_start
;
3573 nursery_frag_real_end
= frag
->fragment_end
;
3575 DEBUG (4, fprintf (gc_debug_file
, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next
, nursery_frag_real_end
, nursery_frag_real_end
- nursery_next
, size
));
3576 frag
->next
= fragment_freelist
;
3577 fragment_freelist
= frag
;
3580 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3581 * an object of size @size
3582 * Return FALSE if not found (which means we need a collection)
3585 search_fragment_for_size (size_t size
)
3587 Fragment
*frag
, *prev
;
3588 DEBUG (4, fprintf (gc_debug_file
, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end
, size
));
3590 if (nursery_frag_real_end
> nursery_next
&& nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
3591 /* Clear the remaining space, pinning depends on this */
3592 memset (nursery_next
, 0, nursery_frag_real_end
- nursery_next
);
3596 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
3597 if (size
<= (frag
->fragment_end
- frag
->fragment_start
)) {
3598 setup_fragment (frag
, prev
, size
);
3607 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3608 * This improves nursery usage.
3611 search_fragment_for_size_range (size_t desired_size
, size_t minimum_size
)
3613 Fragment
*frag
, *prev
, *min_prev
;
3614 DEBUG (4, fprintf (gc_debug_file
, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end
, desired_size
, minimum_size
));
3616 if (nursery_frag_real_end
> nursery_next
&& nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
3617 /* Clear the remaining space, pinning depends on this */
3618 memset (nursery_next
, 0, nursery_frag_real_end
- nursery_next
);
3621 min_prev
= GINT_TO_POINTER (-1);
3624 for (frag
= nursery_fragments
; frag
; frag
= frag
->next
) {
3625 int frag_size
= frag
->fragment_end
- frag
->fragment_start
;
3626 if (desired_size
<= frag_size
) {
3627 setup_fragment (frag
, prev
, desired_size
);
3628 return desired_size
;
3630 if (minimum_size
<= frag_size
)
3636 if (min_prev
!= GINT_TO_POINTER (-1)) {
3639 frag
= min_prev
->next
;
3641 frag
= nursery_fragments
;
3643 frag_size
= frag
->fragment_end
- frag
->fragment_start
;
3644 HEAVY_STAT (++stat_wasted_fragments_used
);
3645 HEAVY_STAT (stat_wasted_fragments_bytes
+= frag_size
);
3647 setup_fragment (frag
, min_prev
, minimum_size
);
3655 alloc_degraded (MonoVTable
*vtable
, size_t size
)
3657 if (need_major_collection (0)) {
3658 mono_profiler_gc_event (MONO_GC_EVENT_START
, 1);
3660 major_collection ("degraded overflow");
3662 mono_profiler_gc_event (MONO_GC_EVENT_END
, 1);
3665 return major_collector
.alloc_degraded (vtable
, size
);
3669 * Provide a variant that takes just the vtable for small fixed-size objects.
3670 * The aligned size is already computed and stored in vt->gc_descr.
3671 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3672 * processing. We can keep track of where objects start, for example,
3673 * so when we scan the thread stacks for pinned objects, we can start
3674 * a search for the pinned object in SCAN_START_SIZE chunks.
3677 mono_gc_alloc_obj_nolock (MonoVTable
*vtable
, size_t size
)
3679 /* FIXME: handle OOM */
3684 HEAVY_STAT (++stat_objects_alloced
);
3685 if (size
<= MAX_SMALL_OBJ_SIZE
)
3686 HEAVY_STAT (stat_bytes_alloced
+= size
);
3688 HEAVY_STAT (stat_bytes_alloced_los
+= size
);
3690 size
= ALIGN_UP (size
);
3692 g_assert (vtable
->gc_descr
);
3694 if (G_UNLIKELY (collect_before_allocs
)) {
3695 static int alloc_count
;
3697 InterlockedIncrement (&alloc_count
);
3698 if (((alloc_count
% collect_before_allocs
) == 0) && nursery_section
) {
3699 mono_profiler_gc_event (MONO_GC_EVENT_START
, 0);
3701 collect_nursery (0);
3703 mono_profiler_gc_event (MONO_GC_EVENT_END
, 0);
3704 if (!degraded_mode
&& !search_fragment_for_size (size
) && size
<= MAX_SMALL_OBJ_SIZE
) {
3706 g_assert_not_reached ();
3712 * We must already have the lock here instead of after the
3713 * fast path because we might be interrupted in the fast path
3714 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
3715 * and we'll end up allocating an object in a fragment which
3716 * no longer belongs to us.
3718 * The managed allocator does not do this, but it's treated
3719 * specially by the world-stopping code.
3722 if (size
> MAX_SMALL_OBJ_SIZE
) {
3723 p
= mono_sgen_los_alloc_large_inner (vtable
, size
);
3725 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3727 p
= (void**)TLAB_NEXT
;
3728 /* FIXME: handle overflow */
3729 new_next
= (char*)p
+ size
;
3730 TLAB_NEXT
= new_next
;
3732 if (G_LIKELY (new_next
< TLAB_TEMP_END
)) {
3736 * FIXME: We might need a memory barrier here so the change to tlab_next is
3737 * visible before the vtable store.
3740 DEBUG (6, fprintf (gc_debug_file
, "Allocated object %p, vtable: %p (%s), size: %zd\n", p
, vtable
, vtable
->klass
->name
, size
));
3741 binary_protocol_alloc (p
, vtable
, size
);
3742 g_assert (*p
== NULL
);
3745 g_assert (TLAB_NEXT
== new_next
);
3752 /* there are two cases: the object is too big or we run out of space in the TLAB */
3753 /* we also reach here when the thread does its first allocation after a minor
3754 * collection, since the tlab_ variables are initialized to NULL.
3755 * there can be another case (from ORP), if we cooperate with the runtime a bit:
3756 * objects that need finalizers can have the high bit set in their size
3757 * so the above check fails and we can readily add the object to the queue.
3758 * This avoids taking again the GC lock when registering, but this is moot when
3759 * doing thread-local allocation, so it may not be a good idea.
3761 g_assert (TLAB_NEXT
== new_next
);
3762 if (TLAB_NEXT
>= TLAB_REAL_END
) {
3764 * Run out of space in the TLAB. When this happens, some amount of space
3765 * remains in the TLAB, but not enough to satisfy the current allocation
3766 * request. Currently, we retire the TLAB in all cases, later we could
3767 * keep it if the remaining space is above a treshold, and satisfy the
3768 * allocation directly from the nursery.
3771 /* when running in degraded mode, we continue allocing that way
3772 * for a while, to decrease the number of useless nursery collections.
3774 if (degraded_mode
&& degraded_mode
< DEFAULT_NURSERY_SIZE
) {
3775 p
= alloc_degraded (vtable
, size
);
3776 binary_protocol_alloc_degraded (p
, vtable
, size
);
3780 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
3781 if (size
> tlab_size
) {
3782 /* Allocate directly from the nursery */
3783 if (nursery_next
+ size
>= nursery_frag_real_end
) {
3784 if (!search_fragment_for_size (size
)) {
3785 minor_collect_or_expand_inner (size
);
3786 if (degraded_mode
) {
3787 p
= alloc_degraded (vtable
, size
);
3788 binary_protocol_alloc_degraded (p
, vtable
, size
);
3794 p
= (void*)nursery_next
;
3795 nursery_next
+= size
;
3796 if (nursery_next
> nursery_frag_real_end
) {
3801 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
3802 memset (p
, 0, size
);
3805 int alloc_size
= tlab_size
;
3806 int available_in_nursery
= nursery_frag_real_end
- nursery_next
;
3808 DEBUG (3, fprintf (gc_debug_file
, "Retire TLAB: %p-%p [%ld]\n", TLAB_START
, TLAB_REAL_END
, (long)(TLAB_REAL_END
- TLAB_NEXT
- size
)));
3810 if (alloc_size
>= available_in_nursery
) {
3811 if (available_in_nursery
> MAX_NURSERY_TLAB_WASTE
&& available_in_nursery
> size
) {
3812 alloc_size
= available_in_nursery
;
3814 alloc_size
= search_fragment_for_size_range (tlab_size
, size
);
3816 alloc_size
= tlab_size
;
3817 minor_collect_or_expand_inner (tlab_size
);
3818 if (degraded_mode
) {
3819 p
= alloc_degraded (vtable
, size
);
3820 binary_protocol_alloc_degraded (p
, vtable
, size
);
3827 /* Allocate a new TLAB from the current nursery fragment */
3828 TLAB_START
= nursery_next
;
3829 nursery_next
+= alloc_size
;
3830 TLAB_NEXT
= TLAB_START
;
3831 TLAB_REAL_END
= TLAB_START
+ alloc_size
;
3832 TLAB_TEMP_END
= TLAB_START
+ MIN (SCAN_START_SIZE
, alloc_size
);
3834 if (nursery_clear_policy
== CLEAR_AT_TLAB_CREATION
) {
3835 memset (TLAB_START
, 0, alloc_size
);
3838 /* Allocate from the TLAB */
3839 p
= (void*)TLAB_NEXT
;
3841 g_assert (TLAB_NEXT
<= TLAB_REAL_END
);
3843 nursery_section
->scan_starts
[((char*)p
- (char*)nursery_section
->data
)/SCAN_START_SIZE
] = (char*)p
;
3846 /* Reached tlab_temp_end */
3848 /* record the scan start so we can find pinned objects more easily */
3849 nursery_section
->scan_starts
[((char*)p
- (char*)nursery_section
->data
)/SCAN_START_SIZE
] = (char*)p
;
3850 /* we just bump tlab_temp_end as well */
3851 TLAB_TEMP_END
= MIN (TLAB_REAL_END
, TLAB_NEXT
+ SCAN_START_SIZE
);
3852 DEBUG (5, fprintf (gc_debug_file
, "Expanding local alloc: %p-%p\n", TLAB_NEXT
, TLAB_TEMP_END
));
3857 DEBUG (6, fprintf (gc_debug_file
, "Allocated object %p, vtable: %p (%s), size: %zd\n", p
, vtable
, vtable
->klass
->name
, size
));
3858 binary_protocol_alloc (p
, vtable
, size
);
3866 mono_gc_try_alloc_obj_nolock (MonoVTable
*vtable
, size_t size
)
3872 size
= ALIGN_UP (size
);
3874 g_assert (vtable
->gc_descr
);
3875 if (size
<= MAX_SMALL_OBJ_SIZE
) {
3876 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3878 p
= (void**)TLAB_NEXT
;
3879 /* FIXME: handle overflow */
3880 new_next
= (char*)p
+ size
;
3881 TLAB_NEXT
= new_next
;
3883 if (G_LIKELY (new_next
< TLAB_TEMP_END
)) {
3887 * FIXME: We might need a memory barrier here so the change to tlab_next is
3888 * visible before the vtable store.
3891 HEAVY_STAT (++stat_objects_alloced
);
3892 HEAVY_STAT (stat_bytes_alloced
+= size
);
3894 DEBUG (6, fprintf (gc_debug_file
, "Allocated object %p, vtable: %p (%s), size: %zd\n", p
, vtable
, vtable
->klass
->name
, size
));
3895 binary_protocol_alloc (p
, vtable
, size
);
3896 g_assert (*p
== NULL
);
3899 g_assert (TLAB_NEXT
== new_next
);
3908 mono_gc_alloc_obj (MonoVTable
*vtable
, size_t size
)
3911 #ifndef DISABLE_CRITICAL_REGION
3913 ENTER_CRITICAL_REGION
;
3914 res
= mono_gc_try_alloc_obj_nolock (vtable
, size
);
3916 EXIT_CRITICAL_REGION
;
3919 EXIT_CRITICAL_REGION
;
3922 res
= mono_gc_alloc_obj_nolock (vtable
, size
);
3924 if (G_UNLIKELY (!res
))
3925 return mono_gc_out_of_memory (size
);
3930 mono_gc_alloc_vector (MonoVTable
*vtable
, size_t size
, uintptr_t max_length
)
3933 #ifndef DISABLE_CRITICAL_REGION
3935 ENTER_CRITICAL_REGION
;
3936 arr
= mono_gc_try_alloc_obj_nolock (vtable
, size
);
3938 arr
->max_length
= max_length
;
3939 EXIT_CRITICAL_REGION
;
3942 EXIT_CRITICAL_REGION
;
3947 arr
= mono_gc_alloc_obj_nolock (vtable
, size
);
3948 if (G_UNLIKELY (!arr
)) {
3950 return mono_gc_out_of_memory (size
);
3953 arr
->max_length
= max_length
;
3961 mono_gc_alloc_array (MonoVTable
*vtable
, size_t size
, uintptr_t max_length
, uintptr_t bounds_size
)
3964 MonoArrayBounds
*bounds
;
3968 arr
= mono_gc_alloc_obj_nolock (vtable
, size
);
3969 if (G_UNLIKELY (!arr
)) {
3971 return mono_gc_out_of_memory (size
);
3974 arr
->max_length
= max_length
;
3976 bounds
= (MonoArrayBounds
*)((char*)arr
+ size
- bounds_size
);
3977 arr
->bounds
= bounds
;
3985 mono_gc_alloc_string (MonoVTable
*vtable
, size_t size
, gint32 len
)
3988 #ifndef DISABLE_CRITICAL_REGION
3990 ENTER_CRITICAL_REGION
;
3991 str
= mono_gc_try_alloc_obj_nolock (vtable
, size
);
3994 EXIT_CRITICAL_REGION
;
3997 EXIT_CRITICAL_REGION
;
4002 str
= mono_gc_alloc_obj_nolock (vtable
, size
);
4003 if (G_UNLIKELY (!str
)) {
4005 return mono_gc_out_of_memory (size
);
4016 * To be used for interned strings and possibly MonoThread, reflection handles.
4017 * We may want to explicitly free these objects.
4020 mono_gc_alloc_pinned_obj (MonoVTable
*vtable
, size_t size
)
4023 size
= ALIGN_UP (size
);
4026 if (size
> MAX_SMALL_OBJ_SIZE
) {
4027 /* large objects are always pinned anyway */
4028 p
= mono_sgen_los_alloc_large_inner (vtable
, size
);
4030 DEBUG (9, g_assert (vtable
->klass
->inited
));
4031 p
= major_collector
.alloc_small_pinned_obj (size
, SGEN_VTABLE_HAS_REFERENCES (vtable
));
4034 DEBUG (6, fprintf (gc_debug_file
, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p
, vtable
, vtable
->klass
->name
, size
));
4035 binary_protocol_alloc_pinned (p
, vtable
, size
);
4043 mono_gc_alloc_mature (MonoVTable
*vtable
)
4046 size_t size
= ALIGN_UP (vtable
->klass
->instance_size
);
4048 res
= alloc_degraded (vtable
, size
);
4051 if (G_UNLIKELY (vtable
->klass
->has_finalize
))
4052 mono_object_register_finalizer ((MonoObject
*)res
);
4058 * ######################################################################
4059 * ######## Finalization support
4060 * ######################################################################
4064 * this is valid for the nursery: if the object has been forwarded it means it's
4065 * still refrenced from a root. If it is pinned it's still alive as well.
4066 * Return TRUE if @obj is ready to be finalized.
4068 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4071 is_critical_finalizer (FinalizeEntry
*entry
)
4076 if (!mono_defaults
.critical_finalizer_object
)
4079 obj
= finalize_entry_get_object (entry
);
4080 class = ((MonoVTable
*)LOAD_VTABLE (obj
))->klass
;
4082 return mono_class_has_parent (class, mono_defaults
.critical_finalizer_object
);
4086 queue_finalization_entry (FinalizeEntry
*entry
) {
4087 if (is_critical_finalizer (entry
)) {
4088 entry
->next
= critical_fin_list
;
4089 critical_fin_list
= entry
;
4091 entry
->next
= fin_ready_list
;
4092 fin_ready_list
= entry
;
4096 /* LOCKING: requires that the GC lock is held */
4098 rehash_fin_table (FinalizeEntryHashTable
*hash_table
)
4100 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4101 mword finalizable_hash_size
= hash_table
->size
;
4104 FinalizeEntry
**new_hash
;
4105 FinalizeEntry
*entry
, *next
;
4106 int new_size
= g_spaced_primes_closest (hash_table
->num_registered
);
4108 new_hash
= mono_sgen_alloc_internal_dynamic (new_size
* sizeof (FinalizeEntry
*), INTERNAL_MEM_FIN_TABLE
);
4109 for (i
= 0; i
< finalizable_hash_size
; ++i
) {
4110 for (entry
= finalizable_hash
[i
]; entry
; entry
= next
) {
4111 hash
= mono_object_hash (finalize_entry_get_object (entry
)) % new_size
;
4113 entry
->next
= new_hash
[hash
];
4114 new_hash
[hash
] = entry
;
4117 mono_sgen_free_internal_dynamic (finalizable_hash
, finalizable_hash_size
* sizeof (FinalizeEntry
*), INTERNAL_MEM_FIN_TABLE
);
4118 hash_table
->table
= new_hash
;
4119 hash_table
->size
= new_size
;
4122 /* LOCKING: requires that the GC lock is held */
4124 rehash_fin_table_if_necessary (FinalizeEntryHashTable
*hash_table
)
4126 if (hash_table
->num_registered
>= hash_table
->size
* 2)
4127 rehash_fin_table (hash_table
);
4132 /* LOCKING: requires that the GC lock is held */
4134 mono_sgen_mark_bridge_object (MonoObject
*obj
)
4136 FinalizeEntryHashTable
*hash_table
= get_finalize_entry_hash_table (ptr_in_nursery (obj
) ? GENERATION_NURSERY
: GENERATION_OLD
);
4137 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4138 FinalizeEntry
*entry
;
4141 hash
= mono_object_hash (obj
);
4142 hash
%= hash_table
->size
;
4144 for (entry
= finalizable_hash
[hash
]; entry
; entry
= entry
->next
) {
4145 if (finalize_entry_get_object (entry
) == obj
)
4146 finalize_entry_set_object (entry
, obj
, TRUE
);
4150 /* LOCKING: requires that the GC lock is held */
4152 collect_bridge_objects (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, int generation
, GrayQueue
*queue
)
4154 FinalizeEntryHashTable
*hash_table
= get_finalize_entry_hash_table (generation
);
4155 FinalizeEntry
*entry
, *prev
;
4157 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4158 mword finalizable_hash_size
= hash_table
->size
;
4163 for (i
= 0; i
< finalizable_hash_size
; ++i
) {
4165 for (entry
= finalizable_hash
[i
]; entry
;) {
4166 MonoObject
*object
= finalize_entry_get_object (entry
);
4167 gboolean ignore_obj
= finalize_entry_get_bridge_bit (entry
);
4170 /* Bridge code told us to ignore this one */
4174 /* Object is a bridge object and major heap says it's dead */
4175 if (!((char*)object
>= start
&& (char*)object
< end
&& !major_collector
.is_object_live ((char*)object
)))
4178 /* Nursery says the object is dead. */
4179 if (!object_is_fin_ready (object
))
4182 if (!mono_sgen_is_bridge_object (object
))
4185 copy
= (char*)object
;
4186 copy_func ((void**)©
, queue
);
4188 bridge_register_finalized_object ((MonoObject
*)copy
);
4190 if (hash_table
== &minor_finalizable_hash
&& !ptr_in_nursery (copy
)) {
4191 FinalizeEntry
*next
= entry
->next
;
4192 unsigned int major_hash
;
4193 /* remove from the list */
4195 prev
->next
= entry
->next
;
4197 finalizable_hash
[i
] = entry
->next
;
4198 hash_table
->num_registered
--;
4200 finalize_entry_set_object (entry
, (MonoObject
*)copy
, ignore_obj
);
4202 /* insert it into the major hash */
4203 rehash_fin_table_if_necessary (&major_finalizable_hash
);
4204 major_hash
= mono_object_hash ((MonoObject
*) copy
) %
4205 major_finalizable_hash
.size
;
4206 entry
->next
= major_finalizable_hash
.table
[major_hash
];
4207 major_finalizable_hash
.table
[major_hash
] = entry
;
4208 major_finalizable_hash
.num_registered
++;
4213 /* update pointer */
4214 finalize_entry_set_object (entry
, (MonoObject
*)copy
, ignore_obj
);
4218 entry
= entry
->next
;
4222 drain_gray_stack (queue
);
4225 /* LOCKING: requires that the GC lock is held */
4227 finalize_in_range (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, int generation
, GrayQueue
*queue
)
4229 FinalizeEntryHashTable
*hash_table
= get_finalize_entry_hash_table (generation
);
4230 FinalizeEntry
*entry
, *prev
;
4232 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4233 mword finalizable_hash_size
= hash_table
->size
;
4237 for (i
= 0; i
< finalizable_hash_size
; ++i
) {
4239 for (entry
= finalizable_hash
[i
]; entry
;) {
4240 MonoObject
*object
= finalize_entry_get_object (entry
);
4241 gboolean bridge_bit
= finalize_entry_get_bridge_bit (entry
);
4243 if ((char*)object
>= start
&& (char*)object
< end
&& !major_collector
.is_object_live ((char*)object
)) {
4244 gboolean is_fin_ready
= object_is_fin_ready (object
);
4245 char *copy
= (char*)object
;
4246 copy_func ((void**)©
, queue
);
4249 FinalizeEntry
*next
;
4250 /* Make it survive */
4251 from
= (char*)object
;
4252 finalize_entry_set_object (entry
, (MonoObject
*)copy
, bridge_bit
);
4253 /* remove and put in fin_ready_list */
4255 prev
->next
= entry
->next
;
4257 finalizable_hash
[i
] = entry
->next
;
4259 num_ready_finalizers
++;
4260 hash_table
->num_registered
--;
4261 queue_finalization_entry (entry
);
4262 DEBUG (5, fprintf (gc_debug_file
, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", object
, safe_name (object
), from
, num_ready_finalizers
, hash_table
->num_registered
));
4266 char *from
= (char*)object
;
4267 if (hash_table
== &minor_finalizable_hash
&& !ptr_in_nursery (copy
)) {
4268 FinalizeEntry
*next
= entry
->next
;
4269 unsigned int major_hash
;
4270 /* remove from the list */
4272 prev
->next
= entry
->next
;
4274 finalizable_hash
[i
] = entry
->next
;
4275 hash_table
->num_registered
--;
4277 finalize_entry_set_object (entry
, (MonoObject
*)copy
, bridge_bit
);
4279 /* insert it into the major hash */
4280 rehash_fin_table_if_necessary (&major_finalizable_hash
);
4281 major_hash
= mono_object_hash ((MonoObject
*) copy
) %
4282 major_finalizable_hash
.size
;
4283 entry
->next
= major_finalizable_hash
.table
[major_hash
];
4284 major_finalizable_hash
.table
[major_hash
] = entry
;
4285 major_finalizable_hash
.num_registered
++;
4287 DEBUG (5, fprintf (gc_debug_file
, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy
, safe_name (copy
), from
));
4292 /* update pointer */
4293 DEBUG (5, fprintf (gc_debug_file
, "Updating object for finalization: %p (%s) (was at %p)\n", object
, safe_name (object
), from
));
4294 finalize_entry_set_object (entry
, (MonoObject
*)copy
, bridge_bit
);
4299 entry
= entry
->next
;
4305 object_is_reachable (char *object
, char *start
, char *end
)
4307 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
4308 if (object
< start
|| object
>= end
)
4310 return !object_is_fin_ready (object
) || major_collector
.is_object_live (object
);
4314 mono_sgen_object_is_live (void *obj
)
4316 if (ptr_in_nursery (obj
))
4317 return object_is_pinned (obj
);
4318 if (current_collection_generation
== GENERATION_NURSERY
)
4320 return major_collector
.is_object_live (obj
);
4323 /* LOCKING: requires that the GC lock is held */
4325 null_ephemerons_for_domain (MonoDomain
*domain
)
4327 EphemeronLinkNode
*current
= ephemeron_list
, *prev
= NULL
;
4330 MonoObject
*object
= (MonoObject
*)current
->array
;
4332 if (object
&& !object
->vtable
) {
4333 EphemeronLinkNode
*tmp
= current
;
4336 prev
->next
= current
->next
;
4338 ephemeron_list
= current
->next
;
4340 current
= current
->next
;
4341 mono_sgen_free_internal (tmp
, INTERNAL_MEM_EPHEMERON_LINK
);
4344 current
= current
->next
;
4349 /* LOCKING: requires that the GC lock is held */
4351 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, GrayQueue
*queue
)
4353 int was_in_nursery
, was_promoted
;
4354 EphemeronLinkNode
*current
= ephemeron_list
, *prev
= NULL
;
4356 Ephemeron
*cur
, *array_end
;
4360 char *object
= current
->array
;
4362 if (!object_is_reachable (object
, start
, end
)) {
4363 EphemeronLinkNode
*tmp
= current
;
4365 DEBUG (5, fprintf (gc_debug_file
, "Dead Ephemeron array at %p\n", object
));
4368 prev
->next
= current
->next
;
4370 ephemeron_list
= current
->next
;
4372 current
= current
->next
;
4373 mono_sgen_free_internal (tmp
, INTERNAL_MEM_EPHEMERON_LINK
);
4378 was_in_nursery
= ptr_in_nursery (object
);
4379 copy_func ((void**)&object
, queue
);
4380 current
->array
= object
;
4382 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
4383 was_promoted
= was_in_nursery
&& !ptr_in_nursery (object
);
4385 DEBUG (5, fprintf (gc_debug_file
, "Clearing unreachable entries for ephemeron array at %p\n", object
));
4387 array
= (MonoArray
*)object
;
4388 cur
= mono_array_addr (array
, Ephemeron
, 0);
4389 array_end
= cur
+ mono_array_length_fast (array
);
4390 tombstone
= (char*)((MonoVTable
*)LOAD_VTABLE (object
))->domain
->ephemeron_tombstone
;
4392 for (; cur
< array_end
; ++cur
) {
4393 char *key
= (char*)cur
->key
;
4395 if (!key
|| key
== tombstone
)
4398 DEBUG (5, fprintf (gc_debug_file
, "[%td] key %p (%s) value %p (%s)\n", cur
- mono_array_addr (array
, Ephemeron
, 0),
4399 key
, object_is_reachable (key
, start
, end
) ? "reachable" : "unreachable",
4400 cur
->value
, cur
->value
&& object_is_reachable (cur
->value
, start
, end
) ? "reachable" : "unreachable"));
4402 if (!object_is_reachable (key
, start
, end
)) {
4403 cur
->key
= tombstone
;
4409 if (ptr_in_nursery (key
)) {/*key was not promoted*/
4410 DEBUG (5, fprintf (gc_debug_file
, "\tAdded remset to key %p\n", key
));
4411 mono_sgen_add_to_global_remset (&cur
->key
);
4413 if (ptr_in_nursery (cur
->value
)) {/*value was not promoted*/
4414 DEBUG (5, fprintf (gc_debug_file
, "\tAdded remset to value %p\n", cur
->value
));
4415 mono_sgen_add_to_global_remset (&cur
->value
);
4420 current
= current
->next
;
4424 /* LOCKING: requires that the GC lock is held */
4426 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, GrayQueue
*queue
)
4428 int nothing_marked
= 1;
4429 EphemeronLinkNode
*current
= ephemeron_list
;
4431 Ephemeron
*cur
, *array_end
;
4434 for (current
= ephemeron_list
; current
; current
= current
->next
) {
4435 char *object
= current
->array
;
4436 DEBUG (5, fprintf (gc_debug_file
, "Ephemeron array at %p\n", object
));
4438 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
4439 if (object
< start
|| object
>= end
)
4442 /*It has to be alive*/
4443 if (!object_is_reachable (object
, start
, end
)) {
4444 DEBUG (5, fprintf (gc_debug_file
, "\tnot reachable\n"));
4448 copy_func ((void**)&object
, queue
);
4450 array
= (MonoArray
*)object
;
4451 cur
= mono_array_addr (array
, Ephemeron
, 0);
4452 array_end
= cur
+ mono_array_length_fast (array
);
4453 tombstone
= (char*)((MonoVTable
*)LOAD_VTABLE (object
))->domain
->ephemeron_tombstone
;
4455 for (; cur
< array_end
; ++cur
) {
4456 char *key
= cur
->key
;
4458 if (!key
|| key
== tombstone
)
4461 DEBUG (5, fprintf (gc_debug_file
, "[%td] key %p (%s) value %p (%s)\n", cur
- mono_array_addr (array
, Ephemeron
, 0),
4462 key
, object_is_reachable (key
, start
, end
) ? "reachable" : "unreachable",
4463 cur
->value
, cur
->value
&& object_is_reachable (cur
->value
, start
, end
) ? "reachable" : "unreachable"));
4465 if (object_is_reachable (key
, start
, end
)) {
4466 char *value
= cur
->value
;
4468 copy_func ((void**)&cur
->key
, queue
);
4470 if (!object_is_reachable (value
, start
, end
))
4472 copy_func ((void**)&cur
->value
, queue
);
4478 DEBUG (5, fprintf (gc_debug_file
, "Ephemeron run finished. Is it done %d\n", nothing_marked
));
4479 return nothing_marked
;
4482 /* LOCKING: requires that the GC lock is held */
4484 null_link_in_range (CopyOrMarkObjectFunc copy_func
, char *start
, char *end
, int generation
, gboolean before_finalization
, GrayQueue
*queue
)
4486 DisappearingLinkHashTable
*hash
= get_dislink_hash_table (generation
);
4487 DisappearingLink
**disappearing_link_hash
= hash
->table
;
4488 int disappearing_link_hash_size
= hash
->size
;
4489 DisappearingLink
*entry
, *prev
;
4491 if (!hash
->num_links
)
4493 for (i
= 0; i
< disappearing_link_hash_size
; ++i
) {
4495 for (entry
= disappearing_link_hash
[i
]; entry
;) {
4497 gboolean track
= DISLINK_TRACK (entry
);
4500 * Tracked references are processed after
4501 * finalization handling whereas standard weak
4502 * references are processed before. If an
4503 * object is still not marked after finalization
4504 * handling it means that it either doesn't have
4505 * a finalizer or the finalizer has already run,
4506 * so we must null a tracking reference.
4508 if (track
== before_finalization
) {
4510 entry
= entry
->next
;
4514 object
= DISLINK_OBJECT (entry
);
4516 if (object
>= start
&& object
< end
&& !major_collector
.is_object_live (object
)) {
4517 if (object_is_fin_ready (object
)) {
4518 void **p
= entry
->link
;
4519 DisappearingLink
*old
;
4521 /* remove from list */
4523 prev
->next
= entry
->next
;
4525 disappearing_link_hash
[i
] = entry
->next
;
4526 DEBUG (5, fprintf (gc_debug_file
, "Dislink nullified at %p to GCed object %p\n", p
, object
));
4528 mono_sgen_free_internal (entry
, INTERNAL_MEM_DISLINK
);
4533 char *copy
= object
;
4534 copy_func ((void**)©
, queue
);
4536 /* Update pointer if it's moved. If the object
4537 * has been moved out of the nursery, we need to
4538 * remove the link from the minor hash table to
4541 * FIXME: what if an object is moved earlier?
4544 if (hash
== &minor_disappearing_link_hash
&& !ptr_in_nursery (copy
)) {
4545 void **link
= entry
->link
;
4546 DisappearingLink
*old
;
4547 /* remove from list */
4549 prev
->next
= entry
->next
;
4551 disappearing_link_hash
[i
] = entry
->next
;
4553 mono_sgen_free_internal (entry
, INTERNAL_MEM_DISLINK
);
4557 add_or_remove_disappearing_link ((MonoObject
*)copy
, link
,
4558 track
, GENERATION_OLD
);
4560 DEBUG (5, fprintf (gc_debug_file
, "Upgraded dislink at %p to major because object %p moved to %p\n", link
, object
, copy
));
4564 *entry
->link
= HIDE_POINTER (copy
, track
);
4565 DEBUG (5, fprintf (gc_debug_file
, "Updated dislink at %p to %p\n", entry
->link
, DISLINK_OBJECT (entry
)));
4570 entry
= entry
->next
;
4575 /* LOCKING: requires that the GC lock is held */
4577 null_links_for_domain (MonoDomain
*domain
, int generation
)
4579 DisappearingLinkHashTable
*hash
= get_dislink_hash_table (generation
);
4580 DisappearingLink
**disappearing_link_hash
= hash
->table
;
4581 int disappearing_link_hash_size
= hash
->size
;
4582 DisappearingLink
*entry
, *prev
;
4584 for (i
= 0; i
< disappearing_link_hash_size
; ++i
) {
4586 for (entry
= disappearing_link_hash
[i
]; entry
; ) {
4587 char *object
= DISLINK_OBJECT (entry
);
4588 if (object
&& !((MonoObject
*)object
)->vtable
) {
4589 DisappearingLink
*next
= entry
->next
;
4594 disappearing_link_hash
[i
] = next
;
4596 if (*(entry
->link
)) {
4597 *(entry
->link
) = NULL
;
4598 g_warning ("Disappearing link %p not freed", entry
->link
);
4600 mono_sgen_free_internal (entry
, INTERNAL_MEM_DISLINK
);
4607 entry
= entry
->next
;
4612 /* LOCKING: requires that the GC lock is held */
4614 finalizers_for_domain (MonoDomain
*domain
, MonoObject
**out_array
, int out_size
,
4615 FinalizeEntryHashTable
*hash_table
)
4617 FinalizeEntry
**finalizable_hash
= hash_table
->table
;
4618 mword finalizable_hash_size
= hash_table
->size
;
4619 FinalizeEntry
*entry
, *prev
;
4622 if (no_finalize
|| !out_size
|| !out_array
)
4625 for (i
= 0; i
< finalizable_hash_size
; ++i
) {
4627 for (entry
= finalizable_hash
[i
]; entry
;) {
4628 MonoObject
*object
= finalize_entry_get_object (entry
);
4629 if (mono_object_domain (object
) == domain
) {
4630 FinalizeEntry
*next
;
4631 /* remove and put in out_array */
4633 prev
->next
= entry
->next
;
4635 finalizable_hash
[i
] = entry
->next
;
4637 hash_table
->num_registered
--;
4638 out_array
[count
++] = object
;
4639 DEBUG (5, fprintf (gc_debug_file
, "Collecting object for finalization: %p (%s) (%d/%d)\n", object
, safe_name (object
), num_ready_finalizers
, hash_table
->num_registered
));
4641 if (count
== out_size
)
4646 entry
= entry
->next
;
4653 * mono_gc_finalizers_for_domain:
4654 * @domain: the unloading appdomain
4655 * @out_array: output array
4656 * @out_size: size of output array
4658 * Store inside @out_array up to @out_size objects that belong to the unloading
4659 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4660 * until it returns 0.
4661 * The items are removed from the finalizer data structure, so the caller is supposed
4663 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4666 mono_gc_finalizers_for_domain (MonoDomain
*domain
, MonoObject
**out_array
, int out_size
)
4671 result
= finalizers_for_domain (domain
, out_array
, out_size
, &minor_finalizable_hash
);
4672 if (result
< out_size
) {
4673 result
+= finalizers_for_domain (domain
, out_array
+ result
, out_size
- result
,
4674 &major_finalizable_hash
);
4682 register_for_finalization (MonoObject
*obj
, void *user_data
, int generation
)
4684 FinalizeEntryHashTable
*hash_table
= get_finalize_entry_hash_table (generation
);
4685 FinalizeEntry
**finalizable_hash
;
4686 mword finalizable_hash_size
;
4687 FinalizeEntry
*entry
, *prev
;
4691 g_assert (user_data
== NULL
|| user_data
== mono_gc_run_finalize
);
4692 hash
= mono_object_hash (obj
);
4694 rehash_fin_table_if_necessary (hash_table
);
4695 finalizable_hash
= hash_table
->table
;
4696 finalizable_hash_size
= hash_table
->size
;
4697 hash
%= finalizable_hash_size
;
4699 for (entry
= finalizable_hash
[hash
]; entry
; entry
= entry
->next
) {
4700 if (finalize_entry_get_object (entry
) == obj
) {
4702 /* remove from the list */
4704 prev
->next
= entry
->next
;
4706 finalizable_hash
[hash
] = entry
->next
;
4707 hash_table
->num_registered
--;
4708 DEBUG (5, fprintf (gc_debug_file
, "Removed finalizer %p for object: %p (%s) (%d)\n", entry
, obj
, obj
->vtable
->klass
->name
, hash_table
->num_registered
));
4709 mono_sgen_free_internal (entry
, INTERNAL_MEM_FINALIZE_ENTRY
);
4717 /* request to deregister, but already out of the list */
4721 entry
= mono_sgen_alloc_internal (INTERNAL_MEM_FINALIZE_ENTRY
);
4722 finalize_entry_set_object (entry
, obj
, FALSE
);
4723 entry
->next
= finalizable_hash
[hash
];
4724 finalizable_hash
[hash
] = entry
;
4725 hash_table
->num_registered
++;
4726 DEBUG (5, fprintf (gc_debug_file
, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry
, obj
, obj
->vtable
->klass
->name
, hash_table
->num_registered
, generation_name (generation
)));
4731 mono_gc_register_for_finalization (MonoObject
*obj
, void *user_data
)
4733 if (ptr_in_nursery (obj
))
4734 register_for_finalization (obj
, user_data
, GENERATION_NURSERY
);
4736 register_for_finalization (obj
, user_data
, GENERATION_OLD
);
4740 rehash_dislink (DisappearingLinkHashTable
*hash_table
)
4742 DisappearingLink
**disappearing_link_hash
= hash_table
->table
;
4743 int disappearing_link_hash_size
= hash_table
->size
;
4746 DisappearingLink
**new_hash
;
4747 DisappearingLink
*entry
, *next
;
4748 int new_size
= g_spaced_primes_closest (hash_table
->num_links
);
4750 new_hash
= mono_sgen_alloc_internal_dynamic (new_size
* sizeof (DisappearingLink
*), INTERNAL_MEM_DISLINK_TABLE
);
4751 for (i
= 0; i
< disappearing_link_hash_size
; ++i
) {
4752 for (entry
= disappearing_link_hash
[i
]; entry
; entry
= next
) {
4753 hash
= mono_aligned_addr_hash (entry
->link
) % new_size
;
4755 entry
->next
= new_hash
[hash
];
4756 new_hash
[hash
] = entry
;
4759 mono_sgen_free_internal_dynamic (disappearing_link_hash
,
4760 disappearing_link_hash_size
* sizeof (DisappearingLink
*), INTERNAL_MEM_DISLINK_TABLE
);
4761 hash_table
->table
= new_hash
;
4762 hash_table
->size
= new_size
;
4765 /* LOCKING: assumes the GC lock is held */
4767 add_or_remove_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
, int generation
)
4769 DisappearingLinkHashTable
*hash_table
= get_dislink_hash_table (generation
);
4770 DisappearingLink
*entry
, *prev
;
4772 DisappearingLink
**disappearing_link_hash
= hash_table
->table
;
4773 int disappearing_link_hash_size
= hash_table
->size
;
4775 if (hash_table
->num_links
>= disappearing_link_hash_size
* 2) {
4776 rehash_dislink (hash_table
);
4777 disappearing_link_hash
= hash_table
->table
;
4778 disappearing_link_hash_size
= hash_table
->size
;
4780 /* FIXME: add check that link is not in the heap */
4781 hash
= mono_aligned_addr_hash (link
) % disappearing_link_hash_size
;
4782 entry
= disappearing_link_hash
[hash
];
4784 for (; entry
; entry
= entry
->next
) {
4785 /* link already added */
4786 if (link
== entry
->link
) {
4787 /* NULL obj means remove */
4790 prev
->next
= entry
->next
;
4792 disappearing_link_hash
[hash
] = entry
->next
;
4793 hash_table
->num_links
--;
4794 DEBUG (5, fprintf (gc_debug_file
, "Removed dislink %p (%d) from %s table\n", entry
, hash_table
->num_links
, generation_name (generation
)));
4795 mono_sgen_free_internal (entry
, INTERNAL_MEM_DISLINK
);
4798 *link
= HIDE_POINTER (obj
, track
); /* we allow the change of object */
4806 entry
= mono_sgen_alloc_internal (INTERNAL_MEM_DISLINK
);
4807 *link
= HIDE_POINTER (obj
, track
);
4809 entry
->next
= disappearing_link_hash
[hash
];
4810 disappearing_link_hash
[hash
] = entry
;
4811 hash_table
->num_links
++;
4812 DEBUG (5, fprintf (gc_debug_file
, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry
, obj
, obj
->vtable
->klass
->name
, link
, generation_name (generation
)));
4815 /* LOCKING: assumes the GC lock is held */
4817 mono_gc_register_disappearing_link (MonoObject
*obj
, void **link
, gboolean track
)
4819 add_or_remove_disappearing_link (NULL
, link
, FALSE
, GENERATION_NURSERY
);
4820 add_or_remove_disappearing_link (NULL
, link
, FALSE
, GENERATION_OLD
);
4822 if (ptr_in_nursery (obj
))
4823 add_or_remove_disappearing_link (obj
, link
, track
, GENERATION_NURSERY
);
4825 add_or_remove_disappearing_link (obj
, link
, track
, GENERATION_OLD
);
4830 mono_gc_invoke_finalizers (void)
4832 FinalizeEntry
*entry
= NULL
;
4833 gboolean entry_is_critical
= FALSE
;
4836 /* FIXME: batch to reduce lock contention */
4837 while (fin_ready_list
|| critical_fin_list
) {
4841 FinalizeEntry
**list
= entry_is_critical
? &critical_fin_list
: &fin_ready_list
;
4843 /* We have finalized entry in the last
4844 interation, now we need to remove it from
4847 *list
= entry
->next
;
4849 FinalizeEntry
*e
= *list
;
4850 while (e
->next
!= entry
)
4852 e
->next
= entry
->next
;
4854 mono_sgen_free_internal (entry
, INTERNAL_MEM_FINALIZE_ENTRY
);
4858 /* Now look for the first non-null entry. */
4859 for (entry
= fin_ready_list
; entry
&& !finalize_entry_get_object (entry
); entry
= entry
->next
)
4862 entry_is_critical
= FALSE
;
4864 entry_is_critical
= TRUE
;
4865 for (entry
= critical_fin_list
; entry
&& !finalize_entry_get_object (entry
); entry
= entry
->next
)
4870 obj
= finalize_entry_get_object (entry
);
4872 num_ready_finalizers
--;
4873 finalize_entry_set_object (entry
, NULL
, FALSE
);
4874 DEBUG (7, fprintf (gc_debug_file
, "Finalizing object %p (%s)\n", obj
, safe_name (obj
)));
4882 g_assert (finalize_entry_get_object (entry
) == NULL
);
4884 /* the object is on the stack so it is pinned */
4885 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
4886 mono_gc_run_finalize (obj
, NULL
);
4893 mono_gc_pending_finalizers (void)
4895 return fin_ready_list
|| critical_fin_list
;
4898 /* Negative value to remove */
4900 mono_gc_add_memory_pressure (gint64 value
)
4902 /* FIXME: Use interlocked functions */
4904 memory_pressure
+= value
;
4909 mono_sgen_register_major_sections_alloced (int num_sections
)
4911 minor_collection_sections_alloced
+= num_sections
;
4915 mono_sgen_get_minor_collection_allowance (void)
4917 return minor_collection_allowance
;
4921 * ######################################################################
4922 * ######## registered roots support
4923 * ######################################################################
4927 rehash_roots (gboolean pinned
)
4931 RootRecord
**new_hash
;
4932 RootRecord
*entry
, *next
;
4935 new_size
= g_spaced_primes_closest (num_roots_entries
[pinned
]);
4936 new_hash
= mono_sgen_alloc_internal_dynamic (new_size
* sizeof (RootRecord
*), INTERNAL_MEM_ROOTS_TABLE
);
4937 for (i
= 0; i
< roots_hash_size
[pinned
]; ++i
) {
4938 for (entry
= roots_hash
[pinned
][i
]; entry
; entry
= next
) {
4939 hash
= mono_aligned_addr_hash (entry
->start_root
) % new_size
;
4941 entry
->next
= new_hash
[hash
];
4942 new_hash
[hash
] = entry
;
4945 mono_sgen_free_internal_dynamic (roots_hash
[pinned
], roots_hash_size
[pinned
] * sizeof (RootRecord
*), INTERNAL_MEM_ROOTS_TABLE
);
4946 roots_hash
[pinned
] = new_hash
;
4947 roots_hash_size
[pinned
] = new_size
;
4951 find_root (int root_type
, char *start
, guint32 addr_hash
)
4953 RootRecord
*new_root
;
4955 guint32 hash
= addr_hash
% roots_hash_size
[root_type
];
4956 for (new_root
= roots_hash
[root_type
][hash
]; new_root
; new_root
= new_root
->next
) {
4957 /* we allow changing the size and the descriptor (for thread statics etc) */
4958 if (new_root
->start_root
== start
) {
4967 * We do not coalesce roots.
4970 mono_gc_register_root_inner (char *start
, size_t size
, void *descr
, int root_type
)
4972 RootRecord
*new_root
;
4973 unsigned int hash
, addr_hash
= mono_aligned_addr_hash (start
);
4976 for (i
= 0; i
< ROOT_TYPE_NUM
; ++i
) {
4977 if (num_roots_entries
[i
] >= roots_hash_size
[i
] * 2)
4980 for (i
= 0; i
< ROOT_TYPE_NUM
; ++i
) {
4981 new_root
= find_root (i
, start
, addr_hash
);
4982 /* we allow changing the size and the descriptor (for thread statics etc) */
4984 size_t old_size
= new_root
->end_root
- new_root
->start_root
;
4985 new_root
->end_root
= new_root
->start_root
+ size
;
4986 g_assert (((new_root
->root_desc
!= 0) && (descr
!= NULL
)) ||
4987 ((new_root
->root_desc
== 0) && (descr
== NULL
)));
4988 new_root
->root_desc
= (mword
)descr
;
4990 roots_size
-= old_size
;
4995 new_root
= mono_sgen_alloc_internal (INTERNAL_MEM_ROOT_RECORD
);
4997 new_root
->start_root
= start
;
4998 new_root
->end_root
= new_root
->start_root
+ size
;
4999 new_root
->root_desc
= (mword
)descr
;
5001 hash
= addr_hash
% roots_hash_size
[root_type
];
5002 num_roots_entries
[root_type
]++;
5003 new_root
->next
= roots_hash
[root_type
] [hash
];
5004 roots_hash
[root_type
][hash
] = new_root
;
5005 DEBUG (3, fprintf (gc_debug_file
, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root
, new_root
->start_root
, new_root
->end_root
, descr
, (int)size
, (int)roots_size
));
5015 mono_gc_register_root (char *start
, size_t size
, void *descr
)
5017 return mono_gc_register_root_inner (start
, size
, descr
, descr
? ROOT_TYPE_NORMAL
: ROOT_TYPE_PINNED
);
5021 mono_gc_register_root_wbarrier (char *start
, size_t size
, void *descr
)
5023 return mono_gc_register_root_inner (start
, size
, descr
, ROOT_TYPE_WBARRIER
);
5027 mono_gc_deregister_root (char* addr
)
5029 RootRecord
*tmp
, *prev
;
5030 unsigned int hash
, addr_hash
= mono_aligned_addr_hash (addr
);
5034 for (root_type
= 0; root_type
< ROOT_TYPE_NUM
; ++root_type
) {
5035 hash
= addr_hash
% roots_hash_size
[root_type
];
5036 tmp
= roots_hash
[root_type
][hash
];
5039 if (tmp
->start_root
== (char*)addr
) {
5041 prev
->next
= tmp
->next
;
5043 roots_hash
[root_type
][hash
] = tmp
->next
;
5044 roots_size
-= (tmp
->end_root
- tmp
->start_root
);
5045 num_roots_entries
[root_type
]--;
5046 DEBUG (3, fprintf (gc_debug_file
, "Removed root %p for range: %p-%p\n", tmp
, tmp
->start_root
, tmp
->end_root
));
5047 mono_sgen_free_internal (tmp
, INTERNAL_MEM_ROOT_RECORD
);
5058 * ######################################################################
5059 * ######## Thread handling (stop/start code)
5060 * ######################################################################
5063 /* FIXME: handle large/small config */
5064 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5066 static SgenThreadInfo
* thread_table
[THREAD_HASH_SIZE
];
5068 #if USE_SIGNAL_BASED_START_STOP_WORLD
5070 static MonoSemType suspend_ack_semaphore
;
5071 static MonoSemType
*suspend_ack_semaphore_ptr
;
5072 static unsigned int global_stop_count
= 0;
5074 static sigset_t suspend_signal_mask
;
5075 static mword cur_thread_regs
[ARCH_NUM_REGS
] = {0};
5077 /* LOCKING: assumes the GC lock is held */
5079 mono_sgen_get_thread_table (void)
5081 return thread_table
;
5085 mono_sgen_thread_info_lookup (ARCH_THREAD_TYPE id
)
5087 unsigned int hash
= HASH_PTHREAD_T (id
) % THREAD_HASH_SIZE
;
5088 SgenThreadInfo
*info
;
5090 info
= thread_table
[hash
];
5091 while (info
&& !ARCH_THREAD_EQUALS (info
->id
, id
)) {
5098 update_current_thread_stack (void *start
)
5100 void *ptr
= cur_thread_regs
;
5101 SgenThreadInfo
*info
= mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5103 info
->stack_start
= align_pointer (&ptr
);
5104 g_assert (info
->stack_start
>= info
->stack_start_limit
&& info
->stack_start
< info
->stack_end
);
5105 ARCH_STORE_REGS (ptr
);
5106 info
->stopped_regs
= ptr
;
5107 if (gc_callbacks
.thread_suspend_func
)
5108 gc_callbacks
.thread_suspend_func (info
->runtime_data
, NULL
);
5112 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5113 * have cross-domain checks in the write barrier.
5115 //#define XDOMAIN_CHECKS_IN_WBARRIER
5117 #ifndef SGEN_BINARY_PROTOCOL
5118 #ifndef HEAVY_STATISTICS
5119 #define MANAGED_ALLOCATION
5120 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5121 #define MANAGED_WBARRIER
5127 is_ip_in_managed_allocator (MonoDomain
*domain
, gpointer ip
);
5130 mono_sgen_wait_for_suspend_ack (int count
)
5134 for (i
= 0; i
< count
; ++i
) {
5135 while ((result
= MONO_SEM_WAIT (suspend_ack_semaphore_ptr
)) != 0) {
5136 if (errno
!= EINTR
) {
5137 g_error ("sem_wait ()");
5144 restart_threads_until_none_in_managed_allocator (void)
5146 SgenThreadInfo
*info
;
5147 int i
, num_threads_died
= 0;
5148 int sleep_duration
= -1;
5151 int restart_count
= 0, restarted_count
= 0;
5152 /* restart all threads that stopped in the
5154 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5155 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5160 if (!info
->stack_start
|| info
->in_critical_region
||
5161 is_ip_in_managed_allocator (info
->stopped_domain
, info
->stopped_ip
)) {
5162 binary_protocol_thread_restart ((gpointer
)info
->id
);
5163 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5164 result
= thread_resume (pthread_mach_thread_np (info
->id
)) == KERN_SUCCESS
;
5166 result
= mono_sgen_pthread_kill (info
, restart_signal_num
) == 0;
5174 /* we set the stopped_ip to
5175 NULL for threads which
5176 we're not restarting so
5177 that we can easily identify
5179 info
->stopped_ip
= NULL
;
5180 info
->stopped_domain
= NULL
;
5184 /* if no threads were restarted, we're done */
5185 if (restart_count
== 0)
5188 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5189 /* mach thread_resume is synchronous so we dont need to wait for them */
5191 /* wait for the threads to signal their restart */
5192 mono_sgen_wait_for_suspend_ack (restart_count
);
5195 if (sleep_duration
< 0) {
5199 g_usleep (sleep_duration
);
5200 sleep_duration
+= 10;
5203 /* stop them again */
5204 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5205 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5207 if (info
->skip
|| info
->stopped_ip
== NULL
)
5209 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5210 result
= mono_sgen_suspend_thread (info
);
5212 result
= mono_sgen_pthread_kill (info
, suspend_signal_num
) == 0;
5221 /* some threads might have died */
5222 num_threads_died
+= restart_count
- restarted_count
;
5223 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5224 /* mach thread_resume is synchronous so we dont need to wait for them */
5226 /* wait for the threads to signal their suspension
5228 mono_sgen_wait_for_suspend_ack (restart_count
);
5232 return num_threads_died
;
5235 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5237 suspend_handler (int sig
, siginfo_t
*siginfo
, void *context
)
5239 SgenThreadInfo
*info
;
5242 int old_errno
= errno
;
5243 gpointer regs
[ARCH_NUM_REGS
];
5244 gpointer stack_start
;
5246 id
= pthread_self ();
5247 info
= mono_sgen_thread_info_lookup (id
);
5248 info
->stopped_domain
= mono_domain_get ();
5249 info
->stopped_ip
= (gpointer
) ARCH_SIGCTX_IP (context
);
5250 stop_count
= global_stop_count
;
5251 /* duplicate signal */
5252 if (0 && info
->stop_count
== stop_count
) {
5256 #ifdef HAVE_KW_THREAD
5257 /* update the remset info in the thread data structure */
5258 info
->remset
= remembered_set
;
5260 stack_start
= (char*) ARCH_SIGCTX_SP (context
) - REDZONE_SIZE
;
5261 /* If stack_start is not within the limits, then don't set it
5262 in info and we will be restarted. */
5263 if (stack_start
>= info
->stack_start_limit
&& info
->stack_start
<= info
->stack_end
) {
5264 info
->stack_start
= stack_start
;
5266 ARCH_COPY_SIGCTX_REGS (regs
, context
);
5267 info
->stopped_regs
= regs
;
5269 g_assert (!info
->stack_start
);
5272 /* Notify the JIT */
5273 if (gc_callbacks
.thread_suspend_func
)
5274 gc_callbacks
.thread_suspend_func (info
->runtime_data
, context
);
5276 DEBUG (4, fprintf (gc_debug_file
, "Posting suspend_ack_semaphore for suspend from %p %p\n", info
, (gpointer
)ARCH_GET_THREAD ()));
5277 /* notify the waiting thread */
5278 MONO_SEM_POST (suspend_ack_semaphore_ptr
);
5279 info
->stop_count
= stop_count
;
5281 /* wait until we receive the restart signal */
5284 sigsuspend (&suspend_signal_mask
);
5285 } while (info
->signal
!= restart_signal_num
);
5287 DEBUG (4, fprintf (gc_debug_file
, "Posting suspend_ack_semaphore for resume from %p %p\n", info
, (gpointer
)ARCH_GET_THREAD ()));
5288 /* notify the waiting thread */
5289 MONO_SEM_POST (suspend_ack_semaphore_ptr
);
5295 restart_handler (int sig
)
5297 SgenThreadInfo
*info
;
5298 int old_errno
= errno
;
5300 info
= mono_sgen_thread_info_lookup (pthread_self ());
5301 info
->signal
= restart_signal_num
;
5302 DEBUG (4, fprintf (gc_debug_file
, "Restart handler in %p %p\n", info
, (gpointer
)ARCH_GET_THREAD ()));
5308 acquire_gc_locks (void)
5314 release_gc_locks (void)
5316 UNLOCK_INTERRUPTION
;
5319 static TV_DECLARE (stop_world_time
);
5320 static unsigned long max_pause_usec
= 0;
5322 /* LOCKING: assumes the GC lock is held */
5324 stop_world (int generation
)
5328 mono_profiler_gc_event (MONO_GC_EVENT_PRE_STOP_WORLD
, generation
);
5329 acquire_gc_locks ();
5331 update_current_thread_stack (&count
);
5333 global_stop_count
++;
5334 DEBUG (3, fprintf (gc_debug_file
, "stopping world n %d from %p %p\n", global_stop_count
, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer
)ARCH_GET_THREAD ()));
5335 TV_GETTIME (stop_world_time
);
5336 count
= mono_sgen_thread_handshake (suspend_signal_num
);
5337 count
-= restart_threads_until_none_in_managed_allocator ();
5338 g_assert (count
>= 0);
5339 DEBUG (3, fprintf (gc_debug_file
, "world stopped %d thread(s)\n", count
));
5340 mono_profiler_gc_event (MONO_GC_EVENT_POST_STOP_WORLD
, generation
);
5342 last_major_num_sections
= major_collector
.get_num_major_sections ();
5343 last_los_memory_usage
= los_memory_usage
;
5344 major_collection_hapenned
= FALSE
;
5348 /* LOCKING: assumes the GC lock is held */
5350 restart_world (int generation
)
5352 int count
, i
, num_major_sections
;
5353 SgenThreadInfo
*info
;
5354 TV_DECLARE (end_sw
);
5355 TV_DECLARE (end_bridge
);
5356 unsigned long usec
, bridge_usec
;
5358 /* notify the profiler of the leftovers */
5359 if (G_UNLIKELY (mono_profiler_events
& MONO_PROFILE_GC_MOVES
)) {
5360 if (moved_objects_idx
) {
5361 mono_profiler_gc_moves (moved_objects
, moved_objects_idx
);
5362 moved_objects_idx
= 0;
5365 mono_profiler_gc_event (MONO_GC_EVENT_PRE_START_WORLD
, generation
);
5366 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5367 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5368 info
->stack_start
= NULL
;
5369 info
->stopped_regs
= NULL
;
5373 release_gc_locks ();
5375 count
= mono_sgen_thread_handshake (restart_signal_num
);
5376 TV_GETTIME (end_sw
);
5377 usec
= TV_ELAPSED (stop_world_time
, end_sw
);
5378 max_pause_usec
= MAX (usec
, max_pause_usec
);
5379 DEBUG (2, fprintf (gc_debug_file
, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count
, (int)usec
, (int)max_pause_usec
));
5380 mono_profiler_gc_event (MONO_GC_EVENT_POST_START_WORLD
, generation
);
5382 TV_GETTIME (end_bridge
);
5383 bridge_usec
= TV_ELAPSED (end_sw
, end_bridge
);
5385 num_major_sections
= major_collector
.get_num_major_sections ();
5386 if (major_collection_hapenned
)
5387 mono_trace (G_LOG_LEVEL_INFO
, MONO_TRACE_GC
, "GC_MAJOR: %s pause %.2fms, bridge %.2fms major %dK/%dK los %dK/%dK",
5388 generation
? "" : "(minor overflow)",
5389 (int)usec
/ 1000.0f
, (int)bridge_usec
/ 1000.0f
,
5390 major_collector
.section_size
* num_major_sections
/ 1024,
5391 major_collector
.section_size
* last_major_num_sections
/ 1024,
5392 los_memory_usage
/ 1024,
5393 last_los_memory_usage
/ 1024);
5395 mono_trace (G_LOG_LEVEL_INFO
, MONO_TRACE_GC
, "GC_MINOR: pause %.2fms, bridge %.2fms promoted %dK major %dK los %dK",
5396 (int)usec
/ 1000.0f
, (int)bridge_usec
/ 1000.0f
,
5397 (num_major_sections
- last_major_num_sections
) * major_collector
.section_size
/ 1024,
5398 major_collector
.section_size
* num_major_sections
/ 1024,
5399 los_memory_usage
/ 1024);
5404 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5407 mono_sgen_get_current_collection_generation (void)
5409 return current_collection_generation
;
5413 mono_gc_set_gc_callbacks (MonoGCCallbacks
*callbacks
)
5415 gc_callbacks
= *callbacks
;
5419 mono_gc_get_gc_callbacks ()
5421 return &gc_callbacks
;
5424 /* Variables holding start/end nursery so it won't have to be passed at every call */
5425 static void *scan_area_arg_start
, *scan_area_arg_end
;
5428 mono_gc_conservatively_scan_area (void *start
, void *end
)
5430 conservatively_pin_objects_from (start
, end
, scan_area_arg_start
, scan_area_arg_end
, PIN_TYPE_STACK
);
5434 mono_gc_scan_object (void *obj
)
5436 if (current_collection_generation
== GENERATION_NURSERY
)
5437 major_collector
.copy_object (&obj
, &gray_queue
);
5439 major_collector
.copy_or_mark_object (&obj
, &gray_queue
);
5444 * Mark from thread stacks and registers.
5447 scan_thread_data (void *start_nursery
, void *end_nursery
, gboolean precise
)
5450 SgenThreadInfo
*info
;
5452 scan_area_arg_start
= start_nursery
;
5453 scan_area_arg_end
= end_nursery
;
5455 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5456 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5458 DEBUG (3, fprintf (gc_debug_file
, "Skipping dead thread %p, range: %p-%p, size: %td\n", info
, info
->stack_start
, info
->stack_end
, (char*)info
->stack_end
- (char*)info
->stack_start
));
5461 DEBUG (3, fprintf (gc_debug_file
, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info
, info
->stack_start
, info
->stack_end
, (char*)info
->stack_end
- (char*)info
->stack_start
, next_pin_slot
));
5462 if (gc_callbacks
.thread_mark_func
&& !conservative_stack_mark
)
5463 gc_callbacks
.thread_mark_func (info
->runtime_data
, info
->stack_start
, info
->stack_end
, precise
);
5465 conservatively_pin_objects_from (info
->stack_start
, info
->stack_end
, start_nursery
, end_nursery
, PIN_TYPE_STACK
);
5468 conservatively_pin_objects_from (info
->stopped_regs
, info
->stopped_regs
+ ARCH_NUM_REGS
,
5469 start_nursery
, end_nursery
, PIN_TYPE_STACK
);
5475 find_pinning_ref_from_thread (char *obj
, size_t size
)
5478 SgenThreadInfo
*info
;
5479 char *endobj
= obj
+ size
;
5481 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5482 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5483 char **start
= (char**)info
->stack_start
;
5486 while (start
< (char**)info
->stack_end
) {
5487 if (*start
>= obj
&& *start
< endobj
) {
5488 DEBUG (0, fprintf (gc_debug_file
, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj
, info
, (gpointer
)info
->id
, start
, info
->stack_start
, info
->stack_end
));
5493 for (j
= 0; j
< ARCH_NUM_REGS
; ++j
) {
5494 mword w
= (mword
)info
->stopped_regs
[j
];
5496 if (w
>= (mword
)obj
&& w
< (mword
)obj
+ size
)
5497 DEBUG (0, fprintf (gc_debug_file
, "Object %p referenced in saved reg %d of thread %p (id %p)\n", obj
, j
, info
, (gpointer
)info
->id
));
5504 ptr_on_stack (void *ptr
)
5506 gpointer stack_start
= &stack_start
;
5507 SgenThreadInfo
*info
= mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5509 if (ptr
>= stack_start
&& ptr
< (gpointer
)info
->stack_end
)
5515 handle_remset (mword
*p
, void *start_nursery
, void *end_nursery
, gboolean global
, GrayQueue
*queue
)
5522 HEAVY_STAT (++stat_global_remsets_processed
);
5524 HEAVY_STAT (++stat_local_remsets_processed
);
5526 /* FIXME: exclude stack locations */
5527 switch ((*p
) & REMSET_TYPE_MASK
) {
5528 case REMSET_LOCATION
:
5530 //__builtin_prefetch (ptr);
5531 if (((void*)ptr
< start_nursery
|| (void*)ptr
>= end_nursery
)) {
5532 gpointer old
= *ptr
;
5533 major_collector
.copy_object (ptr
, queue
);
5534 DEBUG (9, fprintf (gc_debug_file
, "Overwrote remset at %p with %p\n", ptr
, *ptr
));
5536 binary_protocol_ptr_update (ptr
, old
, *ptr
, (gpointer
)LOAD_VTABLE (*ptr
), safe_object_get_size (*ptr
));
5537 if (!global
&& *ptr
>= start_nursery
&& *ptr
< end_nursery
) {
5539 * If the object is pinned, each reference to it from nonpinned objects
5540 * becomes part of the global remset, which can grow very large.
5542 DEBUG (9, fprintf (gc_debug_file
, "Add to global remset because of pinning %p (%p %s)\n", ptr
, *ptr
, safe_name (*ptr
)));
5543 mono_sgen_add_to_global_remset (ptr
);
5546 DEBUG (9, fprintf (gc_debug_file
, "Skipping remset at %p holding %p\n", ptr
, *ptr
));
5550 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
5551 if (((void*)ptr
>= start_nursery
&& (void*)ptr
< end_nursery
))
5554 while (count
-- > 0) {
5555 major_collector
.copy_object (ptr
, queue
);
5556 DEBUG (9, fprintf (gc_debug_file
, "Overwrote remset at %p with %p (count: %d)\n", ptr
, *ptr
, (int)count
));
5557 if (!global
&& *ptr
>= start_nursery
&& *ptr
< end_nursery
)
5558 mono_sgen_add_to_global_remset (ptr
);
5563 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
5564 if (((void*)ptr
>= start_nursery
&& (void*)ptr
< end_nursery
))
5566 major_collector
.minor_scan_object ((char*)ptr
, queue
);
5568 case REMSET_VTYPE
: {
5569 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
5570 if (((void*)ptr
>= start_nursery
&& (void*)ptr
< end_nursery
))
5575 ptr
= (void**) major_collector
.minor_scan_vtype ((char*)ptr
, desc
, start_nursery
, end_nursery
, queue
);
5579 g_assert_not_reached ();
5584 #ifdef HEAVY_STATISTICS
5586 collect_store_remsets (RememberedSet
*remset
, mword
*bumper
)
5588 mword
*p
= remset
->data
;
5593 while (p
< remset
->store_next
) {
5594 switch ((*p
) & REMSET_TYPE_MASK
) {
5595 case REMSET_LOCATION
:
5598 ++stat_saved_remsets_1
;
5600 if (*p
== last1
|| *p
== last2
) {
5601 ++stat_saved_remsets_2
;
5618 g_assert_not_reached ();
5628 RememberedSet
*remset
;
5630 SgenThreadInfo
*info
;
5632 mword
*addresses
, *bumper
, *p
, *r
;
5634 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5635 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5636 for (remset
= info
->remset
; remset
; remset
= remset
->next
)
5637 size
+= remset
->store_next
- remset
->data
;
5640 for (remset
= freed_thread_remsets
; remset
; remset
= remset
->next
)
5641 size
+= remset
->store_next
- remset
->data
;
5642 for (remset
= global_remset
; remset
; remset
= remset
->next
)
5643 size
+= remset
->store_next
- remset
->data
;
5645 bumper
= addresses
= mono_sgen_alloc_internal_dynamic (sizeof (mword
) * size
, INTERNAL_MEM_STATISTICS
);
5647 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5648 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5649 for (remset
= info
->remset
; remset
; remset
= remset
->next
)
5650 bumper
= collect_store_remsets (remset
, bumper
);
5653 for (remset
= global_remset
; remset
; remset
= remset
->next
)
5654 bumper
= collect_store_remsets (remset
, bumper
);
5655 for (remset
= freed_thread_remsets
; remset
; remset
= remset
->next
)
5656 bumper
= collect_store_remsets (remset
, bumper
);
5658 g_assert (bumper
<= addresses
+ size
);
5660 stat_store_remsets
+= bumper
- addresses
;
5662 sort_addresses ((void**)addresses
, bumper
- addresses
);
5665 while (r
< bumper
) {
5671 stat_store_remsets_unique
+= p
- addresses
;
5673 mono_sgen_free_internal_dynamic (addresses
, sizeof (mword
) * size
, INTERNAL_MEM_STATISTICS
);
5678 clear_thread_store_remset_buffer (SgenThreadInfo
*info
)
5680 *info
->store_remset_buffer_index_addr
= 0;
5681 memset (*info
->store_remset_buffer_addr
, 0, sizeof (gpointer
) * STORE_REMSET_BUFFER_SIZE
);
5685 remset_byte_size (RememberedSet
*remset
)
5687 return sizeof (RememberedSet
) + (remset
->end_set
- remset
->data
) * sizeof (gpointer
);
5691 scan_from_remsets (void *start_nursery
, void *end_nursery
, GrayQueue
*queue
)
5694 SgenThreadInfo
*info
;
5695 RememberedSet
*remset
;
5696 GenericStoreRememberedSet
*store_remset
;
5697 mword
*p
, *next_p
, *store_pos
;
5699 #ifdef HEAVY_STATISTICS
5703 /* the global one */
5704 for (remset
= global_remset
; remset
; remset
= remset
->next
) {
5705 DEBUG (4, fprintf (gc_debug_file
, "Scanning global remset range: %p-%p, size: %td\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
5706 store_pos
= remset
->data
;
5707 for (p
= remset
->data
; p
< remset
->store_next
; p
= next_p
) {
5708 void **ptr
= (void**)p
[0];
5710 /*Ignore previously processed remset.*/
5711 if (!global_remset_location_was_not_added (ptr
)) {
5716 next_p
= handle_remset (p
, start_nursery
, end_nursery
, TRUE
, queue
);
5719 * Clear global remsets of locations which no longer point to the
5720 * nursery. Otherwise, they could grow indefinitely between major
5723 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5725 if (ptr_in_nursery (*ptr
)) {
5726 *store_pos
++ = p
[0];
5727 HEAVY_STAT (++stat_global_remsets_readded
);
5731 /* Truncate the remset */
5732 remset
->store_next
= store_pos
;
5735 /* the generic store ones */
5736 store_remset
= generic_store_remsets
;
5737 while (store_remset
) {
5738 GenericStoreRememberedSet
*next
= store_remset
->next
;
5740 for (i
= 0; i
< STORE_REMSET_BUFFER_SIZE
- 1; ++i
) {
5741 gpointer addr
= store_remset
->data
[i
];
5743 handle_remset ((mword
*)&addr
, start_nursery
, end_nursery
, FALSE
, queue
);
5746 mono_sgen_free_internal (store_remset
, INTERNAL_MEM_STORE_REMSET
);
5748 store_remset
= next
;
5750 generic_store_remsets
= NULL
;
5752 /* the per-thread ones */
5753 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5754 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5755 RememberedSet
*next
;
5757 for (remset
= info
->remset
; remset
; remset
= next
) {
5758 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info
, remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
5759 for (p
= remset
->data
; p
< remset
->store_next
;)
5760 p
= handle_remset (p
, start_nursery
, end_nursery
, FALSE
, queue
);
5761 remset
->store_next
= remset
->data
;
5762 next
= remset
->next
;
5763 remset
->next
= NULL
;
5764 if (remset
!= info
->remset
) {
5765 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5766 mono_sgen_free_internal_dynamic (remset
, remset_byte_size (remset
), INTERNAL_MEM_REMSET
);
5769 for (j
= 0; j
< *info
->store_remset_buffer_index_addr
; ++j
)
5770 handle_remset ((mword
*)*info
->store_remset_buffer_addr
+ j
+ 1, start_nursery
, end_nursery
, FALSE
, queue
);
5771 clear_thread_store_remset_buffer (info
);
5775 /* the freed thread ones */
5776 while (freed_thread_remsets
) {
5777 RememberedSet
*next
;
5778 remset
= freed_thread_remsets
;
5779 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
5780 for (p
= remset
->data
; p
< remset
->store_next
;)
5781 p
= handle_remset (p
, start_nursery
, end_nursery
, FALSE
, queue
);
5782 next
= remset
->next
;
5783 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5784 mono_sgen_free_internal_dynamic (remset
, remset_byte_size (remset
), INTERNAL_MEM_REMSET
);
5785 freed_thread_remsets
= next
;
5790 * Clear the info in the remembered sets: we're doing a major collection, so
5791 * the per-thread ones are not needed and the global ones will be reconstructed
5795 clear_remsets (void)
5798 SgenThreadInfo
*info
;
5799 RememberedSet
*remset
, *next
;
5801 /* the global list */
5802 for (remset
= global_remset
; remset
; remset
= next
) {
5803 remset
->store_next
= remset
->data
;
5804 next
= remset
->next
;
5805 remset
->next
= NULL
;
5806 if (remset
!= global_remset
) {
5807 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5808 mono_sgen_free_internal_dynamic (remset
, remset_byte_size (remset
), INTERNAL_MEM_REMSET
);
5811 /* the generic store ones */
5812 while (generic_store_remsets
) {
5813 GenericStoreRememberedSet
*gs_next
= generic_store_remsets
->next
;
5814 mono_sgen_free_internal (generic_store_remsets
, INTERNAL_MEM_STORE_REMSET
);
5815 generic_store_remsets
= gs_next
;
5817 /* the per-thread ones */
5818 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5819 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5820 for (remset
= info
->remset
; remset
; remset
= next
) {
5821 remset
->store_next
= remset
->data
;
5822 next
= remset
->next
;
5823 remset
->next
= NULL
;
5824 if (remset
!= info
->remset
) {
5825 DEBUG (3, fprintf (gc_debug_file
, "Freed remset at %p\n", remset
->data
));
5826 mono_sgen_free_internal_dynamic (remset
, remset_byte_size (remset
), INTERNAL_MEM_REMSET
);
5829 clear_thread_store_remset_buffer (info
);
5833 /* the freed thread ones */
5834 while (freed_thread_remsets
) {
5835 next
= freed_thread_remsets
->next
;
5836 DEBUG (4, fprintf (gc_debug_file
, "Freed remset at %p\n", freed_thread_remsets
->data
));
5837 mono_sgen_free_internal_dynamic (freed_thread_remsets
, remset_byte_size (freed_thread_remsets
), INTERNAL_MEM_REMSET
);
5838 freed_thread_remsets
= next
;
5843 * Clear the thread local TLAB variables for all threads.
5848 SgenThreadInfo
*info
;
5851 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
5852 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
5853 /* A new TLAB will be allocated when the thread does its first allocation */
5854 *info
->tlab_start_addr
= NULL
;
5855 *info
->tlab_next_addr
= NULL
;
5856 *info
->tlab_temp_end_addr
= NULL
;
5857 *info
->tlab_real_end_addr
= NULL
;
5862 /* LOCKING: assumes the GC lock is held */
5863 static SgenThreadInfo
*
5864 gc_register_current_thread (void *addr
)
5867 SgenThreadInfo
* info
= malloc (sizeof (SgenThreadInfo
));
5868 #ifndef HAVE_KW_THREAD
5869 SgenThreadInfo
*__thread_info__
= info
;
5875 memset (info
, 0, sizeof (SgenThreadInfo
));
5876 #ifndef HAVE_KW_THREAD
5877 info
->tlab_start
= info
->tlab_next
= info
->tlab_temp_end
= info
->tlab_real_end
= NULL
;
5879 g_assert (!pthread_getspecific (thread_info_key
));
5880 pthread_setspecific (thread_info_key
, info
);
5885 info
->id
= ARCH_GET_THREAD ();
5886 info
->stop_count
= -1;
5889 info
->stack_start
= NULL
;
5890 info
->tlab_start_addr
= &TLAB_START
;
5891 info
->tlab_next_addr
= &TLAB_NEXT
;
5892 info
->tlab_temp_end_addr
= &TLAB_TEMP_END
;
5893 info
->tlab_real_end_addr
= &TLAB_REAL_END
;
5894 info
->store_remset_buffer_addr
= &STORE_REMSET_BUFFER
;
5895 info
->store_remset_buffer_index_addr
= &STORE_REMSET_BUFFER_INDEX
;
5896 info
->stopped_ip
= NULL
;
5897 info
->stopped_domain
= NULL
;
5898 info
->stopped_regs
= NULL
;
5900 binary_protocol_thread_register ((gpointer
)info
->id
);
5902 #ifdef HAVE_KW_THREAD
5903 tlab_next_addr
= &tlab_next
;
5904 store_remset_buffer_index_addr
= &store_remset_buffer_index
;
5907 #if defined(__MACH__)
5908 info
->mach_port
= mach_thread_self ();
5911 #if defined(PLATFORM_ANDROID)
5912 info
->android_tid
= (gpointer
) gettid ();
5915 /* try to get it with attributes first */
5916 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
5920 pthread_attr_t attr
;
5921 pthread_getattr_np (pthread_self (), &attr
);
5922 pthread_attr_getstack (&attr
, &sstart
, &size
);
5923 info
->stack_start_limit
= sstart
;
5924 info
->stack_end
= (char*)sstart
+ size
;
5925 pthread_attr_destroy (&attr
);
5927 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
5928 info
->stack_end
= (char*)pthread_get_stackaddr_np (pthread_self ());
5929 info
->stack_start_limit
= (char*)info
->stack_end
- pthread_get_stacksize_np (pthread_self ());
5932 /* FIXME: we assume the stack grows down */
5933 gsize stack_bottom
= (gsize
)addr
;
5934 stack_bottom
+= 4095;
5935 stack_bottom
&= ~4095;
5936 info
->stack_end
= (char*)stack_bottom
;
5940 #ifdef HAVE_KW_THREAD
5941 stack_end
= info
->stack_end
;
5944 /* hash into the table */
5945 hash
= HASH_PTHREAD_T (info
->id
) % THREAD_HASH_SIZE
;
5946 info
->next
= thread_table
[hash
];
5947 thread_table
[hash
] = info
;
5949 info
->remset
= alloc_remset (DEFAULT_REMSET_SIZE
, info
);
5950 pthread_setspecific (remembered_set_key
, info
->remset
);
5951 #ifdef HAVE_KW_THREAD
5952 remembered_set
= info
->remset
;
5955 STORE_REMSET_BUFFER
= mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET
);
5956 STORE_REMSET_BUFFER_INDEX
= 0;
5958 DEBUG (3, fprintf (gc_debug_file
, "registered thread %p (%p) (hash: %d)\n", info
, (gpointer
)info
->id
, hash
));
5960 if (gc_callbacks
.thread_attach_func
)
5961 info
->runtime_data
= gc_callbacks
.thread_attach_func ();
5967 add_generic_store_remset_from_buffer (gpointer
*buffer
)
5969 GenericStoreRememberedSet
*remset
= mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET
);
5970 memcpy (remset
->data
, buffer
+ 1, sizeof (gpointer
) * (STORE_REMSET_BUFFER_SIZE
- 1));
5971 remset
->next
= generic_store_remsets
;
5972 generic_store_remsets
= remset
;
5976 unregister_current_thread (void)
5979 SgenThreadInfo
*prev
= NULL
;
5981 RememberedSet
*rset
;
5982 ARCH_THREAD_TYPE id
= ARCH_GET_THREAD ();
5984 binary_protocol_thread_unregister ((gpointer
)id
);
5986 hash
= HASH_PTHREAD_T (id
) % THREAD_HASH_SIZE
;
5987 p
= thread_table
[hash
];
5989 DEBUG (3, fprintf (gc_debug_file
, "unregister thread %p (%p)\n", p
, (gpointer
)p
->id
));
5990 while (!ARCH_THREAD_EQUALS (p
->id
, id
)) {
5995 thread_table
[hash
] = p
->next
;
5997 prev
->next
= p
->next
;
6000 #if defined(__MACH__)
6001 mach_port_deallocate (current_task (), p
->mach_port
);
6004 if (gc_callbacks
.thread_detach_func
) {
6005 gc_callbacks
.thread_detach_func (p
->runtime_data
);
6006 p
->runtime_data
= NULL
;
6010 if (freed_thread_remsets
) {
6011 for (rset
= p
->remset
; rset
->next
; rset
= rset
->next
)
6013 rset
->next
= freed_thread_remsets
;
6014 freed_thread_remsets
= p
->remset
;
6016 freed_thread_remsets
= p
->remset
;
6019 if (*p
->store_remset_buffer_index_addr
)
6020 add_generic_store_remset_from_buffer (*p
->store_remset_buffer_addr
);
6021 mono_sgen_free_internal (*p
->store_remset_buffer_addr
, INTERNAL_MEM_STORE_REMSET
);
6026 unregister_thread (void *k
)
6028 /* If a delegate is passed to native code and invoked on a thread we dont
6029 * know about, the jit will register it with mono_jit_thead_attach, but
6030 * we have no way of knowing when that thread goes away. SGen has a TSD
6031 * so we assume that if the domain is still registered, we can detach
6034 if (mono_domain_get ())
6035 mono_thread_detach (mono_thread_current ());
6038 unregister_current_thread ();
6043 mono_gc_register_thread (void *baseptr
)
6045 SgenThreadInfo
*info
;
6049 info
= mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
6051 info
= gc_register_current_thread (baseptr
);
6053 /* The main thread might get registered before callbacks are set */
6054 if (gc_callbacks
.thread_attach_func
&& !info
->runtime_data
)
6055 info
->runtime_data
= gc_callbacks
.thread_attach_func ();
6059 /* Need a better place to initialize this */
6060 if (!array_fill_vtable
&& mono_get_root_domain ()) {
6061 array_fill_vtable
= mono_class_vtable (mono_get_root_domain (), mono_array_class_get (mono_defaults
.byte_class
, 1));
6064 return info
!= NULL
;
6068 * mono_gc_set_stack_end:
6070 * Set the end of the current threads stack to STACK_END. The stack space between
6071 * STACK_END and the real end of the threads stack will not be scanned during collections.
6074 mono_gc_set_stack_end (void *stack_end
)
6076 SgenThreadInfo
*info
;
6079 info
= mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
6081 g_assert (stack_end
< info
->stack_end
);
6082 info
->stack_end
= stack_end
;
6087 #if USE_PTHREAD_INTERCEPT
6090 void *(*start_routine
) (void *);
6093 MonoSemType registered
;
6094 } SgenThreadStartInfo
;
6097 gc_start_thread (void *arg
)
6099 SgenThreadStartInfo
*start_info
= arg
;
6100 SgenThreadInfo
* info
;
6101 void *t_arg
= start_info
->arg
;
6102 void *(*start_func
) (void*) = start_info
->start_routine
;
6107 info
= gc_register_current_thread (&result
);
6109 post_result
= MONO_SEM_POST (&(start_info
->registered
));
6110 g_assert (!post_result
);
6111 result
= start_func (t_arg
);
6112 g_assert (!mono_domain_get ());
6114 * this is done by the pthread key dtor
6116 unregister_current_thread ();
6124 mono_gc_pthread_create (pthread_t
*new_thread
, const pthread_attr_t
*attr
, void *(*start_routine
)(void *), void *arg
)
6126 SgenThreadStartInfo
*start_info
;
6129 start_info
= malloc (sizeof (SgenThreadStartInfo
));
6132 MONO_SEM_INIT (&(start_info
->registered
), 0);
6133 start_info
->arg
= arg
;
6134 start_info
->start_routine
= start_routine
;
6136 result
= pthread_create (new_thread
, attr
, gc_start_thread
, start_info
);
6138 while (MONO_SEM_WAIT (&(start_info
->registered
)) != 0) {
6139 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6142 MONO_SEM_DESTROY (&(start_info
->registered
));
6148 mono_gc_pthread_join (pthread_t thread
, void **retval
)
6150 return pthread_join (thread
, retval
);
6154 mono_gc_pthread_detach (pthread_t thread
)
6156 return pthread_detach (thread
);
6159 #endif /* USE_PTHREAD_INTERCEPT */
6162 * ######################################################################
6163 * ######## Write barriers
6164 * ######################################################################
6168 * This causes the compile to extend the liveness of 'v' till the call to dummy_use
6171 dummy_use (gpointer v
) {
6172 __asm__
volatile ("" : "=r"(v
) : "r"(v
));
6176 static RememberedSet
*
6177 alloc_remset (int size
, gpointer id
) {
6178 RememberedSet
* res
= mono_sgen_alloc_internal_dynamic (sizeof (RememberedSet
) + (size
* sizeof (gpointer
)), INTERNAL_MEM_REMSET
);
6179 res
->store_next
= res
->data
;
6180 res
->end_set
= res
->data
+ size
;
6182 DEBUG (4, fprintf (gc_debug_file
, "Allocated remset size %d at %p for %p\n", size
, res
->data
, id
));
6187 * Note: the write barriers first do the needed GC work and then do the actual store:
6188 * this way the value is visible to the conservative GC scan after the write barrier
6189 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6190 * the conservative scan, otherwise by the remembered set scan.
6193 mono_gc_wbarrier_set_field (MonoObject
*obj
, gpointer field_ptr
, MonoObject
* value
)
6195 HEAVY_STAT (++stat_wbarrier_set_field
);
6196 if (ptr_in_nursery (field_ptr
)) {
6197 *(void**)field_ptr
= value
;
6200 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p\n", field_ptr
));
6201 if (use_cardtable
) {
6202 *(void**)field_ptr
= value
;
6203 if (ptr_in_nursery (value
))
6204 sgen_card_table_mark_address ((mword
)field_ptr
);
6211 rs
= REMEMBERED_SET
;
6212 if (rs
->store_next
< rs
->end_set
) {
6213 *(rs
->store_next
++) = (mword
)field_ptr
;
6214 *(void**)field_ptr
= value
;
6218 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6219 rs
->next
= REMEMBERED_SET
;
6220 REMEMBERED_SET
= rs
;
6221 #ifdef HAVE_KW_THREAD
6222 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6224 *(rs
->store_next
++) = (mword
)field_ptr
;
6225 *(void**)field_ptr
= value
;
6231 mono_gc_wbarrier_set_arrayref (MonoArray
*arr
, gpointer slot_ptr
, MonoObject
* value
)
6233 HEAVY_STAT (++stat_wbarrier_set_arrayref
);
6234 if (ptr_in_nursery (slot_ptr
)) {
6235 *(void**)slot_ptr
= value
;
6238 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p\n", slot_ptr
));
6239 if (use_cardtable
) {
6240 *(void**)slot_ptr
= value
;
6241 if (ptr_in_nursery (value
))
6242 sgen_card_table_mark_address ((mword
)slot_ptr
);
6249 rs
= REMEMBERED_SET
;
6250 if (rs
->store_next
< rs
->end_set
) {
6251 *(rs
->store_next
++) = (mword
)slot_ptr
;
6252 *(void**)slot_ptr
= value
;
6256 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6257 rs
->next
= REMEMBERED_SET
;
6258 REMEMBERED_SET
= rs
;
6259 #ifdef HAVE_KW_THREAD
6260 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6262 *(rs
->store_next
++) = (mword
)slot_ptr
;
6263 *(void**)slot_ptr
= value
;
6269 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr
, gpointer src_ptr
, int count
)
6271 HEAVY_STAT (++stat_wbarrier_arrayref_copy
);
6272 /*This check can be done without taking a lock since dest_ptr array is pinned*/
6273 if (ptr_in_nursery (dest_ptr
) || count
<= 0) {
6274 memmove (dest_ptr
, src_ptr
, count
* sizeof (gpointer
));
6278 if (use_cardtable
) {
6279 gpointer
*dest
= dest_ptr
;
6280 gpointer
*src
= src_ptr
;
6282 /*overlapping that required backward copying*/
6283 if (src
< dest
&& (src
+ count
) > dest
) {
6284 gpointer
*start
= dest
;
6288 for (; dest
>= start
; --src
, --dest
) {
6289 gpointer value
= *src
;
6291 if (ptr_in_nursery (value
))
6292 sgen_card_table_mark_address ((mword
)dest
);
6296 gpointer
*end
= dest
+ count
;
6297 for (; dest
< end
; ++src
, ++dest
) {
6298 gpointer value
= *src
;
6300 if (ptr_in_nursery (value
))
6301 sgen_card_table_mark_address ((mword
)dest
);
6309 memmove (dest_ptr
, src_ptr
, count
* sizeof (gpointer
));
6311 rs
= REMEMBERED_SET
;
6312 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p, %d\n", dest_ptr
, count
));
6313 if (rs
->store_next
+ 1 < rs
->end_set
) {
6314 *(rs
->store_next
++) = (mword
)dest_ptr
| REMSET_RANGE
;
6315 *(rs
->store_next
++) = count
;
6319 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6320 rs
->next
= REMEMBERED_SET
;
6321 REMEMBERED_SET
= rs
;
6322 #ifdef HAVE_KW_THREAD
6323 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6325 *(rs
->store_next
++) = (mword
)dest_ptr
| REMSET_RANGE
;
6326 *(rs
->store_next
++) = count
;
6332 static char *found_obj
;
6335 find_object_for_ptr_callback (char *obj
, size_t size
, void *user_data
)
6337 char *ptr
= user_data
;
6339 if (ptr
>= obj
&& ptr
< obj
+ size
) {
6340 g_assert (!found_obj
);
6345 /* for use in the debugger */
6346 char* find_object_for_ptr (char *ptr
);
6348 find_object_for_ptr (char *ptr
)
6350 if (ptr
>= nursery_section
->data
&& ptr
< nursery_section
->end_data
) {
6352 mono_sgen_scan_area_with_callback (nursery_section
->data
, nursery_section
->end_data
,
6353 find_object_for_ptr_callback
, ptr
, TRUE
);
6359 mono_sgen_los_iterate_objects (find_object_for_ptr_callback
, ptr
);
6364 * Very inefficient, but this is debugging code, supposed to
6365 * be called from gdb, so we don't care.
6368 major_collector
.iterate_objects (TRUE
, TRUE
, find_object_for_ptr_callback
, ptr
);
6373 evacuate_remset_buffer (void)
6378 buffer
= STORE_REMSET_BUFFER
;
6380 add_generic_store_remset_from_buffer (buffer
);
6381 memset (buffer
, 0, sizeof (gpointer
) * STORE_REMSET_BUFFER_SIZE
);
6383 STORE_REMSET_BUFFER_INDEX
= 0;
6387 mono_gc_wbarrier_generic_nostore (gpointer ptr
)
6393 HEAVY_STAT (++stat_wbarrier_generic_store
);
6395 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6396 /* FIXME: ptr_in_heap must be called with the GC lock held */
6397 if (xdomain_checks
&& *(MonoObject
**)ptr
&& ptr_in_heap (ptr
)) {
6398 char *start
= find_object_for_ptr (ptr
);
6399 MonoObject
*value
= *(MonoObject
**)ptr
;
6403 MonoObject
*obj
= (MonoObject
*)start
;
6404 if (obj
->vtable
->domain
!= value
->vtable
->domain
)
6405 g_assert (is_xdomain_ref_allowed (ptr
, start
, obj
->vtable
->domain
));
6411 if (*(gpointer
*)ptr
)
6412 binary_protocol_wbarrier (ptr
, *(gpointer
*)ptr
, (gpointer
)LOAD_VTABLE (*(gpointer
*)ptr
));
6414 if (ptr_in_nursery (ptr
) || ptr_on_stack (ptr
) || !ptr_in_nursery (*(gpointer
*)ptr
)) {
6415 DEBUG (8, fprintf (gc_debug_file
, "Skipping remset at %p\n", ptr
));
6419 if (use_cardtable
) {
6420 if (ptr_in_nursery(*(gpointer
*)ptr
))
6421 sgen_card_table_mark_address ((mword
)ptr
);
6427 buffer
= STORE_REMSET_BUFFER
;
6428 index
= STORE_REMSET_BUFFER_INDEX
;
6429 /* This simple optimization eliminates a sizable portion of
6430 entries. Comparing it to the last but one entry as well
6431 doesn't eliminate significantly more entries. */
6432 if (buffer
[index
] == ptr
) {
6437 DEBUG (8, fprintf (gc_debug_file
, "Adding remset at %p\n", ptr
));
6438 HEAVY_STAT (++stat_wbarrier_generic_store_remset
);
6441 if (index
>= STORE_REMSET_BUFFER_SIZE
) {
6442 evacuate_remset_buffer ();
6443 index
= STORE_REMSET_BUFFER_INDEX
;
6444 g_assert (index
== 0);
6447 buffer
[index
] = ptr
;
6448 STORE_REMSET_BUFFER_INDEX
= index
;
6454 mono_gc_wbarrier_generic_store (gpointer ptr
, MonoObject
* value
)
6456 DEBUG (8, fprintf (gc_debug_file
, "Wbarrier store at %p to %p (%s)\n", ptr
, value
, value
? safe_name (value
) : "null"));
6457 *(void**)ptr
= value
;
6458 if (ptr_in_nursery (value
))
6459 mono_gc_wbarrier_generic_nostore (ptr
);
6463 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest
, gpointer _src
, int size
, unsigned bitmap
)
6465 mword
*dest
= _dest
;
6470 mono_gc_wbarrier_generic_store (dest
, (MonoObject
*)*src
);
6475 size
-= SIZEOF_VOID_P
;
6482 mono_gc_wbarrier_value_copy (gpointer dest
, gpointer src
, int count
, MonoClass
*klass
)
6485 size_t size
= count
* mono_class_value_size (klass
, NULL
);
6487 HEAVY_STAT (++stat_wbarrier_value_copy
);
6488 g_assert (klass
->valuetype
);
6490 memmove (dest
, src
, size
);
6491 if (use_cardtable
) {
6492 sgen_card_table_mark_range ((mword
)dest
, size
);
6494 rs
= REMEMBERED_SET
;
6495 if (ptr_in_nursery (dest
) || ptr_on_stack (dest
) || !SGEN_CLASS_HAS_REFERENCES (klass
)) {
6499 g_assert (klass
->gc_descr_inited
);
6500 DEBUG (8, fprintf (gc_debug_file
, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest
, count
, klass
->gc_descr
, klass
->name
, klass
));
6502 if (rs
->store_next
+ 3 < rs
->end_set
) {
6503 *(rs
->store_next
++) = (mword
)dest
| REMSET_VTYPE
;
6504 *(rs
->store_next
++) = (mword
)klass
->gc_descr
;
6505 *(rs
->store_next
++) = (mword
)count
;
6509 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6510 rs
->next
= REMEMBERED_SET
;
6511 REMEMBERED_SET
= rs
;
6512 #ifdef HAVE_KW_THREAD
6513 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6515 *(rs
->store_next
++) = (mword
)dest
| REMSET_VTYPE
;
6516 *(rs
->store_next
++) = (mword
)klass
->gc_descr
;
6517 *(rs
->store_next
++) = (mword
)count
;
6523 * mono_gc_wbarrier_object_copy:
6525 * Write barrier to call when obj is the result of a clone or copy of an object.
6528 mono_gc_wbarrier_object_copy (MonoObject
* obj
, MonoObject
*src
)
6534 HEAVY_STAT (++stat_wbarrier_object_copy
);
6535 rs
= REMEMBERED_SET
;
6536 DEBUG (6, fprintf (gc_debug_file
, "Adding object remset for %p\n", obj
));
6537 size
= mono_object_class (obj
)->instance_size
;
6539 /* do not copy the sync state */
6540 memcpy ((char*)obj
+ sizeof (MonoObject
), (char*)src
+ sizeof (MonoObject
),
6541 size
- sizeof (MonoObject
));
6542 if (ptr_in_nursery (obj
) || ptr_on_stack (obj
)) {
6546 if (rs
->store_next
< rs
->end_set
) {
6547 *(rs
->store_next
++) = (mword
)obj
| REMSET_OBJECT
;
6551 rs
= alloc_remset (rs
->end_set
- rs
->data
, (void*)1);
6552 rs
->next
= REMEMBERED_SET
;
6553 REMEMBERED_SET
= rs
;
6554 #ifdef HAVE_KW_THREAD
6555 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset
= rs
;
6557 *(rs
->store_next
++) = (mword
)obj
| REMSET_OBJECT
;
6562 * ######################################################################
6563 * ######## Collector debugging
6564 * ######################################################################
6567 const char*descriptor_types
[] = {
6579 describe_ptr (char *ptr
)
6586 if (ptr_in_nursery (ptr
)) {
6587 printf ("Pointer inside nursery.\n");
6589 if (mono_sgen_ptr_is_in_los (ptr
, &start
)) {
6591 printf ("Pointer is the start of object %p in LOS space.\n", start
);
6593 printf ("Pointer is at offset 0x%x of object %p in LOS space.\n", (int)(ptr
- start
), start
);
6595 } else if (major_collector
.ptr_is_in_non_pinned_space (ptr
)) {
6596 printf ("Pointer inside oldspace.\n");
6597 } else if (major_collector
.obj_is_from_pinned_alloc (ptr
)) {
6598 printf ("Pointer is inside a pinned chunk.\n");
6600 printf ("Pointer unknown.\n");
6605 if (object_is_pinned (ptr
))
6606 printf ("Object is pinned.\n");
6608 if (object_is_forwarded (ptr
))
6609 printf ("Object is forwared.\n");
6611 // FIXME: Handle pointers to the inside of objects
6612 vtable
= (MonoVTable
*)LOAD_VTABLE (ptr
);
6614 printf ("VTable: %p\n", vtable
);
6615 if (vtable
== NULL
) {
6616 printf ("VTable is invalid (empty).\n");
6619 if (ptr_in_nursery (vtable
)) {
6620 printf ("VTable is invalid (points inside nursery).\n");
6623 printf ("Class: %s\n", vtable
->klass
->name
);
6625 desc
= ((GCVTable
*)vtable
)->desc
;
6626 printf ("Descriptor: %lx\n", (long)desc
);
6629 printf ("Descriptor type: %d (%s)\n", type
, descriptor_types
[type
]);
6633 find_in_remset_loc (mword
*p
, char *addr
, gboolean
*found
)
6639 switch ((*p
) & REMSET_TYPE_MASK
) {
6640 case REMSET_LOCATION
:
6641 if (*p
== (mword
)addr
)
6645 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
6647 if ((void**)addr
>= ptr
&& (void**)addr
< ptr
+ count
)
6651 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
6652 count
= safe_object_get_size ((MonoObject
*)ptr
);
6653 count
= ALIGN_UP (count
);
6654 count
/= sizeof (mword
);
6655 if ((void**)addr
>= ptr
&& (void**)addr
< ptr
+ count
)
6659 ptr
= (void**)(*p
& ~REMSET_TYPE_MASK
);
6663 switch (desc
& 0x7) {
6664 case DESC_TYPE_RUN_LENGTH
:
6665 OBJ_RUN_LEN_SIZE (skip_size
, desc
, ptr
);
6667 case DESC_TYPE_SMALL_BITMAP
:
6668 OBJ_BITMAP_SIZE (skip_size
, desc
, start
);
6672 g_assert_not_reached ();
6675 /* The descriptor includes the size of MonoObject */
6676 skip_size
-= sizeof (MonoObject
);
6678 if ((void**)addr
>= ptr
&& (void**)addr
< ptr
+ (skip_size
/ sizeof (gpointer
)))
6683 g_assert_not_reached ();
6689 * Return whenever ADDR occurs in the remembered sets
6692 find_in_remsets (char *addr
)
6695 SgenThreadInfo
*info
;
6696 RememberedSet
*remset
;
6697 GenericStoreRememberedSet
*store_remset
;
6699 gboolean found
= FALSE
;
6701 /* the global one */
6702 for (remset
= global_remset
; remset
; remset
= remset
->next
) {
6703 DEBUG (4, fprintf (gc_debug_file
, "Scanning global remset range: %p-%p, size: %td\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
6704 for (p
= remset
->data
; p
< remset
->store_next
;) {
6705 p
= find_in_remset_loc (p
, addr
, &found
);
6711 /* the generic store ones */
6712 for (store_remset
= generic_store_remsets
; store_remset
; store_remset
= store_remset
->next
) {
6713 for (i
= 0; i
< STORE_REMSET_BUFFER_SIZE
- 1; ++i
) {
6714 if (store_remset
->data
[i
] == addr
)
6719 /* the per-thread ones */
6720 for (i
= 0; i
< THREAD_HASH_SIZE
; ++i
) {
6721 for (info
= thread_table
[i
]; info
; info
= info
->next
) {
6723 for (remset
= info
->remset
; remset
; remset
= remset
->next
) {
6724 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info
, remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
6725 for (p
= remset
->data
; p
< remset
->store_next
;) {
6726 p
= find_in_remset_loc (p
, addr
, &found
);
6731 for (j
= 0; j
< *info
->store_remset_buffer_index_addr
; ++j
) {
6732 if ((*info
->store_remset_buffer_addr
) [j
+ 1] == addr
)
6738 /* the freed thread ones */
6739 for (remset
= freed_thread_remsets
; remset
; remset
= remset
->next
) {
6740 DEBUG (4, fprintf (gc_debug_file
, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset
->data
, remset
->store_next
, remset
->store_next
- remset
->data
));
6741 for (p
= remset
->data
; p
< remset
->store_next
;) {
6742 p
= find_in_remset_loc (p
, addr
, &found
);
6751 static gboolean missing_remsets
;
6754 * We let a missing remset slide if the target object is pinned,
6755 * because the store might have happened but the remset not yet added,
6756 * but in that case the target must be pinned. We might theoretically
6757 * miss some missing remsets this way, but it's very unlikely.
6760 #define HANDLE_PTR(ptr,obj) do { \
6761 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6762 if (!find_in_remsets ((char*)(ptr)) && (!use_cardtable || !sgen_card_table_address_is_marked ((mword)ptr))) { \
6763 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6764 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6765 if (!object_is_pinned (*(ptr))) \
6766 missing_remsets = TRUE; \
6772 * Check that each object reference which points into the nursery can
6773 * be found in the remembered sets.
6776 check_consistency_callback (char *start
, size_t size
, void *dummy
)
6778 GCVTable
*vt
= (GCVTable
*)LOAD_VTABLE (start
);
6779 DEBUG (8, fprintf (gc_debug_file
, "Scanning object %p, vtable: %p (%s)\n", start
, vt
, vt
->klass
->name
));
6781 #define SCAN_OBJECT_ACTION
6782 #include "sgen-scan-object.h"
6786 * Perform consistency check of the heap.
6788 * Assumes the world is stopped.
6791 check_consistency (void)
6793 // Need to add more checks
6795 missing_remsets
= FALSE
;
6797 DEBUG (1, fprintf (gc_debug_file
, "Begin heap consistency check...\n"));
6799 // Check that oldspace->newspace pointers are registered with the collector
6800 major_collector
.iterate_objects (TRUE
, TRUE
, (IterateObjectCallbackFunc
)check_consistency_callback
, NULL
);
6802 mono_sgen_los_iterate_objects ((IterateObjectCallbackFunc
)check_consistency_callback
, NULL
);
6804 DEBUG (1, fprintf (gc_debug_file
, "Heap consistency check done.\n"));
6806 #ifdef SGEN_BINARY_PROTOCOL
6807 if (!binary_protocol_file
)
6809 g_assert (!missing_remsets
);
6814 #define HANDLE_PTR(ptr,obj) do { \
6815 if (*(ptr) && !LOAD_VTABLE (*(ptr))) \
6816 g_error ("Could not load vtable for obj %p slot %d (size %d)", obj, (char*)ptr - (char*)obj, safe_object_get_size ((MonoObject*)obj)); \
6820 check_major_refs_callback (char *start
, size_t size
, void *dummy
)
6822 #define SCAN_OBJECT_ACTION
6823 #include "sgen-scan-object.h"
6827 check_major_refs (void)
6829 major_collector
.iterate_objects (TRUE
, TRUE
, (IterateObjectCallbackFunc
)check_major_refs_callback
, NULL
);
6830 mono_sgen_los_iterate_objects ((IterateObjectCallbackFunc
)check_major_refs_callback
, NULL
);
6833 /* Check that the reference is valid */
6835 #define HANDLE_PTR(ptr,obj) do { \
6837 g_assert (safe_name (*(ptr)) != NULL); \
6844 * Perform consistency check on an object. Currently we only check that the
6845 * reference fields are valid.
6848 check_object (char *start
)
6853 #include "sgen-scan-object.h"
6857 * ######################################################################
6858 * ######## Other mono public interface functions.
6859 * ######################################################################
6862 #define REFS_SIZE 128
6865 MonoGCReferences callback
;
6869 MonoObject
*refs
[REFS_SIZE
];
6870 uintptr_t offsets
[REFS_SIZE
];
6874 #define HANDLE_PTR(ptr,obj) do { \
6876 if (hwi->count == REFS_SIZE) { \
6877 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data); \
6881 hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start; \
6882 hwi->refs [hwi->count++] = *(ptr); \
6887 collect_references (HeapWalkInfo
*hwi
, char *start
, size_t size
)
6889 #include "sgen-scan-object.h"
6893 walk_references (char *start
, size_t size
, void *data
)
6895 HeapWalkInfo
*hwi
= data
;
6898 collect_references (hwi
, start
, size
);
6899 if (hwi
->count
|| !hwi
->called
)
6900 hwi
->callback ((MonoObject
*)start
, mono_object_class (start
), hwi
->called
? 0: size
, hwi
->count
, hwi
->refs
, hwi
->offsets
, hwi
->data
);
6904 * mono_gc_walk_heap:
6905 * @flags: flags for future use
6906 * @callback: a function pointer called for each object in the heap
6907 * @data: a user data pointer that is passed to callback
6909 * This function can be used to iterate over all the live objects in the heap:
6910 * for each object, @callback is invoked, providing info about the object's
6911 * location in memory, its class, its size and the objects it references.
6912 * For each referenced object it's offset from the object address is
6913 * reported in the offsets array.
6914 * The object references may be buffered, so the callback may be invoked
6915 * multiple times for the same object: in all but the first call, the size
6916 * argument will be zero.
6917 * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
6918 * profiler event handler.
6920 * Returns: a non-zero value if the GC doesn't support heap walking
6923 mono_gc_walk_heap (int flags
, MonoGCReferences callback
, void *data
)
6928 hwi
.callback
= callback
;
6931 clear_nursery_fragments (nursery_next
);
6932 mono_sgen_scan_area_with_callback (nursery_section
->data
, nursery_section
->end_data
, walk_references
, &hwi
, FALSE
);
6934 major_collector
.iterate_objects (TRUE
, TRUE
, walk_references
, &hwi
);
6935 mono_sgen_los_iterate_objects (walk_references
, &hwi
);
6941 mono_gc_collect (int generation
)
6946 mono_profiler_gc_event (MONO_GC_EVENT_START
, generation
);
6947 stop_world (generation
);
6948 if (generation
== 0) {
6949 collect_nursery (0);
6951 major_collection ("user request");
6953 restart_world (generation
);
6954 mono_profiler_gc_event (MONO_GC_EVENT_END
, generation
);
6959 mono_gc_max_generation (void)
6965 mono_gc_collection_count (int generation
)
6967 if (generation
== 0)
6968 return num_minor_gcs
;
6969 return num_major_gcs
;
6973 mono_gc_get_used_size (void)
6977 tot
= los_memory_usage
;
6978 tot
+= nursery_section
->next_data
- nursery_section
->data
;
6979 tot
+= major_collector
.get_used_size ();
6980 /* FIXME: account for pinned objects */
6986 mono_gc_get_heap_size (void)
6992 mono_gc_disable (void)
7000 mono_gc_enable (void)
7008 mono_gc_get_los_limit (void)
7010 return MAX_SMALL_OBJ_SIZE
;
7014 mono_object_is_alive (MonoObject
* o
)
7020 mono_gc_get_generation (MonoObject
*obj
)
7022 if (ptr_in_nursery (obj
))
7028 mono_gc_enable_events (void)
7033 mono_gc_weak_link_add (void **link_addr
, MonoObject
*obj
, gboolean track
)
7036 mono_gc_register_disappearing_link (obj
, link_addr
, track
);
7041 mono_gc_weak_link_remove (void **link_addr
)
7044 mono_gc_register_disappearing_link (NULL
, link_addr
, FALSE
);
7049 mono_gc_weak_link_get (void **link_addr
)
7053 return (MonoObject
*) REVEAL_POINTER (*link_addr
);
7057 mono_gc_ephemeron_array_add (MonoObject
*obj
)
7059 EphemeronLinkNode
*node
;
7063 node
= mono_sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK
);
7068 node
->array
= (char*)obj
;
7069 node
->next
= ephemeron_list
;
7070 ephemeron_list
= node
;
7072 DEBUG (5, fprintf (gc_debug_file
, "Registered ephemeron array %p\n", obj
));
7079 mono_gc_make_descr_from_bitmap (gsize
*bitmap
, int numbits
)
7082 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP
, 0);
7083 } else if (numbits
< ((sizeof (*bitmap
) * 8) - ROOT_DESC_TYPE_SHIFT
)) {
7084 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP
, bitmap
[0]);
7086 mword
complex = alloc_complex_descriptor (bitmap
, numbits
);
7087 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX
, complex);
7091 static void *all_ref_root_descrs
[32];
7094 mono_gc_make_root_descr_all_refs (int numbits
)
7099 if (numbits
< 32 && all_ref_root_descrs
[numbits
])
7100 return all_ref_root_descrs
[numbits
];
7102 gc_bitmap
= g_malloc0 (ALIGN_TO (numbits
, 8) + 1);
7103 memset (gc_bitmap
, 0xff, numbits
/ 8);
7105 gc_bitmap
[numbits
/ 8] = (1 << (numbits
% 8)) - 1;
7106 descr
= mono_gc_make_descr_from_bitmap (gc_bitmap
, numbits
);
7110 all_ref_root_descrs
[numbits
] = descr
;
7116 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker
)
7120 g_assert (user_descriptors_next
< MAX_USER_DESCRIPTORS
);
7121 descr
= (void*)MAKE_ROOT_DESC (ROOT_DESC_USER
, (mword
)user_descriptors_next
);
7122 user_descriptors
[user_descriptors_next
++] = marker
;
7128 mono_gc_alloc_fixed (size_t size
, void *descr
)
7130 /* FIXME: do a single allocation */
7131 void *res
= calloc (1, size
);
7134 if (!mono_gc_register_root (res
, size
, descr
)) {
7142 mono_gc_free_fixed (void* addr
)
7144 mono_gc_deregister_root (addr
);
7149 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func
, void *data
)
7153 result
= func (data
);
7154 UNLOCK_INTERRUPTION
;
7159 mono_gc_is_gc_thread (void)
7163 result
= mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()) != NULL
;
7169 mono_gc_base_init (void)
7173 char *major_collector_opt
= NULL
;
7174 struct sigaction sinfo
;
7177 /* the gc_initialized guard seems to imply this method is
7178 idempotent, but LOCK_INIT(gc_mutex) might not be. It's
7179 defined in sgen-gc.h as nothing, so there's no danger at
7181 LOCK_INIT (gc_mutex
);
7183 if (gc_initialized
) {
7187 pagesize
= mono_pagesize ();
7188 gc_debug_file
= stdout
;
7190 LOCK_INIT (interruption_mutex
);
7191 LOCK_INIT (global_remset_mutex
);
7192 LOCK_INIT (pin_queue_mutex
);
7194 if ((env
= getenv ("MONO_GC_PARAMS"))) {
7195 opts
= g_strsplit (env
, ",", -1);
7196 for (ptr
= opts
; *ptr
; ++ptr
) {
7198 if (g_str_has_prefix (opt
, "major=")) {
7199 opt
= strchr (opt
, '=') + 1;
7200 major_collector_opt
= g_strdup (opt
);
7208 mono_sgen_init_internal_allocator ();
7210 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FRAGMENT
, sizeof (Fragment
));
7211 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION
, SGEN_SIZEOF_GC_MEM_SECTION
);
7212 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_ENTRY
, sizeof (FinalizeEntry
));
7213 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_DISLINK
, sizeof (DisappearingLink
));
7214 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_ROOT_RECORD
, sizeof (RootRecord
));
7215 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE
, sizeof (GrayQueueSection
));
7216 g_assert (sizeof (GenericStoreRememberedSet
) == sizeof (gpointer
) * STORE_REMSET_BUFFER_SIZE
);
7217 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_STORE_REMSET
, sizeof (GenericStoreRememberedSet
));
7218 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK
, sizeof (EphemeronLinkNode
));
7220 if (!major_collector_opt
|| !strcmp (major_collector_opt
, "marksweep")) {
7221 mono_sgen_marksweep_init (&major_collector
);
7222 } else if (!major_collector_opt
|| !strcmp (major_collector_opt
, "marksweep-fixed")) {
7223 mono_sgen_marksweep_fixed_init (&major_collector
);
7224 } else if (!major_collector_opt
|| !strcmp (major_collector_opt
, "marksweep-par")) {
7225 mono_sgen_marksweep_par_init (&major_collector
);
7226 workers_init (mono_cpu_count ());
7227 } else if (!major_collector_opt
|| !strcmp (major_collector_opt
, "marksweep-fixed-par")) {
7228 mono_sgen_marksweep_fixed_par_init (&major_collector
);
7229 workers_init (mono_cpu_count ());
7230 } else if (!strcmp (major_collector_opt
, "copying")) {
7231 mono_sgen_copying_init (&major_collector
);
7233 fprintf (stderr
, "Unknown major collector `%s'.\n", major_collector_opt
);
7237 #ifdef SGEN_HAVE_CARDTABLE
7238 use_cardtable
= major_collector
.supports_cardtable
;
7240 use_cardtable
= FALSE
;
7243 /* Keep this the default for now */
7244 conservative_stack_mark
= TRUE
;
7247 for (ptr
= opts
; *ptr
; ++ptr
) {
7249 if (g_str_has_prefix (opt
, "major="))
7251 if (g_str_has_prefix (opt
, "wbarrier=")) {
7252 opt
= strchr (opt
, '=') + 1;
7253 if (strcmp (opt
, "remset") == 0) {
7254 use_cardtable
= FALSE
;
7255 } else if (strcmp (opt
, "cardtable") == 0) {
7256 if (!use_cardtable
) {
7257 if (major_collector
.supports_cardtable
)
7258 fprintf (stderr
, "The cardtable write barrier is not supported on this platform.\n");
7260 fprintf (stderr
, "The major collector does not support the cardtable write barrier.\n");
7266 if (g_str_has_prefix (opt
, "max-heap-size=")) {
7267 opt
= strchr (opt
, '=') + 1;
7268 if (*opt
&& mono_gc_parse_environment_string_extract_number (opt
, &max_heap
)) {
7269 if ((max_heap
& (mono_pagesize () - 1))) {
7270 fprintf (stderr
, "max-heap-size size must be a multiple of %d.\n", mono_pagesize ());
7274 fprintf (stderr
, "max-heap-size must be an integer.\n");
7279 if (g_str_has_prefix (opt
, "stack-mark=")) {
7280 opt
= strchr (opt
, '=') + 1;
7281 if (!strcmp (opt
, "precise")) {
7282 conservative_stack_mark
= FALSE
;
7283 } else if (!strcmp (opt
, "conservative")) {
7284 conservative_stack_mark
= TRUE
;
7286 fprintf (stderr
, "Invalid value '%s' for stack-mark= option, possible values are: 'precise', 'conservative'.\n", opt
);
7292 if (g_str_has_prefix (opt
, "nursery-size=")) {
7294 opt
= strchr (opt
, '=') + 1;
7295 if (*opt
&& mono_gc_parse_environment_string_extract_number (opt
, &val
)) {
7296 default_nursery_size
= val
;
7297 #ifdef SGEN_ALIGN_NURSERY
7298 if ((val
& (val
- 1))) {
7299 fprintf (stderr
, "The nursery size must be a power of two.\n");
7303 default_nursery_bits
= 0;
7304 while (1 << (++ default_nursery_bits
) != default_nursery_size
)
7308 fprintf (stderr
, "nursery-size must be an integer.\n");
7314 if (!(major_collector
.handle_gc_param
&& major_collector
.handle_gc_param (opt
))) {
7315 fprintf (stderr
, "MONO_GC_PARAMS must be a comma-delimited list of one or more of the following:\n");
7316 fprintf (stderr
, " max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
7317 fprintf (stderr
, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
7318 fprintf (stderr
, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-par' or `copying')\n");
7319 fprintf (stderr
, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
7320 fprintf (stderr
, " stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
7321 if (major_collector
.print_gc_param_usage
)
7322 major_collector
.print_gc_param_usage ();
7329 if (major_collector_opt
)
7330 g_free (major_collector_opt
);
7332 nursery_size
= DEFAULT_NURSERY_SIZE
;
7333 minor_collection_allowance
= MIN_MINOR_COLLECTION_ALLOWANCE
;
7334 init_heap_size_limits (max_heap
);
7338 if ((env
= getenv ("MONO_GC_DEBUG"))) {
7339 opts
= g_strsplit (env
, ",", -1);
7340 for (ptr
= opts
; ptr
&& *ptr
; ptr
++) {
7342 if (opt
[0] >= '0' && opt
[0] <= '9') {
7343 gc_debug_level
= atoi (opt
);
7348 char *rf
= g_strdup_printf ("%s.%d", opt
, getpid ());
7349 gc_debug_file
= fopen (rf
, "wb");
7351 gc_debug_file
= stderr
;
7354 } else if (!strcmp (opt
, "collect-before-allocs")) {
7355 collect_before_allocs
= 1;
7356 } else if (g_str_has_prefix (opt
, "collect-before-allocs=")) {
7357 char *arg
= strchr (opt
, '=') + 1;
7358 collect_before_allocs
= atoi (arg
);
7359 } else if (!strcmp (opt
, "check-at-minor-collections")) {
7360 consistency_check_at_minor_collection
= TRUE
;
7361 nursery_clear_policy
= CLEAR_AT_GC
;
7362 } else if (!strcmp (opt
, "xdomain-checks")) {
7363 xdomain_checks
= TRUE
;
7364 } else if (!strcmp (opt
, "clear-at-gc")) {
7365 nursery_clear_policy
= CLEAR_AT_GC
;
7366 } else if (!strcmp (opt
, "clear-nursery-at-gc")) {
7367 nursery_clear_policy
= CLEAR_AT_GC
;
7368 } else if (!strcmp (opt
, "check-scan-starts")) {
7369 do_scan_starts_check
= TRUE
;
7370 } else if (!strcmp (opt
, "disable-minor")) {
7371 disable_minor_collections
= TRUE
;
7372 } else if (!strcmp (opt
, "disable-major")) {
7373 disable_major_collections
= TRUE
;
7374 } else if (g_str_has_prefix (opt
, "heap-dump=")) {
7375 char *filename
= strchr (opt
, '=') + 1;
7376 nursery_clear_policy
= CLEAR_AT_GC
;
7377 heap_dump_file
= fopen (filename
, "w");
7379 fprintf (heap_dump_file
, "<sgen-dump>\n");
7380 #ifdef SGEN_BINARY_PROTOCOL
7381 } else if (g_str_has_prefix (opt
, "binary-protocol=")) {
7382 char *filename
= strchr (opt
, '=') + 1;
7383 binary_protocol_file
= fopen (filename
, "w");
7386 fprintf (stderr
, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env
);
7387 fprintf (stderr
, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7388 fprintf (stderr
, "Valid options are:\n");
7389 fprintf (stderr
, " collect-before-allocs[=<n>]\n");
7390 fprintf (stderr
, " check-at-minor-collections\n");
7391 fprintf (stderr
, " disable-minor\n");
7392 fprintf (stderr
, " disable-major\n");
7393 fprintf (stderr
, " xdomain-checks\n");
7394 fprintf (stderr
, " clear-at-gc\n");
7401 suspend_ack_semaphore_ptr
= &suspend_ack_semaphore
;
7402 MONO_SEM_INIT (&suspend_ack_semaphore
, 0);
7404 sigfillset (&sinfo
.sa_mask
);
7405 sinfo
.sa_flags
= SA_RESTART
| SA_SIGINFO
;
7406 sinfo
.sa_sigaction
= suspend_handler
;
7407 if (sigaction (suspend_signal_num
, &sinfo
, NULL
) != 0) {
7408 g_error ("failed sigaction");
7411 sinfo
.sa_handler
= restart_handler
;
7412 if (sigaction (restart_signal_num
, &sinfo
, NULL
) != 0) {
7413 g_error ("failed sigaction");
7416 sigfillset (&suspend_signal_mask
);
7417 sigdelset (&suspend_signal_mask
, restart_signal_num
);
7419 global_remset
= alloc_remset (1024, NULL
);
7420 global_remset
->next
= NULL
;
7422 pthread_key_create (&remembered_set_key
, unregister_thread
);
7424 #ifndef HAVE_KW_THREAD
7425 pthread_key_create (&thread_info_key
, NULL
);
7431 gc_initialized
= TRUE
;
7433 mono_gc_register_thread (&sinfo
);
7437 mono_gc_get_suspend_signal (void)
7439 return suspend_signal_num
;
7449 #ifdef HAVE_KW_THREAD
7450 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7451 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7452 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7453 mono_mb_emit_i4 ((mb), (offset)); \
7456 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7457 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7458 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7459 mono_mb_emit_i4 ((mb), thread_info_key); \
7460 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7461 mono_mb_emit_byte ((mb), CEE_ADD); \
7462 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7466 #ifdef MANAGED_ALLOCATION
7467 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7468 * for each class. This is currently not easy to do, as it is hard to generate basic
7469 * blocks + branches, but it is easy with the linear IL codebase.
7471 * For this to work we'd need to solve the TLAB race, first. Now we
7472 * require the allocator to be in a few known methods to make sure
7473 * that they are executed atomically via the restart mechanism.
7476 create_allocator (int atype
)
7478 int p_var
, size_var
;
7479 guint32 slowpath_branch
, max_size_branch
;
7480 MonoMethodBuilder
*mb
;
7482 MonoMethodSignature
*csig
;
7483 static gboolean registered
= FALSE
;
7484 int tlab_next_addr_var
, new_next_var
;
7486 const char *name
= NULL
;
7487 AllocatorWrapperInfo
*info
;
7489 #ifdef HAVE_KW_THREAD
7490 int tlab_next_addr_offset
= -1;
7491 int tlab_temp_end_offset
= -1;
7493 MONO_THREAD_VAR_OFFSET (tlab_next_addr
, tlab_next_addr_offset
);
7494 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
7496 g_assert (tlab_next_addr_offset
!= -1);
7497 g_assert (tlab_temp_end_offset
!= -1);
7501 mono_register_jit_icall (mono_gc_alloc_obj
, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE
);
7502 mono_register_jit_icall (mono_gc_alloc_vector
, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE
);
7506 if (atype
== ATYPE_SMALL
) {
7508 name
= "AllocSmall";
7509 } else if (atype
== ATYPE_NORMAL
) {
7512 } else if (atype
== ATYPE_VECTOR
) {
7514 name
= "AllocVector";
7516 g_assert_not_reached ();
7519 csig
= mono_metadata_signature_alloc (mono_defaults
.corlib
, num_params
);
7520 csig
->ret
= &mono_defaults
.object_class
->byval_arg
;
7521 for (i
= 0; i
< num_params
; ++i
)
7522 csig
->params
[i
] = &mono_defaults
.int_class
->byval_arg
;
7524 mb
= mono_mb_new (mono_defaults
.object_class
, name
, MONO_WRAPPER_ALLOC
);
7525 size_var
= mono_mb_add_local (mb
, &mono_defaults
.int32_class
->byval_arg
);
7526 if (atype
== ATYPE_NORMAL
|| atype
== ATYPE_SMALL
) {
7527 /* size = vtable->klass->instance_size; */
7528 mono_mb_emit_ldarg (mb
, 0);
7529 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoVTable
, klass
));
7530 mono_mb_emit_byte (mb
, CEE_ADD
);
7531 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7532 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoClass
, instance_size
));
7533 mono_mb_emit_byte (mb
, CEE_ADD
);
7534 /* FIXME: assert instance_size stays a 4 byte integer */
7535 mono_mb_emit_byte (mb
, CEE_LDIND_U4
);
7536 mono_mb_emit_stloc (mb
, size_var
);
7537 } else if (atype
== ATYPE_VECTOR
) {
7538 MonoExceptionClause
*clause
;
7540 MonoClass
*oom_exc_class
;
7543 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7544 mono_mb_emit_ldarg (mb
, 1);
7545 mono_mb_emit_icon (mb
, MONO_ARRAY_MAX_INDEX
);
7546 pos
= mono_mb_emit_short_branch (mb
, CEE_BLE_UN_S
);
7547 mono_mb_emit_exception (mb
, "OverflowException", NULL
);
7548 mono_mb_patch_short_branch (mb
, pos
);
7550 clause
= mono_image_alloc0 (mono_defaults
.corlib
, sizeof (MonoExceptionClause
));
7551 clause
->try_offset
= mono_mb_get_label (mb
);
7553 /* vtable->klass->sizes.element_size */
7554 mono_mb_emit_ldarg (mb
, 0);
7555 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoVTable
, klass
));
7556 mono_mb_emit_byte (mb
, CEE_ADD
);
7557 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7558 mono_mb_emit_icon (mb
, G_STRUCT_OFFSET (MonoClass
, sizes
.element_size
));
7559 mono_mb_emit_byte (mb
, CEE_ADD
);
7560 mono_mb_emit_byte (mb
, CEE_LDIND_U4
);
7563 mono_mb_emit_ldarg (mb
, 1);
7564 mono_mb_emit_byte (mb
, CEE_MUL_OVF_UN
);
7565 /* + sizeof (MonoArray) */
7566 mono_mb_emit_icon (mb
, sizeof (MonoArray
));
7567 mono_mb_emit_byte (mb
, CEE_ADD_OVF_UN
);
7568 mono_mb_emit_stloc (mb
, size_var
);
7570 pos_leave
= mono_mb_emit_branch (mb
, CEE_LEAVE
);
7573 clause
->flags
= MONO_EXCEPTION_CLAUSE_NONE
;
7574 clause
->try_len
= mono_mb_get_pos (mb
) - clause
->try_offset
;
7575 clause
->data
.catch_class
= mono_class_from_name (mono_defaults
.corlib
,
7576 "System", "OverflowException");
7577 g_assert (clause
->data
.catch_class
);
7578 clause
->handler_offset
= mono_mb_get_label (mb
);
7580 oom_exc_class
= mono_class_from_name (mono_defaults
.corlib
,
7581 "System", "OutOfMemoryException");
7582 g_assert (oom_exc_class
);
7583 ctor
= mono_class_get_method_from_name (oom_exc_class
, ".ctor", 0);
7586 mono_mb_emit_byte (mb
, CEE_POP
);
7587 mono_mb_emit_op (mb
, CEE_NEWOBJ
, ctor
);
7588 mono_mb_emit_byte (mb
, CEE_THROW
);
7590 clause
->handler_len
= mono_mb_get_pos (mb
) - clause
->handler_offset
;
7591 mono_mb_set_clauses (mb
, 1, clause
);
7592 mono_mb_patch_branch (mb
, pos_leave
);
7595 g_assert_not_reached ();
7598 /* size += ALLOC_ALIGN - 1; */
7599 mono_mb_emit_ldloc (mb
, size_var
);
7600 mono_mb_emit_icon (mb
, ALLOC_ALIGN
- 1);
7601 mono_mb_emit_byte (mb
, CEE_ADD
);
7602 /* size &= ~(ALLOC_ALIGN - 1); */
7603 mono_mb_emit_icon (mb
, ~(ALLOC_ALIGN
- 1));
7604 mono_mb_emit_byte (mb
, CEE_AND
);
7605 mono_mb_emit_stloc (mb
, size_var
);
7607 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7608 if (atype
!= ATYPE_SMALL
) {
7609 mono_mb_emit_ldloc (mb
, size_var
);
7610 mono_mb_emit_icon (mb
, MAX_SMALL_OBJ_SIZE
);
7611 max_size_branch
= mono_mb_emit_short_branch (mb
, MONO_CEE_BGT_S
);
7615 * We need to modify tlab_next, but the JIT only supports reading, so we read
7616 * another tls var holding its address instead.
7619 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7620 tlab_next_addr_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7621 EMIT_TLS_ACCESS (mb
, tlab_next_addr
, tlab_next_addr_offset
);
7622 mono_mb_emit_stloc (mb
, tlab_next_addr_var
);
7624 /* p = (void**)tlab_next; */
7625 p_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7626 mono_mb_emit_ldloc (mb
, tlab_next_addr_var
);
7627 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7628 mono_mb_emit_stloc (mb
, p_var
);
7630 /* new_next = (char*)p + size; */
7631 new_next_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7632 mono_mb_emit_ldloc (mb
, p_var
);
7633 mono_mb_emit_ldloc (mb
, size_var
);
7634 mono_mb_emit_byte (mb
, CEE_CONV_I
);
7635 mono_mb_emit_byte (mb
, CEE_ADD
);
7636 mono_mb_emit_stloc (mb
, new_next_var
);
7638 /* tlab_next = new_next */
7639 mono_mb_emit_ldloc (mb
, tlab_next_addr_var
);
7640 mono_mb_emit_ldloc (mb
, new_next_var
);
7641 mono_mb_emit_byte (mb
, CEE_STIND_I
);
7643 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7644 mono_mb_emit_ldloc (mb
, new_next_var
);
7645 EMIT_TLS_ACCESS (mb
, tlab_temp_end
, tlab_temp_end_offset
);
7646 slowpath_branch
= mono_mb_emit_short_branch (mb
, MONO_CEE_BLT_UN_S
);
7649 if (atype
!= ATYPE_SMALL
)
7650 mono_mb_patch_short_branch (mb
, max_size_branch
);
7652 mono_mb_emit_byte (mb
, MONO_CUSTOM_PREFIX
);
7653 mono_mb_emit_byte (mb
, CEE_MONO_NOT_TAKEN
);
7655 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7656 mono_mb_emit_ldarg (mb
, 0);
7657 mono_mb_emit_ldloc (mb
, size_var
);
7658 if (atype
== ATYPE_NORMAL
|| atype
== ATYPE_SMALL
) {
7659 mono_mb_emit_icall (mb
, mono_gc_alloc_obj
);
7660 } else if (atype
== ATYPE_VECTOR
) {
7661 mono_mb_emit_ldarg (mb
, 1);
7662 mono_mb_emit_icall (mb
, mono_gc_alloc_vector
);
7664 g_assert_not_reached ();
7666 mono_mb_emit_byte (mb
, CEE_RET
);
7669 mono_mb_patch_short_branch (mb
, slowpath_branch
);
7671 /* FIXME: Memory barrier */
7674 mono_mb_emit_ldloc (mb
, p_var
);
7675 mono_mb_emit_ldarg (mb
, 0);
7676 mono_mb_emit_byte (mb
, CEE_STIND_I
);
7678 if (atype
== ATYPE_VECTOR
) {
7679 /* arr->max_length = max_length; */
7680 mono_mb_emit_ldloc (mb
, p_var
);
7681 mono_mb_emit_ldflda (mb
, G_STRUCT_OFFSET (MonoArray
, max_length
));
7682 mono_mb_emit_ldarg (mb
, 1);
7683 mono_mb_emit_byte (mb
, CEE_STIND_I
);
7687 mono_mb_emit_ldloc (mb
, p_var
);
7688 mono_mb_emit_byte (mb
, CEE_RET
);
7690 res
= mono_mb_create_method (mb
, csig
, 8);
7692 mono_method_get_header (res
)->init_locals
= FALSE
;
7694 info
= mono_image_alloc0 (mono_defaults
.corlib
, sizeof (AllocatorWrapperInfo
));
7695 info
->gc_name
= "sgen";
7696 info
->alloc_type
= atype
;
7697 mono_marshal_set_wrapper_info (res
, info
);
7704 mono_gc_get_gc_name (void)
7709 static MonoMethod
* alloc_method_cache
[ATYPE_NUM
];
7710 static MonoMethod
*write_barrier_method
;
7713 is_ip_in_managed_allocator (MonoDomain
*domain
, gpointer ip
)
7719 if (!mono_thread_internal_current ())
7720 /* Happens during thread attach */
7725 ji
= mono_jit_info_table_find (domain
, ip
);
7728 method
= ji
->method
;
7730 if (method
== write_barrier_method
)
7732 for (i
= 0; i
< ATYPE_NUM
; ++i
)
7733 if (method
== alloc_method_cache
[i
])
7739 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7740 * The signature of the called method is:
7741 * object allocate (MonoVTable *vtable)
7744 mono_gc_get_managed_allocator (MonoVTable
*vtable
, gboolean for_box
)
7746 #ifdef MANAGED_ALLOCATION
7747 MonoClass
*klass
= vtable
->klass
;
7749 #ifdef HAVE_KW_THREAD
7750 int tlab_next_offset
= -1;
7751 int tlab_temp_end_offset
= -1;
7752 MONO_THREAD_VAR_OFFSET (tlab_next
, tlab_next_offset
);
7753 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
7755 if (tlab_next_offset
== -1 || tlab_temp_end_offset
== -1)
7759 if (!mono_runtime_has_tls_get ())
7761 if (klass
->instance_size
> tlab_size
)
7763 if (klass
->has_finalize
|| klass
->marshalbyref
|| (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS
))
7767 if (klass
->byval_arg
.type
== MONO_TYPE_STRING
)
7769 if (collect_before_allocs
)
7772 if (ALIGN_TO (klass
->instance_size
, ALLOC_ALIGN
) < MAX_SMALL_OBJ_SIZE
)
7773 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL
);
7775 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL
);
7782 mono_gc_get_managed_array_allocator (MonoVTable
*vtable
, int rank
)
7784 #ifdef MANAGED_ALLOCATION
7785 MonoClass
*klass
= vtable
->klass
;
7787 #ifdef HAVE_KW_THREAD
7788 int tlab_next_offset
= -1;
7789 int tlab_temp_end_offset
= -1;
7790 MONO_THREAD_VAR_OFFSET (tlab_next
, tlab_next_offset
);
7791 MONO_THREAD_VAR_OFFSET (tlab_temp_end
, tlab_temp_end_offset
);
7793 if (tlab_next_offset
== -1 || tlab_temp_end_offset
== -1)
7799 if (!mono_runtime_has_tls_get ())
7801 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS
)
7803 if (collect_before_allocs
)
7805 g_assert (!mono_class_has_finalizer (klass
) && !klass
->marshalbyref
);
7807 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR
);
7814 mono_gc_get_managed_allocator_by_type (int atype
)
7816 #ifdef MANAGED_ALLOCATION
7819 if (!mono_runtime_has_tls_get ())
7822 mono_loader_lock ();
7823 res
= alloc_method_cache
[atype
];
7825 res
= alloc_method_cache
[atype
] = create_allocator (atype
);
7826 mono_loader_unlock ();
7834 mono_gc_get_managed_allocator_types (void)
7841 mono_gc_get_write_barrier (void)
7844 MonoMethodBuilder
*mb
;
7845 MonoMethodSignature
*sig
;
7846 #ifdef MANAGED_WBARRIER
7847 int label_no_wb_1
, label_no_wb_2
, label_no_wb_3
, label_no_wb_4
, label_need_wb
, label_slow_path
;
7848 #ifndef SGEN_ALIGN_NURSERY
7849 int label_continue_1
, label_continue_2
, label_no_wb_5
;
7850 int dereferenced_var
;
7852 int buffer_var
, buffer_index_var
, dummy_var
;
7854 #ifdef HAVE_KW_THREAD
7855 int stack_end_offset
= -1, store_remset_buffer_offset
= -1;
7856 int store_remset_buffer_index_offset
= -1, store_remset_buffer_index_addr_offset
= -1;
7858 MONO_THREAD_VAR_OFFSET (stack_end
, stack_end_offset
);
7859 g_assert (stack_end_offset
!= -1);
7860 MONO_THREAD_VAR_OFFSET (store_remset_buffer
, store_remset_buffer_offset
);
7861 g_assert (store_remset_buffer_offset
!= -1);
7862 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index
, store_remset_buffer_index_offset
);
7863 g_assert (store_remset_buffer_index_offset
!= -1);
7864 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr
, store_remset_buffer_index_addr_offset
);
7865 g_assert (store_remset_buffer_index_addr_offset
!= -1);
7869 g_assert (!use_cardtable
);
7871 // FIXME: Maybe create a separate version for ctors (the branch would be
7872 // correctly predicted more times)
7873 if (write_barrier_method
)
7874 return write_barrier_method
;
7876 /* Create the IL version of mono_gc_barrier_generic_store () */
7877 sig
= mono_metadata_signature_alloc (mono_defaults
.corlib
, 1);
7878 sig
->ret
= &mono_defaults
.void_class
->byval_arg
;
7879 sig
->params
[0] = &mono_defaults
.int_class
->byval_arg
;
7881 mb
= mono_mb_new (mono_defaults
.object_class
, "wbarrier", MONO_WRAPPER_WRITE_BARRIER
);
7883 #ifdef MANAGED_WBARRIER
7884 if (mono_runtime_has_tls_get ()) {
7885 #ifdef SGEN_ALIGN_NURSERY
7886 // if (ptr_in_nursery (ptr)) return;
7888 * Masking out the bits might be faster, but we would have to use 64 bit
7889 * immediates, which might be slower.
7891 mono_mb_emit_ldarg (mb
, 0);
7892 mono_mb_emit_icon (mb
, DEFAULT_NURSERY_BITS
);
7893 mono_mb_emit_byte (mb
, CEE_SHR_UN
);
7894 mono_mb_emit_icon (mb
, (mword
)nursery_start
>> DEFAULT_NURSERY_BITS
);
7895 label_no_wb_1
= mono_mb_emit_branch (mb
, CEE_BEQ
);
7897 // if (!ptr_in_nursery (*ptr)) return;
7898 mono_mb_emit_ldarg (mb
, 0);
7899 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7900 mono_mb_emit_icon (mb
, DEFAULT_NURSERY_BITS
);
7901 mono_mb_emit_byte (mb
, CEE_SHR_UN
);
7902 mono_mb_emit_icon (mb
, (mword
)nursery_start
>> DEFAULT_NURSERY_BITS
);
7903 label_no_wb_2
= mono_mb_emit_branch (mb
, CEE_BNE_UN
);
7906 // if (ptr < (nursery_start)) goto continue;
7907 mono_mb_emit_ldarg (mb
, 0);
7908 mono_mb_emit_ptr (mb
, (gpointer
) nursery_start
);
7909 label_continue_1
= mono_mb_emit_branch (mb
, CEE_BLT
);
7911 // if (ptr >= nursery_real_end)) goto continue;
7912 mono_mb_emit_ldarg (mb
, 0);
7913 mono_mb_emit_ptr (mb
, (gpointer
) nursery_real_end
);
7914 label_continue_2
= mono_mb_emit_branch (mb
, CEE_BGE
);
7917 label_no_wb_1
= mono_mb_emit_branch (mb
, CEE_BR
);
7920 mono_mb_patch_branch (mb
, label_continue_1
);
7921 mono_mb_patch_branch (mb
, label_continue_2
);
7923 // Dereference and store in local var
7924 dereferenced_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7925 mono_mb_emit_ldarg (mb
, 0);
7926 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7927 mono_mb_emit_stloc (mb
, dereferenced_var
);
7929 // if (*ptr < nursery_start) return;
7930 mono_mb_emit_ldloc (mb
, dereferenced_var
);
7931 mono_mb_emit_ptr (mb
, (gpointer
) nursery_start
);
7932 label_no_wb_2
= mono_mb_emit_branch (mb
, CEE_BLT
);
7934 // if (*ptr >= nursery_end) return;
7935 mono_mb_emit_ldloc (mb
, dereferenced_var
);
7936 mono_mb_emit_ptr (mb
, (gpointer
) nursery_real_end
);
7937 label_no_wb_5
= mono_mb_emit_branch (mb
, CEE_BGE
);
7940 // if (ptr >= stack_end) goto need_wb;
7941 mono_mb_emit_ldarg (mb
, 0);
7942 EMIT_TLS_ACCESS (mb
, stack_end
, stack_end_offset
);
7943 label_need_wb
= mono_mb_emit_branch (mb
, CEE_BGE_UN
);
7945 // if (ptr >= stack_start) return;
7946 dummy_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7947 mono_mb_emit_ldarg (mb
, 0);
7948 mono_mb_emit_ldloc_addr (mb
, dummy_var
);
7949 label_no_wb_3
= mono_mb_emit_branch (mb
, CEE_BGE_UN
);
7952 mono_mb_patch_branch (mb
, label_need_wb
);
7954 // buffer = STORE_REMSET_BUFFER;
7955 buffer_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7956 EMIT_TLS_ACCESS (mb
, store_remset_buffer
, store_remset_buffer_offset
);
7957 mono_mb_emit_stloc (mb
, buffer_var
);
7959 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7960 buffer_index_var
= mono_mb_add_local (mb
, &mono_defaults
.int_class
->byval_arg
);
7961 EMIT_TLS_ACCESS (mb
, store_remset_buffer_index
, store_remset_buffer_index_offset
);
7962 mono_mb_emit_stloc (mb
, buffer_index_var
);
7964 // if (buffer [buffer_index] == ptr) return;
7965 mono_mb_emit_ldloc (mb
, buffer_var
);
7966 mono_mb_emit_ldloc (mb
, buffer_index_var
);
7967 g_assert (sizeof (gpointer
) == 4 || sizeof (gpointer
) == 8);
7968 mono_mb_emit_icon (mb
, sizeof (gpointer
) == 4 ? 2 : 3);
7969 mono_mb_emit_byte (mb
, CEE_SHL
);
7970 mono_mb_emit_byte (mb
, CEE_ADD
);
7971 mono_mb_emit_byte (mb
, CEE_LDIND_I
);
7972 mono_mb_emit_ldarg (mb
, 0);
7973 label_no_wb_4
= mono_mb_emit_branch (mb
, CEE_BEQ
);
7976 mono_mb_emit_ldloc (mb
, buffer_index_var
);
7977 mono_mb_emit_icon (mb
, 1);
7978 mono_mb_emit_byte (mb
, CEE_ADD
);
7979 mono_mb_emit_stloc (mb
, buffer_index_var
);
7981 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7982 mono_mb_emit_ldloc (mb
, buffer_index_var
);
7983 mono_mb_emit_icon (mb
, STORE_REMSET_BUFFER_SIZE
);
7984 label_slow_path
= mono_mb_emit_branch (mb
, CEE_BGE
);
7986 // buffer [buffer_index] = ptr;
7987 mono_mb_emit_ldloc (mb
, buffer_var
);
7988 mono_mb_emit_ldloc (mb
, buffer_index_var
);
7989 g_assert (sizeof (gpointer
) == 4 || sizeof (gpointer
) == 8);
7990 mono_mb_emit_icon (mb
, sizeof (gpointer
) == 4 ? 2 : 3);
7991 mono_mb_emit_byte (mb
, CEE_SHL
);
7992 mono_mb_emit_byte (mb
, CEE_ADD
);
7993 mono_mb_emit_ldarg (mb
, 0);
7994 mono_mb_emit_byte (mb
, CEE_STIND_I
);
7996 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7997 EMIT_TLS_ACCESS (mb
, store_remset_buffer_index_addr
, store_remset_buffer_index_addr_offset
);
7998 mono_mb_emit_ldloc (mb
, buffer_index_var
);
7999 mono_mb_emit_byte (mb
, CEE_STIND_I
);
8002 mono_mb_patch_branch (mb
, label_no_wb_1
);
8003 mono_mb_patch_branch (mb
, label_no_wb_2
);
8004 mono_mb_patch_branch (mb
, label_no_wb_3
);
8005 mono_mb_patch_branch (mb
, label_no_wb_4
);
8006 #ifndef SGEN_ALIGN_NURSERY
8007 mono_mb_patch_branch (mb
, label_no_wb_5
);
8009 mono_mb_emit_byte (mb
, CEE_RET
);
8012 mono_mb_patch_branch (mb
, label_slow_path
);
8016 mono_mb_emit_ldarg (mb
, 0);
8017 mono_mb_emit_icall (mb
, mono_gc_wbarrier_generic_nostore
);
8018 mono_mb_emit_byte (mb
, CEE_RET
);
8020 res
= mono_mb_create_method (mb
, sig
, 16);
8023 mono_loader_lock ();
8024 if (write_barrier_method
) {
8025 /* Already created */
8026 mono_free_method (res
);
8028 /* double-checked locking */
8029 mono_memory_barrier ();
8030 write_barrier_method
= res
;
8032 mono_loader_unlock ();
8034 return write_barrier_method
;
8038 mono_gc_get_description (void)
8040 return g_strdup ("sgen");
8044 mono_gc_set_desktop_mode (void)
8049 mono_gc_is_moving (void)
8055 mono_gc_is_disabled (void)
8061 mono_sgen_debug_printf (int level
, const char *format
, ...)
8065 if (level
> gc_debug_level
)
8068 va_start (ap
, format
);
8069 vfprintf (gc_debug_file
, format
, ap
);
8074 mono_sgen_get_logfile (void)
8076 return gc_debug_file
;
8079 #endif /* HAVE_SGEN_GC */