mono/sgen/sgen-marksweep-drain-gray-stack.h

   1 /*
   2  * sgen-marksweep-drain-gray-stack.h: The copy/mark and gray stack
   3  *     draining functions of the M&S major collector.
   4  *
   5  * Copyright (C) 2014 Xamarin Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License 2.0 as published by the Free Software Foundation;
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Library General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Library General Public
  17  * License 2.0 along with this library; if not, write to the Free
  18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19  */
  20
  21 /*
  22  * COPY_OR_MARK_FUNCTION_NAME must be defined to be the function name of the copy/mark
  23  * function.
  24  *
  25  * SCAN_OBJECT_FUNCTION_NAME must be defined to be the function name of the object scanning
  26  * function.
  27  *
  28  * DRAIN_GRAY_STACK_FUNCTION_NAME must be defined to be the function name of the gray stack
  29  * draining function.
  30  *
  31  * Define COPY_OR_MARK_WITH_EVACUATION to support evacuation.
  32  */
  33
  34 /* Returns whether the object is still in the nursery. */
  35 static inline MONO_ALWAYS_INLINE gboolean
  36 COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
  37 {
  38         MSBlockInfo *block;
  39
  40 #ifdef HEAVY_STATISTICS
  41         ++stat_optimized_copy;
  42         {
  43                 GCObject *forwarded;
  44                 SgenDescriptor desc;
  45                 if ((forwarded = SGEN_OBJECT_IS_FORWARDED (obj)))
  46                         desc = sgen_obj_get_descriptor_safe (forwarded);
  47                 else
  48                         desc = sgen_obj_get_descriptor_safe (obj);
  49
  50                 sgen_descriptor_count_copied_object (desc);
  51         }
  52 #endif
  53
  54         SGEN_ASSERT (9, obj, "null object from pointer %p", ptr);
  55 #if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
  56         SGEN_ASSERT (9, current_collection_generation == GENERATION_OLD, "old gen parallel allocator called from a %d collection", current_collection_generation);
  57 #endif
  58
  59         if (sgen_ptr_in_nursery (obj)) {
  60 #if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
  61                 int word, bit;
  62                 GCObject *forwarded, *old_obj;
  63                 mword vtable_word = *(mword*)obj;
  64
  65                 HEAVY_STAT (++stat_optimized_copy_nursery);
  66
  67 #if SGEN_MAX_DEBUG_LEVEL >= 9
  68                 if (sgen_nursery_is_to_space (obj))
  69                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word) && !SGEN_VTABLE_IS_FORWARDED (vtable_word), "To-space object can't be pinned or forwarded.");
  70 #endif
  71
  72                 if (SGEN_VTABLE_IS_PINNED (vtable_word)) {
  73                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_FORWARDED (vtable_word), "Cannot be both pinned and forwarded.");
  74                         HEAVY_STAT (++stat_optimized_copy_nursery_pinned);
  75                         return TRUE;
  76                 }
  77                 if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
  78                         HEAVY_STAT (++stat_optimized_copy_nursery_forwarded);
  79                         SGEN_UPDATE_REFERENCE (ptr, forwarded);
  80                         return sgen_ptr_in_nursery (forwarded);
  81                 }
  82
  83                 /* An object in the nursery To Space has already been copied and grayed. Nothing to do. */
  84                 if (sgen_nursery_is_to_space (obj))
  85                         return TRUE;
  86
  87 #ifdef COPY_OR_MARK_WITH_EVACUATION
  88         do_copy_object:
  89 #endif
  90                 old_obj = obj;
  91                 obj = copy_object_no_checks (obj, queue);
  92                 if (G_UNLIKELY (old_obj == obj)) {
  93                         /*
  94                          * If we fail to evacuate an object we just stop doing it for a
  95                          * given block size as all other will surely fail too.
  96                          */
  97                         /* FIXME: test this case somehow. */
  98                         if (!sgen_ptr_in_nursery (obj)) {
  99                                 int size_index;
 100                                 block = MS_BLOCK_FOR_OBJ (obj);
 101                                 size_index = block->obj_size_index;
 102                                 evacuate_block_obj_sizes [size_index] = FALSE;
 103                                 MS_MARK_OBJECT_AND_ENQUEUE (obj, sgen_obj_get_descriptor (obj), block, queue);
 104                                 return FALSE;
 105                         }
 106                         return TRUE;
 107                 }
 108                 HEAVY_STAT (++stat_objects_copied_major);
 109                 SGEN_UPDATE_REFERENCE (ptr, obj);
 110
 111                 if (sgen_ptr_in_nursery (obj))
 112                         return TRUE;
 113
 114                 /*
 115                  * FIXME: See comment for copy_object_no_checks().  If
 116                  * we have that, we can let the allocation function
 117                  * give us the block info, too, and we won't have to
 118                  * re-fetch it.
 119                  *
 120                  * FIXME (2): We should rework this to avoid all those nursery checks.
 121                  */
 122                 /*
 123                  * For the split nursery allocator the object might
 124                  * still be in the nursery despite having being
 125                  * promoted, in which case we can't mark it.
 126                  */
 127                 block = MS_BLOCK_FOR_OBJ (obj);
 128                 MS_CALC_MARK_BIT (word, bit, obj);
 129                 SGEN_ASSERT (9, !MS_MARK_BIT (block, word, bit), "object %p already marked", obj);
 130                 MS_SET_MARK_BIT (block, word, bit);
 131                 binary_protocol_mark (obj, (gpointer)LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 132
 133                 return FALSE;
 134 #endif
 135         } else {
 136                 mword vtable_word = *(mword*)obj;
 137                 SgenDescriptor desc;
 138                 int type;
 139
 140                 HEAVY_STAT (++stat_optimized_copy_major);
 141
 142 #ifdef COPY_OR_MARK_WITH_EVACUATION
 143                 {
 144                         GCObject *forwarded;
 145                         if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
 146                                 HEAVY_STAT (++stat_optimized_copy_major_forwarded);
 147                                 SGEN_UPDATE_REFERENCE (ptr, forwarded);
 148                                 SGEN_ASSERT (9, !sgen_ptr_in_nursery (forwarded), "Cannot be forwarded to nursery.");
 149                                 return FALSE;
 150                         }
 151                 }
 152 #endif
 153
 154                 SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word), "Pinned object in non-pinned block?");
 155
 156                 desc = sgen_vtable_get_descriptor ((GCVTable)vtable_word);
 157                 type = desc & DESC_TYPE_MASK;
 158
 159                 if (sgen_safe_object_is_small (obj, type)) {
 160 #ifdef HEAVY_STATISTICS
 161                         if (type <= DESC_TYPE_MAX_SMALL_OBJ)
 162                                 ++stat_optimized_copy_major_small_fast;
 163                         else
 164                                 ++stat_optimized_copy_major_small_slow;
 165 #endif
 166
 167                         block = MS_BLOCK_FOR_OBJ (obj);
 168
 169 #ifdef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
 170                         if (G_UNLIKELY (major_block_is_evacuating (block))) {
 171                                 /*
 172                                  * We don't copy within the concurrent phase. These objects will
 173                                  * be handled below in the finishing pause, by scanning the mod-union
 174                                  * card table.
 175                                  */
 176                                 return FALSE;
 177                         }
 178 #endif
 179
 180 #ifdef COPY_OR_MARK_WITH_EVACUATION
 181                         if (major_block_is_evacuating (block)) {
 182                                 HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
 183                                 goto do_copy_object;
 184                         }
 185 #endif
 186
 187                         MS_MARK_OBJECT_AND_ENQUEUE (obj, desc, block, queue);
 188                 } else {
 189                         HEAVY_STAT (++stat_optimized_copy_major_large);
 190
 191                         if (sgen_los_object_is_pinned (obj))
 192                                 return FALSE;
 193                         binary_protocol_pin (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 194
 195                         sgen_los_pin_object (obj);
 196                         if (SGEN_OBJECT_HAS_REFERENCES (obj))
 197                                 GRAY_OBJECT_ENQUEUE (queue, obj, sgen_obj_get_descriptor (obj));
 198                 }
 199                 return FALSE;
 200         }
 201
 202         return TRUE;
 203 }
 204
 205 static void
 206 SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQueue *queue)
 207 {
 208         char *start = (char*)full_object;
 209
 210 #ifdef HEAVY_STATISTICS
 211         ++stat_optimized_major_scan;
 212         if (!sgen_gc_descr_has_references (desc))
 213                 ++stat_optimized_major_scan_no_refs;
 214         sgen_descriptor_count_scanned_object (desc);
 215 #endif
 216 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
 217         add_scanned_object (start);
 218 #endif
 219
 220         /* Now scan the object. */
 221
 222 #undef HANDLE_PTR
 223 #if defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
 224 #define HANDLE_PTR(ptr,obj)     do {                                    \
 225                 GCObject *__old = *(ptr);                               \
 226                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 227                 if (__old && !sgen_ptr_in_nursery (__old)) {            \
 228                         if (G_UNLIKELY (!sgen_ptr_in_nursery (ptr) &&   \
 229                                         sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \
 230                                         major_block_is_evacuating (MS_BLOCK_FOR_OBJ (__old)))) { \
 231                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 232                         } else {                                        \
 233                                 PREFETCH_READ (__old);                  \
 234                                 COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 235                         }                                               \
 236                 } else {                                                \
 237                         if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
 238                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 239                         }                                               \
 240                 } while (0)
 241 #elif defined(COPY_OR_MARK_CONCURRENT)
 242 #define HANDLE_PTR(ptr,obj)     do {                                    \
 243                 GCObject *__old = *(ptr);                               \
 244                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 245                 if (__old && !sgen_ptr_in_nursery (__old)) {            \
 246                         PREFETCH_READ (__old);                  \
 247                         COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 248                 } else {                                                \
 249                         if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
 250                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 251                         }                                               \
 252                 } while (0)
 253 #else
 254 #define HANDLE_PTR(ptr,obj)     do {                                    \
 255                 GCObject *__old = *(ptr);                                       \
 256                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 257                 if (__old) {                                            \
 258                         gboolean __still_in_nursery = COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 259                         if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
 260                                 GCObject *__copy = *(ptr);                      \
 261                                 sgen_add_to_global_remset ((ptr), __copy); \
 262                         }                                               \
 263                 }                                                       \
 264         } while (0)
 265 #endif
 266
 267 #define SCAN_OBJECT_PROTOCOL
 268 #include "sgen-scan-object.h"
 269 }
 270
 271 static gboolean
 272 DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
 273 {
 274 #if defined(COPY_OR_MARK_CONCURRENT) || defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
 275         int i;
 276         for (i = 0; i < 32; i++) {
 277 #else
 278         for (;;) {
 279 #endif
 280                 GCObject *obj;
 281                 SgenDescriptor desc;
 282
 283                 HEAVY_STAT (++stat_drain_loops);
 284
 285                 GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
 286                 if (!obj)
 287                         return TRUE;
 288
 289                 SCAN_OBJECT_FUNCTION_NAME (obj, desc, queue);
 290         }
 291         return FALSE;
 292 }
 293
 294 #undef COPY_OR_MARK_FUNCTION_NAME
 295 #undef COPY_OR_MARK_WITH_EVACUATION
 296 #undef SCAN_OBJECT_FUNCTION_NAME
 297 #undef DRAIN_GRAY_STACK_FUNCTION_NAME