?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[ruby/irb] Change debug test workaround only enabled when output is
[ruby.git] / gc.c
blob474454e487ed7c6bb701786617d303061907fd5a
1 /**********************************************************************
2
3 gc.c -
4
5 $Author$
6 created at: Tue Oct 5 09:44:46 JST 1993
7
8 Copyright (C) 1993-2007 Yukihiro Matsumoto
9 Copyright (C) 2000 Network Applied Communication Laboratory, Inc.
10 Copyright (C) 2000 Information-technology Promotion Agency, Japan
11
12 **********************************************************************/
13
14 #define rb_data_object_alloc rb_data_object_alloc
15 #define rb_data_typed_object_alloc rb_data_typed_object_alloc
16
17 #include "ruby/internal/config.h"
18 #ifdef _WIN32
19 # include "ruby/ruby.h"
20 #endif
21
22 #include <signal.h>
23
24 #ifndef _WIN32
25 #include <unistd.h>
26 #include <sys/mman.h>
27 #endif
28
29 #if defined(__wasm__) && !defined(__EMSCRIPTEN__)
30 # include "wasm/setjmp.h"
31 # include "wasm/machine.h"
32 #else
33 # include <setjmp.h>
34 #endif
35 #include <stdarg.h>
36 #include <stdio.h>
37
38 /* MALLOC_HEADERS_BEGIN */
39 #ifndef HAVE_MALLOC_USABLE_SIZE
40 # ifdef _WIN32
41 # define HAVE_MALLOC_USABLE_SIZE
42 # define malloc_usable_size(a) _msize(a)
43 # elif defined HAVE_MALLOC_SIZE
44 # define HAVE_MALLOC_USABLE_SIZE
45 # define malloc_usable_size(a) malloc_size(a)
46 # endif
47 #endif
48
49 #ifdef HAVE_MALLOC_USABLE_SIZE
50 # ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
51 /* Alternative malloc header is included in ruby/missing.h */
52 # elif defined(HAVE_MALLOC_H)
53 # include <malloc.h>
54 # elif defined(HAVE_MALLOC_NP_H)
55 # include <malloc_np.h>
56 # elif defined(HAVE_MALLOC_MALLOC_H)
57 # include <malloc/malloc.h>
58 # endif
59 #endif
60
61 #ifdef HAVE_MALLOC_TRIM
62 # include <malloc.h>
63
64 # ifdef __EMSCRIPTEN__
65 /* malloc_trim is defined in emscripten/emmalloc.h on emscripten. */
66 # include <emscripten/emmalloc.h>
67 # endif
68 #endif
69
70 #if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
71 /* LIST_HEAD conflicts with sys/queue.h on macOS */
72 # include <sys/user.h>
73 #endif
74 /* MALLOC_HEADERS_END */
75
76 #ifdef HAVE_SYS_TIME_H
77 # include <sys/time.h>
78 #endif
79
80 #ifdef HAVE_SYS_RESOURCE_H
81 # include <sys/resource.h>
82 #endif
83
84 #if defined _WIN32 || defined __CYGWIN__
85 # include <windows.h>
86 #elif defined(HAVE_POSIX_MEMALIGN)
87 #elif defined(HAVE_MEMALIGN)
88 # include <malloc.h>
89 #endif
90
91 #include <sys/types.h>
92
93 #ifdef __EMSCRIPTEN__
94 #include <emscripten.h>
95 #endif
96
97 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
98 # include <mach/task.h>
99 # include <mach/mach_init.h>
100 # include <mach/mach_port.h>
101 #endif
102 #undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */
103
104 #include "constant.h"
105 #include "darray.h"
106 #include "debug_counter.h"
107 #include "eval_intern.h"
108 #include "id_table.h"
109 #include "internal.h"
110 #include "internal/class.h"
111 #include "internal/compile.h"
112 #include "internal/complex.h"
113 #include "internal/cont.h"
114 #include "internal/error.h"
115 #include "internal/eval.h"
116 #include "internal/gc.h"
117 #include "internal/hash.h"
118 #include "internal/imemo.h"
119 #include "internal/io.h"
120 #include "internal/numeric.h"
121 #include "internal/object.h"
122 #include "internal/proc.h"
123 #include "internal/rational.h"
124 #include "internal/sanitizers.h"
125 #include "internal/struct.h"
126 #include "internal/symbol.h"
127 #include "internal/thread.h"
128 #include "internal/variable.h"
129 #include "internal/warnings.h"
130 #include "rjit.h"
131 #include "probes.h"
132 #include "regint.h"
133 #include "ruby/debug.h"
134 #include "ruby/io.h"
135 #include "ruby/re.h"
136 #include "ruby/st.h"
137 #include "ruby/thread.h"
138 #include "ruby/util.h"
139 #include "ruby_assert.h"
140 #include "ruby_atomic.h"
141 #include "symbol.h"
142 #include "vm_core.h"
143 #include "vm_sync.h"
144 #include "vm_callinfo.h"
145 #include "ractor_core.h"
146
147 #include "builtin.h"
148 #include "shape.h"
149
150 #define rb_setjmp(env) RUBY_SETJMP(env)
151 #define rb_jmp_buf rb_jmpbuf_t
152 #undef rb_data_object_wrap
153
154 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
155 #define MAP_ANONYMOUS MAP_ANON
156 #endif
157
158
159 static size_t malloc_offset = 0;
160 #if defined(HAVE_MALLOC_USABLE_SIZE)
161 static size_t
162 gc_compute_malloc_offset(void)
163 {
164 // Different allocators use different metadata storage strategies which result in different
165 // ideal sizes.
166 // For instance malloc(64) will waste 8B with glibc, but waste 0B with jemalloc.
167 // But malloc(56) will waste 0B with glibc, but waste 8B with jemalloc.
168 // So we try allocating 64, 56 and 48 bytes and select the first offset that doesn't
169 // waste memory.
170 // This was tested on Linux with glibc 2.35 and jemalloc 5, and for both it result in
171 // no wasted memory.
172 size_t offset = 0;
173 for (offset = 0; offset <= 16; offset += 8) {
174 size_t allocated = (64 - offset);
175 void *test_ptr = malloc(allocated);
176 size_t wasted = malloc_usable_size(test_ptr) - allocated;
177 free(test_ptr);
178
179 if (wasted == 0) {
180 return offset;
181 }
182 }
183 return 0;
184 }
185 #else
186 static size_t
187 gc_compute_malloc_offset(void)
188 {
189 // If we don't have malloc_usable_size, we use powers of 2.
190 return 0;
191 }
192 #endif
193
194 size_t
195 rb_malloc_grow_capa(size_t current, size_t type_size)
196 {
197 size_t current_capacity = current;
198 if (current_capacity < 4) {
199 current_capacity = 4;
200 }
201 current_capacity *= type_size;
202
203 // We double the current capacity.
204 size_t new_capacity = (current_capacity * 2);
205
206 // And round up to the next power of 2 if it's not already one.
207 if (rb_popcount64(new_capacity) != 1) {
208 new_capacity = (size_t)(1 << (64 - nlz_int64(new_capacity)));
209 }
210
211 new_capacity -= malloc_offset;
212 new_capacity /= type_size;
213 if (current > new_capacity) {
214 rb_bug("rb_malloc_grow_capa: current_capacity=%zu, new_capacity=%zu, malloc_offset=%zu", current, new_capacity, malloc_offset);
215 }
216 RUBY_ASSERT(new_capacity > current);
217 return new_capacity;
218 }
219
220 static inline struct rbimpl_size_mul_overflow_tag
221 size_add_overflow(size_t x, size_t y)
222 {
223 size_t z;
224 bool p;
225 #if 0
226
227 #elif defined(ckd_add)
228 p = ckd_add(&z, x, y);
229
230 #elif __has_builtin(__builtin_add_overflow)
231 p = __builtin_add_overflow(x, y, &z);
232
233 #elif defined(DSIZE_T)
234 RB_GNUC_EXTENSION DSIZE_T dx = x;
235 RB_GNUC_EXTENSION DSIZE_T dy = y;
236 RB_GNUC_EXTENSION DSIZE_T dz = dx + dy;
237 p = dz > SIZE_MAX;
238 z = (size_t)dz;
239
240 #else
241 z = x + y;
242 p = z < y;
243
244 #endif
245 return (struct rbimpl_size_mul_overflow_tag) { p, z, };
246 }
247
248 static inline struct rbimpl_size_mul_overflow_tag
249 size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */
250 {
251 struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
252 struct rbimpl_size_mul_overflow_tag u = size_add_overflow(t.right, z);
253 return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left, u.right };
254 }
255
256 static inline struct rbimpl_size_mul_overflow_tag
257 size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
258 {
259 struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
260 struct rbimpl_size_mul_overflow_tag u = rbimpl_size_mul_overflow(z, w);
261 struct rbimpl_size_mul_overflow_tag v = size_add_overflow(t.right, u.right);
262 return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left || v.left, v.right };
263 }
264
265 PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3);
266
267 static inline size_t
268 size_mul_or_raise(size_t x, size_t y, VALUE exc)
269 {
270 struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
271 if (LIKELY(!t.left)) {
272 return t.right;
273 }
274 else if (rb_during_gc()) {
275 rb_memerror(); /* or...? */
276 }
277 else {
278 gc_raise(
279 exc,
280 "integer overflow: %"PRIuSIZE
281 " * %"PRIuSIZE
282 " > %"PRIuSIZE,
283 x, y, (size_t)SIZE_MAX);
284 }
285 }
286
287 size_t
288 rb_size_mul_or_raise(size_t x, size_t y, VALUE exc)
289 {
290 return size_mul_or_raise(x, y, exc);
291 }
292
293 static inline size_t
294 size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
295 {
296 struct rbimpl_size_mul_overflow_tag t = size_mul_add_overflow(x, y, z);
297 if (LIKELY(!t.left)) {
298 return t.right;
299 }
300 else if (rb_during_gc()) {
301 rb_memerror(); /* or...? */
302 }
303 else {
304 gc_raise(
305 exc,
306 "integer overflow: %"PRIuSIZE
307 " * %"PRIuSIZE
308 " + %"PRIuSIZE
309 " > %"PRIuSIZE,
310 x, y, z, (size_t)SIZE_MAX);
311 }
312 }
313
314 size_t
315 rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
316 {
317 return size_mul_add_or_raise(x, y, z, exc);
318 }
319
320 static inline size_t
321 size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc)
322 {
323 struct rbimpl_size_mul_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w);
324 if (LIKELY(!t.left)) {
325 return t.right;
326 }
327 else if (rb_during_gc()) {
328 rb_memerror(); /* or...? */
329 }
330 else {
331 gc_raise(
332 exc,
333 "integer overflow: %"PRIdSIZE
334 " * %"PRIdSIZE
335 " + %"PRIdSIZE
336 " * %"PRIdSIZE
337 " > %"PRIdSIZE,
338 x, y, z, w, (size_t)SIZE_MAX);
339 }
340 }
341
342 #if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL
343 /* trick the compiler into thinking a external signal handler uses this */
344 volatile VALUE rb_gc_guarded_val;
345 volatile VALUE *
346 rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val)
347 {
348 rb_gc_guarded_val = val;
349
350 return ptr;
351 }
352 #endif
353
354 #ifndef GC_HEAP_INIT_SLOTS
355 #define GC_HEAP_INIT_SLOTS 10000
356 #endif
357 #ifndef GC_HEAP_FREE_SLOTS
358 #define GC_HEAP_FREE_SLOTS 4096
359 #endif
360 #ifndef GC_HEAP_GROWTH_FACTOR
361 #define GC_HEAP_GROWTH_FACTOR 1.8
362 #endif
363 #ifndef GC_HEAP_GROWTH_MAX_SLOTS
364 #define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */
365 #endif
366 #ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
367 # define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
368 #endif
369 #ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
370 #define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
371 #endif
372
373 #ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
374 #define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
375 #endif
376 #ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
377 #define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
378 #endif
379 #ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
380 #define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
381 #endif
382
383 #ifndef GC_MALLOC_LIMIT_MIN
384 #define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
385 #endif
386 #ifndef GC_MALLOC_LIMIT_MAX
387 #define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
388 #endif
389 #ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
390 #define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
391 #endif
392
393 #ifndef GC_OLDMALLOC_LIMIT_MIN
394 #define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
395 #endif
396 #ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
397 #define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
398 #endif
399 #ifndef GC_OLDMALLOC_LIMIT_MAX
400 #define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
401 #endif
402
403 #ifndef GC_CAN_COMPILE_COMPACTION
404 #if defined(__wasi__) /* WebAssembly doesn't support signals */
405 # define GC_CAN_COMPILE_COMPACTION 0
406 #else
407 # define GC_CAN_COMPILE_COMPACTION 1
408 #endif
409 #endif
410
411 #ifndef PRINT_MEASURE_LINE
412 #define PRINT_MEASURE_LINE 0
413 #endif
414 #ifndef PRINT_ENTER_EXIT_TICK
415 #define PRINT_ENTER_EXIT_TICK 0
416 #endif
417 #ifndef PRINT_ROOT_TICKS
418 #define PRINT_ROOT_TICKS 0
419 #endif
420
421 #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_MEASURE_LINE || PRINT_ROOT_TICKS)
422
423 typedef struct {
424 size_t size_pool_init_slots[SIZE_POOL_COUNT];
425 size_t heap_free_slots;
426 double growth_factor;
427 size_t growth_max_slots;
428
429 double heap_free_slots_min_ratio;
430 double heap_free_slots_goal_ratio;
431 double heap_free_slots_max_ratio;
432 double uncollectible_wb_unprotected_objects_limit_ratio;
433 double oldobject_limit_factor;
434
435 size_t malloc_limit_min;
436 size_t malloc_limit_max;
437 double malloc_limit_growth_factor;
438
439 size_t oldmalloc_limit_min;
440 size_t oldmalloc_limit_max;
441 double oldmalloc_limit_growth_factor;
442 } ruby_gc_params_t;
443
444 static ruby_gc_params_t gc_params = {
445 { 0 },
446 GC_HEAP_FREE_SLOTS,
447 GC_HEAP_GROWTH_FACTOR,
448 GC_HEAP_GROWTH_MAX_SLOTS,
449
450 GC_HEAP_FREE_SLOTS_MIN_RATIO,
451 GC_HEAP_FREE_SLOTS_GOAL_RATIO,
452 GC_HEAP_FREE_SLOTS_MAX_RATIO,
453 GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
454 GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
455
456 GC_MALLOC_LIMIT_MIN,
457 GC_MALLOC_LIMIT_MAX,
458 GC_MALLOC_LIMIT_GROWTH_FACTOR,
459
460 GC_OLDMALLOC_LIMIT_MIN,
461 GC_OLDMALLOC_LIMIT_MAX,
462 GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
463 };
464
465 /* GC_DEBUG:
466 * enable to embed GC debugging information.
467 */
468 #ifndef GC_DEBUG
469 #define GC_DEBUG 0
470 #endif
471
472 /* RGENGC_DEBUG:
473 * 1: basic information
474 * 2: remember set operation
475 * 3: mark
476 * 4:
477 * 5: sweep
478 */
479 #ifndef RGENGC_DEBUG
480 #ifdef RUBY_DEVEL
481 #define RGENGC_DEBUG -1
482 #else
483 #define RGENGC_DEBUG 0
484 #endif
485 #endif
486 #if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
487 # define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
488 #elif defined(HAVE_VA_ARGS_MACRO)
489 # define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
490 #else
491 # define RGENGC_DEBUG_ENABLED(level) 0
492 #endif
493 int ruby_rgengc_debug;
494
495 /* RGENGC_CHECK_MODE
496 * 0: disable all assertions
497 * 1: enable assertions (to debug RGenGC)
498 * 2: enable internal consistency check at each GC (for debugging)
499 * 3: enable internal consistency check at each GC steps (for debugging)
500 * 4: enable liveness check
501 * 5: show all references
502 */
503 #ifndef RGENGC_CHECK_MODE
504 #define RGENGC_CHECK_MODE 0
505 #endif
506
507 // Note: using RUBY_ASSERT_WHEN() extend a macro in expr (info by nobu).
508 #define GC_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr)
509
510 /* RGENGC_PROFILE
511 * 0: disable RGenGC profiling
512 * 1: enable profiling for basic information
513 * 2: enable profiling for each types
514 */
515 #ifndef RGENGC_PROFILE
516 #define RGENGC_PROFILE 0
517 #endif
518
519 /* RGENGC_ESTIMATE_OLDMALLOC
520 * Enable/disable to estimate increase size of malloc'ed size by old objects.
521 * If estimation exceeds threshold, then will invoke full GC.
522 * 0: disable estimation.
523 * 1: enable estimation.
524 */
525 #ifndef RGENGC_ESTIMATE_OLDMALLOC
526 #define RGENGC_ESTIMATE_OLDMALLOC 1
527 #endif
528
529 /* RGENGC_FORCE_MAJOR_GC
530 * Force major/full GC if this macro is not 0.
531 */
532 #ifndef RGENGC_FORCE_MAJOR_GC
533 #define RGENGC_FORCE_MAJOR_GC 0
534 #endif
535
536 #ifndef GC_PROFILE_MORE_DETAIL
537 #define GC_PROFILE_MORE_DETAIL 0
538 #endif
539 #ifndef GC_PROFILE_DETAIL_MEMORY
540 #define GC_PROFILE_DETAIL_MEMORY 0
541 #endif
542 #ifndef GC_ENABLE_LAZY_SWEEP
543 #define GC_ENABLE_LAZY_SWEEP 1
544 #endif
545 #ifndef CALC_EXACT_MALLOC_SIZE
546 #define CALC_EXACT_MALLOC_SIZE USE_GC_MALLOC_OBJ_INFO_DETAILS
547 #endif
548 #if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
549 #ifndef MALLOC_ALLOCATED_SIZE
550 #define MALLOC_ALLOCATED_SIZE 0
551 #endif
552 #else
553 #define MALLOC_ALLOCATED_SIZE 0
554 #endif
555 #ifndef MALLOC_ALLOCATED_SIZE_CHECK
556 #define MALLOC_ALLOCATED_SIZE_CHECK 0
557 #endif
558
559 #ifndef GC_DEBUG_STRESS_TO_CLASS
560 #define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
561 #endif
562
563 #ifndef RGENGC_OBJ_INFO
564 #define RGENGC_OBJ_INFO (RGENGC_DEBUG | RGENGC_CHECK_MODE)
565 #endif
566
567 typedef enum {
568 GPR_FLAG_NONE = 0x000,
569 /* major reason */
570 GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
571 GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
572 GPR_FLAG_MAJOR_BY_SHADY = 0x004,
573 GPR_FLAG_MAJOR_BY_FORCE = 0x008,
574 #if RGENGC_ESTIMATE_OLDMALLOC
575 GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
576 #endif
577 GPR_FLAG_MAJOR_MASK = 0x0ff,
578
579 /* gc reason */
580 GPR_FLAG_NEWOBJ = 0x100,
581 GPR_FLAG_MALLOC = 0x200,
582 GPR_FLAG_METHOD = 0x400,
583 GPR_FLAG_CAPI = 0x800,
584 GPR_FLAG_STRESS = 0x1000,
585
586 /* others */
587 GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
588 GPR_FLAG_HAVE_FINALIZE = 0x4000,
589 GPR_FLAG_IMMEDIATE_MARK = 0x8000,
590 GPR_FLAG_FULL_MARK = 0x10000,
591 GPR_FLAG_COMPACT = 0x20000,
592
593 GPR_DEFAULT_REASON =
594 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
595 GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
596 } gc_profile_record_flag;
597
598 typedef struct gc_profile_record {
599 unsigned int flags;
600
601 double gc_time;
602 double gc_invoke_time;
603
604 size_t heap_total_objects;
605 size_t heap_use_size;
606 size_t heap_total_size;
607 size_t moved_objects;
608
609 #if GC_PROFILE_MORE_DETAIL
610 double gc_mark_time;
611 double gc_sweep_time;
612
613 size_t heap_use_pages;
614 size_t heap_live_objects;
615 size_t heap_free_objects;
616
617 size_t allocate_increase;
618 size_t allocate_limit;
619
620 double prepare_time;
621 size_t removing_objects;
622 size_t empty_objects;
623 #if GC_PROFILE_DETAIL_MEMORY
624 long maxrss;
625 long minflt;
626 long majflt;
627 #endif
628 #endif
629 #if MALLOC_ALLOCATED_SIZE
630 size_t allocated_size;
631 #endif
632
633 #if RGENGC_PROFILE > 0
634 size_t old_objects;
635 size_t remembered_normal_objects;
636 size_t remembered_shady_objects;
637 #endif
638 } gc_profile_record;
639
640 struct RMoved {
641 VALUE flags;
642 VALUE dummy;
643 VALUE destination;
644 shape_id_t original_shape_id;
645 };
646
647 #define RMOVED(obj) ((struct RMoved *)(obj))
648
649 typedef struct RVALUE {
650 union {
651 struct {
652 VALUE flags; /* always 0 for freed obj */
653 struct RVALUE *next;
654 } free;
655 struct RMoved moved;
656 struct RBasic basic;
657 struct RObject object;
658 struct RClass klass;
659 struct RFloat flonum;
660 struct RString string;
661 struct RArray array;
662 struct RRegexp regexp;
663 struct RHash hash;
664 struct RData data;
665 struct RTypedData typeddata;
666 struct RStruct rstruct;
667 struct RBignum bignum;
668 struct RFile file;
669 struct RMatch match;
670 struct RRational rational;
671 struct RComplex complex;
672 struct RSymbol symbol;
673 union {
674 rb_cref_t cref;
675 struct vm_svar svar;
676 struct vm_throw_data throw_data;
677 struct vm_ifunc ifunc;
678 struct MEMO memo;
679 struct rb_method_entry_struct ment;
680 const rb_iseq_t iseq;
681 rb_env_t env;
682 struct rb_imemo_tmpbuf_struct alloc;
683 rb_ast_t ast;
684 } imemo;
685 struct {
686 struct RBasic basic;
687 VALUE v1;
688 VALUE v2;
689 VALUE v3;
690 } values;
691 } as;
692 } RVALUE;
693
694 /* These members ae located at the end of the slot that the object is in. */
695 #if RACTOR_CHECK_MODE || GC_DEBUG
696 struct rvalue_overhead {
697 # if RACTOR_CHECK_MODE
698 uint32_t _ractor_belonging_id;
699 # endif
700 # if GC_DEBUG
701 const char *file;
702 int line;
703 # endif
704 };
705
706 // Make sure that RVALUE_OVERHEAD aligns to sizeof(VALUE)
707 # define RVALUE_OVERHEAD (sizeof(struct { \
708 union { \
709 struct rvalue_overhead overhead; \
710 VALUE value; \
711 }; \
712 }))
713 # define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_obj_slot_size(obj)))
714 #else
715 # define RVALUE_OVERHEAD 0
716 #endif
717
718 STATIC_ASSERT(sizeof_rvalue, sizeof(RVALUE) == (SIZEOF_VALUE * 5));
719 STATIC_ASSERT(alignof_rvalue, RUBY_ALIGNOF(RVALUE) == SIZEOF_VALUE);
720
721 typedef uintptr_t bits_t;
722 enum {
723 BITS_SIZE = sizeof(bits_t),
724 BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
725 };
726
727 struct heap_page_header {
728 struct heap_page *page;
729 };
730
731 struct heap_page_body {
732 struct heap_page_header header;
733 /* char gap[]; */
734 /* RVALUE values[]; */
735 };
736
737 #define STACK_CHUNK_SIZE 500
738
739 typedef struct stack_chunk {
740 VALUE data[STACK_CHUNK_SIZE];
741 struct stack_chunk *next;
742 } stack_chunk_t;
743
744 typedef struct mark_stack {
745 stack_chunk_t *chunk;
746 stack_chunk_t *cache;
747 int index;
748 int limit;
749 size_t cache_size;
750 size_t unused_cache_size;
751 } mark_stack_t;
752
753 #define SIZE_POOL_EDEN_HEAP(size_pool) (&(size_pool)->eden_heap)
754 #define SIZE_POOL_TOMB_HEAP(size_pool) (&(size_pool)->tomb_heap)
755
756 typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d);
757
758 typedef struct rb_heap_struct {
759 struct heap_page *free_pages;
760 struct ccan_list_head pages;
761 struct heap_page *sweeping_page; /* iterator for .pages */
762 struct heap_page *compact_cursor;
763 uintptr_t compact_cursor_index;
764 struct heap_page *pooled_pages;
765 size_t total_pages; /* total page count in a heap */
766 size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */
767 } rb_heap_t;
768
769 typedef struct rb_size_pool_struct {
770 short slot_size;
771
772 size_t allocatable_pages;
773
774 /* Basic statistics */
775 size_t total_allocated_pages;
776 size_t total_freed_pages;
777 size_t force_major_gc_count;
778 size_t force_incremental_marking_finish_count;
779 size_t total_allocated_objects;
780 size_t total_freed_objects;
781
782 /* Sweeping statistics */
783 size_t freed_slots;
784 size_t empty_slots;
785
786 rb_heap_t eden_heap;
787 rb_heap_t tomb_heap;
788 } rb_size_pool_t;
789
790 enum gc_mode {
791 gc_mode_none,
792 gc_mode_marking,
793 gc_mode_sweeping,
794 gc_mode_compacting,
795 };
796
797 typedef struct rb_objspace {
798 struct {
799 size_t limit;
800 size_t increase;
801 #if MALLOC_ALLOCATED_SIZE
802 size_t allocated_size;
803 size_t allocations;
804 #endif
805
806 } malloc_params;
807
808 struct {
809 unsigned int mode : 2;
810 unsigned int immediate_sweep : 1;
811 unsigned int dont_gc : 1;
812 unsigned int dont_incremental : 1;
813 unsigned int during_gc : 1;
814 unsigned int during_compacting : 1;
815 unsigned int during_reference_updating : 1;
816 unsigned int gc_stressful: 1;
817 unsigned int has_newobj_hook: 1;
818 unsigned int during_minor_gc : 1;
819 unsigned int during_incremental_marking : 1;
820 unsigned int measure_gc : 1;
821 } flags;
822
823 rb_event_flag_t hook_events;
824 unsigned long long next_object_id;
825
826 rb_size_pool_t size_pools[SIZE_POOL_COUNT];
827
828 struct {
829 rb_atomic_t finalizing;
830 } atomic_flags;
831
832 mark_stack_t mark_stack;
833 size_t marked_slots;
834
835 struct {
836 struct heap_page **sorted;
837 size_t allocated_pages;
838 size_t allocatable_pages;
839 size_t sorted_length;
840 uintptr_t range[2];
841 size_t freeable_pages;
842
843 /* final */
844 size_t final_slots;
845 VALUE deferred_final;
846 } heap_pages;
847
848 st_table *finalizer_table;
849
850 struct {
851 int run;
852 unsigned int latest_gc_info;
853 gc_profile_record *records;
854 gc_profile_record *current_record;
855 size_t next_index;
856 size_t size;
857
858 #if GC_PROFILE_MORE_DETAIL
859 double prepare_time;
860 #endif
861 double invoke_time;
862
863 size_t minor_gc_count;
864 size_t major_gc_count;
865 size_t compact_count;
866 size_t read_barrier_faults;
867 #if RGENGC_PROFILE > 0
868 size_t total_generated_normal_object_count;
869 size_t total_generated_shady_object_count;
870 size_t total_shade_operation_count;
871 size_t total_promoted_count;
872 size_t total_remembered_normal_object_count;
873 size_t total_remembered_shady_object_count;
874
875 #if RGENGC_PROFILE >= 2
876 size_t generated_normal_object_count_types[RUBY_T_MASK];
877 size_t generated_shady_object_count_types[RUBY_T_MASK];
878 size_t shade_operation_count_types[RUBY_T_MASK];
879 size_t promoted_types[RUBY_T_MASK];
880 size_t remembered_normal_object_count_types[RUBY_T_MASK];
881 size_t remembered_shady_object_count_types[RUBY_T_MASK];
882 #endif
883 #endif /* RGENGC_PROFILE */
884
885 /* temporary profiling space */
886 double gc_sweep_start_time;
887 size_t total_allocated_objects_at_gc_start;
888 size_t heap_used_at_gc_start;
889
890 /* basic statistics */
891 size_t count;
892 uint64_t marking_time_ns;
893 struct timespec marking_start_time;
894 uint64_t sweeping_time_ns;
895 struct timespec sweeping_start_time;
896
897 /* Weak references */
898 size_t weak_references_count;
899 size_t retained_weak_references_count;
900 } profile;
901
902 VALUE gc_stress_mode;
903
904 struct {
905 VALUE parent_object;
906 int need_major_gc;
907 size_t last_major_gc;
908 size_t uncollectible_wb_unprotected_objects;
909 size_t uncollectible_wb_unprotected_objects_limit;
910 size_t old_objects;
911 size_t old_objects_limit;
912
913 #if RGENGC_ESTIMATE_OLDMALLOC
914 size_t oldmalloc_increase;
915 size_t oldmalloc_increase_limit;
916 #endif
917
918 #if RGENGC_CHECK_MODE >= 2
919 struct st_table *allrefs_table;
920 size_t error_count;
921 #endif
922 } rgengc;
923
924 struct {
925 size_t considered_count_table[T_MASK];
926 size_t moved_count_table[T_MASK];
927 size_t moved_up_count_table[T_MASK];
928 size_t moved_down_count_table[T_MASK];
929 size_t total_moved;
930
931 /* This function will be used, if set, to sort the heap prior to compaction */
932 gc_compact_compare_func compare_func;
933 } rcompactor;
934
935 struct {
936 size_t pooled_slots;
937 size_t step_slots;
938 } rincgc;
939
940 st_table *id_to_obj_tbl;
941 st_table *obj_to_id_tbl;
942
943 #if GC_DEBUG_STRESS_TO_CLASS
944 VALUE stress_to_class;
945 #endif
946
947 rb_darray(VALUE *) weak_references;
948 rb_postponed_job_handle_t finalize_deferred_pjob;
949
950 #ifdef RUBY_ASAN_ENABLED
951 const rb_execution_context_t *marking_machine_context_ec;
952 #endif
953
954 } rb_objspace_t;
955
956
957 #ifndef HEAP_PAGE_ALIGN_LOG
958 /* default tiny heap size: 64KiB */
959 #define HEAP_PAGE_ALIGN_LOG 16
960 #endif
961
962 #define BASE_SLOT_SIZE (sizeof(RVALUE) + RVALUE_OVERHEAD)
963
964 #define CEILDIV(i, mod) roomof(i, mod)
965 enum {
966 HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
967 HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
968 HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
969 HEAP_PAGE_OBJ_LIMIT = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header)) / BASE_SLOT_SIZE),
970 HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, BASE_SLOT_SIZE), BITS_BITLENGTH),
971 HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
972 };
973 #define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
974 #define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
975
976 #if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
977 # define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
978 #endif
979
980 #undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
981 /* Must define either HEAP_PAGE_ALLOC_USE_MMAP or
982 * INIT_HEAP_PAGE_ALLOC_USE_MMAP. */
983
984 #ifndef HAVE_MMAP
985 /* We can't use mmap of course, if it is not available. */
986 static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
987
988 #elif defined(__wasm__)
989 /* wasmtime does not have proper support for mmap.
990 * See https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-rationale.md#why-no-mmap-and-friends
991 */
992 static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
993
994 #elif HAVE_CONST_PAGE_SIZE
995 /* If we have the PAGE_SIZE and it is a constant, then we can directly use it. */
996 static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
997
998 #elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
999 /* If we can use the maximum page size. */
1000 static const bool HEAP_PAGE_ALLOC_USE_MMAP = true;
1001
1002 #elif defined(PAGE_SIZE)
1003 /* If the PAGE_SIZE macro can be used dynamically. */
1004 # define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
1005
1006 #elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
1007 /* If we can use sysconf to determine the page size. */
1008 # define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
1009
1010 #else
1011 /* Otherwise we can't determine the system page size, so don't use mmap. */
1012 static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
1013 #endif
1014
1015 #ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
1016 /* We can determine the system page size at runtime. */
1017 # define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
1018
1019 static bool heap_page_alloc_use_mmap;
1020 #endif
1021
1022 #define RVALUE_AGE_BIT_COUNT 2
1023 #define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
1024
1025 struct heap_page {
1026 short slot_size;
1027 short total_slots;
1028 short free_slots;
1029 short final_slots;
1030 short pinned_slots;
1031 struct {
1032 unsigned int before_sweep : 1;
1033 unsigned int has_remembered_objects : 1;
1034 unsigned int has_uncollectible_wb_unprotected_objects : 1;
1035 unsigned int in_tomb : 1;
1036 } flags;
1037
1038 rb_size_pool_t *size_pool;
1039
1040 struct heap_page *free_next;
1041 uintptr_t start;
1042 RVALUE *freelist;
1043 struct ccan_list_node page_node;
1044
1045 bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
1046 /* the following three bitmaps are cleared at the beginning of full GC */
1047 bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
1048 bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
1049 bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
1050
1051 bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
1052
1053 /* If set, the object is not movable */
1054 bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
1055 bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
1056 };
1057
1058 /*
1059 * When asan is enabled, this will prohibit writing to the freelist until it is unlocked
1060 */
1061 static void
1062 asan_lock_freelist(struct heap_page *page)
1063 {
1064 asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
1065 }
1066
1067 /*
1068 * When asan is enabled, this will enable the ability to write to the freelist
1069 */
1070 static void
1071 asan_unlock_freelist(struct heap_page *page)
1072 {
1073 asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
1074 }
1075
1076 #define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
1077 #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
1078 #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
1079
1080 #define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE)
1081 #define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
1082 #define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
1083 #define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
1084
1085 /* Bitmap Operations */
1086 #define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
1087 #define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
1088 #define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
1089
1090 /* getting bitmap */
1091 #define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
1092 #define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
1093 #define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
1094 #define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
1095 #define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
1096
1097 #define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3
1098
1099 #define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT))
1100 #define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT)
1101
1102 #define RVALUE_OLD_AGE 3
1103
1104 static int
1105 RVALUE_AGE_GET(VALUE obj)
1106 {
1107 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
1108 return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK;
1109 }
1110
1111 static void
1112 RVALUE_AGE_SET(VALUE obj, int age)
1113 {
1114 RUBY_ASSERT(age <= RVALUE_OLD_AGE);
1115 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
1116 // clear the bits
1117 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj)));
1118 // shift the correct value in
1119 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj));
1120 if (age == RVALUE_OLD_AGE) {
1121 RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED);
1122 }
1123 else {
1124 RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED);
1125 }
1126 }
1127
1128 /* Aliases */
1129 #define rb_objspace (*rb_objspace_of(GET_VM()))
1130 #define rb_objspace_of(vm) ((vm)->objspace)
1131 #define unless_objspace(objspace) \
1132 rb_objspace_t *objspace; \
1133 rb_vm_t *unless_objspace_vm = GET_VM(); \
1134 if (unless_objspace_vm) objspace = unless_objspace_vm->objspace; \
1135 else /* return; or objspace will be warned uninitialized */
1136
1137 #define malloc_limit objspace->malloc_params.limit
1138 #define malloc_increase objspace->malloc_params.increase
1139 #define malloc_allocated_size objspace->malloc_params.allocated_size
1140 #define heap_pages_sorted objspace->heap_pages.sorted
1141 #define heap_allocated_pages objspace->heap_pages.allocated_pages
1142 #define heap_pages_sorted_length objspace->heap_pages.sorted_length
1143 #define heap_pages_lomem objspace->heap_pages.range[0]
1144 #define heap_pages_himem objspace->heap_pages.range[1]
1145 #define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
1146 #define heap_pages_final_slots objspace->heap_pages.final_slots
1147 #define heap_pages_deferred_final objspace->heap_pages.deferred_final
1148 #define size_pools objspace->size_pools
1149 #define during_gc objspace->flags.during_gc
1150 #define finalizing objspace->atomic_flags.finalizing
1151 #define finalizer_table objspace->finalizer_table
1152 #define ruby_gc_stressful objspace->flags.gc_stressful
1153 #define ruby_gc_stress_mode objspace->gc_stress_mode
1154 #if GC_DEBUG_STRESS_TO_CLASS
1155 #define stress_to_class objspace->stress_to_class
1156 #define set_stress_to_class(c) (stress_to_class = (c))
1157 #else
1158 #define stress_to_class (objspace, 0)
1159 #define set_stress_to_class(c) (objspace, (c))
1160 #endif
1161
1162 #if 0
1163 #define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
1164 #define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
1165 #define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), (int)b), objspace->flags.dont_gc = (b))
1166 #define dont_gc_val() (objspace->flags.dont_gc)
1167 #else
1168 #define dont_gc_on() (objspace->flags.dont_gc = 1)
1169 #define dont_gc_off() (objspace->flags.dont_gc = 0)
1170 #define dont_gc_set(b) (((int)b), objspace->flags.dont_gc = (b))
1171 #define dont_gc_val() (objspace->flags.dont_gc)
1172 #endif
1173
1174 static inline enum gc_mode
1175 gc_mode_verify(enum gc_mode mode)
1176 {
1177 #if RGENGC_CHECK_MODE > 0
1178 switch (mode) {
1179 case gc_mode_none:
1180 case gc_mode_marking:
1181 case gc_mode_sweeping:
1182 case gc_mode_compacting:
1183 break;
1184 default:
1185 rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
1186 }
1187 #endif
1188 return mode;
1189 }
1190
1191 static inline bool
1192 has_sweeping_pages(rb_objspace_t *objspace)
1193 {
1194 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1195 if (SIZE_POOL_EDEN_HEAP(&size_pools[i])->sweeping_page) {
1196 return TRUE;
1197 }
1198 }
1199 return FALSE;
1200 }
1201
1202 static inline size_t
1203 heap_eden_total_pages(rb_objspace_t *objspace)
1204 {
1205 size_t count = 0;
1206 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1207 count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_pages;
1208 }
1209 return count;
1210 }
1211
1212 static inline size_t
1213 heap_eden_total_slots(rb_objspace_t *objspace)
1214 {
1215 size_t count = 0;
1216 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1217 count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_slots;
1218 }
1219 return count;
1220 }
1221
1222 static inline size_t
1223 heap_tomb_total_pages(rb_objspace_t *objspace)
1224 {
1225 size_t count = 0;
1226 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1227 count += SIZE_POOL_TOMB_HEAP(&size_pools[i])->total_pages;
1228 }
1229 return count;
1230 }
1231
1232 static inline size_t
1233 heap_allocatable_pages(rb_objspace_t *objspace)
1234 {
1235 size_t count = 0;
1236 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1237 count += size_pools[i].allocatable_pages;
1238 }
1239 return count;
1240 }
1241
1242 static inline size_t
1243 heap_allocatable_slots(rb_objspace_t *objspace)
1244 {
1245 size_t count = 0;
1246 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1247 rb_size_pool_t *size_pool = &size_pools[i];
1248 int slot_size_multiple = size_pool->slot_size / BASE_SLOT_SIZE;
1249 count += size_pool->allocatable_pages * HEAP_PAGE_OBJ_LIMIT / slot_size_multiple;
1250 }
1251 return count;
1252 }
1253
1254 static inline size_t
1255 total_allocated_pages(rb_objspace_t *objspace)
1256 {
1257 size_t count = 0;
1258 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1259 rb_size_pool_t *size_pool = &size_pools[i];
1260 count += size_pool->total_allocated_pages;
1261 }
1262 return count;
1263 }
1264
1265 static inline size_t
1266 total_freed_pages(rb_objspace_t *objspace)
1267 {
1268 size_t count = 0;
1269 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1270 rb_size_pool_t *size_pool = &size_pools[i];
1271 count += size_pool->total_freed_pages;
1272 }
1273 return count;
1274 }
1275
1276 static inline size_t
1277 total_allocated_objects(rb_objspace_t *objspace)
1278 {
1279 size_t count = 0;
1280 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1281 rb_size_pool_t *size_pool = &size_pools[i];
1282 count += size_pool->total_allocated_objects;
1283 }
1284 return count;
1285 }
1286
1287 static inline size_t
1288 total_freed_objects(rb_objspace_t *objspace)
1289 {
1290 size_t count = 0;
1291 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1292 rb_size_pool_t *size_pool = &size_pools[i];
1293 count += size_pool->total_freed_objects;
1294 }
1295 return count;
1296 }
1297
1298 #define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
1299 #define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
1300 #define gc_needs_major_flags objspace->rgengc.need_major_gc
1301
1302 #define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
1303 #define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
1304 #define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
1305 #define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
1306 #define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
1307 #define GC_INCREMENTAL_SWEEP_SLOT_COUNT 2048
1308 #define GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT 1024
1309 #define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
1310
1311 #if SIZEOF_LONG == SIZEOF_VOIDP
1312 # define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
1313 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
1314 # define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
1315 ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
1316 #else
1317 # error not supported
1318 #endif
1319
1320 #define RANY(o) ((RVALUE*)(o))
1321
1322 struct RZombie {
1323 struct RBasic basic;
1324 VALUE next;
1325 void (*dfree)(void *);
1326 void *data;
1327 };
1328
1329 #define RZOMBIE(o) ((struct RZombie *)(o))
1330
1331 #define nomem_error GET_VM()->special_exceptions[ruby_error_nomemory]
1332
1333 #if RUBY_MARK_FREE_DEBUG
1334 int ruby_gc_debug_indent = 0;
1335 #endif
1336 VALUE rb_mGC;
1337 int ruby_disable_gc = 0;
1338 int ruby_enable_autocompact = 0;
1339 #if RGENGC_CHECK_MODE
1340 gc_compact_compare_func ruby_autocompact_compare_func;
1341 #endif
1342
1343 void rb_vm_update_references(void *ptr);
1344
1345 void rb_gcdebug_print_obj_condition(VALUE obj);
1346
1347 NORETURN(static void *gc_vraise(void *ptr));
1348 NORETURN(static void gc_raise(VALUE exc, const char *fmt, ...));
1349 NORETURN(static void negative_size_allocation_error(const char *));
1350
1351 static void init_mark_stack(mark_stack_t *stack);
1352 static int garbage_collect(rb_objspace_t *, unsigned int reason);
1353
1354 static int gc_start(rb_objspace_t *objspace, unsigned int reason);
1355 static void gc_rest(rb_objspace_t *objspace);
1356
1357 enum gc_enter_event {
1358 gc_enter_event_start,
1359 gc_enter_event_continue,
1360 gc_enter_event_rest,
1361 gc_enter_event_finalizer,
1362 gc_enter_event_rb_memerror,
1363 };
1364
1365 static inline void gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
1366 static inline void gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
1367 static void gc_marking_enter(rb_objspace_t *objspace);
1368 static void gc_marking_exit(rb_objspace_t *objspace);
1369 static void gc_sweeping_enter(rb_objspace_t *objspace);
1370 static void gc_sweeping_exit(rb_objspace_t *objspace);
1371 static bool gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap);
1372
1373 static void gc_sweep(rb_objspace_t *objspace);
1374 static void gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool);
1375 static void gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap);
1376
1377 static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
1378 static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
1379 static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
1380 NO_SANITIZE("memory", static void gc_mark_maybe(rb_objspace_t *objspace, VALUE ptr));
1381
1382 static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
1383 NO_SANITIZE("memory", static inline int is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr));
1384
1385 static size_t obj_memsize_of(VALUE obj, int use_all_types);
1386 static void gc_verify_internal_consistency(rb_objspace_t *objspace);
1387
1388 static VALUE gc_disable_no_rest(rb_objspace_t *);
1389
1390 static double getrusage_time(void);
1391 static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason);
1392 static inline void gc_prof_timer_start(rb_objspace_t *);
1393 static inline void gc_prof_timer_stop(rb_objspace_t *);
1394 static inline void gc_prof_mark_timer_start(rb_objspace_t *);
1395 static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
1396 static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
1397 static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
1398 static inline void gc_prof_set_malloc_info(rb_objspace_t *);
1399 static inline void gc_prof_set_heap_info(rb_objspace_t *);
1400
1401 #define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \
1402 if (gc_object_moved_p((_objspace), (VALUE)(_thing))) { \
1403 *(_type *)&(_thing) = (_type)RMOVED(_thing)->destination; \
1404 } \
1405 } while (0)
1406
1407 #define UPDATE_IF_MOVED(_objspace, _thing) TYPED_UPDATE_IF_MOVED(_objspace, VALUE, _thing)
1408
1409 #define gc_prof_record(objspace) (objspace)->profile.current_record
1410 #define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
1411
1412 #ifdef HAVE_VA_ARGS_MACRO
1413 # define gc_report(level, objspace, ...) \
1414 if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
1415 #else
1416 # define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body
1417 #endif
1418 PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
1419 static const char *obj_info(VALUE obj);
1420 static const char *obj_info_basic(VALUE obj);
1421 static const char *obj_type_name(VALUE obj);
1422
1423 static void gc_finalize_deferred(void *dmy);
1424
1425 #if USE_TICK_T
1426 /* the following code is only for internal tuning. */
1427
1428 /* Source code to use RDTSC is quoted and modified from
1429 * https://www.mcs.anl.gov/~kazutomo/rdtsc.html
1430 * written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
1431 */
1432
1433 #if defined(__GNUC__) && defined(__i386__)
1434 typedef unsigned long long tick_t;
1435 #define PRItick "llu"
1436 static inline tick_t
1437 tick(void)
1438 {
1439 unsigned long long int x;
1440 __asm__ __volatile__ ("rdtsc" : "=A" (x));
1441 return x;
1442 }
1443
1444 #elif defined(__GNUC__) && defined(__x86_64__)
1445 typedef unsigned long long tick_t;
1446 #define PRItick "llu"
1447
1448 static __inline__ tick_t
1449 tick(void)
1450 {
1451 unsigned long hi, lo;
1452 __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
1453 return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
1454 }
1455
1456 #elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
1457 typedef unsigned long long tick_t;
1458 #define PRItick "llu"
1459
1460 static __inline__ tick_t
1461 tick(void)
1462 {
1463 unsigned long long val = __builtin_ppc_get_timebase();
1464 return val;
1465 }
1466
1467 /* Implementation for macOS PPC by @nobu
1468 * See: https://github.com/ruby/ruby/pull/5975#discussion_r890045558
1469 */
1470 #elif defined(__POWERPC__) && defined(__APPLE__)
1471 typedef unsigned long long tick_t;
1472 #define PRItick "llu"
1473
1474 static __inline__ tick_t
1475 tick(void)
1476 {
1477 unsigned long int upper, lower, tmp;
1478 # define mftbu(r) __asm__ volatile("mftbu %0" : "=r"(r))
1479 # define mftb(r) __asm__ volatile("mftb %0" : "=r"(r))
1480 do {
1481 mftbu(upper);
1482 mftb(lower);
1483 mftbu(tmp);
1484 } while (tmp != upper);
1485 return ((tick_t)upper << 32) | lower;
1486 }
1487
1488 #elif defined(__aarch64__) && defined(__GNUC__)
1489 typedef unsigned long tick_t;
1490 #define PRItick "lu"
1491
1492 static __inline__ tick_t
1493 tick(void)
1494 {
1495 unsigned long val;
1496 __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (val));
1497 return val;
1498 }
1499
1500
1501 #elif defined(_WIN32) && defined(_MSC_VER)
1502 #include <intrin.h>
1503 typedef unsigned __int64 tick_t;
1504 #define PRItick "llu"
1505
1506 static inline tick_t
1507 tick(void)
1508 {
1509 return __rdtsc();
1510 }
1511
1512 #else /* use clock */
1513 typedef clock_t tick_t;
1514 #define PRItick "llu"
1515
1516 static inline tick_t
1517 tick(void)
1518 {
1519 return clock();
1520 }
1521 #endif /* TSC */
1522 #else /* USE_TICK_T */
1523 #define MEASURE_LINE(expr) expr
1524 #endif /* USE_TICK_T */
1525
1526 #define asan_unpoisoning_object(obj) \
1527 for (void *poisoned = asan_unpoison_object_temporary(obj), \
1528 *unpoisoning = &poisoned; /* flag to loop just once */ \
1529 unpoisoning; \
1530 unpoisoning = asan_poison_object_restore(obj, poisoned))
1531
1532 #define FL_CHECK2(name, x, pred) \
1533 ((RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) ? \
1534 (rb_bug(name": SPECIAL_CONST (%p)", (void *)(x)), 0) : (pred))
1535 #define FL_TEST2(x,f) FL_CHECK2("FL_TEST2", x, FL_TEST_RAW((x),(f)) != 0)
1536 #define FL_SET2(x,f) FL_CHECK2("FL_SET2", x, RBASIC(x)->flags |= (f))
1537 #define FL_UNSET2(x,f) FL_CHECK2("FL_UNSET2", x, RBASIC(x)->flags &= ~(f))
1538
1539 #define RVALUE_MARK_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
1540 #define RVALUE_PIN_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
1541 #define RVALUE_PAGE_MARKED(page, obj) MARKED_IN_BITMAP((page)->mark_bits, (obj))
1542
1543 #define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
1544 #define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
1545 #define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
1546
1547 #define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
1548 #define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
1549 #define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
1550
1551 static int rgengc_remember(rb_objspace_t *objspace, VALUE obj);
1552 static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap);
1553 static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
1554
1555 static int
1556 check_rvalue_consistency_force(const VALUE obj, int terminate)
1557 {
1558 int err = 0;
1559 rb_objspace_t *objspace = &rb_objspace;
1560
1561 RB_VM_LOCK_ENTER_NO_BARRIER();
1562 {
1563 if (SPECIAL_CONST_P(obj)) {
1564 fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
1565 err++;
1566 }
1567 else if (!is_pointer_to_heap(objspace, (void *)obj)) {
1568 /* check if it is in tomb_pages */
1569 struct heap_page *page = NULL;
1570 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1571 rb_size_pool_t *size_pool = &size_pools[i];
1572 ccan_list_for_each(&size_pool->tomb_heap.pages, page, page_node) {
1573 if (page->start <= (uintptr_t)obj &&
1574 (uintptr_t)obj < (page->start + (page->total_slots * size_pool->slot_size))) {
1575 fprintf(stderr, "check_rvalue_consistency: %p is in a tomb_heap (%p).\n",
1576 (void *)obj, (void *)page);
1577 err++;
1578 goto skip;
1579 }
1580 }
1581 }
1582 bp();
1583 fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
1584 err++;
1585 skip:
1586 ;
1587 }
1588 else {
1589 const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1590 const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1591 const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0;
1592 const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
1593 const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1594 const int age = RVALUE_AGE_GET((VALUE)obj);
1595
1596 if (GET_HEAP_PAGE(obj)->flags.in_tomb) {
1597 fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", obj_info(obj));
1598 err++;
1599 }
1600 if (BUILTIN_TYPE(obj) == T_NONE) {
1601 fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", obj_info(obj));
1602 err++;
1603 }
1604 if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
1605 fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", obj_info(obj));
1606 err++;
1607 }
1608
1609 obj_memsize_of((VALUE)obj, FALSE);
1610
1611 /* check generation
1612 *
1613 * OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
1614 */
1615 if (age > 0 && wb_unprotected_bit) {
1616 fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", obj_info(obj), age);
1617 err++;
1618 }
1619
1620 if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
1621 fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", obj_info(obj));
1622 err++;
1623 }
1624
1625 if (!is_full_marking(objspace)) {
1626 if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
1627 fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
1628 obj_info(obj), age);
1629 err++;
1630 }
1631 if (remembered_bit && age != RVALUE_OLD_AGE) {
1632 fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
1633 obj_info(obj), age);
1634 err++;
1635 }
1636 }
1637
1638 /*
1639 * check coloring
1640 *
1641 * marking:false marking:true
1642 * marked:false white *invalid*
1643 * marked:true black grey
1644 */
1645 if (is_incremental_marking(objspace) && marking_bit) {
1646 if (!is_marking(objspace) && !mark_bit) {
1647 fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", obj_info(obj));
1648 err++;
1649 }
1650 }
1651 }
1652 }
1653 RB_VM_LOCK_LEAVE_NO_BARRIER();
1654
1655 if (err > 0 && terminate) {
1656 rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
1657 }
1658 return err;
1659 }
1660
1661 #if RGENGC_CHECK_MODE == 0
1662 static inline VALUE
1663 check_rvalue_consistency(const VALUE obj)
1664 {
1665 return obj;
1666 }
1667 #else
1668 static VALUE
1669 check_rvalue_consistency(const VALUE obj)
1670 {
1671 check_rvalue_consistency_force(obj, TRUE);
1672 return obj;
1673 }
1674 #endif
1675
1676 static inline int
1677 gc_object_moved_p(rb_objspace_t * objspace, VALUE obj)
1678 {
1679 if (RB_SPECIAL_CONST_P(obj)) {
1680 return FALSE;
1681 }
1682 else {
1683 void *poisoned = asan_unpoison_object_temporary(obj);
1684
1685 int ret = BUILTIN_TYPE(obj) == T_MOVED;
1686 /* Re-poison slot if it's not the one we want */
1687 if (poisoned) {
1688 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
1689 asan_poison_object(obj);
1690 }
1691 return ret;
1692 }
1693 }
1694
1695 static inline int
1696 RVALUE_MARKED(VALUE obj)
1697 {
1698 check_rvalue_consistency(obj);
1699 return RVALUE_MARK_BITMAP(obj) != 0;
1700 }
1701
1702 static inline int
1703 RVALUE_PINNED(VALUE obj)
1704 {
1705 check_rvalue_consistency(obj);
1706 return RVALUE_PIN_BITMAP(obj) != 0;
1707 }
1708
1709 static inline int
1710 RVALUE_WB_UNPROTECTED(VALUE obj)
1711 {
1712 check_rvalue_consistency(obj);
1713 return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1714 }
1715
1716 static inline int
1717 RVALUE_MARKING(VALUE obj)
1718 {
1719 check_rvalue_consistency(obj);
1720 return RVALUE_MARKING_BITMAP(obj) != 0;
1721 }
1722
1723 static inline int
1724 RVALUE_REMEMBERED(VALUE obj)
1725 {
1726 check_rvalue_consistency(obj);
1727 return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1728 }
1729
1730 static inline int
1731 RVALUE_UNCOLLECTIBLE(VALUE obj)
1732 {
1733 check_rvalue_consistency(obj);
1734 return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1735 }
1736
1737 static inline int
1738 RVALUE_OLD_P(VALUE obj)
1739 {
1740 GC_ASSERT(!RB_SPECIAL_CONST_P(obj));
1741 check_rvalue_consistency(obj);
1742 // Because this will only ever be called on GC controlled objects,
1743 // we can use the faster _RAW function here
1744 return RB_OBJ_PROMOTED_RAW(obj);
1745 }
1746
1747 static inline void
1748 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
1749 {
1750 MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
1751 objspace->rgengc.old_objects++;
1752
1753 #if RGENGC_PROFILE >= 2
1754 objspace->profile.total_promoted_count++;
1755 objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
1756 #endif
1757 }
1758
1759 static inline void
1760 RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
1761 {
1762 RB_DEBUG_COUNTER_INC(obj_promote);
1763 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
1764 }
1765
1766 /* set age to age+1 */
1767 static inline void
1768 RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
1769 {
1770 int age = RVALUE_AGE_GET((VALUE)obj);
1771
1772 if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
1773 rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj));
1774 }
1775
1776 age++;
1777 RVALUE_AGE_SET(obj, age);
1778
1779 if (age == RVALUE_OLD_AGE) {
1780 RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
1781 }
1782
1783 check_rvalue_consistency(obj);
1784 }
1785
1786 static inline void
1787 RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
1788 {
1789 check_rvalue_consistency(obj);
1790 GC_ASSERT(!RVALUE_OLD_P(obj));
1791 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
1792 check_rvalue_consistency(obj);
1793 }
1794
1795 static inline void
1796 RVALUE_AGE_RESET(VALUE obj)
1797 {
1798 RVALUE_AGE_SET(obj, 0);
1799 }
1800
1801 static inline void
1802 RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
1803 {
1804 check_rvalue_consistency(obj);
1805 GC_ASSERT(RVALUE_OLD_P(obj));
1806
1807 if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(obj)) {
1808 CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj);
1809 }
1810
1811 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
1812 RVALUE_AGE_RESET(obj);
1813
1814 if (RVALUE_MARKED(obj)) {
1815 objspace->rgengc.old_objects--;
1816 }
1817
1818 check_rvalue_consistency(obj);
1819 }
1820
1821 static inline int
1822 RVALUE_BLACK_P(VALUE obj)
1823 {
1824 return RVALUE_MARKED(obj) && !RVALUE_MARKING(obj);
1825 }
1826
1827 #if 0
1828 static inline int
1829 RVALUE_GREY_P(VALUE obj)
1830 {
1831 return RVALUE_MARKED(obj) && RVALUE_MARKING(obj);
1832 }
1833 #endif
1834
1835 static inline int
1836 RVALUE_WHITE_P(VALUE obj)
1837 {
1838 return RVALUE_MARKED(obj) == FALSE;
1839 }
1840
1841 /*
1842 --------------------------- ObjectSpace -----------------------------
1843 */
1844
1845 static inline void *
1846 calloc1(size_t n)
1847 {
1848 return calloc(1, n);
1849 }
1850
1851 static VALUE initial_stress = Qfalse;
1852
1853 void
1854 rb_gc_initial_stress_set(VALUE flag)
1855 {
1856 initial_stress = flag;
1857 }
1858
1859 static void *rb_gc_impl_objspace_alloc(void);
1860
1861 #if USE_SHARED_GC
1862 # include "dln.h"
1863
1864 # define RUBY_GC_LIBRARY_PATH "RUBY_GC_LIBRARY_PATH"
1865
1866 void
1867 ruby_external_gc_init(void)
1868 {
1869 char *gc_so_path = getenv(RUBY_GC_LIBRARY_PATH);
1870 void *handle = NULL;
1871 if (gc_so_path && dln_supported_p()) {
1872 char error[1024];
1873 handle = dln_open(gc_so_path, error, sizeof(error));
1874 if (!handle) {
1875 fprintf(stderr, "%s", error);
1876 rb_bug("ruby_external_gc_init: Shared library %s cannot be opened", gc_so_path);
1877 }
1878 }
1879
1880 # define load_external_gc_func(name) do { \
1881 if (handle) { \
1882 rb_gc_functions->name = dln_symbol(handle, "rb_gc_impl_" #name); \
1883 if (!rb_gc_functions->name) { \
1884 rb_bug("ruby_external_gc_init: " #name " func not exported by library %s", gc_so_path); \
1885 } \
1886 } \
1887 else { \
1888 rb_gc_functions->name = rb_gc_impl_##name; \
1889 } \
1890 } while (0)
1891
1892 load_external_gc_func(objspace_alloc);
1893
1894 # undef load_external_gc_func
1895 }
1896
1897 # define rb_gc_impl_objspace_alloc rb_gc_functions->objspace_alloc
1898 #endif
1899
1900 rb_objspace_t *
1901 rb_objspace_alloc(void)
1902 {
1903 #if USE_SHARED_GC
1904 ruby_external_gc_init();
1905 #endif
1906 return (rb_objspace_t *)rb_gc_impl_objspace_alloc();
1907 }
1908
1909 #if USE_SHARED_GC
1910 # undef rb_gc_impl_objspace_alloc
1911 #endif
1912
1913 static void free_stack_chunks(mark_stack_t *);
1914 static void mark_stack_free_cache(mark_stack_t *);
1915 static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
1916
1917 void
1918 rb_objspace_free(rb_objspace_t *objspace)
1919 {
1920 if (is_lazy_sweeping(objspace))
1921 rb_bug("lazy sweeping underway when freeing object space");
1922
1923 free(objspace->profile.records);
1924 objspace->profile.records = NULL;
1925
1926 if (heap_pages_sorted) {
1927 size_t i;
1928 size_t total_heap_pages = heap_allocated_pages;
1929 for (i = 0; i < total_heap_pages; ++i) {
1930 heap_page_free(objspace, heap_pages_sorted[i]);
1931 }
1932 free(heap_pages_sorted);
1933 heap_allocated_pages = 0;
1934 heap_pages_sorted_length = 0;
1935 heap_pages_lomem = 0;
1936 heap_pages_himem = 0;
1937
1938 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1939 rb_size_pool_t *size_pool = &size_pools[i];
1940 SIZE_POOL_EDEN_HEAP(size_pool)->total_pages = 0;
1941 SIZE_POOL_EDEN_HEAP(size_pool)->total_slots = 0;
1942 }
1943 }
1944 st_free_table(objspace->id_to_obj_tbl);
1945 st_free_table(objspace->obj_to_id_tbl);
1946
1947 free_stack_chunks(&objspace->mark_stack);
1948 mark_stack_free_cache(&objspace->mark_stack);
1949
1950 rb_darray_free(objspace->weak_references);
1951
1952 free(objspace);
1953 }
1954
1955 static void
1956 heap_pages_expand_sorted_to(rb_objspace_t *objspace, size_t next_length)
1957 {
1958 struct heap_page **sorted;
1959 size_t size = size_mul_or_raise(next_length, sizeof(struct heap_page *), rb_eRuntimeError);
1960
1961 gc_report(3, objspace, "heap_pages_expand_sorted: next_length: %"PRIdSIZE", size: %"PRIdSIZE"\n",
1962 next_length, size);
1963
1964 if (heap_pages_sorted_length > 0) {
1965 sorted = (struct heap_page **)realloc(heap_pages_sorted, size);
1966 if (sorted) heap_pages_sorted = sorted;
1967 }
1968 else {
1969 sorted = heap_pages_sorted = (struct heap_page **)malloc(size);
1970 }
1971
1972 if (sorted == 0) {
1973 rb_memerror();
1974 }
1975
1976 heap_pages_sorted_length = next_length;
1977 }
1978
1979 static void
1980 heap_pages_expand_sorted(rb_objspace_t *objspace)
1981 {
1982 /* usually heap_allocatable_pages + heap_eden->total_pages == heap_pages_sorted_length
1983 * because heap_allocatable_pages contains heap_tomb->total_pages (recycle heap_tomb pages).
1984 * however, if there are pages which do not have empty slots, then try to create new pages
1985 * so that the additional allocatable_pages counts (heap_tomb->total_pages) are added.
1986 */
1987 size_t next_length = heap_allocatable_pages(objspace);
1988 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
1989 rb_size_pool_t *size_pool = &size_pools[i];
1990 next_length += SIZE_POOL_EDEN_HEAP(size_pool)->total_pages;
1991 next_length += SIZE_POOL_TOMB_HEAP(size_pool)->total_pages;
1992 }
1993
1994 if (next_length > heap_pages_sorted_length) {
1995 heap_pages_expand_sorted_to(objspace, next_length);
1996 }
1997
1998 GC_ASSERT(heap_allocatable_pages(objspace) + heap_eden_total_pages(objspace) <= heap_pages_sorted_length);
1999 GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
2000 }
2001
2002 static void
2003 size_pool_allocatable_pages_set(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t s)
2004 {
2005 size_pool->allocatable_pages = s;
2006 heap_pages_expand_sorted(objspace);
2007 }
2008
2009 static inline void
2010 heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
2011 {
2012 ASSERT_vm_locking();
2013
2014 RVALUE *p = (RVALUE *)obj;
2015
2016 asan_unpoison_object(obj, false);
2017
2018 asan_unlock_freelist(page);
2019
2020 p->as.free.flags = 0;
2021 p->as.free.next = page->freelist;
2022 page->freelist = p;
2023 asan_lock_freelist(page);
2024
2025 RVALUE_AGE_RESET(obj);
2026
2027 if (RGENGC_CHECK_MODE &&
2028 /* obj should belong to page */
2029 !(page->start <= (uintptr_t)obj &&
2030 (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
2031 obj % BASE_SLOT_SIZE == 0)) {
2032 rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)p);
2033 }
2034
2035 asan_poison_object(obj);
2036 gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj);
2037 }
2038
2039 static inline void
2040 heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
2041 {
2042 asan_unlock_freelist(page);
2043 GC_ASSERT(page->free_slots != 0);
2044 GC_ASSERT(page->freelist != NULL);
2045
2046 page->free_next = heap->free_pages;
2047 heap->free_pages = page;
2048
2049 RUBY_DEBUG_LOG("page:%p freelist:%p", (void *)page, (void *)page->freelist);
2050
2051 asan_lock_freelist(page);
2052 }
2053
2054 static inline void
2055 heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
2056 {
2057 asan_unlock_freelist(page);
2058 GC_ASSERT(page->free_slots != 0);
2059 GC_ASSERT(page->freelist != NULL);
2060
2061 page->free_next = heap->pooled_pages;
2062 heap->pooled_pages = page;
2063 objspace->rincgc.pooled_slots += page->free_slots;
2064
2065 asan_lock_freelist(page);
2066 }
2067
2068 static void
2069 heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
2070 {
2071 ccan_list_del(&page->page_node);
2072 heap->total_pages--;
2073 heap->total_slots -= page->total_slots;
2074 }
2075
2076 static void rb_aligned_free(void *ptr, size_t size);
2077
2078 static void
2079 heap_page_body_free(struct heap_page_body *page_body)
2080 {
2081 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
2082
2083 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2084 #ifdef HAVE_MMAP
2085 GC_ASSERT(HEAP_PAGE_SIZE % sysconf(_SC_PAGE_SIZE) == 0);
2086 if (munmap(page_body, HEAP_PAGE_SIZE)) {
2087 rb_bug("heap_page_body_free: munmap failed");
2088 }
2089 #endif
2090 }
2091 else {
2092 rb_aligned_free(page_body, HEAP_PAGE_SIZE);
2093 }
2094 }
2095
2096 static void
2097 heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
2098 {
2099 heap_allocated_pages--;
2100 page->size_pool->total_freed_pages++;
2101 heap_page_body_free(GET_PAGE_BODY(page->start));
2102 free(page);
2103 }
2104
2105 static void *
2106 rb_aligned_malloc(size_t alignment, size_t size)
2107 {
2108 /* alignment must be a power of 2 */
2109 GC_ASSERT(((alignment - 1) & alignment) == 0);
2110 GC_ASSERT(alignment % sizeof(void*) == 0);
2111
2112 void *res;
2113
2114 #if defined __MINGW32__
2115 res = __mingw_aligned_malloc(size, alignment);
2116 #elif defined _WIN32
2117 void *_aligned_malloc(size_t, size_t);
2118 res = _aligned_malloc(size, alignment);
2119 #elif defined(HAVE_POSIX_MEMALIGN)
2120 if (posix_memalign(&res, alignment, size) != 0) {
2121 return NULL;
2122 }
2123 #elif defined(HAVE_MEMALIGN)
2124 res = memalign(alignment, size);
2125 #else
2126 char* aligned;
2127 res = malloc(alignment + size + sizeof(void*));
2128 aligned = (char*)res + alignment + sizeof(void*);
2129 aligned -= ((VALUE)aligned & (alignment - 1));
2130 ((void**)aligned)[-1] = res;
2131 res = (void*)aligned;
2132 #endif
2133
2134 GC_ASSERT((uintptr_t)res % alignment == 0);
2135
2136 return res;
2137 }
2138
2139 static void
2140 heap_pages_free_unused_pages(rb_objspace_t *objspace)
2141 {
2142 size_t i, j;
2143
2144 bool has_pages_in_tomb_heap = FALSE;
2145 for (i = 0; i < SIZE_POOL_COUNT; i++) {
2146 if (!ccan_list_empty(&SIZE_POOL_TOMB_HEAP(&size_pools[i])->pages)) {
2147 has_pages_in_tomb_heap = TRUE;
2148 break;
2149 }
2150 }
2151
2152 if (has_pages_in_tomb_heap) {
2153 for (i = j = 0; j < heap_allocated_pages; i++) {
2154 struct heap_page *page = heap_pages_sorted[i];
2155
2156 if (page->flags.in_tomb && page->free_slots == page->total_slots) {
2157 heap_unlink_page(objspace, SIZE_POOL_TOMB_HEAP(page->size_pool), page);
2158 heap_page_free(objspace, page);
2159 }
2160 else {
2161 if (i != j) {
2162 heap_pages_sorted[j] = page;
2163 }
2164 j++;
2165 }
2166 }
2167
2168 struct heap_page *hipage = heap_pages_sorted[heap_allocated_pages - 1];
2169 uintptr_t himem = (uintptr_t)hipage->start + (hipage->total_slots * hipage->slot_size);
2170 GC_ASSERT(himem <= heap_pages_himem);
2171 heap_pages_himem = himem;
2172
2173 struct heap_page *lopage = heap_pages_sorted[0];
2174 uintptr_t lomem = (uintptr_t)lopage->start;
2175 GC_ASSERT(lomem >= heap_pages_lomem);
2176 heap_pages_lomem = lomem;
2177
2178 GC_ASSERT(j == heap_allocated_pages);
2179 }
2180 }
2181
2182 static struct heap_page_body *
2183 heap_page_body_allocate(void)
2184 {
2185 struct heap_page_body *page_body;
2186
2187 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2188 #ifdef HAVE_MMAP
2189 GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
2190
2191 char *ptr = mmap(NULL, HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE,
2192 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
2193 if (ptr == MAP_FAILED) {
2194 return NULL;
2195 }
2196
2197 char *aligned = ptr + HEAP_PAGE_ALIGN;
2198 aligned -= ((VALUE)aligned & (HEAP_PAGE_ALIGN - 1));
2199 GC_ASSERT(aligned > ptr);
2200 GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
2201
2202 size_t start_out_of_range_size = aligned - ptr;
2203 GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
2204 if (start_out_of_range_size > 0) {
2205 if (munmap(ptr, start_out_of_range_size)) {
2206 rb_bug("heap_page_body_allocate: munmap failed for start");
2207 }
2208 }
2209
2210 size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
2211 GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
2212 if (end_out_of_range_size > 0) {
2213 if (munmap(aligned + HEAP_PAGE_SIZE, end_out_of_range_size)) {
2214 rb_bug("heap_page_body_allocate: munmap failed for end");
2215 }
2216 }
2217
2218 page_body = (struct heap_page_body *)aligned;
2219 #endif
2220 }
2221 else {
2222 page_body = rb_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
2223 }
2224
2225 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
2226
2227 return page_body;
2228 }
2229
2230 static struct heap_page *
2231 heap_page_allocate(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
2232 {
2233 uintptr_t start, end, p;
2234 struct heap_page *page;
2235 uintptr_t hi, lo, mid;
2236 size_t stride = size_pool->slot_size;
2237 unsigned int limit = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header)))/(int)stride;
2238
2239 /* assign heap_page body (contains heap_page_header and RVALUEs) */
2240 struct heap_page_body *page_body = heap_page_body_allocate();
2241 if (page_body == 0) {
2242 rb_memerror();
2243 }
2244
2245 /* assign heap_page entry */
2246 page = calloc1(sizeof(struct heap_page));
2247 if (page == 0) {
2248 heap_page_body_free(page_body);
2249 rb_memerror();
2250 }
2251
2252 /* adjust obj_limit (object number available in this page) */
2253 start = (uintptr_t)((VALUE)page_body + sizeof(struct heap_page_header));
2254
2255 if (start % BASE_SLOT_SIZE != 0) {
2256 int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE);
2257 start = start + delta;
2258 GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1);
2259
2260 /* Find a num in page that is evenly divisible by `stride`.
2261 * This is to ensure that objects are aligned with bit planes.
2262 * In other words, ensure there are an even number of objects
2263 * per bit plane. */
2264 if (NUM_IN_PAGE(start) == 1) {
2265 start += stride - BASE_SLOT_SIZE;
2266 }
2267
2268 GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % stride == 0);
2269
2270 limit = (HEAP_PAGE_SIZE - (int)(start - (uintptr_t)page_body))/(int)stride;
2271 }
2272 end = start + (limit * (int)stride);
2273
2274 /* setup heap_pages_sorted */
2275 lo = 0;
2276 hi = (uintptr_t)heap_allocated_pages;
2277 while (lo < hi) {
2278 struct heap_page *mid_page;
2279
2280 mid = (lo + hi) / 2;
2281 mid_page = heap_pages_sorted[mid];
2282 if ((uintptr_t)mid_page->start < start) {
2283 lo = mid + 1;
2284 }
2285 else if ((uintptr_t)mid_page->start > start) {
2286 hi = mid;
2287 }
2288 else {
2289 rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
2290 }
2291 }
2292
2293 if (hi < (uintptr_t)heap_allocated_pages) {
2294 MEMMOVE(&heap_pages_sorted[hi+1], &heap_pages_sorted[hi], struct heap_page_header*, heap_allocated_pages - hi);
2295 }
2296
2297 heap_pages_sorted[hi] = page;
2298
2299 heap_allocated_pages++;
2300
2301 GC_ASSERT(heap_eden_total_pages(objspace) + heap_allocatable_pages(objspace) <= heap_pages_sorted_length);
2302 GC_ASSERT(heap_eden_total_pages(objspace) + heap_tomb_total_pages(objspace) == heap_allocated_pages - 1);
2303 GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
2304
2305 size_pool->total_allocated_pages++;
2306
2307 if (heap_allocated_pages > heap_pages_sorted_length) {
2308 rb_bug("heap_page_allocate: allocated(%"PRIdSIZE") > sorted(%"PRIdSIZE")",
2309 heap_allocated_pages, heap_pages_sorted_length);
2310 }
2311
2312 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
2313 if (heap_pages_himem < end) heap_pages_himem = end;
2314
2315 page->start = start;
2316 page->total_slots = limit;
2317 page->slot_size = size_pool->slot_size;
2318 page->size_pool = size_pool;
2319 page_body->header.page = page;
2320
2321 for (p = start; p != end; p += stride) {
2322 gc_report(3, objspace, "assign_heap_page: %p is added to freelist\n", (void *)p);
2323 heap_page_add_freeobj(objspace, page, (VALUE)p);
2324 }
2325 page->free_slots = limit;
2326
2327 asan_lock_freelist(page);
2328 return page;
2329 }
2330
2331 static struct heap_page *
2332 heap_page_resurrect(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
2333 {
2334 struct heap_page *page = 0, *next;
2335
2336 ccan_list_for_each_safe(&SIZE_POOL_TOMB_HEAP(size_pool)->pages, page, next, page_node) {
2337 asan_unlock_freelist(page);
2338 if (page->freelist != NULL) {
2339 heap_unlink_page(objspace, &size_pool->tomb_heap, page);
2340 asan_lock_freelist(page);
2341 return page;
2342 }
2343 }
2344
2345 return NULL;
2346 }
2347
2348 static struct heap_page *
2349 heap_page_create(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
2350 {
2351 struct heap_page *page;
2352 const char *method = "recycle";
2353
2354 size_pool->allocatable_pages--;
2355
2356 page = heap_page_resurrect(objspace, size_pool);
2357
2358 if (page == NULL) {
2359 page = heap_page_allocate(objspace, size_pool);
2360 method = "allocate";
2361 }
2362 if (0) fprintf(stderr, "heap_page_create: %s - %p, "
2363 "heap_allocated_pages: %"PRIdSIZE", "
2364 "heap_allocated_pages: %"PRIdSIZE", "
2365 "tomb->total_pages: %"PRIdSIZE"\n",
2366 method, (void *)page, heap_pages_sorted_length, heap_allocated_pages, SIZE_POOL_TOMB_HEAP(size_pool)->total_pages);
2367 return page;
2368 }
2369
2370 static void
2371 heap_add_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, struct heap_page *page)
2372 {
2373 /* Adding to eden heap during incremental sweeping is forbidden */
2374 GC_ASSERT(!(heap == SIZE_POOL_EDEN_HEAP(size_pool) && heap->sweeping_page));
2375 page->flags.in_tomb = (heap == SIZE_POOL_TOMB_HEAP(size_pool));
2376 ccan_list_add_tail(&heap->pages, &page->page_node);
2377 heap->total_pages++;
2378 heap->total_slots += page->total_slots;
2379 }
2380
2381 static void
2382 heap_assign_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
2383 {
2384 struct heap_page *page = heap_page_create(objspace, size_pool);
2385 heap_add_page(objspace, size_pool, heap, page);
2386 heap_add_freepage(heap, page);
2387 }
2388
2389 #if GC_CAN_COMPILE_COMPACTION
2390 static void
2391 heap_add_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, size_t add)
2392 {
2393 size_t i;
2394
2395 size_pool_allocatable_pages_set(objspace, size_pool, add);
2396
2397 for (i = 0; i < add; i++) {
2398 heap_assign_page(objspace, size_pool, heap);
2399 }
2400
2401 GC_ASSERT(size_pool->allocatable_pages == 0);
2402 }
2403 #endif
2404
2405 static size_t
2406 slots_to_pages_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t slots)
2407 {
2408 size_t multiple = size_pool->slot_size / BASE_SLOT_SIZE;
2409 /* Due to alignment, heap pages may have one less slot. We should
2410 * ensure there is enough pages to guarantee that we will have at
2411 * least the required number of slots after allocating all the pages. */
2412 size_t slots_per_page = (HEAP_PAGE_OBJ_LIMIT / multiple) - 1;
2413 return CEILDIV(slots, slots_per_page);
2414 }
2415
2416 static size_t
2417 minimum_pages_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
2418 {
2419 size_t size_pool_idx = size_pool - size_pools;
2420 size_t init_slots = gc_params.size_pool_init_slots[size_pool_idx];
2421 return slots_to_pages_for_size_pool(objspace, size_pool, init_slots);
2422 }
2423
2424 static size_t
2425 heap_extend_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t free_slots, size_t total_slots, size_t used)
2426 {
2427 double goal_ratio = gc_params.heap_free_slots_goal_ratio;
2428 size_t next_used;
2429
2430 if (goal_ratio == 0.0) {
2431 next_used = (size_t)(used * gc_params.growth_factor);
2432 }
2433 else if (total_slots == 0) {
2434 next_used = minimum_pages_for_size_pool(objspace, size_pool);
2435 }
2436 else {
2437 /* Find `f' where free_slots = f * total_slots * goal_ratio
2438 * => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
2439 */
2440 double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
2441
2442 if (f > gc_params.growth_factor) f = gc_params.growth_factor;
2443 if (f < 1.0) f = 1.1;
2444
2445 next_used = (size_t)(f * used);
2446
2447 if (0) {
2448 fprintf(stderr,
2449 "free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
2450 " G(%1.2f), f(%1.2f),"
2451 " used(%8"PRIuSIZE") => next_used(%8"PRIuSIZE")\n",
2452 free_slots, total_slots, free_slots/(double)total_slots,
2453 goal_ratio, f, used, next_used);
2454 }
2455 }
2456
2457 if (gc_params.growth_max_slots > 0) {
2458 size_t max_used = (size_t)(used + gc_params.growth_max_slots/HEAP_PAGE_OBJ_LIMIT);
2459 if (next_used > max_used) next_used = max_used;
2460 }
2461
2462 size_t extend_page_count = next_used - used;
2463 /* Extend by at least 1 page. */
2464 if (extend_page_count == 0) extend_page_count = 1;
2465
2466 return extend_page_count;
2467 }
2468
2469 static int
2470 heap_increment(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
2471 {
2472 if (size_pool->allocatable_pages > 0) {
2473 gc_report(1, objspace, "heap_increment: heap_pages_sorted_length: %"PRIdSIZE", "
2474 "heap_pages_inc: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
2475 heap_pages_sorted_length, size_pool->allocatable_pages, heap->total_pages);
2476
2477 GC_ASSERT(heap_allocatable_pages(objspace) + heap_eden_total_pages(objspace) <= heap_pages_sorted_length);
2478 GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
2479
2480 heap_assign_page(objspace, size_pool, heap);
2481 return TRUE;
2482 }
2483 return FALSE;
2484 }
2485
2486 static void
2487 gc_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
2488 {
2489 unsigned int lock_lev;
2490 gc_enter(objspace, gc_enter_event_continue, &lock_lev);
2491
2492 /* Continue marking if in incremental marking. */
2493 if (is_incremental_marking(objspace)) {
2494 if (gc_marks_continue(objspace, size_pool, heap)) {
2495 gc_sweep(objspace);
2496 }
2497 }
2498
2499 /* Continue sweeping if in lazy sweeping or the previous incremental
2500 * marking finished and did not yield a free page. */
2501 if (heap->free_pages == NULL && is_lazy_sweeping(objspace)) {
2502 gc_sweep_continue(objspace, size_pool, heap);
2503 }
2504
2505 gc_exit(objspace, gc_enter_event_continue, &lock_lev);
2506 }
2507
2508 static void
2509 heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
2510 {
2511 GC_ASSERT(heap->free_pages == NULL);
2512
2513 /* Continue incremental marking or lazy sweeping, if in any of those steps. */
2514 gc_continue(objspace, size_pool, heap);
2515
2516 /* If we still don't have a free page and not allowed to create a new page,
2517 * we should start a new GC cycle. */
2518 if (heap->free_pages == NULL &&
2519 (will_be_incremental_marking(objspace) ||
2520 (heap_increment(objspace, size_pool, heap) == FALSE))) {
2521 if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2522 rb_memerror();
2523 }
2524 else {
2525 /* Do steps of incremental marking or lazy sweeping if the GC run permits. */
2526 gc_continue(objspace, size_pool, heap);
2527
2528 /* If we're not incremental marking (e.g. a minor GC) or finished
2529 * sweeping and still don't have a free page, then
2530 * gc_sweep_finish_size_pool should allow us to create a new page. */
2531 if (heap->free_pages == NULL && !heap_increment(objspace, size_pool, heap)) {
2532 if (gc_needs_major_flags == GPR_FLAG_NONE) {
2533 rb_bug("cannot create a new page after GC");
2534 }
2535 else { // Major GC is required, which will allow us to create new page
2536 if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2537 rb_memerror();
2538 }
2539 else {
2540 /* Do steps of incremental marking or lazy sweeping. */
2541 gc_continue(objspace, size_pool, heap);
2542
2543 if (heap->free_pages == NULL &&
2544 !heap_increment(objspace, size_pool, heap)) {
2545 rb_bug("cannot create a new page after major GC");
2546 }
2547 }
2548 }
2549 }
2550 }
2551 }
2552
2553 GC_ASSERT(heap->free_pages != NULL);
2554 }
2555
2556 void
2557 rb_objspace_set_event_hook(const rb_event_flag_t event)
2558 {
2559 rb_objspace_t *objspace = &rb_objspace;
2560 objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
2561 objspace->flags.has_newobj_hook = !!(objspace->hook_events & RUBY_INTERNAL_EVENT_NEWOBJ);
2562 }
2563
2564 static void
2565 gc_event_hook_body(rb_execution_context_t *ec, rb_objspace_t *objspace, const rb_event_flag_t event, VALUE data)
2566 {
2567 if (UNLIKELY(!ec->cfp)) return;
2568 EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, data);
2569 }
2570
2571 #define gc_event_newobj_hook_needed_p(objspace) ((objspace)->flags.has_newobj_hook)
2572 #define gc_event_hook_needed_p(objspace, event) ((objspace)->hook_events & (event))
2573
2574 #define gc_event_hook_prep(objspace, event, data, prep) do { \
2575 if (UNLIKELY(gc_event_hook_needed_p(objspace, event))) { \
2576 prep; \
2577 gc_event_hook_body(GET_EC(), (objspace), (event), (data)); \
2578 } \
2579 } while (0)
2580
2581 #define gc_event_hook(objspace, event, data) gc_event_hook_prep(objspace, event, data, (void)0)
2582
2583 static inline VALUE
2584 newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
2585 {
2586 #if !__has_feature(memory_sanitizer)
2587 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
2588 GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
2589 #endif
2590 RVALUE *p = RANY(obj);
2591 p->as.basic.flags = flags;
2592 *((VALUE *)&p->as.basic.klass) = klass;
2593
2594 int t = flags & RUBY_T_MASK;
2595 if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
2596 RVALUE_AGE_SET_CANDIDATE(objspace, obj);
2597 }
2598
2599 #if RACTOR_CHECK_MODE
2600 rb_ractor_setup_belonging(obj);
2601 #endif
2602
2603 #if RGENGC_CHECK_MODE
2604 p->as.values.v1 = p->as.values.v2 = p->as.values.v3 = 0;
2605
2606 RB_VM_LOCK_ENTER_NO_BARRIER();
2607 {
2608 check_rvalue_consistency(obj);
2609
2610 GC_ASSERT(RVALUE_MARKED(obj) == FALSE);
2611 GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
2612 GC_ASSERT(RVALUE_OLD_P(obj) == FALSE);
2613 GC_ASSERT(RVALUE_WB_UNPROTECTED(obj) == FALSE);
2614
2615 if (RVALUE_REMEMBERED((VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj));
2616 }
2617 RB_VM_LOCK_LEAVE_NO_BARRIER();
2618 #endif
2619
2620 if (UNLIKELY(wb_protected == FALSE)) {
2621 ASSERT_vm_locking();
2622 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
2623 }
2624
2625 #if RGENGC_PROFILE
2626 if (wb_protected) {
2627 objspace->profile.total_generated_normal_object_count++;
2628 #if RGENGC_PROFILE >= 2
2629 objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
2630 #endif
2631 }
2632 else {
2633 objspace->profile.total_generated_shady_object_count++;
2634 #if RGENGC_PROFILE >= 2
2635 objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
2636 #endif
2637 }
2638 #endif
2639
2640 #if GC_DEBUG
2641 GET_RVALUE_OVERHEAD(obj)->file = rb_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
2642 GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
2643 #endif
2644
2645 gc_report(5, objspace, "newobj: %s\n", obj_info_basic(obj));
2646
2647 // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, obj_type_name(obj));
2648 return obj;
2649 }
2650
2651 size_t
2652 rb_gc_obj_slot_size(VALUE obj)
2653 {
2654 return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
2655 }
2656
2657 static inline size_t
2658 size_pool_slot_size(unsigned char pool_id)
2659 {
2660 GC_ASSERT(pool_id < SIZE_POOL_COUNT);
2661
2662 size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
2663
2664 #if RGENGC_CHECK_MODE
2665 rb_objspace_t *objspace = &rb_objspace;
2666 GC_ASSERT(size_pools[pool_id].slot_size == (short)slot_size);
2667 #endif
2668
2669 slot_size -= RVALUE_OVERHEAD;
2670
2671 return slot_size;
2672 }
2673
2674 bool
2675 rb_gc_size_allocatable_p(size_t size)
2676 {
2677 return size <= size_pool_slot_size(SIZE_POOL_COUNT - 1);
2678 }
2679
2680 static size_t size_pool_sizes[SIZE_POOL_COUNT + 1] = { 0 };
2681
2682 size_t *
2683 rb_gc_size_pool_sizes(void)
2684 {
2685 if (size_pool_sizes[0] == 0) {
2686 for (unsigned char i = 0; i < SIZE_POOL_COUNT; i++) {
2687 size_pool_sizes[i] = size_pool_slot_size(i);
2688 }
2689 }
2690
2691 return size_pool_sizes;
2692 }
2693
2694 size_t
2695 rb_gc_size_pool_id_for_size(size_t size)
2696 {
2697 size += RVALUE_OVERHEAD;
2698
2699 size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
2700
2701 /* size_pool_idx is ceil(log2(slot_count)) */
2702 size_t size_pool_idx = 64 - nlz_int64(slot_count - 1);
2703
2704 if (size_pool_idx >= SIZE_POOL_COUNT) {
2705 rb_bug("rb_gc_size_pool_id_for_size: allocation size too large "
2706 "(size=%"PRIuSIZE"u, size_pool_idx=%"PRIuSIZE"u)", size, size_pool_idx);
2707 }
2708
2709 #if RGENGC_CHECK_MODE
2710 rb_objspace_t *objspace = &rb_objspace;
2711 GC_ASSERT(size <= (size_t)size_pools[size_pool_idx].slot_size);
2712 if (size_pool_idx > 0) GC_ASSERT(size > (size_t)size_pools[size_pool_idx - 1].slot_size);
2713 #endif
2714
2715 return size_pool_idx;
2716 }
2717
2718 static inline VALUE
2719 ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache,
2720 size_t size_pool_idx)
2721 {
2722 rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx];
2723 RVALUE *p = size_pool_cache->freelist;
2724
2725 if (is_incremental_marking(objspace)) {
2726 // Not allowed to allocate without running an incremental marking step
2727 if (cache->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
2728 return Qfalse;
2729 }
2730
2731 if (p) {
2732 cache->incremental_mark_step_allocated_slots++;
2733 }
2734 }
2735
2736 if (p) {
2737 VALUE obj = (VALUE)p;
2738 MAYBE_UNUSED(const size_t) stride = size_pool_slot_size(size_pool_idx);
2739 size_pool_cache->freelist = p->as.free.next;
2740 asan_unpoison_memory_region(p, stride, true);
2741 #if RGENGC_CHECK_MODE
2742 GC_ASSERT(rb_gc_obj_slot_size(obj) == stride);
2743 // zero clear
2744 MEMZERO((char *)obj, char, stride);
2745 #endif
2746 return obj;
2747 }
2748 else {
2749 return Qfalse;
2750 }
2751 }
2752
2753 static struct heap_page *
2754 heap_next_free_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
2755 {
2756 ASSERT_vm_locking();
2757
2758 struct heap_page *page;
2759
2760 if (heap->free_pages == NULL) {
2761 heap_prepare(objspace, size_pool, heap);
2762 }
2763
2764 page = heap->free_pages;
2765 heap->free_pages = page->free_next;
2766
2767 GC_ASSERT(page->free_slots != 0);
2768 RUBY_DEBUG_LOG("page:%p freelist:%p cnt:%d", (void *)page, (void *)page->freelist, page->free_slots);
2769
2770 asan_unlock_freelist(page);
2771
2772 return page;
2773 }
2774
2775 static inline void
2776 ractor_cache_set_page(rb_ractor_newobj_cache_t *cache, size_t size_pool_idx,
2777 struct heap_page *page)
2778 {
2779 gc_report(3, &rb_objspace, "ractor_set_cache: Using page %p\n", (void *)GET_PAGE_BODY(page->start));
2780
2781 rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx];
2782
2783 GC_ASSERT(size_pool_cache->freelist == NULL);
2784 GC_ASSERT(page->free_slots != 0);
2785 GC_ASSERT(page->freelist != NULL);
2786
2787 size_pool_cache->using_page = page;
2788 size_pool_cache->freelist = page->freelist;
2789 page->free_slots = 0;
2790 page->freelist = NULL;
2791
2792 asan_unpoison_object((VALUE)size_pool_cache->freelist, false);
2793 GC_ASSERT(RB_TYPE_P((VALUE)size_pool_cache->freelist, T_NONE));
2794 asan_poison_object((VALUE)size_pool_cache->freelist);
2795 }
2796
2797 static inline VALUE
2798 newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3)
2799 {
2800 RVALUE *p = (RVALUE *)obj;
2801 p->as.values.v1 = v1;
2802 p->as.values.v2 = v2;
2803 p->as.values.v3 = v3;
2804 return obj;
2805 }
2806
2807 static VALUE
2808 newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, bool vm_locked)
2809 {
2810 rb_size_pool_t *size_pool = &size_pools[size_pool_idx];
2811 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
2812
2813 VALUE obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx);
2814
2815 if (UNLIKELY(obj == Qfalse)) {
2816 unsigned int lev;
2817 bool unlock_vm = false;
2818
2819 if (!vm_locked) {
2820 RB_VM_LOCK_ENTER_CR_LEV(GET_RACTOR(), &lev);
2821 vm_locked = true;
2822 unlock_vm = true;
2823 }
2824
2825 {
2826 ASSERT_vm_locking();
2827
2828 if (is_incremental_marking(objspace)) {
2829 gc_continue(objspace, size_pool, heap);
2830 cache->incremental_mark_step_allocated_slots = 0;
2831
2832 // Retry allocation after resetting incremental_mark_step_allocated_slots
2833 obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx);
2834 }
2835
2836 if (obj == Qfalse) {
2837 // Get next free page (possibly running GC)
2838 struct heap_page *page = heap_next_free_page(objspace, size_pool, heap);
2839 ractor_cache_set_page(cache, size_pool_idx, page);
2840
2841 // Retry allocation after moving to new page
2842 obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx);
2843
2844 GC_ASSERT(obj != Qfalse);
2845 }
2846 }
2847
2848 if (unlock_vm) {
2849 RB_VM_LOCK_LEAVE_CR_LEV(GET_RACTOR(), &lev);
2850 }
2851 }
2852
2853 size_pool->total_allocated_objects++;
2854
2855 return obj;
2856 }
2857
2858 static void
2859 newobj_zero_slot(VALUE obj)
2860 {
2861 memset((char *)obj + sizeof(struct RBasic), 0, rb_gc_obj_slot_size(obj) - sizeof(struct RBasic));
2862 }
2863
2864 ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t size_pool_idx));
2865
2866 static inline VALUE
2867 newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t size_pool_idx)
2868 {
2869 VALUE obj;
2870 unsigned int lev;
2871
2872 RB_VM_LOCK_ENTER_CR_LEV(GET_RACTOR(), &lev);
2873 {
2874 if (UNLIKELY(during_gc || ruby_gc_stressful)) {
2875 if (during_gc) {
2876 dont_gc_on();
2877 during_gc = 0;
2878 rb_bug("object allocation during garbage collection phase");
2879 }
2880
2881 if (ruby_gc_stressful) {
2882 if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
2883 rb_memerror();
2884 }
2885 }
2886 }
2887
2888 obj = newobj_alloc(objspace, cache, size_pool_idx, true);
2889 newobj_init(klass, flags, wb_protected, objspace, obj);
2890
2891 gc_event_hook_prep(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj, newobj_zero_slot(obj));
2892 }
2893 RB_VM_LOCK_LEAVE_CR_LEV(GET_RACTOR(), &lev);
2894
2895 return obj;
2896 }
2897
2898 NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags,
2899 rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx));
2900 NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags,
2901 rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx));
2902
2903 static VALUE
2904 newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx)
2905 {
2906 return newobj_slowpath(klass, flags, objspace, cache, TRUE, size_pool_idx);
2907 }
2908
2909 static VALUE
2910 newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx)
2911 {
2912 return newobj_slowpath(klass, flags, objspace, cache, FALSE, size_pool_idx);
2913 }
2914
2915 static inline VALUE
2916 newobj_of(rb_ractor_t *cr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, size_t alloc_size)
2917 {
2918 VALUE obj;
2919 rb_objspace_t *objspace = &rb_objspace;
2920
2921 RB_DEBUG_COUNTER_INC(obj_newobj);
2922 (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
2923
2924 if (UNLIKELY(stress_to_class)) {
2925 long i, cnt = RARRAY_LEN(stress_to_class);
2926 for (i = 0; i < cnt; ++i) {
2927 if (klass == RARRAY_AREF(stress_to_class, i)) rb_memerror();
2928 }
2929 }
2930
2931 size_t size_pool_idx = rb_gc_size_pool_id_for_size(alloc_size);
2932
2933 rb_ractor_newobj_cache_t *cache = &cr->newobj_cache;
2934
2935 if (!UNLIKELY(during_gc ||
2936 ruby_gc_stressful ||
2937 gc_event_newobj_hook_needed_p(objspace)) &&
2938 wb_protected) {
2939 obj = newobj_alloc(objspace, cache, size_pool_idx, false);
2940 newobj_init(klass, flags, wb_protected, objspace, obj);
2941 }
2942 else {
2943 RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
2944
2945 obj = wb_protected ?
2946 newobj_slowpath_wb_protected(klass, flags, objspace, cache, size_pool_idx) :
2947 newobj_slowpath_wb_unprotected(klass, flags, objspace, cache, size_pool_idx);
2948 }
2949
2950 return newobj_fill(obj, v1, v2, v3);
2951 }
2952
2953 VALUE
2954 rb_wb_unprotected_newobj_of(VALUE klass, VALUE flags, size_t size)
2955 {
2956 GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
2957 return newobj_of(GET_RACTOR(), klass, flags, 0, 0, 0, FALSE, size);
2958 }
2959
2960 VALUE
2961 rb_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size)
2962 {
2963 GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
2964 return newobj_of(rb_ec_ractor_ptr(ec), klass, flags, 0, 0, 0, TRUE, size);
2965 }
2966
2967 #define UNEXPECTED_NODE(func) \
2968 rb_bug(#func"(): GC does not handle T_NODE 0x%x(%p) 0x%"PRIxVALUE, \
2969 BUILTIN_TYPE(obj), (void*)(obj), RBASIC(obj)->flags)
2970
2971 static inline void
2972 rb_data_object_check(VALUE klass)
2973 {
2974 if (klass != rb_cObject && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
2975 rb_undef_alloc_func(klass);
2976 rb_warn("undefining the allocator of T_DATA class %"PRIsVALUE, klass);
2977 }
2978 }
2979
2980 VALUE
2981 rb_data_object_wrap(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
2982 {
2983 RUBY_ASSERT_ALWAYS(dfree != (RUBY_DATA_FUNC)1);
2984 if (klass) rb_data_object_check(klass);
2985 return newobj_of(GET_RACTOR(), klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, !dmark, sizeof(struct RTypedData));
2986 }
2987
2988 VALUE
2989 rb_data_object_zalloc(VALUE klass, size_t size, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
2990 {
2991 VALUE obj = rb_data_object_wrap(klass, 0, dmark, dfree);
2992 DATA_PTR(obj) = xcalloc(1, size);
2993 return obj;
2994 }
2995
2996 static VALUE
2997 typed_data_alloc(VALUE klass, VALUE typed_flag, void *datap, const rb_data_type_t *type, size_t size)
2998 {
2999 RBIMPL_NONNULL_ARG(type);
3000 if (klass) rb_data_object_check(klass);
3001 bool wb_protected = (type->flags & RUBY_FL_WB_PROTECTED) || !type->function.dmark;
3002 return newobj_of(GET_RACTOR(), klass, T_DATA, (VALUE)type, 1 | typed_flag, (VALUE)datap, wb_protected, size);
3003 }
3004
3005 VALUE
3006 rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
3007 {
3008 if (UNLIKELY(type->flags & RUBY_TYPED_EMBEDDABLE)) {
3009 rb_raise(rb_eTypeError, "Cannot wrap an embeddable TypedData");
3010 }
3011
3012 return typed_data_alloc(klass, 0, datap, type, sizeof(struct RTypedData));
3013 }
3014
3015 VALUE
3016 rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type)
3017 {
3018 if (type->flags & RUBY_TYPED_EMBEDDABLE) {
3019 if (!(type->flags & RUBY_TYPED_FREE_IMMEDIATELY)) {
3020 rb_raise(rb_eTypeError, "Embeddable TypedData must be freed immediately");
3021 }
3022
3023 size_t embed_size = offsetof(struct RTypedData, data) + size;
3024 if (rb_gc_size_allocatable_p(embed_size)) {
3025 VALUE obj = typed_data_alloc(klass, TYPED_DATA_EMBEDDED, 0, type, embed_size);
3026 memset((char *)obj + offsetof(struct RTypedData, data), 0, size);
3027 return obj;
3028 }
3029 }
3030
3031 VALUE obj = typed_data_alloc(klass, 0, NULL, type, sizeof(struct RTypedData));
3032 DATA_PTR(obj) = xcalloc(1, size);
3033 return obj;
3034 }
3035
3036 static size_t
3037 rb_objspace_data_type_memsize(VALUE obj)
3038 {
3039 size_t size = 0;
3040 if (RTYPEDDATA_P(obj)) {
3041 const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
3042 const void *ptr = RTYPEDDATA_GET_DATA(obj);
3043
3044 if (RTYPEDDATA_TYPE(obj)->flags & RUBY_TYPED_EMBEDDABLE && !RTYPEDDATA_EMBEDDED_P(obj)) {
3045 #ifdef HAVE_MALLOC_USABLE_SIZE
3046 size += malloc_usable_size((void *)ptr);
3047 #endif
3048 }
3049
3050 if (ptr && type->function.dsize) {
3051 size += type->function.dsize(ptr);
3052 }
3053 }
3054
3055 return size;
3056 }
3057
3058 const char *
3059 rb_objspace_data_type_name(VALUE obj)
3060 {
3061 if (RTYPEDDATA_P(obj)) {
3062 return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
3063 }
3064 else {
3065 return 0;
3066 }
3067 }
3068
3069 static int
3070 ptr_in_page_body_p(const void *ptr, const void *memb)
3071 {
3072 struct heap_page *page = *(struct heap_page **)memb;
3073 uintptr_t p_body = (uintptr_t)GET_PAGE_BODY(page->start);
3074
3075 if ((uintptr_t)ptr >= p_body) {
3076 return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
3077 }
3078 else {
3079 return -1;
3080 }
3081 }
3082
3083 PUREFUNC(static inline struct heap_page * heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr);)
3084 static inline struct heap_page *
3085 heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr)
3086 {
3087 struct heap_page **res;
3088
3089 if (ptr < (uintptr_t)heap_pages_lomem ||
3090 ptr > (uintptr_t)heap_pages_himem) {
3091 return NULL;
3092 }
3093
3094 res = bsearch((void *)ptr, heap_pages_sorted,
3095 (size_t)heap_allocated_pages, sizeof(struct heap_page *),
3096 ptr_in_page_body_p);
3097
3098 if (res) {
3099 return *res;
3100 }
3101 else {
3102 return NULL;
3103 }
3104 }
3105
3106 PUREFUNC(static inline int is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr);)
3107 static inline int
3108 is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
3109 {
3110 register uintptr_t p = (uintptr_t)ptr;
3111 register struct heap_page *page;
3112
3113 RB_DEBUG_COUNTER_INC(gc_isptr_trial);
3114
3115 if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
3116 RB_DEBUG_COUNTER_INC(gc_isptr_range);
3117
3118 if (p % BASE_SLOT_SIZE != 0) return FALSE;
3119 RB_DEBUG_COUNTER_INC(gc_isptr_align);
3120
3121 page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
3122 if (page) {
3123 RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
3124 if (page->flags.in_tomb) {
3125 return FALSE;
3126 }
3127 else {
3128 if (p < page->start) return FALSE;
3129 if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
3130 if ((NUM_IN_PAGE(p) * BASE_SLOT_SIZE) % page->slot_size != 0) return FALSE;
3131
3132 return TRUE;
3133 }
3134 }
3135 return FALSE;
3136 }
3137
3138 static enum rb_id_table_iterator_result
3139 cvar_table_free_i(VALUE value, void *ctx)
3140 {
3141 xfree((void *)value);
3142 return ID_TABLE_CONTINUE;
3143 }
3144
3145 #define ZOMBIE_OBJ_KEPT_FLAGS (FL_SEEN_OBJ_ID | FL_FINALIZE)
3146
3147 static inline void
3148 make_zombie(rb_objspace_t *objspace, VALUE obj, void (*dfree)(void *), void *data)
3149 {
3150 struct RZombie *zombie = RZOMBIE(obj);
3151 zombie->basic.flags = T_ZOMBIE | (zombie->basic.flags & ZOMBIE_OBJ_KEPT_FLAGS);
3152 zombie->dfree = dfree;
3153 zombie->data = data;
3154 VALUE prev, next = heap_pages_deferred_final;
3155 do {
3156 zombie->next = prev = next;
3157 next = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev, obj);
3158 } while (next != prev);
3159
3160 struct heap_page *page = GET_HEAP_PAGE(obj);
3161 page->final_slots++;
3162 heap_pages_final_slots++;
3163 }
3164
3165 static inline void
3166 make_io_zombie(rb_objspace_t *objspace, VALUE obj)
3167 {
3168 rb_io_t *fptr = RANY(obj)->as.file.fptr;
3169 make_zombie(objspace, obj, rb_io_fptr_finalize_internal, fptr);
3170 }
3171
3172 static void
3173 obj_free_object_id(rb_objspace_t *objspace, VALUE obj)
3174 {
3175 ASSERT_vm_locking();
3176 st_data_t o = (st_data_t)obj, id;
3177
3178 GC_ASSERT(FL_TEST(obj, FL_SEEN_OBJ_ID));
3179 FL_UNSET(obj, FL_SEEN_OBJ_ID);
3180
3181 if (st_delete(objspace->obj_to_id_tbl, &o, &id)) {
3182 GC_ASSERT(id);
3183 st_delete(objspace->id_to_obj_tbl, &id, NULL);
3184 }
3185 else {
3186 rb_bug("Object ID seen, but not in mapping table: %s", obj_info(obj));
3187 }
3188 }
3189
3190 static bool
3191 rb_data_free(rb_objspace_t *objspace, VALUE obj)
3192 {
3193 void *data = RTYPEDDATA_P(obj) ? RTYPEDDATA_GET_DATA(obj) : DATA_PTR(obj);
3194 if (data) {
3195 int free_immediately = false;
3196 void (*dfree)(void *);
3197
3198 if (RTYPEDDATA_P(obj)) {
3199 free_immediately = (RANY(obj)->as.typeddata.type->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0;
3200 dfree = RANY(obj)->as.typeddata.type->function.dfree;
3201 }
3202 else {
3203 dfree = RANY(obj)->as.data.dfree;
3204 }
3205
3206 if (dfree) {
3207 if (dfree == RUBY_DEFAULT_FREE) {
3208 if (!RTYPEDDATA_EMBEDDED_P(obj)) {
3209 xfree(data);
3210 RB_DEBUG_COUNTER_INC(obj_data_xfree);
3211 }
3212 }
3213 else if (free_immediately) {
3214 (*dfree)(data);
3215 if (RTYPEDDATA_TYPE(obj)->flags & RUBY_TYPED_EMBEDDABLE && !RTYPEDDATA_EMBEDDED_P(obj)) {
3216 xfree(data);
3217 }
3218
3219 RB_DEBUG_COUNTER_INC(obj_data_imm_free);
3220 }
3221 else {
3222 make_zombie(objspace, obj, dfree, data);
3223 RB_DEBUG_COUNTER_INC(obj_data_zombie);
3224 return FALSE;
3225 }
3226 }
3227 else {
3228 RB_DEBUG_COUNTER_INC(obj_data_empty);
3229 }
3230 }
3231
3232 return true;
3233 }
3234
3235 static int
3236 obj_free(rb_objspace_t *objspace, VALUE obj)
3237 {
3238 RB_DEBUG_COUNTER_INC(obj_free);
3239 // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, obj_type_name(obj));
3240
3241 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_FREEOBJ, obj);
3242
3243 switch (BUILTIN_TYPE(obj)) {
3244 case T_NIL:
3245 case T_FIXNUM:
3246 case T_TRUE:
3247 case T_FALSE:
3248 rb_bug("obj_free() called for broken object");
3249 break;
3250 default:
3251 break;
3252 }
3253
3254 if (FL_TEST(obj, FL_EXIVAR)) {
3255 rb_free_generic_ivar((VALUE)obj);
3256 FL_UNSET(obj, FL_EXIVAR);
3257 }
3258
3259 if (FL_TEST(obj, FL_SEEN_OBJ_ID) && !FL_TEST(obj, FL_FINALIZE)) {
3260 obj_free_object_id(objspace, obj);
3261 }
3262
3263 if (RVALUE_WB_UNPROTECTED(obj)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
3264
3265 #if RGENGC_CHECK_MODE
3266 #define CHECK(x) if (x(obj) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", obj_info(obj))
3267 CHECK(RVALUE_WB_UNPROTECTED);
3268 CHECK(RVALUE_MARKED);
3269 CHECK(RVALUE_MARKING);
3270 CHECK(RVALUE_UNCOLLECTIBLE);
3271 #undef CHECK
3272 #endif
3273
3274 switch (BUILTIN_TYPE(obj)) {
3275 case T_OBJECT:
3276 if (rb_shape_obj_too_complex(obj)) {
3277 RB_DEBUG_COUNTER_INC(obj_obj_too_complex);
3278 st_free_table(ROBJECT_IV_HASH(obj));
3279 }
3280 else if (RANY(obj)->as.basic.flags & ROBJECT_EMBED) {
3281 RB_DEBUG_COUNTER_INC(obj_obj_embed);
3282 }
3283 else {
3284 xfree(RANY(obj)->as.object.as.heap.ivptr);
3285 RB_DEBUG_COUNTER_INC(obj_obj_ptr);
3286 }
3287 break;
3288 case T_MODULE:
3289 case T_CLASS:
3290 rb_id_table_free(RCLASS_M_TBL(obj));
3291 rb_cc_table_free(obj);
3292 if (rb_shape_obj_too_complex(obj)) {
3293 st_free_table((st_table *)RCLASS_IVPTR(obj));
3294 }
3295 else {
3296 xfree(RCLASS_IVPTR(obj));
3297 }
3298
3299 if (RCLASS_CONST_TBL(obj)) {
3300 rb_free_const_table(RCLASS_CONST_TBL(obj));
3301 }
3302 if (RCLASS_CVC_TBL(obj)) {
3303 rb_id_table_foreach_values(RCLASS_CVC_TBL(obj), cvar_table_free_i, NULL);
3304 rb_id_table_free(RCLASS_CVC_TBL(obj));
3305 }
3306 rb_class_remove_subclass_head(obj);
3307 rb_class_remove_from_module_subclasses(obj);
3308 rb_class_remove_from_super_subclasses(obj);
3309 if (FL_TEST_RAW(obj, RCLASS_SUPERCLASSES_INCLUDE_SELF)) {
3310 xfree(RCLASS_SUPERCLASSES(obj));
3311 }
3312
3313 (void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE);
3314 (void)RB_DEBUG_COUNTER_INC_IF(obj_class_ptr, BUILTIN_TYPE(obj) == T_CLASS);
3315 break;
3316 case T_STRING:
3317 rb_str_free(obj);
3318 break;
3319 case T_ARRAY:
3320 rb_ary_free(obj);
3321 break;
3322 case T_HASH:
3323 #if USE_DEBUG_COUNTER
3324 switch (RHASH_SIZE(obj)) {
3325 case 0:
3326 RB_DEBUG_COUNTER_INC(obj_hash_empty);
3327 break;
3328 case 1:
3329 RB_DEBUG_COUNTER_INC(obj_hash_1);
3330 break;
3331 case 2:
3332 RB_DEBUG_COUNTER_INC(obj_hash_2);
3333 break;
3334 case 3:
3335 RB_DEBUG_COUNTER_INC(obj_hash_3);
3336 break;
3337 case 4:
3338 RB_DEBUG_COUNTER_INC(obj_hash_4);
3339 break;
3340 case 5:
3341 case 6:
3342 case 7:
3343 case 8:
3344 RB_DEBUG_COUNTER_INC(obj_hash_5_8);
3345 break;
3346 default:
3347 GC_ASSERT(RHASH_SIZE(obj) > 8);
3348 RB_DEBUG_COUNTER_INC(obj_hash_g8);
3349 }
3350
3351 if (RHASH_AR_TABLE_P(obj)) {
3352 if (RHASH_AR_TABLE(obj) == NULL) {
3353 RB_DEBUG_COUNTER_INC(obj_hash_null);
3354 }
3355 else {
3356 RB_DEBUG_COUNTER_INC(obj_hash_ar);
3357 }
3358 }
3359 else {
3360 RB_DEBUG_COUNTER_INC(obj_hash_st);
3361 }
3362 #endif
3363
3364 rb_hash_free(obj);
3365 break;
3366 case T_REGEXP:
3367 if (RANY(obj)->as.regexp.ptr) {
3368 onig_free(RANY(obj)->as.regexp.ptr);
3369 RB_DEBUG_COUNTER_INC(obj_regexp_ptr);
3370 }
3371 break;
3372 case T_DATA:
3373 if (!rb_data_free(objspace, obj)) return false;
3374 break;
3375 case T_MATCH:
3376 {
3377 rb_matchext_t *rm = RMATCH_EXT(obj);
3378 #if USE_DEBUG_COUNTER
3379 if (rm->regs.num_regs >= 8) {
3380 RB_DEBUG_COUNTER_INC(obj_match_ge8);
3381 }
3382 else if (rm->regs.num_regs >= 4) {
3383 RB_DEBUG_COUNTER_INC(obj_match_ge4);
3384 }
3385 else if (rm->regs.num_regs >= 1) {
3386 RB_DEBUG_COUNTER_INC(obj_match_under4);
3387 }
3388 #endif
3389 onig_region_free(&rm->regs, 0);
3390 xfree(rm->char_offset);
3391
3392 RB_DEBUG_COUNTER_INC(obj_match_ptr);
3393 }
3394 break;
3395 case T_FILE:
3396 if (RANY(obj)->as.file.fptr) {
3397 make_io_zombie(objspace, obj);
3398 RB_DEBUG_COUNTER_INC(obj_file_ptr);
3399 return FALSE;
3400 }
3401 break;
3402 case T_RATIONAL:
3403 RB_DEBUG_COUNTER_INC(obj_rational);
3404 break;
3405 case T_COMPLEX:
3406 RB_DEBUG_COUNTER_INC(obj_complex);
3407 break;
3408 case T_MOVED:
3409 break;
3410 case T_ICLASS:
3411 /* Basically , T_ICLASS shares table with the module */
3412 if (RICLASS_OWNS_M_TBL_P(obj)) {
3413 /* Method table is not shared for origin iclasses of classes */
3414 rb_id_table_free(RCLASS_M_TBL(obj));
3415 }
3416 if (RCLASS_CALLABLE_M_TBL(obj) != NULL) {
3417 rb_id_table_free(RCLASS_CALLABLE_M_TBL(obj));
3418 }
3419 rb_class_remove_subclass_head(obj);
3420 rb_cc_table_free(obj);
3421 rb_class_remove_from_module_subclasses(obj);
3422 rb_class_remove_from_super_subclasses(obj);
3423
3424 RB_DEBUG_COUNTER_INC(obj_iclass_ptr);
3425 break;
3426
3427 case T_FLOAT:
3428 RB_DEBUG_COUNTER_INC(obj_float);
3429 break;
3430
3431 case T_BIGNUM:
3432 if (!BIGNUM_EMBED_P(obj) && BIGNUM_DIGITS(obj)) {
3433 xfree(BIGNUM_DIGITS(obj));
3434 RB_DEBUG_COUNTER_INC(obj_bignum_ptr);
3435 }
3436 else {
3437 RB_DEBUG_COUNTER_INC(obj_bignum_embed);
3438 }
3439 break;
3440
3441 case T_NODE:
3442 UNEXPECTED_NODE(obj_free);
3443 break;
3444
3445 case T_STRUCT:
3446 if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) ||
3447 RANY(obj)->as.rstruct.as.heap.ptr == NULL) {
3448 RB_DEBUG_COUNTER_INC(obj_struct_embed);
3449 }
3450 else {
3451 xfree((void *)RANY(obj)->as.rstruct.as.heap.ptr);
3452 RB_DEBUG_COUNTER_INC(obj_struct_ptr);
3453 }
3454 break;
3455
3456 case T_SYMBOL:
3457 {
3458 rb_gc_free_dsymbol(obj);
3459 RB_DEBUG_COUNTER_INC(obj_symbol);
3460 }
3461 break;
3462
3463 case T_IMEMO:
3464 rb_imemo_free((VALUE)obj);
3465 break;
3466
3467 default:
3468 rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
3469 BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
3470 }
3471
3472 if (FL_TEST(obj, FL_FINALIZE)) {
3473 make_zombie(objspace, obj, 0, 0);
3474 return FALSE;
3475 }
3476 else {
3477 RBASIC(obj)->flags = 0;
3478 return TRUE;
3479 }
3480 }
3481
3482
3483 #define OBJ_ID_INCREMENT (BASE_SLOT_SIZE)
3484 #define OBJ_ID_INITIAL (OBJ_ID_INCREMENT)
3485
3486 static int
3487 object_id_cmp(st_data_t x, st_data_t y)
3488 {
3489 if (RB_BIGNUM_TYPE_P(x)) {
3490 return !rb_big_eql(x, y);
3491 }
3492 else {
3493 return x != y;
3494 }
3495 }
3496
3497 static st_index_t
3498 object_id_hash(st_data_t n)
3499 {
3500 if (RB_BIGNUM_TYPE_P(n)) {
3501 return FIX2LONG(rb_big_hash(n));
3502 }
3503 else {
3504 return st_numhash(n);
3505 }
3506 }
3507 static const struct st_hash_type object_id_hash_type = {
3508 object_id_cmp,
3509 object_id_hash,
3510 };
3511
3512 static void *
3513 rb_gc_impl_objspace_alloc(void)
3514 {
3515 rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
3516 ruby_current_vm_ptr->objspace = objspace;
3517
3518 objspace->flags.gc_stressful = RTEST(initial_stress);
3519 objspace->gc_stress_mode = initial_stress;
3520
3521 objspace->flags.measure_gc = 1;
3522 malloc_limit = gc_params.malloc_limit_min;
3523 objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, objspace);
3524 if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
3525 rb_bug("Could not preregister postponed job for GC");
3526 }
3527
3528 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
3529 rb_size_pool_t *size_pool = &size_pools[i];
3530
3531 size_pool->slot_size = (1 << i) * BASE_SLOT_SIZE;
3532
3533 ccan_list_head_init(&SIZE_POOL_EDEN_HEAP(size_pool)->pages);
3534 ccan_list_head_init(&SIZE_POOL_TOMB_HEAP(size_pool)->pages);
3535 }
3536
3537 rb_darray_make(&objspace->weak_references, 0);
3538
3539 // TODO: debug why on Windows Ruby crashes on boot when GC is on.
3540 #ifdef _WIN32
3541 dont_gc_on();
3542 #endif
3543
3544 #if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
3545 /* Need to determine if we can use mmap at runtime. */
3546 heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
3547 #endif
3548
3549 objspace->next_object_id = OBJ_ID_INITIAL;
3550 objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
3551 objspace->obj_to_id_tbl = st_init_numtable();
3552
3553 #if RGENGC_ESTIMATE_OLDMALLOC
3554 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
3555 #endif
3556
3557 /* Set size pools allocatable pages. */
3558 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
3559 rb_size_pool_t *size_pool = &size_pools[i];
3560
3561 /* Set the default value of size_pool_init_slots. */
3562 gc_params.size_pool_init_slots[i] = GC_HEAP_INIT_SLOTS;
3563
3564 size_pool->allocatable_pages = minimum_pages_for_size_pool(objspace, size_pool);
3565 }
3566 heap_pages_expand_sorted(objspace);
3567
3568 init_mark_stack(&objspace->mark_stack);
3569
3570 objspace->profile.invoke_time = getrusage_time();
3571 finalizer_table = st_init_numtable();
3572 return objspace;
3573 }
3574
3575 typedef int each_obj_callback(void *, void *, size_t, void *);
3576 typedef int each_page_callback(struct heap_page *, void *);
3577
3578 static void objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected);
3579 static void objspace_reachable_objects_from_root(rb_objspace_t *, void (func)(const char *, VALUE, void *), void *);
3580
3581 struct each_obj_data {
3582 rb_objspace_t *objspace;
3583 bool reenable_incremental;
3584
3585 each_obj_callback *each_obj_callback;
3586 each_page_callback *each_page_callback;
3587 void *data;
3588
3589 struct heap_page **pages[SIZE_POOL_COUNT];
3590 size_t pages_counts[SIZE_POOL_COUNT];
3591 };
3592
3593 static VALUE
3594 objspace_each_objects_ensure(VALUE arg)
3595 {
3596 struct each_obj_data *data = (struct each_obj_data *)arg;
3597 rb_objspace_t *objspace = data->objspace;
3598
3599 /* Reenable incremental GC */
3600 if (data->reenable_incremental) {
3601 objspace->flags.dont_incremental = FALSE;
3602 }
3603
3604 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
3605 struct heap_page **pages = data->pages[i];
3606 free(pages);
3607 }
3608
3609 return Qnil;
3610 }
3611
3612 static VALUE
3613 objspace_each_objects_try(VALUE arg)
3614 {
3615 struct each_obj_data *data = (struct each_obj_data *)arg;
3616 rb_objspace_t *objspace = data->objspace;
3617
3618 /* Copy pages from all size_pools to their respective buffers. */
3619 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
3620 rb_size_pool_t *size_pool = &size_pools[i];
3621 size_t size = size_mul_or_raise(SIZE_POOL_EDEN_HEAP(size_pool)->total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
3622
3623 struct heap_page **pages = malloc(size);
3624 if (!pages) rb_memerror();
3625
3626 /* Set up pages buffer by iterating over all pages in the current eden
3627 * heap. This will be a snapshot of the state of the heap before we
3628 * call the callback over each page that exists in this buffer. Thus it
3629 * is safe for the callback to allocate objects without possibly entering
3630 * an infinite loop. */
3631 struct heap_page *page = 0;
3632 size_t pages_count = 0;
3633 ccan_list_for_each(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node) {
3634 pages[pages_count] = page;
3635 pages_count++;
3636 }
3637 data->pages[i] = pages;
3638 data->pages_counts[i] = pages_count;
3639 GC_ASSERT(pages_count == SIZE_POOL_EDEN_HEAP(size_pool)->total_pages);
3640 }
3641
3642 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
3643 rb_size_pool_t *size_pool = &size_pools[i];
3644 size_t pages_count = data->pages_counts[i];
3645 struct heap_page **pages = data->pages[i];
3646
3647 struct heap_page *page = ccan_list_top(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, struct heap_page, page_node);
3648 for (size_t i = 0; i < pages_count; i++) {
3649 /* If we have reached the end of the linked list then there are no
3650 * more pages, so break. */
3651 if (page == NULL) break;
3652
3653 /* If this page does not match the one in the buffer, then move to
3654 * the next page in the buffer. */
3655 if (pages[i] != page) continue;
3656
3657 uintptr_t pstart = (uintptr_t)page->start;
3658 uintptr_t pend = pstart + (page->total_slots * size_pool->slot_size);
3659
3660 if (data->each_obj_callback &&
3661 (*data->each_obj_callback)((void *)pstart, (void *)pend, size_pool->slot_size, data->data)) {
3662 break;
3663 }
3664 if (data->each_page_callback &&
3665 (*data->each_page_callback)(page, data->data)) {
3666 break;
3667 }
3668
3669 page = ccan_list_next(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node);
3670 }
3671 }
3672
3673 return Qnil;
3674 }
3675
3676 /*
3677 * rb_objspace_each_objects() is special C API to walk through
3678 * Ruby object space. This C API is too difficult to use it.
3679 * To be frank, you should not use it. Or you need to read the
3680 * source code of this function and understand what this function does.
3681 *
3682 * 'callback' will be called several times (the number of heap page,
3683 * at current implementation) with:
3684 * vstart: a pointer to the first living object of the heap_page.
3685 * vend: a pointer to next to the valid heap_page area.
3686 * stride: a distance to next VALUE.
3687 *
3688 * If callback() returns non-zero, the iteration will be stopped.
3689 *
3690 * This is a sample callback code to iterate liveness objects:
3691 *
3692 * static int
3693 * sample_callback(void *vstart, void *vend, int stride, void *data)
3694 * {
3695 * VALUE v = (VALUE)vstart;
3696 * for (; v != (VALUE)vend; v += stride) {
3697 * if (!rb_objspace_internal_object_p(v)) { // liveness check
3698 * // do something with live object 'v'
3699 * }
3700 * }
3701 * return 0; // continue to iteration
3702 * }
3703 *
3704 * Note: 'vstart' is not a top of heap_page. This point the first
3705 * living object to grasp at least one object to avoid GC issue.
3706 * This means that you can not walk through all Ruby object page
3707 * including freed object page.
3708 *
3709 * Note: On this implementation, 'stride' is the same as sizeof(RVALUE).
3710 * However, there are possibilities to pass variable values with
3711 * 'stride' with some reasons. You must use stride instead of
3712 * use some constant value in the iteration.
3713 */
3714 void
3715 rb_objspace_each_objects(each_obj_callback *callback, void *data)
3716 {
3717 objspace_each_objects(&rb_objspace, callback, data, TRUE);
3718 }
3719
3720 static void
3721 objspace_each_exec(bool protected, struct each_obj_data *each_obj_data)
3722 {
3723 /* Disable incremental GC */
3724 rb_objspace_t *objspace = each_obj_data->objspace;
3725 bool reenable_incremental = FALSE;
3726 if (protected) {
3727 reenable_incremental = !objspace->flags.dont_incremental;
3728
3729 gc_rest(objspace);
3730 objspace->flags.dont_incremental = TRUE;
3731 }
3732
3733 each_obj_data->reenable_incremental = reenable_incremental;
3734 memset(&each_obj_data->pages, 0, sizeof(each_obj_data->pages));
3735 memset(&each_obj_data->pages_counts, 0, sizeof(each_obj_data->pages_counts));
3736 rb_ensure(objspace_each_objects_try, (VALUE)each_obj_data,
3737 objspace_each_objects_ensure, (VALUE)each_obj_data);
3738 }
3739
3740 static void
3741 objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected)
3742 {
3743 struct each_obj_data each_obj_data = {
3744 .objspace = objspace,
3745 .each_obj_callback = callback,
3746 .each_page_callback = NULL,
3747 .data = data,
3748 };
3749 objspace_each_exec(protected, &each_obj_data);
3750 }
3751
3752 #if GC_CAN_COMPILE_COMPACTION
3753 static void
3754 objspace_each_pages(rb_objspace_t *objspace, each_page_callback *callback, void *data, bool protected)
3755 {
3756 struct each_obj_data each_obj_data = {
3757 .objspace = objspace,
3758 .each_obj_callback = NULL,
3759 .each_page_callback = callback,
3760 .data = data,
3761 };
3762 objspace_each_exec(protected, &each_obj_data);
3763 }
3764 #endif
3765
3766 struct os_each_struct {
3767 size_t num;
3768 VALUE of;
3769 };
3770
3771 static int
3772 internal_object_p(VALUE obj)
3773 {
3774 RVALUE *p = (RVALUE *)obj;
3775 void *ptr = asan_unpoison_object_temporary(obj);
3776 bool used_p = p->as.basic.flags;
3777
3778 if (used_p) {
3779 switch (BUILTIN_TYPE(obj)) {
3780 case T_NODE:
3781 UNEXPECTED_NODE(internal_object_p);
3782 break;
3783 case T_NONE:
3784 case T_MOVED:
3785 case T_IMEMO:
3786 case T_ICLASS:
3787 case T_ZOMBIE:
3788 break;
3789 case T_CLASS:
3790 if (!p->as.basic.klass) break;
3791 if (RCLASS_SINGLETON_P(obj)) {
3792 return rb_singleton_class_internal_p(obj);
3793 }
3794 return 0;
3795 default:
3796 if (!p->as.basic.klass) break;
3797 return 0;
3798 }
3799 }
3800 if (ptr || ! used_p) {
3801 asan_poison_object(obj);
3802 }
3803 return 1;
3804 }
3805
3806 int
3807 rb_objspace_internal_object_p(VALUE obj)
3808 {
3809 return internal_object_p(obj);
3810 }
3811
3812 static int
3813 os_obj_of_i(void *vstart, void *vend, size_t stride, void *data)
3814 {
3815 struct os_each_struct *oes = (struct os_each_struct *)data;
3816
3817 VALUE v = (VALUE)vstart;
3818 for (; v != (VALUE)vend; v += stride) {
3819 if (!internal_object_p(v)) {
3820 if (!oes->of || rb_obj_is_kind_of(v, oes->of)) {
3821 if (!rb_multi_ractor_p() || rb_ractor_shareable_p(v)) {
3822 rb_yield(v);
3823 oes->num++;
3824 }
3825 }
3826 }
3827 }
3828
3829 return 0;
3830 }
3831
3832 static VALUE
3833 os_obj_of(VALUE of)
3834 {
3835 struct os_each_struct oes;
3836
3837 oes.num = 0;
3838 oes.of = of;
3839 rb_objspace_each_objects(os_obj_of_i, &oes);
3840 return SIZET2NUM(oes.num);
3841 }
3842
3843 /*
3844 * call-seq:
3845 * ObjectSpace.each_object([module]) {|obj| ... } -> integer
3846 * ObjectSpace.each_object([module]) -> an_enumerator
3847 *
3848 * Calls the block once for each living, nonimmediate object in this
3849 * Ruby process. If <i>module</i> is specified, calls the block
3850 * for only those classes or modules that match (or are a subclass of)
3851 * <i>module</i>. Returns the number of objects found. Immediate
3852 * objects (<code>Fixnum</code>s, <code>Symbol</code>s
3853 * <code>true</code>, <code>false</code>, and <code>nil</code>) are
3854 * never returned. In the example below, #each_object returns both
3855 * the numbers we defined and several constants defined in the Math
3856 * module.
3857 *
3858 * If no block is given, an enumerator is returned instead.
3859 *
3860 * a = 102.7
3861 * b = 95 # Won't be returned
3862 * c = 12345678987654321
3863 * count = ObjectSpace.each_object(Numeric) {|x| p x }
3864 * puts "Total count: #{count}"
3865 *
3866 * <em>produces:</em>
3867 *
3868 * 12345678987654321
3869 * 102.7
3870 * 2.71828182845905
3871 * 3.14159265358979
3872 * 2.22044604925031e-16
3873 * 1.7976931348623157e+308
3874 * 2.2250738585072e-308
3875 * Total count: 7
3876 *
3877 */
3878
3879 static VALUE
3880 os_each_obj(int argc, VALUE *argv, VALUE os)
3881 {
3882 VALUE of;
3883
3884 of = (!rb_check_arity(argc, 0, 1) ? 0 : argv[0]);
3885 RETURN_ENUMERATOR(os, 1, &of);
3886 return os_obj_of(of);
3887 }
3888
3889 /*
3890 * call-seq:
3891 * ObjectSpace.undefine_finalizer(obj)
3892 *
3893 * Removes all finalizers for <i>obj</i>.
3894 *
3895 */
3896
3897 static VALUE
3898 undefine_final(VALUE os, VALUE obj)
3899 {
3900 return rb_undefine_finalizer(obj);
3901 }
3902
3903 VALUE
3904 rb_undefine_finalizer(VALUE obj)
3905 {
3906 rb_objspace_t *objspace = &rb_objspace;
3907 st_data_t data = obj;
3908 rb_check_frozen(obj);
3909 st_delete(finalizer_table, &data, 0);
3910 FL_UNSET(obj, FL_FINALIZE);
3911 return obj;
3912 }
3913
3914 static void
3915 should_be_callable(VALUE block)
3916 {
3917 if (!rb_obj_respond_to(block, idCall, TRUE)) {
3918 rb_raise(rb_eArgError, "wrong type argument %"PRIsVALUE" (should be callable)",
3919 rb_obj_class(block));
3920 }
3921 }
3922
3923 static void
3924 should_be_finalizable(VALUE obj)
3925 {
3926 if (!FL_ABLE(obj)) {
3927 rb_raise(rb_eArgError, "cannot define finalizer for %s",
3928 rb_obj_classname(obj));
3929 }
3930 rb_check_frozen(obj);
3931 }
3932
3933 static VALUE
3934 rb_define_finalizer_no_check(VALUE obj, VALUE block)
3935 {
3936 rb_objspace_t *objspace = &rb_objspace;
3937 VALUE table;
3938 st_data_t data;
3939
3940 RBASIC(obj)->flags |= FL_FINALIZE;
3941
3942 if (st_lookup(finalizer_table, obj, &data)) {
3943 table = (VALUE)data;
3944
3945 /* avoid duplicate block, table is usually small */
3946 {
3947 long len = RARRAY_LEN(table);
3948 long i;
3949
3950 for (i = 0; i < len; i++) {
3951 VALUE recv = RARRAY_AREF(table, i);
3952 if (rb_equal(recv, block)) {
3953 block = recv;
3954 goto end;
3955 }
3956 }
3957 }
3958
3959 rb_ary_push(table, block);
3960 }
3961 else {
3962 table = rb_ary_new3(1, block);
3963 RBASIC_CLEAR_CLASS(table);
3964 st_add_direct(finalizer_table, obj, table);
3965 }
3966 end:
3967 block = rb_ary_new3(2, INT2FIX(0), block);
3968 OBJ_FREEZE(block);
3969 return block;
3970 }
3971
3972 /*
3973 * call-seq:
3974 * ObjectSpace.define_finalizer(obj, aProc=proc())
3975 *
3976 * Adds <i>aProc</i> as a finalizer, to be called after <i>obj</i>
3977 * was destroyed. The object ID of the <i>obj</i> will be passed
3978 * as an argument to <i>aProc</i>. If <i>aProc</i> is a lambda or
3979 * method, make sure it can be called with a single argument.
3980 *
3981 * The return value is an array <code>[0, aProc]</code>.
3982 *
3983 * The two recommended patterns are to either create the finaliser proc
3984 * in a non-instance method where it can safely capture the needed state,
3985 * or to use a custom callable object that stores the needed state
3986 * explicitly as instance variables.
3987 *
3988 * class Foo
3989 * def initialize(data_needed_for_finalization)
3990 * ObjectSpace.define_finalizer(self, self.class.create_finalizer(data_needed_for_finalization))
3991 * end
3992 *
3993 * def self.create_finalizer(data_needed_for_finalization)
3994 * proc {
3995 * puts "finalizing #{data_needed_for_finalization}"
3996 * }
3997 * end
3998 * end
3999 *
4000 * class Bar
4001 * class Remover
4002 * def initialize(data_needed_for_finalization)
4003 * @data_needed_for_finalization = data_needed_for_finalization
4004 * end
4005 *
4006 * def call(id)
4007 * puts "finalizing #{@data_needed_for_finalization}"
4008 * end
4009 * end
4010 *
4011 * def initialize(data_needed_for_finalization)
4012 * ObjectSpace.define_finalizer(self, Remover.new(data_needed_for_finalization))
4013 * end
4014 * end
4015 *
4016 * Note that if your finalizer references the object to be
4017 * finalized it will never be run on GC, although it will still be
4018 * run at exit. You will get a warning if you capture the object
4019 * to be finalized as the receiver of the finalizer.
4020 *
4021 * class CapturesSelf
4022 * def initialize(name)
4023 * ObjectSpace.define_finalizer(self, proc {
4024 * # this finalizer will only be run on exit
4025 * puts "finalizing #{name}"
4026 * })
4027 * end
4028 * end
4029 *
4030 * Also note that finalization can be unpredictable and is never guaranteed
4031 * to be run except on exit.
4032 */
4033
4034 static VALUE
4035 define_final(int argc, VALUE *argv, VALUE os)
4036 {
4037 VALUE obj, block;
4038
4039 rb_scan_args(argc, argv, "11", &obj, &block);
4040 should_be_finalizable(obj);
4041 if (argc == 1) {
4042 block = rb_block_proc();
4043 }
4044 else {
4045 should_be_callable(block);
4046 }
4047
4048 if (rb_callable_receiver(block) == obj) {
4049 rb_warn("finalizer references object to be finalized");
4050 }
4051
4052 return rb_define_finalizer_no_check(obj, block);
4053 }
4054
4055 VALUE
4056 rb_define_finalizer(VALUE obj, VALUE block)
4057 {
4058 should_be_finalizable(obj);
4059 should_be_callable(block);
4060 return rb_define_finalizer_no_check(obj, block);
4061 }
4062
4063 void
4064 rb_gc_copy_finalizer(VALUE dest, VALUE obj)
4065 {
4066 rb_objspace_t *objspace = &rb_objspace;
4067 VALUE table;
4068 st_data_t data;
4069
4070 if (!FL_TEST(obj, FL_FINALIZE)) return;
4071
4072 if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
4073 table = (VALUE)data;
4074 st_insert(finalizer_table, dest, table);
4075 FL_SET(dest, FL_FINALIZE);
4076 }
4077 else {
4078 rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", obj_info(obj));
4079 }
4080 }
4081
4082 static VALUE
4083 run_single_final(VALUE cmd, VALUE objid)
4084 {
4085 return rb_check_funcall(cmd, idCall, 1, &objid);
4086 }
4087
4088 static void
4089 warn_exception_in_finalizer(rb_execution_context_t *ec, VALUE final)
4090 {
4091 if (!UNDEF_P(final) && !NIL_P(ruby_verbose)) {
4092 VALUE errinfo = ec->errinfo;
4093 rb_warn("Exception in finalizer %+"PRIsVALUE, final);
4094 rb_ec_error_print(ec, errinfo);
4095 }
4096 }
4097
4098 static void
4099 run_finalizer(rb_objspace_t *objspace, VALUE obj, VALUE table)
4100 {
4101 long i;
4102 enum ruby_tag_type state;
4103 volatile struct {
4104 VALUE errinfo;
4105 VALUE objid;
4106 VALUE final;
4107 rb_control_frame_t *cfp;
4108 VALUE *sp;
4109 long finished;
4110 } saved;
4111
4112 rb_execution_context_t * volatile ec = GET_EC();
4113 #define RESTORE_FINALIZER() (\
4114 ec->cfp = saved.cfp, \
4115 ec->cfp->sp = saved.sp, \
4116 ec->errinfo = saved.errinfo)
4117
4118 saved.errinfo = ec->errinfo;
4119 saved.objid = rb_obj_id(obj);
4120 saved.cfp = ec->cfp;
4121 saved.sp = ec->cfp->sp;
4122 saved.finished = 0;
4123 saved.final = Qundef;
4124
4125 EC_PUSH_TAG(ec);
4126 state = EC_EXEC_TAG();
4127 if (state != TAG_NONE) {
4128 ++saved.finished; /* skip failed finalizer */
4129 warn_exception_in_finalizer(ec, ATOMIC_VALUE_EXCHANGE(saved.final, Qundef));
4130 }
4131 for (i = saved.finished;
4132 RESTORE_FINALIZER(), i<RARRAY_LEN(table);
4133 saved.finished = ++i) {
4134 run_single_final(saved.final = RARRAY_AREF(table, i), saved.objid);
4135 }
4136 EC_POP_TAG();
4137 #undef RESTORE_FINALIZER
4138 }
4139
4140 static void
4141 run_final(rb_objspace_t *objspace, VALUE zombie)
4142 {
4143 if (RZOMBIE(zombie)->dfree) {
4144 RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
4145 }
4146
4147 st_data_t key = (st_data_t)zombie;
4148 if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
4149 FL_UNSET(zombie, FL_FINALIZE);
4150 st_data_t table;
4151 if (st_delete(finalizer_table, &key, &table)) {
4152 run_finalizer(objspace, zombie, (VALUE)table);
4153 }
4154 else {
4155 rb_bug("FL_FINALIZE flag is set, but finalizers are not found");
4156 }
4157 }
4158 else {
4159 GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
4160 }
4161 }
4162
4163 static void
4164 finalize_list(rb_objspace_t *objspace, VALUE zombie)
4165 {
4166 while (zombie) {
4167 VALUE next_zombie;
4168 struct heap_page *page;
4169 asan_unpoison_object(zombie, false);
4170 next_zombie = RZOMBIE(zombie)->next;
4171 page = GET_HEAP_PAGE(zombie);
4172
4173 run_final(objspace, zombie);
4174
4175 RB_VM_LOCK_ENTER();
4176 {
4177 GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
4178 if (FL_TEST(zombie, FL_SEEN_OBJ_ID)) {
4179 obj_free_object_id(objspace, zombie);
4180 }
4181
4182 GC_ASSERT(heap_pages_final_slots > 0);
4183 GC_ASSERT(page->final_slots > 0);
4184
4185 heap_pages_final_slots--;
4186 page->final_slots--;
4187 page->free_slots++;
4188 heap_page_add_freeobj(objspace, page, zombie);
4189 page->size_pool->total_freed_objects++;
4190 }
4191 RB_VM_LOCK_LEAVE();
4192
4193 zombie = next_zombie;
4194 }
4195 }
4196
4197 static void
4198 finalize_deferred_heap_pages(rb_objspace_t *objspace)
4199 {
4200 VALUE zombie;
4201 while ((zombie = ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
4202 finalize_list(objspace, zombie);
4203 }
4204 }
4205
4206 static void
4207 finalize_deferred(rb_objspace_t *objspace)
4208 {
4209 rb_execution_context_t *ec = GET_EC();
4210 ec->interrupt_mask |= PENDING_INTERRUPT_MASK;
4211 finalize_deferred_heap_pages(objspace);
4212 ec->interrupt_mask &= ~PENDING_INTERRUPT_MASK;
4213 }
4214
4215 static void
4216 gc_finalize_deferred(void *dmy)
4217 {
4218 rb_objspace_t *objspace = dmy;
4219 if (ATOMIC_EXCHANGE(finalizing, 1)) return;
4220
4221 finalize_deferred(objspace);
4222 ATOMIC_SET(finalizing, 0);
4223 }
4224
4225 static void
4226 gc_finalize_deferred_register(rb_objspace_t *objspace)
4227 {
4228 /* will enqueue a call to gc_finalize_deferred */
4229 rb_postponed_job_trigger(objspace->finalize_deferred_pjob);
4230 }
4231
4232 static int pop_mark_stack(mark_stack_t *stack, VALUE *data);
4233
4234 static void
4235 gc_abort(rb_objspace_t *objspace)
4236 {
4237 if (is_incremental_marking(objspace)) {
4238 /* Remove all objects from the mark stack. */
4239 VALUE obj;
4240 while (pop_mark_stack(&objspace->mark_stack, &obj));
4241
4242 objspace->flags.during_incremental_marking = FALSE;
4243 }
4244
4245 if (is_lazy_sweeping(objspace)) {
4246 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
4247 rb_size_pool_t *size_pool = &size_pools[i];
4248 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
4249
4250 heap->sweeping_page = NULL;
4251 struct heap_page *page = NULL;
4252
4253 ccan_list_for_each(&heap->pages, page, page_node) {
4254 page->flags.before_sweep = false;
4255 }
4256 }
4257 }
4258
4259 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
4260 rb_size_pool_t *size_pool = &size_pools[i];
4261 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
4262 rgengc_mark_and_rememberset_clear(objspace, heap);
4263 }
4264
4265 gc_mode_set(objspace, gc_mode_none);
4266 }
4267
4268 struct force_finalize_list {
4269 VALUE obj;
4270 VALUE table;
4271 struct force_finalize_list *next;
4272 };
4273
4274 static int
4275 force_chain_object(st_data_t key, st_data_t val, st_data_t arg)
4276 {
4277 struct force_finalize_list **prev = (struct force_finalize_list **)arg;
4278 struct force_finalize_list *curr = ALLOC(struct force_finalize_list);
4279 curr->obj = key;
4280 curr->table = val;
4281 curr->next = *prev;
4282 *prev = curr;
4283 return ST_CONTINUE;
4284 }
4285
4286 static void
4287 gc_each_object(rb_objspace_t *objspace, void (*func)(VALUE obj, void *data), void *data)
4288 {
4289 for (size_t i = 0; i < heap_allocated_pages; i++) {
4290 struct heap_page *page = heap_pages_sorted[i];
4291 short stride = page->slot_size;
4292
4293 uintptr_t p = (uintptr_t)page->start;
4294 uintptr_t pend = p + page->total_slots * stride;
4295 for (; p < pend; p += stride) {
4296 VALUE obj = (VALUE)p;
4297
4298 void *poisoned = asan_unpoison_object_temporary(obj);
4299
4300 func(obj, data);
4301
4302 if (poisoned) {
4303 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
4304 asan_poison_object(obj);
4305 }
4306 }
4307 }
4308 }
4309
4310 bool rb_obj_is_main_ractor(VALUE gv);
4311
4312 static void
4313 rb_objspace_free_objects_i(VALUE obj, void *data)
4314 {
4315 rb_objspace_t *objspace = (rb_objspace_t *)data;
4316
4317 switch (BUILTIN_TYPE(obj)) {
4318 case T_NONE:
4319 case T_SYMBOL:
4320 break;
4321 default:
4322 obj_free(objspace, obj);
4323 break;
4324 }
4325 }
4326
4327 void
4328 rb_objspace_free_objects(rb_objspace_t *objspace)
4329 {
4330 gc_each_object(objspace, rb_objspace_free_objects_i, objspace);
4331 }
4332
4333 static void
4334 rb_objspace_call_finalizer_i(VALUE obj, void *data)
4335 {
4336 rb_objspace_t *objspace = (rb_objspace_t *)data;
4337
4338 switch (BUILTIN_TYPE(obj)) {
4339 case T_DATA:
4340 if (!rb_free_at_exit && (!DATA_PTR(obj) || !RANY(obj)->as.data.dfree)) break;
4341 if (rb_obj_is_thread(obj)) break;
4342 if (rb_obj_is_mutex(obj)) break;
4343 if (rb_obj_is_fiber(obj)) break;
4344 if (rb_obj_is_main_ractor(obj)) break;
4345
4346 obj_free(objspace, obj);
4347 break;
4348 case T_FILE:
4349 obj_free(objspace, obj);
4350 break;
4351 case T_SYMBOL:
4352 case T_ARRAY:
4353 case T_NONE:
4354 break;
4355 default:
4356 if (rb_free_at_exit) {
4357 obj_free(objspace, obj);
4358 }
4359 break;
4360 }
4361 }
4362
4363 void
4364 rb_objspace_call_finalizer(rb_objspace_t *objspace)
4365 {
4366 #if RGENGC_CHECK_MODE >= 2
4367 gc_verify_internal_consistency(objspace);
4368 #endif
4369 if (ATOMIC_EXCHANGE(finalizing, 1)) return;
4370
4371 /* run finalizers */
4372 finalize_deferred(objspace);
4373 GC_ASSERT(heap_pages_deferred_final == 0);
4374
4375 /* prohibit incremental GC */
4376 objspace->flags.dont_incremental = 1;
4377
4378 /* force to run finalizer */
4379 while (finalizer_table->num_entries) {
4380 struct force_finalize_list *list = 0;
4381 st_foreach(finalizer_table, force_chain_object, (st_data_t)&list);
4382 while (list) {
4383 struct force_finalize_list *curr = list;
4384
4385 st_data_t obj = (st_data_t)curr->obj;
4386 st_delete(finalizer_table, &obj, 0);
4387 FL_UNSET(curr->obj, FL_FINALIZE);
4388
4389 run_finalizer(objspace, curr->obj, curr->table);
4390
4391 list = curr->next;
4392 xfree(curr);
4393 }
4394 }
4395
4396 /* Abort incremental marking and lazy sweeping to speed up shutdown. */
4397 gc_abort(objspace);
4398
4399 /* prohibit GC because force T_DATA finalizers can break an object graph consistency */
4400 dont_gc_on();
4401
4402 /* running data/file finalizers are part of garbage collection */
4403 unsigned int lock_lev;
4404 gc_enter(objspace, gc_enter_event_finalizer, &lock_lev);
4405
4406 gc_each_object(objspace, rb_objspace_call_finalizer_i, objspace);
4407
4408 gc_exit(objspace, gc_enter_event_finalizer, &lock_lev);
4409
4410 finalize_deferred_heap_pages(objspace);
4411
4412 st_free_table(finalizer_table);
4413 finalizer_table = 0;
4414 ATOMIC_SET(finalizing, 0);
4415 }
4416
4417 /* garbage objects will be collected soon. */
4418 static inline bool
4419 is_garbage_object(rb_objspace_t *objspace, VALUE ptr)
4420 {
4421 return is_lazy_sweeping(objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep &&
4422 !MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(ptr), ptr);
4423 }
4424
4425 static inline bool
4426 is_live_object(rb_objspace_t *objspace, VALUE ptr)
4427 {
4428 switch (BUILTIN_TYPE(ptr)) {
4429 case T_NONE:
4430 case T_MOVED:
4431 case T_ZOMBIE:
4432 return FALSE;
4433 default:
4434 break;
4435 }
4436
4437 return !is_garbage_object(objspace, ptr);
4438 }
4439
4440 static inline int
4441 is_markable_object(VALUE obj)
4442 {
4443 return !RB_SPECIAL_CONST_P(obj);
4444 }
4445
4446 int
4447 rb_objspace_markable_object_p(VALUE obj)
4448 {
4449 rb_objspace_t *objspace = &rb_objspace;
4450 return is_markable_object(obj) && is_live_object(objspace, obj);
4451 }
4452
4453 int
4454 rb_objspace_garbage_object_p(VALUE obj)
4455 {
4456 rb_objspace_t *objspace = &rb_objspace;
4457 return is_garbage_object(objspace, obj);
4458 }
4459
4460 bool
4461 rb_gc_is_ptr_to_obj(const void *ptr)
4462 {
4463 rb_objspace_t *objspace = &rb_objspace;
4464 return is_pointer_to_heap(objspace, ptr);
4465 }
4466
4467 /*
4468 * call-seq:
4469 * ObjectSpace._id2ref(object_id) -> an_object
4470 *
4471 * Converts an object id to a reference to the object. May not be
4472 * called on an object id passed as a parameter to a finalizer.
4473 *
4474 * s = "I am a string" #=> "I am a string"
4475 * r = ObjectSpace._id2ref(s.object_id) #=> "I am a string"
4476 * r == s #=> true
4477 *
4478 * On multi-ractor mode, if the object is not shareable, it raises
4479 * RangeError.
4480 */
4481
4482 static VALUE
4483 id2ref(VALUE objid)
4484 {
4485 #if SIZEOF_LONG == SIZEOF_VOIDP
4486 #define NUM2PTR(x) NUM2ULONG(x)
4487 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
4488 #define NUM2PTR(x) NUM2ULL(x)
4489 #endif
4490 rb_objspace_t *objspace = &rb_objspace;
4491
4492 objid = rb_to_int(objid);
4493 if (FIXNUM_P(objid) || rb_big_size(objid) <= SIZEOF_VOIDP) {
4494 VALUE ptr = NUM2PTR(objid);
4495 if (SPECIAL_CONST_P(ptr)) {
4496 if (ptr == Qtrue) return Qtrue;
4497 if (ptr == Qfalse) return Qfalse;
4498 if (NIL_P(ptr)) return Qnil;
4499 if (FIXNUM_P(ptr)) return ptr;
4500 if (FLONUM_P(ptr)) return ptr;
4501
4502 if (SYMBOL_P(ptr)) {
4503 // Check that the symbol is valid
4504 if (rb_static_id_valid_p(SYM2ID(ptr))) {
4505 return ptr;
4506 }
4507 else {
4508 rb_raise(rb_eRangeError, "%p is not symbol id value", (void *)ptr);
4509 }
4510 }
4511
4512 rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_int2str(objid, 10));
4513 }
4514 }
4515
4516 VALUE orig;
4517 if (st_lookup(objspace->id_to_obj_tbl, objid, &orig) &&
4518 is_live_object(objspace, orig)) {
4519 if (!rb_multi_ractor_p() || rb_ractor_shareable_p(orig)) {
4520 return orig;
4521 }
4522 else {
4523 rb_raise(rb_eRangeError, "%+"PRIsVALUE" is id of the unshareable object on multi-ractor", rb_int2str(objid, 10));
4524 }
4525 }
4526
4527 if (rb_int_ge(objid, ULL2NUM(objspace->next_object_id))) {
4528 rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_int2str(objid, 10));
4529 }
4530 else {
4531 rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_int2str(objid, 10));
4532 }
4533 }
4534
4535 /* :nodoc: */
4536 static VALUE
4537 os_id2ref(VALUE os, VALUE objid)
4538 {
4539 return id2ref(objid);
4540 }
4541
4542 static VALUE
4543 rb_find_object_id(VALUE obj, VALUE (*get_heap_object_id)(VALUE))
4544 {
4545 if (SPECIAL_CONST_P(obj)) {
4546 #if SIZEOF_LONG == SIZEOF_VOIDP
4547 return LONG2NUM((SIGNED_VALUE)obj);
4548 #else
4549 return LL2NUM((SIGNED_VALUE)obj);
4550 #endif
4551 }
4552
4553 return get_heap_object_id(obj);
4554 }
4555
4556 static VALUE
4557 cached_object_id(VALUE obj)
4558 {
4559 VALUE id;
4560 rb_objspace_t *objspace = &rb_objspace;
4561
4562 RB_VM_LOCK_ENTER();
4563 if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &id)) {
4564 GC_ASSERT(FL_TEST(obj, FL_SEEN_OBJ_ID));
4565 }
4566 else {
4567 GC_ASSERT(!FL_TEST(obj, FL_SEEN_OBJ_ID));
4568
4569 id = ULL2NUM(objspace->next_object_id);
4570 objspace->next_object_id += OBJ_ID_INCREMENT;
4571
4572 VALUE already_disabled = rb_gc_disable_no_rest();
4573 st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
4574 st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
4575 if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
4576 FL_SET(obj, FL_SEEN_OBJ_ID);
4577 }
4578 RB_VM_LOCK_LEAVE();
4579
4580 return id;
4581 }
4582
4583 static VALUE
4584 nonspecial_obj_id(VALUE obj)
4585 {
4586 #if SIZEOF_LONG == SIZEOF_VOIDP
4587 return (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG);
4588 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
4589 return LL2NUM((SIGNED_VALUE)(obj) / 2);
4590 #else
4591 # error not supported
4592 #endif
4593 }
4594
4595 VALUE
4596 rb_memory_id(VALUE obj)
4597 {
4598 return rb_find_object_id(obj, nonspecial_obj_id);
4599 }
4600
4601 /*
4602 * Document-method: __id__
4603 * Document-method: object_id
4604 *
4605 * call-seq:
4606 * obj.__id__ -> integer
4607 * obj.object_id -> integer
4608 *
4609 * Returns an integer identifier for +obj+.
4610 *
4611 * The same number will be returned on all calls to +object_id+ for a given
4612 * object, and no two active objects will share an id.
4613 *
4614 * Note: that some objects of builtin classes are reused for optimization.
4615 * This is the case for immediate values and frozen string literals.
4616 *
4617 * BasicObject implements +__id__+, Kernel implements +object_id+.
4618 *
4619 * Immediate values are not passed by reference but are passed by value:
4620 * +nil+, +true+, +false+, Fixnums, Symbols, and some Floats.
4621 *
4622 * Object.new.object_id == Object.new.object_id # => false
4623 * (21 * 2).object_id == (21 * 2).object_id # => true
4624 * "hello".object_id == "hello".object_id # => false
4625 * "hi".freeze.object_id == "hi".freeze.object_id # => true
4626 */
4627
4628 VALUE
4629 rb_obj_id(VALUE obj)
4630 {
4631 /*
4632 * 32-bit VALUE space
4633 * MSB ------------------------ LSB
4634 * false 00000000000000000000000000000000
4635 * true 00000000000000000000000000000010
4636 * nil 00000000000000000000000000000100
4637 * undef 00000000000000000000000000000110
4638 * symbol ssssssssssssssssssssssss00001110
4639 * object oooooooooooooooooooooooooooooo00 = 0 (mod sizeof(RVALUE))
4640 * fixnum fffffffffffffffffffffffffffffff1
4641 *
4642 * object_id space
4643 * LSB
4644 * false 00000000000000000000000000000000
4645 * true 00000000000000000000000000000010
4646 * nil 00000000000000000000000000000100
4647 * undef 00000000000000000000000000000110
4648 * symbol 000SSSSSSSSSSSSSSSSSSSSSSSSSSS0 S...S % A = 4 (S...S = s...s * A + 4)
4649 * object oooooooooooooooooooooooooooooo0 o...o % A = 0
4650 * fixnum fffffffffffffffffffffffffffffff1 bignum if required
4651 *
4652 * where A = sizeof(RVALUE)/4
4653 *
4654 * sizeof(RVALUE) is
4655 * 20 if 32-bit, double is 4-byte aligned
4656 * 24 if 32-bit, double is 8-byte aligned
4657 * 40 if 64-bit
4658 */
4659
4660 return rb_find_object_id(obj, cached_object_id);
4661 }
4662
4663 static enum rb_id_table_iterator_result
4664 cc_table_memsize_i(VALUE ccs_ptr, void *data_ptr)
4665 {
4666 size_t *total_size = data_ptr;
4667 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
4668 *total_size += sizeof(*ccs);
4669 *total_size += sizeof(ccs->entries[0]) * ccs->capa;
4670 return ID_TABLE_CONTINUE;
4671 }
4672
4673 static size_t
4674 cc_table_memsize(struct rb_id_table *cc_table)
4675 {
4676 size_t total = rb_id_table_memsize(cc_table);
4677 rb_id_table_foreach_values(cc_table, cc_table_memsize_i, &total);
4678 return total;
4679 }
4680
4681 static size_t
4682 obj_memsize_of(VALUE obj, int use_all_types)
4683 {
4684 size_t size = 0;
4685
4686 if (SPECIAL_CONST_P(obj)) {
4687 return 0;
4688 }
4689
4690 if (FL_TEST(obj, FL_EXIVAR)) {
4691 size += rb_generic_ivar_memsize(obj);
4692 }
4693
4694 switch (BUILTIN_TYPE(obj)) {
4695 case T_OBJECT:
4696 if (rb_shape_obj_too_complex(obj)) {
4697 size += rb_st_memsize(ROBJECT_IV_HASH(obj));
4698 }
4699 else if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) {
4700 size += ROBJECT_IV_CAPACITY(obj) * sizeof(VALUE);
4701 }
4702 break;
4703 case T_MODULE:
4704 case T_CLASS:
4705 if (RCLASS_M_TBL(obj)) {
4706 size += rb_id_table_memsize(RCLASS_M_TBL(obj));
4707 }
4708 // class IV sizes are allocated as powers of two
4709 size += SIZEOF_VALUE << bit_length(RCLASS_IV_COUNT(obj));
4710 if (RCLASS_CVC_TBL(obj)) {
4711 size += rb_id_table_memsize(RCLASS_CVC_TBL(obj));
4712 }
4713 if (RCLASS_EXT(obj)->const_tbl) {
4714 size += rb_id_table_memsize(RCLASS_EXT(obj)->const_tbl);
4715 }
4716 if (RCLASS_CC_TBL(obj)) {
4717 size += cc_table_memsize(RCLASS_CC_TBL(obj));
4718 }
4719 if (FL_TEST_RAW(obj, RCLASS_SUPERCLASSES_INCLUDE_SELF)) {
4720 size += (RCLASS_SUPERCLASS_DEPTH(obj) + 1) * sizeof(VALUE);
4721 }
4722 break;
4723 case T_ICLASS:
4724 if (RICLASS_OWNS_M_TBL_P(obj)) {
4725 if (RCLASS_M_TBL(obj)) {
4726 size += rb_id_table_memsize(RCLASS_M_TBL(obj));
4727 }
4728 }
4729 if (RCLASS_CC_TBL(obj)) {
4730 size += cc_table_memsize(RCLASS_CC_TBL(obj));
4731 }
4732 break;
4733 case T_STRING:
4734 size += rb_str_memsize(obj);
4735 break;
4736 case T_ARRAY:
4737 size += rb_ary_memsize(obj);
4738 break;
4739 case T_HASH:
4740 if (RHASH_ST_TABLE_P(obj)) {
4741 VM_ASSERT(RHASH_ST_TABLE(obj) != NULL);
4742 /* st_table is in the slot */
4743 size += st_memsize(RHASH_ST_TABLE(obj)) - sizeof(st_table);
4744 }
4745 break;
4746 case T_REGEXP:
4747 if (RREGEXP_PTR(obj)) {
4748 size += onig_memsize(RREGEXP_PTR(obj));
4749 }
4750 break;
4751 case T_DATA:
4752 if (use_all_types) size += rb_objspace_data_type_memsize(obj);
4753 break;
4754 case T_MATCH:
4755 {
4756 rb_matchext_t *rm = RMATCH_EXT(obj);
4757 size += onig_region_memsize(&rm->regs);
4758 size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
4759 }
4760 break;
4761 case T_FILE:
4762 if (RFILE(obj)->fptr) {
4763 size += rb_io_memsize(RFILE(obj)->fptr);
4764 }
4765 break;
4766 case T_RATIONAL:
4767 case T_COMPLEX:
4768 break;
4769 case T_IMEMO:
4770 size += rb_imemo_memsize(obj);
4771 break;
4772
4773 case T_FLOAT:
4774 case T_SYMBOL:
4775 break;
4776
4777 case T_BIGNUM:
4778 if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
4779 size += BIGNUM_LEN(obj) * sizeof(BDIGIT);
4780 }
4781 break;
4782
4783 case T_NODE:
4784 UNEXPECTED_NODE(obj_memsize_of);
4785 break;
4786
4787 case T_STRUCT:
4788 if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
4789 RSTRUCT(obj)->as.heap.ptr) {
4790 size += sizeof(VALUE) * RSTRUCT_LEN(obj);
4791 }
4792 break;
4793
4794 case T_ZOMBIE:
4795 case T_MOVED:
4796 break;
4797
4798 default:
4799 rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
4800 BUILTIN_TYPE(obj), (void*)obj);
4801 }
4802
4803 return size + rb_gc_obj_slot_size(obj);
4804 }
4805
4806 size_t
4807 rb_obj_memsize_of(VALUE obj)
4808 {
4809 return obj_memsize_of(obj, TRUE);
4810 }
4811
4812 static int
4813 set_zero(st_data_t key, st_data_t val, st_data_t arg)
4814 {
4815 VALUE k = (VALUE)key;
4816 VALUE hash = (VALUE)arg;
4817 rb_hash_aset(hash, k, INT2FIX(0));
4818 return ST_CONTINUE;
4819 }
4820
4821 static VALUE
4822 type_sym(size_t type)
4823 {
4824 switch (type) {
4825 #define COUNT_TYPE(t) case (t): return ID2SYM(rb_intern(#t)); break;
4826 COUNT_TYPE(T_NONE);
4827 COUNT_TYPE(T_OBJECT);
4828 COUNT_TYPE(T_CLASS);
4829 COUNT_TYPE(T_MODULE);
4830 COUNT_TYPE(T_FLOAT);
4831 COUNT_TYPE(T_STRING);
4832 COUNT_TYPE(T_REGEXP);
4833 COUNT_TYPE(T_ARRAY);
4834 COUNT_TYPE(T_HASH);
4835 COUNT_TYPE(T_STRUCT);
4836 COUNT_TYPE(T_BIGNUM);
4837 COUNT_TYPE(T_FILE);
4838 COUNT_TYPE(T_DATA);
4839 COUNT_TYPE(T_MATCH);
4840 COUNT_TYPE(T_COMPLEX);
4841 COUNT_TYPE(T_RATIONAL);
4842 COUNT_TYPE(T_NIL);
4843 COUNT_TYPE(T_TRUE);
4844 COUNT_TYPE(T_FALSE);
4845 COUNT_TYPE(T_SYMBOL);
4846 COUNT_TYPE(T_FIXNUM);
4847 COUNT_TYPE(T_IMEMO);
4848 COUNT_TYPE(T_UNDEF);
4849 COUNT_TYPE(T_NODE);
4850 COUNT_TYPE(T_ICLASS);
4851 COUNT_TYPE(T_ZOMBIE);
4852 COUNT_TYPE(T_MOVED);
4853 #undef COUNT_TYPE
4854 default: return SIZET2NUM(type); break;
4855 }
4856 }
4857
4858 struct count_objects_data {
4859 size_t counts[T_MASK+1];
4860 size_t freed;
4861 size_t total;
4862 };
4863
4864 static void
4865 count_objects_i(VALUE obj, void *d)
4866 {
4867 struct count_objects_data *data = (struct count_objects_data *)d;
4868
4869 if (RANY(obj)->as.basic.flags) {
4870 data->counts[BUILTIN_TYPE(obj)]++;
4871 }
4872 else {
4873 data->freed++;
4874 }
4875
4876 data->total++;
4877 }
4878
4879 /*
4880 * call-seq:
4881 * ObjectSpace.count_objects([result_hash]) -> hash
4882 *
4883 * Counts all objects grouped by type.
4884 *
4885 * It returns a hash, such as:
4886 * {
4887 * :TOTAL=>10000,
4888 * :FREE=>3011,
4889 * :T_OBJECT=>6,
4890 * :T_CLASS=>404,
4891 * # ...
4892 * }
4893 *
4894 * The contents of the returned hash are implementation specific.
4895 * It may be changed in future.
4896 *
4897 * The keys starting with +:T_+ means live objects.
4898 * For example, +:T_ARRAY+ is the number of arrays.
4899 * +:FREE+ means object slots which is not used now.
4900 * +:TOTAL+ means sum of above.
4901 *
4902 * If the optional argument +result_hash+ is given,
4903 * it is overwritten and returned. This is intended to avoid probe effect.
4904 *
4905 * h = {}
4906 * ObjectSpace.count_objects(h)
4907 * puts h
4908 * # => { :TOTAL=>10000, :T_CLASS=>158280, :T_MODULE=>20672, :T_STRING=>527249 }
4909 *
4910 * This method is only expected to work on C Ruby.
4911 *
4912 */
4913
4914 static VALUE
4915 count_objects(int argc, VALUE *argv, VALUE os)
4916 {
4917 rb_objspace_t *objspace = &rb_objspace;
4918 struct count_objects_data data = { 0 };
4919 VALUE hash = Qnil;
4920
4921 if (rb_check_arity(argc, 0, 1) == 1) {
4922 hash = argv[0];
4923 if (!RB_TYPE_P(hash, T_HASH))
4924 rb_raise(rb_eTypeError, "non-hash given");
4925 }
4926
4927 gc_each_object(objspace, count_objects_i, &data);
4928
4929 if (NIL_P(hash)) {
4930 hash = rb_hash_new();
4931 }
4932 else if (!RHASH_EMPTY_P(hash)) {
4933 rb_hash_stlike_foreach(hash, set_zero, hash);
4934 }
4935 rb_hash_aset(hash, ID2SYM(rb_intern("TOTAL")), SIZET2NUM(data.total));
4936 rb_hash_aset(hash, ID2SYM(rb_intern("FREE")), SIZET2NUM(data.freed));
4937
4938 for (size_t i = 0; i <= T_MASK; i++) {
4939 VALUE type = type_sym(i);
4940 if (data.counts[i])
4941 rb_hash_aset(hash, type, SIZET2NUM(data.counts[i]));
4942 }
4943
4944 return hash;
4945 }
4946
4947 /*
4948 ------------------------ Garbage Collection ------------------------
4949 */
4950
4951 /* Sweeping */
4952
4953 static size_t
4954 objspace_available_slots(rb_objspace_t *objspace)
4955 {
4956 size_t total_slots = 0;
4957 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
4958 rb_size_pool_t *size_pool = &size_pools[i];
4959 total_slots += SIZE_POOL_EDEN_HEAP(size_pool)->total_slots;
4960 total_slots += SIZE_POOL_TOMB_HEAP(size_pool)->total_slots;
4961 }
4962 return total_slots;
4963 }
4964
4965 static size_t
4966 objspace_live_slots(rb_objspace_t *objspace)
4967 {
4968 return total_allocated_objects(objspace) - total_freed_objects(objspace) - heap_pages_final_slots;
4969 }
4970
4971 static size_t
4972 objspace_free_slots(rb_objspace_t *objspace)
4973 {
4974 return objspace_available_slots(objspace) - objspace_live_slots(objspace) - heap_pages_final_slots;
4975 }
4976
4977 static void
4978 gc_setup_mark_bits(struct heap_page *page)
4979 {
4980 /* copy oldgen bitmap to mark bitmap */
4981 memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
4982 }
4983
4984 static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
4985 static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size);
4986
4987 #if defined(_WIN32)
4988 enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
4989
4990 static BOOL
4991 protect_page_body(struct heap_page_body *body, DWORD protect)
4992 {
4993 DWORD old_protect;
4994 return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
4995 }
4996 #else
4997 enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
4998 #define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
4999 #endif
5000
5001 static void
5002 lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
5003 {
5004 if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
5005 rb_bug("Couldn't protect page %p, errno: %s", (void *)body, strerror(errno));
5006 }
5007 else {
5008 gc_report(5, objspace, "Protecting page in move %p\n", (void *)body);
5009 }
5010 }
5011
5012 static void
5013 unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
5014 {
5015 if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
5016 rb_bug("Couldn't unprotect page %p, errno: %s", (void *)body, strerror(errno));
5017 }
5018 else {
5019 gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body);
5020 }
5021 }
5022
5023 static bool
5024 try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, VALUE src)
5025 {
5026 GC_ASSERT(gc_is_moveable_obj(objspace, src));
5027
5028 struct heap_page *src_page = GET_HEAP_PAGE(src);
5029 if (!free_page) {
5030 return false;
5031 }
5032
5033 /* We should return true if either src is successfully moved, or src is
5034 * unmoveable. A false return will cause the sweeping cursor to be
5035 * incremented to the next page, and src will attempt to move again */
5036 GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(src), src));
5037
5038 asan_unlock_freelist(free_page);
5039 VALUE dest = (VALUE)free_page->freelist;
5040 asan_lock_freelist(free_page);
5041 asan_unpoison_object(dest, false);
5042 if (!dest) {
5043 /* if we can't get something from the freelist then the page must be
5044 * full */
5045 return false;
5046 }
5047 asan_unlock_freelist(free_page);
5048 free_page->freelist = RANY(dest)->as.free.next;
5049 asan_lock_freelist(free_page);
5050
5051 GC_ASSERT(RB_BUILTIN_TYPE(dest) == T_NONE);
5052
5053 if (src_page->slot_size > free_page->slot_size) {
5054 objspace->rcompactor.moved_down_count_table[BUILTIN_TYPE(src)]++;
5055 }
5056 else if (free_page->slot_size > src_page->slot_size) {
5057 objspace->rcompactor.moved_up_count_table[BUILTIN_TYPE(src)]++;
5058 }
5059 objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++;
5060 objspace->rcompactor.total_moved++;
5061
5062 gc_move(objspace, src, dest, src_page->slot_size, free_page->slot_size);
5063 gc_pin(objspace, src);
5064 free_page->free_slots--;
5065
5066 return true;
5067 }
5068
5069 static void
5070 gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap)
5071 {
5072 struct heap_page *cursor = heap->compact_cursor;
5073
5074 while (cursor) {
5075 unlock_page_body(objspace, GET_PAGE_BODY(cursor->start));
5076 cursor = ccan_list_next(&heap->pages, cursor, page_node);
5077 }
5078 }
5079
5080 static void gc_update_references(rb_objspace_t * objspace);
5081 #if GC_CAN_COMPILE_COMPACTION
5082 static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page);
5083 #endif
5084
5085 #if defined(__MINGW32__) || defined(_WIN32)
5086 # define GC_COMPACTION_SUPPORTED 1
5087 #else
5088 /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
5089 * the read barrier, so we must disable compaction. */
5090 # define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
5091 #endif
5092
5093 #if GC_CAN_COMPILE_COMPACTION
5094 static void
5095 read_barrier_handler(uintptr_t original_address)
5096 {
5097 VALUE obj;
5098 rb_objspace_t * objspace = &rb_objspace;
5099
5100 /* Calculate address aligned to slots. */
5101 uintptr_t address = original_address - (original_address % BASE_SLOT_SIZE);
5102
5103 obj = (VALUE)address;
5104
5105 struct heap_page_body *page_body = GET_PAGE_BODY(obj);
5106
5107 /* If the page_body is NULL, then mprotect cannot handle it and will crash
5108 * with "Cannot allocate memory". */
5109 if (page_body == NULL) {
5110 rb_bug("read_barrier_handler: segmentation fault at %p", (void *)original_address);
5111 }
5112
5113 RB_VM_LOCK_ENTER();
5114 {
5115 unlock_page_body(objspace, page_body);
5116
5117 objspace->profile.read_barrier_faults++;
5118
5119 invalidate_moved_page(objspace, GET_HEAP_PAGE(obj));
5120 }
5121 RB_VM_LOCK_LEAVE();
5122 }
5123 #endif
5124
5125 #if !GC_CAN_COMPILE_COMPACTION
5126 static void
5127 uninstall_handlers(void)
5128 {
5129 /* no-op */
5130 }
5131
5132 static void
5133 install_handlers(void)
5134 {
5135 /* no-op */
5136 }
5137 #elif defined(_WIN32)
5138 static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
5139 typedef void (*signal_handler)(int);
5140 static signal_handler old_sigsegv_handler;
5141
5142 static LONG WINAPI
5143 read_barrier_signal(EXCEPTION_POINTERS * info)
5144 {
5145 /* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */
5146 if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
5147 /* > The second array element specifies the virtual address of the inaccessible data.
5148 * https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
5149 *
5150 * Use this address to invalidate the page */
5151 read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
5152 return EXCEPTION_CONTINUE_EXECUTION;
5153 }
5154 else {
5155 return EXCEPTION_CONTINUE_SEARCH;
5156 }
5157 }
5158
5159 static void
5160 uninstall_handlers(void)
5161 {
5162 signal(SIGSEGV, old_sigsegv_handler);
5163 SetUnhandledExceptionFilter(old_handler);
5164 }
5165
5166 static void
5167 install_handlers(void)
5168 {
5169 /* Remove SEGV handler so that the Unhandled Exception Filter handles it */
5170 old_sigsegv_handler = signal(SIGSEGV, NULL);
5171 /* Unhandled Exception Filter has access to the violation address similar
5172 * to si_addr from sigaction */
5173 old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
5174 }
5175 #else
5176 static struct sigaction old_sigbus_handler;
5177 static struct sigaction old_sigsegv_handler;
5178
5179 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
5180 static exception_mask_t old_exception_masks[32];
5181 static mach_port_t old_exception_ports[32];
5182 static exception_behavior_t old_exception_behaviors[32];
5183 static thread_state_flavor_t old_exception_flavors[32];
5184 static mach_msg_type_number_t old_exception_count;
5185
5186 static void
5187 disable_mach_bad_access_exc(void)
5188 {
5189 old_exception_count = sizeof(old_exception_masks) / sizeof(old_exception_masks[0]);
5190 task_swap_exception_ports(
5191 mach_task_self(), EXC_MASK_BAD_ACCESS,
5192 MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
5193 old_exception_masks, &old_exception_count,
5194 old_exception_ports, old_exception_behaviors, old_exception_flavors
5195 );
5196 }
5197
5198 static void
5199 restore_mach_bad_access_exc(void)
5200 {
5201 for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
5202 task_set_exception_ports(
5203 mach_task_self(),
5204 old_exception_masks[i], old_exception_ports[i],
5205 old_exception_behaviors[i], old_exception_flavors[i]
5206 );
5207 }
5208 }
5209 #endif
5210
5211 static void
5212 read_barrier_signal(int sig, siginfo_t * info, void * data)
5213 {
5214 // setup SEGV/BUS handlers for errors
5215 struct sigaction prev_sigbus, prev_sigsegv;
5216 sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
5217 sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
5218
5219 // enable SIGBUS/SEGV
5220 sigset_t set, prev_set;
5221 sigemptyset(&set);
5222 sigaddset(&set, SIGBUS);
5223 sigaddset(&set, SIGSEGV);
5224 sigprocmask(SIG_UNBLOCK, &set, &prev_set);
5225 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
5226 disable_mach_bad_access_exc();
5227 #endif
5228 // run handler
5229 read_barrier_handler((uintptr_t)info->si_addr);
5230
5231 // reset SEGV/BUS handlers
5232 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
5233 restore_mach_bad_access_exc();
5234 #endif
5235 sigaction(SIGBUS, &prev_sigbus, NULL);
5236 sigaction(SIGSEGV, &prev_sigsegv, NULL);
5237 sigprocmask(SIG_SETMASK, &prev_set, NULL);
5238 }
5239
5240 static void
5241 uninstall_handlers(void)
5242 {
5243 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
5244 restore_mach_bad_access_exc();
5245 #endif
5246 sigaction(SIGBUS, &old_sigbus_handler, NULL);
5247 sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
5248 }
5249
5250 static void
5251 install_handlers(void)
5252 {
5253 struct sigaction action;
5254 memset(&action, 0, sizeof(struct sigaction));
5255 sigemptyset(&action.sa_mask);
5256 action.sa_sigaction = read_barrier_signal;
5257 action.sa_flags = SA_SIGINFO | SA_ONSTACK;
5258
5259 sigaction(SIGBUS, &action, &old_sigbus_handler);
5260 sigaction(SIGSEGV, &action, &old_sigsegv_handler);
5261 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
5262 disable_mach_bad_access_exc();
5263 #endif
5264 }
5265 #endif
5266
5267 static void
5268 gc_compact_finish(rb_objspace_t *objspace)
5269 {
5270 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5271 rb_size_pool_t *size_pool = &size_pools[i];
5272 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
5273 gc_unprotect_pages(objspace, heap);
5274 }
5275
5276 uninstall_handlers();
5277
5278 gc_update_references(objspace);
5279 objspace->profile.compact_count++;
5280
5281 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5282 rb_size_pool_t *size_pool = &size_pools[i];
5283 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
5284 heap->compact_cursor = NULL;
5285 heap->free_pages = NULL;
5286 heap->compact_cursor_index = 0;
5287 }
5288
5289 if (gc_prof_enabled(objspace)) {
5290 gc_profile_record *record = gc_prof_record(objspace);
5291 record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects;
5292 }
5293 objspace->flags.during_compacting = FALSE;
5294 }
5295
5296 struct gc_sweep_context {
5297 struct heap_page *page;
5298 int final_slots;
5299 int freed_slots;
5300 int empty_slots;
5301 };
5302
5303 static inline void
5304 gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
5305 {
5306 struct heap_page * sweep_page = ctx->page;
5307 short slot_size = sweep_page->slot_size;
5308 short slot_bits = slot_size / BASE_SLOT_SIZE;
5309 GC_ASSERT(slot_bits > 0);
5310
5311 do {
5312 VALUE vp = (VALUE)p;
5313 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
5314
5315 asan_unpoison_object(vp, false);
5316 if (bitset & 1) {
5317 switch (BUILTIN_TYPE(vp)) {
5318 default: /* majority case */
5319 gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
5320 #if RGENGC_CHECK_MODE
5321 if (!is_full_marking(objspace)) {
5322 if (RVALUE_OLD_P(vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
5323 if (RVALUE_REMEMBERED(vp)) rb_bug("page_sweep: %p - remembered.", (void *)p);
5324 }
5325 #endif
5326 if (obj_free(objspace, vp)) {
5327 // always add free slots back to the swept pages freelist,
5328 // so that if we're comapacting, we can re-use the slots
5329 (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE);
5330 heap_page_add_freeobj(objspace, sweep_page, vp);
5331 gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
5332 ctx->freed_slots++;
5333 }
5334 else {
5335 ctx->final_slots++;
5336 }
5337 break;
5338
5339 case T_MOVED:
5340 if (objspace->flags.during_compacting) {
5341 /* The sweep cursor shouldn't have made it to any
5342 * T_MOVED slots while the compact flag is enabled.
5343 * The sweep cursor and compact cursor move in
5344 * opposite directions, and when they meet references will
5345 * get updated and "during_compacting" should get disabled */
5346 rb_bug("T_MOVED shouldn't be seen until compaction is finished");
5347 }
5348 gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
5349 ctx->empty_slots++;
5350 heap_page_add_freeobj(objspace, sweep_page, vp);
5351 break;
5352 case T_ZOMBIE:
5353 /* already counted */
5354 break;
5355 case T_NONE:
5356 ctx->empty_slots++; /* already freed */
5357 break;
5358 }
5359 }
5360 p += slot_size;
5361 bitset >>= slot_bits;
5362 } while (bitset);
5363 }
5364
5365 static inline void
5366 gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx)
5367 {
5368 struct heap_page *sweep_page = ctx->page;
5369 GC_ASSERT(SIZE_POOL_EDEN_HEAP(sweep_page->size_pool) == heap);
5370
5371 uintptr_t p;
5372 bits_t *bits, bitset;
5373
5374 gc_report(2, objspace, "page_sweep: start.\n");
5375
5376 #if RGENGC_CHECK_MODE
5377 if (!objspace->flags.immediate_sweep) {
5378 GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
5379 }
5380 #endif
5381 sweep_page->flags.before_sweep = FALSE;
5382 sweep_page->free_slots = 0;
5383
5384 p = (uintptr_t)sweep_page->start;
5385 bits = sweep_page->mark_bits;
5386
5387 int page_rvalue_count = sweep_page->total_slots * (sweep_page->slot_size / BASE_SLOT_SIZE);
5388 int out_of_range_bits = (NUM_IN_PAGE(p) + page_rvalue_count) % BITS_BITLENGTH;
5389 if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64
5390 bits[BITMAP_INDEX(p) + page_rvalue_count / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
5391 }
5392
5393 /* The last bitmap plane may not be used if the last plane does not
5394 * have enough space for the slot_size. In that case, the last plane must
5395 * be skipped since none of the bits will be set. */
5396 int bitmap_plane_count = CEILDIV(NUM_IN_PAGE(p) + page_rvalue_count, BITS_BITLENGTH);
5397 GC_ASSERT(bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT - 1 ||
5398 bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT);
5399
5400 // Skip out of range slots at the head of the page
5401 bitset = ~bits[0];
5402 bitset >>= NUM_IN_PAGE(p);
5403 if (bitset) {
5404 gc_sweep_plane(objspace, heap, p, bitset, ctx);
5405 }
5406 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5407
5408 for (int i = 1; i < bitmap_plane_count; i++) {
5409 bitset = ~bits[i];
5410 if (bitset) {
5411 gc_sweep_plane(objspace, heap, p, bitset, ctx);
5412 }
5413 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5414 }
5415
5416 if (!heap->compact_cursor) {
5417 gc_setup_mark_bits(sweep_page);
5418 }
5419
5420 #if GC_PROFILE_MORE_DETAIL
5421 if (gc_prof_enabled(objspace)) {
5422 gc_profile_record *record = gc_prof_record(objspace);
5423 record->removing_objects += ctx->final_slots + ctx->freed_slots;
5424 record->empty_objects += ctx->empty_slots;
5425 }
5426 #endif
5427 if (0) fprintf(stderr, "gc_sweep_page(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
5428 rb_gc_count(),
5429 sweep_page->total_slots,
5430 ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
5431
5432 sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
5433 sweep_page->size_pool->total_freed_objects += ctx->freed_slots;
5434
5435 if (heap_pages_deferred_final && !finalizing) {
5436 gc_finalize_deferred_register(objspace);
5437 }
5438
5439 #if RGENGC_CHECK_MODE
5440 short freelist_len = 0;
5441 asan_unlock_freelist(sweep_page);
5442 RVALUE *ptr = sweep_page->freelist;
5443 while (ptr) {
5444 freelist_len++;
5445 ptr = ptr->as.free.next;
5446 }
5447 asan_lock_freelist(sweep_page);
5448 if (freelist_len != sweep_page->free_slots) {
5449 rb_bug("inconsistent freelist length: expected %d but was %d", sweep_page->free_slots, freelist_len);
5450 }
5451 #endif
5452
5453 gc_report(2, objspace, "page_sweep: end.\n");
5454 }
5455
5456 static const char *
5457 gc_mode_name(enum gc_mode mode)
5458 {
5459 switch (mode) {
5460 case gc_mode_none: return "none";
5461 case gc_mode_marking: return "marking";
5462 case gc_mode_sweeping: return "sweeping";
5463 case gc_mode_compacting: return "compacting";
5464 default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode);
5465 }
5466 }
5467
5468 static void
5469 gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
5470 {
5471 #if RGENGC_CHECK_MODE
5472 enum gc_mode prev_mode = gc_mode(objspace);
5473 switch (prev_mode) {
5474 case gc_mode_none: GC_ASSERT(mode == gc_mode_marking); break;
5475 case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping); break;
5476 case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting); break;
5477 case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none); break;
5478 }
5479 #endif
5480 if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode));
5481 gc_mode_set(objspace, mode);
5482 }
5483
5484 static void
5485 heap_page_freelist_append(struct heap_page *page, RVALUE *freelist)
5486 {
5487 if (freelist) {
5488 asan_unlock_freelist(page);
5489 if (page->freelist) {
5490 RVALUE *p = page->freelist;
5491 asan_unpoison_object((VALUE)p, false);
5492 while (p->as.free.next) {
5493 RVALUE *prev = p;
5494 p = p->as.free.next;
5495 asan_poison_object((VALUE)prev);
5496 asan_unpoison_object((VALUE)p, false);
5497 }
5498 p->as.free.next = freelist;
5499 asan_poison_object((VALUE)p);
5500 }
5501 else {
5502 page->freelist = freelist;
5503 }
5504 asan_lock_freelist(page);
5505 }
5506 }
5507
5508 static void
5509 gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
5510 {
5511 heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node);
5512 heap->free_pages = NULL;
5513 heap->pooled_pages = NULL;
5514 if (!objspace->flags.immediate_sweep) {
5515 struct heap_page *page = NULL;
5516
5517 ccan_list_for_each(&heap->pages, page, page_node) {
5518 page->flags.before_sweep = TRUE;
5519 }
5520 }
5521 }
5522
5523 #if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4
5524 __attribute__((noinline))
5525 #endif
5526
5527 #if GC_CAN_COMPILE_COMPACTION
5528 static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func);
5529 static int compare_pinned_slots(const void *left, const void *right, void *d);
5530 #endif
5531
5532 static void
5533 gc_sweep_start(rb_objspace_t *objspace)
5534 {
5535 gc_mode_transition(objspace, gc_mode_sweeping);
5536 objspace->rincgc.pooled_slots = 0;
5537
5538 #if GC_CAN_COMPILE_COMPACTION
5539 if (objspace->flags.during_compacting) {
5540 gc_sort_heap_by_compare_func(
5541 objspace,
5542 objspace->rcompactor.compare_func ? objspace->rcompactor.compare_func : compare_pinned_slots
5543 );
5544 }
5545 #endif
5546
5547 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5548 rb_size_pool_t *size_pool = &size_pools[i];
5549 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
5550
5551 gc_sweep_start_heap(objspace, heap);
5552
5553 /* We should call gc_sweep_finish_size_pool for size pools with no pages. */
5554 if (heap->sweeping_page == NULL) {
5555 GC_ASSERT(heap->total_pages == 0);
5556 GC_ASSERT(heap->total_slots == 0);
5557 gc_sweep_finish_size_pool(objspace, size_pool);
5558 }
5559 }
5560
5561 rb_ractor_t *r = NULL;
5562 ccan_list_for_each(&GET_VM()->ractor.set, r, vmlr_node) {
5563 rb_gc_ractor_newobj_cache_clear(&r->newobj_cache);
5564 }
5565 }
5566
5567 static void
5568 gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
5569 {
5570 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
5571 size_t total_slots = heap->total_slots + SIZE_POOL_TOMB_HEAP(size_pool)->total_slots;
5572 size_t total_pages = heap->total_pages + SIZE_POOL_TOMB_HEAP(size_pool)->total_pages;
5573 size_t swept_slots = size_pool->freed_slots + size_pool->empty_slots;
5574
5575 size_t init_slots = gc_params.size_pool_init_slots[size_pool - size_pools];
5576 size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
5577
5578 /* If we don't have enough slots and we have pages on the tomb heap, move
5579 * pages from the tomb heap to the eden heap. This may prevent page
5580 * creation thrashing (frequently allocating and deallocting pages) and
5581 * GC thrashing (running GC more frequently than required). */
5582 struct heap_page *resurrected_page;
5583 while (swept_slots < min_free_slots &&
5584 (resurrected_page = heap_page_resurrect(objspace, size_pool))) {
5585 swept_slots += resurrected_page->free_slots;
5586
5587 heap_add_page(objspace, size_pool, heap, resurrected_page);
5588 heap_add_freepage(heap, resurrected_page);
5589 }
5590
5591 if (swept_slots < min_free_slots) {
5592 bool grow_heap = is_full_marking(objspace);
5593
5594 /* Consider growing or starting a major GC if we are not currently in a
5595 * major GC and we can't allocate any more pages. */
5596 if (!is_full_marking(objspace) && size_pool->allocatable_pages == 0) {
5597 /* The heap is a growth heap if it freed more slots than had empty slots. */
5598 bool is_growth_heap = size_pool->empty_slots == 0 || size_pool->freed_slots > size_pool->empty_slots;
5599
5600 /* Grow this heap if we haven't run at least RVALUE_OLD_AGE minor
5601 * GC since the last major GC or if this heap is smaller than the
5602 * the configured initial size. */
5603 if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE ||
5604 total_slots < init_slots) {
5605 grow_heap = TRUE;
5606 }
5607 else if (is_growth_heap) { /* Only growth heaps are allowed to start a major GC. */
5608 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
5609 size_pool->force_major_gc_count++;
5610 }
5611 }
5612
5613 if (grow_heap) {
5614 size_t extend_page_count = heap_extend_pages(objspace, size_pool, swept_slots, total_slots, total_pages);
5615
5616 if (extend_page_count > size_pool->allocatable_pages) {
5617 size_pool_allocatable_pages_set(objspace, size_pool, extend_page_count);
5618 }
5619 }
5620 }
5621 }
5622
5623 static void
5624 gc_sweep_finish(rb_objspace_t *objspace)
5625 {
5626 gc_report(1, objspace, "gc_sweep_finish\n");
5627
5628 gc_prof_set_heap_info(objspace);
5629 heap_pages_free_unused_pages(objspace);
5630
5631 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5632 rb_size_pool_t *size_pool = &size_pools[i];
5633
5634 /* if heap_pages has unused pages, then assign them to increment */
5635 size_t tomb_pages = SIZE_POOL_TOMB_HEAP(size_pool)->total_pages;
5636 if (size_pool->allocatable_pages < tomb_pages) {
5637 size_pool->allocatable_pages = tomb_pages;
5638 }
5639
5640 size_pool->freed_slots = 0;
5641 size_pool->empty_slots = 0;
5642
5643 if (!will_be_incremental_marking(objspace)) {
5644 rb_heap_t *eden_heap = SIZE_POOL_EDEN_HEAP(size_pool);
5645 struct heap_page *end_page = eden_heap->free_pages;
5646 if (end_page) {
5647 while (end_page->free_next) end_page = end_page->free_next;
5648 end_page->free_next = eden_heap->pooled_pages;
5649 }
5650 else {
5651 eden_heap->free_pages = eden_heap->pooled_pages;
5652 }
5653 eden_heap->pooled_pages = NULL;
5654 objspace->rincgc.pooled_slots = 0;
5655 }
5656 }
5657 heap_pages_expand_sorted(objspace);
5658
5659 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_SWEEP, 0);
5660 gc_mode_transition(objspace, gc_mode_none);
5661
5662 #if RGENGC_CHECK_MODE >= 2
5663 gc_verify_internal_consistency(objspace);
5664 #endif
5665 }
5666
5667 static int
5668 gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
5669 {
5670 struct heap_page *sweep_page = heap->sweeping_page;
5671 int unlink_limit = GC_SWEEP_PAGES_FREEABLE_PER_STEP;
5672 int swept_slots = 0;
5673 int pooled_slots = 0;
5674
5675 if (sweep_page == NULL) return FALSE;
5676
5677 #if GC_ENABLE_LAZY_SWEEP
5678 gc_prof_sweep_timer_start(objspace);
5679 #endif
5680
5681 do {
5682 RUBY_DEBUG_LOG("sweep_page:%p", (void *)sweep_page);
5683
5684 struct gc_sweep_context ctx = {
5685 .page = sweep_page,
5686 .final_slots = 0,
5687 .freed_slots = 0,
5688 .empty_slots = 0,
5689 };
5690 gc_sweep_page(objspace, heap, &ctx);
5691 int free_slots = ctx.freed_slots + ctx.empty_slots;
5692
5693 heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
5694
5695 if (sweep_page->final_slots + free_slots == sweep_page->total_slots &&
5696 heap_pages_freeable_pages > 0 &&
5697 unlink_limit > 0) {
5698 heap_pages_freeable_pages--;
5699 unlink_limit--;
5700 /* there are no living objects -> move this page to tomb heap */
5701 heap_unlink_page(objspace, heap, sweep_page);
5702 heap_add_page(objspace, size_pool, SIZE_POOL_TOMB_HEAP(size_pool), sweep_page);
5703 }
5704 else if (free_slots > 0) {
5705 size_pool->freed_slots += ctx.freed_slots;
5706 size_pool->empty_slots += ctx.empty_slots;
5707
5708 if (pooled_slots < GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT) {
5709 heap_add_poolpage(objspace, heap, sweep_page);
5710 pooled_slots += free_slots;
5711 }
5712 else {
5713 heap_add_freepage(heap, sweep_page);
5714 swept_slots += free_slots;
5715 if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) {
5716 break;
5717 }
5718 }
5719 }
5720 else {
5721 sweep_page->free_next = NULL;
5722 }
5723 } while ((sweep_page = heap->sweeping_page));
5724
5725 if (!heap->sweeping_page) {
5726 gc_sweep_finish_size_pool(objspace, size_pool);
5727
5728 if (!has_sweeping_pages(objspace)) {
5729 gc_sweep_finish(objspace);
5730 }
5731 }
5732
5733 #if GC_ENABLE_LAZY_SWEEP
5734 gc_prof_sweep_timer_stop(objspace);
5735 #endif
5736
5737 return heap->free_pages != NULL;
5738 }
5739
5740 static void
5741 gc_sweep_rest(rb_objspace_t *objspace)
5742 {
5743 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5744 rb_size_pool_t *size_pool = &size_pools[i];
5745
5746 while (SIZE_POOL_EDEN_HEAP(size_pool)->sweeping_page) {
5747 gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
5748 }
5749 }
5750 }
5751
5752 static void
5753 gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *sweep_size_pool, rb_heap_t *heap)
5754 {
5755 GC_ASSERT(dont_gc_val() == FALSE);
5756 if (!GC_ENABLE_LAZY_SWEEP) return;
5757
5758 gc_sweeping_enter(objspace);
5759
5760 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5761 rb_size_pool_t *size_pool = &size_pools[i];
5762 if (!gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool))) {
5763 /* sweep_size_pool requires a free slot but sweeping did not yield any. */
5764 if (size_pool == sweep_size_pool) {
5765 if (size_pool->allocatable_pages > 0) {
5766 heap_increment(objspace, size_pool, heap);
5767 }
5768 else {
5769 /* Not allowed to create a new page so finish sweeping. */
5770 gc_sweep_rest(objspace);
5771 break;
5772 }
5773 }
5774 }
5775 }
5776
5777 gc_sweeping_exit(objspace);
5778 }
5779
5780 #if GC_CAN_COMPILE_COMPACTION
5781 static void
5782 invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_t p, bits_t bitset)
5783 {
5784 if (bitset) {
5785 do {
5786 if (bitset & 1) {
5787 VALUE forwarding_object = (VALUE)p;
5788 VALUE object;
5789
5790 if (BUILTIN_TYPE(forwarding_object) == T_MOVED) {
5791 GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object));
5792 GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(forwarding_object), forwarding_object));
5793
5794 CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
5795
5796 object = rb_gc_location(forwarding_object);
5797
5798 shape_id_t original_shape_id = 0;
5799 if (RB_TYPE_P(object, T_OBJECT)) {
5800 original_shape_id = RMOVED(forwarding_object)->original_shape_id;
5801 }
5802
5803 gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object)->slot_size, page->slot_size);
5804 /* forwarding_object is now our actual object, and "object"
5805 * is the free slot for the original page */
5806
5807 if (original_shape_id) {
5808 ROBJECT_SET_SHAPE_ID(forwarding_object, original_shape_id);
5809 }
5810
5811 struct heap_page *orig_page = GET_HEAP_PAGE(object);
5812 orig_page->free_slots++;
5813 heap_page_add_freeobj(objspace, orig_page, object);
5814
5815 GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(forwarding_object), forwarding_object));
5816 GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED);
5817 GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
5818 }
5819 }
5820 p += BASE_SLOT_SIZE;
5821 bitset >>= 1;
5822 } while (bitset);
5823 }
5824 }
5825
5826 static void
5827 invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
5828 {
5829 int i;
5830 bits_t *mark_bits, *pin_bits;
5831 bits_t bitset;
5832
5833 mark_bits = page->mark_bits;
5834 pin_bits = page->pinned_bits;
5835
5836 uintptr_t p = page->start;
5837
5838 // Skip out of range slots at the head of the page
5839 bitset = pin_bits[0] & ~mark_bits[0];
5840 bitset >>= NUM_IN_PAGE(p);
5841 invalidate_moved_plane(objspace, page, p, bitset);
5842 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5843
5844 for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
5845 /* Moved objects are pinned but never marked. We reuse the pin bits
5846 * to indicate there is a moved object in this slot. */
5847 bitset = pin_bits[i] & ~mark_bits[i];
5848
5849 invalidate_moved_plane(objspace, page, p, bitset);
5850 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5851 }
5852 }
5853 #endif
5854
5855 static void
5856 gc_compact_start(rb_objspace_t *objspace)
5857 {
5858 struct heap_page *page = NULL;
5859 gc_mode_transition(objspace, gc_mode_compacting);
5860
5861 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5862 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(&size_pools[i]);
5863 ccan_list_for_each(&heap->pages, page, page_node) {
5864 page->flags.before_sweep = TRUE;
5865 }
5866
5867 heap->compact_cursor = ccan_list_tail(&heap->pages, struct heap_page, page_node);
5868 heap->compact_cursor_index = 0;
5869 }
5870
5871 if (gc_prof_enabled(objspace)) {
5872 gc_profile_record *record = gc_prof_record(objspace);
5873 record->moved_objects = objspace->rcompactor.total_moved;
5874 }
5875
5876 memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
5877 memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
5878 memset(objspace->rcompactor.moved_up_count_table, 0, T_MASK * sizeof(size_t));
5879 memset(objspace->rcompactor.moved_down_count_table, 0, T_MASK * sizeof(size_t));
5880
5881 /* Set up read barrier for pages containing MOVED objects */
5882 install_handlers();
5883 }
5884
5885 static void gc_sweep_compact(rb_objspace_t *objspace);
5886
5887 static void
5888 gc_sweep(rb_objspace_t *objspace)
5889 {
5890 gc_sweeping_enter(objspace);
5891
5892 const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
5893
5894 gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
5895
5896 gc_sweep_start(objspace);
5897 if (objspace->flags.during_compacting) {
5898 gc_sweep_compact(objspace);
5899 }
5900
5901 if (immediate_sweep) {
5902 #if !GC_ENABLE_LAZY_SWEEP
5903 gc_prof_sweep_timer_start(objspace);
5904 #endif
5905 gc_sweep_rest(objspace);
5906 #if !GC_ENABLE_LAZY_SWEEP
5907 gc_prof_sweep_timer_stop(objspace);
5908 #endif
5909 }
5910 else {
5911
5912 /* Sweep every size pool. */
5913 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
5914 rb_size_pool_t *size_pool = &size_pools[i];
5915 gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
5916 }
5917 }
5918
5919 gc_sweeping_exit(objspace);
5920 }
5921
5922 /* Marking - Marking stack */
5923
5924 static stack_chunk_t *
5925 stack_chunk_alloc(void)
5926 {
5927 stack_chunk_t *res;
5928
5929 res = malloc(sizeof(stack_chunk_t));
5930 if (!res)
5931 rb_memerror();
5932
5933 return res;
5934 }
5935
5936 static inline int
5937 is_mark_stack_empty(mark_stack_t *stack)
5938 {
5939 return stack->chunk == NULL;
5940 }
5941
5942 static size_t
5943 mark_stack_size(mark_stack_t *stack)
5944 {
5945 size_t size = stack->index;
5946 stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
5947
5948 while (chunk) {
5949 size += stack->limit;
5950 chunk = chunk->next;
5951 }
5952 return size;
5953 }
5954
5955 static void
5956 add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
5957 {
5958 chunk->next = stack->cache;
5959 stack->cache = chunk;
5960 stack->cache_size++;
5961 }
5962
5963 static void
5964 shrink_stack_chunk_cache(mark_stack_t *stack)
5965 {
5966 stack_chunk_t *chunk;
5967
5968 if (stack->unused_cache_size > (stack->cache_size/2)) {
5969 chunk = stack->cache;
5970 stack->cache = stack->cache->next;
5971 stack->cache_size--;
5972 free(chunk);
5973 }
5974 stack->unused_cache_size = stack->cache_size;
5975 }
5976
5977 static void
5978 push_mark_stack_chunk(mark_stack_t *stack)
5979 {
5980 stack_chunk_t *next;
5981
5982 GC_ASSERT(stack->index == stack->limit);
5983
5984 if (stack->cache_size > 0) {
5985 next = stack->cache;
5986 stack->cache = stack->cache->next;
5987 stack->cache_size--;
5988 if (stack->unused_cache_size > stack->cache_size)
5989 stack->unused_cache_size = stack->cache_size;
5990 }
5991 else {
5992 next = stack_chunk_alloc();
5993 }
5994 next->next = stack->chunk;
5995 stack->chunk = next;
5996 stack->index = 0;
5997 }
5998
5999 static void
6000 pop_mark_stack_chunk(mark_stack_t *stack)
6001 {
6002 stack_chunk_t *prev;
6003
6004 prev = stack->chunk->next;
6005 GC_ASSERT(stack->index == 0);
6006 add_stack_chunk_cache(stack, stack->chunk);
6007 stack->chunk = prev;
6008 stack->index = stack->limit;
6009 }
6010
6011 static void
6012 mark_stack_chunk_list_free(stack_chunk_t *chunk)
6013 {
6014 stack_chunk_t *next = NULL;
6015
6016 while (chunk != NULL) {
6017 next = chunk->next;
6018 free(chunk);
6019 chunk = next;
6020 }
6021 }
6022
6023 static void
6024 free_stack_chunks(mark_stack_t *stack)
6025 {
6026 mark_stack_chunk_list_free(stack->chunk);
6027 }
6028
6029 static void
6030 mark_stack_free_cache(mark_stack_t *stack)
6031 {
6032 mark_stack_chunk_list_free(stack->cache);
6033 stack->cache_size = 0;
6034 stack->unused_cache_size = 0;
6035 }
6036
6037 static void
6038 push_mark_stack(mark_stack_t *stack, VALUE obj)
6039 {
6040 switch (BUILTIN_TYPE(obj)) {
6041 case T_OBJECT:
6042 case T_CLASS:
6043 case T_MODULE:
6044 case T_FLOAT:
6045 case T_STRING:
6046 case T_REGEXP:
6047 case T_ARRAY:
6048 case T_HASH:
6049 case T_STRUCT:
6050 case T_BIGNUM:
6051 case T_FILE:
6052 case T_DATA:
6053 case T_MATCH:
6054 case T_COMPLEX:
6055 case T_RATIONAL:
6056 case T_TRUE:
6057 case T_FALSE:
6058 case T_SYMBOL:
6059 case T_IMEMO:
6060 case T_ICLASS:
6061 if (stack->index == stack->limit) {
6062 push_mark_stack_chunk(stack);
6063 }
6064 stack->chunk->data[stack->index++] = obj;
6065 return;
6066
6067 case T_NONE:
6068 case T_NIL:
6069 case T_FIXNUM:
6070 case T_MOVED:
6071 case T_ZOMBIE:
6072 case T_UNDEF:
6073 case T_MASK:
6074 rb_bug("push_mark_stack() called for broken object");
6075 break;
6076
6077 case T_NODE:
6078 UNEXPECTED_NODE(push_mark_stack);
6079 break;
6080 }
6081
6082 rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
6083 BUILTIN_TYPE(obj), (void *)obj,
6084 is_pointer_to_heap(&rb_objspace, (void *)obj) ? "corrupted object" : "non object");
6085 }
6086
6087 static int
6088 pop_mark_stack(mark_stack_t *stack, VALUE *data)
6089 {
6090 if (is_mark_stack_empty(stack)) {
6091 return FALSE;
6092 }
6093 if (stack->index == 1) {
6094 *data = stack->chunk->data[--stack->index];
6095 pop_mark_stack_chunk(stack);
6096 }
6097 else {
6098 *data = stack->chunk->data[--stack->index];
6099 }
6100 return TRUE;
6101 }
6102
6103 static void
6104 init_mark_stack(mark_stack_t *stack)
6105 {
6106 int i;
6107
6108 MEMZERO(stack, mark_stack_t, 1);
6109 stack->index = stack->limit = STACK_CHUNK_SIZE;
6110
6111 for (i=0; i < 4; i++) {
6112 add_stack_chunk_cache(stack, stack_chunk_alloc());
6113 }
6114 stack->unused_cache_size = stack->cache_size;
6115 }
6116
6117 /* Marking */
6118
6119 #define SET_STACK_END SET_MACHINE_STACK_END(&ec->machine.stack_end)
6120
6121 #define STACK_START (ec->machine.stack_start)
6122 #define STACK_END (ec->machine.stack_end)
6123 #define STACK_LEVEL_MAX (ec->machine.stack_maxsize/sizeof(VALUE))
6124
6125 #if STACK_GROW_DIRECTION < 0
6126 # define STACK_LENGTH (size_t)(STACK_START - STACK_END)
6127 #elif STACK_GROW_DIRECTION > 0
6128 # define STACK_LENGTH (size_t)(STACK_END - STACK_START + 1)
6129 #else
6130 # define STACK_LENGTH ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \
6131 : (size_t)(STACK_END - STACK_START + 1))
6132 #endif
6133 #if !STACK_GROW_DIRECTION
6134 int ruby_stack_grow_direction;
6135 int
6136 ruby_get_stack_grow_direction(volatile VALUE *addr)
6137 {
6138 VALUE *end;
6139 SET_MACHINE_STACK_END(&end);
6140
6141 if (end > addr) return ruby_stack_grow_direction = 1;
6142 return ruby_stack_grow_direction = -1;
6143 }
6144 #endif
6145
6146 size_t
6147 ruby_stack_length(VALUE **p)
6148 {
6149 rb_execution_context_t *ec = GET_EC();
6150 SET_STACK_END;
6151 if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END);
6152 return STACK_LENGTH;
6153 }
6154
6155 #define PREVENT_STACK_OVERFLOW 1
6156 #ifndef PREVENT_STACK_OVERFLOW
6157 #if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK))
6158 # define PREVENT_STACK_OVERFLOW 1
6159 #else
6160 # define PREVENT_STACK_OVERFLOW 0
6161 #endif
6162 #endif
6163 #if PREVENT_STACK_OVERFLOW && !defined(__EMSCRIPTEN__)
6164 static int
6165 stack_check(rb_execution_context_t *ec, int water_mark)
6166 {
6167 SET_STACK_END;
6168
6169 size_t length = STACK_LENGTH;
6170 size_t maximum_length = STACK_LEVEL_MAX - water_mark;
6171
6172 return length > maximum_length;
6173 }
6174 #else
6175 #define stack_check(ec, water_mark) FALSE
6176 #endif
6177
6178 #define STACKFRAME_FOR_CALL_CFUNC 2048
6179
6180 int
6181 rb_ec_stack_check(rb_execution_context_t *ec)
6182 {
6183 return stack_check(ec, STACKFRAME_FOR_CALL_CFUNC);
6184 }
6185
6186 int
6187 ruby_stack_check(void)
6188 {
6189 return stack_check(GET_EC(), STACKFRAME_FOR_CALL_CFUNC);
6190 }
6191
6192 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void each_location(rb_objspace_t *objspace, register const VALUE *x, register long n, void (*cb)(rb_objspace_t *, VALUE)));
6193 static void
6194 each_location(rb_objspace_t *objspace, register const VALUE *x, register long n, void (*cb)(rb_objspace_t *, VALUE))
6195 {
6196 VALUE v;
6197 while (n--) {
6198 v = *x;
6199 cb(objspace, v);
6200 x++;
6201 }
6202 }
6203
6204 static void
6205 gc_mark_locations(rb_objspace_t *objspace, const VALUE *start, const VALUE *end, void (*cb)(rb_objspace_t *, VALUE))
6206 {
6207 long n;
6208
6209 if (end <= start) return;
6210 n = end - start;
6211 each_location(objspace, start, n, cb);
6212 }
6213
6214 void
6215 rb_gc_mark_locations(const VALUE *start, const VALUE *end)
6216 {
6217 gc_mark_locations(&rb_objspace, start, end, gc_mark_maybe);
6218 }
6219
6220 void
6221 rb_gc_mark_values(long n, const VALUE *values)
6222 {
6223 long i;
6224 rb_objspace_t *objspace = &rb_objspace;
6225
6226 for (i=0; i<n; i++) {
6227 gc_mark(objspace, values[i]);
6228 }
6229 }
6230
6231 void
6232 rb_gc_mark_vm_stack_values(long n, const VALUE *values)
6233 {
6234 rb_objspace_t *objspace = &rb_objspace;
6235
6236 for (long i = 0; i < n; i++) {
6237 gc_mark_and_pin(objspace, values[i]);
6238 }
6239 }
6240
6241 static int
6242 mark_value(st_data_t key, st_data_t value, st_data_t data)
6243 {
6244 rb_objspace_t *objspace = (rb_objspace_t *)data;
6245 gc_mark(objspace, (VALUE)value);
6246 return ST_CONTINUE;
6247 }
6248
6249 static int
6250 mark_value_pin(st_data_t key, st_data_t value, st_data_t data)
6251 {
6252 rb_objspace_t *objspace = (rb_objspace_t *)data;
6253 gc_mark_and_pin(objspace, (VALUE)value);
6254 return ST_CONTINUE;
6255 }
6256
6257 static void
6258 mark_tbl_no_pin(rb_objspace_t *objspace, st_table *tbl)
6259 {
6260 if (!tbl || tbl->num_entries == 0) return;
6261 st_foreach(tbl, mark_value, (st_data_t)objspace);
6262 }
6263
6264 static void
6265 mark_tbl(rb_objspace_t *objspace, st_table *tbl)
6266 {
6267 if (!tbl || tbl->num_entries == 0) return;
6268 st_foreach(tbl, mark_value_pin, (st_data_t)objspace);
6269 }
6270
6271 static int
6272 mark_key(st_data_t key, st_data_t value, st_data_t data)
6273 {
6274 rb_objspace_t *objspace = (rb_objspace_t *)data;
6275 gc_mark_and_pin(objspace, (VALUE)key);
6276 return ST_CONTINUE;
6277 }
6278
6279 static void
6280 mark_set(rb_objspace_t *objspace, st_table *tbl)
6281 {
6282 if (!tbl) return;
6283 st_foreach(tbl, mark_key, (st_data_t)objspace);
6284 }
6285
6286 static int
6287 pin_value(st_data_t key, st_data_t value, st_data_t data)
6288 {
6289 rb_objspace_t *objspace = (rb_objspace_t *)data;
6290 gc_mark_and_pin(objspace, (VALUE)value);
6291 return ST_CONTINUE;
6292 }
6293
6294 static void
6295 mark_finalizer_tbl(rb_objspace_t *objspace, st_table *tbl)
6296 {
6297 if (!tbl) return;
6298 st_foreach(tbl, pin_value, (st_data_t)objspace);
6299 }
6300
6301 void
6302 rb_mark_set(st_table *tbl)
6303 {
6304 mark_set(&rb_objspace, tbl);
6305 }
6306
6307 static int
6308 mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
6309 {
6310 rb_objspace_t *objspace = (rb_objspace_t *)data;
6311
6312 gc_mark(objspace, (VALUE)key);
6313 gc_mark(objspace, (VALUE)value);
6314 return ST_CONTINUE;
6315 }
6316
6317 static int
6318 pin_key_pin_value(st_data_t key, st_data_t value, st_data_t data)
6319 {
6320 rb_objspace_t *objspace = (rb_objspace_t *)data;
6321
6322 gc_mark_and_pin(objspace, (VALUE)key);
6323 gc_mark_and_pin(objspace, (VALUE)value);
6324 return ST_CONTINUE;
6325 }
6326
6327 static int
6328 pin_key_mark_value(st_data_t key, st_data_t value, st_data_t data)
6329 {
6330 rb_objspace_t *objspace = (rb_objspace_t *)data;
6331
6332 gc_mark_and_pin(objspace, (VALUE)key);
6333 gc_mark(objspace, (VALUE)value);
6334 return ST_CONTINUE;
6335 }
6336
6337 static void
6338 mark_hash(rb_objspace_t *objspace, VALUE hash)
6339 {
6340 if (rb_hash_compare_by_id_p(hash)) {
6341 rb_hash_stlike_foreach(hash, pin_key_mark_value, (st_data_t)objspace);
6342 }
6343 else {
6344 rb_hash_stlike_foreach(hash, mark_keyvalue, (st_data_t)objspace);
6345 }
6346
6347 gc_mark(objspace, RHASH(hash)->ifnone);
6348 }
6349
6350 static void
6351 mark_st(rb_objspace_t *objspace, st_table *tbl)
6352 {
6353 if (!tbl) return;
6354 st_foreach(tbl, pin_key_pin_value, (st_data_t)objspace);
6355 }
6356
6357 void
6358 rb_mark_hash(st_table *tbl)
6359 {
6360 mark_st(&rb_objspace, tbl);
6361 }
6362
6363 static enum rb_id_table_iterator_result
6364 mark_method_entry_i(VALUE me, void *data)
6365 {
6366 rb_objspace_t *objspace = (rb_objspace_t *)data;
6367
6368 gc_mark(objspace, me);
6369 return ID_TABLE_CONTINUE;
6370 }
6371
6372 static void
6373 mark_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
6374 {
6375 if (tbl) {
6376 rb_id_table_foreach_values(tbl, mark_method_entry_i, objspace);
6377 }
6378 }
6379
6380 static enum rb_id_table_iterator_result
6381 mark_const_entry_i(VALUE value, void *data)
6382 {
6383 const rb_const_entry_t *ce = (const rb_const_entry_t *)value;
6384 rb_objspace_t *objspace = data;
6385
6386 gc_mark(objspace, ce->value);
6387 gc_mark(objspace, ce->file);
6388 return ID_TABLE_CONTINUE;
6389 }
6390
6391 static void
6392 mark_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
6393 {
6394 if (!tbl) return;
6395 rb_id_table_foreach_values(tbl, mark_const_entry_i, objspace);
6396 }
6397
6398 #if STACK_GROW_DIRECTION < 0
6399 #define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START)
6400 #elif STACK_GROW_DIRECTION > 0
6401 #define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix))
6402 #else
6403 #define GET_STACK_BOUNDS(start, end, appendix) \
6404 ((STACK_END < STACK_START) ? \
6405 ((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix)))
6406 #endif
6407
6408 static void each_stack_location(rb_objspace_t *objspace, const rb_execution_context_t *ec,
6409 const VALUE *stack_start, const VALUE *stack_end, void (*cb)(rb_objspace_t *, VALUE));
6410
6411 static void
6412 gc_mark_machine_stack_location_maybe(rb_objspace_t *objspace, VALUE obj)
6413 {
6414 gc_mark_maybe(objspace, obj);
6415
6416 #ifdef RUBY_ASAN_ENABLED
6417 const rb_execution_context_t *ec = objspace->marking_machine_context_ec;
6418 void *fake_frame_start;
6419 void *fake_frame_end;
6420 bool is_fake_frame = asan_get_fake_stack_extents(
6421 ec->machine.asan_fake_stack_handle, obj,
6422 ec->machine.stack_start, ec->machine.stack_end,
6423 &fake_frame_start, &fake_frame_end
6424 );
6425 if (is_fake_frame) {
6426 each_stack_location(objspace, ec, fake_frame_start, fake_frame_end, gc_mark_maybe);
6427 }
6428 #endif
6429 }
6430
6431 #if defined(__wasm__)
6432
6433
6434 static VALUE *rb_stack_range_tmp[2];
6435
6436 static void
6437 rb_mark_locations(void *begin, void *end)
6438 {
6439 rb_stack_range_tmp[0] = begin;
6440 rb_stack_range_tmp[1] = end;
6441 }
6442
6443 # if defined(__EMSCRIPTEN__)
6444
6445 static void
6446 mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec)
6447 {
6448 emscripten_scan_stack(rb_mark_locations);
6449 each_stack_location(objspace, ec, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe);
6450
6451 emscripten_scan_registers(rb_mark_locations);
6452 each_stack_location(objspace, ec, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe);
6453 }
6454 # else // use Asyncify version
6455
6456 static void
6457 mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec)
6458 {
6459 VALUE *stack_start, *stack_end;
6460 SET_STACK_END;
6461 GET_STACK_BOUNDS(stack_start, stack_end, 1);
6462 each_stack_location(objspace, ec, stack_start, stack_end, gc_mark_maybe);
6463
6464 rb_wasm_scan_locals(rb_mark_locations);
6465 each_stack_location(objspace, ec, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe);
6466 }
6467
6468 # endif
6469
6470 #else // !defined(__wasm__)
6471
6472 static void
6473 mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec)
6474 {
6475 union {
6476 rb_jmp_buf j;
6477 VALUE v[sizeof(rb_jmp_buf) / (sizeof(VALUE))];
6478 } save_regs_gc_mark;
6479 VALUE *stack_start, *stack_end;
6480
6481 FLUSH_REGISTER_WINDOWS;
6482 memset(&save_regs_gc_mark, 0, sizeof(save_regs_gc_mark));
6483 /* This assumes that all registers are saved into the jmp_buf (and stack) */
6484 rb_setjmp(save_regs_gc_mark.j);
6485
6486 /* SET_STACK_END must be called in this function because
6487 * the stack frame of this function may contain
6488 * callee save registers and they should be marked. */
6489 SET_STACK_END;
6490 GET_STACK_BOUNDS(stack_start, stack_end, 1);
6491
6492 #ifdef RUBY_ASAN_ENABLED
6493 objspace->marking_machine_context_ec = ec;
6494 #endif
6495
6496 each_location(objspace, save_regs_gc_mark.v, numberof(save_regs_gc_mark.v), gc_mark_machine_stack_location_maybe);
6497 each_stack_location(objspace, ec, stack_start, stack_end, gc_mark_machine_stack_location_maybe);
6498
6499 #ifdef RUBY_ASAN_ENABLED
6500 objspace->marking_machine_context_ec = NULL;
6501 #endif
6502 }
6503 #endif
6504
6505 void
6506 rb_gc_mark_machine_context(const rb_execution_context_t *ec)
6507 {
6508 rb_objspace_t *objspace = &rb_objspace;
6509 #ifdef RUBY_ASAN_ENABLED
6510 objspace->marking_machine_context_ec = ec;
6511 #endif
6512
6513 VALUE *stack_start, *stack_end;
6514
6515 GET_STACK_BOUNDS(stack_start, stack_end, 0);
6516 RUBY_DEBUG_LOG("ec->th:%u stack_start:%p stack_end:%p", rb_ec_thread_ptr(ec)->serial, stack_start, stack_end);
6517
6518 each_stack_location(objspace, ec, stack_start, stack_end, gc_mark_machine_stack_location_maybe);
6519 int num_regs = sizeof(ec->machine.regs)/(sizeof(VALUE));
6520 each_location(objspace, (VALUE*)&ec->machine.regs, num_regs, gc_mark_machine_stack_location_maybe);
6521
6522 #ifdef RUBY_ASAN_ENABLED
6523 objspace->marking_machine_context_ec = NULL;
6524 #endif
6525 }
6526
6527 static void
6528 each_stack_location(rb_objspace_t *objspace, const rb_execution_context_t *ec,
6529 const VALUE *stack_start, const VALUE *stack_end, void (*cb)(rb_objspace_t *, VALUE))
6530 {
6531
6532 gc_mark_locations(objspace, stack_start, stack_end, cb);
6533
6534 #if defined(__mc68000__)
6535 gc_mark_locations(objspace,
6536 (VALUE*)((char*)stack_start + 2),
6537 (VALUE*)((char*)stack_end - 2), cb);
6538 #endif
6539 }
6540
6541 void
6542 rb_mark_tbl(st_table *tbl)
6543 {
6544 mark_tbl(&rb_objspace, tbl);
6545 }
6546
6547 void
6548 rb_mark_tbl_no_pin(st_table *tbl)
6549 {
6550 mark_tbl_no_pin(&rb_objspace, tbl);
6551 }
6552
6553 static void
6554 gc_mark_maybe(rb_objspace_t *objspace, VALUE obj)
6555 {
6556 (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
6557
6558 if (is_pointer_to_heap(objspace, (void *)obj)) {
6559 void *ptr = asan_unpoison_object_temporary(obj);
6560
6561 /* Garbage can live on the stack, so do not mark or pin */
6562 switch (BUILTIN_TYPE(obj)) {
6563 case T_ZOMBIE:
6564 case T_NONE:
6565 break;
6566 default:
6567 gc_mark_and_pin(objspace, obj);
6568 break;
6569 }
6570
6571 if (ptr) {
6572 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
6573 asan_poison_object(obj);
6574 }
6575 }
6576 }
6577
6578 void
6579 rb_gc_mark_maybe(VALUE obj)
6580 {
6581 gc_mark_maybe(&rb_objspace, obj);
6582 }
6583
6584 static inline int
6585 gc_mark_set(rb_objspace_t *objspace, VALUE obj)
6586 {
6587 ASSERT_vm_locking();
6588 if (RVALUE_MARKED(obj)) return 0;
6589 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
6590 return 1;
6591 }
6592
6593 static int
6594 gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
6595 {
6596 struct heap_page *page = GET_HEAP_PAGE(obj);
6597 bits_t *uncollectible_bits = &page->uncollectible_bits[0];
6598
6599 if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
6600 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
6601 MARK_IN_BITMAP(uncollectible_bits, obj);
6602 objspace->rgengc.uncollectible_wb_unprotected_objects++;
6603
6604 #if RGENGC_PROFILE > 0
6605 objspace->profile.total_remembered_shady_object_count++;
6606 #if RGENGC_PROFILE >= 2
6607 objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
6608 #endif
6609 #endif
6610 return TRUE;
6611 }
6612 else {
6613 return FALSE;
6614 }
6615 }
6616
6617 static void
6618 rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
6619 {
6620 const VALUE old_parent = objspace->rgengc.parent_object;
6621
6622 if (old_parent) { /* parent object is old */
6623 if (RVALUE_WB_UNPROTECTED(obj) || !RVALUE_OLD_P(obj)) {
6624 rgengc_remember(objspace, old_parent);
6625 }
6626 }
6627
6628 GC_ASSERT(old_parent == objspace->rgengc.parent_object);
6629 }
6630
6631 static void
6632 gc_grey(rb_objspace_t *objspace, VALUE obj)
6633 {
6634 #if RGENGC_CHECK_MODE
6635 if (RVALUE_MARKED(obj) == FALSE) rb_bug("gc_grey: %s is not marked.", obj_info(obj));
6636 if (RVALUE_MARKING(obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", obj_info(obj));
6637 #endif
6638
6639 if (is_incremental_marking(objspace)) {
6640 MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
6641 }
6642
6643 push_mark_stack(&objspace->mark_stack, obj);
6644 }
6645
6646 static void
6647 gc_aging(rb_objspace_t *objspace, VALUE obj)
6648 {
6649 struct heap_page *page = GET_HEAP_PAGE(obj);
6650
6651 GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
6652 check_rvalue_consistency(obj);
6653
6654 if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
6655 if (!RVALUE_OLD_P(obj)) {
6656 gc_report(3, objspace, "gc_aging: YOUNG: %s\n", obj_info(obj));
6657 RVALUE_AGE_INC(objspace, obj);
6658 }
6659 else if (is_full_marking(objspace)) {
6660 GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
6661 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
6662 }
6663 }
6664 check_rvalue_consistency(obj);
6665
6666 objspace->marked_slots++;
6667 }
6668
6669 NOINLINE(static void gc_mark_ptr(rb_objspace_t *objspace, VALUE obj));
6670 static void reachable_objects_from_callback(VALUE obj);
6671
6672 static void
6673 gc_mark_ptr(rb_objspace_t *objspace, VALUE obj)
6674 {
6675 if (LIKELY(during_gc)) {
6676 rgengc_check_relation(objspace, obj);
6677 if (!gc_mark_set(objspace, obj)) return; /* already marked */
6678
6679 if (0) { // for debug GC marking miss
6680 if (objspace->rgengc.parent_object) {
6681 RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
6682 (void *)obj, obj_type_name(obj),
6683 (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
6684 }
6685 else {
6686 RUBY_DEBUG_LOG("%p (%s)", (void *)obj, obj_type_name(obj));
6687 }
6688 }
6689
6690 if (UNLIKELY(RB_TYPE_P(obj, T_NONE))) {
6691 rp(obj);
6692 rb_bug("try to mark T_NONE object"); /* check here will help debugging */
6693 }
6694 gc_aging(objspace, obj);
6695 gc_grey(objspace, obj);
6696 }
6697 else {
6698 reachable_objects_from_callback(obj);
6699 }
6700 }
6701
6702 static inline void
6703 gc_pin(rb_objspace_t *objspace, VALUE obj)
6704 {
6705 GC_ASSERT(is_markable_object(obj));
6706 if (UNLIKELY(objspace->flags.during_compacting)) {
6707 if (LIKELY(during_gc)) {
6708 if (!MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj)) {
6709 GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
6710 GET_HEAP_PAGE(obj)->pinned_slots++;
6711 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
6712 }
6713 }
6714 }
6715 }
6716
6717 static inline void
6718 gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj)
6719 {
6720 if (!is_markable_object(obj)) return;
6721 gc_pin(objspace, obj);
6722 gc_mark_ptr(objspace, obj);
6723 }
6724
6725 static inline void
6726 gc_mark(rb_objspace_t *objspace, VALUE obj)
6727 {
6728 if (!is_markable_object(obj)) return;
6729 gc_mark_ptr(objspace, obj);
6730 }
6731
6732 void
6733 rb_gc_mark_movable(VALUE ptr)
6734 {
6735 gc_mark(&rb_objspace, ptr);
6736 }
6737
6738 void
6739 rb_gc_mark(VALUE ptr)
6740 {
6741 gc_mark_and_pin(&rb_objspace, ptr);
6742 }
6743
6744 void
6745 rb_gc_mark_and_move(VALUE *ptr)
6746 {
6747 rb_objspace_t *objspace = &rb_objspace;
6748 if (RB_SPECIAL_CONST_P(*ptr)) return;
6749
6750 if (UNLIKELY(objspace->flags.during_reference_updating)) {
6751 GC_ASSERT(objspace->flags.during_compacting);
6752 GC_ASSERT(during_gc);
6753
6754 *ptr = rb_gc_location(*ptr);
6755 }
6756 else {
6757 gc_mark_ptr(objspace, *ptr);
6758 }
6759 }
6760
6761 void
6762 rb_gc_mark_weak(VALUE *ptr)
6763 {
6764 rb_objspace_t *objspace = &rb_objspace;
6765
6766 if (UNLIKELY(!during_gc)) return;
6767
6768 VALUE obj = *ptr;
6769 if (RB_SPECIAL_CONST_P(obj)) return;
6770
6771 GC_ASSERT(objspace->rgengc.parent_object == 0 || FL_TEST(objspace->rgengc.parent_object, FL_WB_PROTECTED));
6772
6773 if (UNLIKELY(RB_TYPE_P(obj, T_NONE))) {
6774 rp(obj);
6775 rb_bug("try to mark T_NONE object");
6776 }
6777
6778 /* If we are in a minor GC and the other object is old, then obj should
6779 * already be marked and cannot be reclaimed in this GC cycle so we don't
6780 * need to add it to the weak refences list. */
6781 if (!is_full_marking(objspace) && RVALUE_OLD_P(obj)) {
6782 GC_ASSERT(RVALUE_MARKED(obj));
6783 GC_ASSERT(!objspace->flags.during_compacting);
6784
6785 return;
6786 }
6787
6788 rgengc_check_relation(objspace, obj);
6789
6790 DURING_GC_COULD_MALLOC_REGION_START();
6791 {
6792 rb_darray_append(&objspace->weak_references, ptr);
6793 }
6794 DURING_GC_COULD_MALLOC_REGION_END();
6795
6796 objspace->profile.weak_references_count++;
6797 }
6798
6799 void
6800 rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr)
6801 {
6802 rb_objspace_t *objspace = &rb_objspace;
6803
6804 /* If we're not incremental marking, then the state of the objects can't
6805 * change so we don't need to do anything. */
6806 if (!is_incremental_marking(objspace)) return;
6807 /* If parent_obj has not been marked, then ptr has not yet been marked
6808 * weak, so we don't need to do anything. */
6809 if (!RVALUE_MARKED(parent_obj)) return;
6810
6811 VALUE **ptr_ptr;
6812 rb_darray_foreach(objspace->weak_references, i, ptr_ptr) {
6813 if (*ptr_ptr == ptr) {
6814 *ptr_ptr = NULL;
6815 break;
6816 }
6817 }
6818 }
6819
6820 static inline void
6821 gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
6822 {
6823 if (RVALUE_OLD_P(obj)) {
6824 objspace->rgengc.parent_object = obj;
6825 }
6826 else {
6827 objspace->rgengc.parent_object = Qfalse;
6828 }
6829 }
6830
6831 static bool
6832 gc_declarative_marking_p(const rb_data_type_t *type)
6833 {
6834 return (type->flags & RUBY_TYPED_DECL_MARKING) != 0;
6835 }
6836
6837 static void mark_cvc_tbl(rb_objspace_t *objspace, VALUE klass);
6838
6839 static void
6840 gc_mark_children(rb_objspace_t *objspace, VALUE obj)
6841 {
6842 register RVALUE *any = RANY(obj);
6843 gc_mark_set_parent(objspace, obj);
6844
6845 if (FL_TEST(obj, FL_EXIVAR)) {
6846 rb_mark_generic_ivar(obj);
6847 }
6848
6849 switch (BUILTIN_TYPE(obj)) {
6850 case T_FLOAT:
6851 case T_BIGNUM:
6852 case T_SYMBOL:
6853 /* Not immediates, but does not have references and singleton class.
6854 *
6855 * RSYMBOL(obj)->fstr intentionally not marked. See log for 96815f1e
6856 * ("symbol.c: remove rb_gc_mark_symbols()") */
6857 return;
6858
6859 case T_NIL:
6860 case T_FIXNUM:
6861 rb_bug("rb_gc_mark() called for broken object");
6862 break;
6863
6864 case T_NODE:
6865 UNEXPECTED_NODE(rb_gc_mark);
6866 break;
6867
6868 case T_IMEMO:
6869 rb_imemo_mark_and_move(obj, false);
6870 return;
6871
6872 default:
6873 break;
6874 }
6875
6876 gc_mark(objspace, any->as.basic.klass);
6877
6878 switch (BUILTIN_TYPE(obj)) {
6879 case T_CLASS:
6880 if (FL_TEST(obj, FL_SINGLETON)) {
6881 gc_mark(objspace, RCLASS_ATTACHED_OBJECT(obj));
6882 }
6883 // Continue to the shared T_CLASS/T_MODULE
6884 case T_MODULE:
6885 if (RCLASS_SUPER(obj)) {
6886 gc_mark(objspace, RCLASS_SUPER(obj));
6887 }
6888
6889 mark_m_tbl(objspace, RCLASS_M_TBL(obj));
6890 mark_cvc_tbl(objspace, obj);
6891 rb_cc_table_mark(obj);
6892 if (rb_shape_obj_too_complex(obj)) {
6893 mark_tbl_no_pin(objspace, (st_table *)RCLASS_IVPTR(obj));
6894 }
6895 else {
6896 for (attr_index_t i = 0; i < RCLASS_IV_COUNT(obj); i++) {
6897 gc_mark(objspace, RCLASS_IVPTR(obj)[i]);
6898 }
6899 }
6900 mark_const_tbl(objspace, RCLASS_CONST_TBL(obj));
6901
6902 gc_mark(objspace, RCLASS_EXT(obj)->classpath);
6903 break;
6904
6905 case T_ICLASS:
6906 if (RICLASS_OWNS_M_TBL_P(obj)) {
6907 mark_m_tbl(objspace, RCLASS_M_TBL(obj));
6908 }
6909 if (RCLASS_SUPER(obj)) {
6910 gc_mark(objspace, RCLASS_SUPER(obj));
6911 }
6912
6913 if (RCLASS_INCLUDER(obj)) {
6914 gc_mark(objspace, RCLASS_INCLUDER(obj));
6915 }
6916 mark_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
6917 rb_cc_table_mark(obj);
6918 break;
6919
6920 case T_ARRAY:
6921 if (ARY_SHARED_P(obj)) {
6922 VALUE root = ARY_SHARED_ROOT(obj);
6923 gc_mark(objspace, root);
6924 }
6925 else {
6926 long i, len = RARRAY_LEN(obj);
6927 const VALUE *ptr = RARRAY_CONST_PTR(obj);
6928 for (i=0; i < len; i++) {
6929 gc_mark(objspace, ptr[i]);
6930 }
6931 }
6932 break;
6933
6934 case T_HASH:
6935 mark_hash(objspace, obj);
6936 break;
6937
6938 case T_STRING:
6939 if (STR_SHARED_P(obj)) {
6940 if (STR_EMBED_P(any->as.string.as.heap.aux.shared)) {
6941 /* Embedded shared strings cannot be moved because this string
6942 * points into the slot of the shared string. There may be code
6943 * using the RSTRING_PTR on the stack, which would pin this
6944 * string but not pin the shared string, causing it to move. */
6945 gc_mark_and_pin(objspace, any->as.string.as.heap.aux.shared);
6946 }
6947 else {
6948 gc_mark(objspace, any->as.string.as.heap.aux.shared);
6949 }
6950 }
6951 break;
6952
6953 case T_DATA:
6954 {
6955 void *const ptr = RTYPEDDATA_P(obj) ? RTYPEDDATA_GET_DATA(obj) : DATA_PTR(obj);
6956
6957 if (ptr) {
6958 if (RTYPEDDATA_P(obj) && gc_declarative_marking_p(any->as.typeddata.type)) {
6959 size_t *offset_list = (size_t *)RANY(obj)->as.typeddata.type->function.dmark;
6960
6961 for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) {
6962 rb_gc_mark_movable(*(VALUE *)((char *)ptr + offset));
6963 }
6964 }
6965 else {
6966 RUBY_DATA_FUNC mark_func = RTYPEDDATA_P(obj) ?
6967 any->as.typeddata.type->function.dmark :
6968 any->as.data.dmark;
6969 if (mark_func) (*mark_func)(ptr);
6970 }
6971 }
6972 }
6973 break;
6974
6975 case T_OBJECT:
6976 {
6977 rb_shape_t *shape = rb_shape_get_shape_by_id(ROBJECT_SHAPE_ID(obj));
6978 if (rb_shape_obj_too_complex(obj)) {
6979 mark_tbl_no_pin(objspace, ROBJECT_IV_HASH(obj));
6980 }
6981 else {
6982 const VALUE * const ptr = ROBJECT_IVPTR(obj);
6983
6984 uint32_t i, len = ROBJECT_IV_COUNT(obj);
6985 for (i = 0; i < len; i++) {
6986 gc_mark(objspace, ptr[i]);
6987 }
6988 }
6989 if (shape) {
6990 VALUE klass = RBASIC_CLASS(obj);
6991
6992 // Increment max_iv_count if applicable, used to determine size pool allocation
6993 attr_index_t num_of_ivs = shape->next_iv_index;
6994 if (RCLASS_EXT(klass)->max_iv_count < num_of_ivs) {
6995 RCLASS_EXT(klass)->max_iv_count = num_of_ivs;
6996 }
6997 }
6998 }
6999 break;
7000
7001 case T_FILE:
7002 if (any->as.file.fptr) {
7003 gc_mark(objspace, any->as.file.fptr->self);
7004 gc_mark(objspace, any->as.file.fptr->pathv);
7005 gc_mark(objspace, any->as.file.fptr->tied_io_for_writing);
7006 gc_mark(objspace, any->as.file.fptr->writeconv_asciicompat);
7007 gc_mark(objspace, any->as.file.fptr->writeconv_pre_ecopts);
7008 gc_mark(objspace, any->as.file.fptr->encs.ecopts);
7009 gc_mark(objspace, any->as.file.fptr->write_lock);
7010 gc_mark(objspace, any->as.file.fptr->timeout);
7011 }
7012 break;
7013
7014 case T_REGEXP:
7015 gc_mark(objspace, any->as.regexp.src);
7016 break;
7017
7018 case T_MATCH:
7019 gc_mark(objspace, any->as.match.regexp);
7020 if (any->as.match.str) {
7021 gc_mark(objspace, any->as.match.str);
7022 }
7023 break;
7024
7025 case T_RATIONAL:
7026 gc_mark(objspace, any->as.rational.num);
7027 gc_mark(objspace, any->as.rational.den);
7028 break;
7029
7030 case T_COMPLEX:
7031 gc_mark(objspace, any->as.complex.real);
7032 gc_mark(objspace, any->as.complex.imag);
7033 break;
7034
7035 case T_STRUCT:
7036 {
7037 long i;
7038 const long len = RSTRUCT_LEN(obj);
7039 const VALUE * const ptr = RSTRUCT_CONST_PTR(obj);
7040
7041 for (i=0; i<len; i++) {
7042 gc_mark(objspace, ptr[i]);
7043 }
7044 }
7045 break;
7046
7047 default:
7048 #if GC_DEBUG
7049 rb_gcdebug_print_obj_condition((VALUE)obj);
7050 #endif
7051 if (BUILTIN_TYPE(obj) == T_MOVED) rb_bug("rb_gc_mark(): %p is T_MOVED", (void *)obj);
7052 if (BUILTIN_TYPE(obj) == T_NONE) rb_bug("rb_gc_mark(): %p is T_NONE", (void *)obj);
7053 if (BUILTIN_TYPE(obj) == T_ZOMBIE) rb_bug("rb_gc_mark(): %p is T_ZOMBIE", (void *)obj);
7054 rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
7055 BUILTIN_TYPE(obj), (void *)any,
7056 is_pointer_to_heap(objspace, any) ? "corrupted object" : "non object");
7057 }
7058 }
7059
7060 /**
7061 * incremental: 0 -> not incremental (do all)
7062 * incremental: n -> mark at most `n' objects
7063 */
7064 static inline int
7065 gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count)
7066 {
7067 mark_stack_t *mstack = &objspace->mark_stack;
7068 VALUE obj;
7069 size_t marked_slots_at_the_beginning = objspace->marked_slots;
7070 size_t popped_count = 0;
7071
7072 while (pop_mark_stack(mstack, &obj)) {
7073 if (UNDEF_P(obj)) continue; /* skip */
7074
7075 if (RGENGC_CHECK_MODE && !RVALUE_MARKED(obj)) {
7076 rb_bug("gc_mark_stacked_objects: %s is not marked.", obj_info(obj));
7077 }
7078 gc_mark_children(objspace, obj);
7079
7080 if (incremental) {
7081 if (RGENGC_CHECK_MODE && !RVALUE_MARKING(obj)) {
7082 rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0");
7083 }
7084 CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
7085 popped_count++;
7086
7087 if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) {
7088 break;
7089 }
7090 }
7091 else {
7092 /* just ignore marking bits */
7093 }
7094 }
7095
7096 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
7097
7098 if (is_mark_stack_empty(mstack)) {
7099 shrink_stack_chunk_cache(mstack);
7100 return TRUE;
7101 }
7102 else {
7103 return FALSE;
7104 }
7105 }
7106
7107 static int
7108 gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count)
7109 {
7110 return gc_mark_stacked_objects(objspace, TRUE, count);
7111 }
7112
7113 static int
7114 gc_mark_stacked_objects_all(rb_objspace_t *objspace)
7115 {
7116 return gc_mark_stacked_objects(objspace, FALSE, 0);
7117 }
7118
7119 #if PRINT_ROOT_TICKS
7120 #define MAX_TICKS 0x100
7121 static tick_t mark_ticks[MAX_TICKS];
7122 static const char *mark_ticks_categories[MAX_TICKS];
7123
7124 static void
7125 show_mark_ticks(void)
7126 {
7127 int i;
7128 fprintf(stderr, "mark ticks result:\n");
7129 for (i=0; i<MAX_TICKS; i++) {
7130 const char *category = mark_ticks_categories[i];
7131 if (category) {
7132 fprintf(stderr, "%s\t%8lu\n", category, (unsigned long)mark_ticks[i]);
7133 }
7134 else {
7135 break;
7136 }
7137 }
7138 }
7139
7140 #endif /* PRINT_ROOT_TICKS */
7141
7142 static void
7143 gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
7144 {
7145 rb_execution_context_t *ec = GET_EC();
7146 rb_vm_t *vm = rb_ec_vm_ptr(ec);
7147
7148 #if PRINT_ROOT_TICKS
7149 tick_t start_tick = tick();
7150 int tick_count = 0;
7151 const char *prev_category = 0;
7152
7153 if (mark_ticks_categories[0] == 0) {
7154 atexit(show_mark_ticks);
7155 }
7156 #endif
7157
7158 if (categoryp) *categoryp = "xxx";
7159
7160 objspace->rgengc.parent_object = Qfalse;
7161
7162 #if PRINT_ROOT_TICKS
7163 #define MARK_CHECKPOINT_PRINT_TICK(category) do { \
7164 if (prev_category) { \
7165 tick_t t = tick(); \
7166 mark_ticks[tick_count] = t - start_tick; \
7167 mark_ticks_categories[tick_count] = prev_category; \
7168 tick_count++; \
7169 } \
7170 prev_category = category; \
7171 start_tick = tick(); \
7172 } while (0)
7173 #else /* PRINT_ROOT_TICKS */
7174 #define MARK_CHECKPOINT_PRINT_TICK(category)
7175 #endif
7176
7177 #define MARK_CHECKPOINT(category) do { \
7178 if (categoryp) *categoryp = category; \
7179 MARK_CHECKPOINT_PRINT_TICK(category); \
7180 } while (0)
7181
7182 MARK_CHECKPOINT("vm");
7183 SET_STACK_END;
7184 rb_vm_mark(vm);
7185 if (vm->self) gc_mark(objspace, vm->self);
7186
7187 MARK_CHECKPOINT("finalizers");
7188 mark_finalizer_tbl(objspace, finalizer_table);
7189
7190 MARK_CHECKPOINT("machine_context");
7191 mark_current_machine_context(objspace, ec);
7192
7193 /* mark protected global variables */
7194
7195 MARK_CHECKPOINT("end_proc");
7196 rb_mark_end_proc();
7197
7198 MARK_CHECKPOINT("global_tbl");
7199 rb_gc_mark_global_tbl();
7200
7201 MARK_CHECKPOINT("object_id");
7202 mark_tbl_no_pin(objspace, objspace->obj_to_id_tbl); /* Only mark ids */
7203
7204 if (stress_to_class) rb_gc_mark(stress_to_class);
7205
7206 MARK_CHECKPOINT("finish");
7207 #undef MARK_CHECKPOINT
7208 }
7209
7210 #if RGENGC_CHECK_MODE >= 4
7211
7212 #define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
7213 #define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
7214 #define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
7215
7216 struct reflist {
7217 VALUE *list;
7218 int pos;
7219 int size;
7220 };
7221
7222 static struct reflist *
7223 reflist_create(VALUE obj)
7224 {
7225 struct reflist *refs = xmalloc(sizeof(struct reflist));
7226 refs->size = 1;
7227 refs->list = ALLOC_N(VALUE, refs->size);
7228 refs->list[0] = obj;
7229 refs->pos = 1;
7230 return refs;
7231 }
7232
7233 static void
7234 reflist_destruct(struct reflist *refs)
7235 {
7236 xfree(refs->list);
7237 xfree(refs);
7238 }
7239
7240 static void
7241 reflist_add(struct reflist *refs, VALUE obj)
7242 {
7243 if (refs->pos == refs->size) {
7244 refs->size *= 2;
7245 SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2);
7246 }
7247
7248 refs->list[refs->pos++] = obj;
7249 }
7250
7251 static void
7252 reflist_dump(struct reflist *refs)
7253 {
7254 int i;
7255 for (i=0; i<refs->pos; i++) {
7256 VALUE obj = refs->list[i];
7257 if (IS_ROOTSIG(obj)) { /* root */
7258 fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj));
7259 }
7260 else {
7261 fprintf(stderr, "<%s>", obj_info(obj));
7262 }
7263 if (i+1 < refs->pos) fprintf(stderr, ", ");
7264 }
7265 }
7266
7267 static int
7268 reflist_referred_from_machine_context(struct reflist *refs)
7269 {
7270 int i;
7271 for (i=0; i<refs->pos; i++) {
7272 VALUE obj = refs->list[i];
7273 if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1;
7274 }
7275 return 0;
7276 }
7277
7278 struct allrefs {
7279 rb_objspace_t *objspace;
7280 /* a -> obj1
7281 * b -> obj1
7282 * c -> obj1
7283 * c -> obj2
7284 * d -> obj3
7285 * #=> {obj1 => [a, b, c], obj2 => [c, d]}
7286 */
7287 struct st_table *references;
7288 const char *category;
7289 VALUE root_obj;
7290 mark_stack_t mark_stack;
7291 };
7292
7293 static int
7294 allrefs_add(struct allrefs *data, VALUE obj)
7295 {
7296 struct reflist *refs;
7297 st_data_t r;
7298
7299 if (st_lookup(data->references, obj, &r)) {
7300 refs = (struct reflist *)r;
7301 reflist_add(refs, data->root_obj);
7302 return 0;
7303 }
7304 else {
7305 refs = reflist_create(data->root_obj);
7306 st_insert(data->references, obj, (st_data_t)refs);
7307 return 1;
7308 }
7309 }
7310
7311 static void
7312 allrefs_i(VALUE obj, void *ptr)
7313 {
7314 struct allrefs *data = (struct allrefs *)ptr;
7315
7316 if (allrefs_add(data, obj)) {
7317 push_mark_stack(&data->mark_stack, obj);
7318 }
7319 }
7320
7321 static void
7322 allrefs_roots_i(VALUE obj, void *ptr)
7323 {
7324 struct allrefs *data = (struct allrefs *)ptr;
7325 if (strlen(data->category) == 0) rb_bug("!!!");
7326 data->root_obj = MAKE_ROOTSIG(data->category);
7327
7328 if (allrefs_add(data, obj)) {
7329 push_mark_stack(&data->mark_stack, obj);
7330 }
7331 }
7332 #define PUSH_MARK_FUNC_DATA(v) do { \
7333 struct gc_mark_func_data_struct *prev_mark_func_data = GET_RACTOR()->mfd; \
7334 GET_RACTOR()->mfd = (v);
7335
7336 #define POP_MARK_FUNC_DATA() GET_RACTOR()->mfd = prev_mark_func_data;} while (0)
7337
7338 static st_table *
7339 objspace_allrefs(rb_objspace_t *objspace)
7340 {
7341 struct allrefs data;
7342 struct gc_mark_func_data_struct mfd;
7343 VALUE obj;
7344 int prev_dont_gc = dont_gc_val();
7345 dont_gc_on();
7346
7347 data.objspace = objspace;
7348 data.references = st_init_numtable();
7349 init_mark_stack(&data.mark_stack);
7350
7351 mfd.mark_func = allrefs_roots_i;
7352 mfd.data = &data;
7353
7354 /* traverse root objects */
7355 PUSH_MARK_FUNC_DATA(&mfd);
7356 GET_RACTOR()->mfd = &mfd;
7357 gc_mark_roots(objspace, &data.category);
7358 POP_MARK_FUNC_DATA();
7359
7360 /* traverse rest objects reachable from root objects */
7361 while (pop_mark_stack(&data.mark_stack, &obj)) {
7362 rb_objspace_reachable_objects_from(data.root_obj = obj, allrefs_i, &data);
7363 }
7364 free_stack_chunks(&data.mark_stack);
7365
7366 dont_gc_set(prev_dont_gc);
7367 return data.references;
7368 }
7369
7370 static int
7371 objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
7372 {
7373 struct reflist *refs = (struct reflist *)value;
7374 reflist_destruct(refs);
7375 return ST_CONTINUE;
7376 }
7377
7378 static void
7379 objspace_allrefs_destruct(struct st_table *refs)
7380 {
7381 st_foreach(refs, objspace_allrefs_destruct_i, 0);
7382 st_free_table(refs);
7383 }
7384
7385 #if RGENGC_CHECK_MODE >= 5
7386 static int
7387 allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
7388 {
7389 VALUE obj = (VALUE)k;
7390 struct reflist *refs = (struct reflist *)v;
7391 fprintf(stderr, "[allrefs_dump_i] %s <- ", obj_info(obj));
7392 reflist_dump(refs);
7393 fprintf(stderr, "\n");
7394 return ST_CONTINUE;
7395 }
7396
7397 static void
7398 allrefs_dump(rb_objspace_t *objspace)
7399 {
7400 VALUE size = objspace->rgengc.allrefs_table->num_entries;
7401 fprintf(stderr, "[all refs] (size: %"PRIuVALUE")\n", size);
7402 st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
7403 }
7404 #endif
7405
7406 static int
7407 gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
7408 {
7409 VALUE obj = k;
7410 struct reflist *refs = (struct reflist *)v;
7411 rb_objspace_t *objspace = (rb_objspace_t *)ptr;
7412
7413 /* object should be marked or oldgen */
7414 if (!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj)) {
7415 fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", obj_info(obj));
7416 fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj);
7417 reflist_dump(refs);
7418
7419 if (reflist_referred_from_machine_context(refs)) {
7420 fprintf(stderr, " (marked from machine stack).\n");
7421 /* marked from machine context can be false positive */
7422 }
7423 else {
7424 objspace->rgengc.error_count++;
7425 fprintf(stderr, "\n");
7426 }
7427 }
7428 return ST_CONTINUE;
7429 }
7430
7431 static void
7432 gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
7433 {
7434 size_t saved_malloc_increase = objspace->malloc_params.increase;
7435 #if RGENGC_ESTIMATE_OLDMALLOC
7436 size_t saved_oldmalloc_increase = objspace->rgengc.oldmalloc_increase;
7437 #endif
7438 VALUE already_disabled = rb_objspace_gc_disable(objspace);
7439
7440 objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
7441
7442 if (checker_func) {
7443 st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace);
7444 }
7445
7446 if (objspace->rgengc.error_count > 0) {
7447 #if RGENGC_CHECK_MODE >= 5
7448 allrefs_dump(objspace);
7449 #endif
7450 if (checker_name) rb_bug("%s: GC has problem.", checker_name);
7451 }
7452
7453 objspace_allrefs_destruct(objspace->rgengc.allrefs_table);
7454 objspace->rgengc.allrefs_table = 0;
7455
7456 if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
7457 objspace->malloc_params.increase = saved_malloc_increase;
7458 #if RGENGC_ESTIMATE_OLDMALLOC
7459 objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase;
7460 #endif
7461 }
7462 #endif /* RGENGC_CHECK_MODE >= 4 */
7463
7464 struct verify_internal_consistency_struct {
7465 rb_objspace_t *objspace;
7466 int err_count;
7467 size_t live_object_count;
7468 size_t zombie_object_count;
7469
7470 VALUE parent;
7471 size_t old_object_count;
7472 size_t remembered_shady_count;
7473 };
7474
7475 static void
7476 check_generation_i(const VALUE child, void *ptr)
7477 {
7478 struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
7479 const VALUE parent = data->parent;
7480
7481 if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(parent));
7482
7483 if (!RVALUE_OLD_P(child)) {
7484 if (!RVALUE_REMEMBERED(parent) &&
7485 !RVALUE_REMEMBERED(child) &&
7486 !RVALUE_UNCOLLECTIBLE(child)) {
7487 fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", obj_info(parent), obj_info(child));
7488 data->err_count++;
7489 }
7490 }
7491 }
7492
7493 static void
7494 check_color_i(const VALUE child, void *ptr)
7495 {
7496 struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
7497 const VALUE parent = data->parent;
7498
7499 if (!RVALUE_WB_UNPROTECTED(parent) && RVALUE_WHITE_P(child)) {
7500 fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
7501 obj_info(parent), obj_info(child));
7502 data->err_count++;
7503 }
7504 }
7505
7506 static void
7507 check_children_i(const VALUE child, void *ptr)
7508 {
7509 struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
7510 if (check_rvalue_consistency_force(child, FALSE) != 0) {
7511 fprintf(stderr, "check_children_i: %s has error (referenced from %s)",
7512 obj_info(child), obj_info(data->parent));
7513 rb_print_backtrace(stderr); /* C backtrace will help to debug */
7514
7515 data->err_count++;
7516 }
7517 }
7518
7519 static int
7520 verify_internal_consistency_i(void *page_start, void *page_end, size_t stride,
7521 struct verify_internal_consistency_struct *data)
7522 {
7523 VALUE obj;
7524 rb_objspace_t *objspace = data->objspace;
7525
7526 for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) {
7527 void *poisoned = asan_unpoison_object_temporary(obj);
7528
7529 if (is_live_object(objspace, obj)) {
7530 /* count objects */
7531 data->live_object_count++;
7532 data->parent = obj;
7533
7534 /* Normally, we don't expect T_MOVED objects to be in the heap.
7535 * But they can stay alive on the stack, */
7536 if (!gc_object_moved_p(objspace, obj)) {
7537 /* moved slots don't have children */
7538 rb_objspace_reachable_objects_from(obj, check_children_i, (void *)data);
7539 }
7540
7541 /* check health of children */
7542 if (RVALUE_OLD_P(obj)) data->old_object_count++;
7543 if (RVALUE_WB_UNPROTECTED(obj) && RVALUE_UNCOLLECTIBLE(obj)) data->remembered_shady_count++;
7544
7545 if (!is_marking(objspace) && RVALUE_OLD_P(obj)) {
7546 /* reachable objects from an oldgen object should be old or (young with remember) */
7547 data->parent = obj;
7548 rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data);
7549 }
7550
7551 if (is_incremental_marking(objspace)) {
7552 if (RVALUE_BLACK_P(obj)) {
7553 /* reachable objects from black objects should be black or grey objects */
7554 data->parent = obj;
7555 rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data);
7556 }
7557 }
7558 }
7559 else {
7560 if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
7561 data->zombie_object_count++;
7562
7563 if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) {
7564 fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
7565 obj_info(obj));
7566 data->err_count++;
7567 }
7568
7569 if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) {
7570 fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
7571 FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in",
7572 obj_info(obj));
7573 data->err_count++;
7574 }
7575 }
7576 }
7577 if (poisoned) {
7578 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
7579 asan_poison_object(obj);
7580 }
7581 }
7582
7583 return 0;
7584 }
7585
7586 static int
7587 gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
7588 {
7589 unsigned int has_remembered_shady = FALSE;
7590 unsigned int has_remembered_old = FALSE;
7591 int remembered_old_objects = 0;
7592 int free_objects = 0;
7593 int zombie_objects = 0;
7594
7595 short slot_size = page->slot_size;
7596 uintptr_t start = (uintptr_t)page->start;
7597 uintptr_t end = start + page->total_slots * slot_size;
7598
7599 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
7600 VALUE val = (VALUE)ptr;
7601 void *poisoned = asan_unpoison_object_temporary(val);
7602 enum ruby_value_type type = BUILTIN_TYPE(val);
7603
7604 if (type == T_NONE) free_objects++;
7605 if (type == T_ZOMBIE) zombie_objects++;
7606 if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
7607 has_remembered_shady = TRUE;
7608 }
7609 if (RVALUE_PAGE_MARKING(page, val)) {
7610 has_remembered_old = TRUE;
7611 remembered_old_objects++;
7612 }
7613
7614 if (poisoned) {
7615 GC_ASSERT(BUILTIN_TYPE(val) == T_NONE);
7616 asan_poison_object(val);
7617 }
7618 }
7619
7620 if (!is_incremental_marking(objspace) &&
7621 page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
7622
7623 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
7624 VALUE val = (VALUE)ptr;
7625 if (RVALUE_PAGE_MARKING(page, val)) {
7626 fprintf(stderr, "marking -> %s\n", obj_info(val));
7627 }
7628 }
7629 rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
7630 (void *)page, remembered_old_objects, obj ? obj_info(obj) : "");
7631 }
7632
7633 if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
7634 rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
7635 (void *)page, obj ? obj_info(obj) : "");
7636 }
7637
7638 if (0) {
7639 /* free_slots may not equal to free_objects */
7640 if (page->free_slots != free_objects) {
7641 rb_bug("page %p's free_slots should be %d, but %d", (void *)page, page->free_slots, free_objects);
7642 }
7643 }
7644 if (page->final_slots != zombie_objects) {
7645 rb_bug("page %p's final_slots should be %d, but %d", (void *)page, page->final_slots, zombie_objects);
7646 }
7647
7648 return remembered_old_objects;
7649 }
7650
7651 static int
7652 gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head)
7653 {
7654 int remembered_old_objects = 0;
7655 struct heap_page *page = 0;
7656
7657 ccan_list_for_each(head, page, page_node) {
7658 asan_unlock_freelist(page);
7659 RVALUE *p = page->freelist;
7660 while (p) {
7661 VALUE vp = (VALUE)p;
7662 VALUE prev = vp;
7663 asan_unpoison_object(vp, false);
7664 if (BUILTIN_TYPE(vp) != T_NONE) {
7665 fprintf(stderr, "freelist slot expected to be T_NONE but was: %s\n", obj_info(vp));
7666 }
7667 p = p->as.free.next;
7668 asan_poison_object(prev);
7669 }
7670 asan_lock_freelist(page);
7671
7672 if (page->flags.has_remembered_objects == FALSE) {
7673 remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse);
7674 }
7675 }
7676
7677 return remembered_old_objects;
7678 }
7679
7680 static int
7681 gc_verify_heap_pages(rb_objspace_t *objspace)
7682 {
7683 int remembered_old_objects = 0;
7684 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
7685 remembered_old_objects += gc_verify_heap_pages_(objspace, &(SIZE_POOL_EDEN_HEAP(&size_pools[i])->pages));
7686 remembered_old_objects += gc_verify_heap_pages_(objspace, &(SIZE_POOL_TOMB_HEAP(&size_pools[i])->pages));
7687 }
7688 return remembered_old_objects;
7689 }
7690
7691 /*
7692 * call-seq:
7693 * GC.verify_internal_consistency -> nil
7694 *
7695 * Verify internal consistency.
7696 *
7697 * This method is implementation specific.
7698 * Now this method checks generational consistency
7699 * if RGenGC is supported.
7700 */
7701 static VALUE
7702 gc_verify_internal_consistency_m(VALUE dummy)
7703 {
7704 gc_verify_internal_consistency(&rb_objspace);
7705 return Qnil;
7706 }
7707
7708 static void
7709 gc_verify_internal_consistency_(rb_objspace_t *objspace)
7710 {
7711 struct verify_internal_consistency_struct data = {0};
7712
7713 data.objspace = objspace;
7714 gc_report(5, objspace, "gc_verify_internal_consistency: start\n");
7715
7716 /* check relations */
7717 for (size_t i = 0; i < heap_allocated_pages; i++) {
7718 struct heap_page *page = heap_pages_sorted[i];
7719 short slot_size = page->slot_size;
7720
7721 uintptr_t start = (uintptr_t)page->start;
7722 uintptr_t end = start + page->total_slots * slot_size;
7723
7724 verify_internal_consistency_i((void *)start, (void *)end, slot_size, &data);
7725 }
7726
7727 if (data.err_count != 0) {
7728 #if RGENGC_CHECK_MODE >= 5
7729 objspace->rgengc.error_count = data.err_count;
7730 gc_marks_check(objspace, NULL, NULL);
7731 allrefs_dump(objspace);
7732 #endif
7733 rb_bug("gc_verify_internal_consistency: found internal inconsistency.");
7734 }
7735
7736 /* check heap_page status */
7737 gc_verify_heap_pages(objspace);
7738
7739 /* check counters */
7740
7741 if (!is_lazy_sweeping(objspace) &&
7742 !finalizing &&
7743 ruby_single_main_ractor != NULL) {
7744 if (objspace_live_slots(objspace) != data.live_object_count) {
7745 fprintf(stderr, "heap_pages_final_slots: %"PRIdSIZE", total_freed_objects: %"PRIdSIZE"\n",
7746 heap_pages_final_slots, total_freed_objects(objspace));
7747 rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
7748 objspace_live_slots(objspace), data.live_object_count);
7749 }
7750 }
7751
7752 if (!is_marking(objspace)) {
7753 if (objspace->rgengc.old_objects != data.old_object_count) {
7754 rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
7755 objspace->rgengc.old_objects, data.old_object_count);
7756 }
7757 if (objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
7758 rb_bug("inconsistent number of wb unprotected objects: expect %"PRIuSIZE", but %"PRIuSIZE".",
7759 objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
7760 }
7761 }
7762
7763 if (!finalizing) {
7764 size_t list_count = 0;
7765
7766 {
7767 VALUE z = heap_pages_deferred_final;
7768 while (z) {
7769 list_count++;
7770 z = RZOMBIE(z)->next;
7771 }
7772 }
7773
7774 if (heap_pages_final_slots != data.zombie_object_count ||
7775 heap_pages_final_slots != list_count) {
7776
7777 rb_bug("inconsistent finalizing object count:\n"
7778 " expect %"PRIuSIZE"\n"
7779 " but %"PRIuSIZE" zombies\n"
7780 " heap_pages_deferred_final list has %"PRIuSIZE" items.",
7781 heap_pages_final_slots,
7782 data.zombie_object_count,
7783 list_count);
7784 }
7785 }
7786
7787 gc_report(5, objspace, "gc_verify_internal_consistency: OK\n");
7788 }
7789
7790 static void
7791 gc_verify_internal_consistency(rb_objspace_t *objspace)
7792 {
7793 RB_VM_LOCK_ENTER();
7794 {
7795 rb_vm_barrier(); // stop other ractors
7796
7797 unsigned int prev_during_gc = during_gc;
7798 during_gc = FALSE; // stop gc here
7799 {
7800 gc_verify_internal_consistency_(objspace);
7801 }
7802 during_gc = prev_during_gc;
7803 }
7804 RB_VM_LOCK_LEAVE();
7805 }
7806
7807 void
7808 rb_gc_verify_internal_consistency(void)
7809 {
7810 gc_verify_internal_consistency(&rb_objspace);
7811 }
7812
7813 static void
7814 heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
7815 {
7816 if (heap->pooled_pages) {
7817 if (heap->free_pages) {
7818 struct heap_page *free_pages_tail = heap->free_pages;
7819 while (free_pages_tail->free_next) {
7820 free_pages_tail = free_pages_tail->free_next;
7821 }
7822 free_pages_tail->free_next = heap->pooled_pages;
7823 }
7824 else {
7825 heap->free_pages = heap->pooled_pages;
7826 }
7827
7828 heap->pooled_pages = NULL;
7829 }
7830 }
7831
7832 /* marks */
7833
7834 static void
7835 gc_marks_start(rb_objspace_t *objspace, int full_mark)
7836 {
7837 /* start marking */
7838 gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor");
7839 gc_mode_transition(objspace, gc_mode_marking);
7840
7841 if (full_mark) {
7842 size_t incremental_marking_steps = (objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
7843 objspace->rincgc.step_slots = (objspace->marked_slots * 2) / incremental_marking_steps;
7844
7845 if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE", "
7846 "objspace->rincgc.pooled_page_num: %"PRIdSIZE", "
7847 "objspace->rincgc.step_slots: %"PRIdSIZE", \n",
7848 objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots);
7849 objspace->flags.during_minor_gc = FALSE;
7850 if (ruby_enable_autocompact) {
7851 objspace->flags.during_compacting |= TRUE;
7852 }
7853 objspace->profile.major_gc_count++;
7854 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
7855 objspace->rgengc.old_objects = 0;
7856 objspace->rgengc.last_major_gc = objspace->profile.count;
7857 objspace->marked_slots = 0;
7858
7859 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
7860 rb_size_pool_t *size_pool = &size_pools[i];
7861 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
7862 rgengc_mark_and_rememberset_clear(objspace, heap);
7863 heap_move_pooled_pages_to_free_pages(heap);
7864
7865 if (objspace->flags.during_compacting) {
7866 struct heap_page *page = NULL;
7867
7868 ccan_list_for_each(&heap->pages, page, page_node) {
7869 page->pinned_slots = 0;
7870 }
7871 }
7872 }
7873 }
7874 else {
7875 objspace->flags.during_minor_gc = TRUE;
7876 objspace->marked_slots =
7877 objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */
7878 objspace->profile.minor_gc_count++;
7879
7880 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
7881 rgengc_rememberset_mark(objspace, SIZE_POOL_EDEN_HEAP(&size_pools[i]));
7882 }
7883 }
7884
7885 gc_mark_roots(objspace, NULL);
7886
7887 gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %"PRIdSIZE"\n",
7888 full_mark ? "full" : "minor", mark_stack_size(&objspace->mark_stack));
7889 }
7890
7891 static inline void
7892 gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits)
7893 {
7894 if (bits) {
7895 do {
7896 if (bits & 1) {
7897 gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", obj_info((VALUE)p));
7898 GC_ASSERT(RVALUE_WB_UNPROTECTED((VALUE)p));
7899 GC_ASSERT(RVALUE_MARKED((VALUE)p));
7900 gc_mark_children(objspace, (VALUE)p);
7901 }
7902 p += BASE_SLOT_SIZE;
7903 bits >>= 1;
7904 } while (bits);
7905 }
7906 }
7907
7908 static void
7909 gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap)
7910 {
7911 struct heap_page *page = 0;
7912
7913 ccan_list_for_each(&heap->pages, page, page_node) {
7914 bits_t *mark_bits = page->mark_bits;
7915 bits_t *wbun_bits = page->wb_unprotected_bits;
7916 uintptr_t p = page->start;
7917 size_t j;
7918
7919 bits_t bits = mark_bits[0] & wbun_bits[0];
7920 bits >>= NUM_IN_PAGE(p);
7921 gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
7922 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
7923
7924 for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
7925 bits_t bits = mark_bits[j] & wbun_bits[j];
7926
7927 gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
7928 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
7929 }
7930 }
7931
7932 gc_mark_stacked_objects_all(objspace);
7933 }
7934
7935 static void
7936 gc_update_weak_references(rb_objspace_t *objspace)
7937 {
7938 size_t retained_weak_references_count = 0;
7939 VALUE **ptr_ptr;
7940 rb_darray_foreach(objspace->weak_references, i, ptr_ptr) {
7941 if (!*ptr_ptr) continue;
7942
7943 VALUE obj = **ptr_ptr;
7944
7945 if (RB_SPECIAL_CONST_P(obj)) continue;
7946
7947 if (!RVALUE_MARKED(obj)) {
7948 **ptr_ptr = Qundef;
7949 }
7950 else {
7951 retained_weak_references_count++;
7952 }
7953 }
7954
7955 objspace->profile.retained_weak_references_count = retained_weak_references_count;
7956
7957 rb_darray_clear(objspace->weak_references);
7958
7959 DURING_GC_COULD_MALLOC_REGION_START();
7960 {
7961 rb_darray_resize_capa(&objspace->weak_references, retained_weak_references_count);
7962 }
7963 DURING_GC_COULD_MALLOC_REGION_END();
7964 }
7965
7966 static void
7967 gc_marks_finish(rb_objspace_t *objspace)
7968 {
7969 /* finish incremental GC */
7970 if (is_incremental_marking(objspace)) {
7971 if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) {
7972 rb_bug("gc_marks_finish: mark stack is not empty (%"PRIdSIZE").",
7973 mark_stack_size(&objspace->mark_stack));
7974 }
7975
7976 gc_mark_roots(objspace, 0);
7977 while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == false);
7978
7979 #if RGENGC_CHECK_MODE >= 2
7980 if (gc_verify_heap_pages(objspace) != 0) {
7981 rb_bug("gc_marks_finish (incremental): there are remembered old objects.");
7982 }
7983 #endif
7984
7985 objspace->flags.during_incremental_marking = FALSE;
7986 /* check children of all marked wb-unprotected objects */
7987 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
7988 gc_marks_wb_unprotected_objects(objspace, SIZE_POOL_EDEN_HEAP(&size_pools[i]));
7989 }
7990 }
7991
7992 gc_update_weak_references(objspace);
7993
7994 #if RGENGC_CHECK_MODE >= 2
7995 gc_verify_internal_consistency(objspace);
7996 #endif
7997
7998 #if RGENGC_CHECK_MODE >= 4
7999 during_gc = FALSE;
8000 gc_marks_check(objspace, gc_check_after_marks_i, "after_marks");
8001 during_gc = TRUE;
8002 #endif
8003
8004 {
8005 /* decide full GC is needed or not */
8006 size_t total_slots = heap_allocatable_slots(objspace) + heap_eden_total_slots(objspace);
8007 size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */
8008 size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
8009 size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
8010 int full_marking = is_full_marking(objspace);
8011 const int r_cnt = GET_VM()->ractor.cnt;
8012 const int r_mul = r_cnt > 8 ? 8 : r_cnt; // upto 8
8013
8014 GC_ASSERT(heap_eden_total_slots(objspace) >= objspace->marked_slots);
8015
8016 /* Setup freeable slots. */
8017 size_t total_init_slots = 0;
8018 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
8019 total_init_slots += gc_params.size_pool_init_slots[i] * r_mul;
8020 }
8021
8022 if (max_free_slots < total_init_slots) {
8023 max_free_slots = total_init_slots;
8024 }
8025
8026 if (sweep_slots > max_free_slots) {
8027 heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
8028 }
8029 else {
8030 heap_pages_freeable_pages = 0;
8031 }
8032
8033 /* check free_min */
8034 if (min_free_slots < gc_params.heap_free_slots * r_mul) {
8035 min_free_slots = gc_params.heap_free_slots * r_mul;
8036 }
8037
8038 if (sweep_slots < min_free_slots) {
8039 if (!full_marking) {
8040 if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
8041 full_marking = TRUE;
8042 /* do not update last_major_gc, because full marking is not done. */
8043 /* goto increment; */
8044 }
8045 else {
8046 gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n");
8047 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
8048 }
8049 }
8050 }
8051
8052 if (full_marking) {
8053 /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
8054 const double r = gc_params.oldobject_limit_factor;
8055 objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
8056 (size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r),
8057 (size_t)(objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
8058 );
8059 objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
8060 }
8061
8062 if (objspace->rgengc.uncollectible_wb_unprotected_objects > objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
8063 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
8064 }
8065 if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) {
8066 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
8067 }
8068 if (RGENGC_FORCE_MAJOR_GC) {
8069 gc_needs_major_flags = GPR_FLAG_MAJOR_BY_FORCE;
8070 }
8071
8072 gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, "
8073 "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, "
8074 "sweep %"PRIdSIZE" slots, increment: %"PRIdSIZE", next GC: %s)\n",
8075 objspace->marked_slots, objspace->rgengc.old_objects, heap_eden_total_slots(objspace), sweep_slots, heap_allocatable_pages(objspace),
8076 gc_needs_major_flags ? "major" : "minor");
8077 }
8078
8079 rb_ractor_finish_marking();
8080
8081 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0);
8082 }
8083
8084 static bool
8085 gc_compact_heap_cursors_met_p(rb_heap_t *heap)
8086 {
8087 return heap->sweeping_page == heap->compact_cursor;
8088 }
8089
8090 static rb_size_pool_t *
8091 gc_compact_destination_pool(rb_objspace_t *objspace, rb_size_pool_t *src_pool, VALUE src)
8092 {
8093 size_t obj_size;
8094 size_t idx = 0;
8095
8096 switch (BUILTIN_TYPE(src)) {
8097 case T_ARRAY:
8098 obj_size = rb_ary_size_as_embedded(src);
8099 break;
8100
8101 case T_OBJECT:
8102 if (rb_shape_obj_too_complex(src)) {
8103 return &size_pools[0];
8104 }
8105 else {
8106 obj_size = rb_obj_embedded_size(ROBJECT_IV_CAPACITY(src));
8107 }
8108 break;
8109
8110 case T_STRING:
8111 obj_size = rb_str_size_as_embedded(src);
8112 break;
8113
8114 case T_HASH:
8115 obj_size = sizeof(struct RHash) + (RHASH_ST_TABLE_P(src) ? sizeof(st_table) : sizeof(ar_table));
8116 break;
8117
8118 default:
8119 return src_pool;
8120 }
8121
8122 if (rb_gc_size_allocatable_p(obj_size)){
8123 idx = rb_gc_size_pool_id_for_size(obj_size);
8124 }
8125 return &size_pools[idx];
8126 }
8127
8128 static bool
8129 gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, rb_size_pool_t *size_pool, VALUE src)
8130 {
8131 GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED);
8132 GC_ASSERT(gc_is_moveable_obj(objspace, src));
8133
8134 rb_size_pool_t *dest_pool = gc_compact_destination_pool(objspace, size_pool, src);
8135 rb_heap_t *dheap = SIZE_POOL_EDEN_HEAP(dest_pool);
8136 rb_shape_t *new_shape = NULL;
8137 rb_shape_t *orig_shape = NULL;
8138
8139 if (gc_compact_heap_cursors_met_p(dheap)) {
8140 return dheap != heap;
8141 }
8142
8143 if (RB_TYPE_P(src, T_OBJECT)) {
8144 orig_shape = rb_shape_get_shape(src);
8145 if (dheap != heap && !rb_shape_obj_too_complex(src)) {
8146 rb_shape_t *initial_shape = rb_shape_get_shape_by_id((shape_id_t)((dest_pool - size_pools) + FIRST_T_OBJECT_SHAPE_ID));
8147 new_shape = rb_shape_traverse_from_new_root(initial_shape, orig_shape);
8148
8149 if (!new_shape) {
8150 dest_pool = size_pool;
8151 dheap = heap;
8152 }
8153 }
8154 }
8155
8156 while (!try_move(objspace, dheap, dheap->free_pages, src)) {
8157 struct gc_sweep_context ctx = {
8158 .page = dheap->sweeping_page,
8159 .final_slots = 0,
8160 .freed_slots = 0,
8161 .empty_slots = 0,
8162 };
8163
8164 /* The page of src could be partially compacted, so it may contain
8165 * T_MOVED. Sweeping a page may read objects on this page, so we
8166 * need to lock the page. */
8167 lock_page_body(objspace, GET_PAGE_BODY(src));
8168 gc_sweep_page(objspace, dheap, &ctx);
8169 unlock_page_body(objspace, GET_PAGE_BODY(src));
8170
8171 if (dheap->sweeping_page->free_slots > 0) {
8172 heap_add_freepage(dheap, dheap->sweeping_page);
8173 }
8174
8175 dheap->sweeping_page = ccan_list_next(&dheap->pages, dheap->sweeping_page, page_node);
8176 if (gc_compact_heap_cursors_met_p(dheap)) {
8177 return dheap != heap;
8178 }
8179 }
8180
8181 if (orig_shape) {
8182 if (new_shape) {
8183 VALUE dest = rb_gc_location(src);
8184 rb_shape_set_shape(dest, new_shape);
8185 }
8186 RMOVED(src)->original_shape_id = rb_shape_id(orig_shape);
8187 }
8188
8189 return true;
8190 }
8191
8192 static bool
8193 gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page)
8194 {
8195 short slot_size = page->slot_size;
8196 short slot_bits = slot_size / BASE_SLOT_SIZE;
8197 GC_ASSERT(slot_bits > 0);
8198
8199 do {
8200 VALUE vp = (VALUE)p;
8201 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
8202
8203 if (bitset & 1) {
8204 objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
8205
8206 if (gc_is_moveable_obj(objspace, vp)) {
8207 if (!gc_compact_move(objspace, heap, size_pool, vp)) {
8208 //the cursors met. bubble up
8209 return false;
8210 }
8211 }
8212 }
8213 p += slot_size;
8214 bitset >>= slot_bits;
8215 } while (bitset);
8216
8217 return true;
8218 }
8219
8220 // Iterate up all the objects in page, moving them to where they want to go
8221 static bool
8222 gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, struct heap_page *page)
8223 {
8224 GC_ASSERT(page == heap->compact_cursor);
8225
8226 bits_t *mark_bits, *pin_bits;
8227 bits_t bitset;
8228 uintptr_t p = page->start;
8229
8230 mark_bits = page->mark_bits;
8231 pin_bits = page->pinned_bits;
8232
8233 // objects that can be moved are marked and not pinned
8234 bitset = (mark_bits[0] & ~pin_bits[0]);
8235 bitset >>= NUM_IN_PAGE(p);
8236 if (bitset) {
8237 if (!gc_compact_plane(objspace, size_pool, heap, (uintptr_t)p, bitset, page))
8238 return false;
8239 }
8240 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
8241
8242 for (int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
8243 bitset = (mark_bits[j] & ~pin_bits[j]);
8244 if (bitset) {
8245 if (!gc_compact_plane(objspace, size_pool, heap, (uintptr_t)p, bitset, page))
8246 return false;
8247 }
8248 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
8249 }
8250
8251 return true;
8252 }
8253
8254 static bool
8255 gc_compact_all_compacted_p(rb_objspace_t *objspace)
8256 {
8257 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
8258 rb_size_pool_t *size_pool = &size_pools[i];
8259 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
8260
8261 if (heap->total_pages > 0 &&
8262 !gc_compact_heap_cursors_met_p(heap)) {
8263 return false;
8264 }
8265 }
8266
8267 return true;
8268 }
8269
8270 static void
8271 gc_sweep_compact(rb_objspace_t *objspace)
8272 {
8273 gc_compact_start(objspace);
8274 #if RGENGC_CHECK_MODE >= 2
8275 gc_verify_internal_consistency(objspace);
8276 #endif
8277
8278 while (!gc_compact_all_compacted_p(objspace)) {
8279 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
8280 rb_size_pool_t *size_pool = &size_pools[i];
8281 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
8282
8283 if (gc_compact_heap_cursors_met_p(heap)) {
8284 continue;
8285 }
8286
8287 struct heap_page *start_page = heap->compact_cursor;
8288
8289 if (!gc_compact_page(objspace, size_pool, heap, start_page)) {
8290 lock_page_body(objspace, GET_PAGE_BODY(start_page->start));
8291
8292 continue;
8293 }
8294
8295 // If we get here, we've finished moving all objects on the compact_cursor page
8296 // So we can lock it and move the cursor on to the next one.
8297 lock_page_body(objspace, GET_PAGE_BODY(start_page->start));
8298 heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
8299 }
8300 }
8301
8302 gc_compact_finish(objspace);
8303
8304 #if RGENGC_CHECK_MODE >= 2
8305 gc_verify_internal_consistency(objspace);
8306 #endif
8307 }
8308
8309 static void
8310 gc_marks_rest(rb_objspace_t *objspace)
8311 {
8312 gc_report(1, objspace, "gc_marks_rest\n");
8313
8314 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
8315 SIZE_POOL_EDEN_HEAP(&size_pools[i])->pooled_pages = NULL;
8316 }
8317
8318 if (is_incremental_marking(objspace)) {
8319 while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE);
8320 }
8321 else {
8322 gc_mark_stacked_objects_all(objspace);
8323 }
8324
8325 gc_marks_finish(objspace);
8326 }
8327
8328 static bool
8329 gc_marks_step(rb_objspace_t *objspace, size_t slots)
8330 {
8331 bool marking_finished = false;
8332
8333 GC_ASSERT(is_marking(objspace));
8334 if (gc_mark_stacked_objects_incremental(objspace, slots)) {
8335 gc_marks_finish(objspace);
8336
8337 marking_finished = true;
8338 }
8339
8340 return marking_finished;
8341 }
8342
8343 static bool
8344 gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
8345 {
8346 GC_ASSERT(dont_gc_val() == FALSE);
8347 bool marking_finished = true;
8348
8349 gc_marking_enter(objspace);
8350
8351 if (heap->free_pages) {
8352 gc_report(2, objspace, "gc_marks_continue: has pooled pages");
8353
8354 marking_finished = gc_marks_step(objspace, objspace->rincgc.step_slots);
8355 }
8356 else {
8357 gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n",
8358 mark_stack_size(&objspace->mark_stack));
8359 size_pool->force_incremental_marking_finish_count++;
8360 gc_marks_rest(objspace);
8361 }
8362
8363 gc_marking_exit(objspace);
8364
8365 return marking_finished;
8366 }
8367
8368 static bool
8369 gc_marks(rb_objspace_t *objspace, int full_mark)
8370 {
8371 gc_prof_mark_timer_start(objspace);
8372 gc_marking_enter(objspace);
8373
8374 bool marking_finished = false;
8375
8376 /* setup marking */
8377
8378 gc_marks_start(objspace, full_mark);
8379 if (!is_incremental_marking(objspace)) {
8380 gc_marks_rest(objspace);
8381 marking_finished = true;
8382 }
8383
8384 #if RGENGC_PROFILE > 0
8385 if (gc_prof_record(objspace)) {
8386 gc_profile_record *record = gc_prof_record(objspace);
8387 record->old_objects = objspace->rgengc.old_objects;
8388 }
8389 #endif
8390
8391 gc_marking_exit(objspace);
8392 gc_prof_mark_timer_stop(objspace);
8393
8394 return marking_finished;
8395 }
8396
8397 /* RGENGC */
8398
8399 static void
8400 gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...)
8401 {
8402 if (level <= RGENGC_DEBUG) {
8403 char buf[1024];
8404 FILE *out = stderr;
8405 va_list args;
8406 const char *status = " ";
8407
8408 if (during_gc) {
8409 status = is_full_marking(objspace) ? "+" : "-";
8410 }
8411 else {
8412 if (is_lazy_sweeping(objspace)) {
8413 status = "S";
8414 }
8415 if (is_incremental_marking(objspace)) {
8416 status = "M";
8417 }
8418 }
8419
8420 va_start(args, fmt);
8421 vsnprintf(buf, 1024, fmt, args);
8422 va_end(args);
8423
8424 fprintf(out, "%s|", status);
8425 fputs(buf, out);
8426 }
8427 }
8428
8429 /* bit operations */
8430
8431 static int
8432 rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj)
8433 {
8434 struct heap_page *page = GET_HEAP_PAGE(obj);
8435 bits_t *bits = &page->remembered_bits[0];
8436
8437 if (MARKED_IN_BITMAP(bits, obj)) {
8438 return FALSE;
8439 }
8440 else {
8441 page->flags.has_remembered_objects = TRUE;
8442 MARK_IN_BITMAP(bits, obj);
8443 return TRUE;
8444 }
8445 }
8446
8447 /* wb, etc */
8448
8449 /* return FALSE if already remembered */
8450 static int
8451 rgengc_remember(rb_objspace_t *objspace, VALUE obj)
8452 {
8453 gc_report(6, objspace, "rgengc_remember: %s %s\n", obj_info(obj),
8454 RVALUE_REMEMBERED(obj) ? "was already remembered" : "is remembered now");
8455
8456 check_rvalue_consistency(obj);
8457
8458 if (RGENGC_CHECK_MODE) {
8459 if (RVALUE_WB_UNPROTECTED(obj)) rb_bug("rgengc_remember: %s is not wb protected.", obj_info(obj));
8460 }
8461
8462 #if RGENGC_PROFILE > 0
8463 if (!RVALUE_REMEMBERED(obj)) {
8464 if (RVALUE_WB_UNPROTECTED(obj) == 0) {
8465 objspace->profile.total_remembered_normal_object_count++;
8466 #if RGENGC_PROFILE >= 2
8467 objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++;
8468 #endif
8469 }
8470 }
8471 #endif /* RGENGC_PROFILE > 0 */
8472
8473 return rgengc_remembersetbits_set(objspace, obj);
8474 }
8475
8476 #ifndef PROFILE_REMEMBERSET_MARK
8477 #define PROFILE_REMEMBERSET_MARK 0
8478 #endif
8479
8480 static inline void
8481 rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset)
8482 {
8483 if (bitset) {
8484 do {
8485 if (bitset & 1) {
8486 VALUE obj = (VALUE)p;
8487 gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", obj_info(obj));
8488 GC_ASSERT(RVALUE_UNCOLLECTIBLE(obj));
8489 GC_ASSERT(RVALUE_OLD_P(obj) || RVALUE_WB_UNPROTECTED(obj));
8490
8491 gc_mark_children(objspace, obj);
8492 }
8493 p += BASE_SLOT_SIZE;
8494 bitset >>= 1;
8495 } while (bitset);
8496 }
8497 }
8498
8499 static void
8500 rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
8501 {
8502 size_t j;
8503 struct heap_page *page = 0;
8504 #if PROFILE_REMEMBERSET_MARK
8505 int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
8506 #endif
8507 gc_report(1, objspace, "rgengc_rememberset_mark: start\n");
8508
8509 ccan_list_for_each(&heap->pages, page, page_node) {
8510 if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
8511 uintptr_t p = page->start;
8512 bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
8513 bits_t *remembered_bits = page->remembered_bits;
8514 bits_t *uncollectible_bits = page->uncollectible_bits;
8515 bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
8516 #if PROFILE_REMEMBERSET_MARK
8517 if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
8518 else if (page->flags.has_remembered_objects) has_old++;
8519 else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
8520 #endif
8521 for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
8522 bits[j] = remembered_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]);
8523 remembered_bits[j] = 0;
8524 }
8525 page->flags.has_remembered_objects = FALSE;
8526
8527 bitset = bits[0];
8528 bitset >>= NUM_IN_PAGE(p);
8529 rgengc_rememberset_mark_plane(objspace, p, bitset);
8530 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
8531
8532 for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
8533 bitset = bits[j];
8534 rgengc_rememberset_mark_plane(objspace, p, bitset);
8535 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
8536 }
8537 }
8538 #if PROFILE_REMEMBERSET_MARK
8539 else {
8540 skip++;
8541 }
8542 #endif
8543 }
8544
8545 #if PROFILE_REMEMBERSET_MARK
8546 fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
8547 #endif
8548 gc_report(1, objspace, "rgengc_rememberset_mark: finished\n");
8549 }
8550
8551 static void
8552 rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap)
8553 {
8554 struct heap_page *page = 0;
8555
8556 ccan_list_for_each(&heap->pages, page, page_node) {
8557 memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8558 memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8559 memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8560 memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8561 memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8562 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
8563 page->flags.has_remembered_objects = FALSE;
8564 }
8565 }
8566
8567 /* RGENGC: APIs */
8568
8569 NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace));
8570
8571 static void
8572 gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
8573 {
8574 if (RGENGC_CHECK_MODE) {
8575 if (!RVALUE_OLD_P(a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", obj_info(a));
8576 if ( RVALUE_OLD_P(b)) rb_bug("gc_writebarrier_generational: %s is an old object.", obj_info(b));
8577 if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", obj_info(a), obj_info(b));
8578 }
8579
8580 /* mark `a' and remember (default behavior) */
8581 if (!RVALUE_REMEMBERED(a)) {
8582 RB_VM_LOCK_ENTER_NO_BARRIER();
8583 {
8584 rgengc_remember(objspace, a);
8585 }
8586 RB_VM_LOCK_LEAVE_NO_BARRIER();
8587 gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", obj_info(a), obj_info(b));
8588 }
8589
8590 check_rvalue_consistency(a);
8591 check_rvalue_consistency(b);
8592 }
8593
8594 static void
8595 gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
8596 {
8597 gc_mark_set_parent(objspace, parent);
8598 rgengc_check_relation(objspace, obj);
8599 if (gc_mark_set(objspace, obj) == FALSE) return;
8600 gc_aging(objspace, obj);
8601 gc_grey(objspace, obj);
8602 }
8603
8604 NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
8605
8606 static void
8607 gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
8608 {
8609 gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, obj_info(b));
8610
8611 if (RVALUE_BLACK_P(a)) {
8612 if (RVALUE_WHITE_P(b)) {
8613 if (!RVALUE_WB_UNPROTECTED(a)) {
8614 gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, obj_info(b));
8615 gc_mark_from(objspace, b, a);
8616 }
8617 }
8618 else if (RVALUE_OLD_P(a) && !RVALUE_OLD_P(b)) {
8619 rgengc_remember(objspace, a);
8620 }
8621
8622 if (UNLIKELY(objspace->flags.during_compacting)) {
8623 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
8624 }
8625 }
8626 }
8627
8628 void
8629 rb_gc_writebarrier(VALUE a, VALUE b)
8630 {
8631 rb_objspace_t *objspace = &rb_objspace;
8632
8633 if (RGENGC_CHECK_MODE) {
8634 if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
8635 if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
8636 }
8637
8638 retry:
8639 if (!is_incremental_marking(objspace)) {
8640 if (!RVALUE_OLD_P(a) || RVALUE_OLD_P(b)) {
8641 // do nothing
8642 }
8643 else {
8644 gc_writebarrier_generational(a, b, objspace);
8645 }
8646 }
8647 else {
8648 bool retry = false;
8649 /* slow path */
8650 RB_VM_LOCK_ENTER_NO_BARRIER();
8651 {
8652 if (is_incremental_marking(objspace)) {
8653 gc_writebarrier_incremental(a, b, objspace);
8654 }
8655 else {
8656 retry = true;
8657 }
8658 }
8659 RB_VM_LOCK_LEAVE_NO_BARRIER();
8660
8661 if (retry) goto retry;
8662 }
8663 return;
8664 }
8665
8666 void
8667 rb_gc_writebarrier_unprotect(VALUE obj)
8668 {
8669 if (RVALUE_WB_UNPROTECTED(obj)) {
8670 return;
8671 }
8672 else {
8673 rb_objspace_t *objspace = &rb_objspace;
8674
8675 gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", obj_info(obj),
8676 RVALUE_REMEMBERED(obj) ? " (already remembered)" : "");
8677
8678 RB_VM_LOCK_ENTER_NO_BARRIER();
8679 {
8680 if (RVALUE_OLD_P(obj)) {
8681 gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", obj_info(obj));
8682 RVALUE_DEMOTE(objspace, obj);
8683 gc_mark_set(objspace, obj);
8684 gc_remember_unprotected(objspace, obj);
8685
8686 #if RGENGC_PROFILE
8687 objspace->profile.total_shade_operation_count++;
8688 #if RGENGC_PROFILE >= 2
8689 objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++;
8690 #endif /* RGENGC_PROFILE >= 2 */
8691 #endif /* RGENGC_PROFILE */
8692 }
8693 else {
8694 RVALUE_AGE_RESET(obj);
8695 }
8696
8697 RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
8698 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
8699 }
8700 RB_VM_LOCK_LEAVE_NO_BARRIER();
8701 }
8702 }
8703
8704 /*
8705 * remember `obj' if needed.
8706 */
8707 void
8708 rb_gc_writebarrier_remember(VALUE obj)
8709 {
8710 rb_objspace_t *objspace = &rb_objspace;
8711
8712 gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", obj_info(obj));
8713
8714 if (is_incremental_marking(objspace)) {
8715 if (RVALUE_BLACK_P(obj)) {
8716 gc_grey(objspace, obj);
8717 }
8718 }
8719 else {
8720 if (RVALUE_OLD_P(obj)) {
8721 rgengc_remember(objspace, obj);
8722 }
8723 }
8724 }
8725
8726 void
8727 rb_gc_copy_attributes(VALUE dest, VALUE obj)
8728 {
8729 if (RVALUE_WB_UNPROTECTED(obj)) {
8730 rb_gc_writebarrier_unprotect(dest);
8731 }
8732 rb_gc_copy_finalizer(dest, obj);
8733 }
8734
8735 size_t
8736 rb_obj_gc_flags(VALUE obj, ID* flags, size_t max)
8737 {
8738 size_t n = 0;
8739 static ID ID_marked;
8740 static ID ID_wb_protected, ID_old, ID_marking, ID_uncollectible, ID_pinned;
8741
8742 if (!ID_marked) {
8743 #define I(s) ID_##s = rb_intern(#s);
8744 I(marked);
8745 I(wb_protected);
8746 I(old);
8747 I(marking);
8748 I(uncollectible);
8749 I(pinned);
8750 #undef I
8751 }
8752
8753 if (RVALUE_WB_UNPROTECTED(obj) == 0 && n<max) flags[n++] = ID_wb_protected;
8754 if (RVALUE_OLD_P(obj) && n<max) flags[n++] = ID_old;
8755 if (RVALUE_UNCOLLECTIBLE(obj) && n<max) flags[n++] = ID_uncollectible;
8756 if (MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj) && n<max) flags[n++] = ID_marking;
8757 if (MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) && n<max) flags[n++] = ID_marked;
8758 if (MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) && n<max) flags[n++] = ID_pinned;
8759 return n;
8760 }
8761
8762 /* GC */
8763
8764 void
8765 rb_gc_ractor_newobj_cache_clear(rb_ractor_newobj_cache_t *newobj_cache)
8766 {
8767 newobj_cache->incremental_mark_step_allocated_slots = 0;
8768
8769 for (size_t size_pool_idx = 0; size_pool_idx < SIZE_POOL_COUNT; size_pool_idx++) {
8770 rb_ractor_newobj_size_pool_cache_t *cache = &newobj_cache->size_pool_caches[size_pool_idx];
8771
8772 struct heap_page *page = cache->using_page;
8773 RVALUE *freelist = cache->freelist;
8774 RUBY_DEBUG_LOG("ractor using_page:%p freelist:%p", (void *)page, (void *)freelist);
8775
8776 heap_page_freelist_append(page, freelist);
8777
8778 cache->using_page = NULL;
8779 cache->freelist = NULL;
8780 }
8781 }
8782
8783 void
8784 rb_gc_register_mark_object(VALUE obj)
8785 {
8786 if (!is_pointer_to_heap(&rb_objspace, (void *)obj))
8787 return;
8788
8789 rb_vm_register_global_object(obj);
8790 }
8791
8792 void
8793 rb_gc_register_address(VALUE *addr)
8794 {
8795 rb_vm_t *vm = GET_VM();
8796
8797 VALUE obj = *addr;
8798
8799 struct global_object_list *tmp = ALLOC(struct global_object_list);
8800 tmp->next = vm->global_object_list;
8801 tmp->varptr = addr;
8802 vm->global_object_list = tmp;
8803
8804 /*
8805 * Because some C extensions have assignment-then-register bugs,
8806 * we guard `obj` here so that it would not get swept defensively.
8807 */
8808 RB_GC_GUARD(obj);
8809 if (0 && !SPECIAL_CONST_P(obj)) {
8810 rb_warn("Object is assigned to registering address already: %"PRIsVALUE,
8811 rb_obj_class(obj));
8812 rb_print_backtrace(stderr);
8813 }
8814 }
8815
8816 void
8817 rb_gc_unregister_address(VALUE *addr)
8818 {
8819 rb_vm_t *vm = GET_VM();
8820 struct global_object_list *tmp = vm->global_object_list;
8821
8822 if (tmp->varptr == addr) {
8823 vm->global_object_list = tmp->next;
8824 xfree(tmp);
8825 return;
8826 }
8827 while (tmp->next) {
8828 if (tmp->next->varptr == addr) {
8829 struct global_object_list *t = tmp->next;
8830
8831 tmp->next = tmp->next->next;
8832 xfree(t);
8833 break;
8834 }
8835 tmp = tmp->next;
8836 }
8837 }
8838
8839 void
8840 rb_global_variable(VALUE *var)
8841 {
8842 rb_gc_register_address(var);
8843 }
8844
8845 enum {
8846 gc_stress_no_major,
8847 gc_stress_no_immediate_sweep,
8848 gc_stress_full_mark_after_malloc,
8849 gc_stress_max
8850 };
8851
8852 #define gc_stress_full_mark_after_malloc_p() \
8853 (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
8854
8855 static void
8856 heap_ready_to_gc(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
8857 {
8858 if (!heap->free_pages) {
8859 if (!heap_increment(objspace, size_pool, heap)) {
8860 size_pool_allocatable_pages_set(objspace, size_pool, 1);
8861 heap_increment(objspace, size_pool, heap);
8862 }
8863 }
8864 }
8865
8866 static int
8867 ready_to_gc(rb_objspace_t *objspace)
8868 {
8869 if (dont_gc_val() || during_gc || ruby_disable_gc) {
8870 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
8871 rb_size_pool_t *size_pool = &size_pools[i];
8872 heap_ready_to_gc(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
8873 }
8874 return FALSE;
8875 }
8876 else {
8877 return TRUE;
8878 }
8879 }
8880
8881 static void
8882 gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
8883 {
8884 gc_prof_set_malloc_info(objspace);
8885 {
8886 size_t inc = ATOMIC_SIZE_EXCHANGE(malloc_increase, 0);
8887 size_t old_limit = malloc_limit;
8888
8889 if (inc > malloc_limit) {
8890 malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
8891 if (malloc_limit > gc_params.malloc_limit_max) {
8892 malloc_limit = gc_params.malloc_limit_max;
8893 }
8894 }
8895 else {
8896 malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */
8897 if (malloc_limit < gc_params.malloc_limit_min) {
8898 malloc_limit = gc_params.malloc_limit_min;
8899 }
8900 }
8901
8902 if (0) {
8903 if (old_limit != malloc_limit) {
8904 fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n",
8905 rb_gc_count(), old_limit, malloc_limit);
8906 }
8907 else {
8908 fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n",
8909 rb_gc_count(), malloc_limit);
8910 }
8911 }
8912 }
8913
8914 /* reset oldmalloc info */
8915 #if RGENGC_ESTIMATE_OLDMALLOC
8916 if (!full_mark) {
8917 if (objspace->rgengc.oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
8918 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
8919 objspace->rgengc.oldmalloc_increase_limit =
8920 (size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
8921
8922 if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
8923 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
8924 }
8925 }
8926
8927 if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRIuSIZE"\t%"PRIuSIZE"\t%"PRIdSIZE"\n",
8928 rb_gc_count(),
8929 gc_needs_major_flags,
8930 objspace->rgengc.oldmalloc_increase,
8931 objspace->rgengc.oldmalloc_increase_limit,
8932 gc_params.oldmalloc_limit_max);
8933 }
8934 else {
8935 /* major GC */
8936 objspace->rgengc.oldmalloc_increase = 0;
8937
8938 if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
8939 objspace->rgengc.oldmalloc_increase_limit =
8940 (size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
8941 if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
8942 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
8943 }
8944 }
8945 }
8946 #endif
8947 }
8948
8949 static int
8950 garbage_collect(rb_objspace_t *objspace, unsigned int reason)
8951 {
8952 int ret;
8953
8954 RB_VM_LOCK_ENTER();
8955 {
8956 #if GC_PROFILE_MORE_DETAIL
8957 objspace->profile.prepare_time = getrusage_time();
8958 #endif
8959
8960 gc_rest(objspace);
8961
8962 #if GC_PROFILE_MORE_DETAIL
8963 objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time;
8964 #endif
8965
8966 ret = gc_start(objspace, reason);
8967 }
8968 RB_VM_LOCK_LEAVE();
8969
8970 return ret;
8971 }
8972
8973 static int
8974 gc_start(rb_objspace_t *objspace, unsigned int reason)
8975 {
8976 unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
8977
8978 /* reason may be clobbered, later, so keep set immediate_sweep here */
8979 objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
8980
8981 if (!heap_allocated_pages) return TRUE; /* heap is not ready */
8982 if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
8983
8984 GC_ASSERT(gc_mode(objspace) == gc_mode_none);
8985 GC_ASSERT(!is_lazy_sweeping(objspace));
8986 GC_ASSERT(!is_incremental_marking(objspace));
8987
8988 unsigned int lock_lev;
8989 gc_enter(objspace, gc_enter_event_start, &lock_lev);
8990
8991 #if RGENGC_CHECK_MODE >= 2
8992 gc_verify_internal_consistency(objspace);
8993 #endif
8994
8995 if (ruby_gc_stressful) {
8996 int flag = FIXNUM_P(ruby_gc_stress_mode) ? FIX2INT(ruby_gc_stress_mode) : 0;
8997
8998 if ((flag & (1<<gc_stress_no_major)) == 0) {
8999 do_full_mark = TRUE;
9000 }
9001
9002 objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
9003 }
9004
9005 if (gc_needs_major_flags) {
9006 reason |= gc_needs_major_flags;
9007 do_full_mark = TRUE;
9008 }
9009 else if (RGENGC_FORCE_MAJOR_GC) {
9010 reason = GPR_FLAG_MAJOR_BY_FORCE;
9011 do_full_mark = TRUE;
9012 }
9013
9014 gc_needs_major_flags = GPR_FLAG_NONE;
9015
9016 if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
9017 reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */
9018 }
9019
9020 if (objspace->flags.dont_incremental ||
9021 reason & GPR_FLAG_IMMEDIATE_MARK ||
9022 ruby_gc_stressful) {
9023 objspace->flags.during_incremental_marking = FALSE;
9024 }
9025 else {
9026 objspace->flags.during_incremental_marking = do_full_mark;
9027 }
9028
9029 /* Explicitly enable compaction (GC.compact) */
9030 if (do_full_mark && ruby_enable_autocompact) {
9031 objspace->flags.during_compacting = TRUE;
9032 #if RGENGC_CHECK_MODE
9033 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
9034 #endif
9035 }
9036 else {
9037 objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
9038 }
9039
9040 if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) {
9041 objspace->flags.immediate_sweep = TRUE;
9042 }
9043
9044 if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
9045
9046 gc_report(1, objspace, "gc_start(reason: %x) => %u, %d, %d\n",
9047 reason,
9048 do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
9049
9050 #if USE_DEBUG_COUNTER
9051 RB_DEBUG_COUNTER_INC(gc_count);
9052
9053 if (reason & GPR_FLAG_MAJOR_MASK) {
9054 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
9055 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
9056 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
9057 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
9058 #if RGENGC_ESTIMATE_OLDMALLOC
9059 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
9060 #endif
9061 }
9062 else {
9063 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
9064 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
9065 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
9066 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
9067 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
9068 }
9069 #endif
9070
9071 objspace->profile.count++;
9072 objspace->profile.latest_gc_info = reason;
9073 objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace);
9074 objspace->profile.heap_used_at_gc_start = heap_allocated_pages;
9075 objspace->profile.weak_references_count = 0;
9076 objspace->profile.retained_weak_references_count = 0;
9077 gc_prof_setup_new_record(objspace, reason);
9078 gc_reset_malloc_info(objspace, do_full_mark);
9079
9080 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */);
9081 GC_ASSERT(during_gc);
9082
9083 gc_prof_timer_start(objspace);
9084 {
9085 if (gc_marks(objspace, do_full_mark)) {
9086 gc_sweep(objspace);
9087 }
9088 }
9089 gc_prof_timer_stop(objspace);
9090
9091 gc_exit(objspace, gc_enter_event_start, &lock_lev);
9092 return TRUE;
9093 }
9094
9095 static void
9096 gc_rest(rb_objspace_t *objspace)
9097 {
9098 if (is_incremental_marking(objspace) || is_lazy_sweeping(objspace)) {
9099 unsigned int lock_lev;
9100 gc_enter(objspace, gc_enter_event_rest, &lock_lev);
9101
9102 if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace);
9103
9104 if (is_incremental_marking(objspace)) {
9105 gc_marking_enter(objspace);
9106 gc_marks_rest(objspace);
9107 gc_marking_exit(objspace);
9108
9109 gc_sweep(objspace);
9110 }
9111
9112 if (is_lazy_sweeping(objspace)) {
9113 gc_sweeping_enter(objspace);
9114 gc_sweep_rest(objspace);
9115 gc_sweeping_exit(objspace);
9116 }
9117
9118 gc_exit(objspace, gc_enter_event_rest, &lock_lev);
9119 }
9120 }
9121
9122 struct objspace_and_reason {
9123 rb_objspace_t *objspace;
9124 unsigned int reason;
9125 };
9126
9127 static void
9128 gc_current_status_fill(rb_objspace_t *objspace, char *buff)
9129 {
9130 int i = 0;
9131 if (is_marking(objspace)) {
9132 buff[i++] = 'M';
9133 if (is_full_marking(objspace)) buff[i++] = 'F';
9134 if (is_incremental_marking(objspace)) buff[i++] = 'I';
9135 }
9136 else if (is_sweeping(objspace)) {
9137 buff[i++] = 'S';
9138 if (is_lazy_sweeping(objspace)) buff[i++] = 'L';
9139 }
9140 else {
9141 buff[i++] = 'N';
9142 }
9143 buff[i] = '\0';
9144 }
9145
9146 static const char *
9147 gc_current_status(rb_objspace_t *objspace)
9148 {
9149 static char buff[0x10];
9150 gc_current_status_fill(objspace, buff);
9151 return buff;
9152 }
9153
9154 #if PRINT_ENTER_EXIT_TICK
9155
9156 static tick_t last_exit_tick;
9157 static tick_t enter_tick;
9158 static int enter_count = 0;
9159 static char last_gc_status[0x10];
9160
9161 static inline void
9162 gc_record(rb_objspace_t *objspace, int direction, const char *event)
9163 {
9164 if (direction == 0) { /* enter */
9165 enter_count++;
9166 enter_tick = tick();
9167 gc_current_status_fill(objspace, last_gc_status);
9168 }
9169 else { /* exit */
9170 tick_t exit_tick = tick();
9171 char current_gc_status[0x10];
9172 gc_current_status_fill(objspace, current_gc_status);
9173 #if 1
9174 /* [last mutator time] [gc time] [event] */
9175 fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
9176 enter_tick - last_exit_tick,
9177 exit_tick - enter_tick,
9178 event,
9179 last_gc_status, current_gc_status,
9180 (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
9181 last_exit_tick = exit_tick;
9182 #else
9183 /* [enter_tick] [gc time] [event] */
9184 fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
9185 enter_tick,
9186 exit_tick - enter_tick,
9187 event,
9188 last_gc_status, current_gc_status,
9189 (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
9190 #endif
9191 }
9192 }
9193 #else /* PRINT_ENTER_EXIT_TICK */
9194 static inline void
9195 gc_record(rb_objspace_t *objspace, int direction, const char *event)
9196 {
9197 /* null */
9198 }
9199 #endif /* PRINT_ENTER_EXIT_TICK */
9200
9201 static const char *
9202 gc_enter_event_cstr(enum gc_enter_event event)
9203 {
9204 switch (event) {
9205 case gc_enter_event_start: return "start";
9206 case gc_enter_event_continue: return "continue";
9207 case gc_enter_event_rest: return "rest";
9208 case gc_enter_event_finalizer: return "finalizer";
9209 case gc_enter_event_rb_memerror: return "rb_memerror";
9210 }
9211 return NULL;
9212 }
9213
9214 static void
9215 gc_enter_count(enum gc_enter_event event)
9216 {
9217 switch (event) {
9218 case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
9219 case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue); break;
9220 case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest); break;
9221 case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer); break;
9222 case gc_enter_event_rb_memerror: /* nothing */ break;
9223 }
9224 }
9225
9226 static bool current_process_time(struct timespec *ts);
9227
9228 static void
9229 gc_clock_start(struct timespec *ts)
9230 {
9231 if (!current_process_time(ts)) {
9232 ts->tv_sec = 0;
9233 ts->tv_nsec = 0;
9234 }
9235 }
9236
9237 static uint64_t
9238 gc_clock_end(struct timespec *ts)
9239 {
9240 struct timespec end_time;
9241
9242 if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
9243 current_process_time(&end_time) &&
9244 end_time.tv_sec >= ts->tv_sec) {
9245 return (uint64_t)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
9246 (end_time.tv_nsec - ts->tv_nsec);
9247 }
9248
9249 return 0;
9250 }
9251
9252 static inline void
9253 gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
9254 {
9255 RB_VM_LOCK_ENTER_LEV(lock_lev);
9256
9257 switch (event) {
9258 case gc_enter_event_rest:
9259 if (!is_marking(objspace)) break;
9260 // fall through
9261 case gc_enter_event_start:
9262 case gc_enter_event_continue:
9263 // stop other ractors
9264 rb_vm_barrier();
9265 break;
9266 default:
9267 break;
9268 }
9269
9270 gc_enter_count(event);
9271 if (UNLIKELY(during_gc != 0)) rb_bug("during_gc != 0");
9272 if (RGENGC_CHECK_MODE >= 3 && (dont_gc_val() == 0)) gc_verify_internal_consistency(objspace);
9273
9274 during_gc = TRUE;
9275 RUBY_DEBUG_LOG("%s (%s)",gc_enter_event_cstr(event), gc_current_status(objspace));
9276 gc_report(1, objspace, "gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
9277 gc_record(objspace, 0, gc_enter_event_cstr(event));
9278 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_ENTER, 0); /* TODO: which parameter should be passed? */
9279 }
9280
9281 static inline void
9282 gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
9283 {
9284 GC_ASSERT(during_gc != 0);
9285
9286 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_EXIT, 0); /* TODO: which parameter should be passed? */
9287 gc_record(objspace, 1, gc_enter_event_cstr(event));
9288 RUBY_DEBUG_LOG("%s (%s)", gc_enter_event_cstr(event), gc_current_status(objspace));
9289 gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
9290 during_gc = FALSE;
9291
9292 RB_VM_LOCK_LEAVE_LEV(lock_lev);
9293 }
9294
9295 #ifndef MEASURE_GC
9296 #define MEASURE_GC (objspace->flags.measure_gc)
9297 #endif
9298
9299 static void
9300 gc_marking_enter(rb_objspace_t *objspace)
9301 {
9302 GC_ASSERT(during_gc != 0);
9303
9304 if (MEASURE_GC) {
9305 gc_clock_start(&objspace->profile.marking_start_time);
9306 }
9307 }
9308
9309 static void
9310 gc_marking_exit(rb_objspace_t *objspace)
9311 {
9312 GC_ASSERT(during_gc != 0);
9313
9314 if (MEASURE_GC) {
9315 objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time);
9316 }
9317 }
9318
9319 static void
9320 gc_sweeping_enter(rb_objspace_t *objspace)
9321 {
9322 GC_ASSERT(during_gc != 0);
9323
9324 if (MEASURE_GC) {
9325 gc_clock_start(&objspace->profile.sweeping_start_time);
9326 }
9327 }
9328
9329 static void
9330 gc_sweeping_exit(rb_objspace_t *objspace)
9331 {
9332 GC_ASSERT(during_gc != 0);
9333
9334 if (MEASURE_GC) {
9335 objspace->profile.sweeping_time_ns += gc_clock_end(&objspace->profile.sweeping_start_time);
9336 }
9337 }
9338
9339 static void *
9340 gc_with_gvl(void *ptr)
9341 {
9342 struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr;
9343 return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
9344 }
9345
9346 static int
9347 garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason)
9348 {
9349 if (dont_gc_val()) return TRUE;
9350 if (ruby_thread_has_gvl_p()) {
9351 return garbage_collect(objspace, reason);
9352 }
9353 else {
9354 if (ruby_native_thread_p()) {
9355 struct objspace_and_reason oar;
9356 oar.objspace = objspace;
9357 oar.reason = reason;
9358 return (int)(VALUE)rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar);
9359 }
9360 else {
9361 /* no ruby thread */
9362 fprintf(stderr, "[FATAL] failed to allocate memory\n");
9363 exit(EXIT_FAILURE);
9364 }
9365 }
9366 }
9367
9368 static int
9369 gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data)
9370 {
9371 rb_objspace_t *objspace = &rb_objspace;
9372 VALUE v = (VALUE)vstart;
9373 for (; v != (VALUE)vend; v += stride) {
9374 asan_unpoisoning_object(v) {
9375 switch (BUILTIN_TYPE(v)) {
9376 case T_NONE:
9377 case T_ZOMBIE:
9378 break;
9379 case T_STRING:
9380 // precompute the string coderange. This both save time for when it will be
9381 // eventually needed, and avoid mutating heap pages after a potential fork.
9382 rb_enc_str_coderange(v);
9383 // fall through
9384 default:
9385 if (!RVALUE_OLD_P(v) && !RVALUE_WB_UNPROTECTED(v)) {
9386 RVALUE_AGE_SET_CANDIDATE(objspace, v);
9387 }
9388 }
9389 }
9390 }
9391
9392 return 0;
9393 }
9394
9395 static VALUE
9396 gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep, VALUE compact)
9397 {
9398 rb_objspace_t *objspace = &rb_objspace;
9399 unsigned int reason = (GPR_FLAG_FULL_MARK |
9400 GPR_FLAG_IMMEDIATE_MARK |
9401 GPR_FLAG_IMMEDIATE_SWEEP |
9402 GPR_FLAG_METHOD);
9403
9404 /* For now, compact implies full mark / sweep, so ignore other flags */
9405 if (RTEST(compact)) {
9406 GC_ASSERT(GC_COMPACTION_SUPPORTED);
9407
9408 reason |= GPR_FLAG_COMPACT;
9409 }
9410 else {
9411 if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
9412 if (!RTEST(immediate_mark)) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
9413 if (!RTEST(immediate_sweep)) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
9414 }
9415
9416 garbage_collect(objspace, reason);
9417 gc_finalize_deferred(objspace);
9418
9419 return Qnil;
9420 }
9421
9422 static void
9423 free_empty_pages(void)
9424 {
9425 rb_objspace_t *objspace = &rb_objspace;
9426
9427 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
9428 /* Move all empty pages to the tomb heap for freeing. */
9429 rb_size_pool_t *size_pool = &size_pools[i];
9430 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
9431 rb_heap_t *tomb_heap = SIZE_POOL_TOMB_HEAP(size_pool);
9432
9433 size_t freed_pages = 0;
9434
9435 struct heap_page **next_page_ptr = &heap->free_pages;
9436 struct heap_page *page = heap->free_pages;
9437 while (page) {
9438 /* All finalizers should have been ran in gc_start_internal, so there
9439 * should be no objects that require finalization. */
9440 GC_ASSERT(page->final_slots == 0);
9441
9442 struct heap_page *next_page = page->free_next;
9443
9444 if (page->free_slots == page->total_slots) {
9445 heap_unlink_page(objspace, heap, page);
9446 heap_add_page(objspace, size_pool, tomb_heap, page);
9447 freed_pages++;
9448 }
9449 else {
9450 *next_page_ptr = page;
9451 next_page_ptr = &page->free_next;
9452 }
9453
9454 page = next_page;
9455 }
9456
9457 *next_page_ptr = NULL;
9458
9459 size_pool_allocatable_pages_set(objspace, size_pool, size_pool->allocatable_pages + freed_pages);
9460 }
9461
9462 heap_pages_free_unused_pages(objspace);
9463 }
9464
9465 void
9466 rb_gc_prepare_heap(void)
9467 {
9468 rb_objspace_each_objects(gc_set_candidate_object_i, NULL);
9469 gc_start_internal(NULL, Qtrue, Qtrue, Qtrue, Qtrue, Qtrue);
9470 free_empty_pages();
9471
9472 #if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
9473 malloc_trim(0);
9474 #endif
9475 }
9476
9477 static int
9478 gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
9479 {
9480 GC_ASSERT(!SPECIAL_CONST_P(obj));
9481
9482 switch (BUILTIN_TYPE(obj)) {
9483 case T_NONE:
9484 case T_MOVED:
9485 case T_ZOMBIE:
9486 return FALSE;
9487 case T_SYMBOL:
9488 if (RSYMBOL(obj)->id & ~ID_SCOPE_MASK) {
9489 return FALSE;
9490 }
9491 /* fall through */
9492 case T_STRING:
9493 case T_OBJECT:
9494 case T_FLOAT:
9495 case T_IMEMO:
9496 case T_ARRAY:
9497 case T_BIGNUM:
9498 case T_ICLASS:
9499 case T_MODULE:
9500 case T_REGEXP:
9501 case T_DATA:
9502 case T_MATCH:
9503 case T_STRUCT:
9504 case T_HASH:
9505 case T_FILE:
9506 case T_COMPLEX:
9507 case T_RATIONAL:
9508 case T_NODE:
9509 case T_CLASS:
9510 if (FL_TEST(obj, FL_FINALIZE)) {
9511 /* The finalizer table is a numtable. It looks up objects by address.
9512 * We can't mark the keys in the finalizer table because that would
9513 * prevent the objects from being collected. This check prevents
9514 * objects that are keys in the finalizer table from being moved
9515 * without directly pinning them. */
9516 GC_ASSERT(st_is_member(finalizer_table, obj));
9517
9518 return FALSE;
9519 }
9520 GC_ASSERT(RVALUE_MARKED(obj));
9521 GC_ASSERT(!RVALUE_PINNED(obj));
9522
9523 return TRUE;
9524
9525 default:
9526 rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
9527 break;
9528 }
9529
9530 return FALSE;
9531 }
9532
9533 static VALUE
9534 gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size)
9535 {
9536 int marked;
9537 int wb_unprotected;
9538 int uncollectible;
9539 int age;
9540 RVALUE *dest = (RVALUE *)free;
9541 RVALUE *src = (RVALUE *)scan;
9542
9543 gc_report(4, objspace, "Moving object: %p -> %p\n", (void*)scan, (void *)free);
9544
9545 GC_ASSERT(BUILTIN_TYPE(scan) != T_NONE);
9546 GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(free), free));
9547
9548 GC_ASSERT(!RVALUE_MARKING((VALUE)src));
9549
9550 /* Save off bits for current object. */
9551 marked = RVALUE_MARKED((VALUE)src);
9552 wb_unprotected = RVALUE_WB_UNPROTECTED((VALUE)src);
9553 uncollectible = RVALUE_UNCOLLECTIBLE((VALUE)src);
9554 bool remembered = RVALUE_REMEMBERED((VALUE)src);
9555 age = RVALUE_AGE_GET((VALUE)src);
9556
9557 /* Clear bits for eventual T_MOVED */
9558 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)src), (VALUE)src);
9559 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS((VALUE)src), (VALUE)src);
9560 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)src), (VALUE)src);
9561 CLEAR_IN_BITMAP(GET_HEAP_PAGE((VALUE)src)->remembered_bits, (VALUE)src);
9562
9563 if (FL_TEST((VALUE)src, FL_EXIVAR)) {
9564 /* Resizing the st table could cause a malloc */
9565 DURING_GC_COULD_MALLOC_REGION_START();
9566 {
9567 rb_mv_generic_ivar((VALUE)src, (VALUE)dest);
9568 }
9569 DURING_GC_COULD_MALLOC_REGION_END();
9570 }
9571
9572 if (FL_TEST((VALUE)src, FL_SEEN_OBJ_ID)) {
9573 /* If the source object's object_id has been seen, we need to update
9574 * the object to object id mapping. */
9575 st_data_t srcid = (st_data_t)src, id;
9576
9577 gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest);
9578 /* Resizing the st table could cause a malloc */
9579 DURING_GC_COULD_MALLOC_REGION_START();
9580 {
9581 if (!st_delete(objspace->obj_to_id_tbl, &srcid, &id)) {
9582 rb_bug("gc_move: object ID seen, but not in mapping table: %s", obj_info((VALUE)src));
9583 }
9584
9585 st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id);
9586 }
9587 DURING_GC_COULD_MALLOC_REGION_END();
9588 }
9589 else {
9590 GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)src, NULL));
9591 }
9592
9593 /* Move the object */
9594 memcpy(dest, src, MIN(src_slot_size, slot_size));
9595
9596 if (RVALUE_OVERHEAD > 0) {
9597 void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
9598 void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
9599
9600 memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
9601 }
9602
9603 memset(src, 0, src_slot_size);
9604 RVALUE_AGE_RESET((VALUE)src);
9605
9606 /* Set bits for object in new location */
9607 if (remembered) {
9608 MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, (VALUE)dest);
9609 }
9610 else {
9611 CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, (VALUE)dest);
9612 }
9613
9614 if (marked) {
9615 MARK_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)dest), (VALUE)dest);
9616 }
9617 else {
9618 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)dest), (VALUE)dest);
9619 }
9620
9621 if (wb_unprotected) {
9622 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS((VALUE)dest), (VALUE)dest);
9623 }
9624 else {
9625 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS((VALUE)dest), (VALUE)dest);
9626 }
9627
9628 if (uncollectible) {
9629 MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)dest), (VALUE)dest);
9630 }
9631 else {
9632 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)dest), (VALUE)dest);
9633 }
9634
9635 RVALUE_AGE_SET((VALUE)dest, age);
9636 /* Assign forwarding address */
9637 src->as.moved.flags = T_MOVED;
9638 src->as.moved.dummy = Qundef;
9639 src->as.moved.destination = (VALUE)dest;
9640 GC_ASSERT(BUILTIN_TYPE((VALUE)dest) != T_NONE);
9641
9642 return (VALUE)src;
9643 }
9644
9645 #if GC_CAN_COMPILE_COMPACTION
9646 static int
9647 compare_pinned_slots(const void *left, const void *right, void *dummy)
9648 {
9649 struct heap_page *left_page;
9650 struct heap_page *right_page;
9651
9652 left_page = *(struct heap_page * const *)left;
9653 right_page = *(struct heap_page * const *)right;
9654
9655 return left_page->pinned_slots - right_page->pinned_slots;
9656 }
9657
9658 static int
9659 compare_free_slots(const void *left, const void *right, void *dummy)
9660 {
9661 struct heap_page *left_page;
9662 struct heap_page *right_page;
9663
9664 left_page = *(struct heap_page * const *)left;
9665 right_page = *(struct heap_page * const *)right;
9666
9667 return left_page->free_slots - right_page->free_slots;
9668 }
9669
9670 static void
9671 gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func)
9672 {
9673 for (int j = 0; j < SIZE_POOL_COUNT; j++) {
9674 rb_size_pool_t *size_pool = &size_pools[j];
9675
9676 size_t total_pages = SIZE_POOL_EDEN_HEAP(size_pool)->total_pages;
9677 size_t size = size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
9678 struct heap_page *page = 0, **page_list = malloc(size);
9679 size_t i = 0;
9680
9681 SIZE_POOL_EDEN_HEAP(size_pool)->free_pages = NULL;
9682 ccan_list_for_each(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node) {
9683 page_list[i++] = page;
9684 GC_ASSERT(page);
9685 }
9686
9687 GC_ASSERT((size_t)i == total_pages);
9688
9689 /* Sort the heap so "filled pages" are first. `heap_add_page` adds to the
9690 * head of the list, so empty pages will end up at the start of the heap */
9691 ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL);
9692
9693 /* Reset the eden heap */
9694 ccan_list_head_init(&SIZE_POOL_EDEN_HEAP(size_pool)->pages);
9695
9696 for (i = 0; i < total_pages; i++) {
9697 ccan_list_add(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, &page_list[i]->page_node);
9698 if (page_list[i]->free_slots != 0) {
9699 heap_add_freepage(SIZE_POOL_EDEN_HEAP(size_pool), page_list[i]);
9700 }
9701 }
9702
9703 free(page_list);
9704 }
9705 }
9706 #endif
9707
9708 static void
9709 gc_ref_update_array(rb_objspace_t * objspace, VALUE v)
9710 {
9711 if (ARY_SHARED_P(v)) {
9712 VALUE old_root = RARRAY(v)->as.heap.aux.shared_root;
9713
9714 UPDATE_IF_MOVED(objspace, RARRAY(v)->as.heap.aux.shared_root);
9715
9716 VALUE new_root = RARRAY(v)->as.heap.aux.shared_root;
9717 // If the root is embedded and its location has changed
9718 if (ARY_EMBED_P(new_root) && new_root != old_root) {
9719 size_t offset = (size_t)(RARRAY(v)->as.heap.ptr - RARRAY(old_root)->as.ary);
9720 GC_ASSERT(RARRAY(v)->as.heap.ptr >= RARRAY(old_root)->as.ary);
9721 RARRAY(v)->as.heap.ptr = RARRAY(new_root)->as.ary + offset;
9722 }
9723 }
9724 else {
9725 long len = RARRAY_LEN(v);
9726
9727 if (len > 0) {
9728 VALUE *ptr = (VALUE *)RARRAY_CONST_PTR(v);
9729 for (long i = 0; i < len; i++) {
9730 UPDATE_IF_MOVED(objspace, ptr[i]);
9731 }
9732 }
9733
9734 if (rb_gc_obj_slot_size(v) >= rb_ary_size_as_embedded(v)) {
9735 if (rb_ary_embeddable_p(v)) {
9736 rb_ary_make_embedded(v);
9737 }
9738 }
9739 }
9740 }
9741
9742 static void gc_ref_update_table_values_only(rb_objspace_t *objspace, st_table *tbl);
9743
9744 static void
9745 gc_ref_update_object(rb_objspace_t *objspace, VALUE v)
9746 {
9747 VALUE *ptr = ROBJECT_IVPTR(v);
9748
9749 if (rb_shape_obj_too_complex(v)) {
9750 gc_ref_update_table_values_only(objspace, ROBJECT_IV_HASH(v));
9751 return;
9752 }
9753
9754 size_t slot_size = rb_gc_obj_slot_size(v);
9755 size_t embed_size = rb_obj_embedded_size(ROBJECT_IV_CAPACITY(v));
9756 if (slot_size >= embed_size && !RB_FL_TEST_RAW(v, ROBJECT_EMBED)) {
9757 // Object can be re-embedded
9758 memcpy(ROBJECT(v)->as.ary, ptr, sizeof(VALUE) * ROBJECT_IV_COUNT(v));
9759 RB_FL_SET_RAW(v, ROBJECT_EMBED);
9760 xfree(ptr);
9761 ptr = ROBJECT(v)->as.ary;
9762 }
9763
9764 for (uint32_t i = 0; i < ROBJECT_IV_COUNT(v); i++) {
9765 UPDATE_IF_MOVED(objspace, ptr[i]);
9766 }
9767 }
9768
9769 static int
9770 hash_replace_ref(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
9771 {
9772 rb_objspace_t *objspace = (rb_objspace_t *)argp;
9773
9774 if (gc_object_moved_p(objspace, (VALUE)*key)) {
9775 *key = rb_gc_location((VALUE)*key);
9776 }
9777
9778 if (gc_object_moved_p(objspace, (VALUE)*value)) {
9779 *value = rb_gc_location((VALUE)*value);
9780 }
9781
9782 return ST_CONTINUE;
9783 }
9784
9785 static int
9786 hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error)
9787 {
9788 rb_objspace_t *objspace;
9789
9790 objspace = (rb_objspace_t *)argp;
9791
9792 if (gc_object_moved_p(objspace, (VALUE)key)) {
9793 return ST_REPLACE;
9794 }
9795
9796 if (gc_object_moved_p(objspace, (VALUE)value)) {
9797 return ST_REPLACE;
9798 }
9799 return ST_CONTINUE;
9800 }
9801
9802 static int
9803 hash_replace_ref_value(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
9804 {
9805 rb_objspace_t *objspace = (rb_objspace_t *)argp;
9806
9807 if (gc_object_moved_p(objspace, (VALUE)*value)) {
9808 *value = rb_gc_location((VALUE)*value);
9809 }
9810
9811 return ST_CONTINUE;
9812 }
9813
9814 static int
9815 hash_foreach_replace_value(st_data_t key, st_data_t value, st_data_t argp, int error)
9816 {
9817 rb_objspace_t *objspace;
9818
9819 objspace = (rb_objspace_t *)argp;
9820
9821 if (gc_object_moved_p(objspace, (VALUE)value)) {
9822 return ST_REPLACE;
9823 }
9824 return ST_CONTINUE;
9825 }
9826
9827 static void
9828 gc_ref_update_table_values_only(rb_objspace_t *objspace, st_table *tbl)
9829 {
9830 if (!tbl || tbl->num_entries == 0) return;
9831
9832 if (st_foreach_with_replace(tbl, hash_foreach_replace_value, hash_replace_ref_value, (st_data_t)objspace)) {
9833 rb_raise(rb_eRuntimeError, "hash modified during iteration");
9834 }
9835 }
9836
9837 void
9838 rb_gc_ref_update_table_values_only(st_table *tbl)
9839 {
9840 gc_ref_update_table_values_only(&rb_objspace, tbl);
9841 }
9842
9843 static void
9844 gc_update_table_refs(rb_objspace_t * objspace, st_table *tbl)
9845 {
9846 if (!tbl || tbl->num_entries == 0) return;
9847
9848 if (st_foreach_with_replace(tbl, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace)) {
9849 rb_raise(rb_eRuntimeError, "hash modified during iteration");
9850 }
9851 }
9852
9853 /* Update MOVED references in a VALUE=>VALUE st_table */
9854 void
9855 rb_gc_update_tbl_refs(st_table *ptr)
9856 {
9857 rb_objspace_t *objspace = &rb_objspace;
9858 gc_update_table_refs(objspace, ptr);
9859 }
9860
9861 static void
9862 gc_ref_update_hash(rb_objspace_t * objspace, VALUE v)
9863 {
9864 rb_hash_stlike_foreach_with_replace(v, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace);
9865 }
9866
9867 static void
9868 gc_update_values(rb_objspace_t *objspace, long n, VALUE *values)
9869 {
9870 long i;
9871
9872 for (i=0; i<n; i++) {
9873 UPDATE_IF_MOVED(objspace, values[i]);
9874 }
9875 }
9876
9877 void
9878 rb_gc_update_values(long n, VALUE *values)
9879 {
9880 gc_update_values(&rb_objspace, n, values);
9881 }
9882
9883 static enum rb_id_table_iterator_result
9884 check_id_table_move(VALUE value, void *data)
9885 {
9886 rb_objspace_t *objspace = (rb_objspace_t *)data;
9887
9888 if (gc_object_moved_p(objspace, (VALUE)value)) {
9889 return ID_TABLE_REPLACE;
9890 }
9891
9892 return ID_TABLE_CONTINUE;
9893 }
9894
9895 /* Returns the new location of an object, if it moved. Otherwise returns
9896 * the existing location. */
9897 VALUE
9898 rb_gc_location(VALUE value)
9899 {
9900
9901 VALUE destination;
9902
9903 if (!SPECIAL_CONST_P(value)) {
9904 void *poisoned = asan_unpoison_object_temporary(value);
9905
9906 if (BUILTIN_TYPE(value) == T_MOVED) {
9907 destination = (VALUE)RMOVED(value)->destination;
9908 GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE);
9909 }
9910 else {
9911 destination = value;
9912 }
9913
9914 /* Re-poison slot if it's not the one we want */
9915 if (poisoned) {
9916 GC_ASSERT(BUILTIN_TYPE(value) == T_NONE);
9917 asan_poison_object(value);
9918 }
9919 }
9920 else {
9921 destination = value;
9922 }
9923
9924 return destination;
9925 }
9926
9927 static enum rb_id_table_iterator_result
9928 update_id_table(VALUE *value, void *data, int existing)
9929 {
9930 rb_objspace_t *objspace = (rb_objspace_t *)data;
9931
9932 if (gc_object_moved_p(objspace, (VALUE)*value)) {
9933 *value = rb_gc_location((VALUE)*value);
9934 }
9935
9936 return ID_TABLE_CONTINUE;
9937 }
9938
9939 static void
9940 update_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
9941 {
9942 if (tbl) {
9943 rb_id_table_foreach_values_with_replace(tbl, check_id_table_move, update_id_table, objspace);
9944 }
9945 }
9946
9947 static enum rb_id_table_iterator_result
9948 update_cc_tbl_i(VALUE ccs_ptr, void *data)
9949 {
9950 rb_objspace_t *objspace = (rb_objspace_t *)data;
9951 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
9952 VM_ASSERT(vm_ccs_p(ccs));
9953
9954 if (gc_object_moved_p(objspace, (VALUE)ccs->cme)) {
9955 ccs->cme = (const rb_callable_method_entry_t *)rb_gc_location((VALUE)ccs->cme);
9956 }
9957
9958 for (int i=0; i<ccs->len; i++) {
9959 if (gc_object_moved_p(objspace, (VALUE)ccs->entries[i].cc)) {
9960 ccs->entries[i].cc = (struct rb_callcache *)rb_gc_location((VALUE)ccs->entries[i].cc);
9961 }
9962 }
9963
9964 // do not replace
9965 return ID_TABLE_CONTINUE;
9966 }
9967
9968 static void
9969 update_cc_tbl(rb_objspace_t *objspace, VALUE klass)
9970 {
9971 struct rb_id_table *tbl = RCLASS_CC_TBL(klass);
9972 if (tbl) {
9973 rb_id_table_foreach_values(tbl, update_cc_tbl_i, objspace);
9974 }
9975 }
9976
9977 static enum rb_id_table_iterator_result
9978 update_cvc_tbl_i(VALUE cvc_entry, void *data)
9979 {
9980 struct rb_cvar_class_tbl_entry *entry;
9981 rb_objspace_t * objspace = (rb_objspace_t *)data;
9982
9983 entry = (struct rb_cvar_class_tbl_entry *)cvc_entry;
9984
9985 if (entry->cref) {
9986 TYPED_UPDATE_IF_MOVED(objspace, rb_cref_t *, entry->cref);
9987 }
9988
9989 entry->class_value = rb_gc_location(entry->class_value);
9990
9991 return ID_TABLE_CONTINUE;
9992 }
9993
9994 static void
9995 update_cvc_tbl(rb_objspace_t *objspace, VALUE klass)
9996 {
9997 struct rb_id_table *tbl = RCLASS_CVC_TBL(klass);
9998 if (tbl) {
9999 rb_id_table_foreach_values(tbl, update_cvc_tbl_i, objspace);
10000 }
10001 }
10002
10003 static enum rb_id_table_iterator_result
10004 mark_cvc_tbl_i(VALUE cvc_entry, void *data)
10005 {
10006 rb_objspace_t *objspace = (rb_objspace_t *)data;
10007 struct rb_cvar_class_tbl_entry *entry;
10008
10009 entry = (struct rb_cvar_class_tbl_entry *)cvc_entry;
10010
10011 RUBY_ASSERT(entry->cref == 0 || (BUILTIN_TYPE((VALUE)entry->cref) == T_IMEMO && IMEMO_TYPE_P(entry->cref, imemo_cref)));
10012 gc_mark(objspace, (VALUE) entry->cref);
10013
10014 return ID_TABLE_CONTINUE;
10015 }
10016
10017 static void
10018 mark_cvc_tbl(rb_objspace_t *objspace, VALUE klass)
10019 {
10020 struct rb_id_table *tbl = RCLASS_CVC_TBL(klass);
10021 if (tbl) {
10022 rb_id_table_foreach_values(tbl, mark_cvc_tbl_i, objspace);
10023 }
10024 }
10025
10026 static enum rb_id_table_iterator_result
10027 update_const_table(VALUE value, void *data)
10028 {
10029 rb_const_entry_t *ce = (rb_const_entry_t *)value;
10030 rb_objspace_t * objspace = (rb_objspace_t *)data;
10031
10032 if (gc_object_moved_p(objspace, ce->value)) {
10033 ce->value = rb_gc_location(ce->value);
10034 }
10035
10036 if (gc_object_moved_p(objspace, ce->file)) {
10037 ce->file = rb_gc_location(ce->file);
10038 }
10039
10040 return ID_TABLE_CONTINUE;
10041 }
10042
10043 static void
10044 update_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
10045 {
10046 if (!tbl) return;
10047 rb_id_table_foreach_values(tbl, update_const_table, objspace);
10048 }
10049
10050 static void
10051 update_subclass_entries(rb_objspace_t *objspace, rb_subclass_entry_t *entry)
10052 {
10053 while (entry) {
10054 UPDATE_IF_MOVED(objspace, entry->klass);
10055 entry = entry->next;
10056 }
10057 }
10058
10059 static void
10060 update_class_ext(rb_objspace_t *objspace, rb_classext_t *ext)
10061 {
10062 UPDATE_IF_MOVED(objspace, ext->origin_);
10063 UPDATE_IF_MOVED(objspace, ext->includer);
10064 UPDATE_IF_MOVED(objspace, ext->refined_class);
10065 update_subclass_entries(objspace, ext->subclasses);
10066 }
10067
10068 static void
10069 update_superclasses(rb_objspace_t *objspace, VALUE obj)
10070 {
10071 if (FL_TEST_RAW(obj, RCLASS_SUPERCLASSES_INCLUDE_SELF)) {
10072 for (size_t i = 0; i < RCLASS_SUPERCLASS_DEPTH(obj) + 1; i++) {
10073 UPDATE_IF_MOVED(objspace, RCLASS_SUPERCLASSES(obj)[i]);
10074 }
10075 }
10076 }
10077
10078 static void
10079 gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
10080 {
10081 RVALUE *any = RANY(obj);
10082
10083 gc_report(4, objspace, "update-refs: %p ->\n", (void *)obj);
10084
10085 if (FL_TEST(obj, FL_EXIVAR)) {
10086 rb_ref_update_generic_ivar(obj);
10087 }
10088
10089 switch (BUILTIN_TYPE(obj)) {
10090 case T_CLASS:
10091 if (FL_TEST(obj, FL_SINGLETON)) {
10092 UPDATE_IF_MOVED(objspace, RCLASS_ATTACHED_OBJECT(obj));
10093 }
10094 // Continue to the shared T_CLASS/T_MODULE
10095 case T_MODULE:
10096 if (RCLASS_SUPER((VALUE)obj)) {
10097 UPDATE_IF_MOVED(objspace, RCLASS(obj)->super);
10098 }
10099 update_m_tbl(objspace, RCLASS_M_TBL(obj));
10100 update_cc_tbl(objspace, obj);
10101 update_cvc_tbl(objspace, obj);
10102 update_superclasses(objspace, obj);
10103
10104 if (rb_shape_obj_too_complex(obj)) {
10105 gc_ref_update_table_values_only(objspace, RCLASS_IV_HASH(obj));
10106 }
10107 else {
10108 for (attr_index_t i = 0; i < RCLASS_IV_COUNT(obj); i++) {
10109 UPDATE_IF_MOVED(objspace, RCLASS_IVPTR(obj)[i]);
10110 }
10111 }
10112
10113 update_class_ext(objspace, RCLASS_EXT(obj));
10114 update_const_tbl(objspace, RCLASS_CONST_TBL(obj));
10115
10116 UPDATE_IF_MOVED(objspace, RCLASS_EXT(obj)->classpath);
10117 break;
10118
10119 case T_ICLASS:
10120 if (RICLASS_OWNS_M_TBL_P(obj)) {
10121 update_m_tbl(objspace, RCLASS_M_TBL(obj));
10122 }
10123 if (RCLASS_SUPER((VALUE)obj)) {
10124 UPDATE_IF_MOVED(objspace, RCLASS(obj)->super);
10125 }
10126 update_class_ext(objspace, RCLASS_EXT(obj));
10127 update_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
10128 update_cc_tbl(objspace, obj);
10129 break;
10130
10131 case T_IMEMO:
10132 rb_imemo_mark_and_move(obj, true);
10133 return;
10134
10135 case T_NIL:
10136 case T_FIXNUM:
10137 case T_NODE:
10138 case T_MOVED:
10139 case T_NONE:
10140 /* These can't move */
10141 return;
10142
10143 case T_ARRAY:
10144 gc_ref_update_array(objspace, obj);
10145 break;
10146
10147 case T_HASH:
10148 gc_ref_update_hash(objspace, obj);
10149 UPDATE_IF_MOVED(objspace, any->as.hash.ifnone);
10150 break;
10151
10152 case T_STRING:
10153 {
10154 if (STR_SHARED_P(obj)) {
10155 UPDATE_IF_MOVED(objspace, any->as.string.as.heap.aux.shared);
10156 }
10157
10158 /* If, after move the string is not embedded, and can fit in the
10159 * slot it's been placed in, then re-embed it. */
10160 if (rb_gc_obj_slot_size(obj) >= rb_str_size_as_embedded(obj)) {
10161 if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) {
10162 rb_str_make_embedded(obj);
10163 }
10164 }
10165
10166 break;
10167 }
10168 case T_DATA:
10169 /* Call the compaction callback, if it exists */
10170 {
10171 void *const ptr = RTYPEDDATA_P(obj) ? RTYPEDDATA_GET_DATA(obj) : DATA_PTR(obj);
10172 if (ptr) {
10173 if (RTYPEDDATA_P(obj) && gc_declarative_marking_p(any->as.typeddata.type)) {
10174 size_t *offset_list = (size_t *)RANY(obj)->as.typeddata.type->function.dmark;
10175
10176 for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) {
10177 VALUE *ref = (VALUE *)((char *)ptr + offset);
10178 if (SPECIAL_CONST_P(*ref)) continue;
10179 *ref = rb_gc_location(*ref);
10180 }
10181 }
10182 else if (RTYPEDDATA_P(obj)) {
10183 RUBY_DATA_FUNC compact_func = any->as.typeddata.type->function.dcompact;
10184 if (compact_func) (*compact_func)(ptr);
10185 }
10186 }
10187 }
10188 break;
10189
10190 case T_OBJECT:
10191 gc_ref_update_object(objspace, obj);
10192 break;
10193
10194 case T_FILE:
10195 if (any->as.file.fptr) {
10196 UPDATE_IF_MOVED(objspace, any->as.file.fptr->self);
10197 UPDATE_IF_MOVED(objspace, any->as.file.fptr->pathv);
10198 UPDATE_IF_MOVED(objspace, any->as.file.fptr->tied_io_for_writing);
10199 UPDATE_IF_MOVED(objspace, any->as.file.fptr->writeconv_asciicompat);
10200 UPDATE_IF_MOVED(objspace, any->as.file.fptr->writeconv_pre_ecopts);
10201 UPDATE_IF_MOVED(objspace, any->as.file.fptr->encs.ecopts);
10202 UPDATE_IF_MOVED(objspace, any->as.file.fptr->write_lock);
10203 }
10204 break;
10205 case T_REGEXP:
10206 UPDATE_IF_MOVED(objspace, any->as.regexp.src);
10207 break;
10208
10209 case T_SYMBOL:
10210 UPDATE_IF_MOVED(objspace, RSYMBOL(any)->fstr);
10211 break;
10212
10213 case T_FLOAT:
10214 case T_BIGNUM:
10215 break;
10216
10217 case T_MATCH:
10218 UPDATE_IF_MOVED(objspace, any->as.match.regexp);
10219
10220 if (any->as.match.str) {
10221 UPDATE_IF_MOVED(objspace, any->as.match.str);
10222 }
10223 break;
10224
10225 case T_RATIONAL:
10226 UPDATE_IF_MOVED(objspace, any->as.rational.num);
10227 UPDATE_IF_MOVED(objspace, any->as.rational.den);
10228 break;
10229
10230 case T_COMPLEX:
10231 UPDATE_IF_MOVED(objspace, any->as.complex.real);
10232 UPDATE_IF_MOVED(objspace, any->as.complex.imag);
10233
10234 break;
10235
10236 case T_STRUCT:
10237 {
10238 long i, len = RSTRUCT_LEN(obj);
10239 VALUE *ptr = (VALUE *)RSTRUCT_CONST_PTR(obj);
10240
10241 for (i = 0; i < len; i++) {
10242 UPDATE_IF_MOVED(objspace, ptr[i]);
10243 }
10244 }
10245 break;
10246 default:
10247 #if GC_DEBUG
10248 rb_gcdebug_print_obj_condition((VALUE)obj);
10249 rb_obj_info_dump(obj);
10250 rb_bug("unreachable");
10251 #endif
10252 break;
10253
10254 }
10255
10256 UPDATE_IF_MOVED(objspace, RBASIC(obj)->klass);
10257
10258 gc_report(4, objspace, "update-refs: %p <-\n", (void *)obj);
10259 }
10260
10261 static int
10262 gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t * objspace, struct heap_page *page)
10263 {
10264 VALUE v = (VALUE)vstart;
10265 asan_unlock_freelist(page);
10266 asan_lock_freelist(page);
10267 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
10268 page->flags.has_remembered_objects = FALSE;
10269
10270 /* For each object on the page */
10271 for (; v != (VALUE)vend; v += stride) {
10272 void *poisoned = asan_unpoison_object_temporary(v);
10273
10274 switch (BUILTIN_TYPE(v)) {
10275 case T_NONE:
10276 case T_MOVED:
10277 case T_ZOMBIE:
10278 break;
10279 default:
10280 if (RVALUE_WB_UNPROTECTED(v)) {
10281 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
10282 }
10283 if (RVALUE_REMEMBERED(v)) {
10284 page->flags.has_remembered_objects = TRUE;
10285 }
10286 if (page->flags.before_sweep) {
10287 if (RVALUE_MARKED(v)) {
10288 gc_update_object_references(objspace, v);
10289 }
10290 }
10291 else {
10292 gc_update_object_references(objspace, v);
10293 }
10294 }
10295
10296 if (poisoned) {
10297 asan_poison_object(v);
10298 }
10299 }
10300
10301 return 0;
10302 }
10303
10304 extern rb_symbols_t ruby_global_symbols;
10305 #define global_symbols ruby_global_symbols
10306
10307 static void
10308 gc_update_references(rb_objspace_t *objspace)
10309 {
10310 objspace->flags.during_reference_updating = true;
10311
10312 rb_execution_context_t *ec = GET_EC();
10313 rb_vm_t *vm = rb_ec_vm_ptr(ec);
10314
10315 struct heap_page *page = NULL;
10316
10317 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
10318 bool should_set_mark_bits = TRUE;
10319 rb_size_pool_t *size_pool = &size_pools[i];
10320 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
10321
10322 ccan_list_for_each(&heap->pages, page, page_node) {
10323 uintptr_t start = (uintptr_t)page->start;
10324 uintptr_t end = start + (page->total_slots * size_pool->slot_size);
10325
10326 gc_ref_update((void *)start, (void *)end, size_pool->slot_size, objspace, page);
10327 if (page == heap->sweeping_page) {
10328 should_set_mark_bits = FALSE;
10329 }
10330 if (should_set_mark_bits) {
10331 gc_setup_mark_bits(page);
10332 }
10333 }
10334 }
10335 rb_vm_update_references(vm);
10336 rb_gc_update_global_tbl();
10337 global_symbols.ids = rb_gc_location(global_symbols.ids);
10338 global_symbols.dsymbol_fstr_hash = rb_gc_location(global_symbols.dsymbol_fstr_hash);
10339 gc_ref_update_table_values_only(objspace, objspace->obj_to_id_tbl);
10340 gc_update_table_refs(objspace, objspace->id_to_obj_tbl);
10341 gc_update_table_refs(objspace, global_symbols.str_sym);
10342 gc_update_table_refs(objspace, finalizer_table);
10343
10344 objspace->flags.during_reference_updating = false;
10345 }
10346
10347 #if GC_CAN_COMPILE_COMPACTION
10348 /*
10349 * call-seq:
10350 * GC.latest_compact_info -> hash
10351 *
10352 * Returns information about object moved in the most recent \GC compaction.
10353 *
10354 * The returned +hash+ contains the following keys:
10355 *
10356 * [considered]
10357 * Hash containing the type of the object as the key and the number of
10358 * objects of that type that were considered for movement.
10359 * [moved]
10360 * Hash containing the type of the object as the key and the number of
10361 * objects of that type that were actually moved.
10362 * [moved_up]
10363 * Hash containing the type of the object as the key and the number of
10364 * objects of that type that were increased in size.
10365 * [moved_down]
10366 * Hash containing the type of the object as the key and the number of
10367 * objects of that type that were decreased in size.
10368 *
10369 * Some objects can't be moved (due to pinning) so these numbers can be used to
10370 * calculate compaction efficiency.
10371 */
10372 static VALUE
10373 gc_compact_stats(VALUE self)
10374 {
10375 size_t i;
10376 rb_objspace_t *objspace = &rb_objspace;
10377 VALUE h = rb_hash_new();
10378 VALUE considered = rb_hash_new();
10379 VALUE moved = rb_hash_new();
10380 VALUE moved_up = rb_hash_new();
10381 VALUE moved_down = rb_hash_new();
10382
10383 for (i=0; i<T_MASK; i++) {
10384 if (objspace->rcompactor.considered_count_table[i]) {
10385 rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i]));
10386 }
10387
10388 if (objspace->rcompactor.moved_count_table[i]) {
10389 rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i]));
10390 }
10391
10392 if (objspace->rcompactor.moved_up_count_table[i]) {
10393 rb_hash_aset(moved_up, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_up_count_table[i]));
10394 }
10395
10396 if (objspace->rcompactor.moved_down_count_table[i]) {
10397 rb_hash_aset(moved_down, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_down_count_table[i]));
10398 }
10399 }
10400
10401 rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered);
10402 rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved);
10403 rb_hash_aset(h, ID2SYM(rb_intern("moved_up")), moved_up);
10404 rb_hash_aset(h, ID2SYM(rb_intern("moved_down")), moved_down);
10405
10406 return h;
10407 }
10408 #else
10409 # define gc_compact_stats rb_f_notimplement
10410 #endif
10411
10412 #if GC_CAN_COMPILE_COMPACTION
10413 static void
10414 root_obj_check_moved_i(const char *category, VALUE obj, void *data)
10415 {
10416 rb_objspace_t *objspace = data;
10417
10418 if (gc_object_moved_p(objspace, obj)) {
10419 rb_bug("ROOT %s points to MOVED: %p -> %s", category, (void *)obj, obj_info(rb_gc_location(obj)));
10420 }
10421 }
10422
10423 static void
10424 reachable_object_check_moved_i(VALUE ref, void *data)
10425 {
10426 VALUE parent = (VALUE)data;
10427 if (gc_object_moved_p(&rb_objspace, ref)) {
10428 rb_bug("Object %s points to MOVED: %p -> %s", obj_info(parent), (void *)ref, obj_info(rb_gc_location(ref)));
10429 }
10430 }
10431
10432 static int
10433 heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
10434 {
10435 rb_objspace_t *objspace = data;
10436
10437 VALUE v = (VALUE)vstart;
10438 for (; v != (VALUE)vend; v += stride) {
10439 if (gc_object_moved_p(objspace, v)) {
10440 /* Moved object still on the heap, something may have a reference. */
10441 }
10442 else {
10443 void *poisoned = asan_unpoison_object_temporary(v);
10444
10445 switch (BUILTIN_TYPE(v)) {
10446 case T_NONE:
10447 case T_ZOMBIE:
10448 break;
10449 default:
10450 if (!rb_objspace_garbage_object_p(v)) {
10451 rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v);
10452 }
10453 }
10454
10455 if (poisoned) {
10456 GC_ASSERT(BUILTIN_TYPE(v) == T_NONE);
10457 asan_poison_object(v);
10458 }
10459 }
10460 }
10461
10462 return 0;
10463 }
10464
10465 /*
10466 * call-seq:
10467 * GC.compact -> hash
10468 *
10469 * This function compacts objects together in Ruby's heap. It eliminates
10470 * unused space (or fragmentation) in the heap by moving objects in to that
10471 * unused space.
10472 *
10473 * The returned +hash+ contains statistics about the objects that were moved;
10474 * see GC.latest_compact_info.
10475 *
10476 * This method is only expected to work on CRuby.
10477 *
10478 * To test whether \GC compaction is supported, use the idiom:
10479 *
10480 * GC.respond_to?(:compact)
10481 */
10482 static VALUE
10483 gc_compact(VALUE self)
10484 {
10485 /* Run GC with compaction enabled */
10486 gc_start_internal(NULL, self, Qtrue, Qtrue, Qtrue, Qtrue);
10487
10488 return gc_compact_stats(self);
10489 }
10490 #else
10491 # define gc_compact rb_f_notimplement
10492 #endif
10493
10494 #if GC_CAN_COMPILE_COMPACTION
10495
10496 struct desired_compaction_pages_i_data {
10497 rb_objspace_t *objspace;
10498 size_t required_slots[SIZE_POOL_COUNT];
10499 };
10500
10501 static int
10502 desired_compaction_pages_i(struct heap_page *page, void *data)
10503 {
10504 struct desired_compaction_pages_i_data *tdata = data;
10505 rb_objspace_t *objspace = tdata->objspace;
10506 VALUE vstart = (VALUE)page->start;
10507 VALUE vend = vstart + (VALUE)(page->total_slots * page->size_pool->slot_size);
10508
10509
10510 for (VALUE v = vstart; v != vend; v += page->size_pool->slot_size) {
10511 /* skip T_NONEs; they won't be moved */
10512 void *poisoned = asan_unpoison_object_temporary(v);
10513 if (BUILTIN_TYPE(v) == T_NONE) {
10514 if (poisoned) {
10515 asan_poison_object(v);
10516 }
10517 continue;
10518 }
10519
10520 rb_size_pool_t *dest_pool = gc_compact_destination_pool(objspace, page->size_pool, v);
10521 size_t dest_pool_idx = dest_pool - size_pools;
10522 tdata->required_slots[dest_pool_idx]++;
10523 }
10524
10525 return 0;
10526 }
10527
10528 static VALUE
10529 gc_verify_compaction_references(rb_execution_context_t *ec, VALUE self, VALUE double_heap, VALUE expand_heap, VALUE toward_empty)
10530 {
10531 rb_objspace_t *objspace = &rb_objspace;
10532
10533 /* Clear the heap. */
10534 gc_start_internal(NULL, self, Qtrue, Qtrue, Qtrue, Qfalse);
10535
10536 if (RTEST(double_heap)) {
10537 rb_warn("double_heap is deprecated, please use expand_heap instead");
10538 }
10539
10540 RB_VM_LOCK_ENTER();
10541 {
10542 gc_rest(objspace);
10543
10544 /* if both double_heap and expand_heap are set, expand_heap takes precedence */
10545 if (RTEST(expand_heap)) {
10546 struct desired_compaction_pages_i_data desired_compaction = {
10547 .objspace = objspace,
10548 .required_slots = {0},
10549 };
10550 /* Work out how many objects want to be in each size pool, taking account of moves */
10551 objspace_each_pages(objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
10552
10553 /* Find out which pool has the most pages */
10554 size_t max_existing_pages = 0;
10555 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
10556 rb_size_pool_t *size_pool = &size_pools[i];
10557 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
10558 max_existing_pages = MAX(max_existing_pages, heap->total_pages);
10559 }
10560 /* Add pages to each size pool so that compaction is guaranteed to move every object */
10561 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
10562 rb_size_pool_t *size_pool = &size_pools[i];
10563 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
10564
10565 size_t pages_to_add = 0;
10566 /*
10567 * Step 1: Make sure every pool has the same number of pages, by adding empty pages
10568 * to smaller pools. This is required to make sure the compact cursor can advance
10569 * through all of the pools in `gc_sweep_compact` without hitting the "sweep &
10570 * compact cursors met" condition on some pools before fully compacting others
10571 */
10572 pages_to_add += max_existing_pages - heap->total_pages;
10573 /*
10574 * Step 2: Now add additional free pages to each size pool sufficient to hold all objects
10575 * that want to be in that size pool, whether moved into it or moved within it
10576 */
10577 pages_to_add += slots_to_pages_for_size_pool(objspace, size_pool, desired_compaction.required_slots[i]);
10578 /*
10579 * Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects
10580 * have been moved, and not on the last iteration of the `gc_sweep_compact` loop
10581 */
10582 pages_to_add += 2;
10583
10584 heap_add_pages(objspace, size_pool, heap, pages_to_add);
10585 }
10586 }
10587 else if (RTEST(double_heap)) {
10588 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
10589 rb_size_pool_t *size_pool = &size_pools[i];
10590 rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
10591 heap_add_pages(objspace, size_pool, heap, heap->total_pages);
10592 }
10593
10594 }
10595
10596 if (RTEST(toward_empty)) {
10597 objspace->rcompactor.compare_func = compare_free_slots;
10598 }
10599 }
10600 RB_VM_LOCK_LEAVE();
10601
10602 gc_start_internal(NULL, self, Qtrue, Qtrue, Qtrue, Qtrue);
10603
10604 objspace_reachable_objects_from_root(objspace, root_obj_check_moved_i, objspace);
10605 objspace_each_objects(objspace, heap_check_moved_i, objspace, TRUE);
10606
10607 objspace->rcompactor.compare_func = NULL;
10608 return gc_compact_stats(self);
10609 }
10610 #else
10611 # define gc_verify_compaction_references (rb_builtin_arity3_function_type)rb_f_notimplement
10612 #endif
10613
10614 VALUE
10615 rb_gc_start(void)
10616 {
10617 rb_gc();
10618 return Qnil;
10619 }
10620
10621 void
10622 rb_gc(void)
10623 {
10624 unless_objspace(objspace) { return; }
10625 unsigned int reason = GPR_DEFAULT_REASON;
10626 garbage_collect(objspace, reason);
10627 }
10628
10629 int
10630 rb_during_gc(void)
10631 {
10632 unless_objspace(objspace) { return FALSE; }
10633 return during_gc;
10634 }
10635
10636 #if RGENGC_PROFILE >= 2
10637
10638 static const char *type_name(int type, VALUE obj);
10639
10640 static void
10641 gc_count_add_each_types(VALUE hash, const char *name, const size_t *types)
10642 {
10643 VALUE result = rb_hash_new_with_size(T_MASK);
10644 int i;
10645 for (i=0; i<T_MASK; i++) {
10646 const char *type = type_name(i, 0);
10647 rb_hash_aset(result, ID2SYM(rb_intern(type)), SIZET2NUM(types[i]));
10648 }
10649 rb_hash_aset(hash, ID2SYM(rb_intern(name)), result);
10650 }
10651 #endif
10652
10653 size_t
10654 rb_gc_count(void)
10655 {
10656 return rb_objspace.profile.count;
10657 }
10658
10659 static VALUE
10660 gc_count(rb_execution_context_t *ec, VALUE self)
10661 {
10662 return SIZET2NUM(rb_gc_count());
10663 }
10664
10665 static VALUE
10666 gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned int orig_flags)
10667 {
10668 static VALUE sym_major_by = Qnil, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
10669 static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
10670 #if RGENGC_ESTIMATE_OLDMALLOC
10671 static VALUE sym_oldmalloc;
10672 #endif
10673 static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
10674 static VALUE sym_none, sym_marking, sym_sweeping;
10675 static VALUE sym_weak_references_count, sym_retained_weak_references_count;
10676 VALUE hash = Qnil, key = Qnil;
10677 VALUE major_by, need_major_by;
10678 unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
10679
10680 if (SYMBOL_P(hash_or_key)) {
10681 key = hash_or_key;
10682 }
10683 else if (RB_TYPE_P(hash_or_key, T_HASH)) {
10684 hash = hash_or_key;
10685 }
10686 else {
10687 rb_raise(rb_eTypeError, "non-hash or symbol given");
10688 }
10689
10690 if (NIL_P(sym_major_by)) {
10691 #define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
10692 S(major_by);
10693 S(gc_by);
10694 S(immediate_sweep);
10695 S(have_finalizer);
10696 S(state);
10697 S(need_major_by);
10698
10699 S(stress);
10700 S(nofree);
10701 S(oldgen);
10702 S(shady);
10703 S(force);
10704 #if RGENGC_ESTIMATE_OLDMALLOC
10705 S(oldmalloc);
10706 #endif
10707 S(newobj);
10708 S(malloc);
10709 S(method);
10710 S(capi);
10711
10712 S(none);
10713 S(marking);
10714 S(sweeping);
10715
10716 S(weak_references_count);
10717 S(retained_weak_references_count);
10718 #undef S
10719 }
10720
10721 #define SET(name, attr) \
10722 if (key == sym_##name) \
10723 return (attr); \
10724 else if (hash != Qnil) \
10725 rb_hash_aset(hash, sym_##name, (attr));
10726
10727 major_by =
10728 (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
10729 (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
10730 (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
10731 (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
10732 #if RGENGC_ESTIMATE_OLDMALLOC
10733 (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
10734 #endif
10735 Qnil;
10736 SET(major_by, major_by);
10737
10738 if (orig_flags == 0) { /* set need_major_by only if flags not set explicitly */
10739 unsigned int need_major_flags = gc_needs_major_flags;
10740 need_major_by =
10741 (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
10742 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
10743 (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
10744 (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
10745 #if RGENGC_ESTIMATE_OLDMALLOC
10746 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
10747 #endif
10748 Qnil;
10749 SET(need_major_by, need_major_by);
10750 }
10751
10752 SET(gc_by,
10753 (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
10754 (flags & GPR_FLAG_MALLOC) ? sym_malloc :
10755 (flags & GPR_FLAG_METHOD) ? sym_method :
10756 (flags & GPR_FLAG_CAPI) ? sym_capi :
10757 (flags & GPR_FLAG_STRESS) ? sym_stress :
10758 Qnil
10759 );
10760
10761 SET(have_finalizer, RBOOL(flags & GPR_FLAG_HAVE_FINALIZE));
10762 SET(immediate_sweep, RBOOL(flags & GPR_FLAG_IMMEDIATE_SWEEP));
10763
10764 if (orig_flags == 0) {
10765 SET(state, gc_mode(objspace) == gc_mode_none ? sym_none :
10766 gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
10767 }
10768
10769 SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count));
10770 SET(retained_weak_references_count, LONG2FIX(objspace->profile.retained_weak_references_count));
10771 #undef SET
10772
10773 if (!NIL_P(key)) {/* matched key should return above */
10774 rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
10775 }
10776
10777 return hash;
10778 }
10779
10780 VALUE
10781 rb_gc_latest_gc_info(VALUE key)
10782 {
10783 rb_objspace_t *objspace = &rb_objspace;
10784 return gc_info_decode(objspace, key, 0);
10785 }
10786
10787 static VALUE
10788 gc_latest_gc_info(rb_execution_context_t *ec, VALUE self, VALUE arg)
10789 {
10790 rb_objspace_t *objspace = &rb_objspace;
10791
10792 if (NIL_P(arg)) {
10793 arg = rb_hash_new();
10794 }
10795 else if (!SYMBOL_P(arg) && !RB_TYPE_P(arg, T_HASH)) {
10796 rb_raise(rb_eTypeError, "non-hash or symbol given");
10797 }
10798
10799 return gc_info_decode(objspace, arg, 0);
10800 }
10801
10802 enum gc_stat_sym {
10803 gc_stat_sym_count,
10804 gc_stat_sym_time,
10805 gc_stat_sym_marking_time,
10806 gc_stat_sym_sweeping_time,
10807 gc_stat_sym_heap_allocated_pages,
10808 gc_stat_sym_heap_sorted_length,
10809 gc_stat_sym_heap_allocatable_pages,
10810 gc_stat_sym_heap_available_slots,
10811 gc_stat_sym_heap_live_slots,
10812 gc_stat_sym_heap_free_slots,
10813 gc_stat_sym_heap_final_slots,
10814 gc_stat_sym_heap_marked_slots,
10815 gc_stat_sym_heap_eden_pages,
10816 gc_stat_sym_heap_tomb_pages,
10817 gc_stat_sym_total_allocated_pages,
10818 gc_stat_sym_total_freed_pages,
10819 gc_stat_sym_total_allocated_objects,
10820 gc_stat_sym_total_freed_objects,
10821 gc_stat_sym_malloc_increase_bytes,
10822 gc_stat_sym_malloc_increase_bytes_limit,
10823 gc_stat_sym_minor_gc_count,
10824 gc_stat_sym_major_gc_count,
10825 gc_stat_sym_compact_count,
10826 gc_stat_sym_read_barrier_faults,
10827 gc_stat_sym_total_moved_objects,
10828 gc_stat_sym_remembered_wb_unprotected_objects,
10829 gc_stat_sym_remembered_wb_unprotected_objects_limit,
10830 gc_stat_sym_old_objects,
10831 gc_stat_sym_old_objects_limit,
10832 #if RGENGC_ESTIMATE_OLDMALLOC
10833 gc_stat_sym_oldmalloc_increase_bytes,
10834 gc_stat_sym_oldmalloc_increase_bytes_limit,
10835 #endif
10836 gc_stat_sym_weak_references_count,
10837 #if RGENGC_PROFILE
10838 gc_stat_sym_total_generated_normal_object_count,
10839 gc_stat_sym_total_generated_shady_object_count,
10840 gc_stat_sym_total_shade_operation_count,
10841 gc_stat_sym_total_promoted_count,
10842 gc_stat_sym_total_remembered_normal_object_count,
10843 gc_stat_sym_total_remembered_shady_object_count,
10844 #endif
10845 gc_stat_sym_last
10846 };
10847
10848 static VALUE gc_stat_symbols[gc_stat_sym_last];
10849
10850 static void
10851 setup_gc_stat_symbols(void)
10852 {
10853 if (gc_stat_symbols[0] == 0) {
10854 #define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
10855 S(count);
10856 S(time);
10857 S(marking_time),
10858 S(sweeping_time),
10859 S(heap_allocated_pages);
10860 S(heap_sorted_length);
10861 S(heap_allocatable_pages);
10862 S(heap_available_slots);
10863 S(heap_live_slots);
10864 S(heap_free_slots);
10865 S(heap_final_slots);
10866 S(heap_marked_slots);
10867 S(heap_eden_pages);
10868 S(heap_tomb_pages);
10869 S(total_allocated_pages);
10870 S(total_freed_pages);
10871 S(total_allocated_objects);
10872 S(total_freed_objects);
10873 S(malloc_increase_bytes);
10874 S(malloc_increase_bytes_limit);
10875 S(minor_gc_count);
10876 S(major_gc_count);
10877 S(compact_count);
10878 S(read_barrier_faults);
10879 S(total_moved_objects);
10880 S(remembered_wb_unprotected_objects);
10881 S(remembered_wb_unprotected_objects_limit);
10882 S(old_objects);
10883 S(old_objects_limit);
10884 #if RGENGC_ESTIMATE_OLDMALLOC
10885 S(oldmalloc_increase_bytes);
10886 S(oldmalloc_increase_bytes_limit);
10887 #endif
10888 S(weak_references_count);
10889 #if RGENGC_PROFILE
10890 S(total_generated_normal_object_count);
10891 S(total_generated_shady_object_count);
10892 S(total_shade_operation_count);
10893 S(total_promoted_count);
10894 S(total_remembered_normal_object_count);
10895 S(total_remembered_shady_object_count);
10896 #endif /* RGENGC_PROFILE */
10897 #undef S
10898 }
10899 }
10900
10901 static uint64_t
10902 ns_to_ms(uint64_t ns)
10903 {
10904 return ns / (1000 * 1000);
10905 }
10906
10907 static size_t
10908 gc_stat_internal(VALUE hash_or_sym)
10909 {
10910 rb_objspace_t *objspace = &rb_objspace;
10911 VALUE hash = Qnil, key = Qnil;
10912
10913 setup_gc_stat_symbols();
10914
10915 if (RB_TYPE_P(hash_or_sym, T_HASH)) {
10916 hash = hash_or_sym;
10917 }
10918 else if (SYMBOL_P(hash_or_sym)) {
10919 key = hash_or_sym;
10920 }
10921 else {
10922 rb_raise(rb_eTypeError, "non-hash or symbol argument");
10923 }
10924
10925 #define SET(name, attr) \
10926 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
10927 return attr; \
10928 else if (hash != Qnil) \
10929 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
10930
10931 SET(count, objspace->profile.count);
10932 SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM
10933 SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns));
10934 SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns));
10935
10936 /* implementation dependent counters */
10937 SET(heap_allocated_pages, heap_allocated_pages);
10938 SET(heap_sorted_length, heap_pages_sorted_length);
10939 SET(heap_allocatable_pages, heap_allocatable_pages(objspace));
10940 SET(heap_available_slots, objspace_available_slots(objspace));
10941 SET(heap_live_slots, objspace_live_slots(objspace));
10942 SET(heap_free_slots, objspace_free_slots(objspace));
10943 SET(heap_final_slots, heap_pages_final_slots);
10944 SET(heap_marked_slots, objspace->marked_slots);
10945 SET(heap_eden_pages, heap_eden_total_pages(objspace));
10946 SET(heap_tomb_pages, heap_tomb_total_pages(objspace));
10947 SET(total_allocated_pages, total_allocated_pages(objspace));
10948 SET(total_freed_pages, total_freed_pages(objspace));
10949 SET(total_allocated_objects, total_allocated_objects(objspace));
10950 SET(total_freed_objects, total_freed_objects(objspace));
10951 SET(malloc_increase_bytes, malloc_increase);
10952 SET(malloc_increase_bytes_limit, malloc_limit);
10953 SET(minor_gc_count, objspace->profile.minor_gc_count);
10954 SET(major_gc_count, objspace->profile.major_gc_count);
10955 SET(compact_count, objspace->profile.compact_count);
10956 SET(read_barrier_faults, objspace->profile.read_barrier_faults);
10957 SET(total_moved_objects, objspace->rcompactor.total_moved);
10958 SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
10959 SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
10960 SET(old_objects, objspace->rgengc.old_objects);
10961 SET(old_objects_limit, objspace->rgengc.old_objects_limit);
10962 #if RGENGC_ESTIMATE_OLDMALLOC
10963 SET(oldmalloc_increase_bytes, objspace->rgengc.oldmalloc_increase);
10964 SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
10965 #endif
10966
10967 #if RGENGC_PROFILE
10968 SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
10969 SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
10970 SET(total_shade_operation_count, objspace->profile.total_shade_operation_count);
10971 SET(total_promoted_count, objspace->profile.total_promoted_count);
10972 SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count);
10973 SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
10974 #endif /* RGENGC_PROFILE */
10975 #undef SET
10976
10977 if (!NIL_P(key)) { /* matched key should return above */
10978 rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
10979 }
10980
10981 #if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
10982 if (hash != Qnil) {
10983 gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types);
10984 gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types);
10985 gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types);
10986 gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types);
10987 gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types);
10988 gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types);
10989 }
10990 #endif
10991
10992 return 0;
10993 }
10994
10995 static VALUE
10996 gc_stat(rb_execution_context_t *ec, VALUE self, VALUE arg) // arg is (nil || hash || symbol)
10997 {
10998 if (NIL_P(arg)) {
10999 arg = rb_hash_new();
11000 }
11001 else if (SYMBOL_P(arg)) {
11002 size_t value = gc_stat_internal(arg);
11003 return SIZET2NUM(value);
11004 }
11005 else if (RB_TYPE_P(arg, T_HASH)) {
11006 // ok
11007 }
11008 else {
11009 rb_raise(rb_eTypeError, "non-hash or symbol given");
11010 }
11011
11012 gc_stat_internal(arg);
11013 return arg;
11014 }
11015
11016 size_t
11017 rb_gc_stat(VALUE key)
11018 {
11019 if (SYMBOL_P(key)) {
11020 size_t value = gc_stat_internal(key);
11021 return value;
11022 }
11023 else {
11024 gc_stat_internal(key);
11025 return 0;
11026 }
11027 }
11028
11029
11030 enum gc_stat_heap_sym {
11031 gc_stat_heap_sym_slot_size,
11032 gc_stat_heap_sym_heap_allocatable_pages,
11033 gc_stat_heap_sym_heap_eden_pages,
11034 gc_stat_heap_sym_heap_eden_slots,
11035 gc_stat_heap_sym_heap_tomb_pages,
11036 gc_stat_heap_sym_heap_tomb_slots,
11037 gc_stat_heap_sym_total_allocated_pages,
11038 gc_stat_heap_sym_total_freed_pages,
11039 gc_stat_heap_sym_force_major_gc_count,
11040 gc_stat_heap_sym_force_incremental_marking_finish_count,
11041 gc_stat_heap_sym_total_allocated_objects,
11042 gc_stat_heap_sym_total_freed_objects,
11043 gc_stat_heap_sym_last
11044 };
11045
11046 static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
11047
11048 static void
11049 setup_gc_stat_heap_symbols(void)
11050 {
11051 if (gc_stat_heap_symbols[0] == 0) {
11052 #define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
11053 S(slot_size);
11054 S(heap_allocatable_pages);
11055 S(heap_eden_pages);
11056 S(heap_eden_slots);
11057 S(heap_tomb_pages);
11058 S(heap_tomb_slots);
11059 S(total_allocated_pages);
11060 S(total_freed_pages);
11061 S(force_major_gc_count);
11062 S(force_incremental_marking_finish_count);
11063 S(total_allocated_objects);
11064 S(total_freed_objects);
11065 #undef S
11066 }
11067 }
11068
11069 static size_t
11070 gc_stat_heap_internal(int size_pool_idx, VALUE hash_or_sym)
11071 {
11072 rb_objspace_t *objspace = &rb_objspace;
11073 VALUE hash = Qnil, key = Qnil;
11074
11075 setup_gc_stat_heap_symbols();
11076
11077 if (RB_TYPE_P(hash_or_sym, T_HASH)) {
11078 hash = hash_or_sym;
11079 }
11080 else if (SYMBOL_P(hash_or_sym)) {
11081 key = hash_or_sym;
11082 }
11083 else {
11084 rb_raise(rb_eTypeError, "non-hash or symbol argument");
11085 }
11086
11087 if (size_pool_idx < 0 || size_pool_idx >= SIZE_POOL_COUNT) {
11088 rb_raise(rb_eArgError, "size pool index out of range");
11089 }
11090
11091 rb_size_pool_t *size_pool = &size_pools[size_pool_idx];
11092
11093 #define SET(name, attr) \
11094 if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
11095 return attr; \
11096 else if (hash != Qnil) \
11097 rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
11098
11099 SET(slot_size, size_pool->slot_size);
11100 SET(heap_allocatable_pages, size_pool->allocatable_pages);
11101 SET(heap_eden_pages, SIZE_POOL_EDEN_HEAP(size_pool)->total_pages);
11102 SET(heap_eden_slots, SIZE_POOL_EDEN_HEAP(size_pool)->total_slots);
11103 SET(heap_tomb_pages, SIZE_POOL_TOMB_HEAP(size_pool)->total_pages);
11104 SET(heap_tomb_slots, SIZE_POOL_TOMB_HEAP(size_pool)->total_slots);
11105 SET(total_allocated_pages, size_pool->total_allocated_pages);
11106 SET(total_freed_pages, size_pool->total_freed_pages);
11107 SET(force_major_gc_count, size_pool->force_major_gc_count);
11108 SET(force_incremental_marking_finish_count, size_pool->force_incremental_marking_finish_count);
11109 SET(total_allocated_objects, size_pool->total_allocated_objects);
11110 SET(total_freed_objects, size_pool->total_freed_objects);
11111 #undef SET
11112
11113 if (!NIL_P(key)) { /* matched key should return above */
11114 rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
11115 }
11116
11117 return 0;
11118 }
11119
11120 static VALUE
11121 gc_stat_heap(rb_execution_context_t *ec, VALUE self, VALUE heap_name, VALUE arg)
11122 {
11123 if (NIL_P(heap_name)) {
11124 if (NIL_P(arg)) {
11125 arg = rb_hash_new();
11126 }
11127 else if (RB_TYPE_P(arg, T_HASH)) {
11128 // ok
11129 }
11130 else {
11131 rb_raise(rb_eTypeError, "non-hash given");
11132 }
11133
11134 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
11135 VALUE hash = rb_hash_aref(arg, INT2FIX(i));
11136 if (NIL_P(hash)) {
11137 hash = rb_hash_new();
11138 rb_hash_aset(arg, INT2FIX(i), hash);
11139 }
11140 gc_stat_heap_internal(i, hash);
11141 }
11142 }
11143 else if (FIXNUM_P(heap_name)) {
11144 int size_pool_idx = FIX2INT(heap_name);
11145
11146 if (NIL_P(arg)) {
11147 arg = rb_hash_new();
11148 }
11149 else if (SYMBOL_P(arg)) {
11150 size_t value = gc_stat_heap_internal(size_pool_idx, arg);
11151 return SIZET2NUM(value);
11152 }
11153 else if (RB_TYPE_P(arg, T_HASH)) {
11154 // ok
11155 }
11156 else {
11157 rb_raise(rb_eTypeError, "non-hash or symbol given");
11158 }
11159
11160 gc_stat_heap_internal(size_pool_idx, arg);
11161 }
11162 else {
11163 rb_raise(rb_eTypeError, "heap_name must be nil or an Integer");
11164 }
11165
11166 return arg;
11167 }
11168
11169 static VALUE
11170 gc_stress_get(rb_execution_context_t *ec, VALUE self)
11171 {
11172 rb_objspace_t *objspace = &rb_objspace;
11173 return ruby_gc_stress_mode;
11174 }
11175
11176 static VALUE
11177 gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag)
11178 {
11179 rb_objspace_t *objspace = &rb_objspace;
11180
11181 objspace->flags.gc_stressful = RTEST(flag);
11182 objspace->gc_stress_mode = flag;
11183
11184 return flag;
11185 }
11186
11187 VALUE
11188 rb_gc_enable(void)
11189 {
11190 rb_objspace_t *objspace = &rb_objspace;
11191 return rb_objspace_gc_enable(objspace);
11192 }
11193
11194 VALUE
11195 rb_objspace_gc_enable(rb_objspace_t *objspace)
11196 {
11197 int old = dont_gc_val();
11198
11199 dont_gc_off();
11200 return RBOOL(old);
11201 }
11202
11203 static VALUE
11204 gc_enable(rb_execution_context_t *ec, VALUE _)
11205 {
11206 return rb_gc_enable();
11207 }
11208
11209 VALUE
11210 rb_gc_disable_no_rest(void)
11211 {
11212 rb_objspace_t *objspace = &rb_objspace;
11213 return gc_disable_no_rest(objspace);
11214 }
11215
11216 static VALUE
11217 gc_disable_no_rest(rb_objspace_t *objspace)
11218 {
11219 int old = dont_gc_val();
11220 dont_gc_on();
11221 return RBOOL(old);
11222 }
11223
11224 VALUE
11225 rb_gc_disable(void)
11226 {
11227 rb_objspace_t *objspace = &rb_objspace;
11228 return rb_objspace_gc_disable(objspace);
11229 }
11230
11231 VALUE
11232 rb_objspace_gc_disable(rb_objspace_t *objspace)
11233 {
11234 gc_rest(objspace);
11235 return gc_disable_no_rest(objspace);
11236 }
11237
11238 static VALUE
11239 gc_disable(rb_execution_context_t *ec, VALUE _)
11240 {
11241 return rb_gc_disable();
11242 }
11243
11244 #if GC_CAN_COMPILE_COMPACTION
11245 /*
11246 * call-seq:
11247 * GC.auto_compact = flag
11248 *
11249 * Updates automatic compaction mode.
11250 *
11251 * When enabled, the compactor will execute on every major collection.
11252 *
11253 * Enabling compaction will degrade performance on major collections.
11254 */
11255 static VALUE
11256 gc_set_auto_compact(VALUE _, VALUE v)
11257 {
11258 GC_ASSERT(GC_COMPACTION_SUPPORTED);
11259
11260 ruby_enable_autocompact = RTEST(v);
11261
11262 #if RGENGC_CHECK_MODE
11263 ruby_autocompact_compare_func = NULL;
11264
11265 if (SYMBOL_P(v)) {
11266 ID id = RB_SYM2ID(v);
11267 if (id == rb_intern("empty")) {
11268 ruby_autocompact_compare_func = compare_free_slots;
11269 }
11270 }
11271 #endif
11272
11273 return v;
11274 }
11275 #else
11276 # define gc_set_auto_compact rb_f_notimplement
11277 #endif
11278
11279 #if GC_CAN_COMPILE_COMPACTION
11280 /*
11281 * call-seq:
11282 * GC.auto_compact -> true or false
11283 *
11284 * Returns whether or not automatic compaction has been enabled.
11285 */
11286 static VALUE
11287 gc_get_auto_compact(VALUE _)
11288 {
11289 return RBOOL(ruby_enable_autocompact);
11290 }
11291 #else
11292 # define gc_get_auto_compact rb_f_notimplement
11293 #endif
11294
11295 static int
11296 get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
11297 {
11298 const char *ptr = getenv(name);
11299 ssize_t val;
11300
11301 if (ptr != NULL && *ptr) {
11302 size_t unit = 0;
11303 char *end;
11304 #if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
11305 val = strtoll(ptr, &end, 0);
11306 #else
11307 val = strtol(ptr, &end, 0);
11308 #endif
11309 switch (*end) {
11310 case 'k': case 'K':
11311 unit = 1024;
11312 ++end;
11313 break;
11314 case 'm': case 'M':
11315 unit = 1024*1024;
11316 ++end;
11317 break;
11318 case 'g': case 'G':
11319 unit = 1024*1024*1024;
11320 ++end;
11321 break;
11322 }
11323 while (*end && isspace((unsigned char)*end)) end++;
11324 if (*end) {
11325 if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
11326 return 0;
11327 }
11328 if (unit > 0) {
11329 if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
11330 if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr);
11331 return 0;
11332 }
11333 val *= unit;
11334 }
11335 if (val > 0 && (size_t)val > lower_bound) {
11336 if (RTEST(ruby_verbose)) {
11337 fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value);
11338 }
11339 *default_value = (size_t)val;
11340 return 1;
11341 }
11342 else {
11343 if (RTEST(ruby_verbose)) {
11344 fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n",
11345 name, val, *default_value, lower_bound);
11346 }
11347 return 0;
11348 }
11349 }
11350 return 0;
11351 }
11352
11353 static int
11354 get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero)
11355 {
11356 const char *ptr = getenv(name);
11357 double val;
11358
11359 if (ptr != NULL && *ptr) {
11360 char *end;
11361 val = strtod(ptr, &end);
11362 if (!*ptr || *end) {
11363 if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
11364 return 0;
11365 }
11366
11367 if (accept_zero && val == 0.0) {
11368 goto accept;
11369 }
11370 else if (val <= lower_bound) {
11371 if (RTEST(ruby_verbose)) {
11372 fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
11373 name, val, *default_value, lower_bound);
11374 }
11375 }
11376 else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */
11377 val > upper_bound) {
11378 if (RTEST(ruby_verbose)) {
11379 fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
11380 name, val, *default_value, upper_bound);
11381 }
11382 }
11383 else {
11384 goto accept;
11385 }
11386 }
11387 return 0;
11388
11389 accept:
11390 if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value);
11391 *default_value = val;
11392 return 1;
11393 }
11394
11395 static void
11396 gc_set_initial_pages(rb_objspace_t *objspace)
11397 {
11398 gc_rest(objspace);
11399
11400 for (int i = 0; i < SIZE_POOL_COUNT; i++) {
11401 rb_size_pool_t *size_pool = &size_pools[i];
11402 char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)];
11403 snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i);
11404
11405 size_t size_pool_init_slots = gc_params.size_pool_init_slots[i];
11406 if (get_envparam_size(env_key, &size_pool_init_slots, 0)) {
11407 gc_params.size_pool_init_slots[i] = size_pool_init_slots;
11408 }
11409
11410 if (size_pool_init_slots > size_pool->eden_heap.total_slots) {
11411 size_t slots = size_pool_init_slots - size_pool->eden_heap.total_slots;
11412 size_pool->allocatable_pages = slots_to_pages_for_size_pool(objspace, size_pool, slots);
11413 }
11414 else {
11415 /* We already have more slots than size_pool_init_slots allows, so
11416 * prevent creating more pages. */
11417 size_pool->allocatable_pages = 0;
11418 }
11419 }
11420 heap_pages_expand_sorted(objspace);
11421 }
11422
11423 /*
11424 * GC tuning environment variables
11425 *
11426 * * RUBY_GC_HEAP_FREE_SLOTS
11427 * - Prepare at least this amount of slots after GC.
11428 * - Allocate slots if there are not enough slots.
11429 * * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
11430 * - Allocate slots by this factor.
11431 * - (next slots number) = (current slots number) * (this factor)
11432 * * RUBY_GC_HEAP_GROWTH_MAX_SLOTS (new from 2.1)
11433 * - Allocation rate is limited to this number of slots.
11434 * * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
11435 * - Allocate additional pages when the number of free slots is
11436 * lower than the value (total_slots * (this ratio)).
11437 * * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4)
11438 * - Allocate slots to satisfy this formula:
11439 * free_slots = total_slots * goal_ratio
11440 * - In other words, prepare (total_slots * goal_ratio) free slots.
11441 * - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
11442 * * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4)
11443 * - Allow to free pages when the number of free slots is
11444 * greater than the value (total_slots * (this ratio)).
11445 * * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1)
11446 * - Do full GC when the number of old objects is more than R * N
11447 * where R is this factor and
11448 * N is the number of old objects just after last full GC.
11449 *
11450 * * obsolete
11451 * * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
11452 * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1)
11453 *
11454 * * RUBY_GC_MALLOC_LIMIT
11455 * * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
11456 * * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
11457 *
11458 * * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1)
11459 * * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1)
11460 * * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
11461 */
11462
11463 void
11464 ruby_gc_set_params(void)
11465 {
11466 rb_objspace_t *objspace = &rb_objspace;
11467 /* RUBY_GC_HEAP_FREE_SLOTS */
11468 if (get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0)) {
11469 /* ok */
11470 }
11471
11472 gc_set_initial_pages(objspace);
11473
11474 get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
11475 get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0);
11476 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
11477 0.0, 1.0, FALSE);
11478 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
11479 gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
11480 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
11481 gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
11482 get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
11483 get_envparam_double("RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
11484
11485 if (get_envparam_size("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
11486 malloc_limit = gc_params.malloc_limit_min;
11487 }
11488 get_envparam_size ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
11489 if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */
11490 gc_params.malloc_limit_max = SIZE_MAX;
11491 }
11492 get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
11493
11494 #if RGENGC_ESTIMATE_OLDMALLOC
11495 if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
11496 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
11497 }
11498 get_envparam_size ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
11499 get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
11500 #endif
11501 }
11502
11503 static void
11504 reachable_objects_from_callback(VALUE obj)
11505 {
11506 rb_ractor_t *cr = GET_RACTOR();
11507 cr->mfd->mark_func(obj, cr->mfd->data);
11508 }
11509
11510 void
11511 rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data)
11512 {
11513 rb_objspace_t *objspace = &rb_objspace;
11514
11515 RB_VM_LOCK_ENTER();
11516 {
11517 if (during_gc) rb_bug("rb_objspace_reachable_objects_from() is not supported while during_gc == true");
11518
11519 if (is_markable_object(obj)) {
11520 rb_ractor_t *cr = GET_RACTOR();
11521 struct gc_mark_func_data_struct mfd = {
11522 .mark_func = func,
11523 .data = data,
11524 }, *prev_mfd = cr->mfd;
11525
11526 cr->mfd = &mfd;
11527 gc_mark_children(objspace, obj);
11528 cr->mfd = prev_mfd;
11529 }
11530 }
11531 RB_VM_LOCK_LEAVE();
11532 }
11533
11534 struct root_objects_data {
11535 const char *category;
11536 void (*func)(const char *category, VALUE, void *);
11537 void *data;
11538 };
11539
11540 static void
11541 root_objects_from(VALUE obj, void *ptr)
11542 {
11543 const struct root_objects_data *data = (struct root_objects_data *)ptr;
11544 (*data->func)(data->category, obj, data->data);
11545 }
11546
11547 void
11548 rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data)
11549 {
11550 rb_objspace_t *objspace = &rb_objspace;
11551 objspace_reachable_objects_from_root(objspace, func, passing_data);
11552 }
11553
11554 static void
11555 objspace_reachable_objects_from_root(rb_objspace_t *objspace, void (func)(const char *category, VALUE, void *), void *passing_data)
11556 {
11557 if (during_gc) rb_bug("objspace_reachable_objects_from_root() is not supported while during_gc == true");
11558
11559 rb_ractor_t *cr = GET_RACTOR();
11560 struct root_objects_data data = {
11561 .func = func,
11562 .data = passing_data,
11563 };
11564 struct gc_mark_func_data_struct mfd = {
11565 .mark_func = root_objects_from,
11566 .data = &data,
11567 }, *prev_mfd = cr->mfd;
11568
11569 cr->mfd = &mfd;
11570 gc_mark_roots(objspace, &data.category);
11571 cr->mfd = prev_mfd;
11572 }
11573
11574 /*
11575 ------------------------ Extended allocator ------------------------
11576 */
11577
11578 struct gc_raise_tag {
11579 VALUE exc;
11580 const char *fmt;
11581 va_list *ap;
11582 };
11583
11584 static void *
11585 gc_vraise(void *ptr)
11586 {
11587 struct gc_raise_tag *argv = ptr;
11588 rb_vraise(argv->exc, argv->fmt, *argv->ap);
11589 UNREACHABLE_RETURN(NULL);
11590 }
11591
11592 static void
11593 gc_raise(VALUE exc, const char *fmt, ...)
11594 {
11595 va_list ap;
11596 va_start(ap, fmt);
11597 struct gc_raise_tag argv = {
11598 exc, fmt, &ap,
11599 };
11600
11601 if (ruby_thread_has_gvl_p()) {
11602 gc_vraise(&argv);
11603 UNREACHABLE;
11604 }
11605 else if (ruby_native_thread_p()) {
11606 rb_thread_call_with_gvl(gc_vraise, &argv);
11607 UNREACHABLE;
11608 }
11609 else {
11610 /* Not in a ruby thread */
11611 fprintf(stderr, "%s", "[FATAL] ");
11612 vfprintf(stderr, fmt, ap);
11613 }
11614
11615 va_end(ap);
11616 abort();
11617 }
11618
11619 static void objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t size);
11620
11621 static void
11622 negative_size_allocation_error(const char *msg)
11623 {
11624 gc_raise(rb_eNoMemError, "%s", msg);
11625 }
11626
11627 static void *
11628 ruby_memerror_body(void *dummy)
11629 {
11630 rb_memerror();
11631 return 0;
11632 }
11633
11634 NORETURN(static void ruby_memerror(void));
11635 RBIMPL_ATTR_MAYBE_UNUSED()
11636 static void
11637 ruby_memerror(void)
11638 {
11639 if (ruby_thread_has_gvl_p()) {
11640 rb_memerror();
11641 }
11642 else {
11643 if (ruby_native_thread_p()) {
11644 rb_thread_call_with_gvl(ruby_memerror_body, 0);
11645 }
11646 else {
11647 /* no ruby thread */
11648 fprintf(stderr, "[FATAL] failed to allocate memory\n");
11649 }
11650 }
11651 exit(EXIT_FAILURE);
11652 }
11653
11654 void
11655 rb_memerror(void)
11656 {
11657 rb_execution_context_t *ec = GET_EC();
11658 rb_objspace_t *objspace = rb_objspace_of(rb_ec_vm_ptr(ec));
11659 VALUE exc;
11660
11661 if (0) {
11662 // Print out pid, sleep, so you can attach debugger to see what went wrong:
11663 fprintf(stderr, "rb_memerror pid=%"PRI_PIDT_PREFIX"d\n", getpid());
11664 sleep(60);
11665 }
11666
11667 if (during_gc) {
11668 // TODO: OMG!! How to implement it?
11669 gc_exit(objspace, gc_enter_event_rb_memerror, NULL);
11670 }
11671
11672 exc = nomem_error;
11673 if (!exc ||
11674 rb_ec_raised_p(ec, RAISED_NOMEMORY)) {
11675 fprintf(stderr, "[FATAL] failed to allocate memory\n");
11676 exit(EXIT_FAILURE);
11677 }
11678 if (rb_ec_raised_p(ec, RAISED_NOMEMORY)) {
11679 rb_ec_raised_clear(ec);
11680 }
11681 else {
11682 rb_ec_raised_set(ec, RAISED_NOMEMORY);
11683 exc = ruby_vm_special_exception_copy(exc);
11684 }
11685 ec->errinfo = exc;
11686 EC_JUMP_TAG(ec, TAG_RAISE);
11687 }
11688
11689 static void
11690 rb_aligned_free(void *ptr, size_t size)
11691 {
11692 #if defined __MINGW32__
11693 __mingw_aligned_free(ptr);
11694 #elif defined _WIN32
11695 _aligned_free(ptr);
11696 #elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
11697 free(ptr);
11698 #else
11699 free(((void**)ptr)[-1]);
11700 #endif
11701 }
11702
11703 static inline size_t
11704 objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint)
11705 {
11706 #ifdef HAVE_MALLOC_USABLE_SIZE
11707 return malloc_usable_size(ptr);
11708 #else
11709 return hint;
11710 #endif
11711 }
11712
11713 enum memop_type {
11714 MEMOP_TYPE_MALLOC = 0,
11715 MEMOP_TYPE_FREE,
11716 MEMOP_TYPE_REALLOC
11717 };
11718
11719 static inline void
11720 atomic_sub_nounderflow(size_t *var, size_t sub)
11721 {
11722 if (sub == 0) return;
11723
11724 while (1) {
11725 size_t val = *var;
11726 if (val < sub) sub = val;
11727 if (ATOMIC_SIZE_CAS(*var, val, val-sub) == val) break;
11728 }
11729 }
11730
11731 static void
11732 objspace_malloc_gc_stress(rb_objspace_t *objspace)
11733 {
11734 if (ruby_gc_stressful && ruby_native_thread_p()) {
11735 unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
11736 GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
11737
11738 if (gc_stress_full_mark_after_malloc_p()) {
11739 reason |= GPR_FLAG_FULL_MARK;
11740 }
11741 garbage_collect_with_gvl(objspace, reason);
11742 }
11743 }
11744
11745 static inline bool
11746 objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
11747 {
11748 if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n",
11749 mem,
11750 type == MEMOP_TYPE_MALLOC ? "malloc" :
11751 type == MEMOP_TYPE_FREE ? "free " :
11752 type == MEMOP_TYPE_REALLOC ? "realloc": "error",
11753 new_size, old_size);
11754 return false;
11755 }
11756
11757 static bool
11758 objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
11759 {
11760 if (new_size > old_size) {
11761 ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size);
11762 #if RGENGC_ESTIMATE_OLDMALLOC
11763 ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size);
11764 #endif
11765 }
11766 else {
11767 atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
11768 #if RGENGC_ESTIMATE_OLDMALLOC
11769 atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size);
11770 #endif
11771 }
11772
11773 if (type == MEMOP_TYPE_MALLOC) {
11774 retry:
11775 if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) {
11776 if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) {
11777 gc_rest(objspace); /* gc_rest can reduce malloc_increase */
11778 goto retry;
11779 }
11780 garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
11781 }
11782 }
11783
11784 #if MALLOC_ALLOCATED_SIZE
11785 if (new_size >= old_size) {
11786 ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size);
11787 }
11788 else {
11789 size_t dec_size = old_size - new_size;
11790 size_t allocated_size = objspace->malloc_params.allocated_size;
11791
11792 #if MALLOC_ALLOCATED_SIZE_CHECK
11793 if (allocated_size < dec_size) {
11794 rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size.");
11795 }
11796 #endif
11797 atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size);
11798 }
11799
11800 switch (type) {
11801 case MEMOP_TYPE_MALLOC:
11802 ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
11803 break;
11804 case MEMOP_TYPE_FREE:
11805 {
11806 size_t allocations = objspace->malloc_params.allocations;
11807 if (allocations > 0) {
11808 atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1);
11809 }
11810 #if MALLOC_ALLOCATED_SIZE_CHECK
11811 else {
11812 GC_ASSERT(objspace->malloc_params.allocations > 0);
11813 }
11814 #endif
11815 }
11816 break;
11817 case MEMOP_TYPE_REALLOC: /* ignore */ break;
11818 }
11819 #endif
11820 return true;
11821 }
11822
11823 #define objspace_malloc_increase(...) \
11824 for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
11825 !malloc_increase_done; \
11826 malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
11827
11828 struct malloc_obj_info { /* 4 words */
11829 size_t size;
11830 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11831 size_t gen;
11832 const char *file;
11833 size_t line;
11834 #endif
11835 };
11836
11837 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11838 const char *ruby_malloc_info_file;
11839 int ruby_malloc_info_line;
11840 #endif
11841
11842 static inline size_t
11843 objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
11844 {
11845 if (size == 0) size = 1;
11846
11847 #if CALC_EXACT_MALLOC_SIZE
11848 size += sizeof(struct malloc_obj_info);
11849 #endif
11850
11851 return size;
11852 }
11853
11854 static bool
11855 malloc_during_gc_p(rb_objspace_t *objspace)
11856 {
11857 /* malloc is not allowed during GC when we're not using multiple ractors
11858 * (since ractors can run while another thread is sweeping) and when we
11859 * have the GVL (since if we don't have the GVL, we'll try to acquire the
11860 * GVL which will block and ensure the other thread finishes GC). */
11861 return during_gc && !dont_gc_val() && !rb_multi_ractor_p() && ruby_thread_has_gvl_p();
11862 }
11863
11864 static inline void *
11865 objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
11866 {
11867 size = objspace_malloc_size(objspace, mem, size);
11868 objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC) {}
11869
11870 #if CALC_EXACT_MALLOC_SIZE
11871 {
11872 struct malloc_obj_info *info = mem;
11873 info->size = size;
11874 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11875 info->gen = objspace->profile.count;
11876 info->file = ruby_malloc_info_file;
11877 info->line = info->file ? ruby_malloc_info_line : 0;
11878 #endif
11879 mem = info + 1;
11880 }
11881 #endif
11882
11883 return mem;
11884 }
11885
11886 #if defined(__GNUC__) && RUBY_DEBUG
11887 #define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
11888 #endif
11889
11890 #ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
11891 # define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
11892 #endif
11893
11894 #define GC_MEMERROR(...) \
11895 ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : rb_memerror())
11896
11897 #define TRY_WITH_GC(siz, expr) do { \
11898 const gc_profile_record_flag gpr = \
11899 GPR_FLAG_FULL_MARK | \
11900 GPR_FLAG_IMMEDIATE_MARK | \
11901 GPR_FLAG_IMMEDIATE_SWEEP | \
11902 GPR_FLAG_MALLOC; \
11903 objspace_malloc_gc_stress(objspace); \
11904 \
11905 if (LIKELY((expr))) { \
11906 /* Success on 1st try */ \
11907 } \
11908 else if (!garbage_collect_with_gvl(objspace, gpr)) { \
11909 /* @shyouhei thinks this doesn't happen */ \
11910 GC_MEMERROR("TRY_WITH_GC: could not GC"); \
11911 } \
11912 else if ((expr)) { \
11913 /* Success on 2nd try */ \
11914 } \
11915 else { \
11916 GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
11917 "%"PRIdSIZE" bytes for %s", \
11918 siz, # expr); \
11919 } \
11920 } while (0)
11921
11922 static void
11923 check_malloc_not_in_gc(rb_objspace_t *objspace, const char *msg)
11924 {
11925 if (UNLIKELY(malloc_during_gc_p(objspace))) {
11926 dont_gc_on();
11927 during_gc = false;
11928 rb_bug("Cannot %s during GC", msg);
11929 }
11930 }
11931
11932 /* these shouldn't be called directly.
11933 * objspace_* functions do not check allocation size.
11934 */
11935 static void *
11936 objspace_xmalloc0(rb_objspace_t *objspace, size_t size)
11937 {
11938 check_malloc_not_in_gc(objspace, "malloc");
11939
11940 void *mem;
11941
11942 size = objspace_malloc_prepare(objspace, size);
11943 TRY_WITH_GC(size, mem = malloc(size));
11944 RB_DEBUG_COUNTER_INC(heap_xmalloc);
11945 return objspace_malloc_fixup(objspace, mem, size);
11946 }
11947
11948 static inline size_t
11949 xmalloc2_size(const size_t count, const size_t elsize)
11950 {
11951 return size_mul_or_raise(count, elsize, rb_eArgError);
11952 }
11953
11954 static void *
11955 objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t old_size)
11956 {
11957 check_malloc_not_in_gc(objspace, "realloc");
11958
11959 void *mem;
11960
11961 if (!ptr) return objspace_xmalloc0(objspace, new_size);
11962
11963 /*
11964 * The behavior of realloc(ptr, 0) is implementation defined.
11965 * Therefore we don't use realloc(ptr, 0) for portability reason.
11966 * see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
11967 */
11968 if (new_size == 0) {
11969 if ((mem = objspace_xmalloc0(objspace, 0)) != NULL) {
11970 /*
11971 * - OpenBSD's malloc(3) man page says that when 0 is passed, it
11972 * returns a non-NULL pointer to an access-protected memory page.
11973 * The returned pointer cannot be read / written at all, but
11974 * still be a valid argument of free().
11975 *
11976 * https://man.openbsd.org/malloc.3
11977 *
11978 * - Linux's malloc(3) man page says that it _might_ perhaps return
11979 * a non-NULL pointer when its argument is 0. That return value
11980 * is safe (and is expected) to be passed to free().
11981 *
11982 * https://man7.org/linux/man-pages/man3/malloc.3.html
11983 *
11984 * - As I read the implementation jemalloc's malloc() returns fully
11985 * normal 16 bytes memory region when its argument is 0.
11986 *
11987 * - As I read the implementation musl libc's malloc() returns
11988 * fully normal 32 bytes memory region when its argument is 0.
11989 *
11990 * - Other malloc implementations can also return non-NULL.
11991 */
11992 objspace_xfree(objspace, ptr, old_size);
11993 return mem;
11994 }
11995 else {
11996 /*
11997 * It is dangerous to return NULL here, because that could lead to
11998 * RCE. Fallback to 1 byte instead of zero.
11999 *
12000 * https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932
12001 */
12002 new_size = 1;
12003 }
12004 }
12005
12006 #if CALC_EXACT_MALLOC_SIZE
12007 {
12008 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
12009 new_size += sizeof(struct malloc_obj_info);
12010 ptr = info;
12011 old_size = info->size;
12012 }
12013 #endif
12014
12015 old_size = objspace_malloc_size(objspace, ptr, old_size);
12016 TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size)));
12017 new_size = objspace_malloc_size(objspace, mem, new_size);
12018
12019 #if CALC_EXACT_MALLOC_SIZE
12020 {
12021 struct malloc_obj_info *info = mem;
12022 info->size = new_size;
12023 mem = info + 1;
12024 }
12025 #endif
12026
12027 objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
12028
12029 RB_DEBUG_COUNTER_INC(heap_xrealloc);
12030 return mem;
12031 }
12032
12033 #if CALC_EXACT_MALLOC_SIZE && USE_GC_MALLOC_OBJ_INFO_DETAILS
12034
12035 #define MALLOC_INFO_GEN_SIZE 100
12036 #define MALLOC_INFO_SIZE_SIZE 10
12037 static size_t malloc_info_gen_cnt[MALLOC_INFO_GEN_SIZE];
12038 static size_t malloc_info_gen_size[MALLOC_INFO_GEN_SIZE];
12039 static size_t malloc_info_size[MALLOC_INFO_SIZE_SIZE+1];
12040 static st_table *malloc_info_file_table;
12041
12042 static int
12043 mmalloc_info_file_i(st_data_t key, st_data_t val, st_data_t dmy)
12044 {
12045 const char *file = (void *)key;
12046 const size_t *data = (void *)val;
12047
12048 fprintf(stderr, "%s\t%"PRIdSIZE"\t%"PRIdSIZE"\n", file, data[0], data[1]);
12049
12050 return ST_CONTINUE;
12051 }
12052
12053 __attribute__((destructor))
12054 void
12055 rb_malloc_info_show_results(void)
12056 {
12057 int i;
12058
12059 fprintf(stderr, "* malloc_info gen statistics\n");
12060 for (i=0; i<MALLOC_INFO_GEN_SIZE; i++) {
12061 if (i == MALLOC_INFO_GEN_SIZE-1) {
12062 fprintf(stderr, "more\t%"PRIdSIZE"\t%"PRIdSIZE"\n", malloc_info_gen_cnt[i], malloc_info_gen_size[i]);
12063 }
12064 else {
12065 fprintf(stderr, "%d\t%"PRIdSIZE"\t%"PRIdSIZE"\n", i, malloc_info_gen_cnt[i], malloc_info_gen_size[i]);
12066 }
12067 }
12068
12069 fprintf(stderr, "* malloc_info size statistics\n");
12070 for (i=0; i<MALLOC_INFO_SIZE_SIZE; i++) {
12071 int s = 16 << i;
12072 fprintf(stderr, "%d\t%"PRIdSIZE"\n", s, malloc_info_size[i]);
12073 }
12074 fprintf(stderr, "more\t%"PRIdSIZE"\n", malloc_info_size[i]);
12075
12076 if (malloc_info_file_table) {
12077 fprintf(stderr, "* malloc_info file statistics\n");
12078 st_foreach(malloc_info_file_table, mmalloc_info_file_i, 0);
12079 }
12080 }
12081 #else
12082 void
12083 rb_malloc_info_show_results(void)
12084 {
12085 }
12086 #endif
12087
12088 static void
12089 objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t old_size)
12090 {
12091 if (!ptr) {
12092 /*
12093 * ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since
12094 * its first version. We would better follow.
12095 */
12096 return;
12097 }
12098 #if CALC_EXACT_MALLOC_SIZE
12099 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
12100 ptr = info;
12101 old_size = info->size;
12102
12103 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
12104 {
12105 int gen = (int)(objspace->profile.count - info->gen);
12106 int gen_index = gen >= MALLOC_INFO_GEN_SIZE ? MALLOC_INFO_GEN_SIZE-1 : gen;
12107 int i;
12108
12109 malloc_info_gen_cnt[gen_index]++;
12110 malloc_info_gen_size[gen_index] += info->size;
12111
12112 for (i=0; i<MALLOC_INFO_SIZE_SIZE; i++) {
12113 size_t s = 16 << i;
12114 if (info->size <= s) {
12115 malloc_info_size[i]++;
12116 goto found;
12117 }
12118 }
12119 malloc_info_size[i]++;
12120 found:;
12121
12122 {
12123 st_data_t key = (st_data_t)info->file, d;
12124 size_t *data;
12125
12126 if (malloc_info_file_table == NULL) {
12127 malloc_info_file_table = st_init_numtable_with_size(1024);
12128 }
12129 if (st_lookup(malloc_info_file_table, key, &d)) {
12130 /* hit */
12131 data = (size_t *)d;
12132 }
12133 else {
12134 data = malloc(xmalloc2_size(2, sizeof(size_t)));
12135 if (data == NULL) rb_bug("objspace_xfree: can not allocate memory");
12136 data[0] = data[1] = 0;
12137 st_insert(malloc_info_file_table, key, (st_data_t)data);
12138 }
12139 data[0] ++;
12140 data[1] += info->size;
12141 };
12142 if (0 && gen >= 2) { /* verbose output */
12143 if (info->file) {
12144 fprintf(stderr, "free - size:%"PRIdSIZE", gen:%d, pos: %s:%"PRIdSIZE"\n",
12145 info->size, gen, info->file, info->line);
12146 }
12147 else {
12148 fprintf(stderr, "free - size:%"PRIdSIZE", gen:%d\n",
12149 info->size, gen);
12150 }
12151 }
12152 }
12153 #endif
12154 #endif
12155 old_size = objspace_malloc_size(objspace, ptr, old_size);
12156
12157 objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE) {
12158 free(ptr);
12159 ptr = NULL;
12160 RB_DEBUG_COUNTER_INC(heap_xfree);
12161 }
12162 }
12163
12164 static void *
12165 ruby_xmalloc0(size_t size)
12166 {
12167 return objspace_xmalloc0(&rb_objspace, size);
12168 }
12169
12170 void *
12171 ruby_xmalloc_body(size_t size)
12172 {
12173 if ((ssize_t)size < 0) {
12174 negative_size_allocation_error("too large allocation size");
12175 }
12176 return ruby_xmalloc0(size);
12177 }
12178
12179 void
12180 ruby_malloc_size_overflow(size_t count, size_t elsize)
12181 {
12182 rb_raise(rb_eArgError,
12183 "malloc: possible integer overflow (%"PRIuSIZE"*%"PRIuSIZE")",
12184 count, elsize);
12185 }
12186
12187 void *
12188 ruby_xmalloc2_body(size_t n, size_t size)
12189 {
12190 return objspace_xmalloc0(&rb_objspace, xmalloc2_size(n, size));
12191 }
12192
12193 static void *
12194 objspace_xcalloc(rb_objspace_t *objspace, size_t size)
12195 {
12196 if (UNLIKELY(malloc_during_gc_p(objspace))) {
12197 rb_warn("calloc during GC detected, this could cause crashes if it triggers another GC");
12198 #if RGENGC_CHECK_MODE || RUBY_DEBUG
12199 rb_bug("Cannot calloc during GC");
12200 #endif
12201 }
12202
12203 void *mem;
12204
12205 size = objspace_malloc_prepare(objspace, size);
12206 TRY_WITH_GC(size, mem = calloc1(size));
12207 return objspace_malloc_fixup(objspace, mem, size);
12208 }
12209
12210 void *
12211 ruby_xcalloc_body(size_t n, size_t size)
12212 {
12213 return objspace_xcalloc(&rb_objspace, xmalloc2_size(n, size));
12214 }
12215
12216 #ifdef ruby_sized_xrealloc
12217 #undef ruby_sized_xrealloc
12218 #endif
12219 void *
12220 ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size)
12221 {
12222 if ((ssize_t)new_size < 0) {
12223 negative_size_allocation_error("too large allocation size");
12224 }
12225
12226 return objspace_xrealloc(&rb_objspace, ptr, new_size, old_size);
12227 }
12228
12229 void *
12230 ruby_xrealloc_body(void *ptr, size_t new_size)
12231 {
12232 return ruby_sized_xrealloc(ptr, new_size, 0);
12233 }
12234
12235 #ifdef ruby_sized_xrealloc2
12236 #undef ruby_sized_xrealloc2
12237 #endif
12238 void *
12239 ruby_sized_xrealloc2(void *ptr, size_t n, size_t size, size_t old_n)
12240 {
12241 size_t len = xmalloc2_size(n, size);
12242 return objspace_xrealloc(&rb_objspace, ptr, len, old_n * size);
12243 }
12244
12245 void *
12246 ruby_xrealloc2_body(void *ptr, size_t n, size_t size)
12247 {
12248 return ruby_sized_xrealloc2(ptr, n, size, 0);
12249 }
12250
12251 #ifdef ruby_sized_xfree
12252 #undef ruby_sized_xfree
12253 #endif
12254 void
12255 ruby_sized_xfree(void *x, size_t size)
12256 {
12257 if (LIKELY(x)) {
12258 /* It's possible for a C extension's pthread destructor function set by pthread_key_create
12259 * to be called after ruby_vm_destruct and attempt to free memory. Fall back to mimfree in
12260 * that case. */
12261 if (LIKELY(GET_VM())) {
12262 objspace_xfree(&rb_objspace, x, size);
12263 }
12264 else {
12265 ruby_mimfree(x);
12266 }
12267 }
12268 }
12269
12270 void
12271 ruby_xfree(void *x)
12272 {
12273 ruby_sized_xfree(x, 0);
12274 }
12275
12276 void *
12277 rb_xmalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
12278 {
12279 size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
12280 return ruby_xmalloc(w);
12281 }
12282
12283 void *
12284 rb_xcalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
12285 {
12286 size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
12287 return ruby_xcalloc(w, 1);
12288 }
12289
12290 void *
12291 rb_xrealloc_mul_add(const void *p, size_t x, size_t y, size_t z) /* x * y + z */
12292 {
12293 size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
12294 return ruby_xrealloc((void *)p, w);
12295 }
12296
12297 void *
12298 rb_xmalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
12299 {
12300 size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
12301 return ruby_xmalloc(u);
12302 }
12303
12304 void *
12305 rb_xcalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
12306 {
12307 size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
12308 return ruby_xcalloc(u, 1);
12309 }
12310
12311 /* Mimic ruby_xmalloc, but need not rb_objspace.
12312 * should return pointer suitable for ruby_xfree
12313 */
12314 void *
12315 ruby_mimmalloc(size_t size)
12316 {
12317 void *mem;
12318 #if CALC_EXACT_MALLOC_SIZE
12319 size += sizeof(struct malloc_obj_info);
12320 #endif
12321 mem = malloc(size);
12322 #if CALC_EXACT_MALLOC_SIZE
12323 if (!mem) {
12324 return NULL;
12325 }
12326 else
12327 /* set 0 for consistency of allocated_size/allocations */
12328 {
12329 struct malloc_obj_info *info = mem;
12330 info->size = 0;
12331 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
12332 info->gen = 0;
12333 info->file = NULL;
12334 info->line = 0;
12335 #endif
12336 mem = info + 1;
12337 }
12338 #endif
12339 return mem;
12340 }
12341
12342 void *
12343 ruby_mimcalloc(size_t num, size_t size)
12344 {
12345 void *mem;
12346 #if CALC_EXACT_MALLOC_SIZE
12347 struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(num, size);
12348 if (UNLIKELY(t.left)) {
12349 return NULL;
12350 }
12351 size = t.right + sizeof(struct malloc_obj_info);
12352 mem = calloc1(size);
12353 if (!mem) {
12354 return NULL;
12355 }
12356 else
12357 /* set 0 for consistency of allocated_size/allocations */
12358 {
12359 struct malloc_obj_info *info = mem;
12360 info->size = 0;
12361 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
12362 info->gen = 0;
12363 info->file = NULL;
12364 info->line = 0;
12365 #endif
12366 mem = info + 1;
12367 }
12368 #else
12369 mem = calloc(num, size);
12370 #endif
12371 return mem;
12372 }
12373
12374 void
12375 ruby_mimfree(void *ptr)
12376 {
12377 #if CALC_EXACT_MALLOC_SIZE
12378 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
12379 ptr = info;
12380 #endif
12381 free(ptr);
12382 }
12383
12384 #if MALLOC_ALLOCATED_SIZE
12385 /*
12386 * call-seq:
12387 * GC.malloc_allocated_size -> Integer
12388 *
12389 * Returns the size of memory allocated by malloc().
12390 *
12391 * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
12392 */
12393
12394 static VALUE
12395 gc_malloc_allocated_size(VALUE self)
12396 {
12397 return UINT2NUM(rb_objspace.malloc_params.allocated_size);
12398 }
12399
12400 /*
12401 * call-seq:
12402 * GC.malloc_allocations -> Integer
12403 *
12404 * Returns the number of malloc() allocations.
12405 *
12406 * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
12407 */
12408
12409 static VALUE
12410 gc_malloc_allocations(VALUE self)
12411 {
12412 return UINT2NUM(rb_objspace.malloc_params.allocations);
12413 }
12414 #endif
12415
12416 void
12417 rb_gc_adjust_memory_usage(ssize_t diff)
12418 {
12419 unless_objspace(objspace) { return; }
12420
12421 if (diff > 0) {
12422 objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
12423 }
12424 else if (diff < 0) {
12425 objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
12426 }
12427 }
12428
12429 /*
12430 ------------------------------ GC profiler ------------------------------
12431 */
12432
12433 #define GC_PROFILE_RECORD_DEFAULT_SIZE 100
12434
12435 static bool
12436 current_process_time(struct timespec *ts)
12437 {
12438 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
12439 {
12440 static int try_clock_gettime = 1;
12441 if (try_clock_gettime && clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
12442 return true;
12443 }
12444 else {
12445 try_clock_gettime = 0;
12446 }
12447 }
12448 #endif
12449
12450 #ifdef RUSAGE_SELF
12451 {
12452 struct rusage usage;
12453 struct timeval time;
12454 if (getrusage(RUSAGE_SELF, &usage) == 0) {
12455 time = usage.ru_utime;
12456 ts->tv_sec = time.tv_sec;
12457 ts->tv_nsec = (int32_t)time.tv_usec * 1000;
12458 return true;
12459 }
12460 }
12461 #endif
12462
12463 #ifdef _WIN32
12464 {
12465 FILETIME creation_time, exit_time, kernel_time, user_time;
12466 ULARGE_INTEGER ui;
12467
12468 if (GetProcessTimes(GetCurrentProcess(),
12469 &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
12470 memcpy(&ui, &user_time, sizeof(FILETIME));
12471 #define PER100NSEC (uint64_t)(1000 * 1000 * 10)
12472 ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
12473 ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
12474 return true;
12475 }
12476 }
12477 #endif
12478
12479 return false;
12480 }
12481
12482 static double
12483 getrusage_time(void)
12484 {
12485 struct timespec ts;
12486 if (current_process_time(&ts)) {
12487 return ts.tv_sec + ts.tv_nsec * 1e-9;
12488 }
12489 else {
12490 return 0.0;
12491 }
12492 }
12493
12494
12495 static inline void
12496 gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason)
12497 {
12498 if (objspace->profile.run) {
12499 size_t index = objspace->profile.next_index;
12500 gc_profile_record *record;
12501
12502 /* create new record */
12503 objspace->profile.next_index++;
12504
12505 if (!objspace->profile.records) {
12506 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
12507 objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
12508 }
12509 if (index >= objspace->profile.size) {
12510 void *ptr;
12511 objspace->profile.size += 1000;
12512 ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
12513 if (!ptr) rb_memerror();
12514 objspace->profile.records = ptr;
12515 }
12516 if (!objspace->profile.records) {
12517 rb_bug("gc_profile malloc or realloc miss");
12518 }
12519 record = objspace->profile.current_record = &objspace->profile.records[objspace->profile.next_index - 1];
12520 MEMZERO(record, gc_profile_record, 1);
12521
12522 /* setup before-GC parameter */
12523 record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
12524 #if MALLOC_ALLOCATED_SIZE
12525 record->allocated_size = malloc_allocated_size;
12526 #endif
12527 #if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
12528 #ifdef RUSAGE_SELF
12529 {
12530 struct rusage usage;
12531 if (getrusage(RUSAGE_SELF, &usage) == 0) {
12532 record->maxrss = usage.ru_maxrss;
12533 record->minflt = usage.ru_minflt;
12534 record->majflt = usage.ru_majflt;
12535 }
12536 }
12537 #endif
12538 #endif
12539 }
12540 }
12541
12542 static inline void
12543 gc_prof_timer_start(rb_objspace_t *objspace)
12544 {
12545 if (gc_prof_enabled(objspace)) {
12546 gc_profile_record *record = gc_prof_record(objspace);
12547 #if GC_PROFILE_MORE_DETAIL
12548 record->prepare_time = objspace->profile.prepare_time;
12549 #endif
12550 record->gc_time = 0;
12551 record->gc_invoke_time = getrusage_time();
12552 }
12553 }
12554
12555 static double
12556 elapsed_time_from(double time)
12557 {
12558 double now = getrusage_time();
12559 if (now > time) {
12560 return now - time;
12561 }
12562 else {
12563 return 0;
12564 }
12565 }
12566
12567 static inline void
12568 gc_prof_timer_stop(rb_objspace_t *objspace)
12569 {
12570 if (gc_prof_enabled(objspace)) {
12571 gc_profile_record *record = gc_prof_record(objspace);
12572 record->gc_time = elapsed_time_from(record->gc_invoke_time);
12573 record->gc_invoke_time -= objspace->profile.invoke_time;
12574 }
12575 }
12576
12577 #define RUBY_DTRACE_GC_HOOK(name) \
12578 do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_##name();} while (0)
12579 static inline void
12580 gc_prof_mark_timer_start(rb_objspace_t *objspace)
12581 {
12582 RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
12583 #if GC_PROFILE_MORE_DETAIL
12584 if (gc_prof_enabled(objspace)) {
12585 gc_prof_record(objspace)->gc_mark_time = getrusage_time();
12586 }
12587 #endif
12588 }
12589
12590 static inline void
12591 gc_prof_mark_timer_stop(rb_objspace_t *objspace)
12592 {
12593 RUBY_DTRACE_GC_HOOK(MARK_END);
12594 #if GC_PROFILE_MORE_DETAIL
12595 if (gc_prof_enabled(objspace)) {
12596 gc_profile_record *record = gc_prof_record(objspace);
12597 record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
12598 }
12599 #endif
12600 }
12601
12602 static inline void
12603 gc_prof_sweep_timer_start(rb_objspace_t *objspace)
12604 {
12605 RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
12606 if (gc_prof_enabled(objspace)) {
12607 gc_profile_record *record = gc_prof_record(objspace);
12608
12609 if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
12610 objspace->profile.gc_sweep_start_time = getrusage_time();
12611 }
12612 }
12613 }
12614
12615 static inline void
12616 gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
12617 {
12618 RUBY_DTRACE_GC_HOOK(SWEEP_END);
12619
12620 if (gc_prof_enabled(objspace)) {
12621 double sweep_time;
12622 gc_profile_record *record = gc_prof_record(objspace);
12623
12624 if (record->gc_time > 0) {
12625 sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
12626 /* need to accumulate GC time for lazy sweep after gc() */
12627 record->gc_time += sweep_time;
12628 }
12629 else if (GC_PROFILE_MORE_DETAIL) {
12630 sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
12631 }
12632
12633 #if GC_PROFILE_MORE_DETAIL
12634 record->gc_sweep_time += sweep_time;
12635 if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
12636 #endif
12637 if (heap_pages_deferred_final) objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
12638 }
12639 }
12640
12641 static inline void
12642 gc_prof_set_malloc_info(rb_objspace_t *objspace)
12643 {
12644 #if GC_PROFILE_MORE_DETAIL
12645 if (gc_prof_enabled(objspace)) {
12646 gc_profile_record *record = gc_prof_record(objspace);
12647 record->allocate_increase = malloc_increase;
12648 record->allocate_limit = malloc_limit;
12649 }
12650 #endif
12651 }
12652
12653 static inline void
12654 gc_prof_set_heap_info(rb_objspace_t *objspace)
12655 {
12656 if (gc_prof_enabled(objspace)) {
12657 gc_profile_record *record = gc_prof_record(objspace);
12658 size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace);
12659 size_t total = objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT;
12660
12661 #if GC_PROFILE_MORE_DETAIL
12662 record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
12663 record->heap_live_objects = live;
12664 record->heap_free_objects = total - live;
12665 #endif
12666
12667 record->heap_total_objects = total;
12668 record->heap_use_size = live * sizeof(RVALUE);
12669 record->heap_total_size = total * sizeof(RVALUE);
12670 }
12671 }
12672
12673 /*
12674 * call-seq:
12675 * GC::Profiler.clear -> nil
12676 *
12677 * Clears the \GC profiler data.
12678 *
12679 */
12680
12681 static VALUE
12682 gc_profile_clear(VALUE _)
12683 {
12684 rb_objspace_t *objspace = &rb_objspace;
12685 void *p = objspace->profile.records;
12686 objspace->profile.records = NULL;
12687 objspace->profile.size = 0;
12688 objspace->profile.next_index = 0;
12689 objspace->profile.current_record = 0;
12690 free(p);
12691 return Qnil;
12692 }
12693
12694 /*
12695 * call-seq:
12696 * GC::Profiler.raw_data -> [Hash, ...]
12697 *
12698 * Returns an Array of individual raw profile data Hashes ordered
12699 * from earliest to latest by +:GC_INVOKE_TIME+.
12700 *
12701 * For example:
12702 *
12703 * [
12704 * {
12705 * :GC_TIME=>1.3000000000000858e-05,
12706 * :GC_INVOKE_TIME=>0.010634999999999999,
12707 * :HEAP_USE_SIZE=>289640,
12708 * :HEAP_TOTAL_SIZE=>588960,
12709 * :HEAP_TOTAL_OBJECTS=>14724,
12710 * :GC_IS_MARKED=>false
12711 * },
12712 * # ...
12713 * ]
12714 *
12715 * The keys mean:
12716 *
12717 * +:GC_TIME+::
12718 * Time elapsed in seconds for this GC run
12719 * +:GC_INVOKE_TIME+::
12720 * Time elapsed in seconds from startup to when the GC was invoked
12721 * +:HEAP_USE_SIZE+::
12722 * Total bytes of heap used
12723 * +:HEAP_TOTAL_SIZE+::
12724 * Total size of heap in bytes
12725 * +:HEAP_TOTAL_OBJECTS+::
12726 * Total number of objects
12727 * +:GC_IS_MARKED+::
12728 * Returns +true+ if the GC is in mark phase
12729 *
12730 * If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access
12731 * to the following hash keys:
12732 *
12733 * +:GC_MARK_TIME+::
12734 * +:GC_SWEEP_TIME+::
12735 * +:ALLOCATE_INCREASE+::
12736 * +:ALLOCATE_LIMIT+::
12737 * +:HEAP_USE_PAGES+::
12738 * +:HEAP_LIVE_OBJECTS+::
12739 * +:HEAP_FREE_OBJECTS+::
12740 * +:HAVE_FINALIZE+::
12741 *
12742 */
12743
12744 static VALUE
12745 gc_profile_record_get(VALUE _)
12746 {
12747 VALUE prof;
12748 VALUE gc_profile = rb_ary_new();
12749 size_t i;
12750 rb_objspace_t *objspace = (&rb_objspace);
12751
12752 if (!objspace->profile.run) {
12753 return Qnil;
12754 }
12755
12756 for (i =0; i < objspace->profile.next_index; i++) {
12757 gc_profile_record *record = &objspace->profile.records[i];
12758
12759 prof = rb_hash_new();
12760 rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(objspace, rb_hash_new(), record->flags));
12761 rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time));
12762 rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time));
12763 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
12764 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
12765 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
12766 rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects));
12767 rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
12768 #if GC_PROFILE_MORE_DETAIL
12769 rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
12770 rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time));
12771 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase));
12772 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit));
12773 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages));
12774 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects));
12775 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects));
12776
12777 rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects));
12778 rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects));
12779
12780 rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), RBOOL(record->flags & GPR_FLAG_HAVE_FINALIZE));
12781 #endif
12782
12783 #if RGENGC_PROFILE > 0
12784 rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects));
12785 rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects));
12786 rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects));
12787 #endif
12788 rb_ary_push(gc_profile, prof);
12789 }
12790
12791 return gc_profile;
12792 }
12793
12794 #if GC_PROFILE_MORE_DETAIL
12795 #define MAJOR_REASON_MAX 0x10
12796
12797 static char *
12798 gc_profile_dump_major_reason(unsigned int flags, char *buff)
12799 {
12800 unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
12801 int i = 0;
12802
12803 if (reason == GPR_FLAG_NONE) {
12804 buff[0] = '-';
12805 buff[1] = 0;
12806 }
12807 else {
12808 #define C(x, s) \
12809 if (reason & GPR_FLAG_MAJOR_BY_##x) { \
12810 buff[i++] = #x[0]; \
12811 if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
12812 buff[i] = 0; \
12813 }
12814 C(NOFREE, N);
12815 C(OLDGEN, O);
12816 C(SHADY, S);
12817 #if RGENGC_ESTIMATE_OLDMALLOC
12818 C(OLDMALLOC, M);
12819 #endif
12820 #undef C
12821 }
12822 return buff;
12823 }
12824 #endif
12825
12826 static void
12827 gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
12828 {
12829 rb_objspace_t *objspace = &rb_objspace;
12830 size_t count = objspace->profile.next_index;
12831 #ifdef MAJOR_REASON_MAX
12832 char reason_str[MAJOR_REASON_MAX];
12833 #endif
12834
12835 if (objspace->profile.run && count /* > 1 */) {
12836 size_t i;
12837 const gc_profile_record *record;
12838
12839 append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count));
12840 append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
12841
12842 for (i = 0; i < count; i++) {
12843 record = &objspace->profile.records[i];
12844 append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
12845 i+1, record->gc_invoke_time, record->heap_use_size,
12846 record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
12847 }
12848
12849 #if GC_PROFILE_MORE_DETAIL
12850 const char *str = "\n\n" \
12851 "More detail.\n" \
12852 "Prepare Time = Previously GC's rest sweep time\n"
12853 "Index Flags Allocate Inc. Allocate Limit"
12854 #if CALC_EXACT_MALLOC_SIZE
12855 " Allocated Size"
12856 #endif
12857 " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
12858 #if RGENGC_PROFILE
12859 " OldgenObj RemNormObj RemShadObj"
12860 #endif
12861 #if GC_PROFILE_DETAIL_MEMORY
12862 " MaxRSS(KB) MinorFLT MajorFLT"
12863 #endif
12864 "\n";
12865 append(out, rb_str_new_cstr(str));
12866
12867 for (i = 0; i < count; i++) {
12868 record = &objspace->profile.records[i];
12869 append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE
12870 #if CALC_EXACT_MALLOC_SIZE
12871 " %15"PRIuSIZE
12872 #endif
12873 " %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
12874 #if RGENGC_PROFILE
12875 "%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
12876 #endif
12877 #if GC_PROFILE_DETAIL_MEMORY
12878 "%11ld %8ld %8ld"
12879 #endif
12880
12881 "\n",
12882 i+1,
12883 gc_profile_dump_major_reason(record->flags, reason_str),
12884 (record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.',
12885 (record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" :
12886 (record->flags & GPR_FLAG_MALLOC) ? "MALLOC" :
12887 (record->flags & GPR_FLAG_METHOD) ? "METHOD" :
12888 (record->flags & GPR_FLAG_CAPI) ? "CAPI__" : "??????",
12889 (record->flags & GPR_FLAG_STRESS) ? '!' : ' ',
12890 record->allocate_increase, record->allocate_limit,
12891 #if CALC_EXACT_MALLOC_SIZE
12892 record->allocated_size,
12893 #endif
12894 record->heap_use_pages,
12895 record->gc_mark_time*1000,
12896 record->gc_sweep_time*1000,
12897 record->prepare_time*1000,
12898
12899 record->heap_live_objects,
12900 record->heap_free_objects,
12901 record->removing_objects,
12902 record->empty_objects
12903 #if RGENGC_PROFILE
12904 ,
12905 record->old_objects,
12906 record->remembered_normal_objects,
12907 record->remembered_shady_objects
12908 #endif
12909 #if GC_PROFILE_DETAIL_MEMORY
12910 ,
12911 record->maxrss / 1024,
12912 record->minflt,
12913 record->majflt
12914 #endif
12915
12916 ));
12917 }
12918 #endif
12919 }
12920 }
12921
12922 /*
12923 * call-seq:
12924 * GC::Profiler.result -> String
12925 *
12926 * Returns a profile data report such as:
12927 *
12928 * GC 1 invokes.
12929 * Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC time(ms)
12930 * 1 0.012 159240 212940 10647 0.00000000000001530000
12931 */
12932
12933 static VALUE
12934 gc_profile_result(VALUE _)
12935 {
12936 VALUE str = rb_str_buf_new(0);
12937 gc_profile_dump_on(str, rb_str_buf_append);
12938 return str;
12939 }
12940
12941 /*
12942 * call-seq:
12943 * GC::Profiler.report
12944 * GC::Profiler.report(io)
12945 *
12946 * Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object.
12947 *
12948 */
12949
12950 static VALUE
12951 gc_profile_report(int argc, VALUE *argv, VALUE self)
12952 {
12953 VALUE out;
12954
12955 out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]);
12956 gc_profile_dump_on(out, rb_io_write);
12957
12958 return Qnil;
12959 }
12960
12961 /*
12962 * call-seq:
12963 * GC::Profiler.total_time -> float
12964 *
12965 * The total time used for garbage collection in seconds
12966 */
12967
12968 static VALUE
12969 gc_profile_total_time(VALUE self)
12970 {
12971 double time = 0;
12972 rb_objspace_t *objspace = &rb_objspace;
12973
12974 if (objspace->profile.run && objspace->profile.next_index > 0) {
12975 size_t i;
12976 size_t count = objspace->profile.next_index;
12977
12978 for (i = 0; i < count; i++) {
12979 time += objspace->profile.records[i].gc_time;
12980 }
12981 }
12982 return DBL2NUM(time);
12983 }
12984
12985 /*
12986 * call-seq:
12987 * GC::Profiler.enabled? -> true or false
12988 *
12989 * The current status of \GC profile mode.
12990 */
12991
12992 static VALUE
12993 gc_profile_enable_get(VALUE self)
12994 {
12995 rb_objspace_t *objspace = &rb_objspace;
12996 return RBOOL(objspace->profile.run);
12997 }
12998
12999 /*
13000 * call-seq:
13001 * GC::Profiler.enable -> nil
13002 *
13003 * Starts the \GC profiler.
13004 *
13005 */
13006
13007 static VALUE
13008 gc_profile_enable(VALUE _)
13009 {
13010 rb_objspace_t *objspace = &rb_objspace;
13011 objspace->profile.run = TRUE;
13012 objspace->profile.current_record = 0;
13013 return Qnil;
13014 }
13015
13016 /*
13017 * call-seq:
13018 * GC::Profiler.disable -> nil
13019 *
13020 * Stops the \GC profiler.
13021 *
13022 */
13023
13024 static VALUE
13025 gc_profile_disable(VALUE _)
13026 {
13027 rb_objspace_t *objspace = &rb_objspace;
13028
13029 objspace->profile.run = FALSE;
13030 objspace->profile.current_record = 0;
13031 return Qnil;
13032 }
13033
13034 /*
13035 ------------------------------ DEBUG ------------------------------
13036 */
13037
13038 static const char *
13039 type_name(int type, VALUE obj)
13040 {
13041 switch (type) {
13042 #define TYPE_NAME(t) case (t): return #t;
13043 TYPE_NAME(T_NONE);
13044 TYPE_NAME(T_OBJECT);
13045 TYPE_NAME(T_CLASS);
13046 TYPE_NAME(T_MODULE);
13047 TYPE_NAME(T_FLOAT);
13048 TYPE_NAME(T_STRING);
13049 TYPE_NAME(T_REGEXP);
13050 TYPE_NAME(T_ARRAY);
13051 TYPE_NAME(T_HASH);
13052 TYPE_NAME(T_STRUCT);
13053 TYPE_NAME(T_BIGNUM);
13054 TYPE_NAME(T_FILE);
13055 TYPE_NAME(T_MATCH);
13056 TYPE_NAME(T_COMPLEX);
13057 TYPE_NAME(T_RATIONAL);
13058 TYPE_NAME(T_NIL);
13059 TYPE_NAME(T_TRUE);
13060 TYPE_NAME(T_FALSE);
13061 TYPE_NAME(T_SYMBOL);
13062 TYPE_NAME(T_FIXNUM);
13063 TYPE_NAME(T_UNDEF);
13064 TYPE_NAME(T_IMEMO);
13065 TYPE_NAME(T_ICLASS);
13066 TYPE_NAME(T_MOVED);
13067 TYPE_NAME(T_ZOMBIE);
13068 case T_DATA:
13069 if (obj && rb_objspace_data_type_name(obj)) {
13070 return rb_objspace_data_type_name(obj);
13071 }
13072 return "T_DATA";
13073 #undef TYPE_NAME
13074 }
13075 return "unknown";
13076 }
13077
13078 static const char *
13079 obj_type_name(VALUE obj)
13080 {
13081 return type_name(TYPE(obj), obj);
13082 }
13083
13084 const char *
13085 rb_method_type_name(rb_method_type_t type)
13086 {
13087 switch (type) {
13088 case VM_METHOD_TYPE_ISEQ: return "iseq";
13089 case VM_METHOD_TYPE_ATTRSET: return "attrest";
13090 case VM_METHOD_TYPE_IVAR: return "ivar";
13091 case VM_METHOD_TYPE_BMETHOD: return "bmethod";
13092 case VM_METHOD_TYPE_ALIAS: return "alias";
13093 case VM_METHOD_TYPE_REFINED: return "refined";
13094 case VM_METHOD_TYPE_CFUNC: return "cfunc";
13095 case VM_METHOD_TYPE_ZSUPER: return "zsuper";
13096 case VM_METHOD_TYPE_MISSING: return "missing";
13097 case VM_METHOD_TYPE_OPTIMIZED: return "optimized";
13098 case VM_METHOD_TYPE_UNDEF: return "undef";
13099 case VM_METHOD_TYPE_NOTIMPLEMENTED: return "notimplemented";
13100 }
13101 rb_bug("rb_method_type_name: unreachable (type: %d)", type);
13102 }
13103
13104 static void
13105 rb_raw_iseq_info(char *const buff, const size_t buff_size, const rb_iseq_t *iseq)
13106 {
13107 if (buff_size > 0 && ISEQ_BODY(iseq) && ISEQ_BODY(iseq)->location.label && !RB_TYPE_P(ISEQ_BODY(iseq)->location.pathobj, T_MOVED)) {
13108 VALUE path = rb_iseq_path(iseq);
13109 int n = ISEQ_BODY(iseq)->location.first_lineno;
13110 snprintf(buff, buff_size, " %s@%s:%d",
13111 RSTRING_PTR(ISEQ_BODY(iseq)->location.label),
13112 RSTRING_PTR(path), n);
13113 }
13114 }
13115
13116 static int
13117 str_len_no_raise(VALUE str)
13118 {
13119 long len = RSTRING_LEN(str);
13120 if (len < 0) return 0;
13121 if (len > INT_MAX) return INT_MAX;
13122 return (int)len;
13123 }
13124
13125 #define BUFF_ARGS buff + pos, buff_size - pos
13126 #define APPEND_F(...) if ((pos += snprintf(BUFF_ARGS, "" __VA_ARGS__)) >= buff_size) goto end
13127 #define APPEND_S(s) do { \
13128 if ((pos + (int)rb_strlen_lit(s)) >= buff_size) { \
13129 goto end; \
13130 } \
13131 else { \
13132 memcpy(buff + pos, (s), rb_strlen_lit(s) + 1); \
13133 } \
13134 } while (0)
13135 #define C(c, s) ((c) != 0 ? (s) : " ")
13136
13137 static size_t
13138 rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj)
13139 {
13140 size_t pos = 0;
13141
13142 if (SPECIAL_CONST_P(obj)) {
13143 APPEND_F("%s", obj_type_name(obj));
13144
13145 if (FIXNUM_P(obj)) {
13146 APPEND_F(" %ld", FIX2LONG(obj));
13147 }
13148 else if (SYMBOL_P(obj)) {
13149 APPEND_F(" %s", rb_id2name(SYM2ID(obj)));
13150 }
13151 }
13152 else {
13153 const int age = RVALUE_AGE_GET(obj);
13154
13155 if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
13156 APPEND_F("%p [%d%s%s%s%s%s%s] %s ",
13157 (void *)obj, age,
13158 C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
13159 C(RVALUE_MARK_BITMAP(obj), "M"),
13160 C(RVALUE_PIN_BITMAP(obj), "P"),
13161 C(RVALUE_MARKING_BITMAP(obj), "R"),
13162 C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
13163 C(rb_objspace_garbage_object_p(obj), "G"),
13164 obj_type_name(obj));
13165 }
13166 else {
13167 /* fake */
13168 APPEND_F("%p [%dXXXX] %s",
13169 (void *)obj, age,
13170 obj_type_name(obj));
13171 }
13172
13173 if (internal_object_p(obj)) {
13174 /* ignore */
13175 }
13176 else if (RBASIC(obj)->klass == 0) {
13177 APPEND_S("(temporary internal)");
13178 }
13179 else if (RTEST(RBASIC(obj)->klass)) {
13180 VALUE class_path = rb_class_path_cached(RBASIC(obj)->klass);
13181 if (!NIL_P(class_path)) {
13182 APPEND_F("(%s)", RSTRING_PTR(class_path));
13183 }
13184 }
13185
13186 #if GC_DEBUG
13187 APPEND_F("@%s:%d", GET_RVALUE_OVERHEAD(obj)->file, GET_RVALUE_OVERHEAD(obj)->line);
13188 #endif
13189 }
13190 end:
13191
13192 return pos;
13193 }
13194
13195 static size_t
13196 rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALUE obj, size_t pos)
13197 {
13198 if (LIKELY(pos < buff_size) && !SPECIAL_CONST_P(obj)) {
13199 const enum ruby_value_type type = BUILTIN_TYPE(obj);
13200
13201 switch (type) {
13202 case T_NODE:
13203 UNEXPECTED_NODE(rb_raw_obj_info);
13204 break;
13205 case T_ARRAY:
13206 if (ARY_SHARED_P(obj)) {
13207 APPEND_S("shared -> ");
13208 rb_raw_obj_info(BUFF_ARGS, ARY_SHARED_ROOT(obj));
13209 }
13210 else if (ARY_EMBED_P(obj)) {
13211 APPEND_F("[%s%s] len: %ld (embed)",
13212 C(ARY_EMBED_P(obj), "E"),
13213 C(ARY_SHARED_P(obj), "S"),
13214 RARRAY_LEN(obj));
13215 }
13216 else {
13217 APPEND_F("[%s%s] len: %ld, capa:%ld ptr:%p",
13218 C(ARY_EMBED_P(obj), "E"),
13219 C(ARY_SHARED_P(obj), "S"),
13220 RARRAY_LEN(obj),
13221 ARY_EMBED_P(obj) ? -1L : RARRAY(obj)->as.heap.aux.capa,
13222 (void *)RARRAY_CONST_PTR(obj));
13223 }
13224 break;
13225 case T_STRING: {
13226 if (STR_SHARED_P(obj)) {
13227 APPEND_F(" [shared] len: %ld", RSTRING_LEN(obj));
13228 }
13229 else {
13230 if (STR_EMBED_P(obj)) APPEND_S(" [embed]");
13231
13232 APPEND_F(" len: %ld, capa: %" PRIdSIZE, RSTRING_LEN(obj), rb_str_capacity(obj));
13233 }
13234 APPEND_F(" \"%.*s\"", str_len_no_raise(obj), RSTRING_PTR(obj));
13235 break;
13236 }
13237 case T_SYMBOL: {
13238 VALUE fstr = RSYMBOL(obj)->fstr;
13239 ID id = RSYMBOL(obj)->id;
13240 if (RB_TYPE_P(fstr, T_STRING)) {
13241 APPEND_F(":%s id:%d", RSTRING_PTR(fstr), (unsigned int)id);
13242 }
13243 else {
13244 APPEND_F("(%p) id:%d", (void *)fstr, (unsigned int)id);
13245 }
13246 break;
13247 }
13248 case T_MOVED: {
13249 APPEND_F("-> %p", (void*)rb_gc_location(obj));
13250 break;
13251 }
13252 case T_HASH: {
13253 APPEND_F("[%c] %"PRIdSIZE,
13254 RHASH_AR_TABLE_P(obj) ? 'A' : 'S',
13255 RHASH_SIZE(obj));
13256 break;
13257 }
13258 case T_CLASS:
13259 case T_MODULE:
13260 {
13261 VALUE class_path = rb_class_path_cached(obj);
13262 if (!NIL_P(class_path)) {
13263 APPEND_F("%s", RSTRING_PTR(class_path));
13264 }
13265 else {
13266 APPEND_S("(anon)");
13267 }
13268 break;
13269 }
13270 case T_ICLASS:
13271 {
13272 VALUE class_path = rb_class_path_cached(RBASIC_CLASS(obj));
13273 if (!NIL_P(class_path)) {
13274 APPEND_F("src:%s", RSTRING_PTR(class_path));
13275 }
13276 break;
13277 }
13278 case T_OBJECT:
13279 {
13280 if (rb_shape_obj_too_complex(obj)) {
13281 size_t hash_len = rb_st_table_size(ROBJECT_IV_HASH(obj));
13282 APPEND_F("(too_complex) len:%zu", hash_len);
13283 }
13284 else {
13285 uint32_t len = ROBJECT_IV_CAPACITY(obj);
13286
13287 if (RANY(obj)->as.basic.flags & ROBJECT_EMBED) {
13288 APPEND_F("(embed) len:%d", len);
13289 }
13290 else {
13291 VALUE *ptr = ROBJECT_IVPTR(obj);
13292 APPEND_F("len:%d ptr:%p", len, (void *)ptr);
13293 }
13294 }
13295 }
13296 break;
13297 case T_DATA: {
13298 const struct rb_block *block;
13299 const rb_iseq_t *iseq;
13300 if (rb_obj_is_proc(obj) &&
13301 (block = vm_proc_block(obj)) != NULL &&
13302 (vm_block_type(block) == block_type_iseq) &&
13303 (iseq = vm_block_iseq(block)) != NULL) {
13304 rb_raw_iseq_info(BUFF_ARGS, iseq);
13305 }
13306 else if (rb_ractor_p(obj)) {
13307 rb_ractor_t *r = (void *)DATA_PTR(obj);
13308 if (r) {
13309 APPEND_F("r:%d", r->pub.id);
13310 }
13311 }
13312 else {
13313 const char * const type_name = rb_objspace_data_type_name(obj);
13314 if (type_name) {
13315 APPEND_F("%s", type_name);
13316 }
13317 }
13318 break;
13319 }
13320 case T_IMEMO: {
13321 APPEND_F("<%s> ", rb_imemo_name(imemo_type(obj)));
13322
13323 switch (imemo_type(obj)) {
13324 case imemo_ment:
13325 {
13326 const rb_method_entry_t *me = &RANY(obj)->as.imemo.ment;
13327
13328 APPEND_F(":%s (%s%s%s%s) type:%s aliased:%d owner:%p defined_class:%p",
13329 rb_id2name(me->called_id),
13330 METHOD_ENTRY_VISI(me) == METHOD_VISI_PUBLIC ? "pub" :
13331 METHOD_ENTRY_VISI(me) == METHOD_VISI_PRIVATE ? "pri" : "pro",
13332 METHOD_ENTRY_COMPLEMENTED(me) ? ",cmp" : "",
13333 METHOD_ENTRY_CACHED(me) ? ",cc" : "",
13334 METHOD_ENTRY_INVALIDATED(me) ? ",inv" : "",
13335 me->def ? rb_method_type_name(me->def->type) : "NULL",
13336 me->def ? me->def->aliased : -1,
13337 (void *)me->owner, // obj_info(me->owner),
13338 (void *)me->defined_class); //obj_info(me->defined_class)));
13339
13340 if (me->def) {
13341 switch (me->def->type) {
13342 case VM_METHOD_TYPE_ISEQ:
13343 APPEND_S(" (iseq:");
13344 rb_raw_obj_info(BUFF_ARGS, (VALUE)me->def->body.iseq.iseqptr);
13345 APPEND_S(")");
13346 break;
13347 default:
13348 break;
13349 }
13350 }
13351
13352 break;
13353 }
13354 case imemo_iseq: {
13355 const rb_iseq_t *iseq = (const rb_iseq_t *)obj;
13356 rb_raw_iseq_info(BUFF_ARGS, iseq);
13357 break;
13358 }
13359 case imemo_callinfo:
13360 {
13361 const struct rb_callinfo *ci = (const struct rb_callinfo *)obj;
13362 APPEND_F("(mid:%s, flag:%x argc:%d, kwarg:%s)",
13363 rb_id2name(vm_ci_mid(ci)),
13364 vm_ci_flag(ci),
13365 vm_ci_argc(ci),
13366 vm_ci_kwarg(ci) ? "available" : "NULL");
13367 break;
13368 }
13369 case imemo_callcache:
13370 {
13371 const struct rb_callcache *cc = (const struct rb_callcache *)obj;
13372 VALUE class_path = cc->klass ? rb_class_path_cached(cc->klass) : Qnil;
13373 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
13374
13375 APPEND_F("(klass:%s cme:%s%s (%p) call:%p",
13376 NIL_P(class_path) ? (cc->klass ? "??" : "<NULL>") : RSTRING_PTR(class_path),
13377 cme ? rb_id2name(cme->called_id) : "<NULL>",
13378 cme ? (METHOD_ENTRY_INVALIDATED(cme) ? " [inv]" : "") : "",
13379 (void *)cme,
13380 (void *)vm_cc_call(cc));
13381 break;
13382 }
13383 default:
13384 break;
13385 }
13386 }
13387 default:
13388 break;
13389 }
13390 }
13391 end:
13392
13393 return pos;
13394 }
13395
13396 #undef C
13397
13398 const char *
13399 rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj)
13400 {
13401 asan_unpoisoning_object(obj) {
13402 size_t pos = rb_raw_obj_info_common(buff, buff_size, obj);
13403 pos = rb_raw_obj_info_buitin_type(buff, buff_size, obj, pos);
13404 if (pos >= buff_size) {} // truncated
13405 }
13406
13407 return buff;
13408 }
13409
13410 #undef APPEND_S
13411 #undef APPEND_F
13412 #undef BUFF_ARGS
13413
13414 #if RGENGC_OBJ_INFO
13415 #define OBJ_INFO_BUFFERS_NUM 10
13416 #define OBJ_INFO_BUFFERS_SIZE 0x100
13417 static rb_atomic_t obj_info_buffers_index = 0;
13418 static char obj_info_buffers[OBJ_INFO_BUFFERS_NUM][OBJ_INFO_BUFFERS_SIZE];
13419
13420 /* Increments *var atomically and resets *var to 0 when maxval is
13421 * reached. Returns the wraparound old *var value (0...maxval). */
13422 static rb_atomic_t
13423 atomic_inc_wraparound(rb_atomic_t *var, const rb_atomic_t maxval)
13424 {
13425 rb_atomic_t oldval = RUBY_ATOMIC_FETCH_ADD(*var, 1);
13426 if (UNLIKELY(oldval >= maxval - 1)) { // wraparound *var
13427 const rb_atomic_t newval = oldval + 1;
13428 RUBY_ATOMIC_CAS(*var, newval, newval % maxval);
13429 oldval %= maxval;
13430 }
13431 return oldval;
13432 }
13433
13434 static const char *
13435 obj_info(VALUE obj)
13436 {
13437 rb_atomic_t index = atomic_inc_wraparound(&obj_info_buffers_index, OBJ_INFO_BUFFERS_NUM);
13438 char *const buff = obj_info_buffers[index];
13439 return rb_raw_obj_info(buff, OBJ_INFO_BUFFERS_SIZE, obj);
13440 }
13441
13442 static const char *
13443 obj_info_basic(VALUE obj)
13444 {
13445 rb_atomic_t index = atomic_inc_wraparound(&obj_info_buffers_index, OBJ_INFO_BUFFERS_NUM);
13446 char *const buff = obj_info_buffers[index];
13447
13448 asan_unpoisoning_object(obj) {
13449 rb_raw_obj_info_common(buff, OBJ_INFO_BUFFERS_SIZE, obj);
13450 }
13451
13452 return buff;
13453 }
13454 #else
13455 static const char *
13456 obj_info(VALUE obj)
13457 {
13458 return obj_type_name(obj);
13459 }
13460
13461 static const char *
13462 obj_info_basic(VALUE obj)
13463 {
13464 return obj_type_name(obj);
13465 }
13466
13467 #endif
13468
13469 const char *
13470 rb_obj_info(VALUE obj)
13471 {
13472 return obj_info(obj);
13473 }
13474
13475 void
13476 rb_obj_info_dump(VALUE obj)
13477 {
13478 char buff[0x100];
13479 fprintf(stderr, "rb_obj_info_dump: %s\n", rb_raw_obj_info(buff, 0x100, obj));
13480 }
13481
13482 void
13483 rb_obj_info_dump_loc(VALUE obj, const char *file, int line, const char *func)
13484 {
13485 char buff[0x100];
13486 fprintf(stderr, "<OBJ_INFO:%s@%s:%d> %s\n", func, file, line, rb_raw_obj_info(buff, 0x100, obj));
13487 }
13488
13489 #if GC_DEBUG
13490
13491 void
13492 rb_gcdebug_print_obj_condition(VALUE obj)
13493 {
13494 rb_objspace_t *objspace = &rb_objspace;
13495
13496 fprintf(stderr, "created at: %s:%d\n", GET_RVALUE_OVERHEAD(obj)->file, GET_RVALUE_OVERHEAD(obj)->line);
13497
13498 if (BUILTIN_TYPE(obj) == T_MOVED) {
13499 fprintf(stderr, "moved?: true\n");
13500 }
13501 else {
13502 fprintf(stderr, "moved?: false\n");
13503 }
13504 if (is_pointer_to_heap(objspace, (void *)obj)) {
13505 fprintf(stderr, "pointer to heap?: true\n");
13506 }
13507 else {
13508 fprintf(stderr, "pointer to heap?: false\n");
13509 return;
13510 }
13511
13512 fprintf(stderr, "marked? : %s\n", MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) ? "true" : "false");
13513 fprintf(stderr, "pinned? : %s\n", MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) ? "true" : "false");
13514 fprintf(stderr, "age? : %d\n", RVALUE_AGE_GET(obj));
13515 fprintf(stderr, "old? : %s\n", RVALUE_OLD_P(obj) ? "true" : "false");
13516 fprintf(stderr, "WB-protected?: %s\n", RVALUE_WB_UNPROTECTED(obj) ? "false" : "true");
13517 fprintf(stderr, "remembered? : %s\n", RVALUE_REMEMBERED(obj) ? "true" : "false");
13518
13519 if (is_lazy_sweeping(objspace)) {
13520 fprintf(stderr, "lazy sweeping?: true\n");
13521 fprintf(stderr, "page swept?: %s\n", GET_HEAP_PAGE(obj)->flags.before_sweep ? "false" : "true");
13522 }
13523 else {
13524 fprintf(stderr, "lazy sweeping?: false\n");
13525 }
13526 }
13527
13528 static VALUE
13529 gcdebug_sentinel(RB_BLOCK_CALL_FUNC_ARGLIST(obj, name))
13530 {
13531 fprintf(stderr, "WARNING: object %s(%p) is inadvertently collected\n", (char *)name, (void *)obj);
13532 return Qnil;
13533 }
13534
13535 void
13536 rb_gcdebug_sentinel(VALUE obj, const char *name)
13537 {
13538 rb_define_finalizer(obj, rb_proc_new(gcdebug_sentinel, (VALUE)name));
13539 }
13540
13541 #endif /* GC_DEBUG */
13542
13543 /* :nodoc:
13544 *
13545 * call-seq:
13546 * GC.add_stress_to_class(class[, ...])
13547 *
13548 * Raises NoMemoryError when allocating an instance of the given classes.
13549 *
13550 */
13551 static VALUE
13552 rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self)
13553 {
13554 rb_objspace_t *objspace = &rb_objspace;
13555
13556 if (!stress_to_class) {
13557 set_stress_to_class(rb_ary_hidden_new(argc));
13558 }
13559 rb_ary_cat(stress_to_class, argv, argc);
13560 return self;
13561 }
13562
13563 /* :nodoc:
13564 *
13565 * call-seq:
13566 * GC.remove_stress_to_class(class[, ...])
13567 *
13568 * No longer raises NoMemoryError when allocating an instance of the
13569 * given classes.
13570 *
13571 */
13572 static VALUE
13573 rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self)
13574 {
13575 rb_objspace_t *objspace = &rb_objspace;
13576 int i;
13577
13578 if (stress_to_class) {
13579 for (i = 0; i < argc; ++i) {
13580 rb_ary_delete_same(stress_to_class, argv[i]);
13581 }
13582 if (RARRAY_LEN(stress_to_class) == 0) {
13583 set_stress_to_class(0);
13584 }
13585 }
13586 return Qnil;
13587 }
13588
13589 /*
13590 * Document-module: ObjectSpace
13591 *
13592 * The ObjectSpace module contains a number of routines
13593 * that interact with the garbage collection facility and allow you to
13594 * traverse all living objects with an iterator.
13595 *
13596 * ObjectSpace also provides support for object finalizers, procs that will be
13597 * called after a specific object was destroyed by garbage collection. See
13598 * the documentation for +ObjectSpace.define_finalizer+ for important
13599 * information on how to use this method correctly.
13600 *
13601 * a = "A"
13602 * b = "B"
13603 *
13604 * ObjectSpace.define_finalizer(a, proc {|id| puts "Finalizer one on #{id}" })
13605 * ObjectSpace.define_finalizer(b, proc {|id| puts "Finalizer two on #{id}" })
13606 *
13607 * a = nil
13608 * b = nil
13609 *
13610 * _produces:_
13611 *
13612 * Finalizer two on 537763470
13613 * Finalizer one on 537763480
13614 */
13615
13616 /* Document-class: GC::Profiler
13617 *
13618 * The GC profiler provides access to information on GC runs including time,
13619 * length and object space size.
13620 *
13621 * Example:
13622 *
13623 * GC::Profiler.enable
13624 *
13625 * require 'rdoc/rdoc'
13626 *
13627 * GC::Profiler.report
13628 *
13629 * GC::Profiler.disable
13630 *
13631 * See also GC.count, GC.malloc_allocated_size and GC.malloc_allocations
13632 */
13633
13634 #include "gc.rbinc"
13635
13636 void
13637 Init_GC(void)
13638 {
13639 #if USE_SHARED_GC
13640 if (getenv(RUBY_GC_LIBRARY_PATH) != NULL && !dln_supported_p()) {
13641 rb_warn(RUBY_GC_LIBRARY_PATH " is ignored because this executable file can't load extension libraries");
13642 }
13643 #endif
13644
13645 #undef rb_intern
13646 malloc_offset = gc_compute_malloc_offset();
13647
13648 VALUE rb_mObjSpace;
13649 VALUE rb_mProfiler;
13650 VALUE gc_constants;
13651
13652 rb_mGC = rb_define_module("GC");
13653
13654 gc_constants = rb_hash_new();
13655 rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), RBOOL(GC_DEBUG));
13656 rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
13657 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
13658 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(BASE_SLOT_SIZE));
13659 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));
13660 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
13661 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE));
13662 rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(SIZE_POOL_COUNT));
13663 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(size_pool_slot_size(SIZE_POOL_COUNT - 1)));
13664 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE));
13665 if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
13666 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue);
13667 }
13668 OBJ_FREEZE(gc_constants);
13669 /* Internal constants in the garbage collector. */
13670 rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
13671
13672 rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
13673 rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
13674 rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
13675 rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, 0);
13676 rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
13677 rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
13678 rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
13679 rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
13680 rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
13681
13682 rb_mObjSpace = rb_define_module("ObjectSpace");
13683
13684 rb_define_module_function(rb_mObjSpace, "each_object", os_each_obj, -1);
13685
13686 rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1);
13687 rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1);
13688
13689 rb_define_module_function(rb_mObjSpace, "_id2ref", os_id2ref, 1);
13690
13691 rb_vm_register_special_exception(ruby_error_nomemory, rb_eNoMemError, "failed to allocate memory");
13692
13693 rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0);
13694 rb_define_method(rb_mKernel, "object_id", rb_obj_id, 0);
13695
13696 rb_define_module_function(rb_mObjSpace, "count_objects", count_objects, -1);
13697
13698 /* internal methods */
13699 rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
13700 #if MALLOC_ALLOCATED_SIZE
13701 rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
13702 rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
13703 #endif
13704
13705 if (GC_COMPACTION_SUPPORTED) {
13706 rb_define_singleton_method(rb_mGC, "compact", gc_compact, 0);
13707 rb_define_singleton_method(rb_mGC, "auto_compact", gc_get_auto_compact, 0);
13708 rb_define_singleton_method(rb_mGC, "auto_compact=", gc_set_auto_compact, 1);
13709 rb_define_singleton_method(rb_mGC, "latest_compact_info", gc_compact_stats, 0);
13710 }
13711 else {
13712 rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0);
13713 rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0);
13714 rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1);
13715 rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
13716 /* When !GC_COMPACTION_SUPPORTED, this method is not defined in gc.rb */
13717 rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
13718 }
13719
13720 if (GC_DEBUG_STRESS_TO_CLASS) {
13721 rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
13722 rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
13723 }
13724
13725 {
13726 VALUE opts;
13727 /* \GC build options */
13728 rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new());
13729 #define OPT(o) if (o) rb_ary_push(opts, rb_fstring_lit(#o))
13730 OPT(GC_DEBUG);
13731 OPT(USE_RGENGC);
13732 OPT(RGENGC_DEBUG);
13733 OPT(RGENGC_CHECK_MODE);
13734 OPT(RGENGC_PROFILE);
13735 OPT(RGENGC_ESTIMATE_OLDMALLOC);
13736 OPT(GC_PROFILE_MORE_DETAIL);
13737 OPT(GC_ENABLE_LAZY_SWEEP);
13738 OPT(CALC_EXACT_MALLOC_SIZE);
13739 OPT(MALLOC_ALLOCATED_SIZE);
13740 OPT(MALLOC_ALLOCATED_SIZE_CHECK);
13741 OPT(GC_PROFILE_DETAIL_MEMORY);
13742 OPT(GC_COMPACTION_SUPPORTED);
13743 #undef OPT
13744 OBJ_FREEZE(opts);
13745 }
13746 }
13747
13748 #ifdef ruby_xmalloc
13749 #undef ruby_xmalloc
13750 #endif
13751 #ifdef ruby_xmalloc2
13752 #undef ruby_xmalloc2
13753 #endif
13754 #ifdef ruby_xcalloc
13755 #undef ruby_xcalloc
13756 #endif
13757 #ifdef ruby_xrealloc
13758 #undef ruby_xrealloc
13759 #endif
13760 #ifdef ruby_xrealloc2
13761 #undef ruby_xrealloc2
13762 #endif
13763
13764 void *
13765 ruby_xmalloc(size_t size)
13766 {
13767 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
13768 ruby_malloc_info_file = __FILE__;
13769 ruby_malloc_info_line = __LINE__;
13770 #endif
13771 return ruby_xmalloc_body(size);
13772 }
13773
13774 void *
13775 ruby_xmalloc2(size_t n, size_t size)
13776 {
13777 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
13778 ruby_malloc_info_file = __FILE__;
13779 ruby_malloc_info_line = __LINE__;
13780 #endif
13781 return ruby_xmalloc2_body(n, size);
13782 }
13783
13784 void *
13785 ruby_xcalloc(size_t n, size_t size)
13786 {
13787 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
13788 ruby_malloc_info_file = __FILE__;
13789 ruby_malloc_info_line = __LINE__;
13790 #endif
13791 return ruby_xcalloc_body(n, size);
13792 }
13793
13794 void *
13795 ruby_xrealloc(void *ptr, size_t new_size)
13796 {
13797 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
13798 ruby_malloc_info_file = __FILE__;
13799 ruby_malloc_info_line = __LINE__;
13800 #endif
13801 return ruby_xrealloc_body(ptr, new_size);
13802 }
13803
13804 void *
13805 ruby_xrealloc2(void *ptr, size_t n, size_t new_size)
13806 {
13807 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
13808 ruby_malloc_info_file = __FILE__;
13809 ruby_malloc_info_line = __LINE__;
13810 #endif
13811 return ruby_xrealloc2_body(ptr, n, new_size);
13812 }
The Ruby Programming Language