vm.c

   1 /**********************************************************************
   2
   3   Vm.c -
   4
   5   $Author$
   6
   7   Copyright (C) 2004-2007 Koichi Sasada
   8
   9 **********************************************************************/
  10
  11 #define vm_exec rb_vm_exec
  12
  13 #include "eval_intern.h"
  14 #include "internal.h"
  15 #include "internal/class.h"
  16 #include "internal/compile.h"
  17 #include "internal/cont.h"
  18 #include "internal/error.h"
  19 #include "internal/encoding.h"
  20 #include "internal/eval.h"
  21 #include "internal/gc.h"
  22 #include "internal/inits.h"
  23 #include "internal/missing.h"
  24 #include "internal/object.h"
  25 #include "internal/proc.h"
  26 #include "internal/re.h"
  27 #include "internal/ruby_parser.h"
  28 #include "internal/symbol.h"
  29 #include "internal/thread.h"
  30 #include "internal/transcode.h"
  31 #include "internal/vm.h"
  32 #include "internal/sanitizers.h"
  33 #include "internal/variable.h"
  34 #include "iseq.h"
  35 #include "rjit.h"
  36 #include "yjit.h"
  37 #include "ruby/st.h"
  38 #include "ruby/vm.h"
  39 #include "vm_core.h"
  40 #include "vm_callinfo.h"
  41 #include "vm_debug.h"
  42 #include "vm_exec.h"
  43 #include "vm_insnhelper.h"
  44 #include "ractor_core.h"
  45 #include "vm_sync.h"
  46 #include "shape.h"
  47
  48 #include "builtin.h"
  49
  50 #include "probes.h"
  51 #include "probes_helper.h"
  52
  53 #ifdef RUBY_ASSERT_CRITICAL_SECTION
  54 int ruby_assert_critical_section_entered = 0;
  55 #endif
  56
  57 static void *native_main_thread_stack_top;
  58
  59 VALUE rb_str_concat_literals(size_t, const VALUE*);
  60
  61 VALUE vm_exec(rb_execution_context_t *);
  62
  63 extern const char *const rb_debug_counter_names[];
  64
  65 PUREFUNC(static inline const VALUE *VM_EP_LEP(const VALUE *));
  66 static inline const VALUE *
  67 VM_EP_LEP(const VALUE *ep)
  68 {
  69     while (!VM_ENV_LOCAL_P(ep)) {
  70         ep = VM_ENV_PREV_EP(ep);
  71     }
  72     return ep;
  73 }
  74
  75 static inline const rb_control_frame_t *
  76 rb_vm_search_cf_from_ep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const VALUE * const ep)
  77 {
  78     if (!ep) {
  79         return NULL;
  80     }
  81     else {
  82         const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
  83
  84         while (cfp < eocfp) {
  85             if (cfp->ep == ep) {
  86                 return cfp;
  87             }
  88             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
  89         }
  90
  91         return NULL;
  92     }
  93 }
  94
  95 const VALUE *
  96 rb_vm_ep_local_ep(const VALUE *ep)
  97 {
  98     return VM_EP_LEP(ep);
  99 }
 100
 101 PUREFUNC(static inline const VALUE *VM_CF_LEP(const rb_control_frame_t * const cfp));
 102 static inline const VALUE *
 103 VM_CF_LEP(const rb_control_frame_t * const cfp)
 104 {
 105     return VM_EP_LEP(cfp->ep);
 106 }
 107
 108 static inline const VALUE *
 109 VM_CF_PREV_EP(const rb_control_frame_t * const cfp)
 110 {
 111     return VM_ENV_PREV_EP(cfp->ep);
 112 }
 113
 114 PUREFUNC(static inline VALUE VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp));
 115 static inline VALUE
 116 VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp)
 117 {
 118     const VALUE *ep = VM_CF_LEP(cfp);
 119     return VM_ENV_BLOCK_HANDLER(ep);
 120 }
 121
 122 int
 123 rb_vm_cframe_keyword_p(const rb_control_frame_t *cfp)
 124 {
 125     return VM_FRAME_CFRAME_KW_P(cfp);
 126 }
 127
 128 VALUE
 129 rb_vm_frame_block_handler(const rb_control_frame_t *cfp)
 130 {
 131     return VM_CF_BLOCK_HANDLER(cfp);
 132 }
 133
 134 #if VM_CHECK_MODE > 0
 135 static int
 136 VM_CFP_IN_HEAP_P(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 137 {
 138     const VALUE *start = ec->vm_stack;
 139     const VALUE *end = (VALUE *)ec->vm_stack + ec->vm_stack_size;
 140     VM_ASSERT(start != NULL);
 141
 142     if (start <= (VALUE *)cfp && (VALUE *)cfp < end) {
 143         return FALSE;
 144     }
 145     else {
 146         return TRUE;
 147     }
 148 }
 149
 150 static int
 151 VM_EP_IN_HEAP_P(const rb_execution_context_t *ec, const VALUE *ep)
 152 {
 153     const VALUE *start = ec->vm_stack;
 154     const VALUE *end = (VALUE *)ec->cfp;
 155     VM_ASSERT(start != NULL);
 156
 157     if (start <= ep && ep < end) {
 158         return FALSE;
 159     }
 160     else {
 161         return TRUE;
 162     }
 163 }
 164
 165 static int
 166 vm_ep_in_heap_p_(const rb_execution_context_t *ec, const VALUE *ep)
 167 {
 168     if (VM_EP_IN_HEAP_P(ec, ep)) {
 169         VALUE envval = ep[VM_ENV_DATA_INDEX_ENV]; /* VM_ENV_ENVVAL(ep); */
 170
 171         if (!UNDEF_P(envval)) {
 172             const rb_env_t *env = (const rb_env_t *)envval;
 173
 174             VM_ASSERT(vm_assert_env(envval));
 175             VM_ASSERT(VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED));
 176             VM_ASSERT(env->ep == ep);
 177         }
 178         return TRUE;
 179     }
 180     else {
 181         return FALSE;
 182     }
 183 }
 184
 185 int
 186 rb_vm_ep_in_heap_p(const VALUE *ep)
 187 {
 188     const rb_execution_context_t *ec = GET_EC();
 189     if (ec->vm_stack == NULL) return TRUE;
 190     return vm_ep_in_heap_p_(ec, ep);
 191 }
 192 #endif
 193
 194 static struct rb_captured_block *
 195 VM_CFP_TO_CAPTURED_BLOCK(const rb_control_frame_t *cfp)
 196 {
 197     VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp));
 198     return (struct rb_captured_block *)&cfp->self;
 199 }
 200
 201 static rb_control_frame_t *
 202 VM_CAPTURED_BLOCK_TO_CFP(const struct rb_captured_block *captured)
 203 {
 204     rb_control_frame_t *cfp = ((rb_control_frame_t *)((VALUE *)(captured) - 3));
 205     VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp));
 206     VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 7 + VM_DEBUG_BP_CHECK ? 1 : 0);
 207     return cfp;
 208 }
 209
 210 static int
 211 VM_BH_FROM_CFP_P(VALUE block_handler, const rb_control_frame_t *cfp)
 212 {
 213     const struct rb_captured_block *captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
 214     return VM_TAGGED_PTR_REF(block_handler, 0x03) == captured;
 215 }
 216
 217 static VALUE
 218 vm_passed_block_handler(rb_execution_context_t *ec)
 219 {
 220     VALUE block_handler = ec->passed_block_handler;
 221     ec->passed_block_handler = VM_BLOCK_HANDLER_NONE;
 222     vm_block_handler_verify(block_handler);
 223     return block_handler;
 224 }
 225
 226 static rb_cref_t *
 227 vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int use_prev_prev, int singleton)
 228 {
 229     VALUE refinements = Qnil;
 230     int omod_shared = FALSE;
 231
 232     /* scope */
 233     union {
 234         rb_scope_visibility_t visi;
 235         VALUE value;
 236     } scope_visi;
 237
 238     scope_visi.visi.method_visi = visi;
 239     scope_visi.visi.module_func = module_func;
 240
 241     /* refinements */
 242     if (prev_cref != NULL && prev_cref != (void *)1 /* TODO: why CREF_NEXT(cref) is 1? */) {
 243         refinements = CREF_REFINEMENTS(prev_cref);
 244
 245         if (!NIL_P(refinements)) {
 246             omod_shared = TRUE;
 247             CREF_OMOD_SHARED_SET(prev_cref);
 248         }
 249     }
 250
 251     VM_ASSERT(singleton || klass);
 252
 253     rb_cref_t *cref = IMEMO_NEW(rb_cref_t, imemo_cref, refinements);
 254     cref->klass_or_self = klass;
 255     cref->next = use_prev_prev ? CREF_NEXT(prev_cref) : prev_cref;
 256     *((rb_scope_visibility_t *)&cref->scope_visi) = scope_visi.visi;
 257
 258     if (pushed_by_eval) CREF_PUSHED_BY_EVAL_SET(cref);
 259     if (omod_shared) CREF_OMOD_SHARED_SET(cref);
 260     if (singleton) CREF_SINGLETON_SET(cref);
 261
 262     return cref;
 263 }
 264
 265 static rb_cref_t *
 266 vm_cref_new(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int singleton)
 267 {
 268     return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, FALSE, singleton);
 269 }
 270
 271 static rb_cref_t *
 272 vm_cref_new_use_prev(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval)
 273 {
 274     return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, TRUE, FALSE);
 275 }
 276
 277 static int
 278 ref_delete_symkey(VALUE key, VALUE value, VALUE unused)
 279 {
 280     return SYMBOL_P(key) ? ST_DELETE : ST_CONTINUE;
 281 }
 282
 283 static rb_cref_t *
 284 vm_cref_dup(const rb_cref_t *cref)
 285 {
 286     const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref);
 287     rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref;
 288     int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref);
 289     int singleton = CREF_SINGLETON(cref);
 290
 291     new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton);
 292
 293     if (!NIL_P(CREF_REFINEMENTS(cref))) {
 294         VALUE ref = rb_hash_dup(CREF_REFINEMENTS(cref));
 295         rb_hash_foreach(ref, ref_delete_symkey, Qnil);
 296         CREF_REFINEMENTS_SET(new_cref, ref);
 297         CREF_OMOD_SHARED_UNSET(new_cref);
 298     }
 299
 300     return new_cref;
 301 }
 302
 303
 304 rb_cref_t *
 305 rb_vm_cref_dup_without_refinements(const rb_cref_t *cref)
 306 {
 307     const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref);
 308     rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref;
 309     int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref);
 310     int singleton = CREF_SINGLETON(cref);
 311
 312     new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton);
 313
 314     if (!NIL_P(CREF_REFINEMENTS(cref))) {
 315         CREF_REFINEMENTS_SET(new_cref, Qnil);
 316         CREF_OMOD_SHARED_UNSET(new_cref);
 317     }
 318
 319     return new_cref;
 320 }
 321
 322 static rb_cref_t *
 323 vm_cref_new_toplevel(rb_execution_context_t *ec)
 324 {
 325     rb_cref_t *cref = vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE /* toplevel visibility is private */, FALSE, NULL, FALSE, FALSE);
 326     VALUE top_wrapper = rb_ec_thread_ptr(ec)->top_wrapper;
 327
 328     if (top_wrapper) {
 329         cref = vm_cref_new(top_wrapper, METHOD_VISI_PRIVATE, FALSE, cref, FALSE, FALSE);
 330     }
 331
 332     return cref;
 333 }
 334
 335 rb_cref_t *
 336 rb_vm_cref_new_toplevel(void)
 337 {
 338     return vm_cref_new_toplevel(GET_EC());
 339 }
 340
 341 static void
 342 vm_cref_dump(const char *mesg, const rb_cref_t *cref)
 343 {
 344     ruby_debug_printf("vm_cref_dump: %s (%p)\n", mesg, (void *)cref);
 345
 346     while (cref) {
 347         ruby_debug_printf("= cref| klass: %s\n", RSTRING_PTR(rb_class_path(CREF_CLASS(cref))));
 348         cref = CREF_NEXT(cref);
 349     }
 350 }
 351
 352 void
 353 rb_vm_block_ep_update(VALUE obj, const struct rb_block *dst, const VALUE *ep)
 354 {
 355     *((const VALUE **)&dst->as.captured.ep) = ep;
 356     RB_OBJ_WRITTEN(obj, Qundef, VM_ENV_ENVVAL(ep));
 357 }
 358
 359 static void
 360 vm_bind_update_env(VALUE bindval, rb_binding_t *bind, VALUE envval)
 361 {
 362     const rb_env_t *env = (rb_env_t *)envval;
 363     RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, env->iseq);
 364     rb_vm_block_ep_update(bindval, &bind->block, env->ep);
 365 }
 366
 367 #if VM_COLLECT_USAGE_DETAILS
 368 static void vm_collect_usage_operand(int insn, int n, VALUE op);
 369 static void vm_collect_usage_insn(int insn);
 370 static void vm_collect_usage_register(int reg, int isset);
 371 #endif
 372
 373 static VALUE vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp);
 374 extern VALUE rb_vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
 375                                   int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
 376                                   const rb_callable_method_entry_t *me);
 377 static VALUE vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler);
 378
 379 #if USE_YJIT
 380 // Counter to serve as a proxy for execution time, total number of calls
 381 static uint64_t yjit_total_entry_hits = 0;
 382
 383 // Number of calls used to estimate how hot an ISEQ is
 384 #define YJIT_CALL_COUNT_INTERV 20u
 385
 386 /// Test whether we are ready to compile an ISEQ or not
 387 static inline bool
 388 rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls)
 389 {
 390     yjit_total_entry_hits += 1;
 391
 392     // Record the number of calls at the beginning of the interval
 393     if (entry_calls + YJIT_CALL_COUNT_INTERV == rb_yjit_call_threshold) {
 394         iseq->body->yjit_calls_at_interv = yjit_total_entry_hits;
 395     }
 396
 397     // Try to estimate the total time taken (total number of calls) to reach 20 calls to this ISEQ
 398     // This give us a ratio of how hot/cold this ISEQ is
 399     if (entry_calls == rb_yjit_call_threshold) {
 400         // We expect threshold 1 to compile everything immediately
 401         if (rb_yjit_call_threshold < YJIT_CALL_COUNT_INTERV) {
 402             return true;
 403         }
 404
 405         uint64_t num_calls = yjit_total_entry_hits - iseq->body->yjit_calls_at_interv;
 406
 407         // Reject ISEQs that don't get called often enough
 408         if (num_calls > rb_yjit_cold_threshold) {
 409             rb_yjit_incr_counter("cold_iseq_entry");
 410             return false;
 411         }
 412
 413         return true;
 414     }
 415
 416     return false;
 417 }
 418 #else
 419 #define rb_yjit_threshold_hit(iseq, entry_calls) false
 420 #endif
 421
 422 #if USE_RJIT || USE_YJIT
 423 // Generate JIT code that supports the following kinds of ISEQ entries:
 424 //   * The first ISEQ on vm_exec (e.g. <main>, or Ruby methods/blocks
 425 //     called by a C method). The current frame has VM_FRAME_FLAG_FINISH.
 426 //     The current vm_exec stops if JIT code returns a non-Qundef value.
 427 //   * ISEQs called by the interpreter on vm_sendish (e.g. Ruby methods or
 428 //     blocks called by a Ruby frame that isn't compiled or side-exited).
 429 //     The current frame doesn't have VM_FRAME_FLAG_FINISH. The current
 430 //     vm_exec does NOT stop whether JIT code returns Qundef or not.
 431 static inline rb_jit_func_t
 432 jit_compile(rb_execution_context_t *ec)
 433 {
 434     const rb_iseq_t *iseq = ec->cfp->iseq;
 435     struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
 436     bool yjit_enabled = rb_yjit_enabled_p;
 437     if (!(yjit_enabled || rb_rjit_call_p)) {
 438         return NULL;
 439     }
 440
 441     // Increment the ISEQ's call counter and trigger JIT compilation if not compiled
 442     if (body->jit_entry == NULL) {
 443         body->jit_entry_calls++;
 444         if (yjit_enabled) {
 445             if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) {
 446                 rb_yjit_compile_iseq(iseq, ec, false);
 447             }
 448         }
 449         else if (body->jit_entry_calls == rb_rjit_call_threshold()) {
 450             rb_rjit_compile(iseq);
 451         }
 452     }
 453     return body->jit_entry;
 454 }
 455
 456 // Execute JIT code compiled by jit_compile()
 457 static inline VALUE
 458 jit_exec(rb_execution_context_t *ec)
 459 {
 460     rb_jit_func_t func = jit_compile(ec);
 461     if (func) {
 462         // Call the JIT code
 463         return func(ec, ec->cfp);
 464     }
 465     else {
 466         return Qundef;
 467     }
 468 }
 469 #else
 470 # define jit_compile(ec) ((rb_jit_func_t)0)
 471 # define jit_exec(ec) Qundef
 472 #endif
 473
 474 #if USE_YJIT
 475 // Generate JIT code that supports the following kind of ISEQ entry:
 476 //   * The first ISEQ pushed by vm_exec_handle_exception. The frame would
 477 //     point to a location specified by a catch table, and it doesn't have
 478 //     VM_FRAME_FLAG_FINISH. The current vm_exec stops if JIT code returns
 479 //     a non-Qundef value. So you should not return a non-Qundef value
 480 //     until ec->cfp is changed to a frame with VM_FRAME_FLAG_FINISH.
 481 static inline rb_jit_func_t
 482 jit_compile_exception(rb_execution_context_t *ec)
 483 {
 484     const rb_iseq_t *iseq = ec->cfp->iseq;
 485     struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
 486     if (!rb_yjit_enabled_p) {
 487         return NULL;
 488     }
 489
 490     // Increment the ISEQ's call counter and trigger JIT compilation if not compiled
 491     if (body->jit_exception == NULL) {
 492         body->jit_exception_calls++;
 493         if (body->jit_exception_calls == rb_yjit_call_threshold) {
 494             rb_yjit_compile_iseq(iseq, ec, true);
 495         }
 496     }
 497
 498     return body->jit_exception;
 499 }
 500
 501 // Execute JIT code compiled by jit_compile_exception()
 502 static inline VALUE
 503 jit_exec_exception(rb_execution_context_t *ec)
 504 {
 505     rb_jit_func_t func = jit_compile_exception(ec);
 506     if (func) {
 507         // Call the JIT code
 508         return func(ec, ec->cfp);
 509     }
 510     else {
 511         return Qundef;
 512     }
 513 }
 514 #else
 515 # define jit_compile_exception(ec) ((rb_jit_func_t)0)
 516 # define jit_exec_exception(ec) Qundef
 517 #endif
 518
 519 static void add_opt_method_entry(const rb_method_entry_t *me);
 520
 521 #include "vm_insnhelper.c"
 522
 523 #include "vm_exec.c"
 524
 525 #include "vm_method.c"
 526 #include "vm_eval.c"
 527
 528 #define PROCDEBUG 0
 529
 530 VALUE rb_cRubyVM;
 531 VALUE rb_cThread;
 532 VALUE rb_mRubyVMFrozenCore;
 533 VALUE rb_block_param_proxy;
 534
 535 VALUE ruby_vm_const_missing_count = 0;
 536 rb_vm_t *ruby_current_vm_ptr = NULL;
 537 rb_ractor_t *ruby_single_main_ractor;
 538 bool ruby_vm_keep_script_lines;
 539
 540 #ifdef RB_THREAD_LOCAL_SPECIFIER
 541 RB_THREAD_LOCAL_SPECIFIER rb_execution_context_t *ruby_current_ec;
 542
 543 #ifdef RUBY_NT_SERIAL
 544 RB_THREAD_LOCAL_SPECIFIER rb_atomic_t ruby_nt_serial;
 545 #endif
 546
 547 // no-inline decl on thread_pthread.h
 548 rb_execution_context_t *
 549 rb_current_ec_noinline(void)
 550 {
 551     return ruby_current_ec;
 552 }
 553
 554 void
 555 rb_current_ec_set(rb_execution_context_t *ec)
 556 {
 557     ruby_current_ec = ec;
 558 }
 559
 560
 561 #ifdef __APPLE__
 562 rb_execution_context_t *
 563 rb_current_ec(void)
 564 {
 565     return ruby_current_ec;
 566 }
 567
 568 #endif
 569 #else
 570 native_tls_key_t ruby_current_ec_key;
 571 #endif
 572
 573 rb_event_flag_t ruby_vm_event_flags;
 574 rb_event_flag_t ruby_vm_event_enabled_global_flags;
 575 unsigned int    ruby_vm_event_local_num;
 576
 577 rb_serial_t ruby_vm_constant_cache_invalidations = 0;
 578 rb_serial_t ruby_vm_constant_cache_misses = 0;
 579 rb_serial_t ruby_vm_global_cvar_state = 1;
 580
 581 static const struct rb_callcache vm_empty_cc = {
 582     .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE,
 583     .klass = Qfalse,
 584     .cme_  = NULL,
 585     .call_ = vm_call_general,
 586     .aux_  = {
 587         .v = Qfalse,
 588     }
 589 };
 590
 591 static const struct rb_callcache vm_empty_cc_for_super = {
 592     .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE,
 593     .klass = Qfalse,
 594     .cme_  = NULL,
 595     .call_ = vm_call_super_method,
 596     .aux_  = {
 597         .v = Qfalse,
 598     }
 599 };
 600
 601 static void thread_free(void *ptr);
 602
 603 void
 604 rb_vm_inc_const_missing_count(void)
 605 {
 606     ruby_vm_const_missing_count +=1;
 607 }
 608
 609 int
 610 rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id,
 611                 struct ruby_dtrace_method_hook_args *args)
 612 {
 613     enum ruby_value_type type;
 614     if (!klass) {
 615         if (!ec) ec = GET_EC();
 616         if (!rb_ec_frame_method_id_and_class(ec, &id, 0, &klass) || !klass)
 617             return FALSE;
 618     }
 619     if (RB_TYPE_P(klass, T_ICLASS)) {
 620         klass = RBASIC(klass)->klass;
 621     }
 622     else if (RCLASS_SINGLETON_P(klass)) {
 623         klass = RCLASS_ATTACHED_OBJECT(klass);
 624         if (NIL_P(klass)) return FALSE;
 625     }
 626     type = BUILTIN_TYPE(klass);
 627     if (type == T_CLASS || type == T_ICLASS || type == T_MODULE) {
 628         VALUE name = rb_class_path(klass);
 629         const char *classname, *filename;
 630         const char *methodname = rb_id2name(id);
 631         if (methodname && (filename = rb_source_location_cstr(&args->line_no)) != 0) {
 632             if (NIL_P(name) || !(classname = StringValuePtr(name)))
 633                 classname = "<unknown>";
 634             args->classname = classname;
 635             args->methodname = methodname;
 636             args->filename = filename;
 637             args->klass = klass;
 638             args->name = name;
 639             return TRUE;
 640         }
 641     }
 642     return FALSE;
 643 }
 644
 645 extern unsigned int redblack_buffer_size;
 646
 647 /*
 648  *  call-seq:
 649  *    RubyVM.stat -> Hash
 650  *    RubyVM.stat(hsh) -> hsh
 651  *    RubyVM.stat(Symbol) -> Numeric
 652  *
 653  *  Returns a Hash containing implementation-dependent counters inside the VM.
 654  *
 655  *  This hash includes information about method/constant caches:
 656  *
 657  *    {
 658  *      :constant_cache_invalidations=>2,
 659  *      :constant_cache_misses=>14,
 660  *      :global_cvar_state=>27
 661  *    }
 662  *
 663  *  If <tt>USE_DEBUG_COUNTER</tt> is enabled, debug counters will be included.
 664  *
 665  *  The contents of the hash are implementation specific and may be changed in
 666  *  the future.
 667  *
 668  *  This method is only expected to work on C Ruby.
 669  */
 670 static VALUE
 671 vm_stat(int argc, VALUE *argv, VALUE self)
 672 {
 673     static VALUE sym_constant_cache_invalidations, sym_constant_cache_misses, sym_global_cvar_state, sym_next_shape_id;
 674     static VALUE sym_shape_cache_size;
 675     VALUE arg = Qnil;
 676     VALUE hash = Qnil, key = Qnil;
 677
 678     if (rb_check_arity(argc, 0, 1) == 1) {
 679         arg = argv[0];
 680         if (SYMBOL_P(arg))
 681             key = arg;
 682         else if (RB_TYPE_P(arg, T_HASH))
 683             hash = arg;
 684         else
 685             rb_raise(rb_eTypeError, "non-hash or symbol given");
 686     }
 687     else {
 688         hash = rb_hash_new();
 689     }
 690
 691 #define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
 692     S(constant_cache_invalidations);
 693     S(constant_cache_misses);
 694         S(global_cvar_state);
 695     S(next_shape_id);
 696     S(shape_cache_size);
 697 #undef S
 698
 699 #define SET(name, attr) \
 700     if (key == sym_##name) \
 701         return SERIALT2NUM(attr); \
 702     else if (hash != Qnil) \
 703         rb_hash_aset(hash, sym_##name, SERIALT2NUM(attr));
 704
 705     SET(constant_cache_invalidations, ruby_vm_constant_cache_invalidations);
 706     SET(constant_cache_misses, ruby_vm_constant_cache_misses);
 707     SET(global_cvar_state, ruby_vm_global_cvar_state);
 708     SET(next_shape_id, (rb_serial_t)GET_SHAPE_TREE()->next_shape_id);
 709     SET(shape_cache_size, (rb_serial_t)GET_SHAPE_TREE()->cache_size);
 710 #undef SET
 711
 712 #if USE_DEBUG_COUNTER
 713     ruby_debug_counter_show_at_exit(FALSE);
 714     for (size_t i = 0; i < RB_DEBUG_COUNTER_MAX; i++) {
 715         const VALUE name = rb_sym_intern_ascii_cstr(rb_debug_counter_names[i]);
 716         const VALUE boxed_value = SIZET2NUM(rb_debug_counter[i]);
 717
 718         if (key == name) {
 719             return boxed_value;
 720         }
 721         else if (hash != Qnil) {
 722             rb_hash_aset(hash, name, boxed_value);
 723         }
 724     }
 725 #endif
 726
 727     if (!NIL_P(key)) { /* matched key should return above */
 728         rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
 729     }
 730
 731     return hash;
 732 }
 733
 734 /* control stack frame */
 735
 736 static void
 737 vm_set_top_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq)
 738 {
 739     if (ISEQ_BODY(iseq)->type != ISEQ_TYPE_TOP) {
 740         rb_raise(rb_eTypeError, "Not a toplevel InstructionSequence");
 741     }
 742
 743     /* for return */
 744     vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, rb_ec_thread_ptr(ec)->top_self,
 745                   VM_BLOCK_HANDLER_NONE,
 746                   (VALUE)vm_cref_new_toplevel(ec), /* cref or me */
 747                   ISEQ_BODY(iseq)->iseq_encoded, ec->cfp->sp,
 748                   ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max);
 749 }
 750
 751 static void
 752 vm_set_eval_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq, const rb_cref_t *cref, const struct rb_block *base_block)
 753 {
 754     vm_push_frame(ec, iseq, VM_FRAME_MAGIC_EVAL | VM_FRAME_FLAG_FINISH,
 755                   vm_block_self(base_block), VM_GUARDED_PREV_EP(vm_block_ep(base_block)),
 756                   (VALUE)cref, /* cref or me */
 757                   ISEQ_BODY(iseq)->iseq_encoded,
 758                   ec->cfp->sp, ISEQ_BODY(iseq)->local_table_size,
 759                   ISEQ_BODY(iseq)->stack_max);
 760 }
 761
 762 static void
 763 vm_set_main_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq)
 764 {
 765     VALUE toplevel_binding = rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING"));
 766     rb_binding_t *bind;
 767
 768     GetBindingPtr(toplevel_binding, bind);
 769     RUBY_ASSERT_MESG(bind, "TOPLEVEL_BINDING is not built");
 770
 771     vm_set_eval_stack(ec, iseq, 0, &bind->block);
 772
 773     /* save binding */
 774     if (ISEQ_BODY(iseq)->local_table_size > 0) {
 775         vm_bind_update_env(toplevel_binding, bind, vm_make_env_object(ec, ec->cfp));
 776     }
 777 }
 778
 779 rb_control_frame_t *
 780 rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 781 {
 782     while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
 783         if (cfp->iseq) {
 784             return (rb_control_frame_t *)cfp;
 785         }
 786         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 787     }
 788     return 0;
 789 }
 790
 791 rb_control_frame_t *
 792 rb_vm_get_ruby_level_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 793 {
 794     while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
 795         if (VM_FRAME_RUBYFRAME_P(cfp)) {
 796             return (rb_control_frame_t *)cfp;
 797         }
 798         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 799     }
 800     return 0;
 801 }
 802
 803 static rb_control_frame_t *
 804 vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 805 {
 806     if (VM_FRAME_RUBYFRAME_P(cfp)) {
 807         return (rb_control_frame_t *)cfp;
 808     }
 809
 810     cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 811
 812     while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
 813         if (VM_FRAME_RUBYFRAME_P(cfp)) {
 814             return (rb_control_frame_t *)cfp;
 815         }
 816
 817         if (VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_PASSED) == FALSE) {
 818             break;
 819         }
 820         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 821     }
 822     return 0;
 823 }
 824
 825 void
 826 rb_vm_pop_cfunc_frame(void)
 827 {
 828     rb_execution_context_t *ec = GET_EC();
 829     rb_control_frame_t *cfp = ec->cfp;
 830     const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
 831
 832     EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, Qnil);
 833     RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
 834     vm_pop_frame(ec, cfp, cfp->ep);
 835 }
 836
 837 void
 838 rb_vm_rewind_cfp(rb_execution_context_t *ec, rb_control_frame_t *cfp)
 839 {
 840     /* check skipped frame */
 841     while (ec->cfp != cfp) {
 842 #if VMDEBUG
 843         printf("skipped frame: %s\n", vm_frametype_name(ec->cfp));
 844 #endif
 845         if (VM_FRAME_TYPE(ec->cfp) != VM_FRAME_MAGIC_CFUNC) {
 846             rb_vm_pop_frame(ec);
 847         }
 848         else { /* unlikely path */
 849             rb_vm_pop_cfunc_frame();
 850         }
 851     }
 852 }
 853
 854 /* at exit */
 855
 856 void
 857 ruby_vm_at_exit(void (*func)(rb_vm_t *))
 858 {
 859     rb_vm_t *vm = GET_VM();
 860     rb_at_exit_list *nl = ALLOC(rb_at_exit_list);
 861     nl->func = func;
 862     nl->next = vm->at_exit;
 863     vm->at_exit = nl;
 864 }
 865
 866 static void
 867 ruby_vm_run_at_exit_hooks(rb_vm_t *vm)
 868 {
 869     rb_at_exit_list *l = vm->at_exit;
 870
 871     while (l) {
 872         rb_at_exit_list* t = l->next;
 873         rb_vm_at_exit_func *func = l->func;
 874         ruby_xfree(l);
 875         l = t;
 876         (*func)(vm);
 877     }
 878 }
 879
 880 /* Env */
 881
 882 static VALUE check_env_value(const rb_env_t *env);
 883
 884 static int
 885 check_env(const rb_env_t *env)
 886 {
 887     fputs("---\n", stderr);
 888     ruby_debug_printf("envptr: %p\n", (void *)&env->ep[0]);
 889     ruby_debug_printf("envval: %10p ", (void *)env->ep[1]);
 890     dp(env->ep[1]);
 891     ruby_debug_printf("ep:    %10p\n", (void *)env->ep);
 892     if (rb_vm_env_prev_env(env)) {
 893         fputs(">>\n", stderr);
 894         check_env_value(rb_vm_env_prev_env(env));
 895         fputs("<<\n", stderr);
 896     }
 897     return 1;
 898 }
 899
 900 static VALUE
 901 check_env_value(const rb_env_t *env)
 902 {
 903     if (check_env(env)) {
 904         return (VALUE)env;
 905     }
 906     rb_bug("invalid env");
 907     return Qnil;                /* unreachable */
 908 }
 909
 910 static VALUE
 911 vm_block_handler_escape(const rb_execution_context_t *ec, VALUE block_handler)
 912 {
 913     switch (vm_block_handler_type(block_handler)) {
 914       case block_handler_type_ifunc:
 915       case block_handler_type_iseq:
 916         return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
 917
 918       case block_handler_type_symbol:
 919       case block_handler_type_proc:
 920         return block_handler;
 921     }
 922     VM_UNREACHABLE(vm_block_handler_escape);
 923     return Qnil;
 924 }
 925
 926 static VALUE
 927 vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *const cfp)
 928 {
 929     const VALUE * const ep = cfp->ep;
 930     VALUE *env_body, *env_ep;
 931     int local_size, env_size;
 932
 933     if (VM_ENV_ESCAPED_P(ep)) {
 934         return VM_ENV_ENVVAL(ep);
 935     }
 936
 937     if (!VM_ENV_LOCAL_P(ep)) {
 938         const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
 939         if (!VM_ENV_ESCAPED_P(prev_ep)) {
 940             rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 941
 942             while (prev_cfp->ep != prev_ep) {
 943                 prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(prev_cfp);
 944                 VM_ASSERT(prev_cfp->ep != NULL);
 945             }
 946
 947             vm_make_env_each(ec, prev_cfp);
 948             VM_FORCE_WRITE_SPECIAL_CONST(&ep[VM_ENV_DATA_INDEX_SPECVAL], VM_GUARDED_PREV_EP(prev_cfp->ep));
 949         }
 950     }
 951     else {
 952         VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep);
 953
 954         if (block_handler != VM_BLOCK_HANDLER_NONE) {
 955             VALUE blockprocval = vm_block_handler_escape(ec, block_handler);
 956             VM_STACK_ENV_WRITE(ep, VM_ENV_DATA_INDEX_SPECVAL, blockprocval);
 957         }
 958     }
 959
 960     if (!VM_FRAME_RUBYFRAME_P(cfp)) {
 961         local_size = VM_ENV_DATA_SIZE;
 962     }
 963     else {
 964         local_size = ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE;
 965     }
 966
 967     /*
 968      * # local variables on a stack frame (N == local_size)
 969      * [lvar1, lvar2, ..., lvarN, SPECVAL]
 970      *                            ^
 971      *                            ep[0]
 972      *
 973      * # moved local variables
 974      * [lvar1, lvar2, ..., lvarN, SPECVAL, Envval, BlockProcval (if needed)]
 975      *  ^                         ^
 976      *  env->env[0]               ep[0]
 977      */
 978
 979     env_size = local_size +
 980                1 /* envval */;
 981
 982     // Careful with order in the following sequence. Each allocation can move objects.
 983     env_body = ALLOC_N(VALUE, env_size);
 984     rb_env_t *env = IMEMO_NEW(rb_env_t, imemo_env, 0);
 985
 986     // Set up env without WB since it's brand new (similar to newobj_init(), newobj_fill())
 987     MEMCPY(env_body, ep - (local_size - 1 /* specval */), VALUE, local_size);
 988
 989     env_ep = &env_body[local_size - 1 /* specval */];
 990     env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env;
 991
 992     env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL);
 993     env->ep = env_ep;
 994     env->env = env_body;
 995     env->env_size = env_size;
 996
 997     cfp->ep = env_ep;
 998     VM_ENV_FLAGS_SET(env_ep, VM_ENV_FLAG_ESCAPED | VM_ENV_FLAG_WB_REQUIRED);
 999     VM_STACK_ENV_WRITE(ep, 0, (VALUE)env);              /* GC mark */
1000
1001 #if 0
1002     for (i = 0; i < local_size; i++) {
1003         if (VM_FRAME_RUBYFRAME_P(cfp)) {
1004             /* clear value stack for GC */
1005             ep[-local_size + i] = 0;
1006         }
1007     }
1008 #endif
1009
1010     // Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp).
1011     if (env->iseq) {
1012         rb_yjit_invalidate_ep_is_bp(env->iseq);
1013     }
1014
1015     return (VALUE)env;
1016 }
1017
1018 static VALUE
1019 vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
1020 {
1021     VALUE envval = vm_make_env_each(ec, cfp);
1022
1023     if (PROCDEBUG) {
1024         check_env_value((const rb_env_t *)envval);
1025     }
1026
1027     return envval;
1028 }
1029
1030 void
1031 rb_vm_stack_to_heap(rb_execution_context_t *ec)
1032 {
1033     rb_control_frame_t *cfp = ec->cfp;
1034     while ((cfp = rb_vm_get_binding_creatable_next_cfp(ec, cfp)) != 0) {
1035         vm_make_env_object(ec, cfp);
1036         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1037     }
1038 }
1039
1040 const rb_env_t *
1041 rb_vm_env_prev_env(const rb_env_t *env)
1042 {
1043     const VALUE *ep = env->ep;
1044
1045     if (VM_ENV_LOCAL_P(ep)) {
1046         return NULL;
1047     }
1048     else {
1049         const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
1050         return VM_ENV_ENVVAL_PTR(prev_ep);
1051     }
1052 }
1053
1054 static int
1055 collect_local_variables_in_iseq(const rb_iseq_t *iseq, const struct local_var_list *vars)
1056 {
1057     unsigned int i;
1058     if (!iseq) return 0;
1059     for (i = 0; i < ISEQ_BODY(iseq)->local_table_size; i++) {
1060         local_var_list_add(vars, ISEQ_BODY(iseq)->local_table[i]);
1061     }
1062     return 1;
1063 }
1064
1065 static void
1066 collect_local_variables_in_env(const rb_env_t *env, const struct local_var_list *vars)
1067 {
1068     do {
1069         if (VM_ENV_FLAGS(env->ep, VM_ENV_FLAG_ISOLATED)) break;
1070         collect_local_variables_in_iseq(env->iseq, vars);
1071     } while ((env = rb_vm_env_prev_env(env)) != NULL);
1072 }
1073
1074 static int
1075 vm_collect_local_variables_in_heap(const VALUE *ep, const struct local_var_list *vars)
1076 {
1077     if (VM_ENV_ESCAPED_P(ep)) {
1078         collect_local_variables_in_env(VM_ENV_ENVVAL_PTR(ep), vars);
1079         return 1;
1080     }
1081     else {
1082         return 0;
1083     }
1084 }
1085
1086 VALUE
1087 rb_vm_env_local_variables(const rb_env_t *env)
1088 {
1089     struct local_var_list vars;
1090     local_var_list_init(&vars);
1091     collect_local_variables_in_env(env, &vars);
1092     return local_var_list_finish(&vars);
1093 }
1094
1095 VALUE
1096 rb_iseq_local_variables(const rb_iseq_t *iseq)
1097 {
1098     struct local_var_list vars;
1099     local_var_list_init(&vars);
1100     while (collect_local_variables_in_iseq(iseq, &vars)) {
1101         iseq = ISEQ_BODY(iseq)->parent_iseq;
1102     }
1103     return local_var_list_finish(&vars);
1104 }
1105
1106 /* Proc */
1107
1108 static VALUE
1109 vm_proc_create_from_captured(VALUE klass,
1110                              const struct rb_captured_block *captured,
1111                              enum rb_block_type block_type,
1112                              int8_t is_from_method, int8_t is_lambda)
1113 {
1114     VALUE procval = rb_proc_alloc(klass);
1115     rb_proc_t *proc = RTYPEDDATA_DATA(procval);
1116
1117     VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), captured->ep));
1118
1119     /* copy block */
1120     RB_OBJ_WRITE(procval, &proc->block.as.captured.code.val, captured->code.val);
1121     RB_OBJ_WRITE(procval, &proc->block.as.captured.self, captured->self);
1122     rb_vm_block_ep_update(procval, &proc->block, captured->ep);
1123
1124     vm_block_type_set(&proc->block, block_type);
1125     proc->is_from_method = is_from_method;
1126     proc->is_lambda = is_lambda;
1127
1128     return procval;
1129 }
1130
1131 void
1132 rb_vm_block_copy(VALUE obj, const struct rb_block *dst, const struct rb_block *src)
1133 {
1134     /* copy block */
1135     switch (vm_block_type(src)) {
1136       case block_type_iseq:
1137       case block_type_ifunc:
1138         RB_OBJ_WRITE(obj, &dst->as.captured.self, src->as.captured.self);
1139         RB_OBJ_WRITE(obj, &dst->as.captured.code.val, src->as.captured.code.val);
1140         rb_vm_block_ep_update(obj, dst, src->as.captured.ep);
1141         break;
1142       case block_type_symbol:
1143         RB_OBJ_WRITE(obj, &dst->as.symbol, src->as.symbol);
1144         break;
1145       case block_type_proc:
1146         RB_OBJ_WRITE(obj, &dst->as.proc, src->as.proc);
1147         break;
1148     }
1149 }
1150
1151 static VALUE
1152 proc_create(VALUE klass, const struct rb_block *block, int8_t is_from_method, int8_t is_lambda)
1153 {
1154     VALUE procval = rb_proc_alloc(klass);
1155     rb_proc_t *proc = RTYPEDDATA_DATA(procval);
1156
1157     VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), vm_block_ep(block)));
1158     rb_vm_block_copy(procval, &proc->block, block);
1159     vm_block_type_set(&proc->block, block->type);
1160     proc->is_from_method = is_from_method;
1161     proc->is_lambda = is_lambda;
1162
1163     return procval;
1164 }
1165
1166 VALUE
1167 rb_proc_dup(VALUE self)
1168 {
1169     VALUE procval;
1170     rb_proc_t *src;
1171
1172     GetProcPtr(self, src);
1173     procval = proc_create(rb_obj_class(self), &src->block, src->is_from_method, src->is_lambda);
1174     if (RB_OBJ_SHAREABLE_P(self)) FL_SET_RAW(procval, RUBY_FL_SHAREABLE);
1175     RB_GC_GUARD(self); /* for: body = rb_proc_dup(body) */
1176     return procval;
1177 }
1178
1179 struct collect_outer_variable_name_data {
1180     VALUE ary;
1181     VALUE read_only;
1182     bool yield;
1183     bool isolate;
1184 };
1185
1186 static VALUE
1187 ID2NUM(ID id)
1188 {
1189     if (SIZEOF_VOIDP > SIZEOF_LONG)
1190         return ULL2NUM(id);
1191     else
1192         return ULONG2NUM(id);
1193 }
1194
1195 static ID
1196 NUM2ID(VALUE num)
1197 {
1198     if (SIZEOF_VOIDP > SIZEOF_LONG)
1199         return (ID)NUM2ULL(num);
1200     else
1201         return (ID)NUM2ULONG(num);
1202 }
1203
1204 static enum rb_id_table_iterator_result
1205 collect_outer_variable_names(ID id, VALUE val, void *ptr)
1206 {
1207     struct collect_outer_variable_name_data *data = (struct collect_outer_variable_name_data *)ptr;
1208
1209     if (id == rb_intern("yield")) {
1210         data->yield = true;
1211     }
1212     else {
1213         VALUE *store;
1214         if (data->isolate ||
1215             val == Qtrue /* write */) {
1216             store = &data->ary;
1217         }
1218         else {
1219             store = &data->read_only;
1220         }
1221         if (*store == Qfalse) *store = rb_ary_new();
1222         rb_ary_push(*store, ID2NUM(id));
1223     }
1224     return ID_TABLE_CONTINUE;
1225 }
1226
1227 static const rb_env_t *
1228 env_copy(const VALUE *src_ep, VALUE read_only_variables)
1229 {
1230     const rb_env_t *src_env = (rb_env_t *)VM_ENV_ENVVAL(src_ep);
1231     VM_ASSERT(src_env->ep == src_ep);
1232
1233     VALUE *env_body = ZALLOC_N(VALUE, src_env->env_size); // fill with Qfalse
1234     VALUE *ep = &env_body[src_env->env_size - 2];
1235     const rb_env_t *copied_env = vm_env_new(ep, env_body, src_env->env_size, src_env->iseq);
1236
1237     // Copy after allocations above, since they can move objects in src_ep.
1238     RB_OBJ_WRITE(copied_env, &ep[VM_ENV_DATA_INDEX_ME_CREF], src_ep[VM_ENV_DATA_INDEX_ME_CREF]);
1239     ep[VM_ENV_DATA_INDEX_FLAGS] = src_ep[VM_ENV_DATA_INDEX_FLAGS] | VM_ENV_FLAG_ISOLATED;
1240     if (!VM_ENV_LOCAL_P(src_ep)) {
1241         VM_ENV_FLAGS_SET(ep, VM_ENV_FLAG_LOCAL);
1242     }
1243
1244     if (read_only_variables) {
1245         for (int i=RARRAY_LENINT(read_only_variables)-1; i>=0; i--) {
1246             ID id = NUM2ID(RARRAY_AREF(read_only_variables, i));
1247
1248             for (unsigned int j=0; j<ISEQ_BODY(src_env->iseq)->local_table_size; j++) {
1249                 if (id == ISEQ_BODY(src_env->iseq)->local_table[j]) {
1250                     VALUE v = src_env->env[j];
1251                     if (!rb_ractor_shareable_p(v)) {
1252                         VALUE name = rb_id2str(id);
1253                         VALUE msg = rb_sprintf("can not make shareable Proc because it can refer"
1254                                                " unshareable object %+" PRIsVALUE " from ", v);
1255                         if (name)
1256                             rb_str_catf(msg, "variable '%" PRIsVALUE "'", name);
1257                         else
1258                             rb_str_cat_cstr(msg, "a hidden variable");
1259                         rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg));
1260                     }
1261                     RB_OBJ_WRITE((VALUE)copied_env, &env_body[j], v);
1262                     rb_ary_delete_at(read_only_variables, i);
1263                     break;
1264                 }
1265             }
1266         }
1267     }
1268
1269     if (!VM_ENV_LOCAL_P(src_ep)) {
1270         const VALUE *prev_ep = VM_ENV_PREV_EP(src_env->ep);
1271         const rb_env_t *new_prev_env = env_copy(prev_ep, read_only_variables);
1272         ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_GUARDED_PREV_EP(new_prev_env->ep);
1273         RB_OBJ_WRITTEN(copied_env, Qundef, new_prev_env);
1274         VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_LOCAL);
1275     }
1276     else {
1277         ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_BLOCK_HANDLER_NONE;
1278     }
1279
1280     return copied_env;
1281 }
1282
1283 static void
1284 proc_isolate_env(VALUE self, rb_proc_t *proc, VALUE read_only_variables)
1285 {
1286     const struct rb_captured_block *captured = &proc->block.as.captured;
1287     const rb_env_t *env = env_copy(captured->ep, read_only_variables);
1288     *((const VALUE **)&proc->block.as.captured.ep) = env->ep;
1289     RB_OBJ_WRITTEN(self, Qundef, env);
1290 }
1291
1292 static VALUE
1293 proc_shared_outer_variables(struct rb_id_table *outer_variables, bool isolate, const char *message)
1294 {
1295     struct collect_outer_variable_name_data data = {
1296         .isolate = isolate,
1297         .ary = Qfalse,
1298         .read_only = Qfalse,
1299         .yield = false,
1300     };
1301     rb_id_table_foreach(outer_variables, collect_outer_variable_names, (void *)&data);
1302
1303     if (data.ary != Qfalse) {
1304         VALUE str = rb_sprintf("can not %s because it accesses outer variables", message);
1305         VALUE ary = data.ary;
1306         const char *sep = " (";
1307         for (long i = 0; i < RARRAY_LEN(ary); i++) {
1308             VALUE name = rb_id2str(NUM2ID(RARRAY_AREF(ary, i)));
1309             if (!name) continue;
1310             rb_str_cat_cstr(str, sep);
1311             sep = ", ";
1312             rb_str_append(str, name);
1313         }
1314         if (*sep == ',') rb_str_cat_cstr(str, ")");
1315         rb_str_cat_cstr(str, data.yield ? " and uses 'yield'." : ".");
1316         rb_exc_raise(rb_exc_new_str(rb_eArgError, str));
1317     }
1318     else if (data.yield) {
1319         rb_raise(rb_eArgError, "can not %s because it uses 'yield'.", message);
1320     }
1321
1322     return data.read_only;
1323 }
1324
1325 VALUE
1326 rb_proc_isolate_bang(VALUE self)
1327 {
1328     const rb_iseq_t *iseq = vm_proc_iseq(self);
1329
1330     if (iseq) {
1331         rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self);
1332         if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet");
1333
1334         if (ISEQ_BODY(iseq)->outer_variables) {
1335             proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, true, "isolate a Proc");
1336         }
1337
1338         proc_isolate_env(self, proc, Qfalse);
1339         proc->is_isolated = TRUE;
1340     }
1341
1342     FL_SET_RAW(self, RUBY_FL_SHAREABLE);
1343     return self;
1344 }
1345
1346 VALUE
1347 rb_proc_isolate(VALUE self)
1348 {
1349     VALUE dst = rb_proc_dup(self);
1350     rb_proc_isolate_bang(dst);
1351     return dst;
1352 }
1353
1354 VALUE
1355 rb_proc_ractor_make_shareable(VALUE self)
1356 {
1357     const rb_iseq_t *iseq = vm_proc_iseq(self);
1358
1359     if (iseq) {
1360         rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self);
1361         if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet");
1362
1363         if (!rb_ractor_shareable_p(vm_block_self(&proc->block))) {
1364             rb_raise(rb_eRactorIsolationError,
1365                      "Proc's self is not shareable: %" PRIsVALUE,
1366                      self);
1367         }
1368
1369         VALUE read_only_variables = Qfalse;
1370
1371         if (ISEQ_BODY(iseq)->outer_variables) {
1372             read_only_variables =
1373                 proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, false, "make a Proc shareable");
1374         }
1375
1376         proc_isolate_env(self, proc, read_only_variables);
1377         proc->is_isolated = TRUE;
1378     }
1379
1380     FL_SET_RAW(self, RUBY_FL_SHAREABLE);
1381     return self;
1382 }
1383
1384 VALUE
1385 rb_vm_make_proc_lambda(const rb_execution_context_t *ec, const struct rb_captured_block *captured, VALUE klass, int8_t is_lambda)
1386 {
1387     VALUE procval;
1388     enum imemo_type code_type = imemo_type(captured->code.val);
1389
1390     if (!VM_ENV_ESCAPED_P(captured->ep)) {
1391         rb_control_frame_t *cfp = VM_CAPTURED_BLOCK_TO_CFP(captured);
1392         vm_make_env_object(ec, cfp);
1393     }
1394
1395     VM_ASSERT(VM_EP_IN_HEAP_P(ec, captured->ep));
1396     VM_ASSERT(code_type == imemo_iseq || code_type == imemo_ifunc);
1397
1398     procval = vm_proc_create_from_captured(klass, captured,
1399                                            code_type == imemo_iseq ? block_type_iseq : block_type_ifunc,
1400                                            FALSE, is_lambda);
1401
1402     if (code_type == imemo_ifunc) {
1403         struct vm_ifunc *ifunc = (struct vm_ifunc *)captured->code.val;
1404         if (ifunc->svar_lep) {
1405             VALUE ep0 = ifunc->svar_lep[0];
1406             if (RB_TYPE_P(ep0, T_IMEMO) && imemo_type_p(ep0, imemo_env)) {
1407                 // `ep0 == imemo_env` means this ep is escaped to heap (in env object).
1408                 const rb_env_t *env = (const rb_env_t *)ep0;
1409                 ifunc->svar_lep = (VALUE *)env->ep;
1410             }
1411             else {
1412                 VM_ASSERT(FIXNUM_P(ep0));
1413                 if (ep0 & VM_ENV_FLAG_ESCAPED) {
1414                     // ok. do nothing
1415                 }
1416                 else {
1417                     ifunc->svar_lep = NULL;
1418                 }
1419             }
1420         }
1421     }
1422
1423     return procval;
1424 }
1425
1426 /* Binding */
1427
1428 VALUE
1429 rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *src_cfp)
1430 {
1431     rb_control_frame_t *cfp = rb_vm_get_binding_creatable_next_cfp(ec, src_cfp);
1432     rb_control_frame_t *ruby_level_cfp = rb_vm_get_ruby_level_next_cfp(ec, src_cfp);
1433     VALUE bindval, envval;
1434     rb_binding_t *bind;
1435
1436     if (cfp == 0 || ruby_level_cfp == 0) {
1437         rb_raise(rb_eRuntimeError, "Can't create Binding Object on top of Fiber.");
1438     }
1439     if (!VM_FRAME_RUBYFRAME_P(src_cfp) &&
1440         !VM_FRAME_RUBYFRAME_P(RUBY_VM_PREVIOUS_CONTROL_FRAME(src_cfp))) {
1441         rb_raise(rb_eRuntimeError, "Cannot create Binding object for non-Ruby caller");
1442     }
1443
1444     envval = vm_make_env_object(ec, cfp);
1445     bindval = rb_binding_alloc(rb_cBinding);
1446     GetBindingPtr(bindval, bind);
1447     vm_bind_update_env(bindval, bind, envval);
1448     RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self);
1449     RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, cfp->iseq);
1450     RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(ruby_level_cfp->iseq)->location.pathobj);
1451     bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp);
1452
1453     return bindval;
1454 }
1455
1456 const VALUE *
1457 rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const ID *dynvars)
1458 {
1459     VALUE envval, pathobj = bind->pathobj;
1460     VALUE path = pathobj_path(pathobj);
1461     VALUE realpath = pathobj_realpath(pathobj);
1462     const struct rb_block *base_block;
1463     const rb_env_t *env;
1464     rb_execution_context_t *ec = GET_EC();
1465     const rb_iseq_t *base_iseq, *iseq;
1466     rb_node_scope_t tmp_node;
1467
1468     if (dyncount < 0) return 0;
1469
1470     base_block = &bind->block;
1471     base_iseq = vm_block_iseq(base_block);
1472
1473     VALUE idtmp = 0;
1474     rb_ast_id_table_t *dyns = ALLOCV(idtmp, sizeof(rb_ast_id_table_t) + dyncount * sizeof(ID));
1475     dyns->size = dyncount;
1476     MEMCPY(dyns->ids, dynvars, ID, dyncount);
1477
1478     rb_node_init(RNODE(&tmp_node), NODE_SCOPE);
1479     tmp_node.nd_tbl = dyns;
1480     tmp_node.nd_body = 0;
1481     tmp_node.nd_args = 0;
1482
1483     VALUE ast_value = rb_ruby_ast_new(RNODE(&tmp_node));
1484
1485     if (base_iseq) {
1486         iseq = rb_iseq_new(ast_value, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL);
1487     }
1488     else {
1489         VALUE tempstr = rb_fstring_lit("<temp>");
1490         iseq = rb_iseq_new_top(ast_value, tempstr, tempstr, tempstr, NULL);
1491     }
1492     tmp_node.nd_tbl = 0; /* reset table */
1493     ALLOCV_END(idtmp);
1494
1495     vm_set_eval_stack(ec, iseq, 0, base_block);
1496     vm_bind_update_env(bindval, bind, envval = vm_make_env_object(ec, ec->cfp));
1497     rb_vm_pop_frame(ec);
1498
1499     env = (const rb_env_t *)envval;
1500     return env->env;
1501 }
1502
1503 /* C -> Ruby: block */
1504
1505 static inline VALUE
1506 invoke_block(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_cref_t *cref, VALUE type, int opt_pc)
1507 {
1508     int arg_size = ISEQ_BODY(iseq)->param.size;
1509
1510     vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_FINISH, self,
1511                   VM_GUARDED_PREV_EP(captured->ep),
1512                   (VALUE)cref, /* cref or method */
1513                   ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
1514                   ec->cfp->sp + arg_size,
1515                   ISEQ_BODY(iseq)->local_table_size - arg_size,
1516                   ISEQ_BODY(iseq)->stack_max);
1517     return vm_exec(ec);
1518 }
1519
1520 static VALUE
1521 invoke_bmethod(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_callable_method_entry_t *me, VALUE type, int opt_pc)
1522 {
1523     /* bmethod call from outside the VM */
1524     int arg_size = ISEQ_BODY(iseq)->param.size;
1525     VALUE ret;
1526
1527     VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
1528
1529     vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_BMETHOD, self,
1530                   VM_GUARDED_PREV_EP(captured->ep),
1531                   (VALUE)me,
1532                   ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
1533                   ec->cfp->sp + 1 /* self */ + arg_size,
1534                   ISEQ_BODY(iseq)->local_table_size - arg_size,
1535                   ISEQ_BODY(iseq)->stack_max);
1536
1537     VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);
1538     ret = vm_exec(ec);
1539
1540     return ret;
1541 }
1542
1543 ALWAYS_INLINE(static VALUE
1544               invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured,
1545                                        VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
1546                                        const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me));
1547
1548 static inline VALUE
1549 invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured,
1550                          VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
1551                          const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me)
1552 {
1553     const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
1554     int opt_pc;
1555     VALUE type = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
1556     rb_control_frame_t *cfp = ec->cfp;
1557     VALUE *sp = cfp->sp;
1558     int flags = (kw_splat ? VM_CALL_KW_SPLAT : 0);
1559     VALUE *use_argv = (VALUE *)argv;
1560     VALUE av[2];
1561
1562     stack_check(ec);
1563
1564     if (UNLIKELY(argc > VM_ARGC_STACK_MAX) &&
1565         (VM_ARGC_STACK_MAX >= 1 ||
1566          /* Skip ruby array for potential autosplat case */
1567          (argc != 1 || is_lambda))) {
1568         use_argv = vm_argv_ruby_array(av, argv, &flags, &argc, kw_splat);
1569     }
1570
1571     CHECK_VM_STACK_OVERFLOW(cfp, argc + 1);
1572     vm_check_canary(ec, sp);
1573
1574     VALUE *stack_argv = sp;
1575     if (me) {
1576         *sp = self; // bemthods need `self` on the VM stack
1577         stack_argv++;
1578     }
1579     cfp->sp = stack_argv + argc;
1580     MEMCPY(stack_argv, use_argv, VALUE, argc); // restrict: new stack space
1581
1582     opt_pc = vm_yield_setup_args(ec, iseq, argc, stack_argv, flags, passed_block_handler,
1583                                  (is_lambda ? arg_setup_method : arg_setup_block));
1584     cfp->sp = sp;
1585
1586     if (me == NULL) {
1587         return invoke_block(ec, iseq, self, captured, cref, type, opt_pc);
1588     }
1589     else {
1590         return invoke_bmethod(ec, iseq, self, captured, me, type, opt_pc);
1591     }
1592 }
1593
1594 static inline VALUE
1595 invoke_block_from_c_bh(rb_execution_context_t *ec, VALUE block_handler,
1596                        int argc, const VALUE *argv,
1597                        int kw_splat, VALUE passed_block_handler, const rb_cref_t *cref,
1598                        int is_lambda, int force_blockarg)
1599 {
1600   again:
1601     switch (vm_block_handler_type(block_handler)) {
1602       case block_handler_type_iseq:
1603         {
1604             const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
1605             return invoke_iseq_block_from_c(ec, captured, captured->self,
1606                                             argc, argv, kw_splat, passed_block_handler,
1607                                             cref, is_lambda, NULL);
1608         }
1609       case block_handler_type_ifunc:
1610         return vm_yield_with_cfunc(ec, VM_BH_TO_IFUNC_BLOCK(block_handler),
1611                                    VM_BH_TO_IFUNC_BLOCK(block_handler)->self,
1612                                    argc, argv, kw_splat, passed_block_handler, NULL);
1613       case block_handler_type_symbol:
1614         return vm_yield_with_symbol(ec, VM_BH_TO_SYMBOL(block_handler),
1615                                     argc, argv, kw_splat, passed_block_handler);
1616       case block_handler_type_proc:
1617         if (force_blockarg == FALSE) {
1618             is_lambda = block_proc_is_lambda(VM_BH_TO_PROC(block_handler));
1619         }
1620         block_handler = vm_proc_to_block_handler(VM_BH_TO_PROC(block_handler));
1621         goto again;
1622     }
1623     VM_UNREACHABLE(invoke_block_from_c_splattable);
1624     return Qundef;
1625 }
1626
1627 static inline VALUE
1628 check_block_handler(rb_execution_context_t *ec)
1629 {
1630     VALUE block_handler = VM_CF_BLOCK_HANDLER(ec->cfp);
1631     vm_block_handler_verify(block_handler);
1632     if (UNLIKELY(block_handler == VM_BLOCK_HANDLER_NONE)) {
1633         rb_vm_localjump_error("no block given", Qnil, 0);
1634     }
1635
1636     return block_handler;
1637 }
1638
1639 static VALUE
1640 vm_yield_with_cref(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat, const rb_cref_t *cref, int is_lambda)
1641 {
1642     return invoke_block_from_c_bh(ec, check_block_handler(ec),
1643                                   argc, argv, kw_splat, VM_BLOCK_HANDLER_NONE,
1644                                   cref, is_lambda, FALSE);
1645 }
1646
1647 static VALUE
1648 vm_yield(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat)
1649 {
1650     return vm_yield_with_cref(ec, argc, argv, kw_splat, NULL, FALSE);
1651 }
1652
1653 static VALUE
1654 vm_yield_with_block(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE block_handler, int kw_splat)
1655 {
1656     return invoke_block_from_c_bh(ec, check_block_handler(ec),
1657                                   argc, argv, kw_splat, block_handler,
1658                                   NULL, FALSE, FALSE);
1659 }
1660
1661 static VALUE
1662 vm_yield_force_blockarg(rb_execution_context_t *ec, VALUE args)
1663 {
1664     return invoke_block_from_c_bh(ec, check_block_handler(ec), 1, &args,
1665                                   RB_NO_KEYWORDS, VM_BLOCK_HANDLER_NONE, NULL, FALSE, TRUE);
1666 }
1667
1668 ALWAYS_INLINE(static VALUE
1669               invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc,
1670                                        VALUE self, int argc, const VALUE *argv,
1671                                        int kw_splat, VALUE passed_block_handler, int is_lambda,
1672                                        const rb_callable_method_entry_t *me));
1673
1674 static inline VALUE
1675 invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc,
1676                          VALUE self, int argc, const VALUE *argv,
1677                          int kw_splat, VALUE passed_block_handler, int is_lambda,
1678                          const rb_callable_method_entry_t *me)
1679 {
1680     const struct rb_block *block = &proc->block;
1681
1682   again:
1683     switch (vm_block_type(block)) {
1684       case block_type_iseq:
1685         return invoke_iseq_block_from_c(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, NULL, is_lambda, me);
1686       case block_type_ifunc:
1687         if (kw_splat == 1) {
1688             VALUE keyword_hash = argv[argc-1];
1689             if (!RB_TYPE_P(keyword_hash, T_HASH)) {
1690                 keyword_hash = rb_to_hash_type(keyword_hash);
1691             }
1692             if (RHASH_EMPTY_P(keyword_hash)) {
1693                 argc--;
1694             }
1695             else {
1696                 ((VALUE *)argv)[argc-1] = rb_hash_dup(keyword_hash);
1697             }
1698         }
1699         return vm_yield_with_cfunc(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, me);
1700       case block_type_symbol:
1701         return vm_yield_with_symbol(ec, block->as.symbol, argc, argv, kw_splat, passed_block_handler);
1702       case block_type_proc:
1703         is_lambda = block_proc_is_lambda(block->as.proc);
1704         block = vm_proc_block(block->as.proc);
1705         goto again;
1706     }
1707     VM_UNREACHABLE(invoke_block_from_c_proc);
1708     return Qundef;
1709 }
1710
1711 static VALUE
1712 vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
1713                int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler)
1714 {
1715     return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler, proc->is_lambda, NULL);
1716 }
1717
1718 VALUE
1719 rb_vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
1720                      int argc, const VALUE *argv, int kw_splat, VALUE block_handler, const rb_callable_method_entry_t *me)
1721 {
1722     return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, block_handler, TRUE, me);
1723 }
1724
1725 VALUE
1726 rb_vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc,
1727                   int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler)
1728 {
1729     VALUE self = vm_block_self(&proc->block);
1730     vm_block_handler_verify(passed_block_handler);
1731
1732     if (proc->is_from_method) {
1733         return rb_vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL);
1734     }
1735     else {
1736         return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler);
1737     }
1738 }
1739
1740 VALUE
1741 rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
1742                             int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler)
1743 {
1744     vm_block_handler_verify(passed_block_handler);
1745
1746     if (proc->is_from_method) {
1747         return rb_vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL);
1748     }
1749     else {
1750         return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler);
1751     }
1752 }
1753
1754 /* special variable */
1755
1756 VALUE *
1757 rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
1758 {
1759     while (cfp->pc == 0 || cfp->iseq == 0) {
1760         if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) {
1761             struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq;
1762             return ifunc->svar_lep;
1763         }
1764         else {
1765             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1766         }
1767
1768         if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
1769             return NULL;
1770         }
1771     }
1772
1773     return (VALUE *)VM_CF_LEP(cfp);
1774 }
1775
1776 static VALUE
1777 vm_cfp_svar_get(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key)
1778 {
1779     return lep_svar_get(ec, rb_vm_svar_lep(ec, cfp), key);
1780 }
1781
1782 static void
1783 vm_cfp_svar_set(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key, const VALUE val)
1784 {
1785     lep_svar_set(ec, rb_vm_svar_lep(ec, cfp), key, val);
1786 }
1787
1788 static VALUE
1789 vm_svar_get(const rb_execution_context_t *ec, VALUE key)
1790 {
1791     return vm_cfp_svar_get(ec, ec->cfp, key);
1792 }
1793
1794 static void
1795 vm_svar_set(const rb_execution_context_t *ec, VALUE key, VALUE val)
1796 {
1797     vm_cfp_svar_set(ec, ec->cfp, key, val);
1798 }
1799
1800 VALUE
1801 rb_backref_get(void)
1802 {
1803     return vm_svar_get(GET_EC(), VM_SVAR_BACKREF);
1804 }
1805
1806 void
1807 rb_backref_set(VALUE val)
1808 {
1809     vm_svar_set(GET_EC(), VM_SVAR_BACKREF, val);
1810 }
1811
1812 VALUE
1813 rb_lastline_get(void)
1814 {
1815     return vm_svar_get(GET_EC(), VM_SVAR_LASTLINE);
1816 }
1817
1818 void
1819 rb_lastline_set(VALUE val)
1820 {
1821     vm_svar_set(GET_EC(), VM_SVAR_LASTLINE, val);
1822 }
1823
1824 void
1825 rb_lastline_set_up(VALUE val, unsigned int up)
1826 {
1827     rb_control_frame_t * cfp = GET_EC()->cfp;
1828
1829     for(unsigned int i = 0; i < up; i++) {
1830         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1831     }
1832     vm_cfp_svar_set(GET_EC(), cfp, VM_SVAR_LASTLINE, val);
1833 }
1834
1835 /* misc */
1836
1837 const char *
1838 rb_sourcefile(void)
1839 {
1840     const rb_execution_context_t *ec = GET_EC();
1841     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1842
1843     if (cfp) {
1844         return RSTRING_PTR(rb_iseq_path(cfp->iseq));
1845     }
1846     else {
1847         return 0;
1848     }
1849 }
1850
1851 int
1852 rb_sourceline(void)
1853 {
1854     const rb_execution_context_t *ec = GET_EC();
1855     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1856
1857     if (cfp) {
1858         return rb_vm_get_sourceline(cfp);
1859     }
1860     else {
1861         return 0;
1862     }
1863 }
1864
1865 VALUE
1866 rb_source_location(int *pline)
1867 {
1868     const rb_execution_context_t *ec = GET_EC();
1869     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1870
1871     if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) {
1872         if (pline) *pline = rb_vm_get_sourceline(cfp);
1873         return rb_iseq_path(cfp->iseq);
1874     }
1875     else {
1876         if (pline) *pline = 0;
1877         return Qnil;
1878     }
1879 }
1880
1881 const char *
1882 rb_source_location_cstr(int *pline)
1883 {
1884     VALUE path = rb_source_location(pline);
1885     if (NIL_P(path)) return NULL;
1886     return RSTRING_PTR(path);
1887 }
1888
1889 rb_cref_t *
1890 rb_vm_cref(void)
1891 {
1892     const rb_execution_context_t *ec = GET_EC();
1893     return vm_ec_cref(ec);
1894 }
1895
1896 rb_cref_t *
1897 rb_vm_cref_replace_with_duplicated_cref(void)
1898 {
1899     const rb_execution_context_t *ec = GET_EC();
1900     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1901     rb_cref_t *cref = vm_cref_replace_with_duplicated_cref(cfp->ep);
1902     ASSUME(cref);
1903     return cref;
1904 }
1905
1906 const rb_cref_t *
1907 rb_vm_cref_in_context(VALUE self, VALUE cbase)
1908 {
1909     const rb_execution_context_t *ec = GET_EC();
1910     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1911     const rb_cref_t *cref;
1912     if (!cfp || cfp->self != self) return NULL;
1913     if (!vm_env_cref_by_cref(cfp->ep)) return NULL;
1914     cref = vm_get_cref(cfp->ep);
1915     if (CREF_CLASS(cref) != cbase) return NULL;
1916     return cref;
1917 }
1918
1919 #if 0
1920 void
1921 debug_cref(rb_cref_t *cref)
1922 {
1923     while (cref) {
1924         dp(CREF_CLASS(cref));
1925         printf("%ld\n", CREF_VISI(cref));
1926         cref = CREF_NEXT(cref);
1927     }
1928 }
1929 #endif
1930
1931 VALUE
1932 rb_vm_cbase(void)
1933 {
1934     const rb_execution_context_t *ec = GET_EC();
1935     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1936
1937     if (cfp == 0) {
1938         rb_raise(rb_eRuntimeError, "Can't call on top of Fiber or Thread");
1939     }
1940     return vm_get_cbase(cfp->ep);
1941 }
1942
1943 /* jump */
1944
1945 static VALUE
1946 make_localjump_error(const char *mesg, VALUE value, int reason)
1947 {
1948     extern VALUE rb_eLocalJumpError;
1949     VALUE exc = rb_exc_new2(rb_eLocalJumpError, mesg);
1950     ID id;
1951
1952     switch (reason) {
1953       case TAG_BREAK:
1954         CONST_ID(id, "break");
1955         break;
1956       case TAG_REDO:
1957         CONST_ID(id, "redo");
1958         break;
1959       case TAG_RETRY:
1960         CONST_ID(id, "retry");
1961         break;
1962       case TAG_NEXT:
1963         CONST_ID(id, "next");
1964         break;
1965       case TAG_RETURN:
1966         CONST_ID(id, "return");
1967         break;
1968       default:
1969         CONST_ID(id, "noreason");
1970         break;
1971     }
1972     rb_iv_set(exc, "@exit_value", value);
1973     rb_iv_set(exc, "@reason", ID2SYM(id));
1974     return exc;
1975 }
1976
1977 void
1978 rb_vm_localjump_error(const char *mesg, VALUE value, int reason)
1979 {
1980     VALUE exc = make_localjump_error(mesg, value, reason);
1981     rb_exc_raise(exc);
1982 }
1983
1984 VALUE
1985 rb_vm_make_jump_tag_but_local_jump(enum ruby_tag_type state, VALUE val)
1986 {
1987     const char *mesg;
1988
1989     switch (state) {
1990       case TAG_RETURN:
1991         mesg = "unexpected return";
1992         break;
1993       case TAG_BREAK:
1994         mesg = "unexpected break";
1995         break;
1996       case TAG_NEXT:
1997         mesg = "unexpected next";
1998         break;
1999       case TAG_REDO:
2000         mesg = "unexpected redo";
2001         val = Qnil;
2002         break;
2003       case TAG_RETRY:
2004         mesg = "retry outside of rescue clause";
2005         val = Qnil;
2006         break;
2007       default:
2008         return Qnil;
2009     }
2010     if (UNDEF_P(val)) {
2011         val = GET_EC()->tag->retval;
2012     }
2013     return make_localjump_error(mesg, val, state);
2014 }
2015
2016 void
2017 rb_vm_jump_tag_but_local_jump(enum ruby_tag_type state)
2018 {
2019     VALUE exc = rb_vm_make_jump_tag_but_local_jump(state, Qundef);
2020     if (!NIL_P(exc)) rb_exc_raise(exc);
2021     EC_JUMP_TAG(GET_EC(), state);
2022 }
2023
2024 static rb_control_frame_t *
2025 next_not_local_frame(rb_control_frame_t *cfp)
2026 {
2027     while (VM_ENV_LOCAL_P(cfp->ep)) {
2028         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2029     }
2030     return cfp;
2031 }
2032
2033 NORETURN(static void vm_iter_break(rb_execution_context_t *ec, VALUE val));
2034
2035 static void
2036 vm_iter_break(rb_execution_context_t *ec, VALUE val)
2037 {
2038     rb_control_frame_t *cfp = next_not_local_frame(ec->cfp);
2039     const VALUE *ep = VM_CF_PREV_EP(cfp);
2040     const rb_control_frame_t *target_cfp = rb_vm_search_cf_from_ep(ec, cfp, ep);
2041
2042     if (!target_cfp) {
2043         rb_vm_localjump_error("unexpected break", val, TAG_BREAK);
2044     }
2045
2046     ec->errinfo = (VALUE)THROW_DATA_NEW(val, target_cfp, TAG_BREAK);
2047     EC_JUMP_TAG(ec, TAG_BREAK);
2048 }
2049
2050 void
2051 rb_iter_break(void)
2052 {
2053     vm_iter_break(GET_EC(), Qnil);
2054 }
2055
2056 void
2057 rb_iter_break_value(VALUE val)
2058 {
2059     vm_iter_break(GET_EC(), val);
2060 }
2061
2062 /* optimization: redefine management */
2063
2064 short ruby_vm_redefined_flag[BOP_LAST_];
2065 static st_table *vm_opt_method_def_table = 0;
2066 static st_table *vm_opt_mid_table = 0;
2067
2068 void
2069 rb_free_vm_opt_tables(void)
2070 {
2071     st_free_table(vm_opt_method_def_table);
2072     st_free_table(vm_opt_mid_table);
2073 }
2074
2075 static int
2076 vm_redefinition_check_flag(VALUE klass)
2077 {
2078     if (klass == rb_cInteger) return INTEGER_REDEFINED_OP_FLAG;
2079     if (klass == rb_cFloat)  return FLOAT_REDEFINED_OP_FLAG;
2080     if (klass == rb_cString) return STRING_REDEFINED_OP_FLAG;
2081     if (klass == rb_cArray)  return ARRAY_REDEFINED_OP_FLAG;
2082     if (klass == rb_cHash)   return HASH_REDEFINED_OP_FLAG;
2083     if (klass == rb_cSymbol) return SYMBOL_REDEFINED_OP_FLAG;
2084 #if 0
2085     if (klass == rb_cTime)   return TIME_REDEFINED_OP_FLAG;
2086 #endif
2087     if (klass == rb_cRegexp) return REGEXP_REDEFINED_OP_FLAG;
2088     if (klass == rb_cNilClass) return NIL_REDEFINED_OP_FLAG;
2089     if (klass == rb_cTrueClass) return TRUE_REDEFINED_OP_FLAG;
2090     if (klass == rb_cFalseClass) return FALSE_REDEFINED_OP_FLAG;
2091     if (klass == rb_cProc) return PROC_REDEFINED_OP_FLAG;
2092     return 0;
2093 }
2094
2095 int
2096 rb_vm_check_optimizable_mid(VALUE mid)
2097 {
2098     if (!vm_opt_mid_table) {
2099       return FALSE;
2100     }
2101
2102     return st_lookup(vm_opt_mid_table, mid, NULL);
2103 }
2104
2105 static int
2106 vm_redefinition_check_method_type(const rb_method_entry_t *me)
2107 {
2108     if (me->called_id != me->def->original_id) {
2109         return FALSE;
2110     }
2111
2112     if (METHOD_ENTRY_BASIC(me)) return TRUE;
2113
2114     const rb_method_definition_t *def = me->def;
2115     switch (def->type) {
2116       case VM_METHOD_TYPE_CFUNC:
2117       case VM_METHOD_TYPE_OPTIMIZED:
2118         return TRUE;
2119       default:
2120         return FALSE;
2121     }
2122 }
2123
2124 static void
2125 rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass)
2126 {
2127     st_data_t bop;
2128     if (RB_TYPE_P(klass, T_ICLASS) && FL_TEST(klass, RICLASS_IS_ORIGIN) &&
2129             RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
2130        klass = RBASIC_CLASS(klass);
2131     }
2132     if (vm_redefinition_check_method_type(me)) {
2133         if (st_lookup(vm_opt_method_def_table, (st_data_t)me->def, &bop)) {
2134             int flag = vm_redefinition_check_flag(klass);
2135             if (flag != 0) {
2136                 rb_category_warn(
2137                     RB_WARN_CATEGORY_PERFORMANCE,
2138                     "Redefining '%s#%s' disables interpreter and JIT optimizations",
2139                     rb_class2name(me->owner),
2140                     rb_id2name(me->called_id)
2141                 );
2142                 rb_yjit_bop_redefined(flag, (enum ruby_basic_operators)bop);
2143                 rb_rjit_bop_redefined(flag, (enum ruby_basic_operators)bop);
2144                 ruby_vm_redefined_flag[bop] |= flag;
2145             }
2146         }
2147     }
2148 }
2149
2150 static enum rb_id_table_iterator_result
2151 check_redefined_method(ID mid, VALUE value, void *data)
2152 {
2153     VALUE klass = (VALUE)data;
2154     const rb_method_entry_t *me = (rb_method_entry_t *)value;
2155     const rb_method_entry_t *newme = rb_method_entry(klass, mid);
2156
2157     if (newme != me) rb_vm_check_redefinition_opt_method(me, me->owner);
2158
2159     return ID_TABLE_CONTINUE;
2160 }
2161
2162 void
2163 rb_vm_check_redefinition_by_prepend(VALUE klass)
2164 {
2165     if (!vm_redefinition_check_flag(klass)) return;
2166     rb_id_table_foreach(RCLASS_M_TBL(RCLASS_ORIGIN(klass)), check_redefined_method, (void *)klass);
2167 }
2168
2169 static void
2170 add_opt_method_entry_bop(const rb_method_entry_t *me, ID mid, enum ruby_basic_operators bop)
2171 {
2172     st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop);
2173     st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue);
2174 }
2175
2176 static void
2177 add_opt_method(VALUE klass, ID mid, enum ruby_basic_operators bop)
2178 {
2179     const rb_method_entry_t *me = rb_method_entry_at(klass, mid);
2180
2181     if (me && vm_redefinition_check_method_type(me)) {
2182         add_opt_method_entry_bop(me, mid, bop);
2183     }
2184     else {
2185         rb_bug("undefined optimized method: %s", rb_id2name(mid));
2186     }
2187 }
2188
2189 static enum ruby_basic_operators vm_redefinition_bop_for_id(ID mid);
2190
2191 static void
2192 add_opt_method_entry(const rb_method_entry_t *me)
2193 {
2194     if (me && vm_redefinition_check_method_type(me)) {
2195         ID mid = me->called_id;
2196         enum ruby_basic_operators bop = vm_redefinition_bop_for_id(mid);
2197         if ((int)bop >= 0) {
2198             add_opt_method_entry_bop(me, mid, bop);
2199         }
2200     }
2201 }
2202
2203 static void
2204 vm_init_redefined_flag(void)
2205 {
2206     ID mid;
2207     enum ruby_basic_operators bop;
2208
2209 #define OP(mid_, bop_) (mid = id##mid_, bop = BOP_##bop_, ruby_vm_redefined_flag[bop] = 0)
2210 #define C(k) add_opt_method(rb_c##k, mid, bop)
2211     OP(PLUS, PLUS), (C(Integer), C(Float), C(String), C(Array));
2212     OP(MINUS, MINUS), (C(Integer), C(Float));
2213     OP(MULT, MULT), (C(Integer), C(Float));
2214     OP(DIV, DIV), (C(Integer), C(Float));
2215     OP(MOD, MOD), (C(Integer), C(Float));
2216     OP(Eq, EQ), (C(Integer), C(Float), C(String), C(Symbol));
2217     OP(Eqq, EQQ), (C(Integer), C(Float), C(Symbol), C(String),
2218                    C(NilClass), C(TrueClass), C(FalseClass));
2219     OP(LT, LT), (C(Integer), C(Float));
2220     OP(LE, LE), (C(Integer), C(Float));
2221     OP(GT, GT), (C(Integer), C(Float));
2222     OP(GE, GE), (C(Integer), C(Float));
2223     OP(LTLT, LTLT), (C(String), C(Array));
2224     OP(AREF, AREF), (C(Array), C(Hash), C(Integer));
2225     OP(ASET, ASET), (C(Array), C(Hash));
2226     OP(Length, LENGTH), (C(Array), C(String), C(Hash));
2227     OP(Size, SIZE), (C(Array), C(String), C(Hash));
2228     OP(EmptyP, EMPTY_P), (C(Array), C(String), C(Hash));
2229     OP(Succ, SUCC), (C(Integer), C(String));
2230     OP(EqTilde, MATCH), (C(Regexp), C(String));
2231     OP(Freeze, FREEZE), (C(String));
2232     OP(UMinus, UMINUS), (C(String));
2233     OP(Max, MAX), (C(Array));
2234     OP(Min, MIN), (C(Array));
2235     OP(Hash, HASH), (C(Array));
2236     OP(Call, CALL), (C(Proc));
2237     OP(And, AND), (C(Integer));
2238     OP(Or, OR), (C(Integer));
2239     OP(NilP, NIL_P), (C(NilClass));
2240     OP(Cmp, CMP), (C(Integer), C(Float), C(String));
2241     OP(Default, DEFAULT), (C(Hash));
2242 #undef C
2243 #undef OP
2244 }
2245
2246 static enum ruby_basic_operators
2247 vm_redefinition_bop_for_id(ID mid)
2248 {
2249     switch (mid) {
2250 #define OP(mid_, bop_) case id##mid_: return BOP_##bop_
2251     OP(PLUS, PLUS);
2252     OP(MINUS, MINUS);
2253     OP(MULT, MULT);
2254     OP(DIV, DIV);
2255     OP(MOD, MOD);
2256     OP(Eq, EQ);
2257     OP(Eqq, EQQ);
2258     OP(LT, LT);
2259     OP(LE, LE);
2260     OP(GT, GT);
2261     OP(GE, GE);
2262     OP(LTLT, LTLT);
2263     OP(AREF, AREF);
2264     OP(ASET, ASET);
2265     OP(Length, LENGTH);
2266     OP(Size, SIZE);
2267     OP(EmptyP, EMPTY_P);
2268     OP(Succ, SUCC);
2269     OP(EqTilde, MATCH);
2270     OP(Freeze, FREEZE);
2271     OP(UMinus, UMINUS);
2272     OP(Max, MAX);
2273     OP(Min, MIN);
2274     OP(Hash, HASH);
2275     OP(Call, CALL);
2276     OP(And, AND);
2277     OP(Or, OR);
2278     OP(NilP, NIL_P);
2279     OP(Cmp, CMP);
2280     OP(Default, DEFAULT);
2281 #undef OP
2282     }
2283     return -1;
2284 }
2285
2286 /* for vm development */
2287
2288 #if VMDEBUG
2289 static const char *
2290 vm_frametype_name(const rb_control_frame_t *cfp)
2291 {
2292     switch (VM_FRAME_TYPE(cfp)) {
2293       case VM_FRAME_MAGIC_METHOD: return "method";
2294       case VM_FRAME_MAGIC_BLOCK:  return "block";
2295       case VM_FRAME_MAGIC_CLASS:  return "class";
2296       case VM_FRAME_MAGIC_TOP:    return "top";
2297       case VM_FRAME_MAGIC_CFUNC:  return "cfunc";
2298       case VM_FRAME_MAGIC_IFUNC:  return "ifunc";
2299       case VM_FRAME_MAGIC_EVAL:   return "eval";
2300       case VM_FRAME_MAGIC_RESCUE: return "rescue";
2301       default:
2302         rb_bug("unknown frame");
2303     }
2304 }
2305 #endif
2306
2307 static VALUE
2308 frame_return_value(const struct vm_throw_data *err)
2309 {
2310     if (THROW_DATA_P(err) &&
2311         THROW_DATA_STATE(err) == TAG_BREAK &&
2312         THROW_DATA_CONSUMED_P(err) == FALSE) {
2313         return THROW_DATA_VAL(err);
2314     }
2315     else {
2316         return Qnil;
2317     }
2318 }
2319
2320 #if 0
2321 /* for debug */
2322 static const char *
2323 frame_name(const rb_control_frame_t *cfp)
2324 {
2325     unsigned long type = VM_FRAME_TYPE(cfp);
2326 #define C(t) if (type == VM_FRAME_MAGIC_##t) return #t
2327     C(METHOD);
2328     C(BLOCK);
2329     C(CLASS);
2330     C(TOP);
2331     C(CFUNC);
2332     C(PROC);
2333     C(IFUNC);
2334     C(EVAL);
2335     C(LAMBDA);
2336     C(RESCUE);
2337     C(DUMMY);
2338 #undef C
2339     return "unknown";
2340 }
2341 #endif
2342
2343 // cfp_returning_with_value:
2344 //     Whether cfp is the last frame in the unwinding process for a non-local return.
2345 static void
2346 hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, int state, struct vm_throw_data *err)
2347 {
2348     if (state == TAG_RAISE && RBASIC(err)->klass == rb_eSysStackError) {
2349         return;
2350     }
2351     else {
2352         const rb_iseq_t *iseq = ec->cfp->iseq;
2353         rb_hook_list_t *local_hooks = iseq->aux.exec.local_hooks;
2354
2355         switch (VM_FRAME_TYPE(ec->cfp)) {
2356           case VM_FRAME_MAGIC_METHOD:
2357             RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
2358             EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err));
2359
2360             if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_RETURN)) {
2361                 rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN,
2362                                         ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE);
2363             }
2364
2365             THROW_DATA_CONSUMED_SET(err);
2366             break;
2367           case VM_FRAME_MAGIC_BLOCK:
2368             if (VM_FRAME_BMETHOD_P(ec->cfp)) {
2369                 VALUE bmethod_return_value = frame_return_value(err);
2370                 if (cfp_returning_with_value) {
2371                     // Non-local return terminating at a BMETHOD control frame.
2372                     bmethod_return_value = THROW_DATA_VAL(err);
2373                 }
2374
2375
2376                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, bmethod_return_value);
2377                 if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) {
2378                     rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN,
2379                                             ec->cfp->self, 0, 0, 0, bmethod_return_value, TRUE);
2380                 }
2381
2382                 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(ec->cfp);
2383
2384                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self,
2385                                               rb_vm_frame_method_entry(ec->cfp)->def->original_id,
2386                                               rb_vm_frame_method_entry(ec->cfp)->called_id,
2387                                               rb_vm_frame_method_entry(ec->cfp)->owner,
2388                                               bmethod_return_value);
2389
2390                 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
2391                 local_hooks = me->def->body.bmethod.hooks;
2392
2393                 if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_RETURN)) {
2394                     rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN, ec->cfp->self,
2395                                             rb_vm_frame_method_entry(ec->cfp)->def->original_id,
2396                                             rb_vm_frame_method_entry(ec->cfp)->called_id,
2397                                             rb_vm_frame_method_entry(ec->cfp)->owner,
2398                                             bmethod_return_value, TRUE);
2399                 }
2400                 THROW_DATA_CONSUMED_SET(err);
2401             }
2402             else {
2403                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err));
2404                 if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) {
2405                     rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN,
2406                                             ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE);
2407                 }
2408                 THROW_DATA_CONSUMED_SET(err);
2409             }
2410             break;
2411           case VM_FRAME_MAGIC_CLASS:
2412             EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_END, ec->cfp->self, 0, 0, 0, Qnil);
2413             break;
2414         }
2415     }
2416 }
2417
2418 /* evaluator body */
2419
2420 /*                  finish
2421   VMe (h1)          finish
2422     VM              finish F1 F2
2423       cfunc         finish F1 F2 C1
2424         rb_funcall  finish F1 F2 C1
2425           VMe       finish F1 F2 C1
2426             VM      finish F1 F2 C1 F3
2427
2428   F1 - F3 : pushed by VM
2429   C1      : pushed by send insn (CFUNC)
2430
2431   struct CONTROL_FRAME {
2432     VALUE *pc;                  // cfp[0], program counter
2433     VALUE *sp;                  // cfp[1], stack pointer
2434     rb_iseq_t *iseq;            // cfp[2], iseq
2435     VALUE self;                 // cfp[3], self
2436     const VALUE *ep;            // cfp[4], env pointer
2437     const void *block_code;     // cfp[5], block code
2438   };
2439
2440   struct rb_captured_block {
2441     VALUE self;
2442     VALUE *ep;
2443     union code;
2444   };
2445
2446   struct METHOD_ENV {
2447     VALUE param0;
2448     ...
2449     VALUE paramN;
2450     VALUE lvar1;
2451     ...
2452     VALUE lvarM;
2453     VALUE cref;    // ep[-2]
2454     VALUE special; // ep[-1]
2455     VALUE flags;   // ep[ 0] == lep[0]
2456   };
2457
2458   struct BLOCK_ENV {
2459     VALUE block_param0;
2460     ...
2461     VALUE block_paramN;
2462     VALUE block_lvar1;
2463     ...
2464     VALUE block_lvarM;
2465     VALUE cref;    // ep[-2]
2466     VALUE special; // ep[-1]
2467     VALUE flags;   // ep[ 0]
2468   };
2469
2470   struct CLASS_ENV {
2471     VALUE class_lvar0;
2472     ...
2473     VALUE class_lvarN;
2474     VALUE cref;
2475     VALUE prev_ep; // for frame jump
2476     VALUE flags;
2477   };
2478
2479   struct C_METHOD_CONTROL_FRAME {
2480     VALUE *pc;                       // 0
2481     VALUE *sp;                       // stack pointer
2482     rb_iseq_t *iseq;                 // cmi
2483     VALUE self;                      // ?
2484     VALUE *ep;                       // ep == lep
2485     void *code;                      //
2486   };
2487
2488   struct C_BLOCK_CONTROL_FRAME {
2489     VALUE *pc;                       // point only "finish" insn
2490     VALUE *sp;                       // sp
2491     rb_iseq_t *iseq;                 // ?
2492     VALUE self;                      //
2493     VALUE *ep;                       // ep
2494     void *code;                      //
2495   };
2496  */
2497
2498 static inline VALUE
2499 vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo);
2500 static inline VALUE
2501 vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, struct rb_vm_tag *tag, VALUE result);
2502
2503 // for non-Emscripten Wasm build, use vm_exec with optimized setjmp for runtime performance
2504 #if defined(__wasm__) && !defined(__EMSCRIPTEN__)
2505
2506 struct rb_vm_exec_context {
2507     rb_execution_context_t *const ec;
2508     struct rb_vm_tag *const tag;
2509
2510     VALUE result;
2511 };
2512
2513 static void
2514 vm_exec_bottom_main(void *context)
2515 {
2516     struct rb_vm_exec_context *ctx = context;
2517     rb_execution_context_t *ec = ctx->ec;
2518
2519     ctx->result = vm_exec_loop(ec, TAG_NONE, ctx->tag, vm_exec_core(ec));
2520 }
2521
2522 static void
2523 vm_exec_bottom_rescue(void *context)
2524 {
2525     struct rb_vm_exec_context *ctx = context;
2526     rb_execution_context_t *ec = ctx->ec;
2527
2528     ctx->result = vm_exec_loop(ec, rb_ec_tag_state(ec), ctx->tag, ec->errinfo);
2529 }
2530 #endif
2531
2532 VALUE
2533 vm_exec(rb_execution_context_t *ec)
2534 {
2535     VALUE result = Qundef;
2536
2537     EC_PUSH_TAG(ec);
2538
2539     _tag.retval = Qnil;
2540
2541 #if defined(__wasm__) && !defined(__EMSCRIPTEN__)
2542     struct rb_vm_exec_context ctx = {
2543         .ec = ec,
2544         .tag = &_tag,
2545     };
2546     struct rb_wasm_try_catch try_catch;
2547
2548     EC_REPUSH_TAG();
2549
2550     rb_wasm_try_catch_init(&try_catch, vm_exec_bottom_main, vm_exec_bottom_rescue, &ctx);
2551
2552     rb_wasm_try_catch_loop_run(&try_catch, &RB_VM_TAG_JMPBUF_GET(_tag.buf));
2553
2554     result = ctx.result;
2555 #else
2556     enum ruby_tag_type state;
2557     if ((state = EC_EXEC_TAG()) == TAG_NONE) {
2558         if (UNDEF_P(result = jit_exec(ec))) {
2559             result = vm_exec_core(ec);
2560         }
2561         /* fallback to the VM */
2562         result = vm_exec_loop(ec, TAG_NONE, &_tag, result);
2563     }
2564     else {
2565         result = vm_exec_loop(ec, state, &_tag, ec->errinfo);
2566     }
2567 #endif
2568
2569     EC_POP_TAG();
2570     return result;
2571 }
2572
2573 static inline VALUE
2574 vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state,
2575              struct rb_vm_tag *tag, VALUE result)
2576 {
2577     if (state == TAG_NONE) { /* no jumps, result is discarded */
2578         goto vm_loop_start;
2579     }
2580
2581     rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY);
2582     while (UNDEF_P(result = vm_exec_handle_exception(ec, state, result))) {
2583         // caught a jump, exec the handler. JIT code in jit_exec_exception()
2584         // may return Qundef to run remaining frames with vm_exec_core().
2585         if (UNDEF_P(result = jit_exec_exception(ec))) {
2586             result = vm_exec_core(ec);
2587         }
2588       vm_loop_start:
2589         VM_ASSERT(ec->tag == tag);
2590         /* when caught `throw`, `tag.state` is set. */
2591         if ((state = tag->state) == TAG_NONE) break;
2592         tag->state = TAG_NONE;
2593     }
2594
2595     return result;
2596 }
2597
2598 static inline VALUE
2599 vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo)
2600 {
2601     struct vm_throw_data *err = (struct vm_throw_data *)errinfo;
2602
2603     for (;;) {
2604         unsigned int i;
2605         const struct iseq_catch_table_entry *entry;
2606         const struct iseq_catch_table *ct;
2607         unsigned long epc, cont_pc, cont_sp;
2608         const rb_iseq_t *catch_iseq;
2609         VALUE type;
2610         const rb_control_frame_t *escape_cfp;
2611
2612         cont_pc = cont_sp = 0;
2613         catch_iseq = NULL;
2614
2615         while (ec->cfp->pc == 0 || ec->cfp->iseq == 0) {
2616             if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) {
2617                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self,
2618                                               rb_vm_frame_method_entry(ec->cfp)->def->original_id,
2619                                               rb_vm_frame_method_entry(ec->cfp)->called_id,
2620                                               rb_vm_frame_method_entry(ec->cfp)->owner, Qnil);
2621                 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec,
2622                                                 rb_vm_frame_method_entry(ec->cfp)->owner,
2623                                                 rb_vm_frame_method_entry(ec->cfp)->def->original_id);
2624             }
2625             rb_vm_pop_frame(ec);
2626         }
2627
2628         rb_control_frame_t *const cfp = ec->cfp;
2629         epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded;
2630
2631         escape_cfp = NULL;
2632         if (state == TAG_BREAK || state == TAG_RETURN) {
2633             escape_cfp = THROW_DATA_CATCH_FRAME(err);
2634
2635             if (cfp == escape_cfp) {
2636                 if (state == TAG_RETURN) {
2637                     if (!VM_FRAME_FINISHED_P(cfp)) {
2638                         THROW_DATA_CATCH_FRAME_SET(err, cfp + 1);
2639                         THROW_DATA_STATE_SET(err, state = TAG_BREAK);
2640                     }
2641                     else {
2642                         ct = ISEQ_BODY(cfp->iseq)->catch_table;
2643                         if (ct) for (i = 0; i < ct->size; i++) {
2644                             entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2645                             if (entry->start < epc && entry->end >= epc) {
2646                                 if (entry->type == CATCH_TYPE_ENSURE) {
2647                                     catch_iseq = entry->iseq;
2648                                     cont_pc = entry->cont;
2649                                     cont_sp = entry->sp;
2650                                     break;
2651                                 }
2652                             }
2653                         }
2654                         if (catch_iseq == NULL) {
2655                             ec->errinfo = Qnil;
2656                             THROW_DATA_CATCH_FRAME_SET(err, cfp + 1);
2657                             // cfp == escape_cfp here so calling with cfp_returning_with_value = true
2658                             hook_before_rewind(ec, true, state, err);
2659                             rb_vm_pop_frame(ec);
2660                             return THROW_DATA_VAL(err);
2661                         }
2662                     }
2663                     /* through */
2664                 }
2665                 else {
2666                     /* TAG_BREAK */
2667                     *cfp->sp++ = THROW_DATA_VAL(err);
2668                     ec->errinfo = Qnil;
2669                     return Qundef;
2670                 }
2671             }
2672         }
2673
2674         if (state == TAG_RAISE) {
2675             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2676             if (ct) for (i = 0; i < ct->size; i++) {
2677                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2678                 if (entry->start < epc && entry->end >= epc) {
2679
2680                     if (entry->type == CATCH_TYPE_RESCUE ||
2681                         entry->type == CATCH_TYPE_ENSURE) {
2682                         catch_iseq = entry->iseq;
2683                         cont_pc = entry->cont;
2684                         cont_sp = entry->sp;
2685                         break;
2686                     }
2687                 }
2688             }
2689         }
2690         else if (state == TAG_RETRY) {
2691             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2692             if (ct) for (i = 0; i < ct->size; i++) {
2693                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2694                 if (entry->start < epc && entry->end >= epc) {
2695
2696                     if (entry->type == CATCH_TYPE_ENSURE) {
2697                         catch_iseq = entry->iseq;
2698                         cont_pc = entry->cont;
2699                         cont_sp = entry->sp;
2700                         break;
2701                     }
2702                     else if (entry->type == CATCH_TYPE_RETRY) {
2703                         const rb_control_frame_t *escape_cfp;
2704                         escape_cfp = THROW_DATA_CATCH_FRAME(err);
2705                         if (cfp == escape_cfp) {
2706                             cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont;
2707                             ec->errinfo = Qnil;
2708                             return Qundef;
2709                         }
2710                     }
2711                 }
2712             }
2713         }
2714         else if ((state == TAG_BREAK && !escape_cfp) ||
2715                  (state == TAG_REDO) ||
2716                  (state == TAG_NEXT)) {
2717             type = (const enum rb_catch_type[TAG_MASK]) {
2718                 [TAG_BREAK]  = CATCH_TYPE_BREAK,
2719                 [TAG_NEXT]   = CATCH_TYPE_NEXT,
2720                 [TAG_REDO]   = CATCH_TYPE_REDO,
2721                 /* otherwise = dontcare */
2722             }[state];
2723
2724             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2725             if (ct) for (i = 0; i < ct->size; i++) {
2726                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2727
2728                 if (entry->start < epc && entry->end >= epc) {
2729                     if (entry->type == CATCH_TYPE_ENSURE) {
2730                         catch_iseq = entry->iseq;
2731                         cont_pc = entry->cont;
2732                         cont_sp = entry->sp;
2733                         break;
2734                     }
2735                     else if (entry->type == type) {
2736                         cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont;
2737                         cfp->sp = vm_base_ptr(cfp) + entry->sp;
2738
2739                         if (state != TAG_REDO) {
2740                             *cfp->sp++ = THROW_DATA_VAL(err);
2741                         }
2742                         ec->errinfo = Qnil;
2743                         VM_ASSERT(ec->tag->state == TAG_NONE);
2744                         return Qundef;
2745                     }
2746                 }
2747             }
2748         }
2749         else {
2750             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2751             if (ct) for (i = 0; i < ct->size; i++) {
2752                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2753                 if (entry->start < epc && entry->end >= epc) {
2754
2755                     if (entry->type == CATCH_TYPE_ENSURE) {
2756                         catch_iseq = entry->iseq;
2757                         cont_pc = entry->cont;
2758                         cont_sp = entry->sp;
2759                         break;
2760                     }
2761                 }
2762             }
2763         }
2764
2765         if (catch_iseq != NULL) { /* found catch table */
2766             /* enter catch scope */
2767             const int arg_size = 1;
2768
2769             rb_iseq_check(catch_iseq);
2770             cfp->sp = vm_base_ptr(cfp) + cont_sp;
2771             cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + cont_pc;
2772
2773             /* push block frame */
2774             cfp->sp[0] = (VALUE)err;
2775             vm_push_frame(ec, catch_iseq, VM_FRAME_MAGIC_RESCUE,
2776                           cfp->self,
2777                           VM_GUARDED_PREV_EP(cfp->ep),
2778                           0, /* cref or me */
2779                           ISEQ_BODY(catch_iseq)->iseq_encoded,
2780                           cfp->sp + arg_size /* push value */,
2781                           ISEQ_BODY(catch_iseq)->local_table_size - arg_size,
2782                           ISEQ_BODY(catch_iseq)->stack_max);
2783
2784             state = 0;
2785             ec->tag->state = TAG_NONE;
2786             ec->errinfo = Qnil;
2787
2788             return Qundef;
2789         }
2790         else {
2791             hook_before_rewind(ec, (cfp == escape_cfp), state, err);
2792
2793             if (VM_FRAME_FINISHED_P(ec->cfp)) {
2794                 rb_vm_pop_frame(ec);
2795                 ec->errinfo = (VALUE)err;
2796                 ec->tag = ec->tag->prev;
2797                 EC_JUMP_TAG(ec, state);
2798             }
2799             else {
2800                 rb_vm_pop_frame(ec);
2801             }
2802         }
2803     }
2804 }
2805
2806 /* misc */
2807
2808 VALUE
2809 rb_iseq_eval(const rb_iseq_t *iseq)
2810 {
2811     rb_execution_context_t *ec = GET_EC();
2812     VALUE val;
2813     vm_set_top_stack(ec, iseq);
2814     val = vm_exec(ec);
2815     return val;
2816 }
2817
2818 VALUE
2819 rb_iseq_eval_main(const rb_iseq_t *iseq)
2820 {
2821     rb_execution_context_t *ec = GET_EC();
2822     VALUE val;
2823
2824     vm_set_main_stack(ec, iseq);
2825     val = vm_exec(ec);
2826     return val;
2827 }
2828
2829 int
2830 rb_vm_control_frame_id_and_class(const rb_control_frame_t *cfp, ID *idp, ID *called_idp, VALUE *klassp)
2831 {
2832     const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
2833
2834     if (me) {
2835         if (idp) *idp = me->def->original_id;
2836         if (called_idp) *called_idp = me->called_id;
2837         if (klassp) *klassp = me->owner;
2838         return TRUE;
2839     }
2840     else {
2841         return FALSE;
2842     }
2843 }
2844
2845 int
2846 rb_ec_frame_method_id_and_class(const rb_execution_context_t *ec, ID *idp, ID *called_idp, VALUE *klassp)
2847 {
2848     return rb_vm_control_frame_id_and_class(ec->cfp, idp, called_idp, klassp);
2849 }
2850
2851 int
2852 rb_frame_method_id_and_class(ID *idp, VALUE *klassp)
2853 {
2854     return rb_ec_frame_method_id_and_class(GET_EC(), idp, 0, klassp);
2855 }
2856
2857 VALUE
2858 rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg,
2859                  VALUE block_handler, VALUE filename)
2860 {
2861     rb_execution_context_t *ec = GET_EC();
2862     const rb_control_frame_t *reg_cfp = ec->cfp;
2863     const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP);
2864     VALUE val;
2865
2866     vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH,
2867                   recv, block_handler,
2868                   (VALUE)vm_cref_new_toplevel(ec), /* cref or me */
2869                   0, reg_cfp->sp, 0, 0);
2870
2871     val = (*func)(arg);
2872
2873     rb_vm_pop_frame(ec);
2874     return val;
2875 }
2876
2877 /* vm */
2878
2879 void
2880 rb_vm_update_references(void *ptr)
2881 {
2882     if (ptr) {
2883         rb_vm_t *vm = ptr;
2884
2885         rb_gc_update_tbl_refs(vm->ci_table);
2886         rb_gc_update_tbl_refs(vm->frozen_strings);
2887         vm->mark_object_ary = rb_gc_location(vm->mark_object_ary);
2888         vm->load_path = rb_gc_location(vm->load_path);
2889         vm->load_path_snapshot = rb_gc_location(vm->load_path_snapshot);
2890
2891         if (vm->load_path_check_cache) {
2892             vm->load_path_check_cache = rb_gc_location(vm->load_path_check_cache);
2893         }
2894
2895         vm->expanded_load_path = rb_gc_location(vm->expanded_load_path);
2896         vm->loaded_features = rb_gc_location(vm->loaded_features);
2897         vm->loaded_features_snapshot = rb_gc_location(vm->loaded_features_snapshot);
2898         vm->loaded_features_realpaths = rb_gc_location(vm->loaded_features_realpaths);
2899         vm->loaded_features_realpath_map = rb_gc_location(vm->loaded_features_realpath_map);
2900         vm->top_self = rb_gc_location(vm->top_self);
2901         vm->orig_progname = rb_gc_location(vm->orig_progname);
2902
2903         rb_gc_update_tbl_refs(vm->overloaded_cme_table);
2904
2905         rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd);
2906
2907         if (vm->coverages) {
2908             vm->coverages = rb_gc_location(vm->coverages);
2909             vm->me2counter = rb_gc_location(vm->me2counter);
2910         }
2911     }
2912 }
2913
2914 void
2915 rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx)
2916 {
2917     if (ptr) {
2918         rb_vm_t *vm = ptr;
2919         rb_ractor_t *r = 0;
2920         ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
2921             VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) ||
2922                       rb_ractor_status_p(r, ractor_running));
2923             if (r->threads.cnt > 0) {
2924                 rb_thread_t *th = 0;
2925                 ccan_list_for_each(&r->threads.set, th, lt_node) {
2926                     VM_ASSERT(th != NULL);
2927                     rb_execution_context_t * ec = th->ec;
2928                     if (ec->vm_stack) {
2929                         VALUE *p = ec->vm_stack;
2930                         VALUE *sp = ec->cfp->sp;
2931                         while (p < sp) {
2932                             if (!RB_SPECIAL_CONST_P(*p)) {
2933                                 cb(*p, ctx);
2934                             }
2935                             p++;
2936                         }
2937                     }
2938                 }
2939             }
2940         }
2941     }
2942 }
2943
2944 static enum rb_id_table_iterator_result
2945 vm_mark_negative_cme(VALUE val, void *dmy)
2946 {
2947     rb_gc_mark(val);
2948     return ID_TABLE_CONTINUE;
2949 }
2950
2951 void rb_thread_sched_mark_zombies(rb_vm_t *vm);
2952
2953 void
2954 rb_vm_mark(void *ptr)
2955 {
2956     RUBY_MARK_ENTER("vm");
2957     RUBY_GC_INFO("-------------------------------------------------\n");
2958     if (ptr) {
2959         rb_vm_t *vm = ptr;
2960         rb_ractor_t *r = 0;
2961         long i;
2962
2963         ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
2964             // ractor.set only contains blocking or running ractors
2965             VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) ||
2966                       rb_ractor_status_p(r, ractor_running));
2967             rb_gc_mark(rb_ractor_self(r));
2968         }
2969
2970         for (struct global_object_list *list = vm->global_object_list; list; list = list->next) {
2971             rb_gc_mark_maybe(*list->varptr);
2972         }
2973
2974         rb_gc_mark_movable(vm->mark_object_ary);
2975         rb_gc_mark_movable(vm->load_path);
2976         rb_gc_mark_movable(vm->load_path_snapshot);
2977         rb_gc_mark_movable(vm->load_path_check_cache);
2978         rb_gc_mark_movable(vm->expanded_load_path);
2979         rb_gc_mark_movable(vm->loaded_features);
2980         rb_gc_mark_movable(vm->loaded_features_snapshot);
2981         rb_gc_mark_movable(vm->loaded_features_realpaths);
2982         rb_gc_mark_movable(vm->loaded_features_realpath_map);
2983         rb_gc_mark_movable(vm->top_self);
2984         rb_gc_mark_movable(vm->orig_progname);
2985         rb_gc_mark_movable(vm->coverages);
2986         rb_gc_mark_movable(vm->me2counter);
2987
2988         if (vm->loading_table) {
2989             rb_mark_tbl(vm->loading_table);
2990         }
2991
2992         rb_gc_mark_values(RUBY_NSIG, vm->trap_list.cmd);
2993
2994         rb_id_table_foreach_values(vm->negative_cme_table, vm_mark_negative_cme, NULL);
2995         rb_mark_tbl_no_pin(vm->overloaded_cme_table);
2996         for (i=0; i<VM_GLOBAL_CC_CACHE_TABLE_SIZE; i++) {
2997             const struct rb_callcache *cc = vm->global_cc_cache_table[i];
2998
2999             if (cc != NULL) {
3000                 if (!vm_cc_invalidated_p(cc)) {
3001                     rb_gc_mark((VALUE)cc);
3002                 }
3003                 else {
3004                     vm->global_cc_cache_table[i] = NULL;
3005                 }
3006             }
3007         }
3008
3009         rb_thread_sched_mark_zombies(vm);
3010         rb_rjit_mark();
3011     }
3012
3013     RUBY_MARK_LEAVE("vm");
3014 }
3015
3016 #undef rb_vm_register_special_exception
3017 void
3018 rb_vm_register_special_exception_str(enum ruby_special_exceptions sp, VALUE cls, VALUE mesg)
3019 {
3020     rb_vm_t *vm = GET_VM();
3021     VALUE exc = rb_exc_new3(cls, rb_obj_freeze(mesg));
3022     OBJ_FREEZE(exc);
3023     ((VALUE *)vm->special_exceptions)[sp] = exc;
3024     rb_vm_register_global_object(exc);
3025 }
3026
3027 static int
3028 free_loading_table_entry(st_data_t key, st_data_t value, st_data_t arg)
3029 {
3030     xfree((char *)key);
3031     return ST_DELETE;
3032 }
3033
3034 void rb_free_loaded_features_index(rb_vm_t *vm);
3035 void rb_objspace_free_objects(void *objspace);
3036
3037 int
3038 ruby_vm_destruct(rb_vm_t *vm)
3039 {
3040     RUBY_FREE_ENTER("vm");
3041
3042     if (vm) {
3043         rb_thread_t *th = vm->ractor.main_thread;
3044         VALUE *stack = th->ec->vm_stack;
3045         if (rb_free_at_exit) {
3046             rb_free_encoded_insn_data();
3047             rb_free_global_enc_table();
3048             rb_free_loaded_builtin_table();
3049
3050             rb_free_shared_fiber_pool();
3051             rb_free_static_symid_str();
3052             rb_free_transcoder_table();
3053             rb_free_vm_opt_tables();
3054             rb_free_warning();
3055             rb_free_rb_global_tbl();
3056             rb_free_loaded_features_index(vm);
3057
3058             rb_id_table_free(vm->negative_cme_table);
3059             st_free_table(vm->overloaded_cme_table);
3060
3061             rb_id_table_free(RCLASS(rb_mRubyVMFrozenCore)->m_tbl);
3062
3063             rb_shape_t *cursor = rb_shape_get_root_shape();
3064             rb_shape_t *end = rb_shape_get_shape_by_id(GET_SHAPE_TREE()->next_shape_id);
3065             while (cursor < end) {
3066                 // 0x1 == SINGLE_CHILD_P
3067                 if (cursor->edges && !(((uintptr_t)cursor->edges) & 0x1))
3068                     rb_id_table_free(cursor->edges);
3069                 cursor += 1;
3070             }
3071
3072             xfree(GET_SHAPE_TREE());
3073
3074             st_free_table(vm->static_ext_inits);
3075             st_free_table(vm->ensure_rollback_table);
3076
3077             rb_vm_postponed_job_free();
3078
3079             rb_id_table_free(vm->constant_cache);
3080             st_free_table(vm->unused_block_warning_table);
3081
3082             if (th) {
3083                 xfree(th->nt);
3084                 th->nt = NULL;
3085             }
3086
3087 #ifndef HAVE_SETPROCTITLE
3088             ruby_free_proctitle();
3089 #endif
3090         }
3091         else {
3092             if (th) {
3093                 rb_fiber_reset_root_local_storage(th);
3094                 thread_free(th);
3095             }
3096         }
3097
3098         struct rb_objspace *objspace = vm->objspace;
3099
3100         rb_vm_living_threads_init(vm);
3101         ruby_vm_run_at_exit_hooks(vm);
3102         if (vm->loading_table) {
3103             st_foreach(vm->loading_table, free_loading_table_entry, 0);
3104             st_free_table(vm->loading_table);
3105             vm->loading_table = 0;
3106         }
3107         if (vm->ci_table) {
3108             st_free_table(vm->ci_table);
3109             vm->ci_table = NULL;
3110         }
3111         if (vm->frozen_strings) {
3112             st_free_table(vm->frozen_strings);
3113             vm->frozen_strings = 0;
3114         }
3115         RB_ALTSTACK_FREE(vm->main_altstack);
3116
3117         struct global_object_list *next;
3118         for (struct global_object_list *list = vm->global_object_list; list; list = next) {
3119             next = list->next;
3120             xfree(list);
3121         }
3122
3123         if (objspace) {
3124             if (rb_free_at_exit) {
3125                 rb_objspace_free_objects(objspace);
3126                 rb_free_generic_iv_tbl_();
3127                 rb_free_default_rand_key();
3128                 if (th && vm->fork_gen == 0) {
3129                     /* If we have forked, main_thread may not be the initial thread */
3130                     xfree(stack);
3131                     ruby_mimfree(th);
3132                 }
3133             }
3134             rb_objspace_free(objspace);
3135         }
3136         rb_native_mutex_destroy(&vm->workqueue_lock);
3137         /* after freeing objspace, you *can't* use ruby_xfree() */
3138         ruby_mimfree(vm);
3139         ruby_current_vm_ptr = NULL;
3140     }
3141     RUBY_FREE_LEAVE("vm");
3142     return 0;
3143 }
3144
3145 size_t rb_vm_memsize_waiting_fds(struct ccan_list_head *waiting_fds); // thread.c
3146 size_t rb_vm_memsize_workqueue(struct ccan_list_head *workqueue); // vm_trace.c
3147
3148 // Used for VM memsize reporting. Returns the size of the at_exit list by
3149 // looping through the linked list and adding up the size of the structs.
3150 static enum rb_id_table_iterator_result
3151 vm_memsize_constant_cache_i(ID id, VALUE ics, void *size)
3152 {
3153     *((size_t *) size) += rb_st_memsize((st_table *) ics);
3154     return ID_TABLE_CONTINUE;
3155 }
3156
3157 // Returns a size_t representing the memory footprint of the VM's constant
3158 // cache, which is the memsize of the table as well as the memsize of all of the
3159 // nested tables.
3160 static size_t
3161 vm_memsize_constant_cache(void)
3162 {
3163     rb_vm_t *vm = GET_VM();
3164     size_t size = rb_id_table_memsize(vm->constant_cache);
3165
3166     rb_id_table_foreach(vm->constant_cache, vm_memsize_constant_cache_i, &size);
3167     return size;
3168 }
3169
3170 static size_t
3171 vm_memsize_at_exit_list(rb_at_exit_list *at_exit)
3172 {
3173     size_t size = 0;
3174
3175     while (at_exit) {
3176         size += sizeof(rb_at_exit_list);
3177         at_exit = at_exit->next;
3178     }
3179
3180     return size;
3181 }
3182
3183 // Used for VM memsize reporting. Returns the size of the builtin function
3184 // table if it has been defined.
3185 static size_t
3186 vm_memsize_builtin_function_table(const struct rb_builtin_function *builtin_function_table)
3187 {
3188     return builtin_function_table == NULL ? 0 : sizeof(struct rb_builtin_function);
3189 }
3190
3191 // Reports the memsize of the VM struct object and the structs that are
3192 // associated with it.
3193 static size_t
3194 vm_memsize(const void *ptr)
3195 {
3196     rb_vm_t *vm = GET_VM();
3197
3198     return (
3199         sizeof(rb_vm_t) +
3200         rb_vm_memsize_waiting_fds(&vm->waiting_fds) +
3201         rb_st_memsize(vm->loaded_features_index) +
3202         rb_st_memsize(vm->loading_table) +
3203         rb_st_memsize(vm->ensure_rollback_table) +
3204         rb_vm_memsize_postponed_job_queue() +
3205         rb_vm_memsize_workqueue(&vm->workqueue) +
3206         vm_memsize_at_exit_list(vm->at_exit) +
3207         rb_st_memsize(vm->ci_table) +
3208         rb_st_memsize(vm->frozen_strings) +
3209         vm_memsize_builtin_function_table(vm->builtin_function_table) +
3210         rb_id_table_memsize(vm->negative_cme_table) +
3211         rb_st_memsize(vm->overloaded_cme_table) +
3212         vm_memsize_constant_cache() +
3213         GET_SHAPE_TREE()->cache_size * sizeof(redblack_node_t)
3214     );
3215
3216     // TODO
3217     // struct { struct ccan_list_head set; } ractor;
3218     // void *main_altstack; #ifdef USE_SIGALTSTACK
3219     // struct rb_objspace *objspace;
3220 }
3221
3222 static const rb_data_type_t vm_data_type = {
3223     "VM",
3224     {0, 0, vm_memsize,},
3225     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3226 };
3227
3228
3229 static VALUE
3230 vm_default_params(void)
3231 {
3232     rb_vm_t *vm = GET_VM();
3233     VALUE result = rb_hash_new_with_size(4);
3234 #define SET(name) rb_hash_aset(result, ID2SYM(rb_intern(#name)), SIZET2NUM(vm->default_params.name));
3235     SET(thread_vm_stack_size);
3236     SET(thread_machine_stack_size);
3237     SET(fiber_vm_stack_size);
3238     SET(fiber_machine_stack_size);
3239 #undef SET
3240     rb_obj_freeze(result);
3241     return result;
3242 }
3243
3244 static size_t
3245 get_param(const char *name, size_t default_value, size_t min_value)
3246 {
3247     const char *envval;
3248     size_t result = default_value;
3249     if ((envval = getenv(name)) != 0) {
3250         long val = atol(envval);
3251         if (val < (long)min_value) {
3252             val = (long)min_value;
3253         }
3254         result = (size_t)(((val -1 + RUBY_VM_SIZE_ALIGN) / RUBY_VM_SIZE_ALIGN) * RUBY_VM_SIZE_ALIGN);
3255     }
3256     if (0) ruby_debug_printf("%s: %"PRIuSIZE"\n", name, result); /* debug print */
3257
3258     return result;
3259 }
3260
3261 static void
3262 check_machine_stack_size(size_t *sizep)
3263 {
3264 #ifdef PTHREAD_STACK_MIN
3265     size_t size = *sizep;
3266 #endif
3267
3268 #ifdef PTHREAD_STACK_MIN
3269     if (size < (size_t)PTHREAD_STACK_MIN) {
3270         *sizep = (size_t)PTHREAD_STACK_MIN * 2;
3271     }
3272 #endif
3273 }
3274
3275 static void
3276 vm_default_params_setup(rb_vm_t *vm)
3277 {
3278     vm->default_params.thread_vm_stack_size =
3279       get_param("RUBY_THREAD_VM_STACK_SIZE",
3280                 RUBY_VM_THREAD_VM_STACK_SIZE,
3281                 RUBY_VM_THREAD_VM_STACK_SIZE_MIN);
3282
3283     vm->default_params.thread_machine_stack_size =
3284       get_param("RUBY_THREAD_MACHINE_STACK_SIZE",
3285                 RUBY_VM_THREAD_MACHINE_STACK_SIZE,
3286                 RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN);
3287
3288     vm->default_params.fiber_vm_stack_size =
3289       get_param("RUBY_FIBER_VM_STACK_SIZE",
3290                 RUBY_VM_FIBER_VM_STACK_SIZE,
3291                 RUBY_VM_FIBER_VM_STACK_SIZE_MIN);
3292
3293     vm->default_params.fiber_machine_stack_size =
3294       get_param("RUBY_FIBER_MACHINE_STACK_SIZE",
3295                 RUBY_VM_FIBER_MACHINE_STACK_SIZE,
3296                 RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN);
3297
3298     /* environment dependent check */
3299     check_machine_stack_size(&vm->default_params.thread_machine_stack_size);
3300     check_machine_stack_size(&vm->default_params.fiber_machine_stack_size);
3301 }
3302
3303 static void
3304 vm_init2(rb_vm_t *vm)
3305 {
3306     rb_vm_living_threads_init(vm);
3307     vm->thread_report_on_exception = 1;
3308     vm->src_encoding_index = -1;
3309
3310     vm_default_params_setup(vm);
3311 }
3312
3313 void
3314 rb_execution_context_update(rb_execution_context_t *ec)
3315 {
3316     /* update VM stack */
3317     if (ec->vm_stack) {
3318         long i;
3319         VM_ASSERT(ec->cfp);
3320         VALUE *p = ec->vm_stack;
3321         VALUE *sp = ec->cfp->sp;
3322         rb_control_frame_t *cfp = ec->cfp;
3323         rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
3324
3325         for (i = 0; i < (long)(sp - p); i++) {
3326             VALUE ref = p[i];
3327             VALUE update = rb_gc_location(ref);
3328             if (ref != update) {
3329                 p[i] = update;
3330             }
3331         }
3332
3333         while (cfp != limit_cfp) {
3334             const VALUE *ep = cfp->ep;
3335             cfp->self = rb_gc_location(cfp->self);
3336             cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq);
3337             cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code);
3338
3339             if (!VM_ENV_LOCAL_P(ep)) {
3340                 const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
3341                 if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) {
3342                     VM_FORCE_WRITE(&prev_ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(prev_ep[VM_ENV_DATA_INDEX_ENV]));
3343                 }
3344
3345                 if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) {
3346                     VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(ep[VM_ENV_DATA_INDEX_ENV]));
3347                     VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ME_CREF], rb_gc_location(ep[VM_ENV_DATA_INDEX_ME_CREF]));
3348                 }
3349             }
3350
3351             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
3352         }
3353     }
3354
3355     ec->storage = rb_gc_location(ec->storage);
3356 }
3357
3358 static enum rb_id_table_iterator_result
3359 mark_local_storage_i(VALUE local, void *data)
3360 {
3361     rb_gc_mark(local);
3362     return ID_TABLE_CONTINUE;
3363 }
3364
3365 void
3366 rb_execution_context_mark(const rb_execution_context_t *ec)
3367 {
3368     /* mark VM stack */
3369     if (ec->vm_stack) {
3370         VM_ASSERT(ec->cfp);
3371         VALUE *p = ec->vm_stack;
3372         VALUE *sp = ec->cfp->sp;
3373         rb_control_frame_t *cfp = ec->cfp;
3374         rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
3375
3376         VM_ASSERT(sp == ec->cfp->sp);
3377         rb_gc_mark_vm_stack_values((long)(sp - p), p);
3378
3379         while (cfp != limit_cfp) {
3380             const VALUE *ep = cfp->ep;
3381             VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep));
3382
3383             if (VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) {
3384                 rb_gc_mark_movable(cfp->self);
3385                 rb_gc_mark_movable((VALUE)cfp->iseq);
3386                 rb_gc_mark_movable((VALUE)cfp->block_code);
3387
3388                 if (!VM_ENV_LOCAL_P(ep)) {
3389                     const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
3390                     if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) {
3391                         rb_gc_mark_movable(prev_ep[VM_ENV_DATA_INDEX_ENV]);
3392                     }
3393
3394                     if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) {
3395                         rb_gc_mark_movable(ep[VM_ENV_DATA_INDEX_ENV]);
3396                         rb_gc_mark(ep[VM_ENV_DATA_INDEX_ME_CREF]);
3397                     }
3398                 }
3399             }
3400
3401             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
3402         }
3403     }
3404
3405     /* mark machine stack */
3406     if (ec->machine.stack_start && ec->machine.stack_end &&
3407         ec != GET_EC() /* marked for current ec at the first stage of marking */
3408         ) {
3409         rb_gc_mark_machine_context(ec);
3410     }
3411
3412     rb_gc_mark(ec->errinfo);
3413     rb_gc_mark(ec->root_svar);
3414     if (ec->local_storage) {
3415         rb_id_table_foreach_values(ec->local_storage, mark_local_storage_i, NULL);
3416     }
3417     rb_gc_mark(ec->local_storage_recursive_hash);
3418     rb_gc_mark(ec->local_storage_recursive_hash_for_trace);
3419     rb_gc_mark(ec->private_const_reference);
3420
3421     rb_gc_mark_movable(ec->storage);
3422 }
3423
3424 void rb_fiber_mark_self(rb_fiber_t *fib);
3425 void rb_fiber_update_self(rb_fiber_t *fib);
3426 void rb_threadptr_root_fiber_setup(rb_thread_t *th);
3427 void rb_threadptr_root_fiber_release(rb_thread_t *th);
3428
3429 static void
3430 thread_compact(void *ptr)
3431 {
3432     rb_thread_t *th = ptr;
3433
3434     th->self = rb_gc_location(th->self);
3435
3436     if (!th->root_fiber) {
3437         rb_execution_context_update(th->ec);
3438     }
3439 }
3440
3441 static void
3442 thread_mark(void *ptr)
3443 {
3444     rb_thread_t *th = ptr;
3445     RUBY_MARK_ENTER("thread");
3446     rb_fiber_mark_self(th->ec->fiber_ptr);
3447
3448     /* mark ruby objects */
3449     switch (th->invoke_type) {
3450       case thread_invoke_type_proc:
3451       case thread_invoke_type_ractor_proc:
3452         rb_gc_mark(th->invoke_arg.proc.proc);
3453         rb_gc_mark(th->invoke_arg.proc.args);
3454         break;
3455       case thread_invoke_type_func:
3456         rb_gc_mark_maybe((VALUE)th->invoke_arg.func.arg);
3457         break;
3458       default:
3459         break;
3460     }
3461
3462     rb_gc_mark(rb_ractor_self(th->ractor));
3463     rb_gc_mark(th->thgroup);
3464     rb_gc_mark(th->value);
3465     rb_gc_mark(th->pending_interrupt_queue);
3466     rb_gc_mark(th->pending_interrupt_mask_stack);
3467     rb_gc_mark(th->top_self);
3468     rb_gc_mark(th->top_wrapper);
3469     if (th->root_fiber) rb_fiber_mark_self(th->root_fiber);
3470
3471     RUBY_ASSERT(th->ec == rb_fiberptr_get_ec(th->ec->fiber_ptr));
3472     rb_gc_mark(th->stat_insn_usage);
3473     rb_gc_mark(th->last_status);
3474     rb_gc_mark(th->locking_mutex);
3475     rb_gc_mark(th->name);
3476
3477     rb_gc_mark(th->scheduler);
3478
3479     RUBY_MARK_LEAVE("thread");
3480 }
3481
3482 void rb_threadptr_sched_free(rb_thread_t *th); // thread_*.c
3483
3484 static void
3485 thread_free(void *ptr)
3486 {
3487     rb_thread_t *th = ptr;
3488     RUBY_FREE_ENTER("thread");
3489
3490     rb_threadptr_sched_free(th);
3491
3492     if (th->locking_mutex != Qfalse) {
3493         rb_bug("thread_free: locking_mutex must be NULL (%p:%p)", (void *)th, (void *)th->locking_mutex);
3494     }
3495     if (th->keeping_mutexes != NULL) {
3496         rb_bug("thread_free: keeping_mutexes must be NULL (%p:%p)", (void *)th, (void *)th->keeping_mutexes);
3497     }
3498
3499     ruby_xfree(th->specific_storage);
3500
3501     rb_threadptr_root_fiber_release(th);
3502
3503     if (th->vm && th->vm->ractor.main_thread == th) {
3504         RUBY_GC_INFO("MRI main thread\n");
3505     }
3506     else {
3507         // ruby_xfree(th->nt);
3508         // TODO: MN system collect nt, but without MN system it should be freed here.
3509         ruby_xfree(th);
3510     }
3511
3512     RUBY_FREE_LEAVE("thread");
3513 }
3514
3515 static size_t
3516 thread_memsize(const void *ptr)
3517 {
3518     const rb_thread_t *th = ptr;
3519     size_t size = sizeof(rb_thread_t);
3520
3521     if (!th->root_fiber) {
3522         size += th->ec->vm_stack_size * sizeof(VALUE);
3523     }
3524     if (th->ec->local_storage) {
3525         size += rb_id_table_memsize(th->ec->local_storage);
3526     }
3527     return size;
3528 }
3529
3530 #define thread_data_type ruby_threadptr_data_type
3531 const rb_data_type_t ruby_threadptr_data_type = {
3532     "VM/thread",
3533     {
3534         thread_mark,
3535         thread_free,
3536         thread_memsize,
3537         thread_compact,
3538     },
3539     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3540 };
3541
3542 VALUE
3543 rb_obj_is_thread(VALUE obj)
3544 {
3545     return RBOOL(rb_typeddata_is_kind_of(obj, &thread_data_type));
3546 }
3547
3548 static VALUE
3549 thread_alloc(VALUE klass)
3550 {
3551     rb_thread_t *th;
3552     return TypedData_Make_Struct(klass, rb_thread_t, &thread_data_type, th);
3553 }
3554
3555 inline void
3556 rb_ec_set_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
3557 {
3558     ec->vm_stack = stack;
3559     ec->vm_stack_size = size;
3560 }
3561
3562 void
3563 rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
3564 {
3565     rb_ec_set_vm_stack(ec, stack, size);
3566
3567 #if VM_CHECK_MODE > 0
3568     MEMZERO(stack, VALUE, size); // malloc memory could have the VM canary in it
3569 #endif
3570
3571     ec->cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
3572
3573     vm_push_frame(ec,
3574         NULL /* dummy iseq */,
3575         VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH | VM_FRAME_FLAG_CFRAME /* dummy frame */,
3576         Qnil /* dummy self */, VM_BLOCK_HANDLER_NONE /* dummy block ptr */,
3577         0 /* dummy cref/me */,
3578         0 /* dummy pc */, ec->vm_stack, 0, 0
3579     );
3580 }
3581
3582 void
3583 rb_ec_clear_vm_stack(rb_execution_context_t *ec)
3584 {
3585     rb_ec_set_vm_stack(ec, NULL, 0);
3586
3587     // Avoid dangling pointers:
3588     ec->cfp = NULL;
3589 }
3590
3591 static void
3592 th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm)
3593 {
3594     th->self = self;
3595
3596     rb_threadptr_root_fiber_setup(th);
3597
3598     /* All threads are blocking until a non-blocking fiber is scheduled */
3599     th->blocking = 1;
3600     th->scheduler = Qnil;
3601
3602     if (self == 0) {
3603         size_t size = vm->default_params.thread_vm_stack_size / sizeof(VALUE);
3604         rb_ec_initialize_vm_stack(th->ec, ALLOC_N(VALUE, size), size);
3605     }
3606     else {
3607         VM_ASSERT(th->ec->cfp == NULL);
3608         VM_ASSERT(th->ec->vm_stack == NULL);
3609         VM_ASSERT(th->ec->vm_stack_size == 0);
3610     }
3611
3612     th->status = THREAD_RUNNABLE;
3613     th->last_status = Qnil;
3614     th->top_wrapper = 0;
3615     th->top_self = vm->top_self; // 0 while self == 0
3616     th->value = Qundef;
3617
3618     th->ec->errinfo = Qnil;
3619     th->ec->root_svar = Qfalse;
3620     th->ec->local_storage_recursive_hash = Qnil;
3621     th->ec->local_storage_recursive_hash_for_trace = Qnil;
3622
3623     th->ec->storage = Qnil;
3624
3625 #if OPT_CALL_THREADED_CODE
3626     th->retval = Qundef;
3627 #endif
3628     th->name = Qnil;
3629     th->report_on_exception = vm->thread_report_on_exception;
3630     th->ext_config.ractor_safe = true;
3631
3632 #if USE_RUBY_DEBUG_LOG
3633     static rb_atomic_t thread_serial = 1;
3634     th->serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1);
3635
3636     RUBY_DEBUG_LOG("th:%u", th->serial);
3637 #endif
3638 }
3639
3640 VALUE
3641 rb_thread_alloc(VALUE klass)
3642 {
3643     VALUE self = thread_alloc(klass);
3644     rb_thread_t *target_th = rb_thread_ptr(self);
3645     target_th->ractor = GET_RACTOR();
3646     th_init(target_th, self, target_th->vm = GET_VM());
3647     return self;
3648 }
3649
3650 #define REWIND_CFP(expr) do { \
3651     rb_execution_context_t *ec__ = GET_EC(); \
3652     VALUE *const curr_sp = (ec__->cfp++)->sp; \
3653     VALUE *const saved_sp = ec__->cfp->sp; \
3654     ec__->cfp->sp = curr_sp; \
3655     expr; \
3656     (ec__->cfp--)->sp = saved_sp; \
3657 } while (0)
3658
3659 static VALUE
3660 m_core_set_method_alias(VALUE self, VALUE cbase, VALUE sym1, VALUE sym2)
3661 {
3662     REWIND_CFP({
3663         rb_alias(cbase, SYM2ID(sym1), SYM2ID(sym2));
3664     });
3665     return Qnil;
3666 }
3667
3668 static VALUE
3669 m_core_set_variable_alias(VALUE self, VALUE sym1, VALUE sym2)
3670 {
3671     REWIND_CFP({
3672         rb_alias_variable(SYM2ID(sym1), SYM2ID(sym2));
3673     });
3674     return Qnil;
3675 }
3676
3677 static VALUE
3678 m_core_undef_method(VALUE self, VALUE cbase, VALUE sym)
3679 {
3680     REWIND_CFP({
3681         ID mid = SYM2ID(sym);
3682         rb_undef(cbase, mid);
3683         rb_clear_method_cache(self, mid);
3684     });
3685     return Qnil;
3686 }
3687
3688 static VALUE
3689 m_core_set_postexe(VALUE self)
3690 {
3691     rb_set_end_proc(rb_call_end_proc, rb_block_proc());
3692     return Qnil;
3693 }
3694
3695 static VALUE core_hash_merge_kwd(VALUE hash, VALUE kw);
3696
3697 static VALUE
3698 core_hash_merge(VALUE hash, long argc, const VALUE *argv)
3699 {
3700     Check_Type(hash, T_HASH);
3701     VM_ASSERT(argc % 2 == 0);
3702     rb_hash_bulk_insert(argc, argv, hash);
3703     return hash;
3704 }
3705
3706 static VALUE
3707 m_core_hash_merge_ptr(int argc, VALUE *argv, VALUE recv)
3708 {
3709     VALUE hash = argv[0];
3710
3711     REWIND_CFP(hash = core_hash_merge(hash, argc-1, argv+1));
3712
3713     return hash;
3714 }
3715
3716 static int
3717 kwmerge_i(VALUE key, VALUE value, VALUE hash)
3718 {
3719     rb_hash_aset(hash, key, value);
3720     return ST_CONTINUE;
3721 }
3722
3723 static VALUE
3724 m_core_hash_merge_kwd(VALUE recv, VALUE hash, VALUE kw)
3725 {
3726     if (!NIL_P(kw)) {
3727         REWIND_CFP(hash = core_hash_merge_kwd(hash, kw));
3728     }
3729     return hash;
3730 }
3731
3732 static VALUE
3733 m_core_make_shareable(VALUE recv, VALUE obj)
3734 {
3735     return rb_ractor_make_shareable(obj);
3736 }
3737
3738 static VALUE
3739 m_core_make_shareable_copy(VALUE recv, VALUE obj)
3740 {
3741     return rb_ractor_make_shareable_copy(obj);
3742 }
3743
3744 static VALUE
3745 m_core_ensure_shareable(VALUE recv, VALUE obj, VALUE name)
3746 {
3747     return rb_ractor_ensure_shareable(obj, name);
3748 }
3749
3750 static VALUE
3751 core_hash_merge_kwd(VALUE hash, VALUE kw)
3752 {
3753     rb_hash_foreach(rb_to_hash_type(kw), kwmerge_i, hash);
3754     return hash;
3755 }
3756
3757 extern VALUE *rb_gc_stack_start;
3758 extern size_t rb_gc_stack_maxsize;
3759
3760 /* debug functions */
3761
3762 /* :nodoc: */
3763 static VALUE
3764 sdr(VALUE self)
3765 {
3766     rb_vm_bugreport(NULL, stderr);
3767     return Qnil;
3768 }
3769
3770 /* :nodoc: */
3771 static VALUE
3772 nsdr(VALUE self)
3773 {
3774     VALUE ary = rb_ary_new();
3775 #ifdef HAVE_BACKTRACE
3776 #include <execinfo.h>
3777 #define MAX_NATIVE_TRACE 1024
3778     static void *trace[MAX_NATIVE_TRACE];
3779     int n = (int)backtrace(trace, MAX_NATIVE_TRACE);
3780     char **syms = backtrace_symbols(trace, n);
3781     int i;
3782
3783     if (syms == 0) {
3784         rb_memerror();
3785     }
3786
3787     for (i=0; i<n; i++) {
3788         rb_ary_push(ary, rb_str_new2(syms[i]));
3789     }
3790     free(syms); /* OK */
3791 #endif
3792     return ary;
3793 }
3794
3795 #if VM_COLLECT_USAGE_DETAILS
3796 static VALUE usage_analysis_insn_start(VALUE self);
3797 static VALUE usage_analysis_operand_start(VALUE self);
3798 static VALUE usage_analysis_register_start(VALUE self);
3799 static VALUE usage_analysis_insn_stop(VALUE self);
3800 static VALUE usage_analysis_operand_stop(VALUE self);
3801 static VALUE usage_analysis_register_stop(VALUE self);
3802 static VALUE usage_analysis_insn_running(VALUE self);
3803 static VALUE usage_analysis_operand_running(VALUE self);
3804 static VALUE usage_analysis_register_running(VALUE self);
3805 static VALUE usage_analysis_insn_clear(VALUE self);
3806 static VALUE usage_analysis_operand_clear(VALUE self);
3807 static VALUE usage_analysis_register_clear(VALUE self);
3808 #endif
3809
3810 static VALUE
3811 f_raise(int c, VALUE *v, VALUE _)
3812 {
3813     return rb_f_raise(c, v);
3814 }
3815
3816 static VALUE
3817 f_proc(VALUE _)
3818 {
3819     return rb_block_proc();
3820 }
3821
3822 static VALUE
3823 f_lambda(VALUE _)
3824 {
3825     return rb_block_lambda();
3826 }
3827
3828 static VALUE
3829 f_sprintf(int c, const VALUE *v, VALUE _)
3830 {
3831     return rb_f_sprintf(c, v);
3832 }
3833
3834 /* :nodoc: */
3835 static VALUE
3836 vm_mtbl(VALUE self, VALUE obj, VALUE sym)
3837 {
3838     vm_mtbl_dump(CLASS_OF(obj), RTEST(sym) ? SYM2ID(sym) : 0);
3839     return Qnil;
3840 }
3841
3842 /* :nodoc: */
3843 static VALUE
3844 vm_mtbl2(VALUE self, VALUE obj, VALUE sym)
3845 {
3846     vm_mtbl_dump(obj, RTEST(sym) ? SYM2ID(sym) : 0);
3847     return Qnil;
3848 }
3849
3850 /*
3851  *  call-seq:
3852  *     RubyVM.keep_script_lines -> true or false
3853  *
3854  *  Return current +keep_script_lines+ status. Now it only returns
3855  *  +true+ of +false+, but it can return other objects in future.
3856  *
3857  *  Note that this is an API for ruby internal use, debugging,
3858  *  and research. Do not use this for any other purpose.
3859  *  The compatibility is not guaranteed.
3860  */
3861 static VALUE
3862 vm_keep_script_lines(VALUE self)
3863 {
3864     return RBOOL(ruby_vm_keep_script_lines);
3865 }
3866
3867 /*
3868  *  call-seq:
3869  *     RubyVM.keep_script_lines = true / false
3870  *
3871  *  It set +keep_script_lines+ flag. If the flag is set, all
3872  *  loaded scripts are recorded in a interpreter process.
3873  *
3874  *  Note that this is an API for ruby internal use, debugging,
3875  *  and research. Do not use this for any other purpose.
3876  *  The compatibility is not guaranteed.
3877  */
3878 static VALUE
3879 vm_keep_script_lines_set(VALUE self, VALUE flags)
3880 {
3881     ruby_vm_keep_script_lines = RTEST(flags);
3882     return flags;
3883 }
3884
3885 void
3886 Init_VM(void)
3887 {
3888     VALUE opts;
3889     VALUE klass;
3890     VALUE fcore;
3891
3892     /*
3893      * Document-class: RubyVM
3894      *
3895      * The RubyVM module only exists on MRI. +RubyVM+ is not defined in
3896      * other Ruby implementations such as JRuby and TruffleRuby.
3897      *
3898      * The RubyVM module provides some access to MRI internals.
3899      * This module is for very limited purposes, such as debugging,
3900      * prototyping, and research.  Normal users must not use it.
3901      * This module is not portable between Ruby implementations.
3902      */
3903     rb_cRubyVM = rb_define_class("RubyVM", rb_cObject);
3904     rb_undef_alloc_func(rb_cRubyVM);
3905     rb_undef_method(CLASS_OF(rb_cRubyVM), "new");
3906     rb_define_singleton_method(rb_cRubyVM, "stat", vm_stat, -1);
3907     rb_define_singleton_method(rb_cRubyVM, "keep_script_lines", vm_keep_script_lines, 0);
3908     rb_define_singleton_method(rb_cRubyVM, "keep_script_lines=", vm_keep_script_lines_set, 1);
3909
3910 #if USE_DEBUG_COUNTER
3911     rb_define_singleton_method(rb_cRubyVM, "reset_debug_counters", rb_debug_counter_reset, 0);
3912     rb_define_singleton_method(rb_cRubyVM, "show_debug_counters", rb_debug_counter_show, 0);
3913 #endif
3914
3915     /* FrozenCore (hidden) */
3916     fcore = rb_class_new(rb_cBasicObject);
3917     rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore");
3918     rb_vm_register_global_object(rb_class_path_cached(fcore));
3919     RBASIC(fcore)->flags = T_ICLASS;
3920     klass = rb_singleton_class(fcore);
3921     rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3);
3922     rb_define_method_id(klass, id_core_set_variable_alias, m_core_set_variable_alias, 2);
3923     rb_define_method_id(klass, id_core_undef_method, m_core_undef_method, 2);
3924     rb_define_method_id(klass, id_core_set_postexe, m_core_set_postexe, 0);
3925     rb_define_method_id(klass, id_core_hash_merge_ptr, m_core_hash_merge_ptr, -1);
3926     rb_define_method_id(klass, id_core_hash_merge_kwd, m_core_hash_merge_kwd, 2);
3927     rb_define_method_id(klass, id_core_raise, f_raise, -1);
3928     rb_define_method_id(klass, id_core_sprintf, f_sprintf, -1);
3929     rb_define_method_id(klass, idProc, f_proc, 0);
3930     rb_define_method_id(klass, idLambda, f_lambda, 0);
3931     rb_define_method(klass, "make_shareable", m_core_make_shareable, 1);
3932     rb_define_method(klass, "make_shareable_copy", m_core_make_shareable_copy, 1);
3933     rb_define_method(klass, "ensure_shareable", m_core_ensure_shareable, 2);
3934     rb_obj_freeze(fcore);
3935     RBASIC_CLEAR_CLASS(klass);
3936     rb_obj_freeze(klass);
3937     rb_vm_register_global_object(fcore);
3938     rb_mRubyVMFrozenCore = fcore;
3939
3940     /*
3941      * Document-class: Thread
3942      *
3943      *  Threads are the Ruby implementation for a concurrent programming model.
3944      *
3945      *  Programs that require multiple threads of execution are a perfect
3946      *  candidate for Ruby's Thread class.
3947      *
3948      *  For example, we can create a new thread separate from the main thread's
3949      *  execution using ::new.
3950      *
3951      *      thr = Thread.new { puts "What's the big deal" }
3952      *
3953      *  Then we are able to pause the execution of the main thread and allow
3954      *  our new thread to finish, using #join:
3955      *
3956      *      thr.join #=> "What's the big deal"
3957      *
3958      *  If we don't call +thr.join+ before the main thread terminates, then all
3959      *  other threads including +thr+ will be killed.
3960      *
3961      *  Alternatively, you can use an array for handling multiple threads at
3962      *  once, like in the following example:
3963      *
3964      *      threads = []
3965      *      threads << Thread.new { puts "What's the big deal" }
3966      *      threads << Thread.new { 3.times { puts "Threads are fun!" } }
3967      *
3968      *  After creating a few threads we wait for them all to finish
3969      *  consecutively.
3970      *
3971      *      threads.each { |thr| thr.join }
3972      *
3973      *  To retrieve the last value of a thread, use #value
3974      *
3975      *      thr = Thread.new { sleep 1; "Useful value" }
3976      *      thr.value #=> "Useful value"
3977      *
3978      *  === Thread initialization
3979      *
3980      *  In order to create new threads, Ruby provides ::new, ::start, and
3981      *  ::fork. A block must be provided with each of these methods, otherwise
3982      *  a ThreadError will be raised.
3983      *
3984      *  When subclassing the Thread class, the +initialize+ method of your
3985      *  subclass will be ignored by ::start and ::fork. Otherwise, be sure to
3986      *  call super in your +initialize+ method.
3987      *
3988      *  === Thread termination
3989      *
3990      *  For terminating threads, Ruby provides a variety of ways to do this.
3991      *
3992      *  The class method ::kill, is meant to exit a given thread:
3993      *
3994      *      thr = Thread.new { sleep }
3995      *      Thread.kill(thr) # sends exit() to thr
3996      *
3997      *  Alternatively, you can use the instance method #exit, or any of its
3998      *  aliases #kill or #terminate.
3999      *
4000      *      thr.exit
4001      *
4002      *  === Thread status
4003      *
4004      *  Ruby provides a few instance methods for querying the state of a given
4005      *  thread. To get a string with the current thread's state use #status
4006      *
4007      *      thr = Thread.new { sleep }
4008      *      thr.status # => "sleep"
4009      *      thr.exit
4010      *      thr.status # => false
4011      *
4012      *  You can also use #alive? to tell if the thread is running or sleeping,
4013      *  and #stop? if the thread is dead or sleeping.
4014      *
4015      *  === Thread variables and scope
4016      *
4017      *  Since threads are created with blocks, the same rules apply to other
4018      *  Ruby blocks for variable scope. Any local variables created within this
4019      *  block are accessible to only this thread.
4020      *
4021      *  ==== Fiber-local vs. Thread-local
4022      *
4023      *  Each fiber has its own bucket for Thread#[] storage. When you set a
4024      *  new fiber-local it is only accessible within this Fiber. To illustrate:
4025      *
4026      *      Thread.new {
4027      *        Thread.current[:foo] = "bar"
4028      *        Fiber.new {
4029      *          p Thread.current[:foo] # => nil
4030      *        }.resume
4031      *      }.join
4032      *
4033      *  This example uses #[] for getting and #[]= for setting fiber-locals,
4034      *  you can also use #keys to list the fiber-locals for a given
4035      *  thread and #key? to check if a fiber-local exists.
4036      *
4037      *  When it comes to thread-locals, they are accessible within the entire
4038      *  scope of the thread. Given the following example:
4039      *
4040      *      Thread.new{
4041      *        Thread.current.thread_variable_set(:foo, 1)
4042      *        p Thread.current.thread_variable_get(:foo) # => 1
4043      *        Fiber.new{
4044      *          Thread.current.thread_variable_set(:foo, 2)
4045      *          p Thread.current.thread_variable_get(:foo) # => 2
4046      *        }.resume
4047      *        p Thread.current.thread_variable_get(:foo)   # => 2
4048      *      }.join
4049      *
4050      *  You can see that the thread-local +:foo+ carried over into the fiber
4051      *  and was changed to +2+ by the end of the thread.
4052      *
4053      *  This example makes use of #thread_variable_set to create new
4054      *  thread-locals, and #thread_variable_get to reference them.
4055      *
4056      *  There is also #thread_variables to list all thread-locals, and
4057      *  #thread_variable? to check if a given thread-local exists.
4058      *
4059      *  === Exception handling
4060      *
4061      *  When an unhandled exception is raised inside a thread, it will
4062      *  terminate. By default, this exception will not propagate to other
4063      *  threads. The exception is stored and when another thread calls #value
4064      *  or #join, the exception will be re-raised in that thread.
4065      *
4066      *      t = Thread.new{ raise 'something went wrong' }
4067      *      t.value #=> RuntimeError: something went wrong
4068      *
4069      *  An exception can be raised from outside the thread using the
4070      *  Thread#raise instance method, which takes the same parameters as
4071      *  Kernel#raise.
4072      *
4073      *  Setting Thread.abort_on_exception = true, Thread#abort_on_exception =
4074      *  true, or $DEBUG = true will cause a subsequent unhandled exception
4075      *  raised in a thread to be automatically re-raised in the main thread.
4076      *
4077      *  With the addition of the class method ::handle_interrupt, you can now
4078      *  handle exceptions asynchronously with threads.
4079      *
4080      *  === Scheduling
4081      *
4082      *  Ruby provides a few ways to support scheduling threads in your program.
4083      *
4084      *  The first way is by using the class method ::stop, to put the current
4085      *  running thread to sleep and schedule the execution of another thread.
4086      *
4087      *  Once a thread is asleep, you can use the instance method #wakeup to
4088      *  mark your thread as eligible for scheduling.
4089      *
4090      *  You can also try ::pass, which attempts to pass execution to another
4091      *  thread but is dependent on the OS whether a running thread will switch
4092      *  or not. The same goes for #priority, which lets you hint to the thread
4093      *  scheduler which threads you want to take precedence when passing
4094      *  execution. This method is also dependent on the OS and may be ignored
4095      *  on some platforms.
4096      *
4097      */
4098     rb_cThread = rb_define_class("Thread", rb_cObject);
4099     rb_undef_alloc_func(rb_cThread);
4100
4101 #if VM_COLLECT_USAGE_DETAILS
4102     /* ::RubyVM::USAGE_ANALYSIS_* */
4103 #define define_usage_analysis_hash(name) /* shut up rdoc -C */ \
4104     rb_define_const(rb_cRubyVM, "USAGE_ANALYSIS_" #name, rb_hash_new())
4105     define_usage_analysis_hash(INSN);
4106     define_usage_analysis_hash(REGS);
4107     define_usage_analysis_hash(INSN_BIGRAM);
4108
4109     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_START", usage_analysis_insn_start, 0);
4110     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_START", usage_analysis_operand_start, 0);
4111     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_START", usage_analysis_register_start, 0);
4112     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_STOP", usage_analysis_insn_stop, 0);
4113     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_STOP", usage_analysis_operand_stop, 0);
4114     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_STOP", usage_analysis_register_stop, 0);
4115     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_RUNNING", usage_analysis_insn_running, 0);
4116     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_RUNNING", usage_analysis_operand_running, 0);
4117     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_RUNNING", usage_analysis_register_running, 0);
4118     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_CLEAR", usage_analysis_insn_clear, 0);
4119     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_CLEAR", usage_analysis_operand_clear, 0);
4120     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_CLEAR", usage_analysis_register_clear, 0);
4121 #endif
4122
4123     /* ::RubyVM::OPTS
4124      * An Array of VM build options.
4125      * This constant is MRI specific.
4126      */
4127     rb_define_const(rb_cRubyVM, "OPTS", opts = rb_ary_new());
4128
4129 #if   OPT_DIRECT_THREADED_CODE
4130     rb_ary_push(opts, rb_str_new2("direct threaded code"));
4131 #elif OPT_TOKEN_THREADED_CODE
4132     rb_ary_push(opts, rb_str_new2("token threaded code"));
4133 #elif OPT_CALL_THREADED_CODE
4134     rb_ary_push(opts, rb_str_new2("call threaded code"));
4135 #endif
4136
4137 #if OPT_OPERANDS_UNIFICATION
4138     rb_ary_push(opts, rb_str_new2("operands unification"));
4139 #endif
4140 #if OPT_INSTRUCTIONS_UNIFICATION
4141     rb_ary_push(opts, rb_str_new2("instructions unification"));
4142 #endif
4143 #if OPT_INLINE_METHOD_CACHE
4144     rb_ary_push(opts, rb_str_new2("inline method cache"));
4145 #endif
4146
4147     /* ::RubyVM::INSTRUCTION_NAMES
4148      * A list of bytecode instruction names in MRI.
4149      * This constant is MRI specific.
4150      */
4151     rb_define_const(rb_cRubyVM, "INSTRUCTION_NAMES", rb_insns_name_array());
4152
4153     /* ::RubyVM::DEFAULT_PARAMS
4154      * This constant exposes the VM's default parameters.
4155      * Note that changing these values does not affect VM execution.
4156      * Specification is not stable and you should not depend on this value.
4157      * Of course, this constant is MRI specific.
4158      */
4159     rb_define_const(rb_cRubyVM, "DEFAULT_PARAMS", vm_default_params());
4160
4161     /* debug functions ::RubyVM::SDR(), ::RubyVM::NSDR() */
4162 #if VMDEBUG
4163     rb_define_singleton_method(rb_cRubyVM, "SDR", sdr, 0);
4164     rb_define_singleton_method(rb_cRubyVM, "NSDR", nsdr, 0);
4165     rb_define_singleton_method(rb_cRubyVM, "mtbl", vm_mtbl, 2);
4166     rb_define_singleton_method(rb_cRubyVM, "mtbl2", vm_mtbl2, 2);
4167 #else
4168     (void)sdr;
4169     (void)nsdr;
4170     (void)vm_mtbl;
4171     (void)vm_mtbl2;
4172 #endif
4173
4174     /* VM bootstrap: phase 2 */
4175     {
4176         rb_vm_t *vm = ruby_current_vm_ptr;
4177         rb_thread_t *th = GET_THREAD();
4178         VALUE filename = rb_fstring_lit("<main>");
4179         const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP);
4180
4181         // Ractor setup
4182         rb_ractor_main_setup(vm, th->ractor, th);
4183
4184         /* create vm object */
4185         vm->self = TypedData_Wrap_Struct(rb_cRubyVM, &vm_data_type, vm);
4186
4187         /* create main thread */
4188         th->self = TypedData_Wrap_Struct(rb_cThread, &thread_data_type, th);
4189         vm->ractor.main_thread = th;
4190         vm->ractor.main_ractor = th->ractor;
4191         th->vm = vm;
4192         th->top_wrapper = 0;
4193         th->top_self = rb_vm_top_self();
4194
4195         rb_vm_register_global_object((VALUE)iseq);
4196         th->ec->cfp->iseq = iseq;
4197         th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded;
4198         th->ec->cfp->self = th->top_self;
4199
4200         VM_ENV_FLAGS_UNSET(th->ec->cfp->ep, VM_FRAME_FLAG_CFRAME);
4201         VM_STACK_ENV_WRITE(th->ec->cfp->ep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE, FALSE, NULL, FALSE, FALSE));
4202
4203         /*
4204          * The Binding of the top level scope
4205          */
4206         rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new());
4207
4208 #ifdef _WIN32
4209         rb_objspace_gc_enable(vm->objspace);
4210 #endif
4211     }
4212     vm_init_redefined_flag();
4213
4214     rb_block_param_proxy = rb_obj_alloc(rb_cObject);
4215     rb_add_method_optimized(rb_singleton_class(rb_block_param_proxy), idCall,
4216                             OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC);
4217     rb_obj_freeze(rb_block_param_proxy);
4218     rb_vm_register_global_object(rb_block_param_proxy);
4219
4220     /* vm_backtrace.c */
4221     Init_vm_backtrace();
4222 }
4223
4224 void
4225 rb_vm_set_progname(VALUE filename)
4226 {
4227     rb_thread_t *th = GET_VM()->ractor.main_thread;
4228     rb_control_frame_t *cfp = (void *)(th->ec->vm_stack + th->ec->vm_stack_size);
4229     --cfp;
4230
4231     filename = rb_str_new_frozen(filename);
4232     rb_iseq_pathobj_set(cfp->iseq, filename, rb_iseq_realpath(cfp->iseq));
4233 }
4234
4235 extern const struct st_hash_type rb_fstring_hash_type;
4236
4237 void
4238 Init_BareVM(void)
4239 {
4240     /* VM bootstrap: phase 1 */
4241     rb_vm_t *vm = ruby_mimcalloc(1, sizeof(*vm));
4242     rb_thread_t *th = ruby_mimcalloc(1, sizeof(*th));
4243     if (!vm || !th) {
4244         fputs("[FATAL] failed to allocate memory\n", stderr);
4245         exit(EXIT_FAILURE);
4246     }
4247
4248     // setup the VM
4249     vm_init2(vm);
4250
4251     rb_vm_postponed_job_queue_init(vm);
4252     ruby_current_vm_ptr = vm;
4253     rb_objspace_alloc();
4254     vm->negative_cme_table = rb_id_table_create(16);
4255     vm->overloaded_cme_table = st_init_numtable();
4256     vm->constant_cache = rb_id_table_create(0);
4257     vm->unused_block_warning_table = st_init_numtable();
4258
4259     // TODO: remove before Ruby 3.4.0 release
4260     const char *s = getenv("RUBY_TRY_UNUSED_BLOCK_WARNING_STRICT");
4261     if (s && strcmp(s, "1") == 0) {
4262         vm->unused_block_warning_strict = true;
4263     }
4264
4265     // setup main thread
4266     th->nt = ZALLOC(struct rb_native_thread);
4267     th->vm = vm;
4268     th->ractor = vm->ractor.main_ractor = rb_ractor_main_alloc();
4269     Init_native_thread(th);
4270     rb_jit_cont_init();
4271     th_init(th, 0, vm);
4272
4273     rb_ractor_set_current_ec(th->ractor, th->ec);
4274     /* n.b. native_main_thread_stack_top is set by the INIT_STACK macro */
4275     ruby_thread_init_stack(th, native_main_thread_stack_top);
4276
4277     // setup ractor system
4278     rb_native_mutex_initialize(&vm->ractor.sync.lock);
4279     rb_native_cond_initialize(&vm->ractor.sync.terminate_cond);
4280
4281     vm_opt_method_def_table = st_init_numtable();
4282     vm_opt_mid_table = st_init_numtable();
4283
4284 #ifdef RUBY_THREAD_WIN32_H
4285     rb_native_cond_initialize(&vm->ractor.sync.barrier_cond);
4286 #endif
4287 }
4288
4289 void
4290 ruby_init_stack(void *addr)
4291 {
4292     native_main_thread_stack_top = addr;
4293 }
4294
4295 #ifndef _WIN32
4296 #include <unistd.h>
4297 #include <sys/mman.h>
4298 #endif
4299
4300
4301 #ifndef MARK_OBJECT_ARY_BUCKET_SIZE
4302 #define MARK_OBJECT_ARY_BUCKET_SIZE 1024
4303 #endif
4304
4305 struct pin_array_list {
4306     VALUE next;
4307     long len;
4308     VALUE *array;
4309 };
4310
4311 static void
4312 pin_array_list_mark(void *data)
4313 {
4314     struct pin_array_list *array = (struct pin_array_list *)data;
4315     rb_gc_mark_movable(array->next);
4316
4317     rb_gc_mark_vm_stack_values(array->len, array->array);
4318 }
4319
4320 static void
4321 pin_array_list_free(void *data)
4322 {
4323     struct pin_array_list *array = (struct pin_array_list *)data;
4324     xfree(array->array);
4325 }
4326
4327 static size_t
4328 pin_array_list_memsize(const void *data)
4329 {
4330     return sizeof(struct pin_array_list) + (MARK_OBJECT_ARY_BUCKET_SIZE * sizeof(VALUE));
4331 }
4332
4333 static void
4334 pin_array_list_update_references(void *data)
4335 {
4336     struct pin_array_list *array = (struct pin_array_list *)data;
4337     array->next = rb_gc_location(array->next);
4338 }
4339
4340 static const rb_data_type_t pin_array_list_type = {
4341     .wrap_struct_name = "VM/pin_array_list",
4342     .function = {
4343         .dmark = pin_array_list_mark,
4344         .dfree = pin_array_list_free,
4345         .dsize = pin_array_list_memsize,
4346         .dcompact = pin_array_list_update_references,
4347     },
4348     .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
4349 };
4350
4351 static VALUE
4352 pin_array_list_new(VALUE next)
4353 {
4354     struct pin_array_list *array_list;
4355     VALUE obj = TypedData_Make_Struct(0, struct pin_array_list, &pin_array_list_type, array_list);
4356     RB_OBJ_WRITE(obj, &array_list->next, next);
4357     array_list->array = ALLOC_N(VALUE, MARK_OBJECT_ARY_BUCKET_SIZE);
4358     return obj;
4359 }
4360
4361 static VALUE
4362 pin_array_list_append(VALUE obj, VALUE item)
4363 {
4364     struct pin_array_list *array_list;
4365     TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list);
4366
4367     if (array_list->len >= MARK_OBJECT_ARY_BUCKET_SIZE) {
4368         obj = pin_array_list_new(obj);
4369         TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list);
4370     }
4371
4372     RB_OBJ_WRITE(obj, &array_list->array[array_list->len], item);
4373     array_list->len++;
4374     return obj;
4375 }
4376
4377 void
4378 rb_vm_register_global_object(VALUE obj)
4379 {
4380     RUBY_ASSERT(!RB_SPECIAL_CONST_P(obj));
4381     if (RB_SPECIAL_CONST_P(obj)) {
4382         return;
4383     }
4384
4385     switch (RB_BUILTIN_TYPE(obj)) {
4386       case T_CLASS:
4387       case T_MODULE:
4388         if (FL_TEST(obj, RCLASS_IS_ROOT)) {
4389             return;
4390         }
4391         FL_SET(obj, RCLASS_IS_ROOT);
4392         break;
4393       default:
4394         break;
4395     }
4396     RB_VM_LOCK_ENTER();
4397     {
4398         VALUE list = GET_VM()->mark_object_ary;
4399         VALUE head = pin_array_list_append(list, obj);
4400         if (head != list) {
4401             GET_VM()->mark_object_ary = head;
4402         }
4403         RB_GC_GUARD(obj);
4404     }
4405     RB_VM_LOCK_LEAVE();
4406 }
4407
4408 void
4409 Init_vm_objects(void)
4410 {
4411     rb_vm_t *vm = GET_VM();
4412
4413     /* initialize mark object array, hash */
4414     vm->mark_object_ary = pin_array_list_new(Qnil);
4415     vm->loading_table = st_init_strtable();
4416     vm->ci_table = st_init_table(&vm_ci_hashtype);
4417     vm->frozen_strings = st_init_table_with_size(&rb_fstring_hash_type, 10000);
4418 }
4419
4420 /* Stub for builtin function when not building YJIT units*/
4421 #if !USE_YJIT
4422 void Init_builtin_yjit(void) {}
4423 #endif
4424
4425 /* top self */
4426
4427 static VALUE
4428 main_to_s(VALUE obj)
4429 {
4430     return rb_str_new2("main");
4431 }
4432
4433 VALUE
4434 rb_vm_top_self(void)
4435 {
4436     return GET_VM()->top_self;
4437 }
4438
4439 void
4440 Init_top_self(void)
4441 {
4442     rb_vm_t *vm = GET_VM();
4443
4444     vm->top_self = rb_obj_alloc(rb_cObject);
4445     rb_define_singleton_method(rb_vm_top_self(), "to_s", main_to_s, 0);
4446     rb_define_alias(rb_singleton_class(rb_vm_top_self()), "inspect", "to_s");
4447 }
4448
4449 VALUE *
4450 rb_ruby_verbose_ptr(void)
4451 {
4452     rb_ractor_t *cr = GET_RACTOR();
4453     return &cr->verbose;
4454 }
4455
4456 static bool prism;
4457
4458 bool *
4459 rb_ruby_prism_ptr(void)
4460 {
4461     return &prism;
4462 }
4463
4464 VALUE *
4465 rb_ruby_debug_ptr(void)
4466 {
4467     rb_ractor_t *cr = GET_RACTOR();
4468     return &cr->debug;
4469 }
4470
4471 bool rb_free_at_exit = false;
4472
4473 bool
4474 ruby_free_at_exit_p(void)
4475 {
4476     return rb_free_at_exit;
4477 }
4478
4479 /* iseq.c */
4480 VALUE rb_insn_operand_intern(const rb_iseq_t *iseq,
4481                              VALUE insn, int op_no, VALUE op,
4482                              int len, size_t pos, VALUE *pnop, VALUE child);
4483
4484 st_table *
4485 rb_vm_fstring_table(void)
4486 {
4487     return GET_VM()->frozen_strings;
4488 }
4489
4490 #if VM_COLLECT_USAGE_DETAILS
4491
4492 #define HASH_ASET(h, k, v) rb_hash_aset((h), (st_data_t)(k), (st_data_t)(v))
4493
4494 /* uh = {
4495  *   insn(Fixnum) => ihash(Hash)
4496  * }
4497  * ihash = {
4498  *   -1(Fixnum) => count,      # insn usage
4499  *    0(Fixnum) => ophash,     # operand usage
4500  * }
4501  * ophash = {
4502  *   val(interned string) => count(Fixnum)
4503  * }
4504  */
4505 static void
4506 vm_analysis_insn(int insn)
4507 {
4508     ID usage_hash;
4509     ID bigram_hash;
4510     static int prev_insn = -1;
4511
4512     VALUE uh;
4513     VALUE ihash;
4514     VALUE cv;
4515
4516     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4517     CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM");
4518     uh = rb_const_get(rb_cRubyVM, usage_hash);
4519     if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) {
4520         ihash = rb_hash_new();
4521         HASH_ASET(uh, INT2FIX(insn), ihash);
4522     }
4523     if (NIL_P(cv = rb_hash_aref(ihash, INT2FIX(-1)))) {
4524         cv = INT2FIX(0);
4525     }
4526     HASH_ASET(ihash, INT2FIX(-1), INT2FIX(FIX2INT(cv) + 1));
4527
4528     /* calc bigram */
4529     if (prev_insn != -1) {
4530         VALUE bi;
4531         VALUE ary[2];
4532         VALUE cv;
4533
4534         ary[0] = INT2FIX(prev_insn);
4535         ary[1] = INT2FIX(insn);
4536         bi = rb_ary_new4(2, &ary[0]);
4537
4538         uh = rb_const_get(rb_cRubyVM, bigram_hash);
4539         if (NIL_P(cv = rb_hash_aref(uh, bi))) {
4540             cv = INT2FIX(0);
4541         }
4542         HASH_ASET(uh, bi, INT2FIX(FIX2INT(cv) + 1));
4543     }
4544     prev_insn = insn;
4545 }
4546
4547 static void
4548 vm_analysis_operand(int insn, int n, VALUE op)
4549 {
4550     ID usage_hash;
4551
4552     VALUE uh;
4553     VALUE ihash;
4554     VALUE ophash;
4555     VALUE valstr;
4556     VALUE cv;
4557
4558     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4559
4560     uh = rb_const_get(rb_cRubyVM, usage_hash);
4561     if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) {
4562         ihash = rb_hash_new();
4563         HASH_ASET(uh, INT2FIX(insn), ihash);
4564     }
4565     if (NIL_P(ophash = rb_hash_aref(ihash, INT2FIX(n)))) {
4566         ophash = rb_hash_new();
4567         HASH_ASET(ihash, INT2FIX(n), ophash);
4568     }
4569     /* intern */
4570     valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0);
4571
4572     /* set count */
4573     if (NIL_P(cv = rb_hash_aref(ophash, valstr))) {
4574         cv = INT2FIX(0);
4575     }
4576     HASH_ASET(ophash, valstr, INT2FIX(FIX2INT(cv) + 1));
4577 }
4578
4579 static void
4580 vm_analysis_register(int reg, int isset)
4581 {
4582     ID usage_hash;
4583     VALUE uh;
4584     VALUE valstr;
4585     static const char regstrs[][5] = {
4586         "pc",                   /* 0 */
4587         "sp",                   /* 1 */
4588         "ep",                   /* 2 */
4589         "cfp",                  /* 3 */
4590         "self",                 /* 4 */
4591         "iseq",                 /* 5 */
4592     };
4593     static const char getsetstr[][4] = {
4594         "get",
4595         "set",
4596     };
4597     static VALUE syms[sizeof(regstrs) / sizeof(regstrs[0])][2];
4598
4599     VALUE cv;
4600
4601     CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS");
4602     if (syms[0] == 0) {
4603         char buff[0x10];
4604         int i;
4605
4606         for (i = 0; i < (int)(sizeof(regstrs) / sizeof(regstrs[0])); i++) {
4607             int j;
4608             for (j = 0; j < 2; j++) {
4609                 snprintf(buff, 0x10, "%d %s %-4s", i, getsetstr[j], regstrs[i]);
4610                 syms[i][j] = ID2SYM(rb_intern(buff));
4611             }
4612         }
4613     }
4614     valstr = syms[reg][isset];
4615
4616     uh = rb_const_get(rb_cRubyVM, usage_hash);
4617     if (NIL_P(cv = rb_hash_aref(uh, valstr))) {
4618         cv = INT2FIX(0);
4619     }
4620     HASH_ASET(uh, valstr, INT2FIX(FIX2INT(cv) + 1));
4621 }
4622
4623 #undef HASH_ASET
4624
4625 static void (*ruby_vm_collect_usage_func_insn)(int insn) = NULL;
4626 static void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op) = NULL;
4627 static void (*ruby_vm_collect_usage_func_register)(int reg, int isset) = NULL;
4628
4629 /* :nodoc: */
4630 static VALUE
4631 usage_analysis_insn_start(VALUE self)
4632 {
4633     ruby_vm_collect_usage_func_insn = vm_analysis_insn;
4634     return Qnil;
4635 }
4636
4637 /* :nodoc: */
4638 static VALUE
4639 usage_analysis_operand_start(VALUE self)
4640 {
4641     ruby_vm_collect_usage_func_operand = vm_analysis_operand;
4642     return Qnil;
4643 }
4644
4645 /* :nodoc: */
4646 static VALUE
4647 usage_analysis_register_start(VALUE self)
4648 {
4649     ruby_vm_collect_usage_func_register = vm_analysis_register;
4650     return Qnil;
4651 }
4652
4653 /* :nodoc: */
4654 static VALUE
4655 usage_analysis_insn_stop(VALUE self)
4656 {
4657     ruby_vm_collect_usage_func_insn = 0;
4658     return Qnil;
4659 }
4660
4661 /* :nodoc: */
4662 static VALUE
4663 usage_analysis_operand_stop(VALUE self)
4664 {
4665     ruby_vm_collect_usage_func_operand = 0;
4666     return Qnil;
4667 }
4668
4669 /* :nodoc: */
4670 static VALUE
4671 usage_analysis_register_stop(VALUE self)
4672 {
4673     ruby_vm_collect_usage_func_register = 0;
4674     return Qnil;
4675 }
4676
4677 /* :nodoc: */
4678 static VALUE
4679 usage_analysis_insn_running(VALUE self)
4680 {
4681     return RBOOL(ruby_vm_collect_usage_func_insn != 0);
4682 }
4683
4684 /* :nodoc: */
4685 static VALUE
4686 usage_analysis_operand_running(VALUE self)
4687 {
4688     return RBOOL(ruby_vm_collect_usage_func_operand != 0);
4689 }
4690
4691 /* :nodoc: */
4692 static VALUE
4693 usage_analysis_register_running(VALUE self)
4694 {
4695     return RBOOL(ruby_vm_collect_usage_func_register != 0);
4696 }
4697
4698 static VALUE
4699 usage_analysis_clear(VALUE self, ID usage_hash)
4700 {
4701     VALUE uh;
4702     uh = rb_const_get(self, usage_hash);
4703     rb_hash_clear(uh);
4704
4705     return Qtrue;
4706 }
4707
4708
4709 /* :nodoc: */
4710 static VALUE
4711 usage_analysis_insn_clear(VALUE self)
4712 {
4713     ID usage_hash;
4714     ID bigram_hash;
4715
4716     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4717     CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM");
4718     usage_analysis_clear(rb_cRubyVM, usage_hash);
4719     return usage_analysis_clear(rb_cRubyVM, bigram_hash);
4720 }
4721
4722 /* :nodoc: */
4723 static VALUE
4724 usage_analysis_operand_clear(VALUE self)
4725 {
4726     ID usage_hash;
4727
4728     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4729     return usage_analysis_clear(self, usage_hash);
4730 }
4731
4732 /* :nodoc: */
4733 static VALUE
4734 usage_analysis_register_clear(VALUE self)
4735 {
4736       ID usage_hash;
4737
4738     CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS");
4739     return usage_analysis_clear(self, usage_hash);
4740 }
4741
4742 #else
4743
4744 MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_insn)(int insn)) = 0;
4745 MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op)) = 0;
4746 MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_register)(int reg, int isset)) = 0;
4747
4748 #endif
4749
4750 #if VM_COLLECT_USAGE_DETAILS
4751 /* @param insn instruction number */
4752 static void
4753 vm_collect_usage_insn(int insn)
4754 {
4755     if (RUBY_DTRACE_INSN_ENABLED()) {
4756         RUBY_DTRACE_INSN(rb_insns_name(insn));
4757     }
4758     if (ruby_vm_collect_usage_func_insn)
4759         (*ruby_vm_collect_usage_func_insn)(insn);
4760 }
4761
4762 /* @param insn instruction number
4763  * @param n    n-th operand
4764  * @param op   operand value
4765  */
4766 static void
4767 vm_collect_usage_operand(int insn, int n, VALUE op)
4768 {
4769     if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) {
4770         VALUE valstr;
4771
4772         valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0);
4773
4774         RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn));
4775         RB_GC_GUARD(valstr);
4776     }
4777     if (ruby_vm_collect_usage_func_operand)
4778         (*ruby_vm_collect_usage_func_operand)(insn, n, op);
4779 }
4780
4781 /* @param reg register id. see code of vm_analysis_register() */
4782 /* @param isset 0: read, 1: write */
4783 static void
4784 vm_collect_usage_register(int reg, int isset)
4785 {
4786     if (ruby_vm_collect_usage_func_register)
4787         (*ruby_vm_collect_usage_func_register)(reg, isset);
4788 }
4789 #endif
4790
4791 const struct rb_callcache *
4792 rb_vm_empty_cc(void)
4793 {
4794     return &vm_empty_cc;
4795 }
4796
4797 const struct rb_callcache *
4798 rb_vm_empty_cc_for_super(void)
4799 {
4800     return &vm_empty_cc_for_super;
4801 }
4802
4803 #include "vm_call_iseq_optimized.inc" /* required from vm_insnhelper.c */