vm.c

   1 /**********************************************************************
   2
   3   Vm.c -
   4
   5   $Author$
   6
   7   Copyright (C) 2004-2007 Koichi Sasada
   8
   9 **********************************************************************/
  10
  11 #define vm_exec rb_vm_exec
  12
  13 #include "eval_intern.h"
  14 #include "internal.h"
  15 #include "internal/class.h"
  16 #include "internal/compile.h"
  17 #include "internal/cont.h"
  18 #include "internal/error.h"
  19 #include "internal/encoding.h"
  20 #include "internal/eval.h"
  21 #include "internal/gc.h"
  22 #include "internal/inits.h"
  23 #include "internal/missing.h"
  24 #include "internal/object.h"
  25 #include "internal/proc.h"
  26 #include "internal/re.h"
  27 #include "internal/ruby_parser.h"
  28 #include "internal/symbol.h"
  29 #include "internal/thread.h"
  30 #include "internal/transcode.h"
  31 #include "internal/vm.h"
  32 #include "internal/sanitizers.h"
  33 #include "internal/variable.h"
  34 #include "iseq.h"
  35 #include "rjit.h"
  36 #include "yjit.h"
  37 #include "ruby/st.h"
  38 #include "ruby/vm.h"
  39 #include "vm_core.h"
  40 #include "vm_callinfo.h"
  41 #include "vm_debug.h"
  42 #include "vm_exec.h"
  43 #include "vm_insnhelper.h"
  44 #include "ractor_core.h"
  45 #include "vm_sync.h"
  46 #include "shape.h"
  47
  48 #include "builtin.h"
  49
  50 #include "probes.h"
  51 #include "probes_helper.h"
  52
  53 #ifdef RUBY_ASSERT_CRITICAL_SECTION
  54 int ruby_assert_critical_section_entered = 0;
  55 #endif
  56
  57 static void *native_main_thread_stack_top;
  58
  59 VALUE rb_str_concat_literals(size_t, const VALUE*);
  60
  61 VALUE vm_exec(rb_execution_context_t *);
  62
  63 extern const char *const rb_debug_counter_names[];
  64
  65 PUREFUNC(static inline const VALUE *VM_EP_LEP(const VALUE *));
  66 static inline const VALUE *
  67 VM_EP_LEP(const VALUE *ep)
  68 {
  69     while (!VM_ENV_LOCAL_P(ep)) {
  70         ep = VM_ENV_PREV_EP(ep);
  71     }
  72     return ep;
  73 }
  74
  75 static inline const rb_control_frame_t *
  76 rb_vm_search_cf_from_ep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const VALUE * const ep)
  77 {
  78     if (!ep) {
  79         return NULL;
  80     }
  81     else {
  82         const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
  83
  84         while (cfp < eocfp) {
  85             if (cfp->ep == ep) {
  86                 return cfp;
  87             }
  88             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
  89         }
  90
  91         return NULL;
  92     }
  93 }
  94
  95 const VALUE *
  96 rb_vm_ep_local_ep(const VALUE *ep)
  97 {
  98     return VM_EP_LEP(ep);
  99 }
 100
 101 PUREFUNC(static inline const VALUE *VM_CF_LEP(const rb_control_frame_t * const cfp));
 102 static inline const VALUE *
 103 VM_CF_LEP(const rb_control_frame_t * const cfp)
 104 {
 105     return VM_EP_LEP(cfp->ep);
 106 }
 107
 108 static inline const VALUE *
 109 VM_CF_PREV_EP(const rb_control_frame_t * const cfp)
 110 {
 111     return VM_ENV_PREV_EP(cfp->ep);
 112 }
 113
 114 PUREFUNC(static inline VALUE VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp));
 115 static inline VALUE
 116 VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp)
 117 {
 118     const VALUE *ep = VM_CF_LEP(cfp);
 119     return VM_ENV_BLOCK_HANDLER(ep);
 120 }
 121
 122 int
 123 rb_vm_cframe_keyword_p(const rb_control_frame_t *cfp)
 124 {
 125     return VM_FRAME_CFRAME_KW_P(cfp);
 126 }
 127
 128 VALUE
 129 rb_vm_frame_block_handler(const rb_control_frame_t *cfp)
 130 {
 131     return VM_CF_BLOCK_HANDLER(cfp);
 132 }
 133
 134 #if VM_CHECK_MODE > 0
 135 static int
 136 VM_CFP_IN_HEAP_P(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 137 {
 138     const VALUE *start = ec->vm_stack;
 139     const VALUE *end = (VALUE *)ec->vm_stack + ec->vm_stack_size;
 140     VM_ASSERT(start != NULL);
 141
 142     if (start <= (VALUE *)cfp && (VALUE *)cfp < end) {
 143         return FALSE;
 144     }
 145     else {
 146         return TRUE;
 147     }
 148 }
 149
 150 static int
 151 VM_EP_IN_HEAP_P(const rb_execution_context_t *ec, const VALUE *ep)
 152 {
 153     const VALUE *start = ec->vm_stack;
 154     const VALUE *end = (VALUE *)ec->cfp;
 155     VM_ASSERT(start != NULL);
 156
 157     if (start <= ep && ep < end) {
 158         return FALSE;
 159     }
 160     else {
 161         return TRUE;
 162     }
 163 }
 164
 165 static int
 166 vm_ep_in_heap_p_(const rb_execution_context_t *ec, const VALUE *ep)
 167 {
 168     if (VM_EP_IN_HEAP_P(ec, ep)) {
 169         VALUE envval = ep[VM_ENV_DATA_INDEX_ENV]; /* VM_ENV_ENVVAL(ep); */
 170
 171         if (!UNDEF_P(envval)) {
 172             const rb_env_t *env = (const rb_env_t *)envval;
 173
 174             VM_ASSERT(vm_assert_env(envval));
 175             VM_ASSERT(VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED));
 176             VM_ASSERT(env->ep == ep);
 177         }
 178         return TRUE;
 179     }
 180     else {
 181         return FALSE;
 182     }
 183 }
 184
 185 int
 186 rb_vm_ep_in_heap_p(const VALUE *ep)
 187 {
 188     const rb_execution_context_t *ec = GET_EC();
 189     if (ec->vm_stack == NULL) return TRUE;
 190     return vm_ep_in_heap_p_(ec, ep);
 191 }
 192 #endif
 193
 194 static struct rb_captured_block *
 195 VM_CFP_TO_CAPTURED_BLOCK(const rb_control_frame_t *cfp)
 196 {
 197     VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp));
 198     return (struct rb_captured_block *)&cfp->self;
 199 }
 200
 201 static rb_control_frame_t *
 202 VM_CAPTURED_BLOCK_TO_CFP(const struct rb_captured_block *captured)
 203 {
 204     rb_control_frame_t *cfp = ((rb_control_frame_t *)((VALUE *)(captured) - 3));
 205     VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp));
 206     VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 7 + VM_DEBUG_BP_CHECK ? 1 : 0);
 207     return cfp;
 208 }
 209
 210 static int
 211 VM_BH_FROM_CFP_P(VALUE block_handler, const rb_control_frame_t *cfp)
 212 {
 213     const struct rb_captured_block *captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
 214     return VM_TAGGED_PTR_REF(block_handler, 0x03) == captured;
 215 }
 216
 217 static VALUE
 218 vm_passed_block_handler(rb_execution_context_t *ec)
 219 {
 220     VALUE block_handler = ec->passed_block_handler;
 221     ec->passed_block_handler = VM_BLOCK_HANDLER_NONE;
 222     vm_block_handler_verify(block_handler);
 223     return block_handler;
 224 }
 225
 226 static rb_cref_t *
 227 vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int use_prev_prev, int singleton)
 228 {
 229     VALUE refinements = Qnil;
 230     int omod_shared = FALSE;
 231
 232     /* scope */
 233     union {
 234         rb_scope_visibility_t visi;
 235         VALUE value;
 236     } scope_visi;
 237
 238     scope_visi.visi.method_visi = visi;
 239     scope_visi.visi.module_func = module_func;
 240
 241     /* refinements */
 242     if (prev_cref != NULL && prev_cref != (void *)1 /* TODO: why CREF_NEXT(cref) is 1? */) {
 243         refinements = CREF_REFINEMENTS(prev_cref);
 244
 245         if (!NIL_P(refinements)) {
 246             omod_shared = TRUE;
 247             CREF_OMOD_SHARED_SET(prev_cref);
 248         }
 249     }
 250
 251     VM_ASSERT(singleton || klass);
 252
 253     rb_cref_t *cref = IMEMO_NEW(rb_cref_t, imemo_cref, refinements);
 254     cref->klass_or_self = klass;
 255     cref->next = use_prev_prev ? CREF_NEXT(prev_cref) : prev_cref;
 256     *((rb_scope_visibility_t *)&cref->scope_visi) = scope_visi.visi;
 257
 258     if (pushed_by_eval) CREF_PUSHED_BY_EVAL_SET(cref);
 259     if (omod_shared) CREF_OMOD_SHARED_SET(cref);
 260     if (singleton) CREF_SINGLETON_SET(cref);
 261
 262     return cref;
 263 }
 264
 265 static rb_cref_t *
 266 vm_cref_new(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int singleton)
 267 {
 268     return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, FALSE, singleton);
 269 }
 270
 271 static rb_cref_t *
 272 vm_cref_new_use_prev(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval)
 273 {
 274     return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, TRUE, FALSE);
 275 }
 276
 277 static int
 278 ref_delete_symkey(VALUE key, VALUE value, VALUE unused)
 279 {
 280     return SYMBOL_P(key) ? ST_DELETE : ST_CONTINUE;
 281 }
 282
 283 static rb_cref_t *
 284 vm_cref_dup(const rb_cref_t *cref)
 285 {
 286     const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref);
 287     rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref;
 288     int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref);
 289     int singleton = CREF_SINGLETON(cref);
 290
 291     new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton);
 292
 293     if (!NIL_P(CREF_REFINEMENTS(cref))) {
 294         VALUE ref = rb_hash_dup(CREF_REFINEMENTS(cref));
 295         rb_hash_foreach(ref, ref_delete_symkey, Qnil);
 296         CREF_REFINEMENTS_SET(new_cref, ref);
 297         CREF_OMOD_SHARED_UNSET(new_cref);
 298     }
 299
 300     return new_cref;
 301 }
 302
 303
 304 rb_cref_t *
 305 rb_vm_cref_dup_without_refinements(const rb_cref_t *cref)
 306 {
 307     const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref);
 308     rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref;
 309     int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref);
 310     int singleton = CREF_SINGLETON(cref);
 311
 312     new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton);
 313
 314     if (!NIL_P(CREF_REFINEMENTS(cref))) {
 315         CREF_REFINEMENTS_SET(new_cref, Qnil);
 316         CREF_OMOD_SHARED_UNSET(new_cref);
 317     }
 318
 319     return new_cref;
 320 }
 321
 322 static rb_cref_t *
 323 vm_cref_new_toplevel(rb_execution_context_t *ec)
 324 {
 325     rb_cref_t *cref = vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE /* toplevel visibility is private */, FALSE, NULL, FALSE, FALSE);
 326     VALUE top_wrapper = rb_ec_thread_ptr(ec)->top_wrapper;
 327
 328     if (top_wrapper) {
 329         cref = vm_cref_new(top_wrapper, METHOD_VISI_PRIVATE, FALSE, cref, FALSE, FALSE);
 330     }
 331
 332     return cref;
 333 }
 334
 335 rb_cref_t *
 336 rb_vm_cref_new_toplevel(void)
 337 {
 338     return vm_cref_new_toplevel(GET_EC());
 339 }
 340
 341 static void
 342 vm_cref_dump(const char *mesg, const rb_cref_t *cref)
 343 {
 344     ruby_debug_printf("vm_cref_dump: %s (%p)\n", mesg, (void *)cref);
 345
 346     while (cref) {
 347         ruby_debug_printf("= cref| klass: %s\n", RSTRING_PTR(rb_class_path(CREF_CLASS(cref))));
 348         cref = CREF_NEXT(cref);
 349     }
 350 }
 351
 352 void
 353 rb_vm_block_ep_update(VALUE obj, const struct rb_block *dst, const VALUE *ep)
 354 {
 355     *((const VALUE **)&dst->as.captured.ep) = ep;
 356     RB_OBJ_WRITTEN(obj, Qundef, VM_ENV_ENVVAL(ep));
 357 }
 358
 359 static void
 360 vm_bind_update_env(VALUE bindval, rb_binding_t *bind, VALUE envval)
 361 {
 362     const rb_env_t *env = (rb_env_t *)envval;
 363     RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, env->iseq);
 364     rb_vm_block_ep_update(bindval, &bind->block, env->ep);
 365 }
 366
 367 #if VM_COLLECT_USAGE_DETAILS
 368 static void vm_collect_usage_operand(int insn, int n, VALUE op);
 369 static void vm_collect_usage_insn(int insn);
 370 static void vm_collect_usage_register(int reg, int isset);
 371 #endif
 372
 373 static VALUE vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp);
 374 extern VALUE rb_vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
 375                                   int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
 376                                   const rb_callable_method_entry_t *me);
 377 static VALUE vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler);
 378
 379 #if USE_YJIT
 380 // Counter to serve as a proxy for execution time, total number of calls
 381 static uint64_t yjit_total_entry_hits = 0;
 382
 383 // Number of calls used to estimate how hot an ISEQ is
 384 #define YJIT_CALL_COUNT_INTERV 20u
 385
 386 /// Test whether we are ready to compile an ISEQ or not
 387 static inline bool
 388 rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls)
 389 {
 390     yjit_total_entry_hits += 1;
 391
 392     // Record the number of calls at the beginning of the interval
 393     if (entry_calls + YJIT_CALL_COUNT_INTERV == rb_yjit_call_threshold) {
 394         iseq->body->yjit_calls_at_interv = yjit_total_entry_hits;
 395     }
 396
 397     // Try to estimate the total time taken (total number of calls) to reach 20 calls to this ISEQ
 398     // This give us a ratio of how hot/cold this ISEQ is
 399     if (entry_calls == rb_yjit_call_threshold) {
 400         // We expect threshold 1 to compile everything immediately
 401         if (rb_yjit_call_threshold < YJIT_CALL_COUNT_INTERV) {
 402             return true;
 403         }
 404
 405         uint64_t num_calls = yjit_total_entry_hits - iseq->body->yjit_calls_at_interv;
 406
 407         // Reject ISEQs that don't get called often enough
 408         if (num_calls > rb_yjit_cold_threshold) {
 409             rb_yjit_incr_counter("cold_iseq_entry");
 410             return false;
 411         }
 412
 413         return true;
 414     }
 415
 416     return false;
 417 }
 418 #else
 419 #define rb_yjit_threshold_hit(iseq, entry_calls) false
 420 #endif
 421
 422 #if USE_RJIT || USE_YJIT
 423 // Generate JIT code that supports the following kinds of ISEQ entries:
 424 //   * The first ISEQ on vm_exec (e.g. <main>, or Ruby methods/blocks
 425 //     called by a C method). The current frame has VM_FRAME_FLAG_FINISH.
 426 //     The current vm_exec stops if JIT code returns a non-Qundef value.
 427 //   * ISEQs called by the interpreter on vm_sendish (e.g. Ruby methods or
 428 //     blocks called by a Ruby frame that isn't compiled or side-exited).
 429 //     The current frame doesn't have VM_FRAME_FLAG_FINISH. The current
 430 //     vm_exec does NOT stop whether JIT code returns Qundef or not.
 431 static inline rb_jit_func_t
 432 jit_compile(rb_execution_context_t *ec)
 433 {
 434     const rb_iseq_t *iseq = ec->cfp->iseq;
 435     struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
 436     bool yjit_enabled = rb_yjit_enabled_p;
 437     if (!(yjit_enabled || rb_rjit_call_p)) {
 438         return NULL;
 439     }
 440
 441     // Increment the ISEQ's call counter and trigger JIT compilation if not compiled
 442     if (body->jit_entry == NULL) {
 443         body->jit_entry_calls++;
 444         if (yjit_enabled) {
 445             if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) {
 446                 rb_yjit_compile_iseq(iseq, ec, false);
 447             }
 448         }
 449         else if (body->jit_entry_calls == rb_rjit_call_threshold()) {
 450             rb_rjit_compile(iseq);
 451         }
 452     }
 453     return body->jit_entry;
 454 }
 455
 456 // Execute JIT code compiled by jit_compile()
 457 static inline VALUE
 458 jit_exec(rb_execution_context_t *ec)
 459 {
 460     rb_jit_func_t func = jit_compile(ec);
 461     if (func) {
 462         // Call the JIT code
 463         return func(ec, ec->cfp);
 464     }
 465     else {
 466         return Qundef;
 467     }
 468 }
 469 #else
 470 # define jit_compile(ec) ((rb_jit_func_t)0)
 471 # define jit_exec(ec) Qundef
 472 #endif
 473
 474 #if USE_YJIT
 475 // Generate JIT code that supports the following kind of ISEQ entry:
 476 //   * The first ISEQ pushed by vm_exec_handle_exception. The frame would
 477 //     point to a location specified by a catch table, and it doesn't have
 478 //     VM_FRAME_FLAG_FINISH. The current vm_exec stops if JIT code returns
 479 //     a non-Qundef value. So you should not return a non-Qundef value
 480 //     until ec->cfp is changed to a frame with VM_FRAME_FLAG_FINISH.
 481 static inline rb_jit_func_t
 482 jit_compile_exception(rb_execution_context_t *ec)
 483 {
 484     const rb_iseq_t *iseq = ec->cfp->iseq;
 485     struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
 486     if (!rb_yjit_enabled_p) {
 487         return NULL;
 488     }
 489
 490     // Increment the ISEQ's call counter and trigger JIT compilation if not compiled
 491     if (body->jit_exception == NULL) {
 492         body->jit_exception_calls++;
 493         if (body->jit_exception_calls == rb_yjit_call_threshold) {
 494             rb_yjit_compile_iseq(iseq, ec, true);
 495         }
 496     }
 497
 498     return body->jit_exception;
 499 }
 500
 501 // Execute JIT code compiled by jit_compile_exception()
 502 static inline VALUE
 503 jit_exec_exception(rb_execution_context_t *ec)
 504 {
 505     rb_jit_func_t func = jit_compile_exception(ec);
 506     if (func) {
 507         // Call the JIT code
 508         return func(ec, ec->cfp);
 509     }
 510     else {
 511         return Qundef;
 512     }
 513 }
 514 #else
 515 # define jit_compile_exception(ec) ((rb_jit_func_t)0)
 516 # define jit_exec_exception(ec) Qundef
 517 #endif
 518
 519 static void add_opt_method_entry(const rb_method_entry_t *me);
 520
 521 #include "vm_insnhelper.c"
 522
 523 #include "vm_exec.c"
 524
 525 #include "vm_method.c"
 526 #include "vm_eval.c"
 527
 528 #define PROCDEBUG 0
 529
 530 VALUE rb_cRubyVM;
 531 VALUE rb_cThread;
 532 VALUE rb_mRubyVMFrozenCore;
 533 VALUE rb_block_param_proxy;
 534
 535 VALUE ruby_vm_const_missing_count = 0;
 536 rb_vm_t *ruby_current_vm_ptr = NULL;
 537 rb_ractor_t *ruby_single_main_ractor;
 538 bool ruby_vm_keep_script_lines;
 539
 540 #ifdef RB_THREAD_LOCAL_SPECIFIER
 541 RB_THREAD_LOCAL_SPECIFIER rb_execution_context_t *ruby_current_ec;
 542
 543 #ifdef RUBY_NT_SERIAL
 544 RB_THREAD_LOCAL_SPECIFIER rb_atomic_t ruby_nt_serial;
 545 #endif
 546
 547 // no-inline decl on thread_pthread.h
 548 rb_execution_context_t *
 549 rb_current_ec_noinline(void)
 550 {
 551     return ruby_current_ec;
 552 }
 553
 554 void
 555 rb_current_ec_set(rb_execution_context_t *ec)
 556 {
 557     ruby_current_ec = ec;
 558 }
 559
 560
 561 #ifdef __APPLE__
 562 rb_execution_context_t *
 563 rb_current_ec(void)
 564 {
 565     return ruby_current_ec;
 566 }
 567
 568 #endif
 569 #else
 570 native_tls_key_t ruby_current_ec_key;
 571 #endif
 572
 573 rb_event_flag_t ruby_vm_event_flags;
 574 rb_event_flag_t ruby_vm_event_enabled_global_flags;
 575 unsigned int    ruby_vm_event_local_num;
 576
 577 rb_serial_t ruby_vm_constant_cache_invalidations = 0;
 578 rb_serial_t ruby_vm_constant_cache_misses = 0;
 579 rb_serial_t ruby_vm_global_cvar_state = 1;
 580
 581 static const struct rb_callcache vm_empty_cc = {
 582     .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE,
 583     .klass = Qfalse,
 584     .cme_  = NULL,
 585     .call_ = vm_call_general,
 586     .aux_  = {
 587         .v = Qfalse,
 588     }
 589 };
 590
 591 static const struct rb_callcache vm_empty_cc_for_super = {
 592     .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE,
 593     .klass = Qfalse,
 594     .cme_  = NULL,
 595     .call_ = vm_call_super_method,
 596     .aux_  = {
 597         .v = Qfalse,
 598     }
 599 };
 600
 601 static void thread_free(void *ptr);
 602
 603 void
 604 rb_vm_inc_const_missing_count(void)
 605 {
 606     ruby_vm_const_missing_count +=1;
 607 }
 608
 609 int
 610 rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id,
 611                 struct ruby_dtrace_method_hook_args *args)
 612 {
 613     enum ruby_value_type type;
 614     if (!klass) {
 615         if (!ec) ec = GET_EC();
 616         if (!rb_ec_frame_method_id_and_class(ec, &id, 0, &klass) || !klass)
 617             return FALSE;
 618     }
 619     if (RB_TYPE_P(klass, T_ICLASS)) {
 620         klass = RBASIC(klass)->klass;
 621     }
 622     else if (RCLASS_SINGLETON_P(klass)) {
 623         klass = RCLASS_ATTACHED_OBJECT(klass);
 624         if (NIL_P(klass)) return FALSE;
 625     }
 626     type = BUILTIN_TYPE(klass);
 627     if (type == T_CLASS || type == T_ICLASS || type == T_MODULE) {
 628         VALUE name = rb_class_path(klass);
 629         const char *classname, *filename;
 630         const char *methodname = rb_id2name(id);
 631         if (methodname && (filename = rb_source_location_cstr(&args->line_no)) != 0) {
 632             if (NIL_P(name) || !(classname = StringValuePtr(name)))
 633                 classname = "<unknown>";
 634             args->classname = classname;
 635             args->methodname = methodname;
 636             args->filename = filename;
 637             args->klass = klass;
 638             args->name = name;
 639             return TRUE;
 640         }
 641     }
 642     return FALSE;
 643 }
 644
 645 extern unsigned int redblack_buffer_size;
 646
 647 /*
 648  *  call-seq:
 649  *    RubyVM.stat -> Hash
 650  *    RubyVM.stat(hsh) -> hsh
 651  *    RubyVM.stat(Symbol) -> Numeric
 652  *
 653  *  Returns a Hash containing implementation-dependent counters inside the VM.
 654  *
 655  *  This hash includes information about method/constant caches:
 656  *
 657  *    {
 658  *      :constant_cache_invalidations=>2,
 659  *      :constant_cache_misses=>14,
 660  *      :global_cvar_state=>27
 661  *    }
 662  *
 663  *  If <tt>USE_DEBUG_COUNTER</tt> is enabled, debug counters will be included.
 664  *
 665  *  The contents of the hash are implementation specific and may be changed in
 666  *  the future.
 667  *
 668  *  This method is only expected to work on C Ruby.
 669  */
 670 static VALUE
 671 vm_stat(int argc, VALUE *argv, VALUE self)
 672 {
 673     static VALUE sym_constant_cache_invalidations, sym_constant_cache_misses, sym_global_cvar_state, sym_next_shape_id;
 674     static VALUE sym_shape_cache_size;
 675     VALUE arg = Qnil;
 676     VALUE hash = Qnil, key = Qnil;
 677
 678     if (rb_check_arity(argc, 0, 1) == 1) {
 679         arg = argv[0];
 680         if (SYMBOL_P(arg))
 681             key = arg;
 682         else if (RB_TYPE_P(arg, T_HASH))
 683             hash = arg;
 684         else
 685             rb_raise(rb_eTypeError, "non-hash or symbol given");
 686     }
 687     else {
 688         hash = rb_hash_new();
 689     }
 690
 691 #define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
 692     S(constant_cache_invalidations);
 693     S(constant_cache_misses);
 694         S(global_cvar_state);
 695     S(next_shape_id);
 696     S(shape_cache_size);
 697 #undef S
 698
 699 #define SET(name, attr) \
 700     if (key == sym_##name) \
 701         return SERIALT2NUM(attr); \
 702     else if (hash != Qnil) \
 703         rb_hash_aset(hash, sym_##name, SERIALT2NUM(attr));
 704
 705     SET(constant_cache_invalidations, ruby_vm_constant_cache_invalidations);
 706     SET(constant_cache_misses, ruby_vm_constant_cache_misses);
 707     SET(global_cvar_state, ruby_vm_global_cvar_state);
 708     SET(next_shape_id, (rb_serial_t)GET_SHAPE_TREE()->next_shape_id);
 709     SET(shape_cache_size, (rb_serial_t)GET_SHAPE_TREE()->cache_size);
 710 #undef SET
 711
 712 #if USE_DEBUG_COUNTER
 713     ruby_debug_counter_show_at_exit(FALSE);
 714     for (size_t i = 0; i < RB_DEBUG_COUNTER_MAX; i++) {
 715         const VALUE name = rb_sym_intern_ascii_cstr(rb_debug_counter_names[i]);
 716         const VALUE boxed_value = SIZET2NUM(rb_debug_counter[i]);
 717
 718         if (key == name) {
 719             return boxed_value;
 720         }
 721         else if (hash != Qnil) {
 722             rb_hash_aset(hash, name, boxed_value);
 723         }
 724     }
 725 #endif
 726
 727     if (!NIL_P(key)) { /* matched key should return above */
 728         rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
 729     }
 730
 731     return hash;
 732 }
 733
 734 /* control stack frame */
 735
 736 static void
 737 vm_set_top_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq)
 738 {
 739     if (ISEQ_BODY(iseq)->type != ISEQ_TYPE_TOP) {
 740         rb_raise(rb_eTypeError, "Not a toplevel InstructionSequence");
 741     }
 742
 743     /* for return */
 744     vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, rb_ec_thread_ptr(ec)->top_self,
 745                   VM_BLOCK_HANDLER_NONE,
 746                   (VALUE)vm_cref_new_toplevel(ec), /* cref or me */
 747                   ISEQ_BODY(iseq)->iseq_encoded, ec->cfp->sp,
 748                   ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max);
 749 }
 750
 751 static void
 752 vm_set_eval_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq, const rb_cref_t *cref, const struct rb_block *base_block)
 753 {
 754     vm_push_frame(ec, iseq, VM_FRAME_MAGIC_EVAL | VM_FRAME_FLAG_FINISH,
 755                   vm_block_self(base_block), VM_GUARDED_PREV_EP(vm_block_ep(base_block)),
 756                   (VALUE)cref, /* cref or me */
 757                   ISEQ_BODY(iseq)->iseq_encoded,
 758                   ec->cfp->sp, ISEQ_BODY(iseq)->local_table_size,
 759                   ISEQ_BODY(iseq)->stack_max);
 760 }
 761
 762 static void
 763 vm_set_main_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq)
 764 {
 765     VALUE toplevel_binding = rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING"));
 766     rb_binding_t *bind;
 767
 768     GetBindingPtr(toplevel_binding, bind);
 769     RUBY_ASSERT_MESG(bind, "TOPLEVEL_BINDING is not built");
 770
 771     vm_set_eval_stack(ec, iseq, 0, &bind->block);
 772
 773     /* save binding */
 774     if (ISEQ_BODY(iseq)->local_table_size > 0) {
 775         vm_bind_update_env(toplevel_binding, bind, vm_make_env_object(ec, ec->cfp));
 776     }
 777 }
 778
 779 rb_control_frame_t *
 780 rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 781 {
 782     while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
 783         if (cfp->iseq) {
 784             return (rb_control_frame_t *)cfp;
 785         }
 786         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 787     }
 788     return 0;
 789 }
 790
 791 rb_control_frame_t *
 792 rb_vm_get_ruby_level_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 793 {
 794     while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
 795         if (VM_FRAME_RUBYFRAME_P(cfp)) {
 796             return (rb_control_frame_t *)cfp;
 797         }
 798         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 799     }
 800     return 0;
 801 }
 802
 803 static rb_control_frame_t *
 804 vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
 805 {
 806     if (VM_FRAME_RUBYFRAME_P(cfp)) {
 807         return (rb_control_frame_t *)cfp;
 808     }
 809
 810     cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 811
 812     while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
 813         if (VM_FRAME_RUBYFRAME_P(cfp)) {
 814             return (rb_control_frame_t *)cfp;
 815         }
 816
 817         if (VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_PASSED) == FALSE) {
 818             break;
 819         }
 820         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 821     }
 822     return 0;
 823 }
 824
 825 void
 826 rb_vm_pop_cfunc_frame(void)
 827 {
 828     rb_execution_context_t *ec = GET_EC();
 829     rb_control_frame_t *cfp = ec->cfp;
 830     const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
 831
 832     EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, Qnil);
 833     RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
 834     vm_pop_frame(ec, cfp, cfp->ep);
 835 }
 836
 837 void
 838 rb_vm_rewind_cfp(rb_execution_context_t *ec, rb_control_frame_t *cfp)
 839 {
 840     /* check skipped frame */
 841     while (ec->cfp != cfp) {
 842 #if VMDEBUG
 843         printf("skipped frame: %s\n", vm_frametype_name(ec->cfp));
 844 #endif
 845         if (VM_FRAME_TYPE(ec->cfp) != VM_FRAME_MAGIC_CFUNC) {
 846             rb_vm_pop_frame(ec);
 847         }
 848         else { /* unlikely path */
 849             rb_vm_pop_cfunc_frame();
 850         }
 851     }
 852 }
 853
 854 /* at exit */
 855
 856 void
 857 ruby_vm_at_exit(void (*func)(rb_vm_t *))
 858 {
 859     rb_vm_t *vm = GET_VM();
 860     rb_at_exit_list *nl = ALLOC(rb_at_exit_list);
 861     nl->func = func;
 862     nl->next = vm->at_exit;
 863     vm->at_exit = nl;
 864 }
 865
 866 static void
 867 ruby_vm_run_at_exit_hooks(rb_vm_t *vm)
 868 {
 869     rb_at_exit_list *l = vm->at_exit;
 870
 871     while (l) {
 872         rb_at_exit_list* t = l->next;
 873         rb_vm_at_exit_func *func = l->func;
 874         ruby_xfree(l);
 875         l = t;
 876         (*func)(vm);
 877     }
 878 }
 879
 880 /* Env */
 881
 882 static VALUE check_env_value(const rb_env_t *env);
 883
 884 static int
 885 check_env(const rb_env_t *env)
 886 {
 887     fputs("---\n", stderr);
 888     ruby_debug_printf("envptr: %p\n", (void *)&env->ep[0]);
 889     ruby_debug_printf("envval: %10p ", (void *)env->ep[1]);
 890     dp(env->ep[1]);
 891     ruby_debug_printf("ep:    %10p\n", (void *)env->ep);
 892     if (rb_vm_env_prev_env(env)) {
 893         fputs(">>\n", stderr);
 894         check_env_value(rb_vm_env_prev_env(env));
 895         fputs("<<\n", stderr);
 896     }
 897     return 1;
 898 }
 899
 900 static VALUE
 901 check_env_value(const rb_env_t *env)
 902 {
 903     if (check_env(env)) {
 904         return (VALUE)env;
 905     }
 906     rb_bug("invalid env");
 907     return Qnil;                /* unreachable */
 908 }
 909
 910 static VALUE
 911 vm_block_handler_escape(const rb_execution_context_t *ec, VALUE block_handler)
 912 {
 913     switch (vm_block_handler_type(block_handler)) {
 914       case block_handler_type_ifunc:
 915       case block_handler_type_iseq:
 916         return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
 917
 918       case block_handler_type_symbol:
 919       case block_handler_type_proc:
 920         return block_handler;
 921     }
 922     VM_UNREACHABLE(vm_block_handler_escape);
 923     return Qnil;
 924 }
 925
 926 static VALUE
 927 vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *const cfp)
 928 {
 929     const VALUE * const ep = cfp->ep;
 930     VALUE *env_body, *env_ep;
 931     int local_size, env_size;
 932
 933     if (VM_ENV_ESCAPED_P(ep)) {
 934         return VM_ENV_ENVVAL(ep);
 935     }
 936
 937     if (!VM_ENV_LOCAL_P(ep)) {
 938         const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
 939         if (!VM_ENV_ESCAPED_P(prev_ep)) {
 940             rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
 941
 942             while (prev_cfp->ep != prev_ep) {
 943                 prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(prev_cfp);
 944                 VM_ASSERT(prev_cfp->ep != NULL);
 945             }
 946
 947             vm_make_env_each(ec, prev_cfp);
 948             VM_FORCE_WRITE_SPECIAL_CONST(&ep[VM_ENV_DATA_INDEX_SPECVAL], VM_GUARDED_PREV_EP(prev_cfp->ep));
 949         }
 950     }
 951     else {
 952         VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep);
 953
 954         if (block_handler != VM_BLOCK_HANDLER_NONE) {
 955             VALUE blockprocval = vm_block_handler_escape(ec, block_handler);
 956             VM_STACK_ENV_WRITE(ep, VM_ENV_DATA_INDEX_SPECVAL, blockprocval);
 957         }
 958     }
 959
 960     if (!VM_FRAME_RUBYFRAME_P(cfp)) {
 961         local_size = VM_ENV_DATA_SIZE;
 962     }
 963     else {
 964         local_size = ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE;
 965     }
 966
 967     /*
 968      * # local variables on a stack frame (N == local_size)
 969      * [lvar1, lvar2, ..., lvarN, SPECVAL]
 970      *                            ^
 971      *                            ep[0]
 972      *
 973      * # moved local variables
 974      * [lvar1, lvar2, ..., lvarN, SPECVAL, Envval, BlockProcval (if needed)]
 975      *  ^                         ^
 976      *  env->env[0]               ep[0]
 977      */
 978
 979     env_size = local_size +
 980                1 /* envval */;
 981
 982     // Careful with order in the following sequence. Each allocation can move objects.
 983     env_body = ALLOC_N(VALUE, env_size);
 984     rb_env_t *env = IMEMO_NEW(rb_env_t, imemo_env, 0);
 985
 986     // Set up env without WB since it's brand new (similar to newobj_init(), newobj_fill())
 987     MEMCPY(env_body, ep - (local_size - 1 /* specval */), VALUE, local_size);
 988
 989     env_ep = &env_body[local_size - 1 /* specval */];
 990     env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env;
 991
 992     env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL);
 993     env->ep = env_ep;
 994     env->env = env_body;
 995     env->env_size = env_size;
 996
 997     cfp->ep = env_ep;
 998     VM_ENV_FLAGS_SET(env_ep, VM_ENV_FLAG_ESCAPED | VM_ENV_FLAG_WB_REQUIRED);
 999     VM_STACK_ENV_WRITE(ep, 0, (VALUE)env);              /* GC mark */
1000
1001 #if 0
1002     for (i = 0; i < local_size; i++) {
1003         if (VM_FRAME_RUBYFRAME_P(cfp)) {
1004             /* clear value stack for GC */
1005             ep[-local_size + i] = 0;
1006         }
1007     }
1008 #endif
1009
1010     // Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp).
1011     if (env->iseq) {
1012         rb_yjit_invalidate_ep_is_bp(env->iseq);
1013     }
1014
1015     return (VALUE)env;
1016 }
1017
1018 static VALUE
1019 vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
1020 {
1021     VALUE envval = vm_make_env_each(ec, cfp);
1022
1023     if (PROCDEBUG) {
1024         check_env_value((const rb_env_t *)envval);
1025     }
1026
1027     return envval;
1028 }
1029
1030 void
1031 rb_vm_stack_to_heap(rb_execution_context_t *ec)
1032 {
1033     rb_control_frame_t *cfp = ec->cfp;
1034     while ((cfp = rb_vm_get_binding_creatable_next_cfp(ec, cfp)) != 0) {
1035         vm_make_env_object(ec, cfp);
1036         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1037     }
1038 }
1039
1040 const rb_env_t *
1041 rb_vm_env_prev_env(const rb_env_t *env)
1042 {
1043     const VALUE *ep = env->ep;
1044
1045     if (VM_ENV_LOCAL_P(ep)) {
1046         return NULL;
1047     }
1048     else {
1049         const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
1050         return VM_ENV_ENVVAL_PTR(prev_ep);
1051     }
1052 }
1053
1054 static int
1055 collect_local_variables_in_iseq(const rb_iseq_t *iseq, const struct local_var_list *vars)
1056 {
1057     unsigned int i;
1058     if (!iseq) return 0;
1059     for (i = 0; i < ISEQ_BODY(iseq)->local_table_size; i++) {
1060         local_var_list_add(vars, ISEQ_BODY(iseq)->local_table[i]);
1061     }
1062     return 1;
1063 }
1064
1065 static void
1066 collect_local_variables_in_env(const rb_env_t *env, const struct local_var_list *vars)
1067 {
1068     do {
1069         if (VM_ENV_FLAGS(env->ep, VM_ENV_FLAG_ISOLATED)) break;
1070         collect_local_variables_in_iseq(env->iseq, vars);
1071     } while ((env = rb_vm_env_prev_env(env)) != NULL);
1072 }
1073
1074 static int
1075 vm_collect_local_variables_in_heap(const VALUE *ep, const struct local_var_list *vars)
1076 {
1077     if (VM_ENV_ESCAPED_P(ep)) {
1078         collect_local_variables_in_env(VM_ENV_ENVVAL_PTR(ep), vars);
1079         return 1;
1080     }
1081     else {
1082         return 0;
1083     }
1084 }
1085
1086 VALUE
1087 rb_vm_env_local_variables(const rb_env_t *env)
1088 {
1089     struct local_var_list vars;
1090     local_var_list_init(&vars);
1091     collect_local_variables_in_env(env, &vars);
1092     return local_var_list_finish(&vars);
1093 }
1094
1095 VALUE
1096 rb_iseq_local_variables(const rb_iseq_t *iseq)
1097 {
1098     struct local_var_list vars;
1099     local_var_list_init(&vars);
1100     while (collect_local_variables_in_iseq(iseq, &vars)) {
1101         iseq = ISEQ_BODY(iseq)->parent_iseq;
1102     }
1103     return local_var_list_finish(&vars);
1104 }
1105
1106 /* Proc */
1107
1108 static VALUE
1109 vm_proc_create_from_captured(VALUE klass,
1110                              const struct rb_captured_block *captured,
1111                              enum rb_block_type block_type,
1112                              int8_t is_from_method, int8_t is_lambda)
1113 {
1114     VALUE procval = rb_proc_alloc(klass);
1115     rb_proc_t *proc = RTYPEDDATA_DATA(procval);
1116
1117     VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), captured->ep));
1118
1119     /* copy block */
1120     RB_OBJ_WRITE(procval, &proc->block.as.captured.code.val, captured->code.val);
1121     RB_OBJ_WRITE(procval, &proc->block.as.captured.self, captured->self);
1122     rb_vm_block_ep_update(procval, &proc->block, captured->ep);
1123
1124     vm_block_type_set(&proc->block, block_type);
1125     proc->is_from_method = is_from_method;
1126     proc->is_lambda = is_lambda;
1127
1128     return procval;
1129 }
1130
1131 void
1132 rb_vm_block_copy(VALUE obj, const struct rb_block *dst, const struct rb_block *src)
1133 {
1134     /* copy block */
1135     switch (vm_block_type(src)) {
1136       case block_type_iseq:
1137       case block_type_ifunc:
1138         RB_OBJ_WRITE(obj, &dst->as.captured.self, src->as.captured.self);
1139         RB_OBJ_WRITE(obj, &dst->as.captured.code.val, src->as.captured.code.val);
1140         rb_vm_block_ep_update(obj, dst, src->as.captured.ep);
1141         break;
1142       case block_type_symbol:
1143         RB_OBJ_WRITE(obj, &dst->as.symbol, src->as.symbol);
1144         break;
1145       case block_type_proc:
1146         RB_OBJ_WRITE(obj, &dst->as.proc, src->as.proc);
1147         break;
1148     }
1149 }
1150
1151 static VALUE
1152 proc_create(VALUE klass, const struct rb_block *block, int8_t is_from_method, int8_t is_lambda)
1153 {
1154     VALUE procval = rb_proc_alloc(klass);
1155     rb_proc_t *proc = RTYPEDDATA_DATA(procval);
1156
1157     VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), vm_block_ep(block)));
1158     rb_vm_block_copy(procval, &proc->block, block);
1159     vm_block_type_set(&proc->block, block->type);
1160     proc->is_from_method = is_from_method;
1161     proc->is_lambda = is_lambda;
1162
1163     return procval;
1164 }
1165
1166 VALUE
1167 rb_proc_dup(VALUE self)
1168 {
1169     VALUE procval;
1170     rb_proc_t *src;
1171
1172     GetProcPtr(self, src);
1173     procval = proc_create(rb_obj_class(self), &src->block, src->is_from_method, src->is_lambda);
1174     if (RB_OBJ_SHAREABLE_P(self)) FL_SET_RAW(procval, RUBY_FL_SHAREABLE);
1175     RB_GC_GUARD(self); /* for: body = rb_proc_dup(body) */
1176     return procval;
1177 }
1178
1179 struct collect_outer_variable_name_data {
1180     VALUE ary;
1181     VALUE read_only;
1182     bool yield;
1183     bool isolate;
1184 };
1185
1186 static VALUE
1187 ID2NUM(ID id)
1188 {
1189     if (SIZEOF_VOIDP > SIZEOF_LONG)
1190         return ULL2NUM(id);
1191     else
1192         return ULONG2NUM(id);
1193 }
1194
1195 static ID
1196 NUM2ID(VALUE num)
1197 {
1198     if (SIZEOF_VOIDP > SIZEOF_LONG)
1199         return (ID)NUM2ULL(num);
1200     else
1201         return (ID)NUM2ULONG(num);
1202 }
1203
1204 static enum rb_id_table_iterator_result
1205 collect_outer_variable_names(ID id, VALUE val, void *ptr)
1206 {
1207     struct collect_outer_variable_name_data *data = (struct collect_outer_variable_name_data *)ptr;
1208
1209     if (id == rb_intern("yield")) {
1210         data->yield = true;
1211     }
1212     else {
1213         VALUE *store;
1214         if (data->isolate ||
1215             val == Qtrue /* write */) {
1216             store = &data->ary;
1217         }
1218         else {
1219             store = &data->read_only;
1220         }
1221         if (*store == Qfalse) *store = rb_ary_new();
1222         rb_ary_push(*store, ID2NUM(id));
1223     }
1224     return ID_TABLE_CONTINUE;
1225 }
1226
1227 static const rb_env_t *
1228 env_copy(const VALUE *src_ep, VALUE read_only_variables)
1229 {
1230     const rb_env_t *src_env = (rb_env_t *)VM_ENV_ENVVAL(src_ep);
1231     VM_ASSERT(src_env->ep == src_ep);
1232
1233     VALUE *env_body = ZALLOC_N(VALUE, src_env->env_size); // fill with Qfalse
1234     VALUE *ep = &env_body[src_env->env_size - 2];
1235     const rb_env_t *copied_env = vm_env_new(ep, env_body, src_env->env_size, src_env->iseq);
1236
1237     // Copy after allocations above, since they can move objects in src_ep.
1238     RB_OBJ_WRITE(copied_env, &ep[VM_ENV_DATA_INDEX_ME_CREF], src_ep[VM_ENV_DATA_INDEX_ME_CREF]);
1239     ep[VM_ENV_DATA_INDEX_FLAGS] = src_ep[VM_ENV_DATA_INDEX_FLAGS] | VM_ENV_FLAG_ISOLATED;
1240     if (!VM_ENV_LOCAL_P(src_ep)) {
1241         VM_ENV_FLAGS_SET(ep, VM_ENV_FLAG_LOCAL);
1242     }
1243
1244     if (read_only_variables) {
1245         for (int i=RARRAY_LENINT(read_only_variables)-1; i>=0; i--) {
1246             ID id = NUM2ID(RARRAY_AREF(read_only_variables, i));
1247
1248             for (unsigned int j=0; j<ISEQ_BODY(src_env->iseq)->local_table_size; j++) {
1249                 if (id == ISEQ_BODY(src_env->iseq)->local_table[j]) {
1250                     VALUE v = src_env->env[j];
1251                     if (!rb_ractor_shareable_p(v)) {
1252                         VALUE name = rb_id2str(id);
1253                         VALUE msg = rb_sprintf("can not make shareable Proc because it can refer"
1254                                                " unshareable object %+" PRIsVALUE " from ", v);
1255                         if (name)
1256                             rb_str_catf(msg, "variable '%" PRIsVALUE "'", name);
1257                         else
1258                             rb_str_cat_cstr(msg, "a hidden variable");
1259                         rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg));
1260                     }
1261                     RB_OBJ_WRITE((VALUE)copied_env, &env_body[j], v);
1262                     rb_ary_delete_at(read_only_variables, i);
1263                     break;
1264                 }
1265             }
1266         }
1267     }
1268
1269     if (!VM_ENV_LOCAL_P(src_ep)) {
1270         const VALUE *prev_ep = VM_ENV_PREV_EP(src_env->ep);
1271         const rb_env_t *new_prev_env = env_copy(prev_ep, read_only_variables);
1272         ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_GUARDED_PREV_EP(new_prev_env->ep);
1273         RB_OBJ_WRITTEN(copied_env, Qundef, new_prev_env);
1274         VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_LOCAL);
1275     }
1276     else {
1277         ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_BLOCK_HANDLER_NONE;
1278     }
1279
1280     return copied_env;
1281 }
1282
1283 static void
1284 proc_isolate_env(VALUE self, rb_proc_t *proc, VALUE read_only_variables)
1285 {
1286     const struct rb_captured_block *captured = &proc->block.as.captured;
1287     const rb_env_t *env = env_copy(captured->ep, read_only_variables);
1288     *((const VALUE **)&proc->block.as.captured.ep) = env->ep;
1289     RB_OBJ_WRITTEN(self, Qundef, env);
1290 }
1291
1292 static VALUE
1293 proc_shared_outer_variables(struct rb_id_table *outer_variables, bool isolate, const char *message)
1294 {
1295     struct collect_outer_variable_name_data data = {
1296         .isolate = isolate,
1297         .ary = Qfalse,
1298         .read_only = Qfalse,
1299         .yield = false,
1300     };
1301     rb_id_table_foreach(outer_variables, collect_outer_variable_names, (void *)&data);
1302
1303     if (data.ary != Qfalse) {
1304         VALUE str = rb_sprintf("can not %s because it accesses outer variables", message);
1305         VALUE ary = data.ary;
1306         const char *sep = " (";
1307         for (long i = 0; i < RARRAY_LEN(ary); i++) {
1308             VALUE name = rb_id2str(NUM2ID(RARRAY_AREF(ary, i)));
1309             if (!name) continue;
1310             rb_str_cat_cstr(str, sep);
1311             sep = ", ";
1312             rb_str_append(str, name);
1313         }
1314         if (*sep == ',') rb_str_cat_cstr(str, ")");
1315         rb_str_cat_cstr(str, data.yield ? " and uses 'yield'." : ".");
1316         rb_exc_raise(rb_exc_new_str(rb_eArgError, str));
1317     }
1318     else if (data.yield) {
1319         rb_raise(rb_eArgError, "can not %s because it uses 'yield'.", message);
1320     }
1321
1322     return data.read_only;
1323 }
1324
1325 VALUE
1326 rb_proc_isolate_bang(VALUE self)
1327 {
1328     const rb_iseq_t *iseq = vm_proc_iseq(self);
1329
1330     if (iseq) {
1331         rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self);
1332         if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet");
1333
1334         if (ISEQ_BODY(iseq)->outer_variables) {
1335             proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, true, "isolate a Proc");
1336         }
1337
1338         proc_isolate_env(self, proc, Qfalse);
1339         proc->is_isolated = TRUE;
1340     }
1341
1342     FL_SET_RAW(self, RUBY_FL_SHAREABLE);
1343     return self;
1344 }
1345
1346 VALUE
1347 rb_proc_isolate(VALUE self)
1348 {
1349     VALUE dst = rb_proc_dup(self);
1350     rb_proc_isolate_bang(dst);
1351     return dst;
1352 }
1353
1354 VALUE
1355 rb_proc_ractor_make_shareable(VALUE self)
1356 {
1357     const rb_iseq_t *iseq = vm_proc_iseq(self);
1358
1359     if (iseq) {
1360         rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self);
1361         if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet");
1362
1363         if (!rb_ractor_shareable_p(vm_block_self(&proc->block))) {
1364             rb_raise(rb_eRactorIsolationError,
1365                      "Proc's self is not shareable: %" PRIsVALUE,
1366                      self);
1367         }
1368
1369         VALUE read_only_variables = Qfalse;
1370
1371         if (ISEQ_BODY(iseq)->outer_variables) {
1372             read_only_variables =
1373                 proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, false, "make a Proc shareable");
1374         }
1375
1376         proc_isolate_env(self, proc, read_only_variables);
1377         proc->is_isolated = TRUE;
1378     }
1379
1380     FL_SET_RAW(self, RUBY_FL_SHAREABLE);
1381     return self;
1382 }
1383
1384 VALUE
1385 rb_vm_make_proc_lambda(const rb_execution_context_t *ec, const struct rb_captured_block *captured, VALUE klass, int8_t is_lambda)
1386 {
1387     VALUE procval;
1388     enum imemo_type code_type = imemo_type(captured->code.val);
1389
1390     if (!VM_ENV_ESCAPED_P(captured->ep)) {
1391         rb_control_frame_t *cfp = VM_CAPTURED_BLOCK_TO_CFP(captured);
1392         vm_make_env_object(ec, cfp);
1393     }
1394
1395     VM_ASSERT(VM_EP_IN_HEAP_P(ec, captured->ep));
1396     VM_ASSERT(code_type == imemo_iseq || code_type == imemo_ifunc);
1397
1398     procval = vm_proc_create_from_captured(klass, captured,
1399                                            code_type == imemo_iseq ? block_type_iseq : block_type_ifunc,
1400                                            FALSE, is_lambda);
1401
1402     if (code_type == imemo_ifunc) {
1403         struct vm_ifunc *ifunc = (struct vm_ifunc *)captured->code.val;
1404         if (ifunc->svar_lep) {
1405             VALUE ep0 = ifunc->svar_lep[0];
1406             if (RB_TYPE_P(ep0, T_IMEMO) && imemo_type_p(ep0, imemo_env)) {
1407                 // `ep0 == imemo_env` means this ep is escaped to heap (in env object).
1408                 const rb_env_t *env = (const rb_env_t *)ep0;
1409                 ifunc->svar_lep = (VALUE *)env->ep;
1410             }
1411             else {
1412                 VM_ASSERT(FIXNUM_P(ep0));
1413                 if (ep0 & VM_ENV_FLAG_ESCAPED) {
1414                     // ok. do nothing
1415                 }
1416                 else {
1417                     ifunc->svar_lep = NULL;
1418                 }
1419             }
1420         }
1421     }
1422
1423     return procval;
1424 }
1425
1426 /* Binding */
1427
1428 VALUE
1429 rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *src_cfp)
1430 {
1431     rb_control_frame_t *cfp = rb_vm_get_binding_creatable_next_cfp(ec, src_cfp);
1432     rb_control_frame_t *ruby_level_cfp = rb_vm_get_ruby_level_next_cfp(ec, src_cfp);
1433     VALUE bindval, envval;
1434     rb_binding_t *bind;
1435
1436     if (cfp == 0 || ruby_level_cfp == 0) {
1437         rb_raise(rb_eRuntimeError, "Can't create Binding Object on top of Fiber.");
1438     }
1439     if (!VM_FRAME_RUBYFRAME_P(src_cfp) &&
1440         !VM_FRAME_RUBYFRAME_P(RUBY_VM_PREVIOUS_CONTROL_FRAME(src_cfp))) {
1441         rb_raise(rb_eRuntimeError, "Cannot create Binding object for non-Ruby caller");
1442     }
1443
1444     envval = vm_make_env_object(ec, cfp);
1445     bindval = rb_binding_alloc(rb_cBinding);
1446     GetBindingPtr(bindval, bind);
1447     vm_bind_update_env(bindval, bind, envval);
1448     RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self);
1449     RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, cfp->iseq);
1450     RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(ruby_level_cfp->iseq)->location.pathobj);
1451     bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp);
1452
1453     return bindval;
1454 }
1455
1456 const VALUE *
1457 rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const ID *dynvars)
1458 {
1459     VALUE envval, pathobj = bind->pathobj;
1460     VALUE path = pathobj_path(pathobj);
1461     VALUE realpath = pathobj_realpath(pathobj);
1462     const struct rb_block *base_block;
1463     const rb_env_t *env;
1464     rb_execution_context_t *ec = GET_EC();
1465     const rb_iseq_t *base_iseq, *iseq;
1466     rb_node_scope_t tmp_node;
1467
1468     if (dyncount < 0) return 0;
1469
1470     base_block = &bind->block;
1471     base_iseq = vm_block_iseq(base_block);
1472
1473     VALUE idtmp = 0;
1474     rb_ast_id_table_t *dyns = ALLOCV(idtmp, sizeof(rb_ast_id_table_t) + dyncount * sizeof(ID));
1475     dyns->size = dyncount;
1476     MEMCPY(dyns->ids, dynvars, ID, dyncount);
1477
1478     rb_node_init(RNODE(&tmp_node), NODE_SCOPE);
1479     tmp_node.nd_tbl = dyns;
1480     tmp_node.nd_body = 0;
1481     tmp_node.nd_args = 0;
1482
1483     VALUE ast_value = rb_ruby_ast_new(RNODE(&tmp_node));
1484
1485     if (base_iseq) {
1486         iseq = rb_iseq_new(ast_value, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL);
1487     }
1488     else {
1489         VALUE tempstr = rb_fstring_lit("<temp>");
1490         iseq = rb_iseq_new_top(ast_value, tempstr, tempstr, tempstr, NULL);
1491     }
1492     tmp_node.nd_tbl = 0; /* reset table */
1493     ALLOCV_END(idtmp);
1494
1495     vm_set_eval_stack(ec, iseq, 0, base_block);
1496     vm_bind_update_env(bindval, bind, envval = vm_make_env_object(ec, ec->cfp));
1497     rb_vm_pop_frame(ec);
1498
1499     env = (const rb_env_t *)envval;
1500     return env->env;
1501 }
1502
1503 /* C -> Ruby: block */
1504
1505 static inline VALUE
1506 invoke_block(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_cref_t *cref, VALUE type, int opt_pc)
1507 {
1508     int arg_size = ISEQ_BODY(iseq)->param.size;
1509
1510     vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_FINISH, self,
1511                   VM_GUARDED_PREV_EP(captured->ep),
1512                   (VALUE)cref, /* cref or method */
1513                   ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
1514                   ec->cfp->sp + arg_size,
1515                   ISEQ_BODY(iseq)->local_table_size - arg_size,
1516                   ISEQ_BODY(iseq)->stack_max);
1517     return vm_exec(ec);
1518 }
1519
1520 static VALUE
1521 invoke_bmethod(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_callable_method_entry_t *me, VALUE type, int opt_pc)
1522 {
1523     /* bmethod call from outside the VM */
1524     int arg_size = ISEQ_BODY(iseq)->param.size;
1525     VALUE ret;
1526
1527     VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
1528
1529     vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_BMETHOD, self,
1530                   VM_GUARDED_PREV_EP(captured->ep),
1531                   (VALUE)me,
1532                   ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
1533                   ec->cfp->sp + 1 /* self */ + arg_size,
1534                   ISEQ_BODY(iseq)->local_table_size - arg_size,
1535                   ISEQ_BODY(iseq)->stack_max);
1536
1537     VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);
1538     ret = vm_exec(ec);
1539
1540     return ret;
1541 }
1542
1543 ALWAYS_INLINE(static VALUE
1544               invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured,
1545                                        VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
1546                                        const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me));
1547
1548 static inline VALUE
1549 invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured,
1550                          VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
1551                          const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me)
1552 {
1553     const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
1554     int opt_pc;
1555     VALUE type = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
1556     rb_control_frame_t *cfp = ec->cfp;
1557     VALUE *sp = cfp->sp;
1558     int flags = (kw_splat ? VM_CALL_KW_SPLAT : 0);
1559     VALUE *use_argv = (VALUE *)argv;
1560     VALUE av[2];
1561
1562     stack_check(ec);
1563
1564     if (UNLIKELY(argc > VM_ARGC_STACK_MAX) &&
1565         (VM_ARGC_STACK_MAX >= 1 ||
1566          /* Skip ruby array for potential autosplat case */
1567          (argc != 1 || is_lambda))) {
1568         use_argv = vm_argv_ruby_array(av, argv, &flags, &argc, kw_splat);
1569     }
1570
1571     CHECK_VM_STACK_OVERFLOW(cfp, argc + 1);
1572     vm_check_canary(ec, sp);
1573
1574     VALUE *stack_argv = sp;
1575     if (me) {
1576         *sp = self; // bemthods need `self` on the VM stack
1577         stack_argv++;
1578     }
1579     cfp->sp = stack_argv + argc;
1580     MEMCPY(stack_argv, use_argv, VALUE, argc); // restrict: new stack space
1581
1582     opt_pc = vm_yield_setup_args(ec, iseq, argc, stack_argv, flags, passed_block_handler,
1583                                  (is_lambda ? arg_setup_method : arg_setup_block));
1584     cfp->sp = sp;
1585
1586     if (me == NULL) {
1587         return invoke_block(ec, iseq, self, captured, cref, type, opt_pc);
1588     }
1589     else {
1590         return invoke_bmethod(ec, iseq, self, captured, me, type, opt_pc);
1591     }
1592 }
1593
1594 static inline VALUE
1595 invoke_block_from_c_bh(rb_execution_context_t *ec, VALUE block_handler,
1596                        int argc, const VALUE *argv,
1597                        int kw_splat, VALUE passed_block_handler, const rb_cref_t *cref,
1598                        int is_lambda, int force_blockarg)
1599 {
1600   again:
1601     switch (vm_block_handler_type(block_handler)) {
1602       case block_handler_type_iseq:
1603         {
1604             const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
1605             return invoke_iseq_block_from_c(ec, captured, captured->self,
1606                                             argc, argv, kw_splat, passed_block_handler,
1607                                             cref, is_lambda, NULL);
1608         }
1609       case block_handler_type_ifunc:
1610         return vm_yield_with_cfunc(ec, VM_BH_TO_IFUNC_BLOCK(block_handler),
1611                                    VM_BH_TO_IFUNC_BLOCK(block_handler)->self,
1612                                    argc, argv, kw_splat, passed_block_handler, NULL);
1613       case block_handler_type_symbol:
1614         return vm_yield_with_symbol(ec, VM_BH_TO_SYMBOL(block_handler),
1615                                     argc, argv, kw_splat, passed_block_handler);
1616       case block_handler_type_proc:
1617         if (force_blockarg == FALSE) {
1618             is_lambda = block_proc_is_lambda(VM_BH_TO_PROC(block_handler));
1619         }
1620         block_handler = vm_proc_to_block_handler(VM_BH_TO_PROC(block_handler));
1621         goto again;
1622     }
1623     VM_UNREACHABLE(invoke_block_from_c_splattable);
1624     return Qundef;
1625 }
1626
1627 static inline VALUE
1628 check_block_handler(rb_execution_context_t *ec)
1629 {
1630     VALUE block_handler = VM_CF_BLOCK_HANDLER(ec->cfp);
1631     vm_block_handler_verify(block_handler);
1632     if (UNLIKELY(block_handler == VM_BLOCK_HANDLER_NONE)) {
1633         rb_vm_localjump_error("no block given", Qnil, 0);
1634     }
1635
1636     return block_handler;
1637 }
1638
1639 static VALUE
1640 vm_yield_with_cref(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat, const rb_cref_t *cref, int is_lambda)
1641 {
1642     return invoke_block_from_c_bh(ec, check_block_handler(ec),
1643                                   argc, argv, kw_splat, VM_BLOCK_HANDLER_NONE,
1644                                   cref, is_lambda, FALSE);
1645 }
1646
1647 static VALUE
1648 vm_yield(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat)
1649 {
1650     return vm_yield_with_cref(ec, argc, argv, kw_splat, NULL, FALSE);
1651 }
1652
1653 static VALUE
1654 vm_yield_with_block(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE block_handler, int kw_splat)
1655 {
1656     return invoke_block_from_c_bh(ec, check_block_handler(ec),
1657                                   argc, argv, kw_splat, block_handler,
1658                                   NULL, FALSE, FALSE);
1659 }
1660
1661 static VALUE
1662 vm_yield_force_blockarg(rb_execution_context_t *ec, VALUE args)
1663 {
1664     return invoke_block_from_c_bh(ec, check_block_handler(ec), 1, &args,
1665                                   RB_NO_KEYWORDS, VM_BLOCK_HANDLER_NONE, NULL, FALSE, TRUE);
1666 }
1667
1668 ALWAYS_INLINE(static VALUE
1669               invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc,
1670                                        VALUE self, int argc, const VALUE *argv,
1671                                        int kw_splat, VALUE passed_block_handler, int is_lambda,
1672                                        const rb_callable_method_entry_t *me));
1673
1674 static inline VALUE
1675 invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc,
1676                          VALUE self, int argc, const VALUE *argv,
1677                          int kw_splat, VALUE passed_block_handler, int is_lambda,
1678                          const rb_callable_method_entry_t *me)
1679 {
1680     const struct rb_block *block = &proc->block;
1681
1682   again:
1683     switch (vm_block_type(block)) {
1684       case block_type_iseq:
1685         return invoke_iseq_block_from_c(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, NULL, is_lambda, me);
1686       case block_type_ifunc:
1687         if (kw_splat == 1) {
1688             VALUE keyword_hash = argv[argc-1];
1689             if (!RB_TYPE_P(keyword_hash, T_HASH)) {
1690                 keyword_hash = rb_to_hash_type(keyword_hash);
1691             }
1692             if (RHASH_EMPTY_P(keyword_hash)) {
1693                 argc--;
1694             }
1695             else {
1696                 ((VALUE *)argv)[argc-1] = rb_hash_dup(keyword_hash);
1697             }
1698         }
1699         return vm_yield_with_cfunc(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, me);
1700       case block_type_symbol:
1701         return vm_yield_with_symbol(ec, block->as.symbol, argc, argv, kw_splat, passed_block_handler);
1702       case block_type_proc:
1703         is_lambda = block_proc_is_lambda(block->as.proc);
1704         block = vm_proc_block(block->as.proc);
1705         goto again;
1706     }
1707     VM_UNREACHABLE(invoke_block_from_c_proc);
1708     return Qundef;
1709 }
1710
1711 static VALUE
1712 vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
1713                int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler)
1714 {
1715     return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler, proc->is_lambda, NULL);
1716 }
1717
1718 VALUE
1719 rb_vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
1720                      int argc, const VALUE *argv, int kw_splat, VALUE block_handler, const rb_callable_method_entry_t *me)
1721 {
1722     return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, block_handler, TRUE, me);
1723 }
1724
1725 VALUE
1726 rb_vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc,
1727                   int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler)
1728 {
1729     VALUE self = vm_block_self(&proc->block);
1730     vm_block_handler_verify(passed_block_handler);
1731
1732     if (proc->is_from_method) {
1733         return rb_vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL);
1734     }
1735     else {
1736         return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler);
1737     }
1738 }
1739
1740 VALUE
1741 rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
1742                             int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler)
1743 {
1744     vm_block_handler_verify(passed_block_handler);
1745
1746     if (proc->is_from_method) {
1747         return rb_vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL);
1748     }
1749     else {
1750         return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler);
1751     }
1752 }
1753
1754 /* special variable */
1755
1756 VALUE *
1757 rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
1758 {
1759     while (cfp->pc == 0 || cfp->iseq == 0) {
1760         if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) {
1761             struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq;
1762             return ifunc->svar_lep;
1763         }
1764         else {
1765             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1766         }
1767
1768         if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
1769             return NULL;
1770         }
1771     }
1772
1773     return (VALUE *)VM_CF_LEP(cfp);
1774 }
1775
1776 static VALUE
1777 vm_cfp_svar_get(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key)
1778 {
1779     return lep_svar_get(ec, rb_vm_svar_lep(ec, cfp), key);
1780 }
1781
1782 static void
1783 vm_cfp_svar_set(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key, const VALUE val)
1784 {
1785     lep_svar_set(ec, rb_vm_svar_lep(ec, cfp), key, val);
1786 }
1787
1788 static VALUE
1789 vm_svar_get(const rb_execution_context_t *ec, VALUE key)
1790 {
1791     return vm_cfp_svar_get(ec, ec->cfp, key);
1792 }
1793
1794 static void
1795 vm_svar_set(const rb_execution_context_t *ec, VALUE key, VALUE val)
1796 {
1797     vm_cfp_svar_set(ec, ec->cfp, key, val);
1798 }
1799
1800 VALUE
1801 rb_backref_get(void)
1802 {
1803     return vm_svar_get(GET_EC(), VM_SVAR_BACKREF);
1804 }
1805
1806 void
1807 rb_backref_set(VALUE val)
1808 {
1809     vm_svar_set(GET_EC(), VM_SVAR_BACKREF, val);
1810 }
1811
1812 VALUE
1813 rb_lastline_get(void)
1814 {
1815     return vm_svar_get(GET_EC(), VM_SVAR_LASTLINE);
1816 }
1817
1818 void
1819 rb_lastline_set(VALUE val)
1820 {
1821     vm_svar_set(GET_EC(), VM_SVAR_LASTLINE, val);
1822 }
1823
1824 void
1825 rb_lastline_set_up(VALUE val, unsigned int up)
1826 {
1827     rb_control_frame_t * cfp = GET_EC()->cfp;
1828
1829     for(unsigned int i = 0; i < up; i++) {
1830         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1831     }
1832     vm_cfp_svar_set(GET_EC(), cfp, VM_SVAR_LASTLINE, val);
1833 }
1834
1835 /* misc */
1836
1837 const char *
1838 rb_sourcefile(void)
1839 {
1840     const rb_execution_context_t *ec = GET_EC();
1841     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1842
1843     if (cfp) {
1844         return RSTRING_PTR(rb_iseq_path(cfp->iseq));
1845     }
1846     else {
1847         return 0;
1848     }
1849 }
1850
1851 int
1852 rb_sourceline(void)
1853 {
1854     const rb_execution_context_t *ec = GET_EC();
1855     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1856
1857     if (cfp) {
1858         return rb_vm_get_sourceline(cfp);
1859     }
1860     else {
1861         return 0;
1862     }
1863 }
1864
1865 VALUE
1866 rb_source_location(int *pline)
1867 {
1868     const rb_execution_context_t *ec = GET_EC();
1869     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1870
1871     if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) {
1872         if (pline) *pline = rb_vm_get_sourceline(cfp);
1873         return rb_iseq_path(cfp->iseq);
1874     }
1875     else {
1876         if (pline) *pline = 0;
1877         return Qnil;
1878     }
1879 }
1880
1881 const char *
1882 rb_source_location_cstr(int *pline)
1883 {
1884     VALUE path = rb_source_location(pline);
1885     if (NIL_P(path)) return NULL;
1886     return RSTRING_PTR(path);
1887 }
1888
1889 rb_cref_t *
1890 rb_vm_cref(void)
1891 {
1892     const rb_execution_context_t *ec = GET_EC();
1893     return vm_ec_cref(ec);
1894 }
1895
1896 rb_cref_t *
1897 rb_vm_cref_replace_with_duplicated_cref(void)
1898 {
1899     const rb_execution_context_t *ec = GET_EC();
1900     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1901     rb_cref_t *cref = vm_cref_replace_with_duplicated_cref(cfp->ep);
1902     ASSUME(cref);
1903     return cref;
1904 }
1905
1906 const rb_cref_t *
1907 rb_vm_cref_in_context(VALUE self, VALUE cbase)
1908 {
1909     const rb_execution_context_t *ec = GET_EC();
1910     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1911     const rb_cref_t *cref;
1912     if (!cfp || cfp->self != self) return NULL;
1913     if (!vm_env_cref_by_cref(cfp->ep)) return NULL;
1914     cref = vm_get_cref(cfp->ep);
1915     if (CREF_CLASS(cref) != cbase) return NULL;
1916     return cref;
1917 }
1918
1919 #if 0
1920 void
1921 debug_cref(rb_cref_t *cref)
1922 {
1923     while (cref) {
1924         dp(CREF_CLASS(cref));
1925         printf("%ld\n", CREF_VISI(cref));
1926         cref = CREF_NEXT(cref);
1927     }
1928 }
1929 #endif
1930
1931 VALUE
1932 rb_vm_cbase(void)
1933 {
1934     const rb_execution_context_t *ec = GET_EC();
1935     const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1936
1937     if (cfp == 0) {
1938         rb_raise(rb_eRuntimeError, "Can't call on top of Fiber or Thread");
1939     }
1940     return vm_get_cbase(cfp->ep);
1941 }
1942
1943 /* jump */
1944
1945 static VALUE
1946 make_localjump_error(const char *mesg, VALUE value, int reason)
1947 {
1948     extern VALUE rb_eLocalJumpError;
1949     VALUE exc = rb_exc_new2(rb_eLocalJumpError, mesg);
1950     ID id;
1951
1952     switch (reason) {
1953       case TAG_BREAK:
1954         CONST_ID(id, "break");
1955         break;
1956       case TAG_REDO:
1957         CONST_ID(id, "redo");
1958         break;
1959       case TAG_RETRY:
1960         CONST_ID(id, "retry");
1961         break;
1962       case TAG_NEXT:
1963         CONST_ID(id, "next");
1964         break;
1965       case TAG_RETURN:
1966         CONST_ID(id, "return");
1967         break;
1968       default:
1969         CONST_ID(id, "noreason");
1970         break;
1971     }
1972     rb_iv_set(exc, "@exit_value", value);
1973     rb_iv_set(exc, "@reason", ID2SYM(id));
1974     return exc;
1975 }
1976
1977 void
1978 rb_vm_localjump_error(const char *mesg, VALUE value, int reason)
1979 {
1980     VALUE exc = make_localjump_error(mesg, value, reason);
1981     rb_exc_raise(exc);
1982 }
1983
1984 VALUE
1985 rb_vm_make_jump_tag_but_local_jump(enum ruby_tag_type state, VALUE val)
1986 {
1987     const char *mesg;
1988
1989     switch (state) {
1990       case TAG_RETURN:
1991         mesg = "unexpected return";
1992         break;
1993       case TAG_BREAK:
1994         mesg = "unexpected break";
1995         break;
1996       case TAG_NEXT:
1997         mesg = "unexpected next";
1998         break;
1999       case TAG_REDO:
2000         mesg = "unexpected redo";
2001         val = Qnil;
2002         break;
2003       case TAG_RETRY:
2004         mesg = "retry outside of rescue clause";
2005         val = Qnil;
2006         break;
2007       default:
2008         return Qnil;
2009     }
2010     if (UNDEF_P(val)) {
2011         val = GET_EC()->tag->retval;
2012     }
2013     return make_localjump_error(mesg, val, state);
2014 }
2015
2016 void
2017 rb_vm_jump_tag_but_local_jump(enum ruby_tag_type state)
2018 {
2019     VALUE exc = rb_vm_make_jump_tag_but_local_jump(state, Qundef);
2020     if (!NIL_P(exc)) rb_exc_raise(exc);
2021     EC_JUMP_TAG(GET_EC(), state);
2022 }
2023
2024 static rb_control_frame_t *
2025 next_not_local_frame(rb_control_frame_t *cfp)
2026 {
2027     while (VM_ENV_LOCAL_P(cfp->ep)) {
2028         cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2029     }
2030     return cfp;
2031 }
2032
2033 NORETURN(static void vm_iter_break(rb_execution_context_t *ec, VALUE val));
2034
2035 static void
2036 vm_iter_break(rb_execution_context_t *ec, VALUE val)
2037 {
2038     rb_control_frame_t *cfp = next_not_local_frame(ec->cfp);
2039     const VALUE *ep = VM_CF_PREV_EP(cfp);
2040     const rb_control_frame_t *target_cfp = rb_vm_search_cf_from_ep(ec, cfp, ep);
2041
2042     if (!target_cfp) {
2043         rb_vm_localjump_error("unexpected break", val, TAG_BREAK);
2044     }
2045
2046     ec->errinfo = (VALUE)THROW_DATA_NEW(val, target_cfp, TAG_BREAK);
2047     EC_JUMP_TAG(ec, TAG_BREAK);
2048 }
2049
2050 void
2051 rb_iter_break(void)
2052 {
2053     vm_iter_break(GET_EC(), Qnil);
2054 }
2055
2056 void
2057 rb_iter_break_value(VALUE val)
2058 {
2059     vm_iter_break(GET_EC(), val);
2060 }
2061
2062 /* optimization: redefine management */
2063
2064 short ruby_vm_redefined_flag[BOP_LAST_];
2065 static st_table *vm_opt_method_def_table = 0;
2066 static st_table *vm_opt_mid_table = 0;
2067
2068 void
2069 rb_free_vm_opt_tables(void)
2070 {
2071     st_free_table(vm_opt_method_def_table);
2072     st_free_table(vm_opt_mid_table);
2073 }
2074
2075 static int
2076 vm_redefinition_check_flag(VALUE klass)
2077 {
2078     if (klass == rb_cInteger) return INTEGER_REDEFINED_OP_FLAG;
2079     if (klass == rb_cFloat)  return FLOAT_REDEFINED_OP_FLAG;
2080     if (klass == rb_cString) return STRING_REDEFINED_OP_FLAG;
2081     if (klass == rb_cArray)  return ARRAY_REDEFINED_OP_FLAG;
2082     if (klass == rb_cHash)   return HASH_REDEFINED_OP_FLAG;
2083     if (klass == rb_cSymbol) return SYMBOL_REDEFINED_OP_FLAG;
2084 #if 0
2085     if (klass == rb_cTime)   return TIME_REDEFINED_OP_FLAG;
2086 #endif
2087     if (klass == rb_cRegexp) return REGEXP_REDEFINED_OP_FLAG;
2088     if (klass == rb_cNilClass) return NIL_REDEFINED_OP_FLAG;
2089     if (klass == rb_cTrueClass) return TRUE_REDEFINED_OP_FLAG;
2090     if (klass == rb_cFalseClass) return FALSE_REDEFINED_OP_FLAG;
2091     if (klass == rb_cProc) return PROC_REDEFINED_OP_FLAG;
2092     return 0;
2093 }
2094
2095 int
2096 rb_vm_check_optimizable_mid(VALUE mid)
2097 {
2098     if (!vm_opt_mid_table) {
2099       return FALSE;
2100     }
2101
2102     return st_lookup(vm_opt_mid_table, mid, NULL);
2103 }
2104
2105 static int
2106 vm_redefinition_check_method_type(const rb_method_entry_t *me)
2107 {
2108     if (me->called_id != me->def->original_id) {
2109         return FALSE;
2110     }
2111
2112     if (METHOD_ENTRY_BASIC(me)) return TRUE;
2113
2114     const rb_method_definition_t *def = me->def;
2115     switch (def->type) {
2116       case VM_METHOD_TYPE_CFUNC:
2117       case VM_METHOD_TYPE_OPTIMIZED:
2118         return TRUE;
2119       default:
2120         return FALSE;
2121     }
2122 }
2123
2124 static void
2125 rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass)
2126 {
2127     st_data_t bop;
2128     if (RB_TYPE_P(klass, T_ICLASS) && FL_TEST(klass, RICLASS_IS_ORIGIN) &&
2129             RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
2130        klass = RBASIC_CLASS(klass);
2131     }
2132     if (vm_redefinition_check_method_type(me)) {
2133         if (st_lookup(vm_opt_method_def_table, (st_data_t)me->def, &bop)) {
2134             int flag = vm_redefinition_check_flag(klass);
2135             if (flag != 0) {
2136                 rb_category_warn(
2137                     RB_WARN_CATEGORY_PERFORMANCE,
2138                     "Redefining '%s#%s' disables interpreter and JIT optimizations",
2139                     rb_class2name(me->owner),
2140                     rb_id2name(me->called_id)
2141                 );
2142                 rb_yjit_bop_redefined(flag, (enum ruby_basic_operators)bop);
2143                 rb_rjit_bop_redefined(flag, (enum ruby_basic_operators)bop);
2144                 ruby_vm_redefined_flag[bop] |= flag;
2145             }
2146         }
2147     }
2148 }
2149
2150 static enum rb_id_table_iterator_result
2151 check_redefined_method(ID mid, VALUE value, void *data)
2152 {
2153     VALUE klass = (VALUE)data;
2154     const rb_method_entry_t *me = (rb_method_entry_t *)value;
2155     const rb_method_entry_t *newme = rb_method_entry(klass, mid);
2156
2157     if (newme != me) rb_vm_check_redefinition_opt_method(me, me->owner);
2158
2159     return ID_TABLE_CONTINUE;
2160 }
2161
2162 void
2163 rb_vm_check_redefinition_by_prepend(VALUE klass)
2164 {
2165     if (!vm_redefinition_check_flag(klass)) return;
2166     rb_id_table_foreach(RCLASS_M_TBL(RCLASS_ORIGIN(klass)), check_redefined_method, (void *)klass);
2167 }
2168
2169 static void
2170 add_opt_method_entry_bop(const rb_method_entry_t *me, ID mid, enum ruby_basic_operators bop)
2171 {
2172     st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop);
2173     st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue);
2174 }
2175
2176 static void
2177 add_opt_method(VALUE klass, ID mid, enum ruby_basic_operators bop)
2178 {
2179     const rb_method_entry_t *me = rb_method_entry_at(klass, mid);
2180
2181     if (me && vm_redefinition_check_method_type(me)) {
2182         add_opt_method_entry_bop(me, mid, bop);
2183     }
2184     else {
2185         rb_bug("undefined optimized method: %s", rb_id2name(mid));
2186     }
2187 }
2188
2189 static enum ruby_basic_operators vm_redefinition_bop_for_id(ID mid);
2190
2191 static void
2192 add_opt_method_entry(const rb_method_entry_t *me)
2193 {
2194     if (me && vm_redefinition_check_method_type(me)) {
2195         ID mid = me->called_id;
2196         enum ruby_basic_operators bop = vm_redefinition_bop_for_id(mid);
2197         if ((int)bop >= 0) {
2198             add_opt_method_entry_bop(me, mid, bop);
2199         }
2200     }
2201 }
2202
2203 static void
2204 vm_init_redefined_flag(void)
2205 {
2206     ID mid;
2207     enum ruby_basic_operators bop;
2208
2209 #define OP(mid_, bop_) (mid = id##mid_, bop = BOP_##bop_, ruby_vm_redefined_flag[bop] = 0)
2210 #define C(k) add_opt_method(rb_c##k, mid, bop)
2211     OP(PLUS, PLUS), (C(Integer), C(Float), C(String), C(Array));
2212     OP(MINUS, MINUS), (C(Integer), C(Float));
2213     OP(MULT, MULT), (C(Integer), C(Float));
2214     OP(DIV, DIV), (C(Integer), C(Float));
2215     OP(MOD, MOD), (C(Integer), C(Float));
2216     OP(Eq, EQ), (C(Integer), C(Float), C(String), C(Symbol));
2217     OP(Eqq, EQQ), (C(Integer), C(Float), C(Symbol), C(String),
2218                    C(NilClass), C(TrueClass), C(FalseClass));
2219     OP(LT, LT), (C(Integer), C(Float));
2220     OP(LE, LE), (C(Integer), C(Float));
2221     OP(GT, GT), (C(Integer), C(Float));
2222     OP(GE, GE), (C(Integer), C(Float));
2223     OP(LTLT, LTLT), (C(String), C(Array));
2224     OP(AREF, AREF), (C(Array), C(Hash), C(Integer));
2225     OP(ASET, ASET), (C(Array), C(Hash));
2226     OP(Length, LENGTH), (C(Array), C(String), C(Hash));
2227     OP(Size, SIZE), (C(Array), C(String), C(Hash));
2228     OP(EmptyP, EMPTY_P), (C(Array), C(String), C(Hash));
2229     OP(Succ, SUCC), (C(Integer), C(String));
2230     OP(EqTilde, MATCH), (C(Regexp), C(String));
2231     OP(Freeze, FREEZE), (C(String));
2232     OP(UMinus, UMINUS), (C(String));
2233     OP(Max, MAX), (C(Array));
2234     OP(Min, MIN), (C(Array));
2235     OP(Hash, HASH), (C(Array));
2236     OP(Call, CALL), (C(Proc));
2237     OP(And, AND), (C(Integer));
2238     OP(Or, OR), (C(Integer));
2239     OP(NilP, NIL_P), (C(NilClass));
2240     OP(Cmp, CMP), (C(Integer), C(Float), C(String));
2241     OP(Default, DEFAULT), (C(Hash));
2242 #undef C
2243 #undef OP
2244 }
2245
2246 static enum ruby_basic_operators
2247 vm_redefinition_bop_for_id(ID mid)
2248 {
2249     switch (mid) {
2250 #define OP(mid_, bop_) case id##mid_: return BOP_##bop_
2251     OP(PLUS, PLUS);
2252     OP(MINUS, MINUS);
2253     OP(MULT, MULT);
2254     OP(DIV, DIV);
2255     OP(MOD, MOD);
2256     OP(Eq, EQ);
2257     OP(Eqq, EQQ);
2258     OP(LT, LT);
2259     OP(LE, LE);
2260     OP(GT, GT);
2261     OP(GE, GE);
2262     OP(LTLT, LTLT);
2263     OP(AREF, AREF);
2264     OP(ASET, ASET);
2265     OP(Length, LENGTH);
2266     OP(Size, SIZE);
2267     OP(EmptyP, EMPTY_P);
2268     OP(Succ, SUCC);
2269     OP(EqTilde, MATCH);
2270     OP(Freeze, FREEZE);
2271     OP(UMinus, UMINUS);
2272     OP(Max, MAX);
2273     OP(Min, MIN);
2274     OP(Hash, HASH);
2275     OP(Call, CALL);
2276     OP(And, AND);
2277     OP(Or, OR);
2278     OP(NilP, NIL_P);
2279     OP(Cmp, CMP);
2280     OP(Default, DEFAULT);
2281     OP(Pack, PACK);
2282 #undef OP
2283     }
2284     return -1;
2285 }
2286
2287 /* for vm development */
2288
2289 #if VMDEBUG
2290 static const char *
2291 vm_frametype_name(const rb_control_frame_t *cfp)
2292 {
2293     switch (VM_FRAME_TYPE(cfp)) {
2294       case VM_FRAME_MAGIC_METHOD: return "method";
2295       case VM_FRAME_MAGIC_BLOCK:  return "block";
2296       case VM_FRAME_MAGIC_CLASS:  return "class";
2297       case VM_FRAME_MAGIC_TOP:    return "top";
2298       case VM_FRAME_MAGIC_CFUNC:  return "cfunc";
2299       case VM_FRAME_MAGIC_IFUNC:  return "ifunc";
2300       case VM_FRAME_MAGIC_EVAL:   return "eval";
2301       case VM_FRAME_MAGIC_RESCUE: return "rescue";
2302       default:
2303         rb_bug("unknown frame");
2304     }
2305 }
2306 #endif
2307
2308 static VALUE
2309 frame_return_value(const struct vm_throw_data *err)
2310 {
2311     if (THROW_DATA_P(err) &&
2312         THROW_DATA_STATE(err) == TAG_BREAK &&
2313         THROW_DATA_CONSUMED_P(err) == FALSE) {
2314         return THROW_DATA_VAL(err);
2315     }
2316     else {
2317         return Qnil;
2318     }
2319 }
2320
2321 #if 0
2322 /* for debug */
2323 static const char *
2324 frame_name(const rb_control_frame_t *cfp)
2325 {
2326     unsigned long type = VM_FRAME_TYPE(cfp);
2327 #define C(t) if (type == VM_FRAME_MAGIC_##t) return #t
2328     C(METHOD);
2329     C(BLOCK);
2330     C(CLASS);
2331     C(TOP);
2332     C(CFUNC);
2333     C(PROC);
2334     C(IFUNC);
2335     C(EVAL);
2336     C(LAMBDA);
2337     C(RESCUE);
2338     C(DUMMY);
2339 #undef C
2340     return "unknown";
2341 }
2342 #endif
2343
2344 // cfp_returning_with_value:
2345 //     Whether cfp is the last frame in the unwinding process for a non-local return.
2346 static void
2347 hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, int state, struct vm_throw_data *err)
2348 {
2349     if (state == TAG_RAISE && RBASIC(err)->klass == rb_eSysStackError) {
2350         return;
2351     }
2352     else {
2353         const rb_iseq_t *iseq = ec->cfp->iseq;
2354         rb_hook_list_t *local_hooks = iseq->aux.exec.local_hooks;
2355
2356         switch (VM_FRAME_TYPE(ec->cfp)) {
2357           case VM_FRAME_MAGIC_METHOD:
2358             RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
2359             EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err));
2360
2361             if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_RETURN)) {
2362                 rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN,
2363                                         ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE);
2364             }
2365
2366             THROW_DATA_CONSUMED_SET(err);
2367             break;
2368           case VM_FRAME_MAGIC_BLOCK:
2369             if (VM_FRAME_BMETHOD_P(ec->cfp)) {
2370                 VALUE bmethod_return_value = frame_return_value(err);
2371                 if (cfp_returning_with_value) {
2372                     // Non-local return terminating at a BMETHOD control frame.
2373                     bmethod_return_value = THROW_DATA_VAL(err);
2374                 }
2375
2376
2377                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, bmethod_return_value);
2378                 if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) {
2379                     rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN,
2380                                             ec->cfp->self, 0, 0, 0, bmethod_return_value, TRUE);
2381                 }
2382
2383                 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(ec->cfp);
2384
2385                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self,
2386                                               rb_vm_frame_method_entry(ec->cfp)->def->original_id,
2387                                               rb_vm_frame_method_entry(ec->cfp)->called_id,
2388                                               rb_vm_frame_method_entry(ec->cfp)->owner,
2389                                               bmethod_return_value);
2390
2391                 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
2392                 local_hooks = me->def->body.bmethod.hooks;
2393
2394                 if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_RETURN)) {
2395                     rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN, ec->cfp->self,
2396                                             rb_vm_frame_method_entry(ec->cfp)->def->original_id,
2397                                             rb_vm_frame_method_entry(ec->cfp)->called_id,
2398                                             rb_vm_frame_method_entry(ec->cfp)->owner,
2399                                             bmethod_return_value, TRUE);
2400                 }
2401                 THROW_DATA_CONSUMED_SET(err);
2402             }
2403             else {
2404                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err));
2405                 if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) {
2406                     rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN,
2407                                             ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE);
2408                 }
2409                 THROW_DATA_CONSUMED_SET(err);
2410             }
2411             break;
2412           case VM_FRAME_MAGIC_CLASS:
2413             EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_END, ec->cfp->self, 0, 0, 0, Qnil);
2414             break;
2415         }
2416     }
2417 }
2418
2419 /* evaluator body */
2420
2421 /*                  finish
2422   VMe (h1)          finish
2423     VM              finish F1 F2
2424       cfunc         finish F1 F2 C1
2425         rb_funcall  finish F1 F2 C1
2426           VMe       finish F1 F2 C1
2427             VM      finish F1 F2 C1 F3
2428
2429   F1 - F3 : pushed by VM
2430   C1      : pushed by send insn (CFUNC)
2431
2432   struct CONTROL_FRAME {
2433     VALUE *pc;                  // cfp[0], program counter
2434     VALUE *sp;                  // cfp[1], stack pointer
2435     rb_iseq_t *iseq;            // cfp[2], iseq
2436     VALUE self;                 // cfp[3], self
2437     const VALUE *ep;            // cfp[4], env pointer
2438     const void *block_code;     // cfp[5], block code
2439   };
2440
2441   struct rb_captured_block {
2442     VALUE self;
2443     VALUE *ep;
2444     union code;
2445   };
2446
2447   struct METHOD_ENV {
2448     VALUE param0;
2449     ...
2450     VALUE paramN;
2451     VALUE lvar1;
2452     ...
2453     VALUE lvarM;
2454     VALUE cref;    // ep[-2]
2455     VALUE special; // ep[-1]
2456     VALUE flags;   // ep[ 0] == lep[0]
2457   };
2458
2459   struct BLOCK_ENV {
2460     VALUE block_param0;
2461     ...
2462     VALUE block_paramN;
2463     VALUE block_lvar1;
2464     ...
2465     VALUE block_lvarM;
2466     VALUE cref;    // ep[-2]
2467     VALUE special; // ep[-1]
2468     VALUE flags;   // ep[ 0]
2469   };
2470
2471   struct CLASS_ENV {
2472     VALUE class_lvar0;
2473     ...
2474     VALUE class_lvarN;
2475     VALUE cref;
2476     VALUE prev_ep; // for frame jump
2477     VALUE flags;
2478   };
2479
2480   struct C_METHOD_CONTROL_FRAME {
2481     VALUE *pc;                       // 0
2482     VALUE *sp;                       // stack pointer
2483     rb_iseq_t *iseq;                 // cmi
2484     VALUE self;                      // ?
2485     VALUE *ep;                       // ep == lep
2486     void *code;                      //
2487   };
2488
2489   struct C_BLOCK_CONTROL_FRAME {
2490     VALUE *pc;                       // point only "finish" insn
2491     VALUE *sp;                       // sp
2492     rb_iseq_t *iseq;                 // ?
2493     VALUE self;                      //
2494     VALUE *ep;                       // ep
2495     void *code;                      //
2496   };
2497  */
2498
2499 static inline VALUE
2500 vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo);
2501 static inline VALUE
2502 vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, struct rb_vm_tag *tag, VALUE result);
2503
2504 // for non-Emscripten Wasm build, use vm_exec with optimized setjmp for runtime performance
2505 #if defined(__wasm__) && !defined(__EMSCRIPTEN__)
2506
2507 struct rb_vm_exec_context {
2508     rb_execution_context_t *const ec;
2509     struct rb_vm_tag *const tag;
2510
2511     VALUE result;
2512 };
2513
2514 static void
2515 vm_exec_bottom_main(void *context)
2516 {
2517     struct rb_vm_exec_context *ctx = context;
2518     rb_execution_context_t *ec = ctx->ec;
2519
2520     ctx->result = vm_exec_loop(ec, TAG_NONE, ctx->tag, vm_exec_core(ec));
2521 }
2522
2523 static void
2524 vm_exec_bottom_rescue(void *context)
2525 {
2526     struct rb_vm_exec_context *ctx = context;
2527     rb_execution_context_t *ec = ctx->ec;
2528
2529     ctx->result = vm_exec_loop(ec, rb_ec_tag_state(ec), ctx->tag, ec->errinfo);
2530 }
2531 #endif
2532
2533 VALUE
2534 vm_exec(rb_execution_context_t *ec)
2535 {
2536     VALUE result = Qundef;
2537
2538     EC_PUSH_TAG(ec);
2539
2540     _tag.retval = Qnil;
2541
2542 #if defined(__wasm__) && !defined(__EMSCRIPTEN__)
2543     struct rb_vm_exec_context ctx = {
2544         .ec = ec,
2545         .tag = &_tag,
2546     };
2547     struct rb_wasm_try_catch try_catch;
2548
2549     EC_REPUSH_TAG();
2550
2551     rb_wasm_try_catch_init(&try_catch, vm_exec_bottom_main, vm_exec_bottom_rescue, &ctx);
2552
2553     rb_wasm_try_catch_loop_run(&try_catch, &RB_VM_TAG_JMPBUF_GET(_tag.buf));
2554
2555     result = ctx.result;
2556 #else
2557     enum ruby_tag_type state;
2558     if ((state = EC_EXEC_TAG()) == TAG_NONE) {
2559         if (UNDEF_P(result = jit_exec(ec))) {
2560             result = vm_exec_core(ec);
2561         }
2562         /* fallback to the VM */
2563         result = vm_exec_loop(ec, TAG_NONE, &_tag, result);
2564     }
2565     else {
2566         result = vm_exec_loop(ec, state, &_tag, ec->errinfo);
2567     }
2568 #endif
2569
2570     EC_POP_TAG();
2571     return result;
2572 }
2573
2574 static inline VALUE
2575 vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state,
2576              struct rb_vm_tag *tag, VALUE result)
2577 {
2578     if (state == TAG_NONE) { /* no jumps, result is discarded */
2579         goto vm_loop_start;
2580     }
2581
2582     rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY);
2583     while (UNDEF_P(result = vm_exec_handle_exception(ec, state, result))) {
2584         // caught a jump, exec the handler. JIT code in jit_exec_exception()
2585         // may return Qundef to run remaining frames with vm_exec_core().
2586         if (UNDEF_P(result = jit_exec_exception(ec))) {
2587             result = vm_exec_core(ec);
2588         }
2589       vm_loop_start:
2590         VM_ASSERT(ec->tag == tag);
2591         /* when caught `throw`, `tag.state` is set. */
2592         if ((state = tag->state) == TAG_NONE) break;
2593         tag->state = TAG_NONE;
2594     }
2595
2596     return result;
2597 }
2598
2599 static inline VALUE
2600 vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo)
2601 {
2602     struct vm_throw_data *err = (struct vm_throw_data *)errinfo;
2603
2604     for (;;) {
2605         unsigned int i;
2606         const struct iseq_catch_table_entry *entry;
2607         const struct iseq_catch_table *ct;
2608         unsigned long epc, cont_pc, cont_sp;
2609         const rb_iseq_t *catch_iseq;
2610         VALUE type;
2611         const rb_control_frame_t *escape_cfp;
2612
2613         cont_pc = cont_sp = 0;
2614         catch_iseq = NULL;
2615
2616         while (ec->cfp->pc == 0 || ec->cfp->iseq == 0) {
2617             if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) {
2618                 EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self,
2619                                               rb_vm_frame_method_entry(ec->cfp)->def->original_id,
2620                                               rb_vm_frame_method_entry(ec->cfp)->called_id,
2621                                               rb_vm_frame_method_entry(ec->cfp)->owner, Qnil);
2622                 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec,
2623                                                 rb_vm_frame_method_entry(ec->cfp)->owner,
2624                                                 rb_vm_frame_method_entry(ec->cfp)->def->original_id);
2625             }
2626             rb_vm_pop_frame(ec);
2627         }
2628
2629         rb_control_frame_t *const cfp = ec->cfp;
2630         epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded;
2631
2632         escape_cfp = NULL;
2633         if (state == TAG_BREAK || state == TAG_RETURN) {
2634             escape_cfp = THROW_DATA_CATCH_FRAME(err);
2635
2636             if (cfp == escape_cfp) {
2637                 if (state == TAG_RETURN) {
2638                     if (!VM_FRAME_FINISHED_P(cfp)) {
2639                         THROW_DATA_CATCH_FRAME_SET(err, cfp + 1);
2640                         THROW_DATA_STATE_SET(err, state = TAG_BREAK);
2641                     }
2642                     else {
2643                         ct = ISEQ_BODY(cfp->iseq)->catch_table;
2644                         if (ct) for (i = 0; i < ct->size; i++) {
2645                             entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2646                             if (entry->start < epc && entry->end >= epc) {
2647                                 if (entry->type == CATCH_TYPE_ENSURE) {
2648                                     catch_iseq = entry->iseq;
2649                                     cont_pc = entry->cont;
2650                                     cont_sp = entry->sp;
2651                                     break;
2652                                 }
2653                             }
2654                         }
2655                         if (catch_iseq == NULL) {
2656                             ec->errinfo = Qnil;
2657                             THROW_DATA_CATCH_FRAME_SET(err, cfp + 1);
2658                             // cfp == escape_cfp here so calling with cfp_returning_with_value = true
2659                             hook_before_rewind(ec, true, state, err);
2660                             rb_vm_pop_frame(ec);
2661                             return THROW_DATA_VAL(err);
2662                         }
2663                     }
2664                     /* through */
2665                 }
2666                 else {
2667                     /* TAG_BREAK */
2668                     *cfp->sp++ = THROW_DATA_VAL(err);
2669                     ec->errinfo = Qnil;
2670                     return Qundef;
2671                 }
2672             }
2673         }
2674
2675         if (state == TAG_RAISE) {
2676             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2677             if (ct) for (i = 0; i < ct->size; i++) {
2678                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2679                 if (entry->start < epc && entry->end >= epc) {
2680
2681                     if (entry->type == CATCH_TYPE_RESCUE ||
2682                         entry->type == CATCH_TYPE_ENSURE) {
2683                         catch_iseq = entry->iseq;
2684                         cont_pc = entry->cont;
2685                         cont_sp = entry->sp;
2686                         break;
2687                     }
2688                 }
2689             }
2690         }
2691         else if (state == TAG_RETRY) {
2692             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2693             if (ct) for (i = 0; i < ct->size; i++) {
2694                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2695                 if (entry->start < epc && entry->end >= epc) {
2696
2697                     if (entry->type == CATCH_TYPE_ENSURE) {
2698                         catch_iseq = entry->iseq;
2699                         cont_pc = entry->cont;
2700                         cont_sp = entry->sp;
2701                         break;
2702                     }
2703                     else if (entry->type == CATCH_TYPE_RETRY) {
2704                         const rb_control_frame_t *escape_cfp;
2705                         escape_cfp = THROW_DATA_CATCH_FRAME(err);
2706                         if (cfp == escape_cfp) {
2707                             cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont;
2708                             ec->errinfo = Qnil;
2709                             return Qundef;
2710                         }
2711                     }
2712                 }
2713             }
2714         }
2715         else if ((state == TAG_BREAK && !escape_cfp) ||
2716                  (state == TAG_REDO) ||
2717                  (state == TAG_NEXT)) {
2718             type = (const enum rb_catch_type[TAG_MASK]) {
2719                 [TAG_BREAK]  = CATCH_TYPE_BREAK,
2720                 [TAG_NEXT]   = CATCH_TYPE_NEXT,
2721                 [TAG_REDO]   = CATCH_TYPE_REDO,
2722                 /* otherwise = dontcare */
2723             }[state];
2724
2725             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2726             if (ct) for (i = 0; i < ct->size; i++) {
2727                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2728
2729                 if (entry->start < epc && entry->end >= epc) {
2730                     if (entry->type == CATCH_TYPE_ENSURE) {
2731                         catch_iseq = entry->iseq;
2732                         cont_pc = entry->cont;
2733                         cont_sp = entry->sp;
2734                         break;
2735                     }
2736                     else if (entry->type == type) {
2737                         cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont;
2738                         cfp->sp = vm_base_ptr(cfp) + entry->sp;
2739
2740                         if (state != TAG_REDO) {
2741                             *cfp->sp++ = THROW_DATA_VAL(err);
2742                         }
2743                         ec->errinfo = Qnil;
2744                         VM_ASSERT(ec->tag->state == TAG_NONE);
2745                         return Qundef;
2746                     }
2747                 }
2748             }
2749         }
2750         else {
2751             ct = ISEQ_BODY(cfp->iseq)->catch_table;
2752             if (ct) for (i = 0; i < ct->size; i++) {
2753                 entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
2754                 if (entry->start < epc && entry->end >= epc) {
2755
2756                     if (entry->type == CATCH_TYPE_ENSURE) {
2757                         catch_iseq = entry->iseq;
2758                         cont_pc = entry->cont;
2759                         cont_sp = entry->sp;
2760                         break;
2761                     }
2762                 }
2763             }
2764         }
2765
2766         if (catch_iseq != NULL) { /* found catch table */
2767             /* enter catch scope */
2768             const int arg_size = 1;
2769
2770             rb_iseq_check(catch_iseq);
2771             cfp->sp = vm_base_ptr(cfp) + cont_sp;
2772             cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + cont_pc;
2773
2774             /* push block frame */
2775             cfp->sp[0] = (VALUE)err;
2776             vm_push_frame(ec, catch_iseq, VM_FRAME_MAGIC_RESCUE,
2777                           cfp->self,
2778                           VM_GUARDED_PREV_EP(cfp->ep),
2779                           0, /* cref or me */
2780                           ISEQ_BODY(catch_iseq)->iseq_encoded,
2781                           cfp->sp + arg_size /* push value */,
2782                           ISEQ_BODY(catch_iseq)->local_table_size - arg_size,
2783                           ISEQ_BODY(catch_iseq)->stack_max);
2784
2785             state = 0;
2786             ec->tag->state = TAG_NONE;
2787             ec->errinfo = Qnil;
2788
2789             return Qundef;
2790         }
2791         else {
2792             hook_before_rewind(ec, (cfp == escape_cfp), state, err);
2793
2794             if (VM_FRAME_FINISHED_P(ec->cfp)) {
2795                 rb_vm_pop_frame(ec);
2796                 ec->errinfo = (VALUE)err;
2797                 ec->tag = ec->tag->prev;
2798                 EC_JUMP_TAG(ec, state);
2799             }
2800             else {
2801                 rb_vm_pop_frame(ec);
2802             }
2803         }
2804     }
2805 }
2806
2807 /* misc */
2808
2809 VALUE
2810 rb_iseq_eval(const rb_iseq_t *iseq)
2811 {
2812     rb_execution_context_t *ec = GET_EC();
2813     VALUE val;
2814     vm_set_top_stack(ec, iseq);
2815     val = vm_exec(ec);
2816     return val;
2817 }
2818
2819 VALUE
2820 rb_iseq_eval_main(const rb_iseq_t *iseq)
2821 {
2822     rb_execution_context_t *ec = GET_EC();
2823     VALUE val;
2824
2825     vm_set_main_stack(ec, iseq);
2826     val = vm_exec(ec);
2827     return val;
2828 }
2829
2830 int
2831 rb_vm_control_frame_id_and_class(const rb_control_frame_t *cfp, ID *idp, ID *called_idp, VALUE *klassp)
2832 {
2833     const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
2834
2835     if (me) {
2836         if (idp) *idp = me->def->original_id;
2837         if (called_idp) *called_idp = me->called_id;
2838         if (klassp) *klassp = me->owner;
2839         return TRUE;
2840     }
2841     else {
2842         return FALSE;
2843     }
2844 }
2845
2846 int
2847 rb_ec_frame_method_id_and_class(const rb_execution_context_t *ec, ID *idp, ID *called_idp, VALUE *klassp)
2848 {
2849     return rb_vm_control_frame_id_and_class(ec->cfp, idp, called_idp, klassp);
2850 }
2851
2852 int
2853 rb_frame_method_id_and_class(ID *idp, VALUE *klassp)
2854 {
2855     return rb_ec_frame_method_id_and_class(GET_EC(), idp, 0, klassp);
2856 }
2857
2858 VALUE
2859 rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg,
2860                  VALUE block_handler, VALUE filename)
2861 {
2862     rb_execution_context_t *ec = GET_EC();
2863     const rb_control_frame_t *reg_cfp = ec->cfp;
2864     const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP);
2865     VALUE val;
2866
2867     vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH,
2868                   recv, block_handler,
2869                   (VALUE)vm_cref_new_toplevel(ec), /* cref or me */
2870                   0, reg_cfp->sp, 0, 0);
2871
2872     val = (*func)(arg);
2873
2874     rb_vm_pop_frame(ec);
2875     return val;
2876 }
2877
2878 /* vm */
2879
2880 void
2881 rb_vm_update_references(void *ptr)
2882 {
2883     if (ptr) {
2884         rb_vm_t *vm = ptr;
2885
2886         rb_gc_update_tbl_refs(vm->ci_table);
2887         rb_gc_update_tbl_refs(vm->frozen_strings);
2888         vm->mark_object_ary = rb_gc_location(vm->mark_object_ary);
2889         vm->load_path = rb_gc_location(vm->load_path);
2890         vm->load_path_snapshot = rb_gc_location(vm->load_path_snapshot);
2891
2892         if (vm->load_path_check_cache) {
2893             vm->load_path_check_cache = rb_gc_location(vm->load_path_check_cache);
2894         }
2895
2896         vm->expanded_load_path = rb_gc_location(vm->expanded_load_path);
2897         vm->loaded_features = rb_gc_location(vm->loaded_features);
2898         vm->loaded_features_snapshot = rb_gc_location(vm->loaded_features_snapshot);
2899         vm->loaded_features_realpaths = rb_gc_location(vm->loaded_features_realpaths);
2900         vm->loaded_features_realpath_map = rb_gc_location(vm->loaded_features_realpath_map);
2901         vm->top_self = rb_gc_location(vm->top_self);
2902         vm->orig_progname = rb_gc_location(vm->orig_progname);
2903
2904         rb_gc_update_tbl_refs(vm->overloaded_cme_table);
2905
2906         rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd);
2907
2908         if (vm->coverages) {
2909             vm->coverages = rb_gc_location(vm->coverages);
2910             vm->me2counter = rb_gc_location(vm->me2counter);
2911         }
2912     }
2913 }
2914
2915 void
2916 rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx)
2917 {
2918     if (ptr) {
2919         rb_vm_t *vm = ptr;
2920         rb_ractor_t *r = 0;
2921         ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
2922             VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) ||
2923                       rb_ractor_status_p(r, ractor_running));
2924             if (r->threads.cnt > 0) {
2925                 rb_thread_t *th = 0;
2926                 ccan_list_for_each(&r->threads.set, th, lt_node) {
2927                     VM_ASSERT(th != NULL);
2928                     rb_execution_context_t * ec = th->ec;
2929                     if (ec->vm_stack) {
2930                         VALUE *p = ec->vm_stack;
2931                         VALUE *sp = ec->cfp->sp;
2932                         while (p < sp) {
2933                             if (!RB_SPECIAL_CONST_P(*p)) {
2934                                 cb(*p, ctx);
2935                             }
2936                             p++;
2937                         }
2938                     }
2939                 }
2940             }
2941         }
2942     }
2943 }
2944
2945 static enum rb_id_table_iterator_result
2946 vm_mark_negative_cme(VALUE val, void *dmy)
2947 {
2948     rb_gc_mark(val);
2949     return ID_TABLE_CONTINUE;
2950 }
2951
2952 void rb_thread_sched_mark_zombies(rb_vm_t *vm);
2953
2954 void
2955 rb_vm_mark(void *ptr)
2956 {
2957     RUBY_MARK_ENTER("vm");
2958     RUBY_GC_INFO("-------------------------------------------------\n");
2959     if (ptr) {
2960         rb_vm_t *vm = ptr;
2961         rb_ractor_t *r = 0;
2962         long i;
2963
2964         ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
2965             // ractor.set only contains blocking or running ractors
2966             VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) ||
2967                       rb_ractor_status_p(r, ractor_running));
2968             rb_gc_mark(rb_ractor_self(r));
2969         }
2970
2971         for (struct global_object_list *list = vm->global_object_list; list; list = list->next) {
2972             rb_gc_mark_maybe(*list->varptr);
2973         }
2974
2975         rb_gc_mark_movable(vm->mark_object_ary);
2976         rb_gc_mark_movable(vm->load_path);
2977         rb_gc_mark_movable(vm->load_path_snapshot);
2978         rb_gc_mark_movable(vm->load_path_check_cache);
2979         rb_gc_mark_movable(vm->expanded_load_path);
2980         rb_gc_mark_movable(vm->loaded_features);
2981         rb_gc_mark_movable(vm->loaded_features_snapshot);
2982         rb_gc_mark_movable(vm->loaded_features_realpaths);
2983         rb_gc_mark_movable(vm->loaded_features_realpath_map);
2984         rb_gc_mark_movable(vm->top_self);
2985         rb_gc_mark_movable(vm->orig_progname);
2986         rb_gc_mark_movable(vm->coverages);
2987         rb_gc_mark_movable(vm->me2counter);
2988
2989         if (vm->loading_table) {
2990             rb_mark_tbl(vm->loading_table);
2991         }
2992
2993         rb_gc_mark_values(RUBY_NSIG, vm->trap_list.cmd);
2994
2995         rb_id_table_foreach_values(vm->negative_cme_table, vm_mark_negative_cme, NULL);
2996         rb_mark_tbl_no_pin(vm->overloaded_cme_table);
2997         for (i=0; i<VM_GLOBAL_CC_CACHE_TABLE_SIZE; i++) {
2998             const struct rb_callcache *cc = vm->global_cc_cache_table[i];
2999
3000             if (cc != NULL) {
3001                 if (!vm_cc_invalidated_p(cc)) {
3002                     rb_gc_mark((VALUE)cc);
3003                 }
3004                 else {
3005                     vm->global_cc_cache_table[i] = NULL;
3006                 }
3007             }
3008         }
3009
3010         rb_thread_sched_mark_zombies(vm);
3011         rb_rjit_mark();
3012     }
3013
3014     RUBY_MARK_LEAVE("vm");
3015 }
3016
3017 #undef rb_vm_register_special_exception
3018 void
3019 rb_vm_register_special_exception_str(enum ruby_special_exceptions sp, VALUE cls, VALUE mesg)
3020 {
3021     rb_vm_t *vm = GET_VM();
3022     VALUE exc = rb_exc_new3(cls, rb_obj_freeze(mesg));
3023     OBJ_FREEZE(exc);
3024     ((VALUE *)vm->special_exceptions)[sp] = exc;
3025     rb_vm_register_global_object(exc);
3026 }
3027
3028 static int
3029 free_loading_table_entry(st_data_t key, st_data_t value, st_data_t arg)
3030 {
3031     xfree((char *)key);
3032     return ST_DELETE;
3033 }
3034
3035 void rb_free_loaded_features_index(rb_vm_t *vm);
3036 void rb_objspace_free_objects(void *objspace);
3037
3038 int
3039 ruby_vm_destruct(rb_vm_t *vm)
3040 {
3041     RUBY_FREE_ENTER("vm");
3042
3043     if (vm) {
3044         rb_thread_t *th = vm->ractor.main_thread;
3045         VALUE *stack = th->ec->vm_stack;
3046         if (rb_free_at_exit) {
3047             rb_free_encoded_insn_data();
3048             rb_free_global_enc_table();
3049             rb_free_loaded_builtin_table();
3050
3051             rb_free_shared_fiber_pool();
3052             rb_free_static_symid_str();
3053             rb_free_transcoder_table();
3054             rb_free_vm_opt_tables();
3055             rb_free_warning();
3056             rb_free_rb_global_tbl();
3057             rb_free_loaded_features_index(vm);
3058
3059             rb_id_table_free(vm->negative_cme_table);
3060             st_free_table(vm->overloaded_cme_table);
3061
3062             rb_id_table_free(RCLASS(rb_mRubyVMFrozenCore)->m_tbl);
3063
3064             rb_shape_t *cursor = rb_shape_get_root_shape();
3065             rb_shape_t *end = rb_shape_get_shape_by_id(GET_SHAPE_TREE()->next_shape_id);
3066             while (cursor < end) {
3067                 // 0x1 == SINGLE_CHILD_P
3068                 if (cursor->edges && !(((uintptr_t)cursor->edges) & 0x1))
3069                     rb_id_table_free(cursor->edges);
3070                 cursor += 1;
3071             }
3072
3073             xfree(GET_SHAPE_TREE());
3074
3075             st_free_table(vm->static_ext_inits);
3076             st_free_table(vm->ensure_rollback_table);
3077
3078             rb_vm_postponed_job_free();
3079
3080             rb_id_table_free(vm->constant_cache);
3081             st_free_table(vm->unused_block_warning_table);
3082
3083             if (th) {
3084                 xfree(th->nt);
3085                 th->nt = NULL;
3086             }
3087
3088 #ifndef HAVE_SETPROCTITLE
3089             ruby_free_proctitle();
3090 #endif
3091         }
3092         else {
3093             if (th) {
3094                 rb_fiber_reset_root_local_storage(th);
3095                 thread_free(th);
3096             }
3097         }
3098
3099         struct rb_objspace *objspace = vm->objspace;
3100
3101         rb_vm_living_threads_init(vm);
3102         ruby_vm_run_at_exit_hooks(vm);
3103         if (vm->loading_table) {
3104             st_foreach(vm->loading_table, free_loading_table_entry, 0);
3105             st_free_table(vm->loading_table);
3106             vm->loading_table = 0;
3107         }
3108         if (vm->ci_table) {
3109             st_free_table(vm->ci_table);
3110             vm->ci_table = NULL;
3111         }
3112         if (vm->frozen_strings) {
3113             st_free_table(vm->frozen_strings);
3114             vm->frozen_strings = 0;
3115         }
3116         RB_ALTSTACK_FREE(vm->main_altstack);
3117
3118         struct global_object_list *next;
3119         for (struct global_object_list *list = vm->global_object_list; list; list = next) {
3120             next = list->next;
3121             xfree(list);
3122         }
3123
3124         if (objspace) {
3125             if (rb_free_at_exit) {
3126                 rb_objspace_free_objects(objspace);
3127                 rb_free_generic_iv_tbl_();
3128                 rb_free_default_rand_key();
3129                 if (th && vm->fork_gen == 0) {
3130                     /* If we have forked, main_thread may not be the initial thread */
3131                     xfree(stack);
3132                     ruby_mimfree(th);
3133                 }
3134             }
3135             rb_objspace_free(objspace);
3136         }
3137         rb_native_mutex_destroy(&vm->workqueue_lock);
3138         /* after freeing objspace, you *can't* use ruby_xfree() */
3139         ruby_mimfree(vm);
3140         ruby_current_vm_ptr = NULL;
3141     }
3142     RUBY_FREE_LEAVE("vm");
3143     return 0;
3144 }
3145
3146 size_t rb_vm_memsize_waiting_fds(struct ccan_list_head *waiting_fds); // thread.c
3147 size_t rb_vm_memsize_workqueue(struct ccan_list_head *workqueue); // vm_trace.c
3148
3149 // Used for VM memsize reporting. Returns the size of the at_exit list by
3150 // looping through the linked list and adding up the size of the structs.
3151 static enum rb_id_table_iterator_result
3152 vm_memsize_constant_cache_i(ID id, VALUE ics, void *size)
3153 {
3154     *((size_t *) size) += rb_st_memsize((st_table *) ics);
3155     return ID_TABLE_CONTINUE;
3156 }
3157
3158 // Returns a size_t representing the memory footprint of the VM's constant
3159 // cache, which is the memsize of the table as well as the memsize of all of the
3160 // nested tables.
3161 static size_t
3162 vm_memsize_constant_cache(void)
3163 {
3164     rb_vm_t *vm = GET_VM();
3165     size_t size = rb_id_table_memsize(vm->constant_cache);
3166
3167     rb_id_table_foreach(vm->constant_cache, vm_memsize_constant_cache_i, &size);
3168     return size;
3169 }
3170
3171 static size_t
3172 vm_memsize_at_exit_list(rb_at_exit_list *at_exit)
3173 {
3174     size_t size = 0;
3175
3176     while (at_exit) {
3177         size += sizeof(rb_at_exit_list);
3178         at_exit = at_exit->next;
3179     }
3180
3181     return size;
3182 }
3183
3184 // Used for VM memsize reporting. Returns the size of the builtin function
3185 // table if it has been defined.
3186 static size_t
3187 vm_memsize_builtin_function_table(const struct rb_builtin_function *builtin_function_table)
3188 {
3189     return builtin_function_table == NULL ? 0 : sizeof(struct rb_builtin_function);
3190 }
3191
3192 // Reports the memsize of the VM struct object and the structs that are
3193 // associated with it.
3194 static size_t
3195 vm_memsize(const void *ptr)
3196 {
3197     rb_vm_t *vm = GET_VM();
3198
3199     return (
3200         sizeof(rb_vm_t) +
3201         rb_vm_memsize_waiting_fds(&vm->waiting_fds) +
3202         rb_st_memsize(vm->loaded_features_index) +
3203         rb_st_memsize(vm->loading_table) +
3204         rb_st_memsize(vm->ensure_rollback_table) +
3205         rb_vm_memsize_postponed_job_queue() +
3206         rb_vm_memsize_workqueue(&vm->workqueue) +
3207         vm_memsize_at_exit_list(vm->at_exit) +
3208         rb_st_memsize(vm->ci_table) +
3209         rb_st_memsize(vm->frozen_strings) +
3210         vm_memsize_builtin_function_table(vm->builtin_function_table) +
3211         rb_id_table_memsize(vm->negative_cme_table) +
3212         rb_st_memsize(vm->overloaded_cme_table) +
3213         vm_memsize_constant_cache() +
3214         GET_SHAPE_TREE()->cache_size * sizeof(redblack_node_t)
3215     );
3216
3217     // TODO
3218     // struct { struct ccan_list_head set; } ractor;
3219     // void *main_altstack; #ifdef USE_SIGALTSTACK
3220     // struct rb_objspace *objspace;
3221 }
3222
3223 static const rb_data_type_t vm_data_type = {
3224     "VM",
3225     {0, 0, vm_memsize,},
3226     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3227 };
3228
3229
3230 static VALUE
3231 vm_default_params(void)
3232 {
3233     rb_vm_t *vm = GET_VM();
3234     VALUE result = rb_hash_new_with_size(4);
3235 #define SET(name) rb_hash_aset(result, ID2SYM(rb_intern(#name)), SIZET2NUM(vm->default_params.name));
3236     SET(thread_vm_stack_size);
3237     SET(thread_machine_stack_size);
3238     SET(fiber_vm_stack_size);
3239     SET(fiber_machine_stack_size);
3240 #undef SET
3241     rb_obj_freeze(result);
3242     return result;
3243 }
3244
3245 static size_t
3246 get_param(const char *name, size_t default_value, size_t min_value)
3247 {
3248     const char *envval;
3249     size_t result = default_value;
3250     if ((envval = getenv(name)) != 0) {
3251         long val = atol(envval);
3252         if (val < (long)min_value) {
3253             val = (long)min_value;
3254         }
3255         result = (size_t)(((val -1 + RUBY_VM_SIZE_ALIGN) / RUBY_VM_SIZE_ALIGN) * RUBY_VM_SIZE_ALIGN);
3256     }
3257     if (0) ruby_debug_printf("%s: %"PRIuSIZE"\n", name, result); /* debug print */
3258
3259     return result;
3260 }
3261
3262 static void
3263 check_machine_stack_size(size_t *sizep)
3264 {
3265 #ifdef PTHREAD_STACK_MIN
3266     size_t size = *sizep;
3267 #endif
3268
3269 #ifdef PTHREAD_STACK_MIN
3270     if (size < (size_t)PTHREAD_STACK_MIN) {
3271         *sizep = (size_t)PTHREAD_STACK_MIN * 2;
3272     }
3273 #endif
3274 }
3275
3276 static void
3277 vm_default_params_setup(rb_vm_t *vm)
3278 {
3279     vm->default_params.thread_vm_stack_size =
3280       get_param("RUBY_THREAD_VM_STACK_SIZE",
3281                 RUBY_VM_THREAD_VM_STACK_SIZE,
3282                 RUBY_VM_THREAD_VM_STACK_SIZE_MIN);
3283
3284     vm->default_params.thread_machine_stack_size =
3285       get_param("RUBY_THREAD_MACHINE_STACK_SIZE",
3286                 RUBY_VM_THREAD_MACHINE_STACK_SIZE,
3287                 RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN);
3288
3289     vm->default_params.fiber_vm_stack_size =
3290       get_param("RUBY_FIBER_VM_STACK_SIZE",
3291                 RUBY_VM_FIBER_VM_STACK_SIZE,
3292                 RUBY_VM_FIBER_VM_STACK_SIZE_MIN);
3293
3294     vm->default_params.fiber_machine_stack_size =
3295       get_param("RUBY_FIBER_MACHINE_STACK_SIZE",
3296                 RUBY_VM_FIBER_MACHINE_STACK_SIZE,
3297                 RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN);
3298
3299     /* environment dependent check */
3300     check_machine_stack_size(&vm->default_params.thread_machine_stack_size);
3301     check_machine_stack_size(&vm->default_params.fiber_machine_stack_size);
3302 }
3303
3304 static void
3305 vm_init2(rb_vm_t *vm)
3306 {
3307     rb_vm_living_threads_init(vm);
3308     vm->thread_report_on_exception = 1;
3309     vm->src_encoding_index = -1;
3310
3311     vm_default_params_setup(vm);
3312 }
3313
3314 void
3315 rb_execution_context_update(rb_execution_context_t *ec)
3316 {
3317     /* update VM stack */
3318     if (ec->vm_stack) {
3319         long i;
3320         VM_ASSERT(ec->cfp);
3321         VALUE *p = ec->vm_stack;
3322         VALUE *sp = ec->cfp->sp;
3323         rb_control_frame_t *cfp = ec->cfp;
3324         rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
3325
3326         for (i = 0; i < (long)(sp - p); i++) {
3327             VALUE ref = p[i];
3328             VALUE update = rb_gc_location(ref);
3329             if (ref != update) {
3330                 p[i] = update;
3331             }
3332         }
3333
3334         while (cfp != limit_cfp) {
3335             const VALUE *ep = cfp->ep;
3336             cfp->self = rb_gc_location(cfp->self);
3337             cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq);
3338             cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code);
3339
3340             if (!VM_ENV_LOCAL_P(ep)) {
3341                 const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
3342                 if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) {
3343                     VM_FORCE_WRITE(&prev_ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(prev_ep[VM_ENV_DATA_INDEX_ENV]));
3344                 }
3345
3346                 if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) {
3347                     VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(ep[VM_ENV_DATA_INDEX_ENV]));
3348                     VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ME_CREF], rb_gc_location(ep[VM_ENV_DATA_INDEX_ME_CREF]));
3349                 }
3350             }
3351
3352             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
3353         }
3354     }
3355
3356     ec->storage = rb_gc_location(ec->storage);
3357 }
3358
3359 static enum rb_id_table_iterator_result
3360 mark_local_storage_i(VALUE local, void *data)
3361 {
3362     rb_gc_mark(local);
3363     return ID_TABLE_CONTINUE;
3364 }
3365
3366 void
3367 rb_execution_context_mark(const rb_execution_context_t *ec)
3368 {
3369     /* mark VM stack */
3370     if (ec->vm_stack) {
3371         VM_ASSERT(ec->cfp);
3372         VALUE *p = ec->vm_stack;
3373         VALUE *sp = ec->cfp->sp;
3374         rb_control_frame_t *cfp = ec->cfp;
3375         rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
3376
3377         VM_ASSERT(sp == ec->cfp->sp);
3378         rb_gc_mark_vm_stack_values((long)(sp - p), p);
3379
3380         while (cfp != limit_cfp) {
3381             const VALUE *ep = cfp->ep;
3382             VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep));
3383
3384             if (VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) {
3385                 rb_gc_mark_movable(cfp->self);
3386                 rb_gc_mark_movable((VALUE)cfp->iseq);
3387                 rb_gc_mark_movable((VALUE)cfp->block_code);
3388
3389                 if (!VM_ENV_LOCAL_P(ep)) {
3390                     const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
3391                     if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) {
3392                         rb_gc_mark_movable(prev_ep[VM_ENV_DATA_INDEX_ENV]);
3393                     }
3394
3395                     if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) {
3396                         rb_gc_mark_movable(ep[VM_ENV_DATA_INDEX_ENV]);
3397                         rb_gc_mark(ep[VM_ENV_DATA_INDEX_ME_CREF]);
3398                     }
3399                 }
3400             }
3401
3402             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
3403         }
3404     }
3405
3406     /* mark machine stack */
3407     if (ec->machine.stack_start && ec->machine.stack_end &&
3408         ec != GET_EC() /* marked for current ec at the first stage of marking */
3409         ) {
3410         rb_gc_mark_machine_context(ec);
3411     }
3412
3413     rb_gc_mark(ec->errinfo);
3414     rb_gc_mark(ec->root_svar);
3415     if (ec->local_storage) {
3416         rb_id_table_foreach_values(ec->local_storage, mark_local_storage_i, NULL);
3417     }
3418     rb_gc_mark(ec->local_storage_recursive_hash);
3419     rb_gc_mark(ec->local_storage_recursive_hash_for_trace);
3420     rb_gc_mark(ec->private_const_reference);
3421
3422     rb_gc_mark_movable(ec->storage);
3423 }
3424
3425 void rb_fiber_mark_self(rb_fiber_t *fib);
3426 void rb_fiber_update_self(rb_fiber_t *fib);
3427 void rb_threadptr_root_fiber_setup(rb_thread_t *th);
3428 void rb_threadptr_root_fiber_release(rb_thread_t *th);
3429
3430 static void
3431 thread_compact(void *ptr)
3432 {
3433     rb_thread_t *th = ptr;
3434
3435     th->self = rb_gc_location(th->self);
3436
3437     if (!th->root_fiber) {
3438         rb_execution_context_update(th->ec);
3439     }
3440 }
3441
3442 static void
3443 thread_mark(void *ptr)
3444 {
3445     rb_thread_t *th = ptr;
3446     RUBY_MARK_ENTER("thread");
3447     rb_fiber_mark_self(th->ec->fiber_ptr);
3448
3449     /* mark ruby objects */
3450     switch (th->invoke_type) {
3451       case thread_invoke_type_proc:
3452       case thread_invoke_type_ractor_proc:
3453         rb_gc_mark(th->invoke_arg.proc.proc);
3454         rb_gc_mark(th->invoke_arg.proc.args);
3455         break;
3456       case thread_invoke_type_func:
3457         rb_gc_mark_maybe((VALUE)th->invoke_arg.func.arg);
3458         break;
3459       default:
3460         break;
3461     }
3462
3463     rb_gc_mark(rb_ractor_self(th->ractor));
3464     rb_gc_mark(th->thgroup);
3465     rb_gc_mark(th->value);
3466     rb_gc_mark(th->pending_interrupt_queue);
3467     rb_gc_mark(th->pending_interrupt_mask_stack);
3468     rb_gc_mark(th->top_self);
3469     rb_gc_mark(th->top_wrapper);
3470     if (th->root_fiber) rb_fiber_mark_self(th->root_fiber);
3471
3472     RUBY_ASSERT(th->ec == rb_fiberptr_get_ec(th->ec->fiber_ptr));
3473     rb_gc_mark(th->stat_insn_usage);
3474     rb_gc_mark(th->last_status);
3475     rb_gc_mark(th->locking_mutex);
3476     rb_gc_mark(th->name);
3477
3478     rb_gc_mark(th->scheduler);
3479
3480     RUBY_MARK_LEAVE("thread");
3481 }
3482
3483 void rb_threadptr_sched_free(rb_thread_t *th); // thread_*.c
3484
3485 static void
3486 thread_free(void *ptr)
3487 {
3488     rb_thread_t *th = ptr;
3489     RUBY_FREE_ENTER("thread");
3490
3491     rb_threadptr_sched_free(th);
3492
3493     if (th->locking_mutex != Qfalse) {
3494         rb_bug("thread_free: locking_mutex must be NULL (%p:%p)", (void *)th, (void *)th->locking_mutex);
3495     }
3496     if (th->keeping_mutexes != NULL) {
3497         rb_bug("thread_free: keeping_mutexes must be NULL (%p:%p)", (void *)th, (void *)th->keeping_mutexes);
3498     }
3499
3500     ruby_xfree(th->specific_storage);
3501
3502     rb_threadptr_root_fiber_release(th);
3503
3504     if (th->vm && th->vm->ractor.main_thread == th) {
3505         RUBY_GC_INFO("MRI main thread\n");
3506     }
3507     else {
3508         // ruby_xfree(th->nt);
3509         // TODO: MN system collect nt, but without MN system it should be freed here.
3510         ruby_xfree(th);
3511     }
3512
3513     RUBY_FREE_LEAVE("thread");
3514 }
3515
3516 static size_t
3517 thread_memsize(const void *ptr)
3518 {
3519     const rb_thread_t *th = ptr;
3520     size_t size = sizeof(rb_thread_t);
3521
3522     if (!th->root_fiber) {
3523         size += th->ec->vm_stack_size * sizeof(VALUE);
3524     }
3525     if (th->ec->local_storage) {
3526         size += rb_id_table_memsize(th->ec->local_storage);
3527     }
3528     return size;
3529 }
3530
3531 #define thread_data_type ruby_threadptr_data_type
3532 const rb_data_type_t ruby_threadptr_data_type = {
3533     "VM/thread",
3534     {
3535         thread_mark,
3536         thread_free,
3537         thread_memsize,
3538         thread_compact,
3539     },
3540     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3541 };
3542
3543 VALUE
3544 rb_obj_is_thread(VALUE obj)
3545 {
3546     return RBOOL(rb_typeddata_is_kind_of(obj, &thread_data_type));
3547 }
3548
3549 static VALUE
3550 thread_alloc(VALUE klass)
3551 {
3552     rb_thread_t *th;
3553     return TypedData_Make_Struct(klass, rb_thread_t, &thread_data_type, th);
3554 }
3555
3556 inline void
3557 rb_ec_set_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
3558 {
3559     ec->vm_stack = stack;
3560     ec->vm_stack_size = size;
3561 }
3562
3563 void
3564 rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
3565 {
3566     rb_ec_set_vm_stack(ec, stack, size);
3567
3568 #if VM_CHECK_MODE > 0
3569     MEMZERO(stack, VALUE, size); // malloc memory could have the VM canary in it
3570 #endif
3571
3572     ec->cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
3573
3574     vm_push_frame(ec,
3575         NULL /* dummy iseq */,
3576         VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH | VM_FRAME_FLAG_CFRAME /* dummy frame */,
3577         Qnil /* dummy self */, VM_BLOCK_HANDLER_NONE /* dummy block ptr */,
3578         0 /* dummy cref/me */,
3579         0 /* dummy pc */, ec->vm_stack, 0, 0
3580     );
3581 }
3582
3583 void
3584 rb_ec_clear_vm_stack(rb_execution_context_t *ec)
3585 {
3586     rb_ec_set_vm_stack(ec, NULL, 0);
3587
3588     // Avoid dangling pointers:
3589     ec->cfp = NULL;
3590 }
3591
3592 static void
3593 th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm)
3594 {
3595     th->self = self;
3596
3597     rb_threadptr_root_fiber_setup(th);
3598
3599     /* All threads are blocking until a non-blocking fiber is scheduled */
3600     th->blocking = 1;
3601     th->scheduler = Qnil;
3602
3603     if (self == 0) {
3604         size_t size = vm->default_params.thread_vm_stack_size / sizeof(VALUE);
3605         rb_ec_initialize_vm_stack(th->ec, ALLOC_N(VALUE, size), size);
3606     }
3607     else {
3608         VM_ASSERT(th->ec->cfp == NULL);
3609         VM_ASSERT(th->ec->vm_stack == NULL);
3610         VM_ASSERT(th->ec->vm_stack_size == 0);
3611     }
3612
3613     th->status = THREAD_RUNNABLE;
3614     th->last_status = Qnil;
3615     th->top_wrapper = 0;
3616     th->top_self = vm->top_self; // 0 while self == 0
3617     th->value = Qundef;
3618
3619     th->ec->errinfo = Qnil;
3620     th->ec->root_svar = Qfalse;
3621     th->ec->local_storage_recursive_hash = Qnil;
3622     th->ec->local_storage_recursive_hash_for_trace = Qnil;
3623
3624     th->ec->storage = Qnil;
3625
3626 #if OPT_CALL_THREADED_CODE
3627     th->retval = Qundef;
3628 #endif
3629     th->name = Qnil;
3630     th->report_on_exception = vm->thread_report_on_exception;
3631     th->ext_config.ractor_safe = true;
3632
3633 #if USE_RUBY_DEBUG_LOG
3634     static rb_atomic_t thread_serial = 1;
3635     th->serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1);
3636
3637     RUBY_DEBUG_LOG("th:%u", th->serial);
3638 #endif
3639 }
3640
3641 VALUE
3642 rb_thread_alloc(VALUE klass)
3643 {
3644     VALUE self = thread_alloc(klass);
3645     rb_thread_t *target_th = rb_thread_ptr(self);
3646     target_th->ractor = GET_RACTOR();
3647     th_init(target_th, self, target_th->vm = GET_VM());
3648     return self;
3649 }
3650
3651 #define REWIND_CFP(expr) do { \
3652     rb_execution_context_t *ec__ = GET_EC(); \
3653     VALUE *const curr_sp = (ec__->cfp++)->sp; \
3654     VALUE *const saved_sp = ec__->cfp->sp; \
3655     ec__->cfp->sp = curr_sp; \
3656     expr; \
3657     (ec__->cfp--)->sp = saved_sp; \
3658 } while (0)
3659
3660 static VALUE
3661 m_core_set_method_alias(VALUE self, VALUE cbase, VALUE sym1, VALUE sym2)
3662 {
3663     REWIND_CFP({
3664         rb_alias(cbase, SYM2ID(sym1), SYM2ID(sym2));
3665     });
3666     return Qnil;
3667 }
3668
3669 static VALUE
3670 m_core_set_variable_alias(VALUE self, VALUE sym1, VALUE sym2)
3671 {
3672     REWIND_CFP({
3673         rb_alias_variable(SYM2ID(sym1), SYM2ID(sym2));
3674     });
3675     return Qnil;
3676 }
3677
3678 static VALUE
3679 m_core_undef_method(VALUE self, VALUE cbase, VALUE sym)
3680 {
3681     REWIND_CFP({
3682         ID mid = SYM2ID(sym);
3683         rb_undef(cbase, mid);
3684         rb_clear_method_cache(self, mid);
3685     });
3686     return Qnil;
3687 }
3688
3689 static VALUE
3690 m_core_set_postexe(VALUE self)
3691 {
3692     rb_set_end_proc(rb_call_end_proc, rb_block_proc());
3693     return Qnil;
3694 }
3695
3696 static VALUE core_hash_merge_kwd(VALUE hash, VALUE kw);
3697
3698 static VALUE
3699 core_hash_merge(VALUE hash, long argc, const VALUE *argv)
3700 {
3701     Check_Type(hash, T_HASH);
3702     VM_ASSERT(argc % 2 == 0);
3703     rb_hash_bulk_insert(argc, argv, hash);
3704     return hash;
3705 }
3706
3707 static VALUE
3708 m_core_hash_merge_ptr(int argc, VALUE *argv, VALUE recv)
3709 {
3710     VALUE hash = argv[0];
3711
3712     REWIND_CFP(hash = core_hash_merge(hash, argc-1, argv+1));
3713
3714     return hash;
3715 }
3716
3717 static int
3718 kwmerge_i(VALUE key, VALUE value, VALUE hash)
3719 {
3720     rb_hash_aset(hash, key, value);
3721     return ST_CONTINUE;
3722 }
3723
3724 static VALUE
3725 m_core_hash_merge_kwd(VALUE recv, VALUE hash, VALUE kw)
3726 {
3727     if (!NIL_P(kw)) {
3728         REWIND_CFP(hash = core_hash_merge_kwd(hash, kw));
3729     }
3730     return hash;
3731 }
3732
3733 static VALUE
3734 m_core_make_shareable(VALUE recv, VALUE obj)
3735 {
3736     return rb_ractor_make_shareable(obj);
3737 }
3738
3739 static VALUE
3740 m_core_make_shareable_copy(VALUE recv, VALUE obj)
3741 {
3742     return rb_ractor_make_shareable_copy(obj);
3743 }
3744
3745 static VALUE
3746 m_core_ensure_shareable(VALUE recv, VALUE obj, VALUE name)
3747 {
3748     return rb_ractor_ensure_shareable(obj, name);
3749 }
3750
3751 static VALUE
3752 core_hash_merge_kwd(VALUE hash, VALUE kw)
3753 {
3754     rb_hash_foreach(rb_to_hash_type(kw), kwmerge_i, hash);
3755     return hash;
3756 }
3757
3758 extern VALUE *rb_gc_stack_start;
3759 extern size_t rb_gc_stack_maxsize;
3760
3761 /* debug functions */
3762
3763 /* :nodoc: */
3764 static VALUE
3765 sdr(VALUE self)
3766 {
3767     rb_vm_bugreport(NULL, stderr);
3768     return Qnil;
3769 }
3770
3771 /* :nodoc: */
3772 static VALUE
3773 nsdr(VALUE self)
3774 {
3775     VALUE ary = rb_ary_new();
3776 #ifdef HAVE_BACKTRACE
3777 #include <execinfo.h>
3778 #define MAX_NATIVE_TRACE 1024
3779     static void *trace[MAX_NATIVE_TRACE];
3780     int n = (int)backtrace(trace, MAX_NATIVE_TRACE);
3781     char **syms = backtrace_symbols(trace, n);
3782     int i;
3783
3784     if (syms == 0) {
3785         rb_memerror();
3786     }
3787
3788     for (i=0; i<n; i++) {
3789         rb_ary_push(ary, rb_str_new2(syms[i]));
3790     }
3791     free(syms); /* OK */
3792 #endif
3793     return ary;
3794 }
3795
3796 #if VM_COLLECT_USAGE_DETAILS
3797 static VALUE usage_analysis_insn_start(VALUE self);
3798 static VALUE usage_analysis_operand_start(VALUE self);
3799 static VALUE usage_analysis_register_start(VALUE self);
3800 static VALUE usage_analysis_insn_stop(VALUE self);
3801 static VALUE usage_analysis_operand_stop(VALUE self);
3802 static VALUE usage_analysis_register_stop(VALUE self);
3803 static VALUE usage_analysis_insn_running(VALUE self);
3804 static VALUE usage_analysis_operand_running(VALUE self);
3805 static VALUE usage_analysis_register_running(VALUE self);
3806 static VALUE usage_analysis_insn_clear(VALUE self);
3807 static VALUE usage_analysis_operand_clear(VALUE self);
3808 static VALUE usage_analysis_register_clear(VALUE self);
3809 #endif
3810
3811 static VALUE
3812 f_raise(int c, VALUE *v, VALUE _)
3813 {
3814     return rb_f_raise(c, v);
3815 }
3816
3817 static VALUE
3818 f_proc(VALUE _)
3819 {
3820     return rb_block_proc();
3821 }
3822
3823 static VALUE
3824 f_lambda(VALUE _)
3825 {
3826     return rb_block_lambda();
3827 }
3828
3829 static VALUE
3830 f_sprintf(int c, const VALUE *v, VALUE _)
3831 {
3832     return rb_f_sprintf(c, v);
3833 }
3834
3835 /* :nodoc: */
3836 static VALUE
3837 vm_mtbl(VALUE self, VALUE obj, VALUE sym)
3838 {
3839     vm_mtbl_dump(CLASS_OF(obj), RTEST(sym) ? SYM2ID(sym) : 0);
3840     return Qnil;
3841 }
3842
3843 /* :nodoc: */
3844 static VALUE
3845 vm_mtbl2(VALUE self, VALUE obj, VALUE sym)
3846 {
3847     vm_mtbl_dump(obj, RTEST(sym) ? SYM2ID(sym) : 0);
3848     return Qnil;
3849 }
3850
3851 /*
3852  *  call-seq:
3853  *     RubyVM.keep_script_lines -> true or false
3854  *
3855  *  Return current +keep_script_lines+ status. Now it only returns
3856  *  +true+ of +false+, but it can return other objects in future.
3857  *
3858  *  Note that this is an API for ruby internal use, debugging,
3859  *  and research. Do not use this for any other purpose.
3860  *  The compatibility is not guaranteed.
3861  */
3862 static VALUE
3863 vm_keep_script_lines(VALUE self)
3864 {
3865     return RBOOL(ruby_vm_keep_script_lines);
3866 }
3867
3868 /*
3869  *  call-seq:
3870  *     RubyVM.keep_script_lines = true / false
3871  *
3872  *  It set +keep_script_lines+ flag. If the flag is set, all
3873  *  loaded scripts are recorded in a interpreter process.
3874  *
3875  *  Note that this is an API for ruby internal use, debugging,
3876  *  and research. Do not use this for any other purpose.
3877  *  The compatibility is not guaranteed.
3878  */
3879 static VALUE
3880 vm_keep_script_lines_set(VALUE self, VALUE flags)
3881 {
3882     ruby_vm_keep_script_lines = RTEST(flags);
3883     return flags;
3884 }
3885
3886 void
3887 Init_VM(void)
3888 {
3889     VALUE opts;
3890     VALUE klass;
3891     VALUE fcore;
3892
3893     /*
3894      * Document-class: RubyVM
3895      *
3896      * The RubyVM module only exists on MRI. +RubyVM+ is not defined in
3897      * other Ruby implementations such as JRuby and TruffleRuby.
3898      *
3899      * The RubyVM module provides some access to MRI internals.
3900      * This module is for very limited purposes, such as debugging,
3901      * prototyping, and research.  Normal users must not use it.
3902      * This module is not portable between Ruby implementations.
3903      */
3904     rb_cRubyVM = rb_define_class("RubyVM", rb_cObject);
3905     rb_undef_alloc_func(rb_cRubyVM);
3906     rb_undef_method(CLASS_OF(rb_cRubyVM), "new");
3907     rb_define_singleton_method(rb_cRubyVM, "stat", vm_stat, -1);
3908     rb_define_singleton_method(rb_cRubyVM, "keep_script_lines", vm_keep_script_lines, 0);
3909     rb_define_singleton_method(rb_cRubyVM, "keep_script_lines=", vm_keep_script_lines_set, 1);
3910
3911 #if USE_DEBUG_COUNTER
3912     rb_define_singleton_method(rb_cRubyVM, "reset_debug_counters", rb_debug_counter_reset, 0);
3913     rb_define_singleton_method(rb_cRubyVM, "show_debug_counters", rb_debug_counter_show, 0);
3914 #endif
3915
3916     /* FrozenCore (hidden) */
3917     fcore = rb_class_new(rb_cBasicObject);
3918     rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore");
3919     rb_vm_register_global_object(rb_class_path_cached(fcore));
3920     RBASIC(fcore)->flags = T_ICLASS;
3921     klass = rb_singleton_class(fcore);
3922     rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3);
3923     rb_define_method_id(klass, id_core_set_variable_alias, m_core_set_variable_alias, 2);
3924     rb_define_method_id(klass, id_core_undef_method, m_core_undef_method, 2);
3925     rb_define_method_id(klass, id_core_set_postexe, m_core_set_postexe, 0);
3926     rb_define_method_id(klass, id_core_hash_merge_ptr, m_core_hash_merge_ptr, -1);
3927     rb_define_method_id(klass, id_core_hash_merge_kwd, m_core_hash_merge_kwd, 2);
3928     rb_define_method_id(klass, id_core_raise, f_raise, -1);
3929     rb_define_method_id(klass, id_core_sprintf, f_sprintf, -1);
3930     rb_define_method_id(klass, idProc, f_proc, 0);
3931     rb_define_method_id(klass, idLambda, f_lambda, 0);
3932     rb_define_method(klass, "make_shareable", m_core_make_shareable, 1);
3933     rb_define_method(klass, "make_shareable_copy", m_core_make_shareable_copy, 1);
3934     rb_define_method(klass, "ensure_shareable", m_core_ensure_shareable, 2);
3935     rb_obj_freeze(fcore);
3936     RBASIC_CLEAR_CLASS(klass);
3937     rb_obj_freeze(klass);
3938     rb_vm_register_global_object(fcore);
3939     rb_mRubyVMFrozenCore = fcore;
3940
3941     /*
3942      * Document-class: Thread
3943      *
3944      *  Threads are the Ruby implementation for a concurrent programming model.
3945      *
3946      *  Programs that require multiple threads of execution are a perfect
3947      *  candidate for Ruby's Thread class.
3948      *
3949      *  For example, we can create a new thread separate from the main thread's
3950      *  execution using ::new.
3951      *
3952      *      thr = Thread.new { puts "What's the big deal" }
3953      *
3954      *  Then we are able to pause the execution of the main thread and allow
3955      *  our new thread to finish, using #join:
3956      *
3957      *      thr.join #=> "What's the big deal"
3958      *
3959      *  If we don't call +thr.join+ before the main thread terminates, then all
3960      *  other threads including +thr+ will be killed.
3961      *
3962      *  Alternatively, you can use an array for handling multiple threads at
3963      *  once, like in the following example:
3964      *
3965      *      threads = []
3966      *      threads << Thread.new { puts "What's the big deal" }
3967      *      threads << Thread.new { 3.times { puts "Threads are fun!" } }
3968      *
3969      *  After creating a few threads we wait for them all to finish
3970      *  consecutively.
3971      *
3972      *      threads.each { |thr| thr.join }
3973      *
3974      *  To retrieve the last value of a thread, use #value
3975      *
3976      *      thr = Thread.new { sleep 1; "Useful value" }
3977      *      thr.value #=> "Useful value"
3978      *
3979      *  === Thread initialization
3980      *
3981      *  In order to create new threads, Ruby provides ::new, ::start, and
3982      *  ::fork. A block must be provided with each of these methods, otherwise
3983      *  a ThreadError will be raised.
3984      *
3985      *  When subclassing the Thread class, the +initialize+ method of your
3986      *  subclass will be ignored by ::start and ::fork. Otherwise, be sure to
3987      *  call super in your +initialize+ method.
3988      *
3989      *  === Thread termination
3990      *
3991      *  For terminating threads, Ruby provides a variety of ways to do this.
3992      *
3993      *  The class method ::kill, is meant to exit a given thread:
3994      *
3995      *      thr = Thread.new { sleep }
3996      *      Thread.kill(thr) # sends exit() to thr
3997      *
3998      *  Alternatively, you can use the instance method #exit, or any of its
3999      *  aliases #kill or #terminate.
4000      *
4001      *      thr.exit
4002      *
4003      *  === Thread status
4004      *
4005      *  Ruby provides a few instance methods for querying the state of a given
4006      *  thread. To get a string with the current thread's state use #status
4007      *
4008      *      thr = Thread.new { sleep }
4009      *      thr.status # => "sleep"
4010      *      thr.exit
4011      *      thr.status # => false
4012      *
4013      *  You can also use #alive? to tell if the thread is running or sleeping,
4014      *  and #stop? if the thread is dead or sleeping.
4015      *
4016      *  === Thread variables and scope
4017      *
4018      *  Since threads are created with blocks, the same rules apply to other
4019      *  Ruby blocks for variable scope. Any local variables created within this
4020      *  block are accessible to only this thread.
4021      *
4022      *  ==== Fiber-local vs. Thread-local
4023      *
4024      *  Each fiber has its own bucket for Thread#[] storage. When you set a
4025      *  new fiber-local it is only accessible within this Fiber. To illustrate:
4026      *
4027      *      Thread.new {
4028      *        Thread.current[:foo] = "bar"
4029      *        Fiber.new {
4030      *          p Thread.current[:foo] # => nil
4031      *        }.resume
4032      *      }.join
4033      *
4034      *  This example uses #[] for getting and #[]= for setting fiber-locals,
4035      *  you can also use #keys to list the fiber-locals for a given
4036      *  thread and #key? to check if a fiber-local exists.
4037      *
4038      *  When it comes to thread-locals, they are accessible within the entire
4039      *  scope of the thread. Given the following example:
4040      *
4041      *      Thread.new{
4042      *        Thread.current.thread_variable_set(:foo, 1)
4043      *        p Thread.current.thread_variable_get(:foo) # => 1
4044      *        Fiber.new{
4045      *          Thread.current.thread_variable_set(:foo, 2)
4046      *          p Thread.current.thread_variable_get(:foo) # => 2
4047      *        }.resume
4048      *        p Thread.current.thread_variable_get(:foo)   # => 2
4049      *      }.join
4050      *
4051      *  You can see that the thread-local +:foo+ carried over into the fiber
4052      *  and was changed to +2+ by the end of the thread.
4053      *
4054      *  This example makes use of #thread_variable_set to create new
4055      *  thread-locals, and #thread_variable_get to reference them.
4056      *
4057      *  There is also #thread_variables to list all thread-locals, and
4058      *  #thread_variable? to check if a given thread-local exists.
4059      *
4060      *  === Exception handling
4061      *
4062      *  When an unhandled exception is raised inside a thread, it will
4063      *  terminate. By default, this exception will not propagate to other
4064      *  threads. The exception is stored and when another thread calls #value
4065      *  or #join, the exception will be re-raised in that thread.
4066      *
4067      *      t = Thread.new{ raise 'something went wrong' }
4068      *      t.value #=> RuntimeError: something went wrong
4069      *
4070      *  An exception can be raised from outside the thread using the
4071      *  Thread#raise instance method, which takes the same parameters as
4072      *  Kernel#raise.
4073      *
4074      *  Setting Thread.abort_on_exception = true, Thread#abort_on_exception =
4075      *  true, or $DEBUG = true will cause a subsequent unhandled exception
4076      *  raised in a thread to be automatically re-raised in the main thread.
4077      *
4078      *  With the addition of the class method ::handle_interrupt, you can now
4079      *  handle exceptions asynchronously with threads.
4080      *
4081      *  === Scheduling
4082      *
4083      *  Ruby provides a few ways to support scheduling threads in your program.
4084      *
4085      *  The first way is by using the class method ::stop, to put the current
4086      *  running thread to sleep and schedule the execution of another thread.
4087      *
4088      *  Once a thread is asleep, you can use the instance method #wakeup to
4089      *  mark your thread as eligible for scheduling.
4090      *
4091      *  You can also try ::pass, which attempts to pass execution to another
4092      *  thread but is dependent on the OS whether a running thread will switch
4093      *  or not. The same goes for #priority, which lets you hint to the thread
4094      *  scheduler which threads you want to take precedence when passing
4095      *  execution. This method is also dependent on the OS and may be ignored
4096      *  on some platforms.
4097      *
4098      */
4099     rb_cThread = rb_define_class("Thread", rb_cObject);
4100     rb_undef_alloc_func(rb_cThread);
4101
4102 #if VM_COLLECT_USAGE_DETAILS
4103     /* ::RubyVM::USAGE_ANALYSIS_* */
4104 #define define_usage_analysis_hash(name) /* shut up rdoc -C */ \
4105     rb_define_const(rb_cRubyVM, "USAGE_ANALYSIS_" #name, rb_hash_new())
4106     define_usage_analysis_hash(INSN);
4107     define_usage_analysis_hash(REGS);
4108     define_usage_analysis_hash(INSN_BIGRAM);
4109
4110     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_START", usage_analysis_insn_start, 0);
4111     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_START", usage_analysis_operand_start, 0);
4112     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_START", usage_analysis_register_start, 0);
4113     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_STOP", usage_analysis_insn_stop, 0);
4114     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_STOP", usage_analysis_operand_stop, 0);
4115     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_STOP", usage_analysis_register_stop, 0);
4116     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_RUNNING", usage_analysis_insn_running, 0);
4117     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_RUNNING", usage_analysis_operand_running, 0);
4118     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_RUNNING", usage_analysis_register_running, 0);
4119     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_CLEAR", usage_analysis_insn_clear, 0);
4120     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_CLEAR", usage_analysis_operand_clear, 0);
4121     rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_CLEAR", usage_analysis_register_clear, 0);
4122 #endif
4123
4124     /* ::RubyVM::OPTS
4125      * An Array of VM build options.
4126      * This constant is MRI specific.
4127      */
4128     rb_define_const(rb_cRubyVM, "OPTS", opts = rb_ary_new());
4129
4130 #if   OPT_DIRECT_THREADED_CODE
4131     rb_ary_push(opts, rb_str_new2("direct threaded code"));
4132 #elif OPT_TOKEN_THREADED_CODE
4133     rb_ary_push(opts, rb_str_new2("token threaded code"));
4134 #elif OPT_CALL_THREADED_CODE
4135     rb_ary_push(opts, rb_str_new2("call threaded code"));
4136 #endif
4137
4138 #if OPT_OPERANDS_UNIFICATION
4139     rb_ary_push(opts, rb_str_new2("operands unification"));
4140 #endif
4141 #if OPT_INSTRUCTIONS_UNIFICATION
4142     rb_ary_push(opts, rb_str_new2("instructions unification"));
4143 #endif
4144 #if OPT_INLINE_METHOD_CACHE
4145     rb_ary_push(opts, rb_str_new2("inline method cache"));
4146 #endif
4147
4148     /* ::RubyVM::INSTRUCTION_NAMES
4149      * A list of bytecode instruction names in MRI.
4150      * This constant is MRI specific.
4151      */
4152     rb_define_const(rb_cRubyVM, "INSTRUCTION_NAMES", rb_insns_name_array());
4153
4154     /* ::RubyVM::DEFAULT_PARAMS
4155      * This constant exposes the VM's default parameters.
4156      * Note that changing these values does not affect VM execution.
4157      * Specification is not stable and you should not depend on this value.
4158      * Of course, this constant is MRI specific.
4159      */
4160     rb_define_const(rb_cRubyVM, "DEFAULT_PARAMS", vm_default_params());
4161
4162     /* debug functions ::RubyVM::SDR(), ::RubyVM::NSDR() */
4163 #if VMDEBUG
4164     rb_define_singleton_method(rb_cRubyVM, "SDR", sdr, 0);
4165     rb_define_singleton_method(rb_cRubyVM, "NSDR", nsdr, 0);
4166     rb_define_singleton_method(rb_cRubyVM, "mtbl", vm_mtbl, 2);
4167     rb_define_singleton_method(rb_cRubyVM, "mtbl2", vm_mtbl2, 2);
4168 #else
4169     (void)sdr;
4170     (void)nsdr;
4171     (void)vm_mtbl;
4172     (void)vm_mtbl2;
4173 #endif
4174
4175     /* VM bootstrap: phase 2 */
4176     {
4177         rb_vm_t *vm = ruby_current_vm_ptr;
4178         rb_thread_t *th = GET_THREAD();
4179         VALUE filename = rb_fstring_lit("<main>");
4180         const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP);
4181
4182         // Ractor setup
4183         rb_ractor_main_setup(vm, th->ractor, th);
4184
4185         /* create vm object */
4186         vm->self = TypedData_Wrap_Struct(rb_cRubyVM, &vm_data_type, vm);
4187
4188         /* create main thread */
4189         th->self = TypedData_Wrap_Struct(rb_cThread, &thread_data_type, th);
4190         vm->ractor.main_thread = th;
4191         vm->ractor.main_ractor = th->ractor;
4192         th->vm = vm;
4193         th->top_wrapper = 0;
4194         th->top_self = rb_vm_top_self();
4195
4196         rb_vm_register_global_object((VALUE)iseq);
4197         th->ec->cfp->iseq = iseq;
4198         th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded;
4199         th->ec->cfp->self = th->top_self;
4200
4201         VM_ENV_FLAGS_UNSET(th->ec->cfp->ep, VM_FRAME_FLAG_CFRAME);
4202         VM_STACK_ENV_WRITE(th->ec->cfp->ep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE, FALSE, NULL, FALSE, FALSE));
4203
4204         /*
4205          * The Binding of the top level scope
4206          */
4207         rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new());
4208
4209 #ifdef _WIN32
4210         rb_objspace_gc_enable(vm->objspace);
4211 #endif
4212     }
4213     vm_init_redefined_flag();
4214
4215     rb_block_param_proxy = rb_obj_alloc(rb_cObject);
4216     rb_add_method_optimized(rb_singleton_class(rb_block_param_proxy), idCall,
4217                             OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC);
4218     rb_obj_freeze(rb_block_param_proxy);
4219     rb_vm_register_global_object(rb_block_param_proxy);
4220
4221     /* vm_backtrace.c */
4222     Init_vm_backtrace();
4223 }
4224
4225 void
4226 rb_vm_set_progname(VALUE filename)
4227 {
4228     rb_thread_t *th = GET_VM()->ractor.main_thread;
4229     rb_control_frame_t *cfp = (void *)(th->ec->vm_stack + th->ec->vm_stack_size);
4230     --cfp;
4231
4232     filename = rb_str_new_frozen(filename);
4233     rb_iseq_pathobj_set(cfp->iseq, filename, rb_iseq_realpath(cfp->iseq));
4234 }
4235
4236 extern const struct st_hash_type rb_fstring_hash_type;
4237
4238 void
4239 Init_BareVM(void)
4240 {
4241     /* VM bootstrap: phase 1 */
4242     rb_vm_t *vm = ruby_mimcalloc(1, sizeof(*vm));
4243     rb_thread_t *th = ruby_mimcalloc(1, sizeof(*th));
4244     if (!vm || !th) {
4245         fputs("[FATAL] failed to allocate memory\n", stderr);
4246         exit(EXIT_FAILURE);
4247     }
4248
4249     // setup the VM
4250     vm_init2(vm);
4251
4252     rb_vm_postponed_job_queue_init(vm);
4253     ruby_current_vm_ptr = vm;
4254     rb_objspace_alloc();
4255     vm->negative_cme_table = rb_id_table_create(16);
4256     vm->overloaded_cme_table = st_init_numtable();
4257     vm->constant_cache = rb_id_table_create(0);
4258     vm->unused_block_warning_table = st_init_numtable();
4259
4260     // TODO: remove before Ruby 3.4.0 release
4261     const char *s = getenv("RUBY_TRY_UNUSED_BLOCK_WARNING_STRICT");
4262     if (s && strcmp(s, "1") == 0) {
4263         vm->unused_block_warning_strict = true;
4264     }
4265
4266     // setup main thread
4267     th->nt = ZALLOC(struct rb_native_thread);
4268     th->vm = vm;
4269     th->ractor = vm->ractor.main_ractor = rb_ractor_main_alloc();
4270     Init_native_thread(th);
4271     rb_jit_cont_init();
4272     th_init(th, 0, vm);
4273
4274     rb_ractor_set_current_ec(th->ractor, th->ec);
4275     /* n.b. native_main_thread_stack_top is set by the INIT_STACK macro */
4276     ruby_thread_init_stack(th, native_main_thread_stack_top);
4277
4278     // setup ractor system
4279     rb_native_mutex_initialize(&vm->ractor.sync.lock);
4280     rb_native_cond_initialize(&vm->ractor.sync.terminate_cond);
4281
4282     vm_opt_method_def_table = st_init_numtable();
4283     vm_opt_mid_table = st_init_numtable();
4284
4285 #ifdef RUBY_THREAD_WIN32_H
4286     rb_native_cond_initialize(&vm->ractor.sync.barrier_cond);
4287 #endif
4288 }
4289
4290 void
4291 ruby_init_stack(void *addr)
4292 {
4293     native_main_thread_stack_top = addr;
4294 }
4295
4296 #ifndef _WIN32
4297 #include <unistd.h>
4298 #include <sys/mman.h>
4299 #endif
4300
4301
4302 #ifndef MARK_OBJECT_ARY_BUCKET_SIZE
4303 #define MARK_OBJECT_ARY_BUCKET_SIZE 1024
4304 #endif
4305
4306 struct pin_array_list {
4307     VALUE next;
4308     long len;
4309     VALUE *array;
4310 };
4311
4312 static void
4313 pin_array_list_mark(void *data)
4314 {
4315     struct pin_array_list *array = (struct pin_array_list *)data;
4316     rb_gc_mark_movable(array->next);
4317
4318     rb_gc_mark_vm_stack_values(array->len, array->array);
4319 }
4320
4321 static void
4322 pin_array_list_free(void *data)
4323 {
4324     struct pin_array_list *array = (struct pin_array_list *)data;
4325     xfree(array->array);
4326 }
4327
4328 static size_t
4329 pin_array_list_memsize(const void *data)
4330 {
4331     return sizeof(struct pin_array_list) + (MARK_OBJECT_ARY_BUCKET_SIZE * sizeof(VALUE));
4332 }
4333
4334 static void
4335 pin_array_list_update_references(void *data)
4336 {
4337     struct pin_array_list *array = (struct pin_array_list *)data;
4338     array->next = rb_gc_location(array->next);
4339 }
4340
4341 static const rb_data_type_t pin_array_list_type = {
4342     .wrap_struct_name = "VM/pin_array_list",
4343     .function = {
4344         .dmark = pin_array_list_mark,
4345         .dfree = pin_array_list_free,
4346         .dsize = pin_array_list_memsize,
4347         .dcompact = pin_array_list_update_references,
4348     },
4349     .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
4350 };
4351
4352 static VALUE
4353 pin_array_list_new(VALUE next)
4354 {
4355     struct pin_array_list *array_list;
4356     VALUE obj = TypedData_Make_Struct(0, struct pin_array_list, &pin_array_list_type, array_list);
4357     RB_OBJ_WRITE(obj, &array_list->next, next);
4358     array_list->array = ALLOC_N(VALUE, MARK_OBJECT_ARY_BUCKET_SIZE);
4359     return obj;
4360 }
4361
4362 static VALUE
4363 pin_array_list_append(VALUE obj, VALUE item)
4364 {
4365     struct pin_array_list *array_list;
4366     TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list);
4367
4368     if (array_list->len >= MARK_OBJECT_ARY_BUCKET_SIZE) {
4369         obj = pin_array_list_new(obj);
4370         TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list);
4371     }
4372
4373     RB_OBJ_WRITE(obj, &array_list->array[array_list->len], item);
4374     array_list->len++;
4375     return obj;
4376 }
4377
4378 void
4379 rb_vm_register_global_object(VALUE obj)
4380 {
4381     RUBY_ASSERT(!RB_SPECIAL_CONST_P(obj));
4382     if (RB_SPECIAL_CONST_P(obj)) {
4383         return;
4384     }
4385
4386     switch (RB_BUILTIN_TYPE(obj)) {
4387       case T_CLASS:
4388       case T_MODULE:
4389         if (FL_TEST(obj, RCLASS_IS_ROOT)) {
4390             return;
4391         }
4392         FL_SET(obj, RCLASS_IS_ROOT);
4393         break;
4394       default:
4395         break;
4396     }
4397     RB_VM_LOCK_ENTER();
4398     {
4399         VALUE list = GET_VM()->mark_object_ary;
4400         VALUE head = pin_array_list_append(list, obj);
4401         if (head != list) {
4402             GET_VM()->mark_object_ary = head;
4403         }
4404         RB_GC_GUARD(obj);
4405     }
4406     RB_VM_LOCK_LEAVE();
4407 }
4408
4409 void
4410 Init_vm_objects(void)
4411 {
4412     rb_vm_t *vm = GET_VM();
4413
4414     /* initialize mark object array, hash */
4415     vm->mark_object_ary = pin_array_list_new(Qnil);
4416     vm->loading_table = st_init_strtable();
4417     vm->ci_table = st_init_table(&vm_ci_hashtype);
4418     vm->frozen_strings = st_init_table_with_size(&rb_fstring_hash_type, 10000);
4419 }
4420
4421 /* Stub for builtin function when not building YJIT units*/
4422 #if !USE_YJIT
4423 void Init_builtin_yjit(void) {}
4424 #endif
4425
4426 /* top self */
4427
4428 static VALUE
4429 main_to_s(VALUE obj)
4430 {
4431     return rb_str_new2("main");
4432 }
4433
4434 VALUE
4435 rb_vm_top_self(void)
4436 {
4437     return GET_VM()->top_self;
4438 }
4439
4440 void
4441 Init_top_self(void)
4442 {
4443     rb_vm_t *vm = GET_VM();
4444
4445     vm->top_self = rb_obj_alloc(rb_cObject);
4446     rb_define_singleton_method(rb_vm_top_self(), "to_s", main_to_s, 0);
4447     rb_define_alias(rb_singleton_class(rb_vm_top_self()), "inspect", "to_s");
4448 }
4449
4450 VALUE *
4451 rb_ruby_verbose_ptr(void)
4452 {
4453     rb_ractor_t *cr = GET_RACTOR();
4454     return &cr->verbose;
4455 }
4456
4457 static bool prism;
4458
4459 bool *
4460 rb_ruby_prism_ptr(void)
4461 {
4462     return &prism;
4463 }
4464
4465 VALUE *
4466 rb_ruby_debug_ptr(void)
4467 {
4468     rb_ractor_t *cr = GET_RACTOR();
4469     return &cr->debug;
4470 }
4471
4472 bool rb_free_at_exit = false;
4473
4474 bool
4475 ruby_free_at_exit_p(void)
4476 {
4477     return rb_free_at_exit;
4478 }
4479
4480 /* iseq.c */
4481 VALUE rb_insn_operand_intern(const rb_iseq_t *iseq,
4482                              VALUE insn, int op_no, VALUE op,
4483                              int len, size_t pos, VALUE *pnop, VALUE child);
4484
4485 st_table *
4486 rb_vm_fstring_table(void)
4487 {
4488     return GET_VM()->frozen_strings;
4489 }
4490
4491 #if VM_COLLECT_USAGE_DETAILS
4492
4493 #define HASH_ASET(h, k, v) rb_hash_aset((h), (st_data_t)(k), (st_data_t)(v))
4494
4495 /* uh = {
4496  *   insn(Fixnum) => ihash(Hash)
4497  * }
4498  * ihash = {
4499  *   -1(Fixnum) => count,      # insn usage
4500  *    0(Fixnum) => ophash,     # operand usage
4501  * }
4502  * ophash = {
4503  *   val(interned string) => count(Fixnum)
4504  * }
4505  */
4506 static void
4507 vm_analysis_insn(int insn)
4508 {
4509     ID usage_hash;
4510     ID bigram_hash;
4511     static int prev_insn = -1;
4512
4513     VALUE uh;
4514     VALUE ihash;
4515     VALUE cv;
4516
4517     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4518     CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM");
4519     uh = rb_const_get(rb_cRubyVM, usage_hash);
4520     if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) {
4521         ihash = rb_hash_new();
4522         HASH_ASET(uh, INT2FIX(insn), ihash);
4523     }
4524     if (NIL_P(cv = rb_hash_aref(ihash, INT2FIX(-1)))) {
4525         cv = INT2FIX(0);
4526     }
4527     HASH_ASET(ihash, INT2FIX(-1), INT2FIX(FIX2INT(cv) + 1));
4528
4529     /* calc bigram */
4530     if (prev_insn != -1) {
4531         VALUE bi;
4532         VALUE ary[2];
4533         VALUE cv;
4534
4535         ary[0] = INT2FIX(prev_insn);
4536         ary[1] = INT2FIX(insn);
4537         bi = rb_ary_new4(2, &ary[0]);
4538
4539         uh = rb_const_get(rb_cRubyVM, bigram_hash);
4540         if (NIL_P(cv = rb_hash_aref(uh, bi))) {
4541             cv = INT2FIX(0);
4542         }
4543         HASH_ASET(uh, bi, INT2FIX(FIX2INT(cv) + 1));
4544     }
4545     prev_insn = insn;
4546 }
4547
4548 static void
4549 vm_analysis_operand(int insn, int n, VALUE op)
4550 {
4551     ID usage_hash;
4552
4553     VALUE uh;
4554     VALUE ihash;
4555     VALUE ophash;
4556     VALUE valstr;
4557     VALUE cv;
4558
4559     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4560
4561     uh = rb_const_get(rb_cRubyVM, usage_hash);
4562     if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) {
4563         ihash = rb_hash_new();
4564         HASH_ASET(uh, INT2FIX(insn), ihash);
4565     }
4566     if (NIL_P(ophash = rb_hash_aref(ihash, INT2FIX(n)))) {
4567         ophash = rb_hash_new();
4568         HASH_ASET(ihash, INT2FIX(n), ophash);
4569     }
4570     /* intern */
4571     valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0);
4572
4573     /* set count */
4574     if (NIL_P(cv = rb_hash_aref(ophash, valstr))) {
4575         cv = INT2FIX(0);
4576     }
4577     HASH_ASET(ophash, valstr, INT2FIX(FIX2INT(cv) + 1));
4578 }
4579
4580 static void
4581 vm_analysis_register(int reg, int isset)
4582 {
4583     ID usage_hash;
4584     VALUE uh;
4585     VALUE valstr;
4586     static const char regstrs[][5] = {
4587         "pc",                   /* 0 */
4588         "sp",                   /* 1 */
4589         "ep",                   /* 2 */
4590         "cfp",                  /* 3 */
4591         "self",                 /* 4 */
4592         "iseq",                 /* 5 */
4593     };
4594     static const char getsetstr[][4] = {
4595         "get",
4596         "set",
4597     };
4598     static VALUE syms[sizeof(regstrs) / sizeof(regstrs[0])][2];
4599
4600     VALUE cv;
4601
4602     CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS");
4603     if (syms[0] == 0) {
4604         char buff[0x10];
4605         int i;
4606
4607         for (i = 0; i < (int)(sizeof(regstrs) / sizeof(regstrs[0])); i++) {
4608             int j;
4609             for (j = 0; j < 2; j++) {
4610                 snprintf(buff, 0x10, "%d %s %-4s", i, getsetstr[j], regstrs[i]);
4611                 syms[i][j] = ID2SYM(rb_intern(buff));
4612             }
4613         }
4614     }
4615     valstr = syms[reg][isset];
4616
4617     uh = rb_const_get(rb_cRubyVM, usage_hash);
4618     if (NIL_P(cv = rb_hash_aref(uh, valstr))) {
4619         cv = INT2FIX(0);
4620     }
4621     HASH_ASET(uh, valstr, INT2FIX(FIX2INT(cv) + 1));
4622 }
4623
4624 #undef HASH_ASET
4625
4626 static void (*ruby_vm_collect_usage_func_insn)(int insn) = NULL;
4627 static void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op) = NULL;
4628 static void (*ruby_vm_collect_usage_func_register)(int reg, int isset) = NULL;
4629
4630 /* :nodoc: */
4631 static VALUE
4632 usage_analysis_insn_start(VALUE self)
4633 {
4634     ruby_vm_collect_usage_func_insn = vm_analysis_insn;
4635     return Qnil;
4636 }
4637
4638 /* :nodoc: */
4639 static VALUE
4640 usage_analysis_operand_start(VALUE self)
4641 {
4642     ruby_vm_collect_usage_func_operand = vm_analysis_operand;
4643     return Qnil;
4644 }
4645
4646 /* :nodoc: */
4647 static VALUE
4648 usage_analysis_register_start(VALUE self)
4649 {
4650     ruby_vm_collect_usage_func_register = vm_analysis_register;
4651     return Qnil;
4652 }
4653
4654 /* :nodoc: */
4655 static VALUE
4656 usage_analysis_insn_stop(VALUE self)
4657 {
4658     ruby_vm_collect_usage_func_insn = 0;
4659     return Qnil;
4660 }
4661
4662 /* :nodoc: */
4663 static VALUE
4664 usage_analysis_operand_stop(VALUE self)
4665 {
4666     ruby_vm_collect_usage_func_operand = 0;
4667     return Qnil;
4668 }
4669
4670 /* :nodoc: */
4671 static VALUE
4672 usage_analysis_register_stop(VALUE self)
4673 {
4674     ruby_vm_collect_usage_func_register = 0;
4675     return Qnil;
4676 }
4677
4678 /* :nodoc: */
4679 static VALUE
4680 usage_analysis_insn_running(VALUE self)
4681 {
4682     return RBOOL(ruby_vm_collect_usage_func_insn != 0);
4683 }
4684
4685 /* :nodoc: */
4686 static VALUE
4687 usage_analysis_operand_running(VALUE self)
4688 {
4689     return RBOOL(ruby_vm_collect_usage_func_operand != 0);
4690 }
4691
4692 /* :nodoc: */
4693 static VALUE
4694 usage_analysis_register_running(VALUE self)
4695 {
4696     return RBOOL(ruby_vm_collect_usage_func_register != 0);
4697 }
4698
4699 static VALUE
4700 usage_analysis_clear(VALUE self, ID usage_hash)
4701 {
4702     VALUE uh;
4703     uh = rb_const_get(self, usage_hash);
4704     rb_hash_clear(uh);
4705
4706     return Qtrue;
4707 }
4708
4709
4710 /* :nodoc: */
4711 static VALUE
4712 usage_analysis_insn_clear(VALUE self)
4713 {
4714     ID usage_hash;
4715     ID bigram_hash;
4716
4717     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4718     CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM");
4719     usage_analysis_clear(rb_cRubyVM, usage_hash);
4720     return usage_analysis_clear(rb_cRubyVM, bigram_hash);
4721 }
4722
4723 /* :nodoc: */
4724 static VALUE
4725 usage_analysis_operand_clear(VALUE self)
4726 {
4727     ID usage_hash;
4728
4729     CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
4730     return usage_analysis_clear(self, usage_hash);
4731 }
4732
4733 /* :nodoc: */
4734 static VALUE
4735 usage_analysis_register_clear(VALUE self)
4736 {
4737       ID usage_hash;
4738
4739     CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS");
4740     return usage_analysis_clear(self, usage_hash);
4741 }
4742
4743 #else
4744
4745 MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_insn)(int insn)) = 0;
4746 MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op)) = 0;
4747 MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_register)(int reg, int isset)) = 0;
4748
4749 #endif
4750
4751 #if VM_COLLECT_USAGE_DETAILS
4752 /* @param insn instruction number */
4753 static void
4754 vm_collect_usage_insn(int insn)
4755 {
4756     if (RUBY_DTRACE_INSN_ENABLED()) {
4757         RUBY_DTRACE_INSN(rb_insns_name(insn));
4758     }
4759     if (ruby_vm_collect_usage_func_insn)
4760         (*ruby_vm_collect_usage_func_insn)(insn);
4761 }
4762
4763 /* @param insn instruction number
4764  * @param n    n-th operand
4765  * @param op   operand value
4766  */
4767 static void
4768 vm_collect_usage_operand(int insn, int n, VALUE op)
4769 {
4770     if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) {
4771         VALUE valstr;
4772
4773         valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0);
4774
4775         RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn));
4776         RB_GC_GUARD(valstr);
4777     }
4778     if (ruby_vm_collect_usage_func_operand)
4779         (*ruby_vm_collect_usage_func_operand)(insn, n, op);
4780 }
4781
4782 /* @param reg register id. see code of vm_analysis_register() */
4783 /* @param isset 0: read, 1: write */
4784 static void
4785 vm_collect_usage_register(int reg, int isset)
4786 {
4787     if (ruby_vm_collect_usage_func_register)
4788         (*ruby_vm_collect_usage_func_register)(reg, isset);
4789 }
4790 #endif
4791
4792 const struct rb_callcache *
4793 rb_vm_empty_cc(void)
4794 {
4795     return &vm_empty_cc;
4796 }
4797
4798 const struct rb_callcache *
4799 rb_vm_empty_cc_for_super(void)
4800 {
4801     return &vm_empty_cc_for_super;
4802 }
4803
4804 #include "vm_call_iseq_optimized.inc" /* required from vm_insnhelper.c */