rjit.c

   1 /**********************************************************************
   2
   3   rjit.c - Ruby JIT compiler functions
   4
   5   Copyright (C) 2023 Takashi Kokubun <k0kubun@ruby-lang.org>.
   6
   7 **********************************************************************/
   8
   9 #include "rjit.h" // defines USE_RJIT
  10
  11 #if USE_RJIT
  12
  13 #include "constant.h"
  14 #include "id_table.h"
  15 #include "internal.h"
  16 #include "internal/class.h"
  17 #include "internal/cmdlineopt.h"
  18 #include "internal/cont.h"
  19 #include "internal/file.h"
  20 #include "internal/hash.h"
  21 #include "internal/process.h"
  22 #include "internal/warnings.h"
  23 #include "vm_sync.h"
  24 #include "ractor_core.h"
  25
  26 #ifdef __sun
  27 #define __EXTENSIONS__ 1
  28 #endif
  29
  30 #include "vm_core.h"
  31 #include "vm_callinfo.h"
  32 #include "rjit_c.h"
  33 #include "ruby_assert.h"
  34 #include "ruby/debug.h"
  35 #include "ruby/thread.h"
  36 #include "ruby/version.h"
  37 #include "builtin.h"
  38 #include "insns.inc"
  39 #include "insns_info.inc"
  40 #include "internal/compile.h"
  41 #include "internal/gc.h"
  42
  43 #include <sys/wait.h>
  44 #include <sys/time.h>
  45 #include <dlfcn.h>
  46 #include <errno.h>
  47 #ifdef HAVE_FCNTL_H
  48 #include <fcntl.h>
  49 #endif
  50 #ifdef HAVE_SYS_PARAM_H
  51 # include <sys/param.h>
  52 #endif
  53 #include "dln.h"
  54
  55 // For mmapp(), sysconf()
  56 #ifndef _WIN32
  57 #include <unistd.h>
  58 #include <sys/mman.h>
  59 #endif
  60
  61 #include "ruby/util.h"
  62
  63 // A copy of RJIT portion of MRI options since RJIT initialization.  We
  64 // need them as RJIT threads still can work when the most MRI data were
  65 // freed.
  66 struct rb_rjit_options rb_rjit_opts;
  67
  68 // true if RJIT is enabled.
  69 bool rb_rjit_enabled = false;
  70 // true if --rjit-stats (used before rb_rjit_opts is set)
  71 bool rb_rjit_stats_enabled = false;
  72 // true if --rjit-trace-exits (used before rb_rjit_opts is set)
  73 bool rb_rjit_trace_exits_enabled = false;
  74 // true if JIT-ed code should be called. When `ruby_vm_event_enabled_global_flags & ISEQ_TRACE_EVENTS`
  75 // and `rb_rjit_call_p == false`, any JIT-ed code execution is cancelled as soon as possible.
  76 bool rb_rjit_call_p = false;
  77 // A flag to communicate that rb_rjit_call_p should be disabled while it's temporarily false.
  78 static bool rjit_cancel_p = false;
  79
  80 // `rb_ec_ractor_hooks(ec)->events` is moved to this variable during compilation.
  81 rb_event_flag_t rb_rjit_global_events = 0;
  82
  83 // Basically rb_rjit_opts.stats, but this becomes false during RJIT compilation.
  84 static bool rjit_stats_p = false;
  85
  86 // RubyVM::RJIT
  87 static VALUE rb_mRJIT = 0;
  88 // RubyVM::RJIT::C
  89 static VALUE rb_mRJITC = 0;
  90 // RubyVM::RJIT::Compiler
  91 static VALUE rb_RJITCompiler = 0;
  92 // RubyVM::RJIT::CPointer::Struct_rb_iseq_t
  93 static VALUE rb_cRJITIseqPtr = 0;
  94 // RubyVM::RJIT::CPointer::Struct_rb_control_frame_t
  95 static VALUE rb_cRJITCfpPtr = 0;
  96 // RubyVM::RJIT::Hooks
  97 static VALUE rb_mRJITHooks = 0;
  98
  99 // Frames for --rjit-trace-exits
 100 VALUE rb_rjit_raw_samples = 0;
 101 // Line numbers for --rjit-trace-exits
 102 VALUE rb_rjit_line_samples = 0;
 103
 104 // Postponed job handle for triggering rjit_iseq_update_references
 105 static rb_postponed_job_handle_t rjit_iseq_update_references_pjob;
 106
 107 // A default threshold used to add iseq to JIT.
 108 #define DEFAULT_CALL_THRESHOLD 10
 109 // Size of executable memory block in MiB.
 110 #define DEFAULT_EXEC_MEM_SIZE 64
 111
 112 #define opt_match_noarg(s, l, name) \
 113     opt_match(s, l, name) && (*(s) ? (rb_warn("argument to --rjit-" name " is ignored"), 1) : 1)
 114 #define opt_match_arg(s, l, name) \
 115     opt_match(s, l, name) && (*(s) ? 1 : (rb_raise(rb_eRuntimeError, "--rjit-" name " needs an argument"), 0))
 116
 117 void
 118 rb_rjit_setup_options(const char *s, struct rb_rjit_options *rjit_opt)
 119 {
 120     const size_t l = strlen(s);
 121     if (l == 0) {
 122         return;
 123     }
 124     else if (opt_match_arg(s, l, "exec-mem-size")) {
 125         rjit_opt->exec_mem_size = atoi(s + 1);
 126     }
 127     else if (opt_match_arg(s, l, "call-threshold")) {
 128         rjit_opt->call_threshold = atoi(s + 1);
 129     }
 130     else if (opt_match_noarg(s, l, "stats")) {
 131         rjit_opt->stats = true;
 132     }
 133     else if (opt_match_noarg(s, l, "disable")) {
 134         rjit_opt->disable = true;
 135     }
 136     else if (opt_match_noarg(s, l, "trace")) {
 137         rjit_opt->trace = true;
 138     }
 139     else if (opt_match_noarg(s, l, "trace-exits")) {
 140         rjit_opt->trace_exits = true;
 141     }
 142     else if (opt_match_noarg(s, l, "dump-disasm")) {
 143         rjit_opt->dump_disasm = true;
 144     }
 145     else if (opt_match_noarg(s, l, "verify-ctx")) {
 146         rjit_opt->verify_ctx = true;
 147     }
 148     else {
 149         rb_raise(rb_eRuntimeError,
 150                  "invalid RJIT option '%s' (--help will show valid RJIT options)", s);
 151     }
 152 }
 153
 154 #define M(shortopt, longopt, desc) RUBY_OPT_MESSAGE(shortopt, longopt, desc)
 155 const struct ruby_opt_message rb_rjit_option_messages[] = {
 156     M("--rjit-exec-mem-size=num",  "", "Size of executable memory block in MiB (default: " STRINGIZE(DEFAULT_EXEC_MEM_SIZE) ")."),
 157     M("--rjit-call-threshold=num", "", "Number of calls to trigger JIT (default: " STRINGIZE(DEFAULT_CALL_THRESHOLD) ")."),
 158     M("--rjit-stats",              "", "Enable collecting RJIT statistics."),
 159     M("--rjit-disable",            "", "Disable RJIT for lazily enabling it with RubyVM::RJIT.enable."),
 160     M("--rjit-trace",              "", "Allow TracePoint during JIT compilation."),
 161     M("--rjit-trace-exits",        "", "Trace side exit locations."),
 162 #ifdef HAVE_LIBCAPSTONE
 163     M("--rjit-dump-disasm",        "", "Dump all JIT code"),
 164 #endif
 165     {0}
 166 };
 167 #undef M
 168
 169 struct rb_rjit_runtime_counters rb_rjit_counters = { 0 };
 170
 171 extern VALUE rb_gc_enable(void);
 172 extern VALUE rb_gc_disable(void);
 173 extern uint64_t rb_vm_insns_count;
 174
 175 // Disable GC, TracePoint, JIT, stats, and $!
 176 #define WITH_RJIT_ISOLATED_USING_PC(using_pc, stmt) do { \
 177     VALUE was_disabled = rb_gc_disable(); \
 178     \
 179     rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(GET_EC()); \
 180     rb_rjit_global_events = global_hooks->events; \
 181     \
 182     const VALUE *pc = NULL; \
 183     if (rb_rjit_opts.trace) { \
 184         pc = GET_EC()->cfp->pc; \
 185         if (!using_pc) GET_EC()->cfp->pc = 0; /* avoid crashing on calc_lineno */ \
 186     } \
 187     else global_hooks->events = 0; \
 188     \
 189     bool original_call_p = rb_rjit_call_p; \
 190     rb_rjit_call_p = false; \
 191     \
 192     rjit_stats_p = false; \
 193     uint64_t insns_count = rb_vm_insns_count; \
 194     \
 195     VALUE err = rb_errinfo(); \
 196     \
 197     stmt; \
 198     \
 199     rb_set_errinfo(err); \
 200     \
 201     rb_vm_insns_count = insns_count; \
 202     rjit_stats_p = rb_rjit_opts.stats; \
 203     \
 204     rb_rjit_call_p = (rjit_cancel_p ? false : original_call_p); \
 205     \
 206     if (rb_rjit_opts.trace) GET_EC()->cfp->pc = pc; \
 207     else global_hooks->events = rb_rjit_global_events; \
 208     \
 209     if (!was_disabled) rb_gc_enable(); \
 210 } while (0);
 211 #define WITH_RJIT_ISOLATED(stmt) WITH_RJIT_ISOLATED_USING_PC(false, stmt)
 212
 213 void
 214 rb_rjit_cancel_all(const char *reason)
 215 {
 216     if (!rb_rjit_enabled)
 217         return;
 218
 219     rb_rjit_call_p = false;
 220     rjit_cancel_p = true;
 221 }
 222
 223 void
 224 rb_rjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop)
 225 {
 226     if (!rb_rjit_call_p) return;
 227     rb_rjit_call_p = false;
 228 }
 229
 230 static void
 231 rjit_cme_invalidate(void *data)
 232 {
 233     if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
 234     WITH_RJIT_ISOLATED({
 235         rb_funcall(rb_mRJITHooks, rb_intern("on_cme_invalidate"), 1, SIZET2NUM((size_t)data));
 236     });
 237 }
 238
 239 extern int rb_workqueue_register(unsigned flags, rb_postponed_job_func_t func, void *data);
 240
 241 void
 242 rb_rjit_cme_invalidate(rb_callable_method_entry_t *cme)
 243 {
 244     if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
 245     // Asynchronously hook the Ruby code since running Ruby in the middle of cme invalidation is dangerous.
 246     rb_workqueue_register(0, rjit_cme_invalidate, (void *)cme);
 247 }
 248
 249 void
 250 rb_rjit_before_ractor_spawn(void)
 251 {
 252     if (!rb_rjit_call_p) return;
 253     rb_rjit_call_p = false;
 254 }
 255
 256 static void
 257 rjit_constant_state_changed(void *data)
 258 {
 259     if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
 260     RB_VM_LOCK_ENTER();
 261     rb_vm_barrier();
 262
 263     WITH_RJIT_ISOLATED({
 264         rb_funcall(rb_mRJITHooks, rb_intern("on_constant_state_changed"), 1, SIZET2NUM((size_t)data));
 265     });
 266
 267     RB_VM_LOCK_LEAVE();
 268 }
 269
 270 void
 271 rb_rjit_constant_state_changed(ID id)
 272 {
 273     if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
 274     // Asynchronously hook the Ruby code since this is hooked during a "Ruby critical section".
 275     rb_workqueue_register(0, rjit_constant_state_changed, (void *)id);
 276 }
 277
 278 void
 279 rb_rjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx)
 280 {
 281     if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
 282
 283     RB_VM_LOCK_ENTER();
 284     rb_vm_barrier();
 285
 286     WITH_RJIT_ISOLATED({
 287         rb_funcall(rb_mRJITHooks, rb_intern("on_constant_ic_update"), 3,
 288                    SIZET2NUM((size_t)iseq), SIZET2NUM((size_t)ic), UINT2NUM(insn_idx));
 289     });
 290
 291     RB_VM_LOCK_LEAVE();
 292 }
 293
 294 void
 295 rb_rjit_tracing_invalidate_all(rb_event_flag_t new_iseq_events)
 296 {
 297     if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
 298     WITH_RJIT_ISOLATED({
 299         rb_funcall(rb_mRJITHooks, rb_intern("on_tracing_invalidate_all"), 1, UINT2NUM(new_iseq_events));
 300     });
 301 }
 302
 303 static void
 304 rjit_iseq_update_references(void *data)
 305 {
 306     if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
 307     WITH_RJIT_ISOLATED({
 308         rb_funcall(rb_mRJITHooks, rb_intern("on_update_references"), 0);
 309     });
 310 }
 311
 312 void
 313 rb_rjit_iseq_update_references(struct rb_iseq_constant_body *const body)
 314 {
 315     if (!rb_rjit_enabled) return;
 316
 317     if (body->rjit_blocks) {
 318         body->rjit_blocks = rb_gc_location(body->rjit_blocks);
 319     }
 320
 321     // Asynchronously hook the Ruby code to avoid allocation during GC.compact.
 322     // Using _one because it's too slow to invalidate all for each ISEQ. Thus
 323     // not giving an ISEQ pointer.
 324     rb_postponed_job_trigger(rjit_iseq_update_references_pjob);
 325 }
 326
 327 void
 328 rb_rjit_iseq_mark(VALUE rjit_blocks)
 329 {
 330     if (!rb_rjit_enabled) return;
 331
 332     // Note: This wasn't enough for some reason.
 333     // We actually rely on RubyVM::RJIT::GC_REFS to mark this.
 334     if (rjit_blocks) {
 335         rb_gc_mark_movable(rjit_blocks);
 336     }
 337 }
 338
 339 // Called by rb_vm_mark()
 340 void
 341 rb_rjit_mark(void)
 342 {
 343     if (!rb_rjit_enabled)
 344         return;
 345     RUBY_MARK_ENTER("rjit");
 346
 347     // Pin object pointers used in this file
 348     rb_gc_mark(rb_RJITCompiler);
 349     rb_gc_mark(rb_cRJITIseqPtr);
 350     rb_gc_mark(rb_cRJITCfpPtr);
 351     rb_gc_mark(rb_mRJITHooks);
 352     rb_gc_mark(rb_rjit_raw_samples);
 353     rb_gc_mark(rb_rjit_line_samples);
 354
 355     RUBY_MARK_LEAVE("rjit");
 356 }
 357
 358 void
 359 rb_rjit_free_iseq(const rb_iseq_t *iseq)
 360 {
 361     // TODO: implement this. GC_REFS should remove this iseq's mjit_blocks
 362 }
 363
 364 // TODO: Use this in more places
 365 VALUE
 366 rb_rjit_iseq_new(rb_iseq_t *iseq)
 367 {
 368     return rb_funcall(rb_cRJITIseqPtr, rb_intern("new"), 1, SIZET2NUM((size_t)iseq));
 369 }
 370
 371 void
 372 rb_rjit_compile(const rb_iseq_t *iseq)
 373 {
 374     RB_VM_LOCK_ENTER();
 375     rb_vm_barrier();
 376
 377     WITH_RJIT_ISOLATED_USING_PC(true, {
 378         VALUE iseq_ptr = rb_funcall(rb_cRJITIseqPtr, rb_intern("new"), 1, SIZET2NUM((size_t)iseq));
 379         VALUE cfp_ptr = rb_funcall(rb_cRJITCfpPtr, rb_intern("new"), 1, SIZET2NUM((size_t)GET_EC()->cfp));
 380         rb_funcall(rb_RJITCompiler, rb_intern("compile"), 2, iseq_ptr, cfp_ptr);
 381     });
 382
 383     RB_VM_LOCK_LEAVE();
 384 }
 385
 386 void *
 387 rb_rjit_entry_stub_hit(VALUE branch_stub)
 388 {
 389     VALUE result;
 390
 391     RB_VM_LOCK_ENTER();
 392     rb_vm_barrier();
 393
 394     rb_control_frame_t *cfp = GET_EC()->cfp;
 395
 396     WITH_RJIT_ISOLATED_USING_PC(true, {
 397         VALUE cfp_ptr = rb_funcall(rb_cRJITCfpPtr, rb_intern("new"), 1, SIZET2NUM((size_t)cfp));
 398         result = rb_funcall(rb_RJITCompiler, rb_intern("entry_stub_hit"), 2, branch_stub, cfp_ptr);
 399     });
 400
 401     RB_VM_LOCK_LEAVE();
 402
 403     return (void *)NUM2SIZET(result);
 404 }
 405
 406 void *
 407 rb_rjit_branch_stub_hit(VALUE branch_stub, int sp_offset, int target0_p)
 408 {
 409     VALUE result;
 410
 411     RB_VM_LOCK_ENTER();
 412     rb_vm_barrier();
 413
 414     rb_control_frame_t *cfp = GET_EC()->cfp;
 415     cfp->sp += sp_offset; // preserve stack values, also using the actual sp_offset to make jit.peek_at_stack work
 416
 417     WITH_RJIT_ISOLATED({
 418         VALUE cfp_ptr = rb_funcall(rb_cRJITCfpPtr, rb_intern("new"), 1, SIZET2NUM((size_t)cfp));
 419         result = rb_funcall(rb_RJITCompiler, rb_intern("branch_stub_hit"), 3, branch_stub, cfp_ptr, RBOOL(target0_p));
 420     });
 421
 422     cfp->sp -= sp_offset; // reset for consistency with the code without the stub
 423
 424     RB_VM_LOCK_LEAVE();
 425
 426     return (void *)NUM2SIZET(result);
 427 }
 428
 429 void
 430 rb_rjit_init(const struct rb_rjit_options *opts)
 431 {
 432     VM_ASSERT(rb_rjit_enabled);
 433
 434     // Normalize options
 435     rb_rjit_opts = *opts;
 436     if (rb_rjit_opts.exec_mem_size == 0)
 437         rb_rjit_opts.exec_mem_size = DEFAULT_EXEC_MEM_SIZE;
 438     if (rb_rjit_opts.call_threshold == 0)
 439         rb_rjit_opts.call_threshold = DEFAULT_CALL_THRESHOLD;
 440 #ifndef HAVE_LIBCAPSTONE
 441     if (rb_rjit_opts.dump_disasm)
 442         rb_warn("libcapstone has not been linked. Ignoring --rjit-dump-disasm.");
 443 #endif
 444
 445     // RJIT doesn't support miniruby, but it might reach here by RJIT_FORCE_ENABLE.
 446     rb_mRJIT = rb_const_get(rb_cRubyVM, rb_intern("RJIT"));
 447     if (!rb_const_defined(rb_mRJIT, rb_intern("Compiler"))) {
 448         rb_warn("Disabling RJIT because RubyVM::RJIT::Compiler is not defined");
 449         rb_rjit_enabled = false;
 450         return;
 451     }
 452     rjit_iseq_update_references_pjob = rb_postponed_job_preregister(0, rjit_iseq_update_references, NULL);
 453     if (rjit_iseq_update_references_pjob == POSTPONED_JOB_HANDLE_INVALID) {
 454         rb_bug("Could not preregister postponed job for RJIT");
 455     }
 456     rb_mRJITC = rb_const_get(rb_mRJIT, rb_intern("C"));
 457     VALUE rb_cRJITCompiler = rb_const_get(rb_mRJIT, rb_intern("Compiler"));
 458     rb_RJITCompiler = rb_funcall(rb_cRJITCompiler, rb_intern("new"), 0);
 459     rb_cRJITIseqPtr = rb_funcall(rb_mRJITC, rb_intern("rb_iseq_t"), 0);
 460     rb_cRJITCfpPtr = rb_funcall(rb_mRJITC, rb_intern("rb_control_frame_t"), 0);
 461     rb_mRJITHooks = rb_const_get(rb_mRJIT, rb_intern("Hooks"));
 462     if (rb_rjit_opts.trace_exits) {
 463         rb_rjit_raw_samples = rb_ary_new();
 464         rb_rjit_line_samples = rb_ary_new();
 465     }
 466
 467     // Enable RJIT and stats from here
 468     rb_rjit_call_p = !rb_rjit_opts.disable;
 469     rjit_stats_p = rb_rjit_opts.stats;
 470 }
 471
 472 //
 473 // Primitive for rjit.rb
 474 //
 475
 476 // Same as `rb_rjit_opts.stats`, but this is used before rb_rjit_opts is set.
 477 static VALUE
 478 rjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self)
 479 {
 480     return RBOOL(rb_rjit_stats_enabled);
 481 }
 482
 483 // Same as `rb_rjit_opts.trace_exits`, but this is used before rb_rjit_opts is set.
 484 static VALUE
 485 rjit_trace_exits_enabled_p(rb_execution_context_t *ec, VALUE self)
 486 {
 487     return RBOOL(rb_rjit_trace_exits_enabled);
 488 }
 489
 490 // Disable anything that could impact stats. It ends up disabling JIT calls as well.
 491 static VALUE
 492 rjit_stop_stats(rb_execution_context_t *ec, VALUE self)
 493 {
 494     rb_rjit_call_p = false;
 495     rjit_stats_p = false;
 496     return Qnil;
 497 }
 498
 499 #include "rjit.rbinc"
 500
 501 #endif // USE_RJIT