1 // This part of YJIT helps interfacing with the rest of CRuby and with the OS.
2 // Sometimes our FFI binding generation tool gives undesirable outputs when it
3 // sees C features that Rust doesn't support well. We mitigate that by binding
4 // functions which have simple parameter types. The boilerplate C functions for
5 // that purpose are in this file.
6 // Similarly, we wrap OS facilities we need in simple functions to help with
7 // FFI and to avoid the need to use external crates.io Rust libraries.
10 #include "internal/sanitizers.h"
11 #include "internal/string.h"
12 #include "internal/hash.h"
13 #include "internal/variable.h"
14 #include "internal/compile.h"
15 #include "internal/class.h"
16 #include "internal/fixnum.h"
17 #include "internal/numeric.h"
18 #include "internal/gc.h"
20 #include "vm_callinfo.h"
23 #include "insns_info.inc"
26 #include "vm_insnhelper.h"
28 #include "probes_helper.h"
30 #include "ruby/debug.h"
31 #include "internal/cont.h"
33 // For mmapp(), sysconf()
41 // Field offsets for the RObject struct
42 enum robject_offsets
{
43 ROBJECT_OFFSET_AS_HEAP_IVPTR
= offsetof(struct RObject
, as
.heap
.ivptr
),
44 ROBJECT_OFFSET_AS_HEAP_IV_INDEX_TBL
= offsetof(struct RObject
, as
.heap
.iv_index_tbl
),
45 ROBJECT_OFFSET_AS_ARY
= offsetof(struct RObject
, as
.ary
),
48 // Field offsets for the RString struct
49 enum rstring_offsets
{
50 RUBY_OFFSET_RSTRING_LEN
= offsetof(struct RString
, len
)
53 // We need size_t to have a known size to simplify code generation and FFI.
54 // TODO(alan): check this in configure.ac to fail fast on 32 bit platforms.
55 STATIC_ASSERT(64b_size_t
, SIZE_MAX
== UINT64_MAX
);
56 // I don't know any C implementation that has uint64_t and puts padding bits
57 // into size_t but the standard seems to allow it.
58 STATIC_ASSERT(size_t_no_padding_bits
, sizeof(size_t) == sizeof(uint64_t));
60 // This build config impacts the pointer tagging scheme and we only want to
61 // support one scheme for simplicity.
62 STATIC_ASSERT(pointer_tagging_scheme
, USE_FLONUM
);
64 // NOTE: We can trust that uint8_t has no "padding bits" since the C spec
65 // guarantees it. Wording about padding bits is more explicit in C11 compared
66 // to C99. See C11 7.20.1.1p2. All this is to say we have _some_ standards backing to
67 // use a Rust `*mut u8` to represent a C `uint8_t *`.
69 // If we don't want to trust that we can interpreter the C standard correctly, we
70 // could outsource that work to the Rust standard library by sticking to fundamental
71 // types in C such as int, long, etc. and use `std::os::raw::c_long` and friends on
74 // What's up with the long prefix? Even though we build with `-fvisibility=hidden`
75 // we are sometimes a static library where the option doesn't prevent name collision.
76 // The "_yjit_" part is for trying to be informative. We might want different
77 // suffixes for symbols meant for Rust and symbols meant for broader CRuby.
80 rb_yjit_mark_writable(void *mem_block
, uint32_t mem_size
)
82 return mprotect(mem_block
, mem_size
, PROT_READ
| PROT_WRITE
) == 0;
86 rb_yjit_mark_executable(void *mem_block
, uint32_t mem_size
)
88 // Do not call mprotect when mem_size is zero. Some platforms may return
89 // an error for it. https://github.com/Shopify/ruby/issues/450
93 if (mprotect(mem_block
, mem_size
, PROT_READ
| PROT_EXEC
)) {
94 rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s",
95 mem_block
, (unsigned long)mem_size
, strerror(errno
));
99 // Free the specified memory block.
101 rb_yjit_mark_unused(void *mem_block
, uint32_t mem_size
)
103 // On Linux, you need to use madvise MADV_DONTNEED to free memory.
104 // We might not need to call this on macOS, but it's not really documented.
105 // We generally prefer to do the same thing on both to ease testing too.
106 madvise(mem_block
, mem_size
, MADV_DONTNEED
);
108 // On macOS, mprotect PROT_NONE seems to reduce RSS.
109 // We also call this on Linux to avoid executing unused pages.
110 return mprotect(mem_block
, mem_size
, PROT_NONE
) == 0;
114 rb_yjit_array_len(VALUE a
)
116 return rb_array_len(a
);
119 // `start` is inclusive and `end` is exclusive.
121 rb_yjit_icache_invalidate(void *start
, void *end
)
123 // Clear/invalidate the instruction cache. Compiles to nothing on x86_64
124 // but required on ARM before running freshly written code.
125 // On Darwin it's the same as calling sys_icache_invalidate().
127 __builtin___clear_cache(start
, end
);
128 #elif defined(__aarch64__)
129 #error No instruction cache clear available with this compiler on Aarch64!
133 # define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x)))
135 // For a given raw_sample (frame), set the hash with the caller's
136 // name, file, and line number. Return the hash with collected frame_info.
138 rb_yjit_add_frame(VALUE hash
, VALUE frame
)
140 VALUE frame_id
= PTR2NUM(frame
);
142 if (RTEST(rb_hash_aref(hash
, frame_id
))) {
146 VALUE frame_info
= rb_hash_new();
147 // Full label for the frame
148 VALUE name
= rb_profile_frame_full_label(frame
);
149 // Absolute path of the frame from rb_iseq_realpath
150 VALUE file
= rb_profile_frame_absolute_path(frame
);
151 // Line number of the frame
152 VALUE line
= rb_profile_frame_first_lineno(frame
);
154 // If absolute path isn't available use the rb_iseq_path
156 file
= rb_profile_frame_path(frame
);
159 rb_hash_aset(frame_info
, ID2SYM(rb_intern("name")), name
);
160 rb_hash_aset(frame_info
, ID2SYM(rb_intern("file")), file
);
161 rb_hash_aset(frame_info
, ID2SYM(rb_intern("samples")), INT2NUM(0));
162 rb_hash_aset(frame_info
, ID2SYM(rb_intern("total_samples")), INT2NUM(0));
163 rb_hash_aset(frame_info
, ID2SYM(rb_intern("edges")), rb_hash_new());
164 rb_hash_aset(frame_info
, ID2SYM(rb_intern("lines")), rb_hash_new());
166 if (line
!= INT2FIX(0)) {
167 rb_hash_aset(frame_info
, ID2SYM(rb_intern("line")), line
);
170 rb_hash_aset(hash
, frame_id
, frame_info
);
174 // Parses the YjitExitLocations raw_samples and line_samples collected by
175 // rb_yjit_record_exit_stack and turns them into 3 hashes (raw, lines, and frames) to
176 // be used by RubyVM::YJIT.exit_locations. yjit_raw_samples represents the raw frames information
177 // (without name, file, and line), and yjit_line_samples represents the line information
178 // of the iseq caller.
180 rb_yjit_exit_locations_dict(VALUE
*yjit_raw_samples
, int *yjit_line_samples
, int samples_len
)
182 VALUE result
= rb_hash_new();
183 VALUE raw_samples
= rb_ary_new_capa(samples_len
);
184 VALUE line_samples
= rb_ary_new_capa(samples_len
);
185 VALUE frames
= rb_hash_new();
188 // While the index is less than samples_len, parse yjit_raw_samples and
189 // yjit_line_samples, then add casted values to raw_samples and line_samples array.
190 while (idx
< samples_len
) {
191 int num
= (int)yjit_raw_samples
[idx
];
192 int line_num
= (int)yjit_line_samples
[idx
];
195 // + 1 as we append an additional sample for the insn
196 rb_ary_push(raw_samples
, SIZET2NUM(num
+ 1));
197 rb_ary_push(line_samples
, INT2NUM(line_num
+ 1));
199 // Loop through the length of samples_len and add data to the
200 // frames hash. Also push the current value onto the raw_samples
201 // and line_samples array respectively.
202 for (int o
= 0; o
< num
; o
++) {
203 rb_yjit_add_frame(frames
, yjit_raw_samples
[idx
]);
204 rb_ary_push(raw_samples
, SIZET2NUM(yjit_raw_samples
[idx
]));
205 rb_ary_push(line_samples
, INT2NUM(yjit_line_samples
[idx
]));
209 rb_ary_push(raw_samples
, SIZET2NUM(yjit_raw_samples
[idx
]));
210 rb_ary_push(line_samples
, INT2NUM(yjit_line_samples
[idx
]));
213 rb_ary_push(raw_samples
, SIZET2NUM(yjit_raw_samples
[idx
]));
214 rb_ary_push(line_samples
, INT2NUM(yjit_line_samples
[idx
]));
218 // Set add the raw_samples, line_samples, and frames to the results
220 rb_hash_aset(result
, ID2SYM(rb_intern("raw")), raw_samples
);
221 rb_hash_aset(result
, ID2SYM(rb_intern("lines")), line_samples
);
222 rb_hash_aset(result
, ID2SYM(rb_intern("frames")), frames
);
228 rb_yjit_get_page_size(void)
230 #if defined(_SC_PAGESIZE)
231 long page_size
= sysconf(_SC_PAGESIZE
);
232 if (page_size
<= 0) rb_bug("yjit: failed to get page size");
234 // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected.
235 // Though our design sort of assume we have fine grained control over memory protection
236 // which require small page sizes.
237 if (page_size
> 0x40000000l
) rb_bug("yjit page size too large");
239 return (uint32_t)page_size
;
241 #error "YJIT supports POSIX only for now"
245 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
246 // Align the current write position to a multiple of bytes
248 align_ptr(uint8_t *ptr
, uint32_t multiple
)
250 // Compute the pointer modulo the given alignment boundary
251 uint32_t rem
= ((uint32_t)(uintptr_t)ptr
) % multiple
;
253 // If the pointer is already aligned, stop
257 // Pad the pointer by the necessary amount to align it
258 uint32_t pad
= multiple
- rem
;
264 // Address space reservation. Memory pages are mapped on an as needed basis.
265 // See the Rust mm module for details.
267 rb_yjit_reserve_addr_space(uint32_t mem_size
)
273 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
274 uint32_t const page_size
= (uint32_t)sysconf(_SC_PAGESIZE
);
275 uint8_t *const cfunc_sample_addr
= (void *)&rb_yjit_reserve_addr_space
;
276 uint8_t *const probe_region_end
= cfunc_sample_addr
+ INT32_MAX
;
277 // Align the requested address to page size
278 uint8_t *req_addr
= align_ptr(cfunc_sample_addr
, page_size
);
280 // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
281 // to improve odds of being in range for 32-bit relative call instructions.
287 MAP_PRIVATE
| MAP_ANONYMOUS
| MAP_FIXED_NOREPLACE
,
292 // If we succeeded, stop
293 if (mem_block
!= MAP_FAILED
) {
298 req_addr
+= 4 * 1024 * 1024;
299 } while (req_addr
< probe_region_end
);
301 // On MacOS and other platforms
303 // Try to map a chunk of memory as executable
305 (void *)rb_yjit_reserve_addr_space
,
308 MAP_PRIVATE
| MAP_ANONYMOUS
,
315 if (mem_block
== MAP_FAILED
) {
316 // Try again without the address hint (e.g., valgrind)
321 MAP_PRIVATE
| MAP_ANONYMOUS
,
327 // Check that the memory mapping was successful
328 if (mem_block
== MAP_FAILED
) {
329 perror("ruby: yjit: mmap:");
330 if(errno
== ENOMEM
) {
331 // No crash report if it's only insufficient memory
334 rb_bug("mmap failed");
339 // Windows not supported for now
344 // Is anyone listening for :c_call and :c_return event currently?
346 rb_c_method_tracing_currently_enabled(const rb_execution_context_t
*ec
)
348 rb_event_flag_t tracing_events
;
349 if (rb_multi_ractor_p()) {
350 tracing_events
= ruby_vm_event_enabled_global_flags
;
353 // At the time of writing, events are never removed from
354 // ruby_vm_event_enabled_global_flags so always checking using it would
355 // mean we don't compile even after tracing is disabled.
356 tracing_events
= rb_ec_ractor_hooks(ec
)->events
;
359 return tracing_events
& (RUBY_EVENT_C_CALL
| RUBY_EVENT_C_RETURN
);
362 // The code we generate in gen_send_cfunc() doesn't fire the c_return TracePoint event
363 // like the interpreter. When tracing for c_return is enabled, we patch the code after
364 // the C method return to call into this to fire the event.
366 rb_full_cfunc_return(rb_execution_context_t
*ec
, VALUE return_value
)
368 rb_control_frame_t
*cfp
= ec
->cfp
;
369 RUBY_ASSERT_ALWAYS(cfp
== GET_EC()->cfp
);
370 const rb_callable_method_entry_t
*me
= rb_vm_frame_method_entry(cfp
);
372 RUBY_ASSERT_ALWAYS(RUBYVM_CFUNC_FRAME_P(cfp
));
373 RUBY_ASSERT_ALWAYS(me
->def
->type
== VM_METHOD_TYPE_CFUNC
);
375 // CHECK_CFP_CONSISTENCY("full_cfunc_return"); TODO revive this
377 // Pop the C func's frame and fire the c_return TracePoint event
378 // Note that this is the same order as vm_call_cfunc_with_frame().
380 EXEC_EVENT_HOOK(ec
, RUBY_EVENT_C_RETURN
, cfp
->self
, me
->def
->original_id
, me
->called_id
, me
->owner
, return_value
);
381 // Note, this deviates from the interpreter in that users need to enable
382 // a c_return TracePoint for this DTrace hook to work. A reasonable change
383 // since the Ruby return event works this way as well.
384 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec
, me
->owner
, me
->def
->original_id
);
386 // Push return value into the caller's stack. We know that it's a frame that
387 // uses cfp->sp because we are patching a call done with gen_send_cfunc().
388 ec
->cfp
->sp
[0] = return_value
;
393 rb_iseq_encoded_size(const rb_iseq_t
*iseq
)
395 return iseq
->body
->iseq_size
;
398 // TODO(alan): consider using an opaque pointer for the payload rather than a void pointer
400 rb_iseq_get_yjit_payload(const rb_iseq_t
*iseq
)
402 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq
, imemo_iseq
));
404 return iseq
->body
->yjit_payload
;
407 // Body is NULL when constructing the iseq.
413 rb_iseq_set_yjit_payload(const rb_iseq_t
*iseq
, void *payload
)
415 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq
, imemo_iseq
));
416 RUBY_ASSERT_ALWAYS(iseq
->body
);
417 RUBY_ASSERT_ALWAYS(NULL
== iseq
->body
->yjit_payload
);
418 iseq
->body
->yjit_payload
= payload
;
422 rb_iseq_reset_jit_func(const rb_iseq_t
*iseq
)
424 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq
, imemo_iseq
));
425 iseq
->body
->jit_entry
= NULL
;
426 iseq
->body
->jit_exception
= NULL
;
427 // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint,
428 // we'd like to re-compile ISEQs that haven't been converted to trace_* insns.
429 iseq
->body
->jit_entry_calls
= 0;
430 iseq
->body
->jit_exception_calls
= 0;
433 // Get the PC for a given index in an iseq
435 rb_iseq_pc_at_idx(const rb_iseq_t
*iseq
, uint32_t insn_idx
)
437 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq
, imemo_iseq
));
438 RUBY_ASSERT_ALWAYS(insn_idx
< iseq
->body
->iseq_size
);
439 VALUE
*encoded
= iseq
->body
->iseq_encoded
;
440 VALUE
*pc
= &encoded
[insn_idx
];
444 // Get the opcode given a program counter. Can return trace opcode variants.
446 rb_iseq_opcode_at_pc(const rb_iseq_t
*iseq
, const VALUE
*pc
)
448 // YJIT should only use iseqs after AST to bytecode compilation
449 RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE
)iseq
, ISEQ_TRANSLATED
));
451 const VALUE at_pc
= *pc
;
452 return rb_vm_insn_addr2opcode((const void *)at_pc
);
456 rb_RSTRING_LEN(VALUE str
)
458 return RSTRING_LEN(str
);
462 rb_RSTRING_PTR(VALUE str
)
464 return RSTRING_PTR(str
);
468 rb_yjit_get_proc_ptr(VALUE procv
)
471 GetProcPtr(procv
, proc
);
475 // This is defined only as a named struct inside rb_iseq_constant_body.
476 // By giving it a separate typedef, we make it nameable by rust-bindgen.
477 // Bindgen's temp/anon name isn't guaranteed stable.
478 typedef struct rb_iseq_param_keyword rb_seq_param_keyword_struct
;
481 rb_insn_name(VALUE insn
)
483 return insn_name(insn
);
487 rb_vm_ci_argc(const struct rb_callinfo
*ci
)
489 return vm_ci_argc(ci
);
493 rb_vm_ci_mid(const struct rb_callinfo
*ci
)
495 return vm_ci_mid(ci
);
499 rb_vm_ci_flag(const struct rb_callinfo
*ci
)
501 return vm_ci_flag(ci
);
504 const struct rb_callinfo_kwarg
*
505 rb_vm_ci_kwarg(const struct rb_callinfo
*ci
)
507 return vm_ci_kwarg(ci
);
511 rb_get_cikw_keyword_len(const struct rb_callinfo_kwarg
*cikw
)
513 return cikw
->keyword_len
;
517 rb_get_cikw_keywords_idx(const struct rb_callinfo_kwarg
*cikw
, int idx
)
519 return cikw
->keywords
[idx
];
522 rb_method_visibility_t
523 rb_METHOD_ENTRY_VISI(const rb_callable_method_entry_t
*me
)
525 return METHOD_ENTRY_VISI(me
);
529 rb_get_cme_def_type(const rb_callable_method_entry_t
*cme
)
531 if (UNDEFINED_METHOD_ENTRY_P(cme
)) {
532 return VM_METHOD_TYPE_UNDEF
;
535 return cme
->def
->type
;
540 rb_get_cme_def_body_attr_id(const rb_callable_method_entry_t
*cme
)
542 return cme
->def
->body
.attr
.id
;
545 ID
rb_get_symbol_id(VALUE namep
);
547 enum method_optimized_type
548 rb_get_cme_def_body_optimized_type(const rb_callable_method_entry_t
*cme
)
550 return cme
->def
->body
.optimized
.type
;
554 rb_get_cme_def_body_optimized_index(const rb_callable_method_entry_t
*cme
)
556 return cme
->def
->body
.optimized
.index
;
560 rb_get_cme_def_body_cfunc(const rb_callable_method_entry_t
*cme
)
562 return UNALIGNED_MEMBER_PTR(cme
->def
, body
.cfunc
);
566 rb_get_def_method_serial(const rb_method_definition_t
*def
)
568 return def
->method_serial
;
572 rb_get_def_original_id(const rb_method_definition_t
*def
)
574 return def
->original_id
;
578 rb_get_mct_argc(const rb_method_cfunc_t
*mct
)
584 rb_get_mct_func(const rb_method_cfunc_t
*mct
)
586 return (void*)mct
->func
; // this field is defined as type VALUE (*func)(ANYARGS)
590 rb_get_def_iseq_ptr(rb_method_definition_t
*def
)
592 return def_iseq_ptr(def
);
596 rb_get_def_bmethod_proc(rb_method_definition_t
*def
)
598 RUBY_ASSERT(def
->type
== VM_METHOD_TYPE_BMETHOD
);
599 return def
->body
.bmethod
.proc
;
603 rb_get_iseq_body_local_iseq(const rb_iseq_t
*iseq
)
605 return iseq
->body
->local_iseq
;
609 rb_get_iseq_body_parent_iseq(const rb_iseq_t
*iseq
)
611 return iseq
->body
->parent_iseq
;
615 rb_get_iseq_body_local_table_size(const rb_iseq_t
*iseq
)
617 return iseq
->body
->local_table_size
;
621 rb_get_iseq_body_iseq_encoded(const rb_iseq_t
*iseq
)
623 return iseq
->body
->iseq_encoded
;
627 rb_get_iseq_body_stack_max(const rb_iseq_t
*iseq
)
629 return iseq
->body
->stack_max
;
633 rb_get_iseq_body_type(const rb_iseq_t
*iseq
)
635 return iseq
->body
->type
;
639 rb_get_iseq_flags_has_lead(const rb_iseq_t
*iseq
)
641 return iseq
->body
->param
.flags
.has_lead
;
645 rb_get_iseq_flags_has_opt(const rb_iseq_t
*iseq
)
647 return iseq
->body
->param
.flags
.has_opt
;
651 rb_get_iseq_flags_has_kw(const rb_iseq_t
*iseq
)
653 return iseq
->body
->param
.flags
.has_kw
;
657 rb_get_iseq_flags_has_post(const rb_iseq_t
*iseq
)
659 return iseq
->body
->param
.flags
.has_post
;
663 rb_get_iseq_flags_has_kwrest(const rb_iseq_t
*iseq
)
665 return iseq
->body
->param
.flags
.has_kwrest
;
669 rb_get_iseq_flags_anon_kwrest(const rb_iseq_t
*iseq
)
671 return iseq
->body
->param
.flags
.anon_kwrest
;
675 rb_get_iseq_flags_has_rest(const rb_iseq_t
*iseq
)
677 return iseq
->body
->param
.flags
.has_rest
;
681 rb_get_iseq_flags_ruby2_keywords(const rb_iseq_t
*iseq
)
683 return iseq
->body
->param
.flags
.ruby2_keywords
;
687 rb_get_iseq_flags_has_block(const rb_iseq_t
*iseq
)
689 return iseq
->body
->param
.flags
.has_block
;
693 rb_get_iseq_flags_ambiguous_param0(const rb_iseq_t
*iseq
)
695 return iseq
->body
->param
.flags
.ambiguous_param0
;
699 rb_get_iseq_flags_accepts_no_kwarg(const rb_iseq_t
*iseq
)
701 return iseq
->body
->param
.flags
.accepts_no_kwarg
;
704 const rb_seq_param_keyword_struct
*
705 rb_get_iseq_body_param_keyword(const rb_iseq_t
*iseq
)
707 return iseq
->body
->param
.keyword
;
711 rb_get_iseq_body_param_size(const rb_iseq_t
*iseq
)
713 return iseq
->body
->param
.size
;
717 rb_get_iseq_body_param_lead_num(const rb_iseq_t
*iseq
)
719 return iseq
->body
->param
.lead_num
;
723 rb_get_iseq_body_param_opt_num(const rb_iseq_t
*iseq
)
725 return iseq
->body
->param
.opt_num
;
729 rb_get_iseq_body_param_opt_table(const rb_iseq_t
*iseq
)
731 return iseq
->body
->param
.opt_table
;
735 rb_optimized_call(VALUE
*recv
, rb_execution_context_t
*ec
, int argc
, VALUE
*argv
, int kw_splat
, VALUE block_handler
)
738 GetProcPtr(recv
, proc
);
739 return rb_vm_invoke_proc(ec
, proc
, argc
, argv
, kw_splat
, block_handler
);
743 rb_yjit_iseq_builtin_attrs(const rb_iseq_t
*iseq
)
745 return iseq
->body
->builtin_attrs
;
748 // If true, the iseq has only opt_invokebuiltin_delegate(_leave) and leave insns.
750 invokebuiltin_delegate_leave_p(const rb_iseq_t
*iseq
)
752 int insn1
= rb_vm_insn_addr2opcode((void *)iseq
->body
->iseq_encoded
[0]);
753 if ((int)iseq
->body
->iseq_size
!= insn_len(insn1
) + insn_len(BIN(leave
))) {
756 int insn2
= rb_vm_insn_addr2opcode((void *)iseq
->body
->iseq_encoded
[insn_len(insn1
)]);
757 return (insn1
== BIN(opt_invokebuiltin_delegate
) || insn1
== BIN(opt_invokebuiltin_delegate_leave
)) &&
761 // Return an rb_builtin_function if the iseq contains only that builtin function.
762 const struct rb_builtin_function
*
763 rb_yjit_builtin_function(const rb_iseq_t
*iseq
)
765 if (invokebuiltin_delegate_leave_p(iseq
)) {
766 return (const struct rb_builtin_function
*)iseq
->body
->iseq_encoded
[1];
774 rb_yjit_str_simple_append(VALUE str1
, VALUE str2
)
776 return rb_str_cat(str1
, RSTRING_PTR(str2
), RSTRING_LEN(str2
));
779 struct rb_control_frame_struct
*
780 rb_get_ec_cfp(const rb_execution_context_t
*ec
)
786 rb_get_cfp_iseq(struct rb_control_frame_struct
*cfp
)
792 rb_get_cfp_pc(struct rb_control_frame_struct
*cfp
)
794 return (VALUE
*)cfp
->pc
;
798 rb_get_cfp_sp(struct rb_control_frame_struct
*cfp
)
804 rb_set_cfp_pc(struct rb_control_frame_struct
*cfp
, const VALUE
*pc
)
810 rb_set_cfp_sp(struct rb_control_frame_struct
*cfp
, VALUE
*sp
)
816 rb_get_cfp_self(struct rb_control_frame_struct
*cfp
)
822 rb_get_cfp_ep(struct rb_control_frame_struct
*cfp
)
824 return (VALUE
*)cfp
->ep
;
828 rb_get_cfp_ep_level(struct rb_control_frame_struct
*cfp
, uint32_t lv
)
831 const VALUE
*ep
= (VALUE
*)cfp
->ep
;
832 for (i
= 0; i
< lv
; i
++) {
833 ep
= VM_ENV_PREV_EP(ep
);
838 extern VALUE
*rb_vm_base_ptr(struct rb_control_frame_struct
*cfp
);
841 rb_yarv_class_of(VALUE obj
)
843 return rb_class_of(obj
);
846 // YJIT needs this function to never allocate and never raise
848 rb_yarv_str_eql_internal(VALUE str1
, VALUE str2
)
850 // We wrap this since it's static inline
851 return rb_str_eql_internal(str1
, str2
);
855 rb_str_neq_internal(VALUE str1
, VALUE str2
)
857 return rb_str_eql_internal(str1
, str2
) == Qtrue
? Qfalse
: Qtrue
;
860 // YJIT needs this function to never allocate and never raise
862 rb_yarv_ary_entry_internal(VALUE ary
, long offset
)
864 return rb_ary_entry_internal(ary
, offset
);
867 extern VALUE
rb_ary_unshift_m(int argc
, VALUE
*argv
, VALUE ary
);
870 rb_yjit_rb_ary_subseq_length(VALUE ary
, long beg
)
872 long len
= RARRAY_LEN(ary
);
873 return rb_ary_subseq(ary
, beg
, len
);
877 rb_yjit_fix_div_fix(VALUE recv
, VALUE obj
)
879 return rb_fix_div_fix(recv
, obj
);
883 rb_yjit_fix_mod_fix(VALUE recv
, VALUE obj
)
885 return rb_fix_mod_fix(recv
, obj
);
888 // Return non-zero when `obj` is an array and its last item is a
889 // `ruby2_keywords` hash. We don't support this kind of splat.
891 rb_yjit_ruby2_keywords_splat_p(VALUE obj
)
893 if (!RB_TYPE_P(obj
, T_ARRAY
)) return 0;
894 long len
= RARRAY_LEN(obj
);
895 if (len
== 0) return 0;
896 VALUE last
= RARRAY_AREF(obj
, len
- 1);
897 if (!RB_TYPE_P(last
, T_HASH
)) return 0;
898 return FL_TEST_RAW(last
, RHASH_PASS_AS_KEYWORDS
);
901 // Checks to establish preconditions for rb_yjit_splat_varg_cfunc()
903 rb_yjit_splat_varg_checks(VALUE
*sp
, VALUE splat_array
, rb_control_frame_t
*cfp
)
905 // We inserted a T_ARRAY guard before this call
906 long len
= RARRAY_LEN(splat_array
);
908 // Large splat arrays need a separate allocation
909 if (len
< 0 || len
> VM_ARGC_STACK_MAX
) return Qfalse
;
911 // Would we overflow if we put the contents of the array onto the stack?
912 if (sp
+ len
> (VALUE
*)(cfp
- 2)) return Qfalse
;
914 // Reject keywords hash since that requires duping it sometimes
916 VALUE last_hash
= RARRAY_AREF(splat_array
, len
- 1);
917 if (RB_TYPE_P(last_hash
, T_HASH
) &&
918 FL_TEST_RAW(last_hash
, RHASH_PASS_AS_KEYWORDS
)) {
926 // Push array elements to the stack for a C method that has a variable number
927 // of parameters. Returns the number of arguments the splat array contributes.
929 rb_yjit_splat_varg_cfunc(VALUE
*stack_splat_array
)
931 VALUE splat_array
= *stack_splat_array
;
934 // We already checked that length fits in `int`
935 RUBY_ASSERT(RB_TYPE_P(splat_array
, T_ARRAY
));
936 len
= (int)RARRAY_LEN(splat_array
);
938 // Push the contents of the array onto the stack
939 MEMCPY(stack_splat_array
, RARRAY_CONST_PTR(splat_array
), VALUE
, len
);
944 // Print the Ruby source location of some ISEQ for debugging purposes
946 rb_yjit_dump_iseq_loc(const rb_iseq_t
*iseq
, uint32_t insn_idx
)
950 VALUE path
= rb_iseq_path(iseq
);
951 RSTRING_GETMEM(path
, ptr
, len
);
952 fprintf(stderr
, "%s %.*s:%u\n", __func__
, (int)len
, ptr
, rb_iseq_line_no(iseq
, insn_idx
));
955 // Get the number of digits required to print an integer
957 num_digits(int integer
)
960 while (integer
/= 10) {
966 // Allocate a C string that formats an ISEQ label like iseq_inspect()
968 rb_yjit_iseq_inspect(const rb_iseq_t
*iseq
)
970 const char *label
= RSTRING_PTR(iseq
->body
->location
.label
);
971 const char *path
= RSTRING_PTR(rb_iseq_path(iseq
));
972 int lineno
= iseq
->body
->location
.code_location
.beg_pos
.lineno
;
974 char *buf
= ZALLOC_N(char, strlen(label
) + strlen(path
) + num_digits(lineno
) + 3);
975 sprintf(buf
, "%s@%s:%d", label
, path
, lineno
);
979 // The FL_TEST() macro
981 rb_FL_TEST(VALUE obj
, VALUE flags
)
983 return RB_FL_TEST(obj
, flags
);
986 // The FL_TEST_RAW() macro, normally an internal implementation detail
988 rb_FL_TEST_RAW(VALUE obj
, VALUE flags
)
990 return FL_TEST_RAW(obj
, flags
);
993 // The RB_TYPE_P macro
995 rb_RB_TYPE_P(VALUE obj
, enum ruby_value_type t
)
997 return RB_TYPE_P(obj
, t
);
1001 rb_RSTRUCT_LEN(VALUE st
)
1003 return RSTRUCT_LEN(st
);
1006 // There are RSTRUCT_SETs in ruby/internal/core/rstruct.h and internal/struct.h
1007 // with different types (int vs long) for k. Here we use the one from ruby/internal/core/rstruct.h,
1008 // which takes an int.
1010 rb_RSTRUCT_SET(VALUE st
, int k
, VALUE v
)
1012 RSTRUCT_SET(st
, k
, v
);
1015 const struct rb_callinfo
*
1016 rb_get_call_data_ci(const struct rb_call_data
*cd
)
1022 rb_BASIC_OP_UNREDEFINED_P(enum ruby_basic_operators bop
, uint32_t klass
)
1024 return BASIC_OP_UNREDEFINED_P(bop
, klass
);
1028 rb_RCLASS_ORIGIN(VALUE c
)
1030 return RCLASS_ORIGIN(c
);
1033 // Return the string encoding index
1035 rb_ENCODING_GET(VALUE obj
)
1037 return RB_ENCODING_GET(obj
);
1041 rb_yjit_multi_ractor_p(void)
1043 return rb_multi_ractor_p();
1048 rb_assert_iseq_handle(VALUE handle
)
1050 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle
, imemo_iseq
));
1054 rb_IMEMO_TYPE_P(VALUE imemo
, enum imemo_type imemo_type
)
1056 return IMEMO_TYPE_P(imemo
, imemo_type
);
1060 rb_assert_cme_handle(VALUE handle
)
1062 RUBY_ASSERT_ALWAYS(rb_objspace_markable_object_p(handle
));
1063 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle
, imemo_ment
));
1066 // Used for passing a callback and other data over rb_objspace_each_objects
1067 struct iseq_callback_data
{
1068 rb_iseq_callback callback
;
1072 // Heap-walking callback for rb_yjit_for_each_iseq().
1074 for_each_iseq_i(void *vstart
, void *vend
, size_t stride
, void *data
)
1076 const struct iseq_callback_data
*callback_data
= (struct iseq_callback_data
*)data
;
1077 VALUE v
= (VALUE
)vstart
;
1078 for (; v
!= (VALUE
)vend
; v
+= stride
) {
1079 void *ptr
= asan_poisoned_object_p(v
);
1080 asan_unpoison_object(v
, false);
1082 if (rb_obj_is_iseq(v
)) {
1083 rb_iseq_t
*iseq
= (rb_iseq_t
*)v
;
1084 callback_data
->callback(iseq
, callback_data
->data
);
1087 asan_poison_object_if(ptr
, v
);
1092 // Iterate through the whole GC heap and invoke a callback for each iseq.
1093 // Used for global code invalidation.
1095 rb_yjit_for_each_iseq(rb_iseq_callback callback
, void *data
)
1097 struct iseq_callback_data callback_data
= { .callback
= callback
, .data
= data
};
1098 rb_objspace_each_objects(for_each_iseq_i
, (void *)&callback_data
);
1101 // For running write barriers from Rust. Required when we add a new edge in the
1102 // object graph from `old` to `young`.
1104 rb_yjit_obj_written(VALUE old
, VALUE young
, const char *file
, int line
)
1106 rb_obj_written(old
, Qundef
, young
, file
, line
);
1109 // Acquire the VM lock and then signal all other Ruby threads (ractors) to
1110 // contend for the VM lock, putting them to sleep. YJIT uses this to evict
1111 // threads running inside generated code so among other things, it can
1112 // safely change memory protection of regions housing generated code.
1114 rb_yjit_vm_lock_then_barrier(unsigned int *recursive_lock_level
, const char *file
, int line
)
1116 rb_vm_lock_enter(recursive_lock_level
, file
, line
);
1120 // Release the VM lock. The lock level must point to the same integer used to
1121 // acquire the lock.
1123 rb_yjit_vm_unlock(unsigned int *recursive_lock_level
, const char *file
, int line
)
1125 rb_vm_lock_leave(recursive_lock_level
, file
, line
);
1129 rb_yjit_compile_iseq(const rb_iseq_t
*iseq
, rb_execution_context_t
*ec
, bool jit_exception
)
1134 // Compile a block version starting at the current instruction
1135 uint8_t *rb_yjit_iseq_gen_entry_point(const rb_iseq_t
*iseq
, rb_execution_context_t
*ec
, bool jit_exception
); // defined in Rust
1136 uint8_t *code_ptr
= rb_yjit_iseq_gen_entry_point(iseq
, ec
, jit_exception
);
1138 if (jit_exception
) {
1139 iseq
->body
->jit_exception
= (rb_jit_func_t
)code_ptr
;
1142 iseq
->body
->jit_entry
= (rb_jit_func_t
)code_ptr
;
1148 // GC root for interacting with the GC
1149 struct yjit_root_struct
{
1150 bool unused
; // empty structs are not legal in C99
1154 yjit_root_free(void *ptr
)
1156 // Do nothing. The root lives as long as the process.
1160 yjit_root_memsize(const void *ptr
)
1162 // Count off-gc-heap allocation size of the dependency table
1163 return 0; // TODO: more accurate accounting
1166 void rb_yjit_root_mark(void *ptr
); // in Rust
1167 void rb_yjit_root_update_references(void *ptr
); // in Rust
1169 // Custom type for interacting with the GC
1170 // TODO: make this write barrier protected
1171 static const rb_data_type_t yjit_root_type
= {
1173 {rb_yjit_root_mark
, yjit_root_free
, yjit_root_memsize
, rb_yjit_root_update_references
},
1174 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1177 // For dealing with refinements
1179 rb_yjit_invalidate_all_method_lookup_assumptions(void)
1181 // It looks like Module#using actually doesn't need to invalidate all the
1182 // method caches, so we do nothing here for now.
1185 // Number of object shapes, which might be useful for investigating YJIT exit reasons.
1187 object_shape_count(rb_execution_context_t
*ec
, VALUE self
)
1189 // next_shape_id starts from 0, so it's the same as the count
1190 return ULONG2NUM((unsigned long)GET_SHAPE_TREE()->next_shape_id
);
1193 // Assert that we have the VM lock. Relevant mostly for multi ractor situations.
1194 // The GC takes the lock before calling us, and this asserts that it indeed happens.
1196 rb_yjit_assert_holding_vm_lock(void)
1198 ASSERT_vm_locking();
1201 // The number of stack slots that vm_sendish() pops for send and invokesuper.
1203 rb_yjit_sendish_sp_pops(const struct rb_callinfo
*ci
)
1205 return 1 - sp_inc_of_sendish(ci
); // + 1 to ignore return value push
1208 // The number of stack slots that vm_sendish() pops for invokeblock.
1210 rb_yjit_invokeblock_sp_pops(const struct rb_callinfo
*ci
)
1212 return 1 - sp_inc_of_invokeblock(ci
); // + 1 to ignore return value push
1215 // Setup jit_return to avoid returning a non-Qundef value on a non-FINISH frame.
1216 // See [jit_compile_exception] for details.
1218 rb_yjit_set_exception_return(rb_control_frame_t
*cfp
, void *leave_exit
, void *leave_exception
)
1220 if (VM_FRAME_FINISHED_P(cfp
)) {
1221 // If it's a FINISH frame, just normally exit with a non-Qundef value.
1222 cfp
->jit_return
= leave_exit
;
1224 else if (cfp
->jit_return
) {
1225 while (!VM_FRAME_FINISHED_P(cfp
)) {
1226 if (cfp
->jit_return
== leave_exit
) {
1227 // Unlike jit_exec(), leave_exit is not safe on a non-FINISH frame on
1228 // jit_exec_exception(). See [jit_exec] and [jit_exec_exception] for
1229 // details. Exit to the interpreter with Qundef to let it keep executing
1230 // other Ruby frames.
1231 cfp
->jit_return
= leave_exception
;
1234 cfp
= RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp
);
1238 // If the caller was not JIT code, exit to the interpreter with Qundef
1239 // to keep executing Ruby frames with the interpreter.
1240 cfp
->jit_return
= leave_exception
;
1244 // Primitives used by yjit.rb
1245 VALUE
rb_yjit_stats_enabled_p(rb_execution_context_t
*ec
, VALUE self
);
1246 VALUE
rb_yjit_print_stats_p(rb_execution_context_t
*ec
, VALUE self
);
1247 VALUE
rb_yjit_trace_exit_locations_enabled_p(rb_execution_context_t
*ec
, VALUE self
);
1248 VALUE
rb_yjit_get_stats(rb_execution_context_t
*ec
, VALUE self
, VALUE context
);
1249 VALUE
rb_yjit_reset_stats_bang(rb_execution_context_t
*ec
, VALUE self
);
1250 VALUE
rb_yjit_disasm_iseq(rb_execution_context_t
*ec
, VALUE self
, VALUE iseq
);
1251 VALUE
rb_yjit_insns_compiled(rb_execution_context_t
*ec
, VALUE self
, VALUE iseq
);
1252 VALUE
rb_yjit_code_gc(rb_execution_context_t
*ec
, VALUE self
);
1253 VALUE
rb_yjit_simulate_oom_bang(rb_execution_context_t
*ec
, VALUE self
);
1254 VALUE
rb_yjit_get_exit_locations(rb_execution_context_t
*ec
, VALUE self
);
1255 VALUE
rb_yjit_enable(rb_execution_context_t
*ec
, VALUE self
, VALUE gen_stats
, VALUE print_stats
);
1257 // Preprocessed yjit.rb generated during build
1258 #include "yjit.rbinc"
1260 // Initialize the GC hooks
1262 rb_yjit_init_gc_hooks(void)
1264 struct yjit_root_struct
*root
;
1265 VALUE yjit_root
= TypedData_Make_Struct(0, struct yjit_root_struct
, &yjit_root_type
, root
);
1266 rb_vm_register_global_object(yjit_root
);