2 * tramp-amd64.c: JIT trampoline code for amd64
5 * Dietmar Maurer (dietmar@ximian.com)
6 * Zoltan Varga (vargaz@gmail.com)
8 * (C) 2001 Ximian, Inc.
14 #include <mono/metadata/appdomain.h>
15 #include <mono/metadata/marshal.h>
16 #include <mono/metadata/tabledefs.h>
17 #include <mono/metadata/mono-debug-debugger.h>
18 #include <mono/metadata/monitor.h>
19 #include <mono/arch/amd64/amd64-codegen.h>
21 #include <mono/utils/memcheck.h>
24 #include "mini-amd64.h"
26 #define IS_REX(inst) (((inst) >= 0x40) && ((inst) <= 0x4f))
28 static guint8
* nullified_class_init_trampoline
;
31 * mono_arch_get_unbox_trampoline:
32 * @gsctx: the generic sharing context
34 * @addr: pointer to native code for @m
36 * when value type methods are called through the vtable we need to unbox the
37 * this argument. This method returns a pointer to a trampoline which does
38 * unboxing before calling the method
41 mono_arch_get_unbox_trampoline (MonoGenericSharingContext
*gsctx
, MonoMethod
*m
, gpointer addr
)
46 MonoDomain
*domain
= mono_domain_get ();
48 this_reg
= mono_arch_get_this_arg_reg (mono_method_signature (m
), gsctx
, NULL
);
50 start
= code
= mono_domain_code_reserve (domain
, 20);
52 amd64_alu_reg_imm (code
, X86_ADD
, this_reg
, sizeof (MonoObject
));
53 /* FIXME: Optimize this */
54 amd64_mov_reg_imm (code
, AMD64_RAX
, addr
);
55 amd64_jump_reg (code
, AMD64_RAX
);
56 g_assert ((code
- start
) < 20);
58 mono_arch_flush_icache (start
, code
- start
);
64 * mono_arch_get_static_rgctx_trampoline:
66 * Create a trampoline which sets RGCTX_REG to MRGCTX, then jumps to ADDR.
69 mono_arch_get_static_rgctx_trampoline (MonoMethod
*m
, MonoMethodRuntimeGenericContext
*mrgctx
, gpointer addr
)
74 MonoDomain
*domain
= mono_domain_get ();
76 #ifdef MONO_ARCH_NOMAP32BIT
79 /* AOTed code could still have a non-32 bit address */
80 if ((((guint64
)addr
) >> 32) == 0)
86 start
= code
= mono_domain_code_reserve (domain
, buf_len
);
88 amd64_mov_reg_imm (code
, MONO_ARCH_RGCTX_REG
, mrgctx
);
89 amd64_jump_code (code
, addr
);
90 g_assert ((code
- start
) < buf_len
);
92 mono_arch_flush_icache (start
, code
- start
);
98 mono_arch_get_llvm_imt_trampoline (MonoDomain
*domain
, MonoMethod
*m
, int vt_offset
)
100 guint8
*code
, *start
;
106 start
= code
= mono_domain_code_reserve (domain
, buf_len
);
108 this_reg
= mono_arch_get_this_arg_reg (mono_method_signature (m
), NULL
, NULL
);
111 amd64_mov_reg_imm (code
, MONO_ARCH_IMT_REG
, m
);
112 /* Load vtable address */
113 amd64_mov_reg_membase (code
, AMD64_RAX
, this_reg
, 0, 8);
114 amd64_jump_membase (code
, AMD64_RAX
, vt_offset
);
117 g_assert ((code
- start
) < buf_len
);
119 mono_arch_flush_icache (start
, code
- start
);
125 * mono_arch_patch_callsite:
127 * Patch the callsite whose address is given by ORIG_CODE so it calls ADDR. ORIG_CODE
128 * points to the pc right after the call.
131 mono_arch_patch_callsite (guint8
*method_start
, guint8
*orig_code
, guint8
*addr
)
135 gboolean can_write
= mono_breakpoint_clean_code (method_start
, orig_code
, 14, buf
, sizeof (buf
));
139 if (((code
[-13] == 0x49) && (code
[-12] == 0xbb)) || (code
[-5] == 0xe8)) {
140 if (code
[-5] != 0xe8) {
142 InterlockedExchangePointer ((gpointer
*)(orig_code
- 11), addr
);
143 VALGRIND_DISCARD_TRANSLATIONS (orig_code
- 11, sizeof (gpointer
));
146 if ((((guint64
)(addr
)) >> 32) != 0) {
147 #ifdef MONO_ARCH_NOMAP32BIT
148 /* Print some diagnostics */
149 MonoJitInfo
*ji
= mono_jit_info_table_find (mono_domain_get (), (char*)orig_code
);
151 fprintf (stderr
, "At %s, offset 0x%zx\n", mono_method_full_name (ji
->method
, TRUE
), (guint8
*)orig_code
- (guint8
*)ji
->code_start
);
152 fprintf (stderr
, "Addr: %p\n", addr
);
153 ji
= mono_jit_info_table_find (mono_domain_get (), (char*)addr
);
155 fprintf (stderr
, "Callee: %s\n", mono_method_full_name (ji
->method
, TRUE
));
156 g_assert_not_reached ();
159 * This might happen when calling AOTed code. Create a thunk.
161 guint8
*thunk_start
, *thunk_code
;
163 thunk_start
= thunk_code
= mono_domain_code_reserve (mono_domain_get (), 32);
164 amd64_jump_membase (thunk_code
, AMD64_RIP
, 0);
165 *(guint64
*)thunk_code
= (guint64
)addr
;
167 g_assert ((((guint64
)(addr
)) >> 32) == 0);
168 mono_arch_flush_icache (thunk_start
, thunk_code
- thunk_start
);
171 g_assert ((((guint64
)(orig_code
)) >> 32) == 0);
173 InterlockedExchange ((gint32
*)(orig_code
- 4), ((gint64
)addr
- (gint64
)orig_code
));
174 VALGRIND_DISCARD_TRANSLATIONS (orig_code
- 5, 4);
178 else if ((code
[-7] == 0x41) && (code
[-6] == 0xff) && (code
[-5] == 0x15)) {
179 /* call *<OFFSET>(%rip) */
180 gpointer
*got_entry
= (gpointer
*)((guint8
*)orig_code
+ (*(guint32
*)(orig_code
- 4)));
182 InterlockedExchangePointer (got_entry
, addr
);
183 VALGRIND_DISCARD_TRANSLATIONS (orig_code
- 5, sizeof (gpointer
));
189 mono_arch_patch_plt_entry (guint8
*code
, gpointer
*got
, mgreg_t
*regs
, guint8
*addr
)
192 gpointer
*plt_jump_table_entry
;
194 /* A PLT entry: jmp *<DISP>(%rip) */
195 g_assert (code
[0] == 0xff);
196 g_assert (code
[1] == 0x25);
198 disp
= *(gint32
*)(code
+ 2);
200 plt_jump_table_entry
= (gpointer
*)(code
+ 6 + disp
);
202 InterlockedExchangePointer (plt_jump_table_entry
, addr
);
206 mono_arch_nullify_class_init_trampoline (guint8
*code
, mgreg_t
*regs
)
209 MonoJitInfo
*ji
= NULL
;
213 /* code - 7 might be before the start of the method */
214 /* FIXME: Avoid this expensive call somehow */
215 ji
= mono_jit_info_table_find (mono_domain_get (), (char*)code
);
218 can_write
= mono_breakpoint_clean_code (ji
? ji
->code_start
: NULL
, code
, 7, buf
, sizeof (buf
));
226 * A given byte sequence can match more than case here, so we have to be
227 * really careful about the ordering of the cases. Longer sequences
230 if ((buf
[0] == 0x41) && (buf
[1] == 0xff) && (buf
[2] == 0x15)) {
231 gpointer
*vtable_slot
;
233 /* call *<OFFSET>(%rip) */
234 vtable_slot
= mono_get_vcall_slot_addr (code
+ 3, regs
);
235 g_assert (vtable_slot
);
237 *vtable_slot
= nullified_class_init_trampoline
;
238 } else if (buf
[2] == 0xe8) {
240 //guint8 *buf = code - 2;
243 * It would be better to replace the call with nops, but that doesn't seem
244 * to work on SMP machines even when the whole call is inside a cache line.
245 * Patching the call address seems to work.
255 mono_arch_patch_callsite (code
- 2, code
- 2 + 5, nullified_class_init_trampoline
);
256 } else if ((buf
[5] == 0xff) && x86_modrm_mod (buf
[6]) == 3 && x86_modrm_reg (buf
[6]) == 2) {
258 /* Generated by the LLVM JIT or on platforms without MAP_32BIT set */
261 /* FIXME: Not thread safe */
264 } else if (buf
[4] == 0x90 || buf
[5] == 0xeb || buf
[6] == 0x66) {
265 /* Already changed by another thread */
268 printf ("Invalid trampoline sequence: %x %x %x %x %x %x %x\n", buf
[0], buf
[1], buf
[2], buf
[3],
269 buf
[4], buf
[5], buf
[6]);
270 g_assert_not_reached ();
275 mono_arch_nullify_plt_entry (guint8
*code
, mgreg_t
*regs
)
277 if (mono_aot_only
&& !nullified_class_init_trampoline
)
278 nullified_class_init_trampoline
= mono_aot_get_named_code ("nullified_class_init_trampoline");
280 mono_arch_patch_plt_entry (code
, NULL
, regs
, nullified_class_init_trampoline
);
284 mono_arch_create_trampoline_code (MonoTrampolineType tramp_type
)
289 GSList
*unwind_ops
, *l
;
291 code
= mono_arch_create_trampoline_code_full (tramp_type
, &code_size
, &ji
, &unwind_ops
, FALSE
);
293 mono_save_trampoline_xdebug_info ("<generic_trampoline>", code
, code_size
, unwind_ops
);
295 for (l
= unwind_ops
; l
; l
= l
->next
)
297 g_slist_free (unwind_ops
);
303 mono_arch_create_trampoline_code_full (MonoTrampolineType tramp_type
, guint32
*code_size
, MonoJumpInfo
**ji
, GSList
**out_unwind_ops
, gboolean aot
)
305 guint8
*buf
, *code
, *tramp
, *br
[2], *r11_save_code
, *after_r11_save_code
;
306 int i
, lmf_offset
, offset
, res_offset
, arg_offset
, rax_offset
, tramp_offset
, saved_regs_offset
;
307 int saved_fpregs_offset
, rbp_offset
, framesize
, orig_rsp_to_rbp_offset
, cfa_offset
;
309 GSList
*unwind_ops
= NULL
;
311 if (tramp_type
== MONO_TRAMPOLINE_JUMP
)
316 code
= buf
= mono_global_codeman_reserve (538);
320 framesize
= 538 + sizeof (MonoLMF
);
321 framesize
= (framesize
+ (MONO_ARCH_FRAME_ALIGNMENT
- 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT
- 1);
323 orig_rsp_to_rbp_offset
= 0;
324 r11_save_code
= code
;
325 /* Reserve 5 bytes for the mov_membase_reg to save R11 */
327 after_r11_save_code
= code
;
329 // CFA = sp + 16 (the trampoline address is on the stack)
331 mono_add_unwind_op_def_cfa (unwind_ops
, code
, buf
, AMD64_RSP
, 16);
332 // IP saved at CFA - 8
333 mono_add_unwind_op_offset (unwind_ops
, code
, buf
, AMD64_RIP
, -8);
335 /* Pop the return address off the stack */
336 amd64_pop_reg (code
, AMD64_R11
);
337 orig_rsp_to_rbp_offset
+= 8;
340 mono_add_unwind_op_def_cfa_offset (unwind_ops
, code
, buf
, cfa_offset
);
343 * Allocate a new stack frame
345 amd64_push_reg (code
, AMD64_RBP
);
347 mono_add_unwind_op_def_cfa_offset (unwind_ops
, code
, buf
, cfa_offset
);
348 mono_add_unwind_op_offset (unwind_ops
, code
, buf
, AMD64_RBP
, - cfa_offset
);
350 orig_rsp_to_rbp_offset
-= 8;
351 amd64_mov_reg_reg (code
, AMD64_RBP
, AMD64_RSP
, 8);
352 mono_add_unwind_op_def_cfa_reg (unwind_ops
, code
, buf
, AMD64_RBP
);
353 amd64_alu_reg_imm (code
, X86_SUB
, AMD64_RSP
, framesize
);
356 rbp_offset
= - offset
;
359 rax_offset
= - offset
;
362 tramp_offset
= - offset
;
365 arg_offset
= - offset
;
367 /* Compute the trampoline address from the return address */
369 /* 7 = length of call *<offset>(rip) */
370 amd64_alu_reg_imm (code
, X86_SUB
, AMD64_R11
, 7);
372 /* 5 = length of amd64_call_membase () */
373 amd64_alu_reg_imm (code
, X86_SUB
, AMD64_R11
, 5);
375 amd64_mov_membase_reg (code
, AMD64_RBP
, tramp_offset
, AMD64_R11
, 8);
378 res_offset
= - offset
;
380 /* Save all registers */
382 offset
+= AMD64_NREG
* 8;
383 saved_regs_offset
= - offset
;
384 for (i
= 0; i
< AMD64_NREG
; ++i
) {
385 if (i
== AMD64_RBP
) {
386 /* RAX is already saved */
387 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RBP
, rbp_offset
, 8);
388 amd64_mov_membase_reg (code
, AMD64_RBP
, saved_regs_offset
+ (i
* 8), AMD64_RAX
, 8);
389 } else if (i
!= AMD64_R11
) {
390 amd64_mov_membase_reg (code
, AMD64_RBP
, saved_regs_offset
+ (i
* 8), i
, 8);
392 /* We have to save R11 right at the start of
393 the trampoline code because it's used as a
395 amd64_mov_membase_reg (r11_save_code
, AMD64_RSP
, saved_regs_offset
+ orig_rsp_to_rbp_offset
+ (i
* 8), i
, 8);
396 g_assert (r11_save_code
== after_r11_save_code
);
400 saved_fpregs_offset
= - offset
;
401 for (i
= 0; i
< 8; ++i
)
402 amd64_movsd_membase_reg (code
, AMD64_RBP
, saved_fpregs_offset
+ (i
* 8), i
);
404 if (tramp_type
!= MONO_TRAMPOLINE_GENERIC_CLASS_INIT
&&
405 tramp_type
!= MONO_TRAMPOLINE_MONITOR_ENTER
&&
406 tramp_type
!= MONO_TRAMPOLINE_MONITOR_EXIT
) {
407 /* Obtain the trampoline argument which is encoded in the instruction stream */
409 /* Load the GOT offset */
410 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RBP
, tramp_offset
, 8);
411 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_R11
, 7, 4);
412 /* Compute the address of the GOT slot */
413 amd64_alu_reg_reg_size (code
, X86_ADD
, AMD64_R11
, AMD64_RAX
, 8);
415 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_R11
, 0, 8);
417 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RBP
, tramp_offset
, 8);
418 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_R11
, 5, 1);
419 amd64_widen_reg (code
, AMD64_RAX
, AMD64_RAX
, TRUE
, FALSE
);
420 amd64_alu_reg_imm_size (code
, X86_CMP
, AMD64_RAX
, 4, 1);
422 x86_branch8 (code
, X86_CC_NE
, 6, FALSE
);
423 /* 32 bit immediate */
424 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_R11
, 6, 4);
426 x86_jump8 (code
, 10);
427 /* 64 bit immediate */
428 mono_amd64_patch (br
[0], code
);
429 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_R11
, 6, 8);
430 mono_amd64_patch (br
[1], code
);
432 amd64_mov_membase_reg (code
, AMD64_RBP
, arg_offset
, AMD64_R11
, 8);
434 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RBP
, saved_regs_offset
+ (MONO_AMD64_ARG_REG1
* 8), 8);
435 amd64_mov_membase_reg (code
, AMD64_RBP
, arg_offset
, AMD64_R11
, 8);
440 offset
+= sizeof (MonoLMF
);
441 lmf_offset
= - offset
;
445 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RBP
, 8, 8);
447 amd64_mov_reg_imm (code
, AMD64_R11
, 0);
448 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, rip
), AMD64_R11
, 8);
450 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RSP
, framesize
, 8);
451 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, rbp
), AMD64_R11
, 8);
453 amd64_mov_reg_reg (code
, AMD64_R11
, AMD64_RSP
, 8);
454 amd64_alu_reg_imm (code
, X86_ADD
, AMD64_R11
, framesize
+ 16);
455 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, rsp
), AMD64_R11
, 8);
457 if (tramp_type
== MONO_TRAMPOLINE_JIT
|| tramp_type
== MONO_TRAMPOLINE_JUMP
) {
458 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RBP
, arg_offset
, 8);
459 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, method
), AMD64_R11
, 8);
461 amd64_mov_membase_imm (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, method
), 0, 8);
463 /* Save callee saved regs */
465 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, rdi
), AMD64_RDI
, 8);
466 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, rsi
), AMD64_RSI
, 8);
468 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, rbx
), AMD64_RBX
, 8);
469 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, r12
), AMD64_R12
, 8);
470 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, r13
), AMD64_R13
, 8);
471 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, r14
), AMD64_R14
, 8);
472 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, r15
), AMD64_R15
, 8);
475 *ji
= mono_patch_info_list_prepend (*ji
, code
- buf
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, "mono_get_lmf_addr");
476 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RIP
, 0, 8);
478 amd64_mov_reg_imm (code
, AMD64_R11
, mono_get_lmf_addr
);
480 amd64_call_reg (code
, AMD64_R11
);
483 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, lmf_addr
), AMD64_RAX
, 8);
484 /* Save previous_lmf */
485 /* Set the lowest bit to 1 to signal that this LMF has the ip field set */
486 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RAX
, 0, 8);
487 amd64_alu_reg_imm_size (code
, X86_ADD
, AMD64_R11
, 1, 8);
488 amd64_mov_membase_reg (code
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, previous_lmf
), AMD64_R11
, 8);
490 amd64_lea_membase (code
, AMD64_R11
, AMD64_RBP
, lmf_offset
);
491 amd64_mov_membase_reg (code
, AMD64_RAX
, 0, AMD64_R11
, 8);
495 /* Arg1 is the pointer to the saved registers */
496 amd64_lea_membase (code
, AMD64_ARG_REG1
, AMD64_RBP
, saved_regs_offset
);
498 /* Arg2 is the address of the calling code */
500 amd64_mov_reg_membase (code
, AMD64_ARG_REG2
, AMD64_RBP
, 8, 8);
502 amd64_mov_reg_imm (code
, AMD64_ARG_REG2
, 0);
504 /* Arg3 is the method/vtable ptr */
505 amd64_mov_reg_membase (code
, AMD64_ARG_REG3
, AMD64_RBP
, arg_offset
, 8);
507 /* Arg4 is the trampoline address */
508 amd64_mov_reg_membase (code
, AMD64_ARG_REG4
, AMD64_RBP
, tramp_offset
, 8);
511 char *icall_name
= g_strdup_printf ("trampoline_func_%d", tramp_type
);
512 *ji
= mono_patch_info_list_prepend (*ji
, code
- buf
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, icall_name
);
513 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RIP
, 0, 8);
515 tramp
= (guint8
*)mono_get_trampoline_func (tramp_type
);
516 amd64_mov_reg_imm (code
, AMD64_RAX
, tramp
);
518 amd64_call_reg (code
, AMD64_RAX
);
520 /* Check for thread interruption */
521 /* This is not perf critical code so no need to check the interrupt flag */
523 * Have to call the _force_ variant, since there could be a protected wrapper on the top of the stack.
525 amd64_mov_membase_reg (code
, AMD64_RBP
, res_offset
, AMD64_RAX
, 8);
527 *ji
= mono_patch_info_list_prepend (*ji
, code
- buf
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, "mono_thread_force_interruption_checkpoint");
528 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RIP
, 0, 8);
530 amd64_mov_reg_imm (code
, AMD64_RAX
, (guint8
*)mono_thread_force_interruption_checkpoint
);
532 amd64_call_reg (code
, AMD64_RAX
);
533 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RBP
, res_offset
, 8);
537 amd64_mov_reg_membase (code
, AMD64_RCX
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, previous_lmf
), 8);
538 amd64_alu_reg_imm_size (code
, X86_SUB
, AMD64_RCX
, 1, 8);
539 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RBP
, lmf_offset
+ G_STRUCT_OFFSET (MonoLMF
, lmf_addr
), 8);
540 amd64_mov_membase_reg (code
, AMD64_R11
, 0, AMD64_RCX
, 8);
543 * Save rax to the stack, after the leave instruction, this will become part of
546 amd64_mov_membase_reg (code
, AMD64_RBP
, rax_offset
, AMD64_RAX
, 8);
548 /* Restore argument registers, r10 (needed to pass rgctx to
549 static shared generic methods), r11 (imt register for
550 interface calls), and rax (needed for direct calls to C vararg functions). */
551 for (i
= 0; i
< AMD64_NREG
; ++i
)
552 if (AMD64_IS_ARGUMENT_REG (i
) || i
== AMD64_R10
|| i
== AMD64_R11
|| i
== AMD64_RAX
)
553 amd64_mov_reg_membase (code
, i
, AMD64_RBP
, saved_regs_offset
+ (i
* 8), 8);
555 for (i
= 0; i
< 8; ++i
)
556 amd64_movsd_reg_membase (code
, i
, AMD64_RBP
, saved_fpregs_offset
+ (i
* 8));
561 if (MONO_TRAMPOLINE_TYPE_MUST_RETURN (tramp_type
)) {
563 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RSP
, rax_offset
- 0x8, 8);
566 /* call the compiled method using the saved rax */
567 amd64_jump_membase (code
, AMD64_RSP
, rax_offset
- 0x8);
570 g_assert ((code
- buf
) <= 538);
572 mono_arch_flush_icache (buf
, code
- buf
);
574 *code_size
= code
- buf
;
576 if (tramp_type
== MONO_TRAMPOLINE_CLASS_INIT
) {
579 /* Initialize the nullified class init trampoline used in the AOT case */
580 nullified_class_init_trampoline
= mono_arch_get_nullified_class_init_trampoline (&code_len
);
583 *out_unwind_ops
= unwind_ops
;
589 mono_arch_get_nullified_class_init_trampoline (guint32
*code_len
)
593 code
= buf
= mono_global_codeman_reserve (16);
596 mono_arch_flush_icache (buf
, code
- buf
);
598 *code_len
= code
- buf
;
604 mono_arch_create_specific_trampoline (gpointer arg1
, MonoTrampolineType tramp_type
, MonoDomain
*domain
, guint32
*code_len
)
606 guint8
*code
, *buf
, *tramp
;
609 tramp
= mono_get_trampoline_code (tramp_type
);
611 if ((((guint64
)arg1
) >> 32) == 0)
616 code
= buf
= mono_domain_code_reserve_align (domain
, size
, 1);
618 amd64_call_code (code
, tramp
);
619 /* The trampoline code will obtain the argument from the instruction stream */
620 if ((((guint64
)arg1
) >> 32) == 0) {
622 *(guint32
*)(code
+ 1) = (gint64
)arg1
;
626 *(guint64
*)(code
+ 1) = (gint64
)arg1
;
630 g_assert ((code
- buf
) <= size
);
635 mono_arch_flush_icache (buf
, size
);
641 mono_arch_create_rgctx_lazy_fetch_trampoline (guint32 slot
)
646 return mono_arch_create_rgctx_lazy_fetch_trampoline_full (slot
, &code_size
, &ji
, FALSE
);
650 mono_arch_create_rgctx_lazy_fetch_trampoline_full (guint32 slot
, guint32
*code_size
, MonoJumpInfo
**ji
, gboolean aot
)
654 guint8
**rgctx_null_jumps
;
662 mrgctx
= MONO_RGCTX_SLOT_IS_MRGCTX (slot
);
663 index
= MONO_RGCTX_SLOT_INDEX (slot
);
665 index
+= MONO_SIZEOF_METHOD_RUNTIME_GENERIC_CONTEXT
/ sizeof (gpointer
);
666 for (depth
= 0; ; ++depth
) {
667 int size
= mono_class_rgctx_get_array_size (depth
, mrgctx
);
669 if (index
< size
- 1)
674 tramp_size
= 64 + 8 * depth
;
676 code
= buf
= mono_global_codeman_reserve (tramp_size
);
678 rgctx_null_jumps
= g_malloc (sizeof (guint8
*) * (depth
+ 2));
682 amd64_mov_reg_reg (code
, AMD64_RAX
, AMD64_ARG_REG1
, 8);
684 /* load rgctx ptr from vtable */
685 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_ARG_REG1
, G_STRUCT_OFFSET (MonoVTable
, runtime_generic_context
), 8);
686 /* is the rgctx ptr null? */
687 amd64_test_reg_reg (code
, AMD64_RAX
, AMD64_RAX
);
688 /* if yes, jump to actual trampoline */
689 rgctx_null_jumps
[0] = code
;
690 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
693 for (i
= 0; i
< depth
; ++i
) {
694 /* load ptr to next array */
695 if (mrgctx
&& i
== 0)
696 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RAX
, MONO_SIZEOF_METHOD_RUNTIME_GENERIC_CONTEXT
, 8);
698 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RAX
, 0, 8);
699 /* is the ptr null? */
700 amd64_test_reg_reg (code
, AMD64_RAX
, AMD64_RAX
);
701 /* if yes, jump to actual trampoline */
702 rgctx_null_jumps
[i
+ 1] = code
;
703 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
707 amd64_mov_reg_membase (code
, AMD64_RAX
, AMD64_RAX
, sizeof (gpointer
) * (index
+ 1), 8);
708 /* is the slot null? */
709 amd64_test_reg_reg (code
, AMD64_RAX
, AMD64_RAX
);
710 /* if yes, jump to actual trampoline */
711 rgctx_null_jumps
[depth
+ 1] = code
;
712 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
713 /* otherwise return */
716 for (i
= mrgctx
? 1 : 0; i
<= depth
+ 1; ++i
)
717 x86_patch (rgctx_null_jumps
[i
], code
);
719 g_free (rgctx_null_jumps
);
721 /* move the rgctx pointer to the VTABLE register */
722 amd64_mov_reg_reg (code
, MONO_ARCH_VTABLE_REG
, AMD64_ARG_REG1
, 8);
725 *ji
= mono_patch_info_list_prepend (*ji
, code
- buf
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, g_strdup_printf ("specific_trampoline_lazy_fetch_%u", slot
));
726 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RIP
, 0, 8);
727 amd64_jump_reg (code
, AMD64_R11
);
729 tramp
= mono_arch_create_specific_trampoline (GUINT_TO_POINTER (slot
), MONO_TRAMPOLINE_RGCTX_LAZY_FETCH
, mono_get_root_domain (), NULL
);
731 /* jump to the actual trampoline */
732 amd64_jump_code (code
, tramp
);
735 mono_arch_flush_icache (buf
, code
- buf
);
737 g_assert (code
- buf
<= tramp_size
);
739 *code_size
= code
- buf
;
745 mono_arch_create_generic_class_init_trampoline (void)
750 return mono_arch_create_generic_class_init_trampoline_full (&code_size
, &ji
, FALSE
);
754 mono_arch_create_generic_class_init_trampoline_full (guint32
*code_size
, MonoJumpInfo
**ji
, gboolean aot
)
758 static int byte_offset
= -1;
759 static guint8 bitmask
;
767 code
= buf
= mono_global_codeman_reserve (tramp_size
);
770 mono_marshal_find_bitfield_offset (MonoVTable
, initialized
, &byte_offset
, &bitmask
);
772 amd64_test_membase_imm_size (code
, MONO_AMD64_ARG_REG1
, byte_offset
, bitmask
, 1);
774 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
778 x86_patch (jump
, code
);
781 *ji
= mono_patch_info_list_prepend (*ji
, code
- buf
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, "specific_trampoline_generic_class_init");
782 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RIP
, 0, 8);
783 amd64_jump_reg (code
, AMD64_R11
);
785 tramp
= mono_arch_create_specific_trampoline (NULL
, MONO_TRAMPOLINE_GENERIC_CLASS_INIT
, mono_get_root_domain (), NULL
);
787 /* jump to the actual trampoline */
788 amd64_jump_code (code
, tramp
);
791 mono_arch_flush_icache (buf
, code
- buf
);
793 g_assert (code
- buf
<= tramp_size
);
795 *code_size
= code
- buf
;
800 #ifdef MONO_ARCH_MONITOR_OBJECT_REG
803 mono_arch_create_monitor_enter_trampoline (void)
808 return mono_arch_create_monitor_enter_trampoline_full (&code_size
, &ji
, FALSE
);
812 mono_arch_create_monitor_enter_trampoline_full (guint32
*code_size
, MonoJumpInfo
**ji
, gboolean aot
)
817 guint8
*jump_obj_null
, *jump_sync_null
, *jump_cmpxchg_failed
, *jump_other_owner
, *jump_tid
;
819 int owner_offset
, nest_offset
, dummy
;
823 g_assert (MONO_ARCH_MONITOR_OBJECT_REG
== AMD64_RDI
);
825 mono_monitor_threads_sync_members_offset (&owner_offset
, &nest_offset
, &dummy
);
826 g_assert (MONO_THREADS_SYNC_MEMBER_SIZE (owner_offset
) == sizeof (gpointer
));
827 g_assert (MONO_THREADS_SYNC_MEMBER_SIZE (nest_offset
) == sizeof (guint32
));
828 owner_offset
= MONO_THREADS_SYNC_MEMBER_OFFSET (owner_offset
);
829 nest_offset
= MONO_THREADS_SYNC_MEMBER_OFFSET (nest_offset
);
833 code
= buf
= mono_global_codeman_reserve (tramp_size
);
835 if (mono_thread_get_tls_offset () != -1) {
836 /* MonoObject* obj is in RDI */
838 amd64_test_reg_reg (code
, AMD64_RDI
, AMD64_RDI
);
839 /* if yes, jump to actual trampoline */
840 jump_obj_null
= code
;
841 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
843 /* load obj->synchronization to RCX */
844 amd64_mov_reg_membase (code
, AMD64_RCX
, AMD64_RDI
, G_STRUCT_OFFSET (MonoObject
, synchronisation
), 8);
845 /* is synchronization null? */
846 amd64_test_reg_reg (code
, AMD64_RCX
, AMD64_RCX
);
847 /* if yes, jump to actual trampoline */
848 jump_sync_null
= code
;
849 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
851 /* load MonoInternalThread* into RDX */
852 code
= mono_amd64_emit_tls_get (code
, AMD64_RDX
, mono_thread_get_tls_offset ());
853 /* load TID into RDX */
854 amd64_mov_reg_membase (code
, AMD64_RDX
, AMD64_RDX
, G_STRUCT_OFFSET (MonoInternalThread
, tid
), 8);
856 /* is synchronization->owner null? */
857 amd64_alu_membase_imm_size (code
, X86_CMP
, AMD64_RCX
, owner_offset
, 0, 8);
858 /* if not, jump to next case */
860 amd64_branch8 (code
, X86_CC_NZ
, -1, 1);
862 /* if yes, try a compare-exchange with the TID */
864 amd64_alu_reg_reg (code
, X86_XOR
, AMD64_RAX
, AMD64_RAX
);
865 /* compare and exchange */
866 amd64_prefix (code
, X86_LOCK_PREFIX
);
867 amd64_cmpxchg_membase_reg_size (code
, AMD64_RCX
, owner_offset
, AMD64_RDX
, 8);
868 /* if not successful, jump to actual trampoline */
869 jump_cmpxchg_failed
= code
;
870 amd64_branch8 (code
, X86_CC_NZ
, -1, 1);
871 /* if successful, return */
874 /* next case: synchronization->owner is not null */
875 x86_patch (jump_tid
, code
);
876 /* is synchronization->owner == TID? */
877 amd64_alu_membase_reg_size (code
, X86_CMP
, AMD64_RCX
, owner_offset
, AMD64_RDX
, 8);
878 /* if not, jump to actual trampoline */
879 jump_other_owner
= code
;
880 amd64_branch8 (code
, X86_CC_NZ
, -1, 1);
881 /* if yes, increment nest */
882 amd64_inc_membase_size (code
, AMD64_RCX
, nest_offset
, 4);
886 x86_patch (jump_obj_null
, code
);
887 x86_patch (jump_sync_null
, code
);
888 x86_patch (jump_cmpxchg_failed
, code
);
889 x86_patch (jump_other_owner
, code
);
892 /* jump to the actual trampoline */
893 #if MONO_AMD64_ARG_REG1 != AMD64_RDI
894 amd64_mov_reg_reg (code
, MONO_AMD64_ARG_REG1
, AMD64_RDI
);
898 *ji
= mono_patch_info_list_prepend (*ji
, code
- buf
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, "specific_trampoline_monitor_enter");
899 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RIP
, 0, 8);
900 amd64_jump_reg (code
, AMD64_R11
);
902 tramp
= mono_arch_create_specific_trampoline (NULL
, MONO_TRAMPOLINE_MONITOR_ENTER
, mono_get_root_domain (), NULL
);
904 /* jump to the actual trampoline */
905 amd64_jump_code (code
, tramp
);
908 mono_arch_flush_icache (code
, code
- buf
);
909 g_assert (code
- buf
<= tramp_size
);
911 *code_size
= code
- buf
;
917 mono_arch_create_monitor_exit_trampoline (void)
922 return mono_arch_create_monitor_exit_trampoline_full (&code_size
, &ji
, FALSE
);
926 mono_arch_create_monitor_exit_trampoline_full (guint32
*code_size
, MonoJumpInfo
**ji
, gboolean aot
)
930 guint8
*jump_obj_null
, *jump_have_waiters
;
933 int owner_offset
, nest_offset
, entry_count_offset
;
937 g_assert (MONO_ARCH_MONITOR_OBJECT_REG
== AMD64_RDI
);
939 mono_monitor_threads_sync_members_offset (&owner_offset
, &nest_offset
, &entry_count_offset
);
940 g_assert (MONO_THREADS_SYNC_MEMBER_SIZE (owner_offset
) == sizeof (gpointer
));
941 g_assert (MONO_THREADS_SYNC_MEMBER_SIZE (nest_offset
) == sizeof (guint32
));
942 g_assert (MONO_THREADS_SYNC_MEMBER_SIZE (entry_count_offset
) == sizeof (gint32
));
943 owner_offset
= MONO_THREADS_SYNC_MEMBER_OFFSET (owner_offset
);
944 nest_offset
= MONO_THREADS_SYNC_MEMBER_OFFSET (nest_offset
);
945 entry_count_offset
= MONO_THREADS_SYNC_MEMBER_OFFSET (entry_count_offset
);
949 code
= buf
= mono_global_codeman_reserve (tramp_size
);
951 if (mono_thread_get_tls_offset () != -1) {
952 /* MonoObject* obj is in RDI */
954 amd64_test_reg_reg (code
, AMD64_RDI
, AMD64_RDI
);
955 /* if yes, jump to actual trampoline */
956 jump_obj_null
= code
;
957 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
959 /* load obj->synchronization to RCX */
960 amd64_mov_reg_membase (code
, AMD64_RCX
, AMD64_RDI
, G_STRUCT_OFFSET (MonoObject
, synchronisation
), 8);
961 /* is synchronization null? */
962 amd64_test_reg_reg (code
, AMD64_RCX
, AMD64_RCX
);
963 /* if not, jump to next case */
965 amd64_branch8 (code
, X86_CC_NZ
, -1, 1);
966 /* if yes, just return */
969 /* next case: synchronization is not null */
970 x86_patch (jump_next
, code
);
971 /* load MonoInternalThread* into RDX */
972 code
= mono_amd64_emit_tls_get (code
, AMD64_RDX
, mono_thread_get_tls_offset ());
973 /* load TID into RDX */
974 amd64_mov_reg_membase (code
, AMD64_RDX
, AMD64_RDX
, G_STRUCT_OFFSET (MonoInternalThread
, tid
), 8);
975 /* is synchronization->owner == TID */
976 amd64_alu_membase_reg_size (code
, X86_CMP
, AMD64_RCX
, owner_offset
, AMD64_RDX
, 8);
977 /* if yes, jump to next case */
979 amd64_branch8 (code
, X86_CC_Z
, -1, 1);
980 /* if not, just return */
983 /* next case: synchronization->owner == TID */
984 x86_patch (jump_next
, code
);
985 /* is synchronization->nest == 1 */
986 amd64_alu_membase_imm_size (code
, X86_CMP
, AMD64_RCX
, nest_offset
, 1, 4);
987 /* if not, jump to next case */
989 amd64_branch8 (code
, X86_CC_NZ
, -1, 1);
990 /* if yes, is synchronization->entry_count zero? */
991 amd64_alu_membase_imm_size (code
, X86_CMP
, AMD64_RCX
, entry_count_offset
, 0, 4);
992 /* if not, jump to actual trampoline */
993 jump_have_waiters
= code
;
994 amd64_branch8 (code
, X86_CC_NZ
, -1 , 1);
995 /* if yes, set synchronization->owner to null and return */
996 amd64_mov_membase_imm (code
, AMD64_RCX
, owner_offset
, 0, 8);
999 /* next case: synchronization->nest is not 1 */
1000 x86_patch (jump_next
, code
);
1001 /* decrease synchronization->nest and return */
1002 amd64_dec_membase_size (code
, AMD64_RCX
, nest_offset
, 4);
1005 x86_patch (jump_obj_null
, code
);
1006 x86_patch (jump_have_waiters
, code
);
1009 /* jump to the actual trampoline */
1010 #if MONO_AMD64_ARG_REG1 != AMD64_RDI
1011 amd64_mov_reg_reg (code
, MONO_AMD64_ARG_REG1
, AMD64_RDI
);
1015 *ji
= mono_patch_info_list_prepend (*ji
, code
- buf
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, "specific_trampoline_monitor_exit");
1016 amd64_mov_reg_membase (code
, AMD64_R11
, AMD64_RIP
, 0, 8);
1017 amd64_jump_reg (code
, AMD64_R11
);
1019 tramp
= mono_arch_create_specific_trampoline (NULL
, MONO_TRAMPOLINE_MONITOR_EXIT
, mono_get_root_domain (), NULL
);
1020 amd64_jump_code (code
, tramp
);
1023 mono_arch_flush_icache (code
, code
- buf
);
1024 g_assert (code
- buf
<= tramp_size
);
1026 *code_size
= code
- buf
;
1033 mono_arch_invalidate_method (MonoJitInfo
*ji
, void *func
, gpointer func_arg
)
1035 /* FIXME: This is not thread safe */
1036 guint8
*code
= ji
->code_start
;
1038 amd64_mov_reg_imm (code
, AMD64_ARG_REG1
, func_arg
);
1039 amd64_mov_reg_imm (code
, AMD64_R11
, func
);
1041 x86_push_imm (code
, (guint64
)func_arg
);
1042 amd64_call_reg (code
, AMD64_R11
);