3 * llvm "Backend" for the mono JIT
5 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
6 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
12 #include <mono/metadata/debug-helpers.h>
13 #include <mono/metadata/debug-internals.h>
14 #include <mono/metadata/mempool-internals.h>
15 #include <mono/metadata/environment.h>
16 #include <mono/metadata/object-internals.h>
17 #include <mono/metadata/abi-details.h>
18 #include <mono/utils/mono-tls.h>
19 #include <mono/utils/mono-dl.h>
20 #include <mono/utils/mono-time.h>
21 #include <mono/utils/freebsd-dwarf.h>
23 #ifndef __STDC_LIMIT_MACROS
24 #define __STDC_LIMIT_MACROS
26 #ifndef __STDC_CONSTANT_MACROS
27 #define __STDC_CONSTANT_MACROS
30 #include "llvm-c/BitWriter.h"
31 #include "llvm-c/Analysis.h"
33 #include "mini-llvm-cpp.h"
35 #include "aot-compiler.h"
36 #include "mini-llvm.h"
37 #include "mini-runtime.h"
38 #include <mono/utils/mono-math.h>
42 #if defined(TARGET_AMD64) && defined(TARGET_WIN32) && defined(HOST_WIN32) && defined(_MSC_VER)
43 #define TARGET_X86_64_WIN32_MSVC
46 #if defined(TARGET_X86_64_WIN32_MSVC)
47 #define TARGET_WIN32_MSVC
50 #if LLVM_API_VERSION < 610
51 #error "The version of the mono llvm repository is too old."
55 * Information associated by mono with LLVM modules.
58 LLVMModuleRef lmodule
;
59 LLVMValueRef throw_icall
, rethrow
, throw_corlib_exception
;
60 GHashTable
*llvm_types
;
61 LLVMValueRef dummy_got_var
;
62 const char *get_method_symbol
;
63 const char *get_unbox_tramp_symbol
;
64 const char *init_aotconst_symbol
;
65 GHashTable
*plt_entries
;
66 GHashTable
*plt_entries_ji
;
67 GHashTable
*method_to_lmethod
;
68 GHashTable
*method_to_call_info
;
69 GHashTable
*lvalue_to_lcalls
;
70 GHashTable
*direct_callables
;
71 /* Maps got slot index -> LLVMValueRef */
72 GHashTable
*aotconst_vars
;
77 GPtrArray
*subprogram_mds
;
79 LLVMExecutionEngineRef ee
;
80 gboolean external_symbols
;
83 LLVMValueRef personality
;
84 LLVMValueRef
*intrins_by_id
;
85 gpointer gc_poll_cold_wrapper_compiled
;
88 MonoAssembly
*assembly
;
90 MonoAotFileInfo aot_info
;
91 const char *eh_frame_symbol
;
92 LLVMValueRef get_method
, get_unbox_tramp
, init_aotconst_func
;
93 LLVMValueRef init_method
, init_method_gshared_mrgctx
, init_method_gshared_this
, init_method_gshared_vtable
;
94 LLVMValueRef code_start
, code_end
;
95 LLVMValueRef inited_var
;
96 LLVMValueRef unbox_tramp_indexes
;
97 LLVMValueRef unbox_trampolines
;
98 LLVMValueRef gc_poll_cold_wrapper
;
99 LLVMValueRef info_var
;
100 LLVMTypeRef
*info_var_eltypes
;
101 int max_inited_idx
, max_method_idx
;
102 gboolean has_jitted_code
;
103 gboolean static_link
;
106 GHashTable
*idx_to_lmethod
;
107 GHashTable
*idx_to_unbox_tramp
;
108 GPtrArray
*callsite_list
;
109 LLVMContextRef context
;
110 LLVMValueRef sentinel_exception
;
111 void *di_builder
, *cu
;
112 GHashTable
*objc_selector_to_var
;
114 int unbox_tramp_num
, unbox_tramp_elemsize
;
115 GHashTable
*got_idx_to_type
;
116 GHashTable
*no_method_table_lmethods
;
120 * Information associated by the backend with mono basic blocks.
123 LLVMBasicBlockRef bblock
, end_bblock
;
124 LLVMValueRef finally_ind
;
125 gboolean added
, invoke_target
;
127 * If this bblock is the start of a finally clause, this is a list of bblocks it
128 * needs to branch to in ENDFINALLY.
130 GSList
*call_handler_return_bbs
;
132 * If this bblock is the start of a finally clause, this is the bblock that
133 * CALL_HANDLER needs to branch to.
135 LLVMBasicBlockRef call_handler_target_bb
;
136 /* The list of switch statements generated by ENDFINALLY instructions */
137 GSList
*endfinally_switch_ins_list
;
142 * Structure containing emit state
145 MonoMemPool
*mempool
;
147 /* Maps method names to the corresponding LLVMValueRef */
148 GHashTable
*emitted_method_decls
;
151 LLVMValueRef lmethod
;
152 MonoLLVMModule
*module
;
153 LLVMModuleRef lmodule
;
155 int sindex
, default_index
, ex_index
;
156 LLVMBuilderRef builder
;
157 LLVMValueRef
*values
, *addresses
;
158 MonoType
**vreg_cli_types
;
160 MonoMethodSignature
*sig
;
162 GHashTable
*region_to_handler
;
163 GHashTable
*clause_to_handler
;
164 LLVMBuilderRef alloca_builder
;
165 LLVMValueRef last_alloca
;
166 LLVMValueRef rgctx_arg
;
167 LLVMValueRef this_arg
;
168 LLVMTypeRef
*vreg_types
;
170 LLVMTypeRef method_type
;
171 LLVMBasicBlockRef init_bb
, inited_bb
;
173 gboolean
*unreachable
;
175 gboolean has_got_access
;
176 gboolean emit_dummy_arg
;
177 gboolean has_safepoints
;
178 int this_arg_pindex
, rgctx_arg_pindex
;
179 LLVMValueRef imt_rgctx_loc
;
180 GHashTable
*llvm_types
;
182 MonoDebugMethodInfo
*minfo
;
183 /* For every clause, the clauses it is nested in */
186 GHashTable
*exc_meta
;
187 GPtrArray
*callsite_list
;
188 GPtrArray
*phi_values
;
189 GPtrArray
*bblock_list
;
191 GHashTable
*jit_callees
;
192 LLVMValueRef long_bb_break_var
;
198 MonoBasicBlock
*in_bb
;
203 * Instruction metadata
204 * This is the same as ins_info, but LREG != IREG.
212 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
213 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
220 /* keep in sync with the enum in mini.h */
222 mini_llvm_ins_info
[] = {
223 #include "mini-ops.h"
228 #if TARGET_SIZEOF_VOID_P == 4
229 #define GET_LONG_IMM(ins) ((ins)->inst_l)
231 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
234 #define LLVM_INS_INFO(opcode) (&mini_llvm_ins_info [((opcode) - OP_START - 1) * 4])
237 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
239 #define TRACE_FAILURE(msg)
243 #define IS_TARGET_X86 1
245 #define IS_TARGET_X86 0
249 #define IS_TARGET_AMD64 1
251 #define IS_TARGET_AMD64 0
254 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
256 static LLVMIntPredicate cond_to_llvm_cond
[] = {
269 static LLVMRealPredicate fpcond_to_llvm_cond
[] = {
284 static MonoLLVMModule aot_module
;
286 static GHashTable
*intrins_id_to_intrins
;
287 static LLVMTypeRef sse_i1_t
, sse_i2_t
, sse_i4_t
, sse_i8_t
, sse_r4_t
, sse_r8_t
;
289 static void init_jit_module (MonoDomain
*domain
);
291 static void emit_dbg_loc (EmitContext
*ctx
, LLVMBuilderRef builder
, const unsigned char *cil_code
);
292 static void emit_default_dbg_loc (EmitContext
*ctx
, LLVMBuilderRef builder
);
293 static LLVMValueRef
emit_dbg_subprogram (EmitContext
*ctx
, MonoCompile
*cfg
, LLVMValueRef method
, const char *name
);
294 static void emit_dbg_info (MonoLLVMModule
*module
, const char *filename
, const char *cu_name
);
295 static void emit_cond_system_exception (EmitContext
*ctx
, MonoBasicBlock
*bb
, const char *exc_type
, LLVMValueRef cmp
, gboolean force_explicit
);
296 static LLVMValueRef
get_intrins (EmitContext
*ctx
, int id
);
297 static LLVMValueRef
get_intrins_from_module (LLVMModuleRef lmodule
, int id
);
298 static void llvm_jit_finalize_method (EmitContext
*ctx
);
299 static void mono_llvm_nonnull_state_update (EmitContext
*ctx
, LLVMValueRef lcall
, MonoMethod
*call_method
, LLVMValueRef
*args
, int num_params
);
300 static void mono_llvm_propagate_nonnull_final (GHashTable
*all_specializable
, MonoLLVMModule
*module
);
301 static void create_aot_info_var (MonoLLVMModule
*module
);
302 static void set_invariant_load_flag (LLVMValueRef v
);
303 static void set_nonnull_load_flag (LLVMValueRef v
);
306 set_failure (EmitContext
*ctx
, const char *message
)
308 TRACE_FAILURE (reason
);
309 ctx
->cfg
->exception_message
= g_strdup (message
);
310 ctx
->cfg
->disable_llvm
= TRUE
;
316 return LLVMConstInt (LLVMInt32Type (), v
, FALSE
);
322 * The LLVM type with width == TARGET_SIZEOF_VOID_P
327 return TARGET_SIZEOF_VOID_P
== 8 ? LLVMInt64Type () : LLVMInt32Type ();
333 return TARGET_SIZEOF_VOID_P
== 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
339 return TARGET_SIZEOF_VOID_P
== 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
345 * Return the size of the LLVM representation of the vtype T.
348 get_vtype_size (MonoType
*t
)
352 size
= mono_class_value_size (mono_class_from_mono_type_internal (t
), NULL
);
354 /* LLVMArgAsIArgs depends on this since it stores whole words */
355 while (size
< 2 * TARGET_SIZEOF_VOID_P
&& mono_is_power_of_two (size
) == -1)
362 * simd_class_to_llvm_type:
364 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
367 simd_class_to_llvm_type (EmitContext
*ctx
, MonoClass
*klass
)
369 const char *klass_name
= m_class_get_name (klass
);
370 if (!strcmp (klass_name
, "Vector2d")) {
371 return LLVMVectorType (LLVMDoubleType (), 2);
372 } else if (!strcmp (klass_name
, "Vector2l")) {
373 return LLVMVectorType (LLVMInt64Type (), 2);
374 } else if (!strcmp (klass_name
, "Vector2ul")) {
375 return LLVMVectorType (LLVMInt64Type (), 2);
376 } else if (!strcmp (klass_name
, "Vector4i")) {
377 return LLVMVectorType (LLVMInt32Type (), 4);
378 } else if (!strcmp (klass_name
, "Vector4ui")) {
379 return LLVMVectorType (LLVMInt32Type (), 4);
380 } else if (!strcmp (klass_name
, "Vector4f")) {
381 return LLVMVectorType (LLVMFloatType (), 4);
382 } else if (!strcmp (klass_name
, "Vector8s")) {
383 return LLVMVectorType (LLVMInt16Type (), 8);
384 } else if (!strcmp (klass_name
, "Vector8us")) {
385 return LLVMVectorType (LLVMInt16Type (), 8);
386 } else if (!strcmp (klass_name
, "Vector16sb")) {
387 return LLVMVectorType (LLVMInt8Type (), 16);
388 } else if (!strcmp (klass_name
, "Vector16b")) {
389 return LLVMVectorType (LLVMInt8Type (), 16);
390 } else if (!strcmp (klass_name
, "Vector2")) {
391 /* System.Numerics */
392 return LLVMVectorType (LLVMFloatType (), 4);
393 } else if (!strcmp (klass_name
, "Vector3")) {
394 return LLVMVectorType (LLVMFloatType (), 4);
395 } else if (!strcmp (klass_name
, "Vector4")) {
396 return LLVMVectorType (LLVMFloatType (), 4);
397 } else if (!strcmp (klass_name
, "Vector`1") || !strcmp (klass_name
, "Vector128`1") || !strcmp (klass_name
, "Vector256`1")) {
398 MonoType
*etype
= mono_class_get_generic_class (klass
)->context
.class_inst
->type_argv
[0];
399 int size
= mono_class_value_size (klass
, NULL
);
400 switch (etype
->type
) {
403 return LLVMVectorType (LLVMInt8Type (), size
);
406 return LLVMVectorType (LLVMInt16Type (), size
/ 2);
409 return LLVMVectorType (LLVMInt32Type (), size
/ 4);
412 return LLVMVectorType (LLVMInt64Type (), size
/ 8);
414 return LLVMVectorType (LLVMFloatType (), size
/ 4);
416 return LLVMVectorType (LLVMDoubleType (), size
/ 8);
418 g_assert_not_reached ();
422 printf ("%s\n", klass_name
);
428 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
429 static inline G_GNUC_UNUSED LLVMTypeRef
430 type_to_sse_type (int type
)
435 return LLVMVectorType (LLVMInt8Type (), 16);
438 return LLVMVectorType (LLVMInt16Type (), 8);
441 return LLVMVectorType (LLVMInt32Type (), 4);
444 return LLVMVectorType (LLVMInt64Type (), 2);
446 return LLVMVectorType (LLVMDoubleType (), 2);
448 return LLVMVectorType (LLVMFloatType (), 4);
450 g_assert_not_reached ();
456 create_llvm_type_for_type (MonoLLVMModule
*module
, MonoClass
*klass
)
458 int i
, size
, nfields
, esize
;
459 LLVMTypeRef
*eltypes
;
464 t
= m_class_get_byval_arg (klass
);
466 if (mini_type_is_hfa (t
, &nfields
, &esize
)) {
468 * This is needed on arm64 where HFAs are returned in
471 /* SIMD types have size 16 in mono_class_value_size () */
472 if (m_class_is_simd_type (klass
))
475 eltypes
= g_new (LLVMTypeRef
, size
);
476 for (i
= 0; i
< size
; ++i
)
477 eltypes
[i
] = esize
== 4 ? LLVMFloatType () : LLVMDoubleType ();
479 size
= get_vtype_size (t
);
481 eltypes
= g_new (LLVMTypeRef
, size
);
482 for (i
= 0; i
< size
; ++i
)
483 eltypes
[i
] = LLVMInt8Type ();
486 name
= mono_type_full_name (m_class_get_byval_arg (klass
));
487 ltype
= LLVMStructCreateNamed (module
->context
, name
);
488 LLVMStructSetBody (ltype
, eltypes
, size
, FALSE
);
496 primitive_type_to_llvm_type (MonoTypeEnum type
)
501 return LLVMInt8Type ();
504 return LLVMInt16Type ();
507 return LLVMInt32Type ();
510 return LLVMInt64Type ();
512 return LLVMFloatType ();
514 return LLVMDoubleType ();
517 return IntPtrType ();
524 inst_c1_type (const MonoInst
*ins
)
526 return (MonoTypeEnum
)ins
->inst_c1
;
532 * Return the LLVM type corresponding to T.
535 type_to_llvm_type (EmitContext
*ctx
, MonoType
*t
)
540 t
= mini_get_underlying_type (t
);
542 LLVMTypeRef prim_llvm_type
= primitive_type_to_llvm_type (t
->type
);
543 if (prim_llvm_type
!= NULL
)
544 return prim_llvm_type
;
548 return LLVMVoidType ();
549 case MONO_TYPE_OBJECT
:
550 return ObjRefType ();
551 case MONO_TYPE_PTR
: {
552 MonoClass
*klass
= mono_class_from_mono_type_internal (t
);
553 MonoClass
*ptr_klass
= m_class_get_element_class (klass
);
554 MonoType
*ptr_type
= m_class_get_byval_arg (ptr_klass
);
555 /* Handle primitive pointers */
556 switch (ptr_type
->type
) {
563 return LLVMPointerType (type_to_llvm_type (ctx
, ptr_type
), 0);
566 return ObjRefType ();
570 /* Because of generic sharing */
571 return ObjRefType ();
572 case MONO_TYPE_GENERICINST
:
573 if (!mono_type_generic_inst_is_valuetype (t
))
574 return ObjRefType ();
576 case MONO_TYPE_VALUETYPE
:
577 case MONO_TYPE_TYPEDBYREF
: {
581 klass
= mono_class_from_mono_type_internal (t
);
583 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, klass
))
584 return simd_class_to_llvm_type (ctx
, klass
);
586 if (m_class_is_enumtype (klass
))
587 return type_to_llvm_type (ctx
, mono_class_enum_basetype_internal (klass
));
589 ltype
= (LLVMTypeRef
)g_hash_table_lookup (ctx
->module
->llvm_types
, klass
);
591 ltype
= create_llvm_type_for_type (ctx
->module
, klass
);
592 g_hash_table_insert (ctx
->module
->llvm_types
, klass
, ltype
);
598 printf ("X: %d\n", t
->type
);
599 ctx
->cfg
->exception_message
= g_strdup_printf ("type %s", mono_type_full_name (t
));
600 ctx
->cfg
->disable_llvm
= TRUE
;
606 primitive_type_is_unsigned (MonoTypeEnum t
)
623 * Return whenever T is an unsigned int type.
626 type_is_unsigned (EmitContext
*ctx
, MonoType
*t
)
628 t
= mini_get_underlying_type (t
);
631 return primitive_type_is_unsigned (t
->type
);
635 * type_to_llvm_arg_type:
637 * Same as type_to_llvm_type, but treat i8/i16 as i32.
640 type_to_llvm_arg_type (EmitContext
*ctx
, MonoType
*t
)
642 LLVMTypeRef ptype
= type_to_llvm_type (ctx
, t
);
644 if (ctx
->cfg
->llvm_only
)
648 * This works on all abis except arm64/ios which passes multiple
649 * arguments in one stack slot.
652 if (ptype
== LLVMInt8Type () || ptype
== LLVMInt16Type ()) {
654 * LLVM generates code which only sets the lower bits, while JITted
655 * code expects all the bits to be set.
657 ptype
= LLVMInt32Type ();
665 * llvm_type_to_stack_type:
667 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
670 static G_GNUC_UNUSED LLVMTypeRef
671 llvm_type_to_stack_type (MonoCompile
*cfg
, LLVMTypeRef type
)
675 if (type
== LLVMInt8Type ())
676 return LLVMInt32Type ();
677 else if (type
== LLVMInt16Type ())
678 return LLVMInt32Type ();
679 else if (!cfg
->r4fp
&& type
== LLVMFloatType ())
680 return LLVMDoubleType ();
686 * regtype_to_llvm_type:
688 * Return the LLVM type corresponding to the regtype C used in instruction
692 regtype_to_llvm_type (char c
)
696 return LLVMInt32Type ();
698 return LLVMInt64Type ();
700 return LLVMDoubleType ();
709 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
712 op_to_llvm_type (int opcode
)
717 return LLVMInt8Type ();
720 return LLVMInt8Type ();
723 return LLVMInt16Type ();
726 return LLVMInt16Type ();
729 return LLVMInt32Type ();
732 return LLVMInt32Type ();
734 return LLVMInt64Type ();
736 return LLVMFloatType ();
738 return LLVMDoubleType ();
740 return LLVMInt64Type ();
742 return LLVMInt32Type ();
744 return LLVMInt64Type ();
749 return LLVMInt8Type ();
754 return LLVMInt16Type ();
757 return LLVMInt32Type ();
760 return LLVMInt64Type ();
763 return TARGET_SIZEOF_VOID_P
== 8 ? LLVMInt64Type () : LLVMInt32Type ();
770 return LLVMInt32Type ();
777 return LLVMInt64Type ();
779 printf ("%s\n", mono_inst_name (opcode
));
780 g_assert_not_reached ();
785 #define CLAUSE_START(clause) ((clause)->try_offset)
786 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
789 * load_store_to_llvm_type:
791 * Return the size/sign/zero extension corresponding to the load/store opcode
795 load_store_to_llvm_type (int opcode
, int *size
, gboolean
*sext
, gboolean
*zext
)
801 case OP_LOADI1_MEMBASE
:
802 case OP_STOREI1_MEMBASE_REG
:
803 case OP_STOREI1_MEMBASE_IMM
:
804 case OP_ATOMIC_LOAD_I1
:
805 case OP_ATOMIC_STORE_I1
:
808 return LLVMInt8Type ();
809 case OP_LOADU1_MEMBASE
:
811 case OP_ATOMIC_LOAD_U1
:
812 case OP_ATOMIC_STORE_U1
:
815 return LLVMInt8Type ();
816 case OP_LOADI2_MEMBASE
:
817 case OP_STOREI2_MEMBASE_REG
:
818 case OP_STOREI2_MEMBASE_IMM
:
819 case OP_ATOMIC_LOAD_I2
:
820 case OP_ATOMIC_STORE_I2
:
823 return LLVMInt16Type ();
824 case OP_LOADU2_MEMBASE
:
826 case OP_ATOMIC_LOAD_U2
:
827 case OP_ATOMIC_STORE_U2
:
830 return LLVMInt16Type ();
831 case OP_LOADI4_MEMBASE
:
832 case OP_LOADU4_MEMBASE
:
835 case OP_STOREI4_MEMBASE_REG
:
836 case OP_STOREI4_MEMBASE_IMM
:
837 case OP_ATOMIC_LOAD_I4
:
838 case OP_ATOMIC_STORE_I4
:
839 case OP_ATOMIC_LOAD_U4
:
840 case OP_ATOMIC_STORE_U4
:
842 return LLVMInt32Type ();
843 case OP_LOADI8_MEMBASE
:
845 case OP_STOREI8_MEMBASE_REG
:
846 case OP_STOREI8_MEMBASE_IMM
:
847 case OP_ATOMIC_LOAD_I8
:
848 case OP_ATOMIC_STORE_I8
:
849 case OP_ATOMIC_LOAD_U8
:
850 case OP_ATOMIC_STORE_U8
:
852 return LLVMInt64Type ();
853 case OP_LOADR4_MEMBASE
:
854 case OP_STORER4_MEMBASE_REG
:
855 case OP_ATOMIC_LOAD_R4
:
856 case OP_ATOMIC_STORE_R4
:
858 return LLVMFloatType ();
859 case OP_LOADR8_MEMBASE
:
860 case OP_STORER8_MEMBASE_REG
:
861 case OP_ATOMIC_LOAD_R8
:
862 case OP_ATOMIC_STORE_R8
:
864 return LLVMDoubleType ();
865 case OP_LOAD_MEMBASE
:
867 case OP_STORE_MEMBASE_REG
:
868 case OP_STORE_MEMBASE_IMM
:
869 *size
= TARGET_SIZEOF_VOID_P
;
870 return IntPtrType ();
872 g_assert_not_reached ();
880 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
883 ovf_op_to_intrins (int opcode
)
887 return INTRINS_SADD_OVF_I32
;
889 return INTRINS_UADD_OVF_I32
;
891 return INTRINS_SSUB_OVF_I32
;
893 return INTRINS_USUB_OVF_I32
;
895 return INTRINS_SMUL_OVF_I32
;
897 return INTRINS_UMUL_OVF_I32
;
899 return INTRINS_SADD_OVF_I64
;
901 return INTRINS_UADD_OVF_I64
;
903 return INTRINS_SSUB_OVF_I64
;
905 return INTRINS_USUB_OVF_I64
;
907 return INTRINS_SMUL_OVF_I64
;
909 return INTRINS_UMUL_OVF_I64
;
911 g_assert_not_reached ();
912 return (IntrinsicId
)0;
917 simd_ins_to_intrins (int opcode
)
920 #if defined(TARGET_X86) || defined(TARGET_AMD64)
922 return INTRINS_SSE_MINPD
;
924 return INTRINS_SSE_MINPS
;
926 return INTRINS_SSE_MAXPD
;
928 return INTRINS_SSE_MAXPS
;
930 return INTRINS_SSE_HADDPD
;
932 return INTRINS_SSE_HADDPS
;
934 return INTRINS_SSE_HSUBPD
;
936 return INTRINS_SSE_HSUBPS
;
938 return INTRINS_SSE_ADDSUBPS
;
940 return INTRINS_SSE_ADDSUBPD
;
941 case OP_EXTRACT_MASK
:
942 return INTRINS_SSE_PMOVMSKB
;
945 return INTRINS_SSE_PSRLI_W
;
948 return INTRINS_SSE_PSRLI_D
;
951 return INTRINS_SSE_PSRLI_Q
;
954 return INTRINS_SSE_PSLLI_W
;
957 return INTRINS_SSE_PSLLI_D
;
960 return INTRINS_SSE_PSLLI_Q
;
963 return INTRINS_SSE_PSRAI_W
;
966 return INTRINS_SSE_PSRAI_D
;
968 return INTRINS_SSE_RSQRT_PS
;
970 return INTRINS_SSE_RCP_PS
;
972 return INTRINS_SSE_CVTPD2DQ
;
974 return INTRINS_SSE_CVTPS2DQ
;
976 return INTRINS_SSE_CVTPD2PS
;
978 return INTRINS_SSE_CVTTPD2DQ
;
980 return INTRINS_SSE_CVTTPS2DQ
;
982 return INTRINS_SSE_PACKSSWB
;
984 return INTRINS_SSE_PACKSSDW
;
986 return INTRINS_SSE_PACKUSWB
;
988 return INTRINS_SSE_PACKUSDW
;
990 return INTRINS_SSE_PMULHW
;
991 case OP_PMULW_HIGH_UN
:
992 return INTRINS_SSE_PMULHU
;
994 return INTRINS_SSE_DPPS
;
996 return INTRINS_SSE_SQRT_SS
;
998 return INTRINS_SSE_SQRT_SD
;
1000 return INTRINS_SSE_SQRT_PS
;
1002 return INTRINS_SSE_SQRT_PD
;
1005 g_assert_not_reached ();
1006 return (IntrinsicId
)0;
1011 simd_op_to_llvm_type (int opcode
)
1013 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1026 case OP_EXTRACTX_U2
:
1043 case OP_EXTRACT_MASK
:
1055 g_assert_not_reached ();
1064 set_cold_cconv (LLVMValueRef func
)
1067 * xcode10 (watchOS) and ARM/ARM64 doesn't seem to support preserveall, it fails with:
1068 * fatal error: error in backend: Unsupported calling convention
1070 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1071 LLVMSetFunctionCallConv (func
, LLVMColdCallConv
);
1076 set_call_cold_cconv (LLVMValueRef func
)
1078 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1079 LLVMSetInstructionCallConv (func
, LLVMColdCallConv
);
1086 * Return the LLVM basic block corresponding to BB.
1088 static LLVMBasicBlockRef
1089 get_bb (EmitContext
*ctx
, MonoBasicBlock
*bb
)
1091 char bb_name_buf
[128];
1094 if (ctx
->bblocks
[bb
->block_num
].bblock
== NULL
) {
1095 if (bb
->flags
& BB_EXCEPTION_HANDLER
) {
1096 int clause_index
= (mono_get_block_region_notry (ctx
->cfg
, bb
->region
) >> 8) - 1;
1097 sprintf (bb_name_buf
, "EH_CLAUSE%d_BB%d", clause_index
, bb
->block_num
);
1098 bb_name
= bb_name_buf
;
1099 } else if (bb
->block_num
< 256) {
1100 if (!ctx
->module
->bb_names
) {
1101 ctx
->module
->bb_names_len
= 256;
1102 ctx
->module
->bb_names
= g_new0 (char*, ctx
->module
->bb_names_len
);
1104 if (!ctx
->module
->bb_names
[bb
->block_num
]) {
1107 n
= g_strdup_printf ("BB%d", bb
->block_num
);
1108 mono_memory_barrier ();
1109 ctx
->module
->bb_names
[bb
->block_num
] = n
;
1111 bb_name
= ctx
->module
->bb_names
[bb
->block_num
];
1113 sprintf (bb_name_buf
, "BB%d", bb
->block_num
);
1114 bb_name
= bb_name_buf
;
1117 ctx
->bblocks
[bb
->block_num
].bblock
= LLVMAppendBasicBlock (ctx
->lmethod
, bb_name
);
1118 ctx
->bblocks
[bb
->block_num
].end_bblock
= ctx
->bblocks
[bb
->block_num
].bblock
;
1121 return ctx
->bblocks
[bb
->block_num
].bblock
;
1127 * Return the last LLVM bblock corresponding to BB.
1128 * This might not be equal to the bb returned by get_bb () since we need to generate
1129 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1131 static LLVMBasicBlockRef
1132 get_end_bb (EmitContext
*ctx
, MonoBasicBlock
*bb
)
1135 return ctx
->bblocks
[bb
->block_num
].end_bblock
;
1138 static LLVMBasicBlockRef
1139 gen_bb (EmitContext
*ctx
, const char *prefix
)
1143 sprintf (bb_name
, "%s%d", prefix
, ++ ctx
->ex_index
);
1144 return LLVMAppendBasicBlock (ctx
->lmethod
, bb_name
);
1150 * Return the target of the patch identified by TYPE and TARGET.
1153 resolve_patch (MonoCompile
*cfg
, MonoJumpInfoType type
, gconstpointer target
)
1159 memset (&ji
, 0, sizeof (ji
));
1161 ji
.data
.target
= target
;
1163 res
= mono_resolve_patch_target (cfg
->method
, cfg
->domain
, NULL
, &ji
, FALSE
, error
);
1164 mono_error_assert_ok (error
);
1172 * Emit code to convert the LLVM value V to DTYPE.
1175 convert_full (EmitContext
*ctx
, LLVMValueRef v
, LLVMTypeRef dtype
, gboolean is_unsigned
)
1177 LLVMTypeRef stype
= LLVMTypeOf (v
);
1179 if (stype
!= dtype
) {
1180 gboolean ext
= FALSE
;
1183 if (dtype
== LLVMInt64Type () && (stype
== LLVMInt32Type () || stype
== LLVMInt16Type () || stype
== LLVMInt8Type ()))
1185 else if (dtype
== LLVMInt32Type () && (stype
== LLVMInt16Type () || stype
== LLVMInt8Type ()))
1187 else if (dtype
== LLVMInt16Type () && (stype
== LLVMInt8Type ()))
1191 return is_unsigned
? LLVMBuildZExt (ctx
->builder
, v
, dtype
, "") : LLVMBuildSExt (ctx
->builder
, v
, dtype
, "");
1193 if (dtype
== LLVMDoubleType () && stype
== LLVMFloatType ())
1194 return LLVMBuildFPExt (ctx
->builder
, v
, dtype
, "");
1197 if (stype
== LLVMInt64Type () && (dtype
== LLVMInt32Type () || dtype
== LLVMInt16Type () || dtype
== LLVMInt8Type ()))
1198 return LLVMBuildTrunc (ctx
->builder
, v
, dtype
, "");
1199 if (stype
== LLVMInt32Type () && (dtype
== LLVMInt16Type () || dtype
== LLVMInt8Type ()))
1200 return LLVMBuildTrunc (ctx
->builder
, v
, dtype
, "");
1201 if (stype
== LLVMInt16Type () && dtype
== LLVMInt8Type ())
1202 return LLVMBuildTrunc (ctx
->builder
, v
, dtype
, "");
1203 if (stype
== LLVMDoubleType () && dtype
== LLVMFloatType ())
1204 return LLVMBuildFPTrunc (ctx
->builder
, v
, dtype
, "");
1206 if (LLVMGetTypeKind (stype
) == LLVMPointerTypeKind
&& LLVMGetTypeKind (dtype
) == LLVMPointerTypeKind
)
1207 return LLVMBuildBitCast (ctx
->builder
, v
, dtype
, "");
1208 if (LLVMGetTypeKind (dtype
) == LLVMPointerTypeKind
)
1209 return LLVMBuildIntToPtr (ctx
->builder
, v
, dtype
, "");
1210 if (LLVMGetTypeKind (stype
) == LLVMPointerTypeKind
)
1211 return LLVMBuildPtrToInt (ctx
->builder
, v
, dtype
, "");
1213 if (mono_arch_is_soft_float ()) {
1214 if (stype
== LLVMInt32Type () && dtype
== LLVMFloatType ())
1215 return LLVMBuildBitCast (ctx
->builder
, v
, dtype
, "");
1216 if (stype
== LLVMInt32Type () && dtype
== LLVMDoubleType ())
1217 return LLVMBuildBitCast (ctx
->builder
, LLVMBuildZExt (ctx
->builder
, v
, LLVMInt64Type (), ""), dtype
, "");
1220 if (LLVMGetTypeKind (stype
) == LLVMVectorTypeKind
&& LLVMGetTypeKind (dtype
) == LLVMVectorTypeKind
)
1221 return LLVMBuildBitCast (ctx
->builder
, v
, dtype
, "");
1223 mono_llvm_dump_value (v
);
1224 mono_llvm_dump_value (LLVMConstNull (dtype
));
1226 g_assert_not_reached ();
1234 convert (EmitContext
*ctx
, LLVMValueRef v
, LLVMTypeRef dtype
)
1236 return convert_full (ctx
, v
, dtype
, FALSE
);
1240 emit_memset (EmitContext
*ctx
, LLVMBuilderRef builder
, LLVMValueRef v
, LLVMValueRef size
, int alignment
)
1242 LLVMValueRef args
[5];
1245 args
[aindex
++] = v
;
1246 args
[aindex
++] = LLVMConstInt (LLVMInt8Type (), 0, FALSE
);
1247 args
[aindex
++] = size
;
1248 #if LLVM_API_VERSION < 900
1249 args
[aindex
++] = LLVMConstInt (LLVMInt32Type (), alignment
, FALSE
);
1251 args
[aindex
++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE
);
1252 LLVMBuildCall (builder
, get_intrins (ctx
, INTRINS_MEMSET
), args
, aindex
, "");
1256 * emit_volatile_load:
1258 * If vreg is volatile, emit a load from its address.
1261 emit_volatile_load (EmitContext
*ctx
, int vreg
)
1266 // On arm64, we pass the rgctx in a callee saved
1267 // register on arm64 (x15), and llvm might keep the value in that register
1268 // even through the register is marked as 'reserved' inside llvm.
1270 v
= mono_llvm_build_load (ctx
->builder
, ctx
->addresses
[vreg
], "", TRUE
);
1271 t
= ctx
->vreg_cli_types
[vreg
];
1272 if (t
&& !t
->byref
) {
1274 * Might have to zero extend since llvm doesn't have
1277 if (t
->type
== MONO_TYPE_U1
|| t
->type
== MONO_TYPE_U2
|| t
->type
== MONO_TYPE_CHAR
|| t
->type
== MONO_TYPE_BOOLEAN
)
1278 v
= LLVMBuildZExt (ctx
->builder
, v
, LLVMInt32Type (), "");
1279 else if (t
->type
== MONO_TYPE_I1
|| t
->type
== MONO_TYPE_I2
)
1280 v
= LLVMBuildSExt (ctx
->builder
, v
, LLVMInt32Type (), "");
1281 else if (t
->type
== MONO_TYPE_U8
)
1282 v
= LLVMBuildZExt (ctx
->builder
, v
, LLVMInt64Type (), "");
1289 * emit_volatile_store:
1291 * If VREG is volatile, emit a store from its value to its address.
1294 emit_volatile_store (EmitContext
*ctx
, int vreg
)
1296 MonoInst
*var
= get_vreg_to_inst (ctx
->cfg
, vreg
);
1298 if (var
&& var
->flags
& (MONO_INST_VOLATILE
|MONO_INST_INDIRECT
)) {
1299 g_assert (ctx
->addresses
[vreg
]);
1300 LLVMBuildStore (ctx
->builder
, convert (ctx
, ctx
->values
[vreg
], type_to_llvm_type (ctx
, var
->inst_vtype
)), ctx
->addresses
[vreg
]);
1305 sig_to_llvm_sig_no_cinfo (EmitContext
*ctx
, MonoMethodSignature
*sig
)
1307 LLVMTypeRef ret_type
;
1308 LLVMTypeRef
*param_types
= NULL
;
1313 ret_type
= type_to_llvm_type (ctx
, sig
->ret
);
1316 rtype
= mini_get_underlying_type (sig
->ret
);
1318 param_types
= g_new0 (LLVMTypeRef
, (sig
->param_count
* 8) + 3);
1322 param_types
[pindex
++] = ThisType ();
1323 for (i
= 0; i
< sig
->param_count
; ++i
)
1324 param_types
[pindex
++] = type_to_llvm_arg_type (ctx
, sig
->params
[i
]);
1326 if (!ctx_ok (ctx
)) {
1327 g_free (param_types
);
1331 res
= LLVMFunctionType (ret_type
, param_types
, pindex
, FALSE
);
1332 g_free (param_types
);
1338 * sig_to_llvm_sig_full:
1340 * Return the LLVM signature corresponding to the mono signature SIG using the
1341 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1344 sig_to_llvm_sig_full (EmitContext
*ctx
, MonoMethodSignature
*sig
, LLVMCallInfo
*cinfo
)
1346 LLVMTypeRef ret_type
;
1347 LLVMTypeRef
*param_types
= NULL
;
1349 int i
, j
, pindex
, vret_arg_pindex
= 0;
1350 gboolean vretaddr
= FALSE
;
1354 return sig_to_llvm_sig_no_cinfo (ctx
, sig
);
1356 ret_type
= type_to_llvm_type (ctx
, sig
->ret
);
1359 rtype
= mini_get_underlying_type (sig
->ret
);
1361 switch (cinfo
->ret
.storage
) {
1362 case LLVMArgVtypeInReg
:
1363 /* LLVM models this by returning an aggregate value */
1364 if (cinfo
->ret
.pair_storage
[0] == LLVMArgInIReg
&& cinfo
->ret
.pair_storage
[1] == LLVMArgNone
) {
1365 LLVMTypeRef members
[2];
1367 members
[0] = IntPtrType ();
1368 ret_type
= LLVMStructType (members
, 1, FALSE
);
1369 } else if (cinfo
->ret
.pair_storage
[0] == LLVMArgNone
&& cinfo
->ret
.pair_storage
[1] == LLVMArgNone
) {
1371 ret_type
= LLVMVoidType ();
1372 } else if (cinfo
->ret
.pair_storage
[0] == LLVMArgInIReg
&& cinfo
->ret
.pair_storage
[1] == LLVMArgInIReg
) {
1373 LLVMTypeRef members
[2];
1375 members
[0] = IntPtrType ();
1376 members
[1] = IntPtrType ();
1377 ret_type
= LLVMStructType (members
, 2, FALSE
);
1379 g_assert_not_reached ();
1382 case LLVMArgVtypeByVal
:
1383 /* Vtype returned normally by val */
1385 case LLVMArgVtypeAsScalar
: {
1386 int size
= mono_class_value_size (mono_class_from_mono_type_internal (rtype
), NULL
);
1387 /* LLVM models this by returning an int */
1388 if (size
< TARGET_SIZEOF_VOID_P
) {
1389 g_assert (cinfo
->ret
.nslots
== 1);
1390 ret_type
= LLVMIntType (size
* 8);
1392 g_assert (cinfo
->ret
.nslots
== 1 || cinfo
->ret
.nslots
== 2);
1393 ret_type
= LLVMIntType (cinfo
->ret
.nslots
* sizeof (target_mgreg_t
) * 8);
1397 case LLVMArgAsIArgs
:
1398 ret_type
= LLVMArrayType (IntPtrType (), cinfo
->ret
.nslots
);
1400 case LLVMArgFpStruct
: {
1401 /* Vtype returned as a fp struct */
1402 LLVMTypeRef members
[16];
1404 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1405 for (i
= 0; i
< cinfo
->ret
.nslots
; ++i
)
1406 members
[i
] = cinfo
->ret
.esize
== 8 ? LLVMDoubleType () : LLVMFloatType ();
1407 ret_type
= LLVMStructType (members
, cinfo
->ret
.nslots
, FALSE
);
1410 case LLVMArgVtypeByRef
:
1411 /* Vtype returned using a hidden argument */
1412 ret_type
= LLVMVoidType ();
1414 case LLVMArgVtypeRetAddr
:
1415 case LLVMArgGsharedvtFixed
:
1416 case LLVMArgGsharedvtFixedVtype
:
1417 case LLVMArgGsharedvtVariable
:
1419 ret_type
= LLVMVoidType ();
1425 param_types
= g_new0 (LLVMTypeRef
, (sig
->param_count
* 8) + 3);
1427 if (cinfo
->ret
.storage
== LLVMArgVtypeByRef
) {
1429 * Has to be the first argument because of the sret argument attribute
1430 * FIXME: This might conflict with passing 'this' as the first argument, but
1431 * this is only used on arm64 which has a dedicated struct return register.
1433 cinfo
->vret_arg_pindex
= pindex
;
1434 param_types
[pindex
] = type_to_llvm_arg_type (ctx
, sig
->ret
);
1435 if (!ctx_ok (ctx
)) {
1436 g_free (param_types
);
1439 param_types
[pindex
] = LLVMPointerType (param_types
[pindex
], 0);
1442 if (!ctx
->llvm_only
&& cinfo
->rgctx_arg
) {
1443 cinfo
->rgctx_arg_pindex
= pindex
;
1444 param_types
[pindex
] = ctx
->module
->ptr_type
;
1447 if (cinfo
->imt_arg
) {
1448 cinfo
->imt_arg_pindex
= pindex
;
1449 param_types
[pindex
] = ctx
->module
->ptr_type
;
1453 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1454 vret_arg_pindex
= pindex
;
1455 if (cinfo
->vret_arg_index
== 1) {
1456 /* Add the slots consumed by the first argument */
1457 LLVMArgInfo
*ainfo
= &cinfo
->args
[0];
1458 switch (ainfo
->storage
) {
1459 case LLVMArgVtypeInReg
:
1460 for (j
= 0; j
< 2; ++j
) {
1461 if (ainfo
->pair_storage
[j
] == LLVMArgInIReg
)
1470 cinfo
->vret_arg_pindex
= vret_arg_pindex
;
1473 if (vretaddr
&& vret_arg_pindex
== pindex
)
1474 param_types
[pindex
++] = IntPtrType ();
1476 cinfo
->this_arg_pindex
= pindex
;
1477 param_types
[pindex
++] = ThisType ();
1478 cinfo
->args
[0].pindex
= cinfo
->this_arg_pindex
;
1480 if (vretaddr
&& vret_arg_pindex
== pindex
)
1481 param_types
[pindex
++] = IntPtrType ();
1482 for (i
= 0; i
< sig
->param_count
; ++i
) {
1483 LLVMArgInfo
*ainfo
= &cinfo
->args
[i
+ sig
->hasthis
];
1485 if (vretaddr
&& vret_arg_pindex
== pindex
)
1486 param_types
[pindex
++] = IntPtrType ();
1487 ainfo
->pindex
= pindex
;
1489 switch (ainfo
->storage
) {
1490 case LLVMArgVtypeInReg
:
1491 for (j
= 0; j
< 2; ++j
) {
1492 switch (ainfo
->pair_storage
[j
]) {
1494 param_types
[pindex
++] = LLVMIntType (TARGET_SIZEOF_VOID_P
* 8);
1499 g_assert_not_reached ();
1503 case LLVMArgVtypeByVal
:
1504 param_types
[pindex
] = type_to_llvm_arg_type (ctx
, ainfo
->type
);
1507 param_types
[pindex
] = LLVMPointerType (param_types
[pindex
], 0);
1510 case LLVMArgAsIArgs
:
1511 if (ainfo
->esize
== 8)
1512 param_types
[pindex
] = LLVMArrayType (LLVMInt64Type (), ainfo
->nslots
);
1514 param_types
[pindex
] = LLVMArrayType (IntPtrType (), ainfo
->nslots
);
1517 case LLVMArgVtypeAddr
:
1518 case LLVMArgVtypeByRef
:
1519 param_types
[pindex
] = type_to_llvm_arg_type (ctx
, ainfo
->type
);
1522 param_types
[pindex
] = LLVMPointerType (param_types
[pindex
], 0);
1525 case LLVMArgAsFpArgs
: {
1528 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1529 for (j
= 0; j
< ainfo
->ndummy_fpargs
; ++j
)
1530 param_types
[pindex
++] = LLVMDoubleType ();
1531 for (j
= 0; j
< ainfo
->nslots
; ++j
)
1532 param_types
[pindex
++] = ainfo
->esize
== 8 ? LLVMDoubleType () : LLVMFloatType ();
1535 case LLVMArgVtypeAsScalar
:
1536 g_assert_not_reached ();
1538 case LLVMArgGsharedvtFixed
:
1539 case LLVMArgGsharedvtFixedVtype
:
1540 param_types
[pindex
++] = LLVMPointerType (type_to_llvm_arg_type (ctx
, ainfo
->type
), 0);
1542 case LLVMArgGsharedvtVariable
:
1543 param_types
[pindex
++] = LLVMPointerType (IntPtrType (), 0);
1546 param_types
[pindex
++] = type_to_llvm_arg_type (ctx
, ainfo
->type
);
1550 if (!ctx_ok (ctx
)) {
1551 g_free (param_types
);
1554 if (vretaddr
&& vret_arg_pindex
== pindex
)
1555 param_types
[pindex
++] = IntPtrType ();
1556 if (ctx
->llvm_only
&& cinfo
->rgctx_arg
) {
1557 /* Pass the rgctx as the last argument */
1558 cinfo
->rgctx_arg_pindex
= pindex
;
1559 param_types
[pindex
] = ctx
->module
->ptr_type
;
1561 } else if (ctx
->llvm_only
&& cinfo
->dummy_arg
) {
1562 /* Pass a dummy arg last */
1563 cinfo
->dummy_arg_pindex
= pindex
;
1564 param_types
[pindex
] = ctx
->module
->ptr_type
;
1568 res
= LLVMFunctionType (ret_type
, param_types
, pindex
, FALSE
);
1569 g_free (param_types
);
1575 sig_to_llvm_sig (EmitContext
*ctx
, MonoMethodSignature
*sig
)
1577 return sig_to_llvm_sig_full (ctx
, sig
, NULL
);
1581 * LLVMFunctionType1:
1583 * Create an LLVM function type from the arguments.
1585 static G_GNUC_UNUSED LLVMTypeRef
1586 LLVMFunctionType0 (LLVMTypeRef ReturnType
,
1589 return LLVMFunctionType (ReturnType
, NULL
, 0, IsVarArg
);
1593 * LLVMFunctionType1:
1595 * Create an LLVM function type from the arguments.
1597 static G_GNUC_UNUSED LLVMTypeRef
1598 LLVMFunctionType1 (LLVMTypeRef ReturnType
,
1599 LLVMTypeRef ParamType1
,
1602 LLVMTypeRef param_types
[1];
1604 param_types
[0] = ParamType1
;
1606 return LLVMFunctionType (ReturnType
, param_types
, 1, IsVarArg
);
1610 * LLVMFunctionType2:
1612 * Create an LLVM function type from the arguments.
1614 static G_GNUC_UNUSED LLVMTypeRef
1615 LLVMFunctionType2 (LLVMTypeRef ReturnType
,
1616 LLVMTypeRef ParamType1
,
1617 LLVMTypeRef ParamType2
,
1620 LLVMTypeRef param_types
[2];
1622 param_types
[0] = ParamType1
;
1623 param_types
[1] = ParamType2
;
1625 return LLVMFunctionType (ReturnType
, param_types
, 2, IsVarArg
);
1629 * LLVMFunctionType3:
1631 * Create an LLVM function type from the arguments.
1633 static G_GNUC_UNUSED LLVMTypeRef
1634 LLVMFunctionType3 (LLVMTypeRef ReturnType
,
1635 LLVMTypeRef ParamType1
,
1636 LLVMTypeRef ParamType2
,
1637 LLVMTypeRef ParamType3
,
1640 LLVMTypeRef param_types
[3];
1642 param_types
[0] = ParamType1
;
1643 param_types
[1] = ParamType2
;
1644 param_types
[2] = ParamType3
;
1646 return LLVMFunctionType (ReturnType
, param_types
, 3, IsVarArg
);
1649 static G_GNUC_UNUSED LLVMTypeRef
1650 LLVMFunctionType4 (LLVMTypeRef ReturnType
,
1651 LLVMTypeRef ParamType1
,
1652 LLVMTypeRef ParamType2
,
1653 LLVMTypeRef ParamType3
,
1654 LLVMTypeRef ParamType4
,
1657 LLVMTypeRef param_types
[4];
1659 param_types
[0] = ParamType1
;
1660 param_types
[1] = ParamType2
;
1661 param_types
[2] = ParamType3
;
1662 param_types
[3] = ParamType4
;
1664 return LLVMFunctionType (ReturnType
, param_types
, 4, IsVarArg
);
1667 static G_GNUC_UNUSED LLVMTypeRef
1668 LLVMFunctionType5 (LLVMTypeRef ReturnType
,
1669 LLVMTypeRef ParamType1
,
1670 LLVMTypeRef ParamType2
,
1671 LLVMTypeRef ParamType3
,
1672 LLVMTypeRef ParamType4
,
1673 LLVMTypeRef ParamType5
,
1676 LLVMTypeRef param_types
[5];
1678 param_types
[0] = ParamType1
;
1679 param_types
[1] = ParamType2
;
1680 param_types
[2] = ParamType3
;
1681 param_types
[3] = ParamType4
;
1682 param_types
[4] = ParamType5
;
1684 return LLVMFunctionType (ReturnType
, param_types
, 5, IsVarArg
);
1690 * Create an LLVM builder and remember it so it can be freed later.
1692 static LLVMBuilderRef
1693 create_builder (EmitContext
*ctx
)
1695 LLVMBuilderRef builder
= LLVMCreateBuilder ();
1696 if (mono_use_fast_math
)
1697 mono_llvm_set_fast_math (builder
);
1699 ctx
->builders
= g_slist_prepend_mempool (ctx
->cfg
->mempool
, ctx
->builders
, builder
);
1701 emit_default_dbg_loc (ctx
, builder
);
1707 get_aotconst_name (MonoJumpInfoType type
, gconstpointer data
, int got_offset
)
1713 case MONO_PATCH_INFO_JIT_ICALL_ID
:
1714 name
= g_strdup_printf ("jit_icall_%s", mono_find_jit_icall_info ((MonoJitICallId
)(gsize
)data
)->name
);
1716 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX
: {
1717 MonoJumpInfoRgctxEntry
*entry
= (MonoJumpInfoRgctxEntry
*)data
;
1718 name
= g_strdup_printf ("rgctx_slot_index_%s", mono_rgctx_info_type_to_str (entry
->info_type
));
1721 case MONO_PATCH_INFO_AOT_MODULE
:
1722 case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG
:
1723 case MONO_PATCH_INFO_GC_CARD_TABLE_ADDR
:
1724 case MONO_PATCH_INFO_GC_NURSERY_START
:
1725 case MONO_PATCH_INFO_GC_NURSERY_BITS
:
1726 case MONO_PATCH_INFO_INTERRUPTION_REQUEST_FLAG
:
1727 name
= g_strdup_printf ("%s", mono_ji_type_to_string (type
));
1728 len
= strlen (name
);
1729 for (int i
= 0; i
< len
; ++i
)
1730 name
[i
] = tolower (name
[i
]);
1733 name
= g_strdup_printf ("%s_%d", mono_ji_type_to_string (type
), got_offset
);
1734 len
= strlen (name
);
1735 for (int i
= 0; i
< len
; ++i
)
1736 name
[i
] = tolower (name
[i
]);
1744 compute_aot_got_offset (MonoLLVMModule
*module
, MonoJumpInfo
*ji
, LLVMTypeRef llvm_type
)
1746 guint32 got_offset
= mono_aot_get_got_offset (ji
);
1748 LLVMTypeRef lookup_type
= (LLVMTypeRef
) g_hash_table_lookup (module
->got_idx_to_type
, GINT_TO_POINTER (got_offset
));
1751 lookup_type
= llvm_type
;
1752 } else if (llvm_type
!= lookup_type
) {
1753 lookup_type
= module
->ptr_type
;
1758 g_hash_table_insert (module
->got_idx_to_type
, GINT_TO_POINTER (got_offset
), lookup_type
);
1762 /* Allocate a GOT slot for TYPE/DATA, and emit IR to load it */
1764 get_aotconst_module (MonoLLVMModule
*module
, LLVMBuilderRef builder
, MonoJumpInfoType type
, gconstpointer data
, LLVMTypeRef llvm_type
,
1765 guint32
*out_got_offset
, MonoJumpInfo
**out_ji
)
1770 MonoJumpInfo tmp_ji
;
1772 tmp_ji
.data
.target
= data
;
1774 MonoJumpInfo
*ji
= mono_aot_patch_info_dup (&tmp_ji
);
1779 got_offset
= compute_aot_got_offset (module
, ji
, llvm_type
);
1780 module
->max_got_offset
= MAX (module
->max_got_offset
, got_offset
);
1783 *out_got_offset
= got_offset
;
1785 LLVMValueRef const_var
= g_hash_table_lookup (module
->aotconst_vars
, GINT_TO_POINTER (got_offset
));
1787 LLVMTypeRef type
= llvm_type
;
1789 char *name
= get_aotconst_name (ji
->type
, ji
->data
.target
, got_offset
);
1790 char *symbol
= g_strdup_printf ("aotconst_%s", name
);
1792 LLVMValueRef v
= LLVMAddGlobal (module
->lmodule
, type
, symbol
);
1793 LLVMSetVisibility (v
, LLVMHiddenVisibility
);
1794 LLVMSetLinkage (v
, LLVMInternalLinkage
);
1795 LLVMSetInitializer (v
, LLVMConstNull (type
));
1797 LLVMSetAlignment (v
, 8);
1799 g_hash_table_insert (module
->aotconst_vars
, GINT_TO_POINTER (got_offset
), v
);
1803 load
= LLVMBuildLoad (builder
, const_var
, "");
1805 if (mono_aot_is_shared_got_offset (got_offset
))
1806 set_invariant_load_flag (load
);
1807 if (type
== MONO_PATCH_INFO_LDSTR
)
1808 set_nonnull_load_flag (load
);
1810 load
= LLVMBuildBitCast (builder
, load
, llvm_type
, "");
1816 get_aotconst (EmitContext
*ctx
, MonoJumpInfoType type
, gconstpointer data
, LLVMTypeRef llvm_type
)
1825 MonoJumpInfo tmp_ji
;
1827 tmp_ji
.data
.target
= data
;
1829 load
= get_aotconst_module (ctx
->module
, ctx
->builder
, type
, data
, llvm_type
, &got_offset
, &ji
);
1831 ji
->next
= cfg
->patch_info
;
1832 cfg
->patch_info
= ji
;
1835 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1836 * explicitly initialize it.
1838 if (!mono_aot_is_shared_got_offset (got_offset
)) {
1839 //mono_print_ji (ji);
1841 ctx
->cfg
->got_access_count
++;
1848 get_dummy_aotconst (EmitContext
*ctx
, LLVMTypeRef llvm_type
)
1850 LLVMValueRef indexes
[2];
1851 LLVMValueRef got_entry_addr
, load
;
1853 LLVMBuilderRef builder
= ctx
->builder
;
1854 indexes
[0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
1855 indexes
[1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
1856 got_entry_addr
= LLVMBuildGEP (builder
, ctx
->module
->dummy_got_var
, indexes
, 2, "");
1858 load
= LLVMBuildLoad (builder
, got_entry_addr
, "");
1859 load
= convert (ctx
, load
, llvm_type
);
1871 get_callee_llvmonly (EmitContext
*ctx
, LLVMTypeRef llvm_sig
, MonoJumpInfoType type
, gconstpointer data
)
1873 LLVMValueRef callee
;
1874 char *callee_name
= NULL
;
1876 if (ctx
->module
->static_link
&& ctx
->module
->assembly
->image
!= mono_get_corlib ()) {
1877 if (type
== MONO_PATCH_INFO_JIT_ICALL_ID
) {
1878 MonoJitICallInfo
* const info
= mono_find_jit_icall_info ((MonoJitICallId
)(gsize
)data
);
1880 if (info
->func
!= info
->wrapper
) {
1881 type
= MONO_PATCH_INFO_METHOD
;
1882 data
= mono_icall_get_wrapper_method (info
);
1883 callee_name
= mono_aot_get_mangled_method_name ((MonoMethod
*)data
);
1885 } else if (type
== MONO_PATCH_INFO_METHOD
) {
1886 MonoMethod
*method
= (MonoMethod
*)data
;
1887 if (m_class_get_image (method
->klass
) != ctx
->module
->assembly
->image
&& mono_aot_is_externally_callable (method
))
1888 callee_name
= mono_aot_get_mangled_method_name (method
);
1893 callee_name
= mono_aot_get_direct_call_symbol (type
, data
);
1895 /* Directly callable */
1897 callee
= (LLVMValueRef
)g_hash_table_lookup (ctx
->module
->direct_callables
, callee_name
);
1899 callee
= LLVMAddFunction (ctx
->lmodule
, callee_name
, llvm_sig
);
1901 LLVMSetVisibility (callee
, LLVMHiddenVisibility
);
1903 g_hash_table_insert (ctx
->module
->direct_callables
, (char*)callee_name
, callee
);
1905 /* LLVMTypeRef's are uniqued */
1906 if (LLVMGetElementType (LLVMTypeOf (callee
)) != llvm_sig
)
1907 return LLVMConstBitCast (callee
, LLVMPointerType (llvm_sig
, 0));
1909 g_free (callee_name
);
1915 * Change references to icalls/pinvokes/jit icalls to their wrappers when in corlib, so
1916 * they can be called directly.
1918 if (ctx
->module
->assembly
->image
== mono_get_corlib () && type
== MONO_PATCH_INFO_JIT_ICALL_ID
) {
1919 MonoJitICallInfo
* const info
= mono_find_jit_icall_info ((MonoJitICallId
)(gsize
)data
);
1921 if (info
->func
!= info
->wrapper
) {
1922 type
= MONO_PATCH_INFO_METHOD
;
1923 data
= mono_icall_get_wrapper_method (info
);
1926 if (ctx
->module
->assembly
->image
== mono_get_corlib () && type
== MONO_PATCH_INFO_METHOD
) {
1927 MonoMethod
*method
= (MonoMethod
*)data
;
1928 if (m_method_is_icall (method
) || m_method_is_pinvoke (method
))
1929 data
= mono_marshal_get_native_wrapper (method
, TRUE
, TRUE
);
1933 * Instead of emitting an indirect call through a got slot, emit a placeholder, and
1934 * replace it with a direct call or an indirect call in mono_llvm_fixup_aot_module ()
1935 * after all methods have been emitted.
1937 if (type
== MONO_PATCH_INFO_METHOD
) {
1938 MonoMethod
*method
= (MonoMethod
*)data
;
1939 if (m_class_get_image (method
->klass
)->assembly
== ctx
->module
->assembly
) {
1940 MonoJumpInfo tmp_ji
;
1943 tmp_ji
.data
.target
= method
;
1945 MonoJumpInfo
*ji
= mono_aot_patch_info_dup (&tmp_ji
);
1946 ji
->next
= ctx
->cfg
->patch_info
;
1947 ctx
->cfg
->patch_info
= ji
;
1948 LLVMTypeRef llvm_type
= LLVMPointerType (llvm_sig
, 0);
1950 ctx
->cfg
->got_access_count
++;
1952 CallSite
*info
= g_new0 (CallSite
, 1);
1953 info
->method
= method
;
1955 info
->type
= llvm_type
;
1958 * Emit a dummy load to represent the callee, and either replace it with
1959 * a reference to the llvm method for the callee, or from a load from the
1962 LLVMValueRef load
= get_dummy_aotconst (ctx
, llvm_type
);
1965 g_ptr_array_add (ctx
->callsite_list
, info
);
1972 * All other calls are made through the GOT.
1974 callee
= get_aotconst (ctx
, type
, data
, LLVMPointerType (llvm_sig
, 0));
1982 * Return an llvm value representing the callee given by the arguments.
1985 get_callee (EmitContext
*ctx
, LLVMTypeRef llvm_sig
, MonoJumpInfoType type
, gconstpointer data
)
1987 LLVMValueRef callee
;
1989 MonoJumpInfo
*ji
= NULL
;
1992 return get_callee_llvmonly (ctx
, llvm_sig
, type
, data
);
1995 /* Cross-assembly direct calls */
1996 if (type
== MONO_PATCH_INFO_METHOD
) {
1997 MonoMethod
*cmethod
= (MonoMethod
*)data
;
1999 if (m_class_get_image (cmethod
->klass
) != ctx
->module
->assembly
->image
) {
2000 MonoJumpInfo tmp_ji
;
2002 memset (&tmp_ji
, 0, sizeof (MonoJumpInfo
));
2004 tmp_ji
.data
.target
= data
;
2005 if (mono_aot_is_direct_callable (&tmp_ji
)) {
2007 * This will add a reference to cmethod's image so it will
2008 * be loaded when the current AOT image is loaded, so
2009 * the GOT slots used by the init method code are initialized.
2011 tmp_ji
.type
= MONO_PATCH_INFO_IMAGE
;
2012 tmp_ji
.data
.image
= m_class_get_image (cmethod
->klass
);
2013 ji
= mono_aot_patch_info_dup (&tmp_ji
);
2014 mono_aot_get_got_offset (ji
);
2016 callee_name
= mono_aot_get_mangled_method_name (cmethod
);
2018 callee
= (LLVMValueRef
)g_hash_table_lookup (ctx
->module
->direct_callables
, callee_name
);
2020 callee
= LLVMAddFunction (ctx
->lmodule
, callee_name
, llvm_sig
);
2022 LLVMSetLinkage (callee
, LLVMExternalLinkage
);
2024 g_hash_table_insert (ctx
->module
->direct_callables
, callee_name
, callee
);
2026 /* LLVMTypeRef's are uniqued */
2027 if (LLVMGetElementType (LLVMTypeOf (callee
)) != llvm_sig
)
2028 callee
= LLVMConstBitCast (callee
, LLVMPointerType (llvm_sig
, 0));
2030 g_free (callee_name
);
2037 callee_name
= mono_aot_get_plt_symbol (type
, data
);
2041 if (ctx
->cfg
->compile_aot
)
2042 /* Add a patch so referenced wrappers can be compiled in full aot mode */
2043 mono_add_patch_info (ctx
->cfg
, 0, type
, data
);
2046 callee
= (LLVMValueRef
)g_hash_table_lookup (ctx
->module
->plt_entries
, callee_name
);
2048 callee
= LLVMAddFunction (ctx
->lmodule
, callee_name
, llvm_sig
);
2050 LLVMSetVisibility (callee
, LLVMHiddenVisibility
);
2052 g_hash_table_insert (ctx
->module
->plt_entries
, (char*)callee_name
, callee
);
2055 if (ctx
->cfg
->compile_aot
) {
2056 ji
= g_new0 (MonoJumpInfo
, 1);
2058 ji
->data
.target
= data
;
2060 g_hash_table_insert (ctx
->module
->plt_entries_ji
, ji
, callee
);
2067 get_jit_callee (EmitContext
*ctx
, const char *name
, LLVMTypeRef llvm_sig
, MonoJumpInfoType type
, gconstpointer data
)
2071 // This won't be patched so compile the wrapper immediately
2072 if (type
== MONO_PATCH_INFO_JIT_ICALL_ID
) {
2073 MonoJitICallInfo
* const info
= mono_find_jit_icall_info ((MonoJitICallId
)(gsize
)data
);
2074 target
= (gpointer
)mono_icall_get_wrapper_full (info
, TRUE
);
2076 target
= resolve_patch (ctx
->cfg
, type
, data
);
2079 LLVMValueRef tramp_var
= LLVMAddGlobal (ctx
->lmodule
, LLVMPointerType (llvm_sig
, 0), name
);
2080 LLVMSetInitializer (tramp_var
, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64
)(size_t)target
, FALSE
), LLVMPointerType (llvm_sig
, 0)));
2081 LLVMSetLinkage (tramp_var
, LLVMExternalLinkage
);
2082 LLVMValueRef callee
= LLVMBuildLoad (ctx
->builder
, tramp_var
, "");
2087 get_handler_clause (MonoCompile
*cfg
, MonoBasicBlock
*bb
)
2089 MonoMethodHeader
*header
= cfg
->header
;
2090 MonoExceptionClause
*clause
;
2094 if (bb
->region
!= -1 && MONO_BBLOCK_IS_IN_REGION (bb
, MONO_REGION_TRY
))
2095 return (bb
->region
>> 8) - 1;
2098 for (i
= 0; i
< header
->num_clauses
; ++i
) {
2099 clause
= &header
->clauses
[i
];
2101 if (MONO_OFFSET_IN_CLAUSE (clause
, bb
->real_offset
) && clause
->flags
== MONO_EXCEPTION_CLAUSE_NONE
)
2108 static MonoExceptionClause
*
2109 get_most_deep_clause (MonoCompile
*cfg
, EmitContext
*ctx
, MonoBasicBlock
*bb
)
2111 if (bb
== cfg
->bb_init
)
2113 // Since they're sorted by nesting we just need
2114 // the first one that the bb is a member of
2115 for (int i
= 0; i
< cfg
->header
->num_clauses
; i
++) {
2116 MonoExceptionClause
*curr
= &cfg
->header
->clauses
[i
];
2118 if (MONO_OFFSET_IN_CLAUSE (curr
, bb
->real_offset
))
2126 set_metadata_flag (LLVMValueRef v
, const char *flag_name
)
2128 LLVMValueRef md_arg
;
2131 md_kind
= LLVMGetMDKindID (flag_name
, strlen (flag_name
));
2132 md_arg
= LLVMMDString ("mono", 4);
2133 LLVMSetMetadata (v
, md_kind
, LLVMMDNode (&md_arg
, 1));
2137 set_nonnull_load_flag (LLVMValueRef v
)
2139 LLVMValueRef md_arg
;
2141 const char *flag_name
;
2143 flag_name
= "nonnull";
2144 md_kind
= LLVMGetMDKindID (flag_name
, strlen (flag_name
));
2145 md_arg
= LLVMMDString ("<index>", strlen ("<index>"));
2146 LLVMSetMetadata (v
, md_kind
, LLVMMDNode (&md_arg
, 1));
2150 set_nontemporal_flag (LLVMValueRef v
)
2152 LLVMValueRef md_arg
;
2154 const char *flag_name
;
2156 // FIXME: Cache this
2157 flag_name
= "nontemporal";
2158 md_kind
= LLVMGetMDKindID (flag_name
, strlen (flag_name
));
2159 md_arg
= const_int32 (1);
2160 LLVMSetMetadata (v
, md_kind
, LLVMMDNode (&md_arg
, 1));
2164 set_invariant_load_flag (LLVMValueRef v
)
2166 LLVMValueRef md_arg
;
2168 const char *flag_name
;
2170 // FIXME: Cache this
2171 flag_name
= "invariant.load";
2172 md_kind
= LLVMGetMDKindID (flag_name
, strlen (flag_name
));
2173 md_arg
= LLVMMDString ("<index>", strlen ("<index>"));
2174 LLVMSetMetadata (v
, md_kind
, LLVMMDNode (&md_arg
, 1));
2180 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
2184 emit_call (EmitContext
*ctx
, MonoBasicBlock
*bb
, LLVMBuilderRef
*builder_ref
, LLVMValueRef callee
, LLVMValueRef
*args
, int pindex
)
2186 MonoCompile
*cfg
= ctx
->cfg
;
2187 LLVMValueRef lcall
= NULL
;
2188 LLVMBuilderRef builder
= *builder_ref
;
2189 MonoExceptionClause
*clause
;
2191 if (ctx
->llvm_only
) {
2192 clause
= get_most_deep_clause (cfg
, ctx
, bb
);
2195 g_assert (clause
->flags
== MONO_EXCEPTION_CLAUSE_NONE
|| clause
->flags
== MONO_EXCEPTION_CLAUSE_FINALLY
|| clause
->flags
== MONO_EXCEPTION_CLAUSE_FAULT
);
2198 * Have to use an invoke instead of a call, branching to the
2199 * handler bblock of the clause containing this bblock.
2201 intptr_t key
= CLAUSE_END(clause
);
2203 LLVMBasicBlockRef lpad_bb
= (LLVMBasicBlockRef
)g_hash_table_lookup (ctx
->exc_meta
, (gconstpointer
)key
);
2205 // FIXME: Find the one that has the lowest end bound for the right start address
2206 // FIXME: Finally + nesting
2209 LLVMBasicBlockRef noex_bb
= gen_bb (ctx
, "CALL_NOEX_BB");
2212 lcall
= LLVMBuildInvoke (builder
, callee
, args
, pindex
, noex_bb
, lpad_bb
, "");
2214 builder
= ctx
->builder
= create_builder (ctx
);
2215 LLVMPositionBuilderAtEnd (ctx
->builder
, noex_bb
);
2217 ctx
->bblocks
[bb
->block_num
].end_bblock
= noex_bb
;
2221 int clause_index
= get_handler_clause (cfg
, bb
);
2223 if (clause_index
!= -1) {
2224 MonoMethodHeader
*header
= cfg
->header
;
2225 MonoExceptionClause
*ec
= &header
->clauses
[clause_index
];
2226 MonoBasicBlock
*tblock
;
2227 LLVMBasicBlockRef ex_bb
, noex_bb
;
2230 * Have to use an invoke instead of a call, branching to the
2231 * handler bblock of the clause containing this bblock.
2234 g_assert (ec
->flags
== MONO_EXCEPTION_CLAUSE_NONE
|| ec
->flags
== MONO_EXCEPTION_CLAUSE_FINALLY
|| ec
->flags
== MONO_EXCEPTION_CLAUSE_FAULT
);
2236 tblock
= cfg
->cil_offset_to_bb
[ec
->handler_offset
];
2239 ctx
->bblocks
[tblock
->block_num
].invoke_target
= TRUE
;
2241 ex_bb
= get_bb (ctx
, tblock
);
2243 noex_bb
= gen_bb (ctx
, "NOEX_BB");
2246 lcall
= LLVMBuildInvoke (builder
, callee
, args
, pindex
, noex_bb
, ex_bb
, "");
2248 builder
= ctx
->builder
= create_builder (ctx
);
2249 LLVMPositionBuilderAtEnd (ctx
->builder
, noex_bb
);
2251 ctx
->bblocks
[bb
->block_num
].end_bblock
= noex_bb
;
2256 lcall
= LLVMBuildCall (builder
, callee
, args
, pindex
, "");
2257 ctx
->builder
= builder
;
2261 *builder_ref
= ctx
->builder
;
2267 emit_load (EmitContext
*ctx
, MonoBasicBlock
*bb
, LLVMBuilderRef
*builder_ref
, int size
, LLVMValueRef addr
, LLVMValueRef base
, const char *name
, gboolean is_faulting
, gboolean is_volatile
, BarrierKind barrier
)
2272 * We emit volatile loads for loads which can fault, because otherwise
2273 * LLVM will generate invalid code when encountering a load from a
2276 if (barrier
!= LLVM_BARRIER_NONE
)
2277 res
= mono_llvm_build_atomic_load (*builder_ref
, addr
, name
, is_volatile
, size
, barrier
);
2279 res
= mono_llvm_build_load (*builder_ref
, addr
, name
, is_volatile
);
2285 emit_store_general (EmitContext
*ctx
, MonoBasicBlock
*bb
, LLVMBuilderRef
*builder_ref
, int size
, LLVMValueRef value
, LLVMValueRef addr
, LLVMValueRef base
, gboolean is_faulting
, gboolean is_volatile
, BarrierKind barrier
)
2287 if (barrier
!= LLVM_BARRIER_NONE
)
2288 mono_llvm_build_aligned_store (*builder_ref
, value
, addr
, barrier
, size
);
2290 mono_llvm_build_store (*builder_ref
, value
, addr
, is_volatile
, barrier
);
2294 emit_store (EmitContext
*ctx
, MonoBasicBlock
*bb
, LLVMBuilderRef
*builder_ref
, int size
, LLVMValueRef value
, LLVMValueRef addr
, LLVMValueRef base
, gboolean is_faulting
, gboolean is_volatile
)
2296 emit_store_general (ctx
, bb
, builder_ref
, size
, value
, addr
, base
, is_faulting
, is_volatile
, LLVM_BARRIER_NONE
);
2300 * emit_cond_system_exception:
2302 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2303 * Might set the ctx exception.
2306 emit_cond_system_exception (EmitContext
*ctx
, MonoBasicBlock
*bb
, const char *exc_type
, LLVMValueRef cmp
, gboolean force_explicit
)
2308 LLVMBasicBlockRef ex_bb
, ex2_bb
= NULL
, noex_bb
;
2309 LLVMBuilderRef builder
;
2310 MonoClass
*exc_class
;
2311 LLVMValueRef args
[2];
2312 LLVMValueRef callee
;
2313 gboolean no_pc
= FALSE
;
2314 static MonoClass
*exc_classes
[MONO_EXC_INTRINS_NUM
];
2316 if (IS_TARGET_AMD64
)
2317 /* Some platforms don't require the pc argument */
2320 int exc_id
= mini_exception_id_by_name (exc_type
);
2321 if (!exc_classes
[exc_id
])
2322 exc_classes
[exc_id
] = mono_class_load_from_name (mono_get_corlib (), "System", exc_type
);
2323 exc_class
= exc_classes
[exc_id
];
2325 ex_bb
= gen_bb (ctx
, "EX_BB");
2327 ex2_bb
= gen_bb (ctx
, "EX2_BB");
2328 noex_bb
= gen_bb (ctx
, "NOEX_BB");
2330 LLVMValueRef branch
= LLVMBuildCondBr (ctx
->builder
, cmp
, ex_bb
, noex_bb
);
2331 if (exc_id
== MONO_EXC_NULL_REF
&& !ctx
->cfg
->disable_llvm_implicit_null_checks
&& !force_explicit
) {
2332 mono_llvm_set_implicit_branch (ctx
->builder
, branch
);
2335 /* Emit exception throwing code */
2336 ctx
->builder
= builder
= create_builder (ctx
);
2337 LLVMPositionBuilderAtEnd (builder
, ex_bb
);
2339 if (ctx
->cfg
->llvm_only
) {
2340 LLVMBuildBr (builder
, ex2_bb
);
2342 ctx
->builder
= builder
= create_builder (ctx
);
2343 LLVMPositionBuilderAtEnd (ctx
->builder
, ex2_bb
);
2345 if (exc_id
== MONO_EXC_NULL_REF
) {
2346 static LLVMTypeRef sig
;
2349 sig
= LLVMFunctionType0 (LLVMVoidType (), FALSE
);
2350 /* Can't cache this */
2351 callee
= get_callee (ctx
, sig
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception
));
2352 emit_call (ctx
, bb
, &builder
, callee
, NULL
, 0);
2354 static LLVMTypeRef sig
;
2357 sig
= LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE
);
2358 callee
= get_callee (ctx
, sig
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_throw_corlib_exception
));
2359 args
[0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class
) - MONO_TOKEN_TYPE_DEF
, FALSE
);
2360 emit_call (ctx
, bb
, &builder
, callee
, args
, 1);
2363 LLVMBuildUnreachable (builder
);
2365 ctx
->builder
= builder
= create_builder (ctx
);
2366 LLVMPositionBuilderAtEnd (ctx
->builder
, noex_bb
);
2368 ctx
->bblocks
[bb
->block_num
].end_bblock
= noex_bb
;
2374 callee
= ctx
->module
->throw_corlib_exception
;
2379 sig
= LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE
);
2381 sig
= LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE
);
2383 const MonoJitICallId icall_id
= MONO_JIT_ICALL_mono_llvm_throw_corlib_exception_abs_trampoline
;
2385 if (ctx
->cfg
->compile_aot
) {
2386 callee
= get_callee (ctx
, sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
2389 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2390 * - On x86, LLVM generated code doesn't push the arguments
2391 * - The trampoline takes the throw address as an arguments, not a pc offset.
2393 callee
= get_jit_callee (ctx
, "llvm_throw_corlib_exception_trampoline", sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
2396 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2397 * added by get_jit_callee ().
2399 ex2_bb
= gen_bb (ctx
, "EX2_BB");
2400 LLVMBuildBr (builder
, ex2_bb
);
2403 ctx
->builder
= builder
= create_builder (ctx
);
2404 LLVMPositionBuilderAtEnd (ctx
->builder
, ex2_bb
);
2408 args
[0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class
) - MONO_TOKEN_TYPE_DEF
, FALSE
);
2411 * The LLVM mono branch contains changes so a block address can be passed as an
2412 * argument to a call.
2415 emit_call (ctx
, bb
, &builder
, callee
, args
, 1);
2417 args
[1] = LLVMBlockAddress (ctx
->lmethod
, ex_bb
);
2418 emit_call (ctx
, bb
, &builder
, callee
, args
, 2);
2421 LLVMBuildUnreachable (builder
);
2423 ctx
->builder
= builder
= create_builder (ctx
);
2424 LLVMPositionBuilderAtEnd (ctx
->builder
, noex_bb
);
2426 ctx
->bblocks
[bb
->block_num
].end_bblock
= noex_bb
;
2433 * emit_args_to_vtype:
2435 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2438 emit_args_to_vtype (EmitContext
*ctx
, LLVMBuilderRef builder
, MonoType
*t
, LLVMValueRef address
, LLVMArgInfo
*ainfo
, LLVMValueRef
*args
)
2440 int j
, size
, nslots
;
2443 t
= mini_get_underlying_type (t
);
2444 klass
= mono_class_from_mono_type_internal (t
);
2445 size
= mono_class_value_size (klass
, NULL
);
2447 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, klass
))
2448 address
= LLVMBuildBitCast (ctx
->builder
, address
, LLVMPointerType (LLVMInt8Type (), 0), "");
2450 if (ainfo
->storage
== LLVMArgAsFpArgs
)
2451 nslots
= ainfo
->nslots
;
2455 for (j
= 0; j
< nslots
; ++j
) {
2456 LLVMValueRef index
[2], addr
, daddr
;
2457 int part_size
= size
> TARGET_SIZEOF_VOID_P
? TARGET_SIZEOF_VOID_P
: size
;
2458 LLVMTypeRef part_type
;
2460 while (part_size
!= 1 && part_size
!= 2 && part_size
!= 4 && part_size
< 8)
2463 if (ainfo
->pair_storage
[j
] == LLVMArgNone
)
2466 switch (ainfo
->pair_storage
[j
]) {
2467 case LLVMArgInIReg
: {
2468 part_type
= LLVMIntType (part_size
* 8);
2469 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, klass
)) {
2470 index
[0] = LLVMConstInt (LLVMInt32Type (), j
* TARGET_SIZEOF_VOID_P
, FALSE
);
2471 addr
= LLVMBuildGEP (builder
, address
, index
, 1, "");
2473 daddr
= LLVMBuildBitCast (ctx
->builder
, address
, LLVMPointerType (IntPtrType (), 0), "");
2474 index
[0] = LLVMConstInt (LLVMInt32Type (), j
, FALSE
);
2475 addr
= LLVMBuildGEP (builder
, daddr
, index
, 1, "");
2477 LLVMBuildStore (builder
, convert (ctx
, args
[j
], part_type
), LLVMBuildBitCast (ctx
->builder
, addr
, LLVMPointerType (part_type
, 0), ""));
2480 case LLVMArgInFPReg
: {
2481 LLVMTypeRef arg_type
;
2483 if (ainfo
->esize
== 8)
2484 arg_type
= LLVMDoubleType ();
2486 arg_type
= LLVMFloatType ();
2488 index
[0] = LLVMConstInt (LLVMInt32Type (), j
, FALSE
);
2489 daddr
= LLVMBuildBitCast (ctx
->builder
, address
, LLVMPointerType (arg_type
, 0), "");
2490 addr
= LLVMBuildGEP (builder
, daddr
, index
, 1, "");
2491 LLVMBuildStore (builder
, args
[j
], addr
);
2497 g_assert_not_reached ();
2500 size
-= TARGET_SIZEOF_VOID_P
;
2505 * emit_vtype_to_args:
2507 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2508 * into ARGS, and the number of arguments into NARGS.
2511 emit_vtype_to_args (EmitContext
*ctx
, LLVMBuilderRef builder
, MonoType
*t
, LLVMValueRef address
, LLVMArgInfo
*ainfo
, LLVMValueRef
*args
, guint32
*nargs
)
2514 int j
, size
, nslots
;
2515 LLVMTypeRef arg_type
;
2517 t
= mini_get_underlying_type (t
);
2518 size
= get_vtype_size (t
);
2520 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, mono_class_from_mono_type_internal (t
)))
2521 address
= LLVMBuildBitCast (ctx
->builder
, address
, LLVMPointerType (LLVMInt8Type (), 0), "");
2523 if (ainfo
->storage
== LLVMArgAsFpArgs
)
2524 nslots
= ainfo
->nslots
;
2527 for (j
= 0; j
< nslots
; ++j
) {
2528 LLVMValueRef index
[2], addr
, daddr
;
2529 int partsize
= size
> TARGET_SIZEOF_VOID_P
? TARGET_SIZEOF_VOID_P
: size
;
2531 if (ainfo
->pair_storage
[j
] == LLVMArgNone
)
2534 switch (ainfo
->pair_storage
[j
]) {
2536 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, mono_class_from_mono_type_internal (t
))) {
2537 index
[0] = LLVMConstInt (LLVMInt32Type (), j
* TARGET_SIZEOF_VOID_P
, FALSE
);
2538 addr
= LLVMBuildGEP (builder
, address
, index
, 1, "");
2540 daddr
= LLVMBuildBitCast (ctx
->builder
, address
, LLVMPointerType (IntPtrType (), 0), "");
2541 index
[0] = LLVMConstInt (LLVMInt32Type (), j
, FALSE
);
2542 addr
= LLVMBuildGEP (builder
, daddr
, index
, 1, "");
2544 args
[pindex
++] = convert (ctx
, LLVMBuildLoad (builder
, LLVMBuildBitCast (ctx
->builder
, addr
, LLVMPointerType (LLVMIntType (partsize
* 8), 0), ""), ""), IntPtrType ());
2546 case LLVMArgInFPReg
:
2547 if (ainfo
->esize
== 8)
2548 arg_type
= LLVMDoubleType ();
2550 arg_type
= LLVMFloatType ();
2551 daddr
= LLVMBuildBitCast (ctx
->builder
, address
, LLVMPointerType (arg_type
, 0), "");
2552 index
[0] = LLVMConstInt (LLVMInt32Type (), j
, FALSE
);
2553 addr
= LLVMBuildGEP (builder
, daddr
, index
, 1, "");
2554 args
[pindex
++] = LLVMBuildLoad (builder
, addr
, "");
2559 g_assert_not_reached ();
2561 size
-= TARGET_SIZEOF_VOID_P
;
2568 build_alloca_llvm_type_name (EmitContext
*ctx
, LLVMTypeRef t
, int align
, const char *name
)
2571 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2572 * get executed every time control reaches them.
2574 LLVMPositionBuilder (ctx
->alloca_builder
, get_bb (ctx
, ctx
->cfg
->bb_entry
), ctx
->last_alloca
);
2576 ctx
->last_alloca
= mono_llvm_build_alloca (ctx
->alloca_builder
, t
, NULL
, align
, name
);
2577 return ctx
->last_alloca
;
2581 build_alloca_llvm_type (EmitContext
*ctx
, LLVMTypeRef t
, int align
)
2583 return build_alloca_llvm_type_name (ctx
, t
, align
, "");
2587 build_alloca (EmitContext
*ctx
, MonoType
*t
)
2589 MonoClass
*k
= mono_class_from_mono_type_internal (t
);
2592 g_assert (!mini_is_gsharedvt_variable_type (t
));
2594 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, k
))
2595 align
= mono_class_value_size (k
, NULL
);
2597 align
= mono_class_min_align (k
);
2599 /* Sometimes align is not a power of 2 */
2600 while (mono_is_power_of_two (align
) == -1)
2603 return build_alloca_llvm_type (ctx
, type_to_llvm_type (ctx
, t
), align
);
2607 emit_gsharedvt_ldaddr (EmitContext
*ctx
, int vreg
)
2611 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2613 MonoCompile
*cfg
= ctx
->cfg
;
2614 LLVMBuilderRef builder
= ctx
->builder
;
2615 LLVMValueRef offset
, offset_var
;
2616 LLVMValueRef info_var
= ctx
->values
[cfg
->gsharedvt_info_var
->dreg
];
2617 LLVMValueRef locals_var
= ctx
->values
[cfg
->gsharedvt_locals_var
->dreg
];
2621 g_assert (info_var
);
2622 g_assert (locals_var
);
2624 int idx
= cfg
->gsharedvt_vreg_to_idx
[vreg
] - 1;
2626 offset
= LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo
, entries
) + (idx
* TARGET_SIZEOF_VOID_P
), FALSE
);
2627 ptr
= LLVMBuildAdd (builder
, convert (ctx
, info_var
, IntPtrType ()), convert (ctx
, offset
, IntPtrType ()), "");
2629 name
= g_strdup_printf ("gsharedvt_local_%d_offset", vreg
);
2630 offset_var
= LLVMBuildLoad (builder
, convert (ctx
, ptr
, LLVMPointerType (LLVMInt32Type (), 0)), name
);
2632 return LLVMBuildAdd (builder
, convert (ctx
, locals_var
, IntPtrType ()), convert (ctx
, offset_var
, IntPtrType ()), "");
2636 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2639 mark_as_used (MonoLLVMModule
*module
, LLVMValueRef global
)
2642 module
->used
= g_ptr_array_sized_new (16);
2643 g_ptr_array_add (module
->used
, global
);
2647 emit_llvm_used (MonoLLVMModule
*module
)
2649 LLVMModuleRef lmodule
= module
->lmodule
;
2650 LLVMTypeRef used_type
;
2651 LLVMValueRef used
, *used_elem
;
2657 used_type
= LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module
->used
->len
);
2658 used
= LLVMAddGlobal (lmodule
, used_type
, "llvm.used");
2659 used_elem
= g_new0 (LLVMValueRef
, module
->used
->len
);
2660 for (i
= 0; i
< module
->used
->len
; ++i
)
2661 used_elem
[i
] = LLVMConstBitCast ((LLVMValueRef
)g_ptr_array_index (module
->used
, i
), LLVMPointerType (LLVMInt8Type (), 0));
2662 LLVMSetInitializer (used
, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem
, module
->used
->len
));
2663 LLVMSetLinkage (used
, LLVMAppendingLinkage
);
2664 LLVMSetSection (used
, "llvm.metadata");
2670 * Emit a function mapping method indexes to their code
2673 emit_get_method (MonoLLVMModule
*module
)
2675 LLVMModuleRef lmodule
= module
->lmodule
;
2676 LLVMValueRef func
, switch_ins
, m
;
2677 LLVMBasicBlockRef entry_bb
, fail_bb
, bb
, code_start_bb
, code_end_bb
;
2678 LLVMBasicBlockRef
*bbs
= NULL
;
2680 LLVMBuilderRef builder
= LLVMCreateBuilder ();
2681 LLVMValueRef table
= NULL
;
2684 gboolean emit_table
= FALSE
;
2688 * Emit a table of functions instead of a switch statement,
2689 * its very efficient on wasm. This might be usable on
2690 * other platforms too.
2695 rtype
= LLVMPointerType (LLVMInt8Type (), 0);
2698 LLVMTypeRef table_type
;
2699 LLVMValueRef
*table_elems
;
2702 int table_len
= module
->max_method_idx
+ 1;
2703 table_type
= LLVMArrayType (rtype
, table_len
);
2704 table_name
= g_strdup_printf ("%s_method_table", module
->assembly
->aname
.name
);
2705 table
= LLVMAddGlobal (lmodule
, table_type
, table_name
);
2706 table_elems
= g_new0 (LLVMValueRef
, table_len
);
2707 for (i
= 0; i
< table_len
; ++i
) {
2708 m
= (LLVMValueRef
)g_hash_table_lookup (module
->idx_to_lmethod
, GINT_TO_POINTER (i
));
2709 if (m
&& !g_hash_table_lookup (module
->no_method_table_lmethods
, m
))
2710 table_elems
[i
] = LLVMBuildBitCast (builder
, m
, rtype
, "");
2712 table_elems
[i
] = LLVMConstNull (rtype
);
2714 LLVMSetInitializer (table
, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), table_elems
, table_len
));
2718 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2719 * but generating code seems safer.
2721 func
= LLVMAddFunction (lmodule
, module
->get_method_symbol
, LLVMFunctionType1 (rtype
, LLVMInt32Type (), FALSE
));
2722 LLVMSetLinkage (func
, LLVMExternalLinkage
);
2723 LLVMSetVisibility (func
, LLVMHiddenVisibility
);
2724 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_UNWIND
);
2725 module
->get_method
= func
;
2727 entry_bb
= LLVMAppendBasicBlock (func
, "ENTRY");
2730 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2731 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2732 * then we will have to find another solution.
2735 name
= g_strdup_printf ("BB_CODE_START");
2736 code_start_bb
= LLVMAppendBasicBlock (func
, name
);
2738 LLVMPositionBuilderAtEnd (builder
, code_start_bb
);
2739 LLVMBuildRet (builder
, LLVMBuildBitCast (builder
, module
->code_start
, rtype
, ""));
2741 name
= g_strdup_printf ("BB_CODE_END");
2742 code_end_bb
= LLVMAppendBasicBlock (func
, name
);
2744 LLVMPositionBuilderAtEnd (builder
, code_end_bb
);
2745 LLVMBuildRet (builder
, LLVMBuildBitCast (builder
, module
->code_end
, rtype
, ""));
2750 * case -1: return code_start;
2751 * case -2: return code_end;
2752 * default: return method_table [index];
2754 LLVMBasicBlockRef default_bb
= LLVMAppendBasicBlock (func
, "DEFAULT");
2755 LLVMPositionBuilderAtEnd (builder
, default_bb
);
2756 LLVMValueRef base
= table
;
2757 LLVMValueRef indexes
[2];
2758 indexes
[0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
2759 indexes
[1] = LLVMGetParam (func
, 0);
2760 LLVMValueRef addr
= LLVMBuildGEP (builder
, base
, indexes
, 2, "");
2761 LLVMValueRef res
= mono_llvm_build_load (builder
, addr
, "", FALSE
);
2762 LLVMBuildRet (builder
, res
);
2764 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
2766 switch_ins
= LLVMBuildSwitch (builder
, LLVMGetParam (func
, 0), default_bb
, 0);
2767 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), -1, FALSE
), code_start_bb
);
2768 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), -2, FALSE
), code_end_bb
);
2770 bbs
= g_new0 (LLVMBasicBlockRef
, module
->max_method_idx
+ 1);
2771 for (i
= 0; i
< module
->max_method_idx
+ 1; ++i
) {
2772 name
= g_strdup_printf ("BB_%d", i
);
2773 bb
= LLVMAppendBasicBlock (func
, name
);
2777 LLVMPositionBuilderAtEnd (builder
, bb
);
2779 m
= (LLVMValueRef
)g_hash_table_lookup (module
->idx_to_lmethod
, GINT_TO_POINTER (i
));
2780 if (m
&& !g_hash_table_lookup (module
->no_method_table_lmethods
, m
))
2781 LLVMBuildRet (builder
, LLVMBuildBitCast (builder
, m
, rtype
, ""));
2783 LLVMBuildRet (builder
, LLVMConstNull (rtype
));
2786 fail_bb
= LLVMAppendBasicBlock (func
, "FAIL");
2787 LLVMPositionBuilderAtEnd (builder
, fail_bb
);
2788 LLVMBuildRet (builder
, LLVMConstNull (rtype
));
2790 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
2792 switch_ins
= LLVMBuildSwitch (builder
, LLVMGetParam (func
, 0), fail_bb
, 0);
2793 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), -1, FALSE
), code_start_bb
);
2794 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), -2, FALSE
), code_end_bb
);
2795 for (i
= 0; i
< module
->max_method_idx
+ 1; ++i
) {
2796 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), bbs
[i
]);
2800 mark_as_used (module
, func
);
2802 LLVMDisposeBuilder (builder
);
2806 * emit_get_unbox_tramp:
2808 * Emit a function mapping method indexes to their unbox trampoline
2811 emit_get_unbox_tramp (MonoLLVMModule
*module
)
2813 LLVMModuleRef lmodule
= module
->lmodule
;
2814 LLVMValueRef func
, switch_ins
, m
;
2815 LLVMBasicBlockRef entry_bb
, fail_bb
, bb
;
2816 LLVMBasicBlockRef
*bbs
;
2818 LLVMBuilderRef builder
= LLVMCreateBuilder ();
2821 gboolean emit_table
= FALSE
;
2823 /* Similar to emit_get_method () */
2825 #ifndef TARGET_WATCHOS
2829 rtype
= LLVMPointerType (LLVMInt8Type (), 0);
2832 // About 10% of methods have an unbox tramp, so emit a table of indexes for them
2833 // that the runtime can search using a binary search
2835 for (i
= 0; i
< module
->max_method_idx
+ 1; ++i
) {
2836 m
= (LLVMValueRef
)g_hash_table_lookup (module
->idx_to_unbox_tramp
, GINT_TO_POINTER (i
));
2841 LLVMTypeRef table_type
, elemtype
;
2842 LLVMValueRef
*table_elems
;
2849 elemsize
= module
->max_method_idx
< 65000 ? 2 : 4;
2852 elemtype
= elemsize
== 2 ? LLVMInt16Type () : LLVMInt32Type ();
2853 table_type
= LLVMArrayType (elemtype
, table_len
);
2854 table_name
= g_strdup_printf ("%s_unbox_tramp_indexes", module
->assembly
->aname
.name
);
2855 table
= LLVMAddGlobal (lmodule
, table_type
, table_name
);
2856 table_elems
= g_new0 (LLVMValueRef
, table_len
);
2858 for (i
= 0; i
< module
->max_method_idx
+ 1; ++i
) {
2859 m
= (LLVMValueRef
)g_hash_table_lookup (module
->idx_to_unbox_tramp
, GINT_TO_POINTER (i
));
2861 table_elems
[idx
++] = LLVMConstInt (elemtype
, i
, FALSE
);
2863 LLVMSetInitializer (table
, LLVMConstArray (elemtype
, table_elems
, table_len
));
2864 module
->unbox_tramp_indexes
= table
;
2866 // The trampoline table
2868 table_type
= LLVMArrayType (elemtype
, table_len
);
2869 table_name
= g_strdup_printf ("%s_unbox_trampolines", module
->assembly
->aname
.name
);
2870 table
= LLVMAddGlobal (lmodule
, table_type
, table_name
);
2871 table_elems
= g_new0 (LLVMValueRef
, table_len
);
2873 for (i
= 0; i
< module
->max_method_idx
+ 1; ++i
) {
2874 m
= (LLVMValueRef
)g_hash_table_lookup (module
->idx_to_unbox_tramp
, GINT_TO_POINTER (i
));
2876 table_elems
[idx
++] = LLVMBuildBitCast (builder
, m
, rtype
, "");
2878 LLVMSetInitializer (table
, LLVMConstArray (elemtype
, table_elems
, table_len
));
2879 module
->unbox_trampolines
= table
;
2881 module
->unbox_tramp_num
= table_len
;
2882 module
->unbox_tramp_elemsize
= elemsize
;
2886 func
= LLVMAddFunction (lmodule
, module
->get_unbox_tramp_symbol
, LLVMFunctionType1 (rtype
, LLVMInt32Type (), FALSE
));
2887 LLVMSetLinkage (func
, LLVMExternalLinkage
);
2888 LLVMSetVisibility (func
, LLVMHiddenVisibility
);
2889 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_UNWIND
);
2890 module
->get_unbox_tramp
= func
;
2892 entry_bb
= LLVMAppendBasicBlock (func
, "ENTRY");
2894 bbs
= g_new0 (LLVMBasicBlockRef
, module
->max_method_idx
+ 1);
2895 for (i
= 0; i
< module
->max_method_idx
+ 1; ++i
) {
2896 m
= (LLVMValueRef
)g_hash_table_lookup (module
->idx_to_unbox_tramp
, GINT_TO_POINTER (i
));
2900 name
= g_strdup_printf ("BB_%d", i
);
2901 bb
= LLVMAppendBasicBlock (func
, name
);
2905 LLVMPositionBuilderAtEnd (builder
, bb
);
2907 LLVMBuildRet (builder
, LLVMBuildBitCast (builder
, m
, rtype
, ""));
2910 fail_bb
= LLVMAppendBasicBlock (func
, "FAIL");
2911 LLVMPositionBuilderAtEnd (builder
, fail_bb
);
2912 LLVMBuildRet (builder
, LLVMConstNull (rtype
));
2914 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
2916 switch_ins
= LLVMBuildSwitch (builder
, LLVMGetParam (func
, 0), fail_bb
, 0);
2917 for (i
= 0; i
< module
->max_method_idx
+ 1; ++i
) {
2918 m
= (LLVMValueRef
)g_hash_table_lookup (module
->idx_to_unbox_tramp
, GINT_TO_POINTER (i
));
2922 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), bbs
[i
]);
2925 mark_as_used (module
, func
);
2926 LLVMDisposeBuilder (builder
);
2930 * emit_init_aotconst:
2932 * Emit a function to initialize the aotconst_ variables. Called by the runtime.
2935 emit_init_aotconst (MonoLLVMModule
*module
)
2937 LLVMModuleRef lmodule
= module
->lmodule
;
2938 LLVMValueRef func
, switch_ins
;
2939 LLVMBasicBlockRef entry_bb
, fail_bb
, bb
;
2940 LLVMBasicBlockRef
*bbs
= NULL
;
2941 LLVMBuilderRef builder
= LLVMCreateBuilder ();
2944 func
= LLVMAddFunction (lmodule
, module
->init_aotconst_symbol
, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE
));
2945 LLVMSetLinkage (func
, LLVMExternalLinkage
);
2946 LLVMSetVisibility (func
, LLVMHiddenVisibility
);
2947 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_UNWIND
);
2948 module
->init_aotconst_func
= func
;
2950 entry_bb
= LLVMAppendBasicBlock (func
, "ENTRY");
2952 bbs
= g_new0 (LLVMBasicBlockRef
, module
->max_got_offset
+ 1);
2953 for (int i
= 0; i
< module
->max_got_offset
+ 1; ++i
) {
2954 name
= g_strdup_printf ("BB_%d", i
);
2955 bb
= LLVMAppendBasicBlock (func
, name
);
2959 LLVMPositionBuilderAtEnd (builder
, bb
);
2961 LLVMValueRef var
= g_hash_table_lookup (module
->aotconst_vars
, GINT_TO_POINTER (i
));
2963 LLVMValueRef addr
= LLVMBuildBitCast (builder
, var
, LLVMPointerType (IntPtrType (), 0), "");
2964 LLVMBuildStore (builder
, LLVMGetParam (func
, 1), addr
);
2966 LLVMBuildRetVoid (builder
);
2969 fail_bb
= LLVMAppendBasicBlock (func
, "FAIL");
2970 LLVMPositionBuilderAtEnd (builder
, fail_bb
);
2971 LLVMBuildRetVoid (builder
);
2973 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
2975 switch_ins
= LLVMBuildSwitch (builder
, LLVMGetParam (func
, 0), fail_bb
, 0);
2976 for (int i
= 0; i
< module
->max_got_offset
+ 1; ++i
)
2977 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), bbs
[i
]);
2979 LLVMDisposeBuilder (builder
);
2982 /* Add a function to mark the beginning of LLVM code */
2984 emit_llvm_code_start (MonoLLVMModule
*module
)
2986 LLVMModuleRef lmodule
= module
->lmodule
;
2988 LLVMBasicBlockRef entry_bb
;
2989 LLVMBuilderRef builder
;
2991 func
= LLVMAddFunction (lmodule
, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL
, 0, FALSE
));
2992 LLVMSetLinkage (func
, LLVMInternalLinkage
);
2993 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_UNWIND
);
2994 module
->code_start
= func
;
2995 entry_bb
= LLVMAppendBasicBlock (func
, "ENTRY");
2996 builder
= LLVMCreateBuilder ();
2997 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
2998 LLVMBuildRetVoid (builder
);
2999 LLVMDisposeBuilder (builder
);
3005 * Emit functions to initialize LLVM methods.
3006 * These are wrappers around the mini_llvm_init_method () JIT icall.
3007 * The wrappers handle adding the 'amodule' argument, loading the vtable from different locations, and they have
3008 * a cold calling convention.
3011 emit_init_func (MonoLLVMModule
*module
, MonoAotInitSubtype subtype
)
3013 LLVMModuleRef lmodule
= module
->lmodule
;
3014 LLVMValueRef func
, indexes
[2], args
[16], callee
, info_var
, index_var
, inited_var
, cmp
;
3015 LLVMBasicBlockRef entry_bb
, inited_bb
, notinited_bb
;
3016 LLVMBuilderRef builder
;
3017 LLVMTypeRef icall_sig
;
3018 const char *wrapper_name
= mono_marshal_get_aot_init_wrapper_name (subtype
);
3019 LLVMTypeRef func_type
= NULL
;
3020 LLVMTypeRef arg_type
= module
->ptr_type
;
3022 char *name
= g_strdup_printf ("%s_%s", module
->global_prefix
, wrapper_name
);
3025 case AOT_INIT_METHOD
:
3026 func_type
= LLVMFunctionType1 (LLVMVoidType (), arg_type
, FALSE
);
3028 case AOT_INIT_METHOD_GSHARED_MRGCTX
:
3029 case AOT_INIT_METHOD_GSHARED_VTABLE
:
3030 func_type
= LLVMFunctionType2 (LLVMVoidType (), arg_type
, IntPtrType (), FALSE
);
3032 case AOT_INIT_METHOD_GSHARED_THIS
:
3033 func_type
= LLVMFunctionType2 (LLVMVoidType (), arg_type
, ObjRefType (), FALSE
);
3036 g_assert_not_reached ();
3039 func
= LLVMAddFunction (lmodule
, name
, func_type
);
3041 info_var
= LLVMGetParam (func
, 0);
3043 LLVMSetLinkage (func
, LLVMInternalLinkage
);
3045 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_INLINE
);
3047 set_cold_cconv (func
);
3049 entry_bb
= LLVMAppendBasicBlock (func
, "ENTRY");
3051 builder
= LLVMCreateBuilder ();
3052 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
3054 /* Load method_index which is emitted at the start of the method info */
3055 indexes
[0] = const_int32 (0);
3056 indexes
[1] = const_int32 (0);
3057 // FIXME: Make sure its aligned
3058 index_var
= LLVMBuildLoad (builder
, LLVMBuildGEP (builder
, LLVMBuildBitCast (builder
, info_var
, LLVMPointerType (LLVMInt32Type (), 0), ""), indexes
, 1, ""), "method_index");
3060 /* Check for is_inited here as well, since this can be called from JITted code which might not check it */
3061 indexes
[0] = const_int32 (0);
3062 indexes
[1] = index_var
;
3063 inited_var
= LLVMBuildLoad (builder
, LLVMBuildGEP (builder
, module
->inited_var
, indexes
, 2, ""), "is_inited");
3065 cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, inited_var
, LLVMConstInt (LLVMTypeOf (inited_var
), 0, FALSE
), "");
3067 inited_bb
= LLVMAppendBasicBlock (func
, "INITED");
3068 notinited_bb
= LLVMAppendBasicBlock (func
, "NOT_INITED");
3070 LLVMBuildCondBr (builder
, cmp
, notinited_bb
, inited_bb
);
3072 LLVMPositionBuilderAtEnd (builder
, notinited_bb
);
3074 LLVMValueRef amodule_var
= get_aotconst_module (module
, builder
, MONO_PATCH_INFO_AOT_MODULE
, NULL
, LLVMPointerType (IntPtrType (), 0), NULL
, NULL
);
3076 args
[0] = LLVMBuildPtrToInt (builder
, module
->info_var
, IntPtrType (), "");
3077 args
[1] = LLVMBuildPtrToInt (builder
, amodule_var
, IntPtrType (), "");
3078 args
[2] = info_var
;
3081 case AOT_INIT_METHOD
:
3082 args
[3] = LLVMConstNull (IntPtrType ());
3084 case AOT_INIT_METHOD_GSHARED_VTABLE
:
3085 args
[3] = LLVMGetParam (func
, 1);
3087 case AOT_INIT_METHOD_GSHARED_THIS
:
3088 /* Load this->vtable */
3089 args
[3] = LLVMBuildBitCast (builder
, LLVMGetParam (func
, 1), LLVMPointerType (IntPtrType (), 0), "");
3090 indexes
[0] = const_int32 (MONO_STRUCT_OFFSET (MonoObject
, vtable
) / SIZEOF_VOID_P
);
3091 args
[3] = LLVMBuildLoad (builder
, LLVMBuildGEP (builder
, args
[3], indexes
, 1, ""), "vtable");
3093 case AOT_INIT_METHOD_GSHARED_MRGCTX
:
3094 /* Load mrgctx->vtable */
3095 args
[3] = LLVMBuildIntToPtr (builder
, LLVMGetParam (func
, 1), LLVMPointerType (IntPtrType (), 0), "");
3096 indexes
[0] = const_int32 (MONO_STRUCT_OFFSET (MonoMethodRuntimeGenericContext
, class_vtable
) / SIZEOF_VOID_P
);
3097 args
[3] = LLVMBuildLoad (builder
, LLVMBuildGEP (builder
, args
[3], indexes
, 1, ""), "vtable");
3100 g_assert_not_reached ();
3104 /* Call the mini_llvm_init_method JIT icall */
3105 icall_sig
= LLVMFunctionType4 (LLVMVoidType (), IntPtrType (), IntPtrType (), arg_type
, IntPtrType (), FALSE
);
3106 callee
= get_aotconst_module (module
, builder
, MONO_PATCH_INFO_JIT_ICALL_ID
, GINT_TO_POINTER (MONO_JIT_ICALL_mini_llvm_init_method
), LLVMPointerType (icall_sig
, 0), NULL
, NULL
);
3107 LLVMBuildCall (builder
, callee
, args
, LLVMCountParamTypes (icall_sig
), "");
3110 * Set the inited flag
3111 * This is already done by the LLVM methods themselves, but its needed by JITted methods.
3113 indexes
[0] = const_int32 (0);
3114 indexes
[1] = index_var
;
3115 LLVMBuildStore (builder
, LLVMConstInt (LLVMInt8Type (), 1, FALSE
), LLVMBuildGEP (builder
, module
->inited_var
, indexes
, 2, ""));
3117 LLVMBuildBr (builder
, inited_bb
);
3119 LLVMPositionBuilderAtEnd (builder
, inited_bb
);
3120 LLVMBuildRetVoid (builder
);
3122 LLVMVerifyFunction (func
, LLVMAbortProcessAction
);
3123 LLVMDisposeBuilder (builder
);
3129 /* Emit a wrapper around the parameterless JIT icall ICALL_ID with a cold calling convention */
3131 emit_icall_cold_wrapper (MonoLLVMModule
*module
, LLVMModuleRef lmodule
, MonoJitICallId icall_id
, gboolean aot
)
3133 LLVMValueRef func
, callee
;
3134 LLVMBasicBlockRef entry_bb
;
3135 LLVMBuilderRef builder
;
3139 name
= g_strdup_printf ("%s_icall_cold_wrapper_%d", module
->global_prefix
, icall_id
);
3141 func
= LLVMAddFunction (lmodule
, name
, LLVMFunctionType (LLVMVoidType (), NULL
, 0, FALSE
));
3142 sig
= LLVMFunctionType (LLVMVoidType (), NULL
, 0, FALSE
);
3143 LLVMSetLinkage (func
, LLVMInternalLinkage
);
3144 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_INLINE
);
3145 set_cold_cconv (func
);
3147 entry_bb
= LLVMAppendBasicBlock (func
, "ENTRY");
3148 builder
= LLVMCreateBuilder ();
3149 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
3152 callee
= get_aotconst_module (module
, builder
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
), LLVMPointerType (sig
, 0), NULL
, NULL
);
3154 MonoJitICallInfo
* const info
= mono_find_jit_icall_info (icall_id
);
3155 gpointer target
= (gpointer
)mono_icall_get_wrapper_full (info
, TRUE
);
3157 LLVMValueRef tramp_var
= LLVMAddGlobal (lmodule
, LLVMPointerType (sig
, 0), name
);
3158 LLVMSetInitializer (tramp_var
, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64
)(size_t)target
, FALSE
), LLVMPointerType (sig
, 0)));
3159 LLVMSetLinkage (tramp_var
, LLVMExternalLinkage
);
3160 callee
= LLVMBuildLoad (builder
, tramp_var
, "");
3162 LLVMBuildCall (builder
, callee
, NULL
, 0, "");
3164 LLVMBuildRetVoid (builder
);
3166 LLVMVerifyFunction(func
, LLVMAbortProcessAction
);
3167 LLVMDisposeBuilder (builder
);
3172 * Emit wrappers around the C icalls used to initialize llvm methods, to
3173 * make the calling code smaller and to enable usage of the llvm
3174 * cold calling convention.
3177 emit_init_funcs (MonoLLVMModule
*module
)
3179 module
->init_method
= emit_init_func (module
, AOT_INIT_METHOD
);
3180 module
->init_method_gshared_mrgctx
= emit_init_func (module
, AOT_INIT_METHOD_GSHARED_MRGCTX
);
3181 module
->init_method_gshared_this
= emit_init_func (module
, AOT_INIT_METHOD_GSHARED_THIS
);
3182 module
->init_method_gshared_vtable
= emit_init_func (module
, AOT_INIT_METHOD_GSHARED_VTABLE
);
3186 get_init_func (MonoLLVMModule
*module
, MonoAotInitSubtype subtype
)
3189 case AOT_INIT_METHOD
:
3190 return module
->init_method
;
3191 case AOT_INIT_METHOD_GSHARED_MRGCTX
:
3192 return module
->init_method_gshared_mrgctx
;
3193 case AOT_INIT_METHOD_GSHARED_THIS
:
3194 return module
->init_method_gshared_this
;
3195 case AOT_INIT_METHOD_GSHARED_VTABLE
:
3196 return module
->init_method_gshared_vtable
;
3198 g_assert_not_reached ();
3203 emit_gc_safepoint_poll (MonoLLVMModule
*module
, LLVMModuleRef lmodule
, MonoCompile
*cfg
)
3205 gboolean is_aot
= cfg
== NULL
|| cfg
->compile_aot
;
3206 LLVMValueRef func
= mono_llvm_get_or_insert_gc_safepoint_poll (lmodule
);
3207 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_UNWIND
);
3209 LLVMSetLinkage (func
, LLVMWeakODRLinkage
);
3211 mono_llvm_add_func_attr (func
, LLVM_ATTR_OPTIMIZE_NONE
); // no need to waste time here, the function is already optimized and will be inlined.
3212 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_INLINE
); // optnone attribute requires noinline (but it will be inlined anyway)
3213 if (!module
->gc_poll_cold_wrapper_compiled
) {
3215 /* Compiling a method here is a bit ugly, but it works */
3216 MonoMethod
*wrapper
= mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL
);
3217 module
->gc_poll_cold_wrapper_compiled
= mono_jit_compile_method (wrapper
, error
);
3218 mono_error_assert_ok (error
);
3221 LLVMBasicBlockRef entry_bb
= LLVMAppendBasicBlock (func
, "gc.safepoint_poll.entry");
3222 LLVMBasicBlockRef poll_bb
= LLVMAppendBasicBlock (func
, "gc.safepoint_poll.poll");
3223 LLVMBasicBlockRef exit_bb
= LLVMAppendBasicBlock (func
, "gc.safepoint_poll.exit");
3224 LLVMTypeRef ptr_type
= LLVMPointerType (IntPtrType (), 0);
3225 LLVMBuilderRef builder
= LLVMCreateBuilder ();
3228 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
3229 LLVMValueRef poll_val_ptr
;
3231 poll_val_ptr
= get_aotconst_module (module
, builder
, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG
, NULL
, ptr_type
, NULL
, NULL
);
3233 LLVMValueRef poll_val_int
= LLVMConstInt (IntPtrType (), (guint64
) &mono_polling_required
, FALSE
);
3234 poll_val_ptr
= LLVMBuildIntToPtr (builder
, poll_val_int
, ptr_type
, "");
3236 LLVMValueRef poll_val_ptr_load
= LLVMBuildLoad (builder
, poll_val_ptr
, ""); // probably needs to be volatile
3237 LLVMValueRef poll_val
= LLVMBuildPtrToInt (builder
, poll_val_ptr_load
, IntPtrType (), "");
3238 LLVMValueRef poll_val_zero
= LLVMConstNull (LLVMTypeOf (poll_val
));
3239 LLVMValueRef cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, poll_val
, poll_val_zero
, "");
3240 mono_llvm_build_weighted_branch (builder
, cmp
, exit_bb
, poll_bb
, 1000 /* weight for exit_bb */, 1 /* weight for poll_bb */);
3243 LLVMPositionBuilderAtEnd (builder
, poll_bb
);
3246 LLVMValueRef icall_wrapper
= emit_icall_cold_wrapper (module
, lmodule
, MONO_JIT_ICALL_mono_threads_state_poll
, TRUE
);
3247 module
->gc_poll_cold_wrapper
= icall_wrapper
;
3248 call
= LLVMBuildCall (builder
, icall_wrapper
, NULL
, 0, "");
3250 // in JIT mode we have to emit @gc.safepoint_poll function for each method (module)
3251 // this function calls gc_poll_cold_wrapper_compiled via a global variable.
3252 // @gc.safepoint_poll will be inlined and can be deleted after -place-safepoints pass.
3253 LLVMTypeRef poll_sig
= LLVMFunctionType0 (LLVMVoidType (), FALSE
);
3254 LLVMTypeRef poll_sig_ptr
= LLVMPointerType (poll_sig
, 0);
3255 gpointer target
= resolve_patch (cfg
, MONO_PATCH_INFO_ABS
, module
->gc_poll_cold_wrapper_compiled
);
3256 LLVMValueRef tramp_var
= LLVMAddGlobal (lmodule
, poll_sig_ptr
, "mono_threads_state_poll");
3257 LLVMValueRef target_val
= LLVMConstInt (LLVMInt64Type (), (guint64
) target
, FALSE
);
3258 LLVMSetInitializer (tramp_var
, LLVMConstIntToPtr (target_val
, poll_sig_ptr
));
3259 LLVMSetLinkage (tramp_var
, LLVMExternalLinkage
);
3260 LLVMValueRef callee
= LLVMBuildLoad (builder
, tramp_var
, "");
3261 call
= LLVMBuildCall (builder
, callee
, NULL
, 0, "");
3263 set_call_cold_cconv (call
);
3264 LLVMBuildBr (builder
, exit_bb
);
3267 LLVMPositionBuilderAtEnd (builder
, exit_bb
);
3268 LLVMBuildRetVoid (builder
);
3269 LLVMDisposeBuilder (builder
);
3273 emit_llvm_code_end (MonoLLVMModule
*module
)
3275 LLVMModuleRef lmodule
= module
->lmodule
;
3277 LLVMBasicBlockRef entry_bb
;
3278 LLVMBuilderRef builder
;
3280 func
= LLVMAddFunction (lmodule
, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL
, 0, FALSE
));
3281 LLVMSetLinkage (func
, LLVMInternalLinkage
);
3282 mono_llvm_add_func_attr (func
, LLVM_ATTR_NO_UNWIND
);
3283 module
->code_end
= func
;
3284 entry_bb
= LLVMAppendBasicBlock (func
, "ENTRY");
3285 builder
= LLVMCreateBuilder ();
3286 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
3287 LLVMBuildRetVoid (builder
);
3288 LLVMDisposeBuilder (builder
);
3292 emit_div_check (EmitContext
*ctx
, LLVMBuilderRef builder
, MonoBasicBlock
*bb
, MonoInst
*ins
, LLVMValueRef lhs
, LLVMValueRef rhs
)
3294 gboolean need_div_check
= ctx
->cfg
->backend
->need_div_check
;
3297 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
3298 need_div_check
= TRUE
;
3300 if (!need_div_check
)
3303 switch (ins
->opcode
) {
3316 case OP_IDIV_UN_IMM
:
3317 case OP_LDIV_UN_IMM
:
3318 case OP_IREM_UN_IMM
:
3319 case OP_LREM_UN_IMM
: {
3321 gboolean is_signed
= (ins
->opcode
== OP_IDIV
|| ins
->opcode
== OP_LDIV
|| ins
->opcode
== OP_IREM
|| ins
->opcode
== OP_LREM
||
3322 ins
->opcode
== OP_IDIV_IMM
|| ins
->opcode
== OP_LDIV_IMM
|| ins
->opcode
== OP_IREM_IMM
|| ins
->opcode
== OP_LREM_IMM
);
3324 cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, rhs
, LLVMConstInt (LLVMTypeOf (rhs
), 0, FALSE
), "");
3325 emit_cond_system_exception (ctx
, bb
, "DivideByZeroException", cmp
, FALSE
);
3328 builder
= ctx
->builder
;
3330 /* b == -1 && a == 0x80000000 */
3332 LLVMValueRef c
= (LLVMTypeOf (lhs
) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs
), 0x80000000, FALSE
) : LLVMConstInt (LLVMTypeOf (lhs
), 0x8000000000000000LL
, FALSE
);
3333 LLVMValueRef cond1
= LLVMBuildICmp (builder
, LLVMIntEQ
, rhs
, LLVMConstInt (LLVMTypeOf (rhs
), -1, FALSE
), "");
3334 LLVMValueRef cond2
= LLVMBuildICmp (builder
, LLVMIntEQ
, lhs
, c
, "");
3336 cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, LLVMBuildAnd (builder
, cond1
, cond2
, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE
), "");
3337 emit_cond_system_exception (ctx
, bb
, "OverflowException", cmp
, FALSE
);
3340 builder
= ctx
->builder
;
3352 * Emit code to initialize the GOT slots used by the method.
3355 emit_method_init (EmitContext
*ctx
)
3357 LLVMValueRef indexes
[16], args
[16], callee
;
3358 LLVMValueRef inited_var
, cmp
, call
;
3359 LLVMBasicBlockRef inited_bb
, notinited_bb
;
3360 LLVMBuilderRef builder
= ctx
->builder
;
3361 MonoCompile
*cfg
= ctx
->cfg
;
3363 ctx
->module
->max_inited_idx
= MAX (ctx
->module
->max_inited_idx
, cfg
->method_index
);
3365 indexes
[0] = const_int32 (0);
3366 indexes
[1] = const_int32 (cfg
->method_index
);
3367 inited_var
= LLVMBuildLoad (builder
, LLVMBuildGEP (builder
, ctx
->module
->inited_var
, indexes
, 2, ""), "is_inited");
3369 args
[0] = inited_var
;
3370 args
[1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE
);
3371 inited_var
= LLVMBuildCall (ctx
->builder
, get_intrins (ctx
, INTRINS_EXPECT_I8
), args
, 2, "");
3373 cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, inited_var
, LLVMConstInt (LLVMTypeOf (inited_var
), 0, FALSE
), "");
3375 inited_bb
= ctx
->inited_bb
;
3376 notinited_bb
= gen_bb (ctx
, "NOTINITED_BB");
3378 ctx
->cfg
->llvmonly_init_cond
= LLVMBuildCondBr (ctx
->builder
, cmp
, notinited_bb
, inited_bb
);
3380 builder
= ctx
->builder
= create_builder (ctx
);
3381 LLVMPositionBuilderAtEnd (ctx
->builder
, notinited_bb
);
3383 LLVMTypeRef type
= LLVMArrayType (LLVMInt8Type (), 0);
3384 char *symbol
= g_strdup_printf ("info_dummy_%s", cfg
->llvm_method_name
);
3385 LLVMValueRef info_var
= LLVMAddGlobal (ctx
->lmodule
, type
, symbol
);
3387 cfg
->llvm_dummy_info_var
= info_var
;
3389 args
[0] = convert (ctx
, info_var
, ctx
->module
->ptr_type
);
3392 if (ctx
->rgctx_arg
&& ((cfg
->method
->is_inflated
&& mono_method_get_context (cfg
->method
)->method_inst
) ||
3393 mini_method_is_default_method (cfg
->method
))) {
3394 args
[1] = convert (ctx
, ctx
->rgctx_arg
, IntPtrType ());
3395 callee
= ctx
->module
->init_method_gshared_mrgctx
;
3396 call
= LLVMBuildCall (builder
, callee
, args
, 2, "");
3397 } else if (ctx
->rgctx_arg
) {
3398 /* A vtable is passed as the rgctx argument */
3399 args
[1] = convert (ctx
, ctx
->rgctx_arg
, IntPtrType ());
3400 callee
= ctx
->module
->init_method_gshared_vtable
;
3401 call
= LLVMBuildCall (builder
, callee
, args
, 2, "");
3402 } else if (cfg
->gshared
) {
3403 args
[1] = convert (ctx
, ctx
->this_arg
, ObjRefType ());
3404 callee
= ctx
->module
->init_method_gshared_this
;
3405 call
= LLVMBuildCall (builder
, callee
, args
, 2, "");
3407 callee
= ctx
->module
->init_method
;
3408 call
= LLVMBuildCall (builder
, callee
, args
, 1, "");
3412 * This enables llvm to keep arguments in their original registers/
3413 * scratch registers, since the call will not clobber them.
3415 set_call_cold_cconv (call
);
3417 // Set the inited flag
3418 indexes
[0] = const_int32 (0);
3419 indexes
[1] = const_int32 (cfg
->method_index
);
3420 LLVMBuildStore (builder
, LLVMConstInt (LLVMInt8Type (), 1, FALSE
), LLVMBuildGEP (builder
, ctx
->module
->inited_var
, indexes
, 2, ""));
3422 LLVMBuildBr (builder
, inited_bb
);
3423 ctx
->bblocks
[cfg
->bb_entry
->block_num
].end_bblock
= inited_bb
;
3425 builder
= ctx
->builder
= create_builder (ctx
);
3426 LLVMPositionBuilderAtEnd (ctx
->builder
, inited_bb
);
3430 emit_unbox_tramp (EmitContext
*ctx
, const char *method_name
, LLVMTypeRef method_type
, LLVMValueRef method
, int method_index
)
3433 * Emit unbox trampoline using a tailcall
3435 LLVMValueRef tramp
, call
, *args
;
3436 LLVMBuilderRef builder
;
3437 LLVMBasicBlockRef lbb
;
3438 LLVMCallInfo
*linfo
;
3442 tramp_name
= g_strdup_printf ("ut_%s", method_name
);
3443 tramp
= LLVMAddFunction (ctx
->module
->lmodule
, tramp_name
, method_type
);
3444 LLVMSetLinkage (tramp
, LLVMInternalLinkage
);
3445 mono_llvm_add_func_attr (tramp
, LLVM_ATTR_OPTIMIZE_FOR_SIZE
);
3446 //mono_llvm_add_func_attr (tramp, LLVM_ATTR_NO_UNWIND);
3448 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
3449 if (!ctx
->llvm_only
&& ctx
->rgctx_arg_pindex
!= -1)
3450 mono_llvm_add_param_attr (LLVMGetParam (tramp
, ctx
->rgctx_arg_pindex
), LLVM_ATTR_IN_REG
);
3451 if (ctx
->cfg
->vret_addr
) {
3452 LLVMSetValueName (LLVMGetParam (tramp
, linfo
->vret_arg_pindex
), "vret");
3453 if (linfo
->ret
.storage
== LLVMArgVtypeByRef
) {
3454 mono_llvm_add_param_attr (LLVMGetParam (tramp
, linfo
->vret_arg_pindex
), LLVM_ATTR_STRUCT_RET
);
3455 mono_llvm_add_param_attr (LLVMGetParam (tramp
, linfo
->vret_arg_pindex
), LLVM_ATTR_NO_ALIAS
);
3459 lbb
= LLVMAppendBasicBlock (tramp
, "");
3460 builder
= LLVMCreateBuilder ();
3461 LLVMPositionBuilderAtEnd (builder
, lbb
);
3463 nargs
= LLVMCountParamTypes (method_type
);
3464 args
= g_new0 (LLVMValueRef
, nargs
);
3465 for (i
= 0; i
< nargs
; ++i
) {
3466 args
[i
] = LLVMGetParam (tramp
, i
);
3467 if (i
== ctx
->this_arg_pindex
) {
3468 LLVMTypeRef arg_type
= LLVMTypeOf (args
[i
]);
3470 args
[i
] = LLVMBuildPtrToInt (builder
, args
[i
], IntPtrType (), "");
3471 args
[i
] = LLVMBuildAdd (builder
, args
[i
], LLVMConstInt (IntPtrType (), MONO_ABI_SIZEOF (MonoObject
), FALSE
), "");
3472 args
[i
] = LLVMBuildIntToPtr (builder
, args
[i
], arg_type
, "");
3475 call
= LLVMBuildCall (builder
, method
, args
, nargs
, "");
3476 if (!ctx
->llvm_only
&& ctx
->rgctx_arg_pindex
!= -1)
3477 mono_llvm_add_instr_attr (call
, 1 + ctx
->rgctx_arg_pindex
, LLVM_ATTR_IN_REG
);
3478 if (linfo
->ret
.storage
== LLVMArgVtypeByRef
)
3479 mono_llvm_add_instr_attr (call
, 1 + linfo
->vret_arg_pindex
, LLVM_ATTR_STRUCT_RET
);
3481 // FIXME: This causes assertions in clang
3482 //mono_llvm_set_must_tailcall (call);
3483 if (LLVMGetReturnType (method_type
) == LLVMVoidType ())
3484 LLVMBuildRetVoid (builder
);
3486 LLVMBuildRet (builder
, call
);
3488 g_hash_table_insert (ctx
->module
->idx_to_unbox_tramp
, GINT_TO_POINTER (method_index
), tramp
);
3489 LLVMDisposeBuilder (builder
);
3495 * Emit code to load/convert arguments.
3498 emit_entry_bb (EmitContext
*ctx
, LLVMBuilderRef builder
)
3501 MonoCompile
*cfg
= ctx
->cfg
;
3502 MonoMethodSignature
*sig
= ctx
->sig
;
3503 LLVMCallInfo
*linfo
= ctx
->linfo
;
3507 LLVMBuilderRef old_builder
= ctx
->builder
;
3508 ctx
->builder
= builder
;
3510 ctx
->alloca_builder
= create_builder (ctx
);
3513 * Handle indirect/volatile variables by allocating memory for them
3514 * using 'alloca', and storing their address in a temporary.
3516 for (i
= 0; i
< cfg
->num_varinfo
; ++i
) {
3517 MonoInst
*var
= cfg
->varinfo
[i
];
3520 if ((var
->opcode
== OP_GSHAREDVT_LOCAL
|| var
->opcode
== OP_GSHAREDVT_ARG_REGOFFSET
))
3524 // For GC stack scanning to work, have to spill all reference variables to the stack
3525 // Some ref variables have type intptr
3526 if (ctx
->has_safepoints
&& (MONO_TYPE_IS_REFERENCE (var
->inst_vtype
) || var
->inst_vtype
->type
== MONO_TYPE_I
) && var
!= ctx
->cfg
->rgctx_var
)
3527 var
->flags
|= MONO_INST_INDIRECT
;
3530 if (var
->flags
& (MONO_INST_VOLATILE
|MONO_INST_INDIRECT
) || (mini_type_is_vtype (var
->inst_vtype
) && !MONO_CLASS_IS_SIMD (ctx
->cfg
, var
->klass
))) {
3531 vtype
= type_to_llvm_type (ctx
, var
->inst_vtype
);
3534 /* Could be already created by an OP_VPHI */
3535 if (!ctx
->addresses
[var
->dreg
]) {
3536 ctx
->addresses
[var
->dreg
] = build_alloca (ctx
, var
->inst_vtype
);
3537 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
3539 ctx
->vreg_cli_types
[var
->dreg
] = var
->inst_vtype
;
3543 names
= g_new (char *, sig
->param_count
);
3544 mono_method_get_param_names (cfg
->method
, (const char **) names
);
3546 for (i
= 0; i
< sig
->param_count
; ++i
) {
3547 LLVMArgInfo
*ainfo
= &linfo
->args
[i
+ sig
->hasthis
];
3548 int reg
= cfg
->args
[i
+ sig
->hasthis
]->dreg
;
3551 pindex
= ainfo
->pindex
;
3553 switch (ainfo
->storage
) {
3554 case LLVMArgVtypeInReg
:
3555 case LLVMArgAsFpArgs
: {
3556 LLVMValueRef args
[8];
3559 pindex
+= ainfo
->ndummy_fpargs
;
3561 /* The argument is received as a set of int/fp arguments, store them into the real argument */
3562 memset (args
, 0, sizeof (args
));
3563 if (ainfo
->storage
== LLVMArgVtypeInReg
) {
3564 args
[0] = LLVMGetParam (ctx
->lmethod
, pindex
);
3565 if (ainfo
->pair_storage
[1] != LLVMArgNone
)
3566 args
[1] = LLVMGetParam (ctx
->lmethod
, pindex
+ 1);
3568 g_assert (ainfo
->nslots
<= 8);
3569 for (j
= 0; j
< ainfo
->nslots
; ++j
)
3570 args
[j
] = LLVMGetParam (ctx
->lmethod
, pindex
+ j
);
3572 ctx
->addresses
[reg
] = build_alloca (ctx
, ainfo
->type
);
3574 emit_args_to_vtype (ctx
, builder
, ainfo
->type
, ctx
->addresses
[reg
], ainfo
, args
);
3577 case LLVMArgVtypeByVal
: {
3578 ctx
->addresses
[reg
] = LLVMGetParam (ctx
->lmethod
, pindex
);
3581 case LLVMArgVtypeAddr
:
3582 case LLVMArgVtypeByRef
: {
3583 /* The argument is passed by ref */
3584 ctx
->addresses
[reg
] = LLVMGetParam (ctx
->lmethod
, pindex
);
3587 case LLVMArgAsIArgs
: {
3588 LLVMValueRef arg
= LLVMGetParam (ctx
->lmethod
, pindex
);
3590 MonoType
*t
= mini_get_underlying_type (ainfo
->type
);
3592 /* The argument is received as an array of ints, store it into the real argument */
3593 ctx
->addresses
[reg
] = build_alloca (ctx
, t
);
3595 size
= mono_class_value_size (mono_class_from_mono_type_internal (t
), NULL
);
3597 } else if (size
< TARGET_SIZEOF_VOID_P
) {
3598 /* The upper bits of the registers might not be valid */
3599 LLVMValueRef val
= LLVMBuildExtractValue (builder
, arg
, 0, "");
3600 LLVMValueRef dest
= convert (ctx
, ctx
->addresses
[reg
], LLVMPointerType (LLVMIntType (size
* 8), 0));
3601 LLVMBuildStore (ctx
->builder
, LLVMBuildTrunc (builder
, val
, LLVMIntType (size
* 8), ""), dest
);
3603 LLVMBuildStore (ctx
->builder
, arg
, convert (ctx
, ctx
->addresses
[reg
], LLVMPointerType (LLVMTypeOf (arg
), 0)));
3607 case LLVMArgVtypeAsScalar
:
3608 g_assert_not_reached ();
3610 case LLVMArgGsharedvtFixed
: {
3611 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
3612 LLVMValueRef arg
= LLVMGetParam (ctx
->lmethod
, pindex
);
3615 name
= g_strdup_printf ("arg_%s", names
[i
]);
3617 name
= g_strdup_printf ("arg_%d", i
);
3619 ctx
->values
[reg
] = LLVMBuildLoad (builder
, convert (ctx
, arg
, LLVMPointerType (type_to_llvm_type (ctx
, ainfo
->type
), 0)), name
);
3622 case LLVMArgGsharedvtFixedVtype
: {
3623 LLVMValueRef arg
= LLVMGetParam (ctx
->lmethod
, pindex
);
3626 name
= g_strdup_printf ("vtype_arg_%s", names
[i
]);
3628 name
= g_strdup_printf ("vtype_arg_%d", i
);
3630 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3631 g_assert (ctx
->addresses
[reg
]);
3632 LLVMSetValueName (ctx
->addresses
[reg
], name
);
3633 LLVMBuildStore (builder
, LLVMBuildLoad (builder
, convert (ctx
, arg
, LLVMPointerType (type_to_llvm_type (ctx
, ainfo
->type
), 0)), ""), ctx
->addresses
[reg
]);
3636 case LLVMArgGsharedvtVariable
:
3637 /* The IR treats these as variables with addresses */
3638 ctx
->addresses
[reg
] = LLVMGetParam (ctx
->lmethod
, pindex
);
3642 /* Needed to avoid phi argument mismatch errors since operations on pointers produce i32/i64 */
3643 if (ainfo
->type
->byref
)
3646 t
= type_to_llvm_type (ctx
, ainfo
->type
);
3647 ctx
->values
[reg
] = convert_full (ctx
, ctx
->values
[reg
], llvm_type_to_stack_type (cfg
, t
), type_is_unsigned (ctx
, ainfo
->type
));
3652 switch (ainfo
->storage
) {
3653 case LLVMArgVtypeInReg
:
3654 case LLVMArgVtypeByVal
:
3655 #ifdef ENABLE_NETCORE
3656 // FIXME: Enabling this fails on windows
3657 case LLVMArgVtypeAddr
:
3658 case LLVMArgVtypeByRef
:
3661 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, mono_class_from_mono_type_internal (ainfo
->type
)))
3662 /* Treat these as normal values */
3663 ctx
->values
[reg
] = LLVMBuildLoad (builder
, ctx
->addresses
[reg
], "");
3673 /* Handle this arguments as inputs to phi nodes */
3674 int reg
= cfg
->args
[0]->dreg
;
3675 if (ctx
->vreg_types
[reg
])
3676 ctx
->values
[reg
] = convert (ctx
, ctx
->values
[reg
], ctx
->vreg_types
[reg
]);
3680 emit_volatile_store (ctx
, cfg
->vret_addr
->dreg
);
3682 emit_volatile_store (ctx
, cfg
->args
[0]->dreg
);
3683 for (i
= 0; i
< sig
->param_count
; ++i
)
3684 if (!mini_type_is_vtype (sig
->params
[i
]))
3685 emit_volatile_store (ctx
, cfg
->args
[i
+ sig
->hasthis
]->dreg
);
3687 if (sig
->hasthis
&& !cfg
->rgctx_var
&& cfg
->gshared
&& !cfg
->llvm_only
) {
3688 LLVMValueRef this_alloc
;
3691 * The exception handling code needs the location where the this argument was
3692 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3693 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3694 * location into the LSDA.
3696 this_alloc
= mono_llvm_build_alloca (builder
, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE
), 0, "");
3697 /* This volatile store will keep the alloca alive */
3698 mono_llvm_build_store (builder
, ctx
->values
[cfg
->args
[0]->dreg
], this_alloc
, TRUE
, LLVM_BARRIER_NONE
);
3700 set_metadata_flag (this_alloc
, "mono.this");
3703 if (cfg
->rgctx_var
) {
3704 if (!(cfg
->rgctx_var
->flags
& MONO_INST_VOLATILE
)) {
3705 /* FIXME: This could be volatile even in llvmonly mode if used inside a clause etc. */
3706 g_assert (!ctx
->addresses
[cfg
->rgctx_var
->dreg
]);
3707 ctx
->values
[cfg
->rgctx_var
->dreg
] = ctx
->rgctx_arg
;
3709 LLVMValueRef rgctx_alloc
, store
;
3712 * We handle the rgctx arg similarly to the this pointer.
3714 g_assert (ctx
->addresses
[cfg
->rgctx_var
->dreg
]);
3715 rgctx_alloc
= ctx
->addresses
[cfg
->rgctx_var
->dreg
];
3716 /* This volatile store will keep the alloca alive */
3717 store
= mono_llvm_build_store (builder
, convert (ctx
, ctx
->rgctx_arg
, IntPtrType ()), rgctx_alloc
, TRUE
, LLVM_BARRIER_NONE
);
3719 set_metadata_flag (rgctx_alloc
, "mono.this");
3723 /* Initialize the method if needed */
3724 if (cfg
->compile_aot
) {
3725 /* Emit a location for the initialization code */
3726 ctx
->init_bb
= gen_bb (ctx
, "INIT_BB");
3727 ctx
->inited_bb
= gen_bb (ctx
, "INITED_BB");
3729 LLVMBuildBr (ctx
->builder
, ctx
->init_bb
);
3730 builder
= ctx
->builder
= create_builder (ctx
);
3731 LLVMPositionBuilderAtEnd (ctx
->builder
, ctx
->inited_bb
);
3732 ctx
->bblocks
[cfg
->bb_entry
->block_num
].end_bblock
= ctx
->inited_bb
;
3735 /* Compute nesting between clauses */
3736 ctx
->nested_in
= (GSList
**)mono_mempool_alloc0 (cfg
->mempool
, sizeof (GSList
*) * cfg
->header
->num_clauses
);
3737 for (i
= 0; i
< cfg
->header
->num_clauses
; ++i
) {
3738 for (j
= 0; j
< cfg
->header
->num_clauses
; ++j
) {
3739 MonoExceptionClause
*clause1
= &cfg
->header
->clauses
[i
];
3740 MonoExceptionClause
*clause2
= &cfg
->header
->clauses
[j
];
3742 if (i
!= j
&& clause1
->try_offset
>= clause2
->try_offset
&& clause1
->handler_offset
<= clause2
->handler_offset
)
3743 ctx
->nested_in
[i
] = g_slist_prepend_mempool (cfg
->mempool
, ctx
->nested_in
[i
], GINT_TO_POINTER (j
));
3748 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3749 * it needs to continue normally, or return back to the exception handling system.
3751 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
3754 if (!(bb
->region
!= -1 && (bb
->flags
& BB_EXCEPTION_HANDLER
)))
3757 if (bb
->in_scount
== 0) {
3760 sprintf (name
, "finally_ind_bb%d", bb
->block_num
);
3761 val
= LLVMBuildAlloca (builder
, LLVMInt32Type (), name
);
3762 LLVMBuildStore (builder
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), val
);
3764 ctx
->bblocks
[bb
->block_num
].finally_ind
= val
;
3766 /* Create a variable to hold the exception var */
3768 ctx
->ex_var
= LLVMBuildAlloca (builder
, ObjRefType (), "exvar");
3771 ctx
->builder
= old_builder
;
3775 needs_extra_arg (EmitContext
*ctx
, MonoMethod
*method
)
3777 WrapperInfo
*info
= NULL
;
3780 * When targeting wasm, the caller and callee signature has to match exactly. This means
3781 * that every method which can be called indirectly need an extra arg since the caller
3782 * will call it through an ftnptr and will pass an extra arg.
3784 if (!ctx
->cfg
->llvm_only
|| !ctx
->emit_dummy_arg
)
3786 if (method
->wrapper_type
)
3787 info
= mono_marshal_get_wrapper_info (method
);
3789 switch (method
->wrapper_type
) {
3790 case MONO_WRAPPER_OTHER
:
3791 if (info
->subtype
== WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG
|| info
->subtype
== WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG
)
3792 /* Already have an explicit extra arg */
3795 case MONO_WRAPPER_MANAGED_TO_NATIVE
:
3796 if (strstr (method
->name
, "icall_wrapper"))
3797 /* These are JIT icall wrappers which are only called from JITted code directly */
3799 /* Normal icalls can be virtual methods which need an extra arg */
3801 case MONO_WRAPPER_RUNTIME_INVOKE
:
3802 case MONO_WRAPPER_ALLOC
:
3803 case MONO_WRAPPER_CASTCLASS
:
3804 case MONO_WRAPPER_WRITE_BARRIER
:
3805 case MONO_WRAPPER_NATIVE_TO_MANAGED
:
3807 case MONO_WRAPPER_STELEMREF
:
3808 if (info
->subtype
!= WRAPPER_SUBTYPE_VIRTUAL_STELEMREF
)
3811 case MONO_WRAPPER_MANAGED_TO_MANAGED
:
3812 if (info
->subtype
== WRAPPER_SUBTYPE_STRING_CTOR
)
3818 if (method
->string_ctor
)
3821 /* These are called from gsharedvt code with an indirect call which doesn't pass an extra arg */
3822 if (method
->klass
== mono_get_string_class () && (strstr (method
->name
, "memcpy") || strstr (method
->name
, "bzero")))
3827 static inline gboolean
3828 is_supported_callconv (EmitContext
*ctx
, MonoCallInst
*call
)
3830 #if defined(TARGET_WIN32) && defined(TARGET_AMD64)
3831 gboolean result
= (call
->signature
->call_convention
== MONO_CALL_DEFAULT
) ||
3832 (call
->signature
->call_convention
== MONO_CALL_C
) ||
3833 (call
->signature
->call_convention
== MONO_CALL_STDCALL
);
3835 gboolean result
= (call
->signature
->call_convention
== MONO_CALL_DEFAULT
) || ((call
->signature
->call_convention
== MONO_CALL_C
) && ctx
->llvm_only
);
3841 process_call (EmitContext
*ctx
, MonoBasicBlock
*bb
, LLVMBuilderRef
*builder_ref
, MonoInst
*ins
)
3843 MonoCompile
*cfg
= ctx
->cfg
;
3844 LLVMValueRef
*values
= ctx
->values
;
3845 LLVMValueRef
*addresses
= ctx
->addresses
;
3846 MonoCallInst
*call
= (MonoCallInst
*)ins
;
3847 MonoMethodSignature
*sig
= call
->signature
;
3848 LLVMValueRef callee
= NULL
, lcall
;
3850 LLVMCallInfo
*cinfo
;
3854 LLVMTypeRef llvm_sig
;
3856 gboolean is_virtual
, calli
;
3857 LLVMBuilderRef builder
= *builder_ref
;
3859 /* If both imt and rgctx arg are required, only pass the imt arg, the rgctx trampoline will pass the rgctx */
3860 if (call
->imt_arg_reg
)
3861 call
->rgctx_arg_reg
= 0;
3863 if (!is_supported_callconv (ctx
, call
)) {
3864 set_failure (ctx
, "non-default callconv");
3868 cinfo
= call
->cinfo
;
3870 if (call
->rgctx_arg_reg
)
3871 cinfo
->rgctx_arg
= TRUE
;
3872 if (call
->imt_arg_reg
)
3873 cinfo
->imt_arg
= TRUE
;
3874 if (!call
->rgctx_arg_reg
&& call
->method
&& needs_extra_arg (ctx
, call
->method
))
3875 cinfo
->dummy_arg
= TRUE
;
3877 vretaddr
= (cinfo
->ret
.storage
== LLVMArgVtypeRetAddr
|| cinfo
->ret
.storage
== LLVMArgVtypeByRef
|| cinfo
->ret
.storage
== LLVMArgGsharedvtFixed
|| cinfo
->ret
.storage
== LLVMArgGsharedvtVariable
|| cinfo
->ret
.storage
== LLVMArgGsharedvtFixedVtype
);
3879 llvm_sig
= sig_to_llvm_sig_full (ctx
, sig
, cinfo
);
3883 int const opcode
= ins
->opcode
;
3885 is_virtual
= opcode
== OP_VOIDCALL_MEMBASE
|| opcode
== OP_CALL_MEMBASE
3886 || opcode
== OP_VCALL_MEMBASE
|| opcode
== OP_LCALL_MEMBASE
3887 || opcode
== OP_FCALL_MEMBASE
|| opcode
== OP_RCALL_MEMBASE
3888 || opcode
== OP_TAILCALL_MEMBASE
;
3889 calli
= !call
->fptr_is_patch
&& (opcode
== OP_VOIDCALL_REG
|| opcode
== OP_CALL_REG
3890 || opcode
== OP_VCALL_REG
|| opcode
== OP_LCALL_REG
|| opcode
== OP_FCALL_REG
3891 || opcode
== OP_RCALL_REG
|| opcode
== OP_TAILCALL_REG
);
3893 /* FIXME: Avoid creating duplicate methods */
3895 if (ins
->flags
& MONO_INST_HAS_METHOD
) {
3899 if (cfg
->compile_aot
) {
3900 callee
= get_callee (ctx
, llvm_sig
, MONO_PATCH_INFO_METHOD
, call
->method
);
3902 set_failure (ctx
, "can't encode patch");
3905 } else if (cfg
->method
== call
->method
) {
3906 callee
= ctx
->lmethod
;
3909 static int tramp_index
;
3912 name
= g_strdup_printf ("[tramp_%d] %s", tramp_index
, mono_method_full_name (call
->method
, TRUE
));
3916 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3917 * Make all calls through a global. The address of the global will be saved in
3918 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3921 LLVMValueRef tramp_var
= (LLVMValueRef
)g_hash_table_lookup (ctx
->jit_callees
, call
->method
);
3924 mono_create_jit_trampoline (mono_domain_get (),
3925 call
->method
, error
);
3926 if (!is_ok (error
)) {
3927 set_failure (ctx
, mono_error_get_message (error
));
3928 mono_error_cleanup (error
);
3932 tramp_var
= LLVMAddGlobal (ctx
->lmodule
, LLVMPointerType (llvm_sig
, 0), name
);
3933 LLVMSetInitializer (tramp_var
, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64
)(size_t)target
, FALSE
), LLVMPointerType (llvm_sig
, 0)));
3934 LLVMSetLinkage (tramp_var
, LLVMExternalLinkage
);
3935 g_hash_table_insert (ctx
->jit_callees
, call
->method
, tramp_var
);
3937 callee
= LLVMBuildLoad (builder
, tramp_var
, "");
3941 if (!cfg
->llvm_only
&& call
->method
&& strstr (m_class_get_name (call
->method
->klass
), "AsyncVoidMethodBuilder")) {
3942 /* LLVM miscompiles async methods */
3943 set_failure (ctx
, "#13734");
3948 const MonoJitICallId jit_icall_id
= call
->jit_icall_id
;
3951 if (cfg
->compile_aot
) {
3952 callee
= get_callee (ctx
, llvm_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (jit_icall_id
));
3954 set_failure (ctx
, "can't encode patch");
3958 callee
= get_jit_callee (ctx
, "", llvm_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (jit_icall_id
));
3961 if (cfg
->compile_aot
) {
3963 if (cfg
->abs_patches
) {
3964 MonoJumpInfo
*abs_ji
= (MonoJumpInfo
*)g_hash_table_lookup (cfg
->abs_patches
, call
->fptr
);
3966 callee
= get_callee (ctx
, llvm_sig
, abs_ji
->type
, abs_ji
->data
.target
);
3968 set_failure (ctx
, "can't encode patch");
3974 set_failure (ctx
, "aot");
3978 if (cfg
->abs_patches
) {
3979 MonoJumpInfo
*abs_ji
= (MonoJumpInfo
*)g_hash_table_lookup (cfg
->abs_patches
, call
->fptr
);
3983 target
= mono_resolve_patch_target (cfg
->method
, cfg
->domain
, NULL
, abs_ji
, FALSE
, error
);
3984 mono_error_assert_ok (error
);
3985 callee
= get_jit_callee (ctx
, "", llvm_sig
, abs_ji
->type
, abs_ji
->data
.target
);
3987 g_assert_not_reached ();
3990 g_assert_not_reached ();
3997 int size
= TARGET_SIZEOF_VOID_P
;
4000 g_assert (ins
->inst_offset
% size
== 0);
4001 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
/ size
, FALSE
);
4003 callee
= convert (ctx
, LLVMBuildLoad (builder
, LLVMBuildGEP (builder
, convert (ctx
, values
[ins
->inst_basereg
], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index
, 1, ""), ""), LLVMPointerType (llvm_sig
, 0));
4005 callee
= convert (ctx
, values
[ins
->sreg1
], LLVMPointerType (llvm_sig
, 0));
4007 if (ins
->flags
& MONO_INST_HAS_METHOD
) {
4012 * Collect and convert arguments
4014 nargs
= (sig
->param_count
* 16) + sig
->hasthis
+ vretaddr
+ call
->rgctx_reg
+ call
->imt_arg_reg
+ call
->cinfo
->dummy_arg
+ 1;
4015 len
= sizeof (LLVMValueRef
) * nargs
;
4016 args
= g_newa (LLVMValueRef
, nargs
);
4017 memset (args
, 0, len
);
4018 l
= call
->out_ireg_args
;
4020 if (call
->rgctx_arg_reg
) {
4021 g_assert (values
[call
->rgctx_arg_reg
]);
4022 g_assert (cinfo
->rgctx_arg_pindex
< nargs
);
4024 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
4025 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
4026 * it using a volatile load.
4029 if (!ctx
->imt_rgctx_loc
)
4030 ctx
->imt_rgctx_loc
= build_alloca_llvm_type (ctx
, ctx
->module
->ptr_type
, TARGET_SIZEOF_VOID_P
);
4031 LLVMBuildStore (builder
, convert (ctx
, ctx
->values
[call
->rgctx_arg_reg
], ctx
->module
->ptr_type
), ctx
->imt_rgctx_loc
);
4032 args
[cinfo
->rgctx_arg_pindex
] = mono_llvm_build_load (builder
, ctx
->imt_rgctx_loc
, "", TRUE
);
4034 args
[cinfo
->rgctx_arg_pindex
] = convert (ctx
, values
[call
->rgctx_arg_reg
], ctx
->module
->ptr_type
);
4037 if (call
->imt_arg_reg
) {
4038 g_assert (!ctx
->llvm_only
);
4039 g_assert (values
[call
->imt_arg_reg
]);
4040 g_assert (cinfo
->imt_arg_pindex
< nargs
);
4042 if (!ctx
->imt_rgctx_loc
)
4043 ctx
->imt_rgctx_loc
= build_alloca_llvm_type (ctx
, ctx
->module
->ptr_type
, TARGET_SIZEOF_VOID_P
);
4044 LLVMBuildStore (builder
, convert (ctx
, ctx
->values
[call
->imt_arg_reg
], ctx
->module
->ptr_type
), ctx
->imt_rgctx_loc
);
4045 args
[cinfo
->imt_arg_pindex
] = mono_llvm_build_load (builder
, ctx
->imt_rgctx_loc
, "", TRUE
);
4047 args
[cinfo
->imt_arg_pindex
] = convert (ctx
, values
[call
->imt_arg_reg
], ctx
->module
->ptr_type
);
4050 switch (cinfo
->ret
.storage
) {
4051 case LLVMArgGsharedvtVariable
: {
4052 MonoInst
*var
= get_vreg_to_inst (cfg
, call
->inst
.dreg
);
4054 if (var
&& var
->opcode
== OP_GSHAREDVT_LOCAL
) {
4055 args
[cinfo
->vret_arg_pindex
] = convert (ctx
, emit_gsharedvt_ldaddr (ctx
, var
->dreg
), IntPtrType ());
4057 g_assert (addresses
[call
->inst
.dreg
]);
4058 args
[cinfo
->vret_arg_pindex
] = convert (ctx
, addresses
[call
->inst
.dreg
], IntPtrType ());
4064 if (!addresses
[call
->inst
.dreg
])
4065 addresses
[call
->inst
.dreg
] = build_alloca (ctx
, sig
->ret
);
4066 g_assert (cinfo
->vret_arg_pindex
< nargs
);
4067 if (cinfo
->ret
.storage
== LLVMArgVtypeByRef
)
4068 args
[cinfo
->vret_arg_pindex
] = addresses
[call
->inst
.dreg
];
4070 args
[cinfo
->vret_arg_pindex
] = LLVMBuildPtrToInt (builder
, addresses
[call
->inst
.dreg
], IntPtrType (), "");
4076 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
4077 * use the real callee for argument type conversion.
4079 LLVMTypeRef callee_type
= LLVMGetElementType (LLVMTypeOf (callee
));
4080 LLVMTypeRef
*param_types
= (LLVMTypeRef
*)g_alloca (sizeof (LLVMTypeRef
) * LLVMCountParamTypes (callee_type
));
4081 LLVMGetParamTypes (callee_type
, param_types
);
4083 for (i
= 0; i
< sig
->param_count
+ sig
->hasthis
; ++i
) {
4086 LLVMArgInfo
*ainfo
= &call
->cinfo
->args
[i
];
4088 pindex
= ainfo
->pindex
;
4090 regpair
= (guint32
)(gssize
)(l
->data
);
4091 reg
= regpair
& 0xffffff;
4092 args
[pindex
] = values
[reg
];
4093 switch (ainfo
->storage
) {
4094 case LLVMArgVtypeInReg
:
4095 case LLVMArgAsFpArgs
: {
4099 for (j
= 0; j
< ainfo
->ndummy_fpargs
; ++j
)
4100 args
[pindex
+ j
] = LLVMConstNull (LLVMDoubleType ());
4101 pindex
+= ainfo
->ndummy_fpargs
;
4103 g_assert (addresses
[reg
]);
4104 emit_vtype_to_args (ctx
, builder
, ainfo
->type
, addresses
[reg
], ainfo
, args
+ pindex
, &nargs
);
4108 // FIXME: Get rid of the VMOVE
4111 case LLVMArgVtypeByVal
:
4112 g_assert (addresses
[reg
]);
4113 args
[pindex
] = addresses
[reg
];
4115 case LLVMArgVtypeAddr
:
4116 case LLVMArgVtypeByRef
: {
4117 g_assert (addresses
[reg
]);
4118 args
[pindex
] = convert (ctx
, addresses
[reg
], LLVMPointerType (type_to_llvm_arg_type (ctx
, ainfo
->type
), 0));
4121 case LLVMArgAsIArgs
:
4122 g_assert (addresses
[reg
]);
4123 if (ainfo
->esize
== 8)
4124 args
[pindex
] = LLVMBuildLoad (ctx
->builder
, convert (ctx
, addresses
[reg
], LLVMPointerType (LLVMArrayType (LLVMInt64Type (), ainfo
->nslots
), 0)), "");
4126 args
[pindex
] = LLVMBuildLoad (ctx
->builder
, convert (ctx
, addresses
[reg
], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo
->nslots
), 0)), "");
4128 case LLVMArgVtypeAsScalar
:
4129 g_assert_not_reached ();
4131 case LLVMArgGsharedvtFixed
:
4132 case LLVMArgGsharedvtFixedVtype
:
4133 g_assert (addresses
[reg
]);
4134 args
[pindex
] = convert (ctx
, addresses
[reg
], LLVMPointerType (type_to_llvm_arg_type (ctx
, ainfo
->type
), 0));
4136 case LLVMArgGsharedvtVariable
:
4137 g_assert (addresses
[reg
]);
4138 args
[pindex
] = convert (ctx
, addresses
[reg
], LLVMPointerType (IntPtrType (), 0));
4141 g_assert (args
[pindex
]);
4142 if (i
== 0 && sig
->hasthis
)
4143 args
[pindex
] = convert (ctx
, args
[pindex
], param_types
[pindex
]);
4145 args
[pindex
] = convert (ctx
, args
[pindex
], type_to_llvm_arg_type (ctx
, ainfo
->type
));
4148 g_assert (pindex
<= nargs
);
4153 if (call
->cinfo
->dummy_arg
) {
4154 g_assert (call
->cinfo
->dummy_arg_pindex
< nargs
);
4155 args
[call
->cinfo
->dummy_arg_pindex
] = LLVMConstNull (ctx
->module
->ptr_type
);
4158 // FIXME: Align call sites
4163 lcall
= emit_call (ctx
, bb
, &builder
, callee
, args
, LLVMCountParamTypes (llvm_sig
));
4165 mono_llvm_nonnull_state_update (ctx
, lcall
, call
->method
, args
, LLVMCountParamTypes (llvm_sig
));
4167 // If we just allocated an object, it's not null.
4168 if (call
->method
&& call
->method
->wrapper_type
== MONO_WRAPPER_ALLOC
) {
4169 mono_llvm_set_call_nonnull_ret (lcall
);
4172 if (ins
->opcode
!= OP_TAILCALL
&& ins
->opcode
!= OP_TAILCALL_MEMBASE
&& LLVMGetInstructionOpcode (lcall
) == LLVMCall
)
4173 mono_llvm_set_call_notailcall (lcall
);
4175 // Add original method name we are currently emitting as a custom string metadata (the only way to leave comments in LLVM IR)
4176 if (mono_debug_enabled () && call
&& call
->method
)
4177 mono_llvm_add_string_metadata (lcall
, "managed_name", mono_method_full_name (call
->method
, TRUE
));
4179 // As per the LLVM docs, a function has a noalias return value if and only if
4180 // it is an allocation function. This is an allocation function.
4181 if (call
->method
&& call
->method
->wrapper_type
== MONO_WRAPPER_ALLOC
) {
4182 mono_llvm_set_call_noalias_ret (lcall
);
4183 // All objects are expected to be 8-byte aligned (SGEN_ALLOC_ALIGN)
4184 mono_llvm_set_alignment_ret (lcall
, 8);
4188 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
4190 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
4191 g_assert (MONO_ARCH_IMT_REG
== MONO_ARCH_RGCTX_REG
);
4193 /* The two can't be used together, so use only one LLVM calling conv to pass them */
4194 g_assert (!(call
->rgctx_arg_reg
&& call
->imt_arg_reg
));
4195 if (!sig
->pinvoke
&& !cfg
->llvm_only
)
4196 LLVMSetInstructionCallConv (lcall
, LLVMMono1CallConv
);
4198 if (cinfo
->ret
.storage
== LLVMArgVtypeByRef
)
4199 mono_llvm_add_instr_attr (lcall
, 1 + cinfo
->vret_arg_pindex
, LLVM_ATTR_STRUCT_RET
);
4200 if (!ctx
->llvm_only
&& call
->rgctx_arg_reg
)
4201 mono_llvm_add_instr_attr (lcall
, 1 + cinfo
->rgctx_arg_pindex
, LLVM_ATTR_IN_REG
);
4202 if (call
->imt_arg_reg
)
4203 mono_llvm_add_instr_attr (lcall
, 1 + cinfo
->imt_arg_pindex
, LLVM_ATTR_IN_REG
);
4205 /* Add byval attributes if needed */
4206 for (i
= 0; i
< sig
->param_count
; ++i
) {
4207 LLVMArgInfo
*ainfo
= &call
->cinfo
->args
[i
+ sig
->hasthis
];
4209 if (ainfo
&& ainfo
->storage
== LLVMArgVtypeByVal
)
4210 mono_llvm_add_instr_attr (lcall
, 1 + ainfo
->pindex
, LLVM_ATTR_BY_VAL
);
4214 * Convert the result
4216 switch (cinfo
->ret
.storage
) {
4217 case LLVMArgVtypeInReg
: {
4218 LLVMValueRef regs
[2];
4220 if (LLVMTypeOf (lcall
) == LLVMVoidType ())
4224 if (!addresses
[ins
->dreg
])
4225 addresses
[ins
->dreg
] = build_alloca (ctx
, sig
->ret
);
4227 regs
[0] = LLVMBuildExtractValue (builder
, lcall
, 0, "");
4228 if (cinfo
->ret
.pair_storage
[1] != LLVMArgNone
)
4229 regs
[1] = LLVMBuildExtractValue (builder
, lcall
, 1, "");
4230 emit_args_to_vtype (ctx
, builder
, sig
->ret
, addresses
[ins
->dreg
], &cinfo
->ret
, regs
);
4233 case LLVMArgVtypeByVal
:
4234 if (!addresses
[call
->inst
.dreg
])
4235 addresses
[call
->inst
.dreg
] = build_alloca (ctx
, sig
->ret
);
4236 LLVMBuildStore (builder
, lcall
, addresses
[call
->inst
.dreg
]);
4238 case LLVMArgAsIArgs
:
4239 case LLVMArgFpStruct
:
4240 if (!addresses
[call
->inst
.dreg
])
4241 addresses
[call
->inst
.dreg
] = build_alloca (ctx
, sig
->ret
);
4242 LLVMBuildStore (builder
, lcall
, convert_full (ctx
, addresses
[call
->inst
.dreg
], LLVMPointerType (LLVMTypeOf (lcall
), 0), FALSE
));
4244 case LLVMArgVtypeAsScalar
:
4245 if (!addresses
[call
->inst
.dreg
])
4246 addresses
[call
->inst
.dreg
] = build_alloca (ctx
, sig
->ret
);
4247 LLVMBuildStore (builder
, lcall
, convert_full (ctx
, addresses
[call
->inst
.dreg
], LLVMPointerType (LLVMTypeOf (lcall
), 0), FALSE
));
4249 case LLVMArgVtypeRetAddr
:
4250 case LLVMArgVtypeByRef
:
4251 if (MONO_CLASS_IS_SIMD (ctx
->cfg
, mono_class_from_mono_type_internal (sig
->ret
))) {
4252 /* Some opcodes like STOREX_MEMBASE access these by value */
4253 g_assert (addresses
[call
->inst
.dreg
]);
4254 values
[ins
->dreg
] = LLVMBuildLoad (builder
, convert_full (ctx
, addresses
[call
->inst
.dreg
], LLVMPointerType (type_to_llvm_type (ctx
, sig
->ret
), 0), FALSE
), "");
4257 case LLVMArgGsharedvtVariable
:
4259 case LLVMArgGsharedvtFixed
:
4260 case LLVMArgGsharedvtFixedVtype
:
4261 values
[ins
->dreg
] = LLVMBuildLoad (builder
, convert_full (ctx
, addresses
[call
->inst
.dreg
], LLVMPointerType (type_to_llvm_type (ctx
, sig
->ret
), 0), FALSE
), "");
4264 if (sig
->ret
->type
!= MONO_TYPE_VOID
)
4265 /* If the method returns an unsigned value, need to zext it */
4266 values
[ins
->dreg
] = convert_full (ctx
, lcall
, llvm_type_to_stack_type (cfg
, type_to_llvm_type (ctx
, sig
->ret
)), type_is_unsigned (ctx
, sig
->ret
));
4270 *builder_ref
= ctx
->builder
;
4274 emit_llvmonly_throw (EmitContext
*ctx
, MonoBasicBlock
*bb
, gboolean rethrow
, LLVMValueRef exc
)
4276 MonoJitICallId icall_id
= rethrow
? MONO_JIT_ICALL_mono_llvm_rethrow_exception
: MONO_JIT_ICALL_mono_llvm_throw_exception
;
4277 LLVMValueRef callee
= rethrow
? ctx
->module
->rethrow
: ctx
->module
->throw_icall
;
4279 LLVMTypeRef exc_type
= type_to_llvm_type (ctx
, m_class_get_byval_arg (mono_get_exception_class ()));
4282 LLVMTypeRef fun_sig
= LLVMFunctionType1 (LLVMVoidType (), exc_type
, FALSE
);
4284 g_assert (ctx
->cfg
->compile_aot
);
4285 callee
= get_callee (ctx
, fun_sig
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, GUINT_TO_POINTER (icall_id
));
4288 LLVMValueRef args
[2];
4290 args
[0] = convert (ctx
, exc
, exc_type
);
4291 emit_call (ctx
, bb
, &ctx
->builder
, callee
, args
, 1);
4293 LLVMBuildUnreachable (ctx
->builder
);
4295 ctx
->builder
= create_builder (ctx
);
4299 emit_throw (EmitContext
*ctx
, MonoBasicBlock
*bb
, gboolean rethrow
, LLVMValueRef exc
)
4301 MonoMethodSignature
*throw_sig
;
4303 LLVMValueRef
* const pcallee
= rethrow
? &ctx
->module
->rethrow
: &ctx
->module
->throw_icall
;
4304 LLVMValueRef callee
= *pcallee
;
4305 char const * const icall_name
= rethrow
? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4309 MonoJitICallId icall_id
= rethrow
? MONO_JIT_ICALL_mono_arch_rethrow_exception
: MONO_JIT_ICALL_mono_arch_throw_exception
;
4312 throw_sig
= mono_metadata_signature_alloc (mono_get_corlib (), 1);
4313 throw_sig
->ret
= m_class_get_byval_arg (mono_get_void_class ());
4314 throw_sig
->params
[0] = m_class_get_byval_arg (mono_get_object_class ());
4315 if (ctx
->cfg
->compile_aot
) {
4316 callee
= get_callee (ctx
, sig_to_llvm_sig (ctx
, throw_sig
), MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
4320 * LLVM doesn't push the exception argument, so we need a different
4323 icall_id
= rethrow
? MONO_JIT_ICALL_mono_llvm_rethrow_exception_trampoline
: MONO_JIT_ICALL_mono_llvm_throw_exception_trampoline
;
4325 callee
= get_jit_callee (ctx
, icall_name
, sig_to_llvm_sig (ctx
, throw_sig
), MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
4328 mono_memory_barrier ();
4331 arg
= convert (ctx
, exc
, type_to_llvm_type (ctx
, m_class_get_byval_arg (mono_get_object_class ())));
4332 emit_call (ctx
, bb
, &ctx
->builder
, callee
, &arg
, 1);
4336 emit_resume_eh (EmitContext
*ctx
, MonoBasicBlock
*bb
)
4338 const MonoJitICallId icall_id
= MONO_JIT_ICALL_mono_llvm_resume_exception
;
4339 LLVMValueRef callee
;
4341 LLVMTypeRef fun_sig
= LLVMFunctionType0 (LLVMVoidType (), FALSE
);
4343 g_assert (ctx
->cfg
->compile_aot
);
4344 callee
= get_callee (ctx
, fun_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
4346 emit_call (ctx
, bb
, &ctx
->builder
, callee
, NULL
, 0);
4348 LLVMBuildUnreachable (ctx
->builder
);
4350 ctx
->builder
= create_builder (ctx
);
4354 mono_llvm_emit_clear_exception_call (EmitContext
*ctx
, LLVMBuilderRef builder
)
4356 const char *icall_name
= "mono_llvm_clear_exception";
4357 const MonoJitICallId icall_id
= MONO_JIT_ICALL_mono_llvm_clear_exception
;
4359 LLVMTypeRef call_sig
= LLVMFunctionType (LLVMVoidType (), NULL
, 0, FALSE
);
4360 LLVMValueRef callee
= NULL
;
4363 if (ctx
->cfg
->compile_aot
) {
4364 callee
= get_callee (ctx
, call_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
4366 // FIXME: This is broken.
4367 callee
= LLVMAddFunction (ctx
->lmodule
, icall_name
, call_sig
);
4371 g_assert (builder
&& callee
);
4373 return LLVMBuildCall (builder
, callee
, NULL
, 0, "");
4377 mono_llvm_emit_load_exception_call (EmitContext
*ctx
, LLVMBuilderRef builder
)
4379 const char *icall_name
= "mono_llvm_load_exception";
4380 const MonoJitICallId icall_id
= MONO_JIT_ICALL_mono_llvm_load_exception
;
4382 LLVMTypeRef call_sig
= LLVMFunctionType (ObjRefType (), NULL
, 0, FALSE
);
4383 LLVMValueRef callee
= NULL
;
4386 if (ctx
->cfg
->compile_aot
) {
4387 callee
= get_callee (ctx
, call_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
4389 // FIXME: This is broken.
4390 callee
= LLVMAddFunction (ctx
->lmodule
, icall_name
, call_sig
);
4394 g_assert (builder
&& callee
);
4396 return LLVMBuildCall (builder
, callee
, NULL
, 0, icall_name
);
4401 mono_llvm_emit_match_exception_call (EmitContext
*ctx
, LLVMBuilderRef builder
, gint32 region_start
, gint32 region_end
)
4403 const char *icall_name
= "mono_llvm_match_exception";
4404 const MonoJitICallId icall_id
= MONO_JIT_ICALL_mono_llvm_match_exception
;
4406 ctx
->builder
= builder
;
4408 LLVMValueRef args
[5];
4409 const int num_args
= G_N_ELEMENTS (args
);
4411 args
[0] = convert (ctx
, get_aotconst (ctx
, MONO_PATCH_INFO_AOT_JIT_INFO
, GINT_TO_POINTER (ctx
->cfg
->method_index
), LLVMPointerType (IntPtrType (), 0)), IntPtrType ());
4412 args
[1] = LLVMConstInt (LLVMInt32Type (), region_start
, 0);
4413 args
[2] = LLVMConstInt (LLVMInt32Type (), region_end
, 0);
4414 if (ctx
->cfg
->rgctx_var
) {
4415 if (ctx
->cfg
->llvm_only
) {
4416 args
[3] = convert (ctx
, ctx
->rgctx_arg
, IntPtrType ());
4418 LLVMValueRef rgctx_alloc
= ctx
->addresses
[ctx
->cfg
->rgctx_var
->dreg
];
4419 g_assert (rgctx_alloc
);
4420 args
[3] = LLVMBuildLoad (builder
, convert (ctx
, rgctx_alloc
, LLVMPointerType (IntPtrType (), 0)), "");
4423 args
[3] = LLVMConstInt (IntPtrType (), 0, 0);
4426 args
[4] = convert (ctx
, ctx
->this_arg
, IntPtrType ());
4428 args
[4] = LLVMConstInt (IntPtrType (), 0, 0);
4430 LLVMTypeRef match_sig
= LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE
);
4431 LLVMValueRef callee
;
4432 g_assert (ctx
->cfg
->compile_aot
);
4433 ctx
->builder
= builder
;
4434 // get_callee expects ctx->builder to be the emitting builder
4435 callee
= get_callee (ctx
, match_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (icall_id
));
4437 g_assert (builder
&& callee
);
4439 g_assert (ctx
->ex_var
);
4441 return LLVMBuildCall (builder
, callee
, args
, num_args
, icall_name
);
4444 // FIXME: This won't work because the code-finding makes this
4446 /*#define MONO_PERSONALITY_DEBUG*/
4448 #ifdef MONO_PERSONALITY_DEBUG
4449 static const gboolean use_mono_personality_debug
= TRUE
;
4450 static const char *default_personality_name
= "mono_debug_personality";
4452 static const gboolean use_mono_personality_debug
= FALSE
;
4453 static const char *default_personality_name
= "__gxx_personality_v0";
4457 default_cpp_lpad_exc_signature (void)
4459 static gboolean inited
= FALSE
;
4460 static LLVMTypeRef sig
;
4463 LLVMTypeRef signature
[2];
4464 signature
[0] = LLVMPointerType (LLVMInt8Type (), 0);
4465 signature
[1] = LLVMInt32Type ();
4466 sig
= LLVMStructType (signature
, 2, FALSE
);
4474 get_mono_personality (EmitContext
*ctx
)
4476 LLVMValueRef personality
= NULL
;
4477 LLVMTypeRef personality_type
= LLVMFunctionType (LLVMInt32Type (), NULL
, 0, TRUE
);
4479 g_assert (ctx
->cfg
->compile_aot
);
4480 if (!use_mono_personality_debug
) {
4481 personality
= LLVMGetNamedFunction (ctx
->lmodule
, default_personality_name
);
4483 personality
= get_callee (ctx
, personality_type
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_debug_personality
));
4486 g_assert (personality
);
4490 static LLVMBasicBlockRef
4491 emit_landing_pad (EmitContext
*ctx
, int group_index
, int group_size
)
4493 MonoCompile
*cfg
= ctx
->cfg
;
4494 LLVMBuilderRef old_builder
= ctx
->builder
;
4495 MonoExceptionClause
*group_start
= cfg
->header
->clauses
+ group_index
;
4497 LLVMBuilderRef lpadBuilder
= create_builder (ctx
);
4498 ctx
->builder
= lpadBuilder
;
4500 MonoBasicBlock
*handler_bb
= cfg
->cil_offset_to_bb
[CLAUSE_START (group_start
)];
4501 g_assert (handler_bb
);
4503 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4504 LLVMValueRef personality
= get_mono_personality (ctx
);
4505 g_assert (personality
);
4507 char *bb_name
= g_strdup_printf ("LPAD%d_BB", group_index
);
4508 LLVMBasicBlockRef lpad_bb
= gen_bb (ctx
, bb_name
);
4510 LLVMPositionBuilderAtEnd (lpadBuilder
, lpad_bb
);
4511 LLVMValueRef landing_pad
= LLVMBuildLandingPad (lpadBuilder
, default_cpp_lpad_exc_signature (), personality
, 0, "");
4512 g_assert (landing_pad
);
4514 LLVMValueRef cast
= LLVMBuildBitCast (lpadBuilder
, ctx
->module
->sentinel_exception
, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
4515 LLVMAddClause (landing_pad
, cast
);
4517 LLVMBasicBlockRef resume_bb
= gen_bb (ctx
, "RESUME_BB");
4518 LLVMBuilderRef resume_builder
= create_builder (ctx
);
4519 ctx
->builder
= resume_builder
;
4520 LLVMPositionBuilderAtEnd (resume_builder
, resume_bb
);
4522 emit_resume_eh (ctx
, handler_bb
);
4525 ctx
->builder
= lpadBuilder
;
4526 LLVMPositionBuilderAtEnd (lpadBuilder
, lpad_bb
);
4528 gboolean finally_only
= TRUE
;
4530 MonoExceptionClause
*group_cursor
= group_start
;
4532 for (int i
= 0; i
< group_size
; i
++) {
4533 if (!(group_cursor
->flags
& MONO_EXCEPTION_CLAUSE_FINALLY
|| group_cursor
->flags
& MONO_EXCEPTION_CLAUSE_FAULT
))
4534 finally_only
= FALSE
;
4540 // Handle landing pad inlining
4542 if (!finally_only
) {
4543 // So at each level of the exception stack we will match the exception again.
4544 // During that match, we need to compare against the handler types for the current
4545 // protected region. We send the try start and end so that we can only check against
4546 // handlers for this lexical protected region.
4547 LLVMValueRef match
= mono_llvm_emit_match_exception_call (ctx
, lpadBuilder
, group_start
->try_offset
, group_start
->try_offset
+ group_start
->try_len
);
4549 // if returns -1, resume
4550 LLVMValueRef switch_ins
= LLVMBuildSwitch (lpadBuilder
, match
, resume_bb
, group_size
);
4552 // else move to that target bb
4553 for (int i
= 0; i
< group_size
; i
++) {
4554 MonoExceptionClause
*clause
= group_start
+ i
;
4555 int clause_index
= clause
- cfg
->header
->clauses
;
4556 MonoBasicBlock
*handler_bb
= (MonoBasicBlock
*)g_hash_table_lookup (ctx
->clause_to_handler
, GINT_TO_POINTER (clause_index
));
4557 g_assert (handler_bb
);
4558 g_assert (ctx
->bblocks
[handler_bb
->block_num
].call_handler_target_bb
);
4559 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), clause_index
, FALSE
), ctx
->bblocks
[handler_bb
->block_num
].call_handler_target_bb
);
4562 int clause_index
= group_start
- cfg
->header
->clauses
;
4563 MonoBasicBlock
*finally_bb
= (MonoBasicBlock
*)g_hash_table_lookup (ctx
->clause_to_handler
, GINT_TO_POINTER (clause_index
));
4564 g_assert (finally_bb
);
4566 LLVMBuildBr (ctx
->builder
, ctx
->bblocks
[finally_bb
->block_num
].call_handler_target_bb
);
4569 ctx
->builder
= old_builder
;
4575 create_const_vector_i32 (const int *mask
, int count
)
4577 LLVMValueRef
*llvm_mask
= g_new (LLVMValueRef
, count
);
4578 for (int i
= 0; i
< count
; i
++)
4579 llvm_mask
[i
] = LLVMConstInt (LLVMInt32Type (), mask
[i
], FALSE
);
4580 LLVMValueRef vec
= LLVMConstVector (llvm_mask
, count
);
4586 create_const_vector_4_i32 (int v0
, int v1
, int v2
, int v3
)
4588 LLVMValueRef mask
[4];
4589 mask
[0] = LLVMConstInt (LLVMInt32Type (), v0
, FALSE
);
4590 mask
[1] = LLVMConstInt (LLVMInt32Type (), v1
, FALSE
);
4591 mask
[2] = LLVMConstInt (LLVMInt32Type (), v2
, FALSE
);
4592 mask
[3] = LLVMConstInt (LLVMInt32Type (), v3
, FALSE
);
4593 return LLVMConstVector (mask
, 4);
4597 create_const_vector_2_i32 (int v0
, int v1
)
4599 LLVMValueRef mask
[2];
4600 mask
[0] = LLVMConstInt (LLVMInt32Type (), v0
, FALSE
);
4601 mask
[1] = LLVMConstInt (LLVMInt32Type (), v1
, FALSE
);
4602 return LLVMConstVector (mask
, 2);
4606 emit_llvmonly_handler_start (EmitContext
*ctx
, MonoBasicBlock
*bb
, LLVMBasicBlockRef cbb
)
4608 int clause_index
= MONO_REGION_CLAUSE_INDEX (bb
->region
);
4609 MonoExceptionClause
*clause
= &ctx
->cfg
->header
->clauses
[clause_index
];
4611 // Make exception available to catch blocks
4612 if (!(clause
->flags
& MONO_EXCEPTION_CLAUSE_FINALLY
|| clause
->flags
& MONO_EXCEPTION_CLAUSE_FAULT
)) {
4613 LLVMValueRef mono_exc
= mono_llvm_emit_load_exception_call (ctx
, ctx
->builder
);
4615 g_assert (ctx
->ex_var
);
4616 LLVMBuildStore (ctx
->builder
, LLVMBuildBitCast (ctx
->builder
, mono_exc
, ObjRefType (), ""), ctx
->ex_var
);
4618 if (bb
->in_scount
== 1) {
4619 MonoInst
*exvar
= bb
->in_stack
[0];
4620 g_assert (!ctx
->values
[exvar
->dreg
]);
4621 g_assert (ctx
->ex_var
);
4622 ctx
->values
[exvar
->dreg
] = LLVMBuildLoad (ctx
->builder
, ctx
->ex_var
, "save_exception");
4623 emit_volatile_store (ctx
, exvar
->dreg
);
4626 mono_llvm_emit_clear_exception_call (ctx
, ctx
->builder
);
4629 LLVMBuilderRef handler_builder
= create_builder (ctx
);
4630 LLVMBasicBlockRef target_bb
= ctx
->bblocks
[bb
->block_num
].call_handler_target_bb
;
4631 LLVMPositionBuilderAtEnd (handler_builder
, target_bb
);
4633 // Make the handler code end with a jump to cbb
4634 LLVMBuildBr (handler_builder
, cbb
);
4638 emit_handler_start (EmitContext
*ctx
, MonoBasicBlock
*bb
, LLVMBuilderRef builder
)
4640 MonoCompile
*cfg
= ctx
->cfg
;
4641 LLVMValueRef
*values
= ctx
->values
;
4642 LLVMModuleRef lmodule
= ctx
->lmodule
;
4643 BBInfo
*bblocks
= ctx
->bblocks
;
4645 LLVMValueRef personality
;
4646 LLVMValueRef landing_pad
;
4647 LLVMBasicBlockRef target_bb
;
4649 static int ti_generator
;
4651 LLVMValueRef type_info
;
4655 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4657 if (cfg
->compile_aot
) {
4658 /* Use a dummy personality function */
4659 personality
= LLVMGetNamedFunction (lmodule
, "mono_personality");
4660 g_assert (personality
);
4662 /* Can't cache this as each method is in its own llvm module */
4663 LLVMTypeRef personality_type
= LLVMFunctionType (LLVMInt32Type (), NULL
, 0, TRUE
);
4664 personality
= LLVMAddFunction (ctx
->lmodule
, "mono_personality", personality_type
);
4665 mono_llvm_add_func_attr (personality
, LLVM_ATTR_NO_UNWIND
);
4666 LLVMBasicBlockRef entry_bb
= LLVMAppendBasicBlock (personality
, "ENTRY");
4667 LLVMBuilderRef builder2
= LLVMCreateBuilder ();
4668 LLVMPositionBuilderAtEnd (builder2
, entry_bb
);
4669 LLVMBuildRet (builder2
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
));
4670 LLVMDisposeBuilder (builder2
);
4673 i8ptr
= LLVMPointerType (LLVMInt8Type (), 0);
4675 clause_index
= (mono_get_block_region_notry (cfg
, bb
->region
) >> 8) - 1;
4678 * Create the type info
4680 sprintf (ti_name
, "type_info_%d", ti_generator
);
4683 if (cfg
->compile_aot
) {
4684 /* decode_eh_frame () in aot-runtime.c will decode this */
4685 type_info
= LLVMAddGlobal (lmodule
, LLVMInt32Type (), ti_name
);
4686 LLVMSetInitializer (type_info
, LLVMConstInt (LLVMInt32Type (), clause_index
, FALSE
));
4689 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4691 LLVMSetLinkage (type_info
, LLVMInternalLinkage
);
4693 type_info
= LLVMAddGlobal (lmodule
, LLVMInt32Type (), ti_name
);
4694 LLVMSetInitializer (type_info
, LLVMConstInt (LLVMInt32Type (), clause_index
, FALSE
));
4698 LLVMTypeRef members
[2], ret_type
;
4700 members
[0] = i8ptr
;
4701 members
[1] = LLVMInt32Type ();
4702 ret_type
= LLVMStructType (members
, 2, FALSE
);
4704 landing_pad
= LLVMBuildLandingPad (builder
, ret_type
, personality
, 1, "");
4705 LLVMAddClause (landing_pad
, type_info
);
4707 /* Store the exception into the exvar */
4709 LLVMBuildStore (builder
, convert (ctx
, LLVMBuildExtractValue (builder
, landing_pad
, 0, "ex_obj"), ObjRefType ()), ctx
->ex_var
);
4713 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4714 * code expects control to be transferred to this landing pad even in the
4715 * presence of nested clauses. The landing pad needs to branch to the landing
4716 * pads belonging to nested clauses based on the selector value returned by
4717 * the landing pad instruction, which is passed to the landing pad in a
4718 * register by the EH code.
4720 target_bb
= bblocks
[bb
->block_num
].call_handler_target_bb
;
4721 g_assert (target_bb
);
4724 * Branch to the correct landing pad
4726 LLVMValueRef ex_selector
= LLVMBuildExtractValue (builder
, landing_pad
, 1, "ex_selector");
4727 LLVMValueRef switch_ins
= LLVMBuildSwitch (builder
, ex_selector
, target_bb
, 0);
4729 for (l
= ctx
->nested_in
[clause_index
]; l
; l
= l
->next
) {
4730 int nesting_clause_index
= GPOINTER_TO_INT (l
->data
);
4731 MonoBasicBlock
*handler_bb
;
4733 handler_bb
= (MonoBasicBlock
*)g_hash_table_lookup (ctx
->clause_to_handler
, GINT_TO_POINTER (nesting_clause_index
));
4734 g_assert (handler_bb
);
4736 g_assert (ctx
->bblocks
[handler_bb
->block_num
].call_handler_target_bb
);
4737 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), nesting_clause_index
, FALSE
), ctx
->bblocks
[handler_bb
->block_num
].call_handler_target_bb
);
4740 /* Start a new bblock which CALL_HANDLER can branch to */
4741 ctx
->builder
= builder
= create_builder (ctx
);
4742 LLVMPositionBuilderAtEnd (ctx
->builder
, target_bb
);
4744 ctx
->bblocks
[bb
->block_num
].end_bblock
= target_bb
;
4746 /* Store the exception into the IL level exvar */
4747 if (bb
->in_scount
== 1) {
4748 g_assert (bb
->in_scount
== 1);
4749 exvar
= bb
->in_stack
[0];
4751 // FIXME: This is shared with filter clauses ?
4752 g_assert (!values
[exvar
->dreg
]);
4754 g_assert (ctx
->ex_var
);
4755 values
[exvar
->dreg
] = LLVMBuildLoad (builder
, ctx
->ex_var
, "");
4756 emit_volatile_store (ctx
, exvar
->dreg
);
4759 /* Make normal branches to the start of the clause branch to the new bblock */
4760 bblocks
[bb
->block_num
].bblock
= target_bb
;
4763 //Wasm requires us to canonicalize NaNs.
4765 get_double_const (MonoCompile
*cfg
, double val
)
4768 if (mono_isnan (val
))
4769 *(gint64
*)&val
= 0x7FF8000000000000ll
;
4771 return LLVMConstReal (LLVMDoubleType (), val
);
4775 get_float_const (MonoCompile
*cfg
, float val
)
4778 if (mono_isnan (val
))
4779 *(int *)&val
= 0x7FC00000;
4782 return LLVMConstReal (LLVMFloatType (), val
);
4784 return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val
), LLVMDoubleType ());
4788 call_intrins (EmitContext
*ctx
, int id
, LLVMValueRef
*args
, const char *name
)
4790 LLVMValueRef intrins
= get_intrins (ctx
, id
);
4791 int nargs
= LLVMCountParamTypes (LLVMGetElementType (LLVMTypeOf (intrins
)));
4792 return LLVMBuildCall (ctx
->builder
, intrins
, args
, nargs
, name
);
4796 process_bb (EmitContext
*ctx
, MonoBasicBlock
*bb
)
4798 MonoCompile
*cfg
= ctx
->cfg
;
4799 MonoMethodSignature
*sig
= ctx
->sig
;
4800 LLVMValueRef method
= ctx
->lmethod
;
4801 LLVMValueRef
*values
= ctx
->values
;
4802 LLVMValueRef
*addresses
= ctx
->addresses
;
4803 LLVMCallInfo
*linfo
= ctx
->linfo
;
4804 BBInfo
*bblocks
= ctx
->bblocks
;
4806 LLVMBasicBlockRef cbb
;
4807 LLVMBuilderRef builder
, starting_builder
;
4808 gboolean has_terminator
;
4810 LLVMValueRef lhs
, rhs
;
4813 cbb
= get_end_bb (ctx
, bb
);
4815 builder
= create_builder (ctx
);
4816 ctx
->builder
= builder
;
4817 LLVMPositionBuilderAtEnd (builder
, cbb
);
4822 if (bb
->flags
& BB_EXCEPTION_HANDLER
) {
4823 if (!ctx
->llvm_only
&& !bblocks
[bb
->block_num
].invoke_target
) {
4824 set_failure (ctx
, "handler without invokes");
4829 emit_llvmonly_handler_start (ctx
, bb
, cbb
);
4831 emit_handler_start (ctx
, bb
, builder
);
4834 builder
= ctx
->builder
;
4837 /* Handle PHI nodes first */
4838 /* They should be grouped at the start of the bb */
4839 for (ins
= bb
->code
; ins
; ins
= ins
->next
) {
4840 emit_dbg_loc (ctx
, builder
, ins
->cil_code
);
4842 if (ins
->opcode
== OP_NOP
)
4844 if (!MONO_IS_PHI (ins
))
4848 gboolean empty
= TRUE
;
4850 /* Check that all input bblocks really branch to us */
4851 for (i
= 0; i
< bb
->in_count
; ++i
) {
4852 if (bb
->in_bb
[i
]->last_ins
&& bb
->in_bb
[i
]->last_ins
->opcode
== OP_NOT_REACHED
)
4853 ins
->inst_phi_args
[i
+ 1] = -1;
4859 /* LLVM doesn't like phi instructions with zero operands */
4860 ctx
->is_dead
[ins
->dreg
] = TRUE
;
4864 /* Created earlier, insert it now */
4865 LLVMInsertIntoBuilder (builder
, values
[ins
->dreg
]);
4867 for (i
= 0; i
< ins
->inst_phi_args
[0]; i
++) {
4868 int sreg1
= ins
->inst_phi_args
[i
+ 1];
4872 * Count the number of times the incoming bblock branches to us,
4873 * since llvm requires a separate entry for each.
4875 if (bb
->in_bb
[i
]->last_ins
&& bb
->in_bb
[i
]->last_ins
->opcode
== OP_SWITCH
) {
4876 MonoInst
*switch_ins
= bb
->in_bb
[i
]->last_ins
;
4879 for (j
= 0; j
< GPOINTER_TO_UINT (switch_ins
->klass
); ++j
) {
4880 if (switch_ins
->inst_many_bb
[j
] == bb
)
4887 /* Remember for later */
4888 for (j
= 0; j
< count
; ++j
) {
4889 PhiNode
*node
= (PhiNode
*)mono_mempool_alloc0 (ctx
->mempool
, sizeof (PhiNode
));
4892 node
->in_bb
= bb
->in_bb
[i
];
4894 bblocks
[bb
->in_bb
[i
]->block_num
].phi_nodes
= g_slist_prepend_mempool (ctx
->mempool
, bblocks
[bb
->in_bb
[i
]->block_num
].phi_nodes
, node
);
4898 // Add volatile stores for PHI nodes
4899 // These need to be emitted after the PHI nodes
4900 for (ins
= bb
->code
; ins
; ins
= ins
->next
) {
4901 const char *spec
= LLVM_INS_INFO (ins
->opcode
);
4903 if (ins
->opcode
== OP_NOP
)
4905 if (!MONO_IS_PHI (ins
))
4908 if (spec
[MONO_INST_DEST
] != 'v')
4909 emit_volatile_store (ctx
, ins
->dreg
);
4912 has_terminator
= FALSE
;
4913 starting_builder
= builder
;
4914 for (ins
= bb
->code
; ins
; ins
= ins
->next
) {
4915 const char *spec
= LLVM_INS_INFO (ins
->opcode
);
4917 char dname_buf
[128];
4919 emit_dbg_loc (ctx
, builder
, ins
->cil_code
);
4924 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4925 * Start a new bblock.
4926 * Prevent the bblocks to be merged by doing a volatile load + cond branch
4927 * from localloc-ed memory.
4929 if (!cfg
->llvm_only
)
4930 ;//set_failure (ctx, "basic block too long");
4932 if (!ctx
->long_bb_break_var
) {
4933 ctx
->long_bb_break_var
= build_alloca_llvm_type_name (ctx
, LLVMInt32Type (), 0, "long_bb_break");
4934 mono_llvm_build_store (ctx
->alloca_builder
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), ctx
->long_bb_break_var
, TRUE
, LLVM_BARRIER_NONE
);
4937 cbb
= gen_bb (ctx
, "CONT_LONG_BB");
4938 LLVMBasicBlockRef dummy_bb
= gen_bb (ctx
, "CONT_LONG_BB_DUMMY");
4940 LLVMValueRef load
= mono_llvm_build_load (builder
, ctx
->long_bb_break_var
, "", TRUE
);
4942 * The long_bb_break_var is initialized to 0 in the prolog, so this branch will always go to 'cbb'
4943 * but llvm doesn't know that, so the branch is not going to be eliminated.
4945 LLVMValueRef cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, load
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
4947 LLVMBuildCondBr (builder
, cmp
, cbb
, dummy_bb
);
4949 /* Emit a dummy false bblock which does nothing but contains a volatile store so it cannot be eliminated */
4950 ctx
->builder
= builder
= create_builder (ctx
);
4951 LLVMPositionBuilderAtEnd (builder
, dummy_bb
);
4952 mono_llvm_build_store (builder
, LLVMConstInt (LLVMInt32Type (), 1, FALSE
), ctx
->long_bb_break_var
, TRUE
, LLVM_BARRIER_NONE
);
4953 LLVMBuildBr (builder
, cbb
);
4955 ctx
->builder
= builder
= create_builder (ctx
);
4956 LLVMPositionBuilderAtEnd (builder
, cbb
);
4957 ctx
->bblocks
[bb
->block_num
].end_bblock
= cbb
;
4960 emit_dbg_loc (ctx
, builder
, ins
->cil_code
);
4964 /* There could be instructions after a terminator, skip them */
4967 if (spec
[MONO_INST_DEST
] != ' ' && !MONO_IS_STORE_MEMBASE (ins
)) {
4968 sprintf (dname_buf
, "t%d", ins
->dreg
);
4972 if (spec
[MONO_INST_SRC1
] != ' ' && spec
[MONO_INST_SRC1
] != 'v') {
4973 MonoInst
*var
= get_vreg_to_inst (cfg
, ins
->sreg1
);
4975 if (var
&& var
->flags
& (MONO_INST_VOLATILE
|MONO_INST_INDIRECT
) && var
->opcode
!= OP_GSHAREDVT_ARG_REGOFFSET
) {
4976 lhs
= emit_volatile_load (ctx
, ins
->sreg1
);
4978 /* It is ok for SETRET to have an uninitialized argument */
4979 if (!values
[ins
->sreg1
] && ins
->opcode
!= OP_SETRET
) {
4980 set_failure (ctx
, "sreg1");
4983 lhs
= values
[ins
->sreg1
];
4989 if (spec
[MONO_INST_SRC2
] != ' ' && spec
[MONO_INST_SRC2
] != ' ') {
4990 MonoInst
*var
= get_vreg_to_inst (cfg
, ins
->sreg2
);
4991 if (var
&& var
->flags
& (MONO_INST_VOLATILE
|MONO_INST_INDIRECT
)) {
4992 rhs
= emit_volatile_load (ctx
, ins
->sreg2
);
4994 if (!values
[ins
->sreg2
]) {
4995 set_failure (ctx
, "sreg2");
4998 rhs
= values
[ins
->sreg2
];
5004 //mono_print_ins (ins);
5005 gboolean skip_volatile_store
= FALSE
;
5006 switch (ins
->opcode
) {
5009 case OP_LIVERANGE_START
:
5010 case OP_LIVERANGE_END
:
5013 values
[ins
->dreg
] = LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
);
5016 #if TARGET_SIZEOF_VOID_P == 4
5017 values
[ins
->dreg
] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins
), FALSE
);
5019 values
[ins
->dreg
] = LLVMConstInt (LLVMInt64Type (), (gint64
)ins
->inst_c0
, FALSE
);
5023 values
[ins
->dreg
] = get_double_const (cfg
, *(double*)ins
->inst_p0
);
5026 values
[ins
->dreg
] = get_float_const (cfg
, *(float*)ins
->inst_p0
);
5028 case OP_DUMMY_ICONST
:
5029 values
[ins
->dreg
] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
5031 case OP_DUMMY_I8CONST
:
5032 values
[ins
->dreg
] = LLVMConstInt (LLVMInt64Type (), 0, FALSE
);
5034 case OP_DUMMY_R8CONST
:
5035 values
[ins
->dreg
] = LLVMConstReal (LLVMDoubleType (), 0.0f
);
5038 LLVMBasicBlockRef target_bb
= get_bb (ctx
, ins
->inst_target_bb
);
5039 LLVMBuildBr (builder
, target_bb
);
5040 has_terminator
= TRUE
;
5047 LLVMBasicBlockRef new_bb
;
5048 LLVMBuilderRef new_builder
;
5050 // The default branch is already handled
5051 // FIXME: Handle it here
5053 /* Start new bblock */
5054 sprintf (bb_name
, "SWITCH_DEFAULT_BB%d", ctx
->default_index
++);
5055 new_bb
= LLVMAppendBasicBlock (ctx
->lmethod
, bb_name
);
5057 lhs
= convert (ctx
, lhs
, LLVMInt32Type ());
5058 v
= LLVMBuildSwitch (builder
, lhs
, new_bb
, GPOINTER_TO_UINT (ins
->klass
));
5059 for (i
= 0; i
< GPOINTER_TO_UINT (ins
->klass
); ++i
) {
5060 MonoBasicBlock
*target_bb
= ins
->inst_many_bb
[i
];
5062 LLVMAddCase (v
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), get_bb (ctx
, target_bb
));
5065 new_builder
= create_builder (ctx
);
5066 LLVMPositionBuilderAtEnd (new_builder
, new_bb
);
5067 LLVMBuildUnreachable (new_builder
);
5069 has_terminator
= TRUE
;
5070 g_assert (!ins
->next
);
5076 switch (linfo
->ret
.storage
) {
5077 case LLVMArgVtypeInReg
: {
5078 LLVMTypeRef ret_type
= LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method
)));
5079 LLVMValueRef val
, addr
, retval
;
5082 retval
= LLVMGetUndef (ret_type
);
5084 if (!addresses
[ins
->sreg1
]) {
5086 * The return type is an LLVM vector type, have to convert between it and the
5087 * real return type which is a struct type.
5089 g_assert (MONO_CLASS_IS_SIMD (ctx
->cfg
, mono_class_from_mono_type_internal (sig
->ret
)));
5090 /* Convert to 2xi64 first */
5091 val
= LLVMBuildBitCast (builder
, values
[ins
->sreg1
], LLVMVectorType (IntPtrType (), 2), "");
5093 for (i
= 0; i
< 2; ++i
) {
5094 if (linfo
->ret
.pair_storage
[i
] == LLVMArgInIReg
) {
5095 retval
= LLVMBuildInsertValue (builder
, retval
, LLVMBuildExtractElement (builder
, val
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), ""), i
, "");
5097 g_assert (linfo
->ret
.pair_storage
[i
] == LLVMArgNone
);
5101 addr
= LLVMBuildBitCast (builder
, addresses
[ins
->sreg1
], LLVMPointerType (ret_type
, 0), "");
5102 for (i
= 0; i
< 2; ++i
) {
5103 if (linfo
->ret
.pair_storage
[i
] == LLVMArgInIReg
) {
5104 LLVMValueRef indexes
[2], part_addr
;
5106 indexes
[0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
5107 indexes
[1] = LLVMConstInt (LLVMInt32Type (), i
, FALSE
);
5108 part_addr
= LLVMBuildGEP (builder
, addr
, indexes
, 2, "");
5110 retval
= LLVMBuildInsertValue (builder
, retval
, LLVMBuildLoad (builder
, part_addr
, ""), i
, "");
5112 g_assert (linfo
->ret
.pair_storage
[i
] == LLVMArgNone
);
5116 LLVMBuildRet (builder
, retval
);
5119 case LLVMArgVtypeAsScalar
: {
5120 LLVMTypeRef ret_type
= LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method
)));
5121 LLVMValueRef retval
;
5123 g_assert (addresses
[ins
->sreg1
]);
5125 retval
= LLVMBuildLoad (builder
, LLVMBuildBitCast (builder
, addresses
[ins
->sreg1
], LLVMPointerType (ret_type
, 0), ""), "");
5126 LLVMBuildRet (builder
, retval
);
5129 case LLVMArgVtypeByVal
: {
5130 LLVMValueRef retval
;
5132 g_assert (addresses
[ins
->sreg1
]);
5133 retval
= LLVMBuildLoad (builder
, addresses
[ins
->sreg1
], "");
5134 LLVMBuildRet (builder
, retval
);
5137 case LLVMArgVtypeByRef
: {
5138 LLVMBuildRetVoid (builder
);
5141 case LLVMArgGsharedvtFixed
: {
5142 LLVMTypeRef ret_type
= type_to_llvm_type (ctx
, sig
->ret
);
5143 /* The return value is in lhs, need to store to the vret argument */
5144 /* sreg1 might not be set */
5146 g_assert (cfg
->vret_addr
);
5147 g_assert (values
[cfg
->vret_addr
->dreg
]);
5148 LLVMBuildStore (builder
, convert (ctx
, lhs
, ret_type
), convert (ctx
, values
[cfg
->vret_addr
->dreg
], LLVMPointerType (ret_type
, 0)));
5150 LLVMBuildRetVoid (builder
);
5153 case LLVMArgGsharedvtFixedVtype
: {
5155 LLVMBuildRetVoid (builder
);
5158 case LLVMArgGsharedvtVariable
: {
5160 LLVMBuildRetVoid (builder
);
5163 case LLVMArgVtypeRetAddr
: {
5164 LLVMBuildRetVoid (builder
);
5167 case LLVMArgAsIArgs
:
5168 case LLVMArgFpStruct
: {
5169 LLVMTypeRef ret_type
= LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method
)));
5170 LLVMValueRef retval
;
5172 g_assert (addresses
[ins
->sreg1
]);
5173 retval
= LLVMBuildLoad (builder
, convert (ctx
, addresses
[ins
->sreg1
], LLVMPointerType (ret_type
, 0)), "");
5174 LLVMBuildRet (builder
, retval
);
5178 case LLVMArgNormal
: {
5179 if (!lhs
|| ctx
->is_dead
[ins
->sreg1
]) {
5181 * The method did not set its return value, probably because it
5182 * ends with a throw.
5185 LLVMBuildRetVoid (builder
);
5187 LLVMBuildRet (builder
, LLVMConstNull (type_to_llvm_type (ctx
, sig
->ret
)));
5189 LLVMBuildRet (builder
, convert (ctx
, lhs
, type_to_llvm_type (ctx
, sig
->ret
)));
5191 has_terminator
= TRUE
;
5195 g_assert_not_reached ();
5204 case OP_ICOMPARE_IMM
:
5205 case OP_LCOMPARE_IMM
:
5206 case OP_COMPARE_IMM
: {
5208 LLVMValueRef cmp
, args
[16];
5209 gboolean likely
= (ins
->flags
& MONO_INST_LIKELY
) != 0;
5210 gboolean unlikely
= FALSE
;
5212 if (MONO_IS_COND_BRANCH_OP (ins
->next
)) {
5213 if (ins
->next
->inst_false_bb
->out_of_line
)
5215 else if (ins
->next
->inst_true_bb
->out_of_line
)
5219 if (ins
->next
->opcode
== OP_NOP
)
5222 if (ins
->next
->opcode
== OP_BR
)
5223 /* The comparison result is not needed */
5226 rel
= mono_opcode_to_cond (ins
->next
->opcode
);
5228 if (ins
->opcode
== OP_ICOMPARE_IMM
) {
5229 lhs
= convert (ctx
, lhs
, LLVMInt32Type ());
5230 rhs
= LLVMConstInt (LLVMInt32Type (), ins
->inst_imm
, FALSE
);
5232 if (ins
->opcode
== OP_LCOMPARE_IMM
) {
5233 lhs
= convert (ctx
, lhs
, LLVMInt64Type ());
5234 rhs
= LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins
), FALSE
);
5236 if (ins
->opcode
== OP_LCOMPARE
) {
5237 lhs
= convert (ctx
, lhs
, LLVMInt64Type ());
5238 rhs
= convert (ctx
, rhs
, LLVMInt64Type ());
5240 if (ins
->opcode
== OP_ICOMPARE
) {
5241 lhs
= convert (ctx
, lhs
, LLVMInt32Type ());
5242 rhs
= convert (ctx
, rhs
, LLVMInt32Type ());
5246 if (LLVMGetTypeKind (LLVMTypeOf (lhs
)) == LLVMPointerTypeKind
)
5247 rhs
= convert (ctx
, rhs
, LLVMTypeOf (lhs
));
5248 else if (LLVMGetTypeKind (LLVMTypeOf (rhs
)) == LLVMPointerTypeKind
)
5249 lhs
= convert (ctx
, lhs
, LLVMTypeOf (rhs
));
5252 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
5253 if (ins
->opcode
== OP_FCOMPARE
) {
5254 cmp
= LLVMBuildFCmp (builder
, fpcond_to_llvm_cond
[rel
], convert (ctx
, lhs
, LLVMDoubleType ()), convert (ctx
, rhs
, LLVMDoubleType ()), "");
5255 } else if (ins
->opcode
== OP_RCOMPARE
) {
5256 cmp
= LLVMBuildFCmp (builder
, fpcond_to_llvm_cond
[rel
], convert (ctx
, lhs
, LLVMFloatType ()), convert (ctx
, rhs
, LLVMFloatType ()), "");
5257 } else if (ins
->opcode
== OP_COMPARE_IMM
) {
5258 LLVMIntPredicate llvm_pred
= cond_to_llvm_cond
[rel
];
5259 if (LLVMGetTypeKind (LLVMTypeOf (lhs
)) == LLVMPointerTypeKind
&& ins
->inst_imm
== 0) {
5260 // We are emitting a NULL check for a pointer
5261 gboolean nonnull
= mono_llvm_is_nonnull (lhs
);
5263 if (nonnull
&& llvm_pred
== LLVMIntEQ
)
5264 cmp
= LLVMConstInt (LLVMInt1Type (), FALSE
, FALSE
);
5265 else if (nonnull
&& llvm_pred
== LLVMIntNE
)
5266 cmp
= LLVMConstInt (LLVMInt1Type (), TRUE
, FALSE
);
5268 cmp
= LLVMBuildICmp (builder
, llvm_pred
, lhs
, LLVMConstNull (LLVMTypeOf (lhs
)), "");
5271 cmp
= LLVMBuildICmp (builder
, llvm_pred
, convert (ctx
, lhs
, IntPtrType ()), LLVMConstInt (IntPtrType (), ins
->inst_imm
, FALSE
), "");
5273 } else if (ins
->opcode
== OP_LCOMPARE_IMM
) {
5274 cmp
= LLVMBuildICmp (builder
, cond_to_llvm_cond
[rel
], lhs
, rhs
, "");
5276 else if (ins
->opcode
== OP_COMPARE
) {
5277 if (LLVMGetTypeKind (LLVMTypeOf (lhs
)) == LLVMPointerTypeKind
&& LLVMTypeOf (lhs
) == LLVMTypeOf (rhs
))
5278 cmp
= LLVMBuildICmp (builder
, cond_to_llvm_cond
[rel
], lhs
, rhs
, "");
5280 cmp
= LLVMBuildICmp (builder
, cond_to_llvm_cond
[rel
], convert (ctx
, lhs
, IntPtrType ()), convert (ctx
, rhs
, IntPtrType ()), "");
5282 cmp
= LLVMBuildICmp (builder
, cond_to_llvm_cond
[rel
], lhs
, rhs
, "");
5284 if (likely
|| unlikely
) {
5286 args
[1] = LLVMConstInt (LLVMInt1Type (), likely
? 1 : 0, FALSE
);
5287 cmp
= call_intrins (ctx
, INTRINS_EXPECT_I1
, args
, "");
5290 if (MONO_IS_COND_BRANCH_OP (ins
->next
)) {
5291 if (ins
->next
->inst_true_bb
== ins
->next
->inst_false_bb
) {
5293 * If the target bb contains PHI instructions, LLVM requires
5294 * two PHI entries for this bblock, while we only generate one.
5295 * So convert this to an unconditional bblock. (bxc #171).
5297 LLVMBuildBr (builder
, get_bb (ctx
, ins
->next
->inst_true_bb
));
5299 LLVMBuildCondBr (builder
, cmp
, get_bb (ctx
, ins
->next
->inst_true_bb
), get_bb (ctx
, ins
->next
->inst_false_bb
));
5301 has_terminator
= TRUE
;
5302 } else if (MONO_IS_SETCC (ins
->next
)) {
5303 sprintf (dname_buf
, "t%d", ins
->next
->dreg
);
5305 values
[ins
->next
->dreg
] = LLVMBuildZExt (builder
, cmp
, LLVMInt32Type (), dname
);
5307 /* Add stores for volatile variables */
5308 emit_volatile_store (ctx
, ins
->next
->dreg
);
5309 } else if (MONO_IS_COND_EXC (ins
->next
)) {
5310 gboolean force_explicit_branch
= FALSE
;
5311 if (bb
->region
!= -1) {
5312 /* Don't tag null check branches in exception-handling
5313 * regions with `make.implicit`.
5315 force_explicit_branch
= TRUE
;
5317 emit_cond_system_exception (ctx
, bb
, (const char*)ins
->next
->inst_p1
, cmp
, force_explicit_branch
);
5320 builder
= ctx
->builder
;
5322 set_failure (ctx
, "next");
5340 rel
= mono_opcode_to_cond (ins
->opcode
);
5342 cmp
= LLVMBuildFCmp (builder
, fpcond_to_llvm_cond
[rel
], convert (ctx
, lhs
, LLVMDoubleType ()), convert (ctx
, rhs
, LLVMDoubleType ()), "");
5343 values
[ins
->dreg
] = LLVMBuildZExt (builder
, cmp
, LLVMInt32Type (), dname
);
5355 rel
= mono_opcode_to_cond (ins
->opcode
);
5357 cmp
= LLVMBuildFCmp (builder
, fpcond_to_llvm_cond
[rel
], convert (ctx
, lhs
, LLVMFloatType ()), convert (ctx
, rhs
, LLVMFloatType ()), "");
5358 values
[ins
->dreg
] = LLVMBuildZExt (builder
, cmp
, LLVMInt32Type (), dname
);
5366 skip_volatile_store
= TRUE
;
5374 values
[ins
->dreg
] = lhs
;
5378 MonoInst
*var
= get_vreg_to_inst (cfg
, ins
->dreg
);
5381 values
[ins
->dreg
] = lhs
;
5383 if (var
&& m_class_get_byval_arg (var
->klass
)->type
== MONO_TYPE_R4
) {
5385 * This is added by the spilling pass in case of the JIT,
5386 * but we have to do it ourselves.
5388 values
[ins
->dreg
] = convert (ctx
, values
[ins
->dreg
], LLVMFloatType ());
5392 case OP_MOVE_F_TO_I4
: {
5393 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, LLVMBuildFPTrunc (builder
, lhs
, LLVMFloatType (), ""), LLVMInt32Type (), "");
5396 case OP_MOVE_I4_TO_F
: {
5397 values
[ins
->dreg
] = LLVMBuildFPExt (builder
, LLVMBuildBitCast (builder
, lhs
, LLVMFloatType (), ""), LLVMDoubleType (), "");
5400 case OP_MOVE_F_TO_I8
: {
5401 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, lhs
, LLVMInt64Type (), "");
5404 case OP_MOVE_I8_TO_F
: {
5405 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, lhs
, LLVMDoubleType (), "");
5438 lhs
= convert (ctx
, lhs
, regtype_to_llvm_type (spec
[MONO_INST_DEST
]));
5439 rhs
= convert (ctx
, rhs
, regtype_to_llvm_type (spec
[MONO_INST_DEST
]));
5441 emit_div_check (ctx
, builder
, bb
, ins
, lhs
, rhs
);
5444 builder
= ctx
->builder
;
5446 switch (ins
->opcode
) {
5449 values
[ins
->dreg
] = LLVMBuildAdd (builder
, lhs
, rhs
, dname
);
5453 values
[ins
->dreg
] = LLVMBuildSub (builder
, lhs
, rhs
, dname
);
5457 values
[ins
->dreg
] = LLVMBuildMul (builder
, lhs
, rhs
, dname
);
5461 values
[ins
->dreg
] = LLVMBuildSRem (builder
, lhs
, rhs
, dname
);
5465 values
[ins
->dreg
] = LLVMBuildURem (builder
, lhs
, rhs
, dname
);
5469 values
[ins
->dreg
] = LLVMBuildSDiv (builder
, lhs
, rhs
, dname
);
5473 values
[ins
->dreg
] = LLVMBuildUDiv (builder
, lhs
, rhs
, dname
);
5477 values
[ins
->dreg
] = LLVMBuildFDiv (builder
, lhs
, rhs
, dname
);
5481 values
[ins
->dreg
] = LLVMBuildAnd (builder
, lhs
, rhs
, dname
);
5485 values
[ins
->dreg
] = LLVMBuildOr (builder
, lhs
, rhs
, dname
);
5489 values
[ins
->dreg
] = LLVMBuildXor (builder
, lhs
, rhs
, dname
);
5493 values
[ins
->dreg
] = LLVMBuildShl (builder
, lhs
, rhs
, dname
);
5497 values
[ins
->dreg
] = LLVMBuildAShr (builder
, lhs
, rhs
, dname
);
5501 values
[ins
->dreg
] = LLVMBuildLShr (builder
, lhs
, rhs
, dname
);
5505 values
[ins
->dreg
] = LLVMBuildFAdd (builder
, lhs
, rhs
, dname
);
5508 values
[ins
->dreg
] = LLVMBuildFSub (builder
, lhs
, rhs
, dname
);
5511 values
[ins
->dreg
] = LLVMBuildFMul (builder
, lhs
, rhs
, dname
);
5515 g_assert_not_reached ();
5522 lhs
= convert (ctx
, lhs
, LLVMFloatType ());
5523 rhs
= convert (ctx
, rhs
, LLVMFloatType ());
5524 switch (ins
->opcode
) {
5526 values
[ins
->dreg
] = LLVMBuildFAdd (builder
, lhs
, rhs
, dname
);
5529 values
[ins
->dreg
] = LLVMBuildFSub (builder
, lhs
, rhs
, dname
);
5532 values
[ins
->dreg
] = LLVMBuildFMul (builder
, lhs
, rhs
, dname
);
5535 values
[ins
->dreg
] = LLVMBuildFDiv (builder
, lhs
, rhs
, dname
);
5538 g_assert_not_reached ();
5547 case OP_IREM_UN_IMM
:
5549 case OP_IDIV_UN_IMM
:
5555 case OP_ISHR_UN_IMM
:
5565 case OP_LSHR_UN_IMM
:
5571 case OP_SHR_UN_IMM
: {
5574 if (spec
[MONO_INST_SRC1
] == 'l') {
5575 imm
= LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins
), FALSE
);
5577 imm
= LLVMConstInt (LLVMInt32Type (), ins
->inst_imm
, FALSE
);
5580 emit_div_check (ctx
, builder
, bb
, ins
, lhs
, imm
);
5583 builder
= ctx
->builder
;
5585 #if TARGET_SIZEOF_VOID_P == 4
5586 if (ins
->opcode
== OP_LSHL_IMM
|| ins
->opcode
== OP_LSHR_IMM
|| ins
->opcode
== OP_LSHR_UN_IMM
)
5587 imm
= LLVMConstInt (LLVMInt32Type (), ins
->inst_imm
, FALSE
);
5590 if (LLVMGetTypeKind (LLVMTypeOf (lhs
)) == LLVMPointerTypeKind
)
5591 lhs
= convert (ctx
, lhs
, IntPtrType ());
5592 imm
= convert (ctx
, imm
, LLVMTypeOf (lhs
));
5593 switch (ins
->opcode
) {
5597 values
[ins
->dreg
] = LLVMBuildAdd (builder
, lhs
, imm
, dname
);
5601 values
[ins
->dreg
] = LLVMBuildSub (builder
, lhs
, imm
, dname
);
5606 values
[ins
->dreg
] = LLVMBuildMul (builder
, lhs
, imm
, dname
);
5610 values
[ins
->dreg
] = LLVMBuildSDiv (builder
, lhs
, imm
, dname
);
5612 case OP_IDIV_UN_IMM
:
5613 case OP_LDIV_UN_IMM
:
5614 values
[ins
->dreg
] = LLVMBuildUDiv (builder
, lhs
, imm
, dname
);
5618 values
[ins
->dreg
] = LLVMBuildSRem (builder
, lhs
, imm
, dname
);
5620 case OP_IREM_UN_IMM
:
5621 values
[ins
->dreg
] = LLVMBuildURem (builder
, lhs
, imm
, dname
);
5626 values
[ins
->dreg
] = LLVMBuildAnd (builder
, lhs
, imm
, dname
);
5630 values
[ins
->dreg
] = LLVMBuildOr (builder
, lhs
, imm
, dname
);
5634 values
[ins
->dreg
] = LLVMBuildXor (builder
, lhs
, imm
, dname
);
5638 values
[ins
->dreg
] = LLVMBuildShl (builder
, lhs
, imm
, dname
);
5641 if (TARGET_SIZEOF_VOID_P
== 8) {
5642 /* The IL is not regular */
5643 lhs
= convert (ctx
, lhs
, LLVMInt64Type ());
5644 imm
= convert (ctx
, imm
, LLVMInt64Type ());
5646 values
[ins
->dreg
] = LLVMBuildShl (builder
, lhs
, imm
, dname
);
5651 values
[ins
->dreg
] = LLVMBuildAShr (builder
, lhs
, imm
, dname
);
5653 case OP_ISHR_UN_IMM
:
5654 /* This is used to implement conv.u4, so the lhs could be an i8 */
5655 lhs
= convert (ctx
, lhs
, LLVMInt32Type ());
5656 imm
= convert (ctx
, imm
, LLVMInt32Type ());
5657 values
[ins
->dreg
] = LLVMBuildLShr (builder
, lhs
, imm
, dname
);
5659 case OP_LSHR_UN_IMM
:
5661 values
[ins
->dreg
] = LLVMBuildLShr (builder
, lhs
, imm
, dname
);
5664 g_assert_not_reached ();
5669 values
[ins
->dreg
] = LLVMBuildSub (builder
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), convert (ctx
, lhs
, LLVMInt32Type ()), dname
);
5672 if (LLVMTypeOf (lhs
) != LLVMInt64Type ())
5673 lhs
= convert (ctx
, lhs
, LLVMInt64Type ());
5674 values
[ins
->dreg
] = LLVMBuildSub (builder
, LLVMConstInt (LLVMInt64Type (), 0, FALSE
), lhs
, dname
);
5677 lhs
= convert (ctx
, lhs
, LLVMDoubleType ());
5678 values
[ins
->dreg
] = LLVMBuildFNeg (builder
, lhs
, dname
);
5681 lhs
= convert (ctx
, lhs
, LLVMFloatType ());
5682 values
[ins
->dreg
] = LLVMBuildFNeg (builder
, lhs
, dname
);
5685 guint32 v
= 0xffffffff;
5686 values
[ins
->dreg
] = LLVMBuildXor (builder
, LLVMConstInt (LLVMInt32Type (), v
, FALSE
), convert (ctx
, lhs
, LLVMInt32Type ()), dname
);
5690 if (LLVMTypeOf (lhs
) != LLVMInt64Type ())
5691 lhs
= convert (ctx
, lhs
, LLVMInt64Type ());
5692 guint64 v
= 0xffffffffffffffffLL
;
5693 values
[ins
->dreg
] = LLVMBuildXor (builder
, LLVMConstInt (LLVMInt64Type (), v
, FALSE
), lhs
, dname
);
5696 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5698 LLVMValueRef v1
, v2
;
5700 rhs
= LLVMBuildSExt (builder
, convert (ctx
, rhs
, LLVMInt32Type ()), LLVMInt64Type (), "");
5702 v1
= LLVMBuildMul (builder
, convert (ctx
, rhs
, IntPtrType ()), LLVMConstInt (IntPtrType (), ((unsigned long long)1 << ins
->backend
.shift_amount
), FALSE
), "");
5703 v2
= LLVMBuildAdd (builder
, convert (ctx
, lhs
, IntPtrType ()), v1
, "");
5704 values
[ins
->dreg
] = LLVMBuildAdd (builder
, v2
, LLVMConstInt (IntPtrType (), ins
->inst_imm
, FALSE
), dname
);
5709 case OP_ICONV_TO_I1
:
5710 case OP_ICONV_TO_I2
:
5711 case OP_ICONV_TO_I4
:
5712 case OP_ICONV_TO_U1
:
5713 case OP_ICONV_TO_U2
:
5714 case OP_ICONV_TO_U4
:
5715 case OP_LCONV_TO_I1
:
5716 case OP_LCONV_TO_I2
:
5717 case OP_LCONV_TO_U1
:
5718 case OP_LCONV_TO_U2
:
5719 case OP_LCONV_TO_U4
: {
5722 sign
= (ins
->opcode
== OP_ICONV_TO_I1
) || (ins
->opcode
== OP_ICONV_TO_I2
) || (ins
->opcode
== OP_ICONV_TO_I4
) || (ins
->opcode
== OP_LCONV_TO_I1
) || (ins
->opcode
== OP_LCONV_TO_I2
);
5724 /* Have to do two casts since our vregs have type int */
5725 v
= LLVMBuildTrunc (builder
, lhs
, op_to_llvm_type (ins
->opcode
), "");
5727 values
[ins
->dreg
] = LLVMBuildSExt (builder
, v
, LLVMInt32Type (), dname
);
5729 values
[ins
->dreg
] = LLVMBuildZExt (builder
, v
, LLVMInt32Type (), dname
);
5732 case OP_ICONV_TO_I8
:
5733 values
[ins
->dreg
] = LLVMBuildSExt (builder
, lhs
, LLVMInt64Type (), dname
);
5735 case OP_ICONV_TO_U8
:
5736 values
[ins
->dreg
] = LLVMBuildZExt (builder
, lhs
, LLVMInt64Type (), dname
);
5738 case OP_FCONV_TO_I4
:
5739 case OP_RCONV_TO_I4
:
5740 values
[ins
->dreg
] = LLVMBuildFPToSI (builder
, lhs
, LLVMInt32Type (), dname
);
5742 case OP_FCONV_TO_I1
:
5743 case OP_RCONV_TO_I1
:
5744 values
[ins
->dreg
] = LLVMBuildSExt (builder
, LLVMBuildFPToSI (builder
, lhs
, LLVMInt8Type (), dname
), LLVMInt32Type (), "");
5746 case OP_FCONV_TO_U1
:
5747 case OP_RCONV_TO_U1
:
5748 values
[ins
->dreg
] = LLVMBuildZExt (builder
, LLVMBuildTrunc (builder
, LLVMBuildFPToUI (builder
, lhs
, IntPtrType (), dname
), LLVMInt8Type (), ""), LLVMInt32Type (), "");
5750 case OP_FCONV_TO_I2
:
5751 case OP_RCONV_TO_I2
:
5752 values
[ins
->dreg
] = LLVMBuildSExt (builder
, LLVMBuildFPToSI (builder
, lhs
, LLVMInt16Type (), dname
), LLVMInt32Type (), "");
5754 case OP_FCONV_TO_U2
:
5755 case OP_RCONV_TO_U2
:
5756 values
[ins
->dreg
] = LLVMBuildZExt (builder
, LLVMBuildFPToUI (builder
, lhs
, LLVMInt16Type (), dname
), LLVMInt32Type (), "");
5758 case OP_FCONV_TO_U4
:
5759 case OP_RCONV_TO_U4
:
5760 values
[ins
->dreg
] = LLVMBuildFPToUI (builder
, lhs
, LLVMInt32Type (), dname
);
5762 case OP_FCONV_TO_U8
:
5763 case OP_RCONV_TO_U8
:
5764 values
[ins
->dreg
] = LLVMBuildFPToUI (builder
, lhs
, LLVMInt64Type (), dname
);
5766 case OP_FCONV_TO_I8
:
5767 case OP_RCONV_TO_I8
:
5768 values
[ins
->dreg
] = LLVMBuildFPToSI (builder
, lhs
, LLVMInt64Type (), dname
);
5771 values
[ins
->dreg
] = LLVMBuildFPToSI (builder
, lhs
, IntPtrType (), dname
);
5773 case OP_ICONV_TO_R8
:
5774 case OP_LCONV_TO_R8
:
5775 values
[ins
->dreg
] = LLVMBuildSIToFP (builder
, lhs
, LLVMDoubleType (), dname
);
5777 case OP_ICONV_TO_R_UN
:
5778 case OP_LCONV_TO_R_UN
:
5779 values
[ins
->dreg
] = LLVMBuildUIToFP (builder
, lhs
, LLVMDoubleType (), dname
);
5781 #if TARGET_SIZEOF_VOID_P == 4
5784 case OP_LCONV_TO_I4
:
5785 values
[ins
->dreg
] = LLVMBuildTrunc (builder
, lhs
, LLVMInt32Type (), dname
);
5787 case OP_ICONV_TO_R4
:
5788 case OP_LCONV_TO_R4
:
5789 v
= LLVMBuildSIToFP (builder
, lhs
, LLVMFloatType (), "");
5791 values
[ins
->dreg
] = v
;
5793 values
[ins
->dreg
] = LLVMBuildFPExt (builder
, v
, LLVMDoubleType (), dname
);
5795 case OP_FCONV_TO_R4
:
5796 v
= LLVMBuildFPTrunc (builder
, lhs
, LLVMFloatType (), "");
5798 values
[ins
->dreg
] = v
;
5800 values
[ins
->dreg
] = LLVMBuildFPExt (builder
, v
, LLVMDoubleType (), dname
);
5802 case OP_RCONV_TO_R8
:
5803 values
[ins
->dreg
] = LLVMBuildFPExt (builder
, lhs
, LLVMDoubleType (), dname
);
5805 case OP_RCONV_TO_R4
:
5806 values
[ins
->dreg
] = lhs
;
5809 values
[ins
->dreg
] = LLVMBuildSExt (builder
, convert (ctx
, lhs
, LLVMInt32Type ()), LLVMInt64Type (), dname
);
5812 values
[ins
->dreg
] = LLVMBuildZExt (builder
, convert (ctx
, lhs
, LLVMInt32Type ()), LLVMInt64Type (), dname
);
5815 values
[ins
->dreg
] = LLVMBuildTrunc (builder
, lhs
, LLVMInt32Type (), dname
);
5817 case OP_LOCALLOC_IMM
: {
5820 guint32 size
= ins
->inst_imm
;
5821 size
= (size
+ (MONO_ARCH_FRAME_ALIGNMENT
- 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT
- 1);
5823 v
= mono_llvm_build_alloca (builder
, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size
, FALSE
), MONO_ARCH_FRAME_ALIGNMENT
, "");
5825 if (ins
->flags
& MONO_INST_INIT
)
5826 emit_memset (ctx
, builder
, v
, const_int32 (size
), MONO_ARCH_FRAME_ALIGNMENT
);
5828 values
[ins
->dreg
] = v
;
5832 LLVMValueRef v
, size
;
5834 size
= LLVMBuildAnd (builder
, LLVMBuildAdd (builder
, convert (ctx
, lhs
, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT
- 1, FALSE
), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT
- 1), FALSE
), "");
5836 v
= mono_llvm_build_alloca (builder
, LLVMInt8Type (), size
, MONO_ARCH_FRAME_ALIGNMENT
, "");
5838 if (ins
->flags
& MONO_INST_INIT
)
5839 emit_memset (ctx
, builder
, v
, size
, MONO_ARCH_FRAME_ALIGNMENT
);
5840 values
[ins
->dreg
] = v
;
5844 case OP_LOADI1_MEMBASE
:
5845 case OP_LOADU1_MEMBASE
:
5846 case OP_LOADI2_MEMBASE
:
5847 case OP_LOADU2_MEMBASE
:
5848 case OP_LOADI4_MEMBASE
:
5849 case OP_LOADU4_MEMBASE
:
5850 case OP_LOADI8_MEMBASE
:
5851 case OP_LOADR4_MEMBASE
:
5852 case OP_LOADR8_MEMBASE
:
5853 case OP_LOAD_MEMBASE
:
5861 LLVMValueRef base
, index
, addr
;
5863 gboolean sext
= FALSE
, zext
= FALSE
;
5864 gboolean is_faulting
= (ins
->flags
& MONO_INST_FAULT
) != 0;
5865 gboolean is_volatile
= (ins
->flags
& MONO_INST_VOLATILE
) != 0;
5866 gboolean is_unaligned
= (ins
->flags
& MONO_INST_UNALIGNED
) != 0;
5868 t
= load_store_to_llvm_type (ins
->opcode
, &size
, &sext
, &zext
);
5873 if ((ins
->opcode
== OP_LOADI8_MEM
) || (ins
->opcode
== OP_LOAD_MEM
) || (ins
->opcode
== OP_LOADI4_MEM
) || (ins
->opcode
== OP_LOADU4_MEM
) || (ins
->opcode
== OP_LOADU1_MEM
) || (ins
->opcode
== OP_LOADU2_MEM
)) {
5874 addr
= LLVMConstInt (IntPtrType (), ins
->inst_imm
, FALSE
);
5880 if (ins
->inst_offset
== 0) {
5881 LLVMValueRef gep_base
, gep_offset
;
5882 if (mono_llvm_can_be_gep (base
, &gep_base
, &gep_offset
)) {
5883 addr
= LLVMBuildGEP (builder
, convert (ctx
, gep_base
, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset
, 1, "");
5887 } else if (ins
->inst_offset
% size
!= 0) {
5888 /* Unaligned load */
5889 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
, FALSE
);
5890 addr
= LLVMBuildGEP (builder
, convert (ctx
, base
, LLVMPointerType (LLVMInt8Type (), 0)), &index
, 1, "");
5892 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
/ size
, FALSE
);
5893 addr
= LLVMBuildGEP (builder
, convert (ctx
, base
, LLVMPointerType (t
, 0)), &index
, 1, "");
5897 addr
= convert (ctx
, addr
, LLVMPointerType (t
, 0));
5900 values
[ins
->dreg
] = mono_llvm_build_aligned_load (builder
, addr
, dname
, is_volatile
, 1);
5902 values
[ins
->dreg
] = emit_load (ctx
, bb
, &builder
, size
, addr
, base
, dname
, is_faulting
, is_volatile
, LLVM_BARRIER_NONE
);
5904 if (!(is_faulting
|| is_volatile
) && (ins
->flags
& MONO_INST_INVARIANT_LOAD
)) {
5906 * These will signal LLVM that these loads do not alias any stores, and
5907 * they can't fail, allowing them to be hoisted out of loops.
5909 set_invariant_load_flag (values
[ins
->dreg
]);
5913 values
[ins
->dreg
] = LLVMBuildSExt (builder
, values
[ins
->dreg
], LLVMInt32Type (), dname
);
5915 values
[ins
->dreg
] = LLVMBuildZExt (builder
, values
[ins
->dreg
], LLVMInt32Type (), dname
);
5916 else if (!cfg
->r4fp
&& ins
->opcode
== OP_LOADR4_MEMBASE
)
5917 values
[ins
->dreg
] = LLVMBuildFPExt (builder
, values
[ins
->dreg
], LLVMDoubleType (), dname
);
5921 case OP_STOREI1_MEMBASE_REG
:
5922 case OP_STOREI2_MEMBASE_REG
:
5923 case OP_STOREI4_MEMBASE_REG
:
5924 case OP_STOREI8_MEMBASE_REG
:
5925 case OP_STORER4_MEMBASE_REG
:
5926 case OP_STORER8_MEMBASE_REG
:
5927 case OP_STORE_MEMBASE_REG
: {
5929 LLVMValueRef index
, addr
, base
;
5931 gboolean sext
= FALSE
, zext
= FALSE
;
5932 gboolean is_faulting
= (ins
->flags
& MONO_INST_FAULT
) != 0;
5933 gboolean is_volatile
= (ins
->flags
& MONO_INST_VOLATILE
) != 0;
5934 gboolean is_unaligned
= (ins
->flags
& MONO_INST_UNALIGNED
) != 0;
5936 if (!values
[ins
->inst_destbasereg
]) {
5937 set_failure (ctx
, "inst_destbasereg");
5941 t
= load_store_to_llvm_type (ins
->opcode
, &size
, &sext
, &zext
);
5943 base
= values
[ins
->inst_destbasereg
];
5944 LLVMValueRef gep_base
, gep_offset
;
5945 if (ins
->inst_offset
== 0 && mono_llvm_can_be_gep (base
, &gep_base
, &gep_offset
)) {
5946 addr
= LLVMBuildGEP (builder
, convert (ctx
, gep_base
, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset
, 1, "");
5947 } else if (ins
->inst_offset
% size
!= 0) {
5948 /* Unaligned store */
5949 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
, FALSE
);
5950 addr
= LLVMBuildGEP (builder
, convert (ctx
, base
, LLVMPointerType (LLVMInt8Type (), 0)), &index
, 1, "");
5952 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
/ size
, FALSE
);
5953 addr
= LLVMBuildGEP (builder
, convert (ctx
, base
, LLVMPointerType (t
, 0)), &index
, 1, "");
5955 if (is_volatile
&& LLVMGetInstructionOpcode (base
) == LLVMAlloca
&& !(ins
->flags
& MONO_INST_VOLATILE
))
5956 /* Storing to an alloca cannot fail */
5957 is_volatile
= FALSE
;
5958 LLVMValueRef srcval
= convert (ctx
, values
[ins
->sreg1
], t
);
5959 LLVMValueRef ptrdst
= convert (ctx
, addr
, LLVMPointerType (t
, 0));
5962 mono_llvm_build_aligned_store (builder
, srcval
, ptrdst
, is_volatile
, 1);
5964 emit_store (ctx
, bb
, &builder
, size
, srcval
, ptrdst
, base
, is_faulting
, is_volatile
);
5968 case OP_STOREI1_MEMBASE_IMM
:
5969 case OP_STOREI2_MEMBASE_IMM
:
5970 case OP_STOREI4_MEMBASE_IMM
:
5971 case OP_STOREI8_MEMBASE_IMM
:
5972 case OP_STORE_MEMBASE_IMM
: {
5974 LLVMValueRef index
, addr
, base
;
5976 gboolean sext
= FALSE
, zext
= FALSE
;
5977 gboolean is_faulting
= (ins
->flags
& MONO_INST_FAULT
) != 0;
5978 gboolean is_volatile
= (ins
->flags
& MONO_INST_VOLATILE
) != 0;
5979 gboolean is_unaligned
= (ins
->flags
& MONO_INST_UNALIGNED
) != 0;
5981 t
= load_store_to_llvm_type (ins
->opcode
, &size
, &sext
, &zext
);
5983 base
= values
[ins
->inst_destbasereg
];
5984 LLVMValueRef gep_base
, gep_offset
;
5985 if (ins
->inst_offset
== 0 && mono_llvm_can_be_gep (base
, &gep_base
, &gep_offset
)) {
5986 addr
= LLVMBuildGEP (builder
, convert (ctx
, gep_base
, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset
, 1, "");
5987 } else if (ins
->inst_offset
% size
!= 0) {
5988 /* Unaligned store */
5989 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
, FALSE
);
5990 addr
= LLVMBuildGEP (builder
, convert (ctx
, base
, LLVMPointerType (LLVMInt8Type (), 0)), &index
, 1, "");
5992 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
/ size
, FALSE
);
5993 addr
= LLVMBuildGEP (builder
, convert (ctx
, base
, LLVMPointerType (t
, 0)), &index
, 1, "");
5995 LLVMValueRef srcval
= convert (ctx
, LLVMConstInt (IntPtrType (), ins
->inst_imm
, FALSE
), t
);
5996 LLVMValueRef ptrdst
= convert (ctx
, addr
, LLVMPointerType (t
, 0));
5998 mono_llvm_build_aligned_store (builder
, srcval
, ptrdst
, is_volatile
, 1);
6000 emit_store (ctx
, bb
, &builder
, size
, srcval
, ptrdst
, base
, is_faulting
, is_volatile
);
6005 emit_load (ctx
, bb
, &builder
, TARGET_SIZEOF_VOID_P
, convert (ctx
, lhs
, LLVMPointerType (IntPtrType (), 0)), lhs
, "", TRUE
, FALSE
, LLVM_BARRIER_NONE
);
6007 case OP_OUTARG_VTRETADDR
:
6015 case OP_VOIDCALL_MEMBASE
:
6016 case OP_CALL_MEMBASE
:
6017 case OP_LCALL_MEMBASE
:
6018 case OP_FCALL_MEMBASE
:
6019 case OP_RCALL_MEMBASE
:
6020 case OP_VCALL_MEMBASE
:
6021 case OP_VOIDCALL_REG
:
6026 case OP_VCALL_REG
: {
6027 process_call (ctx
, bb
, &builder
, ins
);
6031 MonoJumpInfoType ji_type
= ins
->inst_c1
;
6032 gpointer ji_data
= ins
->inst_p0
;
6034 if (ji_type
== MONO_PATCH_INFO_ICALL_ADDR
) {
6035 char *symbol
= mono_aot_get_direct_call_symbol (MONO_PATCH_INFO_ICALL_ADDR_CALL
, ji_data
);
6038 * Avoid emitting a got entry for these since the method is directly called, and it might not be
6039 * resolvable at runtime using dlsym ().
6042 values
[ins
->dreg
] = LLVMConstInt (IntPtrType (), 0, FALSE
);
6047 values
[ins
->dreg
] = get_aotconst (ctx
, ji_type
, ji_data
, LLVMPointerType (IntPtrType (), 0));
6052 LLVMValueRef args
[5];
6053 args
[argn
++] = convert (ctx
, values
[ins
->sreg1
], LLVMPointerType (LLVMInt8Type (), 0));
6054 args
[argn
++] = convert (ctx
, values
[ins
->sreg2
], LLVMPointerType (LLVMInt8Type (), 0));
6055 args
[argn
++] = convert (ctx
, values
[ins
->sreg3
], LLVMInt64Type ());
6056 #if LLVM_API_VERSION < 900
6057 args
[argn
++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE
); // alignment
6059 args
[argn
++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE
); // is_volatile
6061 call_intrins (ctx
, INTRINS_MEMMOVE
, args
, "");
6064 case OP_NOT_REACHED
:
6065 LLVMBuildUnreachable (builder
);
6066 has_terminator
= TRUE
;
6067 g_assert (bb
->block_num
< cfg
->max_block_num
);
6068 ctx
->unreachable
[bb
->block_num
] = TRUE
;
6069 /* Might have instructions after this */
6071 MonoInst
*next
= ins
->next
;
6073 * FIXME: If later code uses the regs defined by these instructions,
6074 * compilation will fail.
6076 const char *spec
= INS_INFO (next
->opcode
);
6077 if (spec
[MONO_INST_DEST
] == 'i' && !MONO_IS_STORE_MEMBASE (next
))
6078 ctx
->values
[next
->dreg
] = LLVMConstNull (LLVMInt32Type ());
6079 MONO_DELETE_INS (bb
, next
);
6083 MonoInst
*var
= ins
->inst_i0
;
6085 if (var
->opcode
== OP_VTARG_ADDR
) {
6086 /* The variable contains the vtype address */
6087 values
[ins
->dreg
] = values
[var
->dreg
];
6088 } else if (var
->opcode
== OP_GSHAREDVT_LOCAL
) {
6089 values
[ins
->dreg
] = emit_gsharedvt_ldaddr (ctx
, var
->dreg
);
6091 values
[ins
->dreg
] = addresses
[var
->dreg
];
6096 LLVMValueRef args
[1];
6098 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6099 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SIN
, args
, dname
);
6103 LLVMValueRef args
[1];
6105 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6106 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SINF
, args
, dname
);
6110 LLVMValueRef args
[1];
6112 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6113 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_EXP
, args
, dname
);
6117 LLVMValueRef args
[1];
6119 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6120 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_EXPF
, args
, dname
);
6124 LLVMValueRef args
[1];
6126 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6127 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_LOG2
, args
, dname
);
6131 LLVMValueRef args
[1];
6133 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6134 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_LOG2F
, args
, dname
);
6138 LLVMValueRef args
[1];
6140 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6141 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_LOG10
, args
, dname
);
6145 LLVMValueRef args
[1];
6147 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6148 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_LOG10F
, args
, dname
);
6152 LLVMValueRef args
[1];
6154 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6155 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_LOG
, args
, dname
);
6159 LLVMValueRef args
[1];
6161 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6162 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_TRUNC
, args
, dname
);
6166 LLVMValueRef args
[1];
6168 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6169 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_TRUNCF
, args
, dname
);
6173 LLVMValueRef args
[1];
6175 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6176 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_COS
, args
, dname
);
6180 LLVMValueRef args
[1];
6182 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6183 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_COSF
, args
, dname
);
6187 LLVMValueRef args
[1];
6189 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6190 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SQRT
, args
, dname
);
6194 LLVMValueRef args
[1];
6196 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6197 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SQRTF
, args
, dname
);
6201 LLVMValueRef args
[1];
6203 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6204 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_FLOOR
, args
, dname
);
6208 LLVMValueRef args
[1];
6210 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6211 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_FLOORF
, args
, dname
);
6215 LLVMValueRef args
[1];
6217 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6218 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_CEIL
, args
, dname
);
6222 LLVMValueRef args
[1];
6224 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6225 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_CEILF
, args
, dname
);
6229 LLVMValueRef args
[3];
6231 args
[0] = convert (ctx
, values
[ins
->sreg1
], LLVMDoubleType ());
6232 args
[1] = convert (ctx
, values
[ins
->sreg2
], LLVMDoubleType ());
6233 args
[2] = convert (ctx
, values
[ins
->sreg3
], LLVMDoubleType ());
6235 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_FMA
, args
, dname
);
6239 LLVMValueRef args
[3];
6241 args
[0] = convert (ctx
, values
[ins
->sreg1
], LLVMFloatType ());
6242 args
[1] = convert (ctx
, values
[ins
->sreg2
], LLVMFloatType ());
6243 args
[2] = convert (ctx
, values
[ins
->sreg3
], LLVMFloatType ());
6245 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_FMAF
, args
, dname
);
6249 LLVMValueRef args
[1];
6251 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6252 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_FABS
, args
, dname
);
6256 LLVMValueRef args
[1];
6259 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6260 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_ABSF
, args
, dname
);
6262 /* llvm.fabs not supported on all platforms */
6263 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6264 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_FABS
, args
, dname
);
6265 values
[ins
->dreg
] = convert (ctx
, values
[ins
->dreg
], LLVMFloatType ());
6270 LLVMValueRef args
[2];
6272 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6273 args
[1] = convert (ctx
, rhs
, LLVMFloatType ());
6274 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_POWF
, args
, dname
);
6278 LLVMValueRef args
[2];
6280 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6281 args
[1] = convert (ctx
, rhs
, LLVMDoubleType ());
6282 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_POW
, args
, dname
);
6285 case OP_FCOPYSIGN
: {
6286 LLVMValueRef args
[2];
6288 args
[0] = convert (ctx
, lhs
, LLVMDoubleType ());
6289 args
[1] = convert (ctx
, rhs
, LLVMDoubleType ());
6290 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_COPYSIGN
, args
, dname
);
6293 case OP_RCOPYSIGN
: {
6294 LLVMValueRef args
[2];
6296 args
[0] = convert (ctx
, lhs
, LLVMFloatType ());
6297 args
[1] = convert (ctx
, rhs
, LLVMFloatType ());
6298 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_COPYSIGNF
, args
, dname
);
6316 lhs
= convert (ctx
, lhs
, regtype_to_llvm_type (spec
[MONO_INST_DEST
]));
6317 rhs
= convert (ctx
, rhs
, regtype_to_llvm_type (spec
[MONO_INST_DEST
]));
6319 switch (ins
->opcode
) {
6322 v
= LLVMBuildICmp (builder
, LLVMIntSLE
, lhs
, rhs
, "");
6326 v
= LLVMBuildICmp (builder
, LLVMIntSGE
, lhs
, rhs
, "");
6330 v
= LLVMBuildICmp (builder
, LLVMIntULE
, lhs
, rhs
, "");
6334 v
= LLVMBuildICmp (builder
, LLVMIntUGE
, lhs
, rhs
, "");
6338 v
= LLVMBuildFCmp (builder
, LLVMRealUGE
, lhs
, rhs
, "");
6342 v
= LLVMBuildFCmp (builder
, LLVMRealULE
, lhs
, rhs
, "");
6345 g_assert_not_reached ();
6348 values
[ins
->dreg
] = LLVMBuildSelect (builder
, v
, lhs
, rhs
, dname
);
6353 * See the ARM64 comment in mono/utils/atomic.h for an explanation of why this
6354 * hack is necessary (for now).
6357 #define ARM64_ATOMIC_FENCE_FIX mono_llvm_build_fence (builder, LLVM_BARRIER_SEQ)
6359 #define ARM64_ATOMIC_FENCE_FIX
6362 case OP_ATOMIC_EXCHANGE_I4
:
6363 case OP_ATOMIC_EXCHANGE_I8
: {
6364 LLVMValueRef args
[2];
6367 if (ins
->opcode
== OP_ATOMIC_EXCHANGE_I4
)
6368 t
= LLVMInt32Type ();
6370 t
= LLVMInt64Type ();
6372 g_assert (ins
->inst_offset
== 0);
6374 args
[0] = convert (ctx
, lhs
, LLVMPointerType (t
, 0));
6375 args
[1] = convert (ctx
, rhs
, t
);
6377 ARM64_ATOMIC_FENCE_FIX
;
6378 values
[ins
->dreg
] = mono_llvm_build_atomic_rmw (builder
, LLVM_ATOMICRMW_OP_XCHG
, args
[0], args
[1]);
6379 ARM64_ATOMIC_FENCE_FIX
;
6382 case OP_ATOMIC_ADD_I4
:
6383 case OP_ATOMIC_ADD_I8
:
6384 case OP_ATOMIC_AND_I4
:
6385 case OP_ATOMIC_AND_I8
:
6386 case OP_ATOMIC_OR_I4
:
6387 case OP_ATOMIC_OR_I8
: {
6388 LLVMValueRef args
[2];
6391 if (ins
->type
== STACK_I4
)
6392 t
= LLVMInt32Type ();
6394 t
= LLVMInt64Type ();
6396 g_assert (ins
->inst_offset
== 0);
6398 args
[0] = convert (ctx
, lhs
, LLVMPointerType (t
, 0));
6399 args
[1] = convert (ctx
, rhs
, t
);
6400 ARM64_ATOMIC_FENCE_FIX
;
6401 if (ins
->opcode
== OP_ATOMIC_ADD_I4
|| ins
->opcode
== OP_ATOMIC_ADD_I8
)
6402 // Interlocked.Add returns new value (that's why we emit additional Add here)
6403 // see https://github.com/dotnet/runtime/pull/33102
6404 values
[ins
->dreg
] = LLVMBuildAdd (builder
, mono_llvm_build_atomic_rmw (builder
, LLVM_ATOMICRMW_OP_ADD
, args
[0], args
[1]), args
[1], dname
);
6405 else if (ins
->opcode
== OP_ATOMIC_AND_I4
|| ins
->opcode
== OP_ATOMIC_AND_I8
)
6406 values
[ins
->dreg
] = mono_llvm_build_atomic_rmw (builder
, LLVM_ATOMICRMW_OP_AND
, args
[0], args
[1]);
6407 else if (ins
->opcode
== OP_ATOMIC_OR_I4
|| ins
->opcode
== OP_ATOMIC_OR_I8
)
6408 values
[ins
->dreg
] = mono_llvm_build_atomic_rmw (builder
, LLVM_ATOMICRMW_OP_OR
, args
[0], args
[1]);
6410 g_assert_not_reached ();
6411 ARM64_ATOMIC_FENCE_FIX
;
6414 case OP_ATOMIC_CAS_I4
:
6415 case OP_ATOMIC_CAS_I8
: {
6416 LLVMValueRef args
[3], val
;
6419 if (ins
->opcode
== OP_ATOMIC_CAS_I4
)
6420 t
= LLVMInt32Type ();
6422 t
= LLVMInt64Type ();
6424 args
[0] = convert (ctx
, lhs
, LLVMPointerType (t
, 0));
6426 args
[1] = convert (ctx
, values
[ins
->sreg3
], t
);
6428 args
[2] = convert (ctx
, values
[ins
->sreg2
], t
);
6429 ARM64_ATOMIC_FENCE_FIX
;
6430 val
= mono_llvm_build_cmpxchg (builder
, args
[0], args
[1], args
[2]);
6431 ARM64_ATOMIC_FENCE_FIX
;
6432 /* cmpxchg returns a pair */
6433 values
[ins
->dreg
] = LLVMBuildExtractValue (builder
, val
, 0, "");
6436 case OP_MEMORY_BARRIER
: {
6437 mono_llvm_build_fence (builder
, (BarrierKind
) ins
->backend
.memory_barrier_kind
);
6440 case OP_ATOMIC_LOAD_I1
:
6441 case OP_ATOMIC_LOAD_I2
:
6442 case OP_ATOMIC_LOAD_I4
:
6443 case OP_ATOMIC_LOAD_I8
:
6444 case OP_ATOMIC_LOAD_U1
:
6445 case OP_ATOMIC_LOAD_U2
:
6446 case OP_ATOMIC_LOAD_U4
:
6447 case OP_ATOMIC_LOAD_U8
:
6448 case OP_ATOMIC_LOAD_R4
:
6449 case OP_ATOMIC_LOAD_R8
: {
6451 gboolean sext
, zext
;
6453 gboolean is_faulting
= (ins
->flags
& MONO_INST_FAULT
) != 0;
6454 gboolean is_volatile
= (ins
->flags
& MONO_INST_VOLATILE
) != 0;
6455 BarrierKind barrier
= (BarrierKind
) ins
->backend
.memory_barrier_kind
;
6456 LLVMValueRef index
, addr
;
6458 t
= load_store_to_llvm_type (ins
->opcode
, &size
, &sext
, &zext
);
6463 if (ins
->inst_offset
!= 0) {
6464 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
/ size
, FALSE
);
6465 addr
= LLVMBuildGEP (builder
, convert (ctx
, lhs
, LLVMPointerType (t
, 0)), &index
, 1, "");
6470 addr
= convert (ctx
, addr
, LLVMPointerType (t
, 0));
6472 ARM64_ATOMIC_FENCE_FIX
;
6473 values
[ins
->dreg
] = emit_load (ctx
, bb
, &builder
, size
, addr
, lhs
, dname
, is_faulting
, is_volatile
, barrier
);
6474 ARM64_ATOMIC_FENCE_FIX
;
6477 values
[ins
->dreg
] = LLVMBuildSExt (builder
, values
[ins
->dreg
], LLVMInt32Type (), dname
);
6479 values
[ins
->dreg
] = LLVMBuildZExt (builder
, values
[ins
->dreg
], LLVMInt32Type (), dname
);
6482 case OP_ATOMIC_STORE_I1
:
6483 case OP_ATOMIC_STORE_I2
:
6484 case OP_ATOMIC_STORE_I4
:
6485 case OP_ATOMIC_STORE_I8
:
6486 case OP_ATOMIC_STORE_U1
:
6487 case OP_ATOMIC_STORE_U2
:
6488 case OP_ATOMIC_STORE_U4
:
6489 case OP_ATOMIC_STORE_U8
:
6490 case OP_ATOMIC_STORE_R4
:
6491 case OP_ATOMIC_STORE_R8
: {
6493 gboolean sext
, zext
;
6495 gboolean is_faulting
= (ins
->flags
& MONO_INST_FAULT
) != 0;
6496 gboolean is_volatile
= (ins
->flags
& MONO_INST_VOLATILE
) != 0;
6497 BarrierKind barrier
= (BarrierKind
) ins
->backend
.memory_barrier_kind
;
6498 LLVMValueRef index
, addr
, value
, base
;
6500 if (!values
[ins
->inst_destbasereg
]) {
6501 set_failure (ctx
, "inst_destbasereg");
6505 t
= load_store_to_llvm_type (ins
->opcode
, &size
, &sext
, &zext
);
6507 base
= values
[ins
->inst_destbasereg
];
6508 index
= LLVMConstInt (LLVMInt32Type (), ins
->inst_offset
/ size
, FALSE
);
6509 addr
= LLVMBuildGEP (builder
, convert (ctx
, base
, LLVMPointerType (t
, 0)), &index
, 1, "");
6510 value
= convert (ctx
, values
[ins
->sreg1
], t
);
6512 ARM64_ATOMIC_FENCE_FIX
;
6513 emit_store_general (ctx
, bb
, &builder
, size
, value
, addr
, base
, is_faulting
, is_volatile
, barrier
);
6514 ARM64_ATOMIC_FENCE_FIX
;
6517 case OP_RELAXED_NOP
: {
6518 #if defined(TARGET_AMD64) || defined(TARGET_X86)
6519 call_intrins (ctx
, INTRINS_SSE_PAUSE
, NULL
, "");
6526 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
6528 // 257 == FS segment register
6529 LLVMTypeRef ptrtype
= LLVMPointerType (IntPtrType (), 257);
6531 // 256 == GS segment register
6532 LLVMTypeRef ptrtype
= LLVMPointerType (IntPtrType (), 256);
6535 values
[ins
->dreg
] = LLVMBuildLoad (builder
, LLVMBuildIntToPtr (builder
, LLVMConstInt (IntPtrType (), ins
->inst_offset
, TRUE
), ptrtype
, ""), "");
6536 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
6537 /* See mono_amd64_emit_tls_get () */
6538 int offset
= mono_amd64_get_tls_gs_offset () + (ins
->inst_offset
* 8);
6540 // 256 == GS segment register
6541 LLVMTypeRef ptrtype
= LLVMPointerType (IntPtrType (), 256);
6542 values
[ins
->dreg
] = LLVMBuildLoad (builder
, LLVMBuildIntToPtr (builder
, LLVMConstInt (IntPtrType (), offset
, TRUE
), ptrtype
, ""), "");
6544 set_failure (ctx
, "opcode tls-get");
6550 case OP_GC_SAFE_POINT
: {
6551 LLVMValueRef val
, cmp
, callee
, call
;
6552 LLVMBasicBlockRef poll_bb
, cont_bb
;
6553 LLVMValueRef args
[2];
6554 static LLVMTypeRef sig
;
6555 const char *icall_name
= "mono_threads_state_poll";
6558 * Create the cold wrapper around the icall, along with a managed method for it so
6561 if (!cfg
->compile_aot
&& !ctx
->module
->gc_poll_cold_wrapper_compiled
) {
6563 /* Compiling a method here is a bit ugly, but it works */
6564 MonoMethod
*wrapper
= mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL
);
6565 ctx
->module
->gc_poll_cold_wrapper_compiled
= mono_jit_compile_method (wrapper
, error
);
6566 mono_error_assert_ok (error
);
6570 sig
= LLVMFunctionType0 (LLVMVoidType (), FALSE
);
6574 * mono_threads_state_poll ();
6576 val
= mono_llvm_build_load (builder
, convert (ctx
, lhs
, LLVMPointerType (IntPtrType (), 0)), "", TRUE
);
6577 cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, val
, LLVMConstNull (LLVMTypeOf (val
)), "");
6578 poll_bb
= gen_bb (ctx
, "POLL_BB");
6579 cont_bb
= gen_bb (ctx
, "CONT_BB");
6582 args
[1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE
);
6583 cmp
= call_intrins (ctx
, INTRINS_EXPECT_I1
, args
, "");
6585 mono_llvm_build_weighted_branch (builder
, cmp
, cont_bb
, poll_bb
, 64, 4);
6587 ctx
->builder
= builder
= create_builder (ctx
);
6588 LLVMPositionBuilderAtEnd (builder
, poll_bb
);
6590 if (ctx
->cfg
->compile_aot
) {
6591 callee
= get_callee (ctx
, sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_threads_state_poll
));
6592 call
= LLVMBuildCall (builder
, callee
, NULL
, 0, "");
6594 callee
= get_jit_callee (ctx
, icall_name
, sig
, MONO_PATCH_INFO_ABS
, ctx
->module
->gc_poll_cold_wrapper_compiled
);
6595 call
= LLVMBuildCall (builder
, callee
, NULL
, 0, "");
6596 set_call_cold_cconv (call
);
6598 LLVMBuildBr (builder
, cont_bb
);
6600 ctx
->builder
= builder
= create_builder (ctx
);
6601 LLVMPositionBuilderAtEnd (builder
, cont_bb
);
6602 ctx
->bblocks
[bb
->block_num
].end_bblock
= cont_bb
;
6610 case OP_IADD_OVF_UN
:
6612 case OP_ISUB_OVF_UN
:
6614 case OP_IMUL_OVF_UN
:
6616 case OP_LADD_OVF_UN
:
6618 case OP_LSUB_OVF_UN
:
6620 case OP_LMUL_OVF_UN
: {
6621 LLVMValueRef args
[2], val
, ovf
;
6622 IntrinsicId intrins
;
6624 args
[0] = convert (ctx
, lhs
, op_to_llvm_type (ins
->opcode
));
6625 args
[1] = convert (ctx
, rhs
, op_to_llvm_type (ins
->opcode
));
6626 intrins
= ovf_op_to_intrins (ins
->opcode
);
6627 val
= call_intrins (ctx
, intrins
, args
, "");
6628 values
[ins
->dreg
] = LLVMBuildExtractValue (builder
, val
, 0, dname
);
6629 ovf
= LLVMBuildExtractValue (builder
, val
, 1, "");
6630 emit_cond_system_exception (ctx
, bb
, "OverflowException", ovf
, FALSE
);
6633 builder
= ctx
->builder
;
6639 * We currently model them using arrays. Promotion to local vregs is
6640 * disabled for them in mono_handle_global_vregs () in the LLVM case,
6641 * so we always have an entry in cfg->varinfo for them.
6642 * FIXME: Is this needed ?
6645 MonoClass
*klass
= ins
->klass
;
6649 set_failure (ctx
, "!klass");
6653 if (!addresses
[ins
->dreg
])
6654 addresses
[ins
->dreg
] = build_alloca (ctx
, m_class_get_byval_arg (klass
));
6655 LLVMValueRef ptr
= LLVMBuildBitCast (builder
, addresses
[ins
->dreg
], LLVMPointerType (LLVMInt8Type (), 0), "");
6656 emit_memset (ctx
, builder
, ptr
, const_int32 (mono_class_value_size (klass
, NULL
)), 0);
6659 case OP_DUMMY_VZERO
:
6662 case OP_STOREV_MEMBASE
:
6663 case OP_LOADV_MEMBASE
:
6665 MonoClass
*klass
= ins
->klass
;
6666 LLVMValueRef src
= NULL
, dst
, args
[5];
6667 gboolean done
= FALSE
;
6671 set_failure (ctx
, "!klass");
6675 if (mini_is_gsharedvt_klass (klass
)) {
6677 set_failure (ctx
, "gsharedvt");
6681 switch (ins
->opcode
) {
6682 case OP_STOREV_MEMBASE
:
6683 if (cfg
->gen_write_barriers
&& m_class_has_references (klass
) && ins
->inst_destbasereg
!= cfg
->frame_reg
&&
6684 LLVMGetInstructionOpcode (values
[ins
->inst_destbasereg
]) != LLVMAlloca
) {
6685 /* Decomposed earlier */
6686 g_assert_not_reached ();
6689 if (!addresses
[ins
->sreg1
]) {
6691 g_assert (values
[ins
->sreg1
]);
6692 dst
= convert (ctx
, LLVMBuildAdd (builder
, convert (ctx
, values
[ins
->inst_destbasereg
], IntPtrType ()), LLVMConstInt (IntPtrType (), ins
->inst_offset
, FALSE
), ""), LLVMPointerType (type_to_llvm_type (ctx
, m_class_get_byval_arg (klass
)), 0));
6693 LLVMBuildStore (builder
, values
[ins
->sreg1
], dst
);
6696 src
= LLVMBuildBitCast (builder
, addresses
[ins
->sreg1
], LLVMPointerType (LLVMInt8Type (), 0), "");
6697 dst
= convert (ctx
, LLVMBuildAdd (builder
, convert (ctx
, values
[ins
->inst_destbasereg
], IntPtrType ()), LLVMConstInt (IntPtrType (), ins
->inst_offset
, FALSE
), ""), LLVMPointerType (LLVMInt8Type (), 0));
6700 case OP_LOADV_MEMBASE
:
6701 if (!addresses
[ins
->dreg
])
6702 addresses
[ins
->dreg
] = build_alloca (ctx
, m_class_get_byval_arg (klass
));
6703 src
= convert (ctx
, LLVMBuildAdd (builder
, convert (ctx
, values
[ins
->inst_basereg
], IntPtrType ()), LLVMConstInt (IntPtrType (), ins
->inst_offset
, FALSE
), ""), LLVMPointerType (LLVMInt8Type (), 0));
6704 dst
= LLVMBuildBitCast (builder
, addresses
[ins
->dreg
], LLVMPointerType (LLVMInt8Type (), 0), "");
6707 if (!addresses
[ins
->sreg1
])
6708 addresses
[ins
->sreg1
] = build_alloca (ctx
, m_class_get_byval_arg (klass
));
6709 if (!addresses
[ins
->dreg
])
6710 addresses
[ins
->dreg
] = build_alloca (ctx
, m_class_get_byval_arg (klass
));
6711 src
= LLVMBuildBitCast (builder
, addresses
[ins
->sreg1
], LLVMPointerType (LLVMInt8Type (), 0), "");
6712 dst
= LLVMBuildBitCast (builder
, addresses
[ins
->dreg
], LLVMPointerType (LLVMInt8Type (), 0), "");
6715 g_assert_not_reached ();
6724 args
[aindex
++] = dst
;
6725 args
[aindex
++] = src
;
6726 args
[aindex
++] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass
, NULL
), FALSE
);
6727 #if LLVM_API_VERSION < 900
6729 args
[aindex
++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
6731 args
[aindex
++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE
);
6732 call_intrins (ctx
, INTRINS_MEMCPY
, args
, "");
6735 case OP_LLVM_OUTARG_VT
: {
6736 LLVMArgInfo
*ainfo
= (LLVMArgInfo
*)ins
->inst_p0
;
6737 MonoType
*t
= mini_get_underlying_type (ins
->inst_vtype
);
6739 if (ainfo
->storage
== LLVMArgGsharedvtVariable
) {
6740 MonoInst
*var
= get_vreg_to_inst (cfg
, ins
->sreg1
);
6742 if (var
&& var
->opcode
== OP_GSHAREDVT_LOCAL
) {
6743 addresses
[ins
->dreg
] = convert (ctx
, emit_gsharedvt_ldaddr (ctx
, var
->dreg
), LLVMPointerType (IntPtrType (), 0));
6745 g_assert (addresses
[ins
->sreg1
]);
6746 addresses
[ins
->dreg
] = addresses
[ins
->sreg1
];
6748 } else if (ainfo
->storage
== LLVMArgGsharedvtFixed
) {
6749 if (!addresses
[ins
->sreg1
]) {
6750 addresses
[ins
->sreg1
] = build_alloca (ctx
, t
);
6751 g_assert (values
[ins
->sreg1
]);
6753 LLVMBuildStore (builder
, convert (ctx
, values
[ins
->sreg1
], LLVMGetElementType (LLVMTypeOf (addresses
[ins
->sreg1
]))), addresses
[ins
->sreg1
]);
6754 addresses
[ins
->dreg
] = addresses
[ins
->sreg1
];
6756 if (!addresses
[ins
->sreg1
]) {
6757 addresses
[ins
->sreg1
] = build_alloca (ctx
, t
);
6758 g_assert (values
[ins
->sreg1
]);
6759 LLVMBuildStore (builder
, convert (ctx
, values
[ins
->sreg1
], type_to_llvm_type (ctx
, t
)), addresses
[ins
->sreg1
]);
6760 addresses
[ins
->dreg
] = addresses
[ins
->sreg1
];
6761 } else if (ainfo
->storage
== LLVMArgVtypeAddr
|| values
[ins
->sreg1
] == addresses
[ins
->sreg1
]) {
6762 /* LLVMArgVtypeByRef/LLVMArgVtypeAddr, have to make a copy */
6763 addresses
[ins
->dreg
] = build_alloca (ctx
, t
);
6764 LLVMValueRef v
= LLVMBuildLoad (builder
, addresses
[ins
->sreg1
], "");
6765 LLVMBuildStore (builder
, convert (ctx
, v
, type_to_llvm_type (ctx
, t
)), addresses
[ins
->dreg
]);
6767 addresses
[ins
->dreg
] = addresses
[ins
->sreg1
];
6772 case OP_OBJC_GET_SELECTOR
: {
6773 const char *name
= (const char*)ins
->inst_p0
;
6776 if (!ctx
->module
->objc_selector_to_var
) {
6777 ctx
->module
->objc_selector_to_var
= g_hash_table_new_full (g_str_hash
, g_str_equal
, g_free
, NULL
);
6779 LLVMValueRef info_var
= LLVMAddGlobal (ctx
->lmodule
, LLVMArrayType (LLVMInt8Type (), 8), "@OBJC_IMAGE_INFO");
6780 int32_t objc_imageinfo
[] = { 0, 16 };
6781 LLVMSetInitializer (info_var
, mono_llvm_create_constant_data_array ((uint8_t *) &objc_imageinfo
, 8));
6782 LLVMSetLinkage (info_var
, LLVMPrivateLinkage
);
6783 LLVMSetExternallyInitialized (info_var
, TRUE
);
6784 LLVMSetSection (info_var
, "__DATA, __objc_imageinfo,regular,no_dead_strip");
6785 LLVMSetAlignment (info_var
, sizeof (target_mgreg_t
));
6786 mark_as_used (ctx
->module
, info_var
);
6789 var
= (LLVMValueRef
)g_hash_table_lookup (ctx
->module
->objc_selector_to_var
, name
);
6791 LLVMValueRef indexes
[16];
6793 LLVMValueRef name_var
= LLVMAddGlobal (ctx
->lmodule
, LLVMArrayType (LLVMInt8Type (), strlen (name
) + 1), "@OBJC_METH_VAR_NAME_");
6794 LLVMSetInitializer (name_var
, mono_llvm_create_constant_data_array ((const uint8_t*)name
, strlen (name
) + 1));
6795 LLVMSetLinkage (name_var
, LLVMPrivateLinkage
);
6796 LLVMSetSection (name_var
, "__TEXT,__objc_methname,cstring_literals");
6797 mark_as_used (ctx
->module
, name_var
);
6799 LLVMValueRef ref_var
= LLVMAddGlobal (ctx
->lmodule
, LLVMPointerType (LLVMInt8Type (), 0), "@OBJC_SELECTOR_REFERENCES_");
6801 indexes
[0] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6802 indexes
[1] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6803 LLVMSetInitializer (ref_var
, LLVMConstGEP (name_var
, indexes
, 2));
6804 LLVMSetLinkage (ref_var
, LLVMPrivateLinkage
);
6805 LLVMSetExternallyInitialized (ref_var
, TRUE
);
6806 LLVMSetSection (ref_var
, "__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
6807 LLVMSetAlignment (ref_var
, sizeof (target_mgreg_t
));
6808 mark_as_used (ctx
->module
, ref_var
);
6810 g_hash_table_insert (ctx
->module
->objc_selector_to_var
, g_strdup (name
), ref_var
);
6814 values
[ins
->dreg
] = LLVMBuildLoad (builder
, var
, "");
6821 #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM)
6823 values
[ins
->dreg
] = LLVMConstNull (type_to_llvm_type (ctx
, m_class_get_byval_arg (ins
->klass
)));
6826 case OP_LOADX_MEMBASE
: {
6827 LLVMTypeRef t
= type_to_llvm_type (ctx
, m_class_get_byval_arg (ins
->klass
));
6830 src
= convert (ctx
, LLVMBuildAdd (builder
, convert (ctx
, values
[ins
->inst_basereg
], IntPtrType ()), LLVMConstInt (IntPtrType (), ins
->inst_offset
, FALSE
), ""), LLVMPointerType (t
, 0));
6831 values
[ins
->dreg
] = mono_llvm_build_aligned_load (builder
, src
, "", FALSE
, 1);
6834 case OP_STOREX_MEMBASE
: {
6835 LLVMTypeRef t
= LLVMTypeOf (values
[ins
->sreg1
]);
6838 dest
= convert (ctx
, LLVMBuildAdd (builder
, convert (ctx
, values
[ins
->inst_destbasereg
], IntPtrType ()), LLVMConstInt (IntPtrType (), ins
->inst_offset
, FALSE
), ""), LLVMPointerType (t
, 0));
6839 mono_llvm_build_aligned_store (builder
, values
[ins
->sreg1
], dest
, FALSE
, 1);
6846 values
[ins
->dreg
] = LLVMBuildAdd (builder
, lhs
, rhs
, "");
6850 values
[ins
->dreg
] = LLVMBuildFAdd (builder
, lhs
, rhs
, "");
6856 values
[ins
->dreg
] = LLVMBuildSub (builder
, lhs
, rhs
, "");
6860 values
[ins
->dreg
] = LLVMBuildFSub (builder
, lhs
, rhs
, "");
6864 values
[ins
->dreg
] = LLVMBuildFMul (builder
, lhs
, rhs
, "");
6868 values
[ins
->dreg
] = LLVMBuildFDiv (builder
, lhs
, rhs
, "");
6871 values
[ins
->dreg
] = LLVMBuildAnd (builder
, lhs
, rhs
, "");
6874 values
[ins
->dreg
] = LLVMBuildOr (builder
, lhs
, rhs
, "");
6877 values
[ins
->dreg
] = LLVMBuildXor (builder
, lhs
, rhs
, "");
6881 values
[ins
->dreg
] = LLVMBuildMul (builder
, lhs
, rhs
, "");
6892 LLVMValueRef v
= NULL
;
6894 switch (ins
->opcode
) {
6899 t
= LLVMVectorType (LLVMInt32Type (), 4);
6900 rt
= LLVMVectorType (LLVMFloatType (), 4);
6906 t
= LLVMVectorType (LLVMInt64Type (), 2);
6907 rt
= LLVMVectorType (LLVMDoubleType (), 2);
6910 t
= LLVMInt32Type ();
6911 rt
= LLVMInt32Type ();
6912 g_assert_not_reached ();
6915 lhs
= LLVMBuildBitCast (builder
, lhs
, t
, "");
6916 rhs
= LLVMBuildBitCast (builder
, rhs
, t
, "");
6917 switch (ins
->opcode
) {
6920 v
= LLVMBuildAnd (builder
, lhs
, rhs
, "");
6924 v
= LLVMBuildOr (builder
, lhs
, rhs
, "");
6928 v
= LLVMBuildXor (builder
, lhs
, rhs
, "");
6932 v
= LLVMBuildAnd (builder
, rhs
, LLVMBuildNot (builder
, lhs
, ""), "");
6935 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, v
, rt
, "");
6941 LLVMValueRef cmp
= LLVMBuildICmp (builder
, LLVMIntULT
, lhs
, rhs
, "");
6942 values
[ins
->dreg
] = LLVMBuildSelect (builder
, cmp
, lhs
, rhs
, "");
6948 LLVMValueRef cmp
= LLVMBuildICmp (builder
, LLVMIntUGT
, lhs
, rhs
, "");
6949 values
[ins
->dreg
] = LLVMBuildSelect (builder
, cmp
, lhs
, rhs
, "");
6953 LLVMValueRef cmp
= LLVMBuildICmp (builder
, LLVMIntSLT
, lhs
, rhs
, "");
6954 values
[ins
->dreg
] = LLVMBuildSelect (builder
, cmp
, lhs
, rhs
, "");
6958 LLVMValueRef cmp
= LLVMBuildICmp (builder
, LLVMIntSGT
, lhs
, rhs
, "");
6959 values
[ins
->dreg
] = LLVMBuildSelect (builder
, cmp
, lhs
, rhs
, "");
6977 case OP_PMULW_HIGH_UN
: {
6978 LLVMValueRef args
[2];
6983 values
[ins
->dreg
] = call_intrins (ctx
, simd_ins_to_intrins (ins
->opcode
), args
, "");
6988 LLVMValueRef ones_vec
;
6989 LLVMValueRef ones
[32];
6990 int vector_size
= LLVMGetVectorSize (LLVMTypeOf (lhs
));
6991 LLVMTypeRef ext_elem_type
= vector_size
== 16 ? LLVMInt16Type () : LLVMInt32Type ();
6993 for (int i
= 0; i
< 32; ++i
)
6994 ones
[i
] = LLVMConstInt (ext_elem_type
, 1, FALSE
);
6995 ones_vec
= LLVMConstVector (ones
, vector_size
);
6998 LLVMTypeRef ext_type
= LLVMVectorType (ext_elem_type
, vector_size
);
7000 /* Have to increase the vector element size to prevent overflows */
7001 /* res = trunc ((zext (lhs) + zext (rhs) + 1) >> 1) */
7002 val
= LLVMBuildAdd (builder
, LLVMBuildZExt (builder
, lhs
, ext_type
, ""), LLVMBuildZExt (builder
, rhs
, ext_type
, ""), "");
7003 val
= LLVMBuildAdd (builder
, val
, ones_vec
, "");
7004 val
= LLVMBuildLShr (builder
, val
, ones_vec
, "");
7005 values
[ins
->dreg
] = LLVMBuildTrunc (builder
, val
, LLVMTypeOf (lhs
), "");
7014 LLVMTypeRef retType
;
7015 LLVMIntPredicate cmpOp
;
7017 if (ins
->opcode
== OP_PCMPGTB
)
7022 if (LLVMTypeOf (lhs
) == LLVMTypeOf (rhs
)) {
7023 pcmp
= LLVMBuildICmp (builder
, cmpOp
, lhs
, rhs
, "");
7024 retType
= LLVMTypeOf (lhs
);
7026 LLVMTypeRef flatType
= LLVMVectorType (LLVMInt8Type (), 16);
7027 LLVMValueRef flatRHS
= convert (ctx
, rhs
, flatType
);
7028 LLVMValueRef flatLHS
= convert (ctx
, lhs
, flatType
);
7030 pcmp
= LLVMBuildICmp (builder
, cmpOp
, flatLHS
, flatRHS
, "");
7034 values
[ins
->dreg
] = LLVMBuildSExt (builder
, pcmp
, retType
, "");
7043 case OP_EXTRACTX_U2
:
7045 case OP_EXTRACT_U1
: {
7047 gboolean zext
= FALSE
;
7049 t
= simd_op_to_llvm_type (ins
->opcode
);
7051 switch (ins
->opcode
) {
7060 case OP_EXTRACTX_U2
:
7065 t
= LLVMInt32Type ();
7066 g_assert_not_reached ();
7069 lhs
= LLVMBuildBitCast (builder
, lhs
, t
, "");
7070 values
[ins
->dreg
] = LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
), "");
7072 values
[ins
->dreg
] = LLVMBuildZExt (builder
, values
[ins
->dreg
], LLVMInt32Type (), "");
7081 case OP_EXPAND_R8
: {
7083 LLVMValueRef mask
[32], v
;
7086 #ifdef ENABLE_NETCORE
7087 t
= simd_class_to_llvm_type (ctx
, ins
->klass
);
7089 t
= simd_op_to_llvm_type (ins
->opcode
);
7091 for (i
= 0; i
< 32; ++i
)
7092 mask
[i
] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
7094 v
= convert (ctx
, values
[ins
->sreg1
], LLVMGetElementType (t
));
7096 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, LLVMConstNull (t
), v
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7097 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, values
[ins
->dreg
], LLVMGetUndef (t
), LLVMConstVector (mask
, LLVMGetVectorSize (t
)), "");
7102 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, values
[ins
->sreg1
], convert (ctx
, values
[ins
->sreg2
], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
), dname
);
7105 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, values
[ins
->sreg1
], convert (ctx
, values
[ins
->sreg2
], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
), dname
);
7108 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, values
[ins
->sreg1
], convert (ctx
, values
[ins
->sreg2
], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
), dname
);
7111 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, values
[ins
->sreg1
], convert (ctx
, values
[ins
->sreg2
], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
), dname
);
7114 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, values
[ins
->sreg1
], convert (ctx
, values
[ins
->sreg2
], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
), dname
);
7117 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, values
[ins
->sreg1
], convert (ctx
, values
[ins
->sreg2
], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
), dname
);
7119 case OP_XINSERT_I2
: {
7120 LLVMBasicBlockRef bbs
[64];
7121 LLVMValueRef switch_ins
;
7122 LLVMValueRef vector
= lhs
;
7123 LLVMValueRef value
= rhs
;
7124 LLVMValueRef index
= values
[ins
->sreg3
];
7125 LLVMValueRef phi_values
[64];
7126 int nelems
= LLVMGetVectorSize (LLVMTypeOf (lhs
));
7130 * Many SIMD opcodes require an immediate operand, but can be called with a non-immediate.
7131 * To handle these cases, generate a switch statement with one case for all possible
7132 * values of the immediate.
7135 * res = <op> (val, i)
7139 g_assert (nelems
<= 64);
7140 for (i
= 0; i
< nelems
; ++i
)
7141 bbs
[i
] = gen_bb (ctx
, "XINSERT_CASE_BB");
7142 cbb
= gen_bb (ctx
, "XINSERT_COND_BB");
7144 switch_ins
= LLVMBuildSwitch (builder
, LLVMBuildAnd (builder
, index
, const_int32 (0xf), ""), bbs
[0], 0);
7145 for (i
= 0; i
< nelems
; ++i
) {
7146 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), bbs
[i
]);
7147 LLVMPositionBuilderAtEnd (builder
, bbs
[i
]);
7148 phi_values
[i
] = LLVMBuildInsertElement (builder
, vector
, convert (ctx
, value
, LLVMGetElementType (LLVMTypeOf (vector
))), LLVMConstInt (LLVMInt32Type (), i
, FALSE
), "");
7149 LLVMBuildBr (builder
, cbb
);
7152 LLVMPositionBuilderAtEnd (builder
, cbb
);
7153 values
[ins
->dreg
] = LLVMBuildPhi (builder
, LLVMTypeOf (phi_values
[0]), "");
7154 LLVMAddIncoming (values
[ins
->dreg
], phi_values
, bbs
, nelems
);
7156 ctx
->bblocks
[bb
->block_num
].end_bblock
= cbb
;
7160 LLVMValueRef i4
= LLVMBuildBitCast (builder
, lhs
, sse_i4_t
, "");
7161 values
[ins
->dreg
] = LLVMBuildSIToFP (builder
, i4
, sse_r4_t
, dname
);
7165 LLVMValueRef indexes
[16];
7167 indexes
[0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
7168 indexes
[1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE
);
7169 LLVMValueRef mask
= LLVMConstVector (indexes
, 2);
7170 LLVMValueRef shuffle
= LLVMBuildShuffleVector (builder
, lhs
, LLVMConstNull (LLVMTypeOf (lhs
)), mask
, "");
7171 values
[ins
->dreg
] = LLVMBuildSIToFP (builder
, shuffle
, LLVMVectorType (LLVMDoubleType (), 2), dname
);
7174 case OP_SSE2_CVTSS2SD
: {
7175 LLVMValueRef rhs_elem
= LLVMBuildExtractElement (builder
, rhs
, const_int32 (0), "");
7176 LLVMValueRef fpext
= LLVMBuildFPExt (builder
, rhs_elem
, LLVMDoubleType (), dname
);
7177 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, lhs
, fpext
, const_int32 (0), "");
7181 LLVMValueRef indexes
[16];
7183 indexes
[0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
7184 indexes
[1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE
);
7185 LLVMValueRef mask
= LLVMConstVector (indexes
, 2);
7186 LLVMValueRef shuffle
= LLVMBuildShuffleVector (builder
, lhs
, LLVMConstNull (LLVMTypeOf (lhs
)), mask
, "");
7187 values
[ins
->dreg
] = LLVMBuildFPExt (builder
, shuffle
, LLVMVectorType (LLVMDoubleType (), 2), dname
);
7191 values
[ins
->dreg
] = LLVMBuildFPToSI (builder
, lhs
, LLVMVectorType (LLVMInt32Type (), 4), dname
);
7198 case OP_EXTRACT_MASK
:
7205 v
= convert (ctx
, values
[ins
->sreg1
], simd_op_to_llvm_type (ins
->opcode
));
7207 values
[ins
->dreg
] = call_intrins (ctx
, simd_ins_to_intrins (ins
->opcode
), &v
, dname
);
7212 LLVMRealPredicate op
;
7214 switch (ins
->inst_c0
) {
7224 case SIMD_COMP_UNORD
:
7240 g_assert_not_reached ();
7243 LLVMValueRef cmp
= LLVMBuildFCmp (builder
, op
, lhs
, rhs
, "");
7244 if (ins
->opcode
== OP_COMPPD
)
7245 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, LLVMBuildSExt (builder
, cmp
, LLVMVectorType (LLVMInt64Type (), 2), ""), LLVMTypeOf (lhs
), "");
7247 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, LLVMBuildSExt (builder
, cmp
, LLVMVectorType (LLVMInt32Type (), 4), ""), LLVMTypeOf (lhs
), "");
7251 /* This is only used for implementing shifts by non-immediate */
7252 values
[ins
->dreg
] = lhs
;
7263 LLVMValueRef args
[3];
7266 args
[1] = LLVMConstInt (LLVMInt32Type (), ins
->inst_imm
, FALSE
);
7268 values
[ins
->dreg
] = call_intrins (ctx
, simd_ins_to_intrins (ins
->opcode
), args
, dname
);
7279 case OP_PSHLQ_REG
: {
7280 LLVMValueRef args
[3];
7283 args
[1] = values
[ins
->sreg2
];
7285 values
[ins
->dreg
] = call_intrins (ctx
, simd_ins_to_intrins (ins
->opcode
), args
, dname
);
7292 case OP_PSHUFLEW_LOW
:
7293 case OP_PSHUFLEW_HIGH
: {
7295 LLVMValueRef v1
= NULL
, v2
= NULL
, mask_values
[16];
7296 int i
, mask_size
= 0;
7297 int imask
= ins
->inst_c0
;
7299 /* Convert the x86 shuffle mask to LLVM's */
7300 switch (ins
->opcode
) {
7303 mask
[0] = ((imask
>> 0) & 3);
7304 mask
[1] = ((imask
>> 2) & 3);
7305 mask
[2] = ((imask
>> 4) & 3) + 4;
7306 mask
[3] = ((imask
>> 6) & 3) + 4;
7307 v1
= values
[ins
->sreg1
];
7308 v2
= values
[ins
->sreg2
];
7312 mask
[0] = ((imask
>> 0) & 1);
7313 mask
[1] = ((imask
>> 1) & 1) + 2;
7314 v1
= values
[ins
->sreg1
];
7315 v2
= values
[ins
->sreg2
];
7317 case OP_PSHUFLEW_LOW
:
7319 mask
[0] = ((imask
>> 0) & 3);
7320 mask
[1] = ((imask
>> 2) & 3);
7321 mask
[2] = ((imask
>> 4) & 3);
7322 mask
[3] = ((imask
>> 6) & 3);
7327 v1
= values
[ins
->sreg1
];
7328 v2
= LLVMGetUndef (LLVMTypeOf (v1
));
7330 case OP_PSHUFLEW_HIGH
:
7336 mask
[4] = 4 + ((imask
>> 0) & 3);
7337 mask
[5] = 4 + ((imask
>> 2) & 3);
7338 mask
[6] = 4 + ((imask
>> 4) & 3);
7339 mask
[7] = 4 + ((imask
>> 6) & 3);
7340 v1
= values
[ins
->sreg1
];
7341 v2
= LLVMGetUndef (LLVMTypeOf (v1
));
7345 mask
[0] = ((imask
>> 0) & 3);
7346 mask
[1] = ((imask
>> 2) & 3);
7347 mask
[2] = ((imask
>> 4) & 3);
7348 mask
[3] = ((imask
>> 6) & 3);
7349 v1
= values
[ins
->sreg1
];
7350 v2
= LLVMGetUndef (LLVMTypeOf (v1
));
7353 g_assert_not_reached ();
7355 for (i
= 0; i
< mask_size
; ++i
)
7356 mask_values
[i
] = LLVMConstInt (LLVMInt32Type (), mask
[i
], FALSE
);
7358 values
[ins
->dreg
] =
7359 LLVMBuildShuffleVector (builder
, v1
, v2
,
7360 LLVMConstVector (mask_values
, mask_size
), dname
);
7364 case OP_UNPACK_LOWB
:
7365 case OP_UNPACK_LOWW
:
7366 case OP_UNPACK_LOWD
:
7367 case OP_UNPACK_LOWQ
:
7368 case OP_UNPACK_LOWPS
:
7369 case OP_UNPACK_LOWPD
:
7370 case OP_UNPACK_HIGHB
:
7371 case OP_UNPACK_HIGHW
:
7372 case OP_UNPACK_HIGHD
:
7373 case OP_UNPACK_HIGHQ
:
7374 case OP_UNPACK_HIGHPS
:
7375 case OP_UNPACK_HIGHPD
: {
7377 LLVMValueRef mask_values
[16];
7378 int i
, mask_size
= 0;
7379 gboolean low
= FALSE
;
7381 switch (ins
->opcode
) {
7382 case OP_UNPACK_LOWB
:
7386 case OP_UNPACK_LOWW
:
7390 case OP_UNPACK_LOWD
:
7391 case OP_UNPACK_LOWPS
:
7395 case OP_UNPACK_LOWQ
:
7396 case OP_UNPACK_LOWPD
:
7400 case OP_UNPACK_HIGHB
:
7403 case OP_UNPACK_HIGHW
:
7406 case OP_UNPACK_HIGHD
:
7407 case OP_UNPACK_HIGHPS
:
7410 case OP_UNPACK_HIGHQ
:
7411 case OP_UNPACK_HIGHPD
:
7415 g_assert_not_reached ();
7419 for (i
= 0; i
< (mask_size
/ 2); ++i
) {
7421 mask
[(i
* 2) + 1] = mask_size
+ i
;
7424 for (i
= 0; i
< (mask_size
/ 2); ++i
) {
7425 mask
[(i
* 2)] = (mask_size
/ 2) + i
;
7426 mask
[(i
* 2) + 1] = mask_size
+ (mask_size
/ 2) + i
;
7430 for (i
= 0; i
< mask_size
; ++i
)
7431 mask_values
[i
] = LLVMConstInt (LLVMInt32Type (), mask
[i
], FALSE
);
7433 values
[ins
->dreg
] =
7434 LLVMBuildShuffleVector (builder
, values
[ins
->sreg1
], values
[ins
->sreg2
],
7435 LLVMConstVector (mask_values
, mask_size
), dname
);
7440 LLVMTypeRef t
= simd_op_to_llvm_type (ins
->opcode
);
7441 LLVMValueRef v
, val
;
7443 v
= LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7444 val
= LLVMConstNull (t
);
7445 val
= LLVMBuildInsertElement (builder
, val
, v
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7446 val
= LLVMBuildInsertElement (builder
, val
, v
, LLVMConstInt (LLVMInt32Type (), 1, FALSE
), dname
);
7448 values
[ins
->dreg
] = val
;
7452 case OP_DUPPS_HIGH
: {
7453 LLVMTypeRef t
= simd_op_to_llvm_type (ins
->opcode
);
7454 LLVMValueRef v1
, v2
, val
;
7457 if (ins
->opcode
== OP_DUPPS_LOW
) {
7458 v1
= LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7459 v2
= LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), 2, FALSE
), "");
7461 v1
= LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), 1, FALSE
), "");
7462 v2
= LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), 3, FALSE
), "");
7464 val
= LLVMConstNull (t
);
7465 val
= LLVMBuildInsertElement (builder
, val
, v1
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7466 val
= LLVMBuildInsertElement (builder
, val
, v1
, LLVMConstInt (LLVMInt32Type (), 1, FALSE
), "");
7467 val
= LLVMBuildInsertElement (builder
, val
, v2
, LLVMConstInt (LLVMInt32Type (), 2, FALSE
), "");
7468 val
= LLVMBuildInsertElement (builder
, val
, v2
, LLVMConstInt (LLVMInt32Type (), 3, FALSE
), "");
7470 values
[ins
->dreg
] = val
;
7475 LLVMValueRef args
[3];
7479 /* 0xf1 == multiply all 4 elements, add them together, and store the result to the lowest element */
7480 args
[2] = LLVMConstInt (LLVMInt8Type (), 0xf1, FALSE
);
7482 values
[ins
->dreg
] = call_intrins (ctx
, simd_ins_to_intrins (ins
->opcode
), args
, dname
);
7486 case OP_FCONV_TO_R8_X
: {
7487 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, LLVMConstNull (sse_r8_t
), lhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7491 case OP_FCONV_TO_R4_X
: {
7492 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, LLVMConstNull (sse_r4_t
), lhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7496 #if defined(TARGET_X86) || defined(TARGET_AMD64)
7497 case OP_SSE_MOVMSK
: {
7498 LLVMValueRef args
[1];
7499 if (ins
->inst_c1
== MONO_TYPE_R4
) {
7501 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_MOVMSK_PS
, args
, dname
);
7502 } else if (ins
->inst_c1
== MONO_TYPE_R8
) {
7504 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_MOVMSK_PD
, args
, dname
);
7506 args
[0] = convert (ctx
, lhs
, sse_i1_t
);
7507 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_PMOVMSKB
, args
, dname
);
7513 case OP_SSE_MOVS2
: {
7514 if (ins
->inst_c1
== MONO_TYPE_R4
)
7515 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, rhs
, lhs
, create_const_vector_4_i32 (0, 5, 6, 7), "");
7516 else if (ins
->inst_c1
== MONO_TYPE_R8
)
7517 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, rhs
, lhs
, create_const_vector_2_i32 (0, 3), "");
7518 else if (ins
->inst_c1
== MONO_TYPE_I8
|| ins
->inst_c1
== MONO_TYPE_U8
)
7519 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, lhs
,
7520 LLVMConstInt (LLVMInt64Type (), 0, FALSE
),
7521 LLVMConstInt (LLVMInt32Type (), 1, FALSE
), "");
7523 g_assert_not_reached (); // will be needed for other types later
7527 case OP_SSE_MOVEHL
: {
7528 if (ins
->inst_c1
== MONO_TYPE_R4
)
7529 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, rhs
, create_const_vector_4_i32 (6, 7, 2, 3), "");
7531 g_assert_not_reached ();
7535 case OP_SSE_MOVELH
: {
7536 if (ins
->inst_c1
== MONO_TYPE_R4
)
7537 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, rhs
, create_const_vector_4_i32 (0, 1, 4, 5), "");
7539 g_assert_not_reached ();
7543 case OP_SSE_UNPACKLO
: {
7544 if (ins
->inst_c1
== MONO_TYPE_R8
|| ins
->inst_c1
== MONO_TYPE_I8
|| ins
->inst_c1
== MONO_TYPE_U8
) {
7545 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, rhs
, create_const_vector_2_i32 (0, 2), "");
7546 } else if (ins
->inst_c1
== MONO_TYPE_R4
|| ins
->inst_c1
== MONO_TYPE_I4
|| ins
->inst_c1
== MONO_TYPE_U4
) {
7547 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, rhs
, create_const_vector_4_i32 (0, 4, 1, 5), "");
7548 } else if (ins
->inst_c1
== MONO_TYPE_I2
|| ins
->inst_c1
== MONO_TYPE_U2
) {
7549 const int mask_values
[] = { 0, 8, 1, 9, 2, 10, 3, 11 };
7550 LLVMValueRef shuffled
= LLVMBuildShuffleVector (builder
,
7551 convert (ctx
, lhs
, sse_i2_t
),
7552 convert (ctx
, rhs
, sse_i2_t
),
7553 create_const_vector_i32 (mask_values
, 8), "");
7554 values
[ins
->dreg
] = convert (ctx
, shuffled
, type_to_sse_type (ins
->inst_c1
));
7555 } else if (ins
->inst_c1
== MONO_TYPE_I1
|| ins
->inst_c1
== MONO_TYPE_U1
) {
7556 const int mask_values
[] = { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 };
7557 LLVMValueRef shuffled
= LLVMBuildShuffleVector (builder
,
7558 convert (ctx
, lhs
, sse_i1_t
),
7559 convert (ctx
, rhs
, sse_i1_t
),
7560 create_const_vector_i32 (mask_values
, 16), "");
7561 values
[ins
->dreg
] = convert (ctx
, shuffled
, type_to_sse_type (ins
->inst_c1
));
7563 g_assert_not_reached ();
7568 case OP_SSE_UNPACKHI
: {
7569 if (ins
->inst_c1
== MONO_TYPE_R8
|| ins
->inst_c1
== MONO_TYPE_I8
|| ins
->inst_c1
== MONO_TYPE_U8
) {
7570 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, rhs
, create_const_vector_2_i32 (1, 3), "");
7571 } else if (ins
->inst_c1
== MONO_TYPE_R4
|| ins
->inst_c1
== MONO_TYPE_I4
|| ins
->inst_c1
== MONO_TYPE_U4
) {
7572 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, rhs
, create_const_vector_4_i32 (2, 6, 3, 7), "");
7573 } else if (ins
->inst_c1
== MONO_TYPE_I2
|| ins
->inst_c1
== MONO_TYPE_U2
) {
7574 const int mask_values
[] = { 4, 12, 5, 13, 6, 14, 7, 15 };
7575 LLVMValueRef shuffled
= LLVMBuildShuffleVector (builder
,
7576 convert (ctx
, lhs
, sse_i2_t
),
7577 convert (ctx
, rhs
, sse_i2_t
),
7578 create_const_vector_i32 (mask_values
, 8), "");
7579 values
[ins
->dreg
] = convert (ctx
, shuffled
, type_to_sse_type (ins
->inst_c1
));
7580 } else if (ins
->inst_c1
== MONO_TYPE_I1
|| ins
->inst_c1
== MONO_TYPE_U1
) {
7581 const int mask_values
[] = { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 };
7582 LLVMValueRef shuffled
= LLVMBuildShuffleVector (builder
,
7583 convert (ctx
, lhs
, sse_i1_t
),
7584 convert (ctx
, rhs
, sse_i1_t
),
7585 create_const_vector_i32 (mask_values
, 16), "");
7586 values
[ins
->dreg
] = convert (ctx
, shuffled
, type_to_sse_type (ins
->inst_c1
));
7588 g_assert_not_reached ();
7593 case OP_SSE_LOADU
: {
7594 LLVMValueRef dst_ptr
= convert (ctx
, lhs
, LLVMPointerType (primitive_type_to_llvm_type (inst_c1_type (ins
)), 0));
7595 LLVMValueRef dst_vec
= LLVMBuildBitCast (builder
, dst_ptr
, LLVMPointerType (type_to_sse_type (ins
->inst_c1
), 0), "");
7596 values
[ins
->dreg
] = mono_llvm_build_aligned_load (builder
, dst_vec
, "", FALSE
, ins
->inst_c0
); // inst_c0 is alignment
7599 case OP_SSE_MOVSS
: {
7600 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (LLVMFloatType (), 0));
7601 LLVMValueRef val
= mono_llvm_build_load (builder
, addr
, "", FALSE
);
7602 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, LLVMConstNull (type_to_sse_type (ins
->inst_c1
)), val
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7605 case OP_SSE_MOVSS_STORE
: {
7606 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (LLVMFloatType (), 0));
7607 LLVMValueRef val
= LLVMBuildExtractElement (builder
, rhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7608 mono_llvm_build_store (builder
, val
, addr
, FALSE
, LLVM_BARRIER_NONE
);
7613 case OP_SSE2_MOVUPD
: {
7614 LLVMTypeRef rty
= NULL
;
7615 switch (ins
->opcode
) {
7616 case OP_SSE2_MOVD
: rty
= sse_i4_t
; break;
7617 case OP_SSE2_MOVQ
: rty
= sse_i8_t
; break;
7618 case OP_SSE2_MOVUPD
: rty
= sse_r8_t
; break;
7620 LLVMTypeRef srcty
= LLVMGetElementType (rty
);
7621 LLVMValueRef zero
= LLVMConstNull (rty
);
7622 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (srcty
, 0));
7623 LLVMValueRef val
= mono_llvm_build_aligned_load (builder
, addr
, "", FALSE
, 1);
7624 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, zero
, val
, const_int32 (0), dname
);
7628 case OP_SSE_MOVLPS_LOAD
:
7629 case OP_SSE_MOVHPS_LOAD
: {
7630 LLVMTypeRef t
= LLVMFloatType ();
7632 gboolean high
= ins
->opcode
== OP_SSE_MOVHPS_LOAD
;
7633 /* Load two floats from rhs and store them in the low/high part of lhs */
7634 LLVMValueRef addr
= rhs
;
7635 LLVMValueRef addr1
= convert (ctx
, addr
, LLVMPointerType (t
, 0));
7636 LLVMValueRef addr2
= convert (ctx
, LLVMBuildAdd (builder
, convert (ctx
, addr
, IntPtrType ()), convert (ctx
, LLVMConstInt (LLVMInt32Type (), size
, FALSE
), IntPtrType ()), ""), LLVMPointerType (t
, 0));
7637 LLVMValueRef val1
= mono_llvm_build_load (builder
, addr1
, "", FALSE
);
7638 LLVMValueRef val2
= mono_llvm_build_load (builder
, addr2
, "", FALSE
);
7641 index1
= high
? 2: 0;
7642 index2
= high
? 3 : 1;
7643 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, LLVMBuildInsertElement (builder
, lhs
, val1
, LLVMConstInt (LLVMInt32Type (), index1
, FALSE
), ""), val2
, LLVMConstInt (LLVMInt32Type (), index2
, FALSE
), "");
7647 case OP_SSE2_MOVLPD_LOAD
:
7648 case OP_SSE2_MOVHPD_LOAD
: {
7649 LLVMTypeRef t
= LLVMDoubleType ();
7650 LLVMValueRef addr
= convert (ctx
, rhs
, LLVMPointerType (t
, 0));
7651 LLVMValueRef val
= mono_llvm_build_load (builder
, addr
, "", FALSE
);
7652 int index
= ins
->opcode
== OP_SSE2_MOVHPD_LOAD
? 1 : 0;
7653 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, lhs
, val
, const_int32 (index
), "");
7657 case OP_SSE_MOVLPS_STORE
:
7658 case OP_SSE_MOVHPS_STORE
: {
7659 /* Store two floats from the low/hight part of rhs into lhs */
7660 LLVMValueRef addr
= lhs
;
7661 LLVMValueRef addr1
= convert (ctx
, addr
, LLVMPointerType (LLVMFloatType (), 0));
7662 LLVMValueRef addr2
= convert (ctx
, LLVMBuildAdd (builder
, convert (ctx
, addr
, IntPtrType ()), convert (ctx
, LLVMConstInt (LLVMInt32Type (), 4, FALSE
), IntPtrType ()), ""), LLVMPointerType (LLVMFloatType (), 0));
7663 int index1
= ins
->opcode
== OP_SSE_MOVLPS_STORE
? 0 : 2;
7664 int index2
= ins
->opcode
== OP_SSE_MOVLPS_STORE
? 1 : 3;
7665 LLVMValueRef val1
= LLVMBuildExtractElement (builder
, rhs
, LLVMConstInt (LLVMInt32Type (), index1
, FALSE
), "");
7666 LLVMValueRef val2
= LLVMBuildExtractElement (builder
, rhs
, LLVMConstInt (LLVMInt32Type (), index2
, FALSE
), "");
7667 mono_llvm_build_store (builder
, val1
, addr1
, FALSE
, LLVM_BARRIER_NONE
);
7668 mono_llvm_build_store (builder
, val2
, addr2
, FALSE
, LLVM_BARRIER_NONE
);
7672 case OP_SSE2_MOVLPD_STORE
:
7673 case OP_SSE2_MOVHPD_STORE
: {
7674 LLVMTypeRef t
= LLVMDoubleType ();
7675 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (t
, 0));
7676 int index
= ins
->opcode
== OP_SSE2_MOVHPD_STORE
? 1 : 0;
7677 LLVMValueRef val
= LLVMBuildExtractElement (builder
, rhs
, const_int32 (index
), "");
7678 mono_llvm_build_store (builder
, val
, addr
, FALSE
, LLVM_BARRIER_NONE
);
7682 case OP_SSE_STORE
: {
7683 LLVMValueRef dst_vec
= convert (ctx
, lhs
, LLVMPointerType (LLVMTypeOf (rhs
), 0));
7684 mono_llvm_build_aligned_store (builder
, rhs
, dst_vec
, FALSE
, ins
->inst_c0
);
7688 case OP_SSE_STORES
: {
7689 LLVMValueRef first_elem
= LLVMBuildExtractElement (builder
, rhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7690 LLVMValueRef dst
= convert (ctx
, lhs
, LLVMPointerType (LLVMTypeOf (first_elem
), 0));
7691 mono_llvm_build_aligned_store (builder
, first_elem
, dst
, FALSE
, 1);
7694 case OP_SSE_MOVNTPS
: {
7695 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (LLVMTypeOf (rhs
), 0));
7696 LLVMValueRef store
= mono_llvm_build_aligned_store (builder
, rhs
, addr
, FALSE
, ins
->inst_c0
);
7697 set_nontemporal_flag (store
);
7700 case OP_SSE_PREFETCHT0
: {
7701 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (LLVMInt8Type (), 0));
7702 LLVMValueRef args
[] = { addr
, const_int32 (0), const_int32 (3), const_int32 (1) };
7703 call_intrins (ctx
, INTRINS_PREFETCH
, args
, "");
7706 case OP_SSE_PREFETCHT1
: {
7707 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (LLVMInt8Type (), 0));
7708 LLVMValueRef args
[] = { addr
, const_int32 (0), const_int32 (2), const_int32 (1) };
7709 call_intrins (ctx
, INTRINS_PREFETCH
, args
, "");
7712 case OP_SSE_PREFETCHT2
: {
7713 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (LLVMInt8Type (), 0));
7714 LLVMValueRef args
[] = { addr
, const_int32 (0), const_int32 (1), const_int32 (1) };
7715 call_intrins (ctx
, INTRINS_PREFETCH
, args
, "");
7718 case OP_SSE_PREFETCHNTA
: {
7719 LLVMValueRef addr
= convert (ctx
, lhs
, LLVMPointerType (LLVMInt8Type (), 0));
7720 LLVMValueRef args
[] = { addr
, const_int32 (0), const_int32 (0), const_int32 (1) };
7721 call_intrins (ctx
, INTRINS_PREFETCH
, args
, "");
7725 case OP_SSE_SHUFFLE
: {
7726 LLVMValueRef shuffle_vec
= create_const_vector_4_i32 (
7727 ((ins
->inst_c0
>> 0) & 0x3) + 0, // take two elements from lhs
7728 ((ins
->inst_c0
>> 2) & 0x3) + 0,
7729 ((ins
->inst_c0
>> 4) & 0x3) + 4, // and two from rhs
7730 ((ins
->inst_c0
>> 6) & 0x3) + 4);
7731 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, rhs
, shuffle_vec
, "");
7735 case OP_SSE2_SHUFFLE
: {
7736 LLVMValueRef right_vec
;
7737 LLVMValueRef shuffle_vec
;
7738 if (ins
->inst_c1
== MONO_TYPE_R8
) {
7740 shuffle_vec
= create_const_vector_2_i32 (
7741 ((ins
->inst_c0
>> 0) & 0x1) + 0,
7742 ((ins
->inst_c0
>> 1) & 0x1) + 2);
7744 right_vec
= LLVMGetUndef (LLVMVectorType (LLVMInt32Type (), 4));
7745 shuffle_vec
= create_const_vector_4_i32 (
7746 (ins
->inst_c0
>> 0) & 0x3,
7747 (ins
->inst_c0
>> 2) & 0x3,
7748 (ins
->inst_c0
>> 4) & 0x3,
7749 (ins
->inst_c0
>> 6) & 0x3);
7751 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, right_vec
, shuffle_vec
, "");
7756 LLVMValueRef vec_lhs_i64
= convert (ctx
, lhs
, sse_i8_t
);
7757 LLVMValueRef vec_rhs_i64
= convert (ctx
, rhs
, sse_i8_t
);
7758 LLVMValueRef vec_and
= LLVMBuildOr (builder
, vec_lhs_i64
, vec_rhs_i64
, "");
7759 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, vec_and
, type_to_sse_type (ins
->inst_c1
), "");
7764 LLVMValueRef vec_lhs_i64
= convert (ctx
, lhs
, sse_i8_t
);
7765 LLVMValueRef vec_rhs_i64
= convert (ctx
, rhs
, sse_i8_t
);
7766 LLVMValueRef vec_and
= LLVMBuildXor (builder
, vec_lhs_i64
, vec_rhs_i64
, "");
7767 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, vec_and
, type_to_sse_type (ins
->inst_c1
), "");
7772 LLVMValueRef vec_lhs_i64
= convert (ctx
, lhs
, sse_i8_t
);
7773 LLVMValueRef vec_rhs_i64
= convert (ctx
, rhs
, sse_i8_t
);
7774 LLVMValueRef vec_and
= LLVMBuildAnd (builder
, vec_lhs_i64
, vec_rhs_i64
, "");
7775 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, vec_and
, type_to_sse_type (ins
->inst_c1
), "");
7780 LLVMValueRef minus_one
[2];
7781 minus_one
[0] = LLVMConstInt (LLVMInt64Type (), -1, FALSE
);
7782 minus_one
[1] = LLVMConstInt (LLVMInt64Type (), -1, FALSE
);
7783 LLVMValueRef vec_lhs_i64
= convert (ctx
, lhs
, sse_i8_t
);
7784 LLVMValueRef vec_xor
= LLVMBuildXor (builder
, vec_lhs_i64
, LLVMConstVector (minus_one
, 2), "");
7785 LLVMValueRef vec_rhs_i64
= convert (ctx
, rhs
, sse_i8_t
);
7786 LLVMValueRef vec_and
= LLVMBuildAnd (builder
, vec_rhs_i64
, vec_xor
, "");
7787 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, vec_and
, type_to_sse_type (ins
->inst_c1
), "");
7798 case OP_SSE2_MULSD
: {
7799 LLVMValueRef v1
= LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7800 LLVMValueRef v2
= LLVMBuildExtractElement (builder
, rhs
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7802 LLVMValueRef v
= NULL
;
7803 switch (ins
->opcode
) {
7806 v
= LLVMBuildFAdd (builder
, v1
, v2
, "");
7810 v
= LLVMBuildFSub (builder
, v1
, v2
, "");
7814 v
= LLVMBuildFDiv (builder
, v1
, v2
, "");
7818 v
= LLVMBuildFMul (builder
, v1
, v2
, "");
7821 g_assert_not_reached ();
7823 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, lhs
, v
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
7828 case OP_SSE2_CMPSD
: {
7830 switch (ins
->inst_c0
) {
7831 case CMP_EQ
: imm
= 0; break;
7832 case CMP_GT
: imm
= 6; break;
7833 case CMP_GE
: imm
= 5; break;
7834 case CMP_LT
: imm
= 1; break;
7835 case CMP_LE
: imm
= 2; break;
7836 case CMP_NE
: imm
= 4; break;
7837 case CMP_ORD
: imm
= 7; break;
7838 case CMP_UNORD
: imm
= 3; break;
7839 default: g_assert_not_reached (); break;
7841 LLVMValueRef cmp
= LLVMConstInt (LLVMInt8Type (), imm
, FALSE
);
7842 LLVMValueRef args
[] = { lhs
, rhs
, cmp
};
7843 switch (ins
->opcode
) {
7845 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_CMPSS
, args
, "");
7848 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_CMPSD
, args
, "");
7851 g_assert_not_reached ();
7856 case OP_SSE_COMISS
: {
7857 LLVMValueRef args
[] = { lhs
, rhs
};
7858 IntrinsicId id
= (IntrinsicId
)0;
7859 switch (ins
->inst_c0
) {
7860 case CMP_EQ
: id
= INTRINS_SSE_COMIEQ_SS
; break;
7861 case CMP_GT
: id
= INTRINS_SSE_COMIGT_SS
; break;
7862 case CMP_GE
: id
= INTRINS_SSE_COMIGE_SS
; break;
7863 case CMP_LT
: id
= INTRINS_SSE_COMILT_SS
; break;
7864 case CMP_LE
: id
= INTRINS_SSE_COMILE_SS
; break;
7865 case CMP_NE
: id
= INTRINS_SSE_COMINEQ_SS
; break;
7866 default: g_assert_not_reached (); break;
7868 values
[ins
->dreg
] = call_intrins (ctx
, id
, args
, "");
7871 case OP_SSE_UCOMISS
: {
7872 LLVMValueRef args
[] = { lhs
, rhs
};
7873 IntrinsicId id
= (IntrinsicId
)0;
7874 switch (ins
->inst_c0
) {
7875 case CMP_EQ
: id
= INTRINS_SSE_UCOMIEQ_SS
; break;
7876 case CMP_GT
: id
= INTRINS_SSE_UCOMIGT_SS
; break;
7877 case CMP_GE
: id
= INTRINS_SSE_UCOMIGE_SS
; break;
7878 case CMP_LT
: id
= INTRINS_SSE_UCOMILT_SS
; break;
7879 case CMP_LE
: id
= INTRINS_SSE_UCOMILE_SS
; break;
7880 case CMP_NE
: id
= INTRINS_SSE_UCOMINEQ_SS
; break;
7881 default: g_assert_not_reached (); break;
7883 values
[ins
->dreg
] = call_intrins (ctx
, id
, args
, "");
7886 case OP_SSE2_COMISD
: {
7887 LLVMValueRef args
[] = { lhs
, rhs
};
7888 IntrinsicId id
= (IntrinsicId
)0;
7889 switch (ins
->inst_c0
) {
7890 case CMP_EQ
: id
= INTRINS_SSE_COMIEQ_SD
; break;
7891 case CMP_GT
: id
= INTRINS_SSE_COMIGT_SD
; break;
7892 case CMP_GE
: id
= INTRINS_SSE_COMIGE_SD
; break;
7893 case CMP_LT
: id
= INTRINS_SSE_COMILT_SD
; break;
7894 case CMP_LE
: id
= INTRINS_SSE_COMILE_SD
; break;
7895 case CMP_NE
: id
= INTRINS_SSE_COMINEQ_SD
; break;
7896 default: g_assert_not_reached (); break;
7898 values
[ins
->dreg
] = call_intrins (ctx
, id
, args
, "");
7901 case OP_SSE2_UCOMISD
: {
7902 LLVMValueRef args
[] = { lhs
, rhs
};
7903 IntrinsicId id
= (IntrinsicId
)0;
7904 switch (ins
->inst_c0
) {
7905 case CMP_EQ
: id
= INTRINS_SSE_UCOMIEQ_SD
; break;
7906 case CMP_GT
: id
= INTRINS_SSE_UCOMIGT_SD
; break;
7907 case CMP_GE
: id
= INTRINS_SSE_UCOMIGE_SD
; break;
7908 case CMP_LT
: id
= INTRINS_SSE_UCOMILT_SD
; break;
7909 case CMP_LE
: id
= INTRINS_SSE_UCOMILE_SD
; break;
7910 case CMP_NE
: id
= INTRINS_SSE_UCOMINEQ_SD
; break;
7911 default: g_assert_not_reached (); break;
7913 values
[ins
->dreg
] = call_intrins (ctx
, id
, args
, "");
7916 case OP_SSE_CVTSI2SS
:
7917 case OP_SSE_CVTSI2SS64
:
7918 case OP_SSE2_CVTSI2SD
:
7919 case OP_SSE2_CVTSI2SD64
: {
7920 LLVMTypeRef ty
= LLVMFloatType ();
7921 switch (ins
->opcode
) {
7922 case OP_SSE2_CVTSI2SD
:
7923 case OP_SSE2_CVTSI2SD64
:
7924 ty
= LLVMDoubleType ();
7927 LLVMValueRef fp
= LLVMBuildSIToFP (builder
, rhs
, ty
, "");
7928 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, lhs
, fp
, const_int32 (0), dname
);
7931 case OP_SSE2_PMULUDQ
: {
7932 #if LLVM_API_VERSION < 700
7933 LLVMValueRef args
[] = { lhs
, rhs
};
7934 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_PMULUDQ
, args
, dname
);
7936 LLVMValueRef i32_max
= LLVMConstInt (LLVMInt64Type (), UINT32_MAX
, FALSE
);
7937 LLVMValueRef maskvals
[] = { i32_max
, i32_max
};
7938 LLVMValueRef mask
= LLVMConstVector (maskvals
, 2);
7939 LLVMValueRef l
= LLVMBuildAnd (builder
, convert (ctx
, lhs
, sse_i8_t
), mask
, "");
7940 LLVMValueRef r
= LLVMBuildAnd (builder
, convert (ctx
, rhs
, sse_i8_t
), mask
, "");
7941 values
[ins
->dreg
] = LLVMBuildNUWMul (builder
, l
, r
, dname
);
7946 case OP_SSE2_SQRTSD
: {
7947 #if LLVM_API_VERSION < 700
7948 LLVMValueRef result
= call_intrins (ctx
, simd_ins_to_intrins (ins
->opcode
), &rhs
, dname
);
7949 const int maskf32
[] = { 0, 5, 6, 7 };
7950 const int maskf64
[] = { 0, 1 };
7951 const int *mask
= NULL
;
7953 switch (ins
->opcode
) {
7954 case OP_SSE_SQRTSS
: mask
= maskf32
; mask_len
= 4; break;
7955 case OP_SSE2_SQRTSD
: mask
= maskf64
; mask_len
= 2; break;
7956 default: g_assert_not_reached (); break;
7958 LLVMValueRef shufmask
= create_const_vector_i32 (mask
, mask_len
);
7959 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, result
, lhs
, shufmask
, "");
7961 LLVMValueRef upper
= values
[ins
->sreg1
];
7962 LLVMValueRef lower
= values
[ins
->sreg2
];
7963 LLVMValueRef scalar
= LLVMBuildExtractElement (builder
, lower
, const_int32 (0), "");
7964 LLVMValueRef result
= call_intrins (ctx
, simd_ins_to_intrins (ins
->opcode
), &scalar
, dname
);
7965 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, upper
, result
, const_int32 (0), "");
7970 case OP_SSE_RSQRTSS
: {
7971 IntrinsicId id
= (IntrinsicId
)0;
7972 switch (ins
->opcode
) {
7973 case OP_SSE_RCPSS
: id
= INTRINS_SSE_RCP_SS
; break;
7974 case OP_SSE_RSQRTSS
: id
= INTRINS_SSE_RSQRT_SS
; break;
7975 default: g_assert_not_reached (); break;
7977 LLVMValueRef result
= call_intrins (ctx
, id
, &rhs
, dname
);
7978 const int mask
[] = { 0, 5, 6, 7 };
7979 LLVMValueRef shufmask
= create_const_vector_i32 (mask
, 4);
7980 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, result
, lhs
, shufmask
, "");
7984 IntrinsicId id
= (IntrinsicId
)0;
7985 switch (ins
->inst_c0
) {
7986 case SIMD_OP_SSE_LFENCE
: id
= INTRINS_SSE_LFENCE
; break;
7987 case SIMD_OP_SSE_SFENCE
: id
= INTRINS_SSE_SFENCE
; break;
7988 case SIMD_OP_SSE_MFENCE
: id
= INTRINS_SSE_MFENCE
; break;
7989 default: g_assert_not_reached (); break;
7991 call_intrins (ctx
, id
, NULL
, "");
7996 IntrinsicId id
= (IntrinsicId
)0;
7997 switch (ins
->inst_c0
) {
7998 case SIMD_OP_SSE_SQRTPS
: id
= INTRINS_SSE_SQRT_PS
; break;
7999 case SIMD_OP_SSE_RCPPS
: id
= INTRINS_SSE_RCP_PS
; break;
8000 case SIMD_OP_SSE_RSQRTPS
: id
= INTRINS_SSE_RSQRT_PS
; break;
8001 case SIMD_OP_SSE_SQRTPD
: id
= INTRINS_SSE_SQRT_PD
; break;
8002 case SIMD_OP_SSE_LDDQU
: id
= INTRINS_SSE_LDU_DQ
; break;
8003 default: g_assert_not_reached (); break;
8005 values
[ins
->dreg
] = call_intrins (ctx
, id
, &lhs
, "");
8010 IntrinsicId id
= (IntrinsicId
)0;
8011 switch (ins
->inst_c0
) {
8012 case SIMD_OP_SSE_CVTSS2SI
: id
= INTRINS_SSE_CVTSS2SI
; break;
8013 case SIMD_OP_SSE_CVTTSS2SI
: id
= INTRINS_SSE_CVTTSS2SI
; break;
8014 case SIMD_OP_SSE_CVTSS2SI64
: id
= INTRINS_SSE_CVTSS2SI64
; break;
8015 case SIMD_OP_SSE_CVTTSS2SI64
: id
= INTRINS_SSE_CVTTSS2SI64
; break;
8016 case SIMD_OP_SSE_CVTSD2SI
: id
= INTRINS_SSE_CVTSD2SI
; break;
8017 case SIMD_OP_SSE_CVTTSD2SI
: id
= INTRINS_SSE_CVTTSD2SI
; break;
8018 case SIMD_OP_SSE_CVTSD2SI64
: id
= INTRINS_SSE_CVTSD2SI64
; break;
8019 case SIMD_OP_SSE_CVTTSD2SI64
: id
= INTRINS_SSE_CVTTSD2SI64
; break;
8020 default: g_assert_not_reached (); break;
8022 values
[ins
->dreg
] = call_intrins (ctx
, id
, &lhs
, "");
8027 case OP_XOP_X_X_I8
: {
8028 LLVMValueRef args
[] = { lhs
, rhs
};
8029 IntrinsicId id
= (IntrinsicId
)0;
8030 switch (ins
->inst_c0
) {
8031 case SIMD_OP_SSE_CVTSD2SS
: id
= INTRINS_SSE_CVTSD2SS
; break;
8032 case SIMD_OP_SSE_MAXPS
: id
= INTRINS_SSE_MAXPS
; break;
8033 case SIMD_OP_SSE_MAXSS
: id
= INTRINS_SSE_MAXSS
; break;
8034 case SIMD_OP_SSE_MINPS
: id
= INTRINS_SSE_MINPS
; break;
8035 case SIMD_OP_SSE_MINSS
: id
= INTRINS_SSE_MINSS
; break;
8036 case SIMD_OP_SSE_MAXPD
: id
= INTRINS_SSE_MAXPD
; break;
8037 case SIMD_OP_SSE_MAXSD
: id
= INTRINS_SSE_MAXSD
; break;
8038 case SIMD_OP_SSE_MINPD
: id
= INTRINS_SSE_MINPD
; break;
8039 case SIMD_OP_SSE_MINSD
: id
= INTRINS_SSE_MINSD
; break;
8040 case SIMD_OP_SSE_PMADDWD
: id
= INTRINS_SSE_PMADDWD
; break;
8041 case SIMD_OP_SSE_PMULHW
: id
= INTRINS_SSE_PMULHW
; break;
8042 case SIMD_OP_SSE_PMULHUW
: id
= INTRINS_SSE_PMULHUW
; break;
8043 case SIMD_OP_SSE_PACKSSWB
: id
= INTRINS_SSE_PACKSSWB
; break;
8044 case SIMD_OP_SSE_PACKSSDW
: id
= INTRINS_SSE_PACKSSDW
; break;
8045 case SIMD_OP_SSE_PSRLW_IMM
: id
= INTRINS_SSE_PSRLI_W
; break;
8046 case SIMD_OP_SSE_PSRLD_IMM
: id
= INTRINS_SSE_PSRLI_D
; break;
8047 case SIMD_OP_SSE_PSRLQ_IMM
: id
= INTRINS_SSE_PSRLI_Q
; break;
8048 case SIMD_OP_SSE_PSRLW
: id
= INTRINS_SSE_PSRL_W
; break;
8049 case SIMD_OP_SSE_PSRLD
: id
= INTRINS_SSE_PSRL_D
; break;
8050 case SIMD_OP_SSE_PSRLQ
: id
= INTRINS_SSE_PSRL_Q
; break;
8051 case SIMD_OP_SSE_PSLLW_IMM
: id
= INTRINS_SSE_PSLLI_W
; break;
8052 case SIMD_OP_SSE_PSLLD_IMM
: id
= INTRINS_SSE_PSLLI_D
; break;
8053 case SIMD_OP_SSE_PSLLQ_IMM
: id
= INTRINS_SSE_PSLLI_Q
; break;
8054 case SIMD_OP_SSE_PSLLW
: id
= INTRINS_SSE_PSLL_W
; break;
8055 case SIMD_OP_SSE_PSLLD
: id
= INTRINS_SSE_PSLL_D
; break;
8056 case SIMD_OP_SSE_PSLLQ
: id
= INTRINS_SSE_PSLL_Q
; break;
8057 case SIMD_OP_SSE_PSRAW_IMM
: id
= INTRINS_SSE_PSRAI_W
; break;
8058 case SIMD_OP_SSE_PSRAD_IMM
: id
= INTRINS_SSE_PSRAI_D
; break;
8059 case SIMD_OP_SSE_PSRAW
: id
= INTRINS_SSE_PSRA_W
; break;
8060 case SIMD_OP_SSE_PSRAD
: id
= INTRINS_SSE_PSRA_D
; break;
8061 case SIMD_OP_SSE_PSADBW
: id
= INTRINS_SSE_PSADBW
; break;
8062 case SIMD_OP_SSE_ADDSUBPS
: id
= INTRINS_SSE_ADDSUBPS
; break;
8063 case SIMD_OP_SSE_ADDSUBPD
: id
= INTRINS_SSE_ADDSUBPD
; break;
8064 case SIMD_OP_SSE_HADDPS
: id
= INTRINS_SSE_HADDPS
; break;
8065 case SIMD_OP_SSE_HADDPD
: id
= INTRINS_SSE_HADDPD
; break;
8066 case SIMD_OP_SSE_PHADDW
: id
= INTRINS_SSE_PHADDW
; break;
8067 case SIMD_OP_SSE_PHADDD
: id
= INTRINS_SSE_PHADDD
; break;
8068 case SIMD_OP_SSE_PHSUBW
: id
= INTRINS_SSE_PHSUBW
; break;
8069 case SIMD_OP_SSE_PHSUBD
: id
= INTRINS_SSE_PHSUBD
; break;
8070 case SIMD_OP_SSE_HSUBPS
: id
= INTRINS_SSE_HSUBPS
; break;
8071 case SIMD_OP_SSE_HSUBPD
: id
= INTRINS_SSE_HSUBPD
; break;
8072 case SIMD_OP_SSE_PHADDSW
: id
= INTRINS_SSE_PHADDSW
; break;
8073 case SIMD_OP_SSE_PHSUBSW
: id
= INTRINS_SSE_PHSUBSW
; break;
8074 case SIMD_OP_SSE_PSIGNB
: id
= INTRINS_SSE_PSIGNB
; break;
8075 case SIMD_OP_SSE_PSIGNW
: id
= INTRINS_SSE_PSIGNW
; break;
8076 case SIMD_OP_SSE_PSIGND
: id
= INTRINS_SSE_PSIGND
; break;
8077 case SIMD_OP_SSE_PMADDUBSW
: id
= INTRINS_SSE_PMADDUBSW
; break;
8078 case SIMD_OP_SSE_PMULHRSW
: id
= INTRINS_SSE_PMULHRSW
; break;
8079 default: g_assert_not_reached (); break;
8081 values
[ins
->dreg
] = call_intrins (ctx
, id
, args
, "");
8085 case OP_SSE2_MASKMOVDQU
: {
8086 LLVMTypeRef i8ptr
= LLVMPointerType (LLVMInt8Type (), 0);
8087 LLVMValueRef dstaddr
= convert (ctx
, values
[ins
->sreg3
], i8ptr
);
8088 LLVMValueRef src
= convert (ctx
, lhs
, sse_i1_t
);
8089 LLVMValueRef mask
= convert (ctx
, rhs
, sse_i1_t
);
8090 LLVMValueRef args
[] = { src
, mask
, dstaddr
};
8091 call_intrins (ctx
, INTRINS_SSE_MASKMOVDQU
, args
, "");
8099 case OP_PADDB_SAT_UN
:
8100 case OP_PADDW_SAT_UN
:
8101 case OP_PSUBB_SAT_UN
:
8102 case OP_PSUBW_SAT_UN
:
8104 case OP_SSE2_SUBS
: {
8105 IntrinsicId id
= (IntrinsicId
)0;
8107 gboolean is_add
= TRUE
;
8108 switch (ins
->opcode
) {
8109 case OP_PADDB_SAT
: type
= MONO_TYPE_I1
; break;
8110 case OP_PADDW_SAT
: type
= MONO_TYPE_I2
; break;
8111 case OP_PSUBB_SAT
: type
= MONO_TYPE_I1
; is_add
= FALSE
; break;
8112 case OP_PSUBW_SAT
: type
= MONO_TYPE_I2
; is_add
= FALSE
; break;
8113 case OP_PADDB_SAT_UN
: type
= MONO_TYPE_U1
; break;
8114 case OP_PADDW_SAT_UN
: type
= MONO_TYPE_U2
; break;
8115 case OP_PSUBB_SAT_UN
: type
= MONO_TYPE_U1
; is_add
= FALSE
; break;
8116 case OP_PSUBW_SAT_UN
: type
= MONO_TYPE_U2
; is_add
= FALSE
; break;
8117 case OP_SSE2_ADDS
: type
= ins
->inst_c1
; break;
8118 case OP_SSE2_SUBS
: type
= ins
->inst_c1
; is_add
= FALSE
; break;
8119 default: g_assert_not_reached ();
8123 case MONO_TYPE_I1
: id
= INTRINS_SSE_SADD_SATI8
; break;
8124 case MONO_TYPE_U1
: id
= INTRINS_SSE_UADD_SATI8
; break;
8125 case MONO_TYPE_I2
: id
= INTRINS_SSE_SADD_SATI16
; break;
8126 case MONO_TYPE_U2
: id
= INTRINS_SSE_UADD_SATI16
; break;
8127 default: g_assert_not_reached (); break;
8131 case MONO_TYPE_I1
: id
= INTRINS_SSE_SSUB_SATI8
; break;
8132 case MONO_TYPE_U1
: id
= INTRINS_SSE_USUB_SATI8
; break;
8133 case MONO_TYPE_I2
: id
= INTRINS_SSE_SSUB_SATI16
; break;
8134 case MONO_TYPE_U2
: id
= INTRINS_SSE_USUB_SATI16
; break;
8135 default: g_assert_not_reached (); break;
8138 LLVMTypeRef vecty
= type_to_sse_type (type
);
8139 LLVMValueRef args
[] = { convert (ctx
, lhs
, vecty
), convert (ctx
, rhs
, vecty
) };
8140 LLVMValueRef result
= call_intrins (ctx
, id
, args
, dname
);
8141 values
[ins
->dreg
] = convert (ctx
, result
, vecty
);
8145 case OP_SSE2_PACKUS
: {
8146 LLVMValueRef args
[2];
8147 args
[0] = convert (ctx
, lhs
, sse_i2_t
);
8148 args
[1] = convert (ctx
, rhs
, sse_i2_t
);
8149 values
[ins
->dreg
] = convert (ctx
,
8150 call_intrins (ctx
, INTRINS_SSE_PACKUSWB
, args
, dname
),
8151 type_to_sse_type (ins
->inst_c1
));
8155 case OP_SSE2_SRLI
: {
8156 LLVMValueRef args
[] = { lhs
, rhs
};
8157 values
[ins
->dreg
] = convert (ctx
,
8158 call_intrins (ctx
, INTRINS_SSE_PSRLI_W
, args
, dname
),
8159 type_to_sse_type (ins
->inst_c1
));
8163 case OP_SSE2_PSLLDQ
:
8164 case OP_SSE2_PSRLDQ
: {
8165 LLVMBasicBlockRef bbs
[16 + 1];
8166 LLVMValueRef switch_ins
;
8167 LLVMValueRef value
= lhs
;
8168 LLVMValueRef index
= rhs
;
8169 LLVMValueRef phi_values
[16 + 1];
8170 LLVMTypeRef t
= sse_i1_t
;
8173 gboolean shift_right
= (ins
->opcode
== OP_SSE2_PSRLDQ
);
8175 value
= convert (ctx
, value
, t
);
8177 // No corresponding LLVM intrinsics
8178 // FIXME: Optimize const count
8179 for (i
= 0; i
< nelems
; ++i
)
8180 bbs
[i
] = gen_bb (ctx
, "PSLLDQ_CASE_BB");
8181 bbs
[nelems
] = gen_bb (ctx
, "PSLLDQ_DEF_BB");
8182 cbb
= gen_bb (ctx
, "PSLLDQ_COND_BB");
8184 switch_ins
= LLVMBuildSwitch (builder
, index
, bbs
[nelems
], 0);
8185 for (i
= 0; i
< nelems
; ++i
) {
8186 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), bbs
[i
]);
8187 LLVMPositionBuilderAtEnd (builder
, bbs
[i
]);
8189 int mask_values
[16];
8190 // Implement shift using a shuffle
8192 for (int j
= 0; j
< nelems
- i
; ++j
)
8193 mask_values
[j
] = i
+ j
;
8194 for (int j
= nelems
-i
; j
< nelems
; ++j
)
8195 mask_values
[j
] = nelems
;
8197 for (int j
= 0; j
< i
; ++j
)
8198 mask_values
[j
] = nelems
;
8199 for (int j
= 0; j
< nelems
- i
; ++j
)
8200 mask_values
[j
+ i
] = j
;
8202 phi_values
[i
] = LLVMBuildShuffleVector (builder
, value
, LLVMGetUndef (t
), create_const_vector_i32 (mask_values
, nelems
), "");
8203 LLVMBuildBr (builder
, cbb
);
8206 LLVMPositionBuilderAtEnd (builder
, bbs
[nelems
]);
8207 phi_values
[nelems
] = LLVMConstNull (t
);
8208 LLVMBuildBr (builder
, cbb
);
8210 LLVMPositionBuilderAtEnd (builder
, cbb
);
8211 values
[ins
->dreg
] = LLVMBuildPhi (builder
, LLVMTypeOf (phi_values
[0]), "");
8212 LLVMAddIncoming (values
[ins
->dreg
], phi_values
, bbs
, nelems
+ 1);
8213 values
[ins
->dreg
] = convert (ctx
, values
[ins
->dreg
], type_to_sse_type (ins
->inst_c1
));
8215 ctx
->bblocks
[bb
->block_num
].end_bblock
= cbb
;
8219 case OP_SSE2_PSRAW_IMM
:
8220 case OP_SSE2_PSRAD_IMM
:
8221 case OP_SSE2_PSRLW_IMM
:
8222 case OP_SSE2_PSRLD_IMM
:
8223 case OP_SSE2_PSRLQ_IMM
: {
8224 LLVMValueRef value
= lhs
;
8225 LLVMValueRef index
= rhs
;
8228 // FIXME: Optimize const index case
8230 /* Use the non-immediate version */
8231 switch (ins
->opcode
) {
8232 case OP_SSE2_PSRAW_IMM
: id
= INTRINS_SSE_PSRA_W
; break;
8233 case OP_SSE2_PSRAD_IMM
: id
= INTRINS_SSE_PSRA_D
; break;
8234 case OP_SSE2_PSRLW_IMM
: id
= INTRINS_SSE_PSRL_W
; break;
8235 case OP_SSE2_PSRLD_IMM
: id
= INTRINS_SSE_PSRL_D
; break;
8236 case OP_SSE2_PSRLQ_IMM
: id
= INTRINS_SSE_PSRL_Q
; break;
8237 default: g_assert_not_reached (); break;
8240 LLVMTypeRef t
= LLVMTypeOf (value
);
8241 LLVMValueRef index_vect
= LLVMBuildInsertElement (builder
, LLVMConstNull (t
), convert (ctx
, index
, LLVMGetElementType (t
)), const_int32 (0), "");
8242 LLVMValueRef args
[] = { value
, index_vect
};
8243 values
[ins
->dreg
] = call_intrins (ctx
, id
, args
, "");
8247 case OP_SSE2_SHUFPD
:
8248 case OP_SSE2_PSHUFD
:
8249 case OP_SSE2_PSHUFHW
:
8250 case OP_SSE2_PSHUFLW
: {
8251 LLVMBasicBlockRef bbs
[256 + 1];
8252 LLVMValueRef switch_ins
;
8253 LLVMValueRef v1
, v2
, mask
;
8254 LLVMValueRef phi_values
[256 + 1];
8257 // FIXME: Optimize constant shuffle mask
8259 if (ins
->opcode
== OP_SSE2_SHUFPD
) {
8260 /* 3 parameter version */
8263 mask
= values
[ins
->sreg3
];
8266 /* 2 parameter version */
8272 for (int i
= 0; i
< ncases
; ++i
)
8273 bbs
[i
] = gen_bb (ctx
, "PSHUFHW_CASE_BB");
8274 cbb
= gen_bb (ctx
, "PSHUFHW_COND_BB");
8275 /* No default case */
8276 switch_ins
= LLVMBuildSwitch (builder
, mask
, bbs
[0], 0);
8277 for (int i
= 0; i
< ncases
; ++i
) {
8278 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), bbs
[i
]);
8279 LLVMPositionBuilderAtEnd (builder
, bbs
[i
]);
8281 /* Convert the x86 shuffle mask to LLVM's */
8283 int mask_values
[8];
8285 switch (ins
->opcode
) {
8286 case OP_SSE2_SHUFPD
:
8287 /* Bit 0 selects v1[0] or v1[1], bit 1 selects v2[0] or v2[1] */
8289 mask_values
[0] = ((imask
>> 0) & 1);
8290 mask_values
[1] = ((imask
>> 1) & 1) + 2;
8292 case OP_SSE2_PSHUFD
:
8294 * Each 2 bits in mask selects 1 dword from the the source and copies it to the
8298 for (int j
= 0; j
< 4; ++j
) {
8299 int windex
= (imask
>> (j
* 2)) & 0x3;
8300 mask_values
[j
] = windex
;
8303 case OP_SSE2_PSHUFHW
:
8305 * Each 2 bits in mask selects 1 word from the high quadword of the source and copies it to the
8306 * high quadword of the destination.
8309 /* The low quadword stays the same */
8310 for (int j
= 0; j
< 4; ++j
)
8311 mask_values
[j
] = j
;
8312 for (int j
= 0; j
< 4; ++j
) {
8313 int windex
= (imask
>> (j
* 2)) & 0x3;
8314 mask_values
[j
+ 4] = 4 + windex
;
8317 case OP_SSE2_PSHUFLW
:
8319 /* The high quadword stays the same */
8320 for (int j
= 0; j
< 4; ++j
)
8321 mask_values
[j
+ 4] = j
+ 4;
8322 for (int j
= 0; j
< 4; ++j
) {
8323 int windex
= (imask
>> (j
* 2)) & 0x3;
8324 mask_values
[j
] = windex
;
8328 g_assert_not_reached ();
8331 phi_values
[i
] = LLVMBuildShuffleVector (builder
, v1
, v2
, create_const_vector_i32 (mask_values
, mask_len
), "");
8332 LLVMBuildBr (builder
, cbb
);
8335 LLVMPositionBuilderAtEnd (builder
, cbb
);
8336 values
[ins
->dreg
] = LLVMBuildPhi (builder
, LLVMTypeOf (phi_values
[0]), "");
8337 LLVMAddIncoming (values
[ins
->dreg
], phi_values
, bbs
, ncases
);
8341 case OP_SSE3_MOVDDUP
: {
8342 int mask
[] = { 0, 0 };
8343 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
,
8344 LLVMGetUndef (LLVMTypeOf (lhs
)),
8345 create_const_vector_i32 (mask
, 2), "");
8348 case OP_SSE3_MOVDDUP_MEM
: {
8349 int mask
[] = { 0, 0 };
8350 LLVMTypeRef t
= type_to_sse_type (ins
->inst_c1
);
8351 LLVMValueRef value
= mono_llvm_build_load (builder
, convert (ctx
, lhs
, LLVMPointerType (t
, 0)), "", FALSE
);
8352 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, value
, LLVMGetUndef (LLVMTypeOf (value
)), create_const_vector_i32 (mask
, 2), "");
8355 case OP_SSE3_MOVSHDUP
: {
8356 int mask
[] = { 1, 1, 3, 3 };
8357 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, LLVMConstNull (LLVMTypeOf (lhs
)), create_const_vector_i32 (mask
, 4), "");
8360 case OP_SSE3_MOVSLDUP
: {
8361 int mask
[] = { 0, 0, 2, 2 };
8362 values
[ins
->dreg
] = LLVMBuildShuffleVector (builder
, lhs
, LLVMConstNull (LLVMTypeOf (lhs
)), create_const_vector_i32 (mask
, 4), "");
8366 case OP_SSSE3_SHUFFLE
: {
8367 LLVMValueRef args
[] = { lhs
, rhs
};
8368 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_PSHUFB
, args
, dname
);
8372 case OP_SSSE3_ABS
: {
8373 // %sub = sub <16 x i8> zeroinitializer, %arg
8374 // %cmp = icmp sgt <16 x i8> %arg, zeroinitializer
8375 // %abs = select <16 x i1> %cmp, <16 x i8> %arg, <16 x i8> %sub
8376 LLVMTypeRef typ
= type_to_sse_type (ins
->inst_c1
);
8377 LLVMValueRef sub
= LLVMBuildSub(builder
, LLVMConstNull(typ
), lhs
, "");
8378 LLVMValueRef cmp
= LLVMBuildICmp(builder
, LLVMIntSGT
, lhs
, LLVMConstNull(typ
), "");
8379 LLVMValueRef abs
= LLVMBuildSelect (builder
, cmp
, lhs
, sub
, "");
8380 values
[ins
->dreg
] = convert (ctx
, abs
, typ
);
8384 case OP_SSSE3_ALIGNR
: {
8385 LLVMValueRef mask_values
[16];
8386 for (int i
= 0; i
< 16; i
++)
8387 mask_values
[i
] = LLVMConstInt (LLVMInt8Type (), i
+ ins
->inst_c0
, FALSE
);
8388 LLVMValueRef shuffled
= LLVMBuildShuffleVector (builder
,
8389 convert (ctx
, rhs
, sse_i1_t
),
8390 convert (ctx
, lhs
, sse_i1_t
),
8391 LLVMConstVector (mask_values
, 16), "");
8392 values
[ins
->dreg
] = convert (ctx
, shuffled
, type_to_sse_type (ins
->inst_c1
));
8396 case OP_CREATE_SCALAR
:
8397 case OP_CREATE_SCALAR_UNSAFE
: {
8398 MonoTypeEnum primty
= inst_c1_type (ins
);
8399 LLVMTypeRef type
= type_to_sse_type (primty
);
8400 // use undef vector (most likely empty but may contain garbage values) for OP_CREATE_SCALAR_UNSAFE
8401 // and zero one for OP_CREATE_SCALAR
8402 LLVMValueRef vector
= (ins
->opcode
== OP_CREATE_SCALAR
) ? LLVMConstNull (type
) : LLVMGetUndef (type
);
8403 LLVMValueRef insert_pos
= LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
8404 LLVMValueRef val
= convert_full (ctx
, lhs
, primitive_type_to_llvm_type (primty
), primitive_type_is_unsigned (primty
));
8405 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
, vector
, val
, insert_pos
, "");
8409 case OP_SSE41_ROUNDSS
: {
8410 LLVMValueRef args
[3];
8414 args
[2] = LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
);
8416 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_ROUNDSS
, args
, dname
);
8420 case OP_SSE41_ROUNDPD
: {
8421 LLVMValueRef args
[3];
8424 args
[1] = LLVMConstInt (LLVMInt32Type (), ins
->inst_c0
, FALSE
);
8426 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_ROUNDPD
, args
, dname
);
8430 case OP_SSE41_INSERT
: {
8431 if (ins
->inst_c1
== MONO_TYPE_R4
) {
8432 // special case for <float> overload
8433 LLVMValueRef args
[3];
8434 args
[0] = values
[ins
->sreg1
];
8435 args
[1] = values
[ins
->sreg2
];
8436 args
[2] = convert (ctx
, values
[ins
->sreg3
], LLVMInt8Type ());
8437 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_SSE_INSERTPS
, args
, dname
);
8439 // other overloads are implemented with `insertelement`
8440 values
[ins
->dreg
] = LLVMBuildInsertElement (builder
,
8441 values
[ins
->sreg1
],
8442 convert (ctx
, values
[ins
->sreg2
], primitive_type_to_llvm_type (inst_c1_type (ins
))),
8443 convert (ctx
, values
[ins
->sreg3
], LLVMInt8Type ()), dname
);
8448 case OP_SSE41_PTESTZ
: {
8449 LLVMValueRef args
[2];
8450 args
[0] = convert (ctx
, lhs
, sse_i8_t
);
8451 args
[1] = convert (ctx
, rhs
, sse_i8_t
);
8452 LLVMValueRef call
= call_intrins (ctx
, INTRINS_SSE_PTESTZ
, args
, dname
);
8453 LLVMValueRef cmp_zero
= LLVMBuildICmp (builder
, LLVMIntNE
, call
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
8454 values
[ins
->dreg
] = LLVMBuildZExt (builder
, cmp_zero
, LLVMInt8Type (), "");
8458 case OP_SSE42_CRC32
:
8459 case OP_SSE42_CRC64
: {
8460 LLVMValueRef args
[2];
8462 args
[1] = convert (ctx
, rhs
, primitive_type_to_llvm_type (ins
->inst_c0
));
8464 switch (ins
->inst_c0
) {
8465 case MONO_TYPE_U1
: id
= INTRINS_SSE_CRC32_32_8
; break;
8466 case MONO_TYPE_U2
: id
= INTRINS_SSE_CRC32_32_16
; break;
8467 case MONO_TYPE_U4
: id
= INTRINS_SSE_CRC32_32_32
; break;
8468 case MONO_TYPE_U8
: id
= INTRINS_SSE_CRC32_64_64
; break;
8469 default: g_assert_not_reached (); break;
8471 values
[ins
->dreg
] = call_intrins (ctx
, id
, args
, "");
8476 #ifdef ENABLE_NETCORE
8478 LLVMTypeRef t
= simd_class_to_llvm_type (ctx
, ins
->klass
);
8480 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, lhs
, t
, "");
8483 case OP_XCOMPARE_FP
: {
8484 LLVMRealPredicate pred
= fpcond_to_llvm_cond
[ins
->inst_c0
];
8485 LLVMValueRef cmp
= LLVMBuildFCmp (builder
, pred
, lhs
, rhs
, "");
8486 int nelems
= LLVMGetVectorSize (LLVMTypeOf (cmp
));
8487 g_assert (LLVMTypeOf (lhs
) == LLVMTypeOf (rhs
));
8488 if (ins
->inst_c1
== MONO_TYPE_R8
)
8489 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, LLVMBuildSExt (builder
, cmp
, LLVMVectorType (LLVMInt64Type (), nelems
), ""), LLVMTypeOf (lhs
), "");
8491 values
[ins
->dreg
] = LLVMBuildBitCast (builder
, LLVMBuildSExt (builder
, cmp
, LLVMVectorType (LLVMInt32Type (), nelems
), ""), LLVMTypeOf (lhs
), "");
8495 LLVMIntPredicate pred
= cond_to_llvm_cond
[ins
->inst_c0
];
8496 LLVMValueRef cmp
= LLVMBuildICmp (builder
, pred
, lhs
, rhs
, "");
8497 g_assert (LLVMTypeOf (lhs
) == LLVMTypeOf (rhs
));
8498 values
[ins
->dreg
] = LLVMBuildSExt (builder
, cmp
, LLVMTypeOf (lhs
), "");
8503 LLVMValueRef cmp
, mask
[32], shuffle
;
8506 #if defined(TARGET_WASM) && LLVM_API_VERSION >= 800
8507 /* The wasm code generator doesn't understand the shuffle/and code sequence below */
8509 if (LLVMIsNull (lhs
) || LLVMIsNull (rhs
)) {
8510 val
= LLVMIsNull (lhs
) ? rhs
: lhs
;
8511 nelems
= LLVMGetVectorSize (LLVMTypeOf (lhs
));
8513 IntrinsicId intrins
= (IntrinsicId
)0;
8516 intrins
= INTRINS_WASM_ANYTRUE_V16
;
8519 intrins
= INTRINS_WASM_ANYTRUE_V8
;
8522 intrins
= INTRINS_WASM_ANYTRUE_V4
;
8525 intrins
= INTRINS_WASM_ANYTRUE_V2
;
8528 g_assert_not_reached ();
8530 /* res = !wasm.anytrue (val) */
8531 values
[ins
->dreg
] = call_intrins (ctx
, intrins
, &val
, "");
8532 values
[ins
->dreg
] = LLVMBuildZExt (builder
, LLVMBuildICmp (builder
, LLVMIntEQ
, values
[ins
->dreg
], LLVMConstInt (LLVMInt32Type (), 0, FALSE
), ""), LLVMInt32Type (), dname
);
8536 LLVMTypeRef srcelemt
= LLVMGetElementType (LLVMTypeOf (lhs
));
8538 //%c = icmp sgt <16 x i8> %a0, %a1
8539 if (srcelemt
== LLVMDoubleType () || srcelemt
== LLVMFloatType ())
8540 cmp
= LLVMBuildFCmp (builder
, LLVMRealOEQ
, lhs
, rhs
, "");
8542 cmp
= LLVMBuildICmp (builder
, LLVMIntEQ
, lhs
, rhs
, "");
8543 nelems
= LLVMGetVectorSize (LLVMTypeOf (cmp
));
8546 if (srcelemt
== LLVMDoubleType ())
8547 elemt
= LLVMInt64Type ();
8548 else if (srcelemt
== LLVMFloatType ())
8549 elemt
= LLVMInt32Type ();
8553 t
= LLVMVectorType (elemt
, nelems
);
8554 cmp
= LLVMBuildSExt (builder
, cmp
, t
, "");
8555 // cmp is a <nelems x elemt> vector, each element is either 0xff... or 0
8556 int half
= nelems
/ 2;
8558 // AND the top and bottom halfes into the bottom half
8559 for (int i
= 0; i
< half
; ++i
)
8560 mask
[i
] = LLVMConstInt (LLVMInt32Type (), half
+ i
, FALSE
);
8561 for (int i
= half
; i
< nelems
; ++i
)
8562 mask
[i
] = LLVMConstInt (LLVMInt32Type (), 0, FALSE
);
8563 shuffle
= LLVMBuildShuffleVector (builder
, cmp
, LLVMGetUndef (t
), LLVMConstVector (mask
, LLVMGetVectorSize (t
)), "");
8564 cmp
= LLVMBuildAnd (builder
, cmp
, shuffle
, "");
8568 LLVMValueRef first_elem
= LLVMBuildExtractElement (builder
, cmp
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), "");
8570 LLVMValueRef cmp_zero
= LLVMBuildICmp (builder
, LLVMIntNE
, first_elem
, LLVMConstInt (elemt
, 0, FALSE
), "");
8571 values
[ins
->dreg
] = LLVMBuildZExt (builder
, cmp_zero
, LLVMInt8Type (), "");
8575 switch (ins
->inst_c0
) {
8577 values
[ins
->dreg
] = LLVMBuildAdd (builder
, lhs
, rhs
, "");
8580 values
[ins
->dreg
] = LLVMBuildSub (builder
, lhs
, rhs
, "");
8583 values
[ins
->dreg
] = LLVMBuildAnd (builder
, lhs
, rhs
, "");
8586 values
[ins
->dreg
] = LLVMBuildOr (builder
, lhs
, rhs
, "");
8589 values
[ins
->dreg
] = LLVMBuildXor (builder
, lhs
, rhs
, "");
8592 values
[ins
->dreg
] = LLVMBuildFAdd (builder
, lhs
, rhs
, "");
8595 values
[ins
->dreg
] = LLVMBuildFSub (builder
, lhs
, rhs
, "");
8598 values
[ins
->dreg
] = LLVMBuildFMul (builder
, lhs
, rhs
, "");
8601 values
[ins
->dreg
] = LLVMBuildFDiv (builder
, lhs
, rhs
, "");
8605 #if defined(TARGET_X86) || defined(TARGET_AMD64)
8606 LLVMValueRef args
[] = { lhs
, rhs
};
8608 gboolean is_r4
= ins
->inst_c1
== MONO_TYPE_R4
;
8609 if (ins
->inst_c0
== OP_FMAX
)
8610 values
[ins
->dreg
] = call_intrins (ctx
, is_r4
? INTRINS_SSE_MAXPS
: INTRINS_SSE_MAXPD
, args
, dname
);
8612 values
[ins
->dreg
] = call_intrins (ctx
, is_r4
? INTRINS_SSE_MINPS
: INTRINS_SSE_MINPD
, args
, dname
);
8619 gboolean is_unsigned
= ins
->inst_c1
== MONO_TYPE_U1
|| ins
->inst_c1
== MONO_TYPE_U2
|| ins
->inst_c1
== MONO_TYPE_U4
|| ins
->inst_c1
== MONO_TYPE_U8
;
8620 LLVMValueRef cmp
= LLVMBuildICmp (builder
, is_unsigned
? LLVMIntUGT
: LLVMIntSGT
, lhs
, rhs
, "");
8621 values
[ins
->dreg
] = LLVMBuildSelect (builder
, cmp
, lhs
, rhs
, "");
8625 gboolean is_unsigned
= ins
->inst_c1
== MONO_TYPE_U1
|| ins
->inst_c1
== MONO_TYPE_U2
|| ins
->inst_c1
== MONO_TYPE_U4
|| ins
->inst_c1
== MONO_TYPE_U8
;
8626 LLVMValueRef cmp
= LLVMBuildICmp (builder
, is_unsigned
? LLVMIntULT
: LLVMIntSLT
, lhs
, rhs
, "");
8627 values
[ins
->dreg
] = LLVMBuildSelect (builder
, cmp
, lhs
, rhs
, "");
8632 g_assert_not_reached ();
8636 case OP_XEXTRACT_I32
:
8637 case OP_XEXTRACT_I64
:
8638 case OP_XEXTRACT_R8
:
8639 case OP_XEXTRACT_R4
: {
8640 LLVMBasicBlockRef bbs
[64];
8641 LLVMValueRef switch_ins
;
8642 LLVMValueRef phi_values
[64];
8643 int nelems
= LLVMGetVectorSize (LLVMTypeOf (lhs
));
8646 g_assert (nelems
<= 64);
8647 for (i
= 0; i
< nelems
; ++i
)
8648 bbs
[i
] = gen_bb (ctx
, "XEXTRACT_CASE_BB");
8649 cbb
= gen_bb (ctx
, "XEXTRACT_COND_BB");
8651 switch_ins
= LLVMBuildSwitch (builder
, LLVMBuildAnd (builder
, rhs
, const_int32 (0xf), ""), bbs
[0], 0);
8652 for (i
= 0; i
< nelems
; ++i
) {
8653 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), bbs
[i
]);
8654 LLVMPositionBuilderAtEnd (builder
, bbs
[i
]);
8655 phi_values
[i
] = LLVMBuildExtractElement (builder
, lhs
, LLVMConstInt (LLVMInt32Type (), i
, FALSE
), "");
8656 LLVMBuildBr (builder
, cbb
);
8659 LLVMPositionBuilderAtEnd (builder
, cbb
);
8660 values
[ins
->dreg
] = LLVMBuildPhi (builder
, LLVMTypeOf (phi_values
[0]), "");
8661 LLVMAddIncoming (values
[ins
->dreg
], phi_values
, bbs
, nelems
);
8663 MonoTypeEnum type
= (MonoTypeEnum
)ins
->inst_c0
;
8667 values
[ins
->dreg
] = LLVMBuildZExt (ctx
->builder
, values
[ins
->dreg
], LLVMInt32Type (), "");
8672 ctx
->bblocks
[bb
->block_num
].end_bblock
= cbb
;
8676 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_CTPOP_I32
, &lhs
, "");
8679 values
[ins
->dreg
] = call_intrins (ctx
, INTRINS_CTPOP_I64
, &lhs
, "");
8683 LLVMValueRef args
[2];
8685 args
[1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE
);
8686 values
[ins
->dreg
] = call_intrins (ctx
, ins
->opcode
== OP_LZCNT32
? INTRINS_CTLZ_I32
: INTRINS_CTLZ_I64
, args
, "");
8691 LLVMValueRef args
[2];
8693 args
[1] = LLVMConstInt (LLVMInt1Type (), 0, FALSE
);
8694 values
[ins
->dreg
] = call_intrins (ctx
, ins
->opcode
== OP_CTTZ32
? INTRINS_CTTZ_I32
: INTRINS_CTTZ_I64
, args
, "");
8699 LLVMValueRef args
[2];
8701 args
[1] = convert (ctx
, rhs
, ins
->opcode
== OP_BEXTR32
? LLVMInt32Type () : LLVMInt64Type ()); // cast ushort to u32/u64
8702 values
[ins
->dreg
] = call_intrins (ctx
, ins
->opcode
== OP_BEXTR32
? INTRINS_BEXTR_I32
: INTRINS_BEXTR_I64
, args
, "");
8707 LLVMValueRef args
[2];
8710 values
[ins
->dreg
] = call_intrins (ctx
, ins
->opcode
== OP_BZHI32
? INTRINS_BZHI_I32
: INTRINS_BZHI_I64
, args
, "");
8716 case OP_MULX_HL64
: {
8717 gboolean is_64
= ins
->opcode
== OP_MULX_H64
|| ins
->opcode
== OP_MULX_HL64
;
8718 gboolean only_high
= ins
->opcode
== OP_MULX_H32
|| ins
->opcode
== OP_MULX_H64
;
8719 LLVMValueRef lx
= LLVMBuildZExt (ctx
->builder
, lhs
, LLVMInt128Type (), "");
8720 LLVMValueRef rx
= LLVMBuildZExt (ctx
->builder
, rhs
, LLVMInt128Type (), "");
8721 LLVMValueRef mulx
= LLVMBuildMul (ctx
->builder
, lx
, rx
, "");
8723 LLVMValueRef lowx
= LLVMBuildTrunc (ctx
->builder
, mulx
, is_64
? LLVMInt64Type () : LLVMInt32Type (), "");
8724 LLVMBuildStore (ctx
->builder
, lowx
, values
[ins
->sreg3
]);
8726 LLVMValueRef shift
= LLVMConstInt (LLVMInt128Type (), is_64
? 64 : 32, FALSE
);
8727 LLVMValueRef highx
= LLVMBuildLShr (ctx
->builder
, mulx
, shift
, "");
8728 values
[ins
->dreg
] = LLVMBuildTrunc (ctx
->builder
, highx
, is_64
? LLVMInt64Type () : LLVMInt32Type (), "");
8733 LLVMValueRef args
[2];
8736 values
[ins
->dreg
] = call_intrins (ctx
, ins
->opcode
== OP_PEXT32
? INTRINS_PEXT_I32
: INTRINS_PEXT_I64
, args
, "");
8741 LLVMValueRef args
[2];
8744 values
[ins
->dreg
] = call_intrins (ctx
, ins
->opcode
== OP_PDEP32
? INTRINS_PDEP_I32
: INTRINS_PDEP_I64
, args
, "");
8747 #endif /* ENABLE_NETCORE */
8754 * EXCEPTION HANDLING
8756 case OP_IMPLICIT_EXCEPTION
:
8757 /* This marks a place where an implicit exception can happen */
8758 if (bb
->region
!= -1)
8759 set_failure (ctx
, "implicit-exception");
8763 gboolean rethrow
= (ins
->opcode
== OP_RETHROW
);
8764 if (ctx
->llvm_only
) {
8765 emit_llvmonly_throw (ctx
, bb
, rethrow
, lhs
);
8766 has_terminator
= TRUE
;
8767 ctx
->unreachable
[bb
->block_num
] = TRUE
;
8769 emit_throw (ctx
, bb
, rethrow
, lhs
);
8770 builder
= ctx
->builder
;
8774 case OP_CALL_HANDLER
: {
8776 * We don't 'call' handlers, but instead simply branch to them.
8777 * The code generated by ENDFINALLY will branch back to us.
8779 LLVMBasicBlockRef noex_bb
;
8781 BBInfo
*info
= &bblocks
[ins
->inst_target_bb
->block_num
];
8783 bb_list
= info
->call_handler_return_bbs
;
8786 * Set the indicator variable for the finally clause.
8788 lhs
= info
->finally_ind
;
8790 LLVMBuildStore (builder
, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list
) + 1, FALSE
), lhs
);
8792 /* Branch to the finally clause */
8793 LLVMBuildBr (builder
, info
->call_handler_target_bb
);
8795 noex_bb
= gen_bb (ctx
, "CALL_HANDLER_CONT_BB");
8796 info
->call_handler_return_bbs
= g_slist_append_mempool (cfg
->mempool
, info
->call_handler_return_bbs
, noex_bb
);
8798 builder
= ctx
->builder
= create_builder (ctx
);
8799 LLVMPositionBuilderAtEnd (ctx
->builder
, noex_bb
);
8801 bblocks
[bb
->block_num
].end_bblock
= noex_bb
;
8804 case OP_START_HANDLER
: {
8807 case OP_ENDFINALLY
: {
8808 LLVMBasicBlockRef resume_bb
;
8809 MonoBasicBlock
*handler_bb
;
8810 LLVMValueRef val
, switch_ins
, callee
;
8813 gboolean is_fault
= MONO_REGION_FLAGS (bb
->region
) == MONO_EXCEPTION_CLAUSE_FAULT
;
8816 * Fault clauses are like finally clauses, but they are only called if an exception is thrown.
8819 handler_bb
= (MonoBasicBlock
*)g_hash_table_lookup (ctx
->region_to_handler
, GUINT_TO_POINTER (mono_get_block_region_notry (cfg
, bb
->region
)));
8820 g_assert (handler_bb
);
8821 info
= &bblocks
[handler_bb
->block_num
];
8822 lhs
= info
->finally_ind
;
8825 bb_list
= info
->call_handler_return_bbs
;
8827 resume_bb
= gen_bb (ctx
, "ENDFINALLY_RESUME_BB");
8829 /* Load the finally variable */
8830 val
= LLVMBuildLoad (builder
, lhs
, "");
8832 /* Reset the variable */
8833 LLVMBuildStore (builder
, LLVMConstInt (LLVMInt32Type (), 0, FALSE
), lhs
);
8835 /* Branch to either resume_bb, or to the bblocks in bb_list */
8836 switch_ins
= LLVMBuildSwitch (builder
, val
, resume_bb
, g_slist_length (bb_list
));
8838 * The other targets are added at the end to handle OP_CALL_HANDLER
8839 * opcodes processed later.
8841 info
->endfinally_switch_ins_list
= g_slist_append_mempool (cfg
->mempool
, info
->endfinally_switch_ins_list
, switch_ins
);
8843 builder
= ctx
->builder
= create_builder (ctx
);
8844 LLVMPositionBuilderAtEnd (ctx
->builder
, resume_bb
);
8847 if (ctx
->llvm_only
) {
8848 emit_resume_eh (ctx
, bb
);
8850 LLVMTypeRef icall_sig
= LLVMFunctionType (LLVMVoidType (), NULL
, 0, FALSE
);
8851 if (ctx
->cfg
->compile_aot
) {
8852 callee
= get_callee (ctx
, icall_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline
));
8854 callee
= get_jit_callee (ctx
, "llvm_resume_unwind_trampoline", icall_sig
, MONO_PATCH_INFO_JIT_ICALL_ID
, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline
));
8856 LLVMBuildCall (builder
, callee
, NULL
, 0, "");
8857 LLVMBuildUnreachable (builder
);
8860 has_terminator
= TRUE
;
8863 case OP_IL_SEQ_POINT
:
8868 sprintf (reason
, "opcode %s", mono_inst_name (ins
->opcode
));
8869 set_failure (ctx
, reason
);
8877 /* Convert the value to the type required by phi nodes */
8878 if (spec
[MONO_INST_DEST
] != ' ' && !MONO_IS_STORE_MEMBASE (ins
) && ctx
->vreg_types
[ins
->dreg
]) {
8879 if (ctx
->is_vphi
[ins
->dreg
])
8881 values
[ins
->dreg
] = addresses
[ins
->dreg
];
8883 values
[ins
->dreg
] = convert (ctx
, values
[ins
->dreg
], ctx
->vreg_types
[ins
->dreg
]);
8886 /* Add stores for volatile variables */
8887 if (!skip_volatile_store
&& spec
[MONO_INST_DEST
] != ' ' && spec
[MONO_INST_DEST
] != 'v' && !MONO_IS_STORE_MEMBASE (ins
))
8888 emit_volatile_store (ctx
, ins
->dreg
);
8894 if (!has_terminator
&& bb
->next_bb
&& (bb
== cfg
->bb_entry
|| bb
->in_count
> 0)) {
8895 LLVMBuildBr (builder
, get_bb (ctx
, bb
->next_bb
));
8898 if (bb
== cfg
->bb_exit
&& sig
->ret
->type
== MONO_TYPE_VOID
) {
8899 emit_dbg_loc (ctx
, builder
, cfg
->header
->code
+ cfg
->header
->code_size
- 1);
8900 LLVMBuildRetVoid (builder
);
8903 if (bb
== cfg
->bb_entry
)
8904 ctx
->last_alloca
= LLVMGetLastInstruction (get_bb (ctx
, cfg
->bb_entry
));
8908 * mono_llvm_check_method_supported:
8910 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
8911 * compiling a method twice.
8914 mono_llvm_check_method_supported (MonoCompile
*cfg
)
8919 if (mono_method_signature_internal (cfg
->method
)->call_convention
== MONO_CALL_VARARG
) {
8920 cfg
->exception_message
= g_strdup ("vararg callconv");
8921 cfg
->disable_llvm
= TRUE
;
8929 if (cfg
->method
->save_lmf
) {
8930 cfg
->exception_message
= g_strdup ("lmf");
8931 cfg
->disable_llvm
= TRUE
;
8933 if (cfg
->disable_llvm
)
8937 * Nested clauses where one of the clauses is a finally clause is
8938 * not supported, because LLVM can't figure out the control flow,
8939 * probably because we resume exception handling by calling our
8940 * own function instead of using the 'resume' llvm instruction.
8942 for (i
= 0; i
< cfg
->header
->num_clauses
; ++i
) {
8943 for (j
= 0; j
< cfg
->header
->num_clauses
; ++j
) {
8944 MonoExceptionClause
*clause1
= &cfg
->header
->clauses
[i
];
8945 MonoExceptionClause
*clause2
= &cfg
->header
->clauses
[j
];
8947 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
8948 if (i
!= j
&& clause1
->try_offset
>= clause2
->try_offset
&& clause1
->handler_offset
<= clause2
->handler_offset
) {
8949 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
8950 cfg
->exception_message
= g_strdup ("nested clauses");
8951 cfg
->disable_llvm
= TRUE
;
8956 if (cfg
->disable_llvm
)
8960 if (cfg
->method
->dynamic
) {
8961 cfg
->exception_message
= g_strdup ("dynamic.");
8962 cfg
->disable_llvm
= TRUE
;
8964 if (cfg
->disable_llvm
)
8968 static LLVMCallInfo
*
8969 get_llvm_call_info (MonoCompile
*cfg
, MonoMethodSignature
*sig
)
8971 LLVMCallInfo
*linfo
;
8974 if (cfg
->gsharedvt
&& cfg
->llvm_only
&& mini_is_gsharedvt_variable_signature (sig
)) {
8978 * Gsharedvt methods have the following calling convention:
8979 * - all arguments are passed by ref, even non generic ones
8980 * - the return value is returned by ref too, using a vret
8981 * argument passed after 'this'.
8983 n
= sig
->param_count
+ sig
->hasthis
;
8984 linfo
= (LLVMCallInfo
*)mono_mempool_alloc0 (cfg
->mempool
, sizeof (LLVMCallInfo
) + (sizeof (LLVMArgInfo
) * n
));
8988 linfo
->args
[pindex
++].storage
= LLVMArgNormal
;
8990 if (sig
->ret
->type
!= MONO_TYPE_VOID
) {
8991 if (mini_is_gsharedvt_variable_type (sig
->ret
))
8992 linfo
->ret
.storage
= LLVMArgGsharedvtVariable
;
8993 else if (mini_type_is_vtype (sig
->ret
))
8994 linfo
->ret
.storage
= LLVMArgGsharedvtFixedVtype
;
8996 linfo
->ret
.storage
= LLVMArgGsharedvtFixed
;
8997 linfo
->vret_arg_index
= pindex
;
8999 linfo
->ret
.storage
= LLVMArgNone
;
9002 for (i
= 0; i
< sig
->param_count
; ++i
) {
9003 if (sig
->params
[i
]->byref
)
9004 linfo
->args
[pindex
].storage
= LLVMArgNormal
;
9005 else if (mini_is_gsharedvt_variable_type (sig
->params
[i
]))
9006 linfo
->args
[pindex
].storage
= LLVMArgGsharedvtVariable
;
9007 else if (mini_type_is_vtype (sig
->params
[i
]))
9008 linfo
->args
[pindex
].storage
= LLVMArgGsharedvtFixedVtype
;
9010 linfo
->args
[pindex
].storage
= LLVMArgGsharedvtFixed
;
9011 linfo
->args
[pindex
].type
= sig
->params
[i
];
9017 linfo
= mono_arch_get_llvm_call_info (cfg
, sig
);
9018 linfo
->dummy_arg_pindex
= -1;
9019 for (i
= 0; i
< sig
->param_count
; ++i
)
9020 linfo
->args
[i
+ sig
->hasthis
].type
= sig
->params
[i
];
9026 emit_method_inner (EmitContext
*ctx
);
9029 free_ctx (EmitContext
*ctx
)
9033 g_free (ctx
->values
);
9034 g_free (ctx
->addresses
);
9035 g_free (ctx
->vreg_types
);
9036 g_free (ctx
->is_vphi
);
9037 g_free (ctx
->vreg_cli_types
);
9038 g_free (ctx
->is_dead
);
9039 g_free (ctx
->unreachable
);
9040 g_ptr_array_free (ctx
->phi_values
, TRUE
);
9041 g_free (ctx
->bblocks
);
9042 g_hash_table_destroy (ctx
->region_to_handler
);
9043 g_hash_table_destroy (ctx
->clause_to_handler
);
9044 g_hash_table_destroy (ctx
->jit_callees
);
9046 g_ptr_array_free (ctx
->callsite_list
, TRUE
);
9048 g_free (ctx
->method_name
);
9049 g_ptr_array_free (ctx
->bblock_list
, TRUE
);
9051 for (l
= ctx
->builders
; l
; l
= l
->next
) {
9052 LLVMBuilderRef builder
= (LLVMBuilderRef
)l
->data
;
9053 LLVMDisposeBuilder (builder
);
9060 * mono_llvm_emit_method:
9062 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
9065 mono_llvm_emit_method (MonoCompile
*cfg
)
9074 /* The code below might acquire the loader lock, so use it for global locking */
9075 mono_loader_lock ();
9077 ctx
= g_new0 (EmitContext
, 1);
9079 ctx
->mempool
= cfg
->mempool
;
9082 * This maps vregs to the LLVM instruction defining them
9084 ctx
->values
= g_new0 (LLVMValueRef
, cfg
->next_vreg
);
9086 * This maps vregs for volatile variables to the LLVM instruction defining their
9089 ctx
->addresses
= g_new0 (LLVMValueRef
, cfg
->next_vreg
);
9090 ctx
->vreg_types
= g_new0 (LLVMTypeRef
, cfg
->next_vreg
);
9091 ctx
->is_vphi
= g_new0 (gboolean
, cfg
->next_vreg
);
9092 ctx
->vreg_cli_types
= g_new0 (MonoType
*, cfg
->next_vreg
);
9093 ctx
->phi_values
= g_ptr_array_sized_new (256);
9095 * This signals whenever the vreg was defined by a phi node with no input vars
9096 * (i.e. all its input bblocks end with NOT_REACHABLE).
9098 ctx
->is_dead
= g_new0 (gboolean
, cfg
->next_vreg
);
9099 /* Whenever the bblock is unreachable */
9100 ctx
->unreachable
= g_new0 (gboolean
, cfg
->max_block_num
);
9101 ctx
->bblock_list
= g_ptr_array_sized_new (256);
9103 ctx
->region_to_handler
= g_hash_table_new (NULL
, NULL
);
9104 ctx
->clause_to_handler
= g_hash_table_new (NULL
, NULL
);
9105 ctx
->callsite_list
= g_ptr_array_new ();
9106 ctx
->jit_callees
= g_hash_table_new (NULL
, NULL
);
9107 if (cfg
->compile_aot
) {
9108 ctx
->module
= &aot_module
;
9110 if (mono_aot_is_externally_callable (cfg
->method
))
9111 method_name
= mono_aot_get_mangled_method_name (cfg
->method
);
9113 method_name
= mono_aot_get_method_name (cfg
);
9114 cfg
->llvm_method_name
= g_strdup (method_name
);
9116 init_jit_module (cfg
->domain
);
9117 ctx
->module
= (MonoLLVMModule
*)domain_jit_info (cfg
->domain
)->llvm_module
;
9118 method_name
= mono_method_full_name (cfg
->method
, TRUE
);
9120 ctx
->method_name
= method_name
;
9122 if (cfg
->compile_aot
) {
9123 ctx
->lmodule
= ctx
->module
->lmodule
;
9125 ctx
->lmodule
= LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg
->method
->name
));
9126 /* Reset this as it contains values from lmodule */
9127 memset (ctx
->module
->intrins_by_id
, 0, sizeof (LLVMValueRef
) * INTRINS_NUM
);
9129 ctx
->llvm_only
= ctx
->module
->llvm_only
;
9131 ctx
->emit_dummy_arg
= TRUE
;
9134 emit_method_inner (ctx
);
9136 if (!ctx_ok (ctx
)) {
9138 /* Need to add unused phi nodes as they can be referenced by other values */
9139 LLVMBasicBlockRef phi_bb
= LLVMAppendBasicBlock (ctx
->lmethod
, "PHI_BB");
9140 LLVMBuilderRef builder
;
9142 builder
= create_builder (ctx
);
9143 LLVMPositionBuilderAtEnd (builder
, phi_bb
);
9145 for (i
= 0; i
< ctx
->phi_values
->len
; ++i
) {
9146 LLVMValueRef v
= (LLVMValueRef
)g_ptr_array_index (ctx
->phi_values
, i
);
9147 if (LLVMGetInstructionParent (v
) == NULL
)
9148 LLVMInsertIntoBuilder (builder
, v
);
9151 if (ctx
->module
->llvm_only
&& ctx
->module
->static_link
) {
9152 // Keep a stub for the function since it might be called directly
9153 int nbbs
= LLVMCountBasicBlocks (ctx
->lmethod
);
9154 LLVMBasicBlockRef
*bblocks
= g_new0 (LLVMBasicBlockRef
, nbbs
);
9155 LLVMGetBasicBlocks (ctx
->lmethod
, bblocks
);
9156 for (int i
= 0; i
< nbbs
; ++i
)
9157 LLVMDeleteBasicBlock (bblocks
[i
]);
9159 LLVMBasicBlockRef entry_bb
= LLVMAppendBasicBlock (ctx
->lmethod
, "ENTRY");
9160 builder
= create_builder (ctx
);
9161 LLVMPositionBuilderAtEnd (builder
, entry_bb
);
9162 ctx
->builder
= builder
;
9164 LLVMTypeRef sig
= LLVMFunctionType0 (LLVMVoidType (), FALSE
);
9165 LLVMValueRef callee
= get_callee (ctx
, sig
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception
));
9166 LLVMBuildCall (builder
, callee
, NULL
, 0, "");
9167 LLVMBuildUnreachable (builder
);
9169 LLVMDeleteFunction (ctx
->lmethod
);
9176 mono_loader_unlock ();
9180 emit_method_inner (EmitContext
*ctx
)
9182 MonoCompile
*cfg
= ctx
->cfg
;
9183 MonoMethodSignature
*sig
;
9185 LLVMTypeRef method_type
;
9186 LLVMValueRef method
= NULL
;
9187 LLVMValueRef
*values
= ctx
->values
;
9188 int i
, max_block_num
, bb_index
;
9189 gboolean last
= FALSE
;
9190 gboolean llvmonly_fail
= FALSE
;
9191 LLVMCallInfo
*linfo
;
9192 LLVMModuleRef lmodule
= ctx
->lmodule
;
9194 GPtrArray
*bblock_list
= ctx
->bblock_list
;
9195 MonoMethodHeader
*header
;
9196 MonoExceptionClause
*clause
;
9198 LLVMBuilderRef entry_builder
= NULL
;
9199 LLVMBasicBlockRef entry_bb
= NULL
;
9201 if (cfg
->gsharedvt
&& !cfg
->llvm_only
) {
9202 set_failure (ctx
, "gsharedvt");
9208 static int count
= 0;
9211 char *llvm_count_str
= g_getenv ("LLVM_COUNT");
9212 if (llvm_count_str
) {
9213 int lcount
= atoi (llvm_count_str
);
9214 g_free (llvm_count_str
);
9215 if (count
== lcount
) {
9216 printf ("LAST: %s\n", mono_method_full_name (cfg
->method
, TRUE
));
9220 if (count
> lcount
) {
9221 set_failure (ctx
, "count");
9228 // If we come upon one of the init_method wrappers, we need to find
9229 // the method that we have already emitted and tell LLVM that this
9230 // managed method info for the wrapper is associated with this method
9231 // we constructed ourselves from LLVM IR.
9233 // This is necessary to unwind through the init_method, in the case that
9234 // it has to run a static cctor that throws an exception
9235 if (cfg
->method
->wrapper_type
== MONO_WRAPPER_OTHER
) {
9236 WrapperInfo
*info
= mono_marshal_get_wrapper_info (cfg
->method
);
9237 if (info
->subtype
== WRAPPER_SUBTYPE_AOT_INIT
) {
9238 method
= get_init_func (ctx
->module
, info
->d
.aot_init
.subtype
);
9239 ctx
->lmethod
= method
;
9240 ctx
->module
->max_method_idx
= MAX (ctx
->module
->max_method_idx
, cfg
->method_index
);
9242 const char *init_name
= mono_marshal_get_aot_init_wrapper_name (info
->d
.aot_init
.subtype
);
9243 ctx
->method_name
= g_strdup_printf ("%s_%s", ctx
->module
->global_prefix
, init_name
);
9244 ctx
->cfg
->asm_symbol
= g_strdup (ctx
->method_name
);
9246 if (!cfg
->llvm_only
&& ctx
->module
->external_symbols
) {
9247 LLVMSetLinkage (method
, LLVMExternalLinkage
);
9248 LLVMSetVisibility (method
, LLVMHiddenVisibility
);
9250 /* Not looked up at runtime */
9251 g_hash_table_insert (ctx
->module
->no_method_table_lmethods
, method
, method
);
9254 } else if (info
->subtype
== WRAPPER_SUBTYPE_LLVM_FUNC
) {
9255 g_assert (info
->d
.llvm_func
.subtype
== LLVM_FUNC_WRAPPER_GC_POLL
);
9257 if (cfg
->compile_aot
) {
9258 method
= ctx
->module
->gc_poll_cold_wrapper
;
9261 method
= emit_icall_cold_wrapper (ctx
->module
, lmodule
, MONO_JIT_ICALL_mono_threads_state_poll
, FALSE
);
9263 ctx
->lmethod
= method
;
9264 ctx
->module
->max_method_idx
= MAX (ctx
->module
->max_method_idx
, cfg
->method_index
);
9266 ctx
->method_name
= g_strdup (LLVMGetValueName (method
)); //g_strdup_printf ("%s_%s", ctx->module->global_prefix, LLVMGetValueName (method));
9267 ctx
->cfg
->asm_symbol
= g_strdup (ctx
->method_name
);
9269 if (!cfg
->llvm_only
&& ctx
->module
->external_symbols
) {
9270 LLVMSetLinkage (method
, LLVMExternalLinkage
);
9271 LLVMSetVisibility (method
, LLVMHiddenVisibility
);
9278 sig
= mono_method_signature_internal (cfg
->method
);
9281 linfo
= get_llvm_call_info (cfg
, sig
);
9287 linfo
->rgctx_arg
= TRUE
;
9288 else if (needs_extra_arg (ctx
, cfg
->method
))
9289 linfo
->dummy_arg
= TRUE
;
9290 ctx
->method_type
= method_type
= sig_to_llvm_sig_full (ctx
, sig
, linfo
);
9294 method
= LLVMAddFunction (lmodule
, ctx
->method_name
, method_type
);
9295 ctx
->lmethod
= method
;
9297 if (!cfg
->llvm_only
)
9298 LLVMSetFunctionCallConv (method
, LLVMMono1CallConv
);
9300 /* if the method doesn't contain
9301 * (1) a call (so it's a leaf method)
9303 * we can skip the GC safepoint on method entry. */
9304 gboolean requires_safepoint
;
9305 requires_safepoint
= cfg
->has_calls
;
9306 if (!requires_safepoint
) {
9307 for (bb
= cfg
->bb_entry
->next_bb
; bb
; bb
= bb
->next_bb
) {
9308 if (bb
->loop_body_start
|| (bb
->flags
& BB_EXCEPTION_HANDLER
)) {
9309 requires_safepoint
= TRUE
;
9313 if (cfg
->method
->wrapper_type
) {
9314 WrapperInfo
*info
= mono_marshal_get_wrapper_info (cfg
->method
);
9316 switch (info
->subtype
) {
9317 case WRAPPER_SUBTYPE_GSHAREDVT_IN
:
9318 case WRAPPER_SUBTYPE_GSHAREDVT_OUT
:
9319 case WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG
:
9320 case WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG
:
9321 /* Arguments are not used after the call */
9322 requires_safepoint
= FALSE
;
9326 ctx
->has_safepoints
= requires_safepoint
;
9328 if (!cfg
->llvm_only
&& mono_threads_are_safepoints_enabled () && requires_safepoint
) {
9329 if (!cfg
->compile_aot
&& cfg
->method
->wrapper_type
!= MONO_WRAPPER_ALLOC
) {
9330 LLVMSetGC (method
, "coreclr");
9331 emit_gc_safepoint_poll (ctx
->module
, ctx
->lmodule
, cfg
);
9332 } else if (cfg
->compile_aot
) {
9333 LLVMSetGC (method
, "coreclr");
9336 LLVMSetLinkage (method
, LLVMPrivateLinkage
);
9338 mono_llvm_add_func_attr (method
, LLVM_ATTR_UW_TABLE
);
9340 if (cfg
->disable_omit_fp
)
9341 mono_llvm_add_func_attr_nv (method
, "no-frame-pointer-elim", "true");
9343 if (cfg
->compile_aot
) {
9344 if (mono_aot_is_externally_callable (cfg
->method
)) {
9345 LLVMSetLinkage (method
, LLVMExternalLinkage
);
9347 LLVMSetLinkage (method
, LLVMInternalLinkage
);
9348 //all methods have internal visibility when doing llvm_only
9349 if (!cfg
->llvm_only
&& ctx
->module
->external_symbols
) {
9350 LLVMSetLinkage (method
, LLVMExternalLinkage
);
9351 LLVMSetVisibility (method
, LLVMHiddenVisibility
);
9355 LLVMSetLinkage (method
, LLVMExternalLinkage
);
9358 if (cfg
->method
->save_lmf
&& !cfg
->llvm_only
) {
9359 set_failure (ctx
, "lmf");
9363 if (sig
->pinvoke
&& cfg
->method
->wrapper_type
!= MONO_WRAPPER_RUNTIME_INVOKE
&& !cfg
->llvm_only
) {
9364 set_failure (ctx
, "pinvoke signature");
9368 header
= cfg
->header
;
9369 for (i
= 0; i
< header
->num_clauses
; ++i
) {
9370 clause
= &header
->clauses
[i
];
9371 if (clause
->flags
!= MONO_EXCEPTION_CLAUSE_FINALLY
&& clause
->flags
!= MONO_EXCEPTION_CLAUSE_FAULT
&& clause
->flags
!= MONO_EXCEPTION_CLAUSE_NONE
) {
9372 if (cfg
->llvm_only
) {
9373 // FIXME: Treat unhandled opcodes like __arglist the same way
9374 // It would require deleting the already emitted code
9375 llvmonly_fail
= TRUE
;
9377 set_failure (ctx
, "non-finally/catch/fault clause.");
9382 if (header
->num_clauses
|| (cfg
->method
->iflags
& METHOD_IMPL_ATTRIBUTE_NOINLINING
) || cfg
->no_inline
)
9383 /* We can't handle inlined methods with clauses */
9384 mono_llvm_add_func_attr (method
, LLVM_ATTR_NO_INLINE
);
9386 if (linfo
->rgctx_arg
) {
9387 ctx
->rgctx_arg
= LLVMGetParam (method
, linfo
->rgctx_arg_pindex
);
9388 ctx
->rgctx_arg_pindex
= linfo
->rgctx_arg_pindex
;
9390 * We mark the rgctx parameter with the inreg attribute, which is mapped to
9391 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
9392 * CC_X86_64_Mono in X86CallingConv.td.
9394 if (!ctx
->llvm_only
)
9395 mono_llvm_add_param_attr (ctx
->rgctx_arg
, LLVM_ATTR_IN_REG
);
9396 LLVMSetValueName (ctx
->rgctx_arg
, "rgctx");
9398 ctx
->rgctx_arg_pindex
= -1;
9400 if (cfg
->vret_addr
) {
9401 values
[cfg
->vret_addr
->dreg
] = LLVMGetParam (method
, linfo
->vret_arg_pindex
);
9402 LLVMSetValueName (values
[cfg
->vret_addr
->dreg
], "vret");
9403 if (linfo
->ret
.storage
== LLVMArgVtypeByRef
) {
9404 mono_llvm_add_param_attr (LLVMGetParam (method
, linfo
->vret_arg_pindex
), LLVM_ATTR_STRUCT_RET
);
9405 mono_llvm_add_param_attr (LLVMGetParam (method
, linfo
->vret_arg_pindex
), LLVM_ATTR_NO_ALIAS
);
9410 ctx
->this_arg_pindex
= linfo
->this_arg_pindex
;
9411 ctx
->this_arg
= LLVMGetParam (method
, linfo
->this_arg_pindex
);
9412 values
[cfg
->args
[0]->dreg
] = ctx
->this_arg
;
9413 LLVMSetValueName (values
[cfg
->args
[0]->dreg
], "this");
9415 if (linfo
->dummy_arg
)
9416 LLVMSetValueName (LLVMGetParam (method
, linfo
->dummy_arg_pindex
), "dummy_arg");
9418 names
= g_new (char *, sig
->param_count
);
9419 mono_method_get_param_names (cfg
->method
, (const char **) names
);
9421 /* Set parameter names/attributes */
9422 for (i
= 0; i
< sig
->param_count
; ++i
) {
9423 LLVMArgInfo
*ainfo
= &linfo
->args
[i
+ sig
->hasthis
];
9425 int pindex
= ainfo
->pindex
+ ainfo
->ndummy_fpargs
;
9428 for (j
= 0; j
< ainfo
->ndummy_fpargs
; ++j
) {
9429 name
= g_strdup_printf ("dummy_%d_%d", i
, j
);
9430 LLVMSetValueName (LLVMGetParam (method
, ainfo
->pindex
+ j
), name
);
9434 if (ainfo
->storage
== LLVMArgVtypeInReg
&& ainfo
->pair_storage
[0] == LLVMArgNone
&& ainfo
->pair_storage
[1] == LLVMArgNone
)
9437 values
[cfg
->args
[i
+ sig
->hasthis
]->dreg
] = LLVMGetParam (method
, pindex
);
9438 if (ainfo
->storage
== LLVMArgGsharedvtFixed
|| ainfo
->storage
== LLVMArgGsharedvtFixedVtype
) {
9439 if (names
[i
] && names
[i
][0] != '\0')
9440 name
= g_strdup_printf ("p_arg_%s", names
[i
]);
9442 name
= g_strdup_printf ("p_arg_%d", i
);
9444 if (names
[i
] && names
[i
][0] != '\0')
9445 name
= g_strdup_printf ("arg_%s", names
[i
]);
9447 name
= g_strdup_printf ("arg_%d", i
);
9449 LLVMSetValueName (LLVMGetParam (method
, pindex
), name
);
9451 if (ainfo
->storage
== LLVMArgVtypeByVal
)
9452 mono_llvm_add_param_attr (LLVMGetParam (method
, pindex
), LLVM_ATTR_BY_VAL
);
9454 if (ainfo
->storage
== LLVMArgVtypeByRef
|| ainfo
->storage
== LLVMArgVtypeAddr
) {
9456 cfg
->args
[i
+ sig
->hasthis
]->opcode
= OP_VTARG_ADDR
;
9461 if (ctx
->module
->emit_dwarf
&& cfg
->compile_aot
&& mono_debug_enabled ()) {
9462 ctx
->minfo
= mono_debug_lookup_method (cfg
->method
);
9463 ctx
->dbg_md
= emit_dbg_subprogram (ctx
, cfg
, method
, ctx
->method_name
);
9467 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
)
9468 max_block_num
= MAX (max_block_num
, bb
->block_num
);
9469 ctx
->bblocks
= bblocks
= g_new0 (BBInfo
, max_block_num
+ 1);
9471 /* Add branches between non-consecutive bblocks */
9472 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
9473 if (bb
->last_ins
&& MONO_IS_COND_BRANCH_OP (bb
->last_ins
) &&
9474 bb
->next_bb
!= bb
->last_ins
->inst_false_bb
) {
9476 MonoInst
*inst
= (MonoInst
*)mono_mempool_alloc0 (cfg
->mempool
, sizeof (MonoInst
));
9477 inst
->opcode
= OP_BR
;
9478 inst
->inst_target_bb
= bb
->last_ins
->inst_false_bb
;
9479 mono_bblock_add_inst (bb
, inst
);
9484 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
9486 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
9488 LLVMBuilderRef builder
;
9490 char dname_buf
[128];
9492 builder
= create_builder (ctx
);
9494 for (ins
= bb
->code
; ins
; ins
= ins
->next
) {
9495 switch (ins
->opcode
) {
9500 LLVMTypeRef phi_type
= llvm_type_to_stack_type (cfg
, type_to_llvm_type (ctx
, m_class_get_byval_arg (ins
->klass
)));
9505 if (ins
->opcode
== OP_VPHI
) {
9506 /* Treat valuetype PHI nodes as operating on the address itself */
9507 g_assert (ins
->klass
);
9508 phi_type
= LLVMPointerType (type_to_llvm_type (ctx
, m_class_get_byval_arg (ins
->klass
)), 0);
9512 * Have to precreate these, as they can be referenced by
9513 * earlier instructions.
9515 sprintf (dname_buf
, "t%d", ins
->dreg
);
9517 values
[ins
->dreg
] = LLVMBuildPhi (builder
, phi_type
, dname
);
9519 if (ins
->opcode
== OP_VPHI
)
9520 ctx
->addresses
[ins
->dreg
] = values
[ins
->dreg
];
9522 g_ptr_array_add (ctx
->phi_values
, values
[ins
->dreg
]);
9525 * Set the expected type of the incoming arguments since these have
9526 * to have the same type.
9528 for (i
= 0; i
< ins
->inst_phi_args
[0]; i
++) {
9529 int sreg1
= ins
->inst_phi_args
[i
+ 1];
9532 if (ins
->opcode
== OP_VPHI
)
9533 ctx
->is_vphi
[sreg1
] = TRUE
;
9534 ctx
->vreg_types
[sreg1
] = phi_type
;
9540 ((MonoInst
*)ins
->inst_p0
)->flags
|= MONO_INST_INDIRECT
;
9549 * Create an ordering for bblocks, use the depth first order first, then
9550 * put the exception handling bblocks last.
9552 for (bb_index
= 0; bb_index
< cfg
->num_bblocks
; ++bb_index
) {
9553 bb
= cfg
->bblocks
[bb_index
];
9554 if (!(bb
->region
!= -1 && !MONO_BBLOCK_IS_IN_REGION (bb
, MONO_REGION_TRY
))) {
9555 g_ptr_array_add (bblock_list
, bb
);
9556 bblocks
[bb
->block_num
].added
= TRUE
;
9560 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
9561 if (!bblocks
[bb
->block_num
].added
)
9562 g_ptr_array_add (bblock_list
, bb
);
9566 * Second pass: generate code.
9569 entry_builder
= create_builder (ctx
);
9570 entry_bb
= get_bb (ctx
, cfg
->bb_entry
);
9571 LLVMPositionBuilderAtEnd (entry_builder
, entry_bb
);
9572 emit_entry_bb (ctx
, entry_builder
);
9576 * In llvmonly mode, we want to emit an llvm method for every method even if it fails to compile,
9577 * so direct calls can be made from outside the assembly.
9579 goto after_codegen_1
;
9581 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
9585 if (!(bb
->region
!= -1 && (bb
->flags
& BB_EXCEPTION_HANDLER
)))
9588 clause_index
= MONO_REGION_CLAUSE_INDEX (bb
->region
);
9589 g_hash_table_insert (ctx
->region_to_handler
, GUINT_TO_POINTER (mono_get_block_region_notry (cfg
, bb
->region
)), bb
);
9590 g_hash_table_insert (ctx
->clause_to_handler
, GINT_TO_POINTER (clause_index
), bb
);
9593 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
9594 * LLVM bblock containing a landing pad causes problems for the
9595 * LLVM optimizer passes.
9597 sprintf (name
, "BB%d_CALL_HANDLER_TARGET", bb
->block_num
);
9598 ctx
->bblocks
[bb
->block_num
].call_handler_target_bb
= LLVMAppendBasicBlock (ctx
->lmethod
, name
);
9601 // Make landing pads first
9602 ctx
->exc_meta
= g_hash_table_new_full (NULL
, NULL
, NULL
, NULL
);
9604 if (ctx
->llvm_only
) {
9605 size_t group_index
= 0;
9606 while (group_index
< cfg
->header
->num_clauses
) {
9608 size_t cursor
= group_index
;
9609 while (cursor
< cfg
->header
->num_clauses
&&
9610 CLAUSE_START (&cfg
->header
->clauses
[cursor
]) == CLAUSE_START (&cfg
->header
->clauses
[group_index
]) &&
9611 CLAUSE_END (&cfg
->header
->clauses
[cursor
]) == CLAUSE_END (&cfg
->header
->clauses
[group_index
])) {
9616 LLVMBasicBlockRef lpad_bb
= emit_landing_pad (ctx
, group_index
, count
);
9617 intptr_t key
= CLAUSE_END (&cfg
->header
->clauses
[group_index
]);
9618 g_hash_table_insert (ctx
->exc_meta
, (gpointer
)key
, lpad_bb
);
9620 group_index
= cursor
;
9624 for (bb_index
= 0; bb_index
< bblock_list
->len
; ++bb_index
) {
9625 bb
= (MonoBasicBlock
*)g_ptr_array_index (bblock_list
, bb_index
);
9627 // Prune unreachable mono BBs.
9628 if (!(bb
== cfg
->bb_entry
|| bb
->in_count
> 0))
9631 process_bb (ctx
, bb
);
9635 g_hash_table_destroy (ctx
->exc_meta
);
9637 mono_memory_barrier ();
9639 /* Add incoming phi values */
9640 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
9641 GSList
*l
, *ins_list
;
9643 ins_list
= bblocks
[bb
->block_num
].phi_nodes
;
9645 for (l
= ins_list
; l
; l
= l
->next
) {
9646 PhiNode
*node
= (PhiNode
*)l
->data
;
9647 MonoInst
*phi
= node
->phi
;
9648 int sreg1
= node
->sreg
;
9649 LLVMBasicBlockRef in_bb
;
9654 in_bb
= get_end_bb (ctx
, node
->in_bb
);
9656 if (ctx
->unreachable
[node
->in_bb
->block_num
])
9659 if (phi
->opcode
== OP_VPHI
) {
9660 g_assert (LLVMTypeOf (ctx
->addresses
[sreg1
]) == LLVMTypeOf (values
[phi
->dreg
]));
9661 LLVMAddIncoming (values
[phi
->dreg
], &ctx
->addresses
[sreg1
], &in_bb
, 1);
9663 if (!values
[sreg1
]) {
9664 /* Can happen with values in EH clauses */
9665 set_failure (ctx
, "incoming phi sreg1");
9668 if (LLVMTypeOf (values
[sreg1
]) != LLVMTypeOf (values
[phi
->dreg
])) {
9669 set_failure (ctx
, "incoming phi arg type mismatch");
9672 g_assert (LLVMTypeOf (values
[sreg1
]) == LLVMTypeOf (values
[phi
->dreg
]));
9673 LLVMAddIncoming (values
[phi
->dreg
], &values
[sreg1
], &in_bb
, 1);
9678 /* Nullify empty phi instructions */
9679 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
9680 GSList
*l
, *ins_list
;
9682 ins_list
= bblocks
[bb
->block_num
].phi_nodes
;
9684 for (l
= ins_list
; l
; l
= l
->next
) {
9685 PhiNode
*node
= (PhiNode
*)l
->data
;
9686 MonoInst
*phi
= node
->phi
;
9687 LLVMValueRef phi_ins
= values
[phi
->dreg
];
9690 /* Already removed */
9693 if (LLVMCountIncoming (phi_ins
) == 0) {
9694 mono_llvm_replace_uses_of (phi_ins
, LLVMConstNull (LLVMTypeOf (phi_ins
)));
9695 LLVMInstructionEraseFromParent (phi_ins
);
9696 values
[phi
->dreg
] = NULL
;
9701 /* Create the SWITCH statements for ENDFINALLY instructions */
9702 for (bb
= cfg
->bb_entry
; bb
; bb
= bb
->next_bb
) {
9703 BBInfo
*info
= &bblocks
[bb
->block_num
];
9705 for (l
= info
->endfinally_switch_ins_list
; l
; l
= l
->next
) {
9706 LLVMValueRef switch_ins
= (LLVMValueRef
)l
->data
;
9707 GSList
*bb_list
= info
->call_handler_return_bbs
;
9709 GSList
*bb_list_iter
;
9711 for (bb_list_iter
= bb_list
; bb_list_iter
; bb_list_iter
= g_slist_next (bb_list_iter
)) {
9712 LLVMAddCase (switch_ins
, LLVMConstInt (LLVMInt32Type (), i
+ 1, FALSE
), (LLVMBasicBlockRef
)bb_list_iter
->data
);
9718 ctx
->module
->max_method_idx
= MAX (ctx
->module
->max_method_idx
, cfg
->method_index
);
9722 if (llvmonly_fail
) {
9724 * FIXME: Maybe fallback to interpreter
9726 static LLVMTypeRef sig
;
9728 ctx
->builder
= create_builder (ctx
);
9729 LLVMPositionBuilderAtEnd (ctx
->builder
, ctx
->inited_bb
);
9731 char *name
= mono_method_get_full_name (cfg
->method
);
9732 int len
= strlen (name
);
9734 LLVMTypeRef type
= LLVMArrayType (LLVMInt8Type (), len
+ 1);
9735 LLVMValueRef name_var
= LLVMAddGlobal (ctx
->lmodule
, type
, "missing_method_name");
9736 LLVMSetVisibility (name_var
, LLVMHiddenVisibility
);
9737 LLVMSetLinkage (name_var
, LLVMInternalLinkage
);
9738 LLVMSetInitializer (name_var
, mono_llvm_create_constant_data_array ((guint8
*)name
, len
+ 1));
9739 mono_llvm_set_is_constant (name_var
);
9743 sig
= LLVMFunctionType1 (LLVMVoidType (), ctx
->module
->ptr_type
, FALSE
);
9744 LLVMValueRef callee
= get_callee (ctx
, sig
, MONO_PATCH_INFO_JIT_ICALL_ADDR
, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_aot_failed_exception
));
9745 LLVMValueRef args
[] = { convert (ctx
, name_var
, ctx
->module
->ptr_type
) };
9746 LLVMBuildCall (ctx
->builder
, callee
, args
, 1, "");
9747 LLVMBuildUnreachable (ctx
->builder
);
9750 /* Initialize the method if needed */
9751 if (cfg
->compile_aot
) {
9752 // FIXME: Add more shared got entries
9753 ctx
->builder
= create_builder (ctx
);
9754 LLVMPositionBuilderAtEnd (ctx
->builder
, ctx
->init_bb
);
9756 // FIXME: beforefieldinit
9758 * NATIVE_TO_MANAGED methods might be called on a thread not attached to the runtime, so they are initialized when loaded
9759 * in load_method ().
9761 gboolean needs_init
= ctx
->cfg
->got_access_count
> 0;
9762 MonoMethod
*cctor
= NULL
;
9763 if (!needs_init
&& (cctor
= mono_class_get_cctor (cfg
->method
->klass
))) {
9764 /* Needs init to run the cctor */
9765 if (cfg
->method
->flags
& METHOD_ATTRIBUTE_STATIC
)
9767 if (cctor
== cfg
->method
)
9770 // If we are a constructor, we need to init so the static
9771 // constructor gets called.
9772 if (!strcmp (cfg
->method
->name
, ".ctor"))
9775 if (cfg
->method
->wrapper_type
== MONO_WRAPPER_NATIVE_TO_MANAGED
)
9778 emit_method_init (ctx
);
9780 LLVMBuildBr (ctx
->builder
, ctx
->inited_bb
);
9782 // Was observing LLVM moving field accesses into the caller's method
9783 // body before the init call (the inlined one), leading to NULL derefs
9784 // after the init_method returns (GOT is filled out though)
9786 mono_llvm_add_func_attr (method
, LLVM_ATTR_NO_INLINE
);
9789 if (mini_get_debug_options ()->llvm_disable_inlining
)
9790 mono_llvm_add_func_attr (method
, LLVM_ATTR_NO_INLINE
);
9793 g_ptr_array_add (ctx
->module
->cfgs
, cfg
);
9794 if (cfg
->llvm_only
) {
9796 * Add the contents of ctx->callsite_list to module->callsite_list.
9797 * We can't do this earlier, as it contains llvm instructions which can be
9798 * freed if compilation fails.
9799 * FIXME: Get rid of this when all methods can be llvm compiled.
9801 for (int i
= 0; i
< ctx
->callsite_list
->len
; ++i
)
9802 g_ptr_array_add (ctx
->module
->callsite_list
, g_ptr_array_index (ctx
->callsite_list
, i
));
9805 if (cfg
->verbose_level
> 1) {
9806 g_print ("\n*** Unoptimized LLVM IR for %s ***\n", mono_method_full_name (cfg
->method
, TRUE
));
9807 if (cfg
->compile_aot
) {
9808 mono_llvm_dump_value (method
);
9810 mono_llvm_dump_module (ctx
->lmodule
);
9812 g_print ("***\n\n");
9815 if (cfg
->compile_aot
&& !cfg
->llvm_only
)
9816 mark_as_used (ctx
->module
, method
);
9818 if (!cfg
->llvm_only
) {
9819 LLVMValueRef md_args
[16];
9820 LLVMValueRef md_node
;
9823 if (cfg
->compile_aot
)
9824 method_index
= mono_aot_get_method_index (cfg
->orig_method
);
9827 md_args
[0] = LLVMMDString (ctx
->method_name
, strlen (ctx
->method_name
));
9828 md_args
[1] = LLVMConstInt (LLVMInt32Type (), method_index
, FALSE
);
9829 md_node
= LLVMMDNode (md_args
, 2);
9830 LLVMAddNamedMetadataOperand (lmodule
, "mono.function_indexes", md_node
);
9831 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
9834 if (cfg
->compile_aot
) {
9835 /* Don't generate native code, keep the LLVM IR */
9836 if (cfg
->verbose_level
) {
9837 char *name
= mono_method_get_full_name (cfg
->method
);
9838 printf ("%s emitted as %s\n", name
, ctx
->method_name
);
9842 //LLVMDumpValue (ctx->lmethod);
9843 //int err = LLVMVerifyFunction (ctx->lmethod, LLVMPrintMessageAction);
9844 //g_assert (err == 0);
9846 //LLVMVerifyFunction (method, 0);
9847 llvm_jit_finalize_method (ctx
);
9850 if (ctx
->module
->method_to_lmethod
)
9851 g_hash_table_insert (ctx
->module
->method_to_lmethod
, cfg
->method
, ctx
->lmethod
);
9853 if (ctx
->module
->idx_to_lmethod
)
9854 g_hash_table_insert (ctx
->module
->idx_to_lmethod
, GINT_TO_POINTER (cfg
->method_index
), ctx
->lmethod
);
9856 if (ctx
->llvm_only
&& m_class_is_valuetype (cfg
->orig_method
->klass
) && !(cfg
->orig_method
->flags
& METHOD_ATTRIBUTE_STATIC
))
9857 emit_unbox_tramp (ctx
, ctx
->method_name
, ctx
->method_type
, ctx
->lmethod
, cfg
->method_index
);
9861 * mono_llvm_create_vars:
9863 * Same as mono_arch_create_vars () for LLVM.
9866 mono_llvm_create_vars (MonoCompile
*cfg
)
9868 MonoMethodSignature
*sig
;
9870 sig
= mono_method_signature_internal (cfg
->method
);
9871 if (cfg
->gsharedvt
&& cfg
->llvm_only
) {
9872 gboolean vretaddr
= FALSE
;
9874 if (mini_is_gsharedvt_variable_signature (sig
) && sig
->ret
->type
!= MONO_TYPE_VOID
) {
9877 MonoMethodSignature
*sig
= mono_method_signature_internal (cfg
->method
);
9878 LLVMCallInfo
*linfo
;
9880 linfo
= get_llvm_call_info (cfg
, sig
);
9881 vretaddr
= (linfo
->ret
.storage
== LLVMArgVtypeRetAddr
|| linfo
->ret
.storage
== LLVMArgVtypeByRef
|| linfo
->ret
.storage
== LLVMArgGsharedvtFixed
|| linfo
->ret
.storage
== LLVMArgGsharedvtVariable
|| linfo
->ret
.storage
== LLVMArgGsharedvtFixedVtype
);
9885 * Creating vret_addr forces CEE_SETRET to store the result into it,
9886 * so we don't have to generate any code in our OP_SETRET case.
9888 cfg
->vret_addr
= mono_compile_create_var (cfg
, m_class_get_byval_arg (mono_get_intptr_class ()), OP_ARG
);
9889 if (G_UNLIKELY (cfg
->verbose_level
> 1)) {
9890 printf ("vret_addr = ");
9891 mono_print_ins (cfg
->vret_addr
);
9895 mono_arch_create_vars (cfg
);
9900 * mono_llvm_emit_call:
9902 * Same as mono_arch_emit_call () for LLVM.
9905 mono_llvm_emit_call (MonoCompile
*cfg
, MonoCallInst
*call
)
9908 MonoMethodSignature
*sig
;
9909 int i
, n
, stack_size
;
9914 sig
= call
->signature
;
9915 n
= sig
->param_count
+ sig
->hasthis
;
9917 call
->cinfo
= get_llvm_call_info (cfg
, sig
);
9919 if (cfg
->disable_llvm
)
9922 if (sig
->call_convention
== MONO_CALL_VARARG
) {
9923 cfg
->exception_message
= g_strdup ("varargs");
9924 cfg
->disable_llvm
= TRUE
;
9927 for (i
= 0; i
< n
; ++i
) {
9930 ainfo
= call
->cinfo
->args
+ i
;
9932 in
= call
->args
[i
];
9934 /* Simply remember the arguments */
9935 switch (ainfo
->storage
) {
9936 case LLVMArgNormal
: {
9937 MonoType
*t
= (sig
->hasthis
&& i
== 0) ? m_class_get_byval_arg (mono_get_intptr_class ()) : ainfo
->type
;
9940 opcode
= mono_type_to_regmove (cfg
, t
);
9941 if (opcode
== OP_FMOVE
) {
9942 MONO_INST_NEW (cfg
, ins
, OP_FMOVE
);
9943 ins
->dreg
= mono_alloc_freg (cfg
);
9944 } else if (opcode
== OP_LMOVE
) {
9945 MONO_INST_NEW (cfg
, ins
, OP_LMOVE
);
9946 ins
->dreg
= mono_alloc_lreg (cfg
);
9947 } else if (opcode
== OP_RMOVE
) {
9948 MONO_INST_NEW (cfg
, ins
, OP_RMOVE
);
9949 ins
->dreg
= mono_alloc_freg (cfg
);
9951 MONO_INST_NEW (cfg
, ins
, OP_MOVE
);
9952 ins
->dreg
= mono_alloc_ireg (cfg
);
9954 ins
->sreg1
= in
->dreg
;
9957 case LLVMArgVtypeByVal
:
9958 case LLVMArgVtypeByRef
:
9959 case LLVMArgVtypeInReg
:
9960 case LLVMArgVtypeAddr
:
9961 case LLVMArgVtypeAsScalar
:
9962 case LLVMArgAsIArgs
:
9963 case LLVMArgAsFpArgs
:
9964 case LLVMArgGsharedvtVariable
:
9965 case LLVMArgGsharedvtFixed
:
9966 case LLVMArgGsharedvtFixedVtype
:
9967 MONO_INST_NEW (cfg
, ins
, OP_LLVM_OUTARG_VT
);
9968 ins
->dreg
= mono_alloc_ireg (cfg
);
9969 ins
->sreg1
= in
->dreg
;
9970 ins
->inst_p0
= mono_mempool_alloc0 (cfg
->mempool
, sizeof (LLVMArgInfo
));
9971 memcpy (ins
->inst_p0
, ainfo
, sizeof (LLVMArgInfo
));
9972 ins
->inst_vtype
= ainfo
->type
;
9973 ins
->klass
= mono_class_from_mono_type_internal (ainfo
->type
);
9976 cfg
->exception_message
= g_strdup ("ainfo->storage");
9977 cfg
->disable_llvm
= TRUE
;
9981 if (!cfg
->disable_llvm
) {
9982 MONO_ADD_INS (cfg
->cbb
, ins
);
9983 mono_call_inst_add_outarg_reg (cfg
, call
, ins
->dreg
, 0, FALSE
);
9989 add_func (LLVMModuleRef module
, const char *name
, LLVMTypeRef ret_type
, LLVMTypeRef
*param_types
, int nparams
)
9991 LLVMAddFunction (module
, name
, LLVMFunctionType (ret_type
, param_types
, nparams
, FALSE
));
9995 add_intrins (LLVMModuleRef module
, IntrinsicId id
, LLVMTypeRef
*params
, int nparams
)
9997 return mono_llvm_register_overloaded_intrinsic (module
, id
, params
, nparams
);
10000 static LLVMValueRef
10001 add_intrins1 (LLVMModuleRef module
, IntrinsicId id
, LLVMTypeRef param1
)
10003 return mono_llvm_register_overloaded_intrinsic (module
, id
, ¶m1
, 1);
10006 static LLVMValueRef
10007 add_intrins2 (LLVMModuleRef module
, IntrinsicId id
, LLVMTypeRef param1
, LLVMTypeRef param2
)
10009 LLVMTypeRef params
[] = { param1
, param2
};
10010 return mono_llvm_register_overloaded_intrinsic (module
, id
, params
, 2);
10013 static LLVMValueRef
10014 add_intrins3 (LLVMModuleRef module
, IntrinsicId id
, LLVMTypeRef param1
, LLVMTypeRef param2
, LLVMTypeRef param3
)
10016 LLVMTypeRef params
[] = { param1
, param2
, param3
};
10017 return mono_llvm_register_overloaded_intrinsic (module
, id
, params
, 3);
10021 add_intrinsic (LLVMModuleRef module
, int id
)
10023 /* Register simple intrinsics */
10024 LLVMValueRef intrins
= mono_llvm_register_intrinsic (module
, (IntrinsicId
)id
);
10026 g_hash_table_insert (intrins_id_to_intrins
, GINT_TO_POINTER (id
), intrins
);
10030 /* Register overloaded intrinsics */
10032 case INTRINS_MEMSET
:
10033 intrins
= add_intrins2 (module
, id
, LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ());
10035 case INTRINS_MEMCPY
:
10036 intrins
= add_intrins3 (module
, id
, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ());
10038 case INTRINS_MEMMOVE
:
10039 intrins
= add_intrins3 (module
, id
, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type ());
10041 case INTRINS_SADD_OVF_I32
:
10042 case INTRINS_UADD_OVF_I32
:
10043 case INTRINS_SSUB_OVF_I32
:
10044 case INTRINS_USUB_OVF_I32
:
10045 case INTRINS_SMUL_OVF_I32
:
10046 case INTRINS_UMUL_OVF_I32
:
10047 intrins
= add_intrins1 (module
, id
, LLVMInt32Type ());
10049 case INTRINS_SADD_OVF_I64
:
10050 case INTRINS_UADD_OVF_I64
:
10051 case INTRINS_SSUB_OVF_I64
:
10052 case INTRINS_USUB_OVF_I64
:
10053 case INTRINS_SMUL_OVF_I64
:
10054 case INTRINS_UMUL_OVF_I64
:
10055 intrins
= add_intrins1 (module
, id
, LLVMInt64Type ());
10061 case INTRINS_LOG10
:
10062 case INTRINS_TRUNC
:
10066 case INTRINS_FLOOR
:
10069 case INTRINS_COPYSIGN
:
10071 intrins
= add_intrins1 (module
, id
, LLVMDoubleType ());
10075 case INTRINS_LOG2F
:
10076 case INTRINS_LOG10F
:
10077 case INTRINS_TRUNCF
:
10080 case INTRINS_SQRTF
:
10081 case INTRINS_FLOORF
:
10082 case INTRINS_CEILF
:
10084 case INTRINS_COPYSIGNF
:
10086 intrins
= add_intrins1 (module
, id
, LLVMFloatType ());
10088 case INTRINS_EXPECT_I8
:
10089 intrins
= add_intrins1 (module
, id
, LLVMInt8Type ());
10091 case INTRINS_EXPECT_I1
:
10092 intrins
= add_intrins1 (module
, id
, LLVMInt1Type ());
10094 case INTRINS_CTPOP_I32
:
10095 case INTRINS_CTLZ_I32
:
10096 case INTRINS_CTTZ_I32
:
10097 case INTRINS_BEXTR_I32
:
10098 case INTRINS_BZHI_I32
:
10099 case INTRINS_PEXT_I32
:
10100 case INTRINS_PDEP_I32
:
10101 intrins
= add_intrins1 (module
, id
, LLVMInt32Type ());
10103 case INTRINS_CTPOP_I64
:
10104 case INTRINS_BEXTR_I64
:
10105 case INTRINS_BZHI_I64
:
10106 case INTRINS_PEXT_I64
:
10107 case INTRINS_PDEP_I64
:
10108 case INTRINS_CTLZ_I64
:
10109 case INTRINS_CTTZ_I64
:
10110 intrins
= add_intrins1 (module
, id
, LLVMInt64Type ());
10112 #if defined(TARGET_AMD64) || defined(TARGET_X86)
10113 case INTRINS_SSE_SADD_SATI8
:
10114 case INTRINS_SSE_UADD_SATI8
:
10115 case INTRINS_SSE_SSUB_SATI8
:
10116 case INTRINS_SSE_USUB_SATI8
:
10117 intrins
= add_intrins1 (module
, id
, sse_i1_t
);
10119 case INTRINS_SSE_SADD_SATI16
:
10120 case INTRINS_SSE_UADD_SATI16
:
10121 case INTRINS_SSE_SSUB_SATI16
:
10122 case INTRINS_SSE_USUB_SATI16
:
10123 intrins
= add_intrins1 (module
, id
, sse_i2_t
);
10125 #if LLVM_API_VERSION >= 700
10126 case INTRINS_SSE_SQRT_PS
:
10127 intrins
= add_intrins1 (module
, id
, sse_r4_t
);
10129 case INTRINS_SSE_SQRT_PD
:
10130 intrins
= add_intrins1 (module
, id
, sse_r8_t
);
10132 case INTRINS_SSE_SQRT_SS
:
10133 intrins
= add_intrins1 (module
, id
, LLVMFloatType ());
10135 case INTRINS_SSE_SQRT_SD
:
10136 intrins
= add_intrins1 (module
, id
, LLVMDoubleType ());
10138 #endif /* LLVM_API_VERSION >= 700 */
10139 #endif /* AMD64 || X86 */
10140 #if defined(TARGET_WASM) && LLVM_API_VERSION >= 800
10141 case INTRINS_WASM_ANYTRUE_V16
:
10142 intrins
= add_intrins1 (module
, id
, sse_i1_t
);
10144 case INTRINS_WASM_ANYTRUE_V8
:
10145 intrins
= add_intrins1 (module
, id
, sse_i2_t
);
10147 case INTRINS_WASM_ANYTRUE_V4
:
10148 intrins
= add_intrins1 (module
, id
, sse_i4_t
);
10150 case INTRINS_WASM_ANYTRUE_V2
:
10151 intrins
= add_intrins1 (module
, id
, sse_i8_t
);
10155 g_assert_not_reached ();
10158 g_assert (intrins
);
10159 g_hash_table_insert (intrins_id_to_intrins
, GINT_TO_POINTER (id
), intrins
);
10162 static LLVMValueRef
10163 get_intrins_from_module (LLVMModuleRef lmodule
, int id
)
10167 res
= (LLVMValueRef
)g_hash_table_lookup (intrins_id_to_intrins
, GINT_TO_POINTER (id
));
10172 static LLVMValueRef
10173 get_intrins (EmitContext
*ctx
, int id
)
10175 MonoLLVMModule
*module
= ctx
->module
;
10179 * Every method is emitted into its own module so
10180 * we can add intrinsics on demand.
10182 res
= module
->intrins_by_id
[id
];
10184 res
= get_intrins_from_module (ctx
->lmodule
, id
);
10185 module
->intrins_by_id
[id
] = res
;
10191 add_intrinsics (LLVMModuleRef module
)
10195 /* Emit declarations of instrinsics */
10197 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
10198 * type doesn't seem to do any locking.
10200 for (i
= 0; i
< INTRINS_NUM
; ++i
)
10201 add_intrinsic (module
, i
);
10203 /* EH intrinsics */
10204 add_func (module
, "mono_personality", LLVMVoidType (), NULL
, 0);
10205 add_func (module
, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL
, 0);
10209 add_types (MonoLLVMModule
*module
)
10211 module
->ptr_type
= LLVMPointerType (TARGET_SIZEOF_VOID_P
== 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
10215 mono_llvm_init (gboolean enable_jit
)
10217 sse_i1_t
= type_to_sse_type (MONO_TYPE_I1
);
10218 sse_i2_t
= type_to_sse_type (MONO_TYPE_I2
);
10219 sse_i4_t
= type_to_sse_type (MONO_TYPE_I4
);
10220 sse_i8_t
= type_to_sse_type (MONO_TYPE_I8
);
10221 sse_r4_t
= type_to_sse_type (MONO_TYPE_R4
);
10222 sse_r8_t
= type_to_sse_type (MONO_TYPE_R8
);
10224 intrins_id_to_intrins
= g_hash_table_new (NULL
, NULL
);
10227 mono_llvm_jit_init ();
10231 mono_llvm_cleanup (void)
10233 MonoLLVMModule
*module
= &aot_module
;
10235 if (module
->lmodule
)
10236 LLVMDisposeModule (module
->lmodule
);
10238 if (module
->context
)
10239 LLVMContextDispose (module
->context
);
10243 mono_llvm_free_domain_info (MonoDomain
*domain
)
10245 MonoJitDomainInfo
*info
= domain_jit_info (domain
);
10246 MonoLLVMModule
*module
= (MonoLLVMModule
*)info
->llvm_module
;
10252 g_hash_table_destroy (module
->llvm_types
);
10254 mono_llvm_dispose_ee (module
->mono_ee
);
10256 if (module
->bb_names
) {
10257 for (i
= 0; i
< module
->bb_names_len
; ++i
)
10258 g_free (module
->bb_names
[i
]);
10259 g_free (module
->bb_names
);
10261 //LLVMDisposeModule (module->module);
10265 info
->llvm_module
= NULL
;
10269 mono_llvm_create_aot_module (MonoAssembly
*assembly
, const char *global_prefix
, int initial_got_size
, LLVMModuleFlags flags
)
10271 MonoLLVMModule
*module
= &aot_module
;
10272 gboolean emit_dwarf
= (flags
& LLVM_MODULE_FLAG_DWARF
) ? 1 : 0;
10273 #ifdef TARGET_WIN32_MSVC
10274 gboolean emit_codeview
= (flags
& LLVM_MODULE_FLAG_CODEVIEW
) ? 1 : 0;
10276 gboolean static_link
= (flags
& LLVM_MODULE_FLAG_STATIC
) ? 1 : 0;
10277 gboolean llvm_only
= (flags
& LLVM_MODULE_FLAG_LLVM_ONLY
) ? 1 : 0;
10278 gboolean interp
= (flags
& LLVM_MODULE_FLAG_INTERP
) ? 1 : 0;
10280 /* Delete previous module */
10281 g_hash_table_destroy (module
->plt_entries
);
10282 if (module
->lmodule
)
10283 LLVMDisposeModule (module
->lmodule
);
10285 memset (module
, 0, sizeof (aot_module
));
10287 module
->lmodule
= LLVMModuleCreateWithName ("aot");
10288 module
->assembly
= assembly
;
10289 module
->global_prefix
= g_strdup (global_prefix
);
10290 module
->eh_frame_symbol
= g_strdup_printf ("%s_eh_frame", global_prefix
);
10291 module
->get_method_symbol
= g_strdup_printf ("%s_get_method", global_prefix
);
10292 module
->get_unbox_tramp_symbol
= g_strdup_printf ("%s_get_unbox_tramp", global_prefix
);
10293 module
->init_aotconst_symbol
= g_strdup_printf ("%s_init_aotconst", global_prefix
);
10294 module
->external_symbols
= TRUE
;
10295 module
->emit_dwarf
= emit_dwarf
;
10296 module
->static_link
= static_link
;
10297 module
->llvm_only
= llvm_only
;
10298 module
->interp
= interp
;
10299 /* The first few entries are reserved */
10300 module
->max_got_offset
= initial_got_size
;
10301 module
->context
= LLVMGetGlobalContext ();
10302 module
->cfgs
= g_ptr_array_new ();
10303 module
->intrins_by_id
= g_new0 (LLVMValueRef
, INTRINS_NUM
);
10304 module
->aotconst_vars
= g_hash_table_new (NULL
, NULL
);
10305 module
->llvm_types
= g_hash_table_new (NULL
, NULL
);
10306 module
->plt_entries
= g_hash_table_new (g_str_hash
, g_str_equal
);
10307 module
->plt_entries_ji
= g_hash_table_new (NULL
, NULL
);
10308 module
->direct_callables
= g_hash_table_new (g_str_hash
, g_str_equal
);
10309 module
->idx_to_lmethod
= g_hash_table_new (NULL
, NULL
);
10310 module
->method_to_lmethod
= g_hash_table_new (NULL
, NULL
);
10311 module
->method_to_call_info
= g_hash_table_new (NULL
, NULL
);
10312 module
->idx_to_unbox_tramp
= g_hash_table_new (NULL
, NULL
);
10313 module
->no_method_table_lmethods
= g_hash_table_new (NULL
, NULL
);
10314 module
->callsite_list
= g_ptr_array_new ();
10317 /* clang ignores our debug info because it has an invalid version */
10318 module
->emit_dwarf
= FALSE
;
10320 add_intrinsics (module
->lmodule
);
10321 add_types (module
);
10323 #ifdef MONO_ARCH_LLVM_TARGET_LAYOUT
10324 LLVMSetDataLayout (module
->lmodule
, MONO_ARCH_LLVM_TARGET_LAYOUT
);
10326 g_assert_not_reached ();
10329 #ifdef MONO_ARCH_LLVM_TARGET_TRIPLE
10330 LLVMSetTarget (module
->lmodule
, MONO_ARCH_LLVM_TARGET_TRIPLE
);
10333 if (module
->emit_dwarf
) {
10334 char *dir
, *build_info
, *s
, *cu_name
;
10336 module
->di_builder
= mono_llvm_create_di_builder (module
->lmodule
);
10339 dir
= g_strdup (".");
10340 build_info
= mono_get_runtime_build_info ();
10341 s
= g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info
);
10342 cu_name
= g_path_get_basename (assembly
->image
->name
);
10343 module
->cu
= mono_llvm_di_create_compile_unit (module
->di_builder
, cu_name
, dir
, s
);
10345 g_free (build_info
);
10349 #ifdef TARGET_WIN32_MSVC
10350 if (emit_codeview
) {
10351 LLVMValueRef codeview_option_args
[3];
10353 codeview_option_args
[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE
);
10354 codeview_option_args
[1] = LLVMMDString ("CodeView", 8);
10355 codeview_option_args
[2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE
);
10357 LLVMAddNamedMetadataOperand (module
->lmodule
, "llvm.module.flags", LLVMMDNode (codeview_option_args
, G_N_ELEMENTS (codeview_option_args
)));
10360 if (!static_link
) {
10361 const char linker_options
[] = "Linker Options";
10362 const char *default_dynamic_lib_names
[] = { "/DEFAULTLIB:msvcrt",
10363 "/DEFAULTLIB:ucrt.lib",
10364 "/DEFAULTLIB:vcruntime.lib" };
10366 LLVMValueRef default_lib_args
[G_N_ELEMENTS (default_dynamic_lib_names
)];
10367 LLVMValueRef default_lib_nodes
[G_N_ELEMENTS(default_dynamic_lib_names
)];
10369 const char *default_lib_name
= NULL
;
10370 for (int i
= 0; i
< G_N_ELEMENTS (default_dynamic_lib_names
); ++i
) {
10371 const char *default_lib_name
= default_dynamic_lib_names
[i
];
10372 default_lib_args
[i
] = LLVMMDString (default_lib_name
, strlen (default_lib_name
));
10373 default_lib_nodes
[i
] = LLVMMDNode (default_lib_args
+ i
, 1);
10376 LLVMAddNamedMetadataOperand (module
->lmodule
, "llvm.linker.options", LLVMMDNode (default_lib_args
, G_N_ELEMENTS (default_lib_args
)));
10381 LLVMTypeRef got_type
= LLVMArrayType (module
->ptr_type
, 16);
10383 module
->dummy_got_var
= LLVMAddGlobal (module
->lmodule
, got_type
, "dummy_got");
10384 module
->got_idx_to_type
= g_hash_table_new (NULL
, NULL
);
10385 LLVMSetInitializer (module
->dummy_got_var
, LLVMConstNull (got_type
));
10386 LLVMSetVisibility (module
->dummy_got_var
, LLVMHiddenVisibility
);
10387 LLVMSetLinkage (module
->dummy_got_var
, LLVMInternalLinkage
);
10390 /* Add initialization array */
10391 LLVMTypeRef inited_type
= LLVMArrayType (LLVMInt8Type (), 0);
10393 module
->inited_var
= LLVMAddGlobal (aot_module
.lmodule
, inited_type
, "mono_inited_tmp");
10394 LLVMSetInitializer (module
->inited_var
, LLVMConstNull (inited_type
));
10396 create_aot_info_var (module
);
10398 emit_gc_safepoint_poll (module
, module
->lmodule
, NULL
);
10400 emit_llvm_code_start (module
);
10402 // Needs idx_to_lmethod
10403 emit_init_funcs (module
);
10405 /* Add a dummy personality function */
10406 if (!use_mono_personality_debug
) {
10407 LLVMValueRef personality
= LLVMAddFunction (module
->lmodule
, default_personality_name
, LLVMFunctionType (LLVMInt32Type (), NULL
, 0, TRUE
));
10408 LLVMSetLinkage (personality
, LLVMExternalLinkage
);
10410 //EMCC chockes if the personality function is referenced in the 'used' array
10411 #ifndef TARGET_WASM
10412 mark_as_used (module
, personality
);
10416 /* Add a reference to the c++ exception we throw/catch */
10418 LLVMTypeRef exc
= LLVMPointerType (LLVMInt8Type (), 0);
10419 module
->sentinel_exception
= LLVMAddGlobal (module
->lmodule
, exc
, "_ZTIPi");
10420 LLVMSetLinkage (module
->sentinel_exception
, LLVMExternalLinkage
);
10421 mono_llvm_set_is_constant (module
->sentinel_exception
);
10426 mono_llvm_fixup_aot_module (void)
10428 MonoLLVMModule
*module
= &aot_module
;
10429 MonoMethod
*method
;
10432 * Replace GOT entries for directly callable methods with the methods themselves.
10433 * It would be easier to implement this by predefining all methods before compiling
10434 * their bodies, but that couldn't handle the case when a method fails to compile
10438 GHashTable
*specializable
= g_hash_table_new (NULL
, NULL
);
10440 GHashTable
*patches_to_null
= g_hash_table_new (mono_patch_info_hash
, mono_patch_info_equal
);
10441 for (int sindex
= 0; sindex
< module
->callsite_list
->len
; ++sindex
) {
10442 CallSite
*site
= (CallSite
*)g_ptr_array_index (module
->callsite_list
, sindex
);
10443 method
= site
->method
;
10444 LLVMValueRef lmethod
= (LLVMValueRef
)g_hash_table_lookup (module
->method_to_lmethod
, method
);
10445 LLVMValueRef placeholder
= (LLVMValueRef
)site
->load
;
10448 gboolean can_direct_call
= FALSE
;
10450 /* Replace sharable instances with their shared version */
10451 if (!lmethod
&& method
->is_inflated
) {
10452 if (mono_method_is_generic_sharable_full (method
, FALSE
, TRUE
, FALSE
)) {
10453 ERROR_DECL (error
);
10454 MonoMethod
*shared
= mini_get_shared_method_full (method
, SHARE_MODE_NONE
, error
);
10455 if (is_ok (error
)) {
10456 lmethod
= (LLVMValueRef
)g_hash_table_lookup (module
->method_to_lmethod
, shared
);
10463 if (lmethod
&& !m_method_is_synchronized (method
)) {
10464 can_direct_call
= TRUE
;
10465 } else if (m_method_is_wrapper (method
) && !method
->is_inflated
) {
10466 WrapperInfo
*info
= mono_marshal_get_wrapper_info (method
);
10468 /* This is a call from the synchronized wrapper to the real method */
10469 if (info
->subtype
== WRAPPER_SUBTYPE_SYNCHRONIZED_INNER
) {
10470 method
= info
->d
.synchronized
.method
;
10471 lmethod
= (LLVMValueRef
)g_hash_table_lookup (module
->method_to_lmethod
, method
);
10473 can_direct_call
= TRUE
;
10477 if (can_direct_call
) {
10478 mono_llvm_replace_uses_of (placeholder
, lmethod
);
10480 if (mono_aot_can_specialize (method
))
10481 g_hash_table_insert (specializable
, lmethod
, method
);
10483 g_hash_table_insert (patches_to_null
, site
->ji
, site
->ji
);
10486 LLVMBuilderRef builder
= LLVMCreateBuilder ();
10487 LLVMPositionBuilderBefore (builder
, placeholder
);
10489 load
= get_aotconst_module (module
, builder
, site
->ji
->type
, site
->ji
->data
.target
, site
->type
, NULL
, NULL
);
10490 LLVMReplaceAllUsesWith (placeholder
, load
);
10495 mono_llvm_propagate_nonnull_final (specializable
, module
);
10496 g_hash_table_destroy (specializable
);
10498 for (int i
= 0; i
< module
->cfgs
->len
; ++i
) {
10500 * Nullify the patches pointing to direct calls. This is needed to
10501 * avoid allocating extra got slots, which is a perf problem and it
10502 * makes module->max_got_offset invalid.
10503 * It would be better to just store the patch_info in CallSite, but
10504 * cfg->patch_info is copied in aot-compiler.c.
10506 MonoCompile
*cfg
= (MonoCompile
*)g_ptr_array_index (module
->cfgs
, i
);
10507 for (MonoJumpInfo
*patch_info
= cfg
->patch_info
; patch_info
; patch_info
= patch_info
->next
) {
10508 if (patch_info
->type
== MONO_PATCH_INFO_METHOD
) {
10509 if (g_hash_table_lookup (patches_to_null
, patch_info
)) {
10510 patch_info
->type
= MONO_PATCH_INFO_NONE
;
10511 /* Nullify the call to init_method () if possible */
10512 g_assert (cfg
->got_access_count
);
10513 cfg
->got_access_count
--;
10514 if (cfg
->got_access_count
== 0) {
10515 LLVMValueRef br
= (LLVMValueRef
)cfg
->llvmonly_init_cond
;
10517 LLVMSetSuccessor (br
, 0, LLVMGetSuccessor (br
, 1));
10524 g_hash_table_destroy (patches_to_null
);
10527 static LLVMValueRef
10528 llvm_array_from_uints (LLVMTypeRef el_type
, guint32
*values
, int nvalues
)
10531 LLVMValueRef res
, *vals
;
10533 vals
= g_new0 (LLVMValueRef
, nvalues
);
10534 for (i
= 0; i
< nvalues
; ++i
)
10535 vals
[i
] = LLVMConstInt (LLVMInt32Type (), values
[i
], FALSE
);
10536 res
= LLVMConstArray (LLVMInt32Type (), vals
, nvalues
);
10541 static LLVMValueRef
10542 llvm_array_from_bytes (guint8
*values
, int nvalues
)
10545 LLVMValueRef res
, *vals
;
10547 vals
= g_new0 (LLVMValueRef
, nvalues
);
10548 for (i
= 0; i
< nvalues
; ++i
)
10549 vals
[i
] = LLVMConstInt (LLVMInt8Type (), values
[i
], FALSE
);
10550 res
= LLVMConstArray (LLVMInt8Type (), vals
, nvalues
);
10555 * mono_llvm_emit_aot_file_info:
10557 * Emit the MonoAotFileInfo structure.
10558 * Same as emit_aot_file_info () in aot-compiler.c.
10561 mono_llvm_emit_aot_file_info (MonoAotFileInfo
*info
, gboolean has_jitted_code
)
10563 MonoLLVMModule
*module
= &aot_module
;
10565 /* Save these for later */
10566 memcpy (&module
->aot_info
, info
, sizeof (MonoAotFileInfo
));
10567 module
->has_jitted_code
= has_jitted_code
;
10571 * mono_llvm_emit_aot_data:
10573 * Emit the binary data DATA pointed to by symbol SYMBOL.
10574 * Return the LLVM variable for the data.
10577 mono_llvm_emit_aot_data_aligned (const char *symbol
, guint8
*data
, int data_len
, int align
)
10579 MonoLLVMModule
*module
= &aot_module
;
10583 type
= LLVMArrayType (LLVMInt8Type (), data_len
);
10584 d
= LLVMAddGlobal (module
->lmodule
, type
, symbol
);
10585 LLVMSetVisibility (d
, LLVMHiddenVisibility
);
10586 LLVMSetLinkage (d
, LLVMInternalLinkage
);
10587 LLVMSetInitializer (d
, mono_llvm_create_constant_data_array (data
, data_len
));
10589 LLVMSetAlignment (d
, align
);
10590 mono_llvm_set_is_constant (d
);
10595 mono_llvm_emit_aot_data (const char *symbol
, guint8
*data
, int data_len
)
10597 return mono_llvm_emit_aot_data_aligned (symbol
, data
, data_len
, 8);
10600 /* Add a reference to a global defined in JITted code */
10601 static LLVMValueRef
10602 AddJitGlobal (MonoLLVMModule
*module
, LLVMTypeRef type
, const char *name
)
10607 s
= g_strdup_printf ("%s%s", module
->global_prefix
, name
);
10608 v
= LLVMAddGlobal (module
->lmodule
, LLVMInt8Type (), s
);
10609 LLVMSetVisibility (v
, LLVMHiddenVisibility
);
10613 #define FILE_INFO_NUM_HEADER_FIELDS 2
10614 #define FILE_INFO_NUM_SCALAR_FIELDS 23
10615 #define FILE_INFO_NUM_ARRAY_FIELDS 5
10616 #define FILE_INFO_NUM_AOTID_FIELDS 1
10617 #define FILE_INFO_NFIELDS (FILE_INFO_NUM_HEADER_FIELDS + MONO_AOT_FILE_INFO_NUM_SYMBOLS + FILE_INFO_NUM_SCALAR_FIELDS + FILE_INFO_NUM_ARRAY_FIELDS + FILE_INFO_NUM_AOTID_FIELDS)
10620 create_aot_info_var (MonoLLVMModule
*module
)
10622 LLVMTypeRef file_info_type
;
10623 LLVMTypeRef
*eltypes
;
10624 LLVMValueRef info_var
;
10625 int i
, nfields
, tindex
;
10626 LLVMModuleRef lmodule
= module
->lmodule
;
10628 /* Create an LLVM type to represent MonoAotFileInfo */
10629 nfields
= FILE_INFO_NFIELDS
;
10630 eltypes
= g_new (LLVMTypeRef
, nfields
);
10632 eltypes
[tindex
++] = LLVMInt32Type ();
10633 eltypes
[tindex
++] = LLVMInt32Type ();
10635 for (i
= 0; i
< MONO_AOT_FILE_INFO_NUM_SYMBOLS
; ++i
)
10636 eltypes
[tindex
++] = LLVMPointerType (LLVMInt8Type (), 0);
10638 for (i
= 0; i
< FILE_INFO_NUM_SCALAR_FIELDS
; ++i
)
10639 eltypes
[tindex
++] = LLVMInt32Type ();
10641 eltypes
[tindex
++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM
);
10642 for (i
= 0; i
< FILE_INFO_NUM_ARRAY_FIELDS
- 1; ++i
)
10643 eltypes
[tindex
++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM
);
10644 eltypes
[tindex
++] = LLVMArrayType (LLVMInt8Type (), 16);
10645 g_assert (tindex
== nfields
);
10646 file_info_type
= LLVMStructCreateNamed (module
->context
, "MonoAotFileInfo");
10647 LLVMStructSetBody (file_info_type
, eltypes
, nfields
, FALSE
);
10649 info_var
= LLVMAddGlobal (lmodule
, file_info_type
, "mono_aot_file_info");
10651 module
->info_var
= info_var
;
10652 module
->info_var_eltypes
= eltypes
;
10656 emit_aot_file_info (MonoLLVMModule
*module
)
10658 LLVMTypeRef
*eltypes
, eltype
;
10659 LLVMValueRef info_var
;
10660 LLVMValueRef
*fields
;
10661 int i
, nfields
, tindex
;
10662 MonoAotFileInfo
*info
;
10663 LLVMModuleRef lmodule
= module
->lmodule
;
10665 info
= &module
->aot_info
;
10666 info_var
= module
->info_var
;
10667 eltypes
= module
->info_var_eltypes
;
10668 nfields
= FILE_INFO_NFIELDS
;
10670 if (module
->static_link
) {
10671 LLVMSetVisibility (info_var
, LLVMHiddenVisibility
);
10672 LLVMSetLinkage (info_var
, LLVMInternalLinkage
);
10675 #ifdef TARGET_WIN32
10676 if (!module
->static_link
) {
10677 LLVMSetDLLStorageClass (info_var
, LLVMDLLExportStorageClass
);
10681 fields
= g_new (LLVMValueRef
, nfields
);
10683 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->version
, FALSE
);
10684 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->dummy
, FALSE
);
10688 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
10689 * for symbols defined in the .s file emitted by the aot compiler.
10691 eltype
= eltypes
[tindex
];
10692 if (module
->llvm_only
)
10693 fields
[tindex
++] = LLVMConstNull (eltype
);
10695 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "jit_got");
10696 /* llc defines this directly */
10697 if (!module
->llvm_only
) {
10698 fields
[tindex
++] = LLVMAddGlobal (lmodule
, eltype
, module
->eh_frame_symbol
);
10699 fields
[tindex
++] = LLVMConstNull (eltype
);
10700 fields
[tindex
++] = LLVMConstNull (eltype
);
10702 fields
[tindex
++] = LLVMConstNull (eltype
);
10703 fields
[tindex
++] = module
->get_method
;
10704 fields
[tindex
++] = module
->get_unbox_tramp
? module
->get_unbox_tramp
: LLVMConstNull (eltype
);
10706 fields
[tindex
++] = module
->init_aotconst_func
;
10707 if (module
->has_jitted_code
) {
10708 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "jit_code_start");
10709 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "jit_code_end");
10711 fields
[tindex
++] = LLVMConstNull (eltype
);
10712 fields
[tindex
++] = LLVMConstNull (eltype
);
10714 if (!module
->llvm_only
)
10715 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "method_addresses");
10717 fields
[tindex
++] = LLVMConstNull (eltype
);
10718 if (module
->llvm_only
&& module
->unbox_tramp_indexes
) {
10719 fields
[tindex
++] = module
->unbox_tramp_indexes
;
10720 fields
[tindex
++] = module
->unbox_trampolines
;
10722 fields
[tindex
++] = LLVMConstNull (eltype
);
10723 fields
[tindex
++] = LLVMConstNull (eltype
);
10725 if (info
->flags
& MONO_AOT_FILE_FLAG_SEPARATE_DATA
) {
10726 for (i
= 0; i
< MONO_AOT_TABLE_NUM
; ++i
)
10727 fields
[tindex
++] = LLVMConstNull (eltype
);
10729 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "blob");
10730 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "class_name_table");
10731 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "class_info_offsets");
10732 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "method_info_offsets");
10733 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "ex_info_offsets");
10734 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "extra_method_info_offsets");
10735 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "extra_method_table");
10736 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "got_info_offsets");
10737 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "llvm_got_info_offsets");
10738 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "image_table");
10739 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "weak_field_indexes");
10740 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "method_flags_table");
10742 /* Not needed (mem_end) */
10743 fields
[tindex
++] = LLVMConstNull (eltype
);
10744 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "assembly_guid");
10745 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "runtime_version");
10746 if (info
->trampoline_size
[0]) {
10747 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "specific_trampolines");
10748 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "static_rgctx_trampolines");
10749 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "imt_trampolines");
10750 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "gsharedvt_arg_trampolines");
10751 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "ftnptr_arg_trampolines");
10752 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "unbox_arbitrary_trampolines");
10754 fields
[tindex
++] = LLVMConstNull (eltype
);
10755 fields
[tindex
++] = LLVMConstNull (eltype
);
10756 fields
[tindex
++] = LLVMConstNull (eltype
);
10757 fields
[tindex
++] = LLVMConstNull (eltype
);
10758 fields
[tindex
++] = LLVMConstNull (eltype
);
10759 fields
[tindex
++] = LLVMConstNull (eltype
);
10761 if (module
->static_link
&& !module
->llvm_only
)
10762 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "globals");
10764 fields
[tindex
++] = LLVMConstNull (eltype
);
10765 fields
[tindex
++] = LLVMGetNamedGlobal (lmodule
, "assembly_name");
10766 if (!module
->llvm_only
) {
10767 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "plt");
10768 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "plt_end");
10769 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "unwind_info");
10770 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "unbox_trampolines");
10771 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "unbox_trampolines_end");
10772 fields
[tindex
++] = AddJitGlobal (module
, eltype
, "unbox_trampoline_addresses");
10774 fields
[tindex
++] = LLVMConstNull (eltype
);
10775 fields
[tindex
++] = LLVMConstNull (eltype
);
10776 fields
[tindex
++] = LLVMConstNull (eltype
);
10777 fields
[tindex
++] = LLVMConstNull (eltype
);
10778 fields
[tindex
++] = LLVMConstNull (eltype
);
10779 fields
[tindex
++] = LLVMConstNull (eltype
);
10782 for (i
= 0; i
< MONO_AOT_FILE_INFO_NUM_SYMBOLS
; ++i
) {
10783 g_assert (fields
[FILE_INFO_NUM_HEADER_FIELDS
+ i
]);
10784 fields
[FILE_INFO_NUM_HEADER_FIELDS
+ i
] = LLVMConstBitCast (fields
[FILE_INFO_NUM_HEADER_FIELDS
+ i
], eltype
);
10788 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->plt_got_offset_base
, FALSE
);
10789 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->plt_got_info_offset_base
, FALSE
);
10790 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->got_size
, FALSE
);
10791 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->llvm_got_size
, FALSE
);
10792 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->plt_size
, FALSE
);
10793 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->nmethods
, FALSE
);
10794 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->nextra_methods
, FALSE
);
10795 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->flags
, FALSE
);
10796 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->opts
, FALSE
);
10797 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->simd_opts
, FALSE
);
10798 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->gc_name_index
, FALSE
);
10799 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->num_rgctx_fetch_trampolines
, FALSE
);
10800 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->double_align
, FALSE
);
10801 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->long_align
, FALSE
);
10802 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->generic_tramp_num
, FALSE
);
10803 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->card_table_shift_bits
, FALSE
);
10804 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->card_table_mask
, FALSE
);
10805 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->tramp_page_size
, FALSE
);
10806 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->call_table_entry_size
, FALSE
);
10807 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->nshared_got_entries
, FALSE
);
10808 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), info
->datafile_size
, FALSE
);
10809 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), module
->unbox_tramp_num
, FALSE
);
10810 fields
[tindex
++] = LLVMConstInt (LLVMInt32Type (), module
->unbox_tramp_elemsize
, FALSE
);
10812 fields
[tindex
++] = llvm_array_from_uints (LLVMInt32Type (), info
->table_offsets
, MONO_AOT_TABLE_NUM
);
10813 fields
[tindex
++] = llvm_array_from_uints (LLVMInt32Type (), info
->num_trampolines
, MONO_AOT_TRAMP_NUM
);
10814 fields
[tindex
++] = llvm_array_from_uints (LLVMInt32Type (), info
->trampoline_got_offset_base
, MONO_AOT_TRAMP_NUM
);
10815 fields
[tindex
++] = llvm_array_from_uints (LLVMInt32Type (), info
->trampoline_size
, MONO_AOT_TRAMP_NUM
);
10816 fields
[tindex
++] = llvm_array_from_uints (LLVMInt32Type (), info
->tramp_page_code_offsets
, MONO_AOT_TRAMP_NUM
);
10818 fields
[tindex
++] = llvm_array_from_bytes (info
->aotid
, 16);
10819 g_assert (tindex
== nfields
);
10821 LLVMSetInitializer (info_var
, LLVMConstNamedStruct (LLVMGetElementType (LLVMTypeOf (info_var
)), fields
, nfields
));
10823 if (module
->static_link
) {
10827 s
= g_strdup_printf ("mono_aot_module_%s_info", module
->assembly
->aname
.name
);
10828 /* Get rid of characters which cannot occur in symbols */
10830 for (p
= s
; *p
; ++p
) {
10831 if (!(isalnum (*p
) || *p
== '_'))
10834 var
= LLVMAddGlobal (module
->lmodule
, LLVMPointerType (LLVMInt8Type (), 0), s
);
10836 LLVMSetInitializer (var
, LLVMConstBitCast (LLVMGetNamedGlobal (module
->lmodule
, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
10837 LLVMSetLinkage (var
, LLVMExternalLinkage
);
10842 LLVMValueRef lmethod
;
10844 } NonnullPropWorkItem
;
10847 mono_llvm_nonnull_state_update (EmitContext
*ctx
, LLVMValueRef lcall
, MonoMethod
*call_method
, LLVMValueRef
*args
, int num_params
)
10849 if (mono_aot_can_specialize (call_method
)) {
10850 int num_passed
= LLVMGetNumArgOperands (lcall
);
10851 g_assert (num_params
<= num_passed
);
10853 g_assert (ctx
->module
->method_to_call_info
);
10854 GArray
*call_site_union
= (GArray
*) g_hash_table_lookup (ctx
->module
->method_to_call_info
, call_method
);
10856 if (!call_site_union
) {
10857 call_site_union
= g_array_sized_new (FALSE
, TRUE
, sizeof (gint32
), num_params
);
10859 for (int i
= 0; i
< num_params
; i
++)
10860 g_array_insert_val (call_site_union
, i
, zero
);
10863 for (int i
= 0; i
< num_params
; i
++) {
10864 if (mono_llvm_is_nonnull (args
[i
])) {
10865 g_assert (i
< LLVMGetNumArgOperands (lcall
));
10866 mono_llvm_set_call_nonnull_arg (lcall
, i
);
10868 gint32
*nullable_count
= &g_array_index (call_site_union
, gint32
, i
);
10869 *nullable_count
= *nullable_count
+ 1;
10873 g_hash_table_insert (ctx
->module
->method_to_call_info
, call_method
, call_site_union
);
10878 mono_llvm_propagate_nonnull_final (GHashTable
*all_specializable
, MonoLLVMModule
*module
)
10880 // When we first traverse the mini IL, we mark the things that are
10881 // nonnull (the roots). Then, for all of the methods that can be specialized, we
10882 // see if their call sites have nonnull attributes.
10884 // If so, we mark the function's param. This param has uses to propagate
10885 // the attribute to. This propagation can trigger a need to mark more attributes
10886 // non-null, and so on and so forth.
10887 GSList
*queue
= NULL
;
10889 GHashTableIter iter
;
10890 LLVMValueRef lmethod
;
10891 MonoMethod
*method
;
10892 g_hash_table_iter_init (&iter
, all_specializable
);
10893 while (g_hash_table_iter_next (&iter
, (void**)&lmethod
, (void**)&method
)) {
10894 GArray
*call_site_union
= (GArray
*) g_hash_table_lookup (module
->method_to_call_info
, method
);
10896 // Basic sanity checking
10897 if (call_site_union
)
10898 g_assert (call_site_union
->len
== LLVMCountParams (lmethod
));
10900 // Add root to work queue
10901 for (int i
= 0; call_site_union
&& i
< call_site_union
->len
; i
++) {
10902 if (g_array_index (call_site_union
, gint32
, i
) == 0) {
10903 NonnullPropWorkItem
*item
= g_malloc (sizeof (NonnullPropWorkItem
));
10904 item
->lmethod
= lmethod
;
10905 item
->argument
= i
;
10906 queue
= g_slist_prepend (queue
, item
);
10911 // This is essentially reference counting, and we are propagating
10912 // the refcount decrement here. We have less work to do than we may otherwise
10913 // because we are only working with a set of subgraphs of specializable functions.
10915 // We rely on being able to see all of the references in the graph.
10916 // This is ensured by the function mono_aot_can_specialize. Everything in
10917 // all_specializable is a function that can be specialized, and is the resulting
10918 // node in the graph after all of the subsitutions are done.
10920 // Anything disrupting the direct calls made with self-init will break this optimization.
10923 // Update the queue state.
10924 // Our only other per-iteration responsibility is now to free current
10925 NonnullPropWorkItem
*current
= (NonnullPropWorkItem
*) queue
->data
;
10926 queue
= queue
->next
;
10927 g_assert (current
->argument
< LLVMCountParams (current
->lmethod
));
10929 // Does the actual leaf-node work here
10930 // Mark the function argument as nonnull for LLVM
10931 mono_llvm_set_func_nonnull_arg (current
->lmethod
, current
->argument
);
10933 // The rest of this is for propagating forward nullability changes
10934 // to calls that use the argument that is now nullable.
10936 // Get the actual LLVM value of the argument, so we can see which call instructions
10937 // used that argument
10938 LLVMValueRef caller_argument
= LLVMGetParam (current
->lmethod
, current
->argument
);
10940 // Iterate over the calls using the newly-non-nullable argument
10941 GSList
*calls
= mono_llvm_calls_using (caller_argument
);
10942 for (GSList
*cursor
= calls
; cursor
!= NULL
; cursor
= cursor
->next
) {
10944 LLVMValueRef lcall
= (LLVMValueRef
) cursor
->data
;
10945 LLVMValueRef callee_lmethod
= LLVMGetCalledValue (lcall
);
10947 // If this wasn't a direct call for which mono_aot_can_specialize is true,
10948 // this lookup won't find a MonoMethod.
10949 MonoMethod
*callee_method
= (MonoMethod
*) g_hash_table_lookup (all_specializable
, callee_lmethod
);
10950 if (!callee_method
)
10953 // Decrement number of nullable refs at that func's arg offset
10954 GArray
*call_site_union
= (GArray
*) g_hash_table_lookup (module
->method_to_call_info
, callee_method
);
10956 // It has module-local callers and is specializable, should have seen this call site
10958 g_assert (call_site_union
);
10960 // The function *definition* parameter arity should always be consistent
10961 int max_params
= LLVMCountParams (callee_lmethod
);
10962 if (call_site_union
->len
!= max_params
) {
10963 mono_llvm_dump_value (callee_lmethod
);
10964 g_assert_not_reached ();
10967 // Get the values that correspond to the parameters passed to the call
10968 // that used our argument
10969 LLVMValueRef
*operands
= mono_llvm_call_args (lcall
);
10970 for (int call_argument
= 0; call_argument
< max_params
; call_argument
++) {
10971 // Every time we used the newly-non-nullable argument, decrement the nullable
10972 // refcount for that function.
10973 if (caller_argument
== operands
[call_argument
]) {
10974 gint32
*nullable_count
= &g_array_index (call_site_union
, gint32
, call_argument
);
10975 g_assert (*nullable_count
> 0);
10976 *nullable_count
= *nullable_count
- 1;
10978 // If we caused that callee's parameter to become newly nullable, add to work queue
10979 if (*nullable_count
== 0) {
10980 NonnullPropWorkItem
*item
= g_malloc (sizeof (NonnullPropWorkItem
));
10981 item
->lmethod
= callee_lmethod
;
10982 item
->argument
= call_argument
;
10983 queue
= g_slist_prepend (queue
, item
);
10989 // Update nullability refcount information for the callee now
10990 g_hash_table_insert (module
->method_to_call_info
, callee_method
, call_site_union
);
10992 g_slist_free (calls
);
10999 * Emit the aot module into the LLVM bitcode file FILENAME.
11002 mono_llvm_emit_aot_module (const char *filename
, const char *cu_name
)
11004 LLVMTypeRef inited_type
;
11005 LLVMValueRef real_inited
;
11006 MonoLLVMModule
*module
= &aot_module
;
11008 emit_llvm_code_end (module
);
11011 * Create the real init_var and replace all uses of the dummy variable with
11014 inited_type
= LLVMArrayType (LLVMInt8Type (), module
->max_inited_idx
+ 1);
11015 real_inited
= LLVMAddGlobal (module
->lmodule
, inited_type
, "mono_inited");
11016 LLVMSetInitializer (real_inited
, LLVMConstNull (inited_type
));
11017 LLVMSetLinkage (real_inited
, LLVMInternalLinkage
);
11018 mono_llvm_replace_uses_of (module
->inited_var
, real_inited
);
11019 LLVMDeleteGlobal (module
->inited_var
);
11021 /* Replace the dummy info_ variables with the real ones */
11022 for (int i
= 0; i
< module
->cfgs
->len
; ++i
) {
11023 MonoCompile
*cfg
= (MonoCompile
*)g_ptr_array_index (module
->cfgs
, i
);
11025 // FIXME: Eliminate unused vars
11027 // FIXME: Speed this up
11028 if (cfg
->llvm_dummy_info_var
) {
11029 if (cfg
->llvm_info_var
) {
11030 mono_llvm_replace_uses_of (cfg
->llvm_dummy_info_var
, cfg
->llvm_info_var
);
11031 LLVMDeleteGlobal (cfg
->llvm_dummy_info_var
);
11033 // FIXME: How can this happen ?
11034 LLVMSetInitializer (cfg
->llvm_dummy_info_var
, mono_llvm_create_constant_data_array (NULL
, 0));
11039 if (module
->llvm_only
) {
11040 emit_get_method (&aot_module
);
11041 emit_get_unbox_tramp (&aot_module
);
11043 emit_init_aotconst (module
);
11045 emit_llvm_used (&aot_module
);
11046 emit_dbg_info (&aot_module
, filename
, cu_name
);
11047 emit_aot_file_info (&aot_module
);
11049 /* Replace PLT entries for directly callable methods with the methods themselves */
11051 GHashTableIter iter
;
11053 LLVMValueRef callee
;
11055 GHashTable
*specializable
= g_hash_table_new (NULL
, NULL
);
11057 g_hash_table_iter_init (&iter
, module
->plt_entries_ji
);
11058 while (g_hash_table_iter_next (&iter
, (void**)&ji
, (void**)&callee
)) {
11059 if (mono_aot_is_direct_callable (ji
)) {
11060 LLVMValueRef lmethod
;
11062 lmethod
= (LLVMValueRef
)g_hash_table_lookup (module
->method_to_lmethod
, ji
->data
.method
);
11063 /* The types might not match because the caller might pass an rgctx */
11064 if (lmethod
&& LLVMTypeOf (callee
) == LLVMTypeOf (lmethod
)) {
11065 mono_llvm_replace_uses_of (callee
, lmethod
);
11067 if (mono_aot_can_specialize (ji
->data
.method
))
11068 g_hash_table_insert (specializable
, lmethod
, ji
->data
.method
);
11069 mono_aot_mark_unused_llvm_plt_entry (ji
);
11074 mono_llvm_propagate_nonnull_final (specializable
, module
);
11076 g_hash_table_destroy (specializable
);
11079 /* Note: You can still dump an invalid bitcode file by running `llvm-dis`
11080 * in a debugger, set a breakpoint on `LLVMVerifyModule` and fake its
11081 * result to 0 (indicating success). */
11082 LLVMWriteBitcodeToFile (module
->lmodule
, filename
);
11086 char *verifier_err
;
11088 if (LLVMVerifyModule (module
->lmodule
, LLVMReturnStatusAction
, &verifier_err
)) {
11089 printf ("%s\n", verifier_err
);
11090 g_assert_not_reached ();
11097 static LLVMValueRef
11098 md_string (const char *s
)
11100 return LLVMMDString (s
, strlen (s
));
11103 /* Debugging support */
11106 emit_dbg_info (MonoLLVMModule
*module
, const char *filename
, const char *cu_name
)
11108 LLVMModuleRef lmodule
= module
->lmodule
;
11109 LLVMValueRef args
[16], ver
;
11112 * This can only be enabled when LLVM code is emitted into a separate object
11113 * file, since the AOT compiler also emits dwarf info,
11114 * and the abbrev indexes will not be correct since llvm has added its own
11117 if (!module
->emit_dwarf
)
11120 mono_llvm_di_builder_finalize (module
->di_builder
);
11122 args
[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE
);
11123 args
[1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
11124 args
[2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE
);
11125 ver
= LLVMMDNode (args
, 3);
11126 LLVMAddNamedMetadataOperand (lmodule
, "llvm.module.flags", ver
);
11128 args
[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE
);
11129 args
[1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
11130 args
[2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE
);
11131 ver
= LLVMMDNode (args
, 3);
11132 LLVMAddNamedMetadataOperand (lmodule
, "llvm.module.flags", ver
);
11135 static LLVMValueRef
11136 emit_dbg_subprogram (EmitContext
*ctx
, MonoCompile
*cfg
, LLVMValueRef method
, const char *name
)
11138 MonoLLVMModule
*module
= ctx
->module
;
11139 MonoDebugMethodInfo
*minfo
= ctx
->minfo
;
11140 char *source_file
, *dir
, *filename
;
11141 MonoSymSeqPoint
*sym_seq_points
;
11147 mono_debug_get_seq_points (minfo
, &source_file
, NULL
, NULL
, &sym_seq_points
, &n_seq_points
);
11149 source_file
= g_strdup ("<unknown>");
11150 dir
= g_path_get_dirname (source_file
);
11151 filename
= g_path_get_basename (source_file
);
11152 g_free (source_file
);
11154 return (LLVMValueRef
)mono_llvm_di_create_function (module
->di_builder
, module
->cu
, method
, cfg
->method
->name
, name
, dir
, filename
, n_seq_points
? sym_seq_points
[0].line
: 1);
11158 emit_dbg_loc (EmitContext
*ctx
, LLVMBuilderRef builder
, const unsigned char *cil_code
)
11160 MonoCompile
*cfg
= ctx
->cfg
;
11162 if (ctx
->minfo
&& cil_code
&& cil_code
>= cfg
->header
->code
&& cil_code
< cfg
->header
->code
+ cfg
->header
->code_size
) {
11163 MonoDebugSourceLocation
*loc
;
11164 LLVMValueRef loc_md
;
11166 loc
= mono_debug_method_lookup_location (ctx
->minfo
, cil_code
- cfg
->header
->code
);
11169 loc_md
= (LLVMValueRef
)mono_llvm_di_create_location (ctx
->module
->di_builder
, ctx
->dbg_md
, loc
->row
, loc
->column
);
11170 mono_llvm_di_set_location (builder
, loc_md
);
11171 mono_debug_free_source_location (loc
);
11177 emit_default_dbg_loc (EmitContext
*ctx
, LLVMBuilderRef builder
)
11180 LLVMValueRef loc_md
;
11181 loc_md
= (LLVMValueRef
)mono_llvm_di_create_location (ctx
->module
->di_builder
, ctx
->dbg_md
, 0, 0);
11182 mono_llvm_di_set_location (builder
, loc_md
);
11188 - Emit LLVM IR from the mono IR using the LLVM C API.
11189 - The original arch specific code remains, so we can fall back to it if we run
11190 into something we can't handle.
11194 A partial list of issues:
11195 - Handling of opcodes which can throw exceptions.
11197 In the mono JIT, these are implemented using code like this:
11204 push throw_pos - method
11205 call <exception trampoline>
11207 The problematic part is push throw_pos - method, which cannot be represented
11208 in the LLVM IR, since it does not support label values.
11209 -> this can be implemented in AOT mode using inline asm + labels, but cannot
11210 be implemented in JIT mode ?
11211 -> a possible but slower implementation would use the normal exception
11212 throwing code but it would need to control the placement of the throw code
11213 (it needs to be exactly after the compare+branch).
11214 -> perhaps add a PC offset intrinsics ?
11216 - efficient implementation of .ovf opcodes.
11218 These are currently implemented as:
11219 <ins which sets the condition codes>
11222 Some overflow opcodes are now supported by LLVM SVN.
11224 - exception handling, unwinding.
11225 - SSA is disabled for methods with exception handlers
11226 - How to obtain unwind info for LLVM compiled methods ?
11227 -> this is now solved by converting the unwind info generated by LLVM
11229 - LLVM uses the c++ exception handling framework, while we use our home grown
11230 code, and couldn't use the c++ one:
11231 - its not supported under VC++, other exotic platforms.
11232 - it might be impossible to support filter clauses with it.
11236 The trampolines need a predictable call sequence, since they need to disasm
11237 the calling code to obtain register numbers / offsets.
11239 LLVM currently generates this code in non-JIT mode:
11240 mov -0x98(%rax),%eax
11242 Here, the vtable pointer is lost.
11243 -> solution: use one vtable trampoline per class.
11245 - passing/receiving the IMT pointer/RGCTX.
11246 -> solution: pass them as normal arguments ?
11248 - argument passing.
11250 LLVM does not allow the specification of argument registers etc. This means
11251 that all calls are made according to the platform ABI.
11253 - passing/receiving vtypes.
11255 Vtypes passed/received in registers are handled by the front end by using
11256 a signature with scalar arguments, and loading the parts of the vtype into those
11259 Vtypes passed on the stack are handled using the 'byval' attribute.
11263 Supported though alloca, we need to emit the load/store code.
11267 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
11268 typed registers, so we have to keep track of the precise LLVM type of each vreg.
11269 This is made easier because the IR is already in SSA form.
11270 An additional problem is that our IR is not consistent with types, i.e. i32/i64
11271 types are frequently used incorrectly.
11276 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
11277 it with the file containing the methods emitted by the JIT and the AOT data
11281 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
11282 * - each bblock should end with a branch
11283 * - setting the return value, making cfg->ret non-volatile
11284 * - avoid some transformations in the JIT which make it harder for us to generate
11286 * - use pointer types to help optimizations.
11289 #else /* DISABLE_JIT */
11292 mono_llvm_cleanup (void)
11297 mono_llvm_free_domain_info (MonoDomain
*domain
)
11302 mono_llvm_init (gboolean enable_jit
)
11306 #endif /* DISABLE_JIT */
11308 #if !defined(DISABLE_JIT) && !defined(MONO_CROSS_COMPILE)
11310 /* LLVM JIT support */
11313 * decode_llvm_eh_info:
11315 * Decode the EH table emitted by llvm in jit mode, and store
11316 * the result into cfg.
11319 decode_llvm_eh_info (EmitContext
*ctx
, gpointer eh_frame
)
11321 MonoCompile
*cfg
= ctx
->cfg
;
11324 MonoLLVMFDEInfo info
;
11325 MonoJitExceptionInfo
*ei
;
11326 guint8
*p
= (guint8
*)eh_frame
;
11327 int version
, fde_count
, fde_offset
;
11328 guint32 ei_len
, i
, nested_len
;
11329 gpointer
*type_info
;
11334 * Decode the one element EH table emitted by the MonoException class
11338 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
11341 g_assert (version
== 3);
11344 p
= (guint8
*)ALIGN_PTR_TO (p
, 4);
11346 fde_count
= *(guint32
*)p
;
11348 table
= (gint32
*)p
;
11350 g_assert (fde_count
<= 2);
11352 /* The first entry is the real method */
11353 g_assert (table
[0] == 1);
11354 fde_offset
= table
[1];
11355 table
+= fde_count
* 2;
11357 cfg
->code_len
= table
[0];
11358 fde_len
= table
[1] - fde_offset
;
11361 fde
= (guint8
*)eh_frame
+ fde_offset
;
11362 cie
= (guint8
*)table
;
11364 /* Compute lengths */
11365 mono_unwind_decode_llvm_mono_fde (fde
, fde_len
, cie
, cfg
->native_code
, &info
, NULL
, NULL
, NULL
);
11367 ei
= (MonoJitExceptionInfo
*)g_malloc0 (info
.ex_info_len
* sizeof (MonoJitExceptionInfo
));
11368 type_info
= (gpointer
*)g_malloc0 (info
.ex_info_len
* sizeof (gpointer
));
11369 unw_info
= (guint8
*)g_malloc0 (info
.unw_info_len
);
11371 mono_unwind_decode_llvm_mono_fde (fde
, fde_len
, cie
, cfg
->native_code
, &info
, ei
, type_info
, unw_info
);
11373 cfg
->encoded_unwind_ops
= unw_info
;
11374 cfg
->encoded_unwind_ops_len
= info
.unw_info_len
;
11375 if (cfg
->verbose_level
> 1)
11376 mono_print_unwind_info (cfg
->encoded_unwind_ops
, cfg
->encoded_unwind_ops_len
);
11377 if (info
.this_reg
!= -1) {
11378 cfg
->llvm_this_reg
= info
.this_reg
;
11379 cfg
->llvm_this_offset
= info
.this_offset
;
11382 ei_len
= info
.ex_info_len
;
11384 // Nested clauses are currently disabled
11387 cfg
->llvm_ex_info
= (MonoJitExceptionInfo
*)mono_mempool_alloc0 (cfg
->mempool
, (ei_len
+ nested_len
) * sizeof (MonoJitExceptionInfo
));
11388 cfg
->llvm_ex_info_len
= ei_len
+ nested_len
;
11389 memcpy (cfg
->llvm_ex_info
, ei
, ei_len
* sizeof (MonoJitExceptionInfo
));
11390 /* Fill the rest of the information from the type info */
11391 for (i
= 0; i
< ei_len
; ++i
) {
11392 gint32 clause_index
= *(gint32
*)type_info
[i
];
11393 MonoExceptionClause
*clause
= &cfg
->header
->clauses
[clause_index
];
11395 cfg
->llvm_ex_info
[i
].flags
= clause
->flags
;
11396 cfg
->llvm_ex_info
[i
].data
.catch_class
= clause
->data
.catch_class
;
11397 cfg
->llvm_ex_info
[i
].clause_index
= clause_index
;
11402 init_jit_module (MonoDomain
*domain
)
11404 MonoJitDomainInfo
*dinfo
;
11405 MonoLLVMModule
*module
;
11408 dinfo
= domain_jit_info (domain
);
11409 if (dinfo
->llvm_module
)
11412 mono_loader_lock ();
11414 if (dinfo
->llvm_module
) {
11415 mono_loader_unlock ();
11419 module
= g_new0 (MonoLLVMModule
, 1);
11421 name
= g_strdup_printf ("mono-%s", domain
->friendly_name
);
11422 module
->context
= LLVMGetGlobalContext ();
11423 module
->intrins_by_id
= g_new0 (LLVMValueRef
, INTRINS_NUM
);
11425 module
->mono_ee
= (MonoEERef
*)mono_llvm_create_ee (&module
->ee
);
11427 // This contains just the intrinsics
11428 module
->lmodule
= LLVMModuleCreateWithName ("jit-global-module");
11429 add_intrinsics (module
->lmodule
);
11430 add_types (module
);
11432 module
->llvm_types
= g_hash_table_new (NULL
, NULL
);
11434 mono_memory_barrier ();
11436 dinfo
->llvm_module
= module
;
11438 mono_loader_unlock ();
11442 llvm_jit_finalize_method (EmitContext
*ctx
)
11444 MonoCompile
*cfg
= ctx
->cfg
;
11445 MonoDomain
*domain
= mono_domain_get ();
11446 MonoJitDomainInfo
*domain_info
;
11447 int nvars
= g_hash_table_size (ctx
->jit_callees
);
11448 LLVMValueRef
*callee_vars
= g_new0 (LLVMValueRef
, nvars
);
11449 gpointer
*callee_addrs
= g_new0 (gpointer
, nvars
);
11450 GHashTableIter iter
;
11452 MonoMethod
*callee
;
11457 * Compute the addresses of the LLVM globals pointing to the
11458 * methods called by the current method. Pass it to the trampoline
11459 * code so it can update them after their corresponding method was
11462 g_hash_table_iter_init (&iter
, ctx
->jit_callees
);
11464 while (g_hash_table_iter_next (&iter
, NULL
, (void**)&var
))
11465 callee_vars
[i
++] = var
;
11467 cfg
->native_code
= (guint8
*)mono_llvm_compile_method (ctx
->module
->mono_ee
, cfg
, ctx
->lmethod
, nvars
, callee_vars
, callee_addrs
, &eh_frame
);
11468 mono_llvm_remove_gc_safepoint_poll (ctx
->lmodule
);
11469 if (cfg
->verbose_level
> 1) {
11470 g_print ("\n*** Optimized LLVM IR for %s ***\n", mono_method_full_name (cfg
->method
, TRUE
));
11471 if (cfg
->compile_aot
) {
11472 mono_llvm_dump_value (ctx
->lmethod
);
11474 mono_llvm_dump_module (ctx
->lmodule
);
11476 g_print ("***\n\n");
11479 decode_llvm_eh_info (ctx
, eh_frame
);
11481 mono_domain_lock (domain
);
11482 domain_info
= domain_jit_info (domain
);
11483 if (!domain_info
->llvm_jit_callees
)
11484 domain_info
->llvm_jit_callees
= g_hash_table_new (NULL
, NULL
);
11485 g_hash_table_iter_init (&iter
, ctx
->jit_callees
);
11487 while (g_hash_table_iter_next (&iter
, (void**)&callee
, (void**)&var
)) {
11488 GSList
*addrs
= (GSList
*)g_hash_table_lookup (domain_info
->llvm_jit_callees
, callee
);
11489 addrs
= g_slist_prepend (addrs
, callee_addrs
[i
]);
11490 g_hash_table_insert (domain_info
->llvm_jit_callees
, callee
, addrs
);
11493 mono_domain_unlock (domain
);
11499 init_jit_module (MonoDomain
*domain
)
11501 g_assert_not_reached ();
11505 llvm_jit_finalize_method (EmitContext
*ctx
)
11507 g_assert_not_reached ();
11512 static MonoCPUFeatures cpu_features
;
11514 MonoCPUFeatures
mono_llvm_get_cpu_features (void)
11516 static const CpuFeatureAliasFlag flags_map
[] = {
11517 #if defined(TARGET_X86) || defined(TARGET_AMD64)
11518 { "sse", MONO_CPU_X86_SSE
},
11519 { "sse2", MONO_CPU_X86_SSE2
},
11520 { "pclmul", MONO_CPU_X86_PCLMUL
},
11521 { "aes", MONO_CPU_X86_AES
},
11522 { "sse2", MONO_CPU_X86_SSE2
},
11523 { "sse3", MONO_CPU_X86_SSE3
},
11524 { "ssse3", MONO_CPU_X86_SSSE3
},
11525 { "sse4.1", MONO_CPU_X86_SSE41
},
11526 { "sse4.2", MONO_CPU_X86_SSE42
},
11527 { "popcnt", MONO_CPU_X86_POPCNT
},
11528 { "avx", MONO_CPU_X86_AVX
},
11529 { "avx2", MONO_CPU_X86_AVX2
},
11530 { "fma", MONO_CPU_X86_FMA
},
11531 { "lzcnt", MONO_CPU_X86_LZCNT
},
11532 { "bmi", MONO_CPU_X86_BMI1
},
11533 { "bmi2", MONO_CPU_X86_BMI2
},
11537 cpu_features
= MONO_CPU_INITED
| (MonoCPUFeatures
)mono_llvm_check_cpu_features (flags_map
, G_N_ELEMENTS (flags_map
));
11538 return cpu_features
;