?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[llvm] Define a separate symbol for the aot method info for every method. Pass this...
[mono-project.git] / mono / mini / mini-llvm.c
blob4392f3311857a72dc710d3a594600b89494c63ac
1 /**
2 * \file
3 * llvm "Backend" for the mono JIT
4 *
5 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
6 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
8 */
9
10 #include "config.h"
11
12 #include <mono/metadata/debug-helpers.h>
13 #include <mono/metadata/debug-internals.h>
14 #include <mono/metadata/mempool-internals.h>
15 #include <mono/metadata/environment.h>
16 #include <mono/metadata/object-internals.h>
17 #include <mono/metadata/abi-details.h>
18 #include <mono/utils/mono-tls.h>
19 #include <mono/utils/mono-dl.h>
20 #include <mono/utils/mono-time.h>
21 #include <mono/utils/freebsd-dwarf.h>
22
23 #ifndef __STDC_LIMIT_MACROS
24 #define __STDC_LIMIT_MACROS
25 #endif
26 #ifndef __STDC_CONSTANT_MACROS
27 #define __STDC_CONSTANT_MACROS
28 #endif
29
30 #include "llvm-c/BitWriter.h"
31 #include "llvm-c/Analysis.h"
32
33 #include "mini-llvm-cpp.h"
34 #include "llvm-jit.h"
35 #include "aot-compiler.h"
36 #include "mini-llvm.h"
37 #include "mini-runtime.h"
38 #include <mono/utils/mono-math.h>
39
40 #ifndef DISABLE_JIT
41
42 #if defined(TARGET_AMD64) && defined(TARGET_WIN32) && defined(HOST_WIN32) && defined(_MSC_VER)
43 #define TARGET_X86_64_WIN32_MSVC
44 #endif
45
46 #if defined(TARGET_X86_64_WIN32_MSVC)
47 #define TARGET_WIN32_MSVC
48 #endif
49
50 #if LLVM_API_VERSION < 610
51 #error "The version of the mono llvm repository is too old."
52 #endif
53
54 /*
55 * Information associated by mono with LLVM modules.
56 */
57 typedef struct {
58 LLVMModuleRef lmodule;
59 LLVMValueRef throw_icall, rethrow, throw_corlib_exception;
60 GHashTable *llvm_types;
61 LLVMValueRef dummy_got_var;
62 const char *get_method_symbol;
63 const char *get_unbox_tramp_symbol;
64 const char *init_aotconst_symbol;
65 GHashTable *plt_entries;
66 GHashTable *plt_entries_ji;
67 GHashTable *method_to_lmethod;
68 GHashTable *method_to_call_info;
69 GHashTable *lvalue_to_lcalls;
70 GHashTable *direct_callables;
71 /* Maps got slot index -> LLVMValueRef */
72 GHashTable *aotconst_vars;
73 char **bb_names;
74 int bb_names_len;
75 GPtrArray *used;
76 LLVMTypeRef ptr_type;
77 GPtrArray *subprogram_mds;
78 MonoEERef *mono_ee;
79 LLVMExecutionEngineRef ee;
80 gboolean external_symbols;
81 gboolean emit_dwarf;
82 int max_got_offset;
83 LLVMValueRef personality;
84 LLVMValueRef *intrins_by_id;
85 gpointer gc_poll_cold_wrapper_compiled;
86
87 /* For AOT */
88 MonoAssembly *assembly;
89 char *global_prefix;
90 MonoAotFileInfo aot_info;
91 const char *eh_frame_symbol;
92 LLVMValueRef get_method, get_unbox_tramp, init_aotconst_func;
93 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
94 LLVMValueRef code_start, code_end;
95 LLVMValueRef inited_var;
96 LLVMValueRef unbox_tramp_indexes;
97 LLVMValueRef unbox_trampolines;
98 LLVMValueRef gc_poll_cold_wrapper;
99 LLVMValueRef info_var;
100 LLVMTypeRef *info_var_eltypes;
101 int max_inited_idx, max_method_idx;
102 gboolean has_jitted_code;
103 gboolean static_link;
104 gboolean llvm_only;
105 gboolean interp;
106 GHashTable *idx_to_lmethod;
107 GHashTable *idx_to_unbox_tramp;
108 GPtrArray *callsite_list;
109 LLVMContextRef context;
110 LLVMValueRef sentinel_exception;
111 void *di_builder, *cu;
112 GHashTable *objc_selector_to_var;
113 GPtrArray *cfgs;
114 int unbox_tramp_num, unbox_tramp_elemsize;
115 GHashTable *got_idx_to_type;
116 GHashTable *no_method_table_lmethods;
117 } MonoLLVMModule;
118
119 /*
120 * Information associated by the backend with mono basic blocks.
121 */
122 typedef struct {
123 LLVMBasicBlockRef bblock, end_bblock;
124 LLVMValueRef finally_ind;
125 gboolean added, invoke_target;
126 /*
127 * If this bblock is the start of a finally clause, this is a list of bblocks it
128 * needs to branch to in ENDFINALLY.
129 */
130 GSList *call_handler_return_bbs;
131 /*
132 * If this bblock is the start of a finally clause, this is the bblock that
133 * CALL_HANDLER needs to branch to.
134 */
135 LLVMBasicBlockRef call_handler_target_bb;
136 /* The list of switch statements generated by ENDFINALLY instructions */
137 GSList *endfinally_switch_ins_list;
138 GSList *phi_nodes;
139 } BBInfo;
140
141 /*
142 * Structure containing emit state
143 */
144 typedef struct {
145 MonoMemPool *mempool;
146
147 /* Maps method names to the corresponding LLVMValueRef */
148 GHashTable *emitted_method_decls;
149
150 MonoCompile *cfg;
151 LLVMValueRef lmethod;
152 MonoLLVMModule *module;
153 LLVMModuleRef lmodule;
154 BBInfo *bblocks;
155 int sindex, default_index, ex_index;
156 LLVMBuilderRef builder;
157 LLVMValueRef *values, *addresses;
158 MonoType **vreg_cli_types;
159 LLVMCallInfo *linfo;
160 MonoMethodSignature *sig;
161 GSList *builders;
162 GHashTable *region_to_handler;
163 GHashTable *clause_to_handler;
164 LLVMBuilderRef alloca_builder;
165 LLVMValueRef last_alloca;
166 LLVMValueRef rgctx_arg;
167 LLVMValueRef this_arg;
168 LLVMTypeRef *vreg_types;
169 gboolean *is_vphi;
170 LLVMTypeRef method_type;
171 LLVMBasicBlockRef init_bb, inited_bb;
172 gboolean *is_dead;
173 gboolean *unreachable;
174 gboolean llvm_only;
175 gboolean has_got_access;
176 gboolean emit_dummy_arg;
177 gboolean has_safepoints;
178 int this_arg_pindex, rgctx_arg_pindex;
179 LLVMValueRef imt_rgctx_loc;
180 GHashTable *llvm_types;
181 LLVMValueRef dbg_md;
182 MonoDebugMethodInfo *minfo;
183 /* For every clause, the clauses it is nested in */
184 GSList **nested_in;
185 LLVMValueRef ex_var;
186 GHashTable *exc_meta;
187 GPtrArray *callsite_list;
188 GPtrArray *phi_values;
189 GPtrArray *bblock_list;
190 char *method_name;
191 GHashTable *jit_callees;
192 LLVMValueRef long_bb_break_var;
193 } EmitContext;
194
195 typedef struct {
196 MonoBasicBlock *bb;
197 MonoInst *phi;
198 MonoBasicBlock *in_bb;
199 int sreg;
200 } PhiNode;
201
202 /*
203 * Instruction metadata
204 * This is the same as ins_info, but LREG != IREG.
205 */
206 #ifdef MINI_OP
207 #undef MINI_OP
208 #endif
209 #ifdef MINI_OP3
210 #undef MINI_OP3
211 #endif
212 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
213 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
214 #define NONE ' '
215 #define IREG 'i'
216 #define FREG 'f'
217 #define VREG 'v'
218 #define XREG 'x'
219 #define LREG 'l'
220 /* keep in sync with the enum in mini.h */
221 const char
222 mini_llvm_ins_info[] = {
223 #include "mini-ops.h"
224 };
225 #undef MINI_OP
226 #undef MINI_OP3
227
228 #if TARGET_SIZEOF_VOID_P == 4
229 #define GET_LONG_IMM(ins) ((ins)->inst_l)
230 #else
231 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
232 #endif
233
234 #define LLVM_INS_INFO(opcode) (&mini_llvm_ins_info [((opcode) - OP_START - 1) * 4])
235
236 #if 0
237 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
238 #else
239 #define TRACE_FAILURE(msg)
240 #endif
241
242 #ifdef TARGET_X86
243 #define IS_TARGET_X86 1
244 #else
245 #define IS_TARGET_X86 0
246 #endif
247
248 #ifdef TARGET_AMD64
249 #define IS_TARGET_AMD64 1
250 #else
251 #define IS_TARGET_AMD64 0
252 #endif
253
254 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
255
256 static LLVMIntPredicate cond_to_llvm_cond [] = {
257 LLVMIntEQ,
258 LLVMIntNE,
259 LLVMIntSLE,
260 LLVMIntSGE,
261 LLVMIntSLT,
262 LLVMIntSGT,
263 LLVMIntULE,
264 LLVMIntUGE,
265 LLVMIntULT,
266 LLVMIntUGT,
267 };
268
269 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
270 LLVMRealOEQ,
271 LLVMRealUNE,
272 LLVMRealOLE,
273 LLVMRealOGE,
274 LLVMRealOLT,
275 LLVMRealOGT,
276 LLVMRealULE,
277 LLVMRealUGE,
278 LLVMRealULT,
279 LLVMRealUGT,
280 LLVMRealORD,
281 LLVMRealUNO
282 };
283
284 static MonoLLVMModule aot_module;
285
286 static GHashTable *intrins_id_to_intrins;
287 static LLVMTypeRef sse_i1_t, sse_i2_t, sse_i4_t, sse_i8_t, sse_r4_t, sse_r8_t;
288
289 static void init_jit_module (MonoDomain *domain);
290
291 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
292 static void emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder);
293 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
294 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
295 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp, gboolean force_explicit);
296 static LLVMValueRef get_intrins (EmitContext *ctx, int id);
297 static LLVMValueRef get_intrins_from_module (LLVMModuleRef lmodule, int id);
298 static void llvm_jit_finalize_method (EmitContext *ctx);
299 static void mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params);
300 static void mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module);
301 static void create_aot_info_var (MonoLLVMModule *module);
302 static void set_invariant_load_flag (LLVMValueRef v);
303 static void set_nonnull_load_flag (LLVMValueRef v);
304
305 static inline void
306 set_failure (EmitContext *ctx, const char *message)
307 {
308 TRACE_FAILURE (reason);
309 ctx->cfg->exception_message = g_strdup (message);
310 ctx->cfg->disable_llvm = TRUE;
311 }
312
313 static LLVMValueRef
314 const_int32 (int v)
315 {
316 return LLVMConstInt (LLVMInt32Type (), v, FALSE);
317 }
318
319 /*
320 * IntPtrType:
321 *
322 * The LLVM type with width == TARGET_SIZEOF_VOID_P
323 */
324 static LLVMTypeRef
325 IntPtrType (void)
326 {
327 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
328 }
329
330 static LLVMTypeRef
331 ObjRefType (void)
332 {
333 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
334 }
335
336 static LLVMTypeRef
337 ThisType (void)
338 {
339 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
340 }
341
342 /*
343 * get_vtype_size:
344 *
345 * Return the size of the LLVM representation of the vtype T.
346 */
347 static guint32
348 get_vtype_size (MonoType *t)
349 {
350 int size;
351
352 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
353
354 /* LLVMArgAsIArgs depends on this since it stores whole words */
355 while (size < 2 * TARGET_SIZEOF_VOID_P && mono_is_power_of_two (size) == -1)
356 size ++;
357
358 return size;
359 }
360
361 /*
362 * simd_class_to_llvm_type:
363 *
364 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
365 */
366 static LLVMTypeRef
367 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
368 {
369 const char *klass_name = m_class_get_name (klass);
370 if (!strcmp (klass_name, "Vector2d")) {
371 return LLVMVectorType (LLVMDoubleType (), 2);
372 } else if (!strcmp (klass_name, "Vector2l")) {
373 return LLVMVectorType (LLVMInt64Type (), 2);
374 } else if (!strcmp (klass_name, "Vector2ul")) {
375 return LLVMVectorType (LLVMInt64Type (), 2);
376 } else if (!strcmp (klass_name, "Vector4i")) {
377 return LLVMVectorType (LLVMInt32Type (), 4);
378 } else if (!strcmp (klass_name, "Vector4ui")) {
379 return LLVMVectorType (LLVMInt32Type (), 4);
380 } else if (!strcmp (klass_name, "Vector4f")) {
381 return LLVMVectorType (LLVMFloatType (), 4);
382 } else if (!strcmp (klass_name, "Vector8s")) {
383 return LLVMVectorType (LLVMInt16Type (), 8);
384 } else if (!strcmp (klass_name, "Vector8us")) {
385 return LLVMVectorType (LLVMInt16Type (), 8);
386 } else if (!strcmp (klass_name, "Vector16sb")) {
387 return LLVMVectorType (LLVMInt8Type (), 16);
388 } else if (!strcmp (klass_name, "Vector16b")) {
389 return LLVMVectorType (LLVMInt8Type (), 16);
390 } else if (!strcmp (klass_name, "Vector2")) {
391 /* System.Numerics */
392 return LLVMVectorType (LLVMFloatType (), 4);
393 } else if (!strcmp (klass_name, "Vector3")) {
394 return LLVMVectorType (LLVMFloatType (), 4);
395 } else if (!strcmp (klass_name, "Vector4")) {
396 return LLVMVectorType (LLVMFloatType (), 4);
397 } else if (!strcmp (klass_name, "Vector`1") || !strcmp (klass_name, "Vector128`1") || !strcmp (klass_name, "Vector256`1")) {
398 MonoType *etype = mono_class_get_generic_class (klass)->context.class_inst->type_argv [0];
399 int size = mono_class_value_size (klass, NULL);
400 switch (etype->type) {
401 case MONO_TYPE_I1:
402 case MONO_TYPE_U1:
403 return LLVMVectorType (LLVMInt8Type (), size);
404 case MONO_TYPE_I2:
405 case MONO_TYPE_U2:
406 return LLVMVectorType (LLVMInt16Type (), size / 2);
407 case MONO_TYPE_I4:
408 case MONO_TYPE_U4:
409 return LLVMVectorType (LLVMInt32Type (), size / 4);
410 case MONO_TYPE_I8:
411 case MONO_TYPE_U8:
412 return LLVMVectorType (LLVMInt64Type (), size / 8);
413 case MONO_TYPE_R4:
414 return LLVMVectorType (LLVMFloatType (), size / 4);
415 case MONO_TYPE_R8:
416 return LLVMVectorType (LLVMDoubleType (), size / 8);
417 default:
418 g_assert_not_reached ();
419 return NULL;
420 }
421 } else {
422 printf ("%s\n", klass_name);
423 NOT_IMPLEMENTED;
424 return NULL;
425 }
426 }
427
428 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
429 static inline G_GNUC_UNUSED LLVMTypeRef
430 type_to_sse_type (int type)
431 {
432 switch (type) {
433 case MONO_TYPE_I1:
434 case MONO_TYPE_U1:
435 return LLVMVectorType (LLVMInt8Type (), 16);
436 case MONO_TYPE_U2:
437 case MONO_TYPE_I2:
438 return LLVMVectorType (LLVMInt16Type (), 8);
439 case MONO_TYPE_U4:
440 case MONO_TYPE_I4:
441 return LLVMVectorType (LLVMInt32Type (), 4);
442 case MONO_TYPE_U8:
443 case MONO_TYPE_I8:
444 return LLVMVectorType (LLVMInt64Type (), 2);
445 case MONO_TYPE_R8:
446 return LLVMVectorType (LLVMDoubleType (), 2);
447 case MONO_TYPE_R4:
448 return LLVMVectorType (LLVMFloatType (), 4);
449 default:
450 g_assert_not_reached ();
451 return NULL;
452 }
453 }
454
455 static LLVMTypeRef
456 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
457 {
458 int i, size, nfields, esize;
459 LLVMTypeRef *eltypes;
460 char *name;
461 MonoType *t;
462 LLVMTypeRef ltype;
463
464 t = m_class_get_byval_arg (klass);
465
466 if (mini_type_is_hfa (t, &nfields, &esize)) {
467 /*
468 * This is needed on arm64 where HFAs are returned in
469 * registers.
470 */
471 /* SIMD types have size 16 in mono_class_value_size () */
472 if (m_class_is_simd_type (klass))
473 nfields = 16/ esize;
474 size = nfields;
475 eltypes = g_new (LLVMTypeRef, size);
476 for (i = 0; i < size; ++i)
477 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
478 } else {
479 size = get_vtype_size (t);
480
481 eltypes = g_new (LLVMTypeRef, size);
482 for (i = 0; i < size; ++i)
483 eltypes [i] = LLVMInt8Type ();
484 }
485
486 name = mono_type_full_name (m_class_get_byval_arg (klass));
487 ltype = LLVMStructCreateNamed (module->context, name);
488 LLVMStructSetBody (ltype, eltypes, size, FALSE);
489 g_free (eltypes);
490 g_free (name);
491
492 return ltype;
493 }
494
495 static LLVMTypeRef
496 primitive_type_to_llvm_type (MonoTypeEnum type)
497 {
498 switch (type) {
499 case MONO_TYPE_I1:
500 case MONO_TYPE_U1:
501 return LLVMInt8Type ();
502 case MONO_TYPE_I2:
503 case MONO_TYPE_U2:
504 return LLVMInt16Type ();
505 case MONO_TYPE_I4:
506 case MONO_TYPE_U4:
507 return LLVMInt32Type ();
508 case MONO_TYPE_I8:
509 case MONO_TYPE_U8:
510 return LLVMInt64Type ();
511 case MONO_TYPE_R4:
512 return LLVMFloatType ();
513 case MONO_TYPE_R8:
514 return LLVMDoubleType ();
515 case MONO_TYPE_I:
516 case MONO_TYPE_U:
517 return IntPtrType ();
518 default:
519 return NULL;
520 }
521 }
522
523 static MonoTypeEnum
524 inst_c1_type (const MonoInst *ins)
525 {
526 return (MonoTypeEnum)ins->inst_c1;
527 }
528
529 /*
530 * type_to_llvm_type:
531 *
532 * Return the LLVM type corresponding to T.
533 */
534 static LLVMTypeRef
535 type_to_llvm_type (EmitContext *ctx, MonoType *t)
536 {
537 if (t->byref)
538 return ThisType ();
539
540 t = mini_get_underlying_type (t);
541
542 LLVMTypeRef prim_llvm_type = primitive_type_to_llvm_type (t->type);
543 if (prim_llvm_type != NULL)
544 return prim_llvm_type;
545
546 switch (t->type) {
547 case MONO_TYPE_VOID:
548 return LLVMVoidType ();
549 case MONO_TYPE_OBJECT:
550 return ObjRefType ();
551 case MONO_TYPE_PTR: {
552 MonoClass *klass = mono_class_from_mono_type_internal (t);
553 MonoClass *ptr_klass = m_class_get_element_class (klass);
554 MonoType *ptr_type = m_class_get_byval_arg (ptr_klass);
555 /* Handle primitive pointers */
556 switch (ptr_type->type) {
557 case MONO_TYPE_I1:
558 case MONO_TYPE_I2:
559 case MONO_TYPE_I4:
560 case MONO_TYPE_U1:
561 case MONO_TYPE_U2:
562 case MONO_TYPE_U4:
563 return LLVMPointerType (type_to_llvm_type (ctx, ptr_type), 0);
564 }
565
566 return ObjRefType ();
567 }
568 case MONO_TYPE_VAR:
569 case MONO_TYPE_MVAR:
570 /* Because of generic sharing */
571 return ObjRefType ();
572 case MONO_TYPE_GENERICINST:
573 if (!mono_type_generic_inst_is_valuetype (t))
574 return ObjRefType ();
575 /* Fall through */
576 case MONO_TYPE_VALUETYPE:
577 case MONO_TYPE_TYPEDBYREF: {
578 MonoClass *klass;
579 LLVMTypeRef ltype;
580
581 klass = mono_class_from_mono_type_internal (t);
582
583 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
584 return simd_class_to_llvm_type (ctx, klass);
585
586 if (m_class_is_enumtype (klass))
587 return type_to_llvm_type (ctx, mono_class_enum_basetype_internal (klass));
588
589 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
590 if (!ltype) {
591 ltype = create_llvm_type_for_type (ctx->module, klass);
592 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
593 }
594 return ltype;
595 }
596
597 default:
598 printf ("X: %d\n", t->type);
599 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
600 ctx->cfg->disable_llvm = TRUE;
601 return NULL;
602 }
603 }
604
605 static gboolean
606 primitive_type_is_unsigned (MonoTypeEnum t)
607 {
608 switch (t) {
609 case MONO_TYPE_U1:
610 case MONO_TYPE_U2:
611 case MONO_TYPE_CHAR:
612 case MONO_TYPE_U4:
613 case MONO_TYPE_U8:
614 return TRUE;
615 default:
616 return FALSE;
617 }
618 }
619
620 /*
621 * type_is_unsigned:
622 *
623 * Return whenever T is an unsigned int type.
624 */
625 static gboolean
626 type_is_unsigned (EmitContext *ctx, MonoType *t)
627 {
628 t = mini_get_underlying_type (t);
629 if (t->byref)
630 return FALSE;
631 return primitive_type_is_unsigned (t->type);
632 }
633
634 /*
635 * type_to_llvm_arg_type:
636 *
637 * Same as type_to_llvm_type, but treat i8/i16 as i32.
638 */
639 static LLVMTypeRef
640 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
641 {
642 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
643
644 if (ctx->cfg->llvm_only)
645 return ptype;
646
647 /*
648 * This works on all abis except arm64/ios which passes multiple
649 * arguments in one stack slot.
650 */
651 #ifndef TARGET_ARM64
652 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
653 /*
654 * LLVM generates code which only sets the lower bits, while JITted
655 * code expects all the bits to be set.
656 */
657 ptype = LLVMInt32Type ();
658 }
659 #endif
660
661 return ptype;
662 }
663
664 /*
665 * llvm_type_to_stack_type:
666 *
667 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
668 * on the IL stack.
669 */
670 static G_GNUC_UNUSED LLVMTypeRef
671 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
672 {
673 if (type == NULL)
674 return NULL;
675 if (type == LLVMInt8Type ())
676 return LLVMInt32Type ();
677 else if (type == LLVMInt16Type ())
678 return LLVMInt32Type ();
679 else if (!cfg->r4fp && type == LLVMFloatType ())
680 return LLVMDoubleType ();
681 else
682 return type;
683 }
684
685 /*
686 * regtype_to_llvm_type:
687 *
688 * Return the LLVM type corresponding to the regtype C used in instruction
689 * descriptions.
690 */
691 static LLVMTypeRef
692 regtype_to_llvm_type (char c)
693 {
694 switch (c) {
695 case 'i':
696 return LLVMInt32Type ();
697 case 'l':
698 return LLVMInt64Type ();
699 case 'f':
700 return LLVMDoubleType ();
701 default:
702 return NULL;
703 }
704 }
705
706 /*
707 * op_to_llvm_type:
708 *
709 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
710 */
711 static LLVMTypeRef
712 op_to_llvm_type (int opcode)
713 {
714 switch (opcode) {
715 case OP_ICONV_TO_I1:
716 case OP_LCONV_TO_I1:
717 return LLVMInt8Type ();
718 case OP_ICONV_TO_U1:
719 case OP_LCONV_TO_U1:
720 return LLVMInt8Type ();
721 case OP_ICONV_TO_I2:
722 case OP_LCONV_TO_I2:
723 return LLVMInt16Type ();
724 case OP_ICONV_TO_U2:
725 case OP_LCONV_TO_U2:
726 return LLVMInt16Type ();
727 case OP_ICONV_TO_I4:
728 case OP_LCONV_TO_I4:
729 return LLVMInt32Type ();
730 case OP_ICONV_TO_U4:
731 case OP_LCONV_TO_U4:
732 return LLVMInt32Type ();
733 case OP_ICONV_TO_I8:
734 return LLVMInt64Type ();
735 case OP_ICONV_TO_R4:
736 return LLVMFloatType ();
737 case OP_ICONV_TO_R8:
738 return LLVMDoubleType ();
739 case OP_ICONV_TO_U8:
740 return LLVMInt64Type ();
741 case OP_FCONV_TO_I4:
742 return LLVMInt32Type ();
743 case OP_FCONV_TO_I8:
744 return LLVMInt64Type ();
745 case OP_FCONV_TO_I1:
746 case OP_FCONV_TO_U1:
747 case OP_RCONV_TO_I1:
748 case OP_RCONV_TO_U1:
749 return LLVMInt8Type ();
750 case OP_FCONV_TO_I2:
751 case OP_FCONV_TO_U2:
752 case OP_RCONV_TO_I2:
753 case OP_RCONV_TO_U2:
754 return LLVMInt16Type ();
755 case OP_FCONV_TO_U4:
756 case OP_RCONV_TO_U4:
757 return LLVMInt32Type ();
758 case OP_FCONV_TO_U8:
759 case OP_RCONV_TO_U8:
760 return LLVMInt64Type ();
761 case OP_FCONV_TO_I:
762 case OP_FCONV_TO_U:
763 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
764 case OP_IADD_OVF:
765 case OP_IADD_OVF_UN:
766 case OP_ISUB_OVF:
767 case OP_ISUB_OVF_UN:
768 case OP_IMUL_OVF:
769 case OP_IMUL_OVF_UN:
770 return LLVMInt32Type ();
771 case OP_LADD_OVF:
772 case OP_LADD_OVF_UN:
773 case OP_LSUB_OVF:
774 case OP_LSUB_OVF_UN:
775 case OP_LMUL_OVF:
776 case OP_LMUL_OVF_UN:
777 return LLVMInt64Type ();
778 default:
779 printf ("%s\n", mono_inst_name (opcode));
780 g_assert_not_reached ();
781 return NULL;
782 }
783 }
784
785 #define CLAUSE_START(clause) ((clause)->try_offset)
786 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
787
788 /*
789 * load_store_to_llvm_type:
790 *
791 * Return the size/sign/zero extension corresponding to the load/store opcode
792 * OPCODE.
793 */
794 static LLVMTypeRef
795 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
796 {
797 *sext = FALSE;
798 *zext = FALSE;
799
800 switch (opcode) {
801 case OP_LOADI1_MEMBASE:
802 case OP_STOREI1_MEMBASE_REG:
803 case OP_STOREI1_MEMBASE_IMM:
804 case OP_ATOMIC_LOAD_I1:
805 case OP_ATOMIC_STORE_I1:
806 *size = 1;
807 *sext = TRUE;
808 return LLVMInt8Type ();
809 case OP_LOADU1_MEMBASE:
810 case OP_LOADU1_MEM:
811 case OP_ATOMIC_LOAD_U1:
812 case OP_ATOMIC_STORE_U1:
813 *size = 1;
814 *zext = TRUE;
815 return LLVMInt8Type ();
816 case OP_LOADI2_MEMBASE:
817 case OP_STOREI2_MEMBASE_REG:
818 case OP_STOREI2_MEMBASE_IMM:
819 case OP_ATOMIC_LOAD_I2:
820 case OP_ATOMIC_STORE_I2:
821 *size = 2;
822 *sext = TRUE;
823 return LLVMInt16Type ();
824 case OP_LOADU2_MEMBASE:
825 case OP_LOADU2_MEM:
826 case OP_ATOMIC_LOAD_U2:
827 case OP_ATOMIC_STORE_U2:
828 *size = 2;
829 *zext = TRUE;
830 return LLVMInt16Type ();
831 case OP_LOADI4_MEMBASE:
832 case OP_LOADU4_MEMBASE:
833 case OP_LOADI4_MEM:
834 case OP_LOADU4_MEM:
835 case OP_STOREI4_MEMBASE_REG:
836 case OP_STOREI4_MEMBASE_IMM:
837 case OP_ATOMIC_LOAD_I4:
838 case OP_ATOMIC_STORE_I4:
839 case OP_ATOMIC_LOAD_U4:
840 case OP_ATOMIC_STORE_U4:
841 *size = 4;
842 return LLVMInt32Type ();
843 case OP_LOADI8_MEMBASE:
844 case OP_LOADI8_MEM:
845 case OP_STOREI8_MEMBASE_REG:
846 case OP_STOREI8_MEMBASE_IMM:
847 case OP_ATOMIC_LOAD_I8:
848 case OP_ATOMIC_STORE_I8:
849 case OP_ATOMIC_LOAD_U8:
850 case OP_ATOMIC_STORE_U8:
851 *size = 8;
852 return LLVMInt64Type ();
853 case OP_LOADR4_MEMBASE:
854 case OP_STORER4_MEMBASE_REG:
855 case OP_ATOMIC_LOAD_R4:
856 case OP_ATOMIC_STORE_R4:
857 *size = 4;
858 return LLVMFloatType ();
859 case OP_LOADR8_MEMBASE:
860 case OP_STORER8_MEMBASE_REG:
861 case OP_ATOMIC_LOAD_R8:
862 case OP_ATOMIC_STORE_R8:
863 *size = 8;
864 return LLVMDoubleType ();
865 case OP_LOAD_MEMBASE:
866 case OP_LOAD_MEM:
867 case OP_STORE_MEMBASE_REG:
868 case OP_STORE_MEMBASE_IMM:
869 *size = TARGET_SIZEOF_VOID_P;
870 return IntPtrType ();
871 default:
872 g_assert_not_reached ();
873 return NULL;
874 }
875 }
876
877 /*
878 * ovf_op_to_intrins:
879 *
880 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
881 */
882 static IntrinsicId
883 ovf_op_to_intrins (int opcode)
884 {
885 switch (opcode) {
886 case OP_IADD_OVF:
887 return INTRINS_SADD_OVF_I32;
888 case OP_IADD_OVF_UN:
889 return INTRINS_UADD_OVF_I32;
890 case OP_ISUB_OVF:
891 return INTRINS_SSUB_OVF_I32;
892 case OP_ISUB_OVF_UN:
893 return INTRINS_USUB_OVF_I32;
894 case OP_IMUL_OVF:
895 return INTRINS_SMUL_OVF_I32;
896 case OP_IMUL_OVF_UN:
897 return INTRINS_UMUL_OVF_I32;
898 case OP_LADD_OVF:
899 return INTRINS_SADD_OVF_I64;
900 case OP_LADD_OVF_UN:
901 return INTRINS_UADD_OVF_I64;
902 case OP_LSUB_OVF:
903 return INTRINS_SSUB_OVF_I64;
904 case OP_LSUB_OVF_UN:
905 return INTRINS_USUB_OVF_I64;
906 case OP_LMUL_OVF:
907 return INTRINS_SMUL_OVF_I64;
908 case OP_LMUL_OVF_UN:
909 return INTRINS_UMUL_OVF_I64;
910 default:
911 g_assert_not_reached ();
912 return (IntrinsicId)0;
913 }
914 }
915
916 static IntrinsicId
917 simd_ins_to_intrins (int opcode)
918 {
919 switch (opcode) {
920 #if defined(TARGET_X86) || defined(TARGET_AMD64)
921 case OP_MINPD:
922 return INTRINS_SSE_MINPD;
923 case OP_MINPS:
924 return INTRINS_SSE_MINPS;
925 case OP_MAXPD:
926 return INTRINS_SSE_MAXPD;
927 case OP_MAXPS:
928 return INTRINS_SSE_MAXPS;
929 case OP_HADDPD:
930 return INTRINS_SSE_HADDPD;
931 case OP_HADDPS:
932 return INTRINS_SSE_HADDPS;
933 case OP_HSUBPD:
934 return INTRINS_SSE_HSUBPD;
935 case OP_HSUBPS:
936 return INTRINS_SSE_HSUBPS;
937 case OP_ADDSUBPS:
938 return INTRINS_SSE_ADDSUBPS;
939 case OP_ADDSUBPD:
940 return INTRINS_SSE_ADDSUBPD;
941 case OP_EXTRACT_MASK:
942 return INTRINS_SSE_PMOVMSKB;
943 case OP_PSHRW:
944 case OP_PSHRW_REG:
945 return INTRINS_SSE_PSRLI_W;
946 case OP_PSHRD:
947 case OP_PSHRD_REG:
948 return INTRINS_SSE_PSRLI_D;
949 case OP_PSHRQ:
950 case OP_PSHRQ_REG:
951 return INTRINS_SSE_PSRLI_Q;
952 case OP_PSHLW:
953 case OP_PSHLW_REG:
954 return INTRINS_SSE_PSLLI_W;
955 case OP_PSHLD:
956 case OP_PSHLD_REG:
957 return INTRINS_SSE_PSLLI_D;
958 case OP_PSHLQ:
959 case OP_PSHLQ_REG:
960 return INTRINS_SSE_PSLLI_Q;
961 case OP_PSARW:
962 case OP_PSARW_REG:
963 return INTRINS_SSE_PSRAI_W;
964 case OP_PSARD:
965 case OP_PSARD_REG:
966 return INTRINS_SSE_PSRAI_D;
967 case OP_RSQRTPS:
968 return INTRINS_SSE_RSQRT_PS;
969 case OP_RCPPS:
970 return INTRINS_SSE_RCP_PS;
971 case OP_CVTPD2DQ:
972 return INTRINS_SSE_CVTPD2DQ;
973 case OP_CVTPS2DQ:
974 return INTRINS_SSE_CVTPS2DQ;
975 case OP_CVTPD2PS:
976 return INTRINS_SSE_CVTPD2PS;
977 case OP_CVTTPD2DQ:
978 return INTRINS_SSE_CVTTPD2DQ;
979 case OP_CVTTPS2DQ:
980 return INTRINS_SSE_CVTTPS2DQ;
981 case OP_PACKW:
982 return INTRINS_SSE_PACKSSWB;
983 case OP_PACKD:
984 return INTRINS_SSE_PACKSSDW;
985 case OP_PACKW_UN:
986 return INTRINS_SSE_PACKUSWB;
987 case OP_PACKD_UN:
988 return INTRINS_SSE_PACKUSDW;
989 case OP_PMULW_HIGH:
990 return INTRINS_SSE_PMULHW;
991 case OP_PMULW_HIGH_UN:
992 return INTRINS_SSE_PMULHU;
993 case OP_DPPS:
994 return INTRINS_SSE_DPPS;
995 case OP_SSE_SQRTSS:
996 return INTRINS_SSE_SQRT_SS;
997 case OP_SSE2_SQRTSD:
998 return INTRINS_SSE_SQRT_SD;
999 case OP_SQRTPS:
1000 return INTRINS_SSE_SQRT_PS;
1001 case OP_SQRTPD:
1002 return INTRINS_SSE_SQRT_PD;
1003 #endif
1004 default:
1005 g_assert_not_reached ();
1006 return (IntrinsicId)0;
1007 }
1008 }
1009
1010 static LLVMTypeRef
1011 simd_op_to_llvm_type (int opcode)
1012 {
1013 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1014 switch (opcode) {
1015 case OP_EXTRACT_R8:
1016 case OP_EXPAND_R8:
1017 return sse_r8_t;
1018 case OP_EXTRACT_I8:
1019 case OP_EXPAND_I8:
1020 return sse_i8_t;
1021 case OP_EXTRACT_I4:
1022 case OP_EXPAND_I4:
1023 return sse_i4_t;
1024 case OP_EXTRACT_I2:
1025 case OP_EXTRACT_U2:
1026 case OP_EXTRACTX_U2:
1027 case OP_EXPAND_I2:
1028 return sse_i2_t;
1029 case OP_EXTRACT_I1:
1030 case OP_EXTRACT_U1:
1031 case OP_EXPAND_I1:
1032 return sse_i1_t;
1033 case OP_EXTRACT_R4:
1034 case OP_EXPAND_R4:
1035 return sse_r4_t;
1036 case OP_CVTPD2DQ:
1037 case OP_CVTPD2PS:
1038 case OP_CVTTPD2DQ:
1039 return sse_r8_t;
1040 case OP_CVTPS2DQ:
1041 case OP_CVTTPS2DQ:
1042 return sse_r4_t;
1043 case OP_EXTRACT_MASK:
1044 return sse_i1_t;
1045 case OP_SQRTPS:
1046 case OP_RSQRTPS:
1047 case OP_RCPPS:
1048 case OP_DUPPS_LOW:
1049 case OP_DUPPS_HIGH:
1050 return sse_r4_t;
1051 case OP_SQRTPD:
1052 case OP_DUPPD:
1053 return sse_r8_t;
1054 default:
1055 g_assert_not_reached ();
1056 return NULL;
1057 }
1058 #else
1059 return NULL;
1060 #endif
1061 }
1062
1063 static void
1064 set_cold_cconv (LLVMValueRef func)
1065 {
1066 /*
1067 * xcode10 (watchOS) and ARM/ARM64 doesn't seem to support preserveall, it fails with:
1068 * fatal error: error in backend: Unsupported calling convention
1069 */
1070 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1071 LLVMSetFunctionCallConv (func, LLVMColdCallConv);
1072 #endif
1073 }
1074
1075 static void
1076 set_call_cold_cconv (LLVMValueRef func)
1077 {
1078 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1079 LLVMSetInstructionCallConv (func, LLVMColdCallConv);
1080 #endif
1081 }
1082
1083 /*
1084 * get_bb:
1085 *
1086 * Return the LLVM basic block corresponding to BB.
1087 */
1088 static LLVMBasicBlockRef
1089 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
1090 {
1091 char bb_name_buf [128];
1092 char *bb_name;
1093
1094 if (ctx->bblocks [bb->block_num].bblock == NULL) {
1095 if (bb->flags & BB_EXCEPTION_HANDLER) {
1096 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
1097 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
1098 bb_name = bb_name_buf;
1099 } else if (bb->block_num < 256) {
1100 if (!ctx->module->bb_names) {
1101 ctx->module->bb_names_len = 256;
1102 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
1103 }
1104 if (!ctx->module->bb_names [bb->block_num]) {
1105 char *n;
1106
1107 n = g_strdup_printf ("BB%d", bb->block_num);
1108 mono_memory_barrier ();
1109 ctx->module->bb_names [bb->block_num] = n;
1110 }
1111 bb_name = ctx->module->bb_names [bb->block_num];
1112 } else {
1113 sprintf (bb_name_buf, "BB%d", bb->block_num);
1114 bb_name = bb_name_buf;
1115 }
1116
1117 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1118 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1119 }
1120
1121 return ctx->bblocks [bb->block_num].bblock;
1122 }
1123
1124 /*
1125 * get_end_bb:
1126 *
1127 * Return the last LLVM bblock corresponding to BB.
1128 * This might not be equal to the bb returned by get_bb () since we need to generate
1129 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1130 */
1131 static LLVMBasicBlockRef
1132 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1133 {
1134 get_bb (ctx, bb);
1135 return ctx->bblocks [bb->block_num].end_bblock;
1136 }
1137
1138 static LLVMBasicBlockRef
1139 gen_bb (EmitContext *ctx, const char *prefix)
1140 {
1141 char bb_name [128];
1142
1143 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1144 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1145 }
1146
1147 /*
1148 * resolve_patch:
1149 *
1150 * Return the target of the patch identified by TYPE and TARGET.
1151 */
1152 static gpointer
1153 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1154 {
1155 MonoJumpInfo ji;
1156 ERROR_DECL (error);
1157 gpointer res;
1158
1159 memset (&ji, 0, sizeof (ji));
1160 ji.type = type;
1161 ji.data.target = target;
1162
1163 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, error);
1164 mono_error_assert_ok (error);
1165
1166 return res;
1167 }
1168
1169 /*
1170 * convert_full:
1171 *
1172 * Emit code to convert the LLVM value V to DTYPE.
1173 */
1174 static LLVMValueRef
1175 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1176 {
1177 LLVMTypeRef stype = LLVMTypeOf (v);
1178
1179 if (stype != dtype) {
1180 gboolean ext = FALSE;
1181
1182 /* Extend */
1183 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1184 ext = TRUE;
1185 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1186 ext = TRUE;
1187 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1188 ext = TRUE;
1189
1190 if (ext)
1191 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1192
1193 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1194 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1195
1196 /* Trunc */
1197 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1198 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1199 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1200 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1201 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1202 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1203 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1204 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1205
1206 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1207 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1208 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1209 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1210 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1211 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1212
1213 if (mono_arch_is_soft_float ()) {
1214 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1215 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1216 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1217 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1218 }
1219
1220 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1221 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1222
1223 mono_llvm_dump_value (v);
1224 mono_llvm_dump_value (LLVMConstNull (dtype));
1225 printf ("\n");
1226 g_assert_not_reached ();
1227 return NULL;
1228 } else {
1229 return v;
1230 }
1231 }
1232
1233 static LLVMValueRef
1234 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1235 {
1236 return convert_full (ctx, v, dtype, FALSE);
1237 }
1238
1239 static void
1240 emit_memset (EmitContext *ctx, LLVMBuilderRef builder, LLVMValueRef v, LLVMValueRef size, int alignment)
1241 {
1242 LLVMValueRef args [5];
1243 int aindex = 0;
1244
1245 args [aindex ++] = v;
1246 args [aindex ++] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
1247 args [aindex ++] = size;
1248 #if LLVM_API_VERSION < 900
1249 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), alignment, FALSE);
1250 #endif
1251 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
1252 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMSET), args, aindex, "");
1253 }
1254
1255 /*
1256 * emit_volatile_load:
1257 *
1258 * If vreg is volatile, emit a load from its address.
1259 */
1260 static LLVMValueRef
1261 emit_volatile_load (EmitContext *ctx, int vreg)
1262 {
1263 MonoType *t;
1264 LLVMValueRef v;
1265
1266 // On arm64, we pass the rgctx in a callee saved
1267 // register on arm64 (x15), and llvm might keep the value in that register
1268 // even through the register is marked as 'reserved' inside llvm.
1269
1270 v = mono_llvm_build_load (ctx->builder, ctx->addresses [vreg], "", TRUE);
1271 t = ctx->vreg_cli_types [vreg];
1272 if (t && !t->byref) {
1273 /*
1274 * Might have to zero extend since llvm doesn't have
1275 * unsigned types.
1276 */
1277 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1278 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1279 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1280 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1281 else if (t->type == MONO_TYPE_U8)
1282 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1283 }
1284
1285 return v;
1286 }
1287
1288 /*
1289 * emit_volatile_store:
1290 *
1291 * If VREG is volatile, emit a store from its value to its address.
1292 */
1293 static void
1294 emit_volatile_store (EmitContext *ctx, int vreg)
1295 {
1296 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1297
1298 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1299 g_assert (ctx->addresses [vreg]);
1300 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1301 }
1302 }
1303
1304 static LLVMTypeRef
1305 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1306 {
1307 LLVMTypeRef ret_type;
1308 LLVMTypeRef *param_types = NULL;
1309 LLVMTypeRef res;
1310 int i, pindex;
1311 MonoType *rtype;
1312
1313 ret_type = type_to_llvm_type (ctx, sig->ret);
1314 if (!ctx_ok (ctx))
1315 return NULL;
1316 rtype = mini_get_underlying_type (sig->ret);
1317
1318 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1319 pindex = 0;
1320
1321 if (sig->hasthis)
1322 param_types [pindex ++] = ThisType ();
1323 for (i = 0; i < sig->param_count; ++i)
1324 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1325
1326 if (!ctx_ok (ctx)) {
1327 g_free (param_types);
1328 return NULL;
1329 }
1330
1331 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1332 g_free (param_types);
1333
1334 return res;
1335 }
1336
1337 /*
1338 * sig_to_llvm_sig_full:
1339 *
1340 * Return the LLVM signature corresponding to the mono signature SIG using the
1341 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1342 */
1343 static LLVMTypeRef
1344 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1345 {
1346 LLVMTypeRef ret_type;
1347 LLVMTypeRef *param_types = NULL;
1348 LLVMTypeRef res;
1349 int i, j, pindex, vret_arg_pindex = 0;
1350 gboolean vretaddr = FALSE;
1351 MonoType *rtype;
1352
1353 if (!cinfo)
1354 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1355
1356 ret_type = type_to_llvm_type (ctx, sig->ret);
1357 if (!ctx_ok (ctx))
1358 return NULL;
1359 rtype = mini_get_underlying_type (sig->ret);
1360
1361 switch (cinfo->ret.storage) {
1362 case LLVMArgVtypeInReg:
1363 /* LLVM models this by returning an aggregate value */
1364 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1365 LLVMTypeRef members [2];
1366
1367 members [0] = IntPtrType ();
1368 ret_type = LLVMStructType (members, 1, FALSE);
1369 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1370 /* Empty struct */
1371 ret_type = LLVMVoidType ();
1372 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1373 LLVMTypeRef members [2];
1374
1375 members [0] = IntPtrType ();
1376 members [1] = IntPtrType ();
1377 ret_type = LLVMStructType (members, 2, FALSE);
1378 } else {
1379 g_assert_not_reached ();
1380 }
1381 break;
1382 case LLVMArgVtypeByVal:
1383 /* Vtype returned normally by val */
1384 break;
1385 case LLVMArgVtypeAsScalar: {
1386 int size = mono_class_value_size (mono_class_from_mono_type_internal (rtype), NULL);
1387 /* LLVM models this by returning an int */
1388 if (size < TARGET_SIZEOF_VOID_P) {
1389 g_assert (cinfo->ret.nslots == 1);
1390 ret_type = LLVMIntType (size * 8);
1391 } else {
1392 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1393 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (target_mgreg_t) * 8);
1394 }
1395 break;
1396 }
1397 case LLVMArgAsIArgs:
1398 ret_type = LLVMArrayType (IntPtrType (), cinfo->ret.nslots);
1399 break;
1400 case LLVMArgFpStruct: {
1401 /* Vtype returned as a fp struct */
1402 LLVMTypeRef members [16];
1403
1404 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1405 for (i = 0; i < cinfo->ret.nslots; ++i)
1406 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1407 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1408 break;
1409 }
1410 case LLVMArgVtypeByRef:
1411 /* Vtype returned using a hidden argument */
1412 ret_type = LLVMVoidType ();
1413 break;
1414 case LLVMArgVtypeRetAddr:
1415 case LLVMArgGsharedvtFixed:
1416 case LLVMArgGsharedvtFixedVtype:
1417 case LLVMArgGsharedvtVariable:
1418 vretaddr = TRUE;
1419 ret_type = LLVMVoidType ();
1420 break;
1421 default:
1422 break;
1423 }
1424
1425 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1426 pindex = 0;
1427 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1428 /*
1429 * Has to be the first argument because of the sret argument attribute
1430 * FIXME: This might conflict with passing 'this' as the first argument, but
1431 * this is only used on arm64 which has a dedicated struct return register.
1432 */
1433 cinfo->vret_arg_pindex = pindex;
1434 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1435 if (!ctx_ok (ctx)) {
1436 g_free (param_types);
1437 return NULL;
1438 }
1439 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1440 pindex ++;
1441 }
1442 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1443 cinfo->rgctx_arg_pindex = pindex;
1444 param_types [pindex] = ctx->module->ptr_type;
1445 pindex ++;
1446 }
1447 if (cinfo->imt_arg) {
1448 cinfo->imt_arg_pindex = pindex;
1449 param_types [pindex] = ctx->module->ptr_type;
1450 pindex ++;
1451 }
1452 if (vretaddr) {
1453 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1454 vret_arg_pindex = pindex;
1455 if (cinfo->vret_arg_index == 1) {
1456 /* Add the slots consumed by the first argument */
1457 LLVMArgInfo *ainfo = &cinfo->args [0];
1458 switch (ainfo->storage) {
1459 case LLVMArgVtypeInReg:
1460 for (j = 0; j < 2; ++j) {
1461 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1462 vret_arg_pindex ++;
1463 }
1464 break;
1465 default:
1466 vret_arg_pindex ++;
1467 }
1468 }
1469
1470 cinfo->vret_arg_pindex = vret_arg_pindex;
1471 }
1472
1473 if (vretaddr && vret_arg_pindex == pindex)
1474 param_types [pindex ++] = IntPtrType ();
1475 if (sig->hasthis) {
1476 cinfo->this_arg_pindex = pindex;
1477 param_types [pindex ++] = ThisType ();
1478 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1479 }
1480 if (vretaddr && vret_arg_pindex == pindex)
1481 param_types [pindex ++] = IntPtrType ();
1482 for (i = 0; i < sig->param_count; ++i) {
1483 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1484
1485 if (vretaddr && vret_arg_pindex == pindex)
1486 param_types [pindex ++] = IntPtrType ();
1487 ainfo->pindex = pindex;
1488
1489 switch (ainfo->storage) {
1490 case LLVMArgVtypeInReg:
1491 for (j = 0; j < 2; ++j) {
1492 switch (ainfo->pair_storage [j]) {
1493 case LLVMArgInIReg:
1494 param_types [pindex ++] = LLVMIntType (TARGET_SIZEOF_VOID_P * 8);
1495 break;
1496 case LLVMArgNone:
1497 break;
1498 default:
1499 g_assert_not_reached ();
1500 }
1501 }
1502 break;
1503 case LLVMArgVtypeByVal:
1504 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1505 if (!ctx_ok (ctx))
1506 break;
1507 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1508 pindex ++;
1509 break;
1510 case LLVMArgAsIArgs:
1511 if (ainfo->esize == 8)
1512 param_types [pindex] = LLVMArrayType (LLVMInt64Type (), ainfo->nslots);
1513 else
1514 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1515 pindex ++;
1516 break;
1517 case LLVMArgVtypeAddr:
1518 case LLVMArgVtypeByRef:
1519 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1520 if (!ctx_ok (ctx))
1521 break;
1522 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1523 pindex ++;
1524 break;
1525 case LLVMArgAsFpArgs: {
1526 int j;
1527
1528 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1529 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1530 param_types [pindex ++] = LLVMDoubleType ();
1531 for (j = 0; j < ainfo->nslots; ++j)
1532 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1533 break;
1534 }
1535 case LLVMArgVtypeAsScalar:
1536 g_assert_not_reached ();
1537 break;
1538 case LLVMArgGsharedvtFixed:
1539 case LLVMArgGsharedvtFixedVtype:
1540 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1541 break;
1542 case LLVMArgGsharedvtVariable:
1543 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1544 break;
1545 default:
1546 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1547 break;
1548 }
1549 }
1550 if (!ctx_ok (ctx)) {
1551 g_free (param_types);
1552 return NULL;
1553 }
1554 if (vretaddr && vret_arg_pindex == pindex)
1555 param_types [pindex ++] = IntPtrType ();
1556 if (ctx->llvm_only && cinfo->rgctx_arg) {
1557 /* Pass the rgctx as the last argument */
1558 cinfo->rgctx_arg_pindex = pindex;
1559 param_types [pindex] = ctx->module->ptr_type;
1560 pindex ++;
1561 } else if (ctx->llvm_only && cinfo->dummy_arg) {
1562 /* Pass a dummy arg last */
1563 cinfo->dummy_arg_pindex = pindex;
1564 param_types [pindex] = ctx->module->ptr_type;
1565 pindex ++;
1566 }
1567
1568 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1569 g_free (param_types);
1570
1571 return res;
1572 }
1573
1574 static LLVMTypeRef
1575 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1576 {
1577 return sig_to_llvm_sig_full (ctx, sig, NULL);
1578 }
1579
1580 /*
1581 * LLVMFunctionType1:
1582 *
1583 * Create an LLVM function type from the arguments.
1584 */
1585 static G_GNUC_UNUSED LLVMTypeRef
1586 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1587 int IsVarArg)
1588 {
1589 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1590 }
1591
1592 /*
1593 * LLVMFunctionType1:
1594 *
1595 * Create an LLVM function type from the arguments.
1596 */
1597 static G_GNUC_UNUSED LLVMTypeRef
1598 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1599 LLVMTypeRef ParamType1,
1600 int IsVarArg)
1601 {
1602 LLVMTypeRef param_types [1];
1603
1604 param_types [0] = ParamType1;
1605
1606 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1607 }
1608
1609 /*
1610 * LLVMFunctionType2:
1611 *
1612 * Create an LLVM function type from the arguments.
1613 */
1614 static G_GNUC_UNUSED LLVMTypeRef
1615 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1616 LLVMTypeRef ParamType1,
1617 LLVMTypeRef ParamType2,
1618 int IsVarArg)
1619 {
1620 LLVMTypeRef param_types [2];
1621
1622 param_types [0] = ParamType1;
1623 param_types [1] = ParamType2;
1624
1625 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1626 }
1627
1628 /*
1629 * LLVMFunctionType3:
1630 *
1631 * Create an LLVM function type from the arguments.
1632 */
1633 static G_GNUC_UNUSED LLVMTypeRef
1634 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1635 LLVMTypeRef ParamType1,
1636 LLVMTypeRef ParamType2,
1637 LLVMTypeRef ParamType3,
1638 int IsVarArg)
1639 {
1640 LLVMTypeRef param_types [3];
1641
1642 param_types [0] = ParamType1;
1643 param_types [1] = ParamType2;
1644 param_types [2] = ParamType3;
1645
1646 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1647 }
1648
1649 static G_GNUC_UNUSED LLVMTypeRef
1650 LLVMFunctionType4 (LLVMTypeRef ReturnType,
1651 LLVMTypeRef ParamType1,
1652 LLVMTypeRef ParamType2,
1653 LLVMTypeRef ParamType3,
1654 LLVMTypeRef ParamType4,
1655 int IsVarArg)
1656 {
1657 LLVMTypeRef param_types [4];
1658
1659 param_types [0] = ParamType1;
1660 param_types [1] = ParamType2;
1661 param_types [2] = ParamType3;
1662 param_types [3] = ParamType4;
1663
1664 return LLVMFunctionType (ReturnType, param_types, 4, IsVarArg);
1665 }
1666
1667 static G_GNUC_UNUSED LLVMTypeRef
1668 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1669 LLVMTypeRef ParamType1,
1670 LLVMTypeRef ParamType2,
1671 LLVMTypeRef ParamType3,
1672 LLVMTypeRef ParamType4,
1673 LLVMTypeRef ParamType5,
1674 int IsVarArg)
1675 {
1676 LLVMTypeRef param_types [5];
1677
1678 param_types [0] = ParamType1;
1679 param_types [1] = ParamType2;
1680 param_types [2] = ParamType3;
1681 param_types [3] = ParamType4;
1682 param_types [4] = ParamType5;
1683
1684 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1685 }
1686
1687 /*
1688 * create_builder:
1689 *
1690 * Create an LLVM builder and remember it so it can be freed later.
1691 */
1692 static LLVMBuilderRef
1693 create_builder (EmitContext *ctx)
1694 {
1695 LLVMBuilderRef builder = LLVMCreateBuilder ();
1696 if (mono_use_fast_math)
1697 mono_llvm_set_fast_math (builder);
1698
1699 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1700
1701 emit_default_dbg_loc (ctx, builder);
1702
1703 return builder;
1704 }
1705
1706 static char*
1707 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1708 {
1709 char *name;
1710 int len;
1711
1712 switch (type) {
1713 case MONO_PATCH_INFO_JIT_ICALL_ID:
1714 name = g_strdup_printf ("jit_icall_%s", mono_find_jit_icall_info ((MonoJitICallId)(gsize)data)->name);
1715 break;
1716 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1717 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1718 name = g_strdup_printf ("rgctx_slot_index_%s", mono_rgctx_info_type_to_str (entry->info_type));
1719 break;
1720 }
1721 case MONO_PATCH_INFO_AOT_MODULE:
1722 case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG:
1723 case MONO_PATCH_INFO_GC_CARD_TABLE_ADDR:
1724 case MONO_PATCH_INFO_GC_NURSERY_START:
1725 case MONO_PATCH_INFO_GC_NURSERY_BITS:
1726 case MONO_PATCH_INFO_INTERRUPTION_REQUEST_FLAG:
1727 name = g_strdup_printf ("%s", mono_ji_type_to_string (type));
1728 len = strlen (name);
1729 for (int i = 0; i < len; ++i)
1730 name [i] = tolower (name [i]);
1731 break;
1732 default:
1733 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1734 len = strlen (name);
1735 for (int i = 0; i < len; ++i)
1736 name [i] = tolower (name [i]);
1737 break;
1738 }
1739
1740 return name;
1741 }
1742
1743 static int
1744 compute_aot_got_offset (MonoLLVMModule *module, MonoJumpInfo *ji, LLVMTypeRef llvm_type)
1745 {
1746 guint32 got_offset = mono_aot_get_got_offset (ji);
1747
1748 LLVMTypeRef lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (got_offset));
1749
1750 if (!lookup_type) {
1751 lookup_type = llvm_type;
1752 } else if (llvm_type != lookup_type) {
1753 lookup_type = module->ptr_type;
1754 } else {
1755 return got_offset;
1756 }
1757
1758 g_hash_table_insert (module->got_idx_to_type, GINT_TO_POINTER (got_offset), lookup_type);
1759 return got_offset;
1760 }
1761
1762 /* Allocate a GOT slot for TYPE/DATA, and emit IR to load it */
1763 static LLVMValueRef
1764 get_aotconst_module (MonoLLVMModule *module, LLVMBuilderRef builder, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type,
1765 guint32 *out_got_offset, MonoJumpInfo **out_ji)
1766 {
1767 guint32 got_offset;
1768 LLVMValueRef load;
1769
1770 MonoJumpInfo tmp_ji;
1771 tmp_ji.type = type;
1772 tmp_ji.data.target = data;
1773
1774 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1775
1776 if (out_ji)
1777 *out_ji = ji;
1778
1779 got_offset = compute_aot_got_offset (module, ji, llvm_type);
1780 module->max_got_offset = MAX (module->max_got_offset, got_offset);
1781
1782 if (out_got_offset)
1783 *out_got_offset = got_offset;
1784
1785 LLVMValueRef const_var = g_hash_table_lookup (module->aotconst_vars, GINT_TO_POINTER (got_offset));
1786 if (!const_var) {
1787 LLVMTypeRef type = llvm_type;
1788 // FIXME:
1789 char *name = get_aotconst_name (ji->type, ji->data.target, got_offset);
1790 char *symbol = g_strdup_printf ("aotconst_%s", name);
1791 g_free (name);
1792 LLVMValueRef v = LLVMAddGlobal (module->lmodule, type, symbol);
1793 LLVMSetVisibility (v, LLVMHiddenVisibility);
1794 LLVMSetLinkage (v, LLVMInternalLinkage);
1795 LLVMSetInitializer (v, LLVMConstNull (type));
1796 // FIXME:
1797 LLVMSetAlignment (v, 8);
1798
1799 g_hash_table_insert (module->aotconst_vars, GINT_TO_POINTER (got_offset), v);
1800 const_var = v;
1801 }
1802
1803 load = LLVMBuildLoad (builder, const_var, "");
1804
1805 if (mono_aot_is_shared_got_offset (got_offset))
1806 set_invariant_load_flag (load);
1807 if (type == MONO_PATCH_INFO_LDSTR)
1808 set_nonnull_load_flag (load);
1809
1810 load = LLVMBuildBitCast (builder, load, llvm_type, "");
1811
1812 return load;
1813 }
1814
1815 static LLVMValueRef
1816 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1817 {
1818 MonoCompile *cfg;
1819 guint32 got_offset;
1820 MonoJumpInfo *ji;
1821 LLVMValueRef load;
1822
1823 cfg = ctx->cfg;
1824
1825 MonoJumpInfo tmp_ji;
1826 tmp_ji.type = type;
1827 tmp_ji.data.target = data;
1828
1829 load = get_aotconst_module (ctx->module, ctx->builder, type, data, llvm_type, &got_offset, &ji);
1830
1831 ji->next = cfg->patch_info;
1832 cfg->patch_info = ji;
1833
1834 /*
1835 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1836 * explicitly initialize it.
1837 */
1838 if (!mono_aot_is_shared_got_offset (got_offset)) {
1839 //mono_print_ji (ji);
1840 //printf ("\n");
1841 ctx->cfg->got_access_count ++;
1842 }
1843
1844 return load;
1845 }
1846
1847 static LLVMValueRef
1848 get_dummy_aotconst (EmitContext *ctx, LLVMTypeRef llvm_type)
1849 {
1850 LLVMValueRef indexes [2];
1851 LLVMValueRef got_entry_addr, load;
1852
1853 LLVMBuilderRef builder = ctx->builder;
1854 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1855 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1856 got_entry_addr = LLVMBuildGEP (builder, ctx->module->dummy_got_var, indexes, 2, "");
1857
1858 load = LLVMBuildLoad (builder, got_entry_addr, "");
1859 load = convert (ctx, load, llvm_type);
1860 return load;
1861 }
1862
1863 typedef struct {
1864 MonoJumpInfo *ji;
1865 MonoMethod *method;
1866 LLVMValueRef load;
1867 LLVMTypeRef type;
1868 } CallSite;
1869
1870 static LLVMValueRef
1871 get_callee_llvmonly (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1872 {
1873 LLVMValueRef callee;
1874 char *callee_name = NULL;
1875
1876 if (ctx->module->static_link && ctx->module->assembly->image != mono_get_corlib ()) {
1877 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1878 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1879 g_assert (info);
1880 if (info->func != info->wrapper) {
1881 type = MONO_PATCH_INFO_METHOD;
1882 data = mono_icall_get_wrapper_method (info);
1883 callee_name = mono_aot_get_mangled_method_name ((MonoMethod*)data);
1884 }
1885 } else if (type == MONO_PATCH_INFO_METHOD) {
1886 MonoMethod *method = (MonoMethod*)data;
1887 if (m_class_get_image (method->klass) != ctx->module->assembly->image && mono_aot_is_externally_callable (method))
1888 callee_name = mono_aot_get_mangled_method_name (method);
1889 }
1890 }
1891
1892 if (!callee_name)
1893 callee_name = mono_aot_get_direct_call_symbol (type, data);
1894 if (callee_name) {
1895 /* Directly callable */
1896 // FIXME: Locking
1897 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1898 if (!callee) {
1899 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1900
1901 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1902
1903 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1904 } else {
1905 /* LLVMTypeRef's are uniqued */
1906 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
1907 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
1908
1909 g_free (callee_name);
1910 }
1911 return callee;
1912 }
1913
1914 /*
1915 * Change references to icalls/pinvokes/jit icalls to their wrappers when in corlib, so
1916 * they can be called directly.
1917 */
1918 if (ctx->module->assembly->image == mono_get_corlib () && type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1919 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1920
1921 if (info->func != info->wrapper) {
1922 type = MONO_PATCH_INFO_METHOD;
1923 data = mono_icall_get_wrapper_method (info);
1924 }
1925 }
1926 if (ctx->module->assembly->image == mono_get_corlib () && type == MONO_PATCH_INFO_METHOD) {
1927 MonoMethod *method = (MonoMethod*)data;
1928 if (m_method_is_icall (method) || m_method_is_pinvoke (method))
1929 data = mono_marshal_get_native_wrapper (method, TRUE, TRUE);
1930 }
1931
1932 /*
1933 * Instead of emitting an indirect call through a got slot, emit a placeholder, and
1934 * replace it with a direct call or an indirect call in mono_llvm_fixup_aot_module ()
1935 * after all methods have been emitted.
1936 */
1937 if (type == MONO_PATCH_INFO_METHOD) {
1938 MonoMethod *method = (MonoMethod*)data;
1939 if (m_class_get_image (method->klass)->assembly == ctx->module->assembly) {
1940 MonoJumpInfo tmp_ji;
1941
1942 tmp_ji.type = type;
1943 tmp_ji.data.target = method;
1944
1945 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1946 ji->next = ctx->cfg->patch_info;
1947 ctx->cfg->patch_info = ji;
1948 LLVMTypeRef llvm_type = LLVMPointerType (llvm_sig, 0);
1949
1950 ctx->cfg->got_access_count ++;
1951
1952 CallSite *info = g_new0 (CallSite, 1);
1953 info->method = method;
1954 info->ji = ji;
1955 info->type = llvm_type;
1956
1957 /*
1958 * Emit a dummy load to represent the callee, and either replace it with
1959 * a reference to the llvm method for the callee, or from a load from the
1960 * GOT.
1961 */
1962 LLVMValueRef load = get_dummy_aotconst (ctx, llvm_type);
1963 info->load = load;
1964
1965 g_ptr_array_add (ctx->callsite_list, info);
1966
1967 return load;
1968 }
1969 }
1970
1971 /*
1972 * All other calls are made through the GOT.
1973 */
1974 callee = get_aotconst (ctx, type, data, LLVMPointerType (llvm_sig, 0));
1975
1976 return callee;
1977 }
1978
1979 /*
1980 * get_callee:
1981 *
1982 * Return an llvm value representing the callee given by the arguments.
1983 */
1984 static LLVMValueRef
1985 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1986 {
1987 LLVMValueRef callee;
1988 char *callee_name;
1989 MonoJumpInfo *ji = NULL;
1990
1991 if (ctx->llvm_only)
1992 return get_callee_llvmonly (ctx, llvm_sig, type, data);
1993
1994 callee_name = NULL;
1995 /* Cross-assembly direct calls */
1996 if (type == MONO_PATCH_INFO_METHOD) {
1997 MonoMethod *cmethod = (MonoMethod*)data;
1998
1999 if (m_class_get_image (cmethod->klass) != ctx->module->assembly->image) {
2000 MonoJumpInfo tmp_ji;
2001
2002 memset (&tmp_ji, 0, sizeof (MonoJumpInfo));
2003 tmp_ji.type = type;
2004 tmp_ji.data.target = data;
2005 if (mono_aot_is_direct_callable (&tmp_ji)) {
2006 /*
2007 * This will add a reference to cmethod's image so it will
2008 * be loaded when the current AOT image is loaded, so
2009 * the GOT slots used by the init method code are initialized.
2010 */
2011 tmp_ji.type = MONO_PATCH_INFO_IMAGE;
2012 tmp_ji.data.image = m_class_get_image (cmethod->klass);
2013 ji = mono_aot_patch_info_dup (&tmp_ji);
2014 mono_aot_get_got_offset (ji);
2015
2016 callee_name = mono_aot_get_mangled_method_name (cmethod);
2017
2018 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
2019 if (!callee) {
2020 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
2021
2022 LLVMSetLinkage (callee, LLVMExternalLinkage);
2023
2024 g_hash_table_insert (ctx->module->direct_callables, callee_name, callee);
2025 } else {
2026 /* LLVMTypeRef's are uniqued */
2027 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
2028 callee = LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
2029
2030 g_free (callee_name);
2031 }
2032 return callee;
2033 }
2034 }
2035 }
2036
2037 callee_name = mono_aot_get_plt_symbol (type, data);
2038 if (!callee_name)
2039 return NULL;
2040
2041 if (ctx->cfg->compile_aot)
2042 /* Add a patch so referenced wrappers can be compiled in full aot mode */
2043 mono_add_patch_info (ctx->cfg, 0, type, data);
2044
2045 // FIXME: Locking
2046 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
2047 if (!callee) {
2048 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
2049
2050 LLVMSetVisibility (callee, LLVMHiddenVisibility);
2051
2052 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
2053 }
2054
2055 if (ctx->cfg->compile_aot) {
2056 ji = g_new0 (MonoJumpInfo, 1);
2057 ji->type = type;
2058 ji->data.target = data;
2059
2060 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
2061 }
2062
2063 return callee;
2064 }
2065
2066 static LLVMValueRef
2067 get_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
2068 {
2069 gpointer target;
2070
2071 // This won't be patched so compile the wrapper immediately
2072 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
2073 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
2074 target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
2075 } else {
2076 target = resolve_patch (ctx->cfg, type, data);
2077 }
2078
2079 LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
2080 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
2081 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
2082 LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
2083 return callee;
2084 }
2085
2086 static int
2087 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
2088 {
2089 MonoMethodHeader *header = cfg->header;
2090 MonoExceptionClause *clause;
2091 int i;
2092
2093 /* Directly */
2094 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
2095 return (bb->region >> 8) - 1;
2096
2097 /* Indirectly */
2098 for (i = 0; i < header->num_clauses; ++i) {
2099 clause = &header->clauses [i];
2100
2101 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
2102 return i;
2103 }
2104
2105 return -1;
2106 }
2107
2108 static MonoExceptionClause *
2109 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
2110 {
2111 if (bb == cfg->bb_init)
2112 return NULL;
2113 // Since they're sorted by nesting we just need
2114 // the first one that the bb is a member of
2115 for (int i = 0; i < cfg->header->num_clauses; i++) {
2116 MonoExceptionClause *curr = &cfg->header->clauses [i];
2117
2118 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
2119 return curr;
2120 }
2121
2122 return NULL;
2123 }
2124
2125 static void
2126 set_metadata_flag (LLVMValueRef v, const char *flag_name)
2127 {
2128 LLVMValueRef md_arg;
2129 int md_kind;
2130
2131 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2132 md_arg = LLVMMDString ("mono", 4);
2133 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2134 }
2135
2136 static void
2137 set_nonnull_load_flag (LLVMValueRef v)
2138 {
2139 LLVMValueRef md_arg;
2140 int md_kind;
2141 const char *flag_name;
2142
2143 flag_name = "nonnull";
2144 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2145 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2146 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2147 }
2148
2149 static void
2150 set_nontemporal_flag (LLVMValueRef v)
2151 {
2152 LLVMValueRef md_arg;
2153 int md_kind;
2154 const char *flag_name;
2155
2156 // FIXME: Cache this
2157 flag_name = "nontemporal";
2158 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2159 md_arg = const_int32 (1);
2160 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2161 }
2162
2163 static void
2164 set_invariant_load_flag (LLVMValueRef v)
2165 {
2166 LLVMValueRef md_arg;
2167 int md_kind;
2168 const char *flag_name;
2169
2170 // FIXME: Cache this
2171 flag_name = "invariant.load";
2172 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2173 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2174 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2175 }
2176
2177 /*
2178 * emit_call:
2179 *
2180 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
2181 * a try region.
2182 */
2183 static LLVMValueRef
2184 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
2185 {
2186 MonoCompile *cfg = ctx->cfg;
2187 LLVMValueRef lcall = NULL;
2188 LLVMBuilderRef builder = *builder_ref;
2189 MonoExceptionClause *clause;
2190
2191 if (ctx->llvm_only) {
2192 clause = get_most_deep_clause (cfg, ctx, bb);
2193
2194 if (clause) {
2195 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2196
2197 /*
2198 * Have to use an invoke instead of a call, branching to the
2199 * handler bblock of the clause containing this bblock.
2200 */
2201 intptr_t key = CLAUSE_END(clause);
2202
2203 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
2204
2205 // FIXME: Find the one that has the lowest end bound for the right start address
2206 // FIXME: Finally + nesting
2207
2208 if (lpad_bb) {
2209 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
2210
2211 /* Use an invoke */
2212 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
2213
2214 builder = ctx->builder = create_builder (ctx);
2215 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2216
2217 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2218 }
2219 }
2220 } else {
2221 int clause_index = get_handler_clause (cfg, bb);
2222
2223 if (clause_index != -1) {
2224 MonoMethodHeader *header = cfg->header;
2225 MonoExceptionClause *ec = &header->clauses [clause_index];
2226 MonoBasicBlock *tblock;
2227 LLVMBasicBlockRef ex_bb, noex_bb;
2228
2229 /*
2230 * Have to use an invoke instead of a call, branching to the
2231 * handler bblock of the clause containing this bblock.
2232 */
2233
2234 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY || ec->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2235
2236 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
2237 g_assert (tblock);
2238
2239 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
2240
2241 ex_bb = get_bb (ctx, tblock);
2242
2243 noex_bb = gen_bb (ctx, "NOEX_BB");
2244
2245 /* Use an invoke */
2246 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
2247
2248 builder = ctx->builder = create_builder (ctx);
2249 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2250
2251 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2252 }
2253 }
2254
2255 if (!lcall) {
2256 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
2257 ctx->builder = builder;
2258 }
2259
2260 if (builder_ref)
2261 *builder_ref = ctx->builder;
2262
2263 return lcall;
2264 }
2265
2266 static LLVMValueRef
2267 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, LLVMValueRef base, const char *name, gboolean is_faulting, gboolean is_volatile, BarrierKind barrier)
2268 {
2269 LLVMValueRef res;
2270
2271 /*
2272 * We emit volatile loads for loads which can fault, because otherwise
2273 * LLVM will generate invalid code when encountering a load from a
2274 * NULL address.
2275 */
2276 if (barrier != LLVM_BARRIER_NONE)
2277 res = mono_llvm_build_atomic_load (*builder_ref, addr, name, is_volatile, size, barrier);
2278 else
2279 res = mono_llvm_build_load (*builder_ref, addr, name, is_volatile);
2280
2281 return res;
2282 }
2283
2284 static void
2285 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, gboolean is_volatile, BarrierKind barrier)
2286 {
2287 if (barrier != LLVM_BARRIER_NONE)
2288 mono_llvm_build_aligned_store (*builder_ref, value, addr, barrier, size);
2289 else
2290 mono_llvm_build_store (*builder_ref, value, addr, is_volatile, barrier);
2291 }
2292
2293 static void
2294 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, gboolean is_volatile)
2295 {
2296 emit_store_general (ctx, bb, builder_ref, size, value, addr, base, is_faulting, is_volatile, LLVM_BARRIER_NONE);
2297 }
2298
2299 /*
2300 * emit_cond_system_exception:
2301 *
2302 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2303 * Might set the ctx exception.
2304 */
2305 static void
2306 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp, gboolean force_explicit)
2307 {
2308 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2309 LLVMBuilderRef builder;
2310 MonoClass *exc_class;
2311 LLVMValueRef args [2];
2312 LLVMValueRef callee;
2313 gboolean no_pc = FALSE;
2314 static MonoClass *exc_classes [MONO_EXC_INTRINS_NUM];
2315
2316 if (IS_TARGET_AMD64)
2317 /* Some platforms don't require the pc argument */
2318 no_pc = TRUE;
2319
2320 int exc_id = mini_exception_id_by_name (exc_type);
2321 if (!exc_classes [exc_id])
2322 exc_classes [exc_id] = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
2323 exc_class = exc_classes [exc_id];
2324
2325 ex_bb = gen_bb (ctx, "EX_BB");
2326 if (ctx->llvm_only)
2327 ex2_bb = gen_bb (ctx, "EX2_BB");
2328 noex_bb = gen_bb (ctx, "NOEX_BB");
2329
2330 LLVMValueRef branch = LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2331 if (exc_id == MONO_EXC_NULL_REF && !ctx->cfg->disable_llvm_implicit_null_checks && !force_explicit) {
2332 mono_llvm_set_implicit_branch (ctx->builder, branch);
2333 }
2334
2335 /* Emit exception throwing code */
2336 ctx->builder = builder = create_builder (ctx);
2337 LLVMPositionBuilderAtEnd (builder, ex_bb);
2338
2339 if (ctx->cfg->llvm_only) {
2340 LLVMBuildBr (builder, ex2_bb);
2341
2342 ctx->builder = builder = create_builder (ctx);
2343 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2344
2345 if (exc_id == MONO_EXC_NULL_REF) {
2346 static LLVMTypeRef sig;
2347
2348 if (!sig)
2349 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
2350 /* Can't cache this */
2351 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
2352 emit_call (ctx, bb, &builder, callee, NULL, 0);
2353 } else {
2354 static LLVMTypeRef sig;
2355
2356 if (!sig)
2357 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2358 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_throw_corlib_exception));
2359 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2360 emit_call (ctx, bb, &builder, callee, args, 1);
2361 }
2362
2363 LLVMBuildUnreachable (builder);
2364
2365 ctx->builder = builder = create_builder (ctx);
2366 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2367
2368 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2369
2370 ctx->ex_index ++;
2371 return;
2372 }
2373
2374 callee = ctx->module->throw_corlib_exception;
2375 if (!callee) {
2376 LLVMTypeRef sig;
2377
2378 if (no_pc)
2379 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2380 else
2381 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2382
2383 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_throw_corlib_exception_abs_trampoline;
2384
2385 if (ctx->cfg->compile_aot) {
2386 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2387 } else {
2388 /*
2389 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2390 * - On x86, LLVM generated code doesn't push the arguments
2391 * - The trampoline takes the throw address as an arguments, not a pc offset.
2392 */
2393 callee = get_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2394
2395 /*
2396 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2397 * added by get_jit_callee ().
2398 */
2399 ex2_bb = gen_bb (ctx, "EX2_BB");
2400 LLVMBuildBr (builder, ex2_bb);
2401 ex_bb = ex2_bb;
2402
2403 ctx->builder = builder = create_builder (ctx);
2404 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2405 }
2406 }
2407
2408 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2409
2410 /*
2411 * The LLVM mono branch contains changes so a block address can be passed as an
2412 * argument to a call.
2413 */
2414 if (no_pc) {
2415 emit_call (ctx, bb, &builder, callee, args, 1);
2416 } else {
2417 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2418 emit_call (ctx, bb, &builder, callee, args, 2);
2419 }
2420
2421 LLVMBuildUnreachable (builder);
2422
2423 ctx->builder = builder = create_builder (ctx);
2424 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2425
2426 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2427
2428 ctx->ex_index ++;
2429 return;
2430 }
2431
2432 /*
2433 * emit_args_to_vtype:
2434 *
2435 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2436 */
2437 static void
2438 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2439 {
2440 int j, size, nslots;
2441 MonoClass *klass;
2442
2443 t = mini_get_underlying_type (t);
2444 klass = mono_class_from_mono_type_internal (t);
2445 size = mono_class_value_size (klass, NULL);
2446
2447 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
2448 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2449
2450 if (ainfo->storage == LLVMArgAsFpArgs)
2451 nslots = ainfo->nslots;
2452 else
2453 nslots = 2;
2454
2455 for (j = 0; j < nslots; ++j) {
2456 LLVMValueRef index [2], addr, daddr;
2457 int part_size = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2458 LLVMTypeRef part_type;
2459
2460 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2461 part_size ++;
2462
2463 if (ainfo->pair_storage [j] == LLVMArgNone)
2464 continue;
2465
2466 switch (ainfo->pair_storage [j]) {
2467 case LLVMArgInIReg: {
2468 part_type = LLVMIntType (part_size * 8);
2469 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass)) {
2470 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2471 addr = LLVMBuildGEP (builder, address, index, 1, "");
2472 } else {
2473 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2474 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2475 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2476 }
2477 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2478 break;
2479 }
2480 case LLVMArgInFPReg: {
2481 LLVMTypeRef arg_type;
2482
2483 if (ainfo->esize == 8)
2484 arg_type = LLVMDoubleType ();
2485 else
2486 arg_type = LLVMFloatType ();
2487
2488 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2489 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2490 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2491 LLVMBuildStore (builder, args [j], addr);
2492 break;
2493 }
2494 case LLVMArgNone:
2495 break;
2496 default:
2497 g_assert_not_reached ();
2498 }
2499
2500 size -= TARGET_SIZEOF_VOID_P;
2501 }
2502 }
2503
2504 /*
2505 * emit_vtype_to_args:
2506 *
2507 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2508 * into ARGS, and the number of arguments into NARGS.
2509 */
2510 static void
2511 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2512 {
2513 int pindex = 0;
2514 int j, size, nslots;
2515 LLVMTypeRef arg_type;
2516
2517 t = mini_get_underlying_type (t);
2518 size = get_vtype_size (t);
2519
2520 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t)))
2521 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2522
2523 if (ainfo->storage == LLVMArgAsFpArgs)
2524 nslots = ainfo->nslots;
2525 else
2526 nslots = 2;
2527 for (j = 0; j < nslots; ++j) {
2528 LLVMValueRef index [2], addr, daddr;
2529 int partsize = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2530
2531 if (ainfo->pair_storage [j] == LLVMArgNone)
2532 continue;
2533
2534 switch (ainfo->pair_storage [j]) {
2535 case LLVMArgInIReg:
2536 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
2537 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2538 addr = LLVMBuildGEP (builder, address, index, 1, "");
2539 } else {
2540 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2541 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2542 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2543 }
2544 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2545 break;
2546 case LLVMArgInFPReg:
2547 if (ainfo->esize == 8)
2548 arg_type = LLVMDoubleType ();
2549 else
2550 arg_type = LLVMFloatType ();
2551 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2552 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2553 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2554 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2555 break;
2556 case LLVMArgNone:
2557 break;
2558 default:
2559 g_assert_not_reached ();
2560 }
2561 size -= TARGET_SIZEOF_VOID_P;
2562 }
2563
2564 *nargs = pindex;
2565 }
2566
2567 static LLVMValueRef
2568 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2569 {
2570 /*
2571 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2572 * get executed every time control reaches them.
2573 */
2574 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2575
2576 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2577 return ctx->last_alloca;
2578 }
2579
2580 static LLVMValueRef
2581 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2582 {
2583 return build_alloca_llvm_type_name (ctx, t, align, "");
2584 }
2585
2586 static LLVMValueRef
2587 build_alloca (EmitContext *ctx, MonoType *t)
2588 {
2589 MonoClass *k = mono_class_from_mono_type_internal (t);
2590 int align;
2591
2592 g_assert (!mini_is_gsharedvt_variable_type (t));
2593
2594 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2595 align = mono_class_value_size (k, NULL);
2596 else
2597 align = mono_class_min_align (k);
2598
2599 /* Sometimes align is not a power of 2 */
2600 while (mono_is_power_of_two (align) == -1)
2601 align ++;
2602
2603 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2604 }
2605
2606 static LLVMValueRef
2607 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2608 {
2609 /*
2610 * gsharedvt local.
2611 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2612 */
2613 MonoCompile *cfg = ctx->cfg;
2614 LLVMBuilderRef builder = ctx->builder;
2615 LLVMValueRef offset, offset_var;
2616 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2617 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2618 LLVMValueRef ptr;
2619 char *name;
2620
2621 g_assert (info_var);
2622 g_assert (locals_var);
2623
2624 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2625
2626 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * TARGET_SIZEOF_VOID_P), FALSE);
2627 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2628
2629 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2630 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2631
2632 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2633 }
2634
2635 /*
2636 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2637 */
2638 static void
2639 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2640 {
2641 if (!module->used)
2642 module->used = g_ptr_array_sized_new (16);
2643 g_ptr_array_add (module->used, global);
2644 }
2645
2646 static void
2647 emit_llvm_used (MonoLLVMModule *module)
2648 {
2649 LLVMModuleRef lmodule = module->lmodule;
2650 LLVMTypeRef used_type;
2651 LLVMValueRef used, *used_elem;
2652 int i;
2653
2654 if (!module->used)
2655 return;
2656
2657 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2658 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2659 used_elem = g_new0 (LLVMValueRef, module->used->len);
2660 for (i = 0; i < module->used->len; ++i)
2661 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2662 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2663 LLVMSetLinkage (used, LLVMAppendingLinkage);
2664 LLVMSetSection (used, "llvm.metadata");
2665 }
2666
2667 /*
2668 * emit_get_method:
2669 *
2670 * Emit a function mapping method indexes to their code
2671 */
2672 static void
2673 emit_get_method (MonoLLVMModule *module)
2674 {
2675 LLVMModuleRef lmodule = module->lmodule;
2676 LLVMValueRef func, switch_ins, m;
2677 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2678 LLVMBasicBlockRef *bbs = NULL;
2679 LLVMTypeRef rtype;
2680 LLVMBuilderRef builder = LLVMCreateBuilder ();
2681 LLVMValueRef table = NULL;
2682 char *name;
2683 int i;
2684 gboolean emit_table = FALSE;
2685
2686 #ifdef TARGET_WASM
2687 /*
2688 * Emit a table of functions instead of a switch statement,
2689 * its very efficient on wasm. This might be usable on
2690 * other platforms too.
2691 */
2692 emit_table = TRUE;
2693 #endif
2694
2695 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2696
2697 if (emit_table) {
2698 LLVMTypeRef table_type;
2699 LLVMValueRef *table_elems;
2700 char *table_name;
2701
2702 int table_len = module->max_method_idx + 1;
2703 table_type = LLVMArrayType (rtype, table_len);
2704 table_name = g_strdup_printf ("%s_method_table", module->assembly->aname.name);
2705 table = LLVMAddGlobal (lmodule, table_type, table_name);
2706 table_elems = g_new0 (LLVMValueRef, table_len);
2707 for (i = 0; i < table_len; ++i) {
2708 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2709 if (m && !g_hash_table_lookup (module->no_method_table_lmethods, m))
2710 table_elems [i] = LLVMBuildBitCast (builder, m, rtype, "");
2711 else
2712 table_elems [i] = LLVMConstNull (rtype);
2713 }
2714 LLVMSetInitializer (table, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), table_elems, table_len));
2715 }
2716
2717 /*
2718 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2719 * but generating code seems safer.
2720 */
2721 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2722 LLVMSetLinkage (func, LLVMExternalLinkage);
2723 LLVMSetVisibility (func, LLVMHiddenVisibility);
2724 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2725 module->get_method = func;
2726
2727 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2728
2729 /*
2730 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2731 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2732 * then we will have to find another solution.
2733 */
2734
2735 name = g_strdup_printf ("BB_CODE_START");
2736 code_start_bb = LLVMAppendBasicBlock (func, name);
2737 g_free (name);
2738 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2739 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2740
2741 name = g_strdup_printf ("BB_CODE_END");
2742 code_end_bb = LLVMAppendBasicBlock (func, name);
2743 g_free (name);
2744 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2745 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2746
2747 if (emit_table) {
2748 /*
2749 * switch (index) {
2750 * case -1: return code_start;
2751 * case -2: return code_end;
2752 * default: return method_table [index];
2753 */
2754 LLVMBasicBlockRef default_bb = LLVMAppendBasicBlock (func, "DEFAULT");
2755 LLVMPositionBuilderAtEnd (builder, default_bb);
2756 LLVMValueRef base = table;
2757 LLVMValueRef indexes [2];
2758 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2759 indexes [1] = LLVMGetParam (func, 0);
2760 LLVMValueRef addr = LLVMBuildGEP (builder, base, indexes, 2, "");
2761 LLVMValueRef res = mono_llvm_build_load (builder, addr, "", FALSE);
2762 LLVMBuildRet (builder, res);
2763
2764 LLVMPositionBuilderAtEnd (builder, entry_bb);
2765
2766 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), default_bb, 0);
2767 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2768 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2769 } else {
2770 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2771 for (i = 0; i < module->max_method_idx + 1; ++i) {
2772 name = g_strdup_printf ("BB_%d", i);
2773 bb = LLVMAppendBasicBlock (func, name);
2774 g_free (name);
2775 bbs [i] = bb;
2776
2777 LLVMPositionBuilderAtEnd (builder, bb);
2778
2779 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2780 if (m && !g_hash_table_lookup (module->no_method_table_lmethods, m))
2781 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2782 else
2783 LLVMBuildRet (builder, LLVMConstNull (rtype));
2784 }
2785
2786 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2787 LLVMPositionBuilderAtEnd (builder, fail_bb);
2788 LLVMBuildRet (builder, LLVMConstNull (rtype));
2789
2790 LLVMPositionBuilderAtEnd (builder, entry_bb);
2791
2792 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2793 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2794 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2795 for (i = 0; i < module->max_method_idx + 1; ++i) {
2796 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2797 }
2798 }
2799
2800 mark_as_used (module, func);
2801
2802 LLVMDisposeBuilder (builder);
2803 }
2804
2805 /*
2806 * emit_get_unbox_tramp:
2807 *
2808 * Emit a function mapping method indexes to their unbox trampoline
2809 */
2810 static void
2811 emit_get_unbox_tramp (MonoLLVMModule *module)
2812 {
2813 LLVMModuleRef lmodule = module->lmodule;
2814 LLVMValueRef func, switch_ins, m;
2815 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2816 LLVMBasicBlockRef *bbs;
2817 LLVMTypeRef rtype;
2818 LLVMBuilderRef builder = LLVMCreateBuilder ();
2819 char *name;
2820 int i;
2821 gboolean emit_table = FALSE;
2822
2823 /* Similar to emit_get_method () */
2824
2825 #ifndef TARGET_WATCHOS
2826 emit_table = TRUE;
2827 #endif
2828
2829 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2830
2831 if (emit_table) {
2832 // About 10% of methods have an unbox tramp, so emit a table of indexes for them
2833 // that the runtime can search using a binary search
2834 int len = 0;
2835 for (i = 0; i < module->max_method_idx + 1; ++i) {
2836 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2837 if (m)
2838 len ++;
2839 }
2840
2841 LLVMTypeRef table_type, elemtype;
2842 LLVMValueRef *table_elems;
2843 LLVMValueRef table;
2844 char *table_name;
2845 int table_len;
2846 int elemsize;
2847
2848 table_len = len;
2849 elemsize = module->max_method_idx < 65000 ? 2 : 4;
2850
2851 // The index table
2852 elemtype = elemsize == 2 ? LLVMInt16Type () : LLVMInt32Type ();
2853 table_type = LLVMArrayType (elemtype, table_len);
2854 table_name = g_strdup_printf ("%s_unbox_tramp_indexes", module->assembly->aname.name);
2855 table = LLVMAddGlobal (lmodule, table_type, table_name);
2856 table_elems = g_new0 (LLVMValueRef, table_len);
2857 int idx = 0;
2858 for (i = 0; i < module->max_method_idx + 1; ++i) {
2859 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2860 if (m)
2861 table_elems [idx ++] = LLVMConstInt (elemtype, i, FALSE);
2862 }
2863 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2864 module->unbox_tramp_indexes = table;
2865
2866 // The trampoline table
2867 elemtype = rtype;
2868 table_type = LLVMArrayType (elemtype, table_len);
2869 table_name = g_strdup_printf ("%s_unbox_trampolines", module->assembly->aname.name);
2870 table = LLVMAddGlobal (lmodule, table_type, table_name);
2871 table_elems = g_new0 (LLVMValueRef, table_len);
2872 idx = 0;
2873 for (i = 0; i < module->max_method_idx + 1; ++i) {
2874 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2875 if (m)
2876 table_elems [idx ++] = LLVMBuildBitCast (builder, m, rtype, "");
2877 }
2878 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2879 module->unbox_trampolines = table;
2880
2881 module->unbox_tramp_num = table_len;
2882 module->unbox_tramp_elemsize = elemsize;
2883 return;
2884 }
2885
2886 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2887 LLVMSetLinkage (func, LLVMExternalLinkage);
2888 LLVMSetVisibility (func, LLVMHiddenVisibility);
2889 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2890 module->get_unbox_tramp = func;
2891
2892 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2893
2894 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2895 for (i = 0; i < module->max_method_idx + 1; ++i) {
2896 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2897 if (!m)
2898 continue;
2899
2900 name = g_strdup_printf ("BB_%d", i);
2901 bb = LLVMAppendBasicBlock (func, name);
2902 g_free (name);
2903 bbs [i] = bb;
2904
2905 LLVMPositionBuilderAtEnd (builder, bb);
2906
2907 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2908 }
2909
2910 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2911 LLVMPositionBuilderAtEnd (builder, fail_bb);
2912 LLVMBuildRet (builder, LLVMConstNull (rtype));
2913
2914 LLVMPositionBuilderAtEnd (builder, entry_bb);
2915
2916 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2917 for (i = 0; i < module->max_method_idx + 1; ++i) {
2918 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2919 if (!m)
2920 continue;
2921
2922 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2923 }
2924
2925 mark_as_used (module, func);
2926 LLVMDisposeBuilder (builder);
2927 }
2928
2929 /*
2930 * emit_init_aotconst:
2931 *
2932 * Emit a function to initialize the aotconst_ variables. Called by the runtime.
2933 */
2934 static void
2935 emit_init_aotconst (MonoLLVMModule *module)
2936 {
2937 LLVMModuleRef lmodule = module->lmodule;
2938 LLVMValueRef func, switch_ins;
2939 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2940 LLVMBasicBlockRef *bbs = NULL;
2941 LLVMBuilderRef builder = LLVMCreateBuilder ();
2942 char *name;
2943
2944 func = LLVMAddFunction (lmodule, module->init_aotconst_symbol, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
2945 LLVMSetLinkage (func, LLVMExternalLinkage);
2946 LLVMSetVisibility (func, LLVMHiddenVisibility);
2947 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2948 module->init_aotconst_func = func;
2949
2950 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2951
2952 bbs = g_new0 (LLVMBasicBlockRef, module->max_got_offset + 1);
2953 for (int i = 0; i < module->max_got_offset + 1; ++i) {
2954 name = g_strdup_printf ("BB_%d", i);
2955 bb = LLVMAppendBasicBlock (func, name);
2956 g_free (name);
2957 bbs [i] = bb;
2958
2959 LLVMPositionBuilderAtEnd (builder, bb);
2960
2961 LLVMValueRef var = g_hash_table_lookup (module->aotconst_vars, GINT_TO_POINTER (i));
2962 if (var) {
2963 LLVMValueRef addr = LLVMBuildBitCast (builder, var, LLVMPointerType (IntPtrType (), 0), "");
2964 LLVMBuildStore (builder, LLVMGetParam (func, 1), addr);
2965 }
2966 LLVMBuildRetVoid (builder);
2967 }
2968
2969 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2970 LLVMPositionBuilderAtEnd (builder, fail_bb);
2971 LLVMBuildRetVoid (builder);
2972
2973 LLVMPositionBuilderAtEnd (builder, entry_bb);
2974
2975 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2976 for (int i = 0; i < module->max_got_offset + 1; ++i)
2977 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2978
2979 LLVMDisposeBuilder (builder);
2980 }
2981
2982 /* Add a function to mark the beginning of LLVM code */
2983 static void
2984 emit_llvm_code_start (MonoLLVMModule *module)
2985 {
2986 LLVMModuleRef lmodule = module->lmodule;
2987 LLVMValueRef func;
2988 LLVMBasicBlockRef entry_bb;
2989 LLVMBuilderRef builder;
2990
2991 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2992 LLVMSetLinkage (func, LLVMInternalLinkage);
2993 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2994 module->code_start = func;
2995 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2996 builder = LLVMCreateBuilder ();
2997 LLVMPositionBuilderAtEnd (builder, entry_bb);
2998 LLVMBuildRetVoid (builder);
2999 LLVMDisposeBuilder (builder);
3000 }
3001
3002 /*
3003 * emit_init_func:
3004 *
3005 * Emit functions to initialize LLVM methods.
3006 * These are wrappers around the mini_llvm_init_method () JIT icall.
3007 * The wrappers handle adding the 'amodule' argument, loading the vtable from different locations, and they have
3008 * a cold calling convention.
3009 */
3010 static LLVMValueRef
3011 emit_init_func (MonoLLVMModule *module, MonoAotInitSubtype subtype)
3012 {
3013 LLVMModuleRef lmodule = module->lmodule;
3014 LLVMValueRef func, indexes [2], args [16], callee, info_var, index_var, inited_var, cmp;
3015 LLVMBasicBlockRef entry_bb, inited_bb, notinited_bb;
3016 LLVMBuilderRef builder;
3017 LLVMTypeRef icall_sig;
3018 const char *wrapper_name = mono_marshal_get_aot_init_wrapper_name (subtype);
3019 LLVMTypeRef func_type = NULL;
3020 LLVMTypeRef arg_type = module->ptr_type;
3021
3022 char *name = g_strdup_printf ("%s_%s", module->global_prefix, wrapper_name);
3023
3024 switch (subtype) {
3025 case AOT_INIT_METHOD:
3026 func_type = LLVMFunctionType1 (LLVMVoidType (), arg_type, FALSE);
3027 break;
3028 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3029 case AOT_INIT_METHOD_GSHARED_VTABLE:
3030 func_type = LLVMFunctionType2 (LLVMVoidType (), arg_type, IntPtrType (), FALSE);
3031 break;
3032 case AOT_INIT_METHOD_GSHARED_THIS:
3033 func_type = LLVMFunctionType2 (LLVMVoidType (), arg_type, ObjRefType (), FALSE);
3034 break;
3035 default:
3036 g_assert_not_reached ();
3037 }
3038
3039 func = LLVMAddFunction (lmodule, name, func_type);
3040
3041 info_var = LLVMGetParam (func, 0);
3042
3043 LLVMSetLinkage (func, LLVMInternalLinkage);
3044
3045 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3046
3047 set_cold_cconv (func);
3048
3049 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3050
3051 builder = LLVMCreateBuilder ();
3052 LLVMPositionBuilderAtEnd (builder, entry_bb);
3053
3054 /* Load method_index which is emitted at the start of the method info */
3055 indexes [0] = const_int32 (0);
3056 indexes [1] = const_int32 (0);
3057 // FIXME: Make sure its aligned
3058 index_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, LLVMBuildBitCast (builder, info_var, LLVMPointerType (LLVMInt32Type (), 0), ""), indexes, 1, ""), "method_index");
3059
3060 /* Check for is_inited here as well, since this can be called from JITted code which might not check it */
3061 indexes [0] = const_int32 (0);
3062 indexes [1] = index_var;
3063 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""), "is_inited");
3064
3065 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
3066
3067 inited_bb = LLVMAppendBasicBlock (func, "INITED");
3068 notinited_bb = LLVMAppendBasicBlock (func, "NOT_INITED");
3069
3070 LLVMBuildCondBr (builder, cmp, notinited_bb, inited_bb);
3071
3072 LLVMPositionBuilderAtEnd (builder, notinited_bb);
3073
3074 LLVMValueRef amodule_var = get_aotconst_module (module, builder, MONO_PATCH_INFO_AOT_MODULE, NULL, LLVMPointerType (IntPtrType (), 0), NULL, NULL);
3075
3076 args [0] = LLVMBuildPtrToInt (builder, module->info_var, IntPtrType (), "");
3077 args [1] = LLVMBuildPtrToInt (builder, amodule_var, IntPtrType (), "");
3078 args [2] = info_var;
3079
3080 switch (subtype) {
3081 case AOT_INIT_METHOD:
3082 args [3] = LLVMConstNull (IntPtrType ());
3083 break;
3084 case AOT_INIT_METHOD_GSHARED_VTABLE:
3085 args [3] = LLVMGetParam (func, 1);
3086 break;
3087 case AOT_INIT_METHOD_GSHARED_THIS:
3088 /* Load this->vtable */
3089 args [3] = LLVMBuildBitCast (builder, LLVMGetParam (func, 1), LLVMPointerType (IntPtrType (), 0), "");
3090 indexes [0] = const_int32 (MONO_STRUCT_OFFSET (MonoObject, vtable) / SIZEOF_VOID_P);
3091 args [3] = LLVMBuildLoad (builder, LLVMBuildGEP (builder, args [3], indexes, 1, ""), "vtable");
3092 break;
3093 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3094 /* Load mrgctx->vtable */
3095 args [3] = LLVMBuildIntToPtr (builder, LLVMGetParam (func, 1), LLVMPointerType (IntPtrType (), 0), "");
3096 indexes [0] = const_int32 (MONO_STRUCT_OFFSET (MonoMethodRuntimeGenericContext, class_vtable) / SIZEOF_VOID_P);
3097 args [3] = LLVMBuildLoad (builder, LLVMBuildGEP (builder, args [3], indexes, 1, ""), "vtable");
3098 break;
3099 default:
3100 g_assert_not_reached ();
3101 break;
3102 }
3103
3104 /* Call the mini_llvm_init_method JIT icall */
3105 icall_sig = LLVMFunctionType4 (LLVMVoidType (), IntPtrType (), IntPtrType (), arg_type, IntPtrType (), FALSE);
3106 callee = get_aotconst_module (module, builder, MONO_PATCH_INFO_JIT_ICALL_ID, GINT_TO_POINTER (MONO_JIT_ICALL_mini_llvm_init_method), LLVMPointerType (icall_sig, 0), NULL, NULL);
3107 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (icall_sig), "");
3108
3109 /*
3110 * Set the inited flag
3111 * This is already done by the LLVM methods themselves, but its needed by JITted methods.
3112 */
3113 indexes [0] = const_int32 (0);
3114 indexes [1] = index_var;
3115 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
3116
3117 LLVMBuildBr (builder, inited_bb);
3118
3119 LLVMPositionBuilderAtEnd (builder, inited_bb);
3120 LLVMBuildRetVoid (builder);
3121
3122 LLVMVerifyFunction (func, LLVMAbortProcessAction);
3123 LLVMDisposeBuilder (builder);
3124 g_free (name);
3125
3126 return func;
3127 }
3128
3129 /* Emit a wrapper around the parameterless JIT icall ICALL_ID with a cold calling convention */
3130 static LLVMValueRef
3131 emit_icall_cold_wrapper (MonoLLVMModule *module, LLVMModuleRef lmodule, MonoJitICallId icall_id, gboolean aot)
3132 {
3133 LLVMValueRef func, callee;
3134 LLVMBasicBlockRef entry_bb;
3135 LLVMBuilderRef builder;
3136 LLVMTypeRef sig;
3137 char *name;
3138
3139 name = g_strdup_printf ("%s_icall_cold_wrapper_%d", module->global_prefix, icall_id);
3140
3141 func = LLVMAddFunction (lmodule, name, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3142 sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3143 LLVMSetLinkage (func, LLVMInternalLinkage);
3144 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3145 set_cold_cconv (func);
3146
3147 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3148 builder = LLVMCreateBuilder ();
3149 LLVMPositionBuilderAtEnd (builder, entry_bb);
3150
3151 if (aot) {
3152 callee = get_aotconst_module (module, builder, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id), LLVMPointerType (sig, 0), NULL, NULL);
3153 } else {
3154 MonoJitICallInfo * const info = mono_find_jit_icall_info (icall_id);
3155 gpointer target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
3156
3157 LLVMValueRef tramp_var = LLVMAddGlobal (lmodule, LLVMPointerType (sig, 0), name);
3158 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (sig, 0)));
3159 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3160 callee = LLVMBuildLoad (builder, tramp_var, "");
3161 }
3162 LLVMBuildCall (builder, callee, NULL, 0, "");
3163
3164 LLVMBuildRetVoid (builder);
3165
3166 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3167 LLVMDisposeBuilder (builder);
3168 return func;
3169 }
3170
3171 /*
3172 * Emit wrappers around the C icalls used to initialize llvm methods, to
3173 * make the calling code smaller and to enable usage of the llvm
3174 * cold calling convention.
3175 */
3176 static void
3177 emit_init_funcs (MonoLLVMModule *module)
3178 {
3179 module->init_method = emit_init_func (module, AOT_INIT_METHOD);
3180 module->init_method_gshared_mrgctx = emit_init_func (module, AOT_INIT_METHOD_GSHARED_MRGCTX);
3181 module->init_method_gshared_this = emit_init_func (module, AOT_INIT_METHOD_GSHARED_THIS);
3182 module->init_method_gshared_vtable = emit_init_func (module, AOT_INIT_METHOD_GSHARED_VTABLE);
3183 }
3184
3185 static LLVMValueRef
3186 get_init_func (MonoLLVMModule *module, MonoAotInitSubtype subtype)
3187 {
3188 switch (subtype) {
3189 case AOT_INIT_METHOD:
3190 return module->init_method;
3191 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3192 return module->init_method_gshared_mrgctx;
3193 case AOT_INIT_METHOD_GSHARED_THIS:
3194 return module->init_method_gshared_this;
3195 case AOT_INIT_METHOD_GSHARED_VTABLE:
3196 return module->init_method_gshared_vtable;
3197 default:
3198 g_assert_not_reached ();
3199 }
3200 }
3201
3202 static void
3203 emit_gc_safepoint_poll (MonoLLVMModule *module, LLVMModuleRef lmodule, MonoCompile *cfg)
3204 {
3205 gboolean is_aot = cfg == NULL || cfg->compile_aot;
3206 LLVMValueRef func = mono_llvm_get_or_insert_gc_safepoint_poll (lmodule);
3207 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3208 if (is_aot) {
3209 LLVMSetLinkage (func, LLVMWeakODRLinkage);
3210 } else {
3211 mono_llvm_add_func_attr (func, LLVM_ATTR_OPTIMIZE_NONE); // no need to waste time here, the function is already optimized and will be inlined.
3212 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE); // optnone attribute requires noinline (but it will be inlined anyway)
3213 if (!module->gc_poll_cold_wrapper_compiled) {
3214 ERROR_DECL (error);
3215 /* Compiling a method here is a bit ugly, but it works */
3216 MonoMethod *wrapper = mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL);
3217 module->gc_poll_cold_wrapper_compiled = mono_jit_compile_method (wrapper, error);
3218 mono_error_assert_ok (error);
3219 }
3220 }
3221 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.entry");
3222 LLVMBasicBlockRef poll_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.poll");
3223 LLVMBasicBlockRef exit_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.exit");
3224 LLVMTypeRef ptr_type = LLVMPointerType (IntPtrType (), 0);
3225 LLVMBuilderRef builder = LLVMCreateBuilder ();
3226
3227 /* entry: */
3228 LLVMPositionBuilderAtEnd (builder, entry_bb);
3229 LLVMValueRef poll_val_ptr;
3230 if (is_aot) {
3231 poll_val_ptr = get_aotconst_module (module, builder, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, NULL, ptr_type, NULL, NULL);
3232 } else {
3233 LLVMValueRef poll_val_int = LLVMConstInt (IntPtrType (), (guint64) &mono_polling_required, FALSE);
3234 poll_val_ptr = LLVMBuildIntToPtr (builder, poll_val_int, ptr_type, "");
3235 }
3236 LLVMValueRef poll_val_ptr_load = LLVMBuildLoad (builder, poll_val_ptr, ""); // probably needs to be volatile
3237 LLVMValueRef poll_val = LLVMBuildPtrToInt (builder, poll_val_ptr_load, IntPtrType (), "");
3238 LLVMValueRef poll_val_zero = LLVMConstNull (LLVMTypeOf (poll_val));
3239 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, poll_val, poll_val_zero, "");
3240 mono_llvm_build_weighted_branch (builder, cmp, exit_bb, poll_bb, 1000 /* weight for exit_bb */, 1 /* weight for poll_bb */);
3241
3242 /* poll: */
3243 LLVMPositionBuilderAtEnd (builder, poll_bb);
3244 LLVMValueRef call;
3245 if (is_aot) {
3246 LLVMValueRef icall_wrapper = emit_icall_cold_wrapper (module, lmodule, MONO_JIT_ICALL_mono_threads_state_poll, TRUE);
3247 module->gc_poll_cold_wrapper = icall_wrapper;
3248 call = LLVMBuildCall (builder, icall_wrapper, NULL, 0, "");
3249 } else {
3250 // in JIT mode we have to emit @gc.safepoint_poll function for each method (module)
3251 // this function calls gc_poll_cold_wrapper_compiled via a global variable.
3252 // @gc.safepoint_poll will be inlined and can be deleted after -place-safepoints pass.
3253 LLVMTypeRef poll_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3254 LLVMTypeRef poll_sig_ptr = LLVMPointerType (poll_sig, 0);
3255 gpointer target = resolve_patch (cfg, MONO_PATCH_INFO_ABS, module->gc_poll_cold_wrapper_compiled);
3256 LLVMValueRef tramp_var = LLVMAddGlobal (lmodule, poll_sig_ptr, "mono_threads_state_poll");
3257 LLVMValueRef target_val = LLVMConstInt (LLVMInt64Type (), (guint64) target, FALSE);
3258 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (target_val, poll_sig_ptr));
3259 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3260 LLVMValueRef callee = LLVMBuildLoad (builder, tramp_var, "");
3261 call = LLVMBuildCall (builder, callee, NULL, 0, "");
3262 }
3263 set_call_cold_cconv (call);
3264 LLVMBuildBr (builder, exit_bb);
3265
3266 /* exit: */
3267 LLVMPositionBuilderAtEnd (builder, exit_bb);
3268 LLVMBuildRetVoid (builder);
3269 LLVMDisposeBuilder (builder);
3270 }
3271
3272 static void
3273 emit_llvm_code_end (MonoLLVMModule *module)
3274 {
3275 LLVMModuleRef lmodule = module->lmodule;
3276 LLVMValueRef func;
3277 LLVMBasicBlockRef entry_bb;
3278 LLVMBuilderRef builder;
3279
3280 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3281 LLVMSetLinkage (func, LLVMInternalLinkage);
3282 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3283 module->code_end = func;
3284 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3285 builder = LLVMCreateBuilder ();
3286 LLVMPositionBuilderAtEnd (builder, entry_bb);
3287 LLVMBuildRetVoid (builder);
3288 LLVMDisposeBuilder (builder);
3289 }
3290
3291 static void
3292 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
3293 {
3294 gboolean need_div_check = ctx->cfg->backend->need_div_check;
3295
3296 if (bb->region)
3297 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
3298 need_div_check = TRUE;
3299
3300 if (!need_div_check)
3301 return;
3302
3303 switch (ins->opcode) {
3304 case OP_IDIV:
3305 case OP_LDIV:
3306 case OP_IREM:
3307 case OP_LREM:
3308 case OP_IDIV_UN:
3309 case OP_LDIV_UN:
3310 case OP_IREM_UN:
3311 case OP_LREM_UN:
3312 case OP_IDIV_IMM:
3313 case OP_LDIV_IMM:
3314 case OP_IREM_IMM:
3315 case OP_LREM_IMM:
3316 case OP_IDIV_UN_IMM:
3317 case OP_LDIV_UN_IMM:
3318 case OP_IREM_UN_IMM:
3319 case OP_LREM_UN_IMM: {
3320 LLVMValueRef cmp;
3321 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
3322 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
3323
3324 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
3325 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp, FALSE);
3326 if (!ctx_ok (ctx))
3327 break;
3328 builder = ctx->builder;
3329
3330 /* b == -1 && a == 0x80000000 */
3331 if (is_signed) {
3332 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
3333 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
3334 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
3335
3336 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
3337 emit_cond_system_exception (ctx, bb, "OverflowException", cmp, FALSE);
3338 if (!ctx_ok (ctx))
3339 break;
3340 builder = ctx->builder;
3341 }
3342 break;
3343 }
3344 default:
3345 break;
3346 }
3347 }
3348
3349 /*
3350 * emit_method_init:
3351 *
3352 * Emit code to initialize the GOT slots used by the method.
3353 */
3354 static void
3355 emit_method_init (EmitContext *ctx)
3356 {
3357 LLVMValueRef indexes [16], args [16], callee;
3358 LLVMValueRef inited_var, cmp, call;
3359 LLVMBasicBlockRef inited_bb, notinited_bb;
3360 LLVMBuilderRef builder = ctx->builder;
3361 MonoCompile *cfg = ctx->cfg;
3362
3363 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
3364
3365 indexes [0] = const_int32 (0);
3366 indexes [1] = const_int32 (cfg->method_index);
3367 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
3368
3369 args [0] = inited_var;
3370 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
3371 inited_var = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I8), args, 2, "");
3372
3373 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
3374
3375 inited_bb = ctx->inited_bb;
3376 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
3377
3378 ctx->cfg->llvmonly_init_cond = LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
3379
3380 builder = ctx->builder = create_builder (ctx);
3381 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
3382
3383 LLVMTypeRef type = LLVMArrayType (LLVMInt8Type (), 0);
3384 char *symbol = g_strdup_printf ("info_dummy_%s", cfg->llvm_method_name);
3385 LLVMValueRef info_var = LLVMAddGlobal (ctx->lmodule, type, symbol);
3386 g_free (symbol);
3387 cfg->llvm_dummy_info_var = info_var;
3388
3389 args [0] = convert (ctx, info_var, ctx->module->ptr_type);
3390
3391 // FIXME: Cache
3392 if (ctx->rgctx_arg && ((cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) ||
3393 mini_method_is_default_method (cfg->method))) {
3394 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3395 callee = ctx->module->init_method_gshared_mrgctx;
3396 call = LLVMBuildCall (builder, callee, args, 2, "");
3397 } else if (ctx->rgctx_arg) {
3398 /* A vtable is passed as the rgctx argument */
3399 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3400 callee = ctx->module->init_method_gshared_vtable;
3401 call = LLVMBuildCall (builder, callee, args, 2, "");
3402 } else if (cfg->gshared) {
3403 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
3404 callee = ctx->module->init_method_gshared_this;
3405 call = LLVMBuildCall (builder, callee, args, 2, "");
3406 } else {
3407 callee = ctx->module->init_method;
3408 call = LLVMBuildCall (builder, callee, args, 1, "");
3409 }
3410
3411 /*
3412 * This enables llvm to keep arguments in their original registers/
3413 * scratch registers, since the call will not clobber them.
3414 */
3415 set_call_cold_cconv (call);
3416
3417 // Set the inited flag
3418 indexes [0] = const_int32 (0);
3419 indexes [1] = const_int32 (cfg->method_index);
3420 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""));
3421
3422 LLVMBuildBr (builder, inited_bb);
3423 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
3424
3425 builder = ctx->builder = create_builder (ctx);
3426 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
3427 }
3428
3429 static void
3430 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
3431 {
3432 /*
3433 * Emit unbox trampoline using a tailcall
3434 */
3435 LLVMValueRef tramp, call, *args;
3436 LLVMBuilderRef builder;
3437 LLVMBasicBlockRef lbb;
3438 LLVMCallInfo *linfo;
3439 char *tramp_name;
3440 int i, nargs;
3441
3442 tramp_name = g_strdup_printf ("ut_%s", method_name);
3443 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
3444 LLVMSetLinkage (tramp, LLVMInternalLinkage);
3445 mono_llvm_add_func_attr (tramp, LLVM_ATTR_OPTIMIZE_FOR_SIZE);
3446 //mono_llvm_add_func_attr (tramp, LLVM_ATTR_NO_UNWIND);
3447 linfo = ctx->linfo;
3448 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
3449 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3450 mono_llvm_add_param_attr (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVM_ATTR_IN_REG);
3451 if (ctx->cfg->vret_addr) {
3452 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
3453 if (linfo->ret.storage == LLVMArgVtypeByRef) {
3454 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
3455 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
3456 }
3457 }
3458
3459 lbb = LLVMAppendBasicBlock (tramp, "");
3460 builder = LLVMCreateBuilder ();
3461 LLVMPositionBuilderAtEnd (builder, lbb);
3462
3463 nargs = LLVMCountParamTypes (method_type);
3464 args = g_new0 (LLVMValueRef, nargs);
3465 for (i = 0; i < nargs; ++i) {
3466 args [i] = LLVMGetParam (tramp, i);
3467 if (i == ctx->this_arg_pindex) {
3468 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
3469
3470 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
3471 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), MONO_ABI_SIZEOF (MonoObject), FALSE), "");
3472 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
3473 }
3474 }
3475 call = LLVMBuildCall (builder, method, args, nargs, "");
3476 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3477 mono_llvm_add_instr_attr (call, 1 + ctx->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
3478 if (linfo->ret.storage == LLVMArgVtypeByRef)
3479 mono_llvm_add_instr_attr (call, 1 + linfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
3480
3481 // FIXME: This causes assertions in clang
3482 //mono_llvm_set_must_tailcall (call);
3483 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
3484 LLVMBuildRetVoid (builder);
3485 else
3486 LLVMBuildRet (builder, call);
3487
3488 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
3489 LLVMDisposeBuilder (builder);
3490 }
3491
3492 /*
3493 * emit_entry_bb:
3494 *
3495 * Emit code to load/convert arguments.
3496 */
3497 static void
3498 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
3499 {
3500 int i, j, pindex;
3501 MonoCompile *cfg = ctx->cfg;
3502 MonoMethodSignature *sig = ctx->sig;
3503 LLVMCallInfo *linfo = ctx->linfo;
3504 MonoBasicBlock *bb;
3505 char **names;
3506
3507 LLVMBuilderRef old_builder = ctx->builder;
3508 ctx->builder = builder;
3509
3510 ctx->alloca_builder = create_builder (ctx);
3511
3512 /*
3513 * Handle indirect/volatile variables by allocating memory for them
3514 * using 'alloca', and storing their address in a temporary.
3515 */
3516 for (i = 0; i < cfg->num_varinfo; ++i) {
3517 MonoInst *var = cfg->varinfo [i];
3518 LLVMTypeRef vtype;
3519
3520 if ((var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET))
3521 continue;
3522
3523 #ifdef TARGET_WASM
3524 // For GC stack scanning to work, have to spill all reference variables to the stack
3525 // Some ref variables have type intptr
3526 if (ctx->has_safepoints && (MONO_TYPE_IS_REFERENCE (var->inst_vtype) || var->inst_vtype->type == MONO_TYPE_I) && var != ctx->cfg->rgctx_var)
3527 var->flags |= MONO_INST_INDIRECT;
3528 #endif
3529
3530 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
3531 vtype = type_to_llvm_type (ctx, var->inst_vtype);
3532 if (!ctx_ok (ctx))
3533 return;
3534 /* Could be already created by an OP_VPHI */
3535 if (!ctx->addresses [var->dreg]) {
3536 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
3537 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
3538 }
3539 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
3540 }
3541 }
3542
3543 names = g_new (char *, sig->param_count);
3544 mono_method_get_param_names (cfg->method, (const char **) names);
3545
3546 for (i = 0; i < sig->param_count; ++i) {
3547 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
3548 int reg = cfg->args [i + sig->hasthis]->dreg;
3549 char *name;
3550
3551 pindex = ainfo->pindex;
3552
3553 switch (ainfo->storage) {
3554 case LLVMArgVtypeInReg:
3555 case LLVMArgAsFpArgs: {
3556 LLVMValueRef args [8];
3557 int j;
3558
3559 pindex += ainfo->ndummy_fpargs;
3560
3561 /* The argument is received as a set of int/fp arguments, store them into the real argument */
3562 memset (args, 0, sizeof (args));
3563 if (ainfo->storage == LLVMArgVtypeInReg) {
3564 args [0] = LLVMGetParam (ctx->lmethod, pindex);
3565 if (ainfo->pair_storage [1] != LLVMArgNone)
3566 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
3567 } else {
3568 g_assert (ainfo->nslots <= 8);
3569 for (j = 0; j < ainfo->nslots; ++j)
3570 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
3571 }
3572 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
3573
3574 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
3575 break;
3576 }
3577 case LLVMArgVtypeByVal: {
3578 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3579 break;
3580 }
3581 case LLVMArgVtypeAddr:
3582 case LLVMArgVtypeByRef: {
3583 /* The argument is passed by ref */
3584 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3585 break;
3586 }
3587 case LLVMArgAsIArgs: {
3588 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3589 int size;
3590 MonoType *t = mini_get_underlying_type (ainfo->type);
3591
3592 /* The argument is received as an array of ints, store it into the real argument */
3593 ctx->addresses [reg] = build_alloca (ctx, t);
3594
3595 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
3596 if (size == 0) {
3597 } else if (size < TARGET_SIZEOF_VOID_P) {
3598 /* The upper bits of the registers might not be valid */
3599 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
3600 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
3601 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
3602 } else {
3603 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
3604 }
3605 break;
3606 }
3607 case LLVMArgVtypeAsScalar:
3608 g_assert_not_reached ();
3609 break;
3610 case LLVMArgGsharedvtFixed: {
3611 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
3612 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3613
3614 if (names [i])
3615 name = g_strdup_printf ("arg_%s", names [i]);
3616 else
3617 name = g_strdup_printf ("arg_%d", i);
3618
3619 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
3620 break;
3621 }
3622 case LLVMArgGsharedvtFixedVtype: {
3623 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3624
3625 if (names [i])
3626 name = g_strdup_printf ("vtype_arg_%s", names [i]);
3627 else
3628 name = g_strdup_printf ("vtype_arg_%d", i);
3629
3630 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3631 g_assert (ctx->addresses [reg]);
3632 LLVMSetValueName (ctx->addresses [reg], name);
3633 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
3634 break;
3635 }
3636 case LLVMArgGsharedvtVariable:
3637 /* The IR treats these as variables with addresses */
3638 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3639 break;
3640 default: {
3641 LLVMTypeRef t;
3642 /* Needed to avoid phi argument mismatch errors since operations on pointers produce i32/i64 */
3643 if (ainfo->type->byref)
3644 t = IntPtrType ();
3645 else
3646 t = type_to_llvm_type (ctx, ainfo->type);
3647 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, t), type_is_unsigned (ctx, ainfo->type));
3648 break;
3649 }
3650 }
3651
3652 switch (ainfo->storage) {
3653 case LLVMArgVtypeInReg:
3654 case LLVMArgVtypeByVal:
3655 #ifdef ENABLE_NETCORE
3656 // FIXME: Enabling this fails on windows
3657 case LLVMArgVtypeAddr:
3658 case LLVMArgVtypeByRef:
3659 #endif
3660 {
3661 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type)))
3662 /* Treat these as normal values */
3663 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3664 break;
3665 }
3666 default:
3667 break;
3668 }
3669 }
3670 g_free (names);
3671
3672 if (sig->hasthis) {
3673 /* Handle this arguments as inputs to phi nodes */
3674 int reg = cfg->args [0]->dreg;
3675 if (ctx->vreg_types [reg])
3676 ctx->values [reg] = convert (ctx, ctx->values [reg], ctx->vreg_types [reg]);
3677 }
3678
3679 if (cfg->vret_addr)
3680 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3681 if (sig->hasthis)
3682 emit_volatile_store (ctx, cfg->args [0]->dreg);
3683 for (i = 0; i < sig->param_count; ++i)
3684 if (!mini_type_is_vtype (sig->params [i]))
3685 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3686
3687 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared && !cfg->llvm_only) {
3688 LLVMValueRef this_alloc;
3689
3690 /*
3691 * The exception handling code needs the location where the this argument was
3692 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3693 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3694 * location into the LSDA.
3695 */
3696 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3697 /* This volatile store will keep the alloca alive */
3698 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3699
3700 set_metadata_flag (this_alloc, "mono.this");
3701 }
3702
3703 if (cfg->rgctx_var) {
3704 if (!(cfg->rgctx_var->flags & MONO_INST_VOLATILE)) {
3705 /* FIXME: This could be volatile even in llvmonly mode if used inside a clause etc. */
3706 g_assert (!ctx->addresses [cfg->rgctx_var->dreg]);
3707 ctx->values [cfg->rgctx_var->dreg] = ctx->rgctx_arg;
3708 } else {
3709 LLVMValueRef rgctx_alloc, store;
3710
3711 /*
3712 * We handle the rgctx arg similarly to the this pointer.
3713 */
3714 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3715 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3716 /* This volatile store will keep the alloca alive */
3717 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3718
3719 set_metadata_flag (rgctx_alloc, "mono.this");
3720 }
3721 }
3722
3723 /* Initialize the method if needed */
3724 if (cfg->compile_aot) {
3725 /* Emit a location for the initialization code */
3726 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3727 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3728
3729 LLVMBuildBr (ctx->builder, ctx->init_bb);
3730 builder = ctx->builder = create_builder (ctx);
3731 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3732 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3733 }
3734
3735 /* Compute nesting between clauses */
3736 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3737 for (i = 0; i < cfg->header->num_clauses; ++i) {
3738 for (j = 0; j < cfg->header->num_clauses; ++j) {
3739 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3740 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3741
3742 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3743 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3744 }
3745 }
3746
3747 /*
3748 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3749 * it needs to continue normally, or return back to the exception handling system.
3750 */
3751 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3752 char name [128];
3753
3754 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3755 continue;
3756
3757 if (bb->in_scount == 0) {
3758 LLVMValueRef val;
3759
3760 sprintf (name, "finally_ind_bb%d", bb->block_num);
3761 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3762 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3763
3764 ctx->bblocks [bb->block_num].finally_ind = val;
3765 } else {
3766 /* Create a variable to hold the exception var */
3767 if (!ctx->ex_var)
3768 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3769 }
3770 }
3771 ctx->builder = old_builder;
3772 }
3773
3774 static gboolean
3775 needs_extra_arg (EmitContext *ctx, MonoMethod *method)
3776 {
3777 WrapperInfo *info = NULL;
3778
3779 /*
3780 * When targeting wasm, the caller and callee signature has to match exactly. This means
3781 * that every method which can be called indirectly need an extra arg since the caller
3782 * will call it through an ftnptr and will pass an extra arg.
3783 */
3784 if (!ctx->cfg->llvm_only || !ctx->emit_dummy_arg)
3785 return FALSE;
3786 if (method->wrapper_type)
3787 info = mono_marshal_get_wrapper_info (method);
3788
3789 switch (method->wrapper_type) {
3790 case MONO_WRAPPER_OTHER:
3791 if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
3792 /* Already have an explicit extra arg */
3793 return FALSE;
3794 break;
3795 case MONO_WRAPPER_MANAGED_TO_NATIVE:
3796 if (strstr (method->name, "icall_wrapper"))
3797 /* These are JIT icall wrappers which are only called from JITted code directly */
3798 return FALSE;
3799 /* Normal icalls can be virtual methods which need an extra arg */
3800 break;
3801 case MONO_WRAPPER_RUNTIME_INVOKE:
3802 case MONO_WRAPPER_ALLOC:
3803 case MONO_WRAPPER_CASTCLASS:
3804 case MONO_WRAPPER_WRITE_BARRIER:
3805 case MONO_WRAPPER_NATIVE_TO_MANAGED:
3806 return FALSE;
3807 case MONO_WRAPPER_STELEMREF:
3808 if (info->subtype != WRAPPER_SUBTYPE_VIRTUAL_STELEMREF)
3809 return FALSE;
3810 break;
3811 case MONO_WRAPPER_MANAGED_TO_MANAGED:
3812 if (info->subtype == WRAPPER_SUBTYPE_STRING_CTOR)
3813 return FALSE;
3814 break;
3815 default:
3816 break;
3817 }
3818 if (method->string_ctor)
3819 return FALSE;
3820
3821 /* These are called from gsharedvt code with an indirect call which doesn't pass an extra arg */
3822 if (method->klass == mono_get_string_class () && (strstr (method->name, "memcpy") || strstr (method->name, "bzero")))
3823 return FALSE;
3824 return TRUE;
3825 }
3826
3827 static inline gboolean
3828 is_supported_callconv (EmitContext *ctx, MonoCallInst *call)
3829 {
3830 #if defined(TARGET_WIN32) && defined(TARGET_AMD64)
3831 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) ||
3832 (call->signature->call_convention == MONO_CALL_C) ||
3833 (call->signature->call_convention == MONO_CALL_STDCALL);
3834 #else
3835 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) || ((call->signature->call_convention == MONO_CALL_C) && ctx->llvm_only);
3836 #endif
3837 return result;
3838 }
3839
3840 static void
3841 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3842 {
3843 MonoCompile *cfg = ctx->cfg;
3844 LLVMValueRef *values = ctx->values;
3845 LLVMValueRef *addresses = ctx->addresses;
3846 MonoCallInst *call = (MonoCallInst*)ins;
3847 MonoMethodSignature *sig = call->signature;
3848 LLVMValueRef callee = NULL, lcall;
3849 LLVMValueRef *args;
3850 LLVMCallInfo *cinfo;
3851 GSList *l;
3852 int i, len, nargs;
3853 gboolean vretaddr;
3854 LLVMTypeRef llvm_sig;
3855 gpointer target;
3856 gboolean is_virtual, calli;
3857 LLVMBuilderRef builder = *builder_ref;
3858
3859 /* If both imt and rgctx arg are required, only pass the imt arg, the rgctx trampoline will pass the rgctx */
3860 if (call->imt_arg_reg)
3861 call->rgctx_arg_reg = 0;
3862
3863 if (!is_supported_callconv (ctx, call)) {
3864 set_failure (ctx, "non-default callconv");
3865 return;
3866 }
3867
3868 cinfo = call->cinfo;
3869 g_assert (cinfo);
3870 if (call->rgctx_arg_reg)
3871 cinfo->rgctx_arg = TRUE;
3872 if (call->imt_arg_reg)
3873 cinfo->imt_arg = TRUE;
3874 if (!call->rgctx_arg_reg && call->method && needs_extra_arg (ctx, call->method))
3875 cinfo->dummy_arg = TRUE;
3876
3877 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3878
3879 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3880 if (!ctx_ok (ctx))
3881 return;
3882
3883 int const opcode = ins->opcode;
3884
3885 is_virtual = opcode == OP_VOIDCALL_MEMBASE || opcode == OP_CALL_MEMBASE
3886 || opcode == OP_VCALL_MEMBASE || opcode == OP_LCALL_MEMBASE
3887 || opcode == OP_FCALL_MEMBASE || opcode == OP_RCALL_MEMBASE
3888 || opcode == OP_TAILCALL_MEMBASE;
3889 calli = !call->fptr_is_patch && (opcode == OP_VOIDCALL_REG || opcode == OP_CALL_REG
3890 || opcode == OP_VCALL_REG || opcode == OP_LCALL_REG || opcode == OP_FCALL_REG
3891 || opcode == OP_RCALL_REG || opcode == OP_TAILCALL_REG);
3892
3893 /* FIXME: Avoid creating duplicate methods */
3894
3895 if (ins->flags & MONO_INST_HAS_METHOD) {
3896 if (is_virtual) {
3897 callee = NULL;
3898 } else {
3899 if (cfg->compile_aot) {
3900 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3901 if (!callee) {
3902 set_failure (ctx, "can't encode patch");
3903 return;
3904 }
3905 } else if (cfg->method == call->method) {
3906 callee = ctx->lmethod;
3907 } else {
3908 ERROR_DECL (error);
3909 static int tramp_index;
3910 char *name;
3911
3912 name = g_strdup_printf ("[tramp_%d] %s", tramp_index, mono_method_full_name (call->method, TRUE));
3913 tramp_index ++;
3914
3915 /*
3916 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3917 * Make all calls through a global. The address of the global will be saved in
3918 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3919 * compiled.
3920 */
3921 LLVMValueRef tramp_var = (LLVMValueRef)g_hash_table_lookup (ctx->jit_callees, call->method);
3922 if (!tramp_var) {
3923 target =
3924 mono_create_jit_trampoline (mono_domain_get (),
3925 call->method, error);
3926 if (!is_ok (error)) {
3927 set_failure (ctx, mono_error_get_message (error));
3928 mono_error_cleanup (error);
3929 return;
3930 }
3931
3932 tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
3933 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
3934 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3935 g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
3936 }
3937 callee = LLVMBuildLoad (builder, tramp_var, "");
3938 }
3939 }
3940
3941 if (!cfg->llvm_only && call->method && strstr (m_class_get_name (call->method->klass), "AsyncVoidMethodBuilder")) {
3942 /* LLVM miscompiles async methods */
3943 set_failure (ctx, "#13734");
3944 return;
3945 }
3946 } else if (calli) {
3947 } else {
3948 const MonoJitICallId jit_icall_id = call->jit_icall_id;
3949
3950 if (jit_icall_id) {
3951 if (cfg->compile_aot) {
3952 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3953 if (!callee) {
3954 set_failure (ctx, "can't encode patch");
3955 return;
3956 }
3957 } else {
3958 callee = get_jit_callee (ctx, "", llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3959 }
3960 } else {
3961 if (cfg->compile_aot) {
3962 callee = NULL;
3963 if (cfg->abs_patches) {
3964 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3965 if (abs_ji) {
3966 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3967 if (!callee) {
3968 set_failure (ctx, "can't encode patch");
3969 return;
3970 }
3971 }
3972 }
3973 if (!callee) {
3974 set_failure (ctx, "aot");
3975 return;
3976 }
3977 } else {
3978 if (cfg->abs_patches) {
3979 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3980 if (abs_ji) {
3981 ERROR_DECL (error);
3982
3983 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, error);
3984 mono_error_assert_ok (error);
3985 callee = get_jit_callee (ctx, "", llvm_sig, abs_ji->type, abs_ji->data.target);
3986 } else {
3987 g_assert_not_reached ();
3988 }
3989 } else {
3990 g_assert_not_reached ();
3991 }
3992 }
3993 }
3994 }
3995
3996 if (is_virtual) {
3997 int size = TARGET_SIZEOF_VOID_P;
3998 LLVMValueRef index;
3999
4000 g_assert (ins->inst_offset % size == 0);
4001 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
4002
4003 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
4004 } else if (calli) {
4005 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
4006 } else {
4007 if (ins->flags & MONO_INST_HAS_METHOD) {
4008 }
4009 }
4010
4011 /*
4012 * Collect and convert arguments
4013 */
4014 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg + call->cinfo->dummy_arg + 1;
4015 len = sizeof (LLVMValueRef) * nargs;
4016 args = g_newa (LLVMValueRef, nargs);
4017 memset (args, 0, len);
4018 l = call->out_ireg_args;
4019
4020 if (call->rgctx_arg_reg) {
4021 g_assert (values [call->rgctx_arg_reg]);
4022 g_assert (cinfo->rgctx_arg_pindex < nargs);
4023 /*
4024 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
4025 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
4026 * it using a volatile load.
4027 */
4028 #ifdef TARGET_ARM
4029 if (!ctx->imt_rgctx_loc)
4030 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
4031 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
4032 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
4033 #else
4034 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
4035 #endif
4036 }
4037 if (call->imt_arg_reg) {
4038 g_assert (!ctx->llvm_only);
4039 g_assert (values [call->imt_arg_reg]);
4040 g_assert (cinfo->imt_arg_pindex < nargs);
4041 #ifdef TARGET_ARM
4042 if (!ctx->imt_rgctx_loc)
4043 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
4044 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
4045 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
4046 #else
4047 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
4048 #endif
4049 }
4050 switch (cinfo->ret.storage) {
4051 case LLVMArgGsharedvtVariable: {
4052 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
4053
4054 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
4055 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
4056 } else {
4057 g_assert (addresses [call->inst.dreg]);
4058 args [cinfo->vret_arg_pindex] = convert (ctx, addresses [call->inst.dreg], IntPtrType ());
4059 }
4060 break;
4061 }
4062 default:
4063 if (vretaddr) {
4064 if (!addresses [call->inst.dreg])
4065 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4066 g_assert (cinfo->vret_arg_pindex < nargs);
4067 if (cinfo->ret.storage == LLVMArgVtypeByRef)
4068 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
4069 else
4070 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
4071 }
4072 break;
4073 }
4074
4075 /*
4076 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
4077 * use the real callee for argument type conversion.
4078 */
4079 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
4080 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
4081 LLVMGetParamTypes (callee_type, param_types);
4082
4083 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
4084 guint32 regpair;
4085 int reg, pindex;
4086 LLVMArgInfo *ainfo = &call->cinfo->args [i];
4087
4088 pindex = ainfo->pindex;
4089
4090 regpair = (guint32)(gssize)(l->data);
4091 reg = regpair & 0xffffff;
4092 args [pindex] = values [reg];
4093 switch (ainfo->storage) {
4094 case LLVMArgVtypeInReg:
4095 case LLVMArgAsFpArgs: {
4096 guint32 nargs;
4097 int j;
4098
4099 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
4100 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
4101 pindex += ainfo->ndummy_fpargs;
4102
4103 g_assert (addresses [reg]);
4104 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
4105 pindex += nargs;
4106
4107 // FIXME: alignment
4108 // FIXME: Get rid of the VMOVE
4109 break;
4110 }
4111 case LLVMArgVtypeByVal:
4112 g_assert (addresses [reg]);
4113 args [pindex] = addresses [reg];
4114 break;
4115 case LLVMArgVtypeAddr :
4116 case LLVMArgVtypeByRef: {
4117 g_assert (addresses [reg]);
4118 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4119 break;
4120 }
4121 case LLVMArgAsIArgs:
4122 g_assert (addresses [reg]);
4123 if (ainfo->esize == 8)
4124 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (LLVMInt64Type (), ainfo->nslots), 0)), "");
4125 else
4126 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
4127 break;
4128 case LLVMArgVtypeAsScalar:
4129 g_assert_not_reached ();
4130 break;
4131 case LLVMArgGsharedvtFixed:
4132 case LLVMArgGsharedvtFixedVtype:
4133 g_assert (addresses [reg]);
4134 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4135 break;
4136 case LLVMArgGsharedvtVariable:
4137 g_assert (addresses [reg]);
4138 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
4139 break;
4140 default:
4141 g_assert (args [pindex]);
4142 if (i == 0 && sig->hasthis)
4143 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
4144 else
4145 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
4146 break;
4147 }
4148 g_assert (pindex <= nargs);
4149
4150 l = l->next;
4151 }
4152
4153 if (call->cinfo->dummy_arg) {
4154 g_assert (call->cinfo->dummy_arg_pindex < nargs);
4155 args [call->cinfo->dummy_arg_pindex] = LLVMConstNull (ctx->module->ptr_type);
4156 }
4157
4158 // FIXME: Align call sites
4159
4160 /*
4161 * Emit the call
4162 */
4163 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
4164
4165 mono_llvm_nonnull_state_update (ctx, lcall, call->method, args, LLVMCountParamTypes (llvm_sig));
4166
4167 // If we just allocated an object, it's not null.
4168 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC) {
4169 mono_llvm_set_call_nonnull_ret (lcall);
4170 }
4171
4172 if (ins->opcode != OP_TAILCALL && ins->opcode != OP_TAILCALL_MEMBASE && LLVMGetInstructionOpcode (lcall) == LLVMCall)
4173 mono_llvm_set_call_notailcall (lcall);
4174
4175 // Add original method name we are currently emitting as a custom string metadata (the only way to leave comments in LLVM IR)
4176 if (mono_debug_enabled () && call && call->method)
4177 mono_llvm_add_string_metadata (lcall, "managed_name", mono_method_full_name (call->method, TRUE));
4178
4179 // As per the LLVM docs, a function has a noalias return value if and only if
4180 // it is an allocation function. This is an allocation function.
4181 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC) {
4182 mono_llvm_set_call_noalias_ret (lcall);
4183 // All objects are expected to be 8-byte aligned (SGEN_ALLOC_ALIGN)
4184 mono_llvm_set_alignment_ret (lcall, 8);
4185 }
4186
4187 /*
4188 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
4189 */
4190 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
4191 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
4192 #endif
4193 /* The two can't be used together, so use only one LLVM calling conv to pass them */
4194 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
4195 if (!sig->pinvoke && !cfg->llvm_only)
4196 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
4197
4198 if (cinfo->ret.storage == LLVMArgVtypeByRef)
4199 mono_llvm_add_instr_attr (lcall, 1 + cinfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
4200 if (!ctx->llvm_only && call->rgctx_arg_reg)
4201 mono_llvm_add_instr_attr (lcall, 1 + cinfo->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
4202 if (call->imt_arg_reg)
4203 mono_llvm_add_instr_attr (lcall, 1 + cinfo->imt_arg_pindex, LLVM_ATTR_IN_REG);
4204
4205 /* Add byval attributes if needed */
4206 for (i = 0; i < sig->param_count; ++i) {
4207 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
4208
4209 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
4210 mono_llvm_add_instr_attr (lcall, 1 + ainfo->pindex, LLVM_ATTR_BY_VAL);
4211 }
4212
4213 /*
4214 * Convert the result
4215 */
4216 switch (cinfo->ret.storage) {
4217 case LLVMArgVtypeInReg: {
4218 LLVMValueRef regs [2];
4219
4220 if (LLVMTypeOf (lcall) == LLVMVoidType ())
4221 /* Empty struct */
4222 break;
4223
4224 if (!addresses [ins->dreg])
4225 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
4226
4227 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
4228 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
4229 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
4230 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
4231 break;
4232 }
4233 case LLVMArgVtypeByVal:
4234 if (!addresses [call->inst.dreg])
4235 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4236 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
4237 break;
4238 case LLVMArgAsIArgs:
4239 case LLVMArgFpStruct:
4240 if (!addresses [call->inst.dreg])
4241 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4242 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4243 break;
4244 case LLVMArgVtypeAsScalar:
4245 if (!addresses [call->inst.dreg])
4246 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4247 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4248 break;
4249 case LLVMArgVtypeRetAddr:
4250 case LLVMArgVtypeByRef:
4251 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret))) {
4252 /* Some opcodes like STOREX_MEMBASE access these by value */
4253 g_assert (addresses [call->inst.dreg]);
4254 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4255 }
4256 break;
4257 case LLVMArgGsharedvtVariable:
4258 break;
4259 case LLVMArgGsharedvtFixed:
4260 case LLVMArgGsharedvtFixedVtype:
4261 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4262 break;
4263 default:
4264 if (sig->ret->type != MONO_TYPE_VOID)
4265 /* If the method returns an unsigned value, need to zext it */
4266 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
4267 break;
4268 }
4269
4270 *builder_ref = ctx->builder;
4271 }
4272
4273 static void
4274 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4275 {
4276 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception : MONO_JIT_ICALL_mono_llvm_throw_exception;
4277 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
4278
4279 LLVMTypeRef exc_type = type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_exception_class ()));
4280
4281 if (!callee) {
4282 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
4283
4284 g_assert (ctx->cfg->compile_aot);
4285 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (icall_id));
4286 }
4287
4288 LLVMValueRef args [2];
4289
4290 args [0] = convert (ctx, exc, exc_type);
4291 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
4292
4293 LLVMBuildUnreachable (ctx->builder);
4294
4295 ctx->builder = create_builder (ctx);
4296 }
4297
4298 static void
4299 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4300 {
4301 MonoMethodSignature *throw_sig;
4302
4303 LLVMValueRef * const pcallee = rethrow ? &ctx->module->rethrow : &ctx->module->throw_icall;
4304 LLVMValueRef callee = *pcallee;
4305 char const * const icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4306 #ifndef TARGET_X86
4307 const
4308 #endif
4309 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_arch_rethrow_exception : MONO_JIT_ICALL_mono_arch_throw_exception;
4310
4311 if (!callee) {
4312 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
4313 throw_sig->ret = m_class_get_byval_arg (mono_get_void_class ());
4314 throw_sig->params [0] = m_class_get_byval_arg (mono_get_object_class ());
4315 if (ctx->cfg->compile_aot) {
4316 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4317 } else {
4318 #ifdef TARGET_X86
4319 /*
4320 * LLVM doesn't push the exception argument, so we need a different
4321 * trampoline.
4322 */
4323 icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception_trampoline : MONO_JIT_ICALL_mono_llvm_throw_exception_trampoline;
4324 #endif
4325 callee = get_jit_callee (ctx, icall_name, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4326 }
4327
4328 mono_memory_barrier ();
4329 }
4330 LLVMValueRef arg;
4331 arg = convert (ctx, exc, type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_object_class ())));
4332 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
4333 }
4334
4335 static void
4336 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
4337 {
4338 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_resume_exception;
4339 LLVMValueRef callee;
4340
4341 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
4342
4343 g_assert (ctx->cfg->compile_aot);
4344 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4345
4346 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
4347
4348 LLVMBuildUnreachable (ctx->builder);
4349
4350 ctx->builder = create_builder (ctx);
4351 }
4352
4353 static LLVMValueRef
4354 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4355 {
4356 const char *icall_name = "mono_llvm_clear_exception";
4357 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_clear_exception;
4358
4359 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
4360 LLVMValueRef callee = NULL;
4361
4362 if (!callee) {
4363 if (ctx->cfg->compile_aot) {
4364 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4365 } else {
4366 // FIXME: This is broken.
4367 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4368 }
4369 }
4370
4371 g_assert (builder && callee);
4372
4373 return LLVMBuildCall (builder, callee, NULL, 0, "");
4374 }
4375
4376 static LLVMValueRef
4377 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4378 {
4379 const char *icall_name = "mono_llvm_load_exception";
4380 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_load_exception;
4381
4382 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
4383 LLVMValueRef callee = NULL;
4384
4385 if (!callee) {
4386 if (ctx->cfg->compile_aot) {
4387 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4388 } else {
4389 // FIXME: This is broken.
4390 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4391 }
4392 }
4393
4394 g_assert (builder && callee);
4395
4396 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
4397 }
4398
4399
4400 static LLVMValueRef
4401 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
4402 {
4403 const char *icall_name = "mono_llvm_match_exception";
4404 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_match_exception;
4405
4406 ctx->builder = builder;
4407
4408 LLVMValueRef args[5];
4409 const int num_args = G_N_ELEMENTS (args);
4410
4411 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index), LLVMPointerType (IntPtrType (), 0)), IntPtrType ());
4412 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
4413 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
4414 if (ctx->cfg->rgctx_var) {
4415 if (ctx->cfg->llvm_only) {
4416 args [3] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
4417 } else {
4418 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
4419 g_assert (rgctx_alloc);
4420 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
4421 }
4422 } else {
4423 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
4424 }
4425 if (ctx->this_arg)
4426 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
4427 else
4428 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
4429
4430 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
4431 LLVMValueRef callee;
4432 g_assert (ctx->cfg->compile_aot);
4433 ctx->builder = builder;
4434 // get_callee expects ctx->builder to be the emitting builder
4435 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4436
4437 g_assert (builder && callee);
4438
4439 g_assert (ctx->ex_var);
4440
4441 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
4442 }
4443
4444 // FIXME: This won't work because the code-finding makes this
4445 // not a constant.
4446 /*#define MONO_PERSONALITY_DEBUG*/
4447
4448 #ifdef MONO_PERSONALITY_DEBUG
4449 static const gboolean use_mono_personality_debug = TRUE;
4450 static const char *default_personality_name = "mono_debug_personality";
4451 #else
4452 static const gboolean use_mono_personality_debug = FALSE;
4453 static const char *default_personality_name = "__gxx_personality_v0";
4454 #endif
4455
4456 static LLVMTypeRef
4457 default_cpp_lpad_exc_signature (void)
4458 {
4459 static gboolean inited = FALSE;
4460 static LLVMTypeRef sig;
4461
4462 if (!sig) {
4463 LLVMTypeRef signature [2];
4464 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
4465 signature [1] = LLVMInt32Type ();
4466 sig = LLVMStructType (signature, 2, FALSE);
4467 inited = TRUE;
4468 }
4469
4470 return sig;
4471 }
4472
4473 static LLVMValueRef
4474 get_mono_personality (EmitContext *ctx)
4475 {
4476 LLVMValueRef personality = NULL;
4477 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4478
4479 g_assert (ctx->cfg->compile_aot);
4480 if (!use_mono_personality_debug) {
4481 personality = LLVMGetNamedFunction (ctx->lmodule, default_personality_name);
4482 } else {
4483 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_debug_personality));
4484 }
4485
4486 g_assert (personality);
4487 return personality;
4488 }
4489
4490 static LLVMBasicBlockRef
4491 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
4492 {
4493 MonoCompile *cfg = ctx->cfg;
4494 LLVMBuilderRef old_builder = ctx->builder;
4495 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
4496
4497 LLVMBuilderRef lpadBuilder = create_builder (ctx);
4498 ctx->builder = lpadBuilder;
4499
4500 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
4501 g_assert (handler_bb);
4502
4503 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4504 LLVMValueRef personality = get_mono_personality (ctx);
4505 g_assert (personality);
4506
4507 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
4508 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
4509 g_free (bb_name);
4510 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4511 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
4512 g_assert (landing_pad);
4513
4514 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
4515 LLVMAddClause (landing_pad, cast);
4516
4517 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
4518 LLVMBuilderRef resume_builder = create_builder (ctx);
4519 ctx->builder = resume_builder;
4520 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
4521
4522 emit_resume_eh (ctx, handler_bb);
4523
4524 // Build match
4525 ctx->builder = lpadBuilder;
4526 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4527
4528 gboolean finally_only = TRUE;
4529
4530 MonoExceptionClause *group_cursor = group_start;
4531
4532 for (int i = 0; i < group_size; i ++) {
4533 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY || group_cursor->flags & MONO_EXCEPTION_CLAUSE_FAULT))
4534 finally_only = FALSE;
4535
4536 group_cursor++;
4537 }
4538
4539 // FIXME:
4540 // Handle landing pad inlining
4541
4542 if (!finally_only) {
4543 // So at each level of the exception stack we will match the exception again.
4544 // During that match, we need to compare against the handler types for the current
4545 // protected region. We send the try start and end so that we can only check against
4546 // handlers for this lexical protected region.
4547 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
4548
4549 // if returns -1, resume
4550 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
4551
4552 // else move to that target bb
4553 for (int i = 0; i < group_size; i++) {
4554 MonoExceptionClause *clause = group_start + i;
4555 int clause_index = clause - cfg->header->clauses;
4556 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4557 g_assert (handler_bb);
4558 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4559 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4560 }
4561 } else {
4562 int clause_index = group_start - cfg->header->clauses;
4563 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4564 g_assert (finally_bb);
4565
4566 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
4567 }
4568
4569 ctx->builder = old_builder;
4570
4571 return lpad_bb;
4572 }
4573
4574 static LLVMValueRef
4575 create_const_vector_i32 (const int *mask, int count)
4576 {
4577 LLVMValueRef *llvm_mask = g_new (LLVMValueRef, count);
4578 for (int i = 0; i < count; i++)
4579 llvm_mask [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
4580 LLVMValueRef vec = LLVMConstVector (llvm_mask, count);
4581 g_free (llvm_mask);
4582 return vec;
4583 }
4584
4585 static LLVMValueRef
4586 create_const_vector_4_i32 (int v0, int v1, int v2, int v3)
4587 {
4588 LLVMValueRef mask [4];
4589 mask [0] = LLVMConstInt (LLVMInt32Type (), v0, FALSE);
4590 mask [1] = LLVMConstInt (LLVMInt32Type (), v1, FALSE);
4591 mask [2] = LLVMConstInt (LLVMInt32Type (), v2, FALSE);
4592 mask [3] = LLVMConstInt (LLVMInt32Type (), v3, FALSE);
4593 return LLVMConstVector (mask, 4);
4594 }
4595
4596 static LLVMValueRef
4597 create_const_vector_2_i32 (int v0, int v1)
4598 {
4599 LLVMValueRef mask [2];
4600 mask [0] = LLVMConstInt (LLVMInt32Type (), v0, FALSE);
4601 mask [1] = LLVMConstInt (LLVMInt32Type (), v1, FALSE);
4602 return LLVMConstVector (mask, 2);
4603 }
4604
4605 static void
4606 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
4607 {
4608 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
4609 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
4610
4611 // Make exception available to catch blocks
4612 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags & MONO_EXCEPTION_CLAUSE_FAULT)) {
4613 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
4614
4615 g_assert (ctx->ex_var);
4616 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
4617
4618 if (bb->in_scount == 1) {
4619 MonoInst *exvar = bb->in_stack [0];
4620 g_assert (!ctx->values [exvar->dreg]);
4621 g_assert (ctx->ex_var);
4622 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
4623 emit_volatile_store (ctx, exvar->dreg);
4624 }
4625
4626 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
4627 }
4628
4629 LLVMBuilderRef handler_builder = create_builder (ctx);
4630 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
4631 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
4632
4633 // Make the handler code end with a jump to cbb
4634 LLVMBuildBr (handler_builder, cbb);
4635 }
4636
4637 static void
4638 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
4639 {
4640 MonoCompile *cfg = ctx->cfg;
4641 LLVMValueRef *values = ctx->values;
4642 LLVMModuleRef lmodule = ctx->lmodule;
4643 BBInfo *bblocks = ctx->bblocks;
4644 LLVMTypeRef i8ptr;
4645 LLVMValueRef personality;
4646 LLVMValueRef landing_pad;
4647 LLVMBasicBlockRef target_bb;
4648 MonoInst *exvar;
4649 static int ti_generator;
4650 char ti_name [128];
4651 LLVMValueRef type_info;
4652 int clause_index;
4653 GSList *l;
4654
4655 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4656
4657 if (cfg->compile_aot) {
4658 /* Use a dummy personality function */
4659 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
4660 g_assert (personality);
4661 } else {
4662 /* Can't cache this as each method is in its own llvm module */
4663 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4664 personality = LLVMAddFunction (ctx->lmodule, "mono_personality", personality_type);
4665 mono_llvm_add_func_attr (personality, LLVM_ATTR_NO_UNWIND);
4666 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (personality, "ENTRY");
4667 LLVMBuilderRef builder2 = LLVMCreateBuilder ();
4668 LLVMPositionBuilderAtEnd (builder2, entry_bb);
4669 LLVMBuildRet (builder2, LLVMConstInt (LLVMInt32Type (), 0, FALSE));
4670 LLVMDisposeBuilder (builder2);
4671 }
4672
4673 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
4674
4675 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
4676
4677 /*
4678 * Create the type info
4679 */
4680 sprintf (ti_name, "type_info_%d", ti_generator);
4681 ti_generator ++;
4682
4683 if (cfg->compile_aot) {
4684 /* decode_eh_frame () in aot-runtime.c will decode this */
4685 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4686 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4687
4688 /*
4689 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4690 */
4691 LLVMSetLinkage (type_info, LLVMInternalLinkage);
4692 } else {
4693 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4694 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4695 }
4696
4697 {
4698 LLVMTypeRef members [2], ret_type;
4699
4700 members [0] = i8ptr;
4701 members [1] = LLVMInt32Type ();
4702 ret_type = LLVMStructType (members, 2, FALSE);
4703
4704 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
4705 LLVMAddClause (landing_pad, type_info);
4706
4707 /* Store the exception into the exvar */
4708 if (ctx->ex_var)
4709 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
4710 }
4711
4712 /*
4713 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4714 * code expects control to be transferred to this landing pad even in the
4715 * presence of nested clauses. The landing pad needs to branch to the landing
4716 * pads belonging to nested clauses based on the selector value returned by
4717 * the landing pad instruction, which is passed to the landing pad in a
4718 * register by the EH code.
4719 */
4720 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4721 g_assert (target_bb);
4722
4723 /*
4724 * Branch to the correct landing pad
4725 */
4726 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
4727 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
4728
4729 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
4730 int nesting_clause_index = GPOINTER_TO_INT (l->data);
4731 MonoBasicBlock *handler_bb;
4732
4733 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
4734 g_assert (handler_bb);
4735
4736 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4737 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4738 }
4739
4740 /* Start a new bblock which CALL_HANDLER can branch to */
4741 ctx->builder = builder = create_builder (ctx);
4742 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
4743
4744 ctx->bblocks [bb->block_num].end_bblock = target_bb;
4745
4746 /* Store the exception into the IL level exvar */
4747 if (bb->in_scount == 1) {
4748 g_assert (bb->in_scount == 1);
4749 exvar = bb->in_stack [0];
4750
4751 // FIXME: This is shared with filter clauses ?
4752 g_assert (!values [exvar->dreg]);
4753
4754 g_assert (ctx->ex_var);
4755 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
4756 emit_volatile_store (ctx, exvar->dreg);
4757 }
4758
4759 /* Make normal branches to the start of the clause branch to the new bblock */
4760 bblocks [bb->block_num].bblock = target_bb;
4761 }
4762
4763 //Wasm requires us to canonicalize NaNs.
4764 static LLVMValueRef
4765 get_double_const (MonoCompile *cfg, double val)
4766 {
4767 #ifdef TARGET_WASM
4768 if (mono_isnan (val))
4769 *(gint64 *)&val = 0x7FF8000000000000ll;
4770 #endif
4771 return LLVMConstReal (LLVMDoubleType (), val);
4772 }
4773
4774 static LLVMValueRef
4775 get_float_const (MonoCompile *cfg, float val)
4776 {
4777 #ifdef TARGET_WASM
4778 if (mono_isnan (val))
4779 *(int *)&val = 0x7FC00000;
4780 #endif
4781 if (cfg->r4fp)
4782 return LLVMConstReal (LLVMFloatType (), val);
4783 else
4784 return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val), LLVMDoubleType ());
4785 }
4786
4787 static LLVMValueRef
4788 call_intrins (EmitContext *ctx, int id, LLVMValueRef *args, const char *name)
4789 {
4790 LLVMValueRef intrins = get_intrins (ctx, id);
4791 int nargs = LLVMCountParamTypes (LLVMGetElementType (LLVMTypeOf (intrins)));
4792 return LLVMBuildCall (ctx->builder, intrins, args, nargs, name);
4793 }
4794
4795 static void
4796 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4797 {
4798 MonoCompile *cfg = ctx->cfg;
4799 MonoMethodSignature *sig = ctx->sig;
4800 LLVMValueRef method = ctx->lmethod;
4801 LLVMValueRef *values = ctx->values;
4802 LLVMValueRef *addresses = ctx->addresses;
4803 LLVMCallInfo *linfo = ctx->linfo;
4804 BBInfo *bblocks = ctx->bblocks;
4805 MonoInst *ins;
4806 LLVMBasicBlockRef cbb;
4807 LLVMBuilderRef builder, starting_builder;
4808 gboolean has_terminator;
4809 LLVMValueRef v;
4810 LLVMValueRef lhs, rhs;
4811 int nins = 0;
4812
4813 cbb = get_end_bb (ctx, bb);
4814
4815 builder = create_builder (ctx);
4816 ctx->builder = builder;
4817 LLVMPositionBuilderAtEnd (builder, cbb);
4818
4819 if (!ctx_ok (ctx))
4820 return;
4821
4822 if (bb->flags & BB_EXCEPTION_HANDLER) {
4823 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4824 set_failure (ctx, "handler without invokes");
4825 return;
4826 }
4827
4828 if (ctx->llvm_only)
4829 emit_llvmonly_handler_start (ctx, bb, cbb);
4830 else
4831 emit_handler_start (ctx, bb, builder);
4832 if (!ctx_ok (ctx))
4833 return;
4834 builder = ctx->builder;
4835 }
4836
4837 /* Handle PHI nodes first */
4838 /* They should be grouped at the start of the bb */
4839 for (ins = bb->code; ins; ins = ins->next) {
4840 emit_dbg_loc (ctx, builder, ins->cil_code);
4841
4842 if (ins->opcode == OP_NOP)
4843 continue;
4844 if (!MONO_IS_PHI (ins))
4845 break;
4846
4847 int i;
4848 gboolean empty = TRUE;
4849
4850 /* Check that all input bblocks really branch to us */
4851 for (i = 0; i < bb->in_count; ++i) {
4852 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4853 ins->inst_phi_args [i + 1] = -1;
4854 else
4855 empty = FALSE;
4856 }
4857
4858 if (empty) {
4859 /* LLVM doesn't like phi instructions with zero operands */
4860 ctx->is_dead [ins->dreg] = TRUE;
4861 continue;
4862 }
4863
4864 /* Created earlier, insert it now */
4865 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4866
4867 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4868 int sreg1 = ins->inst_phi_args [i + 1];
4869 int count, j;
4870
4871 /*
4872 * Count the number of times the incoming bblock branches to us,
4873 * since llvm requires a separate entry for each.
4874 */
4875 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4876 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4877
4878 count = 0;
4879 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4880 if (switch_ins->inst_many_bb [j] == bb)
4881 count ++;
4882 }
4883 } else {
4884 count = 1;
4885 }
4886
4887 /* Remember for later */
4888 for (j = 0; j < count; ++j) {
4889 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4890 node->bb = bb;
4891 node->phi = ins;
4892 node->in_bb = bb->in_bb [i];
4893 node->sreg = sreg1;
4894 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4895 }
4896 }
4897 }
4898 // Add volatile stores for PHI nodes
4899 // These need to be emitted after the PHI nodes
4900 for (ins = bb->code; ins; ins = ins->next) {
4901 const char *spec = LLVM_INS_INFO (ins->opcode);
4902
4903 if (ins->opcode == OP_NOP)
4904 continue;
4905 if (!MONO_IS_PHI (ins))
4906 break;
4907
4908 if (spec [MONO_INST_DEST] != 'v')
4909 emit_volatile_store (ctx, ins->dreg);
4910 }
4911
4912 has_terminator = FALSE;
4913 starting_builder = builder;
4914 for (ins = bb->code; ins; ins = ins->next) {
4915 const char *spec = LLVM_INS_INFO (ins->opcode);
4916 char *dname = NULL;
4917 char dname_buf [128];
4918
4919 emit_dbg_loc (ctx, builder, ins->cil_code);
4920
4921 nins ++;
4922 if (nins > 1000) {
4923 /*
4924 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4925 * Start a new bblock.
4926 * Prevent the bblocks to be merged by doing a volatile load + cond branch
4927 * from localloc-ed memory.
4928 */
4929 if (!cfg->llvm_only)
4930 ;//set_failure (ctx, "basic block too long");
4931
4932 if (!ctx->long_bb_break_var) {
4933 ctx->long_bb_break_var = build_alloca_llvm_type_name (ctx, LLVMInt32Type (), 0, "long_bb_break");
4934 mono_llvm_build_store (ctx->alloca_builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4935 }
4936
4937 cbb = gen_bb (ctx, "CONT_LONG_BB");
4938 LLVMBasicBlockRef dummy_bb = gen_bb (ctx, "CONT_LONG_BB_DUMMY");
4939
4940 LLVMValueRef load = mono_llvm_build_load (builder, ctx->long_bb_break_var, "", TRUE);
4941 /*
4942 * The long_bb_break_var is initialized to 0 in the prolog, so this branch will always go to 'cbb'
4943 * but llvm doesn't know that, so the branch is not going to be eliminated.
4944 */
4945 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, load, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4946
4947 LLVMBuildCondBr (builder, cmp, cbb, dummy_bb);
4948
4949 /* Emit a dummy false bblock which does nothing but contains a volatile store so it cannot be eliminated */
4950 ctx->builder = builder = create_builder (ctx);
4951 LLVMPositionBuilderAtEnd (builder, dummy_bb);
4952 mono_llvm_build_store (builder, LLVMConstInt (LLVMInt32Type (), 1, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4953 LLVMBuildBr (builder, cbb);
4954
4955 ctx->builder = builder = create_builder (ctx);
4956 LLVMPositionBuilderAtEnd (builder, cbb);
4957 ctx->bblocks [bb->block_num].end_bblock = cbb;
4958 nins = 0;
4959
4960 emit_dbg_loc (ctx, builder, ins->cil_code);
4961 }
4962
4963 if (has_terminator)
4964 /* There could be instructions after a terminator, skip them */
4965 break;
4966
4967 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4968 sprintf (dname_buf, "t%d", ins->dreg);
4969 dname = dname_buf;
4970 }
4971
4972 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4973 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4974
4975 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4976 lhs = emit_volatile_load (ctx, ins->sreg1);
4977 } else {
4978 /* It is ok for SETRET to have an uninitialized argument */
4979 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4980 set_failure (ctx, "sreg1");
4981 return;
4982 }
4983 lhs = values [ins->sreg1];
4984 }
4985 } else {
4986 lhs = NULL;
4987 }
4988
4989 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4990 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4991 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4992 rhs = emit_volatile_load (ctx, ins->sreg2);
4993 } else {
4994 if (!values [ins->sreg2]) {
4995 set_failure (ctx, "sreg2");
4996 return;
4997 }
4998 rhs = values [ins->sreg2];
4999 }
5000 } else {
5001 rhs = NULL;
5002 }
5003
5004 //mono_print_ins (ins);
5005 gboolean skip_volatile_store = FALSE;
5006 switch (ins->opcode) {
5007 case OP_NOP:
5008 case OP_NOT_NULL:
5009 case OP_LIVERANGE_START:
5010 case OP_LIVERANGE_END:
5011 break;
5012 case OP_ICONST:
5013 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
5014 break;
5015 case OP_I8CONST:
5016 #if TARGET_SIZEOF_VOID_P == 4
5017 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5018 #else
5019 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
5020 #endif
5021 break;
5022 case OP_R8CONST:
5023 values [ins->dreg] = get_double_const (cfg, *(double*)ins->inst_p0);
5024 break;
5025 case OP_R4CONST:
5026 values [ins->dreg] = get_float_const (cfg, *(float*)ins->inst_p0);
5027 break;
5028 case OP_DUMMY_ICONST:
5029 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5030 break;
5031 case OP_DUMMY_I8CONST:
5032 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
5033 break;
5034 case OP_DUMMY_R8CONST:
5035 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
5036 break;
5037 case OP_BR: {
5038 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
5039 LLVMBuildBr (builder, target_bb);
5040 has_terminator = TRUE;
5041 break;
5042 }
5043 case OP_SWITCH: {
5044 int i;
5045 LLVMValueRef v;
5046 char bb_name [128];
5047 LLVMBasicBlockRef new_bb;
5048 LLVMBuilderRef new_builder;
5049
5050 // The default branch is already handled
5051 // FIXME: Handle it here
5052
5053 /* Start new bblock */
5054 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
5055 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
5056
5057 lhs = convert (ctx, lhs, LLVMInt32Type ());
5058 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
5059 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
5060 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
5061
5062 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
5063 }
5064
5065 new_builder = create_builder (ctx);
5066 LLVMPositionBuilderAtEnd (new_builder, new_bb);
5067 LLVMBuildUnreachable (new_builder);
5068
5069 has_terminator = TRUE;
5070 g_assert (!ins->next);
5071
5072 break;
5073 }
5074
5075 case OP_SETRET:
5076 switch (linfo->ret.storage) {
5077 case LLVMArgVtypeInReg: {
5078 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5079 LLVMValueRef val, addr, retval;
5080 int i;
5081
5082 retval = LLVMGetUndef (ret_type);
5083
5084 if (!addresses [ins->sreg1]) {
5085 /*
5086 * The return type is an LLVM vector type, have to convert between it and the
5087 * real return type which is a struct type.
5088 */
5089 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret)));
5090 /* Convert to 2xi64 first */
5091 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
5092
5093 for (i = 0; i < 2; ++i) {
5094 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
5095 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
5096 } else {
5097 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
5098 }
5099 }
5100 } else {
5101 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
5102 for (i = 0; i < 2; ++i) {
5103 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
5104 LLVMValueRef indexes [2], part_addr;
5105
5106 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5107 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
5108 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
5109
5110 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
5111 } else {
5112 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
5113 }
5114 }
5115 }
5116 LLVMBuildRet (builder, retval);
5117 break;
5118 }
5119 case LLVMArgVtypeAsScalar: {
5120 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5121 LLVMValueRef retval;
5122
5123 g_assert (addresses [ins->sreg1]);
5124
5125 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
5126 LLVMBuildRet (builder, retval);
5127 break;
5128 }
5129 case LLVMArgVtypeByVal: {
5130 LLVMValueRef retval;
5131
5132 g_assert (addresses [ins->sreg1]);
5133 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
5134 LLVMBuildRet (builder, retval);
5135 break;
5136 }
5137 case LLVMArgVtypeByRef: {
5138 LLVMBuildRetVoid (builder);
5139 break;
5140 }
5141 case LLVMArgGsharedvtFixed: {
5142 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
5143 /* The return value is in lhs, need to store to the vret argument */
5144 /* sreg1 might not be set */
5145 if (lhs) {
5146 g_assert (cfg->vret_addr);
5147 g_assert (values [cfg->vret_addr->dreg]);
5148 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
5149 }
5150 LLVMBuildRetVoid (builder);
5151 break;
5152 }
5153 case LLVMArgGsharedvtFixedVtype: {
5154 /* Already set */
5155 LLVMBuildRetVoid (builder);
5156 break;
5157 }
5158 case LLVMArgGsharedvtVariable: {
5159 /* Already set */
5160 LLVMBuildRetVoid (builder);
5161 break;
5162 }
5163 case LLVMArgVtypeRetAddr: {
5164 LLVMBuildRetVoid (builder);
5165 break;
5166 }
5167 case LLVMArgAsIArgs:
5168 case LLVMArgFpStruct: {
5169 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5170 LLVMValueRef retval;
5171
5172 g_assert (addresses [ins->sreg1]);
5173 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
5174 LLVMBuildRet (builder, retval);
5175 break;
5176 }
5177 case LLVMArgNone:
5178 case LLVMArgNormal: {
5179 if (!lhs || ctx->is_dead [ins->sreg1]) {
5180 /*
5181 * The method did not set its return value, probably because it
5182 * ends with a throw.
5183 */
5184 if (cfg->vret_addr)
5185 LLVMBuildRetVoid (builder);
5186 else
5187 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
5188 } else {
5189 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
5190 }
5191 has_terminator = TRUE;
5192 break;
5193 }
5194 default:
5195 g_assert_not_reached ();
5196 break;
5197 }
5198 break;
5199 case OP_ICOMPARE:
5200 case OP_FCOMPARE:
5201 case OP_RCOMPARE:
5202 case OP_LCOMPARE:
5203 case OP_COMPARE:
5204 case OP_ICOMPARE_IMM:
5205 case OP_LCOMPARE_IMM:
5206 case OP_COMPARE_IMM: {
5207 CompRelation rel;
5208 LLVMValueRef cmp, args [16];
5209 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
5210 gboolean unlikely = FALSE;
5211
5212 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5213 if (ins->next->inst_false_bb->out_of_line)
5214 likely = TRUE;
5215 else if (ins->next->inst_true_bb->out_of_line)
5216 unlikely = TRUE;
5217 }
5218
5219 if (ins->next->opcode == OP_NOP)
5220 break;
5221
5222 if (ins->next->opcode == OP_BR)
5223 /* The comparison result is not needed */
5224 continue;
5225
5226 rel = mono_opcode_to_cond (ins->next->opcode);
5227
5228 if (ins->opcode == OP_ICOMPARE_IMM) {
5229 lhs = convert (ctx, lhs, LLVMInt32Type ());
5230 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5231 }
5232 if (ins->opcode == OP_LCOMPARE_IMM) {
5233 lhs = convert (ctx, lhs, LLVMInt64Type ());
5234 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5235 }
5236 if (ins->opcode == OP_LCOMPARE) {
5237 lhs = convert (ctx, lhs, LLVMInt64Type ());
5238 rhs = convert (ctx, rhs, LLVMInt64Type ());
5239 }
5240 if (ins->opcode == OP_ICOMPARE) {
5241 lhs = convert (ctx, lhs, LLVMInt32Type ());
5242 rhs = convert (ctx, rhs, LLVMInt32Type ());
5243 }
5244
5245 if (lhs && rhs) {
5246 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5247 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
5248 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
5249 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
5250 }
5251
5252 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
5253 if (ins->opcode == OP_FCOMPARE) {
5254 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5255 } else if (ins->opcode == OP_RCOMPARE) {
5256 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5257 } else if (ins->opcode == OP_COMPARE_IMM) {
5258 LLVMIntPredicate llvm_pred = cond_to_llvm_cond [rel];
5259 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0) {
5260 // We are emitting a NULL check for a pointer
5261 gboolean nonnull = mono_llvm_is_nonnull (lhs);
5262
5263 if (nonnull && llvm_pred == LLVMIntEQ)
5264 cmp = LLVMConstInt (LLVMInt1Type (), FALSE, FALSE);
5265 else if (nonnull && llvm_pred == LLVMIntNE)
5266 cmp = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
5267 else
5268 cmp = LLVMBuildICmp (builder, llvm_pred, lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
5269
5270 } else {
5271 cmp = LLVMBuildICmp (builder, llvm_pred, convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
5272 }
5273 } else if (ins->opcode == OP_LCOMPARE_IMM) {
5274 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5275 }
5276 else if (ins->opcode == OP_COMPARE) {
5277 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
5278 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5279 else
5280 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
5281 } else
5282 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5283
5284 if (likely || unlikely) {
5285 args [0] = cmp;
5286 args [1] = LLVMConstInt (LLVMInt1Type (), likely ? 1 : 0, FALSE);
5287 cmp = call_intrins (ctx, INTRINS_EXPECT_I1, args, "");
5288 }
5289
5290 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5291 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
5292 /*
5293 * If the target bb contains PHI instructions, LLVM requires
5294 * two PHI entries for this bblock, while we only generate one.
5295 * So convert this to an unconditional bblock. (bxc #171).
5296 */
5297 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
5298 } else {
5299 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
5300 }
5301 has_terminator = TRUE;
5302 } else if (MONO_IS_SETCC (ins->next)) {
5303 sprintf (dname_buf, "t%d", ins->next->dreg);
5304 dname = dname_buf;
5305 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5306
5307 /* Add stores for volatile variables */
5308 emit_volatile_store (ctx, ins->next->dreg);
5309 } else if (MONO_IS_COND_EXC (ins->next)) {
5310 gboolean force_explicit_branch = FALSE;
5311 if (bb->region != -1) {
5312 /* Don't tag null check branches in exception-handling
5313 * regions with `make.implicit`.
5314 */
5315 force_explicit_branch = TRUE;
5316 }
5317 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp, force_explicit_branch);
5318 if (!ctx_ok (ctx))
5319 break;
5320 builder = ctx->builder;
5321 } else {
5322 set_failure (ctx, "next");
5323 break;
5324 }
5325
5326 ins = ins->next;
5327 break;
5328 }
5329 case OP_FCEQ:
5330 case OP_FCNEQ:
5331 case OP_FCLT:
5332 case OP_FCLT_UN:
5333 case OP_FCGT:
5334 case OP_FCGT_UN:
5335 case OP_FCGE:
5336 case OP_FCLE: {
5337 CompRelation rel;
5338 LLVMValueRef cmp;
5339
5340 rel = mono_opcode_to_cond (ins->opcode);
5341
5342 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5343 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5344 break;
5345 }
5346 case OP_RCEQ:
5347 case OP_RCNEQ:
5348 case OP_RCLT:
5349 case OP_RCLT_UN:
5350 case OP_RCGT:
5351 case OP_RCGT_UN: {
5352 CompRelation rel;
5353 LLVMValueRef cmp;
5354
5355 rel = mono_opcode_to_cond (ins->opcode);
5356
5357 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5358 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5359 break;
5360 }
5361 case OP_PHI:
5362 case OP_FPHI:
5363 case OP_VPHI:
5364 case OP_XPHI: {
5365 // Handled above
5366 skip_volatile_store = TRUE;
5367 break;
5368 }
5369 case OP_MOVE:
5370 case OP_LMOVE:
5371 case OP_XMOVE:
5372 case OP_SETFRET:
5373 g_assert (lhs);
5374 values [ins->dreg] = lhs;
5375 break;
5376 case OP_FMOVE:
5377 case OP_RMOVE: {
5378 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
5379
5380 g_assert (lhs);
5381 values [ins->dreg] = lhs;
5382
5383 if (var && m_class_get_byval_arg (var->klass)->type == MONO_TYPE_R4) {
5384 /*
5385 * This is added by the spilling pass in case of the JIT,
5386 * but we have to do it ourselves.
5387 */
5388 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
5389 }
5390 break;
5391 }
5392 case OP_MOVE_F_TO_I4: {
5393 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
5394 break;
5395 }
5396 case OP_MOVE_I4_TO_F: {
5397 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
5398 break;
5399 }
5400 case OP_MOVE_F_TO_I8: {
5401 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
5402 break;
5403 }
5404 case OP_MOVE_I8_TO_F: {
5405 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
5406 break;
5407 }
5408 case OP_IADD:
5409 case OP_ISUB:
5410 case OP_IAND:
5411 case OP_IMUL:
5412 case OP_IDIV:
5413 case OP_IDIV_UN:
5414 case OP_IREM:
5415 case OP_IREM_UN:
5416 case OP_IOR:
5417 case OP_IXOR:
5418 case OP_ISHL:
5419 case OP_ISHR:
5420 case OP_ISHR_UN:
5421 case OP_FADD:
5422 case OP_FSUB:
5423 case OP_FMUL:
5424 case OP_FDIV:
5425 case OP_LADD:
5426 case OP_LSUB:
5427 case OP_LMUL:
5428 case OP_LDIV:
5429 case OP_LDIV_UN:
5430 case OP_LREM:
5431 case OP_LREM_UN:
5432 case OP_LAND:
5433 case OP_LOR:
5434 case OP_LXOR:
5435 case OP_LSHL:
5436 case OP_LSHR:
5437 case OP_LSHR_UN:
5438 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5439 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5440
5441 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
5442 if (!ctx_ok (ctx))
5443 break;
5444 builder = ctx->builder;
5445
5446 switch (ins->opcode) {
5447 case OP_IADD:
5448 case OP_LADD:
5449 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
5450 break;
5451 case OP_ISUB:
5452 case OP_LSUB:
5453 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
5454 break;
5455 case OP_IMUL:
5456 case OP_LMUL:
5457 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
5458 break;
5459 case OP_IREM:
5460 case OP_LREM:
5461 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
5462 break;
5463 case OP_IREM_UN:
5464 case OP_LREM_UN:
5465 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
5466 break;
5467 case OP_IDIV:
5468 case OP_LDIV:
5469 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
5470 break;
5471 case OP_IDIV_UN:
5472 case OP_LDIV_UN:
5473 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
5474 break;
5475 case OP_FDIV:
5476 case OP_RDIV:
5477 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5478 break;
5479 case OP_IAND:
5480 case OP_LAND:
5481 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
5482 break;
5483 case OP_IOR:
5484 case OP_LOR:
5485 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
5486 break;
5487 case OP_IXOR:
5488 case OP_LXOR:
5489 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
5490 break;
5491 case OP_ISHL:
5492 case OP_LSHL:
5493 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
5494 break;
5495 case OP_ISHR:
5496 case OP_LSHR:
5497 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
5498 break;
5499 case OP_ISHR_UN:
5500 case OP_LSHR_UN:
5501 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
5502 break;
5503
5504 case OP_FADD:
5505 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5506 break;
5507 case OP_FSUB:
5508 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5509 break;
5510 case OP_FMUL:
5511 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5512 break;
5513
5514 default:
5515 g_assert_not_reached ();
5516 }
5517 break;
5518 case OP_RADD:
5519 case OP_RSUB:
5520 case OP_RMUL:
5521 case OP_RDIV: {
5522 lhs = convert (ctx, lhs, LLVMFloatType ());
5523 rhs = convert (ctx, rhs, LLVMFloatType ());
5524 switch (ins->opcode) {
5525 case OP_RADD:
5526 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5527 break;
5528 case OP_RSUB:
5529 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5530 break;
5531 case OP_RMUL:
5532 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5533 break;
5534 case OP_RDIV:
5535 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5536 break;
5537 default:
5538 g_assert_not_reached ();
5539 break;
5540 }
5541 break;
5542 }
5543 case OP_IADD_IMM:
5544 case OP_ISUB_IMM:
5545 case OP_IMUL_IMM:
5546 case OP_IREM_IMM:
5547 case OP_IREM_UN_IMM:
5548 case OP_IDIV_IMM:
5549 case OP_IDIV_UN_IMM:
5550 case OP_IAND_IMM:
5551 case OP_IOR_IMM:
5552 case OP_IXOR_IMM:
5553 case OP_ISHL_IMM:
5554 case OP_ISHR_IMM:
5555 case OP_ISHR_UN_IMM:
5556 case OP_LADD_IMM:
5557 case OP_LSUB_IMM:
5558 case OP_LMUL_IMM:
5559 case OP_LREM_IMM:
5560 case OP_LAND_IMM:
5561 case OP_LOR_IMM:
5562 case OP_LXOR_IMM:
5563 case OP_LSHL_IMM:
5564 case OP_LSHR_IMM:
5565 case OP_LSHR_UN_IMM:
5566 case OP_ADD_IMM:
5567 case OP_AND_IMM:
5568 case OP_MUL_IMM:
5569 case OP_SHL_IMM:
5570 case OP_SHR_IMM:
5571 case OP_SHR_UN_IMM: {
5572 LLVMValueRef imm;
5573
5574 if (spec [MONO_INST_SRC1] == 'l') {
5575 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5576 } else {
5577 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5578 }
5579
5580 emit_div_check (ctx, builder, bb, ins, lhs, imm);
5581 if (!ctx_ok (ctx))
5582 break;
5583 builder = ctx->builder;
5584
5585 #if TARGET_SIZEOF_VOID_P == 4
5586 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
5587 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5588 #endif
5589
5590 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5591 lhs = convert (ctx, lhs, IntPtrType ());
5592 imm = convert (ctx, imm, LLVMTypeOf (lhs));
5593 switch (ins->opcode) {
5594 case OP_IADD_IMM:
5595 case OP_LADD_IMM:
5596 case OP_ADD_IMM:
5597 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
5598 break;
5599 case OP_ISUB_IMM:
5600 case OP_LSUB_IMM:
5601 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
5602 break;
5603 case OP_IMUL_IMM:
5604 case OP_MUL_IMM:
5605 case OP_LMUL_IMM:
5606 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
5607 break;
5608 case OP_IDIV_IMM:
5609 case OP_LDIV_IMM:
5610 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
5611 break;
5612 case OP_IDIV_UN_IMM:
5613 case OP_LDIV_UN_IMM:
5614 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
5615 break;
5616 case OP_IREM_IMM:
5617 case OP_LREM_IMM:
5618 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
5619 break;
5620 case OP_IREM_UN_IMM:
5621 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
5622 break;
5623 case OP_IAND_IMM:
5624 case OP_LAND_IMM:
5625 case OP_AND_IMM:
5626 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
5627 break;
5628 case OP_IOR_IMM:
5629 case OP_LOR_IMM:
5630 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
5631 break;
5632 case OP_IXOR_IMM:
5633 case OP_LXOR_IMM:
5634 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
5635 break;
5636 case OP_ISHL_IMM:
5637 case OP_LSHL_IMM:
5638 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5639 break;
5640 case OP_SHL_IMM:
5641 if (TARGET_SIZEOF_VOID_P == 8) {
5642 /* The IL is not regular */
5643 lhs = convert (ctx, lhs, LLVMInt64Type ());
5644 imm = convert (ctx, imm, LLVMInt64Type ());
5645 }
5646 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5647 break;
5648 case OP_ISHR_IMM:
5649 case OP_LSHR_IMM:
5650 case OP_SHR_IMM:
5651 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
5652 break;
5653 case OP_ISHR_UN_IMM:
5654 /* This is used to implement conv.u4, so the lhs could be an i8 */
5655 lhs = convert (ctx, lhs, LLVMInt32Type ());
5656 imm = convert (ctx, imm, LLVMInt32Type ());
5657 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5658 break;
5659 case OP_LSHR_UN_IMM:
5660 case OP_SHR_UN_IMM:
5661 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5662 break;
5663 default:
5664 g_assert_not_reached ();
5665 }
5666 break;
5667 }
5668 case OP_INEG:
5669 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5670 break;
5671 case OP_LNEG:
5672 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5673 lhs = convert (ctx, lhs, LLVMInt64Type ());
5674 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
5675 break;
5676 case OP_FNEG:
5677 lhs = convert (ctx, lhs, LLVMDoubleType ());
5678 values [ins->dreg] = LLVMBuildFNeg (builder, lhs, dname);
5679 break;
5680 case OP_RNEG:
5681 lhs = convert (ctx, lhs, LLVMFloatType ());
5682 values [ins->dreg] = LLVMBuildFNeg (builder, lhs, dname);
5683 break;
5684 case OP_INOT: {
5685 guint32 v = 0xffffffff;
5686 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5687 break;
5688 }
5689 case OP_LNOT: {
5690 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5691 lhs = convert (ctx, lhs, LLVMInt64Type ());
5692 guint64 v = 0xffffffffffffffffLL;
5693 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
5694 break;
5695 }
5696 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5697 case OP_X86_LEA: {
5698 LLVMValueRef v1, v2;
5699
5700 rhs = LLVMBuildSExt (builder, convert (ctx, rhs, LLVMInt32Type ()), LLVMInt64Type (), "");
5701
5702 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ((unsigned long long)1 << ins->backend.shift_amount), FALSE), "");
5703 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
5704 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
5705 break;
5706 }
5707 #endif
5708
5709 case OP_ICONV_TO_I1:
5710 case OP_ICONV_TO_I2:
5711 case OP_ICONV_TO_I4:
5712 case OP_ICONV_TO_U1:
5713 case OP_ICONV_TO_U2:
5714 case OP_ICONV_TO_U4:
5715 case OP_LCONV_TO_I1:
5716 case OP_LCONV_TO_I2:
5717 case OP_LCONV_TO_U1:
5718 case OP_LCONV_TO_U2:
5719 case OP_LCONV_TO_U4: {
5720 gboolean sign;
5721
5722 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
5723
5724 /* Have to do two casts since our vregs have type int */
5725 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
5726 if (sign)
5727 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
5728 else
5729 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
5730 break;
5731 }
5732 case OP_ICONV_TO_I8:
5733 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
5734 break;
5735 case OP_ICONV_TO_U8:
5736 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
5737 break;
5738 case OP_FCONV_TO_I4:
5739 case OP_RCONV_TO_I4:
5740 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
5741 break;
5742 case OP_FCONV_TO_I1:
5743 case OP_RCONV_TO_I1:
5744 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
5745 break;
5746 case OP_FCONV_TO_U1:
5747 case OP_RCONV_TO_U1:
5748 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
5749 break;
5750 case OP_FCONV_TO_I2:
5751 case OP_RCONV_TO_I2:
5752 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5753 break;
5754 case OP_FCONV_TO_U2:
5755 case OP_RCONV_TO_U2:
5756 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5757 break;
5758 case OP_FCONV_TO_U4:
5759 case OP_RCONV_TO_U4:
5760 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
5761 break;
5762 case OP_FCONV_TO_U8:
5763 case OP_RCONV_TO_U8:
5764 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt64Type (), dname);
5765 break;
5766 case OP_FCONV_TO_I8:
5767 case OP_RCONV_TO_I8:
5768 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
5769 break;
5770 case OP_FCONV_TO_I:
5771 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
5772 break;
5773 case OP_ICONV_TO_R8:
5774 case OP_LCONV_TO_R8:
5775 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
5776 break;
5777 case OP_ICONV_TO_R_UN:
5778 case OP_LCONV_TO_R_UN:
5779 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
5780 break;
5781 #if TARGET_SIZEOF_VOID_P == 4
5782 case OP_LCONV_TO_U:
5783 #endif
5784 case OP_LCONV_TO_I4:
5785 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5786 break;
5787 case OP_ICONV_TO_R4:
5788 case OP_LCONV_TO_R4:
5789 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
5790 if (cfg->r4fp)
5791 values [ins->dreg] = v;
5792 else
5793 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5794 break;
5795 case OP_FCONV_TO_R4:
5796 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
5797 if (cfg->r4fp)
5798 values [ins->dreg] = v;
5799 else
5800 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5801 break;
5802 case OP_RCONV_TO_R8:
5803 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
5804 break;
5805 case OP_RCONV_TO_R4:
5806 values [ins->dreg] = lhs;
5807 break;
5808 case OP_SEXT_I4:
5809 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5810 break;
5811 case OP_ZEXT_I4:
5812 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5813 break;
5814 case OP_TRUNC_I4:
5815 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5816 break;
5817 case OP_LOCALLOC_IMM: {
5818 LLVMValueRef v;
5819
5820 guint32 size = ins->inst_imm;
5821 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5822
5823 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
5824
5825 if (ins->flags & MONO_INST_INIT)
5826 emit_memset (ctx, builder, v, const_int32 (size), MONO_ARCH_FRAME_ALIGNMENT);
5827
5828 values [ins->dreg] = v;
5829 break;
5830 }
5831 case OP_LOCALLOC: {
5832 LLVMValueRef v, size;
5833
5834 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
5835
5836 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
5837
5838 if (ins->flags & MONO_INST_INIT)
5839 emit_memset (ctx, builder, v, size, MONO_ARCH_FRAME_ALIGNMENT);
5840 values [ins->dreg] = v;
5841 break;
5842 }
5843
5844 case OP_LOADI1_MEMBASE:
5845 case OP_LOADU1_MEMBASE:
5846 case OP_LOADI2_MEMBASE:
5847 case OP_LOADU2_MEMBASE:
5848 case OP_LOADI4_MEMBASE:
5849 case OP_LOADU4_MEMBASE:
5850 case OP_LOADI8_MEMBASE:
5851 case OP_LOADR4_MEMBASE:
5852 case OP_LOADR8_MEMBASE:
5853 case OP_LOAD_MEMBASE:
5854 case OP_LOADI8_MEM:
5855 case OP_LOADU1_MEM:
5856 case OP_LOADU2_MEM:
5857 case OP_LOADI4_MEM:
5858 case OP_LOADU4_MEM:
5859 case OP_LOAD_MEM: {
5860 int size = 8;
5861 LLVMValueRef base, index, addr;
5862 LLVMTypeRef t;
5863 gboolean sext = FALSE, zext = FALSE;
5864 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
5865 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
5866 gboolean is_unaligned = (ins->flags & MONO_INST_UNALIGNED) != 0;
5867
5868 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5869
5870 if (sext || zext)
5871 dname = (char*)"";
5872
5873 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5874 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5875 base = addr;
5876 } else {
5877 /* _MEMBASE */
5878 base = lhs;
5879
5880 if (ins->inst_offset == 0) {
5881 LLVMValueRef gep_base, gep_offset;
5882 if (mono_llvm_can_be_gep (base, &gep_base, &gep_offset)) {
5883 addr = LLVMBuildGEP (builder, convert (ctx, gep_base, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset, 1, "");
5884 } else {
5885 addr = base;
5886 }
5887 } else if (ins->inst_offset % size != 0) {
5888 /* Unaligned load */
5889 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5890 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5891 } else {
5892 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5893 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5894 }
5895 }
5896
5897 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5898
5899 if (is_unaligned)
5900 values [ins->dreg] = mono_llvm_build_aligned_load (builder, addr, dname, is_volatile, 1);
5901 else
5902 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, base, dname, is_faulting, is_volatile, LLVM_BARRIER_NONE);
5903
5904 if (!(is_faulting || is_volatile) && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5905 /*
5906 * These will signal LLVM that these loads do not alias any stores, and
5907 * they can't fail, allowing them to be hoisted out of loops.
5908 */
5909 set_invariant_load_flag (values [ins->dreg]);
5910 }
5911
5912 if (sext)
5913 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5914 else if (zext)
5915 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5916 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5917 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5918 break;
5919 }
5920
5921 case OP_STOREI1_MEMBASE_REG:
5922 case OP_STOREI2_MEMBASE_REG:
5923 case OP_STOREI4_MEMBASE_REG:
5924 case OP_STOREI8_MEMBASE_REG:
5925 case OP_STORER4_MEMBASE_REG:
5926 case OP_STORER8_MEMBASE_REG:
5927 case OP_STORE_MEMBASE_REG: {
5928 int size = 8;
5929 LLVMValueRef index, addr, base;
5930 LLVMTypeRef t;
5931 gboolean sext = FALSE, zext = FALSE;
5932 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
5933 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
5934 gboolean is_unaligned = (ins->flags & MONO_INST_UNALIGNED) != 0;
5935
5936 if (!values [ins->inst_destbasereg]) {
5937 set_failure (ctx, "inst_destbasereg");
5938 break;
5939 }
5940
5941 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5942
5943 base = values [ins->inst_destbasereg];
5944 LLVMValueRef gep_base, gep_offset;
5945 if (ins->inst_offset == 0 && mono_llvm_can_be_gep (base, &gep_base, &gep_offset)) {
5946 addr = LLVMBuildGEP (builder, convert (ctx, gep_base, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset, 1, "");
5947 } else if (ins->inst_offset % size != 0) {
5948 /* Unaligned store */
5949 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5950 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5951 } else {
5952 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5953 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5954 }
5955 if (is_volatile && LLVMGetInstructionOpcode (base) == LLVMAlloca && !(ins->flags & MONO_INST_VOLATILE))
5956 /* Storing to an alloca cannot fail */
5957 is_volatile = FALSE;
5958 LLVMValueRef srcval = convert (ctx, values [ins->sreg1], t);
5959 LLVMValueRef ptrdst = convert (ctx, addr, LLVMPointerType (t, 0));
5960
5961 if (is_unaligned)
5962 mono_llvm_build_aligned_store (builder, srcval, ptrdst, is_volatile, 1);
5963 else
5964 emit_store (ctx, bb, &builder, size, srcval, ptrdst, base, is_faulting, is_volatile);
5965 break;
5966 }
5967
5968 case OP_STOREI1_MEMBASE_IMM:
5969 case OP_STOREI2_MEMBASE_IMM:
5970 case OP_STOREI4_MEMBASE_IMM:
5971 case OP_STOREI8_MEMBASE_IMM:
5972 case OP_STORE_MEMBASE_IMM: {
5973 int size = 8;
5974 LLVMValueRef index, addr, base;
5975 LLVMTypeRef t;
5976 gboolean sext = FALSE, zext = FALSE;
5977 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
5978 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
5979 gboolean is_unaligned = (ins->flags & MONO_INST_UNALIGNED) != 0;
5980
5981 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5982
5983 base = values [ins->inst_destbasereg];
5984 LLVMValueRef gep_base, gep_offset;
5985 if (ins->inst_offset == 0 && mono_llvm_can_be_gep (base, &gep_base, &gep_offset)) {
5986 addr = LLVMBuildGEP (builder, convert (ctx, gep_base, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset, 1, "");
5987 } else if (ins->inst_offset % size != 0) {
5988 /* Unaligned store */
5989 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5990 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5991 } else {
5992 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5993 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5994 }
5995 LLVMValueRef srcval = convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t);
5996 LLVMValueRef ptrdst = convert (ctx, addr, LLVMPointerType (t, 0));
5997 if (is_unaligned)
5998 mono_llvm_build_aligned_store (builder, srcval, ptrdst, is_volatile, 1);
5999 else
6000 emit_store (ctx, bb, &builder, size, srcval, ptrdst, base, is_faulting, is_volatile);
6001 break;
6002 }
6003
6004 case OP_CHECK_THIS:
6005 emit_load (ctx, bb, &builder, TARGET_SIZEOF_VOID_P, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), lhs, "", TRUE, FALSE, LLVM_BARRIER_NONE);
6006 break;
6007 case OP_OUTARG_VTRETADDR:
6008 break;
6009 case OP_VOIDCALL:
6010 case OP_CALL:
6011 case OP_LCALL:
6012 case OP_FCALL:
6013 case OP_RCALL:
6014 case OP_VCALL:
6015 case OP_VOIDCALL_MEMBASE:
6016 case OP_CALL_MEMBASE:
6017 case OP_LCALL_MEMBASE:
6018 case OP_FCALL_MEMBASE:
6019 case OP_RCALL_MEMBASE:
6020 case OP_VCALL_MEMBASE:
6021 case OP_VOIDCALL_REG:
6022 case OP_CALL_REG:
6023 case OP_LCALL_REG:
6024 case OP_FCALL_REG:
6025 case OP_RCALL_REG:
6026 case OP_VCALL_REG: {
6027 process_call (ctx, bb, &builder, ins);
6028 break;
6029 }
6030 case OP_AOTCONST: {
6031 MonoJumpInfoType ji_type = ins->inst_c1;
6032 gpointer ji_data = ins->inst_p0;
6033
6034 if (ji_type == MONO_PATCH_INFO_ICALL_ADDR) {
6035 char *symbol = mono_aot_get_direct_call_symbol (MONO_PATCH_INFO_ICALL_ADDR_CALL, ji_data);
6036 if (symbol) {
6037 /*
6038 * Avoid emitting a got entry for these since the method is directly called, and it might not be
6039 * resolvable at runtime using dlsym ().
6040 */
6041 g_free (symbol);
6042 values [ins->dreg] = LLVMConstInt (IntPtrType (), 0, FALSE);
6043 break;
6044 }
6045 }
6046
6047 values [ins->dreg] = get_aotconst (ctx, ji_type, ji_data, LLVMPointerType (IntPtrType (), 0));
6048 break;
6049 }
6050 case OP_MEMMOVE: {
6051 int argn = 0;
6052 LLVMValueRef args [5];
6053 args [argn++] = convert (ctx, values [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0));
6054 args [argn++] = convert (ctx, values [ins->sreg2], LLVMPointerType (LLVMInt8Type (), 0));
6055 args [argn++] = convert (ctx, values [ins->sreg3], LLVMInt64Type ());
6056 #if LLVM_API_VERSION < 900
6057 args [argn++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE); // alignment
6058 #endif
6059 args [argn++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); // is_volatile
6060
6061 call_intrins (ctx, INTRINS_MEMMOVE, args, "");
6062 break;
6063 }
6064 case OP_NOT_REACHED:
6065 LLVMBuildUnreachable (builder);
6066 has_terminator = TRUE;
6067 g_assert (bb->block_num < cfg->max_block_num);
6068 ctx->unreachable [bb->block_num] = TRUE;
6069 /* Might have instructions after this */
6070 while (ins->next) {
6071 MonoInst *next = ins->next;
6072 /*
6073 * FIXME: If later code uses the regs defined by these instructions,
6074 * compilation will fail.
6075 */
6076 const char *spec = INS_INFO (next->opcode);
6077 if (spec [MONO_INST_DEST] == 'i' && !MONO_IS_STORE_MEMBASE (next))
6078 ctx->values [next->dreg] = LLVMConstNull (LLVMInt32Type ());
6079 MONO_DELETE_INS (bb, next);
6080 }
6081 break;
6082 case OP_LDADDR: {
6083 MonoInst *var = ins->inst_i0;
6084
6085 if (var->opcode == OP_VTARG_ADDR) {
6086 /* The variable contains the vtype address */
6087 values [ins->dreg] = values [var->dreg];
6088 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
6089 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
6090 } else {
6091 values [ins->dreg] = addresses [var->dreg];
6092 }
6093 break;
6094 }
6095 case OP_SIN: {
6096 LLVMValueRef args [1];
6097
6098 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6099 values [ins->dreg] = call_intrins (ctx, INTRINS_SIN, args, dname);
6100 break;
6101 }
6102 case OP_SINF: {
6103 LLVMValueRef args [1];
6104
6105 args [0] = convert (ctx, lhs, LLVMFloatType ());
6106 values [ins->dreg] = call_intrins (ctx, INTRINS_SINF, args, dname);
6107 break;
6108 }
6109 case OP_EXP: {
6110 LLVMValueRef args [1];
6111
6112 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6113 values [ins->dreg] = call_intrins (ctx, INTRINS_EXP, args, dname);
6114 break;
6115 }
6116 case OP_EXPF: {
6117 LLVMValueRef args [1];
6118
6119 args [0] = convert (ctx, lhs, LLVMFloatType ());
6120 values [ins->dreg] = call_intrins (ctx, INTRINS_EXPF, args, dname);
6121 break;
6122 }
6123 case OP_LOG2: {
6124 LLVMValueRef args [1];
6125
6126 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6127 values [ins->dreg] = call_intrins (ctx, INTRINS_LOG2, args, dname);
6128 break;
6129 }
6130 case OP_LOG2F: {
6131 LLVMValueRef args [1];
6132
6133 args [0] = convert (ctx, lhs, LLVMFloatType ());
6134 values [ins->dreg] = call_intrins (ctx, INTRINS_LOG2F, args, dname);
6135 break;
6136 }
6137 case OP_LOG10: {
6138 LLVMValueRef args [1];
6139
6140 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6141 values [ins->dreg] = call_intrins (ctx, INTRINS_LOG10, args, dname);
6142 break;
6143 }
6144 case OP_LOG10F: {
6145 LLVMValueRef args [1];
6146
6147 args [0] = convert (ctx, lhs, LLVMFloatType ());
6148 values [ins->dreg] = call_intrins (ctx, INTRINS_LOG10F, args, dname);
6149 break;
6150 }
6151 case OP_LOG: {
6152 LLVMValueRef args [1];
6153
6154 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6155 values [ins->dreg] = call_intrins (ctx, INTRINS_LOG, args, dname);
6156 break;
6157 }
6158 case OP_TRUNC: {
6159 LLVMValueRef args [1];
6160
6161 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6162 values [ins->dreg] = call_intrins (ctx, INTRINS_TRUNC, args, dname);
6163 break;
6164 }
6165 case OP_TRUNCF: {
6166 LLVMValueRef args [1];
6167
6168 args [0] = convert (ctx, lhs, LLVMFloatType ());
6169 values [ins->dreg] = call_intrins (ctx, INTRINS_TRUNCF, args, dname);
6170 break;
6171 }
6172 case OP_COS: {
6173 LLVMValueRef args [1];
6174
6175 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6176 values [ins->dreg] = call_intrins (ctx, INTRINS_COS, args, dname);
6177 break;
6178 }
6179 case OP_COSF: {
6180 LLVMValueRef args [1];
6181
6182 args [0] = convert (ctx, lhs, LLVMFloatType ());
6183 values [ins->dreg] = call_intrins (ctx, INTRINS_COSF, args, dname);
6184 break;
6185 }
6186 case OP_SQRT: {
6187 LLVMValueRef args [1];
6188
6189 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6190 values [ins->dreg] = call_intrins (ctx, INTRINS_SQRT, args, dname);
6191 break;
6192 }
6193 case OP_SQRTF: {
6194 LLVMValueRef args [1];
6195
6196 args [0] = convert (ctx, lhs, LLVMFloatType ());
6197 values [ins->dreg] = call_intrins (ctx, INTRINS_SQRTF, args, dname);
6198 break;
6199 }
6200 case OP_FLOOR: {
6201 LLVMValueRef args [1];
6202
6203 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6204 values [ins->dreg] = call_intrins (ctx, INTRINS_FLOOR, args, dname);
6205 break;
6206 }
6207 case OP_FLOORF: {
6208 LLVMValueRef args [1];
6209
6210 args [0] = convert (ctx, lhs, LLVMFloatType ());
6211 values [ins->dreg] = call_intrins (ctx, INTRINS_FLOORF, args, dname);
6212 break;
6213 }
6214 case OP_CEIL: {
6215 LLVMValueRef args [1];
6216
6217 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6218 values [ins->dreg] = call_intrins (ctx, INTRINS_CEIL, args, dname);
6219 break;
6220 }
6221 case OP_CEILF: {
6222 LLVMValueRef args [1];
6223
6224 args [0] = convert (ctx, lhs, LLVMFloatType ());
6225 values [ins->dreg] = call_intrins (ctx, INTRINS_CEILF, args, dname);
6226 break;
6227 }
6228 case OP_FMA: {
6229 LLVMValueRef args [3];
6230
6231 args [0] = convert (ctx, values [ins->sreg1], LLVMDoubleType ());
6232 args [1] = convert (ctx, values [ins->sreg2], LLVMDoubleType ());
6233 args [2] = convert (ctx, values [ins->sreg3], LLVMDoubleType ());
6234
6235 values [ins->dreg] = call_intrins (ctx, INTRINS_FMA, args, dname);
6236 break;
6237 }
6238 case OP_FMAF: {
6239 LLVMValueRef args [3];
6240
6241 args [0] = convert (ctx, values [ins->sreg1], LLVMFloatType ());
6242 args [1] = convert (ctx, values [ins->sreg2], LLVMFloatType ());
6243 args [2] = convert (ctx, values [ins->sreg3], LLVMFloatType ());
6244
6245 values [ins->dreg] = call_intrins (ctx, INTRINS_FMAF, args, dname);
6246 break;
6247 }
6248 case OP_ABS: {
6249 LLVMValueRef args [1];
6250
6251 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6252 values [ins->dreg] = call_intrins (ctx, INTRINS_FABS, args, dname);
6253 break;
6254 }
6255 case OP_ABSF: {
6256 LLVMValueRef args [1];
6257
6258 #ifdef TARGET_AMD64
6259 args [0] = convert (ctx, lhs, LLVMFloatType ());
6260 values [ins->dreg] = call_intrins (ctx, INTRINS_ABSF, args, dname);
6261 #else
6262 /* llvm.fabs not supported on all platforms */
6263 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6264 values [ins->dreg] = call_intrins (ctx, INTRINS_FABS, args, dname);
6265 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
6266 #endif
6267 break;
6268 }
6269 case OP_RPOW: {
6270 LLVMValueRef args [2];
6271
6272 args [0] = convert (ctx, lhs, LLVMFloatType ());
6273 args [1] = convert (ctx, rhs, LLVMFloatType ());
6274 values [ins->dreg] = call_intrins (ctx, INTRINS_POWF, args, dname);
6275 break;
6276 }
6277 case OP_FPOW: {
6278 LLVMValueRef args [2];
6279
6280 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6281 args [1] = convert (ctx, rhs, LLVMDoubleType ());
6282 values [ins->dreg] = call_intrins (ctx, INTRINS_POW, args, dname);
6283 break;
6284 }
6285 case OP_FCOPYSIGN: {
6286 LLVMValueRef args [2];
6287
6288 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6289 args [1] = convert (ctx, rhs, LLVMDoubleType ());
6290 values [ins->dreg] = call_intrins (ctx, INTRINS_COPYSIGN, args, dname);
6291 break;
6292 }
6293 case OP_RCOPYSIGN: {
6294 LLVMValueRef args [2];
6295
6296 args [0] = convert (ctx, lhs, LLVMFloatType ());
6297 args [1] = convert (ctx, rhs, LLVMFloatType ());
6298 values [ins->dreg] = call_intrins (ctx, INTRINS_COPYSIGNF, args, dname);
6299 break;
6300 }
6301
6302 case OP_IMIN:
6303 case OP_LMIN:
6304 case OP_IMAX:
6305 case OP_LMAX:
6306 case OP_IMIN_UN:
6307 case OP_LMIN_UN:
6308 case OP_IMAX_UN:
6309 case OP_LMAX_UN:
6310 case OP_FMIN:
6311 case OP_FMAX:
6312 case OP_RMIN:
6313 case OP_RMAX: {
6314 LLVMValueRef v;
6315
6316 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6317 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6318
6319 switch (ins->opcode) {
6320 case OP_IMIN:
6321 case OP_LMIN:
6322 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
6323 break;
6324 case OP_IMAX:
6325 case OP_LMAX:
6326 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
6327 break;
6328 case OP_IMIN_UN:
6329 case OP_LMIN_UN:
6330 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
6331 break;
6332 case OP_IMAX_UN:
6333 case OP_LMAX_UN:
6334 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
6335 break;
6336 case OP_FMAX:
6337 case OP_RMAX:
6338 v = LLVMBuildFCmp (builder, LLVMRealUGE, lhs, rhs, "");
6339 break;
6340 case OP_FMIN:
6341 case OP_RMIN:
6342 v = LLVMBuildFCmp (builder, LLVMRealULE, lhs, rhs, "");
6343 break;
6344 default:
6345 g_assert_not_reached ();
6346 break;
6347 }
6348 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
6349 break;
6350 }
6351
6352 /*
6353 * See the ARM64 comment in mono/utils/atomic.h for an explanation of why this
6354 * hack is necessary (for now).
6355 */
6356 #ifdef TARGET_ARM64
6357 #define ARM64_ATOMIC_FENCE_FIX mono_llvm_build_fence (builder, LLVM_BARRIER_SEQ)
6358 #else
6359 #define ARM64_ATOMIC_FENCE_FIX
6360 #endif
6361
6362 case OP_ATOMIC_EXCHANGE_I4:
6363 case OP_ATOMIC_EXCHANGE_I8: {
6364 LLVMValueRef args [2];
6365 LLVMTypeRef t;
6366
6367 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
6368 t = LLVMInt32Type ();
6369 else
6370 t = LLVMInt64Type ();
6371
6372 g_assert (ins->inst_offset == 0);
6373
6374 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6375 args [1] = convert (ctx, rhs, t);
6376
6377 ARM64_ATOMIC_FENCE_FIX;
6378 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
6379 ARM64_ATOMIC_FENCE_FIX;
6380 break;
6381 }
6382 case OP_ATOMIC_ADD_I4:
6383 case OP_ATOMIC_ADD_I8:
6384 case OP_ATOMIC_AND_I4:
6385 case OP_ATOMIC_AND_I8:
6386 case OP_ATOMIC_OR_I4:
6387 case OP_ATOMIC_OR_I8: {
6388 LLVMValueRef args [2];
6389 LLVMTypeRef t;
6390
6391 if (ins->type == STACK_I4)
6392 t = LLVMInt32Type ();
6393 else
6394 t = LLVMInt64Type ();
6395
6396 g_assert (ins->inst_offset == 0);
6397
6398 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6399 args [1] = convert (ctx, rhs, t);
6400 ARM64_ATOMIC_FENCE_FIX;
6401 if (ins->opcode == OP_ATOMIC_ADD_I4 || ins->opcode == OP_ATOMIC_ADD_I8)
6402 // Interlocked.Add returns new value (that's why we emit additional Add here)
6403 // see https://github.com/dotnet/runtime/pull/33102
6404 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
6405 else if (ins->opcode == OP_ATOMIC_AND_I4 || ins->opcode == OP_ATOMIC_AND_I8)
6406 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_AND, args [0], args [1]);
6407 else if (ins->opcode == OP_ATOMIC_OR_I4 || ins->opcode == OP_ATOMIC_OR_I8)
6408 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_OR, args [0], args [1]);
6409 else
6410 g_assert_not_reached ();
6411 ARM64_ATOMIC_FENCE_FIX;
6412 break;
6413 }
6414 case OP_ATOMIC_CAS_I4:
6415 case OP_ATOMIC_CAS_I8: {
6416 LLVMValueRef args [3], val;
6417 LLVMTypeRef t;
6418
6419 if (ins->opcode == OP_ATOMIC_CAS_I4)
6420 t = LLVMInt32Type ();
6421 else
6422 t = LLVMInt64Type ();
6423
6424 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6425 /* comparand */
6426 args [1] = convert (ctx, values [ins->sreg3], t);
6427 /* new value */
6428 args [2] = convert (ctx, values [ins->sreg2], t);
6429 ARM64_ATOMIC_FENCE_FIX;
6430 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
6431 ARM64_ATOMIC_FENCE_FIX;
6432 /* cmpxchg returns a pair */
6433 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
6434 break;
6435 }
6436 case OP_MEMORY_BARRIER: {
6437 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
6438 break;
6439 }
6440 case OP_ATOMIC_LOAD_I1:
6441 case OP_ATOMIC_LOAD_I2:
6442 case OP_ATOMIC_LOAD_I4:
6443 case OP_ATOMIC_LOAD_I8:
6444 case OP_ATOMIC_LOAD_U1:
6445 case OP_ATOMIC_LOAD_U2:
6446 case OP_ATOMIC_LOAD_U4:
6447 case OP_ATOMIC_LOAD_U8:
6448 case OP_ATOMIC_LOAD_R4:
6449 case OP_ATOMIC_LOAD_R8: {
6450 int size;
6451 gboolean sext, zext;
6452 LLVMTypeRef t;
6453 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
6454 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
6455 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6456 LLVMValueRef index, addr;
6457
6458 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6459
6460 if (sext || zext)
6461 dname = (char *)"";
6462
6463 if (ins->inst_offset != 0) {
6464 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6465 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
6466 } else {
6467 addr = lhs;
6468 }
6469
6470 addr = convert (ctx, addr, LLVMPointerType (t, 0));
6471
6472 ARM64_ATOMIC_FENCE_FIX;
6473 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, lhs, dname, is_faulting, is_volatile, barrier);
6474 ARM64_ATOMIC_FENCE_FIX;
6475
6476 if (sext)
6477 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6478 else if (zext)
6479 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6480 break;
6481 }
6482 case OP_ATOMIC_STORE_I1:
6483 case OP_ATOMIC_STORE_I2:
6484 case OP_ATOMIC_STORE_I4:
6485 case OP_ATOMIC_STORE_I8:
6486 case OP_ATOMIC_STORE_U1:
6487 case OP_ATOMIC_STORE_U2:
6488 case OP_ATOMIC_STORE_U4:
6489 case OP_ATOMIC_STORE_U8:
6490 case OP_ATOMIC_STORE_R4:
6491 case OP_ATOMIC_STORE_R8: {
6492 int size;
6493 gboolean sext, zext;
6494 LLVMTypeRef t;
6495 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
6496 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
6497 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6498 LLVMValueRef index, addr, value, base;
6499
6500 if (!values [ins->inst_destbasereg]) {
6501 set_failure (ctx, "inst_destbasereg");
6502 break;
6503 }
6504
6505 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6506
6507 base = values [ins->inst_destbasereg];
6508 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6509 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
6510 value = convert (ctx, values [ins->sreg1], t);
6511
6512 ARM64_ATOMIC_FENCE_FIX;
6513 emit_store_general (ctx, bb, &builder, size, value, addr, base, is_faulting, is_volatile, barrier);
6514 ARM64_ATOMIC_FENCE_FIX;
6515 break;
6516 }
6517 case OP_RELAXED_NOP: {
6518 #if defined(TARGET_AMD64) || defined(TARGET_X86)
6519 call_intrins (ctx, INTRINS_SSE_PAUSE, NULL, "");
6520 break;
6521 #else
6522 break;
6523 #endif
6524 }
6525 case OP_TLS_GET: {
6526 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
6527 #ifdef TARGET_AMD64
6528 // 257 == FS segment register
6529 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
6530 #else
6531 // 256 == GS segment register
6532 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6533 #endif
6534 // FIXME: XEN
6535 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
6536 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
6537 /* See mono_amd64_emit_tls_get () */
6538 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
6539
6540 // 256 == GS segment register
6541 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6542 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
6543 #else
6544 set_failure (ctx, "opcode tls-get");
6545 break;
6546 #endif
6547
6548 break;
6549 }
6550 case OP_GC_SAFE_POINT: {
6551 LLVMValueRef val, cmp, callee, call;
6552 LLVMBasicBlockRef poll_bb, cont_bb;
6553 LLVMValueRef args [2];
6554 static LLVMTypeRef sig;
6555 const char *icall_name = "mono_threads_state_poll";
6556
6557 /*
6558 * Create the cold wrapper around the icall, along with a managed method for it so
6559 * unwinding works.
6560 */
6561 if (!cfg->compile_aot && !ctx->module->gc_poll_cold_wrapper_compiled) {
6562 ERROR_DECL (error);
6563 /* Compiling a method here is a bit ugly, but it works */
6564 MonoMethod *wrapper = mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL);
6565 ctx->module->gc_poll_cold_wrapper_compiled = mono_jit_compile_method (wrapper, error);
6566 mono_error_assert_ok (error);
6567 }
6568
6569 if (!sig)
6570 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
6571
6572 /*
6573 * if (!*sreg1)
6574 * mono_threads_state_poll ();
6575 */
6576 val = mono_llvm_build_load (builder, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
6577 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
6578 poll_bb = gen_bb (ctx, "POLL_BB");
6579 cont_bb = gen_bb (ctx, "CONT_BB");
6580
6581 args [0] = cmp;
6582 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
6583 cmp = call_intrins (ctx, INTRINS_EXPECT_I1, args, "");
6584
6585 mono_llvm_build_weighted_branch (builder, cmp, cont_bb, poll_bb, 64, 4);
6586
6587 ctx->builder = builder = create_builder (ctx);
6588 LLVMPositionBuilderAtEnd (builder, poll_bb);
6589
6590 if (ctx->cfg->compile_aot) {
6591 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_threads_state_poll));
6592 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6593 } else {
6594 callee = get_jit_callee (ctx, icall_name, sig, MONO_PATCH_INFO_ABS, ctx->module->gc_poll_cold_wrapper_compiled);
6595 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6596 set_call_cold_cconv (call);
6597 }
6598 LLVMBuildBr (builder, cont_bb);
6599
6600 ctx->builder = builder = create_builder (ctx);
6601 LLVMPositionBuilderAtEnd (builder, cont_bb);
6602 ctx->bblocks [bb->block_num].end_bblock = cont_bb;
6603 break;
6604 }
6605
6606 /*
6607 * Overflow opcodes.
6608 */
6609 case OP_IADD_OVF:
6610 case OP_IADD_OVF_UN:
6611 case OP_ISUB_OVF:
6612 case OP_ISUB_OVF_UN:
6613 case OP_IMUL_OVF:
6614 case OP_IMUL_OVF_UN:
6615 case OP_LADD_OVF:
6616 case OP_LADD_OVF_UN:
6617 case OP_LSUB_OVF:
6618 case OP_LSUB_OVF_UN:
6619 case OP_LMUL_OVF:
6620 case OP_LMUL_OVF_UN: {
6621 LLVMValueRef args [2], val, ovf;
6622 IntrinsicId intrins;
6623
6624 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
6625 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
6626 intrins = ovf_op_to_intrins (ins->opcode);
6627 val = call_intrins (ctx, intrins, args, "");
6628 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
6629 ovf = LLVMBuildExtractValue (builder, val, 1, "");
6630 emit_cond_system_exception (ctx, bb, "OverflowException", ovf, FALSE);
6631 if (!ctx_ok (ctx))
6632 break;
6633 builder = ctx->builder;
6634 break;
6635 }
6636
6637 /*
6638 * Valuetypes.
6639 * We currently model them using arrays. Promotion to local vregs is
6640 * disabled for them in mono_handle_global_vregs () in the LLVM case,
6641 * so we always have an entry in cfg->varinfo for them.
6642 * FIXME: Is this needed ?
6643 */
6644 case OP_VZERO: {
6645 MonoClass *klass = ins->klass;
6646
6647 if (!klass) {
6648 // FIXME:
6649 set_failure (ctx, "!klass");
6650 break;
6651 }
6652
6653 if (!addresses [ins->dreg])
6654 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6655 LLVMValueRef ptr = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6656 emit_memset (ctx, builder, ptr, const_int32 (mono_class_value_size (klass, NULL)), 0);
6657 break;
6658 }
6659 case OP_DUMMY_VZERO:
6660 break;
6661
6662 case OP_STOREV_MEMBASE:
6663 case OP_LOADV_MEMBASE:
6664 case OP_VMOVE: {
6665 MonoClass *klass = ins->klass;
6666 LLVMValueRef src = NULL, dst, args [5];
6667 gboolean done = FALSE;
6668
6669 if (!klass) {
6670 // FIXME:
6671 set_failure (ctx, "!klass");
6672 break;
6673 }
6674
6675 if (mini_is_gsharedvt_klass (klass)) {
6676 // FIXME:
6677 set_failure (ctx, "gsharedvt");
6678 break;
6679 }
6680
6681 switch (ins->opcode) {
6682 case OP_STOREV_MEMBASE:
6683 if (cfg->gen_write_barriers && m_class_has_references (klass) && ins->inst_destbasereg != cfg->frame_reg &&
6684 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
6685 /* Decomposed earlier */
6686 g_assert_not_reached ();
6687 break;
6688 }
6689 if (!addresses [ins->sreg1]) {
6690 /* SIMD */
6691 g_assert (values [ins->sreg1]);
6692 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (klass)), 0));
6693 LLVMBuildStore (builder, values [ins->sreg1], dst);
6694 done = TRUE;
6695 } else {
6696 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6697 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6698 }
6699 break;
6700 case OP_LOADV_MEMBASE:
6701 if (!addresses [ins->dreg])
6702 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6703 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6704 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6705 break;
6706 case OP_VMOVE:
6707 if (!addresses [ins->sreg1])
6708 addresses [ins->sreg1] = build_alloca (ctx, m_class_get_byval_arg (klass));
6709 if (!addresses [ins->dreg])
6710 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6711 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6712 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6713 break;
6714 default:
6715 g_assert_not_reached ();
6716 }
6717 if (!ctx_ok (ctx))
6718 break;
6719
6720 if (done)
6721 break;
6722
6723 int aindex = 0;
6724 args [aindex ++] = dst;
6725 args [aindex ++] = src;
6726 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
6727 #if LLVM_API_VERSION < 900
6728 // FIXME: Alignment
6729 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6730 #endif
6731 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6732 call_intrins (ctx, INTRINS_MEMCPY, args, "");
6733 break;
6734 }
6735 case OP_LLVM_OUTARG_VT: {
6736 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
6737 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
6738
6739 if (ainfo->storage == LLVMArgGsharedvtVariable) {
6740 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
6741
6742 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
6743 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
6744 } else {
6745 g_assert (addresses [ins->sreg1]);
6746 addresses [ins->dreg] = addresses [ins->sreg1];
6747 }
6748 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
6749 if (!addresses [ins->sreg1]) {
6750 addresses [ins->sreg1] = build_alloca (ctx, t);
6751 g_assert (values [ins->sreg1]);
6752 }
6753 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
6754 addresses [ins->dreg] = addresses [ins->sreg1];
6755 } else {
6756 if (!addresses [ins->sreg1]) {
6757 addresses [ins->sreg1] = build_alloca (ctx, t);
6758 g_assert (values [ins->sreg1]);
6759 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
6760 addresses [ins->dreg] = addresses [ins->sreg1];
6761 } else if (ainfo->storage == LLVMArgVtypeAddr || values [ins->sreg1] == addresses [ins->sreg1]) {
6762 /* LLVMArgVtypeByRef/LLVMArgVtypeAddr, have to make a copy */
6763 addresses [ins->dreg] = build_alloca (ctx, t);
6764 LLVMValueRef v = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
6765 LLVMBuildStore (builder, convert (ctx, v, type_to_llvm_type (ctx, t)), addresses [ins->dreg]);
6766 } else {
6767 addresses [ins->dreg] = addresses [ins->sreg1];
6768 }
6769 }
6770 break;
6771 }
6772 case OP_OBJC_GET_SELECTOR: {
6773 const char *name = (const char*)ins->inst_p0;
6774 LLVMValueRef var;
6775
6776 if (!ctx->module->objc_selector_to_var) {
6777 ctx->module->objc_selector_to_var = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
6778
6779 LLVMValueRef info_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), 8), "@OBJC_IMAGE_INFO");
6780 int32_t objc_imageinfo [] = { 0, 16 };
6781 LLVMSetInitializer (info_var, mono_llvm_create_constant_data_array ((uint8_t *) &objc_imageinfo, 8));
6782 LLVMSetLinkage (info_var, LLVMPrivateLinkage);
6783 LLVMSetExternallyInitialized (info_var, TRUE);
6784 LLVMSetSection (info_var, "__DATA, __objc_imageinfo,regular,no_dead_strip");
6785 LLVMSetAlignment (info_var, sizeof (target_mgreg_t));
6786 mark_as_used (ctx->module, info_var);
6787 }
6788
6789 var = (LLVMValueRef)g_hash_table_lookup (ctx->module->objc_selector_to_var, name);
6790 if (!var) {
6791 LLVMValueRef indexes [16];
6792
6793 LLVMValueRef name_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), strlen (name) + 1), "@OBJC_METH_VAR_NAME_");
6794 LLVMSetInitializer (name_var, mono_llvm_create_constant_data_array ((const uint8_t*)name, strlen (name) + 1));
6795 LLVMSetLinkage (name_var, LLVMPrivateLinkage);
6796 LLVMSetSection (name_var, "__TEXT,__objc_methname,cstring_literals");
6797 mark_as_used (ctx->module, name_var);
6798
6799 LLVMValueRef ref_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (LLVMInt8Type (), 0), "@OBJC_SELECTOR_REFERENCES_");
6800
6801 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6802 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6803 LLVMSetInitializer (ref_var, LLVMConstGEP (name_var, indexes, 2));
6804 LLVMSetLinkage (ref_var, LLVMPrivateLinkage);
6805 LLVMSetExternallyInitialized (ref_var, TRUE);
6806 LLVMSetSection (ref_var, "__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
6807 LLVMSetAlignment (ref_var, sizeof (target_mgreg_t));
6808 mark_as_used (ctx->module, ref_var);
6809
6810 g_hash_table_insert (ctx->module->objc_selector_to_var, g_strdup (name), ref_var);
6811 var = ref_var;
6812 }
6813
6814 values [ins->dreg] = LLVMBuildLoad (builder, var, "");
6815 break;
6816 }
6817
6818 /*
6819 * SIMD
6820 */
6821 #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM)
6822 case OP_XZERO: {
6823 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
6824 break;
6825 }
6826 case OP_LOADX_MEMBASE: {
6827 LLVMTypeRef t = type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass));
6828 LLVMValueRef src;
6829
6830 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6831 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
6832 break;
6833 }
6834 case OP_STOREX_MEMBASE: {
6835 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
6836 LLVMValueRef dest;
6837
6838 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6839 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
6840 break;
6841 }
6842 case OP_PADDB:
6843 case OP_PADDW:
6844 case OP_PADDD:
6845 case OP_PADDQ:
6846 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
6847 break;
6848 case OP_ADDPD:
6849 case OP_ADDPS:
6850 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
6851 break;
6852 case OP_PSUBB:
6853 case OP_PSUBW:
6854 case OP_PSUBD:
6855 case OP_PSUBQ:
6856 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
6857 break;
6858 case OP_SUBPD:
6859 case OP_SUBPS:
6860 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
6861 break;
6862 case OP_MULPD:
6863 case OP_MULPS:
6864 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
6865 break;
6866 case OP_DIVPD:
6867 case OP_DIVPS:
6868 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
6869 break;
6870 case OP_PAND:
6871 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
6872 break;
6873 case OP_POR:
6874 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
6875 break;
6876 case OP_PXOR:
6877 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
6878 break;
6879 case OP_PMULW:
6880 case OP_PMULD:
6881 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
6882 break;
6883 case OP_ANDPS:
6884 case OP_ANDNPS:
6885 case OP_ORPS:
6886 case OP_XORPS:
6887 case OP_ANDPD:
6888 case OP_ANDNPD:
6889 case OP_ORPD:
6890 case OP_XORPD: {
6891 LLVMTypeRef t, rt;
6892 LLVMValueRef v = NULL;
6893
6894 switch (ins->opcode) {
6895 case OP_ANDPS:
6896 case OP_ANDNPS:
6897 case OP_ORPS:
6898 case OP_XORPS:
6899 t = LLVMVectorType (LLVMInt32Type (), 4);
6900 rt = LLVMVectorType (LLVMFloatType (), 4);
6901 break;
6902 case OP_ANDPD:
6903 case OP_ANDNPD:
6904 case OP_ORPD:
6905 case OP_XORPD:
6906 t = LLVMVectorType (LLVMInt64Type (), 2);
6907 rt = LLVMVectorType (LLVMDoubleType (), 2);
6908 break;
6909 default:
6910 t = LLVMInt32Type ();
6911 rt = LLVMInt32Type ();
6912 g_assert_not_reached ();
6913 }
6914
6915 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6916 rhs = LLVMBuildBitCast (builder, rhs, t, "");
6917 switch (ins->opcode) {
6918 case OP_ANDPS:
6919 case OP_ANDPD:
6920 v = LLVMBuildAnd (builder, lhs, rhs, "");
6921 break;
6922 case OP_ORPS:
6923 case OP_ORPD:
6924 v = LLVMBuildOr (builder, lhs, rhs, "");
6925 break;
6926 case OP_XORPS:
6927 case OP_XORPD:
6928 v = LLVMBuildXor (builder, lhs, rhs, "");
6929 break;
6930 case OP_ANDNPS:
6931 case OP_ANDNPD:
6932 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
6933 break;
6934 }
6935 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
6936 break;
6937 }
6938 case OP_PMIND_UN:
6939 case OP_PMINW_UN:
6940 case OP_PMINB_UN: {
6941 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntULT, lhs, rhs, "");
6942 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6943 break;
6944 }
6945 case OP_PMAXD_UN:
6946 case OP_PMAXW_UN:
6947 case OP_PMAXB_UN: {
6948 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntUGT, lhs, rhs, "");
6949 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6950 break;
6951 }
6952 case OP_PMINW: {
6953 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntSLT, lhs, rhs, "");
6954 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6955 break;
6956 }
6957 case OP_PMAXW: {
6958 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntSGT, lhs, rhs, "");
6959 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6960 break;
6961 }
6962 case OP_MINPD:
6963 case OP_MINPS:
6964 case OP_MAXPD:
6965 case OP_MAXPS:
6966 case OP_ADDSUBPD:
6967 case OP_ADDSUBPS:
6968 case OP_HADDPD:
6969 case OP_HADDPS:
6970 case OP_HSUBPD:
6971 case OP_HSUBPS:
6972 case OP_PACKW:
6973 case OP_PACKD:
6974 case OP_PACKW_UN:
6975 case OP_PACKD_UN:
6976 case OP_PMULW_HIGH:
6977 case OP_PMULW_HIGH_UN: {
6978 LLVMValueRef args [2];
6979
6980 args [0] = lhs;
6981 args [1] = rhs;
6982
6983 values [ins->dreg] = call_intrins (ctx, simd_ins_to_intrins (ins->opcode), args, "");
6984 break;
6985 }
6986 case OP_PAVGB_UN:
6987 case OP_PAVGW_UN: {
6988 LLVMValueRef ones_vec;
6989 LLVMValueRef ones [32];
6990 int vector_size = LLVMGetVectorSize (LLVMTypeOf (lhs));
6991 LLVMTypeRef ext_elem_type = vector_size == 16 ? LLVMInt16Type () : LLVMInt32Type ();
6992
6993 for (int i = 0; i < 32; ++i)
6994 ones [i] = LLVMConstInt (ext_elem_type, 1, FALSE);
6995 ones_vec = LLVMConstVector (ones, vector_size);
6996
6997 LLVMValueRef val;
6998 LLVMTypeRef ext_type = LLVMVectorType (ext_elem_type, vector_size);
6999
7000 /* Have to increase the vector element size to prevent overflows */
7001 /* res = trunc ((zext (lhs) + zext (rhs) + 1) >> 1) */
7002 val = LLVMBuildAdd (builder, LLVMBuildZExt (builder, lhs, ext_type, ""), LLVMBuildZExt (builder, rhs, ext_type, ""), "");
7003 val = LLVMBuildAdd (builder, val, ones_vec, "");
7004 val = LLVMBuildLShr (builder, val, ones_vec, "");
7005 values [ins->dreg] = LLVMBuildTrunc (builder, val, LLVMTypeOf (lhs), "");
7006 break;
7007 }
7008 case OP_PCMPEQB:
7009 case OP_PCMPEQW:
7010 case OP_PCMPEQD:
7011 case OP_PCMPEQQ:
7012 case OP_PCMPGTB: {
7013 LLVMValueRef pcmp;
7014 LLVMTypeRef retType;
7015 LLVMIntPredicate cmpOp;
7016
7017 if (ins->opcode == OP_PCMPGTB)
7018 cmpOp = LLVMIntSGT;
7019 else
7020 cmpOp = LLVMIntEQ;
7021
7022 if (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)) {
7023 pcmp = LLVMBuildICmp (builder, cmpOp, lhs, rhs, "");
7024 retType = LLVMTypeOf (lhs);
7025 } else {
7026 LLVMTypeRef flatType = LLVMVectorType (LLVMInt8Type (), 16);
7027 LLVMValueRef flatRHS = convert (ctx, rhs, flatType);
7028 LLVMValueRef flatLHS = convert (ctx, lhs, flatType);
7029
7030 pcmp = LLVMBuildICmp (builder, cmpOp, flatLHS, flatRHS, "");
7031 retType = flatType;
7032 }
7033
7034 values [ins->dreg] = LLVMBuildSExt (builder, pcmp, retType, "");
7035 break;
7036 }
7037 case OP_EXTRACT_R4:
7038 case OP_EXTRACT_R8:
7039 case OP_EXTRACT_I8:
7040 case OP_EXTRACT_I4:
7041 case OP_EXTRACT_I2:
7042 case OP_EXTRACT_U2:
7043 case OP_EXTRACTX_U2:
7044 case OP_EXTRACT_I1:
7045 case OP_EXTRACT_U1: {
7046 LLVMTypeRef t;
7047 gboolean zext = FALSE;
7048
7049 t = simd_op_to_llvm_type (ins->opcode);
7050
7051 switch (ins->opcode) {
7052 case OP_EXTRACT_R4:
7053 case OP_EXTRACT_R8:
7054 case OP_EXTRACT_I8:
7055 case OP_EXTRACT_I4:
7056 case OP_EXTRACT_I2:
7057 case OP_EXTRACT_I1:
7058 break;
7059 case OP_EXTRACT_U2:
7060 case OP_EXTRACTX_U2:
7061 case OP_EXTRACT_U1:
7062 zext = TRUE;
7063 break;
7064 default:
7065 t = LLVMInt32Type ();
7066 g_assert_not_reached ();
7067 }
7068
7069 lhs = LLVMBuildBitCast (builder, lhs, t, "");
7070 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
7071 if (zext)
7072 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
7073 break;
7074 }
7075
7076 case OP_EXPAND_I1:
7077 case OP_EXPAND_I2:
7078 case OP_EXPAND_I4:
7079 case OP_EXPAND_I8:
7080 case OP_EXPAND_R4:
7081 case OP_EXPAND_R8: {
7082 LLVMTypeRef t;
7083 LLVMValueRef mask [32], v;
7084 int i;
7085
7086 #ifdef ENABLE_NETCORE
7087 t = simd_class_to_llvm_type (ctx, ins->klass);
7088 #else
7089 t = simd_op_to_llvm_type (ins->opcode);
7090 #endif
7091 for (i = 0; i < 32; ++i)
7092 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7093
7094 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
7095
7096 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7097 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
7098 break;
7099 }
7100
7101 case OP_INSERT_I1:
7102 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7103 break;
7104 case OP_INSERT_I2:
7105 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7106 break;
7107 case OP_INSERT_I4:
7108 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7109 break;
7110 case OP_INSERT_I8:
7111 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7112 break;
7113 case OP_INSERT_R4:
7114 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7115 break;
7116 case OP_INSERT_R8:
7117 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7118 break;
7119 case OP_XINSERT_I2: {
7120 LLVMBasicBlockRef bbs [64];
7121 LLVMValueRef switch_ins;
7122 LLVMValueRef vector = lhs;
7123 LLVMValueRef value = rhs;
7124 LLVMValueRef index = values [ins->sreg3];
7125 LLVMValueRef phi_values [64];
7126 int nelems = LLVMGetVectorSize (LLVMTypeOf (lhs));
7127 int i;
7128
7129 /*
7130 * Many SIMD opcodes require an immediate operand, but can be called with a non-immediate.
7131 * To handle these cases, generate a switch statement with one case for all possible
7132 * values of the immediate.
7133 * switch (index) {
7134 * case i:
7135 * res = <op> (val, i)
7136 * break;
7137 * }
7138 */
7139 g_assert (nelems <= 64);
7140 for (i = 0; i < nelems; ++i)
7141 bbs [i] = gen_bb (ctx, "XINSERT_CASE_BB");
7142 cbb = gen_bb (ctx, "XINSERT_COND_BB");
7143
7144 switch_ins = LLVMBuildSwitch (builder, LLVMBuildAnd (builder, index, const_int32 (0xf), ""), bbs [0], 0);
7145 for (i = 0; i < nelems; ++i) {
7146 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
7147 LLVMPositionBuilderAtEnd (builder, bbs [i]);
7148 phi_values [i] = LLVMBuildInsertElement (builder, vector, convert (ctx, value, LLVMGetElementType (LLVMTypeOf (vector))), LLVMConstInt (LLVMInt32Type (), i, FALSE), "");
7149 LLVMBuildBr (builder, cbb);
7150 }
7151
7152 LLVMPositionBuilderAtEnd (builder, cbb);
7153 values [ins->dreg] = LLVMBuildPhi (builder, LLVMTypeOf (phi_values [0]), "");
7154 LLVMAddIncoming (values [ins->dreg], phi_values, bbs, nelems);
7155
7156 ctx->bblocks [bb->block_num].end_bblock = cbb;
7157 break;
7158 }
7159 case OP_CVTDQ2PS: {
7160 LLVMValueRef i4 = LLVMBuildBitCast (builder, lhs, sse_i4_t, "");
7161 values [ins->dreg] = LLVMBuildSIToFP (builder, i4, sse_r4_t, dname);
7162 break;
7163 }
7164 case OP_CVTDQ2PD: {
7165 LLVMValueRef indexes [16];
7166
7167 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7168 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7169 LLVMValueRef mask = LLVMConstVector (indexes, 2);
7170 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
7171 values [ins->dreg] = LLVMBuildSIToFP (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
7172 break;
7173 }
7174 case OP_SSE2_CVTSS2SD: {
7175 LLVMValueRef rhs_elem = LLVMBuildExtractElement (builder, rhs, const_int32 (0), "");
7176 LLVMValueRef fpext = LLVMBuildFPExt (builder, rhs_elem, LLVMDoubleType (), dname);
7177 values [ins->dreg] = LLVMBuildInsertElement (builder, lhs, fpext, const_int32 (0), "");
7178 break;
7179 }
7180 case OP_CVTPS2PD: {
7181 LLVMValueRef indexes [16];
7182
7183 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7184 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7185 LLVMValueRef mask = LLVMConstVector (indexes, 2);
7186 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
7187 values [ins->dreg] = LLVMBuildFPExt (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
7188 break;
7189 }
7190 case OP_CVTTPS2DQ:
7191 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMVectorType (LLVMInt32Type (), 4), dname);
7192 break;
7193
7194 case OP_CVTPD2DQ:
7195 case OP_CVTPS2DQ:
7196 case OP_CVTPD2PS:
7197 case OP_CVTTPD2DQ:
7198 case OP_EXTRACT_MASK:
7199 case OP_SQRTPS:
7200 case OP_SQRTPD:
7201 case OP_RSQRTPS:
7202 case OP_RCPPS: {
7203 LLVMValueRef v;
7204
7205 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
7206
7207 values [ins->dreg] = call_intrins (ctx, simd_ins_to_intrins (ins->opcode), &v, dname);
7208 break;
7209 }
7210 case OP_COMPPS:
7211 case OP_COMPPD: {
7212 LLVMRealPredicate op;
7213
7214 switch (ins->inst_c0) {
7215 case SIMD_COMP_EQ:
7216 op = LLVMRealOEQ;
7217 break;
7218 case SIMD_COMP_LT:
7219 op = LLVMRealOLT;
7220 break;
7221 case SIMD_COMP_LE:
7222 op = LLVMRealOLE;
7223 break;
7224 case SIMD_COMP_UNORD:
7225 op = LLVMRealUNO;
7226 break;
7227 case SIMD_COMP_NEQ:
7228 op = LLVMRealUNE;
7229 break;
7230 case SIMD_COMP_NLT:
7231 op = LLVMRealUGE;
7232 break;
7233 case SIMD_COMP_NLE:
7234 op = LLVMRealUGT;
7235 break;
7236 case SIMD_COMP_ORD:
7237 op = LLVMRealORD;
7238 break;
7239 default:
7240 g_assert_not_reached ();
7241 }
7242
7243 LLVMValueRef cmp = LLVMBuildFCmp (builder, op, lhs, rhs, "");
7244 if (ins->opcode == OP_COMPPD)
7245 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), 2), ""), LLVMTypeOf (lhs), "");
7246 else
7247 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), 4), ""), LLVMTypeOf (lhs), "");
7248 break;
7249 }
7250 case OP_ICONV_TO_X:
7251 /* This is only used for implementing shifts by non-immediate */
7252 values [ins->dreg] = lhs;
7253 break;
7254
7255 case OP_PSHRW:
7256 case OP_PSHRD:
7257 case OP_PSHRQ:
7258 case OP_PSARW:
7259 case OP_PSARD:
7260 case OP_PSHLW:
7261 case OP_PSHLD:
7262 case OP_PSHLQ: {
7263 LLVMValueRef args [3];
7264
7265 args [0] = lhs;
7266 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
7267
7268 values [ins->dreg] = call_intrins (ctx, simd_ins_to_intrins (ins->opcode), args, dname);
7269 break;
7270 }
7271
7272 case OP_PSHRW_REG:
7273 case OP_PSHRD_REG:
7274 case OP_PSHRQ_REG:
7275 case OP_PSARW_REG:
7276 case OP_PSARD_REG:
7277 case OP_PSHLW_REG:
7278 case OP_PSHLD_REG:
7279 case OP_PSHLQ_REG: {
7280 LLVMValueRef args [3];
7281
7282 args [0] = lhs;
7283 args [1] = values [ins->sreg2];
7284
7285 values [ins->dreg] = call_intrins (ctx, simd_ins_to_intrins (ins->opcode), args, dname);
7286 break;
7287 }
7288
7289 case OP_SHUFPS:
7290 case OP_SHUFPD:
7291 case OP_PSHUFLED:
7292 case OP_PSHUFLEW_LOW:
7293 case OP_PSHUFLEW_HIGH: {
7294 int mask [16];
7295 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
7296 int i, mask_size = 0;
7297 int imask = ins->inst_c0;
7298
7299 /* Convert the x86 shuffle mask to LLVM's */
7300 switch (ins->opcode) {
7301 case OP_SHUFPS:
7302 mask_size = 4;
7303 mask [0] = ((imask >> 0) & 3);
7304 mask [1] = ((imask >> 2) & 3);
7305 mask [2] = ((imask >> 4) & 3) + 4;
7306 mask [3] = ((imask >> 6) & 3) + 4;
7307 v1 = values [ins->sreg1];
7308 v2 = values [ins->sreg2];
7309 break;
7310 case OP_SHUFPD:
7311 mask_size = 2;
7312 mask [0] = ((imask >> 0) & 1);
7313 mask [1] = ((imask >> 1) & 1) + 2;
7314 v1 = values [ins->sreg1];
7315 v2 = values [ins->sreg2];
7316 break;
7317 case OP_PSHUFLEW_LOW:
7318 mask_size = 8;
7319 mask [0] = ((imask >> 0) & 3);
7320 mask [1] = ((imask >> 2) & 3);
7321 mask [2] = ((imask >> 4) & 3);
7322 mask [3] = ((imask >> 6) & 3);
7323 mask [4] = 4 + 0;
7324 mask [5] = 4 + 1;
7325 mask [6] = 4 + 2;
7326 mask [7] = 4 + 3;
7327 v1 = values [ins->sreg1];
7328 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7329 break;
7330 case OP_PSHUFLEW_HIGH:
7331 mask_size = 8;
7332 mask [0] = 0;
7333 mask [1] = 1;
7334 mask [2] = 2;
7335 mask [3] = 3;
7336 mask [4] = 4 + ((imask >> 0) & 3);
7337 mask [5] = 4 + ((imask >> 2) & 3);
7338 mask [6] = 4 + ((imask >> 4) & 3);
7339 mask [7] = 4 + ((imask >> 6) & 3);
7340 v1 = values [ins->sreg1];
7341 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7342 break;
7343 case OP_PSHUFLED:
7344 mask_size = 4;
7345 mask [0] = ((imask >> 0) & 3);
7346 mask [1] = ((imask >> 2) & 3);
7347 mask [2] = ((imask >> 4) & 3);
7348 mask [3] = ((imask >> 6) & 3);
7349 v1 = values [ins->sreg1];
7350 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7351 break;
7352 default:
7353 g_assert_not_reached ();
7354 }
7355 for (i = 0; i < mask_size; ++i)
7356 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
7357
7358 values [ins->dreg] =
7359 LLVMBuildShuffleVector (builder, v1, v2,
7360 LLVMConstVector (mask_values, mask_size), dname);
7361 break;
7362 }
7363
7364 case OP_UNPACK_LOWB:
7365 case OP_UNPACK_LOWW:
7366 case OP_UNPACK_LOWD:
7367 case OP_UNPACK_LOWQ:
7368 case OP_UNPACK_LOWPS:
7369 case OP_UNPACK_LOWPD:
7370 case OP_UNPACK_HIGHB:
7371 case OP_UNPACK_HIGHW:
7372 case OP_UNPACK_HIGHD:
7373 case OP_UNPACK_HIGHQ:
7374 case OP_UNPACK_HIGHPS:
7375 case OP_UNPACK_HIGHPD: {
7376 int mask [16];
7377 LLVMValueRef mask_values [16];
7378 int i, mask_size = 0;
7379 gboolean low = FALSE;
7380
7381 switch (ins->opcode) {
7382 case OP_UNPACK_LOWB:
7383 mask_size = 16;
7384 low = TRUE;
7385 break;
7386 case OP_UNPACK_LOWW:
7387 mask_size = 8;
7388 low = TRUE;
7389 break;
7390 case OP_UNPACK_LOWD:
7391 case OP_UNPACK_LOWPS:
7392 mask_size = 4;
7393 low = TRUE;
7394 break;
7395 case OP_UNPACK_LOWQ:
7396 case OP_UNPACK_LOWPD:
7397 mask_size = 2;
7398 low = TRUE;
7399 break;
7400 case OP_UNPACK_HIGHB:
7401 mask_size = 16;
7402 break;
7403 case OP_UNPACK_HIGHW:
7404 mask_size = 8;
7405 break;
7406 case OP_UNPACK_HIGHD:
7407 case OP_UNPACK_HIGHPS:
7408 mask_size = 4;
7409 break;
7410 case OP_UNPACK_HIGHQ:
7411 case OP_UNPACK_HIGHPD:
7412 mask_size = 2;
7413 break;
7414 default:
7415 g_assert_not_reached ();
7416 }
7417
7418 if (low) {
7419 for (i = 0; i < (mask_size / 2); ++i) {
7420 mask [(i * 2)] = i;
7421 mask [(i * 2) + 1] = mask_size + i;
7422 }
7423 } else {
7424 for (i = 0; i < (mask_size / 2); ++i) {
7425 mask [(i * 2)] = (mask_size / 2) + i;
7426 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
7427 }
7428 }
7429
7430 for (i = 0; i < mask_size; ++i)
7431 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
7432
7433 values [ins->dreg] =
7434 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
7435 LLVMConstVector (mask_values, mask_size), dname);
7436 break;
7437 }
7438
7439 case OP_DUPPD: {
7440 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7441 LLVMValueRef v, val;
7442
7443 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7444 val = LLVMConstNull (t);
7445 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7446 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
7447
7448 values [ins->dreg] = val;
7449 break;
7450 }
7451 case OP_DUPPS_LOW:
7452 case OP_DUPPS_HIGH: {
7453 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7454 LLVMValueRef v1, v2, val;
7455
7456
7457 if (ins->opcode == OP_DUPPS_LOW) {
7458 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7459 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7460 } else {
7461 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7462 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7463 }
7464 val = LLVMConstNull (t);
7465 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7466 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7467 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7468 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7469
7470 values [ins->dreg] = val;
7471 break;
7472 }
7473
7474 case OP_DPPS: {
7475 LLVMValueRef args [3];
7476
7477 args [0] = lhs;
7478 args [1] = rhs;
7479 /* 0xf1 == multiply all 4 elements, add them together, and store the result to the lowest element */
7480 args [2] = LLVMConstInt (LLVMInt8Type (), 0xf1, FALSE);
7481
7482 values [ins->dreg] = call_intrins (ctx, simd_ins_to_intrins (ins->opcode), args, dname);
7483 break;
7484 }
7485
7486 case OP_FCONV_TO_R8_X: {
7487 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (sse_r8_t), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7488 break;
7489 }
7490
7491 case OP_FCONV_TO_R4_X: {
7492 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (sse_r4_t), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7493 break;
7494 }
7495
7496 #if defined(TARGET_X86) || defined(TARGET_AMD64)
7497 case OP_SSE_MOVMSK: {
7498 LLVMValueRef args [1];
7499 if (ins->inst_c1 == MONO_TYPE_R4) {
7500 args [0] = lhs;
7501 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_MOVMSK_PS, args, dname);
7502 } else if (ins->inst_c1 == MONO_TYPE_R8) {
7503 args [0] = lhs;
7504 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_MOVMSK_PD, args, dname);
7505 } else {
7506 args [0] = convert (ctx, lhs, sse_i1_t);
7507 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_PMOVMSKB, args, dname);
7508 }
7509 break;
7510 }
7511
7512 case OP_SSE_MOVS:
7513 case OP_SSE_MOVS2: {
7514 if (ins->inst_c1 == MONO_TYPE_R4)
7515 values [ins->dreg] = LLVMBuildShuffleVector (builder, rhs, lhs, create_const_vector_4_i32 (0, 5, 6, 7), "");
7516 else if (ins->inst_c1 == MONO_TYPE_R8)
7517 values [ins->dreg] = LLVMBuildShuffleVector (builder, rhs, lhs, create_const_vector_2_i32 (0, 3), "");
7518 else if (ins->inst_c1 == MONO_TYPE_I8 || ins->inst_c1 == MONO_TYPE_U8)
7519 values [ins->dreg] = LLVMBuildInsertElement (builder, lhs,
7520 LLVMConstInt (LLVMInt64Type (), 0, FALSE),
7521 LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7522 else
7523 g_assert_not_reached (); // will be needed for other types later
7524 break;
7525 }
7526
7527 case OP_SSE_MOVEHL: {
7528 if (ins->inst_c1 == MONO_TYPE_R4)
7529 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_4_i32 (6, 7, 2, 3), "");
7530 else
7531 g_assert_not_reached ();
7532 break;
7533 }
7534
7535 case OP_SSE_MOVELH: {
7536 if (ins->inst_c1 == MONO_TYPE_R4)
7537 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_4_i32 (0, 1, 4, 5), "");
7538 else
7539 g_assert_not_reached ();
7540 break;
7541 }
7542
7543 case OP_SSE_UNPACKLO: {
7544 if (ins->inst_c1 == MONO_TYPE_R8 || ins->inst_c1 == MONO_TYPE_I8 || ins->inst_c1 == MONO_TYPE_U8) {
7545 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_2_i32 (0, 2), "");
7546 } else if (ins->inst_c1 == MONO_TYPE_R4 || ins->inst_c1 == MONO_TYPE_I4 || ins->inst_c1 == MONO_TYPE_U4) {
7547 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_4_i32 (0, 4, 1, 5), "");
7548 } else if (ins->inst_c1 == MONO_TYPE_I2 || ins->inst_c1 == MONO_TYPE_U2) {
7549 const int mask_values [] = { 0, 8, 1, 9, 2, 10, 3, 11 };
7550 LLVMValueRef shuffled = LLVMBuildShuffleVector (builder,
7551 convert (ctx, lhs, sse_i2_t),
7552 convert (ctx, rhs, sse_i2_t),
7553 create_const_vector_i32 (mask_values, 8), "");
7554 values [ins->dreg] = convert (ctx, shuffled, type_to_sse_type (ins->inst_c1));
7555 } else if (ins->inst_c1 == MONO_TYPE_I1 || ins->inst_c1 == MONO_TYPE_U1) {
7556 const int mask_values [] = { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 };
7557 LLVMValueRef shuffled = LLVMBuildShuffleVector (builder,
7558 convert (ctx, lhs, sse_i1_t),
7559 convert (ctx, rhs, sse_i1_t),
7560 create_const_vector_i32 (mask_values, 16), "");
7561 values [ins->dreg] = convert (ctx, shuffled, type_to_sse_type (ins->inst_c1));
7562 } else {
7563 g_assert_not_reached ();
7564 }
7565 break;
7566 }
7567
7568 case OP_SSE_UNPACKHI: {
7569 if (ins->inst_c1 == MONO_TYPE_R8 || ins->inst_c1 == MONO_TYPE_I8 || ins->inst_c1 == MONO_TYPE_U8) {
7570 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_2_i32 (1, 3), "");
7571 } else if (ins->inst_c1 == MONO_TYPE_R4 || ins->inst_c1 == MONO_TYPE_I4 || ins->inst_c1 == MONO_TYPE_U4) {
7572 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_4_i32 (2, 6, 3, 7), "");
7573 } else if (ins->inst_c1 == MONO_TYPE_I2 || ins->inst_c1 == MONO_TYPE_U2) {
7574 const int mask_values [] = { 4, 12, 5, 13, 6, 14, 7, 15 };
7575 LLVMValueRef shuffled = LLVMBuildShuffleVector (builder,
7576 convert (ctx, lhs, sse_i2_t),
7577 convert (ctx, rhs, sse_i2_t),
7578 create_const_vector_i32 (mask_values, 8), "");
7579 values [ins->dreg] = convert (ctx, shuffled, type_to_sse_type (ins->inst_c1));
7580 } else if (ins->inst_c1 == MONO_TYPE_I1 || ins->inst_c1 == MONO_TYPE_U1) {
7581 const int mask_values [] = { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 };
7582 LLVMValueRef shuffled = LLVMBuildShuffleVector (builder,
7583 convert (ctx, lhs, sse_i1_t),
7584 convert (ctx, rhs, sse_i1_t),
7585 create_const_vector_i32 (mask_values, 16), "");
7586 values [ins->dreg] = convert (ctx, shuffled, type_to_sse_type (ins->inst_c1));
7587 } else {
7588 g_assert_not_reached ();
7589 }
7590 break;
7591 }
7592
7593 case OP_SSE_LOADU: {
7594 LLVMValueRef dst_ptr = convert (ctx, lhs, LLVMPointerType (primitive_type_to_llvm_type (inst_c1_type (ins)), 0));
7595 LLVMValueRef dst_vec = LLVMBuildBitCast (builder, dst_ptr, LLVMPointerType (type_to_sse_type (ins->inst_c1), 0), "");
7596 values [ins->dreg] = mono_llvm_build_aligned_load (builder, dst_vec, "", FALSE, ins->inst_c0); // inst_c0 is alignment
7597 break;
7598 }
7599 case OP_SSE_MOVSS: {
7600 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (LLVMFloatType (), 0));
7601 LLVMValueRef val = mono_llvm_build_load (builder, addr, "", FALSE);
7602 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (type_to_sse_type (ins->inst_c1)), val, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7603 break;
7604 }
7605 case OP_SSE_MOVSS_STORE: {
7606 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (LLVMFloatType (), 0));
7607 LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7608 mono_llvm_build_store (builder, val, addr, FALSE, LLVM_BARRIER_NONE);
7609 break;
7610 }
7611 case OP_SSE2_MOVD:
7612 case OP_SSE2_MOVQ:
7613 case OP_SSE2_MOVUPD: {
7614 LLVMTypeRef rty = NULL;
7615 switch (ins->opcode) {
7616 case OP_SSE2_MOVD: rty = sse_i4_t; break;
7617 case OP_SSE2_MOVQ: rty = sse_i8_t; break;
7618 case OP_SSE2_MOVUPD: rty = sse_r8_t; break;
7619 }
7620 LLVMTypeRef srcty = LLVMGetElementType (rty);
7621 LLVMValueRef zero = LLVMConstNull (rty);
7622 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (srcty, 0));
7623 LLVMValueRef val = mono_llvm_build_aligned_load (builder, addr, "", FALSE, 1);
7624 values [ins->dreg] = LLVMBuildInsertElement (builder, zero, val, const_int32 (0), dname);
7625 break;
7626 }
7627
7628 case OP_SSE_MOVLPS_LOAD:
7629 case OP_SSE_MOVHPS_LOAD: {
7630 LLVMTypeRef t = LLVMFloatType ();
7631 int size = 4;
7632 gboolean high = ins->opcode == OP_SSE_MOVHPS_LOAD;
7633 /* Load two floats from rhs and store them in the low/high part of lhs */
7634 LLVMValueRef addr = rhs;
7635 LLVMValueRef addr1 = convert (ctx, addr, LLVMPointerType (t, 0));
7636 LLVMValueRef addr2 = convert (ctx, LLVMBuildAdd (builder, convert (ctx, addr, IntPtrType ()), convert (ctx, LLVMConstInt (LLVMInt32Type (), size, FALSE), IntPtrType ()), ""), LLVMPointerType (t, 0));
7637 LLVMValueRef val1 = mono_llvm_build_load (builder, addr1, "", FALSE);
7638 LLVMValueRef val2 = mono_llvm_build_load (builder, addr2, "", FALSE);
7639 int index1, index2;
7640
7641 index1 = high ? 2: 0;
7642 index2 = high ? 3 : 1;
7643 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMBuildInsertElement (builder, lhs, val1, LLVMConstInt (LLVMInt32Type (), index1, FALSE), ""), val2, LLVMConstInt (LLVMInt32Type (), index2, FALSE), "");
7644 break;
7645 }
7646
7647 case OP_SSE2_MOVLPD_LOAD:
7648 case OP_SSE2_MOVHPD_LOAD: {
7649 LLVMTypeRef t = LLVMDoubleType ();
7650 LLVMValueRef addr = convert (ctx, rhs, LLVMPointerType (t, 0));
7651 LLVMValueRef val = mono_llvm_build_load (builder, addr, "", FALSE);
7652 int index = ins->opcode == OP_SSE2_MOVHPD_LOAD ? 1 : 0;
7653 values [ins->dreg] = LLVMBuildInsertElement (builder, lhs, val, const_int32 (index), "");
7654 break;
7655 }
7656
7657 case OP_SSE_MOVLPS_STORE:
7658 case OP_SSE_MOVHPS_STORE: {
7659 /* Store two floats from the low/hight part of rhs into lhs */
7660 LLVMValueRef addr = lhs;
7661 LLVMValueRef addr1 = convert (ctx, addr, LLVMPointerType (LLVMFloatType (), 0));
7662 LLVMValueRef addr2 = convert (ctx, LLVMBuildAdd (builder, convert (ctx, addr, IntPtrType ()), convert (ctx, LLVMConstInt (LLVMInt32Type (), 4, FALSE), IntPtrType ()), ""), LLVMPointerType (LLVMFloatType (), 0));
7663 int index1 = ins->opcode == OP_SSE_MOVLPS_STORE ? 0 : 2;
7664 int index2 = ins->opcode == OP_SSE_MOVLPS_STORE ? 1 : 3;
7665 LLVMValueRef val1 = LLVMBuildExtractElement (builder, rhs, LLVMConstInt (LLVMInt32Type (), index1, FALSE), "");
7666 LLVMValueRef val2 = LLVMBuildExtractElement (builder, rhs, LLVMConstInt (LLVMInt32Type (), index2, FALSE), "");
7667 mono_llvm_build_store (builder, val1, addr1, FALSE, LLVM_BARRIER_NONE);
7668 mono_llvm_build_store (builder, val2, addr2, FALSE, LLVM_BARRIER_NONE);
7669 break;
7670 }
7671
7672 case OP_SSE2_MOVLPD_STORE:
7673 case OP_SSE2_MOVHPD_STORE: {
7674 LLVMTypeRef t = LLVMDoubleType ();
7675 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (t, 0));
7676 int index = ins->opcode == OP_SSE2_MOVHPD_STORE ? 1 : 0;
7677 LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, const_int32 (index), "");
7678 mono_llvm_build_store (builder, val, addr, FALSE, LLVM_BARRIER_NONE);
7679 break;
7680 }
7681
7682 case OP_SSE_STORE: {
7683 LLVMValueRef dst_vec = convert (ctx, lhs, LLVMPointerType (LLVMTypeOf (rhs), 0));
7684 mono_llvm_build_aligned_store (builder, rhs, dst_vec, FALSE, ins->inst_c0);
7685 break;
7686 }
7687
7688 case OP_SSE_STORES: {
7689 LLVMValueRef first_elem = LLVMBuildExtractElement (builder, rhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7690 LLVMValueRef dst = convert (ctx, lhs, LLVMPointerType (LLVMTypeOf (first_elem), 0));
7691 mono_llvm_build_aligned_store (builder, first_elem, dst, FALSE, 1);
7692 break;
7693 }
7694 case OP_SSE_MOVNTPS: {
7695 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (LLVMTypeOf (rhs), 0));
7696 LLVMValueRef store = mono_llvm_build_aligned_store (builder, rhs, addr, FALSE, ins->inst_c0);
7697 set_nontemporal_flag (store);
7698 break;
7699 }
7700 case OP_SSE_PREFETCHT0: {
7701 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (LLVMInt8Type (), 0));
7702 LLVMValueRef args [] = { addr, const_int32 (0), const_int32 (3), const_int32 (1) };
7703 call_intrins (ctx, INTRINS_PREFETCH, args, "");
7704 break;
7705 }
7706 case OP_SSE_PREFETCHT1: {
7707 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (LLVMInt8Type (), 0));
7708 LLVMValueRef args [] = { addr, const_int32 (0), const_int32 (2), const_int32 (1) };
7709 call_intrins (ctx, INTRINS_PREFETCH, args, "");
7710 break;
7711 }
7712 case OP_SSE_PREFETCHT2: {
7713 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (LLVMInt8Type (), 0));
7714 LLVMValueRef args [] = { addr, const_int32 (0), const_int32 (1), const_int32 (1) };
7715 call_intrins (ctx, INTRINS_PREFETCH, args, "");
7716 break;
7717 }
7718 case OP_SSE_PREFETCHNTA: {
7719 LLVMValueRef addr = convert (ctx, lhs, LLVMPointerType (LLVMInt8Type (), 0));
7720 LLVMValueRef args [] = { addr, const_int32 (0), const_int32 (0), const_int32 (1) };
7721 call_intrins (ctx, INTRINS_PREFETCH, args, "");
7722 break;
7723 }
7724
7725 case OP_SSE_SHUFFLE: {
7726 LLVMValueRef shuffle_vec = create_const_vector_4_i32 (
7727 ((ins->inst_c0 >> 0) & 0x3) + 0, // take two elements from lhs
7728 ((ins->inst_c0 >> 2) & 0x3) + 0,
7729 ((ins->inst_c0 >> 4) & 0x3) + 4, // and two from rhs
7730 ((ins->inst_c0 >> 6) & 0x3) + 4);
7731 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, shuffle_vec, "");
7732 break;
7733 }
7734
7735 case OP_SSE2_SHUFFLE: {
7736 LLVMValueRef right_vec;
7737 LLVMValueRef shuffle_vec;
7738 if (ins->inst_c1 == MONO_TYPE_R8) {
7739 right_vec = rhs;
7740 shuffle_vec = create_const_vector_2_i32 (
7741 ((ins->inst_c0 >> 0) & 0x1) + 0,
7742 ((ins->inst_c0 >> 1) & 0x1) + 2);
7743 } else {
7744 right_vec = LLVMGetUndef (LLVMVectorType (LLVMInt32Type (), 4));
7745 shuffle_vec = create_const_vector_4_i32 (
7746 (ins->inst_c0 >> 0) & 0x3,
7747 (ins->inst_c0 >> 2) & 0x3,
7748 (ins->inst_c0 >> 4) & 0x3,
7749 (ins->inst_c0 >> 6) & 0x3);
7750 }
7751 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, right_vec, shuffle_vec, "");
7752 break;
7753 }
7754
7755 case OP_SSE_OR: {
7756 LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t);
7757 LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t);
7758 LLVMValueRef vec_and = LLVMBuildOr (builder, vec_lhs_i64, vec_rhs_i64, "");
7759 values [ins->dreg] = LLVMBuildBitCast (builder, vec_and, type_to_sse_type (ins->inst_c1), "");
7760 break;
7761 }
7762
7763 case OP_SSE_XOR: {
7764 LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t);
7765 LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t);
7766 LLVMValueRef vec_and = LLVMBuildXor (builder, vec_lhs_i64, vec_rhs_i64, "");
7767 values [ins->dreg] = LLVMBuildBitCast (builder, vec_and, type_to_sse_type (ins->inst_c1), "");
7768 break;
7769 }
7770
7771 case OP_SSE_AND: {
7772 LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t);
7773 LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t);
7774 LLVMValueRef vec_and = LLVMBuildAnd (builder, vec_lhs_i64, vec_rhs_i64, "");
7775 values [ins->dreg] = LLVMBuildBitCast (builder, vec_and, type_to_sse_type (ins->inst_c1), "");
7776 break;
7777 }
7778
7779 case OP_SSE_ANDN: {
7780 LLVMValueRef minus_one [2];
7781 minus_one [0] = LLVMConstInt (LLVMInt64Type (), -1, FALSE);
7782 minus_one [1] = LLVMConstInt (LLVMInt64Type (), -1, FALSE);
7783 LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t);
7784 LLVMValueRef vec_xor = LLVMBuildXor (builder, vec_lhs_i64, LLVMConstVector (minus_one, 2), "");
7785 LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t);
7786 LLVMValueRef vec_and = LLVMBuildAnd (builder, vec_rhs_i64, vec_xor, "");
7787 values [ins->dreg] = LLVMBuildBitCast (builder, vec_and, type_to_sse_type (ins->inst_c1), "");
7788 break;
7789 }
7790
7791 case OP_SSE_ADDSS:
7792 case OP_SSE_SUBSS:
7793 case OP_SSE_DIVSS:
7794 case OP_SSE_MULSS:
7795 case OP_SSE2_ADDSD:
7796 case OP_SSE2_SUBSD:
7797 case OP_SSE2_DIVSD:
7798 case OP_SSE2_MULSD: {
7799 LLVMValueRef v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7800 LLVMValueRef v2 = LLVMBuildExtractElement (builder, rhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7801
7802 LLVMValueRef v = NULL;
7803 switch (ins->opcode) {
7804 case OP_SSE_ADDSS:
7805 case OP_SSE2_ADDSD:
7806 v = LLVMBuildFAdd (builder, v1, v2, "");
7807 break;
7808 case OP_SSE_SUBSS:
7809 case OP_SSE2_SUBSD:
7810 v = LLVMBuildFSub (builder, v1, v2, "");
7811 break;
7812 case OP_SSE_DIVSS:
7813 case OP_SSE2_DIVSD:
7814 v = LLVMBuildFDiv (builder, v1, v2, "");
7815 break;
7816 case OP_SSE_MULSS:
7817 case OP_SSE2_MULSD:
7818 v = LLVMBuildFMul (builder, v1, v2, "");
7819 break;
7820 default:
7821 g_assert_not_reached ();
7822 }
7823 values [ins->dreg] = LLVMBuildInsertElement (builder, lhs, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7824 break;
7825 }
7826
7827 case OP_SSE_CMPSS:
7828 case OP_SSE2_CMPSD: {
7829 int imm = -1;
7830 switch (ins->inst_c0) {
7831 case CMP_EQ: imm = 0; break;
7832 case CMP_GT: imm = 6; break;
7833 case CMP_GE: imm = 5; break;
7834 case CMP_LT: imm = 1; break;
7835 case CMP_LE: imm = 2; break;
7836 case CMP_NE: imm = 4; break;
7837 case CMP_ORD: imm = 7; break;
7838 case CMP_UNORD: imm = 3; break;
7839 default: g_assert_not_reached (); break;
7840 }
7841 LLVMValueRef cmp = LLVMConstInt (LLVMInt8Type (), imm, FALSE);
7842 LLVMValueRef args [] = { lhs, rhs, cmp };
7843 switch (ins->opcode) {
7844 case OP_SSE_CMPSS:
7845 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_CMPSS, args, "");
7846 break;
7847 case OP_SSE2_CMPSD:
7848 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_CMPSD, args, "");
7849 break;
7850 default:
7851 g_assert_not_reached ();
7852 break;
7853 }
7854 break;
7855 }
7856 case OP_SSE_COMISS: {
7857 LLVMValueRef args [] = { lhs, rhs };
7858 IntrinsicId id = (IntrinsicId)0;
7859 switch (ins->inst_c0) {
7860 case CMP_EQ: id = INTRINS_SSE_COMIEQ_SS; break;
7861 case CMP_GT: id = INTRINS_SSE_COMIGT_SS; break;
7862 case CMP_GE: id = INTRINS_SSE_COMIGE_SS; break;
7863 case CMP_LT: id = INTRINS_SSE_COMILT_SS; break;
7864 case CMP_LE: id = INTRINS_SSE_COMILE_SS; break;
7865 case CMP_NE: id = INTRINS_SSE_COMINEQ_SS; break;
7866 default: g_assert_not_reached (); break;
7867 }
7868 values [ins->dreg] = call_intrins (ctx, id, args, "");
7869 break;
7870 }
7871 case OP_SSE_UCOMISS: {
7872 LLVMValueRef args [] = { lhs, rhs };
7873 IntrinsicId id = (IntrinsicId)0;
7874 switch (ins->inst_c0) {
7875 case CMP_EQ: id = INTRINS_SSE_UCOMIEQ_SS; break;
7876 case CMP_GT: id = INTRINS_SSE_UCOMIGT_SS; break;
7877 case CMP_GE: id = INTRINS_SSE_UCOMIGE_SS; break;
7878 case CMP_LT: id = INTRINS_SSE_UCOMILT_SS; break;
7879 case CMP_LE: id = INTRINS_SSE_UCOMILE_SS; break;
7880 case CMP_NE: id = INTRINS_SSE_UCOMINEQ_SS; break;
7881 default: g_assert_not_reached (); break;
7882 }
7883 values [ins->dreg] = call_intrins (ctx, id, args, "");
7884 break;
7885 }
7886 case OP_SSE2_COMISD: {
7887 LLVMValueRef args [] = { lhs, rhs };
7888 IntrinsicId id = (IntrinsicId)0;
7889 switch (ins->inst_c0) {
7890 case CMP_EQ: id = INTRINS_SSE_COMIEQ_SD; break;
7891 case CMP_GT: id = INTRINS_SSE_COMIGT_SD; break;
7892 case CMP_GE: id = INTRINS_SSE_COMIGE_SD; break;
7893 case CMP_LT: id = INTRINS_SSE_COMILT_SD; break;
7894 case CMP_LE: id = INTRINS_SSE_COMILE_SD; break;
7895 case CMP_NE: id = INTRINS_SSE_COMINEQ_SD; break;
7896 default: g_assert_not_reached (); break;
7897 }
7898 values [ins->dreg] = call_intrins (ctx, id, args, "");
7899 break;
7900 }
7901 case OP_SSE2_UCOMISD: {
7902 LLVMValueRef args [] = { lhs, rhs };
7903 IntrinsicId id = (IntrinsicId)0;
7904 switch (ins->inst_c0) {
7905 case CMP_EQ: id = INTRINS_SSE_UCOMIEQ_SD; break;
7906 case CMP_GT: id = INTRINS_SSE_UCOMIGT_SD; break;
7907 case CMP_GE: id = INTRINS_SSE_UCOMIGE_SD; break;
7908 case CMP_LT: id = INTRINS_SSE_UCOMILT_SD; break;
7909 case CMP_LE: id = INTRINS_SSE_UCOMILE_SD; break;
7910 case CMP_NE: id = INTRINS_SSE_UCOMINEQ_SD; break;
7911 default: g_assert_not_reached (); break;
7912 }
7913 values [ins->dreg] = call_intrins (ctx, id, args, "");
7914 break;
7915 }
7916 case OP_SSE_CVTSI2SS:
7917 case OP_SSE_CVTSI2SS64:
7918 case OP_SSE2_CVTSI2SD:
7919 case OP_SSE2_CVTSI2SD64: {
7920 LLVMTypeRef ty = LLVMFloatType ();
7921 switch (ins->opcode) {
7922 case OP_SSE2_CVTSI2SD:
7923 case OP_SSE2_CVTSI2SD64:
7924 ty = LLVMDoubleType ();
7925 break;
7926 }
7927 LLVMValueRef fp = LLVMBuildSIToFP (builder, rhs, ty, "");
7928 values [ins->dreg] = LLVMBuildInsertElement (builder, lhs, fp, const_int32 (0), dname);
7929 break;
7930 }
7931 case OP_SSE2_PMULUDQ: {
7932 #if LLVM_API_VERSION < 700
7933 LLVMValueRef args [] = { lhs, rhs };
7934 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_PMULUDQ, args, dname);
7935 #else
7936 LLVMValueRef i32_max = LLVMConstInt (LLVMInt64Type (), UINT32_MAX, FALSE);
7937 LLVMValueRef maskvals [] = { i32_max, i32_max };
7938 LLVMValueRef mask = LLVMConstVector (maskvals, 2);
7939 LLVMValueRef l = LLVMBuildAnd (builder, convert (ctx, lhs, sse_i8_t), mask, "");
7940 LLVMValueRef r = LLVMBuildAnd (builder, convert (ctx, rhs, sse_i8_t), mask, "");
7941 values [ins->dreg] = LLVMBuildNUWMul (builder, l, r, dname);
7942 #endif
7943 break;
7944 }
7945 case OP_SSE_SQRTSS:
7946 case OP_SSE2_SQRTSD: {
7947 #if LLVM_API_VERSION < 700
7948 LLVMValueRef result = call_intrins (ctx, simd_ins_to_intrins (ins->opcode), &rhs, dname);
7949 const int maskf32[] = { 0, 5, 6, 7 };
7950 const int maskf64[] = { 0, 1 };
7951 const int *mask = NULL;
7952 int mask_len = 0;
7953 switch (ins->opcode) {
7954 case OP_SSE_SQRTSS: mask = maskf32; mask_len = 4; break;
7955 case OP_SSE2_SQRTSD: mask = maskf64; mask_len = 2; break;
7956 default: g_assert_not_reached (); break;
7957 }
7958 LLVMValueRef shufmask = create_const_vector_i32 (mask, mask_len);
7959 values [ins->dreg] = LLVMBuildShuffleVector (builder, result, lhs, shufmask, "");
7960 #else
7961 LLVMValueRef upper = values [ins->sreg1];
7962 LLVMValueRef lower = values [ins->sreg2];
7963 LLVMValueRef scalar = LLVMBuildExtractElement (builder, lower, const_int32 (0), "");
7964 LLVMValueRef result = call_intrins (ctx, simd_ins_to_intrins (ins->opcode), &scalar, dname);
7965 values [ins->dreg] = LLVMBuildInsertElement (builder, upper, result, const_int32 (0), "");
7966 #endif
7967 break;
7968 }
7969 case OP_SSE_RCPSS:
7970 case OP_SSE_RSQRTSS: {
7971 IntrinsicId id = (IntrinsicId)0;
7972 switch (ins->opcode) {
7973 case OP_SSE_RCPSS: id = INTRINS_SSE_RCP_SS; break;
7974 case OP_SSE_RSQRTSS: id = INTRINS_SSE_RSQRT_SS; break;
7975 default: g_assert_not_reached (); break;
7976 };
7977 LLVMValueRef result = call_intrins (ctx, id, &rhs, dname);
7978 const int mask[] = { 0, 5, 6, 7 };
7979 LLVMValueRef shufmask = create_const_vector_i32 (mask, 4);
7980 values [ins->dreg] = LLVMBuildShuffleVector (builder, result, lhs, shufmask, "");
7981 break;
7982 }
7983 case OP_XOP: {
7984 IntrinsicId id = (IntrinsicId)0;
7985 switch (ins->inst_c0) {
7986 case SIMD_OP_SSE_LFENCE: id = INTRINS_SSE_LFENCE; break;
7987 case SIMD_OP_SSE_SFENCE: id = INTRINS_SSE_SFENCE; break;
7988 case SIMD_OP_SSE_MFENCE: id = INTRINS_SSE_MFENCE; break;
7989 default: g_assert_not_reached (); break;
7990 }
7991 call_intrins (ctx, id, NULL, "");
7992 break;
7993 }
7994 case OP_XOP_X_I:
7995 case OP_XOP_X_X: {
7996 IntrinsicId id = (IntrinsicId)0;
7997 switch (ins->inst_c0) {
7998 case SIMD_OP_SSE_SQRTPS: id = INTRINS_SSE_SQRT_PS; break;
7999 case SIMD_OP_SSE_RCPPS: id = INTRINS_SSE_RCP_PS; break;
8000 case SIMD_OP_SSE_RSQRTPS: id = INTRINS_SSE_RSQRT_PS; break;
8001 case SIMD_OP_SSE_SQRTPD: id = INTRINS_SSE_SQRT_PD; break;
8002 case SIMD_OP_SSE_LDDQU: id = INTRINS_SSE_LDU_DQ; break;
8003 default: g_assert_not_reached (); break;
8004 }
8005 values [ins->dreg] = call_intrins (ctx, id, &lhs, "");
8006 break;
8007 }
8008 case OP_XOP_I4_X:
8009 case OP_XOP_I8_X: {
8010 IntrinsicId id = (IntrinsicId)0;
8011 switch (ins->inst_c0) {
8012 case SIMD_OP_SSE_CVTSS2SI: id = INTRINS_SSE_CVTSS2SI; break;
8013 case SIMD_OP_SSE_CVTTSS2SI: id = INTRINS_SSE_CVTTSS2SI; break;
8014 case SIMD_OP_SSE_CVTSS2SI64: id = INTRINS_SSE_CVTSS2SI64; break;
8015 case SIMD_OP_SSE_CVTTSS2SI64: id = INTRINS_SSE_CVTTSS2SI64; break;
8016 case SIMD_OP_SSE_CVTSD2SI: id = INTRINS_SSE_CVTSD2SI; break;
8017 case SIMD_OP_SSE_CVTTSD2SI: id = INTRINS_SSE_CVTTSD2SI; break;
8018 case SIMD_OP_SSE_CVTSD2SI64: id = INTRINS_SSE_CVTSD2SI64; break;
8019 case SIMD_OP_SSE_CVTTSD2SI64: id = INTRINS_SSE_CVTTSD2SI64; break;
8020 default: g_assert_not_reached (); break;
8021 }
8022 values [ins->dreg] = call_intrins (ctx, id, &lhs, "");
8023 break;
8024 }
8025 case OP_XOP_X_X_X:
8026 case OP_XOP_X_X_I4:
8027 case OP_XOP_X_X_I8: {
8028 LLVMValueRef args [] = { lhs, rhs };
8029 IntrinsicId id = (IntrinsicId)0;
8030 switch (ins->inst_c0) {
8031 case SIMD_OP_SSE_CVTSD2SS: id = INTRINS_SSE_CVTSD2SS; break;
8032 case SIMD_OP_SSE_MAXPS: id = INTRINS_SSE_MAXPS; break;
8033 case SIMD_OP_SSE_MAXSS: id = INTRINS_SSE_MAXSS; break;
8034 case SIMD_OP_SSE_MINPS: id = INTRINS_SSE_MINPS; break;
8035 case SIMD_OP_SSE_MINSS: id = INTRINS_SSE_MINSS; break;
8036 case SIMD_OP_SSE_MAXPD: id = INTRINS_SSE_MAXPD; break;
8037 case SIMD_OP_SSE_MAXSD: id = INTRINS_SSE_MAXSD; break;
8038 case SIMD_OP_SSE_MINPD: id = INTRINS_SSE_MINPD; break;
8039 case SIMD_OP_SSE_MINSD: id = INTRINS_SSE_MINSD; break;
8040 case SIMD_OP_SSE_PMADDWD: id = INTRINS_SSE_PMADDWD; break;
8041 case SIMD_OP_SSE_PMULHW: id = INTRINS_SSE_PMULHW; break;
8042 case SIMD_OP_SSE_PMULHUW: id = INTRINS_SSE_PMULHUW; break;
8043 case SIMD_OP_SSE_PACKSSWB: id = INTRINS_SSE_PACKSSWB; break;
8044 case SIMD_OP_SSE_PACKSSDW: id = INTRINS_SSE_PACKSSDW; break;
8045 case SIMD_OP_SSE_PSRLW_IMM: id = INTRINS_SSE_PSRLI_W; break;
8046 case SIMD_OP_SSE_PSRLD_IMM: id = INTRINS_SSE_PSRLI_D; break;
8047 case SIMD_OP_SSE_PSRLQ_IMM: id = INTRINS_SSE_PSRLI_Q; break;
8048 case SIMD_OP_SSE_PSRLW: id = INTRINS_SSE_PSRL_W; break;
8049 case SIMD_OP_SSE_PSRLD: id = INTRINS_SSE_PSRL_D; break;
8050 case SIMD_OP_SSE_PSRLQ: id = INTRINS_SSE_PSRL_Q; break;
8051 case SIMD_OP_SSE_PSLLW_IMM: id = INTRINS_SSE_PSLLI_W; break;
8052 case SIMD_OP_SSE_PSLLD_IMM: id = INTRINS_SSE_PSLLI_D; break;
8053 case SIMD_OP_SSE_PSLLQ_IMM: id = INTRINS_SSE_PSLLI_Q; break;
8054 case SIMD_OP_SSE_PSLLW: id = INTRINS_SSE_PSLL_W; break;
8055 case SIMD_OP_SSE_PSLLD: id = INTRINS_SSE_PSLL_D; break;
8056 case SIMD_OP_SSE_PSLLQ: id = INTRINS_SSE_PSLL_Q; break;
8057 case SIMD_OP_SSE_PSRAW_IMM: id = INTRINS_SSE_PSRAI_W; break;
8058 case SIMD_OP_SSE_PSRAD_IMM: id = INTRINS_SSE_PSRAI_D; break;
8059 case SIMD_OP_SSE_PSRAW: id = INTRINS_SSE_PSRA_W; break;
8060 case SIMD_OP_SSE_PSRAD: id = INTRINS_SSE_PSRA_D; break;
8061 case SIMD_OP_SSE_PSADBW: id = INTRINS_SSE_PSADBW; break;
8062 case SIMD_OP_SSE_ADDSUBPS: id = INTRINS_SSE_ADDSUBPS; break;
8063 case SIMD_OP_SSE_ADDSUBPD: id = INTRINS_SSE_ADDSUBPD; break;
8064 case SIMD_OP_SSE_HADDPS: id = INTRINS_SSE_HADDPS; break;
8065 case SIMD_OP_SSE_HADDPD: id = INTRINS_SSE_HADDPD; break;
8066 case SIMD_OP_SSE_PHADDW: id = INTRINS_SSE_PHADDW; break;
8067 case SIMD_OP_SSE_PHADDD: id = INTRINS_SSE_PHADDD; break;
8068 case SIMD_OP_SSE_PHSUBW: id = INTRINS_SSE_PHSUBW; break;
8069 case SIMD_OP_SSE_PHSUBD: id = INTRINS_SSE_PHSUBD; break;
8070 case SIMD_OP_SSE_HSUBPS: id = INTRINS_SSE_HSUBPS; break;
8071 case SIMD_OP_SSE_HSUBPD: id = INTRINS_SSE_HSUBPD; break;
8072 case SIMD_OP_SSE_PHADDSW: id = INTRINS_SSE_PHADDSW; break;
8073 case SIMD_OP_SSE_PHSUBSW: id = INTRINS_SSE_PHSUBSW; break;
8074 case SIMD_OP_SSE_PSIGNB: id = INTRINS_SSE_PSIGNB; break;
8075 case SIMD_OP_SSE_PSIGNW: id = INTRINS_SSE_PSIGNW; break;
8076 case SIMD_OP_SSE_PSIGND: id = INTRINS_SSE_PSIGND; break;
8077 case SIMD_OP_SSE_PMADDUBSW: id = INTRINS_SSE_PMADDUBSW; break;
8078 case SIMD_OP_SSE_PMULHRSW: id = INTRINS_SSE_PMULHRSW; break;
8079 default: g_assert_not_reached (); break;
8080 }
8081 values [ins->dreg] = call_intrins (ctx, id, args, "");
8082 break;
8083 }
8084
8085 case OP_SSE2_MASKMOVDQU: {
8086 LLVMTypeRef i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
8087 LLVMValueRef dstaddr = convert (ctx, values [ins->sreg3], i8ptr);
8088 LLVMValueRef src = convert (ctx, lhs, sse_i1_t);
8089 LLVMValueRef mask = convert (ctx, rhs, sse_i1_t);
8090 LLVMValueRef args[] = { src, mask, dstaddr };
8091 call_intrins (ctx, INTRINS_SSE_MASKMOVDQU, args, "");
8092 break;
8093 }
8094
8095 case OP_PADDB_SAT:
8096 case OP_PADDW_SAT:
8097 case OP_PSUBB_SAT:
8098 case OP_PSUBW_SAT:
8099 case OP_PADDB_SAT_UN:
8100 case OP_PADDW_SAT_UN:
8101 case OP_PSUBB_SAT_UN:
8102 case OP_PSUBW_SAT_UN:
8103 case OP_SSE2_ADDS:
8104 case OP_SSE2_SUBS: {
8105 IntrinsicId id = (IntrinsicId)0;
8106 int type = 0;
8107 gboolean is_add = TRUE;
8108 switch (ins->opcode) {
8109 case OP_PADDB_SAT: type = MONO_TYPE_I1; break;
8110 case OP_PADDW_SAT: type = MONO_TYPE_I2; break;
8111 case OP_PSUBB_SAT: type = MONO_TYPE_I1; is_add = FALSE; break;
8112 case OP_PSUBW_SAT: type = MONO_TYPE_I2; is_add = FALSE; break;
8113 case OP_PADDB_SAT_UN: type = MONO_TYPE_U1; break;
8114 case OP_PADDW_SAT_UN: type = MONO_TYPE_U2; break;
8115 case OP_PSUBB_SAT_UN: type = MONO_TYPE_U1; is_add = FALSE; break;
8116 case OP_PSUBW_SAT_UN: type = MONO_TYPE_U2; is_add = FALSE; break;
8117 case OP_SSE2_ADDS: type = ins->inst_c1; break;
8118 case OP_SSE2_SUBS: type = ins->inst_c1; is_add = FALSE; break;
8119 default: g_assert_not_reached ();
8120 }
8121 if (is_add) {
8122 switch (type) {
8123 case MONO_TYPE_I1: id = INTRINS_SSE_SADD_SATI8; break;
8124 case MONO_TYPE_U1: id = INTRINS_SSE_UADD_SATI8; break;
8125 case MONO_TYPE_I2: id = INTRINS_SSE_SADD_SATI16; break;
8126 case MONO_TYPE_U2: id = INTRINS_SSE_UADD_SATI16; break;
8127 default: g_assert_not_reached (); break;
8128 }
8129 } else {
8130 switch (type) {
8131 case MONO_TYPE_I1: id = INTRINS_SSE_SSUB_SATI8; break;
8132 case MONO_TYPE_U1: id = INTRINS_SSE_USUB_SATI8; break;
8133 case MONO_TYPE_I2: id = INTRINS_SSE_SSUB_SATI16; break;
8134 case MONO_TYPE_U2: id = INTRINS_SSE_USUB_SATI16; break;
8135 default: g_assert_not_reached (); break;
8136 }
8137 }
8138 LLVMTypeRef vecty = type_to_sse_type (type);
8139 LLVMValueRef args [] = { convert (ctx, lhs, vecty), convert (ctx, rhs, vecty) };
8140 LLVMValueRef result = call_intrins (ctx, id, args, dname);
8141 values [ins->dreg] = convert (ctx, result, vecty);
8142 break;
8143 }
8144
8145 case OP_SSE2_PACKUS: {
8146 LLVMValueRef args [2];
8147 args [0] = convert (ctx, lhs, sse_i2_t);
8148 args [1] = convert (ctx, rhs, sse_i2_t);
8149 values [ins->dreg] = convert (ctx,
8150 call_intrins (ctx, INTRINS_SSE_PACKUSWB, args, dname),
8151 type_to_sse_type (ins->inst_c1));
8152 break;
8153 }
8154
8155 case OP_SSE2_SRLI: {
8156 LLVMValueRef args [] = { lhs, rhs };
8157 values [ins->dreg] = convert (ctx,
8158 call_intrins (ctx, INTRINS_SSE_PSRLI_W, args, dname),
8159 type_to_sse_type (ins->inst_c1));
8160 break;
8161 }
8162
8163 case OP_SSE2_PSLLDQ:
8164 case OP_SSE2_PSRLDQ: {
8165 LLVMBasicBlockRef bbs [16 + 1];
8166 LLVMValueRef switch_ins;
8167 LLVMValueRef value = lhs;
8168 LLVMValueRef index = rhs;
8169 LLVMValueRef phi_values [16 + 1];
8170 LLVMTypeRef t = sse_i1_t;
8171 int nelems = 16;
8172 int i;
8173 gboolean shift_right = (ins->opcode == OP_SSE2_PSRLDQ);
8174
8175 value = convert (ctx, value, t);
8176
8177 // No corresponding LLVM intrinsics
8178 // FIXME: Optimize const count
8179 for (i = 0; i < nelems; ++i)
8180 bbs [i] = gen_bb (ctx, "PSLLDQ_CASE_BB");
8181 bbs [nelems] = gen_bb (ctx, "PSLLDQ_DEF_BB");
8182 cbb = gen_bb (ctx, "PSLLDQ_COND_BB");
8183
8184 switch_ins = LLVMBuildSwitch (builder, index, bbs [nelems], 0);
8185 for (i = 0; i < nelems; ++i) {
8186 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
8187 LLVMPositionBuilderAtEnd (builder, bbs [i]);
8188
8189 int mask_values [16];
8190 // Implement shift using a shuffle
8191 if (shift_right) {
8192 for (int j = 0; j < nelems - i; ++j)
8193 mask_values [j] = i + j;
8194 for (int j = nelems -i ; j < nelems; ++j)
8195 mask_values [j] = nelems;
8196 } else {
8197 for (int j = 0; j < i; ++j)
8198 mask_values [j] = nelems;
8199 for (int j = 0; j < nelems - i; ++j)
8200 mask_values [j + i] = j;
8201 }
8202 phi_values [i] = LLVMBuildShuffleVector (builder, value, LLVMGetUndef (t), create_const_vector_i32 (mask_values, nelems), "");
8203 LLVMBuildBr (builder, cbb);
8204 }
8205 /* Default case */
8206 LLVMPositionBuilderAtEnd (builder, bbs [nelems]);
8207 phi_values [nelems] = LLVMConstNull (t);
8208 LLVMBuildBr (builder, cbb);
8209
8210 LLVMPositionBuilderAtEnd (builder, cbb);
8211 values [ins->dreg] = LLVMBuildPhi (builder, LLVMTypeOf (phi_values [0]), "");
8212 LLVMAddIncoming (values [ins->dreg], phi_values, bbs, nelems + 1);
8213 values [ins->dreg] = convert (ctx, values [ins->dreg], type_to_sse_type (ins->inst_c1));
8214
8215 ctx->bblocks [bb->block_num].end_bblock = cbb;
8216 break;
8217 }
8218
8219 case OP_SSE2_PSRAW_IMM:
8220 case OP_SSE2_PSRAD_IMM:
8221 case OP_SSE2_PSRLW_IMM:
8222 case OP_SSE2_PSRLD_IMM:
8223 case OP_SSE2_PSRLQ_IMM: {
8224 LLVMValueRef value = lhs;
8225 LLVMValueRef index = rhs;
8226 IntrinsicId id;
8227
8228 // FIXME: Optimize const index case
8229
8230 /* Use the non-immediate version */
8231 switch (ins->opcode) {
8232 case OP_SSE2_PSRAW_IMM: id = INTRINS_SSE_PSRA_W; break;
8233 case OP_SSE2_PSRAD_IMM: id = INTRINS_SSE_PSRA_D; break;
8234 case OP_SSE2_PSRLW_IMM: id = INTRINS_SSE_PSRL_W; break;
8235 case OP_SSE2_PSRLD_IMM: id = INTRINS_SSE_PSRL_D; break;
8236 case OP_SSE2_PSRLQ_IMM: id = INTRINS_SSE_PSRL_Q; break;
8237 default: g_assert_not_reached (); break;
8238 }
8239
8240 LLVMTypeRef t = LLVMTypeOf (value);
8241 LLVMValueRef index_vect = LLVMBuildInsertElement (builder, LLVMConstNull (t), convert (ctx, index, LLVMGetElementType (t)), const_int32 (0), "");
8242 LLVMValueRef args [] = { value, index_vect };
8243 values [ins->dreg] = call_intrins (ctx, id, args, "");
8244 break;
8245 }
8246
8247 case OP_SSE2_SHUFPD:
8248 case OP_SSE2_PSHUFD:
8249 case OP_SSE2_PSHUFHW:
8250 case OP_SSE2_PSHUFLW: {
8251 LLVMBasicBlockRef bbs [256 + 1];
8252 LLVMValueRef switch_ins;
8253 LLVMValueRef v1, v2, mask;
8254 LLVMValueRef phi_values [256 + 1];
8255 int ncases;
8256
8257 // FIXME: Optimize constant shuffle mask
8258
8259 if (ins->opcode == OP_SSE2_SHUFPD) {
8260 /* 3 parameter version */
8261 v1 = lhs;
8262 v2 = rhs;
8263 mask = values [ins->sreg3];
8264 ncases = 4;
8265 } else {
8266 /* 2 parameter version */
8267 v1 = v2 = lhs;
8268 mask = rhs;
8269 ncases = 256;
8270 }
8271
8272 for (int i = 0; i < ncases; ++i)
8273 bbs [i] = gen_bb (ctx, "PSHUFHW_CASE_BB");
8274 cbb = gen_bb (ctx, "PSHUFHW_COND_BB");
8275 /* No default case */
8276 switch_ins = LLVMBuildSwitch (builder, mask, bbs [0], 0);
8277 for (int i = 0; i < ncases; ++i) {
8278 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
8279 LLVMPositionBuilderAtEnd (builder, bbs [i]);
8280
8281 /* Convert the x86 shuffle mask to LLVM's */
8282 guint32 imask = i;
8283 int mask_values [8];
8284 int mask_len = 0;
8285 switch (ins->opcode) {
8286 case OP_SSE2_SHUFPD:
8287 /* Bit 0 selects v1[0] or v1[1], bit 1 selects v2[0] or v2[1] */
8288 mask_len = 2;
8289 mask_values [0] = ((imask >> 0) & 1);
8290 mask_values [1] = ((imask >> 1) & 1) + 2;
8291 break;
8292 case OP_SSE2_PSHUFD:
8293 /*
8294 * Each 2 bits in mask selects 1 dword from the the source and copies it to the
8295 * destination.
8296 */
8297 mask_len = 4;
8298 for (int j = 0; j < 4; ++j) {
8299 int windex = (imask >> (j * 2)) & 0x3;
8300 mask_values [j] = windex;
8301 }
8302 break;
8303 case OP_SSE2_PSHUFHW:
8304 /*
8305 * Each 2 bits in mask selects 1 word from the high quadword of the source and copies it to the
8306 * high quadword of the destination.
8307 */
8308 mask_len = 8;
8309 /* The low quadword stays the same */
8310 for (int j = 0; j < 4; ++j)
8311 mask_values [j] = j;
8312 for (int j = 0; j < 4; ++j) {
8313 int windex = (imask >> (j * 2)) & 0x3;
8314 mask_values [j + 4] = 4 + windex;
8315 }
8316 break;
8317 case OP_SSE2_PSHUFLW:
8318 mask_len = 8;
8319 /* The high quadword stays the same */
8320 for (int j = 0; j < 4; ++j)
8321 mask_values [j + 4] = j + 4;
8322 for (int j = 0; j < 4; ++j) {
8323 int windex = (imask >> (j * 2)) & 0x3;
8324 mask_values [j] = windex;
8325 }
8326 break;
8327 default:
8328 g_assert_not_reached ();
8329 break;
8330 }
8331 phi_values [i] = LLVMBuildShuffleVector (builder, v1, v2, create_const_vector_i32 (mask_values, mask_len), "");
8332 LLVMBuildBr (builder, cbb);
8333 }
8334
8335 LLVMPositionBuilderAtEnd (builder, cbb);
8336 values [ins->dreg] = LLVMBuildPhi (builder, LLVMTypeOf (phi_values [0]), "");
8337 LLVMAddIncoming (values [ins->dreg], phi_values, bbs, ncases);
8338 break;
8339 }
8340
8341 case OP_SSE3_MOVDDUP: {
8342 int mask [] = { 0, 0 };
8343 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs,
8344 LLVMGetUndef (LLVMTypeOf (lhs)),
8345 create_const_vector_i32 (mask, 2), "");
8346 break;
8347 }
8348 case OP_SSE3_MOVDDUP_MEM: {
8349 int mask [] = { 0, 0 };
8350 LLVMTypeRef t = type_to_sse_type (ins->inst_c1);
8351 LLVMValueRef value = mono_llvm_build_load (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), "", FALSE);
8352 values [ins->dreg] = LLVMBuildShuffleVector (builder, value, LLVMGetUndef (LLVMTypeOf (value)), create_const_vector_i32 (mask, 2), "");
8353 break;
8354 }
8355 case OP_SSE3_MOVSHDUP: {
8356 int mask [] = { 1, 1, 3, 3 };
8357 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), create_const_vector_i32 (mask, 4), "");
8358 break;
8359 }
8360 case OP_SSE3_MOVSLDUP: {
8361 int mask [] = { 0, 0, 2, 2 };
8362 values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), create_const_vector_i32 (mask, 4), "");
8363 break;
8364 }
8365
8366 case OP_SSSE3_SHUFFLE: {
8367 LLVMValueRef args [] = { lhs, rhs };
8368 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_PSHUFB, args, dname);
8369 break;
8370 }
8371
8372 case OP_SSSE3_ABS: {
8373 // %sub = sub <16 x i8> zeroinitializer, %arg
8374 // %cmp = icmp sgt <16 x i8> %arg, zeroinitializer
8375 // %abs = select <16 x i1> %cmp, <16 x i8> %arg, <16 x i8> %sub
8376 LLVMTypeRef typ = type_to_sse_type (ins->inst_c1);
8377 LLVMValueRef sub = LLVMBuildSub(builder, LLVMConstNull(typ), lhs, "");
8378 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntSGT, lhs, LLVMConstNull(typ), "");
8379 LLVMValueRef abs = LLVMBuildSelect (builder, cmp, lhs, sub, "");
8380 values [ins->dreg] = convert (ctx, abs, typ);
8381 break;
8382 }
8383
8384 case OP_SSSE3_ALIGNR: {
8385 LLVMValueRef mask_values [16];
8386 for (int i = 0; i < 16; i++)
8387 mask_values [i] = LLVMConstInt (LLVMInt8Type (), i + ins->inst_c0, FALSE);
8388 LLVMValueRef shuffled = LLVMBuildShuffleVector (builder,
8389 convert (ctx, rhs, sse_i1_t),
8390 convert (ctx, lhs, sse_i1_t),
8391 LLVMConstVector (mask_values, 16), "");
8392 values [ins->dreg] = convert (ctx, shuffled, type_to_sse_type (ins->inst_c1));
8393 break;
8394 }
8395
8396 case OP_CREATE_SCALAR:
8397 case OP_CREATE_SCALAR_UNSAFE: {
8398 MonoTypeEnum primty = inst_c1_type (ins);
8399 LLVMTypeRef type = type_to_sse_type (primty);
8400 // use undef vector (most likely empty but may contain garbage values) for OP_CREATE_SCALAR_UNSAFE
8401 // and zero one for OP_CREATE_SCALAR
8402 LLVMValueRef vector = (ins->opcode == OP_CREATE_SCALAR) ? LLVMConstNull (type) : LLVMGetUndef (type);
8403 LLVMValueRef insert_pos = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8404 LLVMValueRef val = convert_full (ctx, lhs, primitive_type_to_llvm_type (primty), primitive_type_is_unsigned (primty));
8405 values [ins->dreg] = LLVMBuildInsertElement (builder, vector, val, insert_pos, "");
8406 break;
8407 }
8408
8409 case OP_SSE41_ROUNDSS: {
8410 LLVMValueRef args [3];
8411
8412 args [0] = lhs;
8413 args [1] = lhs;
8414 args [2] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
8415
8416 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_ROUNDSS, args, dname);
8417 break;
8418 }
8419
8420 case OP_SSE41_ROUNDPD: {
8421 LLVMValueRef args [3];
8422
8423 args [0] = lhs;
8424 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
8425
8426 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_ROUNDPD, args, dname);
8427 break;
8428 }
8429
8430 case OP_SSE41_INSERT: {
8431 if (ins->inst_c1 == MONO_TYPE_R4) {
8432 // special case for <float> overload
8433 LLVMValueRef args [3];
8434 args [0] = values [ins->sreg1];
8435 args [1] = values [ins->sreg2];
8436 args [2] = convert (ctx, values [ins->sreg3], LLVMInt8Type ());
8437 values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_INSERTPS, args, dname);
8438 } else {
8439 // other overloads are implemented with `insertelement`
8440 values [ins->dreg] = LLVMBuildInsertElement (builder,
8441 values [ins->sreg1],
8442 convert (ctx, values [ins->sreg2], primitive_type_to_llvm_type (inst_c1_type (ins))),
8443 convert (ctx, values [ins->sreg3], LLVMInt8Type ()), dname);
8444 }
8445 break;
8446 }
8447
8448 case OP_SSE41_PTESTZ: {
8449 LLVMValueRef args [2];
8450 args [0] = convert (ctx, lhs, sse_i8_t);
8451 args [1] = convert (ctx, rhs, sse_i8_t);
8452 LLVMValueRef call = call_intrins (ctx, INTRINS_SSE_PTESTZ, args, dname);
8453 LLVMValueRef cmp_zero = LLVMBuildICmp (builder, LLVMIntNE, call, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
8454 values [ins->dreg] = LLVMBuildZExt (builder, cmp_zero, LLVMInt8Type (), "");
8455 break;
8456 }
8457
8458 case OP_SSE42_CRC32:
8459 case OP_SSE42_CRC64: {
8460 LLVMValueRef args [2];
8461 args [0] = lhs;
8462 args [1] = convert (ctx, rhs, primitive_type_to_llvm_type (ins->inst_c0));
8463 IntrinsicId id;
8464 switch (ins->inst_c0) {
8465 case MONO_TYPE_U1: id = INTRINS_SSE_CRC32_32_8; break;
8466 case MONO_TYPE_U2: id = INTRINS_SSE_CRC32_32_16; break;
8467 case MONO_TYPE_U4: id = INTRINS_SSE_CRC32_32_32; break;
8468 case MONO_TYPE_U8: id = INTRINS_SSE_CRC32_64_64; break;
8469 default: g_assert_not_reached (); break;
8470 }
8471 values [ins->dreg] = call_intrins (ctx, id, args, "");
8472 break;
8473 }
8474 #endif
8475
8476 #ifdef ENABLE_NETCORE
8477 case OP_XCAST: {
8478 LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass);
8479
8480 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, "");
8481 break;
8482 }
8483 case OP_XCOMPARE_FP: {
8484 LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0];
8485 LLVMValueRef cmp = LLVMBuildFCmp (builder, pred, lhs, rhs, "");
8486 int nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
8487 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
8488 if (ins->inst_c1 == MONO_TYPE_R8)
8489 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), nelems), ""), LLVMTypeOf (lhs), "");
8490 else
8491 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), nelems), ""), LLVMTypeOf (lhs), "");
8492 break;
8493 }
8494 case OP_XCOMPARE: {
8495 LLVMIntPredicate pred = cond_to_llvm_cond [ins->inst_c0];
8496 LLVMValueRef cmp = LLVMBuildICmp (builder, pred, lhs, rhs, "");
8497 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
8498 values [ins->dreg] = LLVMBuildSExt (builder, cmp, LLVMTypeOf (lhs), "");
8499 break;
8500 }
8501 case OP_XEQUAL: {
8502 LLVMTypeRef t;
8503 LLVMValueRef cmp, mask [32], shuffle;
8504 int nelems;
8505
8506 #if defined(TARGET_WASM) && LLVM_API_VERSION >= 800
8507 /* The wasm code generator doesn't understand the shuffle/and code sequence below */
8508 LLVMValueRef val;
8509 if (LLVMIsNull (lhs) || LLVMIsNull (rhs)) {
8510 val = LLVMIsNull (lhs) ? rhs : lhs;
8511 nelems = LLVMGetVectorSize (LLVMTypeOf (lhs));
8512
8513 IntrinsicId intrins = (IntrinsicId)0;
8514 switch (nelems) {
8515 case 16:
8516 intrins = INTRINS_WASM_ANYTRUE_V16;
8517 break;
8518 case 8:
8519 intrins = INTRINS_WASM_ANYTRUE_V8;
8520 break;
8521 case 4:
8522 intrins = INTRINS_WASM_ANYTRUE_V4;
8523 break;
8524 case 2:
8525 intrins = INTRINS_WASM_ANYTRUE_V2;
8526 break;
8527 default:
8528 g_assert_not_reached ();
8529 }
8530 /* res = !wasm.anytrue (val) */
8531 values [ins->dreg] = call_intrins (ctx, intrins, &val, "");
8532 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, values [ins->dreg], LLVMConstInt (LLVMInt32Type (), 0, FALSE), ""), LLVMInt32Type (), dname);
8533 break;
8534 }
8535 #endif
8536 LLVMTypeRef srcelemt = LLVMGetElementType (LLVMTypeOf (lhs));
8537
8538 //%c = icmp sgt <16 x i8> %a0, %a1
8539 if (srcelemt == LLVMDoubleType () || srcelemt == LLVMFloatType ())
8540 cmp = LLVMBuildFCmp (builder, LLVMRealOEQ, lhs, rhs, "");
8541 else
8542 cmp = LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, "");
8543 nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
8544
8545 LLVMTypeRef elemt;
8546 if (srcelemt == LLVMDoubleType ())
8547 elemt = LLVMInt64Type ();
8548 else if (srcelemt == LLVMFloatType ())
8549 elemt = LLVMInt32Type ();
8550 else
8551 elemt = srcelemt;
8552
8553 t = LLVMVectorType (elemt, nelems);
8554 cmp = LLVMBuildSExt (builder, cmp, t, "");
8555 // cmp is a <nelems x elemt> vector, each element is either 0xff... or 0
8556 int half = nelems / 2;
8557 while (half >= 1) {
8558 // AND the top and bottom halfes into the bottom half
8559 for (int i = 0; i < half; ++i)
8560 mask [i] = LLVMConstInt (LLVMInt32Type (), half + i, FALSE);
8561 for (int i = half; i < nelems; ++i)
8562 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8563 shuffle = LLVMBuildShuffleVector (builder, cmp, LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
8564 cmp = LLVMBuildAnd (builder, cmp, shuffle, "");
8565 half = half / 2;
8566 }
8567 // Extract [0]
8568 LLVMValueRef first_elem = LLVMBuildExtractElement (builder, cmp, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
8569 // convert to 0/1
8570 LLVMValueRef cmp_zero = LLVMBuildICmp (builder, LLVMIntNE, first_elem, LLVMConstInt (elemt, 0, FALSE), "");
8571 values [ins->dreg] = LLVMBuildZExt (builder, cmp_zero, LLVMInt8Type (), "");
8572 break;
8573 }
8574 case OP_XBINOP: {
8575 switch (ins->inst_c0) {
8576 case OP_IADD:
8577 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
8578 break;
8579 case OP_ISUB:
8580 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
8581 break;
8582 case OP_IAND:
8583 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
8584 break;
8585 case OP_IOR:
8586 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
8587 break;
8588 case OP_IXOR:
8589 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
8590 break;
8591 case OP_FADD:
8592 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
8593 break;
8594 case OP_FSUB:
8595 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
8596 break;
8597 case OP_FMUL:
8598 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
8599 break;
8600 case OP_FDIV:
8601 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
8602 break;
8603 case OP_FMAX:
8604 case OP_FMIN: {
8605 #if defined(TARGET_X86) || defined(TARGET_AMD64)
8606 LLVMValueRef args [] = { lhs, rhs };
8607
8608 gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4;
8609 if (ins->inst_c0 == OP_FMAX)
8610 values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD, args, dname);
8611 else
8612 values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD, args, dname);
8613 #else
8614 NOT_IMPLEMENTED;
8615 #endif
8616 break;
8617 }
8618 case OP_IMAX: {
8619 gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8;
8620 LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, lhs, rhs, "");
8621 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
8622 break;
8623 }
8624 case OP_IMIN: {
8625 gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8;
8626 LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, lhs, rhs, "");
8627 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
8628 }
8629 break;
8630
8631 default:
8632 g_assert_not_reached ();
8633 }
8634 break;
8635 }
8636 case OP_XEXTRACT_I32:
8637 case OP_XEXTRACT_I64:
8638 case OP_XEXTRACT_R8:
8639 case OP_XEXTRACT_R4: {
8640 LLVMBasicBlockRef bbs [64];
8641 LLVMValueRef switch_ins;
8642 LLVMValueRef phi_values [64];
8643 int nelems = LLVMGetVectorSize (LLVMTypeOf (lhs));
8644 int i;
8645
8646 g_assert (nelems <= 64);
8647 for (i = 0; i < nelems; ++i)
8648 bbs [i] = gen_bb (ctx, "XEXTRACT_CASE_BB");
8649 cbb = gen_bb (ctx, "XEXTRACT_COND_BB");
8650
8651 switch_ins = LLVMBuildSwitch (builder, LLVMBuildAnd (builder, rhs, const_int32 (0xf), ""), bbs [0], 0);
8652 for (i = 0; i < nelems; ++i) {
8653 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
8654 LLVMPositionBuilderAtEnd (builder, bbs [i]);
8655 phi_values [i] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), i, FALSE), "");
8656 LLVMBuildBr (builder, cbb);
8657 }
8658
8659 LLVMPositionBuilderAtEnd (builder, cbb);
8660 values [ins->dreg] = LLVMBuildPhi (builder, LLVMTypeOf (phi_values [0]), "");
8661 LLVMAddIncoming (values [ins->dreg], phi_values, bbs, nelems);
8662
8663 MonoTypeEnum type = (MonoTypeEnum)ins->inst_c0;
8664 switch (type) {
8665 case MONO_TYPE_U1:
8666 case MONO_TYPE_U2:
8667 values [ins->dreg] = LLVMBuildZExt (ctx->builder, values [ins->dreg], LLVMInt32Type (), "");
8668 break;
8669 default:
8670 break;
8671 }
8672 ctx->bblocks [bb->block_num].end_bblock = cbb;
8673 break;
8674 }
8675 case OP_POPCNT32:
8676 values [ins->dreg] = call_intrins (ctx, INTRINS_CTPOP_I32, &lhs, "");
8677 break;
8678 case OP_POPCNT64:
8679 values [ins->dreg] = call_intrins (ctx, INTRINS_CTPOP_I64, &lhs, "");
8680 break;
8681 case OP_LZCNT32:
8682 case OP_LZCNT64: {
8683 LLVMValueRef args [2];
8684 args [0] = lhs;
8685 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
8686 values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_LZCNT32 ? INTRINS_CTLZ_I32 : INTRINS_CTLZ_I64, args, "");
8687 break;
8688 }
8689 case OP_CTTZ32:
8690 case OP_CTTZ64: {
8691 LLVMValueRef args [2];
8692 args [0] = lhs;
8693 args [1] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
8694 values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_CTTZ32 ? INTRINS_CTTZ_I32 : INTRINS_CTTZ_I64, args, "");
8695 break;
8696 }
8697 case OP_BEXTR32:
8698 case OP_BEXTR64: {
8699 LLVMValueRef args [2];
8700 args [0] = lhs;
8701 args [1] = convert (ctx, rhs, ins->opcode == OP_BEXTR32 ? LLVMInt32Type () : LLVMInt64Type ()); // cast ushort to u32/u64
8702 values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_BEXTR32 ? INTRINS_BEXTR_I32 : INTRINS_BEXTR_I64, args, "");
8703 break;
8704 }
8705 case OP_BZHI32:
8706 case OP_BZHI64: {
8707 LLVMValueRef args [2];
8708 args [0] = lhs;
8709 args [1] = rhs;
8710 values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_BZHI32 ? INTRINS_BZHI_I32 : INTRINS_BZHI_I64, args, "");
8711 break;
8712 }
8713 case OP_MULX_H32:
8714 case OP_MULX_H64:
8715 case OP_MULX_HL32:
8716 case OP_MULX_HL64: {
8717 gboolean is_64 = ins->opcode == OP_MULX_H64 || ins->opcode == OP_MULX_HL64;
8718 gboolean only_high = ins->opcode == OP_MULX_H32 || ins->opcode == OP_MULX_H64;
8719 LLVMValueRef lx = LLVMBuildZExt (ctx->builder, lhs, LLVMInt128Type (), "");
8720 LLVMValueRef rx = LLVMBuildZExt (ctx->builder, rhs, LLVMInt128Type (), "");
8721 LLVMValueRef mulx = LLVMBuildMul (ctx->builder, lx, rx, "");
8722 if (!only_high) {
8723 LLVMValueRef lowx = LLVMBuildTrunc (ctx->builder, mulx, is_64 ? LLVMInt64Type () : LLVMInt32Type (), "");
8724 LLVMBuildStore (ctx->builder, lowx, values [ins->sreg3]);
8725 }
8726 LLVMValueRef shift = LLVMConstInt (LLVMInt128Type (), is_64 ? 64 : 32, FALSE);
8727 LLVMValueRef highx = LLVMBuildLShr (ctx->builder, mulx, shift, "");
8728 values [ins->dreg] = LLVMBuildTrunc (ctx->builder, highx, is_64 ? LLVMInt64Type () : LLVMInt32Type (), "");
8729 break;
8730 }
8731 case OP_PEXT32:
8732 case OP_PEXT64: {
8733 LLVMValueRef args [2];
8734 args [0] = lhs;
8735 args [1] = rhs;
8736 values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_PEXT32 ? INTRINS_PEXT_I32 : INTRINS_PEXT_I64, args, "");
8737 break;
8738 }
8739 case OP_PDEP32:
8740 case OP_PDEP64: {
8741 LLVMValueRef args [2];
8742 args [0] = lhs;
8743 args [1] = rhs;
8744 values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_PDEP32 ? INTRINS_PDEP_I32 : INTRINS_PDEP_I64, args, "");
8745 break;
8746 }
8747 #endif /* ENABLE_NETCORE */
8748 #endif /* SIMD */
8749
8750 case OP_DUMMY_USE:
8751 break;
8752
8753 /*
8754 * EXCEPTION HANDLING
8755 */
8756 case OP_IMPLICIT_EXCEPTION:
8757 /* This marks a place where an implicit exception can happen */
8758 if (bb->region != -1)
8759 set_failure (ctx, "implicit-exception");
8760 break;
8761 case OP_THROW:
8762 case OP_RETHROW: {
8763 gboolean rethrow = (ins->opcode == OP_RETHROW);
8764 if (ctx->llvm_only) {
8765 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
8766 has_terminator = TRUE;
8767 ctx->unreachable [bb->block_num] = TRUE;
8768 } else {
8769 emit_throw (ctx, bb, rethrow, lhs);
8770 builder = ctx->builder;
8771 }
8772 break;
8773 }
8774 case OP_CALL_HANDLER: {
8775 /*
8776 * We don't 'call' handlers, but instead simply branch to them.
8777 * The code generated by ENDFINALLY will branch back to us.
8778 */
8779 LLVMBasicBlockRef noex_bb;
8780 GSList *bb_list;
8781 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
8782
8783 bb_list = info->call_handler_return_bbs;
8784
8785 /*
8786 * Set the indicator variable for the finally clause.
8787 */
8788 lhs = info->finally_ind;
8789 g_assert (lhs);
8790 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
8791
8792 /* Branch to the finally clause */
8793 LLVMBuildBr (builder, info->call_handler_target_bb);
8794
8795 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
8796 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
8797
8798 builder = ctx->builder = create_builder (ctx);
8799 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
8800
8801 bblocks [bb->block_num].end_bblock = noex_bb;
8802 break;
8803 }
8804 case OP_START_HANDLER: {
8805 break;
8806 }
8807 case OP_ENDFINALLY: {
8808 LLVMBasicBlockRef resume_bb;
8809 MonoBasicBlock *handler_bb;
8810 LLVMValueRef val, switch_ins, callee;
8811 GSList *bb_list;
8812 BBInfo *info;
8813 gboolean is_fault = MONO_REGION_FLAGS (bb->region) == MONO_EXCEPTION_CLAUSE_FAULT;
8814
8815 /*
8816 * Fault clauses are like finally clauses, but they are only called if an exception is thrown.
8817 */
8818 if (!is_fault) {
8819 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
8820 g_assert (handler_bb);
8821 info = &bblocks [handler_bb->block_num];
8822 lhs = info->finally_ind;
8823 g_assert (lhs);
8824
8825 bb_list = info->call_handler_return_bbs;
8826
8827 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
8828
8829 /* Load the finally variable */
8830 val = LLVMBuildLoad (builder, lhs, "");
8831
8832 /* Reset the variable */
8833 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
8834
8835 /* Branch to either resume_bb, or to the bblocks in bb_list */
8836 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
8837 /*
8838 * The other targets are added at the end to handle OP_CALL_HANDLER
8839 * opcodes processed later.
8840 */
8841 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
8842
8843 builder = ctx->builder = create_builder (ctx);
8844 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
8845 }
8846
8847 if (ctx->llvm_only) {
8848 emit_resume_eh (ctx, bb);
8849 } else {
8850 LLVMTypeRef icall_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
8851 if (ctx->cfg->compile_aot) {
8852 callee = get_callee (ctx, icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
8853 } else {
8854 callee = get_jit_callee (ctx, "llvm_resume_unwind_trampoline", icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
8855 }
8856 LLVMBuildCall (builder, callee, NULL, 0, "");
8857 LLVMBuildUnreachable (builder);
8858 }
8859
8860 has_terminator = TRUE;
8861 break;
8862 }
8863 case OP_IL_SEQ_POINT:
8864 break;
8865 default: {
8866 char reason [128];
8867
8868 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
8869 set_failure (ctx, reason);
8870 break;
8871 }
8872 }
8873
8874 if (!ctx_ok (ctx))
8875 break;
8876
8877 /* Convert the value to the type required by phi nodes */
8878 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
8879 if (ctx->is_vphi [ins->dreg])
8880 /* vtypes */
8881 values [ins->dreg] = addresses [ins->dreg];
8882 else
8883 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
8884 }
8885
8886 /* Add stores for volatile variables */
8887 if (!skip_volatile_store && spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
8888 emit_volatile_store (ctx, ins->dreg);
8889 }
8890
8891 if (!ctx_ok (ctx))
8892 return;
8893
8894 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
8895 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
8896 }
8897
8898 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
8899 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
8900 LLVMBuildRetVoid (builder);
8901 }
8902
8903 if (bb == cfg->bb_entry)
8904 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
8905 }
8906
8907 /*
8908 * mono_llvm_check_method_supported:
8909 *
8910 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
8911 * compiling a method twice.
8912 */
8913 void
8914 mono_llvm_check_method_supported (MonoCompile *cfg)
8915 {
8916 int i, j;
8917
8918 #ifdef TARGET_WASM
8919 if (mono_method_signature_internal (cfg->method)->call_convention == MONO_CALL_VARARG) {
8920 cfg->exception_message = g_strdup ("vararg callconv");
8921 cfg->disable_llvm = TRUE;
8922 return;
8923 }
8924 #endif
8925
8926 if (cfg->llvm_only)
8927 return;
8928
8929 if (cfg->method->save_lmf) {
8930 cfg->exception_message = g_strdup ("lmf");
8931 cfg->disable_llvm = TRUE;
8932 }
8933 if (cfg->disable_llvm)
8934 return;
8935
8936 /*
8937 * Nested clauses where one of the clauses is a finally clause is
8938 * not supported, because LLVM can't figure out the control flow,
8939 * probably because we resume exception handling by calling our
8940 * own function instead of using the 'resume' llvm instruction.
8941 */
8942 for (i = 0; i < cfg->header->num_clauses; ++i) {
8943 for (j = 0; j < cfg->header->num_clauses; ++j) {
8944 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
8945 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
8946
8947 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
8948 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
8949 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
8950 cfg->exception_message = g_strdup ("nested clauses");
8951 cfg->disable_llvm = TRUE;
8952 break;
8953 }
8954 }
8955 }
8956 if (cfg->disable_llvm)
8957 return;
8958
8959 /* FIXME: */
8960 if (cfg->method->dynamic) {
8961 cfg->exception_message = g_strdup ("dynamic.");
8962 cfg->disable_llvm = TRUE;
8963 }
8964 if (cfg->disable_llvm)
8965 return;
8966 }
8967
8968 static LLVMCallInfo*
8969 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
8970 {
8971 LLVMCallInfo *linfo;
8972 int i;
8973
8974 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
8975 int i, n, pindex;
8976
8977 /*
8978 * Gsharedvt methods have the following calling convention:
8979 * - all arguments are passed by ref, even non generic ones
8980 * - the return value is returned by ref too, using a vret
8981 * argument passed after 'this'.
8982 */
8983 n = sig->param_count + sig->hasthis;
8984 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
8985
8986 pindex = 0;
8987 if (sig->hasthis)
8988 linfo->args [pindex ++].storage = LLVMArgNormal;
8989
8990 if (sig->ret->type != MONO_TYPE_VOID) {
8991 if (mini_is_gsharedvt_variable_type (sig->ret))
8992 linfo->ret.storage = LLVMArgGsharedvtVariable;
8993 else if (mini_type_is_vtype (sig->ret))
8994 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
8995 else
8996 linfo->ret.storage = LLVMArgGsharedvtFixed;
8997 linfo->vret_arg_index = pindex;
8998 } else {
8999 linfo->ret.storage = LLVMArgNone;
9000 }
9001
9002 for (i = 0; i < sig->param_count; ++i) {
9003 if (sig->params [i]->byref)
9004 linfo->args [pindex].storage = LLVMArgNormal;
9005 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
9006 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
9007 else if (mini_type_is_vtype (sig->params [i]))
9008 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
9009 else
9010 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
9011 linfo->args [pindex].type = sig->params [i];
9012 pindex ++;
9013 }
9014 return linfo;
9015 }
9016
9017 linfo = mono_arch_get_llvm_call_info (cfg, sig);
9018 linfo->dummy_arg_pindex = -1;
9019 for (i = 0; i < sig->param_count; ++i)
9020 linfo->args [i + sig->hasthis].type = sig->params [i];
9021
9022 return linfo;
9023 }
9024
9025 static void
9026 emit_method_inner (EmitContext *ctx);
9027
9028 static void
9029 free_ctx (EmitContext *ctx)
9030 {
9031 GSList *l;
9032
9033 g_free (ctx->values);
9034 g_free (ctx->addresses);
9035 g_free (ctx->vreg_types);
9036 g_free (ctx->is_vphi);
9037 g_free (ctx->vreg_cli_types);
9038 g_free (ctx->is_dead);
9039 g_free (ctx->unreachable);
9040 g_ptr_array_free (ctx->phi_values, TRUE);
9041 g_free (ctx->bblocks);
9042 g_hash_table_destroy (ctx->region_to_handler);
9043 g_hash_table_destroy (ctx->clause_to_handler);
9044 g_hash_table_destroy (ctx->jit_callees);
9045
9046 g_ptr_array_free (ctx->callsite_list, TRUE);
9047
9048 g_free (ctx->method_name);
9049 g_ptr_array_free (ctx->bblock_list, TRUE);
9050
9051 for (l = ctx->builders; l; l = l->next) {
9052 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
9053 LLVMDisposeBuilder (builder);
9054 }
9055
9056 g_free (ctx);
9057 }
9058
9059 /*
9060 * mono_llvm_emit_method:
9061 *
9062 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
9063 */
9064 void
9065 mono_llvm_emit_method (MonoCompile *cfg)
9066 {
9067 EmitContext *ctx;
9068 char *method_name;
9069 int i;
9070
9071 if (cfg->skip)
9072 return;
9073
9074 /* The code below might acquire the loader lock, so use it for global locking */
9075 mono_loader_lock ();
9076
9077 ctx = g_new0 (EmitContext, 1);
9078 ctx->cfg = cfg;
9079 ctx->mempool = cfg->mempool;
9080
9081 /*
9082 * This maps vregs to the LLVM instruction defining them
9083 */
9084 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
9085 /*
9086 * This maps vregs for volatile variables to the LLVM instruction defining their
9087 * address.
9088 */
9089 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
9090 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
9091 ctx->is_vphi = g_new0 (gboolean, cfg->next_vreg);
9092 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
9093 ctx->phi_values = g_ptr_array_sized_new (256);
9094 /*
9095 * This signals whenever the vreg was defined by a phi node with no input vars
9096 * (i.e. all its input bblocks end with NOT_REACHABLE).
9097 */
9098 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
9099 /* Whenever the bblock is unreachable */
9100 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
9101 ctx->bblock_list = g_ptr_array_sized_new (256);
9102
9103 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
9104 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
9105 ctx->callsite_list = g_ptr_array_new ();
9106 ctx->jit_callees = g_hash_table_new (NULL, NULL);
9107 if (cfg->compile_aot) {
9108 ctx->module = &aot_module;
9109
9110 if (mono_aot_is_externally_callable (cfg->method))
9111 method_name = mono_aot_get_mangled_method_name (cfg->method);
9112 else
9113 method_name = mono_aot_get_method_name (cfg);
9114 cfg->llvm_method_name = g_strdup (method_name);
9115 } else {
9116 init_jit_module (cfg->domain);
9117 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
9118 method_name = mono_method_full_name (cfg->method, TRUE);
9119 }
9120 ctx->method_name = method_name;
9121
9122 if (cfg->compile_aot) {
9123 ctx->lmodule = ctx->module->lmodule;
9124 } else {
9125 ctx->lmodule = LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg->method->name));
9126 /* Reset this as it contains values from lmodule */
9127 memset (ctx->module->intrins_by_id, 0, sizeof (LLVMValueRef) * INTRINS_NUM);
9128 }
9129 ctx->llvm_only = ctx->module->llvm_only;
9130 #ifdef TARGET_WASM
9131 ctx->emit_dummy_arg = TRUE;
9132 #endif
9133
9134 emit_method_inner (ctx);
9135
9136 if (!ctx_ok (ctx)) {
9137 if (ctx->lmethod) {
9138 /* Need to add unused phi nodes as they can be referenced by other values */
9139 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
9140 LLVMBuilderRef builder;
9141
9142 builder = create_builder (ctx);
9143 LLVMPositionBuilderAtEnd (builder, phi_bb);
9144
9145 for (i = 0; i < ctx->phi_values->len; ++i) {
9146 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
9147 if (LLVMGetInstructionParent (v) == NULL)
9148 LLVMInsertIntoBuilder (builder, v);
9149 }
9150
9151 if (ctx->module->llvm_only && ctx->module->static_link) {
9152 // Keep a stub for the function since it might be called directly
9153 int nbbs = LLVMCountBasicBlocks (ctx->lmethod);
9154 LLVMBasicBlockRef *bblocks = g_new0 (LLVMBasicBlockRef, nbbs);
9155 LLVMGetBasicBlocks (ctx->lmethod, bblocks);
9156 for (int i = 0; i < nbbs; ++i)
9157 LLVMDeleteBasicBlock (bblocks [i]);
9158
9159 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (ctx->lmethod, "ENTRY");
9160 builder = create_builder (ctx);
9161 LLVMPositionBuilderAtEnd (builder, entry_bb);
9162 ctx->builder = builder;
9163
9164 LLVMTypeRef sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
9165 LLVMValueRef callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
9166 LLVMBuildCall (builder, callee, NULL, 0, "");
9167 LLVMBuildUnreachable (builder);
9168 } else {
9169 LLVMDeleteFunction (ctx->lmethod);
9170 }
9171 }
9172 }
9173
9174 free_ctx (ctx);
9175
9176 mono_loader_unlock ();
9177 }
9178
9179 static void
9180 emit_method_inner (EmitContext *ctx)
9181 {
9182 MonoCompile *cfg = ctx->cfg;
9183 MonoMethodSignature *sig;
9184 MonoBasicBlock *bb;
9185 LLVMTypeRef method_type;
9186 LLVMValueRef method = NULL;
9187 LLVMValueRef *values = ctx->values;
9188 int i, max_block_num, bb_index;
9189 gboolean last = FALSE;
9190 gboolean llvmonly_fail = FALSE;
9191 LLVMCallInfo *linfo;
9192 LLVMModuleRef lmodule = ctx->lmodule;
9193 BBInfo *bblocks;
9194 GPtrArray *bblock_list = ctx->bblock_list;
9195 MonoMethodHeader *header;
9196 MonoExceptionClause *clause;
9197 char **names;
9198 LLVMBuilderRef entry_builder = NULL;
9199 LLVMBasicBlockRef entry_bb = NULL;
9200
9201 if (cfg->gsharedvt && !cfg->llvm_only) {
9202 set_failure (ctx, "gsharedvt");
9203 return;
9204 }
9205
9206 #if 1
9207 {
9208 static int count = 0;
9209 count ++;
9210
9211 char *llvm_count_str = g_getenv ("LLVM_COUNT");
9212 if (llvm_count_str) {
9213 int lcount = atoi (llvm_count_str);
9214 g_free (llvm_count_str);
9215 if (count == lcount) {
9216 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
9217 fflush (stdout);
9218 last = TRUE;
9219 }
9220 if (count > lcount) {
9221 set_failure (ctx, "count");
9222 return;
9223 }
9224 }
9225 }
9226 #endif
9227
9228 // If we come upon one of the init_method wrappers, we need to find
9229 // the method that we have already emitted and tell LLVM that this
9230 // managed method info for the wrapper is associated with this method
9231 // we constructed ourselves from LLVM IR.
9232 //
9233 // This is necessary to unwind through the init_method, in the case that
9234 // it has to run a static cctor that throws an exception
9235 if (cfg->method->wrapper_type == MONO_WRAPPER_OTHER) {
9236 WrapperInfo *info = mono_marshal_get_wrapper_info (cfg->method);
9237 if (info->subtype == WRAPPER_SUBTYPE_AOT_INIT) {
9238 method = get_init_func (ctx->module, info->d.aot_init.subtype);
9239 ctx->lmethod = method;
9240 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
9241
9242 const char *init_name = mono_marshal_get_aot_init_wrapper_name (info->d.aot_init.subtype);
9243 ctx->method_name = g_strdup_printf ("%s_%s", ctx->module->global_prefix, init_name);
9244 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
9245
9246 if (!cfg->llvm_only && ctx->module->external_symbols) {
9247 LLVMSetLinkage (method, LLVMExternalLinkage);
9248 LLVMSetVisibility (method, LLVMHiddenVisibility);
9249 }
9250 /* Not looked up at runtime */
9251 g_hash_table_insert (ctx->module->no_method_table_lmethods, method, method);
9252
9253 goto after_codegen;
9254 } else if (info->subtype == WRAPPER_SUBTYPE_LLVM_FUNC) {
9255 g_assert (info->d.llvm_func.subtype == LLVM_FUNC_WRAPPER_GC_POLL);
9256
9257 if (cfg->compile_aot) {
9258 method = ctx->module->gc_poll_cold_wrapper;
9259 g_assert (method);
9260 } else {
9261 method = emit_icall_cold_wrapper (ctx->module, lmodule, MONO_JIT_ICALL_mono_threads_state_poll, FALSE);
9262 }
9263 ctx->lmethod = method;
9264 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
9265
9266 ctx->method_name = g_strdup (LLVMGetValueName (method)); //g_strdup_printf ("%s_%s", ctx->module->global_prefix, LLVMGetValueName (method));
9267 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
9268
9269 if (!cfg->llvm_only && ctx->module->external_symbols) {
9270 LLVMSetLinkage (method, LLVMExternalLinkage);
9271 LLVMSetVisibility (method, LLVMHiddenVisibility);
9272 }
9273
9274 goto after_codegen;
9275 }
9276 }
9277
9278 sig = mono_method_signature_internal (cfg->method);
9279 ctx->sig = sig;
9280
9281 linfo = get_llvm_call_info (cfg, sig);
9282 ctx->linfo = linfo;
9283 if (!ctx_ok (ctx))
9284 return;
9285
9286 if (cfg->rgctx_var)
9287 linfo->rgctx_arg = TRUE;
9288 else if (needs_extra_arg (ctx, cfg->method))
9289 linfo->dummy_arg = TRUE;
9290 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
9291 if (!ctx_ok (ctx))
9292 return;
9293
9294 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
9295 ctx->lmethod = method;
9296
9297 if (!cfg->llvm_only)
9298 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
9299
9300 /* if the method doesn't contain
9301 * (1) a call (so it's a leaf method)
9302 * (2) and no loops
9303 * we can skip the GC safepoint on method entry. */
9304 gboolean requires_safepoint;
9305 requires_safepoint = cfg->has_calls;
9306 if (!requires_safepoint) {
9307 for (bb = cfg->bb_entry->next_bb; bb; bb = bb->next_bb) {
9308 if (bb->loop_body_start || (bb->flags & BB_EXCEPTION_HANDLER)) {
9309 requires_safepoint = TRUE;
9310 }
9311 }
9312 }
9313 if (cfg->method->wrapper_type) {
9314 WrapperInfo *info = mono_marshal_get_wrapper_info (cfg->method);
9315
9316 switch (info->subtype) {
9317 case WRAPPER_SUBTYPE_GSHAREDVT_IN:
9318 case WRAPPER_SUBTYPE_GSHAREDVT_OUT:
9319 case WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG:
9320 case WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG:
9321 /* Arguments are not used after the call */
9322 requires_safepoint = FALSE;
9323 break;
9324 }
9325 }
9326 ctx->has_safepoints = requires_safepoint;
9327
9328 if (!cfg->llvm_only && mono_threads_are_safepoints_enabled () && requires_safepoint) {
9329 if (!cfg->compile_aot && cfg->method->wrapper_type != MONO_WRAPPER_ALLOC) {
9330 LLVMSetGC (method, "coreclr");
9331 emit_gc_safepoint_poll (ctx->module, ctx->lmodule, cfg);
9332 } else if (cfg->compile_aot) {
9333 LLVMSetGC (method, "coreclr");
9334 }
9335 }
9336 LLVMSetLinkage (method, LLVMPrivateLinkage);
9337
9338 mono_llvm_add_func_attr (method, LLVM_ATTR_UW_TABLE);
9339
9340 if (cfg->disable_omit_fp)
9341 mono_llvm_add_func_attr_nv (method, "no-frame-pointer-elim", "true");
9342
9343 if (cfg->compile_aot) {
9344 if (mono_aot_is_externally_callable (cfg->method)) {
9345 LLVMSetLinkage (method, LLVMExternalLinkage);
9346 } else {
9347 LLVMSetLinkage (method, LLVMInternalLinkage);
9348 //all methods have internal visibility when doing llvm_only
9349 if (!cfg->llvm_only && ctx->module->external_symbols) {
9350 LLVMSetLinkage (method, LLVMExternalLinkage);
9351 LLVMSetVisibility (method, LLVMHiddenVisibility);
9352 }
9353 }
9354 } else {
9355 LLVMSetLinkage (method, LLVMExternalLinkage);
9356 }
9357
9358 if (cfg->method->save_lmf && !cfg->llvm_only) {
9359 set_failure (ctx, "lmf");
9360 return;
9361 }
9362
9363 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
9364 set_failure (ctx, "pinvoke signature");
9365 return;
9366 }
9367
9368 header = cfg->header;
9369 for (i = 0; i < header->num_clauses; ++i) {
9370 clause = &header->clauses [i];
9371 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_FAULT && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
9372 if (cfg->llvm_only) {
9373 // FIXME: Treat unhandled opcodes like __arglist the same way
9374 // It would require deleting the already emitted code
9375 llvmonly_fail = TRUE;
9376 } else {
9377 set_failure (ctx, "non-finally/catch/fault clause.");
9378 return;
9379 }
9380 }
9381 }
9382 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
9383 /* We can't handle inlined methods with clauses */
9384 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
9385
9386 if (linfo->rgctx_arg) {
9387 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
9388 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
9389 /*
9390 * We mark the rgctx parameter with the inreg attribute, which is mapped to
9391 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
9392 * CC_X86_64_Mono in X86CallingConv.td.
9393 */
9394 if (!ctx->llvm_only)
9395 mono_llvm_add_param_attr (ctx->rgctx_arg, LLVM_ATTR_IN_REG);
9396 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
9397 } else {
9398 ctx->rgctx_arg_pindex = -1;
9399 }
9400 if (cfg->vret_addr) {
9401 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
9402 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
9403 if (linfo->ret.storage == LLVMArgVtypeByRef) {
9404 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
9405 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
9406 }
9407 }
9408
9409 if (sig->hasthis) {
9410 ctx->this_arg_pindex = linfo->this_arg_pindex;
9411 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
9412 values [cfg->args [0]->dreg] = ctx->this_arg;
9413 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
9414 }
9415 if (linfo->dummy_arg)
9416 LLVMSetValueName (LLVMGetParam (method, linfo->dummy_arg_pindex), "dummy_arg");
9417
9418 names = g_new (char *, sig->param_count);
9419 mono_method_get_param_names (cfg->method, (const char **) names);
9420
9421 /* Set parameter names/attributes */
9422 for (i = 0; i < sig->param_count; ++i) {
9423 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
9424 char *name;
9425 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
9426 int j;
9427
9428 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
9429 name = g_strdup_printf ("dummy_%d_%d", i, j);
9430 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
9431 g_free (name);
9432 }
9433
9434 if (ainfo->storage == LLVMArgVtypeInReg && ainfo->pair_storage [0] == LLVMArgNone && ainfo->pair_storage [1] == LLVMArgNone)
9435 continue;
9436
9437 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
9438 if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
9439 if (names [i] && names [i][0] != '\0')
9440 name = g_strdup_printf ("p_arg_%s", names [i]);
9441 else
9442 name = g_strdup_printf ("p_arg_%d", i);
9443 } else {
9444 if (names [i] && names [i][0] != '\0')
9445 name = g_strdup_printf ("arg_%s", names [i]);
9446 else
9447 name = g_strdup_printf ("arg_%d", i);
9448 }
9449 LLVMSetValueName (LLVMGetParam (method, pindex), name);
9450 g_free (name);
9451 if (ainfo->storage == LLVMArgVtypeByVal)
9452 mono_llvm_add_param_attr (LLVMGetParam (method, pindex), LLVM_ATTR_BY_VAL);
9453
9454 if (ainfo->storage == LLVMArgVtypeByRef || ainfo->storage == LLVMArgVtypeAddr) {
9455 /* For OP_LDADDR */
9456 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
9457 }
9458 }
9459 g_free (names);
9460
9461 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
9462 ctx->minfo = mono_debug_lookup_method (cfg->method);
9463 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
9464 }
9465
9466 max_block_num = 0;
9467 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
9468 max_block_num = MAX (max_block_num, bb->block_num);
9469 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
9470
9471 /* Add branches between non-consecutive bblocks */
9472 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
9473 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
9474 bb->next_bb != bb->last_ins->inst_false_bb) {
9475
9476 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
9477 inst->opcode = OP_BR;
9478 inst->inst_target_bb = bb->last_ins->inst_false_bb;
9479 mono_bblock_add_inst (bb, inst);
9480 }
9481 }
9482
9483 /*
9484 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
9485 */
9486 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
9487 MonoInst *ins;
9488 LLVMBuilderRef builder;
9489 char *dname;
9490 char dname_buf[128];
9491
9492 builder = create_builder (ctx);
9493
9494 for (ins = bb->code; ins; ins = ins->next) {
9495 switch (ins->opcode) {
9496 case OP_PHI:
9497 case OP_FPHI:
9498 case OP_VPHI:
9499 case OP_XPHI: {
9500 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
9501
9502 if (!ctx_ok (ctx))
9503 return;
9504
9505 if (ins->opcode == OP_VPHI) {
9506 /* Treat valuetype PHI nodes as operating on the address itself */
9507 g_assert (ins->klass);
9508 phi_type = LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)), 0);
9509 }
9510
9511 /*
9512 * Have to precreate these, as they can be referenced by
9513 * earlier instructions.
9514 */
9515 sprintf (dname_buf, "t%d", ins->dreg);
9516 dname = dname_buf;
9517 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
9518
9519 if (ins->opcode == OP_VPHI)
9520 ctx->addresses [ins->dreg] = values [ins->dreg];
9521
9522 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
9523
9524 /*
9525 * Set the expected type of the incoming arguments since these have
9526 * to have the same type.
9527 */
9528 for (i = 0; i < ins->inst_phi_args [0]; i++) {
9529 int sreg1 = ins->inst_phi_args [i + 1];
9530
9531 if (sreg1 != -1) {
9532 if (ins->opcode == OP_VPHI)
9533 ctx->is_vphi [sreg1] = TRUE;
9534 ctx->vreg_types [sreg1] = phi_type;
9535 }
9536 }
9537 break;
9538 }
9539 case OP_LDADDR:
9540 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
9541 break;
9542 default:
9543 break;
9544 }
9545 }
9546 }
9547
9548 /*
9549 * Create an ordering for bblocks, use the depth first order first, then
9550 * put the exception handling bblocks last.
9551 */
9552 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
9553 bb = cfg->bblocks [bb_index];
9554 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
9555 g_ptr_array_add (bblock_list, bb);
9556 bblocks [bb->block_num].added = TRUE;
9557 }
9558 }
9559
9560 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
9561 if (!bblocks [bb->block_num].added)
9562 g_ptr_array_add (bblock_list, bb);
9563 }
9564
9565 /*
9566 * Second pass: generate code.
9567 */
9568 // Emit entry point
9569 entry_builder = create_builder (ctx);
9570 entry_bb = get_bb (ctx, cfg->bb_entry);
9571 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
9572 emit_entry_bb (ctx, entry_builder);
9573
9574 if (llvmonly_fail)
9575 /*
9576 * In llvmonly mode, we want to emit an llvm method for every method even if it fails to compile,
9577 * so direct calls can be made from outside the assembly.
9578 */
9579 goto after_codegen_1;
9580
9581 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
9582 int clause_index;
9583 char name [128];
9584
9585 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
9586 continue;
9587
9588 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
9589 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
9590 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
9591
9592 /*
9593 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
9594 * LLVM bblock containing a landing pad causes problems for the
9595 * LLVM optimizer passes.
9596 */
9597 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
9598 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
9599 }
9600
9601 // Make landing pads first
9602 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
9603
9604 if (ctx->llvm_only) {
9605 size_t group_index = 0;
9606 while (group_index < cfg->header->num_clauses) {
9607 int count = 0;
9608 size_t cursor = group_index;
9609 while (cursor < cfg->header->num_clauses &&
9610 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
9611 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
9612 count++;
9613 cursor++;
9614 }
9615
9616 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
9617 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
9618 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
9619
9620 group_index = cursor;
9621 }
9622 }
9623
9624 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
9625 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
9626
9627 // Prune unreachable mono BBs.
9628 if (!(bb == cfg->bb_entry || bb->in_count > 0))
9629 continue;
9630
9631 process_bb (ctx, bb);
9632 if (!ctx_ok (ctx))
9633 return;
9634 }
9635 g_hash_table_destroy (ctx->exc_meta);
9636
9637 mono_memory_barrier ();
9638
9639 /* Add incoming phi values */
9640 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
9641 GSList *l, *ins_list;
9642
9643 ins_list = bblocks [bb->block_num].phi_nodes;
9644
9645 for (l = ins_list; l; l = l->next) {
9646 PhiNode *node = (PhiNode*)l->data;
9647 MonoInst *phi = node->phi;
9648 int sreg1 = node->sreg;
9649 LLVMBasicBlockRef in_bb;
9650
9651 if (sreg1 == -1)
9652 continue;
9653
9654 in_bb = get_end_bb (ctx, node->in_bb);
9655
9656 if (ctx->unreachable [node->in_bb->block_num])
9657 continue;
9658
9659 if (phi->opcode == OP_VPHI) {
9660 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
9661 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
9662 } else {
9663 if (!values [sreg1]) {
9664 /* Can happen with values in EH clauses */
9665 set_failure (ctx, "incoming phi sreg1");
9666 return;
9667 }
9668 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
9669 set_failure (ctx, "incoming phi arg type mismatch");
9670 return;
9671 }
9672 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
9673 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
9674 }
9675 }
9676 }
9677
9678 /* Nullify empty phi instructions */
9679 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
9680 GSList *l, *ins_list;
9681
9682 ins_list = bblocks [bb->block_num].phi_nodes;
9683
9684 for (l = ins_list; l; l = l->next) {
9685 PhiNode *node = (PhiNode*)l->data;
9686 MonoInst *phi = node->phi;
9687 LLVMValueRef phi_ins = values [phi->dreg];
9688
9689 if (!phi_ins)
9690 /* Already removed */
9691 continue;
9692
9693 if (LLVMCountIncoming (phi_ins) == 0) {
9694 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
9695 LLVMInstructionEraseFromParent (phi_ins);
9696 values [phi->dreg] = NULL;
9697 }
9698 }
9699 }
9700
9701 /* Create the SWITCH statements for ENDFINALLY instructions */
9702 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
9703 BBInfo *info = &bblocks [bb->block_num];
9704 GSList *l;
9705 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
9706 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
9707 GSList *bb_list = info->call_handler_return_bbs;
9708
9709 GSList *bb_list_iter;
9710 i = 0;
9711 for (bb_list_iter = bb_list; bb_list_iter; bb_list_iter = g_slist_next (bb_list_iter)) {
9712 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)bb_list_iter->data);
9713 i ++;
9714 }
9715 }
9716 }
9717
9718 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
9719
9720 after_codegen_1:
9721
9722 if (llvmonly_fail) {
9723 /*
9724 * FIXME: Maybe fallback to interpreter
9725 */
9726 static LLVMTypeRef sig;
9727
9728 ctx->builder = create_builder (ctx);
9729 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
9730
9731 char *name = mono_method_get_full_name (cfg->method);
9732 int len = strlen (name);
9733
9734 LLVMTypeRef type = LLVMArrayType (LLVMInt8Type (), len + 1);
9735 LLVMValueRef name_var = LLVMAddGlobal (ctx->lmodule, type, "missing_method_name");
9736 LLVMSetVisibility (name_var, LLVMHiddenVisibility);
9737 LLVMSetLinkage (name_var, LLVMInternalLinkage);
9738 LLVMSetInitializer (name_var, mono_llvm_create_constant_data_array ((guint8*)name, len + 1));
9739 mono_llvm_set_is_constant (name_var);
9740 g_free (name);
9741
9742 if (!sig)
9743 sig = LLVMFunctionType1 (LLVMVoidType (), ctx->module->ptr_type, FALSE);
9744 LLVMValueRef callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_aot_failed_exception));
9745 LLVMValueRef args [] = { convert (ctx, name_var, ctx->module->ptr_type) };
9746 LLVMBuildCall (ctx->builder, callee, args, 1, "");
9747 LLVMBuildUnreachable (ctx->builder);
9748 }
9749
9750 /* Initialize the method if needed */
9751 if (cfg->compile_aot) {
9752 // FIXME: Add more shared got entries
9753 ctx->builder = create_builder (ctx);
9754 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
9755
9756 // FIXME: beforefieldinit
9757 /*
9758 * NATIVE_TO_MANAGED methods might be called on a thread not attached to the runtime, so they are initialized when loaded
9759 * in load_method ().
9760 */
9761 gboolean needs_init = ctx->cfg->got_access_count > 0;
9762 MonoMethod *cctor = NULL;
9763 if (!needs_init && (cctor = mono_class_get_cctor (cfg->method->klass))) {
9764 /* Needs init to run the cctor */
9765 if (cfg->method->flags & METHOD_ATTRIBUTE_STATIC)
9766 needs_init = TRUE;
9767 if (cctor == cfg->method)
9768 needs_init = FALSE;
9769
9770 // If we are a constructor, we need to init so the static
9771 // constructor gets called.
9772 if (!strcmp (cfg->method->name, ".ctor"))
9773 needs_init = TRUE;
9774 }
9775 if (cfg->method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED)
9776 needs_init = FALSE;
9777 if (needs_init)
9778 emit_method_init (ctx);
9779 else
9780 LLVMBuildBr (ctx->builder, ctx->inited_bb);
9781
9782 // Was observing LLVM moving field accesses into the caller's method
9783 // body before the init call (the inlined one), leading to NULL derefs
9784 // after the init_method returns (GOT is filled out though)
9785 if (needs_init)
9786 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
9787 }
9788
9789 if (mini_get_debug_options ()->llvm_disable_inlining)
9790 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
9791
9792 after_codegen:
9793 g_ptr_array_add (ctx->module->cfgs, cfg);
9794 if (cfg->llvm_only) {
9795 /*
9796 * Add the contents of ctx->callsite_list to module->callsite_list.
9797 * We can't do this earlier, as it contains llvm instructions which can be
9798 * freed if compilation fails.
9799 * FIXME: Get rid of this when all methods can be llvm compiled.
9800 */
9801 for (int i = 0; i < ctx->callsite_list->len; ++i)
9802 g_ptr_array_add (ctx->module->callsite_list, g_ptr_array_index (ctx->callsite_list, i));
9803 }
9804
9805 if (cfg->verbose_level > 1) {
9806 g_print ("\n*** Unoptimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, TRUE));
9807 if (cfg->compile_aot) {
9808 mono_llvm_dump_value (method);
9809 } else {
9810 mono_llvm_dump_module (ctx->lmodule);
9811 }
9812 g_print ("***\n\n");
9813 }
9814
9815 if (cfg->compile_aot && !cfg->llvm_only)
9816 mark_as_used (ctx->module, method);
9817
9818 if (!cfg->llvm_only) {
9819 LLVMValueRef md_args [16];
9820 LLVMValueRef md_node;
9821 int method_index;
9822
9823 if (cfg->compile_aot)
9824 method_index = mono_aot_get_method_index (cfg->orig_method);
9825 else
9826 method_index = 1;
9827 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
9828 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
9829 md_node = LLVMMDNode (md_args, 2);
9830 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
9831 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
9832 }
9833
9834 if (cfg->compile_aot) {
9835 /* Don't generate native code, keep the LLVM IR */
9836 if (cfg->verbose_level) {
9837 char *name = mono_method_get_full_name (cfg->method);
9838 printf ("%s emitted as %s\n", name, ctx->method_name);
9839 g_free (name);
9840 }
9841
9842 //LLVMDumpValue (ctx->lmethod);
9843 //int err = LLVMVerifyFunction (ctx->lmethod, LLVMPrintMessageAction);
9844 //g_assert (err == 0);
9845 } else {
9846 //LLVMVerifyFunction (method, 0);
9847 llvm_jit_finalize_method (ctx);
9848 }
9849
9850 if (ctx->module->method_to_lmethod)
9851 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
9852
9853 if (ctx->module->idx_to_lmethod)
9854 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
9855
9856 if (ctx->llvm_only && m_class_is_valuetype (cfg->orig_method->klass) && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
9857 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
9858 }
9859
9860 /*
9861 * mono_llvm_create_vars:
9862 *
9863 * Same as mono_arch_create_vars () for LLVM.
9864 */
9865 void
9866 mono_llvm_create_vars (MonoCompile *cfg)
9867 {
9868 MonoMethodSignature *sig;
9869
9870 sig = mono_method_signature_internal (cfg->method);
9871 if (cfg->gsharedvt && cfg->llvm_only) {
9872 gboolean vretaddr = FALSE;
9873
9874 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
9875 vretaddr = TRUE;
9876 } else {
9877 MonoMethodSignature *sig = mono_method_signature_internal (cfg->method);
9878 LLVMCallInfo *linfo;
9879
9880 linfo = get_llvm_call_info (cfg, sig);
9881 vretaddr = (linfo->ret.storage == LLVMArgVtypeRetAddr || linfo->ret.storage == LLVMArgVtypeByRef || linfo->ret.storage == LLVMArgGsharedvtFixed || linfo->ret.storage == LLVMArgGsharedvtVariable || linfo->ret.storage == LLVMArgGsharedvtFixedVtype);
9882 }
9883 if (vretaddr) {
9884 /*
9885 * Creating vret_addr forces CEE_SETRET to store the result into it,
9886 * so we don't have to generate any code in our OP_SETRET case.
9887 */
9888 cfg->vret_addr = mono_compile_create_var (cfg, m_class_get_byval_arg (mono_get_intptr_class ()), OP_ARG);
9889 if (G_UNLIKELY (cfg->verbose_level > 1)) {
9890 printf ("vret_addr = ");
9891 mono_print_ins (cfg->vret_addr);
9892 }
9893 }
9894 } else {
9895 mono_arch_create_vars (cfg);
9896 }
9897 }
9898
9899 /*
9900 * mono_llvm_emit_call:
9901 *
9902 * Same as mono_arch_emit_call () for LLVM.
9903 */
9904 void
9905 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
9906 {
9907 MonoInst *in;
9908 MonoMethodSignature *sig;
9909 int i, n, stack_size;
9910 LLVMArgInfo *ainfo;
9911
9912 stack_size = 0;
9913
9914 sig = call->signature;
9915 n = sig->param_count + sig->hasthis;
9916
9917 call->cinfo = get_llvm_call_info (cfg, sig);
9918
9919 if (cfg->disable_llvm)
9920 return;
9921
9922 if (sig->call_convention == MONO_CALL_VARARG) {
9923 cfg->exception_message = g_strdup ("varargs");
9924 cfg->disable_llvm = TRUE;
9925 }
9926
9927 for (i = 0; i < n; ++i) {
9928 MonoInst *ins;
9929
9930 ainfo = call->cinfo->args + i;
9931
9932 in = call->args [i];
9933
9934 /* Simply remember the arguments */
9935 switch (ainfo->storage) {
9936 case LLVMArgNormal: {
9937 MonoType *t = (sig->hasthis && i == 0) ? m_class_get_byval_arg (mono_get_intptr_class ()) : ainfo->type;
9938 int opcode;
9939
9940 opcode = mono_type_to_regmove (cfg, t);
9941 if (opcode == OP_FMOVE) {
9942 MONO_INST_NEW (cfg, ins, OP_FMOVE);
9943 ins->dreg = mono_alloc_freg (cfg);
9944 } else if (opcode == OP_LMOVE) {
9945 MONO_INST_NEW (cfg, ins, OP_LMOVE);
9946 ins->dreg = mono_alloc_lreg (cfg);
9947 } else if (opcode == OP_RMOVE) {
9948 MONO_INST_NEW (cfg, ins, OP_RMOVE);
9949 ins->dreg = mono_alloc_freg (cfg);
9950 } else {
9951 MONO_INST_NEW (cfg, ins, OP_MOVE);
9952 ins->dreg = mono_alloc_ireg (cfg);
9953 }
9954 ins->sreg1 = in->dreg;
9955 break;
9956 }
9957 case LLVMArgVtypeByVal:
9958 case LLVMArgVtypeByRef:
9959 case LLVMArgVtypeInReg:
9960 case LLVMArgVtypeAddr:
9961 case LLVMArgVtypeAsScalar:
9962 case LLVMArgAsIArgs:
9963 case LLVMArgAsFpArgs:
9964 case LLVMArgGsharedvtVariable:
9965 case LLVMArgGsharedvtFixed:
9966 case LLVMArgGsharedvtFixedVtype:
9967 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
9968 ins->dreg = mono_alloc_ireg (cfg);
9969 ins->sreg1 = in->dreg;
9970 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
9971 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
9972 ins->inst_vtype = ainfo->type;
9973 ins->klass = mono_class_from_mono_type_internal (ainfo->type);
9974 break;
9975 default:
9976 cfg->exception_message = g_strdup ("ainfo->storage");
9977 cfg->disable_llvm = TRUE;
9978 return;
9979 }
9980
9981 if (!cfg->disable_llvm) {
9982 MONO_ADD_INS (cfg->cbb, ins);
9983 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
9984 }
9985 }
9986 }
9987
9988 static inline void
9989 add_func (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
9990 {
9991 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
9992 }
9993
9994 static LLVMValueRef
9995 add_intrins (LLVMModuleRef module, IntrinsicId id, LLVMTypeRef *params, int nparams)
9996 {
9997 return mono_llvm_register_overloaded_intrinsic (module, id, params, nparams);
9998 }
9999
10000 static LLVMValueRef
10001 add_intrins1 (LLVMModuleRef module, IntrinsicId id, LLVMTypeRef param1)
10002 {
10003 return mono_llvm_register_overloaded_intrinsic (module, id, &param1, 1);
10004 }
10005
10006 static LLVMValueRef
10007 add_intrins2 (LLVMModuleRef module, IntrinsicId id, LLVMTypeRef param1, LLVMTypeRef param2)
10008 {
10009 LLVMTypeRef params [] = { param1, param2 };
10010 return mono_llvm_register_overloaded_intrinsic (module, id, params, 2);
10011 }
10012
10013 static LLVMValueRef
10014 add_intrins3 (LLVMModuleRef module, IntrinsicId id, LLVMTypeRef param1, LLVMTypeRef param2, LLVMTypeRef param3)
10015 {
10016 LLVMTypeRef params [] = { param1, param2, param3 };
10017 return mono_llvm_register_overloaded_intrinsic (module, id, params, 3);
10018 }
10019
10020 static void
10021 add_intrinsic (LLVMModuleRef module, int id)
10022 {
10023 /* Register simple intrinsics */
10024 LLVMValueRef intrins = mono_llvm_register_intrinsic (module, (IntrinsicId)id);
10025 if (intrins) {
10026 g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (id), intrins);
10027 return;
10028 }
10029
10030 /* Register overloaded intrinsics */
10031 switch (id) {
10032 case INTRINS_MEMSET:
10033 intrins = add_intrins2 (module, id, LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ());
10034 break;
10035 case INTRINS_MEMCPY:
10036 intrins = add_intrins3 (module, id, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ());
10037 break;
10038 case INTRINS_MEMMOVE:
10039 intrins = add_intrins3 (module, id, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type ());
10040 break;
10041 case INTRINS_SADD_OVF_I32:
10042 case INTRINS_UADD_OVF_I32:
10043 case INTRINS_SSUB_OVF_I32:
10044 case INTRINS_USUB_OVF_I32:
10045 case INTRINS_SMUL_OVF_I32:
10046 case INTRINS_UMUL_OVF_I32:
10047 intrins = add_intrins1 (module, id, LLVMInt32Type ());
10048 break;
10049 case INTRINS_SADD_OVF_I64:
10050 case INTRINS_UADD_OVF_I64:
10051 case INTRINS_SSUB_OVF_I64:
10052 case INTRINS_USUB_OVF_I64:
10053 case INTRINS_SMUL_OVF_I64:
10054 case INTRINS_UMUL_OVF_I64:
10055 intrins = add_intrins1 (module, id, LLVMInt64Type ());
10056 break;
10057 case INTRINS_FMA:
10058 case INTRINS_EXP:
10059 case INTRINS_LOG:
10060 case INTRINS_LOG2:
10061 case INTRINS_LOG10:
10062 case INTRINS_TRUNC:
10063 case INTRINS_SIN:
10064 case INTRINS_COS:
10065 case INTRINS_SQRT:
10066 case INTRINS_FLOOR:
10067 case INTRINS_CEIL:
10068 case INTRINS_FABS:
10069 case INTRINS_COPYSIGN:
10070 case INTRINS_POW:
10071 intrins = add_intrins1 (module, id, LLVMDoubleType ());
10072 break;
10073 case INTRINS_FMAF:
10074 case INTRINS_EXPF:
10075 case INTRINS_LOG2F:
10076 case INTRINS_LOG10F:
10077 case INTRINS_TRUNCF:
10078 case INTRINS_SINF:
10079 case INTRINS_COSF:
10080 case INTRINS_SQRTF:
10081 case INTRINS_FLOORF:
10082 case INTRINS_CEILF:
10083 case INTRINS_ABSF:
10084 case INTRINS_COPYSIGNF:
10085 case INTRINS_POWF:
10086 intrins = add_intrins1 (module, id, LLVMFloatType ());
10087 break;
10088 case INTRINS_EXPECT_I8:
10089 intrins = add_intrins1 (module, id, LLVMInt8Type ());
10090 break;
10091 case INTRINS_EXPECT_I1:
10092 intrins = add_intrins1 (module, id, LLVMInt1Type ());
10093 break;
10094 case INTRINS_CTPOP_I32:
10095 case INTRINS_CTLZ_I32:
10096 case INTRINS_CTTZ_I32:
10097 case INTRINS_BEXTR_I32:
10098 case INTRINS_BZHI_I32:
10099 case INTRINS_PEXT_I32:
10100 case INTRINS_PDEP_I32:
10101 intrins = add_intrins1 (module, id, LLVMInt32Type ());
10102 break;
10103 case INTRINS_CTPOP_I64:
10104 case INTRINS_BEXTR_I64:
10105 case INTRINS_BZHI_I64:
10106 case INTRINS_PEXT_I64:
10107 case INTRINS_PDEP_I64:
10108 case INTRINS_CTLZ_I64:
10109 case INTRINS_CTTZ_I64:
10110 intrins = add_intrins1 (module, id, LLVMInt64Type ());
10111 break;
10112 #if defined(TARGET_AMD64) || defined(TARGET_X86)
10113 case INTRINS_SSE_SADD_SATI8:
10114 case INTRINS_SSE_UADD_SATI8:
10115 case INTRINS_SSE_SSUB_SATI8:
10116 case INTRINS_SSE_USUB_SATI8:
10117 intrins = add_intrins1 (module, id, sse_i1_t);
10118 break;
10119 case INTRINS_SSE_SADD_SATI16:
10120 case INTRINS_SSE_UADD_SATI16:
10121 case INTRINS_SSE_SSUB_SATI16:
10122 case INTRINS_SSE_USUB_SATI16:
10123 intrins = add_intrins1 (module, id, sse_i2_t);
10124 break;
10125 #if LLVM_API_VERSION >= 700
10126 case INTRINS_SSE_SQRT_PS:
10127 intrins = add_intrins1 (module, id, sse_r4_t);
10128 break;
10129 case INTRINS_SSE_SQRT_PD:
10130 intrins = add_intrins1 (module, id, sse_r8_t);
10131 break;
10132 case INTRINS_SSE_SQRT_SS:
10133 intrins = add_intrins1 (module, id, LLVMFloatType ());
10134 break;
10135 case INTRINS_SSE_SQRT_SD:
10136 intrins = add_intrins1 (module, id, LLVMDoubleType ());
10137 break;
10138 #endif /* LLVM_API_VERSION >= 700 */
10139 #endif /* AMD64 || X86 */
10140 #if defined(TARGET_WASM) && LLVM_API_VERSION >= 800
10141 case INTRINS_WASM_ANYTRUE_V16:
10142 intrins = add_intrins1 (module, id, sse_i1_t);
10143 break;
10144 case INTRINS_WASM_ANYTRUE_V8:
10145 intrins = add_intrins1 (module, id, sse_i2_t);
10146 break;
10147 case INTRINS_WASM_ANYTRUE_V4:
10148 intrins = add_intrins1 (module, id, sse_i4_t);
10149 break;
10150 case INTRINS_WASM_ANYTRUE_V2:
10151 intrins = add_intrins1 (module, id, sse_i8_t);
10152 break;
10153 #endif
10154 default:
10155 g_assert_not_reached ();
10156 break;
10157 }
10158 g_assert (intrins);
10159 g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (id), intrins);
10160 }
10161
10162 static LLVMValueRef
10163 get_intrins_from_module (LLVMModuleRef lmodule, int id)
10164 {
10165 LLVMValueRef res;
10166
10167 res = (LLVMValueRef)g_hash_table_lookup (intrins_id_to_intrins, GINT_TO_POINTER (id));
10168 g_assert (res);
10169 return res;
10170 }
10171
10172 static LLVMValueRef
10173 get_intrins (EmitContext *ctx, int id)
10174 {
10175 MonoLLVMModule *module = ctx->module;
10176 LLVMValueRef res;
10177
10178 /*
10179 * Every method is emitted into its own module so
10180 * we can add intrinsics on demand.
10181 */
10182 res = module->intrins_by_id [id];
10183 if (!res) {
10184 res = get_intrins_from_module (ctx->lmodule, id);
10185 module->intrins_by_id [id] = res;
10186 }
10187 return res;
10188 }
10189
10190 static void
10191 add_intrinsics (LLVMModuleRef module)
10192 {
10193 int i;
10194
10195 /* Emit declarations of instrinsics */
10196 /*
10197 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
10198 * type doesn't seem to do any locking.
10199 */
10200 for (i = 0; i < INTRINS_NUM; ++i)
10201 add_intrinsic (module, i);
10202
10203 /* EH intrinsics */
10204 add_func (module, "mono_personality", LLVMVoidType (), NULL, 0);
10205 add_func (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
10206 }
10207
10208 static void
10209 add_types (MonoLLVMModule *module)
10210 {
10211 module->ptr_type = LLVMPointerType (TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
10212 }
10213
10214 void
10215 mono_llvm_init (gboolean enable_jit)
10216 {
10217 sse_i1_t = type_to_sse_type (MONO_TYPE_I1);
10218 sse_i2_t = type_to_sse_type (MONO_TYPE_I2);
10219 sse_i4_t = type_to_sse_type (MONO_TYPE_I4);
10220 sse_i8_t = type_to_sse_type (MONO_TYPE_I8);
10221 sse_r4_t = type_to_sse_type (MONO_TYPE_R4);
10222 sse_r8_t = type_to_sse_type (MONO_TYPE_R8);
10223
10224 intrins_id_to_intrins = g_hash_table_new (NULL, NULL);
10225
10226 if (enable_jit)
10227 mono_llvm_jit_init ();
10228 }
10229
10230 void
10231 mono_llvm_cleanup (void)
10232 {
10233 MonoLLVMModule *module = &aot_module;
10234
10235 if (module->lmodule)
10236 LLVMDisposeModule (module->lmodule);
10237
10238 if (module->context)
10239 LLVMContextDispose (module->context);
10240 }
10241
10242 void
10243 mono_llvm_free_domain_info (MonoDomain *domain)
10244 {
10245 MonoJitDomainInfo *info = domain_jit_info (domain);
10246 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
10247 int i;
10248
10249 if (!module)
10250 return;
10251
10252 g_hash_table_destroy (module->llvm_types);
10253
10254 mono_llvm_dispose_ee (module->mono_ee);
10255
10256 if (module->bb_names) {
10257 for (i = 0; i < module->bb_names_len; ++i)
10258 g_free (module->bb_names [i]);
10259 g_free (module->bb_names);
10260 }
10261 //LLVMDisposeModule (module->module);
10262
10263 g_free (module);
10264
10265 info->llvm_module = NULL;
10266 }
10267
10268 void
10269 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, int initial_got_size, LLVMModuleFlags flags)
10270 {
10271 MonoLLVMModule *module = &aot_module;
10272 gboolean emit_dwarf = (flags & LLVM_MODULE_FLAG_DWARF) ? 1 : 0;
10273 #ifdef TARGET_WIN32_MSVC
10274 gboolean emit_codeview = (flags & LLVM_MODULE_FLAG_CODEVIEW) ? 1 : 0;
10275 #endif
10276 gboolean static_link = (flags & LLVM_MODULE_FLAG_STATIC) ? 1 : 0;
10277 gboolean llvm_only = (flags & LLVM_MODULE_FLAG_LLVM_ONLY) ? 1 : 0;
10278 gboolean interp = (flags & LLVM_MODULE_FLAG_INTERP) ? 1 : 0;
10279
10280 /* Delete previous module */
10281 g_hash_table_destroy (module->plt_entries);
10282 if (module->lmodule)
10283 LLVMDisposeModule (module->lmodule);
10284
10285 memset (module, 0, sizeof (aot_module));
10286
10287 module->lmodule = LLVMModuleCreateWithName ("aot");
10288 module->assembly = assembly;
10289 module->global_prefix = g_strdup (global_prefix);
10290 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
10291 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
10292 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
10293 module->init_aotconst_symbol = g_strdup_printf ("%s_init_aotconst", global_prefix);
10294 module->external_symbols = TRUE;
10295 module->emit_dwarf = emit_dwarf;
10296 module->static_link = static_link;
10297 module->llvm_only = llvm_only;
10298 module->interp = interp;
10299 /* The first few entries are reserved */
10300 module->max_got_offset = initial_got_size;
10301 module->context = LLVMGetGlobalContext ();
10302 module->cfgs = g_ptr_array_new ();
10303 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
10304 module->aotconst_vars = g_hash_table_new (NULL, NULL);
10305 module->llvm_types = g_hash_table_new (NULL, NULL);
10306 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
10307 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
10308 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
10309 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
10310 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
10311 module->method_to_call_info = g_hash_table_new (NULL, NULL);
10312 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
10313 module->no_method_table_lmethods = g_hash_table_new (NULL, NULL);
10314 module->callsite_list = g_ptr_array_new ();
10315
10316 if (llvm_only)
10317 /* clang ignores our debug info because it has an invalid version */
10318 module->emit_dwarf = FALSE;
10319
10320 add_intrinsics (module->lmodule);
10321 add_types (module);
10322
10323 #ifdef MONO_ARCH_LLVM_TARGET_LAYOUT
10324 LLVMSetDataLayout (module->lmodule, MONO_ARCH_LLVM_TARGET_LAYOUT);
10325 #else
10326 g_assert_not_reached ();
10327 #endif
10328
10329 #ifdef MONO_ARCH_LLVM_TARGET_TRIPLE
10330 LLVMSetTarget (module->lmodule, MONO_ARCH_LLVM_TARGET_TRIPLE);
10331 #endif
10332
10333 if (module->emit_dwarf) {
10334 char *dir, *build_info, *s, *cu_name;
10335
10336 module->di_builder = mono_llvm_create_di_builder (module->lmodule);
10337
10338 // FIXME:
10339 dir = g_strdup (".");
10340 build_info = mono_get_runtime_build_info ();
10341 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
10342 cu_name = g_path_get_basename (assembly->image->name);
10343 module->cu = mono_llvm_di_create_compile_unit (module->di_builder, cu_name, dir, s);
10344 g_free (dir);
10345 g_free (build_info);
10346 g_free (s);
10347 }
10348
10349 #ifdef TARGET_WIN32_MSVC
10350 if (emit_codeview) {
10351 LLVMValueRef codeview_option_args[3];
10352
10353 codeview_option_args[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
10354 codeview_option_args[1] = LLVMMDString ("CodeView", 8);
10355 codeview_option_args[2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
10356
10357 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.module.flags", LLVMMDNode (codeview_option_args, G_N_ELEMENTS (codeview_option_args)));
10358 }
10359
10360 if (!static_link) {
10361 const char linker_options[] = "Linker Options";
10362 const char *default_dynamic_lib_names[] = { "/DEFAULTLIB:msvcrt",
10363 "/DEFAULTLIB:ucrt.lib",
10364 "/DEFAULTLIB:vcruntime.lib" };
10365
10366 LLVMValueRef default_lib_args[G_N_ELEMENTS (default_dynamic_lib_names)];
10367 LLVMValueRef default_lib_nodes[G_N_ELEMENTS(default_dynamic_lib_names)];
10368
10369 const char *default_lib_name = NULL;
10370 for (int i = 0; i < G_N_ELEMENTS (default_dynamic_lib_names); ++i) {
10371 const char *default_lib_name = default_dynamic_lib_names[i];
10372 default_lib_args[i] = LLVMMDString (default_lib_name, strlen (default_lib_name));
10373 default_lib_nodes[i] = LLVMMDNode (default_lib_args + i, 1);
10374 }
10375
10376 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.linker.options", LLVMMDNode (default_lib_args, G_N_ELEMENTS (default_lib_args)));
10377 }
10378 #endif
10379
10380 {
10381 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 16);
10382
10383 module->dummy_got_var = LLVMAddGlobal (module->lmodule, got_type, "dummy_got");
10384 module->got_idx_to_type = g_hash_table_new (NULL, NULL);
10385 LLVMSetInitializer (module->dummy_got_var, LLVMConstNull (got_type));
10386 LLVMSetVisibility (module->dummy_got_var, LLVMHiddenVisibility);
10387 LLVMSetLinkage (module->dummy_got_var, LLVMInternalLinkage);
10388 }
10389
10390 /* Add initialization array */
10391 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
10392
10393 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
10394 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
10395
10396 create_aot_info_var (module);
10397
10398 emit_gc_safepoint_poll (module, module->lmodule, NULL);
10399
10400 emit_llvm_code_start (module);
10401
10402 // Needs idx_to_lmethod
10403 emit_init_funcs (module);
10404
10405 /* Add a dummy personality function */
10406 if (!use_mono_personality_debug) {
10407 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
10408 LLVMSetLinkage (personality, LLVMExternalLinkage);
10409
10410 //EMCC chockes if the personality function is referenced in the 'used' array
10411 #ifndef TARGET_WASM
10412 mark_as_used (module, personality);
10413 #endif
10414 }
10415
10416 /* Add a reference to the c++ exception we throw/catch */
10417 {
10418 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
10419 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
10420 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
10421 mono_llvm_set_is_constant (module->sentinel_exception);
10422 }
10423 }
10424
10425 void
10426 mono_llvm_fixup_aot_module (void)
10427 {
10428 MonoLLVMModule *module = &aot_module;
10429 MonoMethod *method;
10430
10431 /*
10432 * Replace GOT entries for directly callable methods with the methods themselves.
10433 * It would be easier to implement this by predefining all methods before compiling
10434 * their bodies, but that couldn't handle the case when a method fails to compile
10435 * with llvm.
10436 */
10437
10438 GHashTable *specializable = g_hash_table_new (NULL, NULL);
10439
10440 GHashTable *patches_to_null = g_hash_table_new (mono_patch_info_hash, mono_patch_info_equal);
10441 for (int sindex = 0; sindex < module->callsite_list->len; ++sindex) {
10442 CallSite *site = (CallSite*)g_ptr_array_index (module->callsite_list, sindex);
10443 method = site->method;
10444 LLVMValueRef lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
10445 LLVMValueRef placeholder = (LLVMValueRef)site->load;
10446 LLVMValueRef load;
10447
10448 gboolean can_direct_call = FALSE;
10449
10450 /* Replace sharable instances with their shared version */
10451 if (!lmethod && method->is_inflated) {
10452 if (mono_method_is_generic_sharable_full (method, FALSE, TRUE, FALSE)) {
10453 ERROR_DECL (error);
10454 MonoMethod *shared = mini_get_shared_method_full (method, SHARE_MODE_NONE, error);
10455 if (is_ok (error)) {
10456 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, shared);
10457 if (lmethod)
10458 method = shared;
10459 }
10460 }
10461 }
10462
10463 if (lmethod && !m_method_is_synchronized (method)) {
10464 can_direct_call = TRUE;
10465 } else if (m_method_is_wrapper (method) && !method->is_inflated) {
10466 WrapperInfo *info = mono_marshal_get_wrapper_info (method);
10467
10468 /* This is a call from the synchronized wrapper to the real method */
10469 if (info->subtype == WRAPPER_SUBTYPE_SYNCHRONIZED_INNER) {
10470 method = info->d.synchronized.method;
10471 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
10472 if (lmethod)
10473 can_direct_call = TRUE;
10474 }
10475 }
10476
10477 if (can_direct_call) {
10478 mono_llvm_replace_uses_of (placeholder, lmethod);
10479
10480 if (mono_aot_can_specialize (method))
10481 g_hash_table_insert (specializable, lmethod, method);
10482
10483 g_hash_table_insert (patches_to_null, site->ji, site->ji);
10484 } else {
10485 // FIXME:
10486 LLVMBuilderRef builder = LLVMCreateBuilder ();
10487 LLVMPositionBuilderBefore (builder, placeholder);
10488
10489 load = get_aotconst_module (module, builder, site->ji->type, site->ji->data.target, site->type, NULL, NULL);
10490 LLVMReplaceAllUsesWith (placeholder, load);
10491 }
10492 g_free (site);
10493 }
10494
10495 mono_llvm_propagate_nonnull_final (specializable, module);
10496 g_hash_table_destroy (specializable);
10497
10498 for (int i = 0; i < module->cfgs->len; ++i) {
10499 /*
10500 * Nullify the patches pointing to direct calls. This is needed to
10501 * avoid allocating extra got slots, which is a perf problem and it
10502 * makes module->max_got_offset invalid.
10503 * It would be better to just store the patch_info in CallSite, but
10504 * cfg->patch_info is copied in aot-compiler.c.
10505 */
10506 MonoCompile *cfg = (MonoCompile *)g_ptr_array_index (module->cfgs, i);
10507 for (MonoJumpInfo *patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
10508 if (patch_info->type == MONO_PATCH_INFO_METHOD) {
10509 if (g_hash_table_lookup (patches_to_null, patch_info)) {
10510 patch_info->type = MONO_PATCH_INFO_NONE;
10511 /* Nullify the call to init_method () if possible */
10512 g_assert (cfg->got_access_count);
10513 cfg->got_access_count --;
10514 if (cfg->got_access_count == 0) {
10515 LLVMValueRef br = (LLVMValueRef)cfg->llvmonly_init_cond;
10516 if (br)
10517 LLVMSetSuccessor (br, 0, LLVMGetSuccessor (br, 1));
10518 }
10519 }
10520 }
10521 }
10522 }
10523
10524 g_hash_table_destroy (patches_to_null);
10525 }
10526
10527 static LLVMValueRef
10528 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
10529 {
10530 int i;
10531 LLVMValueRef res, *vals;
10532
10533 vals = g_new0 (LLVMValueRef, nvalues);
10534 for (i = 0; i < nvalues; ++i)
10535 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
10536 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
10537 g_free (vals);
10538 return res;
10539 }
10540
10541 static LLVMValueRef
10542 llvm_array_from_bytes (guint8 *values, int nvalues)
10543 {
10544 int i;
10545 LLVMValueRef res, *vals;
10546
10547 vals = g_new0 (LLVMValueRef, nvalues);
10548 for (i = 0; i < nvalues; ++i)
10549 vals [i] = LLVMConstInt (LLVMInt8Type (), values [i], FALSE);
10550 res = LLVMConstArray (LLVMInt8Type (), vals, nvalues);
10551 g_free (vals);
10552 return res;
10553 }
10554 /*
10555 * mono_llvm_emit_aot_file_info:
10556 *
10557 * Emit the MonoAotFileInfo structure.
10558 * Same as emit_aot_file_info () in aot-compiler.c.
10559 */
10560 void
10561 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
10562 {
10563 MonoLLVMModule *module = &aot_module;
10564
10565 /* Save these for later */
10566 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
10567 module->has_jitted_code = has_jitted_code;
10568 }
10569
10570 /*
10571 * mono_llvm_emit_aot_data:
10572 *
10573 * Emit the binary data DATA pointed to by symbol SYMBOL.
10574 * Return the LLVM variable for the data.
10575 */
10576 gpointer
10577 mono_llvm_emit_aot_data_aligned (const char *symbol, guint8 *data, int data_len, int align)
10578 {
10579 MonoLLVMModule *module = &aot_module;
10580 LLVMTypeRef type;
10581 LLVMValueRef d;
10582
10583 type = LLVMArrayType (LLVMInt8Type (), data_len);
10584 d = LLVMAddGlobal (module->lmodule, type, symbol);
10585 LLVMSetVisibility (d, LLVMHiddenVisibility);
10586 LLVMSetLinkage (d, LLVMInternalLinkage);
10587 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
10588 if (align != 1)
10589 LLVMSetAlignment (d, align);
10590 mono_llvm_set_is_constant (d);
10591 return d;
10592 }
10593
10594 gpointer
10595 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
10596 {
10597 return mono_llvm_emit_aot_data_aligned (symbol, data, data_len, 8);
10598 }
10599
10600 /* Add a reference to a global defined in JITted code */
10601 static LLVMValueRef
10602 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
10603 {
10604 char *s;
10605 LLVMValueRef v;
10606
10607 s = g_strdup_printf ("%s%s", module->global_prefix, name);
10608 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
10609 LLVMSetVisibility (v, LLVMHiddenVisibility);
10610 g_free (s);
10611 return v;
10612 }
10613 #define FILE_INFO_NUM_HEADER_FIELDS 2
10614 #define FILE_INFO_NUM_SCALAR_FIELDS 23
10615 #define FILE_INFO_NUM_ARRAY_FIELDS 5
10616 #define FILE_INFO_NUM_AOTID_FIELDS 1
10617 #define FILE_INFO_NFIELDS (FILE_INFO_NUM_HEADER_FIELDS + MONO_AOT_FILE_INFO_NUM_SYMBOLS + FILE_INFO_NUM_SCALAR_FIELDS + FILE_INFO_NUM_ARRAY_FIELDS + FILE_INFO_NUM_AOTID_FIELDS)
10618
10619 static void
10620 create_aot_info_var (MonoLLVMModule *module)
10621 {
10622 LLVMTypeRef file_info_type;
10623 LLVMTypeRef *eltypes;
10624 LLVMValueRef info_var;
10625 int i, nfields, tindex;
10626 LLVMModuleRef lmodule = module->lmodule;
10627
10628 /* Create an LLVM type to represent MonoAotFileInfo */
10629 nfields = FILE_INFO_NFIELDS;
10630 eltypes = g_new (LLVMTypeRef, nfields);
10631 tindex = 0;
10632 eltypes [tindex ++] = LLVMInt32Type ();
10633 eltypes [tindex ++] = LLVMInt32Type ();
10634 /* Symbols */
10635 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
10636 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
10637 /* Scalars */
10638 for (i = 0; i < FILE_INFO_NUM_SCALAR_FIELDS; ++i)
10639 eltypes [tindex ++] = LLVMInt32Type ();
10640 /* Arrays */
10641 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
10642 for (i = 0; i < FILE_INFO_NUM_ARRAY_FIELDS - 1; ++i)
10643 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
10644 eltypes [tindex ++] = LLVMArrayType (LLVMInt8Type (), 16);
10645 g_assert (tindex == nfields);
10646 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
10647 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
10648
10649 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
10650
10651 module->info_var = info_var;
10652 module->info_var_eltypes = eltypes;
10653 }
10654
10655 static void
10656 emit_aot_file_info (MonoLLVMModule *module)
10657 {
10658 LLVMTypeRef *eltypes, eltype;
10659 LLVMValueRef info_var;
10660 LLVMValueRef *fields;
10661 int i, nfields, tindex;
10662 MonoAotFileInfo *info;
10663 LLVMModuleRef lmodule = module->lmodule;
10664
10665 info = &module->aot_info;
10666 info_var = module->info_var;
10667 eltypes = module->info_var_eltypes;
10668 nfields = FILE_INFO_NFIELDS;
10669
10670 if (module->static_link) {
10671 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
10672 LLVMSetLinkage (info_var, LLVMInternalLinkage);
10673 }
10674
10675 #ifdef TARGET_WIN32
10676 if (!module->static_link) {
10677 LLVMSetDLLStorageClass (info_var, LLVMDLLExportStorageClass);
10678 }
10679 #endif
10680
10681 fields = g_new (LLVMValueRef, nfields);
10682 tindex = 0;
10683 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
10684 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
10685
10686 /* Symbols */
10687 /*
10688 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
10689 * for symbols defined in the .s file emitted by the aot compiler.
10690 */
10691 eltype = eltypes [tindex];
10692 if (module->llvm_only)
10693 fields [tindex ++] = LLVMConstNull (eltype);
10694 else
10695 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
10696 /* llc defines this directly */
10697 if (!module->llvm_only) {
10698 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
10699 fields [tindex ++] = LLVMConstNull (eltype);
10700 fields [tindex ++] = LLVMConstNull (eltype);
10701 } else {
10702 fields [tindex ++] = LLVMConstNull (eltype);
10703 fields [tindex ++] = module->get_method;
10704 fields [tindex ++] = module->get_unbox_tramp ? module->get_unbox_tramp : LLVMConstNull (eltype);
10705 }
10706 fields [tindex ++] = module->init_aotconst_func;
10707 if (module->has_jitted_code) {
10708 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
10709 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
10710 } else {
10711 fields [tindex ++] = LLVMConstNull (eltype);
10712 fields [tindex ++] = LLVMConstNull (eltype);
10713 }
10714 if (!module->llvm_only)
10715 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
10716 else
10717 fields [tindex ++] = LLVMConstNull (eltype);
10718 if (module->llvm_only && module->unbox_tramp_indexes) {
10719 fields [tindex ++] = module->unbox_tramp_indexes;
10720 fields [tindex ++] = module->unbox_trampolines;
10721 } else {
10722 fields [tindex ++] = LLVMConstNull (eltype);
10723 fields [tindex ++] = LLVMConstNull (eltype);
10724 }
10725 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
10726 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
10727 fields [tindex ++] = LLVMConstNull (eltype);
10728 } else {
10729 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
10730 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
10731 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
10732 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
10733 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
10734 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
10735 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
10736 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
10737 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
10738 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
10739 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "weak_field_indexes");
10740 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_flags_table");
10741 }
10742 /* Not needed (mem_end) */
10743 fields [tindex ++] = LLVMConstNull (eltype);
10744 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
10745 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
10746 if (info->trampoline_size [0]) {
10747 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
10748 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
10749 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_trampolines");
10750 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
10751 fields [tindex ++] = AddJitGlobal (module, eltype, "ftnptr_arg_trampolines");
10752 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_arbitrary_trampolines");
10753 } else {
10754 fields [tindex ++] = LLVMConstNull (eltype);
10755 fields [tindex ++] = LLVMConstNull (eltype);
10756 fields [tindex ++] = LLVMConstNull (eltype);
10757 fields [tindex ++] = LLVMConstNull (eltype);
10758 fields [tindex ++] = LLVMConstNull (eltype);
10759 fields [tindex ++] = LLVMConstNull (eltype);
10760 }
10761 if (module->static_link && !module->llvm_only)
10762 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
10763 else
10764 fields [tindex ++] = LLVMConstNull (eltype);
10765 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
10766 if (!module->llvm_only) {
10767 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
10768 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
10769 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
10770 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
10771 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
10772 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
10773 } else {
10774 fields [tindex ++] = LLVMConstNull (eltype);
10775 fields [tindex ++] = LLVMConstNull (eltype);
10776 fields [tindex ++] = LLVMConstNull (eltype);
10777 fields [tindex ++] = LLVMConstNull (eltype);
10778 fields [tindex ++] = LLVMConstNull (eltype);
10779 fields [tindex ++] = LLVMConstNull (eltype);
10780 }
10781
10782 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i) {
10783 g_assert (fields [FILE_INFO_NUM_HEADER_FIELDS + i]);
10784 fields [FILE_INFO_NUM_HEADER_FIELDS + i] = LLVMConstBitCast (fields [FILE_INFO_NUM_HEADER_FIELDS + i], eltype);
10785 }
10786
10787 /* Scalars */
10788 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
10789 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_info_offset_base, FALSE);
10790 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
10791 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->llvm_got_size, FALSE);
10792 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
10793 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
10794 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nextra_methods, FALSE);
10795 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
10796 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
10797 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
10798 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
10799 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
10800 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
10801 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
10802 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
10803 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_shift_bits, FALSE);
10804 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_mask, FALSE);
10805 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
10806 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->call_table_entry_size, FALSE);
10807 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
10808 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
10809 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_num, FALSE);
10810 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_elemsize, FALSE);
10811 /* Arrays */
10812 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
10813 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
10814 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
10815 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
10816 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
10817
10818 fields [tindex ++] = llvm_array_from_bytes (info->aotid, 16);
10819 g_assert (tindex == nfields);
10820
10821 LLVMSetInitializer (info_var, LLVMConstNamedStruct (LLVMGetElementType (LLVMTypeOf (info_var)), fields, nfields));
10822
10823 if (module->static_link) {
10824 char *s, *p;
10825 LLVMValueRef var;
10826
10827 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
10828 /* Get rid of characters which cannot occur in symbols */
10829 p = s;
10830 for (p = s; *p; ++p) {
10831 if (!(isalnum (*p) || *p == '_'))
10832 *p = '_';
10833 }
10834 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
10835 g_free (s);
10836 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
10837 LLVMSetLinkage (var, LLVMExternalLinkage);
10838 }
10839 }
10840
10841 typedef struct {
10842 LLVMValueRef lmethod;
10843 int argument;
10844 } NonnullPropWorkItem;
10845
10846 static void
10847 mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params)
10848 {
10849 if (mono_aot_can_specialize (call_method)) {
10850 int num_passed = LLVMGetNumArgOperands (lcall);
10851 g_assert (num_params <= num_passed);
10852
10853 g_assert (ctx->module->method_to_call_info);
10854 GArray *call_site_union = (GArray *) g_hash_table_lookup (ctx->module->method_to_call_info, call_method);
10855
10856 if (!call_site_union) {
10857 call_site_union = g_array_sized_new (FALSE, TRUE, sizeof (gint32), num_params);
10858 int zero = 0;
10859 for (int i = 0; i < num_params; i++)
10860 g_array_insert_val (call_site_union, i, zero);
10861 }
10862
10863 for (int i = 0; i < num_params; i++) {
10864 if (mono_llvm_is_nonnull (args [i])) {
10865 g_assert (i < LLVMGetNumArgOperands (lcall));
10866 mono_llvm_set_call_nonnull_arg (lcall, i);
10867 } else {
10868 gint32 *nullable_count = &g_array_index (call_site_union, gint32, i);
10869 *nullable_count = *nullable_count + 1;
10870 }
10871 }
10872
10873 g_hash_table_insert (ctx->module->method_to_call_info, call_method, call_site_union);
10874 }
10875 }
10876
10877 static void
10878 mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module)
10879 {
10880 // When we first traverse the mini IL, we mark the things that are
10881 // nonnull (the roots). Then, for all of the methods that can be specialized, we
10882 // see if their call sites have nonnull attributes.
10883
10884 // If so, we mark the function's param. This param has uses to propagate
10885 // the attribute to. This propagation can trigger a need to mark more attributes
10886 // non-null, and so on and so forth.
10887 GSList *queue = NULL;
10888
10889 GHashTableIter iter;
10890 LLVMValueRef lmethod;
10891 MonoMethod *method;
10892 g_hash_table_iter_init (&iter, all_specializable);
10893 while (g_hash_table_iter_next (&iter, (void**)&lmethod, (void**)&method)) {
10894 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, method);
10895
10896 // Basic sanity checking
10897 if (call_site_union)
10898 g_assert (call_site_union->len == LLVMCountParams (lmethod));
10899
10900 // Add root to work queue
10901 for (int i = 0; call_site_union && i < call_site_union->len; i++) {
10902 if (g_array_index (call_site_union, gint32, i) == 0) {
10903 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
10904 item->lmethod = lmethod;
10905 item->argument = i;
10906 queue = g_slist_prepend (queue, item);
10907 }
10908 }
10909 }
10910
10911 // This is essentially reference counting, and we are propagating
10912 // the refcount decrement here. We have less work to do than we may otherwise
10913 // because we are only working with a set of subgraphs of specializable functions.
10914 //
10915 // We rely on being able to see all of the references in the graph.
10916 // This is ensured by the function mono_aot_can_specialize. Everything in
10917 // all_specializable is a function that can be specialized, and is the resulting
10918 // node in the graph after all of the subsitutions are done.
10919 //
10920 // Anything disrupting the direct calls made with self-init will break this optimization.
10921
10922 while (queue) {
10923 // Update the queue state.
10924 // Our only other per-iteration responsibility is now to free current
10925 NonnullPropWorkItem *current = (NonnullPropWorkItem *) queue->data;
10926 queue = queue->next;
10927 g_assert (current->argument < LLVMCountParams (current->lmethod));
10928
10929 // Does the actual leaf-node work here
10930 // Mark the function argument as nonnull for LLVM
10931 mono_llvm_set_func_nonnull_arg (current->lmethod, current->argument);
10932
10933 // The rest of this is for propagating forward nullability changes
10934 // to calls that use the argument that is now nullable.
10935
10936 // Get the actual LLVM value of the argument, so we can see which call instructions
10937 // used that argument
10938 LLVMValueRef caller_argument = LLVMGetParam (current->lmethod, current->argument);
10939
10940 // Iterate over the calls using the newly-non-nullable argument
10941 GSList *calls = mono_llvm_calls_using (caller_argument);
10942 for (GSList *cursor = calls; cursor != NULL; cursor = cursor->next) {
10943
10944 LLVMValueRef lcall = (LLVMValueRef) cursor->data;
10945 LLVMValueRef callee_lmethod = LLVMGetCalledValue (lcall);
10946
10947 // If this wasn't a direct call for which mono_aot_can_specialize is true,
10948 // this lookup won't find a MonoMethod.
10949 MonoMethod *callee_method = (MonoMethod *) g_hash_table_lookup (all_specializable, callee_lmethod);
10950 if (!callee_method)
10951 continue;
10952
10953 // Decrement number of nullable refs at that func's arg offset
10954 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, callee_method);
10955
10956 // It has module-local callers and is specializable, should have seen this call site
10957 // and inited this
10958 g_assert (call_site_union);
10959
10960 // The function *definition* parameter arity should always be consistent
10961 int max_params = LLVMCountParams (callee_lmethod);
10962 if (call_site_union->len != max_params) {
10963 mono_llvm_dump_value (callee_lmethod);
10964 g_assert_not_reached ();
10965 }
10966
10967 // Get the values that correspond to the parameters passed to the call
10968 // that used our argument
10969 LLVMValueRef *operands = mono_llvm_call_args (lcall);
10970 for (int call_argument = 0; call_argument < max_params; call_argument++) {
10971 // Every time we used the newly-non-nullable argument, decrement the nullable
10972 // refcount for that function.
10973 if (caller_argument == operands [call_argument]) {
10974 gint32 *nullable_count = &g_array_index (call_site_union, gint32, call_argument);
10975 g_assert (*nullable_count > 0);
10976 *nullable_count = *nullable_count - 1;
10977
10978 // If we caused that callee's parameter to become newly nullable, add to work queue
10979 if (*nullable_count == 0) {
10980 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
10981 item->lmethod = callee_lmethod;
10982 item->argument = call_argument;
10983 queue = g_slist_prepend (queue, item);
10984 }
10985 }
10986 }
10987 g_free (operands);
10988
10989 // Update nullability refcount information for the callee now
10990 g_hash_table_insert (module->method_to_call_info, callee_method, call_site_union);
10991 }
10992 g_slist_free (calls);
10993
10994 g_free (current);
10995 }
10996 }
10997
10998 /*
10999 * Emit the aot module into the LLVM bitcode file FILENAME.
11000 */
11001 void
11002 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
11003 {
11004 LLVMTypeRef inited_type;
11005 LLVMValueRef real_inited;
11006 MonoLLVMModule *module = &aot_module;
11007
11008 emit_llvm_code_end (module);
11009
11010 /*
11011 * Create the real init_var and replace all uses of the dummy variable with
11012 * the real one.
11013 */
11014 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
11015 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
11016 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
11017 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
11018 mono_llvm_replace_uses_of (module->inited_var, real_inited);
11019 LLVMDeleteGlobal (module->inited_var);
11020
11021 /* Replace the dummy info_ variables with the real ones */
11022 for (int i = 0; i < module->cfgs->len; ++i) {
11023 MonoCompile *cfg = (MonoCompile *)g_ptr_array_index (module->cfgs, i);
11024
11025 // FIXME: Eliminate unused vars
11026
11027 // FIXME: Speed this up
11028 if (cfg->llvm_dummy_info_var) {
11029 if (cfg->llvm_info_var) {
11030 mono_llvm_replace_uses_of (cfg->llvm_dummy_info_var, cfg->llvm_info_var);
11031 LLVMDeleteGlobal (cfg->llvm_dummy_info_var);
11032 } else {
11033 // FIXME: How can this happen ?
11034 LLVMSetInitializer (cfg->llvm_dummy_info_var, mono_llvm_create_constant_data_array (NULL, 0));
11035 }
11036 }
11037 }
11038
11039 if (module->llvm_only) {
11040 emit_get_method (&aot_module);
11041 emit_get_unbox_tramp (&aot_module);
11042 }
11043 emit_init_aotconst (module);
11044
11045 emit_llvm_used (&aot_module);
11046 emit_dbg_info (&aot_module, filename, cu_name);
11047 emit_aot_file_info (&aot_module);
11048
11049 /* Replace PLT entries for directly callable methods with the methods themselves */
11050 {
11051 GHashTableIter iter;
11052 MonoJumpInfo *ji;
11053 LLVMValueRef callee;
11054
11055 GHashTable *specializable = g_hash_table_new (NULL, NULL);
11056
11057 g_hash_table_iter_init (&iter, module->plt_entries_ji);
11058 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
11059 if (mono_aot_is_direct_callable (ji)) {
11060 LLVMValueRef lmethod;
11061
11062 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
11063 /* The types might not match because the caller might pass an rgctx */
11064 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
11065 mono_llvm_replace_uses_of (callee, lmethod);
11066
11067 if (mono_aot_can_specialize (ji->data.method))
11068 g_hash_table_insert (specializable, lmethod, ji->data.method);
11069 mono_aot_mark_unused_llvm_plt_entry (ji);
11070 }
11071 }
11072 }
11073
11074 mono_llvm_propagate_nonnull_final (specializable, module);
11075
11076 g_hash_table_destroy (specializable);
11077 }
11078
11079 /* Note: You can still dump an invalid bitcode file by running `llvm-dis`
11080 * in a debugger, set a breakpoint on `LLVMVerifyModule` and fake its
11081 * result to 0 (indicating success). */
11082 LLVMWriteBitcodeToFile (module->lmodule, filename);
11083
11084 #if 0
11085 {
11086 char *verifier_err;
11087
11088 if (LLVMVerifyModule (module->lmodule, LLVMReturnStatusAction, &verifier_err)) {
11089 printf ("%s\n", verifier_err);
11090 g_assert_not_reached ();
11091 }
11092 }
11093 #endif
11094 }
11095
11096
11097 static LLVMValueRef
11098 md_string (const char *s)
11099 {
11100 return LLVMMDString (s, strlen (s));
11101 }
11102
11103 /* Debugging support */
11104
11105 static void
11106 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
11107 {
11108 LLVMModuleRef lmodule = module->lmodule;
11109 LLVMValueRef args [16], ver;
11110
11111 /*
11112 * This can only be enabled when LLVM code is emitted into a separate object
11113 * file, since the AOT compiler also emits dwarf info,
11114 * and the abbrev indexes will not be correct since llvm has added its own
11115 * abbrevs.
11116 */
11117 if (!module->emit_dwarf)
11118 return;
11119
11120 mono_llvm_di_builder_finalize (module->di_builder);
11121
11122 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
11123 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
11124 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
11125 ver = LLVMMDNode (args, 3);
11126 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
11127
11128 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
11129 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
11130 args [2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE);
11131 ver = LLVMMDNode (args, 3);
11132 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
11133 }
11134
11135 static LLVMValueRef
11136 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
11137 {
11138 MonoLLVMModule *module = ctx->module;
11139 MonoDebugMethodInfo *minfo = ctx->minfo;
11140 char *source_file, *dir, *filename;
11141 MonoSymSeqPoint *sym_seq_points;
11142 int n_seq_points;
11143
11144 if (!minfo)
11145 return NULL;
11146
11147 mono_debug_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
11148 if (!source_file)
11149 source_file = g_strdup ("<unknown>");
11150 dir = g_path_get_dirname (source_file);
11151 filename = g_path_get_basename (source_file);
11152 g_free (source_file);
11153
11154 return (LLVMValueRef)mono_llvm_di_create_function (module->di_builder, module->cu, method, cfg->method->name, name, dir, filename, n_seq_points ? sym_seq_points [0].line : 1);
11155 }
11156
11157 static void
11158 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
11159 {
11160 MonoCompile *cfg = ctx->cfg;
11161
11162 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
11163 MonoDebugSourceLocation *loc;
11164 LLVMValueRef loc_md;
11165
11166 loc = mono_debug_method_lookup_location (ctx->minfo, cil_code - cfg->header->code);
11167
11168 if (loc) {
11169 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, loc->row, loc->column);
11170 mono_llvm_di_set_location (builder, loc_md);
11171 mono_debug_free_source_location (loc);
11172 }
11173 }
11174 }
11175
11176 static void
11177 emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder)
11178 {
11179 if (ctx->minfo) {
11180 LLVMValueRef loc_md;
11181 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, 0, 0);
11182 mono_llvm_di_set_location (builder, loc_md);
11183 }
11184 }
11185
11186 /*
11187 DESIGN:
11188 - Emit LLVM IR from the mono IR using the LLVM C API.
11189 - The original arch specific code remains, so we can fall back to it if we run
11190 into something we can't handle.
11191 */
11192
11193 /*
11194 A partial list of issues:
11195 - Handling of opcodes which can throw exceptions.
11196
11197 In the mono JIT, these are implemented using code like this:
11198 method:
11199 <compare>
11200 throw_pos:
11201 b<cond> ex_label
11202 <rest of code>
11203 ex_label:
11204 push throw_pos - method
11205 call <exception trampoline>
11206
11207 The problematic part is push throw_pos - method, which cannot be represented
11208 in the LLVM IR, since it does not support label values.
11209 -> this can be implemented in AOT mode using inline asm + labels, but cannot
11210 be implemented in JIT mode ?
11211 -> a possible but slower implementation would use the normal exception
11212 throwing code but it would need to control the placement of the throw code
11213 (it needs to be exactly after the compare+branch).
11214 -> perhaps add a PC offset intrinsics ?
11215
11216 - efficient implementation of .ovf opcodes.
11217
11218 These are currently implemented as:
11219 <ins which sets the condition codes>
11220 b<cond> ex_label
11221
11222 Some overflow opcodes are now supported by LLVM SVN.
11223
11224 - exception handling, unwinding.
11225 - SSA is disabled for methods with exception handlers
11226 - How to obtain unwind info for LLVM compiled methods ?
11227 -> this is now solved by converting the unwind info generated by LLVM
11228 into our format.
11229 - LLVM uses the c++ exception handling framework, while we use our home grown
11230 code, and couldn't use the c++ one:
11231 - its not supported under VC++, other exotic platforms.
11232 - it might be impossible to support filter clauses with it.
11233
11234 - trampolines.
11235
11236 The trampolines need a predictable call sequence, since they need to disasm
11237 the calling code to obtain register numbers / offsets.
11238
11239 LLVM currently generates this code in non-JIT mode:
11240 mov -0x98(%rax),%eax
11241 callq *%rax
11242 Here, the vtable pointer is lost.
11243 -> solution: use one vtable trampoline per class.
11244
11245 - passing/receiving the IMT pointer/RGCTX.
11246 -> solution: pass them as normal arguments ?
11247
11248 - argument passing.
11249
11250 LLVM does not allow the specification of argument registers etc. This means
11251 that all calls are made according to the platform ABI.
11252
11253 - passing/receiving vtypes.
11254
11255 Vtypes passed/received in registers are handled by the front end by using
11256 a signature with scalar arguments, and loading the parts of the vtype into those
11257 arguments.
11258
11259 Vtypes passed on the stack are handled using the 'byval' attribute.
11260
11261 - ldaddr.
11262
11263 Supported though alloca, we need to emit the load/store code.
11264
11265 - types.
11266
11267 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
11268 typed registers, so we have to keep track of the precise LLVM type of each vreg.
11269 This is made easier because the IR is already in SSA form.
11270 An additional problem is that our IR is not consistent with types, i.e. i32/i64
11271 types are frequently used incorrectly.
11272 */
11273
11274 /*
11275 AOT SUPPORT:
11276 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
11277 it with the file containing the methods emitted by the JIT and the AOT data
11278 structures.
11279 */
11280
11281 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
11282 * - each bblock should end with a branch
11283 * - setting the return value, making cfg->ret non-volatile
11284 * - avoid some transformations in the JIT which make it harder for us to generate
11285 * code.
11286 * - use pointer types to help optimizations.
11287 */
11288
11289 #else /* DISABLE_JIT */
11290
11291 void
11292 mono_llvm_cleanup (void)
11293 {
11294 }
11295
11296 void
11297 mono_llvm_free_domain_info (MonoDomain *domain)
11298 {
11299 }
11300
11301 void
11302 mono_llvm_init (gboolean enable_jit)
11303 {
11304 }
11305
11306 #endif /* DISABLE_JIT */
11307
11308 #if !defined(DISABLE_JIT) && !defined(MONO_CROSS_COMPILE)
11309
11310 /* LLVM JIT support */
11311
11312 /*
11313 * decode_llvm_eh_info:
11314 *
11315 * Decode the EH table emitted by llvm in jit mode, and store
11316 * the result into cfg.
11317 */
11318 static void
11319 decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame)
11320 {
11321 MonoCompile *cfg = ctx->cfg;
11322 guint8 *cie, *fde;
11323 int fde_len;
11324 MonoLLVMFDEInfo info;
11325 MonoJitExceptionInfo *ei;
11326 guint8 *p = (guint8*)eh_frame;
11327 int version, fde_count, fde_offset;
11328 guint32 ei_len, i, nested_len;
11329 gpointer *type_info;
11330 gint32 *table;
11331 guint8 *unw_info;
11332
11333 /*
11334 * Decode the one element EH table emitted by the MonoException class
11335 * in llvm.
11336 */
11337
11338 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
11339
11340 version = *p;
11341 g_assert (version == 3);
11342 p ++;
11343 p ++;
11344 p = (guint8 *)ALIGN_PTR_TO (p, 4);
11345
11346 fde_count = *(guint32*)p;
11347 p += 4;
11348 table = (gint32*)p;
11349
11350 g_assert (fde_count <= 2);
11351
11352 /* The first entry is the real method */
11353 g_assert (table [0] == 1);
11354 fde_offset = table [1];
11355 table += fde_count * 2;
11356 /* Extra entry */
11357 cfg->code_len = table [0];
11358 fde_len = table [1] - fde_offset;
11359 table += 2;
11360
11361 fde = (guint8*)eh_frame + fde_offset;
11362 cie = (guint8*)table;
11363
11364 /* Compute lengths */
11365 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, NULL, NULL, NULL);
11366
11367 ei = (MonoJitExceptionInfo *)g_malloc0 (info.ex_info_len * sizeof (MonoJitExceptionInfo));
11368 type_info = (gpointer *)g_malloc0 (info.ex_info_len * sizeof (gpointer));
11369 unw_info = (guint8*)g_malloc0 (info.unw_info_len);
11370
11371 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, ei, type_info, unw_info);
11372
11373 cfg->encoded_unwind_ops = unw_info;
11374 cfg->encoded_unwind_ops_len = info.unw_info_len;
11375 if (cfg->verbose_level > 1)
11376 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
11377 if (info.this_reg != -1) {
11378 cfg->llvm_this_reg = info.this_reg;
11379 cfg->llvm_this_offset = info.this_offset;
11380 }
11381
11382 ei_len = info.ex_info_len;
11383
11384 // Nested clauses are currently disabled
11385 nested_len = 0;
11386
11387 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
11388 cfg->llvm_ex_info_len = ei_len + nested_len;
11389 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
11390 /* Fill the rest of the information from the type info */
11391 for (i = 0; i < ei_len; ++i) {
11392 gint32 clause_index = *(gint32*)type_info [i];
11393 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
11394
11395 cfg->llvm_ex_info [i].flags = clause->flags;
11396 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
11397 cfg->llvm_ex_info [i].clause_index = clause_index;
11398 }
11399 }
11400
11401 static void
11402 init_jit_module (MonoDomain *domain)
11403 {
11404 MonoJitDomainInfo *dinfo;
11405 MonoLLVMModule *module;
11406 char *name;
11407
11408 dinfo = domain_jit_info (domain);
11409 if (dinfo->llvm_module)
11410 return;
11411
11412 mono_loader_lock ();
11413
11414 if (dinfo->llvm_module) {
11415 mono_loader_unlock ();
11416 return;
11417 }
11418
11419 module = g_new0 (MonoLLVMModule, 1);
11420
11421 name = g_strdup_printf ("mono-%s", domain->friendly_name);
11422 module->context = LLVMGetGlobalContext ();
11423 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
11424
11425 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (&module->ee);
11426
11427 // This contains just the intrinsics
11428 module->lmodule = LLVMModuleCreateWithName ("jit-global-module");
11429 add_intrinsics (module->lmodule);
11430 add_types (module);
11431
11432 module->llvm_types = g_hash_table_new (NULL, NULL);
11433
11434 mono_memory_barrier ();
11435
11436 dinfo->llvm_module = module;
11437
11438 mono_loader_unlock ();
11439 }
11440
11441 static void
11442 llvm_jit_finalize_method (EmitContext *ctx)
11443 {
11444 MonoCompile *cfg = ctx->cfg;
11445 MonoDomain *domain = mono_domain_get ();
11446 MonoJitDomainInfo *domain_info;
11447 int nvars = g_hash_table_size (ctx->jit_callees);
11448 LLVMValueRef *callee_vars = g_new0 (LLVMValueRef, nvars);
11449 gpointer *callee_addrs = g_new0 (gpointer, nvars);
11450 GHashTableIter iter;
11451 LLVMValueRef var;
11452 MonoMethod *callee;
11453 gpointer eh_frame;
11454 int i;
11455
11456 /*
11457 * Compute the addresses of the LLVM globals pointing to the
11458 * methods called by the current method. Pass it to the trampoline
11459 * code so it can update them after their corresponding method was
11460 * compiled.
11461 */
11462 g_hash_table_iter_init (&iter, ctx->jit_callees);
11463 i = 0;
11464 while (g_hash_table_iter_next (&iter, NULL, (void**)&var))
11465 callee_vars [i ++] = var;
11466
11467 cfg->native_code = (guint8*)mono_llvm_compile_method (ctx->module->mono_ee, cfg, ctx->lmethod, nvars, callee_vars, callee_addrs, &eh_frame);
11468 mono_llvm_remove_gc_safepoint_poll (ctx->lmodule);
11469 if (cfg->verbose_level > 1) {
11470 g_print ("\n*** Optimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, TRUE));
11471 if (cfg->compile_aot) {
11472 mono_llvm_dump_value (ctx->lmethod);
11473 } else {
11474 mono_llvm_dump_module (ctx->lmodule);
11475 }
11476 g_print ("***\n\n");
11477 }
11478
11479 decode_llvm_eh_info (ctx, eh_frame);
11480
11481 mono_domain_lock (domain);
11482 domain_info = domain_jit_info (domain);
11483 if (!domain_info->llvm_jit_callees)
11484 domain_info->llvm_jit_callees = g_hash_table_new (NULL, NULL);
11485 g_hash_table_iter_init (&iter, ctx->jit_callees);
11486 i = 0;
11487 while (g_hash_table_iter_next (&iter, (void**)&callee, (void**)&var)) {
11488 GSList *addrs = (GSList*)g_hash_table_lookup (domain_info->llvm_jit_callees, callee);
11489 addrs = g_slist_prepend (addrs, callee_addrs [i]);
11490 g_hash_table_insert (domain_info->llvm_jit_callees, callee, addrs);
11491 i ++;
11492 }
11493 mono_domain_unlock (domain);
11494 }
11495
11496 #else
11497
11498 static void
11499 init_jit_module (MonoDomain *domain)
11500 {
11501 g_assert_not_reached ();
11502 }
11503
11504 static void
11505 llvm_jit_finalize_method (EmitContext *ctx)
11506 {
11507 g_assert_not_reached ();
11508 }
11509
11510 #endif
11511
11512 static MonoCPUFeatures cpu_features;
11513
11514 MonoCPUFeatures mono_llvm_get_cpu_features (void)
11515 {
11516 static const CpuFeatureAliasFlag flags_map [] = {
11517 #if defined(TARGET_X86) || defined(TARGET_AMD64)
11518 { "sse", MONO_CPU_X86_SSE },
11519 { "sse2", MONO_CPU_X86_SSE2 },
11520 { "pclmul", MONO_CPU_X86_PCLMUL },
11521 { "aes", MONO_CPU_X86_AES },
11522 { "sse2", MONO_CPU_X86_SSE2 },
11523 { "sse3", MONO_CPU_X86_SSE3 },
11524 { "ssse3", MONO_CPU_X86_SSSE3 },
11525 { "sse4.1", MONO_CPU_X86_SSE41 },
11526 { "sse4.2", MONO_CPU_X86_SSE42 },
11527 { "popcnt", MONO_CPU_X86_POPCNT },
11528 { "avx", MONO_CPU_X86_AVX },
11529 { "avx2", MONO_CPU_X86_AVX2 },
11530 { "fma", MONO_CPU_X86_FMA },
11531 { "lzcnt", MONO_CPU_X86_LZCNT },
11532 { "bmi", MONO_CPU_X86_BMI1 },
11533 { "bmi2", MONO_CPU_X86_BMI2 },
11534 #endif
11535 };
11536 if (!cpu_features)
11537 cpu_features = MONO_CPU_INITED | (MonoCPUFeatures)mono_llvm_check_cpu_features (flags_map, G_N_ELEMENTS (flags_map));
11538 return cpu_features;
11539 }
Mirror of the official Mono project git repository