?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[interp] Fix interp logging (#17636)
[mono-project.git] / mono / mini / mini-llvm.c
blobd762cda6a915d8dd6a9f29c680a3d1175860d026
1 /**
2 * \file
3 * llvm "Backend" for the mono JIT
4 *
5 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
6 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
8 */
9
10 #include "config.h"
11
12 #include <mono/metadata/debug-helpers.h>
13 #include <mono/metadata/debug-internals.h>
14 #include <mono/metadata/mempool-internals.h>
15 #include <mono/metadata/environment.h>
16 #include <mono/metadata/object-internals.h>
17 #include <mono/metadata/abi-details.h>
18 #include <mono/utils/mono-tls.h>
19 #include <mono/utils/mono-dl.h>
20 #include <mono/utils/mono-time.h>
21 #include <mono/utils/freebsd-dwarf.h>
22
23 #ifndef __STDC_LIMIT_MACROS
24 #define __STDC_LIMIT_MACROS
25 #endif
26 #ifndef __STDC_CONSTANT_MACROS
27 #define __STDC_CONSTANT_MACROS
28 #endif
29
30 #include "llvm-c/BitWriter.h"
31 #include "llvm-c/Analysis.h"
32
33 #include "mini-llvm-cpp.h"
34 #include "llvm-jit.h"
35 #include "aot-compiler.h"
36 #include "mini-llvm.h"
37 #include "mini-runtime.h"
38 #include <mono/utils/mono-math.h>
39
40 #ifndef DISABLE_JIT
41
42 #if defined(TARGET_AMD64) && defined(TARGET_WIN32) && defined(HOST_WIN32) && defined(_MSC_VER)
43 #define TARGET_X86_64_WIN32_MSVC
44 #endif
45
46 #if defined(TARGET_X86_64_WIN32_MSVC)
47 #define TARGET_WIN32_MSVC
48 #endif
49
50 #if LLVM_API_VERSION < 610
51 #error "The version of the mono llvm repository is too old."
52 #endif
53
54 /*
55 * Information associated by mono with LLVM modules.
56 */
57 typedef struct {
58 LLVMModuleRef lmodule;
59 LLVMValueRef throw_icall, rethrow, throw_corlib_exception;
60 GHashTable *llvm_types;
61 LLVMValueRef got_var;
62 const char *got_symbol;
63 const char *get_method_symbol;
64 const char *get_unbox_tramp_symbol;
65 GHashTable *plt_entries;
66 GHashTable *plt_entries_ji;
67 GHashTable *method_to_lmethod;
68 GHashTable *method_to_call_info;
69 GHashTable *lvalue_to_lcalls;
70 GHashTable *direct_callables;
71 char **bb_names;
72 int bb_names_len;
73 GPtrArray *used;
74 LLVMTypeRef ptr_type;
75 GPtrArray *subprogram_mds;
76 MonoEERef *mono_ee;
77 LLVMExecutionEngineRef ee;
78 gboolean external_symbols;
79 gboolean emit_dwarf;
80 int max_got_offset;
81 LLVMValueRef personality;
82 LLVMValueRef *intrins_by_id;
83 gpointer gc_poll_cold_wrapper_compiled;
84
85 /* For AOT */
86 MonoAssembly *assembly;
87 char *global_prefix;
88 MonoAotFileInfo aot_info;
89 const char *jit_got_symbol;
90 const char *eh_frame_symbol;
91 LLVMValueRef get_method, get_unbox_tramp;
92 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
93 LLVMValueRef code_start, code_end;
94 LLVMValueRef inited_var;
95 LLVMValueRef unbox_tramp_indexes;
96 LLVMValueRef unbox_trampolines;
97 LLVMValueRef gc_poll_cold_wrapper;
98 int max_inited_idx, max_method_idx;
99 gboolean has_jitted_code;
100 gboolean static_link;
101 gboolean llvm_only;
102 gboolean llvm_disable_self_init;
103 gboolean interp;
104 GHashTable *idx_to_lmethod;
105 GHashTable *idx_to_unbox_tramp;
106 GPtrArray *callsite_list;
107 LLVMContextRef context;
108 LLVMValueRef sentinel_exception;
109 void *di_builder, *cu;
110 GHashTable *objc_selector_to_var;
111 GPtrArray *cfgs;
112 int unbox_tramp_num, unbox_tramp_elemsize;
113 GHashTable *got_idx_to_type;
114 } MonoLLVMModule;
115
116 /*
117 * Information associated by the backend with mono basic blocks.
118 */
119 typedef struct {
120 LLVMBasicBlockRef bblock, end_bblock;
121 LLVMValueRef finally_ind;
122 gboolean added, invoke_target;
123 /*
124 * If this bblock is the start of a finally clause, this is a list of bblocks it
125 * needs to branch to in ENDFINALLY.
126 */
127 GSList *call_handler_return_bbs;
128 /*
129 * If this bblock is the start of a finally clause, this is the bblock that
130 * CALL_HANDLER needs to branch to.
131 */
132 LLVMBasicBlockRef call_handler_target_bb;
133 /* The list of switch statements generated by ENDFINALLY instructions */
134 GSList *endfinally_switch_ins_list;
135 GSList *phi_nodes;
136 } BBInfo;
137
138 /*
139 * Structure containing emit state
140 */
141 typedef struct {
142 MonoMemPool *mempool;
143
144 /* Maps method names to the corresponding LLVMValueRef */
145 GHashTable *emitted_method_decls;
146
147 MonoCompile *cfg;
148 LLVMValueRef lmethod;
149 MonoLLVMModule *module;
150 LLVMModuleRef lmodule;
151 BBInfo *bblocks;
152 int sindex, default_index, ex_index;
153 LLVMBuilderRef builder;
154 LLVMValueRef *values, *addresses;
155 MonoType **vreg_cli_types;
156 LLVMCallInfo *linfo;
157 MonoMethodSignature *sig;
158 GSList *builders;
159 GHashTable *region_to_handler;
160 GHashTable *clause_to_handler;
161 LLVMBuilderRef alloca_builder;
162 LLVMValueRef last_alloca;
163 LLVMValueRef rgctx_arg;
164 LLVMValueRef this_arg;
165 LLVMTypeRef *vreg_types;
166 gboolean *is_vphi;
167 LLVMTypeRef method_type;
168 LLVMBasicBlockRef init_bb, inited_bb;
169 gboolean *is_dead;
170 gboolean *unreachable;
171 gboolean llvm_only;
172 gboolean has_got_access;
173 gboolean emit_dummy_arg;
174 int this_arg_pindex, rgctx_arg_pindex;
175 LLVMValueRef imt_rgctx_loc;
176 GHashTable *llvm_types;
177 LLVMValueRef dbg_md;
178 MonoDebugMethodInfo *minfo;
179 /* For every clause, the clauses it is nested in */
180 GSList **nested_in;
181 LLVMValueRef ex_var;
182 GHashTable *exc_meta;
183 GPtrArray *callsite_list;
184 GPtrArray *phi_values;
185 GPtrArray *bblock_list;
186 char *method_name;
187 GHashTable *jit_callees;
188 LLVMValueRef long_bb_break_var;
189 } EmitContext;
190
191 typedef struct {
192 MonoBasicBlock *bb;
193 MonoInst *phi;
194 MonoBasicBlock *in_bb;
195 int sreg;
196 } PhiNode;
197
198 /*
199 * Instruction metadata
200 * This is the same as ins_info, but LREG != IREG.
201 */
202 #ifdef MINI_OP
203 #undef MINI_OP
204 #endif
205 #ifdef MINI_OP3
206 #undef MINI_OP3
207 #endif
208 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
209 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
210 #define NONE ' '
211 #define IREG 'i'
212 #define FREG 'f'
213 #define VREG 'v'
214 #define XREG 'x'
215 #define LREG 'l'
216 /* keep in sync with the enum in mini.h */
217 const char
218 mini_llvm_ins_info[] = {
219 #include "mini-ops.h"
220 };
221 #undef MINI_OP
222 #undef MINI_OP3
223
224 #if TARGET_SIZEOF_VOID_P == 4
225 #define GET_LONG_IMM(ins) ((ins)->inst_l)
226 #else
227 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
228 #endif
229
230 #define LLVM_INS_INFO(opcode) (&mini_llvm_ins_info [((opcode) - OP_START - 1) * 4])
231
232 #if 0
233 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
234 #else
235 #define TRACE_FAILURE(msg)
236 #endif
237
238 #ifdef TARGET_X86
239 #define IS_TARGET_X86 1
240 #else
241 #define IS_TARGET_X86 0
242 #endif
243
244 #ifdef TARGET_AMD64
245 #define IS_TARGET_AMD64 1
246 #else
247 #define IS_TARGET_AMD64 0
248 #endif
249
250 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
251
252 static LLVMIntPredicate cond_to_llvm_cond [] = {
253 LLVMIntEQ,
254 LLVMIntNE,
255 LLVMIntSLE,
256 LLVMIntSGE,
257 LLVMIntSLT,
258 LLVMIntSGT,
259 LLVMIntULE,
260 LLVMIntUGE,
261 LLVMIntULT,
262 LLVMIntUGT,
263 };
264
265 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
266 LLVMRealOEQ,
267 LLVMRealUNE,
268 LLVMRealOLE,
269 LLVMRealOGE,
270 LLVMRealOLT,
271 LLVMRealOGT,
272 LLVMRealULE,
273 LLVMRealUGE,
274 LLVMRealULT,
275 LLVMRealUGT,
276 };
277
278 typedef enum {
279 INTRINS_MEMSET,
280 INTRINS_MEMCPY,
281 INTRINS_MEMMOVE,
282 INTRINS_SADD_OVF_I32,
283 INTRINS_UADD_OVF_I32,
284 INTRINS_SSUB_OVF_I32,
285 INTRINS_USUB_OVF_I32,
286 INTRINS_SMUL_OVF_I32,
287 INTRINS_UMUL_OVF_I32,
288 INTRINS_SADD_OVF_I64,
289 INTRINS_UADD_OVF_I64,
290 INTRINS_SSUB_OVF_I64,
291 INTRINS_USUB_OVF_I64,
292 INTRINS_SMUL_OVF_I64,
293 INTRINS_UMUL_OVF_I64,
294 INTRINS_SIN,
295 INTRINS_COS,
296 INTRINS_SQRT,
297 INTRINS_FLOOR,
298 INTRINS_CEIL,
299 INTRINS_FMA,
300 INTRINS_FABS,
301 INTRINS_ABSF,
302 INTRINS_SINF,
303 INTRINS_COSF,
304 INTRINS_SQRTF,
305 INTRINS_FLOORF,
306 INTRINS_CEILF,
307 INTRINS_FMAF,
308 INTRINS_POW,
309 INTRINS_POWF,
310 INTRINS_EXP,
311 INTRINS_EXPF,
312 INTRINS_LOG,
313 INTRINS_LOG2,
314 INTRINS_LOG2F,
315 INTRINS_LOG10,
316 INTRINS_LOG10F,
317 INTRINS_TRUNC,
318 INTRINS_TRUNCF,
319 INTRINS_COPYSIGN,
320 INTRINS_COPYSIGNF,
321 INTRINS_EXPECT_I8,
322 INTRINS_EXPECT_I1,
323 INTRINS_CTPOP_I32,
324 INTRINS_CTPOP_I64,
325 INTRINS_CTLZ_I32,
326 INTRINS_CTLZ_I64,
327 INTRINS_CTTZ_I32,
328 INTRINS_CTTZ_I64,
329 INTRINS_PEXT_I32,
330 INTRINS_PEXT_I64,
331 INTRINS_PDEP_I32,
332 INTRINS_PDEP_I64,
333 INTRINS_BZHI_I32,
334 INTRINS_BZHI_I64,
335 INTRINS_BEXTR_I32,
336 INTRINS_BEXTR_I64,
337 #if defined(TARGET_AMD64) || defined(TARGET_X86)
338 INTRINS_SSE_PMOVMSKB,
339 INTRINS_SSE_PSRLI_W,
340 INTRINS_SSE_PSRAI_W,
341 INTRINS_SSE_PSLLI_W,
342 INTRINS_SSE_PSRLI_D,
343 INTRINS_SSE_PSRAI_D,
344 INTRINS_SSE_PSLLI_D,
345 INTRINS_SSE_PSRLI_Q,
346 INTRINS_SSE_PSLLI_Q,
347 INTRINS_SSE_SQRT_PD,
348 INTRINS_SSE_SQRT_PS,
349 INTRINS_SSE_RSQRT_PS,
350 INTRINS_SSE_RCP_PS,
351 INTRINS_SSE_CVTTPD2DQ,
352 INTRINS_SSE_CVTTPS2DQ,
353 INTRINS_SSE_CVTDQ2PD,
354 INTRINS_SSE_CVTDQ2PS,
355 INTRINS_SSE_CVTPD2DQ,
356 INTRINS_SSE_CVTPS2DQ,
357 INTRINS_SSE_CVTPD2PS,
358 INTRINS_SSE_CVTPS2PD,
359 INTRINS_SSE_CMPPD,
360 INTRINS_SSE_CMPPS,
361 INTRINS_SSE_PACKSSWB,
362 INTRINS_SSE_PACKUSWB,
363 INTRINS_SSE_PACKSSDW,
364 INTRINS_SSE_PACKUSDW,
365 INTRINS_SSE_MINPS,
366 INTRINS_SSE_MAXPS,
367 INTRINS_SSE_HADDPS,
368 INTRINS_SSE_HSUBPS,
369 INTRINS_SSE_ADDSUBPS,
370 INTRINS_SSE_MINPD,
371 INTRINS_SSE_MAXPD,
372 INTRINS_SSE_HADDPD,
373 INTRINS_SSE_HSUBPD,
374 INTRINS_SSE_ADDSUBPD,
375 INTRINS_SSE_PADDSW,
376 INTRINS_SSE_PSUBSW,
377 INTRINS_SSE_PADDUSW,
378 INTRINS_SSE_PSUBUSW,
379 INTRINS_SSE_PAVGW,
380 INTRINS_SSE_PMULHW,
381 INTRINS_SSE_PMULHU,
382 INTRINS_SE_PADDSB,
383 INTRINS_SSE_PSUBSB,
384 INTRINS_SSE_PADDUSB,
385 INTRINS_SSE_PSUBUSB,
386 INTRINS_SSE_PAVGB,
387 INTRINS_SSE_PAUSE,
388 INTRINS_SSE_DPPS,
389 INTRINS_SSE_ROUNDSS,
390 INTRINS_SSE_ROUNDPD,
391 #endif
392 INTRINS_NUM
393 } IntrinsicId;
394
395 static MonoNativeTlsKey current_cfg_tls_id;
396
397 static MonoLLVMModule aot_module;
398
399 static GHashTable *intrins_id_to_name;
400 static GHashTable *intrins_name_to_id;
401
402 static void init_jit_module (MonoDomain *domain);
403
404 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
405 static void emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder);
406 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
407 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
408 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp);
409 static LLVMValueRef get_intrins_by_name (EmitContext *ctx, const char *name);
410 static LLVMValueRef get_intrins (EmitContext *ctx, int id);
411 static LLVMValueRef get_intrins_from_module (LLVMModuleRef lmodule, int id);
412 static void llvm_jit_finalize_method (EmitContext *ctx);
413 static void mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params);
414 static void mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module);
415
416 static inline void
417 set_failure (EmitContext *ctx, const char *message)
418 {
419 TRACE_FAILURE (reason);
420 ctx->cfg->exception_message = g_strdup (message);
421 ctx->cfg->disable_llvm = TRUE;
422 }
423
424 static LLVMValueRef
425 ConstInt32 (int v)
426 {
427 return LLVMConstInt (LLVMInt32Type (), v, FALSE);
428 }
429
430 /*
431 * IntPtrType:
432 *
433 * The LLVM type with width == TARGET_SIZEOF_VOID_P
434 */
435 static LLVMTypeRef
436 IntPtrType (void)
437 {
438 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
439 }
440
441 static LLVMTypeRef
442 ObjRefType (void)
443 {
444 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
445 }
446
447 static LLVMTypeRef
448 ThisType (void)
449 {
450 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
451 }
452
453 /*
454 * get_vtype_size:
455 *
456 * Return the size of the LLVM representation of the vtype T.
457 */
458 static guint32
459 get_vtype_size (MonoType *t)
460 {
461 int size;
462
463 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
464
465 /* LLVMArgAsIArgs depends on this since it stores whole words */
466 while (size < 2 * TARGET_SIZEOF_VOID_P && mono_is_power_of_two (size) == -1)
467 size ++;
468
469 return size;
470 }
471
472 /*
473 * simd_class_to_llvm_type:
474 *
475 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
476 */
477 static LLVMTypeRef
478 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
479 {
480 const char *klass_name = m_class_get_name (klass);
481 if (!strcmp (klass_name, "Vector2d")) {
482 return LLVMVectorType (LLVMDoubleType (), 2);
483 } else if (!strcmp (klass_name, "Vector2l")) {
484 return LLVMVectorType (LLVMInt64Type (), 2);
485 } else if (!strcmp (klass_name, "Vector2ul")) {
486 return LLVMVectorType (LLVMInt64Type (), 2);
487 } else if (!strcmp (klass_name, "Vector4i")) {
488 return LLVMVectorType (LLVMInt32Type (), 4);
489 } else if (!strcmp (klass_name, "Vector4ui")) {
490 return LLVMVectorType (LLVMInt32Type (), 4);
491 } else if (!strcmp (klass_name, "Vector4f")) {
492 return LLVMVectorType (LLVMFloatType (), 4);
493 } else if (!strcmp (klass_name, "Vector8s")) {
494 return LLVMVectorType (LLVMInt16Type (), 8);
495 } else if (!strcmp (klass_name, "Vector8us")) {
496 return LLVMVectorType (LLVMInt16Type (), 8);
497 } else if (!strcmp (klass_name, "Vector16sb")) {
498 return LLVMVectorType (LLVMInt8Type (), 16);
499 } else if (!strcmp (klass_name, "Vector16b")) {
500 return LLVMVectorType (LLVMInt8Type (), 16);
501 } else if (!strcmp (klass_name, "Vector2")) {
502 /* System.Numerics */
503 return LLVMVectorType (LLVMFloatType (), 4);
504 } else if (!strcmp (klass_name, "Vector3")) {
505 return LLVMVectorType (LLVMFloatType (), 4);
506 } else if (!strcmp (klass_name, "Vector4")) {
507 return LLVMVectorType (LLVMFloatType (), 4);
508 } else if (!strcmp (klass_name, "Vector`1") || !strcmp (klass_name, "Vector128`1") || !strcmp (klass_name, "Vector256`1")) {
509 MonoType *etype = mono_class_get_generic_class (klass)->context.class_inst->type_argv [0];
510 int size = mono_class_value_size (klass, NULL);
511 switch (etype->type) {
512 case MONO_TYPE_I1:
513 case MONO_TYPE_U1:
514 return LLVMVectorType (LLVMInt8Type (), size);
515 case MONO_TYPE_I2:
516 case MONO_TYPE_U2:
517 return LLVMVectorType (LLVMInt16Type (), size / 2);
518 case MONO_TYPE_I4:
519 case MONO_TYPE_U4:
520 return LLVMVectorType (LLVMInt32Type (), size / 4);
521 case MONO_TYPE_I8:
522 case MONO_TYPE_U8:
523 return LLVMVectorType (LLVMInt64Type (), size / 8);
524 case MONO_TYPE_R4:
525 return LLVMVectorType (LLVMFloatType (), size / 4);
526 case MONO_TYPE_R8:
527 return LLVMVectorType (LLVMDoubleType (), size / 8);
528 default:
529 g_assert_not_reached ();
530 return NULL;
531 }
532 } else {
533 printf ("%s\n", klass_name);
534 NOT_IMPLEMENTED;
535 return NULL;
536 }
537 }
538
539 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
540 static inline G_GNUC_UNUSED LLVMTypeRef
541 type_to_simd_type (int type)
542 {
543 switch (type) {
544 case MONO_TYPE_I1:
545 return LLVMVectorType (LLVMInt8Type (), 16);
546 case MONO_TYPE_I2:
547 return LLVMVectorType (LLVMInt16Type (), 8);
548 case MONO_TYPE_I4:
549 return LLVMVectorType (LLVMInt32Type (), 4);
550 case MONO_TYPE_I8:
551 return LLVMVectorType (LLVMInt64Type (), 2);
552 case MONO_TYPE_R8:
553 return LLVMVectorType (LLVMDoubleType (), 2);
554 case MONO_TYPE_R4:
555 return LLVMVectorType (LLVMFloatType (), 4);
556 default:
557 g_assert_not_reached ();
558 return NULL;
559 }
560 }
561
562 static LLVMTypeRef
563 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
564 {
565 int i, size, nfields, esize;
566 LLVMTypeRef *eltypes;
567 char *name;
568 MonoType *t;
569 LLVMTypeRef ltype;
570
571 t = m_class_get_byval_arg (klass);
572
573 if (mini_type_is_hfa (t, &nfields, &esize)) {
574 /*
575 * This is needed on arm64 where HFAs are returned in
576 * registers.
577 */
578 /* SIMD types have size 16 in mono_class_value_size () */
579 if (m_class_is_simd_type (klass))
580 nfields = 16/ esize;
581 size = nfields;
582 eltypes = g_new (LLVMTypeRef, size);
583 for (i = 0; i < size; ++i)
584 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
585 } else {
586 size = get_vtype_size (t);
587
588 eltypes = g_new (LLVMTypeRef, size);
589 for (i = 0; i < size; ++i)
590 eltypes [i] = LLVMInt8Type ();
591 }
592
593 name = mono_type_full_name (m_class_get_byval_arg (klass));
594 ltype = LLVMStructCreateNamed (module->context, name);
595 LLVMStructSetBody (ltype, eltypes, size, FALSE);
596 g_free (eltypes);
597 g_free (name);
598
599 return ltype;
600 }
601
602 /*
603 * type_to_llvm_type:
604 *
605 * Return the LLVM type corresponding to T.
606 */
607 static LLVMTypeRef
608 type_to_llvm_type (EmitContext *ctx, MonoType *t)
609 {
610 if (t->byref)
611 return ThisType ();
612
613 t = mini_get_underlying_type (t);
614
615 switch (t->type) {
616 case MONO_TYPE_VOID:
617 return LLVMVoidType ();
618 case MONO_TYPE_I1:
619 return LLVMInt8Type ();
620 case MONO_TYPE_I2:
621 return LLVMInt16Type ();
622 case MONO_TYPE_I4:
623 return LLVMInt32Type ();
624 case MONO_TYPE_U1:
625 return LLVMInt8Type ();
626 case MONO_TYPE_U2:
627 return LLVMInt16Type ();
628 case MONO_TYPE_U4:
629 return LLVMInt32Type ();
630 case MONO_TYPE_I8:
631 case MONO_TYPE_U8:
632 return LLVMInt64Type ();
633 case MONO_TYPE_R4:
634 return LLVMFloatType ();
635 case MONO_TYPE_R8:
636 return LLVMDoubleType ();
637 case MONO_TYPE_I:
638 case MONO_TYPE_U:
639 return IntPtrType ();
640 case MONO_TYPE_OBJECT:
641 return ObjRefType ();
642 case MONO_TYPE_PTR: {
643 MonoClass *klass = mono_class_from_mono_type_internal (t);
644 MonoClass *ptr_klass = m_class_get_element_class (klass);
645 MonoType *ptr_type = m_class_get_byval_arg (ptr_klass);
646 /* Handle primitive pointers */
647 switch (ptr_type->type) {
648 case MONO_TYPE_I1:
649 case MONO_TYPE_I2:
650 case MONO_TYPE_I4:
651 case MONO_TYPE_U1:
652 case MONO_TYPE_U2:
653 case MONO_TYPE_U4:
654 return LLVMPointerType (type_to_llvm_type (ctx, ptr_type), 0);
655 }
656
657 return ObjRefType ();
658 }
659 case MONO_TYPE_VAR:
660 case MONO_TYPE_MVAR:
661 /* Because of generic sharing */
662 return ObjRefType ();
663 case MONO_TYPE_GENERICINST:
664 if (!mono_type_generic_inst_is_valuetype (t))
665 return ObjRefType ();
666 /* Fall through */
667 case MONO_TYPE_VALUETYPE:
668 case MONO_TYPE_TYPEDBYREF: {
669 MonoClass *klass;
670 LLVMTypeRef ltype;
671
672 klass = mono_class_from_mono_type_internal (t);
673
674 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
675 return simd_class_to_llvm_type (ctx, klass);
676
677 if (m_class_is_enumtype (klass))
678 return type_to_llvm_type (ctx, mono_class_enum_basetype_internal (klass));
679
680 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
681 if (!ltype) {
682 ltype = create_llvm_type_for_type (ctx->module, klass);
683 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
684 }
685 return ltype;
686 }
687
688 default:
689 printf ("X: %d\n", t->type);
690 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
691 ctx->cfg->disable_llvm = TRUE;
692 return NULL;
693 }
694 }
695
696 /*
697 * type_is_unsigned:
698 *
699 * Return whenever T is an unsigned int type.
700 */
701 static gboolean
702 type_is_unsigned (EmitContext *ctx, MonoType *t)
703 {
704 t = mini_get_underlying_type (t);
705 if (t->byref)
706 return FALSE;
707 switch (t->type) {
708 case MONO_TYPE_U1:
709 case MONO_TYPE_U2:
710 case MONO_TYPE_CHAR:
711 case MONO_TYPE_U4:
712 case MONO_TYPE_U8:
713 return TRUE;
714 default:
715 return FALSE;
716 }
717 }
718
719 /*
720 * type_to_llvm_arg_type:
721 *
722 * Same as type_to_llvm_type, but treat i8/i16 as i32.
723 */
724 static LLVMTypeRef
725 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
726 {
727 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
728
729 if (ctx->cfg->llvm_only)
730 return ptype;
731
732 /*
733 * This works on all abis except arm64/ios which passes multiple
734 * arguments in one stack slot.
735 */
736 #ifndef TARGET_ARM64
737 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
738 /*
739 * LLVM generates code which only sets the lower bits, while JITted
740 * code expects all the bits to be set.
741 */
742 ptype = LLVMInt32Type ();
743 }
744 #endif
745
746 return ptype;
747 }
748
749 /*
750 * llvm_type_to_stack_type:
751 *
752 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
753 * on the IL stack.
754 */
755 static G_GNUC_UNUSED LLVMTypeRef
756 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
757 {
758 if (type == NULL)
759 return NULL;
760 if (type == LLVMInt8Type ())
761 return LLVMInt32Type ();
762 else if (type == LLVMInt16Type ())
763 return LLVMInt32Type ();
764 else if (!cfg->r4fp && type == LLVMFloatType ())
765 return LLVMDoubleType ();
766 else
767 return type;
768 }
769
770 /*
771 * regtype_to_llvm_type:
772 *
773 * Return the LLVM type corresponding to the regtype C used in instruction
774 * descriptions.
775 */
776 static LLVMTypeRef
777 regtype_to_llvm_type (char c)
778 {
779 switch (c) {
780 case 'i':
781 return LLVMInt32Type ();
782 case 'l':
783 return LLVMInt64Type ();
784 case 'f':
785 return LLVMDoubleType ();
786 default:
787 return NULL;
788 }
789 }
790
791 /*
792 * op_to_llvm_type:
793 *
794 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
795 */
796 static LLVMTypeRef
797 op_to_llvm_type (int opcode)
798 {
799 switch (opcode) {
800 case OP_ICONV_TO_I1:
801 case OP_LCONV_TO_I1:
802 return LLVMInt8Type ();
803 case OP_ICONV_TO_U1:
804 case OP_LCONV_TO_U1:
805 return LLVMInt8Type ();
806 case OP_ICONV_TO_I2:
807 case OP_LCONV_TO_I2:
808 return LLVMInt16Type ();
809 case OP_ICONV_TO_U2:
810 case OP_LCONV_TO_U2:
811 return LLVMInt16Type ();
812 case OP_ICONV_TO_I4:
813 case OP_LCONV_TO_I4:
814 return LLVMInt32Type ();
815 case OP_ICONV_TO_U4:
816 case OP_LCONV_TO_U4:
817 return LLVMInt32Type ();
818 case OP_ICONV_TO_I8:
819 return LLVMInt64Type ();
820 case OP_ICONV_TO_R4:
821 return LLVMFloatType ();
822 case OP_ICONV_TO_R8:
823 return LLVMDoubleType ();
824 case OP_ICONV_TO_U8:
825 return LLVMInt64Type ();
826 case OP_FCONV_TO_I4:
827 return LLVMInt32Type ();
828 case OP_FCONV_TO_I8:
829 return LLVMInt64Type ();
830 case OP_FCONV_TO_I1:
831 case OP_FCONV_TO_U1:
832 case OP_RCONV_TO_I1:
833 case OP_RCONV_TO_U1:
834 return LLVMInt8Type ();
835 case OP_FCONV_TO_I2:
836 case OP_FCONV_TO_U2:
837 case OP_RCONV_TO_I2:
838 case OP_RCONV_TO_U2:
839 return LLVMInt16Type ();
840 case OP_FCONV_TO_U4:
841 case OP_RCONV_TO_U4:
842 return LLVMInt32Type ();
843 case OP_FCONV_TO_U8:
844 case OP_RCONV_TO_U8:
845 return LLVMInt64Type ();
846 case OP_FCONV_TO_I:
847 case OP_FCONV_TO_U:
848 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
849 case OP_IADD_OVF:
850 case OP_IADD_OVF_UN:
851 case OP_ISUB_OVF:
852 case OP_ISUB_OVF_UN:
853 case OP_IMUL_OVF:
854 case OP_IMUL_OVF_UN:
855 return LLVMInt32Type ();
856 case OP_LADD_OVF:
857 case OP_LADD_OVF_UN:
858 case OP_LSUB_OVF:
859 case OP_LSUB_OVF_UN:
860 case OP_LMUL_OVF:
861 case OP_LMUL_OVF_UN:
862 return LLVMInt64Type ();
863 default:
864 printf ("%s\n", mono_inst_name (opcode));
865 g_assert_not_reached ();
866 return NULL;
867 }
868 }
869
870 #define CLAUSE_START(clause) ((clause)->try_offset)
871 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
872
873 /*
874 * load_store_to_llvm_type:
875 *
876 * Return the size/sign/zero extension corresponding to the load/store opcode
877 * OPCODE.
878 */
879 static LLVMTypeRef
880 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
881 {
882 *sext = FALSE;
883 *zext = FALSE;
884
885 switch (opcode) {
886 case OP_LOADI1_MEMBASE:
887 case OP_STOREI1_MEMBASE_REG:
888 case OP_STOREI1_MEMBASE_IMM:
889 case OP_ATOMIC_LOAD_I1:
890 case OP_ATOMIC_STORE_I1:
891 *size = 1;
892 *sext = TRUE;
893 return LLVMInt8Type ();
894 case OP_LOADU1_MEMBASE:
895 case OP_LOADU1_MEM:
896 case OP_ATOMIC_LOAD_U1:
897 case OP_ATOMIC_STORE_U1:
898 *size = 1;
899 *zext = TRUE;
900 return LLVMInt8Type ();
901 case OP_LOADI2_MEMBASE:
902 case OP_STOREI2_MEMBASE_REG:
903 case OP_STOREI2_MEMBASE_IMM:
904 case OP_ATOMIC_LOAD_I2:
905 case OP_ATOMIC_STORE_I2:
906 *size = 2;
907 *sext = TRUE;
908 return LLVMInt16Type ();
909 case OP_LOADU2_MEMBASE:
910 case OP_LOADU2_MEM:
911 case OP_ATOMIC_LOAD_U2:
912 case OP_ATOMIC_STORE_U2:
913 *size = 2;
914 *zext = TRUE;
915 return LLVMInt16Type ();
916 case OP_LOADI4_MEMBASE:
917 case OP_LOADU4_MEMBASE:
918 case OP_LOADI4_MEM:
919 case OP_LOADU4_MEM:
920 case OP_STOREI4_MEMBASE_REG:
921 case OP_STOREI4_MEMBASE_IMM:
922 case OP_ATOMIC_LOAD_I4:
923 case OP_ATOMIC_STORE_I4:
924 case OP_ATOMIC_LOAD_U4:
925 case OP_ATOMIC_STORE_U4:
926 *size = 4;
927 return LLVMInt32Type ();
928 case OP_LOADI8_MEMBASE:
929 case OP_LOADI8_MEM:
930 case OP_STOREI8_MEMBASE_REG:
931 case OP_STOREI8_MEMBASE_IMM:
932 case OP_ATOMIC_LOAD_I8:
933 case OP_ATOMIC_STORE_I8:
934 case OP_ATOMIC_LOAD_U8:
935 case OP_ATOMIC_STORE_U8:
936 *size = 8;
937 return LLVMInt64Type ();
938 case OP_LOADR4_MEMBASE:
939 case OP_STORER4_MEMBASE_REG:
940 case OP_ATOMIC_LOAD_R4:
941 case OP_ATOMIC_STORE_R4:
942 *size = 4;
943 return LLVMFloatType ();
944 case OP_LOADR8_MEMBASE:
945 case OP_STORER8_MEMBASE_REG:
946 case OP_ATOMIC_LOAD_R8:
947 case OP_ATOMIC_STORE_R8:
948 *size = 8;
949 return LLVMDoubleType ();
950 case OP_LOAD_MEMBASE:
951 case OP_LOAD_MEM:
952 case OP_STORE_MEMBASE_REG:
953 case OP_STORE_MEMBASE_IMM:
954 *size = TARGET_SIZEOF_VOID_P;
955 return IntPtrType ();
956 default:
957 g_assert_not_reached ();
958 return NULL;
959 }
960 }
961
962 /*
963 * ovf_op_to_intrins:
964 *
965 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
966 */
967 static const char*
968 ovf_op_to_intrins (int opcode)
969 {
970 switch (opcode) {
971 case OP_IADD_OVF:
972 return "llvm.sadd.with.overflow.i32";
973 case OP_IADD_OVF_UN:
974 return "llvm.uadd.with.overflow.i32";
975 case OP_ISUB_OVF:
976 return "llvm.ssub.with.overflow.i32";
977 case OP_ISUB_OVF_UN:
978 return "llvm.usub.with.overflow.i32";
979 case OP_IMUL_OVF:
980 return "llvm.smul.with.overflow.i32";
981 case OP_IMUL_OVF_UN:
982 return "llvm.umul.with.overflow.i32";
983 case OP_LADD_OVF:
984 return "llvm.sadd.with.overflow.i64";
985 case OP_LADD_OVF_UN:
986 return "llvm.uadd.with.overflow.i64";
987 case OP_LSUB_OVF:
988 return "llvm.ssub.with.overflow.i64";
989 case OP_LSUB_OVF_UN:
990 return "llvm.usub.with.overflow.i64";
991 case OP_LMUL_OVF:
992 return "llvm.smul.with.overflow.i64";
993 case OP_LMUL_OVF_UN:
994 return "llvm.umul.with.overflow.i64";
995 default:
996 g_assert_not_reached ();
997 return NULL;
998 }
999 }
1000
1001 static const char*
1002 simd_op_to_intrins (int opcode)
1003 {
1004 switch (opcode) {
1005 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1006 case OP_MINPD:
1007 return "llvm.x86.sse2.min.pd";
1008 case OP_MINPS:
1009 return "llvm.x86.sse.min.ps";
1010 case OP_MAXPD:
1011 return "llvm.x86.sse2.max.pd";
1012 case OP_MAXPS:
1013 return "llvm.x86.sse.max.ps";
1014 case OP_HADDPD:
1015 return "llvm.x86.sse3.hadd.pd";
1016 case OP_HADDPS:
1017 return "llvm.x86.sse3.hadd.ps";
1018 case OP_HSUBPD:
1019 return "llvm.x86.sse3.hsub.pd";
1020 case OP_HSUBPS:
1021 return "llvm.x86.sse3.hsub.ps";
1022 case OP_ADDSUBPS:
1023 return "llvm.x86.sse3.addsub.ps";
1024 case OP_ADDSUBPD:
1025 return "llvm.x86.sse3.addsub.pd";
1026 case OP_EXTRACT_MASK:
1027 return "llvm.x86.sse2.pmovmskb.128";
1028 case OP_PSHRW:
1029 case OP_PSHRW_REG:
1030 return "llvm.x86.sse2.psrli.w";
1031 case OP_PSHRD:
1032 case OP_PSHRD_REG:
1033 return "llvm.x86.sse2.psrli.d";
1034 case OP_PSHRQ:
1035 case OP_PSHRQ_REG:
1036 return "llvm.x86.sse2.psrli.q";
1037 case OP_PSHLW:
1038 case OP_PSHLW_REG:
1039 return "llvm.x86.sse2.pslli.w";
1040 case OP_PSHLD:
1041 case OP_PSHLD_REG:
1042 return "llvm.x86.sse2.pslli.d";
1043 case OP_PSHLQ:
1044 case OP_PSHLQ_REG:
1045 return "llvm.x86.sse2.pslli.q";
1046 case OP_PSARW:
1047 case OP_PSARW_REG:
1048 return "llvm.x86.sse2.psrai.w";
1049 case OP_PSARD:
1050 case OP_PSARD_REG:
1051 return "llvm.x86.sse2.psrai.d";
1052 case OP_PADDB_SAT:
1053 return "llvm.x86.sse2.padds.b";
1054 case OP_PADDW_SAT:
1055 return "llvm.x86.sse2.padds.w";
1056 case OP_PSUBB_SAT:
1057 return "llvm.x86.sse2.psubs.b";
1058 case OP_PSUBW_SAT:
1059 return "llvm.x86.sse2.psubs.w";
1060 case OP_PADDB_SAT_UN:
1061 return "llvm.x86.sse2.paddus.b";
1062 case OP_PADDW_SAT_UN:
1063 return "llvm.x86.sse2.paddus.w";
1064 case OP_PSUBB_SAT_UN:
1065 return "llvm.x86.sse2.psubus.b";
1066 case OP_PSUBW_SAT_UN:
1067 return "llvm.x86.sse2.psubus.w";
1068 case OP_PAVGB_UN:
1069 return "llvm.x86.sse2.pavg.b";
1070 case OP_PAVGW_UN:
1071 return "llvm.x86.sse2.pavg.w";
1072 case OP_SQRTPS:
1073 return "llvm.x86.sse.sqrt.ps";
1074 case OP_SQRTPD:
1075 return "llvm.x86.sse2.sqrt.pd";
1076 case OP_RSQRTPS:
1077 return "llvm.x86.sse.rsqrt.ps";
1078 case OP_RCPPS:
1079 return "llvm.x86.sse.rcp.ps";
1080 case OP_CVTDQ2PD:
1081 return "llvm.x86.sse2.cvtdq2pd";
1082 case OP_CVTDQ2PS:
1083 return "llvm.x86.sse2.cvtdq2ps";
1084 case OP_CVTPD2DQ:
1085 return "llvm.x86.sse2.cvtpd2dq";
1086 case OP_CVTPS2DQ:
1087 return "llvm.x86.sse2.cvtps2dq";
1088 case OP_CVTPD2PS:
1089 return "llvm.x86.sse2.cvtpd2ps";
1090 case OP_CVTPS2PD:
1091 return "llvm.x86.sse2.cvtps2pd";
1092 case OP_CVTTPD2DQ:
1093 return "llvm.x86.sse2.cvttpd2dq";
1094 case OP_CVTTPS2DQ:
1095 return "llvm.x86.sse2.cvttps2dq";
1096 case OP_PACKW:
1097 return "llvm.x86.sse2.packsswb.128";
1098 case OP_PACKD:
1099 return "llvm.x86.sse2.packssdw.128";
1100 case OP_PACKW_UN:
1101 return "llvm.x86.sse2.packuswb.128";
1102 case OP_PACKD_UN:
1103 return "llvm.x86.sse41.packusdw";
1104 case OP_PMULW_HIGH:
1105 return "llvm.x86.sse2.pmulh.w";
1106 case OP_PMULW_HIGH_UN:
1107 return "llvm.x86.sse2.pmulhu.w";
1108 case OP_DPPS:
1109 return "llvm.x86.sse41.dpps";
1110 #endif
1111 default:
1112 g_assert_not_reached ();
1113 return NULL;
1114 }
1115 }
1116
1117 static LLVMTypeRef
1118 simd_op_to_llvm_type (int opcode)
1119 {
1120 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1121 switch (opcode) {
1122 case OP_EXTRACT_R8:
1123 case OP_EXPAND_R8:
1124 return type_to_simd_type (MONO_TYPE_R8);
1125 case OP_EXTRACT_I8:
1126 case OP_EXPAND_I8:
1127 return type_to_simd_type (MONO_TYPE_I8);
1128 case OP_EXTRACT_I4:
1129 case OP_EXPAND_I4:
1130 return type_to_simd_type (MONO_TYPE_I4);
1131 case OP_EXTRACT_I2:
1132 case OP_EXTRACT_U2:
1133 case OP_EXTRACTX_U2:
1134 case OP_EXPAND_I2:
1135 return type_to_simd_type (MONO_TYPE_I2);
1136 case OP_EXTRACT_I1:
1137 case OP_EXTRACT_U1:
1138 case OP_EXPAND_I1:
1139 return type_to_simd_type (MONO_TYPE_I1);
1140 case OP_EXTRACT_R4:
1141 case OP_EXPAND_R4:
1142 return type_to_simd_type (MONO_TYPE_R4);
1143 case OP_CVTDQ2PD:
1144 case OP_CVTDQ2PS:
1145 return type_to_simd_type (MONO_TYPE_I4);
1146 case OP_CVTPD2DQ:
1147 case OP_CVTPD2PS:
1148 case OP_CVTTPD2DQ:
1149 return type_to_simd_type (MONO_TYPE_R8);
1150 case OP_CVTPS2DQ:
1151 case OP_CVTPS2PD:
1152 case OP_CVTTPS2DQ:
1153 return type_to_simd_type (MONO_TYPE_R4);
1154 case OP_EXTRACT_MASK:
1155 return type_to_simd_type (MONO_TYPE_I1);
1156 case OP_SQRTPS:
1157 case OP_RSQRTPS:
1158 case OP_RCPPS:
1159 case OP_DUPPS_LOW:
1160 case OP_DUPPS_HIGH:
1161 return type_to_simd_type (MONO_TYPE_R4);
1162 case OP_SQRTPD:
1163 case OP_DUPPD:
1164 return type_to_simd_type (MONO_TYPE_R8);
1165 default:
1166 g_assert_not_reached ();
1167 return NULL;
1168 }
1169 #else
1170 return NULL;
1171 #endif
1172 }
1173
1174 static void
1175 set_cold_cconv (LLVMValueRef func)
1176 {
1177 /*
1178 * xcode10 (watchOS) and ARM/ARM64 doesn't seem to support preserveall, it fails with:
1179 * fatal error: error in backend: Unsupported calling convention
1180 */
1181 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1182 LLVMSetFunctionCallConv (func, LLVMColdCallConv);
1183 #endif
1184 }
1185
1186 static void
1187 set_call_cold_cconv (LLVMValueRef func)
1188 {
1189 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1190 LLVMSetInstructionCallConv (func, LLVMColdCallConv);
1191 #endif
1192 }
1193
1194 /*
1195 * get_bb:
1196 *
1197 * Return the LLVM basic block corresponding to BB.
1198 */
1199 static LLVMBasicBlockRef
1200 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
1201 {
1202 char bb_name_buf [128];
1203 char *bb_name;
1204
1205 if (ctx->bblocks [bb->block_num].bblock == NULL) {
1206 if (bb->flags & BB_EXCEPTION_HANDLER) {
1207 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
1208 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
1209 bb_name = bb_name_buf;
1210 } else if (bb->block_num < 256) {
1211 if (!ctx->module->bb_names) {
1212 ctx->module->bb_names_len = 256;
1213 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
1214 }
1215 if (!ctx->module->bb_names [bb->block_num]) {
1216 char *n;
1217
1218 n = g_strdup_printf ("BB%d", bb->block_num);
1219 mono_memory_barrier ();
1220 ctx->module->bb_names [bb->block_num] = n;
1221 }
1222 bb_name = ctx->module->bb_names [bb->block_num];
1223 } else {
1224 sprintf (bb_name_buf, "BB%d", bb->block_num);
1225 bb_name = bb_name_buf;
1226 }
1227
1228 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1229 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1230 }
1231
1232 return ctx->bblocks [bb->block_num].bblock;
1233 }
1234
1235 /*
1236 * get_end_bb:
1237 *
1238 * Return the last LLVM bblock corresponding to BB.
1239 * This might not be equal to the bb returned by get_bb () since we need to generate
1240 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1241 */
1242 static LLVMBasicBlockRef
1243 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1244 {
1245 get_bb (ctx, bb);
1246 return ctx->bblocks [bb->block_num].end_bblock;
1247 }
1248
1249 static LLVMBasicBlockRef
1250 gen_bb (EmitContext *ctx, const char *prefix)
1251 {
1252 char bb_name [128];
1253
1254 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1255 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1256 }
1257
1258 /*
1259 * resolve_patch:
1260 *
1261 * Return the target of the patch identified by TYPE and TARGET.
1262 */
1263 static gpointer
1264 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1265 {
1266 MonoJumpInfo ji;
1267 ERROR_DECL (error);
1268 gpointer res;
1269
1270 memset (&ji, 0, sizeof (ji));
1271 ji.type = type;
1272 ji.data.target = target;
1273
1274 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, error);
1275 mono_error_assert_ok (error);
1276
1277 return res;
1278 }
1279
1280 /*
1281 * convert_full:
1282 *
1283 * Emit code to convert the LLVM value V to DTYPE.
1284 */
1285 static LLVMValueRef
1286 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1287 {
1288 LLVMTypeRef stype = LLVMTypeOf (v);
1289
1290 if (stype != dtype) {
1291 gboolean ext = FALSE;
1292
1293 /* Extend */
1294 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1295 ext = TRUE;
1296 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1297 ext = TRUE;
1298 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1299 ext = TRUE;
1300
1301 if (ext)
1302 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1303
1304 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1305 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1306
1307 /* Trunc */
1308 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1309 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1310 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1311 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1312 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1313 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1314 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1315 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1316
1317 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1318 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1319 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1320 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1321 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1322 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1323
1324 if (mono_arch_is_soft_float ()) {
1325 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1326 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1327 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1328 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1329 }
1330
1331 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1332 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1333
1334 LLVMDumpValue (v);
1335 printf ("\n");
1336 LLVMDumpValue (LLVMConstNull (dtype));
1337 printf ("\n");
1338 g_assert_not_reached ();
1339 return NULL;
1340 } else {
1341 return v;
1342 }
1343 }
1344
1345 static LLVMValueRef
1346 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1347 {
1348 return convert_full (ctx, v, dtype, FALSE);
1349 }
1350
1351 static void
1352 emit_memset (EmitContext *ctx, LLVMBuilderRef builder, LLVMValueRef v, LLVMValueRef size, int alignment)
1353 {
1354 LLVMValueRef args [5];
1355 int aindex = 0;
1356
1357 args [aindex ++] = v;
1358 args [aindex ++] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
1359 args [aindex ++] = size;
1360 #if LLVM_API_VERSION < 900
1361 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), alignment, FALSE);
1362 #endif
1363 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
1364 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMSET), args, aindex, "");
1365 }
1366
1367 /*
1368 * emit_volatile_load:
1369 *
1370 * If vreg is volatile, emit a load from its address.
1371 */
1372 static LLVMValueRef
1373 emit_volatile_load (EmitContext *ctx, int vreg)
1374 {
1375 MonoType *t;
1376 LLVMValueRef v;
1377
1378 // On arm64, we pass the rgctx in a callee saved
1379 // register on arm64 (x15), and llvm might keep the value in that register
1380 // even through the register is marked as 'reserved' inside llvm.
1381
1382 v = mono_llvm_build_load (ctx->builder, ctx->addresses [vreg], "", TRUE);
1383 t = ctx->vreg_cli_types [vreg];
1384 if (t && !t->byref) {
1385 /*
1386 * Might have to zero extend since llvm doesn't have
1387 * unsigned types.
1388 */
1389 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1390 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1391 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1392 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1393 else if (t->type == MONO_TYPE_U8)
1394 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1395 }
1396
1397 return v;
1398 }
1399
1400 /*
1401 * emit_volatile_store:
1402 *
1403 * If VREG is volatile, emit a store from its value to its address.
1404 */
1405 static void
1406 emit_volatile_store (EmitContext *ctx, int vreg)
1407 {
1408 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1409
1410 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1411 g_assert (ctx->addresses [vreg]);
1412 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1413 }
1414 }
1415
1416 static LLVMTypeRef
1417 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1418 {
1419 LLVMTypeRef ret_type;
1420 LLVMTypeRef *param_types = NULL;
1421 LLVMTypeRef res;
1422 int i, pindex;
1423 MonoType *rtype;
1424
1425 ret_type = type_to_llvm_type (ctx, sig->ret);
1426 if (!ctx_ok (ctx))
1427 return NULL;
1428 rtype = mini_get_underlying_type (sig->ret);
1429
1430 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1431 pindex = 0;
1432
1433 if (sig->hasthis)
1434 param_types [pindex ++] = ThisType ();
1435 for (i = 0; i < sig->param_count; ++i)
1436 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1437
1438 if (!ctx_ok (ctx)) {
1439 g_free (param_types);
1440 return NULL;
1441 }
1442
1443 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1444 g_free (param_types);
1445
1446 return res;
1447 }
1448
1449 /*
1450 * sig_to_llvm_sig_full:
1451 *
1452 * Return the LLVM signature corresponding to the mono signature SIG using the
1453 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1454 */
1455 static LLVMTypeRef
1456 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1457 {
1458 LLVMTypeRef ret_type;
1459 LLVMTypeRef *param_types = NULL;
1460 LLVMTypeRef res;
1461 int i, j, pindex, vret_arg_pindex = 0;
1462 gboolean vretaddr = FALSE;
1463 MonoType *rtype;
1464
1465 if (!cinfo)
1466 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1467
1468 ret_type = type_to_llvm_type (ctx, sig->ret);
1469 if (!ctx_ok (ctx))
1470 return NULL;
1471 rtype = mini_get_underlying_type (sig->ret);
1472
1473 switch (cinfo->ret.storage) {
1474 case LLVMArgVtypeInReg:
1475 /* LLVM models this by returning an aggregate value */
1476 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1477 LLVMTypeRef members [2];
1478
1479 members [0] = IntPtrType ();
1480 ret_type = LLVMStructType (members, 1, FALSE);
1481 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1482 /* Empty struct */
1483 ret_type = LLVMVoidType ();
1484 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1485 LLVMTypeRef members [2];
1486
1487 members [0] = IntPtrType ();
1488 members [1] = IntPtrType ();
1489 ret_type = LLVMStructType (members, 2, FALSE);
1490 } else {
1491 g_assert_not_reached ();
1492 }
1493 break;
1494 case LLVMArgVtypeByVal:
1495 /* Vtype returned normally by val */
1496 break;
1497 case LLVMArgVtypeAsScalar: {
1498 int size = mono_class_value_size (mono_class_from_mono_type_internal (rtype), NULL);
1499 /* LLVM models this by returning an int */
1500 if (size < TARGET_SIZEOF_VOID_P) {
1501 g_assert (cinfo->ret.nslots == 1);
1502 ret_type = LLVMIntType (size * 8);
1503 } else {
1504 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1505 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (target_mgreg_t) * 8);
1506 }
1507 break;
1508 }
1509 case LLVMArgAsIArgs:
1510 ret_type = LLVMArrayType (IntPtrType (), cinfo->ret.nslots);
1511 break;
1512 case LLVMArgFpStruct: {
1513 /* Vtype returned as a fp struct */
1514 LLVMTypeRef members [16];
1515
1516 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1517 for (i = 0; i < cinfo->ret.nslots; ++i)
1518 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1519 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1520 break;
1521 }
1522 case LLVMArgVtypeByRef:
1523 /* Vtype returned using a hidden argument */
1524 ret_type = LLVMVoidType ();
1525 break;
1526 case LLVMArgVtypeRetAddr:
1527 case LLVMArgGsharedvtFixed:
1528 case LLVMArgGsharedvtFixedVtype:
1529 case LLVMArgGsharedvtVariable:
1530 vretaddr = TRUE;
1531 ret_type = LLVMVoidType ();
1532 break;
1533 default:
1534 break;
1535 }
1536
1537 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1538 pindex = 0;
1539 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1540 /*
1541 * Has to be the first argument because of the sret argument attribute
1542 * FIXME: This might conflict with passing 'this' as the first argument, but
1543 * this is only used on arm64 which has a dedicated struct return register.
1544 */
1545 cinfo->vret_arg_pindex = pindex;
1546 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1547 if (!ctx_ok (ctx)) {
1548 g_free (param_types);
1549 return NULL;
1550 }
1551 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1552 pindex ++;
1553 }
1554 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1555 cinfo->rgctx_arg_pindex = pindex;
1556 param_types [pindex] = ctx->module->ptr_type;
1557 pindex ++;
1558 }
1559 if (cinfo->imt_arg) {
1560 cinfo->imt_arg_pindex = pindex;
1561 param_types [pindex] = ctx->module->ptr_type;
1562 pindex ++;
1563 }
1564 if (vretaddr) {
1565 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1566 vret_arg_pindex = pindex;
1567 if (cinfo->vret_arg_index == 1) {
1568 /* Add the slots consumed by the first argument */
1569 LLVMArgInfo *ainfo = &cinfo->args [0];
1570 switch (ainfo->storage) {
1571 case LLVMArgVtypeInReg:
1572 for (j = 0; j < 2; ++j) {
1573 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1574 vret_arg_pindex ++;
1575 }
1576 break;
1577 default:
1578 vret_arg_pindex ++;
1579 }
1580 }
1581
1582 cinfo->vret_arg_pindex = vret_arg_pindex;
1583 }
1584
1585 if (vretaddr && vret_arg_pindex == pindex)
1586 param_types [pindex ++] = IntPtrType ();
1587 if (sig->hasthis) {
1588 cinfo->this_arg_pindex = pindex;
1589 param_types [pindex ++] = ThisType ();
1590 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1591 }
1592 if (vretaddr && vret_arg_pindex == pindex)
1593 param_types [pindex ++] = IntPtrType ();
1594 for (i = 0; i < sig->param_count; ++i) {
1595 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1596
1597 if (vretaddr && vret_arg_pindex == pindex)
1598 param_types [pindex ++] = IntPtrType ();
1599 ainfo->pindex = pindex;
1600
1601 switch (ainfo->storage) {
1602 case LLVMArgVtypeInReg:
1603 for (j = 0; j < 2; ++j) {
1604 switch (ainfo->pair_storage [j]) {
1605 case LLVMArgInIReg:
1606 param_types [pindex ++] = LLVMIntType (TARGET_SIZEOF_VOID_P * 8);
1607 break;
1608 case LLVMArgNone:
1609 break;
1610 default:
1611 g_assert_not_reached ();
1612 }
1613 }
1614 break;
1615 case LLVMArgVtypeByVal:
1616 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1617 if (!ctx_ok (ctx))
1618 break;
1619 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1620 pindex ++;
1621 break;
1622 case LLVMArgAsIArgs:
1623 if (ainfo->esize == 8)
1624 param_types [pindex] = LLVMArrayType (LLVMInt64Type (), ainfo->nslots);
1625 else
1626 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1627 pindex ++;
1628 break;
1629 case LLVMArgVtypeAddr:
1630 case LLVMArgVtypeByRef:
1631 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1632 if (!ctx_ok (ctx))
1633 break;
1634 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1635 pindex ++;
1636 break;
1637 case LLVMArgAsFpArgs: {
1638 int j;
1639
1640 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1641 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1642 param_types [pindex ++] = LLVMDoubleType ();
1643 for (j = 0; j < ainfo->nslots; ++j)
1644 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1645 break;
1646 }
1647 case LLVMArgVtypeAsScalar:
1648 g_assert_not_reached ();
1649 break;
1650 case LLVMArgGsharedvtFixed:
1651 case LLVMArgGsharedvtFixedVtype:
1652 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1653 break;
1654 case LLVMArgGsharedvtVariable:
1655 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1656 break;
1657 default:
1658 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1659 break;
1660 }
1661 }
1662 if (!ctx_ok (ctx)) {
1663 g_free (param_types);
1664 return NULL;
1665 }
1666 if (vretaddr && vret_arg_pindex == pindex)
1667 param_types [pindex ++] = IntPtrType ();
1668 if (ctx->llvm_only && cinfo->rgctx_arg) {
1669 /* Pass the rgctx as the last argument */
1670 cinfo->rgctx_arg_pindex = pindex;
1671 param_types [pindex] = ctx->module->ptr_type;
1672 pindex ++;
1673 } else if (ctx->llvm_only && cinfo->dummy_arg) {
1674 /* Pass a dummy arg last */
1675 cinfo->dummy_arg_pindex = pindex;
1676 param_types [pindex] = ctx->module->ptr_type;
1677 pindex ++;
1678 }
1679
1680 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1681 g_free (param_types);
1682
1683 return res;
1684 }
1685
1686 static LLVMTypeRef
1687 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1688 {
1689 return sig_to_llvm_sig_full (ctx, sig, NULL);
1690 }
1691
1692 /*
1693 * LLVMFunctionType1:
1694 *
1695 * Create an LLVM function type from the arguments.
1696 */
1697 static G_GNUC_UNUSED LLVMTypeRef
1698 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1699 int IsVarArg)
1700 {
1701 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1702 }
1703
1704 /*
1705 * LLVMFunctionType1:
1706 *
1707 * Create an LLVM function type from the arguments.
1708 */
1709 static G_GNUC_UNUSED LLVMTypeRef
1710 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1711 LLVMTypeRef ParamType1,
1712 int IsVarArg)
1713 {
1714 LLVMTypeRef param_types [1];
1715
1716 param_types [0] = ParamType1;
1717
1718 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1719 }
1720
1721 /*
1722 * LLVMFunctionType2:
1723 *
1724 * Create an LLVM function type from the arguments.
1725 */
1726 static G_GNUC_UNUSED LLVMTypeRef
1727 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1728 LLVMTypeRef ParamType1,
1729 LLVMTypeRef ParamType2,
1730 int IsVarArg)
1731 {
1732 LLVMTypeRef param_types [2];
1733
1734 param_types [0] = ParamType1;
1735 param_types [1] = ParamType2;
1736
1737 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1738 }
1739
1740 /*
1741 * LLVMFunctionType3:
1742 *
1743 * Create an LLVM function type from the arguments.
1744 */
1745 static G_GNUC_UNUSED LLVMTypeRef
1746 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1747 LLVMTypeRef ParamType1,
1748 LLVMTypeRef ParamType2,
1749 LLVMTypeRef ParamType3,
1750 int IsVarArg)
1751 {
1752 LLVMTypeRef param_types [3];
1753
1754 param_types [0] = ParamType1;
1755 param_types [1] = ParamType2;
1756 param_types [2] = ParamType3;
1757
1758 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1759 }
1760
1761 static G_GNUC_UNUSED LLVMTypeRef
1762 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1763 LLVMTypeRef ParamType1,
1764 LLVMTypeRef ParamType2,
1765 LLVMTypeRef ParamType3,
1766 LLVMTypeRef ParamType4,
1767 LLVMTypeRef ParamType5,
1768 int IsVarArg)
1769 {
1770 LLVMTypeRef param_types [5];
1771
1772 param_types [0] = ParamType1;
1773 param_types [1] = ParamType2;
1774 param_types [2] = ParamType3;
1775 param_types [3] = ParamType4;
1776 param_types [4] = ParamType5;
1777
1778 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1779 }
1780
1781 /*
1782 * create_builder:
1783 *
1784 * Create an LLVM builder and remember it so it can be freed later.
1785 */
1786 static LLVMBuilderRef
1787 create_builder (EmitContext *ctx)
1788 {
1789 LLVMBuilderRef builder = LLVMCreateBuilder ();
1790 if (mono_use_fast_math)
1791 mono_llvm_set_fast_math (builder);
1792
1793 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1794
1795 emit_default_dbg_loc (ctx, builder);
1796
1797 return builder;
1798 }
1799
1800 static char*
1801 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1802 {
1803 char *name;
1804
1805 switch (type) {
1806 case MONO_PATCH_INFO_JIT_ICALL_ID:
1807 name = g_strdup_printf ("jit_icall_%s", mono_find_jit_icall_info ((MonoJitICallId)(gsize)data)->name);
1808 break;
1809 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1810 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1811 name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
1812 break;
1813 }
1814 default:
1815 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1816 break;
1817 }
1818
1819 return name;
1820 }
1821
1822 static int
1823 compute_aot_got_offset (MonoLLVMModule *module, MonoJumpInfo *ji, LLVMTypeRef llvm_type)
1824 {
1825 guint32 got_offset = mono_aot_get_got_offset (ji);
1826
1827 LLVMTypeRef lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (got_offset));
1828
1829 if (!lookup_type) {
1830 lookup_type = llvm_type;
1831 } else if (llvm_type != lookup_type) {
1832 lookup_type = module->ptr_type;
1833 } else {
1834 return got_offset;
1835 }
1836
1837 g_hash_table_insert (module->got_idx_to_type, GINT_TO_POINTER (got_offset), lookup_type);
1838 return got_offset;
1839 }
1840
1841 /* Allocate a GOT slot for TYPE/DATA, and emit IR to load it */
1842 static LLVMValueRef
1843 get_aotconst_typed_module (MonoLLVMModule *module, LLVMBuilderRef builder, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1844 {
1845 guint32 got_offset;
1846 LLVMValueRef indexes [2];
1847 LLVMValueRef got_entry_addr, load;
1848 char *name = NULL;
1849
1850 MonoJumpInfo tmp_ji;
1851 tmp_ji.type = type;
1852 tmp_ji.data.target = data;
1853
1854 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1855
1856 got_offset = compute_aot_got_offset (module, ji, llvm_type);
1857 module->max_got_offset = MAX (module->max_got_offset, got_offset);
1858
1859 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1860 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1861 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
1862
1863 name = get_aotconst_name (type, data, got_offset);
1864 if (llvm_type) {
1865 load = LLVMBuildLoad (builder, got_entry_addr, "");
1866 load = LLVMBuildBitCast (builder, load, llvm_type, name);
1867 } else {
1868 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1869 }
1870 g_free (name);
1871 //set_invariant_load_flag (load);
1872
1873 return load;
1874 }
1875
1876 static LLVMValueRef
1877 get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1878 {
1879 MonoCompile *cfg;
1880 guint32 got_offset;
1881 LLVMValueRef indexes [2];
1882 LLVMValueRef got_entry_addr, load;
1883 LLVMBuilderRef builder = ctx->builder;
1884 char *name = NULL;
1885
1886 cfg = ctx->cfg;
1887
1888 MonoJumpInfo tmp_ji;
1889 tmp_ji.type = type;
1890 tmp_ji.data.target = data;
1891
1892 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1893
1894 ji->next = cfg->patch_info;
1895 cfg->patch_info = ji;
1896
1897 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, llvm_type);
1898 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1899 /*
1900 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1901 * explicitly initialize it.
1902 */
1903 if (!mono_aot_is_shared_got_offset (got_offset)) {
1904 //mono_print_ji (ji);
1905 //printf ("\n");
1906 ctx->cfg->got_access_count ++;
1907 }
1908
1909 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1910 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1911 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1912
1913 name = get_aotconst_name (type, data, got_offset);
1914 if (llvm_type) {
1915 load = LLVMBuildLoad (builder, got_entry_addr, "");
1916 load = convert (ctx, load, llvm_type);
1917 LLVMSetValueName (load, name ? name : "");
1918 } else {
1919 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1920 }
1921 g_free (name);
1922 //set_invariant_load_flag (load);
1923
1924 return load;
1925 }
1926
1927 static LLVMValueRef
1928 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1929 {
1930 return get_aotconst_typed (ctx, type, data, NULL);
1931 }
1932
1933 typedef struct {
1934 MonoJumpInfo *ji;
1935 MonoMethod *method;
1936 LLVMValueRef load;
1937 LLVMTypeRef type;
1938 } CallSite;
1939
1940 static gboolean
1941 method_is_direct_callable (MonoMethod *method)
1942 {
1943 if (method->wrapper_type == MONO_WRAPPER_ALLOC)
1944 return TRUE;
1945 if (method->string_ctor)
1946 return FALSE;
1947 if (method->wrapper_type)
1948 return FALSE;
1949 if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL))
1950 return FALSE;
1951 /* Can't enable this as the callee might fail llvm compilation */
1952 /*
1953 if (!method->is_inflated && (mono_class_get_flags (method->klass) & TYPE_ATTRIBUTE_PUBLIC) && (method->flags & METHOD_ATTRIBUTE_PUBLIC))
1954 return TRUE;
1955 */
1956 return FALSE;
1957 }
1958
1959 static LLVMValueRef
1960 get_callee_llvmonly (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1961 {
1962 LLVMValueRef callee;
1963 char *callee_name = NULL;
1964
1965 if (ctx->module->static_link && ctx->module->assembly->image != mono_get_corlib ()) {
1966 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1967 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1968 g_assert (info);
1969 if (info->func != info->wrapper) {
1970 type = MONO_PATCH_INFO_METHOD;
1971 data = mono_icall_get_wrapper_method (info);
1972 callee_name = mono_aot_get_mangled_method_name ((MonoMethod*)data);
1973 }
1974 } else if (type == MONO_PATCH_INFO_METHOD) {
1975 MonoMethod *method = (MonoMethod*)data;
1976 if (m_class_get_image (method->klass) != ctx->module->assembly->image && method_is_direct_callable (method))
1977 callee_name = mono_aot_get_mangled_method_name (method);
1978 }
1979 }
1980
1981 if (!callee_name)
1982 callee_name = mono_aot_get_direct_call_symbol (type, data);
1983 if (callee_name) {
1984 /* Directly callable */
1985 // FIXME: Locking
1986 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1987 if (!callee) {
1988 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1989
1990 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1991
1992 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1993 } else {
1994 /* LLVMTypeRef's are uniqued */
1995 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
1996 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
1997
1998 g_free (callee_name);
1999 }
2000 return callee;
2001 }
2002
2003 /*
2004 * Change references to jit icalls to the icall wrappers when in corlib, so
2005 * they can be called directly.
2006 */
2007 if (ctx->module->assembly->image == mono_get_corlib () && type == MONO_PATCH_INFO_JIT_ICALL_ID) {
2008 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
2009
2010 if (info->func != info->wrapper) {
2011 type = MONO_PATCH_INFO_METHOD;
2012 data = mono_icall_get_wrapper_method (info);
2013 }
2014 }
2015
2016 /*
2017 * Instead of emitting an indirect call through a got slot, emit a placeholder, and
2018 * replace it with a direct call or an indirect call in mono_llvm_fixup_aot_module ()
2019 * after all methods have been emitted.
2020 */
2021 if (type == MONO_PATCH_INFO_METHOD) {
2022 MonoMethod *method = (MonoMethod*)data;
2023 if (m_class_get_image (method->klass)->assembly == ctx->module->assembly) {
2024 MonoJumpInfo tmp_ji;
2025 tmp_ji.type = type;
2026 tmp_ji.data.target = data;
2027
2028 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
2029 ji->next = ctx->cfg->patch_info;
2030 ctx->cfg->patch_info = ji;
2031 LLVMTypeRef llvm_type = LLVMPointerType (llvm_sig, 0);
2032
2033 ctx->cfg->got_access_count ++;
2034
2035 CallSite *info = g_new0 (CallSite, 1);
2036 info->method = method;
2037 info->ji = ji;
2038 info->type = llvm_type;
2039
2040 /*
2041 * Emit a dummy load to represent the callee, and either replace it with
2042 * a reference to the llvm method for the callee, or from a load from the
2043 * GOT.
2044 */
2045 LLVMValueRef indexes [2];
2046 LLVMValueRef got_entry_addr, load;
2047
2048 LLVMBuilderRef builder = ctx->builder;
2049 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2050 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2051 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
2052
2053 load = LLVMBuildLoad (builder, got_entry_addr, "");
2054 load = convert (ctx, load, llvm_type);
2055 info->load = load;
2056
2057 g_ptr_array_add (ctx->callsite_list, info);
2058
2059 return load;
2060 }
2061 }
2062
2063 /*
2064 * Calls are made through the GOT.
2065 */
2066 callee = get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
2067
2068 return callee;
2069 }
2070
2071 /*
2072 * get_callee:
2073 *
2074 * Return an llvm value representing the callee given by the arguments.
2075 */
2076 static LLVMValueRef
2077 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
2078 {
2079 LLVMValueRef callee;
2080 char *callee_name;
2081 MonoJumpInfo *ji = NULL;
2082
2083 if (ctx->llvm_only)
2084 return get_callee_llvmonly (ctx, llvm_sig, type, data);
2085
2086 callee_name = mono_aot_get_plt_symbol (type, data);
2087 if (!callee_name)
2088 return NULL;
2089
2090 if (ctx->cfg->compile_aot)
2091 /* Add a patch so referenced wrappers can be compiled in full aot mode */
2092 mono_add_patch_info (ctx->cfg, 0, type, data);
2093
2094 // FIXME: Locking
2095 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
2096 if (!callee) {
2097 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
2098
2099 LLVMSetVisibility (callee, LLVMHiddenVisibility);
2100
2101 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
2102 }
2103
2104 if (ctx->cfg->compile_aot) {
2105 ji = g_new0 (MonoJumpInfo, 1);
2106 ji->type = type;
2107 ji->data.target = data;
2108
2109 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
2110 }
2111
2112 return callee;
2113 }
2114
2115 static LLVMValueRef
2116 get_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
2117 {
2118 gpointer target;
2119
2120 // This won't be patched so compile the wrapper immediately
2121 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
2122 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
2123 target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
2124 } else {
2125 target = resolve_patch (ctx->cfg, type, data);
2126 }
2127
2128 LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
2129 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
2130 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
2131 LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
2132 return callee;
2133 }
2134
2135 static int
2136 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
2137 {
2138 MonoMethodHeader *header = cfg->header;
2139 MonoExceptionClause *clause;
2140 int i;
2141
2142 /* Directly */
2143 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
2144 return (bb->region >> 8) - 1;
2145
2146 /* Indirectly */
2147 for (i = 0; i < header->num_clauses; ++i) {
2148 clause = &header->clauses [i];
2149
2150 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
2151 return i;
2152 }
2153
2154 return -1;
2155 }
2156
2157 static MonoExceptionClause *
2158 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
2159 {
2160 if (bb == cfg->bb_init)
2161 return NULL;
2162 // Since they're sorted by nesting we just need
2163 // the first one that the bb is a member of
2164 for (int i = 0; i < cfg->header->num_clauses; i++) {
2165 MonoExceptionClause *curr = &cfg->header->clauses [i];
2166
2167 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
2168 return curr;
2169 }
2170
2171 return NULL;
2172 }
2173
2174 static void
2175 set_metadata_flag (LLVMValueRef v, const char *flag_name)
2176 {
2177 LLVMValueRef md_arg;
2178 int md_kind;
2179
2180 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2181 md_arg = LLVMMDString ("mono", 4);
2182 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2183 }
2184
2185 static void
2186 set_nonnull_load_flag (LLVMValueRef v)
2187 {
2188 LLVMValueRef md_arg;
2189 int md_kind;
2190 const char *flag_name;
2191
2192 flag_name = "nonnull";
2193 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2194 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2195 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2196 }
2197
2198 static void
2199 set_invariant_load_flag (LLVMValueRef v)
2200 {
2201 LLVMValueRef md_arg;
2202 int md_kind;
2203 const char *flag_name;
2204
2205 // FIXME: Cache this
2206 flag_name = "invariant.load";
2207 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2208 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2209 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2210 }
2211
2212 /*
2213 * emit_call:
2214 *
2215 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
2216 * a try region.
2217 */
2218 static LLVMValueRef
2219 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
2220 {
2221 MonoCompile *cfg = ctx->cfg;
2222 LLVMValueRef lcall = NULL;
2223 LLVMBuilderRef builder = *builder_ref;
2224 MonoExceptionClause *clause;
2225
2226 if (ctx->llvm_only) {
2227 clause = get_most_deep_clause (cfg, ctx, bb);
2228
2229 if (clause) {
2230 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2231
2232 /*
2233 * Have to use an invoke instead of a call, branching to the
2234 * handler bblock of the clause containing this bblock.
2235 */
2236 intptr_t key = CLAUSE_END(clause);
2237
2238 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
2239
2240 // FIXME: Find the one that has the lowest end bound for the right start address
2241 // FIXME: Finally + nesting
2242
2243 if (lpad_bb) {
2244 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
2245
2246 /* Use an invoke */
2247 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
2248
2249 builder = ctx->builder = create_builder (ctx);
2250 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2251
2252 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2253 }
2254 }
2255 } else {
2256 int clause_index = get_handler_clause (cfg, bb);
2257
2258 if (clause_index != -1) {
2259 MonoMethodHeader *header = cfg->header;
2260 MonoExceptionClause *ec = &header->clauses [clause_index];
2261 MonoBasicBlock *tblock;
2262 LLVMBasicBlockRef ex_bb, noex_bb;
2263
2264 /*
2265 * Have to use an invoke instead of a call, branching to the
2266 * handler bblock of the clause containing this bblock.
2267 */
2268
2269 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY || ec->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2270
2271 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
2272 g_assert (tblock);
2273
2274 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
2275
2276 ex_bb = get_bb (ctx, tblock);
2277
2278 noex_bb = gen_bb (ctx, "NOEX_BB");
2279
2280 /* Use an invoke */
2281 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
2282
2283 builder = ctx->builder = create_builder (ctx);
2284 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2285
2286 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2287 }
2288 }
2289
2290 if (!lcall) {
2291 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
2292 ctx->builder = builder;
2293 }
2294
2295 if (builder_ref)
2296 *builder_ref = ctx->builder;
2297
2298 return lcall;
2299 }
2300
2301 static LLVMValueRef
2302 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, LLVMValueRef base, const char *name, gboolean is_faulting, BarrierKind barrier)
2303 {
2304 LLVMValueRef res;
2305
2306 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
2307 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
2308 LLVMValueRef cmp;
2309
2310 cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
2311 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
2312 *builder_ref = ctx->builder;
2313 }
2314
2315 /*
2316 * We emit volatile loads for loads which can fault, because otherwise
2317 * LLVM will generate invalid code when encountering a load from a
2318 * NULL address.
2319 */
2320 if (barrier != LLVM_BARRIER_NONE)
2321 res = mono_llvm_build_atomic_load (*builder_ref, addr, name, is_faulting, size, barrier);
2322 else
2323 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting);
2324
2325 /* Mark it with a custom metadata */
2326 /*
2327 if (is_faulting)
2328 set_metadata_flag (res, "mono.faulting.load");
2329 */
2330
2331 return res;
2332 }
2333
2334 static void
2335 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, BarrierKind barrier)
2336 {
2337 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
2338 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
2339 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
2340 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
2341 *builder_ref = ctx->builder;
2342 }
2343
2344 if (barrier != LLVM_BARRIER_NONE)
2345 mono_llvm_build_aligned_store (*builder_ref, value, addr, barrier, size);
2346 else
2347 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
2348 }
2349
2350 static void
2351 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting)
2352 {
2353 emit_store_general (ctx, bb, builder_ref, size, value, addr, base, is_faulting, LLVM_BARRIER_NONE);
2354 }
2355
2356 /*
2357 * emit_cond_system_exception:
2358 *
2359 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2360 * Might set the ctx exception.
2361 */
2362 static void
2363 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
2364 {
2365 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2366 LLVMBuilderRef builder;
2367 MonoClass *exc_class;
2368 LLVMValueRef args [2];
2369 LLVMValueRef callee;
2370 gboolean no_pc = FALSE;
2371 static MonoClass *exc_classes [MONO_EXC_INTRINS_NUM];
2372
2373 if (IS_TARGET_AMD64)
2374 /* Some platforms don't require the pc argument */
2375 no_pc = TRUE;
2376
2377 int exc_id = mini_exception_id_by_name (exc_type);
2378 if (!exc_classes [exc_id])
2379 exc_classes [exc_id] = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
2380 exc_class = exc_classes [exc_id];
2381
2382 ex_bb = gen_bb (ctx, "EX_BB");
2383 if (ctx->llvm_only)
2384 ex2_bb = gen_bb (ctx, "EX2_BB");
2385 noex_bb = gen_bb (ctx, "NOEX_BB");
2386
2387 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2388
2389 /* Emit exception throwing code */
2390 ctx->builder = builder = create_builder (ctx);
2391 LLVMPositionBuilderAtEnd (builder, ex_bb);
2392
2393 if (ctx->cfg->llvm_only) {
2394 LLVMBuildBr (builder, ex2_bb);
2395
2396 ctx->builder = builder = create_builder (ctx);
2397 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2398
2399 if (exc_id == MONO_EXC_NULL_REF) {
2400 static LLVMTypeRef sig;
2401
2402 if (!sig)
2403 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
2404 /* Can't cache this */
2405 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
2406 emit_call (ctx, bb, &builder, callee, NULL, 0);
2407 } else {
2408 static LLVMTypeRef sig;
2409
2410 if (!sig)
2411 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2412 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_throw_corlib_exception));
2413 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2414 emit_call (ctx, bb, &builder, callee, args, 1);
2415 }
2416
2417 LLVMBuildUnreachable (builder);
2418
2419 ctx->builder = builder = create_builder (ctx);
2420 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2421
2422 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2423
2424 ctx->ex_index ++;
2425 return;
2426 }
2427
2428 callee = ctx->module->throw_corlib_exception;
2429 if (!callee) {
2430 LLVMTypeRef sig;
2431
2432 if (no_pc)
2433 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2434 else
2435 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2436
2437 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_throw_corlib_exception_abs_trampoline;
2438
2439 if (ctx->cfg->compile_aot) {
2440 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2441 } else {
2442 /*
2443 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2444 * - On x86, LLVM generated code doesn't push the arguments
2445 * - The trampoline takes the throw address as an arguments, not a pc offset.
2446 */
2447 callee = get_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2448
2449 /*
2450 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2451 * added by get_jit_callee ().
2452 */
2453 ex2_bb = gen_bb (ctx, "EX2_BB");
2454 LLVMBuildBr (builder, ex2_bb);
2455 ex_bb = ex2_bb;
2456
2457 ctx->builder = builder = create_builder (ctx);
2458 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2459 }
2460 }
2461
2462 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2463
2464 /*
2465 * The LLVM mono branch contains changes so a block address can be passed as an
2466 * argument to a call.
2467 */
2468 if (no_pc) {
2469 emit_call (ctx, bb, &builder, callee, args, 1);
2470 } else {
2471 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2472 emit_call (ctx, bb, &builder, callee, args, 2);
2473 }
2474
2475 LLVMBuildUnreachable (builder);
2476
2477 ctx->builder = builder = create_builder (ctx);
2478 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2479
2480 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2481
2482 ctx->ex_index ++;
2483 return;
2484 }
2485
2486 /*
2487 * emit_args_to_vtype:
2488 *
2489 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2490 */
2491 static void
2492 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2493 {
2494 int j, size, nslots;
2495 MonoClass *klass;
2496
2497 t = mini_get_underlying_type (t);
2498 klass = mono_class_from_mono_type_internal (t);
2499 size = mono_class_value_size (klass, NULL);
2500
2501 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
2502 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2503
2504 if (ainfo->storage == LLVMArgAsFpArgs)
2505 nslots = ainfo->nslots;
2506 else
2507 nslots = 2;
2508
2509 for (j = 0; j < nslots; ++j) {
2510 LLVMValueRef index [2], addr, daddr;
2511 int part_size = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2512 LLVMTypeRef part_type;
2513
2514 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2515 part_size ++;
2516
2517 if (ainfo->pair_storage [j] == LLVMArgNone)
2518 continue;
2519
2520 switch (ainfo->pair_storage [j]) {
2521 case LLVMArgInIReg: {
2522 part_type = LLVMIntType (part_size * 8);
2523 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass)) {
2524 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2525 addr = LLVMBuildGEP (builder, address, index, 1, "");
2526 } else {
2527 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2528 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2529 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2530 }
2531 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2532 break;
2533 }
2534 case LLVMArgInFPReg: {
2535 LLVMTypeRef arg_type;
2536
2537 if (ainfo->esize == 8)
2538 arg_type = LLVMDoubleType ();
2539 else
2540 arg_type = LLVMFloatType ();
2541
2542 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2543 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2544 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2545 LLVMBuildStore (builder, args [j], addr);
2546 break;
2547 }
2548 case LLVMArgNone:
2549 break;
2550 default:
2551 g_assert_not_reached ();
2552 }
2553
2554 size -= TARGET_SIZEOF_VOID_P;
2555 }
2556 }
2557
2558 /*
2559 * emit_vtype_to_args:
2560 *
2561 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2562 * into ARGS, and the number of arguments into NARGS.
2563 */
2564 static void
2565 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2566 {
2567 int pindex = 0;
2568 int j, size, nslots;
2569 LLVMTypeRef arg_type;
2570
2571 t = mini_get_underlying_type (t);
2572 size = get_vtype_size (t);
2573
2574 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t)))
2575 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2576
2577 if (ainfo->storage == LLVMArgAsFpArgs)
2578 nslots = ainfo->nslots;
2579 else
2580 nslots = 2;
2581 for (j = 0; j < nslots; ++j) {
2582 LLVMValueRef index [2], addr, daddr;
2583 int partsize = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2584
2585 if (ainfo->pair_storage [j] == LLVMArgNone)
2586 continue;
2587
2588 switch (ainfo->pair_storage [j]) {
2589 case LLVMArgInIReg:
2590 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
2591 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2592 addr = LLVMBuildGEP (builder, address, index, 1, "");
2593 } else {
2594 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2595 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2596 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2597 }
2598 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2599 break;
2600 case LLVMArgInFPReg:
2601 if (ainfo->esize == 8)
2602 arg_type = LLVMDoubleType ();
2603 else
2604 arg_type = LLVMFloatType ();
2605 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2606 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2607 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2608 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2609 break;
2610 case LLVMArgNone:
2611 break;
2612 default:
2613 g_assert_not_reached ();
2614 }
2615 size -= TARGET_SIZEOF_VOID_P;
2616 }
2617
2618 *nargs = pindex;
2619 }
2620
2621 static LLVMValueRef
2622 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2623 {
2624 /*
2625 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2626 * get executed every time control reaches them.
2627 */
2628 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2629
2630 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2631 return ctx->last_alloca;
2632 }
2633
2634 static LLVMValueRef
2635 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2636 {
2637 return build_alloca_llvm_type_name (ctx, t, align, "");
2638 }
2639
2640 static LLVMValueRef
2641 build_alloca (EmitContext *ctx, MonoType *t)
2642 {
2643 MonoClass *k = mono_class_from_mono_type_internal (t);
2644 int align;
2645
2646 g_assert (!mini_is_gsharedvt_variable_type (t));
2647
2648 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2649 align = mono_class_value_size (k, NULL);
2650 else
2651 align = mono_class_min_align (k);
2652
2653 /* Sometimes align is not a power of 2 */
2654 while (mono_is_power_of_two (align) == -1)
2655 align ++;
2656
2657 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2658 }
2659
2660 static LLVMValueRef
2661 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2662 {
2663 /*
2664 * gsharedvt local.
2665 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2666 */
2667 MonoCompile *cfg = ctx->cfg;
2668 LLVMBuilderRef builder = ctx->builder;
2669 LLVMValueRef offset, offset_var;
2670 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2671 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2672 LLVMValueRef ptr;
2673 char *name;
2674
2675 g_assert (info_var);
2676 g_assert (locals_var);
2677
2678 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2679
2680 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * TARGET_SIZEOF_VOID_P), FALSE);
2681 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2682
2683 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2684 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2685
2686 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2687 }
2688
2689 /*
2690 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2691 */
2692 static void
2693 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2694 {
2695 if (!module->used)
2696 module->used = g_ptr_array_sized_new (16);
2697 g_ptr_array_add (module->used, global);
2698 }
2699
2700 static void
2701 emit_llvm_used (MonoLLVMModule *module)
2702 {
2703 LLVMModuleRef lmodule = module->lmodule;
2704 LLVMTypeRef used_type;
2705 LLVMValueRef used, *used_elem;
2706 int i;
2707
2708 if (!module->used)
2709 return;
2710
2711 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2712 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2713 used_elem = g_new0 (LLVMValueRef, module->used->len);
2714 for (i = 0; i < module->used->len; ++i)
2715 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2716 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2717 LLVMSetLinkage (used, LLVMAppendingLinkage);
2718 LLVMSetSection (used, "llvm.metadata");
2719 }
2720
2721 /*
2722 * emit_get_method:
2723 *
2724 * Emit a function mapping method indexes to their code
2725 */
2726 static void
2727 emit_get_method (MonoLLVMModule *module)
2728 {
2729 LLVMModuleRef lmodule = module->lmodule;
2730 LLVMValueRef func, switch_ins, m;
2731 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2732 LLVMBasicBlockRef *bbs = NULL;
2733 LLVMTypeRef rtype;
2734 LLVMBuilderRef builder = LLVMCreateBuilder ();
2735 LLVMValueRef table = NULL;
2736 char *name;
2737 int i;
2738 gboolean emit_table = FALSE;
2739
2740 #ifdef TARGET_WASM
2741 /*
2742 * Emit a table of functions instead of a switch statement,
2743 * its very efficient on wasm. This might be usable on
2744 * other platforms too.
2745 */
2746 emit_table = TRUE;
2747 #endif
2748
2749 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2750
2751 if (emit_table) {
2752 LLVMTypeRef table_type;
2753 LLVMValueRef *table_elems;
2754 char *table_name;
2755
2756 int table_len = module->max_method_idx + 1;
2757 table_type = LLVMArrayType (rtype, table_len);
2758 table_name = g_strdup_printf ("%s_method_table", module->assembly->aname.name);
2759 table = LLVMAddGlobal (lmodule, table_type, table_name);
2760 table_elems = g_new0 (LLVMValueRef, table_len);
2761 for (i = 0; i < table_len; ++i) {
2762 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2763 if (m)
2764 table_elems [i] = LLVMBuildBitCast (builder, m, rtype, "");
2765 else
2766 table_elems [i] = LLVMConstNull (rtype);
2767 }
2768 LLVMSetInitializer (table, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), table_elems, table_len));
2769 }
2770
2771 /*
2772 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2773 * but generating code seems safer.
2774 */
2775 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2776 LLVMSetLinkage (func, LLVMExternalLinkage);
2777 LLVMSetVisibility (func, LLVMHiddenVisibility);
2778 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2779 module->get_method = func;
2780
2781 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2782
2783 /*
2784 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2785 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2786 * then we will have to find another solution.
2787 */
2788
2789 name = g_strdup_printf ("BB_CODE_START");
2790 code_start_bb = LLVMAppendBasicBlock (func, name);
2791 g_free (name);
2792 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2793 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2794
2795 name = g_strdup_printf ("BB_CODE_END");
2796 code_end_bb = LLVMAppendBasicBlock (func, name);
2797 g_free (name);
2798 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2799 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2800
2801 if (emit_table) {
2802 /*
2803 * switch (index) {
2804 * case -1: return code_start;
2805 * case -2: return code_end;
2806 * default: return method_table [index];
2807 */
2808 LLVMBasicBlockRef default_bb = LLVMAppendBasicBlock (func, "DEFAULT");
2809 LLVMPositionBuilderAtEnd (builder, default_bb);
2810 LLVMValueRef base = table;
2811 LLVMValueRef indexes [2];
2812 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2813 indexes [1] = LLVMGetParam (func, 0);
2814 LLVMValueRef addr = LLVMBuildGEP (builder, base, indexes, 2, "");
2815 LLVMValueRef res = mono_llvm_build_load (builder, addr, "", FALSE);
2816 LLVMBuildRet (builder, res);
2817
2818 LLVMPositionBuilderAtEnd (builder, entry_bb);
2819
2820 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), default_bb, 0);
2821 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2822 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2823 } else {
2824 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2825 for (i = 0; i < module->max_method_idx + 1; ++i) {
2826 name = g_strdup_printf ("BB_%d", i);
2827 bb = LLVMAppendBasicBlock (func, name);
2828 g_free (name);
2829 bbs [i] = bb;
2830
2831 LLVMPositionBuilderAtEnd (builder, bb);
2832
2833 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2834 if (m)
2835 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2836 else
2837 LLVMBuildRet (builder, LLVMConstNull (rtype));
2838 }
2839
2840 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2841 LLVMPositionBuilderAtEnd (builder, fail_bb);
2842 LLVMBuildRet (builder, LLVMConstNull (rtype));
2843
2844 LLVMPositionBuilderAtEnd (builder, entry_bb);
2845
2846 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2847 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2848 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2849 for (i = 0; i < module->max_method_idx + 1; ++i) {
2850 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2851 }
2852 }
2853
2854 mark_as_used (module, func);
2855
2856 LLVMDisposeBuilder (builder);
2857 }
2858
2859 /*
2860 * emit_get_unbox_tramp:
2861 *
2862 * Emit a function mapping method indexes to their unbox trampoline
2863 */
2864 static void
2865 emit_get_unbox_tramp (MonoLLVMModule *module)
2866 {
2867 LLVMModuleRef lmodule = module->lmodule;
2868 LLVMValueRef func, switch_ins, m;
2869 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2870 LLVMBasicBlockRef *bbs;
2871 LLVMTypeRef rtype;
2872 LLVMBuilderRef builder = LLVMCreateBuilder ();
2873 char *name;
2874 int i;
2875 gboolean emit_table = FALSE;
2876
2877 /* Similar to emit_get_method () */
2878
2879 #ifndef TARGET_WATCHOS
2880 emit_table = TRUE;
2881 #endif
2882
2883 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2884
2885 if (emit_table) {
2886 // About 10% of methods have an unbox tramp, so emit a table of indexes for them
2887 // that the runtime can search using a binary search
2888 int len = 0;
2889 for (i = 0; i < module->max_method_idx + 1; ++i) {
2890 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2891 if (m)
2892 len ++;
2893 }
2894
2895 LLVMTypeRef table_type, elemtype;
2896 LLVMValueRef *table_elems;
2897 LLVMValueRef table;
2898 char *table_name;
2899 int table_len;
2900 int elemsize;
2901
2902 table_len = len;
2903 elemsize = module->max_method_idx < 65000 ? 2 : 4;
2904
2905 // The index table
2906 elemtype = elemsize == 2 ? LLVMInt16Type () : LLVMInt32Type ();
2907 table_type = LLVMArrayType (elemtype, table_len);
2908 table_name = g_strdup_printf ("%s_unbox_tramp_indexes", module->assembly->aname.name);
2909 table = LLVMAddGlobal (lmodule, table_type, table_name);
2910 table_elems = g_new0 (LLVMValueRef, table_len);
2911 int idx = 0;
2912 for (i = 0; i < module->max_method_idx + 1; ++i) {
2913 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2914 if (m)
2915 table_elems [idx ++] = LLVMConstInt (elemtype, i, FALSE);
2916 }
2917 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2918 module->unbox_tramp_indexes = table;
2919
2920 // The trampoline table
2921 elemtype = rtype;
2922 table_type = LLVMArrayType (elemtype, table_len);
2923 table_name = g_strdup_printf ("%s_unbox_trampolines", module->assembly->aname.name);
2924 table = LLVMAddGlobal (lmodule, table_type, table_name);
2925 table_elems = g_new0 (LLVMValueRef, table_len);
2926 idx = 0;
2927 for (i = 0; i < module->max_method_idx + 1; ++i) {
2928 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2929 if (m)
2930 table_elems [idx ++] = LLVMBuildBitCast (builder, m, rtype, "");
2931 }
2932 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2933 module->unbox_trampolines = table;
2934
2935 module->unbox_tramp_num = table_len;
2936 module->unbox_tramp_elemsize = elemsize;
2937 return;
2938 }
2939
2940 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2941 LLVMSetLinkage (func, LLVMExternalLinkage);
2942 LLVMSetVisibility (func, LLVMHiddenVisibility);
2943 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2944 module->get_unbox_tramp = func;
2945
2946 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2947
2948 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2949 for (i = 0; i < module->max_method_idx + 1; ++i) {
2950 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2951 if (!m)
2952 continue;
2953
2954 name = g_strdup_printf ("BB_%d", i);
2955 bb = LLVMAppendBasicBlock (func, name);
2956 g_free (name);
2957 bbs [i] = bb;
2958
2959 LLVMPositionBuilderAtEnd (builder, bb);
2960
2961 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2962 }
2963
2964 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2965 LLVMPositionBuilderAtEnd (builder, fail_bb);
2966 LLVMBuildRet (builder, LLVMConstNull (rtype));
2967
2968 LLVMPositionBuilderAtEnd (builder, entry_bb);
2969
2970 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2971 for (i = 0; i < module->max_method_idx + 1; ++i) {
2972 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2973 if (!m)
2974 continue;
2975
2976 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2977 }
2978
2979 mark_as_used (module, func);
2980 LLVMDisposeBuilder (builder);
2981 }
2982
2983 /* Add a function to mark the beginning of LLVM code */
2984 static void
2985 emit_llvm_code_start (MonoLLVMModule *module)
2986 {
2987 LLVMModuleRef lmodule = module->lmodule;
2988 LLVMValueRef func;
2989 LLVMBasicBlockRef entry_bb;
2990 LLVMBuilderRef builder;
2991
2992 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2993 LLVMSetLinkage (func, LLVMInternalLinkage);
2994 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2995 module->code_start = func;
2996 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2997 builder = LLVMCreateBuilder ();
2998 LLVMPositionBuilderAtEnd (builder, entry_bb);
2999 LLVMBuildRetVoid (builder);
3000 LLVMDisposeBuilder (builder);
3001 }
3002
3003 static LLVMValueRef
3004 emit_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
3005 {
3006 LLVMModuleRef lmodule = module->lmodule;
3007 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
3008 LLVMBasicBlockRef entry_bb;
3009 LLVMBuilderRef builder;
3010 LLVMTypeRef sig;
3011 MonoJumpInfo *ji;
3012 int got_offset;
3013 const char *wrapper_name = mono_marshal_get_aot_init_wrapper_name (subtype);
3014 char *name = g_strdup_printf ("%s%s", module->global_prefix, wrapper_name);
3015 MonoJitICallId icall_id = MONO_JIT_ICALL_ZeroIsReserved;
3016
3017 switch (subtype) {
3018 case AOT_INIT_METHOD:
3019 icall_id = MONO_JIT_ICALL_mini_llvm_init_method;
3020 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
3021 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
3022 break;
3023 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3024 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_mrgctx; // Deliberate fall-through
3025 case AOT_INIT_METHOD_GSHARED_VTABLE:
3026 /* mrgctx/vtable */
3027 if (!icall_id)
3028 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_vtable;
3029 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
3030 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
3031 break;
3032 case AOT_INIT_METHOD_GSHARED_THIS:
3033 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_this;
3034 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
3035 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
3036 break;
3037 default:
3038 g_assert_not_reached ();
3039 }
3040
3041 g_assert (icall_id);
3042 LLVMSetLinkage (func, LLVMInternalLinkage);
3043
3044 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3045
3046 set_cold_cconv (func);
3047
3048 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3049 builder = LLVMCreateBuilder ();
3050 LLVMPositionBuilderAtEnd (builder, entry_bb);
3051
3052 /* get_aotconst */
3053 ji = g_new0 (MonoJumpInfo, 1);
3054 ji->type = MONO_PATCH_INFO_AOT_MODULE;
3055 ji = mono_aot_patch_info_dup (ji);
3056 got_offset = compute_aot_got_offset (module, ji, IntPtrType ());
3057 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3058 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3059 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3060 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3061 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
3062 args [1] = LLVMGetParam (func, 0);
3063 if (subtype)
3064 args [2] = LLVMGetParam (func, 1);
3065
3066 ji = g_new0 (MonoJumpInfo, 1);
3067 ji->type = MONO_PATCH_INFO_JIT_ICALL_ID;
3068 ji->data.jit_icall_id = icall_id;
3069 ji = mono_aot_patch_info_dup (ji);
3070 got_offset = compute_aot_got_offset (module, ji, sig);
3071 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3072 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3073 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3074 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3075 callee = LLVMBuildLoad (builder, got_entry_addr, "");
3076 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
3077 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
3078
3079 // Set the inited flag
3080 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3081 indexes [1] = LLVMGetParam (func, 0);
3082 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
3083
3084 LLVMBuildRetVoid (builder);
3085
3086 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3087 LLVMDisposeBuilder (builder);
3088 return func;
3089 }
3090
3091 /* Emit a wrapper around the parameterless JIT icall ICALL_ID with a cold calling convention */
3092 static LLVMValueRef
3093 emit_icall_cold_wrapper (MonoLLVMModule *module, LLVMModuleRef lmodule, MonoJitICallId icall_id, gboolean aot)
3094 {
3095 LLVMValueRef func, callee;
3096 LLVMBasicBlockRef entry_bb;
3097 LLVMBuilderRef builder;
3098 LLVMTypeRef sig;
3099 char *name;
3100
3101 name = g_strdup_printf ("%s_icall_cold_wrapper_%d", module->global_prefix, icall_id);
3102
3103 func = LLVMAddFunction (lmodule, name, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3104 sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3105 LLVMSetLinkage (func, LLVMInternalLinkage);
3106 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3107 set_cold_cconv (func);
3108
3109 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3110 builder = LLVMCreateBuilder ();
3111 LLVMPositionBuilderAtEnd (builder, entry_bb);
3112
3113 if (aot) {
3114 callee = get_aotconst_typed_module (module, builder, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id), LLVMPointerType (sig, 0));
3115 } else {
3116 MonoJitICallInfo * const info = mono_find_jit_icall_info (icall_id);
3117 gpointer target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
3118
3119 LLVMValueRef tramp_var = LLVMAddGlobal (lmodule, LLVMPointerType (sig, 0), name);
3120 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (sig, 0)));
3121 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3122 callee = LLVMBuildLoad (builder, tramp_var, "");
3123 }
3124 LLVMBuildCall (builder, callee, NULL, 0, "");
3125
3126 LLVMBuildRetVoid (builder);
3127
3128 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3129 LLVMDisposeBuilder (builder);
3130 return func;
3131 }
3132
3133 /*
3134 * Emit wrappers around the C icalls used to initialize llvm methods, to
3135 * make the calling code smaller and to enable usage of the llvm
3136 * cold calling convention.
3137 */
3138 static void
3139 emit_init_icall_wrappers (MonoLLVMModule *module)
3140 {
3141 module->init_method = emit_init_icall_wrapper (module, AOT_INIT_METHOD);
3142 module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_MRGCTX);
3143 module->init_method_gshared_this = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_THIS);
3144 module->init_method_gshared_vtable = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_VTABLE);
3145 }
3146
3147 static LLVMValueRef
3148 get_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
3149 {
3150 switch (subtype) {
3151 case AOT_INIT_METHOD:
3152 return module->init_method;
3153 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3154 return module->init_method_gshared_mrgctx;
3155 case AOT_INIT_METHOD_GSHARED_THIS:
3156 return module->init_method_gshared_this;
3157 case AOT_INIT_METHOD_GSHARED_VTABLE:
3158 return module->init_method_gshared_vtable;
3159 default:
3160 g_assert_not_reached ();
3161 }
3162 }
3163
3164 static void
3165 emit_gc_safepoint_poll (MonoLLVMModule *module, LLVMModuleRef lmodule, MonoCompile *cfg)
3166 {
3167 gboolean is_aot = cfg == NULL || cfg->compile_aot;
3168 LLVMValueRef func = mono_llvm_get_or_insert_gc_safepoint_poll (lmodule);
3169 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3170 if (is_aot) {
3171 LLVMSetLinkage (func, LLVMWeakODRLinkage);
3172 } else {
3173 mono_llvm_add_func_attr (func, LLVM_ATTR_OPTIMIZE_NONE); // no need to waste time here, the function is already optimized and will be inlined.
3174 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE); // optnone attribute requires noinline (but it will be inlined anyway)
3175 if (!module->gc_poll_cold_wrapper_compiled) {
3176 ERROR_DECL (error);
3177 /* Compiling a method here is a bit ugly, but it works */
3178 MonoMethod *wrapper = mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL);
3179 module->gc_poll_cold_wrapper_compiled = mono_jit_compile_method (wrapper, error);
3180 mono_error_assert_ok (error);
3181 }
3182 }
3183 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.entry");
3184 LLVMBasicBlockRef poll_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.poll");
3185 LLVMBasicBlockRef exit_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.exit");
3186 LLVMTypeRef ptr_type = LLVMPointerType (IntPtrType (), 0);
3187 LLVMBuilderRef builder = LLVMCreateBuilder ();
3188
3189 /* entry: */
3190 LLVMPositionBuilderAtEnd (builder, entry_bb);
3191 LLVMValueRef poll_val_ptr;
3192 if (is_aot) {
3193 poll_val_ptr = get_aotconst_typed_module (module, builder, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, NULL, ptr_type);
3194 } else {
3195 LLVMValueRef poll_val_int = LLVMConstInt (IntPtrType (), (guint64) &mono_polling_required, FALSE);
3196 poll_val_ptr = LLVMBuildIntToPtr (builder, poll_val_int, ptr_type, "");
3197 }
3198 LLVMValueRef poll_val_ptr_load = LLVMBuildLoad (builder, poll_val_ptr, ""); // probably needs to be volatile
3199 LLVMValueRef poll_val = LLVMBuildPtrToInt (builder, poll_val_ptr_load, IntPtrType (), "");
3200 LLVMValueRef poll_val_zero = LLVMConstNull (LLVMTypeOf (poll_val));
3201 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, poll_val, poll_val_zero, "");
3202 mono_llvm_build_weighted_branch (builder, cmp, exit_bb, poll_bb, 1000 /* weight for exit_bb */, 1 /* weight for poll_bb */);
3203
3204 /* poll: */
3205 LLVMPositionBuilderAtEnd (builder, poll_bb);
3206 LLVMValueRef call;
3207 if (is_aot) {
3208 LLVMValueRef icall_wrapper = emit_icall_cold_wrapper (module, lmodule, MONO_JIT_ICALL_mono_threads_state_poll, TRUE);
3209 module->gc_poll_cold_wrapper = icall_wrapper;
3210 call = LLVMBuildCall (builder, icall_wrapper, NULL, 0, "");
3211 } else {
3212 // in JIT mode we have to emit @gc.safepoint_poll function for each method (module)
3213 // this function calls gc_poll_cold_wrapper_compiled via a global variable.
3214 // @gc.safepoint_poll will be inlined and can be deleted after -place-safepoints pass.
3215 LLVMTypeRef poll_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3216 LLVMTypeRef poll_sig_ptr = LLVMPointerType (poll_sig, 0);
3217 gpointer target = resolve_patch (cfg, MONO_PATCH_INFO_ABS, module->gc_poll_cold_wrapper_compiled);
3218 LLVMValueRef tramp_var = LLVMAddGlobal (lmodule, poll_sig_ptr, "mono_threads_state_poll");
3219 LLVMValueRef target_val = LLVMConstInt (LLVMInt64Type (), (guint64) target, FALSE);
3220 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (target_val, poll_sig_ptr));
3221 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3222 LLVMValueRef callee = LLVMBuildLoad (builder, tramp_var, "");
3223 call = LLVMBuildCall (builder, callee, NULL, 0, "");
3224 }
3225 set_call_cold_cconv (call);
3226 LLVMBuildBr (builder, exit_bb);
3227
3228 /* exit: */
3229 LLVMPositionBuilderAtEnd (builder, exit_bb);
3230 LLVMBuildRetVoid (builder);
3231 LLVMDisposeBuilder (builder);
3232 }
3233
3234 static void
3235 emit_llvm_code_end (MonoLLVMModule *module)
3236 {
3237 LLVMModuleRef lmodule = module->lmodule;
3238 LLVMValueRef func;
3239 LLVMBasicBlockRef entry_bb;
3240 LLVMBuilderRef builder;
3241
3242 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3243 LLVMSetLinkage (func, LLVMInternalLinkage);
3244 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3245 module->code_end = func;
3246 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3247 builder = LLVMCreateBuilder ();
3248 LLVMPositionBuilderAtEnd (builder, entry_bb);
3249 LLVMBuildRetVoid (builder);
3250 LLVMDisposeBuilder (builder);
3251 }
3252
3253 static void
3254 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
3255 {
3256 gboolean need_div_check = ctx->cfg->backend->need_div_check;
3257
3258 if (bb->region)
3259 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
3260 need_div_check = TRUE;
3261
3262 if (!need_div_check)
3263 return;
3264
3265 switch (ins->opcode) {
3266 case OP_IDIV:
3267 case OP_LDIV:
3268 case OP_IREM:
3269 case OP_LREM:
3270 case OP_IDIV_UN:
3271 case OP_LDIV_UN:
3272 case OP_IREM_UN:
3273 case OP_LREM_UN:
3274 case OP_IDIV_IMM:
3275 case OP_LDIV_IMM:
3276 case OP_IREM_IMM:
3277 case OP_LREM_IMM:
3278 case OP_IDIV_UN_IMM:
3279 case OP_LDIV_UN_IMM:
3280 case OP_IREM_UN_IMM:
3281 case OP_LREM_UN_IMM: {
3282 LLVMValueRef cmp;
3283 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
3284 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
3285
3286 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
3287 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
3288 if (!ctx_ok (ctx))
3289 break;
3290 builder = ctx->builder;
3291
3292 /* b == -1 && a == 0x80000000 */
3293 if (is_signed) {
3294 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
3295 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
3296 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
3297
3298 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
3299 emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
3300 if (!ctx_ok (ctx))
3301 break;
3302 builder = ctx->builder;
3303 }
3304 break;
3305 }
3306 default:
3307 break;
3308 }
3309 }
3310
3311 /*
3312 * emit_init_method:
3313 *
3314 * Emit code to initialize the GOT slots used by the method.
3315 */
3316 static void
3317 emit_init_method (EmitContext *ctx)
3318 {
3319 LLVMValueRef indexes [16], args [16], callee;
3320 LLVMValueRef inited_var, cmp, call;
3321 LLVMBasicBlockRef inited_bb, notinited_bb;
3322 LLVMBuilderRef builder = ctx->builder;
3323 MonoCompile *cfg = ctx->cfg;
3324
3325 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
3326
3327 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3328 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
3329 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
3330
3331 //WASM doesn't support the "llvm.expect.i8" intrinsic
3332 #ifndef TARGET_WASM
3333 args [0] = inited_var;
3334 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
3335 inited_var = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I8), args, 2, "");
3336 #endif
3337
3338 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
3339
3340 inited_bb = ctx->inited_bb;
3341 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
3342
3343 ctx->cfg->llvmonly_init_cond = LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
3344
3345 builder = ctx->builder = create_builder (ctx);
3346 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
3347
3348 // FIXME: Cache
3349 if (ctx->rgctx_arg && ((cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) ||
3350 mini_method_is_default_method (cfg->method))) {
3351 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3352 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3353 callee = ctx->module->init_method_gshared_mrgctx;
3354 call = LLVMBuildCall (builder, callee, args, 2, "");
3355 } else if (ctx->rgctx_arg) {
3356 /* A vtable is passed as the rgctx argument */
3357 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3358 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3359 callee = ctx->module->init_method_gshared_vtable;
3360 call = LLVMBuildCall (builder, callee, args, 2, "");
3361 } else if (cfg->gshared) {
3362 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3363 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
3364 callee = ctx->module->init_method_gshared_this;
3365 call = LLVMBuildCall (builder, callee, args, 2, "");
3366 } else {
3367 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3368 callee = ctx->module->init_method;
3369 call = LLVMBuildCall (builder, callee, args, 1, "");
3370 }
3371
3372 /*
3373 * This enables llvm to keep arguments in their original registers/
3374 * scratch registers, since the call will not clobber them.
3375 */
3376 set_call_cold_cconv (call);
3377
3378 LLVMBuildBr (builder, inited_bb);
3379 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
3380
3381 builder = ctx->builder = create_builder (ctx);
3382 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
3383 }
3384
3385 static void
3386 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
3387 {
3388 /*
3389 * Emit unbox trampoline using a tailcall
3390 */
3391 LLVMValueRef tramp, call, *args;
3392 LLVMBuilderRef builder;
3393 LLVMBasicBlockRef lbb;
3394 LLVMCallInfo *linfo;
3395 char *tramp_name;
3396 int i, nargs;
3397
3398 tramp_name = g_strdup_printf ("ut_%s", method_name);
3399 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
3400 LLVMSetLinkage (tramp, LLVMInternalLinkage);
3401 mono_llvm_add_func_attr (tramp, LLVM_ATTR_OPTIMIZE_FOR_SIZE);
3402 //mono_llvm_add_func_attr (tramp, LLVM_ATTR_NO_UNWIND);
3403 linfo = ctx->linfo;
3404 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
3405 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3406 mono_llvm_add_param_attr (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVM_ATTR_IN_REG);
3407 if (ctx->cfg->vret_addr) {
3408 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
3409 if (linfo->ret.storage == LLVMArgVtypeByRef) {
3410 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
3411 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
3412 }
3413 }
3414
3415 lbb = LLVMAppendBasicBlock (tramp, "");
3416 builder = LLVMCreateBuilder ();
3417 LLVMPositionBuilderAtEnd (builder, lbb);
3418
3419 nargs = LLVMCountParamTypes (method_type);
3420 args = g_new0 (LLVMValueRef, nargs);
3421 for (i = 0; i < nargs; ++i) {
3422 args [i] = LLVMGetParam (tramp, i);
3423 if (i == ctx->this_arg_pindex) {
3424 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
3425
3426 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
3427 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), MONO_ABI_SIZEOF (MonoObject), FALSE), "");
3428 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
3429 }
3430 }
3431 call = LLVMBuildCall (builder, method, args, nargs, "");
3432 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3433 mono_llvm_add_instr_attr (call, 1 + ctx->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
3434 if (linfo->ret.storage == LLVMArgVtypeByRef)
3435 mono_llvm_add_instr_attr (call, 1 + linfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
3436
3437 // FIXME: This causes assertions in clang
3438 //mono_llvm_set_must_tailcall (call);
3439 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
3440 LLVMBuildRetVoid (builder);
3441 else
3442 LLVMBuildRet (builder, call);
3443
3444 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
3445 LLVMDisposeBuilder (builder);
3446 }
3447
3448 /*
3449 * emit_entry_bb:
3450 *
3451 * Emit code to load/convert arguments.
3452 */
3453 static void
3454 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
3455 {
3456 int i, j, pindex;
3457 MonoCompile *cfg = ctx->cfg;
3458 MonoMethodSignature *sig = ctx->sig;
3459 LLVMCallInfo *linfo = ctx->linfo;
3460 MonoBasicBlock *bb;
3461 char **names;
3462
3463 LLVMBuilderRef old_builder = ctx->builder;
3464 ctx->builder = builder;
3465
3466 ctx->alloca_builder = create_builder (ctx);
3467
3468 /*
3469 * Handle indirect/volatile variables by allocating memory for them
3470 * using 'alloca', and storing their address in a temporary.
3471 */
3472 for (i = 0; i < cfg->num_varinfo; ++i) {
3473 MonoInst *var = cfg->varinfo [i];
3474 LLVMTypeRef vtype;
3475
3476 if ((var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET))
3477 continue;
3478
3479 #ifdef TARGET_WASM
3480 // For GC stack scanning to work, have to spill all reference variables to the stack
3481 // Some ref variables have type intptr
3482 if (MONO_TYPE_IS_REFERENCE (var->inst_vtype) || var->inst_vtype->type == MONO_TYPE_I)
3483 var->flags |= MONO_INST_INDIRECT;
3484 #endif
3485
3486 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
3487 vtype = type_to_llvm_type (ctx, var->inst_vtype);
3488 if (!ctx_ok (ctx))
3489 return;
3490 /* Could be already created by an OP_VPHI */
3491 if (!ctx->addresses [var->dreg]) {
3492 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
3493 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
3494 }
3495 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
3496 }
3497 }
3498
3499 names = g_new (char *, sig->param_count);
3500 mono_method_get_param_names (cfg->method, (const char **) names);
3501
3502 for (i = 0; i < sig->param_count; ++i) {
3503 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
3504 int reg = cfg->args [i + sig->hasthis]->dreg;
3505 char *name;
3506
3507 pindex = ainfo->pindex;
3508
3509 switch (ainfo->storage) {
3510 case LLVMArgVtypeInReg:
3511 case LLVMArgAsFpArgs: {
3512 LLVMValueRef args [8];
3513 int j;
3514
3515 pindex += ainfo->ndummy_fpargs;
3516
3517 /* The argument is received as a set of int/fp arguments, store them into the real argument */
3518 memset (args, 0, sizeof (args));
3519 if (ainfo->storage == LLVMArgVtypeInReg) {
3520 args [0] = LLVMGetParam (ctx->lmethod, pindex);
3521 if (ainfo->pair_storage [1] != LLVMArgNone)
3522 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
3523 } else {
3524 g_assert (ainfo->nslots <= 8);
3525 for (j = 0; j < ainfo->nslots; ++j)
3526 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
3527 }
3528 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
3529
3530 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
3531
3532 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
3533 /* Treat these as normal values */
3534 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3535 }
3536 break;
3537 }
3538 case LLVMArgVtypeByVal: {
3539 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3540
3541 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
3542 /* Treat these as normal values */
3543 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3544 }
3545 break;
3546 }
3547 case LLVMArgVtypeAddr:
3548 case LLVMArgVtypeByRef: {
3549 /* The argument is passed by ref */
3550 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3551 break;
3552 }
3553 case LLVMArgAsIArgs: {
3554 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3555 int size;
3556 MonoType *t = mini_get_underlying_type (ainfo->type);
3557
3558 /* The argument is received as an array of ints, store it into the real argument */
3559 ctx->addresses [reg] = build_alloca (ctx, t);
3560
3561 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
3562 if (size == 0) {
3563 } else if (size < TARGET_SIZEOF_VOID_P) {
3564 /* The upper bits of the registers might not be valid */
3565 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
3566 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
3567 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
3568 } else {
3569 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
3570 }
3571 break;
3572 }
3573 case LLVMArgVtypeAsScalar:
3574 g_assert_not_reached ();
3575 break;
3576 case LLVMArgGsharedvtFixed: {
3577 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
3578 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3579
3580 if (names [i])
3581 name = g_strdup_printf ("arg_%s", names [i]);
3582 else
3583 name = g_strdup_printf ("arg_%d", i);
3584
3585 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
3586 break;
3587 }
3588 case LLVMArgGsharedvtFixedVtype: {
3589 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3590
3591 if (names [i])
3592 name = g_strdup_printf ("vtype_arg_%s", names [i]);
3593 else
3594 name = g_strdup_printf ("vtype_arg_%d", i);
3595
3596 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3597 g_assert (ctx->addresses [reg]);
3598 LLVMSetValueName (ctx->addresses [reg], name);
3599 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
3600 break;
3601 }
3602 case LLVMArgGsharedvtVariable:
3603 /* The IR treats these as variables with addresses */
3604 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3605 break;
3606 default: {
3607 LLVMTypeRef t;
3608 /* Needed to avoid phi argument mismatch errors since operations on pointers produce i32/i64 */
3609 if (ainfo->type->byref)
3610 t = IntPtrType ();
3611 else
3612 t = type_to_llvm_type (ctx, ainfo->type);
3613 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, t), type_is_unsigned (ctx, ainfo->type));
3614 break;
3615 }
3616 }
3617 }
3618 g_free (names);
3619
3620 if (cfg->vret_addr)
3621 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3622 if (sig->hasthis)
3623 emit_volatile_store (ctx, cfg->args [0]->dreg);
3624 for (i = 0; i < sig->param_count; ++i)
3625 if (!mini_type_is_vtype (sig->params [i]))
3626 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3627
3628 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
3629 LLVMValueRef this_alloc;
3630
3631 /*
3632 * The exception handling code needs the location where the this argument was
3633 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3634 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3635 * location into the LSDA.
3636 */
3637 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3638 /* This volatile store will keep the alloca alive */
3639 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3640
3641 set_metadata_flag (this_alloc, "mono.this");
3642 }
3643
3644 if (cfg->rgctx_var) {
3645 LLVMValueRef rgctx_alloc, store;
3646
3647 /*
3648 * We handle the rgctx arg similarly to the this pointer.
3649 */
3650 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3651 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3652 /* This volatile store will keep the alloca alive */
3653 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3654
3655 set_metadata_flag (rgctx_alloc, "mono.this");
3656 }
3657
3658 /* Initialize the method if needed */
3659 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
3660 /* Emit a location for the initialization code */
3661 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3662 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3663
3664 LLVMBuildBr (ctx->builder, ctx->init_bb);
3665 builder = ctx->builder = create_builder (ctx);
3666 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3667 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3668 }
3669
3670 /* Compute nesting between clauses */
3671 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3672 for (i = 0; i < cfg->header->num_clauses; ++i) {
3673 for (j = 0; j < cfg->header->num_clauses; ++j) {
3674 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3675 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3676
3677 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3678 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3679 }
3680 }
3681
3682 /*
3683 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3684 * it needs to continue normally, or return back to the exception handling system.
3685 */
3686 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3687 int clause_index;
3688 char name [128];
3689
3690 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3691 continue;
3692
3693 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3694 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
3695 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
3696
3697 if (bb->in_scount == 0) {
3698 LLVMValueRef val;
3699
3700 sprintf (name, "finally_ind_bb%d", bb->block_num);
3701 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3702 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3703
3704 ctx->bblocks [bb->block_num].finally_ind = val;
3705 } else {
3706 /* Create a variable to hold the exception var */
3707 if (!ctx->ex_var)
3708 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3709 }
3710
3711 /*
3712 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
3713 * LLVM bblock containing a landing pad causes problems for the
3714 * LLVM optimizer passes.
3715 */
3716 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
3717 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
3718 }
3719 ctx->builder = old_builder;
3720 }
3721
3722 static gboolean
3723 needs_extra_arg (EmitContext *ctx, MonoMethod *method)
3724 {
3725 WrapperInfo *info = NULL;
3726
3727 /*
3728 * When targeting wasm, the caller and callee signature has to match exactly. This means
3729 * that every method which can be called indirectly need an extra arg since the caller
3730 * will call it through an ftnptr and will pass an extra arg.
3731 */
3732 if (!ctx->cfg->llvm_only || !ctx->emit_dummy_arg)
3733 return FALSE;
3734 if (method->wrapper_type)
3735 info = mono_marshal_get_wrapper_info (method);
3736
3737 switch (method->wrapper_type) {
3738 case MONO_WRAPPER_OTHER:
3739 if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
3740 /* Already have an explicit extra arg */
3741 return FALSE;
3742 break;
3743 case MONO_WRAPPER_MANAGED_TO_NATIVE:
3744 if (strstr (method->name, "icall_wrapper"))
3745 /* These are JIT icall wrappers which are only called from JITted code directly */
3746 return FALSE;
3747 /* Normal icalls can be virtual methods which need an extra arg */
3748 break;
3749 case MONO_WRAPPER_RUNTIME_INVOKE:
3750 case MONO_WRAPPER_ALLOC:
3751 case MONO_WRAPPER_CASTCLASS:
3752 case MONO_WRAPPER_WRITE_BARRIER:
3753 case MONO_WRAPPER_NATIVE_TO_MANAGED:
3754 return FALSE;
3755 case MONO_WRAPPER_STELEMREF:
3756 if (info->subtype != WRAPPER_SUBTYPE_VIRTUAL_STELEMREF)
3757 return FALSE;
3758 break;
3759 case MONO_WRAPPER_MANAGED_TO_MANAGED:
3760 if (info->subtype == WRAPPER_SUBTYPE_STRING_CTOR)
3761 return FALSE;
3762 break;
3763 default:
3764 break;
3765 }
3766 if (method->string_ctor)
3767 return FALSE;
3768
3769 /* These are called from gsharedvt code with an indirect call which doesn't pass an extra arg */
3770 if (method->klass == mono_get_string_class () && (strstr (method->name, "memcpy") || strstr (method->name, "bzero")))
3771 return FALSE;
3772 return TRUE;
3773 }
3774
3775 static inline gboolean
3776 is_supported_callconv (EmitContext *ctx, MonoCallInst *call)
3777 {
3778 #if defined(TARGET_WIN32) && defined(TARGET_AMD64)
3779 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) ||
3780 (call->signature->call_convention == MONO_CALL_C) ||
3781 (call->signature->call_convention == MONO_CALL_STDCALL);
3782 #else
3783 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) || ((call->signature->call_convention == MONO_CALL_C) && ctx->llvm_only);
3784 #endif
3785 return result;
3786 }
3787
3788 static void
3789 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3790 {
3791 MonoCompile *cfg = ctx->cfg;
3792 LLVMValueRef *values = ctx->values;
3793 LLVMValueRef *addresses = ctx->addresses;
3794 MonoCallInst *call = (MonoCallInst*)ins;
3795 MonoMethodSignature *sig = call->signature;
3796 LLVMValueRef callee = NULL, lcall;
3797 LLVMValueRef *args;
3798 LLVMCallInfo *cinfo;
3799 GSList *l;
3800 int i, len, nargs;
3801 gboolean vretaddr;
3802 LLVMTypeRef llvm_sig;
3803 gpointer target;
3804 gboolean is_virtual, calli;
3805 LLVMBuilderRef builder = *builder_ref;
3806
3807 /* If both imt and rgctx arg are required, only pass the imt arg, the rgctx trampoline will pass the rgctx */
3808 if (call->imt_arg_reg)
3809 call->rgctx_arg_reg = 0;
3810
3811 if (!is_supported_callconv (ctx, call)) {
3812 set_failure (ctx, "non-default callconv");
3813 return;
3814 }
3815
3816 cinfo = call->cinfo;
3817 g_assert (cinfo);
3818 if (call->rgctx_arg_reg)
3819 cinfo->rgctx_arg = TRUE;
3820 if (call->imt_arg_reg)
3821 cinfo->imt_arg = TRUE;
3822 if (!call->rgctx_arg_reg && call->method && needs_extra_arg (ctx, call->method))
3823 cinfo->dummy_arg = TRUE;
3824
3825 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3826
3827 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3828 if (!ctx_ok (ctx))
3829 return;
3830
3831 int const opcode = ins->opcode;
3832
3833 is_virtual = opcode == OP_VOIDCALL_MEMBASE || opcode == OP_CALL_MEMBASE
3834 || opcode == OP_VCALL_MEMBASE || opcode == OP_LCALL_MEMBASE
3835 || opcode == OP_FCALL_MEMBASE || opcode == OP_RCALL_MEMBASE
3836 || opcode == OP_TAILCALL_MEMBASE;
3837 calli = !call->fptr_is_patch && (opcode == OP_VOIDCALL_REG || opcode == OP_CALL_REG
3838 || opcode == OP_VCALL_REG || opcode == OP_LCALL_REG || opcode == OP_FCALL_REG
3839 || opcode == OP_RCALL_REG || opcode == OP_TAILCALL_REG);
3840
3841 /* FIXME: Avoid creating duplicate methods */
3842
3843 if (ins->flags & MONO_INST_HAS_METHOD) {
3844 if (is_virtual) {
3845 callee = NULL;
3846 } else {
3847 if (cfg->compile_aot) {
3848 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3849 if (!callee) {
3850 set_failure (ctx, "can't encode patch");
3851 return;
3852 }
3853 } else if (cfg->method == call->method) {
3854 callee = ctx->lmethod;
3855 } else {
3856 ERROR_DECL (error);
3857 static int tramp_index;
3858 char *name;
3859
3860 name = g_strdup_printf ("[tramp_%d] %s", tramp_index, mono_method_full_name (call->method, TRUE));
3861 tramp_index ++;
3862
3863 /*
3864 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3865 * Make all calls through a global. The address of the global will be saved in
3866 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3867 * compiled.
3868 */
3869 LLVMValueRef tramp_var = (LLVMValueRef)g_hash_table_lookup (ctx->jit_callees, call->method);
3870 if (!tramp_var) {
3871 target =
3872 mono_create_jit_trampoline (mono_domain_get (),
3873 call->method, error);
3874 if (!is_ok (error)) {
3875 set_failure (ctx, mono_error_get_message (error));
3876 mono_error_cleanup (error);
3877 return;
3878 }
3879
3880 tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
3881 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
3882 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3883 g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
3884 }
3885 callee = LLVMBuildLoad (builder, tramp_var, "");
3886 }
3887 }
3888
3889 if (!cfg->llvm_only && call->method && strstr (m_class_get_name (call->method->klass), "AsyncVoidMethodBuilder")) {
3890 /* LLVM miscompiles async methods */
3891 set_failure (ctx, "#13734");
3892 return;
3893 }
3894 } else if (calli) {
3895 } else {
3896 const MonoJitICallId jit_icall_id = call->jit_icall_id;
3897
3898 if (jit_icall_id) {
3899 if (cfg->compile_aot) {
3900 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3901 if (!callee) {
3902 set_failure (ctx, "can't encode patch");
3903 return;
3904 }
3905 } else {
3906 callee = get_jit_callee (ctx, "", llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3907 }
3908 } else {
3909 if (cfg->compile_aot) {
3910 callee = NULL;
3911 if (cfg->abs_patches) {
3912 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3913 if (abs_ji) {
3914 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3915 if (!callee) {
3916 set_failure (ctx, "can't encode patch");
3917 return;
3918 }
3919 }
3920 }
3921 if (!callee) {
3922 set_failure (ctx, "aot");
3923 return;
3924 }
3925 } else {
3926 if (cfg->abs_patches) {
3927 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3928 if (abs_ji) {
3929 ERROR_DECL (error);
3930
3931 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, error);
3932 mono_error_assert_ok (error);
3933 callee = get_jit_callee (ctx, "", llvm_sig, abs_ji->type, abs_ji->data.target);
3934 } else {
3935 g_assert_not_reached ();
3936 }
3937 } else {
3938 g_assert_not_reached ();
3939 }
3940 }
3941 }
3942 }
3943
3944 if (is_virtual) {
3945 int size = TARGET_SIZEOF_VOID_P;
3946 LLVMValueRef index;
3947
3948 g_assert (ins->inst_offset % size == 0);
3949 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3950
3951 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3952 } else if (calli) {
3953 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3954 } else {
3955 if (ins->flags & MONO_INST_HAS_METHOD) {
3956 }
3957 }
3958
3959 /*
3960 * Collect and convert arguments
3961 */
3962 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg + call->cinfo->dummy_arg + 1;
3963 len = sizeof (LLVMValueRef) * nargs;
3964 args = g_newa (LLVMValueRef, nargs);
3965 memset (args, 0, len);
3966 l = call->out_ireg_args;
3967
3968 if (call->rgctx_arg_reg) {
3969 g_assert (values [call->rgctx_arg_reg]);
3970 g_assert (cinfo->rgctx_arg_pindex < nargs);
3971 /*
3972 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3973 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3974 * it using a volatile load.
3975 */
3976 #ifdef TARGET_ARM
3977 if (!ctx->imt_rgctx_loc)
3978 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3979 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3980 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3981 #else
3982 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3983 #endif
3984 }
3985 if (call->imt_arg_reg) {
3986 g_assert (!ctx->llvm_only);
3987 g_assert (values [call->imt_arg_reg]);
3988 g_assert (cinfo->imt_arg_pindex < nargs);
3989 #ifdef TARGET_ARM
3990 if (!ctx->imt_rgctx_loc)
3991 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3992 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3993 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3994 #else
3995 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3996 #endif
3997 }
3998 switch (cinfo->ret.storage) {
3999 case LLVMArgGsharedvtVariable: {
4000 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
4001
4002 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
4003 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
4004 } else {
4005 g_assert (addresses [call->inst.dreg]);
4006 args [cinfo->vret_arg_pindex] = convert (ctx, addresses [call->inst.dreg], IntPtrType ());
4007 }
4008 break;
4009 }
4010 default:
4011 if (vretaddr) {
4012 if (!addresses [call->inst.dreg])
4013 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4014 g_assert (cinfo->vret_arg_pindex < nargs);
4015 if (cinfo->ret.storage == LLVMArgVtypeByRef)
4016 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
4017 else
4018 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
4019 }
4020 break;
4021 }
4022
4023 /*
4024 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
4025 * use the real callee for argument type conversion.
4026 */
4027 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
4028 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
4029 LLVMGetParamTypes (callee_type, param_types);
4030
4031 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
4032 guint32 regpair;
4033 int reg, pindex;
4034 LLVMArgInfo *ainfo = &call->cinfo->args [i];
4035
4036 pindex = ainfo->pindex;
4037
4038 regpair = (guint32)(gssize)(l->data);
4039 reg = regpair & 0xffffff;
4040 args [pindex] = values [reg];
4041 switch (ainfo->storage) {
4042 case LLVMArgVtypeInReg:
4043 case LLVMArgAsFpArgs: {
4044 guint32 nargs;
4045 int j;
4046
4047 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
4048 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
4049 pindex += ainfo->ndummy_fpargs;
4050
4051 g_assert (addresses [reg]);
4052 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
4053 pindex += nargs;
4054
4055 // FIXME: alignment
4056 // FIXME: Get rid of the VMOVE
4057 break;
4058 }
4059 case LLVMArgVtypeByVal:
4060 g_assert (addresses [reg]);
4061 args [pindex] = addresses [reg];
4062 break;
4063 case LLVMArgVtypeAddr :
4064 case LLVMArgVtypeByRef: {
4065 g_assert (addresses [reg]);
4066 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4067 break;
4068 }
4069 case LLVMArgAsIArgs:
4070 g_assert (addresses [reg]);
4071 if (ainfo->esize == 8)
4072 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (LLVMInt64Type (), ainfo->nslots), 0)), "");
4073 else
4074 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
4075 break;
4076 case LLVMArgVtypeAsScalar:
4077 g_assert_not_reached ();
4078 break;
4079 case LLVMArgGsharedvtFixed:
4080 case LLVMArgGsharedvtFixedVtype:
4081 g_assert (addresses [reg]);
4082 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4083 break;
4084 case LLVMArgGsharedvtVariable:
4085 g_assert (addresses [reg]);
4086 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
4087 break;
4088 default:
4089 g_assert (args [pindex]);
4090 if (i == 0 && sig->hasthis)
4091 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
4092 else
4093 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
4094 break;
4095 }
4096 g_assert (pindex <= nargs);
4097
4098 l = l->next;
4099 }
4100
4101
4102 if (call->cinfo->dummy_arg) {
4103 g_assert (call->cinfo->dummy_arg_pindex < nargs);
4104 args [call->cinfo->dummy_arg_pindex] = LLVMConstNull (ctx->module->ptr_type);
4105 }
4106
4107 // FIXME: Align call sites
4108
4109 /*
4110 * Emit the call
4111 */
4112 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
4113
4114 mono_llvm_nonnull_state_update (ctx, lcall, call->method, args, LLVMCountParamTypes (llvm_sig));
4115
4116 // If we just allocated an object, it's not null.
4117 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC) {
4118 mono_llvm_set_call_nonnull_ret (lcall);
4119 }
4120
4121 if (ins->opcode != OP_TAILCALL && ins->opcode != OP_TAILCALL_MEMBASE && LLVMGetInstructionOpcode (lcall) == LLVMCall)
4122 mono_llvm_set_call_notailcall (lcall);
4123
4124 // Add original method name we are currently emitting as a custom string metadata (the only way to leave comments in LLVM IR)
4125 if (mono_debug_enabled () && call && call->method)
4126 mono_llvm_add_string_metadata (lcall, "managed_name", mono_method_full_name (call->method, TRUE));
4127
4128 // As per the LLVM docs, a function has a noalias return value if and only if
4129 // it is an allocation function. This is an allocation function.
4130 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC)
4131 mono_llvm_set_call_noalias_ret (lcall);
4132
4133 /*
4134 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
4135 */
4136 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
4137 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
4138 #endif
4139 /* The two can't be used together, so use only one LLVM calling conv to pass them */
4140 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
4141 if (!sig->pinvoke && !cfg->llvm_only)
4142 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
4143
4144 if (cinfo->ret.storage == LLVMArgVtypeByRef)
4145 mono_llvm_add_instr_attr (lcall, 1 + cinfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
4146 if (!ctx->llvm_only && call->rgctx_arg_reg)
4147 mono_llvm_add_instr_attr (lcall, 1 + cinfo->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
4148 if (call->imt_arg_reg)
4149 mono_llvm_add_instr_attr (lcall, 1 + cinfo->imt_arg_pindex, LLVM_ATTR_IN_REG);
4150
4151 /* Add byval attributes if needed */
4152 for (i = 0; i < sig->param_count; ++i) {
4153 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
4154
4155 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
4156 mono_llvm_add_instr_attr (lcall, 1 + ainfo->pindex, LLVM_ATTR_BY_VAL);
4157 }
4158
4159 /*
4160 * Convert the result
4161 */
4162 switch (cinfo->ret.storage) {
4163 case LLVMArgVtypeInReg: {
4164 LLVMValueRef regs [2];
4165
4166 if (LLVMTypeOf (lcall) == LLVMVoidType ())
4167 /* Empty struct */
4168 break;
4169
4170 if (!addresses [ins->dreg])
4171 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
4172
4173 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
4174 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
4175 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
4176 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
4177 break;
4178 }
4179 case LLVMArgVtypeByVal:
4180 if (!addresses [call->inst.dreg])
4181 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4182 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
4183 break;
4184 case LLVMArgAsIArgs:
4185 case LLVMArgFpStruct:
4186 if (!addresses [call->inst.dreg])
4187 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4188 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4189 break;
4190 case LLVMArgVtypeAsScalar:
4191 if (!addresses [call->inst.dreg])
4192 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4193 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4194 break;
4195 case LLVMArgVtypeRetAddr:
4196 case LLVMArgVtypeByRef:
4197 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret))) {
4198 /* Some opcodes like STOREX_MEMBASE access these by value */
4199 g_assert (addresses [call->inst.dreg]);
4200 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4201 }
4202 break;
4203 case LLVMArgGsharedvtVariable:
4204 break;
4205 case LLVMArgGsharedvtFixed:
4206 case LLVMArgGsharedvtFixedVtype:
4207 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4208 break;
4209 default:
4210 if (sig->ret->type != MONO_TYPE_VOID)
4211 /* If the method returns an unsigned value, need to zext it */
4212 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
4213 break;
4214 }
4215
4216 *builder_ref = ctx->builder;
4217 }
4218
4219 static void
4220 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4221 {
4222 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception : MONO_JIT_ICALL_mono_llvm_throw_exception;
4223 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
4224
4225 LLVMTypeRef exc_type = type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_exception_class ()));
4226
4227 if (!callee) {
4228 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
4229
4230 g_assert (ctx->cfg->compile_aot);
4231 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (icall_id));
4232 }
4233
4234 LLVMValueRef args [2];
4235
4236 args [0] = convert (ctx, exc, exc_type);
4237 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
4238
4239 LLVMBuildUnreachable (ctx->builder);
4240
4241 ctx->builder = create_builder (ctx);
4242 }
4243
4244 static void
4245 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4246 {
4247 MonoMethodSignature *throw_sig;
4248
4249 LLVMValueRef * const pcallee = rethrow ? &ctx->module->rethrow : &ctx->module->throw_icall;
4250 LLVMValueRef callee = *pcallee;
4251 char const * const icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4252 #ifndef TARGET_X86
4253 const
4254 #endif
4255 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_arch_rethrow_exception : MONO_JIT_ICALL_mono_arch_throw_exception;
4256
4257 if (!callee) {
4258 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
4259 throw_sig->ret = m_class_get_byval_arg (mono_get_void_class ());
4260 throw_sig->params [0] = m_class_get_byval_arg (mono_get_object_class ());
4261 if (ctx->cfg->compile_aot) {
4262 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4263 } else {
4264 #ifdef TARGET_X86
4265 /*
4266 * LLVM doesn't push the exception argument, so we need a different
4267 * trampoline.
4268 */
4269 icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception_trampoline : MONO_JIT_ICALL_mono_llvm_throw_exception_trampoline;
4270 #endif
4271 callee = get_jit_callee (ctx, icall_name, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4272 }
4273
4274 mono_memory_barrier ();
4275 }
4276 LLVMValueRef arg;
4277 arg = convert (ctx, exc, type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_object_class ())));
4278 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
4279 }
4280
4281 static void
4282 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
4283 {
4284 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_resume_exception;
4285 LLVMValueRef callee;
4286
4287 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
4288
4289 g_assert (ctx->cfg->compile_aot);
4290 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4291
4292 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
4293
4294 LLVMBuildUnreachable (ctx->builder);
4295
4296 ctx->builder = create_builder (ctx);
4297 }
4298
4299 static LLVMValueRef
4300 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4301 {
4302 const char *icall_name = "mono_llvm_clear_exception";
4303 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_clear_exception;
4304
4305 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
4306 LLVMValueRef callee = NULL;
4307
4308 if (!callee) {
4309 if (ctx->cfg->compile_aot) {
4310 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4311 } else {
4312 // FIXME: This is broken.
4313 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4314 }
4315 }
4316
4317 g_assert (builder && callee);
4318
4319 return LLVMBuildCall (builder, callee, NULL, 0, "");
4320 }
4321
4322 static LLVMValueRef
4323 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4324 {
4325 const char *icall_name = "mono_llvm_load_exception";
4326 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_load_exception;
4327
4328 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
4329 LLVMValueRef callee = NULL;
4330
4331 if (!callee) {
4332 if (ctx->cfg->compile_aot) {
4333 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4334 } else {
4335 // FIXME: This is broken.
4336 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4337 }
4338 }
4339
4340 g_assert (builder && callee);
4341
4342 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
4343 }
4344
4345
4346 static LLVMValueRef
4347 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
4348 {
4349 const char *icall_name = "mono_llvm_match_exception";
4350 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_match_exception;
4351
4352 ctx->builder = builder;
4353
4354 LLVMValueRef args[5];
4355 const int num_args = G_N_ELEMENTS (args);
4356
4357 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
4358 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
4359 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
4360 if (ctx->cfg->rgctx_var) {
4361 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
4362 g_assert (rgctx_alloc);
4363 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
4364 } else {
4365 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
4366 }
4367 if (ctx->this_arg)
4368 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
4369 else
4370 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
4371
4372 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
4373 LLVMValueRef callee;
4374 g_assert (ctx->cfg->compile_aot);
4375 ctx->builder = builder;
4376 // get_callee expects ctx->builder to be the emitting builder
4377 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4378
4379 g_assert (builder && callee);
4380
4381 g_assert (ctx->ex_var);
4382
4383 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
4384 }
4385
4386 // FIXME: This won't work because the code-finding makes this
4387 // not a constant.
4388 /*#define MONO_PERSONALITY_DEBUG*/
4389
4390 #ifdef MONO_PERSONALITY_DEBUG
4391 static const gboolean use_mono_personality_debug = TRUE;
4392 static const char *default_personality_name = "mono_debug_personality";
4393 #else
4394 static const gboolean use_mono_personality_debug = FALSE;
4395 static const char *default_personality_name = "__gxx_personality_v0";
4396 #endif
4397
4398 static LLVMTypeRef
4399 default_cpp_lpad_exc_signature (void)
4400 {
4401 static gboolean inited = FALSE;
4402 static LLVMTypeRef sig;
4403
4404 if (!sig) {
4405 LLVMTypeRef signature [2];
4406 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
4407 signature [1] = LLVMInt32Type ();
4408 sig = LLVMStructType (signature, 2, FALSE);
4409 inited = TRUE;
4410 }
4411
4412 return sig;
4413 }
4414
4415 static LLVMValueRef
4416 get_mono_personality (EmitContext *ctx)
4417 {
4418 LLVMValueRef personality = NULL;
4419 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4420
4421 g_assert (ctx->cfg->compile_aot);
4422 if (!use_mono_personality_debug) {
4423 personality = get_intrins_by_name (ctx, default_personality_name);
4424 } else {
4425 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_debug_personality));
4426 }
4427
4428 g_assert (personality);
4429 return personality;
4430 }
4431
4432 static LLVMBasicBlockRef
4433 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
4434 {
4435 MonoCompile *cfg = ctx->cfg;
4436 LLVMBuilderRef old_builder = ctx->builder;
4437 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
4438
4439 LLVMBuilderRef lpadBuilder = create_builder (ctx);
4440 ctx->builder = lpadBuilder;
4441
4442 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
4443 g_assert (handler_bb);
4444
4445 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4446 LLVMValueRef personality = get_mono_personality (ctx);
4447 g_assert (personality);
4448
4449 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
4450 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
4451 g_free (bb_name);
4452 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4453 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
4454 g_assert (landing_pad);
4455
4456 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
4457 LLVMAddClause (landing_pad, cast);
4458
4459 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
4460 LLVMBuilderRef resume_builder = create_builder (ctx);
4461 ctx->builder = resume_builder;
4462 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
4463
4464 emit_resume_eh (ctx, handler_bb);
4465
4466 // Build match
4467 ctx->builder = lpadBuilder;
4468 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4469
4470 gboolean finally_only = TRUE;
4471
4472 MonoExceptionClause *group_cursor = group_start;
4473
4474 for (int i = 0; i < group_size; i ++) {
4475 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY || group_cursor->flags & MONO_EXCEPTION_CLAUSE_FAULT))
4476 finally_only = FALSE;
4477
4478 group_cursor++;
4479 }
4480
4481 // FIXME:
4482 // Handle landing pad inlining
4483
4484 if (!finally_only) {
4485 // So at each level of the exception stack we will match the exception again.
4486 // During that match, we need to compare against the handler types for the current
4487 // protected region. We send the try start and end so that we can only check against
4488 // handlers for this lexical protected region.
4489 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
4490
4491 // if returns -1, resume
4492 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
4493
4494 // else move to that target bb
4495 for (int i = 0; i < group_size; i++) {
4496 MonoExceptionClause *clause = group_start + i;
4497 int clause_index = clause - cfg->header->clauses;
4498 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4499 g_assert (handler_bb);
4500 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4501 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4502 }
4503 } else {
4504 int clause_index = group_start - cfg->header->clauses;
4505 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4506 g_assert (finally_bb);
4507
4508 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
4509 }
4510
4511 ctx->builder = old_builder;
4512
4513 return lpad_bb;
4514 }
4515
4516
4517 static void
4518 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
4519 {
4520 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
4521 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
4522
4523 // Make exception available to catch blocks
4524 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags & MONO_EXCEPTION_CLAUSE_FAULT)) {
4525 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
4526
4527 g_assert (ctx->ex_var);
4528 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
4529
4530 if (bb->in_scount == 1) {
4531 MonoInst *exvar = bb->in_stack [0];
4532 g_assert (!ctx->values [exvar->dreg]);
4533 g_assert (ctx->ex_var);
4534 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
4535 emit_volatile_store (ctx, exvar->dreg);
4536 }
4537
4538 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
4539 }
4540
4541 LLVMBuilderRef handler_builder = create_builder (ctx);
4542 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
4543 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
4544
4545 // Make the handler code end with a jump to cbb
4546 LLVMBuildBr (handler_builder, cbb);
4547 }
4548
4549 static void
4550 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
4551 {
4552 MonoCompile *cfg = ctx->cfg;
4553 LLVMValueRef *values = ctx->values;
4554 LLVMModuleRef lmodule = ctx->lmodule;
4555 BBInfo *bblocks = ctx->bblocks;
4556 LLVMTypeRef i8ptr;
4557 LLVMValueRef personality;
4558 LLVMValueRef landing_pad;
4559 LLVMBasicBlockRef target_bb;
4560 MonoInst *exvar;
4561 static int ti_generator;
4562 char ti_name [128];
4563 LLVMValueRef type_info;
4564 int clause_index;
4565 GSList *l;
4566
4567 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4568
4569 if (cfg->compile_aot) {
4570 /* Use a dummy personality function */
4571 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
4572 g_assert (personality);
4573 } else {
4574 /* Can't cache this as each method is in its own llvm module */
4575 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4576 personality = LLVMAddFunction (ctx->lmodule, "mono_personality", personality_type);
4577 mono_llvm_add_func_attr (personality, LLVM_ATTR_NO_UNWIND);
4578 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (personality, "ENTRY");
4579 LLVMBuilderRef builder2 = LLVMCreateBuilder ();
4580 LLVMPositionBuilderAtEnd (builder2, entry_bb);
4581 LLVMBuildRet (builder2, LLVMConstInt (LLVMInt32Type (), 0, FALSE));
4582 LLVMDisposeBuilder (builder2);
4583 }
4584
4585 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
4586
4587 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
4588
4589 /*
4590 * Create the type info
4591 */
4592 sprintf (ti_name, "type_info_%d", ti_generator);
4593 ti_generator ++;
4594
4595 if (cfg->compile_aot) {
4596 /* decode_eh_frame () in aot-runtime.c will decode this */
4597 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4598 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4599
4600 /*
4601 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4602 */
4603 LLVMSetLinkage (type_info, LLVMInternalLinkage);
4604 } else {
4605 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4606 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4607 }
4608
4609 {
4610 LLVMTypeRef members [2], ret_type;
4611
4612 members [0] = i8ptr;
4613 members [1] = LLVMInt32Type ();
4614 ret_type = LLVMStructType (members, 2, FALSE);
4615
4616 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
4617 LLVMAddClause (landing_pad, type_info);
4618
4619 /* Store the exception into the exvar */
4620 if (ctx->ex_var)
4621 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
4622 }
4623
4624 /*
4625 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4626 * code expects control to be transferred to this landing pad even in the
4627 * presence of nested clauses. The landing pad needs to branch to the landing
4628 * pads belonging to nested clauses based on the selector value returned by
4629 * the landing pad instruction, which is passed to the landing pad in a
4630 * register by the EH code.
4631 */
4632 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4633 g_assert (target_bb);
4634
4635 /*
4636 * Branch to the correct landing pad
4637 */
4638 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
4639 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
4640
4641 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
4642 int nesting_clause_index = GPOINTER_TO_INT (l->data);
4643 MonoBasicBlock *handler_bb;
4644
4645 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
4646 g_assert (handler_bb);
4647
4648 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4649 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4650 }
4651
4652 /* Start a new bblock which CALL_HANDLER can branch to */
4653 ctx->builder = builder = create_builder (ctx);
4654 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
4655
4656 ctx->bblocks [bb->block_num].end_bblock = target_bb;
4657
4658 /* Store the exception into the IL level exvar */
4659 if (bb->in_scount == 1) {
4660 g_assert (bb->in_scount == 1);
4661 exvar = bb->in_stack [0];
4662
4663 // FIXME: This is shared with filter clauses ?
4664 g_assert (!values [exvar->dreg]);
4665
4666 g_assert (ctx->ex_var);
4667 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
4668 emit_volatile_store (ctx, exvar->dreg);
4669 }
4670
4671 /* Make normal branches to the start of the clause branch to the new bblock */
4672 bblocks [bb->block_num].bblock = target_bb;
4673 }
4674
4675 //Wasm requires us to canonicalize NaNs.
4676 static LLVMValueRef
4677 get_double_const (MonoCompile *cfg, double val)
4678 {
4679 #ifdef TARGET_WASM
4680 if (mono_isnan (val))
4681 *(gint64 *)&val = 0x7FF8000000000000ll;
4682 #endif
4683 return LLVMConstReal (LLVMDoubleType (), val);
4684 }
4685
4686 static LLVMValueRef
4687 get_float_const (MonoCompile *cfg, float val)
4688 {
4689 #ifdef TARGET_WASM
4690 if (mono_isnan (val))
4691 *(int *)&val = 0x7FC00000;
4692 #endif
4693 if (cfg->r4fp)
4694 return LLVMConstReal (LLVMFloatType (), val);
4695 else
4696 return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val), LLVMDoubleType ());
4697 }
4698
4699 static void
4700 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4701 {
4702 MonoCompile *cfg = ctx->cfg;
4703 MonoMethodSignature *sig = ctx->sig;
4704 LLVMValueRef method = ctx->lmethod;
4705 LLVMValueRef *values = ctx->values;
4706 LLVMValueRef *addresses = ctx->addresses;
4707 LLVMCallInfo *linfo = ctx->linfo;
4708 BBInfo *bblocks = ctx->bblocks;
4709 MonoInst *ins;
4710 LLVMBasicBlockRef cbb;
4711 LLVMBuilderRef builder, starting_builder;
4712 gboolean has_terminator;
4713 LLVMValueRef v;
4714 LLVMValueRef lhs, rhs;
4715 int nins = 0;
4716
4717 cbb = get_end_bb (ctx, bb);
4718
4719 builder = create_builder (ctx);
4720 ctx->builder = builder;
4721 LLVMPositionBuilderAtEnd (builder, cbb);
4722
4723 if (!ctx_ok (ctx))
4724 return;
4725
4726 if (bb->flags & BB_EXCEPTION_HANDLER) {
4727 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4728 set_failure (ctx, "handler without invokes");
4729 return;
4730 }
4731
4732 if (ctx->llvm_only)
4733 emit_llvmonly_handler_start (ctx, bb, cbb);
4734 else
4735 emit_handler_start (ctx, bb, builder);
4736 if (!ctx_ok (ctx))
4737 return;
4738 builder = ctx->builder;
4739 }
4740
4741 /* Handle PHI nodes first */
4742 /* They should be grouped at the start of the bb */
4743 for (ins = bb->code; ins; ins = ins->next) {
4744 emit_dbg_loc (ctx, builder, ins->cil_code);
4745
4746 if (ins->opcode == OP_NOP)
4747 continue;
4748 if (!MONO_IS_PHI (ins))
4749 break;
4750
4751 int i;
4752 gboolean empty = TRUE;
4753
4754 /* Check that all input bblocks really branch to us */
4755 for (i = 0; i < bb->in_count; ++i) {
4756 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4757 ins->inst_phi_args [i + 1] = -1;
4758 else
4759 empty = FALSE;
4760 }
4761
4762 if (empty) {
4763 /* LLVM doesn't like phi instructions with zero operands */
4764 ctx->is_dead [ins->dreg] = TRUE;
4765 continue;
4766 }
4767
4768 /* Created earlier, insert it now */
4769 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4770
4771 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4772 int sreg1 = ins->inst_phi_args [i + 1];
4773 int count, j;
4774
4775 /*
4776 * Count the number of times the incoming bblock branches to us,
4777 * since llvm requires a separate entry for each.
4778 */
4779 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4780 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4781
4782 count = 0;
4783 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4784 if (switch_ins->inst_many_bb [j] == bb)
4785 count ++;
4786 }
4787 } else {
4788 count = 1;
4789 }
4790
4791 /* Remember for later */
4792 for (j = 0; j < count; ++j) {
4793 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4794 node->bb = bb;
4795 node->phi = ins;
4796 node->in_bb = bb->in_bb [i];
4797 node->sreg = sreg1;
4798 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4799 }
4800 }
4801 }
4802 // Add volatile stores for PHI nodes
4803 // These need to be emitted after the PHI nodes
4804 for (ins = bb->code; ins; ins = ins->next) {
4805 const char *spec = LLVM_INS_INFO (ins->opcode);
4806
4807 if (ins->opcode == OP_NOP)
4808 continue;
4809 if (!MONO_IS_PHI (ins))
4810 break;
4811
4812 if (spec [MONO_INST_DEST] != 'v')
4813 emit_volatile_store (ctx, ins->dreg);
4814 }
4815
4816 has_terminator = FALSE;
4817 starting_builder = builder;
4818 for (ins = bb->code; ins; ins = ins->next) {
4819 const char *spec = LLVM_INS_INFO (ins->opcode);
4820 char *dname = NULL;
4821 char dname_buf [128];
4822
4823 emit_dbg_loc (ctx, builder, ins->cil_code);
4824
4825 nins ++;
4826 if (nins > 1000) {
4827 /*
4828 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4829 * Start a new bblock.
4830 * Prevent the bblocks to be merged by doing a volatile load + cond branch
4831 * from localloc-ed memory.
4832 */
4833 if (!cfg->llvm_only)
4834 ;//set_failure (ctx, "basic block too long");
4835
4836 if (!ctx->long_bb_break_var) {
4837 ctx->long_bb_break_var = build_alloca_llvm_type_name (ctx, LLVMInt32Type (), 0, "long_bb_break");
4838 mono_llvm_build_store (ctx->alloca_builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4839 }
4840
4841 cbb = gen_bb (ctx, "CONT_LONG_BB");
4842 LLVMBasicBlockRef dummy_bb = gen_bb (ctx, "CONT_LONG_BB_DUMMY");
4843
4844 LLVMValueRef load = mono_llvm_build_load (builder, ctx->long_bb_break_var, "", TRUE);
4845 /*
4846 * The long_bb_break_var is initialized to 0 in the prolog, so this branch will always go to 'cbb'
4847 * but llvm doesn't know that, so the branch is not going to be eliminated.
4848 */
4849 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, load, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4850
4851 LLVMBuildCondBr (builder, cmp, cbb, dummy_bb);
4852
4853 /* Emit a dummy false bblock which does nothing but contains a volatile store so it cannot be eliminated */
4854 ctx->builder = builder = create_builder (ctx);
4855 LLVMPositionBuilderAtEnd (builder, dummy_bb);
4856 mono_llvm_build_store (builder, LLVMConstInt (LLVMInt32Type (), 1, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4857 LLVMBuildBr (builder, cbb);
4858
4859 ctx->builder = builder = create_builder (ctx);
4860 LLVMPositionBuilderAtEnd (builder, cbb);
4861 ctx->bblocks [bb->block_num].end_bblock = cbb;
4862 nins = 0;
4863
4864 emit_dbg_loc (ctx, builder, ins->cil_code);
4865 }
4866
4867 if (has_terminator)
4868 /* There could be instructions after a terminator, skip them */
4869 break;
4870
4871 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4872 sprintf (dname_buf, "t%d", ins->dreg);
4873 dname = dname_buf;
4874 }
4875
4876 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4877 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4878
4879 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4880 lhs = emit_volatile_load (ctx, ins->sreg1);
4881 } else {
4882 /* It is ok for SETRET to have an uninitialized argument */
4883 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4884 set_failure (ctx, "sreg1");
4885 return;
4886 }
4887 lhs = values [ins->sreg1];
4888 }
4889 } else {
4890 lhs = NULL;
4891 }
4892
4893 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4894 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4895 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4896 rhs = emit_volatile_load (ctx, ins->sreg2);
4897 } else {
4898 if (!values [ins->sreg2]) {
4899 set_failure (ctx, "sreg2");
4900 return;
4901 }
4902 rhs = values [ins->sreg2];
4903 }
4904 } else {
4905 rhs = NULL;
4906 }
4907
4908 //mono_print_ins (ins);
4909 gboolean skip_volatile_store = FALSE;
4910 switch (ins->opcode) {
4911 case OP_NOP:
4912 case OP_NOT_NULL:
4913 case OP_LIVERANGE_START:
4914 case OP_LIVERANGE_END:
4915 break;
4916 case OP_ICONST:
4917 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
4918 break;
4919 case OP_I8CONST:
4920 #if TARGET_SIZEOF_VOID_P == 4
4921 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4922 #else
4923 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
4924 #endif
4925 break;
4926 case OP_R8CONST:
4927 values [ins->dreg] = get_double_const (cfg, *(double*)ins->inst_p0);
4928 break;
4929 case OP_R4CONST:
4930 values [ins->dreg] = get_float_const (cfg, *(float*)ins->inst_p0);
4931 break;
4932 case OP_DUMMY_ICONST:
4933 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4934 break;
4935 case OP_DUMMY_I8CONST:
4936 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
4937 break;
4938 case OP_DUMMY_R8CONST:
4939 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
4940 break;
4941 case OP_BR: {
4942 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
4943 LLVMBuildBr (builder, target_bb);
4944 has_terminator = TRUE;
4945 break;
4946 }
4947 case OP_SWITCH: {
4948 int i;
4949 LLVMValueRef v;
4950 char bb_name [128];
4951 LLVMBasicBlockRef new_bb;
4952 LLVMBuilderRef new_builder;
4953
4954 // The default branch is already handled
4955 // FIXME: Handle it here
4956
4957 /* Start new bblock */
4958 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
4959 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
4960
4961 lhs = convert (ctx, lhs, LLVMInt32Type ());
4962 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
4963 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
4964 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
4965
4966 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
4967 }
4968
4969 new_builder = create_builder (ctx);
4970 LLVMPositionBuilderAtEnd (new_builder, new_bb);
4971 LLVMBuildUnreachable (new_builder);
4972
4973 has_terminator = TRUE;
4974 g_assert (!ins->next);
4975
4976 break;
4977 }
4978
4979 case OP_SETRET:
4980 switch (linfo->ret.storage) {
4981 case LLVMArgVtypeInReg: {
4982 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4983 LLVMValueRef val, addr, retval;
4984 int i;
4985
4986 retval = LLVMGetUndef (ret_type);
4987
4988 if (!addresses [ins->sreg1]) {
4989 /*
4990 * The return type is an LLVM vector type, have to convert between it and the
4991 * real return type which is a struct type.
4992 */
4993 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret)));
4994 /* Convert to 2xi64 first */
4995 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
4996
4997 for (i = 0; i < 2; ++i) {
4998 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4999 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
5000 } else {
5001 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
5002 }
5003 }
5004 } else {
5005 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
5006 for (i = 0; i < 2; ++i) {
5007 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
5008 LLVMValueRef indexes [2], part_addr;
5009
5010 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5011 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
5012 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
5013
5014 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
5015 } else {
5016 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
5017 }
5018 }
5019 }
5020 LLVMBuildRet (builder, retval);
5021 break;
5022 }
5023 case LLVMArgVtypeAsScalar: {
5024 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5025 LLVMValueRef retval;
5026
5027 g_assert (addresses [ins->sreg1]);
5028
5029 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
5030 LLVMBuildRet (builder, retval);
5031 break;
5032 }
5033 case LLVMArgVtypeByVal: {
5034 LLVMValueRef retval;
5035
5036 g_assert (addresses [ins->sreg1]);
5037 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
5038 LLVMBuildRet (builder, retval);
5039 break;
5040 }
5041 case LLVMArgVtypeByRef: {
5042 LLVMBuildRetVoid (builder);
5043 break;
5044 }
5045 case LLVMArgGsharedvtFixed: {
5046 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
5047 /* The return value is in lhs, need to store to the vret argument */
5048 /* sreg1 might not be set */
5049 if (lhs) {
5050 g_assert (cfg->vret_addr);
5051 g_assert (values [cfg->vret_addr->dreg]);
5052 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
5053 }
5054 LLVMBuildRetVoid (builder);
5055 break;
5056 }
5057 case LLVMArgGsharedvtFixedVtype: {
5058 /* Already set */
5059 LLVMBuildRetVoid (builder);
5060 break;
5061 }
5062 case LLVMArgGsharedvtVariable: {
5063 /* Already set */
5064 LLVMBuildRetVoid (builder);
5065 break;
5066 }
5067 case LLVMArgVtypeRetAddr: {
5068 LLVMBuildRetVoid (builder);
5069 break;
5070 }
5071 case LLVMArgAsIArgs:
5072 case LLVMArgFpStruct: {
5073 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5074 LLVMValueRef retval;
5075
5076 g_assert (addresses [ins->sreg1]);
5077 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
5078 LLVMBuildRet (builder, retval);
5079 break;
5080 }
5081 case LLVMArgNone:
5082 case LLVMArgNormal: {
5083 if (!lhs || ctx->is_dead [ins->sreg1]) {
5084 /*
5085 * The method did not set its return value, probably because it
5086 * ends with a throw.
5087 */
5088 if (cfg->vret_addr)
5089 LLVMBuildRetVoid (builder);
5090 else
5091 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
5092 } else {
5093 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
5094 }
5095 has_terminator = TRUE;
5096 break;
5097 }
5098 default:
5099 g_assert_not_reached ();
5100 break;
5101 }
5102 break;
5103 case OP_ICOMPARE:
5104 case OP_FCOMPARE:
5105 case OP_RCOMPARE:
5106 case OP_LCOMPARE:
5107 case OP_COMPARE:
5108 case OP_ICOMPARE_IMM:
5109 case OP_LCOMPARE_IMM:
5110 case OP_COMPARE_IMM: {
5111 CompRelation rel;
5112 LLVMValueRef cmp, args [16];
5113 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
5114 gboolean unlikely = FALSE;
5115
5116 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5117 if (ins->next->inst_false_bb->out_of_line)
5118 likely = TRUE;
5119 else if (ins->next->inst_true_bb->out_of_line)
5120 unlikely = TRUE;
5121 }
5122
5123 if (ins->next->opcode == OP_NOP)
5124 break;
5125
5126 if (ins->next->opcode == OP_BR)
5127 /* The comparison result is not needed */
5128 continue;
5129
5130 rel = mono_opcode_to_cond (ins->next->opcode);
5131
5132 if (ins->opcode == OP_ICOMPARE_IMM) {
5133 lhs = convert (ctx, lhs, LLVMInt32Type ());
5134 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5135 }
5136 if (ins->opcode == OP_LCOMPARE_IMM) {
5137 lhs = convert (ctx, lhs, LLVMInt64Type ());
5138 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5139 }
5140 if (ins->opcode == OP_LCOMPARE) {
5141 lhs = convert (ctx, lhs, LLVMInt64Type ());
5142 rhs = convert (ctx, rhs, LLVMInt64Type ());
5143 }
5144 if (ins->opcode == OP_ICOMPARE) {
5145 lhs = convert (ctx, lhs, LLVMInt32Type ());
5146 rhs = convert (ctx, rhs, LLVMInt32Type ());
5147 }
5148
5149 if (lhs && rhs) {
5150 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5151 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
5152 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
5153 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
5154 }
5155
5156 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
5157 if (ins->opcode == OP_FCOMPARE) {
5158 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5159 } else if (ins->opcode == OP_RCOMPARE) {
5160 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5161 } else if (ins->opcode == OP_COMPARE_IMM) {
5162 LLVMIntPredicate llvm_pred = cond_to_llvm_cond [rel];
5163 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0) {
5164 // We are emitting a NULL check for a pointer
5165 gboolean nonnull = mono_llvm_is_nonnull (lhs);
5166
5167 if (nonnull && llvm_pred == LLVMIntEQ)
5168 cmp = LLVMConstInt (LLVMInt1Type (), FALSE, FALSE);
5169 else if (nonnull && llvm_pred == LLVMIntNE)
5170 cmp = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
5171 else
5172 cmp = LLVMBuildICmp (builder, llvm_pred, lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
5173
5174 } else {
5175 cmp = LLVMBuildICmp (builder, llvm_pred, convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
5176 }
5177 } else if (ins->opcode == OP_LCOMPARE_IMM) {
5178 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5179 }
5180 else if (ins->opcode == OP_COMPARE) {
5181 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
5182 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5183 else
5184 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
5185 } else
5186 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5187
5188 if (likely || unlikely) {
5189 args [0] = cmp;
5190 args [1] = LLVMConstInt (LLVMInt1Type (), likely ? 1 : 0, FALSE);
5191 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
5192 }
5193
5194 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5195 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
5196 /*
5197 * If the target bb contains PHI instructions, LLVM requires
5198 * two PHI entries for this bblock, while we only generate one.
5199 * So convert this to an unconditional bblock. (bxc #171).
5200 */
5201 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
5202 } else {
5203 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
5204 }
5205 has_terminator = TRUE;
5206 } else if (MONO_IS_SETCC (ins->next)) {
5207 sprintf (dname_buf, "t%d", ins->next->dreg);
5208 dname = dname_buf;
5209 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5210
5211 /* Add stores for volatile variables */
5212 emit_volatile_store (ctx, ins->next->dreg);
5213 } else if (MONO_IS_COND_EXC (ins->next)) {
5214 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp);
5215 if (!ctx_ok (ctx))
5216 break;
5217 builder = ctx->builder;
5218 } else {
5219 set_failure (ctx, "next");
5220 break;
5221 }
5222
5223 ins = ins->next;
5224 break;
5225 }
5226 case OP_FCEQ:
5227 case OP_FCNEQ:
5228 case OP_FCLT:
5229 case OP_FCLT_UN:
5230 case OP_FCGT:
5231 case OP_FCGT_UN:
5232 case OP_FCGE:
5233 case OP_FCLE: {
5234 CompRelation rel;
5235 LLVMValueRef cmp;
5236
5237 rel = mono_opcode_to_cond (ins->opcode);
5238
5239 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5240 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5241 break;
5242 }
5243 case OP_RCEQ:
5244 case OP_RCNEQ:
5245 case OP_RCLT:
5246 case OP_RCLT_UN:
5247 case OP_RCGT:
5248 case OP_RCGT_UN: {
5249 CompRelation rel;
5250 LLVMValueRef cmp;
5251
5252 rel = mono_opcode_to_cond (ins->opcode);
5253
5254 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5255 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5256 break;
5257 }
5258 case OP_PHI:
5259 case OP_FPHI:
5260 case OP_VPHI:
5261 case OP_XPHI: {
5262 // Handled above
5263 skip_volatile_store = TRUE;
5264 break;
5265 }
5266 case OP_MOVE:
5267 case OP_LMOVE:
5268 case OP_XMOVE:
5269 case OP_SETFRET:
5270 g_assert (lhs);
5271 values [ins->dreg] = lhs;
5272 break;
5273 case OP_FMOVE:
5274 case OP_RMOVE: {
5275 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
5276
5277 g_assert (lhs);
5278 values [ins->dreg] = lhs;
5279
5280 if (var && m_class_get_byval_arg (var->klass)->type == MONO_TYPE_R4) {
5281 /*
5282 * This is added by the spilling pass in case of the JIT,
5283 * but we have to do it ourselves.
5284 */
5285 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
5286 }
5287 break;
5288 }
5289 case OP_MOVE_F_TO_I4: {
5290 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
5291 break;
5292 }
5293 case OP_MOVE_I4_TO_F: {
5294 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
5295 break;
5296 }
5297 case OP_MOVE_F_TO_I8: {
5298 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
5299 break;
5300 }
5301 case OP_MOVE_I8_TO_F: {
5302 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
5303 break;
5304 }
5305 case OP_IADD:
5306 case OP_ISUB:
5307 case OP_IAND:
5308 case OP_IMUL:
5309 case OP_IDIV:
5310 case OP_IDIV_UN:
5311 case OP_IREM:
5312 case OP_IREM_UN:
5313 case OP_IOR:
5314 case OP_IXOR:
5315 case OP_ISHL:
5316 case OP_ISHR:
5317 case OP_ISHR_UN:
5318 case OP_FADD:
5319 case OP_FSUB:
5320 case OP_FMUL:
5321 case OP_FDIV:
5322 case OP_LADD:
5323 case OP_LSUB:
5324 case OP_LMUL:
5325 case OP_LDIV:
5326 case OP_LDIV_UN:
5327 case OP_LREM:
5328 case OP_LREM_UN:
5329 case OP_LAND:
5330 case OP_LOR:
5331 case OP_LXOR:
5332 case OP_LSHL:
5333 case OP_LSHR:
5334 case OP_LSHR_UN:
5335 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5336 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5337
5338 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
5339 if (!ctx_ok (ctx))
5340 break;
5341 builder = ctx->builder;
5342
5343 switch (ins->opcode) {
5344 case OP_IADD:
5345 case OP_LADD:
5346 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
5347 break;
5348 case OP_ISUB:
5349 case OP_LSUB:
5350 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
5351 break;
5352 case OP_IMUL:
5353 case OP_LMUL:
5354 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
5355 break;
5356 case OP_IREM:
5357 case OP_LREM:
5358 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
5359 break;
5360 case OP_IREM_UN:
5361 case OP_LREM_UN:
5362 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
5363 break;
5364 case OP_IDIV:
5365 case OP_LDIV:
5366 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
5367 break;
5368 case OP_IDIV_UN:
5369 case OP_LDIV_UN:
5370 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
5371 break;
5372 case OP_FDIV:
5373 case OP_RDIV:
5374 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5375 break;
5376 case OP_IAND:
5377 case OP_LAND:
5378 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
5379 break;
5380 case OP_IOR:
5381 case OP_LOR:
5382 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
5383 break;
5384 case OP_IXOR:
5385 case OP_LXOR:
5386 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
5387 break;
5388 case OP_ISHL:
5389 case OP_LSHL:
5390 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
5391 break;
5392 case OP_ISHR:
5393 case OP_LSHR:
5394 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
5395 break;
5396 case OP_ISHR_UN:
5397 case OP_LSHR_UN:
5398 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
5399 break;
5400
5401 case OP_FADD:
5402 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5403 break;
5404 case OP_FSUB:
5405 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5406 break;
5407 case OP_FMUL:
5408 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5409 break;
5410
5411 default:
5412 g_assert_not_reached ();
5413 }
5414 break;
5415 case OP_RADD:
5416 case OP_RSUB:
5417 case OP_RMUL:
5418 case OP_RDIV: {
5419 lhs = convert (ctx, lhs, LLVMFloatType ());
5420 rhs = convert (ctx, rhs, LLVMFloatType ());
5421 switch (ins->opcode) {
5422 case OP_RADD:
5423 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5424 break;
5425 case OP_RSUB:
5426 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5427 break;
5428 case OP_RMUL:
5429 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5430 break;
5431 case OP_RDIV:
5432 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5433 break;
5434 default:
5435 g_assert_not_reached ();
5436 break;
5437 }
5438 break;
5439 }
5440 case OP_IADD_IMM:
5441 case OP_ISUB_IMM:
5442 case OP_IMUL_IMM:
5443 case OP_IREM_IMM:
5444 case OP_IREM_UN_IMM:
5445 case OP_IDIV_IMM:
5446 case OP_IDIV_UN_IMM:
5447 case OP_IAND_IMM:
5448 case OP_IOR_IMM:
5449 case OP_IXOR_IMM:
5450 case OP_ISHL_IMM:
5451 case OP_ISHR_IMM:
5452 case OP_ISHR_UN_IMM:
5453 case OP_LADD_IMM:
5454 case OP_LSUB_IMM:
5455 case OP_LMUL_IMM:
5456 case OP_LREM_IMM:
5457 case OP_LAND_IMM:
5458 case OP_LOR_IMM:
5459 case OP_LXOR_IMM:
5460 case OP_LSHL_IMM:
5461 case OP_LSHR_IMM:
5462 case OP_LSHR_UN_IMM:
5463 case OP_ADD_IMM:
5464 case OP_AND_IMM:
5465 case OP_MUL_IMM:
5466 case OP_SHL_IMM:
5467 case OP_SHR_IMM:
5468 case OP_SHR_UN_IMM: {
5469 LLVMValueRef imm;
5470
5471 if (spec [MONO_INST_SRC1] == 'l') {
5472 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5473 } else {
5474 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5475 }
5476
5477 emit_div_check (ctx, builder, bb, ins, lhs, imm);
5478 if (!ctx_ok (ctx))
5479 break;
5480 builder = ctx->builder;
5481
5482 #if TARGET_SIZEOF_VOID_P == 4
5483 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
5484 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5485 #endif
5486
5487 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5488 lhs = convert (ctx, lhs, IntPtrType ());
5489 imm = convert (ctx, imm, LLVMTypeOf (lhs));
5490 switch (ins->opcode) {
5491 case OP_IADD_IMM:
5492 case OP_LADD_IMM:
5493 case OP_ADD_IMM:
5494 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
5495 break;
5496 case OP_ISUB_IMM:
5497 case OP_LSUB_IMM:
5498 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
5499 break;
5500 case OP_IMUL_IMM:
5501 case OP_MUL_IMM:
5502 case OP_LMUL_IMM:
5503 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
5504 break;
5505 case OP_IDIV_IMM:
5506 case OP_LDIV_IMM:
5507 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
5508 break;
5509 case OP_IDIV_UN_IMM:
5510 case OP_LDIV_UN_IMM:
5511 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
5512 break;
5513 case OP_IREM_IMM:
5514 case OP_LREM_IMM:
5515 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
5516 break;
5517 case OP_IREM_UN_IMM:
5518 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
5519 break;
5520 case OP_IAND_IMM:
5521 case OP_LAND_IMM:
5522 case OP_AND_IMM:
5523 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
5524 break;
5525 case OP_IOR_IMM:
5526 case OP_LOR_IMM:
5527 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
5528 break;
5529 case OP_IXOR_IMM:
5530 case OP_LXOR_IMM:
5531 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
5532 break;
5533 case OP_ISHL_IMM:
5534 case OP_LSHL_IMM:
5535 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5536 break;
5537 case OP_SHL_IMM:
5538 if (TARGET_SIZEOF_VOID_P == 8) {
5539 /* The IL is not regular */
5540 lhs = convert (ctx, lhs, LLVMInt64Type ());
5541 imm = convert (ctx, imm, LLVMInt64Type ());
5542 }
5543 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5544 break;
5545 case OP_ISHR_IMM:
5546 case OP_LSHR_IMM:
5547 case OP_SHR_IMM:
5548 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
5549 break;
5550 case OP_ISHR_UN_IMM:
5551 /* This is used to implement conv.u4, so the lhs could be an i8 */
5552 lhs = convert (ctx, lhs, LLVMInt32Type ());
5553 imm = convert (ctx, imm, LLVMInt32Type ());
5554 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5555 break;
5556 case OP_LSHR_UN_IMM:
5557 case OP_SHR_UN_IMM:
5558 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5559 break;
5560 default:
5561 g_assert_not_reached ();
5562 }
5563 break;
5564 }
5565 case OP_INEG:
5566 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5567 break;
5568 case OP_LNEG:
5569 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5570 lhs = convert (ctx, lhs, LLVMInt64Type ());
5571 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
5572 break;
5573 case OP_FNEG:
5574 lhs = convert (ctx, lhs, LLVMDoubleType ());
5575 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
5576 break;
5577 case OP_RNEG:
5578 lhs = convert (ctx, lhs, LLVMFloatType ());
5579 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
5580 break;
5581 case OP_INOT: {
5582 guint32 v = 0xffffffff;
5583 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5584 break;
5585 }
5586 case OP_LNOT: {
5587 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5588 lhs = convert (ctx, lhs, LLVMInt64Type ());
5589 guint64 v = 0xffffffffffffffffLL;
5590 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
5591 break;
5592 }
5593 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5594 case OP_X86_LEA: {
5595 LLVMValueRef v1, v2;
5596
5597 rhs = LLVMBuildSExt (builder, convert (ctx, rhs, LLVMInt32Type ()), LLVMInt64Type (), "");
5598
5599 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ((unsigned long long)1 << ins->backend.shift_amount), FALSE), "");
5600 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
5601 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
5602 break;
5603 }
5604 #endif
5605
5606 case OP_ICONV_TO_I1:
5607 case OP_ICONV_TO_I2:
5608 case OP_ICONV_TO_I4:
5609 case OP_ICONV_TO_U1:
5610 case OP_ICONV_TO_U2:
5611 case OP_ICONV_TO_U4:
5612 case OP_LCONV_TO_I1:
5613 case OP_LCONV_TO_I2:
5614 case OP_LCONV_TO_U1:
5615 case OP_LCONV_TO_U2:
5616 case OP_LCONV_TO_U4: {
5617 gboolean sign;
5618
5619 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
5620
5621 /* Have to do two casts since our vregs have type int */
5622 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
5623 if (sign)
5624 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
5625 else
5626 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
5627 break;
5628 }
5629 case OP_ICONV_TO_I8:
5630 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
5631 break;
5632 case OP_ICONV_TO_U8:
5633 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
5634 break;
5635 case OP_FCONV_TO_I4:
5636 case OP_RCONV_TO_I4:
5637 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
5638 break;
5639 case OP_FCONV_TO_I1:
5640 case OP_RCONV_TO_I1:
5641 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
5642 break;
5643 case OP_FCONV_TO_U1:
5644 case OP_RCONV_TO_U1:
5645 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
5646 break;
5647 case OP_FCONV_TO_I2:
5648 case OP_RCONV_TO_I2:
5649 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5650 break;
5651 case OP_FCONV_TO_U2:
5652 case OP_RCONV_TO_U2:
5653 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5654 break;
5655 case OP_FCONV_TO_U4:
5656 case OP_RCONV_TO_U4:
5657 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
5658 break;
5659 case OP_FCONV_TO_U8:
5660 case OP_RCONV_TO_U8:
5661 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt64Type (), dname);
5662 break;
5663 case OP_FCONV_TO_I8:
5664 case OP_RCONV_TO_I8:
5665 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
5666 break;
5667 case OP_FCONV_TO_I:
5668 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
5669 break;
5670 case OP_ICONV_TO_R8:
5671 case OP_LCONV_TO_R8:
5672 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
5673 break;
5674 case OP_ICONV_TO_R_UN:
5675 case OP_LCONV_TO_R_UN:
5676 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
5677 break;
5678 #if TARGET_SIZEOF_VOID_P == 4
5679 case OP_LCONV_TO_U:
5680 #endif
5681 case OP_LCONV_TO_I4:
5682 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5683 break;
5684 case OP_ICONV_TO_R4:
5685 case OP_LCONV_TO_R4:
5686 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
5687 if (cfg->r4fp)
5688 values [ins->dreg] = v;
5689 else
5690 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5691 break;
5692 case OP_FCONV_TO_R4:
5693 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
5694 if (cfg->r4fp)
5695 values [ins->dreg] = v;
5696 else
5697 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5698 break;
5699 case OP_RCONV_TO_R8:
5700 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
5701 break;
5702 case OP_RCONV_TO_R4:
5703 values [ins->dreg] = lhs;
5704 break;
5705 case OP_SEXT_I4:
5706 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5707 break;
5708 case OP_ZEXT_I4:
5709 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5710 break;
5711 case OP_TRUNC_I4:
5712 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5713 break;
5714 case OP_LOCALLOC_IMM: {
5715 LLVMValueRef v;
5716
5717 guint32 size = ins->inst_imm;
5718 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5719
5720 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
5721
5722 if (ins->flags & MONO_INST_INIT)
5723 emit_memset (ctx, builder, v, ConstInt32 (size), MONO_ARCH_FRAME_ALIGNMENT);
5724
5725 values [ins->dreg] = v;
5726 break;
5727 }
5728 case OP_LOCALLOC: {
5729 LLVMValueRef v, size;
5730
5731 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
5732
5733 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
5734
5735 if (ins->flags & MONO_INST_INIT)
5736 emit_memset (ctx, builder, v, size, MONO_ARCH_FRAME_ALIGNMENT);
5737 values [ins->dreg] = v;
5738 break;
5739 }
5740
5741 case OP_LOADI1_MEMBASE:
5742 case OP_LOADU1_MEMBASE:
5743 case OP_LOADI2_MEMBASE:
5744 case OP_LOADU2_MEMBASE:
5745 case OP_LOADI4_MEMBASE:
5746 case OP_LOADU4_MEMBASE:
5747 case OP_LOADI8_MEMBASE:
5748 case OP_LOADR4_MEMBASE:
5749 case OP_LOADR8_MEMBASE:
5750 case OP_LOAD_MEMBASE:
5751 case OP_LOADI8_MEM:
5752 case OP_LOADU1_MEM:
5753 case OP_LOADU2_MEM:
5754 case OP_LOADI4_MEM:
5755 case OP_LOADU4_MEM:
5756 case OP_LOAD_MEM: {
5757 int size = 8;
5758 LLVMValueRef base, index, addr;
5759 LLVMTypeRef t;
5760 gboolean sext = FALSE, zext = FALSE;
5761 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5762
5763 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5764
5765 if (sext || zext)
5766 dname = (char*)"";
5767
5768 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5769 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5770 base = addr;
5771 } else {
5772 /* _MEMBASE */
5773 base = lhs;
5774
5775 if (ins->inst_offset == 0) {
5776 addr = base;
5777 } else if (ins->inst_offset % size != 0) {
5778 /* Unaligned load */
5779 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5780 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5781 } else {
5782 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5783 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5784 }
5785 }
5786
5787 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5788
5789 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, base, dname, is_volatile, LLVM_BARRIER_NONE);
5790
5791 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5792 /*
5793 * These will signal LLVM that these loads do not alias any stores, and
5794 * they can't fail, allowing them to be hoisted out of loops.
5795 */
5796 set_invariant_load_flag (values [ins->dreg]);
5797 }
5798
5799 if (sext)
5800 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5801 else if (zext)
5802 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5803 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5804 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5805 break;
5806 }
5807
5808 case OP_STOREI1_MEMBASE_REG:
5809 case OP_STOREI2_MEMBASE_REG:
5810 case OP_STOREI4_MEMBASE_REG:
5811 case OP_STOREI8_MEMBASE_REG:
5812 case OP_STORER4_MEMBASE_REG:
5813 case OP_STORER8_MEMBASE_REG:
5814 case OP_STORE_MEMBASE_REG: {
5815 int size = 8;
5816 LLVMValueRef index, addr, base;
5817 LLVMTypeRef t;
5818 gboolean sext = FALSE, zext = FALSE;
5819 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5820
5821 if (!values [ins->inst_destbasereg]) {
5822 set_failure (ctx, "inst_destbasereg");
5823 break;
5824 }
5825
5826 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5827
5828 base = values [ins->inst_destbasereg];
5829 if (ins->inst_offset % size != 0) {
5830 /* Unaligned store */
5831 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5832 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5833 } else {
5834 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5835 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5836 }
5837 if (is_volatile && LLVMGetInstructionOpcode (base) == LLVMAlloca && !(ins->flags & MONO_INST_VOLATILE))
5838 /* Storing to an alloca cannot fail */
5839 is_volatile = FALSE;
5840 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5841 break;
5842 }
5843
5844 case OP_STOREI1_MEMBASE_IMM:
5845 case OP_STOREI2_MEMBASE_IMM:
5846 case OP_STOREI4_MEMBASE_IMM:
5847 case OP_STOREI8_MEMBASE_IMM:
5848 case OP_STORE_MEMBASE_IMM: {
5849 int size = 8;
5850 LLVMValueRef index, addr, base;
5851 LLVMTypeRef t;
5852 gboolean sext = FALSE, zext = FALSE;
5853 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5854
5855 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5856
5857 base = values [ins->inst_destbasereg];
5858 if (ins->inst_offset % size != 0) {
5859 /* Unaligned store */
5860 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5861 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5862 } else {
5863 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5864 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5865 }
5866 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5867 break;
5868 }
5869
5870 case OP_CHECK_THIS:
5871 emit_load (ctx, bb, &builder, TARGET_SIZEOF_VOID_P, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), lhs, "", TRUE, LLVM_BARRIER_NONE);
5872 break;
5873 case OP_OUTARG_VTRETADDR:
5874 break;
5875 case OP_VOIDCALL:
5876 case OP_CALL:
5877 case OP_LCALL:
5878 case OP_FCALL:
5879 case OP_RCALL:
5880 case OP_VCALL:
5881 case OP_VOIDCALL_MEMBASE:
5882 case OP_CALL_MEMBASE:
5883 case OP_LCALL_MEMBASE:
5884 case OP_FCALL_MEMBASE:
5885 case OP_RCALL_MEMBASE:
5886 case OP_VCALL_MEMBASE:
5887 case OP_VOIDCALL_REG:
5888 case OP_CALL_REG:
5889 case OP_LCALL_REG:
5890 case OP_FCALL_REG:
5891 case OP_RCALL_REG:
5892 case OP_VCALL_REG: {
5893 process_call (ctx, bb, &builder, ins);
5894 break;
5895 }
5896 case OP_AOTCONST: {
5897 guint32 got_offset;
5898 LLVMValueRef indexes [2];
5899 MonoJumpInfo *tmp_ji, *ji;
5900 LLVMValueRef got_entry_addr;
5901 char *name;
5902
5903 /*
5904 * FIXME: Can't allocate from the cfg mempool since that is freed if
5905 * the LLVM compile fails.
5906 */
5907 tmp_ji = g_new0 (MonoJumpInfo, 1);
5908 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
5909 tmp_ji->data.target = ins->inst_p0;
5910
5911 ji = mono_aot_patch_info_dup (tmp_ji);
5912 g_free (tmp_ji);
5913
5914 if (ji->type == MONO_PATCH_INFO_ICALL_ADDR) {
5915 char *symbol = mono_aot_get_direct_call_symbol (MONO_PATCH_INFO_ICALL_ADDR_CALL, ji->data.target);
5916 if (symbol) {
5917 /*
5918 * Avoid emitting a got entry for these since the method is directly called, and it might not be
5919 * resolvable at runtime using dlsym ().
5920 */
5921 g_free (symbol);
5922 values [ins->dreg] = LLVMConstInt (IntPtrType (), 0, FALSE);
5923 break;
5924 }
5925 }
5926
5927 ji->next = cfg->patch_info;
5928 cfg->patch_info = ji;
5929
5930 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
5931 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, NULL);
5932
5933 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
5934 if (!mono_aot_is_shared_got_offset (got_offset)) {
5935 //mono_print_ji (ji);
5936 //printf ("\n");
5937 ctx->cfg->got_access_count ++;
5938 }
5939
5940 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5941 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
5942 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
5943
5944 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
5945 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
5946 g_free (name);
5947 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5948 if (!cfg->llvm_only || mono_aot_is_shared_got_offset (got_offset)) {
5949 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5950 set_invariant_load_flag (values [ins->dreg]);
5951 }
5952
5953 if (ji->type == MONO_PATCH_INFO_LDSTR)
5954 set_nonnull_load_flag (values [ins->dreg]);
5955 break;
5956 }
5957 case OP_MEMMOVE: {
5958 int argn = 0;
5959 LLVMValueRef args [5];
5960 args [argn++] = convert (ctx, values [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0));
5961 args [argn++] = convert (ctx, values [ins->sreg2], LLVMPointerType (LLVMInt8Type (), 0));
5962 args [argn++] = convert (ctx, values [ins->sreg3], LLVMInt64Type ());
5963 #if LLVM_API_VERSION < 900
5964 args [argn++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE); // alignment
5965 #endif
5966 args [argn++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); // is_volatile
5967
5968 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMMOVE), args, argn, "");
5969 break;
5970 }
5971 case OP_NOT_REACHED:
5972 LLVMBuildUnreachable (builder);
5973 has_terminator = TRUE;
5974 g_assert (bb->block_num < cfg->max_block_num);
5975 ctx->unreachable [bb->block_num] = TRUE;
5976 /* Might have instructions after this */
5977 while (ins->next) {
5978 MonoInst *next = ins->next;
5979 /*
5980 * FIXME: If later code uses the regs defined by these instructions,
5981 * compilation will fail.
5982 */
5983 const char *spec = INS_INFO (next->opcode);
5984 if (spec [MONO_INST_DEST] == 'i')
5985 ctx->values [next->dreg] = LLVMConstNull (LLVMInt32Type ());
5986 MONO_DELETE_INS (bb, next);
5987 }
5988 break;
5989 case OP_LDADDR: {
5990 MonoInst *var = ins->inst_i0;
5991
5992 if (var->opcode == OP_VTARG_ADDR) {
5993 /* The variable contains the vtype address */
5994 values [ins->dreg] = values [var->dreg];
5995 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
5996 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
5997 } else {
5998 values [ins->dreg] = addresses [var->dreg];
5999 }
6000 break;
6001 }
6002 case OP_SIN: {
6003 LLVMValueRef args [1];
6004
6005 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6006 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SIN), args, 1, dname);
6007 break;
6008 }
6009 case OP_SINF: {
6010 LLVMValueRef args [1];
6011
6012 args [0] = convert (ctx, lhs, LLVMFloatType ());
6013 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SINF), args, 1, dname);
6014 break;
6015 }
6016 case OP_EXP: {
6017 LLVMValueRef args [1];
6018
6019 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6020 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_EXP), args, 1, dname);
6021 break;
6022 }
6023 case OP_EXPF: {
6024 LLVMValueRef args [1];
6025
6026 args [0] = convert (ctx, lhs, LLVMFloatType ());
6027 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_EXPF), args, 1, dname);
6028 break;
6029 }
6030 case OP_LOG2: {
6031 LLVMValueRef args [1];
6032
6033 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6034 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG2), args, 1, dname);
6035 break;
6036 }
6037 case OP_LOG2F: {
6038 LLVMValueRef args [1];
6039
6040 args [0] = convert (ctx, lhs, LLVMFloatType ());
6041 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG2F), args, 1, dname);
6042 break;
6043 }
6044 case OP_LOG10: {
6045 LLVMValueRef args [1];
6046
6047 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6048 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG10), args, 1, dname);
6049 break;
6050 }
6051 case OP_LOG10F: {
6052 LLVMValueRef args [1];
6053
6054 args [0] = convert (ctx, lhs, LLVMFloatType ());
6055 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG10F), args, 1, dname);
6056 break;
6057 }
6058 case OP_LOG: {
6059 LLVMValueRef args [1];
6060
6061 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6062 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG), args, 1, dname);
6063 break;
6064 }
6065 case OP_TRUNC: {
6066 LLVMValueRef args [1];
6067
6068 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6069 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_TRUNC), args, 1, dname);
6070 break;
6071 }
6072 case OP_TRUNCF: {
6073 LLVMValueRef args [1];
6074
6075 args [0] = convert (ctx, lhs, LLVMFloatType ());
6076 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_TRUNCF), args, 1, dname);
6077 break;
6078 }
6079 case OP_COS: {
6080 LLVMValueRef args [1];
6081
6082 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6083 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COS), args, 1, dname);
6084 break;
6085 }
6086 case OP_COSF: {
6087 LLVMValueRef args [1];
6088
6089 args [0] = convert (ctx, lhs, LLVMFloatType ());
6090 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COSF), args, 1, dname);
6091 break;
6092 }
6093 case OP_SQRT: {
6094 LLVMValueRef args [1];
6095
6096 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6097 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRT), args, 1, dname);
6098 break;
6099 }
6100 case OP_SQRTF: {
6101 LLVMValueRef args [1];
6102
6103 args [0] = convert (ctx, lhs, LLVMFloatType ());
6104 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRTF), args, 1, dname);
6105 break;
6106 }
6107 case OP_FLOOR: {
6108 LLVMValueRef args [1];
6109
6110 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6111 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FLOOR), args, 1, dname);
6112 break;
6113 }
6114 case OP_FLOORF: {
6115 LLVMValueRef args [1];
6116
6117 args [0] = convert (ctx, lhs, LLVMFloatType ());
6118 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FLOORF), args, 1, dname);
6119 break;
6120 }
6121 case OP_CEIL: {
6122 LLVMValueRef args [1];
6123
6124 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6125 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CEIL), args, 1, dname);
6126 break;
6127 }
6128 case OP_CEILF: {
6129 LLVMValueRef args [1];
6130
6131 args [0] = convert (ctx, lhs, LLVMFloatType ());
6132 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CEILF), args, 1, dname);
6133 break;
6134 }
6135 case OP_FMA: {
6136 LLVMValueRef args [3];
6137
6138 args [0] = convert (ctx, values [ins->sreg1], LLVMDoubleType ());
6139 args [1] = convert (ctx, values [ins->sreg2], LLVMDoubleType ());
6140 args [2] = convert (ctx, values [ins->sreg3], LLVMDoubleType ());
6141
6142 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FMA), args, 3, dname);
6143 break;
6144 }
6145 case OP_FMAF: {
6146 LLVMValueRef args [3];
6147
6148 args [0] = convert (ctx, values [ins->sreg1], LLVMFloatType ());
6149 args [1] = convert (ctx, values [ins->sreg2], LLVMFloatType ());
6150 args [2] = convert (ctx, values [ins->sreg3], LLVMFloatType ());
6151
6152 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FMAF), args, 3, dname);
6153 break;
6154 }
6155 case OP_ABS: {
6156 LLVMValueRef args [1];
6157
6158 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6159 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
6160 break;
6161 }
6162 case OP_ABSF: {
6163 LLVMValueRef args [1];
6164
6165 #ifdef TARGET_AMD64
6166 args [0] = convert (ctx, lhs, LLVMFloatType ());
6167 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_ABSF), args, 1, dname);
6168 #else
6169 /* llvm.fabs not supported on all platforms */
6170 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6171 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
6172 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
6173 #endif
6174 break;
6175 }
6176 case OP_RPOW: {
6177 LLVMValueRef args [2];
6178
6179 args [0] = convert (ctx, lhs, LLVMFloatType ());
6180 args [1] = convert (ctx, rhs, LLVMFloatType ());
6181 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_POWF), args, 2, dname);
6182 break;
6183 }
6184 case OP_FPOW: {
6185 LLVMValueRef args [2];
6186
6187 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6188 args [1] = convert (ctx, rhs, LLVMDoubleType ());
6189 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_POW), args, 2, dname);
6190 break;
6191 }
6192 case OP_FCOPYSIGN: {
6193 LLVMValueRef args [2];
6194
6195 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6196 args [1] = convert (ctx, rhs, LLVMDoubleType ());
6197 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COPYSIGN), args, 2, dname);
6198 break;
6199 }
6200 case OP_RCOPYSIGN: {
6201 LLVMValueRef args [2];
6202
6203 args [0] = convert (ctx, lhs, LLVMFloatType ());
6204 args [1] = convert (ctx, rhs, LLVMFloatType ());
6205 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COPYSIGNF), args, 2, dname);
6206 break;
6207 }
6208
6209 case OP_IMIN:
6210 case OP_LMIN:
6211 case OP_IMAX:
6212 case OP_LMAX:
6213 case OP_IMIN_UN:
6214 case OP_LMIN_UN:
6215 case OP_IMAX_UN:
6216 case OP_LMAX_UN:
6217 case OP_FMIN:
6218 case OP_FMAX:
6219 case OP_RMIN:
6220 case OP_RMAX: {
6221 LLVMValueRef v;
6222
6223 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6224 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6225
6226 switch (ins->opcode) {
6227 case OP_IMIN:
6228 case OP_LMIN:
6229 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
6230 break;
6231 case OP_IMAX:
6232 case OP_LMAX:
6233 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
6234 break;
6235 case OP_IMIN_UN:
6236 case OP_LMIN_UN:
6237 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
6238 break;
6239 case OP_IMAX_UN:
6240 case OP_LMAX_UN:
6241 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
6242 break;
6243 case OP_FMAX:
6244 case OP_RMAX:
6245 v = LLVMBuildFCmp (builder, LLVMRealUGE, lhs, rhs, "");
6246 break;
6247 case OP_FMIN:
6248 case OP_RMIN:
6249 v = LLVMBuildFCmp (builder, LLVMRealULE, lhs, rhs, "");
6250 break;
6251 default:
6252 g_assert_not_reached ();
6253 break;
6254 }
6255 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
6256 break;
6257 }
6258
6259 /*
6260 * See the ARM64 comment in mono/utils/atomic.h for an explanation of why this
6261 * hack is necessary (for now).
6262 */
6263 #ifdef TARGET_ARM64
6264 #define ARM64_ATOMIC_FENCE_FIX mono_llvm_build_fence (builder, LLVM_BARRIER_SEQ)
6265 #else
6266 #define ARM64_ATOMIC_FENCE_FIX
6267 #endif
6268
6269 case OP_ATOMIC_EXCHANGE_I4:
6270 case OP_ATOMIC_EXCHANGE_I8: {
6271 LLVMValueRef args [2];
6272 LLVMTypeRef t;
6273
6274 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
6275 t = LLVMInt32Type ();
6276 else
6277 t = LLVMInt64Type ();
6278
6279 g_assert (ins->inst_offset == 0);
6280
6281 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6282 args [1] = convert (ctx, rhs, t);
6283
6284 ARM64_ATOMIC_FENCE_FIX;
6285 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
6286 ARM64_ATOMIC_FENCE_FIX;
6287 break;
6288 }
6289 case OP_ATOMIC_ADD_I4:
6290 case OP_ATOMIC_ADD_I8: {
6291 LLVMValueRef args [2];
6292 LLVMTypeRef t;
6293
6294 if (ins->opcode == OP_ATOMIC_ADD_I4)
6295 t = LLVMInt32Type ();
6296 else
6297 t = LLVMInt64Type ();
6298
6299 g_assert (ins->inst_offset == 0);
6300
6301 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6302 args [1] = convert (ctx, rhs, t);
6303 ARM64_ATOMIC_FENCE_FIX;
6304 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
6305 ARM64_ATOMIC_FENCE_FIX;
6306 break;
6307 }
6308 case OP_ATOMIC_CAS_I4:
6309 case OP_ATOMIC_CAS_I8: {
6310 LLVMValueRef args [3], val;
6311 LLVMTypeRef t;
6312
6313 if (ins->opcode == OP_ATOMIC_CAS_I4)
6314 t = LLVMInt32Type ();
6315 else
6316 t = LLVMInt64Type ();
6317
6318 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6319 /* comparand */
6320 args [1] = convert (ctx, values [ins->sreg3], t);
6321 /* new value */
6322 args [2] = convert (ctx, values [ins->sreg2], t);
6323 ARM64_ATOMIC_FENCE_FIX;
6324 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
6325 ARM64_ATOMIC_FENCE_FIX;
6326 /* cmpxchg returns a pair */
6327 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
6328 break;
6329 }
6330 case OP_MEMORY_BARRIER: {
6331 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
6332 break;
6333 }
6334 case OP_ATOMIC_LOAD_I1:
6335 case OP_ATOMIC_LOAD_I2:
6336 case OP_ATOMIC_LOAD_I4:
6337 case OP_ATOMIC_LOAD_I8:
6338 case OP_ATOMIC_LOAD_U1:
6339 case OP_ATOMIC_LOAD_U2:
6340 case OP_ATOMIC_LOAD_U4:
6341 case OP_ATOMIC_LOAD_U8:
6342 case OP_ATOMIC_LOAD_R4:
6343 case OP_ATOMIC_LOAD_R8: {
6344 int size;
6345 gboolean sext, zext;
6346 LLVMTypeRef t;
6347 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
6348 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6349 LLVMValueRef index, addr;
6350
6351 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6352
6353 if (sext || zext)
6354 dname = (char *)"";
6355
6356 if (ins->inst_offset != 0) {
6357 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6358 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
6359 } else {
6360 addr = lhs;
6361 }
6362
6363 addr = convert (ctx, addr, LLVMPointerType (t, 0));
6364
6365 ARM64_ATOMIC_FENCE_FIX;
6366 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, lhs, dname, is_volatile, barrier);
6367 ARM64_ATOMIC_FENCE_FIX;
6368
6369 if (sext)
6370 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6371 else if (zext)
6372 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6373 break;
6374 }
6375 case OP_ATOMIC_STORE_I1:
6376 case OP_ATOMIC_STORE_I2:
6377 case OP_ATOMIC_STORE_I4:
6378 case OP_ATOMIC_STORE_I8:
6379 case OP_ATOMIC_STORE_U1:
6380 case OP_ATOMIC_STORE_U2:
6381 case OP_ATOMIC_STORE_U4:
6382 case OP_ATOMIC_STORE_U8:
6383 case OP_ATOMIC_STORE_R4:
6384 case OP_ATOMIC_STORE_R8: {
6385 int size;
6386 gboolean sext, zext;
6387 LLVMTypeRef t;
6388 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
6389 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6390 LLVMValueRef index, addr, value, base;
6391
6392 if (!values [ins->inst_destbasereg]) {
6393 set_failure (ctx, "inst_destbasereg");
6394 break;
6395 }
6396
6397 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6398
6399 base = values [ins->inst_destbasereg];
6400 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6401 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
6402 value = convert (ctx, values [ins->sreg1], t);
6403
6404 ARM64_ATOMIC_FENCE_FIX;
6405 emit_store_general (ctx, bb, &builder, size, value, addr, base, is_volatile, barrier);
6406 ARM64_ATOMIC_FENCE_FIX;
6407 break;
6408 }
6409 case OP_RELAXED_NOP: {
6410 #if defined(TARGET_AMD64) || defined(TARGET_X86)
6411 emit_call (ctx, bb, &builder, get_intrins_by_name (ctx, "llvm.x86.sse2.pause"), NULL, 0);
6412 break;
6413 #else
6414 break;
6415 #endif
6416 }
6417 case OP_TLS_GET: {
6418 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
6419 #ifdef TARGET_AMD64
6420 // 257 == FS segment register
6421 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
6422 #else
6423 // 256 == GS segment register
6424 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6425 #endif
6426 // FIXME: XEN
6427 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
6428 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
6429 /* See mono_amd64_emit_tls_get () */
6430 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
6431
6432 // 256 == GS segment register
6433 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6434 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
6435 #else
6436 set_failure (ctx, "opcode tls-get");
6437 break;
6438 #endif
6439
6440 break;
6441 }
6442 case OP_GC_SAFE_POINT: {
6443 LLVMValueRef val, cmp, callee, call;
6444 LLVMBasicBlockRef poll_bb, cont_bb;
6445 LLVMValueRef args [2];
6446 static LLVMTypeRef sig;
6447 const char *icall_name = "mono_threads_state_poll";
6448
6449 /*
6450 * Create the cold wrapper around the icall, along with a managed method for it so
6451 * unwinding works.
6452 */
6453 if (!cfg->compile_aot && !ctx->module->gc_poll_cold_wrapper_compiled) {
6454 ERROR_DECL (error);
6455 /* Compiling a method here is a bit ugly, but it works */
6456 MonoMethod *wrapper = mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL);
6457 ctx->module->gc_poll_cold_wrapper_compiled = mono_jit_compile_method (wrapper, error);
6458 mono_error_assert_ok (error);
6459 }
6460
6461 if (!sig)
6462 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
6463
6464 /*
6465 * if (!*sreg1)
6466 * mono_threads_state_poll ();
6467 */
6468 val = mono_llvm_build_load (builder, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
6469 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
6470 poll_bb = gen_bb (ctx, "POLL_BB");
6471 cont_bb = gen_bb (ctx, "CONT_BB");
6472
6473 args [0] = cmp;
6474 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
6475 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
6476
6477 mono_llvm_build_weighted_branch (builder, cmp, cont_bb, poll_bb, 64, 4);
6478
6479 ctx->builder = builder = create_builder (ctx);
6480 LLVMPositionBuilderAtEnd (builder, poll_bb);
6481
6482 if (ctx->cfg->compile_aot) {
6483 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_threads_state_poll));
6484 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6485 } else {
6486 callee = get_jit_callee (ctx, icall_name, sig, MONO_PATCH_INFO_ABS, ctx->module->gc_poll_cold_wrapper_compiled);
6487 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6488 set_call_cold_cconv (call);
6489 }
6490 LLVMBuildBr (builder, cont_bb);
6491
6492 ctx->builder = builder = create_builder (ctx);
6493 LLVMPositionBuilderAtEnd (builder, cont_bb);
6494 ctx->bblocks [bb->block_num].end_bblock = cont_bb;
6495 break;
6496 }
6497
6498 /*
6499 * Overflow opcodes.
6500 */
6501 case OP_IADD_OVF:
6502 case OP_IADD_OVF_UN:
6503 case OP_ISUB_OVF:
6504 case OP_ISUB_OVF_UN:
6505 case OP_IMUL_OVF:
6506 case OP_IMUL_OVF_UN:
6507 case OP_LADD_OVF:
6508 case OP_LADD_OVF_UN:
6509 case OP_LSUB_OVF:
6510 case OP_LSUB_OVF_UN:
6511 case OP_LMUL_OVF:
6512 case OP_LMUL_OVF_UN:
6513 {
6514 LLVMValueRef args [2], val, ovf, func;
6515
6516 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
6517 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
6518 func = get_intrins_by_name (ctx, ovf_op_to_intrins (ins->opcode));
6519 g_assert (func);
6520 val = LLVMBuildCall (builder, func, args, 2, "");
6521 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
6522 ovf = LLVMBuildExtractValue (builder, val, 1, "");
6523 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
6524 if (!ctx_ok (ctx))
6525 break;
6526 builder = ctx->builder;
6527 break;
6528 }
6529
6530 /*
6531 * Valuetypes.
6532 * We currently model them using arrays. Promotion to local vregs is
6533 * disabled for them in mono_handle_global_vregs () in the LLVM case,
6534 * so we always have an entry in cfg->varinfo for them.
6535 * FIXME: Is this needed ?
6536 */
6537 case OP_VZERO: {
6538 MonoClass *klass = ins->klass;
6539
6540 if (!klass) {
6541 // FIXME:
6542 set_failure (ctx, "!klass");
6543 break;
6544 }
6545
6546 if (!addresses [ins->dreg])
6547 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6548 LLVMValueRef ptr = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6549 emit_memset (ctx, builder, ptr, ConstInt32 (mono_class_value_size (klass, NULL)), 0);
6550 break;
6551 }
6552 case OP_DUMMY_VZERO:
6553 break;
6554
6555 case OP_STOREV_MEMBASE:
6556 case OP_LOADV_MEMBASE:
6557 case OP_VMOVE: {
6558 MonoClass *klass = ins->klass;
6559 LLVMValueRef src = NULL, dst, args [5];
6560 gboolean done = FALSE;
6561
6562 if (!klass) {
6563 // FIXME:
6564 set_failure (ctx, "!klass");
6565 break;
6566 }
6567
6568 if (mini_is_gsharedvt_klass (klass)) {
6569 // FIXME:
6570 set_failure (ctx, "gsharedvt");
6571 break;
6572 }
6573
6574 switch (ins->opcode) {
6575 case OP_STOREV_MEMBASE:
6576 if (cfg->gen_write_barriers && m_class_has_references (klass) && ins->inst_destbasereg != cfg->frame_reg &&
6577 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
6578 /* Decomposed earlier */
6579 g_assert_not_reached ();
6580 break;
6581 }
6582 if (!addresses [ins->sreg1]) {
6583 /* SIMD */
6584 g_assert (values [ins->sreg1]);
6585 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (klass)), 0));
6586 LLVMBuildStore (builder, values [ins->sreg1], dst);
6587 done = TRUE;
6588 } else {
6589 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6590 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6591 }
6592 break;
6593 case OP_LOADV_MEMBASE:
6594 if (!addresses [ins->dreg])
6595 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6596 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6597 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6598 break;
6599 case OP_VMOVE:
6600 if (!addresses [ins->sreg1])
6601 addresses [ins->sreg1] = build_alloca (ctx, m_class_get_byval_arg (klass));
6602 if (!addresses [ins->dreg])
6603 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6604 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6605 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6606 break;
6607 default:
6608 g_assert_not_reached ();
6609 }
6610 if (!ctx_ok (ctx))
6611 break;
6612
6613 if (done)
6614 break;
6615
6616 int aindex = 0;
6617 args [aindex ++] = dst;
6618 args [aindex ++] = src;
6619 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
6620 #if LLVM_API_VERSION < 900
6621 // FIXME: Alignment
6622 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6623 #endif
6624 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6625 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMCPY), args, aindex, "");
6626 break;
6627 }
6628 case OP_LLVM_OUTARG_VT: {
6629 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
6630 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
6631
6632 if (ainfo->storage == LLVMArgGsharedvtVariable) {
6633 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
6634
6635 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
6636 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
6637 } else {
6638 g_assert (addresses [ins->sreg1]);
6639 addresses [ins->dreg] = addresses [ins->sreg1];
6640 }
6641 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
6642 if (!addresses [ins->sreg1]) {
6643 addresses [ins->sreg1] = build_alloca (ctx, t);
6644 g_assert (values [ins->sreg1]);
6645 }
6646 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
6647 addresses [ins->dreg] = addresses [ins->sreg1];
6648 } else {
6649 if (!addresses [ins->sreg1]) {
6650 addresses [ins->sreg1] = build_alloca (ctx, t);
6651 g_assert (values [ins->sreg1]);
6652 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
6653 addresses [ins->dreg] = addresses [ins->sreg1];
6654 } else if (ainfo->storage == LLVMArgVtypeAddr || values [ins->sreg1] == addresses [ins->sreg1]) {
6655 /* LLVMArgVtypeByRef/LLVMArgVtypeAddr, have to make a copy */
6656 addresses [ins->dreg] = build_alloca (ctx, t);
6657 LLVMValueRef v = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
6658 LLVMBuildStore (builder, convert (ctx, v, type_to_llvm_type (ctx, t)), addresses [ins->dreg]);
6659 } else {
6660 addresses [ins->dreg] = addresses [ins->sreg1];
6661 }
6662 }
6663 break;
6664 }
6665 case OP_OBJC_GET_SELECTOR: {
6666 const char *name = (const char*)ins->inst_p0;
6667 LLVMValueRef var;
6668
6669 if (!ctx->module->objc_selector_to_var) {
6670 ctx->module->objc_selector_to_var = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
6671
6672 LLVMValueRef info_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), 8), "@OBJC_IMAGE_INFO");
6673 int32_t objc_imageinfo [] = { 0, 16 };
6674 LLVMSetInitializer (info_var, mono_llvm_create_constant_data_array ((uint8_t *) &objc_imageinfo, 8));
6675 LLVMSetLinkage (info_var, LLVMPrivateLinkage);
6676 LLVMSetExternallyInitialized (info_var, TRUE);
6677 LLVMSetSection (info_var, "__DATA, __objc_imageinfo,regular,no_dead_strip");
6678 LLVMSetAlignment (info_var, sizeof (target_mgreg_t));
6679 mark_as_used (ctx->module, info_var);
6680 }
6681
6682 var = (LLVMValueRef)g_hash_table_lookup (ctx->module->objc_selector_to_var, name);
6683 if (!var) {
6684 LLVMValueRef indexes [16];
6685
6686 LLVMValueRef name_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), strlen (name) + 1), "@OBJC_METH_VAR_NAME_");
6687 LLVMSetInitializer (name_var, mono_llvm_create_constant_data_array ((const uint8_t*)name, strlen (name) + 1));
6688 LLVMSetLinkage (name_var, LLVMPrivateLinkage);
6689 LLVMSetSection (name_var, "__TEXT,__objc_methname,cstring_literals");
6690 mark_as_used (ctx->module, name_var);
6691
6692 LLVMValueRef ref_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (LLVMInt8Type (), 0), "@OBJC_SELECTOR_REFERENCES_");
6693
6694 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6695 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6696 LLVMSetInitializer (ref_var, LLVMConstGEP (name_var, indexes, 2));
6697 LLVMSetLinkage (ref_var, LLVMPrivateLinkage);
6698 LLVMSetExternallyInitialized (ref_var, TRUE);
6699 LLVMSetSection (ref_var, "__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
6700 LLVMSetAlignment (ref_var, sizeof (target_mgreg_t));
6701 mark_as_used (ctx->module, ref_var);
6702
6703 g_hash_table_insert (ctx->module->objc_selector_to_var, g_strdup (name), ref_var);
6704 var = ref_var;
6705 }
6706
6707 values [ins->dreg] = LLVMBuildLoad (builder, var, "");
6708 break;
6709 }
6710
6711 /*
6712 * SIMD
6713 */
6714 #if defined(TARGET_X86) || defined(TARGET_AMD64)
6715 case OP_XZERO: {
6716 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
6717 break;
6718 }
6719 case OP_LOADX_MEMBASE: {
6720 LLVMTypeRef t = type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass));
6721 LLVMValueRef src;
6722
6723 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6724 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
6725 break;
6726 }
6727 case OP_STOREX_MEMBASE: {
6728 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
6729 LLVMValueRef dest;
6730
6731 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6732 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
6733 break;
6734 }
6735 case OP_PADDB:
6736 case OP_PADDW:
6737 case OP_PADDD:
6738 case OP_PADDQ:
6739 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
6740 break;
6741 case OP_ADDPD:
6742 case OP_ADDPS:
6743 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
6744 break;
6745 case OP_PSUBB:
6746 case OP_PSUBW:
6747 case OP_PSUBD:
6748 case OP_PSUBQ:
6749 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
6750 break;
6751 case OP_SUBPD:
6752 case OP_SUBPS:
6753 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
6754 break;
6755 case OP_MULPD:
6756 case OP_MULPS:
6757 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
6758 break;
6759 case OP_DIVPD:
6760 case OP_DIVPS:
6761 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
6762 break;
6763 case OP_PAND:
6764 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
6765 break;
6766 case OP_POR:
6767 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
6768 break;
6769 case OP_PXOR:
6770 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
6771 break;
6772 case OP_PMULW:
6773 case OP_PMULD:
6774 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
6775 break;
6776 case OP_ANDPS:
6777 case OP_ANDNPS:
6778 case OP_ORPS:
6779 case OP_XORPS:
6780 case OP_ANDPD:
6781 case OP_ANDNPD:
6782 case OP_ORPD:
6783 case OP_XORPD: {
6784 LLVMTypeRef t, rt;
6785 LLVMValueRef v = NULL;
6786
6787 switch (ins->opcode) {
6788 case OP_ANDPS:
6789 case OP_ANDNPS:
6790 case OP_ORPS:
6791 case OP_XORPS:
6792 t = LLVMVectorType (LLVMInt32Type (), 4);
6793 rt = LLVMVectorType (LLVMFloatType (), 4);
6794 break;
6795 case OP_ANDPD:
6796 case OP_ANDNPD:
6797 case OP_ORPD:
6798 case OP_XORPD:
6799 t = LLVMVectorType (LLVMInt64Type (), 2);
6800 rt = LLVMVectorType (LLVMDoubleType (), 2);
6801 break;
6802 default:
6803 t = LLVMInt32Type ();
6804 rt = LLVMInt32Type ();
6805 g_assert_not_reached ();
6806 }
6807
6808 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6809 rhs = LLVMBuildBitCast (builder, rhs, t, "");
6810 switch (ins->opcode) {
6811 case OP_ANDPS:
6812 case OP_ANDPD:
6813 v = LLVMBuildAnd (builder, lhs, rhs, "");
6814 break;
6815 case OP_ORPS:
6816 case OP_ORPD:
6817 v = LLVMBuildOr (builder, lhs, rhs, "");
6818 break;
6819 case OP_XORPS:
6820 case OP_XORPD:
6821 v = LLVMBuildXor (builder, lhs, rhs, "");
6822 break;
6823 case OP_ANDNPS:
6824 case OP_ANDNPD:
6825 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
6826 break;
6827 }
6828 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
6829 break;
6830 }
6831 case OP_PMIND_UN:
6832 case OP_PMINW_UN:
6833 case OP_PMINB_UN: {
6834 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntULT, lhs, rhs, "");
6835 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6836 break;
6837 }
6838 case OP_PMAXD_UN:
6839 case OP_PMAXW_UN:
6840 case OP_PMAXB_UN: {
6841 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntUGT, lhs, rhs, "");
6842 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6843 break;
6844 }
6845 case OP_PMINW: {
6846 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntSLT, lhs, rhs, "");
6847 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6848 break;
6849 }
6850 case OP_MINPD:
6851 case OP_MINPS:
6852 case OP_MAXPD:
6853 case OP_MAXPS:
6854 case OP_ADDSUBPD:
6855 case OP_ADDSUBPS:
6856 case OP_HADDPD:
6857 case OP_HADDPS:
6858 case OP_HSUBPD:
6859 case OP_HSUBPS:
6860 case OP_PADDB_SAT:
6861 case OP_PADDW_SAT:
6862 case OP_PSUBB_SAT:
6863 case OP_PSUBW_SAT:
6864 case OP_PADDB_SAT_UN:
6865 case OP_PADDW_SAT_UN:
6866 case OP_PSUBB_SAT_UN:
6867 case OP_PSUBW_SAT_UN:
6868 case OP_PACKW:
6869 case OP_PACKD:
6870 case OP_PACKW_UN:
6871 case OP_PACKD_UN:
6872 case OP_PMULW_HIGH:
6873 case OP_PMULW_HIGH_UN: {
6874 LLVMValueRef args [2];
6875
6876 args [0] = lhs;
6877 args [1] = rhs;
6878
6879 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6880 break;
6881 }
6882 case OP_PAVGB_UN:
6883 case OP_PAVGW_UN: {
6884 LLVMValueRef ones_vec;
6885 LLVMValueRef ones [32];
6886 int vector_size = LLVMGetVectorSize (LLVMTypeOf (lhs));
6887 LLVMTypeRef ext_elem_type = vector_size == 16 ? LLVMInt16Type () : LLVMInt32Type ();
6888
6889 for (int i = 0; i < 32; ++i)
6890 ones [i] = LLVMConstInt (ext_elem_type, 1, FALSE);
6891 ones_vec = LLVMConstVector (ones, vector_size);
6892
6893 LLVMValueRef val;
6894 LLVMTypeRef ext_type = LLVMVectorType (ext_elem_type, vector_size);
6895
6896 /* Have to increase the vector element size to prevent overflows */
6897 /* res = trunc ((zext (lhs) + zext (rhs) + 1) >> 1) */
6898 val = LLVMBuildAdd (builder, LLVMBuildZExt (builder, lhs, ext_type, ""), LLVMBuildZExt (builder, rhs, ext_type, ""), "");
6899 val = LLVMBuildAdd (builder, val, ones_vec, "");
6900 val = LLVMBuildLShr (builder, val, ones_vec, "");
6901 values [ins->dreg] = LLVMBuildTrunc (builder, val, LLVMTypeOf (lhs), "");
6902 break;
6903 }
6904 case OP_PCMPEQB:
6905 case OP_PCMPEQW:
6906 case OP_PCMPEQD:
6907 case OP_PCMPEQQ:
6908 case OP_PCMPGTB: {
6909 LLVMValueRef pcmp;
6910 LLVMTypeRef retType;
6911 LLVMIntPredicate cmpOp;
6912
6913 if (ins->opcode == OP_PCMPGTB)
6914 cmpOp = LLVMIntSGT;
6915 else
6916 cmpOp = LLVMIntEQ;
6917
6918 if (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)) {
6919 pcmp = LLVMBuildICmp (builder, cmpOp, lhs, rhs, "");
6920 retType = LLVMTypeOf (lhs);
6921 } else {
6922 LLVMTypeRef flatType = LLVMVectorType (LLVMInt8Type (), 16);
6923 LLVMValueRef flatRHS = convert (ctx, rhs, flatType);
6924 LLVMValueRef flatLHS = convert (ctx, lhs, flatType);
6925
6926 pcmp = LLVMBuildICmp (builder, cmpOp, flatLHS, flatRHS, "");
6927 retType = flatType;
6928 }
6929
6930 values [ins->dreg] = LLVMBuildSExt (builder, pcmp, retType, "");
6931 break;
6932 }
6933 case OP_EXTRACT_R4:
6934 case OP_EXTRACT_R8:
6935 case OP_EXTRACT_I8:
6936 case OP_EXTRACT_I4:
6937 case OP_EXTRACT_I2:
6938 case OP_EXTRACT_U2:
6939 case OP_EXTRACTX_U2:
6940 case OP_EXTRACT_I1:
6941 case OP_EXTRACT_U1: {
6942 LLVMTypeRef t;
6943 gboolean zext = FALSE;
6944
6945 t = simd_op_to_llvm_type (ins->opcode);
6946
6947 switch (ins->opcode) {
6948 case OP_EXTRACT_R4:
6949 case OP_EXTRACT_R8:
6950 case OP_EXTRACT_I8:
6951 case OP_EXTRACT_I4:
6952 case OP_EXTRACT_I2:
6953 case OP_EXTRACT_I1:
6954 break;
6955 case OP_EXTRACT_U2:
6956 case OP_EXTRACTX_U2:
6957 case OP_EXTRACT_U1:
6958 zext = TRUE;
6959 break;
6960 default:
6961 t = LLVMInt32Type ();
6962 g_assert_not_reached ();
6963 }
6964
6965 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6966 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
6967 if (zext)
6968 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
6969 break;
6970 }
6971
6972 case OP_EXPAND_I1:
6973 case OP_EXPAND_I2:
6974 case OP_EXPAND_I4:
6975 case OP_EXPAND_I8:
6976 case OP_EXPAND_R4:
6977 case OP_EXPAND_R8: {
6978 LLVMTypeRef t;
6979 LLVMValueRef mask [32], v;
6980 int i;
6981
6982 #ifdef ENABLE_NETCORE
6983 t = simd_class_to_llvm_type (ctx, ins->klass);
6984 #else
6985 t = simd_op_to_llvm_type (ins->opcode);
6986 #endif
6987 for (i = 0; i < 32; ++i)
6988 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6989
6990 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
6991
6992 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6993 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
6994 break;
6995 }
6996
6997 case OP_INSERT_I1:
6998 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6999 break;
7000 case OP_INSERT_I2:
7001 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7002 break;
7003 case OP_INSERT_I4:
7004 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7005 break;
7006 case OP_INSERT_I8:
7007 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7008 break;
7009 case OP_INSERT_R4:
7010 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7011 break;
7012 case OP_INSERT_R8:
7013 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7014 break;
7015
7016 case OP_CVTDQ2PD: {
7017 LLVMValueRef indexes [16];
7018
7019 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7020 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7021 LLVMValueRef mask = LLVMConstVector (indexes, 2);
7022 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
7023 values [ins->dreg] = LLVMBuildSIToFP (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
7024 break;
7025 }
7026 case OP_CVTPS2PD: {
7027 LLVMValueRef indexes [16];
7028
7029 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7030 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7031 LLVMValueRef mask = LLVMConstVector (indexes, 2);
7032 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
7033 values [ins->dreg] = LLVMBuildFPExt (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
7034 break;
7035 }
7036 case OP_CVTTPS2DQ:
7037 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMVectorType (LLVMInt32Type (), 4), dname);
7038 break;
7039
7040 case OP_CVTDQ2PS:
7041 case OP_CVTPD2DQ:
7042 case OP_CVTPS2DQ:
7043 case OP_CVTPD2PS:
7044 case OP_CVTTPD2DQ:
7045 case OP_EXTRACT_MASK:
7046 case OP_SQRTPS:
7047 case OP_SQRTPD:
7048 case OP_RSQRTPS:
7049 case OP_RCPPS: {
7050 LLVMValueRef v;
7051
7052 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
7053
7054 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
7055 break;
7056 }
7057 case OP_COMPPS:
7058 case OP_COMPPD: {
7059 LLVMRealPredicate op;
7060
7061 switch (ins->inst_c0) {
7062 case SIMD_COMP_EQ:
7063 op = LLVMRealOEQ;
7064 break;
7065 case SIMD_COMP_LT:
7066 op = LLVMRealOLT;
7067 break;
7068 case SIMD_COMP_LE:
7069 op = LLVMRealOLE;
7070 break;
7071 case SIMD_COMP_UNORD:
7072 op = LLVMRealUNO;
7073 break;
7074 case SIMD_COMP_NEQ:
7075 op = LLVMRealUNE;
7076 break;
7077 case SIMD_COMP_NLT:
7078 op = LLVMRealUGE;
7079 break;
7080 case SIMD_COMP_NLE:
7081 op = LLVMRealUGT;
7082 break;
7083 case SIMD_COMP_ORD:
7084 op = LLVMRealORD;
7085 break;
7086 default:
7087 g_assert_not_reached ();
7088 }
7089
7090 LLVMValueRef cmp = LLVMBuildFCmp (builder, op, lhs, rhs, "");
7091 if (ins->opcode == OP_COMPPD)
7092 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), 2), ""), LLVMTypeOf (lhs), "");
7093 else
7094 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), 4), ""), LLVMTypeOf (lhs), "");
7095 break;
7096 }
7097 case OP_ICONV_TO_X:
7098 /* This is only used for implementing shifts by non-immediate */
7099 values [ins->dreg] = lhs;
7100 break;
7101
7102 case OP_PSHRW:
7103 case OP_PSHRD:
7104 case OP_PSHRQ:
7105 case OP_PSARW:
7106 case OP_PSARD:
7107 case OP_PSHLW:
7108 case OP_PSHLD:
7109 case OP_PSHLQ: {
7110 LLVMValueRef args [3];
7111
7112 args [0] = lhs;
7113 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
7114
7115 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
7116 break;
7117 }
7118
7119 case OP_PSHRW_REG:
7120 case OP_PSHRD_REG:
7121 case OP_PSHRQ_REG:
7122 case OP_PSARW_REG:
7123 case OP_PSARD_REG:
7124 case OP_PSHLW_REG:
7125 case OP_PSHLD_REG:
7126 case OP_PSHLQ_REG: {
7127 LLVMValueRef args [3];
7128
7129 args [0] = lhs;
7130 args [1] = values [ins->sreg2];
7131
7132 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
7133 break;
7134 }
7135
7136 case OP_SHUFPS:
7137 case OP_SHUFPD:
7138 case OP_PSHUFLED:
7139 case OP_PSHUFLEW_LOW:
7140 case OP_PSHUFLEW_HIGH: {
7141 int mask [16];
7142 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
7143 int i, mask_size = 0;
7144 int imask = ins->inst_c0;
7145
7146 /* Convert the x86 shuffle mask to LLVM's */
7147 switch (ins->opcode) {
7148 case OP_SHUFPS:
7149 mask_size = 4;
7150 mask [0] = ((imask >> 0) & 3);
7151 mask [1] = ((imask >> 2) & 3);
7152 mask [2] = ((imask >> 4) & 3) + 4;
7153 mask [3] = ((imask >> 6) & 3) + 4;
7154 v1 = values [ins->sreg1];
7155 v2 = values [ins->sreg2];
7156 break;
7157 case OP_SHUFPD:
7158 mask_size = 2;
7159 mask [0] = ((imask >> 0) & 1);
7160 mask [1] = ((imask >> 1) & 1) + 2;
7161 v1 = values [ins->sreg1];
7162 v2 = values [ins->sreg2];
7163 break;
7164 case OP_PSHUFLEW_LOW:
7165 mask_size = 8;
7166 mask [0] = ((imask >> 0) & 3);
7167 mask [1] = ((imask >> 2) & 3);
7168 mask [2] = ((imask >> 4) & 3);
7169 mask [3] = ((imask >> 6) & 3);
7170 mask [4] = 4 + 0;
7171 mask [5] = 4 + 1;
7172 mask [6] = 4 + 2;
7173 mask [7] = 4 + 3;
7174 v1 = values [ins->sreg1];
7175 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7176 break;
7177 case OP_PSHUFLEW_HIGH:
7178 mask_size = 8;
7179 mask [0] = 0;
7180 mask [1] = 1;
7181 mask [2] = 2;
7182 mask [3] = 3;
7183 mask [4] = 4 + ((imask >> 0) & 3);
7184 mask [5] = 4 + ((imask >> 2) & 3);
7185 mask [6] = 4 + ((imask >> 4) & 3);
7186 mask [7] = 4 + ((imask >> 6) & 3);
7187 v1 = values [ins->sreg1];
7188 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7189 break;
7190 case OP_PSHUFLED:
7191 mask_size = 4;
7192 mask [0] = ((imask >> 0) & 3);
7193 mask [1] = ((imask >> 2) & 3);
7194 mask [2] = ((imask >> 4) & 3);
7195 mask [3] = ((imask >> 6) & 3);
7196 v1 = values [ins->sreg1];
7197 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7198 break;
7199 default:
7200 g_assert_not_reached ();
7201 }
7202 for (i = 0; i < mask_size; ++i)
7203 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
7204
7205 values [ins->dreg] =
7206 LLVMBuildShuffleVector (builder, v1, v2,
7207 LLVMConstVector (mask_values, mask_size), dname);
7208 break;
7209 }
7210
7211 case OP_UNPACK_LOWB:
7212 case OP_UNPACK_LOWW:
7213 case OP_UNPACK_LOWD:
7214 case OP_UNPACK_LOWQ:
7215 case OP_UNPACK_LOWPS:
7216 case OP_UNPACK_LOWPD:
7217 case OP_UNPACK_HIGHB:
7218 case OP_UNPACK_HIGHW:
7219 case OP_UNPACK_HIGHD:
7220 case OP_UNPACK_HIGHQ:
7221 case OP_UNPACK_HIGHPS:
7222 case OP_UNPACK_HIGHPD: {
7223 int mask [16];
7224 LLVMValueRef mask_values [16];
7225 int i, mask_size = 0;
7226 gboolean low = FALSE;
7227
7228 switch (ins->opcode) {
7229 case OP_UNPACK_LOWB:
7230 mask_size = 16;
7231 low = TRUE;
7232 break;
7233 case OP_UNPACK_LOWW:
7234 mask_size = 8;
7235 low = TRUE;
7236 break;
7237 case OP_UNPACK_LOWD:
7238 case OP_UNPACK_LOWPS:
7239 mask_size = 4;
7240 low = TRUE;
7241 break;
7242 case OP_UNPACK_LOWQ:
7243 case OP_UNPACK_LOWPD:
7244 mask_size = 2;
7245 low = TRUE;
7246 break;
7247 case OP_UNPACK_HIGHB:
7248 mask_size = 16;
7249 break;
7250 case OP_UNPACK_HIGHW:
7251 mask_size = 8;
7252 break;
7253 case OP_UNPACK_HIGHD:
7254 case OP_UNPACK_HIGHPS:
7255 mask_size = 4;
7256 break;
7257 case OP_UNPACK_HIGHQ:
7258 case OP_UNPACK_HIGHPD:
7259 mask_size = 2;
7260 break;
7261 default:
7262 g_assert_not_reached ();
7263 }
7264
7265 if (low) {
7266 for (i = 0; i < (mask_size / 2); ++i) {
7267 mask [(i * 2)] = i;
7268 mask [(i * 2) + 1] = mask_size + i;
7269 }
7270 } else {
7271 for (i = 0; i < (mask_size / 2); ++i) {
7272 mask [(i * 2)] = (mask_size / 2) + i;
7273 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
7274 }
7275 }
7276
7277 for (i = 0; i < mask_size; ++i)
7278 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
7279
7280 values [ins->dreg] =
7281 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
7282 LLVMConstVector (mask_values, mask_size), dname);
7283 break;
7284 }
7285
7286 case OP_DUPPD: {
7287 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7288 LLVMValueRef v, val;
7289
7290 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7291 val = LLVMConstNull (t);
7292 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7293 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
7294
7295 values [ins->dreg] = val;
7296 break;
7297 }
7298 case OP_DUPPS_LOW:
7299 case OP_DUPPS_HIGH: {
7300 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7301 LLVMValueRef v1, v2, val;
7302
7303
7304 if (ins->opcode == OP_DUPPS_LOW) {
7305 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7306 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7307 } else {
7308 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7309 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7310 }
7311 val = LLVMConstNull (t);
7312 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7313 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7314 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7315 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7316
7317 values [ins->dreg] = val;
7318 break;
7319 }
7320
7321 case OP_DPPS: {
7322 LLVMValueRef args [3];
7323
7324 args [0] = lhs;
7325 args [1] = rhs;
7326 /* 0xf1 == multiply all 4 elements, add them together, and store the result to the lowest element */
7327 args [2] = LLVMConstInt (LLVMInt8Type (), 0xf1, FALSE);
7328
7329 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 3, dname);
7330 break;
7331 }
7332
7333 case OP_FCONV_TO_R8_X: {
7334 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (type_to_simd_type (MONO_TYPE_R8)), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7335 break;
7336 }
7337
7338 case OP_FCONV_TO_R4_X: {
7339 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (type_to_simd_type (MONO_TYPE_R4)), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7340 break;
7341 }
7342
7343 case OP_SSE41_ROUNDSS: {
7344 LLVMValueRef args [3];
7345
7346 args [0] = lhs;
7347 args [1] = lhs;
7348 args [2] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
7349
7350 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SSE_ROUNDSS), args, 3, dname);
7351 break;
7352 }
7353
7354 case OP_SSE41_ROUNDPD: {
7355 LLVMValueRef args [3];
7356
7357 args [0] = lhs;
7358 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
7359
7360 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SSE_ROUNDPD), args, 2, dname);
7361 break;
7362 }
7363
7364 #ifdef ENABLE_NETCORE
7365 case OP_XCAST: {
7366 LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass);
7367
7368 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, "");
7369 break;
7370 }
7371 case OP_XCOMPARE_FP: {
7372 LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0];
7373 LLVMValueRef cmp = LLVMBuildFCmp (builder, pred, lhs, rhs, "");
7374 int nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
7375 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
7376 if (ins->inst_c1 == MONO_TYPE_R8)
7377 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), nelems), ""), LLVMTypeOf (lhs), "");
7378 else
7379 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), nelems), ""), LLVMTypeOf (lhs), "");
7380 break;
7381 }
7382 case OP_XCOMPARE: {
7383 LLVMIntPredicate pred = cond_to_llvm_cond [ins->inst_c0];
7384 LLVMValueRef cmp = LLVMBuildICmp (builder, pred, lhs, rhs, "");
7385 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
7386 values [ins->dreg] = LLVMBuildSExt (builder, cmp, LLVMTypeOf (lhs), "");
7387 break;
7388 }
7389 case OP_XEQUAL: {
7390 LLVMTypeRef t;
7391 LLVMValueRef cmp, mask [32], shuffle;
7392 int nelems;
7393
7394 LLVMTypeRef srcelemt = LLVMGetElementType (LLVMTypeOf (lhs));
7395
7396 //%c = icmp sgt <16 x i8> %a0, %a1
7397 if (srcelemt == LLVMDoubleType () || srcelemt == LLVMFloatType ())
7398 cmp = LLVMBuildFCmp (builder, LLVMRealOEQ, lhs, rhs, "");
7399 else
7400 cmp = LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, "");
7401 nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
7402
7403 LLVMTypeRef elemt;
7404 if (srcelemt == LLVMDoubleType ())
7405 elemt = LLVMInt64Type ();
7406 else if (srcelemt == LLVMFloatType ())
7407 elemt = LLVMInt32Type ();
7408 else
7409 elemt = srcelemt;
7410
7411 t = LLVMVectorType (elemt, nelems);
7412 cmp = LLVMBuildSExt (builder, cmp, t, "");
7413 // cmp is a <nelems x elemt> vector, each element is either 0xff... or 0
7414 int half = nelems / 2;
7415 while (half >= 1) {
7416 // AND the top and bottom halfes into the bottom half
7417 for (int i = 0; i < half; ++i)
7418 mask [i] = LLVMConstInt (LLVMInt32Type (), half + i, FALSE);
7419 for (int i = half; i < nelems; ++i)
7420 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7421 shuffle = LLVMBuildShuffleVector (builder, cmp, LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
7422 cmp = LLVMBuildAnd (builder, cmp, shuffle, "");
7423 half = half / 2;
7424 }
7425 // Extract [0]
7426 values [ins->dreg] = LLVMBuildExtractElement (builder, cmp, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7427 // Maybe convert to 0/1 ?
7428 break;
7429 }
7430 case OP_XBINOP: {
7431 switch (ins->inst_c0) {
7432 case OP_IADD:
7433 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
7434 break;
7435 case OP_ISUB:
7436 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
7437 break;
7438 case OP_IAND:
7439 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
7440 break;
7441 case OP_IOR:
7442 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
7443 break;
7444 case OP_IXOR:
7445 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
7446 break;
7447 case OP_FADD:
7448 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
7449 break;
7450 case OP_FSUB:
7451 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
7452 break;
7453 case OP_FMUL:
7454 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
7455 break;
7456 case OP_FDIV:
7457 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
7458 break;
7459 case OP_FMAX:
7460 case OP_FMIN: {
7461 LLVMValueRef args [] = { lhs, rhs };
7462
7463 gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4;
7464 if (ins->inst_c0 == OP_FMAX)
7465 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD), args, 2, dname);
7466 else
7467 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD), args, 2, dname);
7468 break;
7469 }
7470 case OP_IMAX: {
7471 gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8;
7472 LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, lhs, rhs, "");
7473 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
7474 break;
7475 }
7476 case OP_IMIN: {
7477 gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8;
7478 LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, lhs, rhs, "");
7479 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
7480 }
7481 break;
7482
7483 default:
7484 g_assert_not_reached ();
7485 }
7486 break;
7487 }
7488 case OP_XEXTRACT_I32:
7489 case OP_XEXTRACT_I64:
7490 case OP_XEXTRACT_R8:
7491 case OP_XEXTRACT_R4: {
7492 LLVMBasicBlockRef bbs [64];
7493 LLVMValueRef switch_ins;
7494 LLVMValueRef phi_values [64];
7495 int nelems = LLVMGetVectorSize (LLVMTypeOf (lhs));
7496 int i;
7497
7498 g_assert (nelems <= 64);
7499 for (i = 0; i < nelems; ++i)
7500 bbs [i] = gen_bb (ctx, "XEXTRACT_CASE_BB");
7501 cbb = gen_bb (ctx, "XEXTRACT_COND_BB");
7502
7503 switch_ins = LLVMBuildSwitch (builder, rhs, bbs [0], 0);
7504 for (i = 0; i < nelems; ++i) {
7505 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
7506 LLVMPositionBuilderAtEnd (builder, bbs [i]);
7507 phi_values [i] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), i, FALSE), "");
7508 LLVMBuildBr (builder, cbb);
7509 }
7510
7511 LLVMPositionBuilderAtEnd (builder, cbb);
7512 values [ins->dreg] = LLVMBuildPhi (builder, LLVMTypeOf (phi_values [0]), "");
7513 LLVMAddIncoming (values [ins->dreg], phi_values, bbs, nelems);
7514
7515 MonoTypeEnum type = (MonoTypeEnum)ins->inst_c0;
7516 switch (type) {
7517 case MONO_TYPE_U1:
7518 case MONO_TYPE_U2:
7519 values [ins->dreg] = LLVMBuildZExt (ctx->builder, values [ins->dreg], LLVMInt32Type (), "");
7520 break;
7521 default:
7522 break;
7523 }
7524 ctx->bblocks [bb->block_num].end_bblock = cbb;
7525 break;
7526 }
7527 case OP_POPCNT32:
7528 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CTPOP_I32), &lhs, 1, "");
7529 break;
7530 case OP_POPCNT64:
7531 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CTPOP_I64), &lhs, 1, "");
7532 break;
7533 case OP_LZCNT32:
7534 case OP_LZCNT64: {
7535 LLVMValueRef args [2];
7536 args [0] = lhs;
7537 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
7538 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_LZCNT32 ? INTRINS_CTLZ_I32 : INTRINS_CTLZ_I64), args, 2, "");
7539 break;
7540 }
7541 case OP_CTTZ32:
7542 case OP_CTTZ64: {
7543 LLVMValueRef args [2];
7544 args [0] = lhs;
7545 args [1] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
7546 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_CTTZ32 ? INTRINS_CTTZ_I32 : INTRINS_CTTZ_I64), args, 2, "");
7547 break;
7548 }
7549 case OP_BEXTR32:
7550 case OP_BEXTR64: {
7551 LLVMValueRef args [2];
7552 args [0] = lhs;
7553 args [1] = convert (ctx, rhs, ins->opcode == OP_BEXTR32 ? LLVMInt32Type () : LLVMInt64Type ()); // cast ushort to u32/u64
7554 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_BEXTR32 ? INTRINS_BEXTR_I32 : INTRINS_BEXTR_I64), args, 2, "");
7555 break;
7556 }
7557 case OP_BZHI32:
7558 case OP_BZHI64: {
7559 LLVMValueRef args [2];
7560 args [0] = lhs;
7561 args [1] = rhs;
7562 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_BZHI32 ? INTRINS_BZHI_I32 : INTRINS_BZHI_I64), args, 2, "");
7563 break;
7564 }
7565 case OP_MULX_H32:
7566 case OP_MULX_H64:
7567 case OP_MULX_HL32:
7568 case OP_MULX_HL64: {
7569 gboolean is_64 = ins->opcode == OP_MULX_H64 || ins->opcode == OP_MULX_HL64;
7570 gboolean only_high = ins->opcode == OP_MULX_H32 || ins->opcode == OP_MULX_H64;
7571 LLVMValueRef lx = LLVMBuildZExt (ctx->builder, lhs, LLVMInt128Type (), "");
7572 LLVMValueRef rx = LLVMBuildZExt (ctx->builder, rhs, LLVMInt128Type (), "");
7573 LLVMValueRef mulx = LLVMBuildMul (ctx->builder, lx, rx, "");
7574 if (!only_high) {
7575 LLVMValueRef lowx = LLVMBuildTrunc (ctx->builder, mulx, is_64 ? LLVMInt64Type () : LLVMInt32Type (), "");
7576 LLVMBuildStore (ctx->builder, lowx, values [ins->sreg3]);
7577 }
7578 LLVMValueRef shift = LLVMConstInt (LLVMInt128Type (), is_64 ? 64 : 32, FALSE);
7579 LLVMValueRef highx = LLVMBuildLShr (ctx->builder, mulx, shift, "");
7580 values [ins->dreg] = LLVMBuildTrunc (ctx->builder, highx, is_64 ? LLVMInt64Type () : LLVMInt32Type (), "");
7581 break;
7582 }
7583 case OP_PEXT32:
7584 case OP_PEXT64: {
7585 LLVMValueRef args [2];
7586 args [0] = lhs;
7587 args [1] = rhs;
7588 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_PEXT32 ? INTRINS_PEXT_I32 : INTRINS_PEXT_I64), args, 2, "");
7589 break;
7590 }
7591 case OP_PDEP32:
7592 case OP_PDEP64: {
7593 LLVMValueRef args [2];
7594 args [0] = lhs;
7595 args [1] = rhs;
7596 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_PDEP32 ? INTRINS_PDEP_I32 : INTRINS_PDEP_I64), args, 2, "");
7597 break;
7598 }
7599 #endif /* ENABLE_NETCORE */
7600 #endif /* SIMD */
7601
7602 case OP_DUMMY_USE:
7603 break;
7604
7605 /*
7606 * EXCEPTION HANDLING
7607 */
7608 case OP_IMPLICIT_EXCEPTION:
7609 /* This marks a place where an implicit exception can happen */
7610 if (bb->region != -1)
7611 set_failure (ctx, "implicit-exception");
7612 break;
7613 case OP_THROW:
7614 case OP_RETHROW: {
7615 gboolean rethrow = (ins->opcode == OP_RETHROW);
7616 if (ctx->llvm_only) {
7617 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
7618 has_terminator = TRUE;
7619 ctx->unreachable [bb->block_num] = TRUE;
7620 } else {
7621 emit_throw (ctx, bb, rethrow, lhs);
7622 builder = ctx->builder;
7623 }
7624 break;
7625 }
7626 case OP_CALL_HANDLER: {
7627 /*
7628 * We don't 'call' handlers, but instead simply branch to them.
7629 * The code generated by ENDFINALLY will branch back to us.
7630 */
7631 LLVMBasicBlockRef noex_bb;
7632 GSList *bb_list;
7633 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
7634
7635 bb_list = info->call_handler_return_bbs;
7636
7637 /*
7638 * Set the indicator variable for the finally clause.
7639 */
7640 lhs = info->finally_ind;
7641 g_assert (lhs);
7642 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
7643
7644 /* Branch to the finally clause */
7645 LLVMBuildBr (builder, info->call_handler_target_bb);
7646
7647 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
7648 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
7649
7650 builder = ctx->builder = create_builder (ctx);
7651 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
7652
7653 bblocks [bb->block_num].end_bblock = noex_bb;
7654 break;
7655 }
7656 case OP_START_HANDLER: {
7657 break;
7658 }
7659 case OP_ENDFINALLY: {
7660 LLVMBasicBlockRef resume_bb;
7661 MonoBasicBlock *handler_bb;
7662 LLVMValueRef val, switch_ins, callee;
7663 GSList *bb_list;
7664 BBInfo *info;
7665 gboolean is_fault = MONO_REGION_FLAGS (bb->region) == MONO_EXCEPTION_CLAUSE_FAULT;
7666
7667 /*
7668 * Fault clauses are like finally clauses, but they are only called if an exception is thrown.
7669 */
7670 if (!is_fault) {
7671 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
7672 g_assert (handler_bb);
7673 info = &bblocks [handler_bb->block_num];
7674 lhs = info->finally_ind;
7675 g_assert (lhs);
7676
7677 bb_list = info->call_handler_return_bbs;
7678
7679 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
7680
7681 /* Load the finally variable */
7682 val = LLVMBuildLoad (builder, lhs, "");
7683
7684 /* Reset the variable */
7685 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
7686
7687 /* Branch to either resume_bb, or to the bblocks in bb_list */
7688 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
7689 /*
7690 * The other targets are added at the end to handle OP_CALL_HANDLER
7691 * opcodes processed later.
7692 */
7693 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
7694
7695 builder = ctx->builder = create_builder (ctx);
7696 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
7697 }
7698
7699 if (ctx->llvm_only) {
7700 emit_resume_eh (ctx, bb);
7701 } else {
7702 LLVMTypeRef icall_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
7703 if (ctx->cfg->compile_aot) {
7704 callee = get_callee (ctx, icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7705 } else {
7706 callee = get_jit_callee (ctx, "llvm_resume_unwind_trampoline", icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7707 }
7708 LLVMBuildCall (builder, callee, NULL, 0, "");
7709 LLVMBuildUnreachable (builder);
7710 }
7711
7712 has_terminator = TRUE;
7713 break;
7714 }
7715 case OP_IL_SEQ_POINT:
7716 break;
7717 default: {
7718 char reason [128];
7719
7720 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
7721 set_failure (ctx, reason);
7722 break;
7723 }
7724 }
7725
7726 if (!ctx_ok (ctx))
7727 break;
7728
7729 /* Convert the value to the type required by phi nodes */
7730 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
7731 if (ctx->is_vphi [ins->dreg])
7732 /* vtypes */
7733 values [ins->dreg] = addresses [ins->dreg];
7734 else
7735 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
7736 }
7737
7738 /* Add stores for volatile variables */
7739 if (!skip_volatile_store && spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
7740 emit_volatile_store (ctx, ins->dreg);
7741 }
7742
7743 if (!ctx_ok (ctx))
7744 return;
7745
7746 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
7747 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
7748 }
7749
7750 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
7751 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
7752 LLVMBuildRetVoid (builder);
7753 }
7754
7755 if (bb == cfg->bb_entry)
7756 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
7757 }
7758
7759 /*
7760 * mono_llvm_check_method_supported:
7761 *
7762 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
7763 * compiling a method twice.
7764 */
7765 void
7766 mono_llvm_check_method_supported (MonoCompile *cfg)
7767 {
7768 int i, j;
7769
7770 #ifdef TARGET_WASM
7771 if (mono_method_signature_internal (cfg->method)->call_convention == MONO_CALL_VARARG) {
7772 cfg->exception_message = g_strdup ("vararg callconv");
7773 cfg->disable_llvm = TRUE;
7774 return;
7775 }
7776 #endif
7777
7778 if (cfg->llvm_only)
7779 return;
7780
7781 if (cfg->method->save_lmf) {
7782 cfg->exception_message = g_strdup ("lmf");
7783 cfg->disable_llvm = TRUE;
7784 }
7785 if (cfg->disable_llvm)
7786 return;
7787
7788 /*
7789 * Nested clauses where one of the clauses is a finally clause is
7790 * not supported, because LLVM can't figure out the control flow,
7791 * probably because we resume exception handling by calling our
7792 * own function instead of using the 'resume' llvm instruction.
7793 */
7794 for (i = 0; i < cfg->header->num_clauses; ++i) {
7795 for (j = 0; j < cfg->header->num_clauses; ++j) {
7796 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
7797 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
7798
7799 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
7800 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7801 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
7802 cfg->exception_message = g_strdup ("nested clauses");
7803 cfg->disable_llvm = TRUE;
7804 break;
7805 }
7806 }
7807 }
7808 if (cfg->disable_llvm)
7809 return;
7810
7811 /* FIXME: */
7812 if (cfg->method->dynamic) {
7813 cfg->exception_message = g_strdup ("dynamic.");
7814 cfg->disable_llvm = TRUE;
7815 }
7816 if (cfg->disable_llvm)
7817 return;
7818 }
7819
7820 static LLVMCallInfo*
7821 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
7822 {
7823 LLVMCallInfo *linfo;
7824 int i;
7825
7826 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
7827 int i, n, pindex;
7828
7829 /*
7830 * Gsharedvt methods have the following calling convention:
7831 * - all arguments are passed by ref, even non generic ones
7832 * - the return value is returned by ref too, using a vret
7833 * argument passed after 'this'.
7834 */
7835 n = sig->param_count + sig->hasthis;
7836 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
7837
7838 pindex = 0;
7839 if (sig->hasthis)
7840 linfo->args [pindex ++].storage = LLVMArgNormal;
7841
7842 if (sig->ret->type != MONO_TYPE_VOID) {
7843 if (mini_is_gsharedvt_variable_type (sig->ret))
7844 linfo->ret.storage = LLVMArgGsharedvtVariable;
7845 else if (mini_type_is_vtype (sig->ret))
7846 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
7847 else
7848 linfo->ret.storage = LLVMArgGsharedvtFixed;
7849 linfo->vret_arg_index = pindex;
7850 } else {
7851 linfo->ret.storage = LLVMArgNone;
7852 }
7853
7854 for (i = 0; i < sig->param_count; ++i) {
7855 if (sig->params [i]->byref)
7856 linfo->args [pindex].storage = LLVMArgNormal;
7857 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
7858 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
7859 else if (mini_type_is_vtype (sig->params [i]))
7860 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
7861 else
7862 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
7863 linfo->args [pindex].type = sig->params [i];
7864 pindex ++;
7865 }
7866 return linfo;
7867 }
7868
7869 linfo = mono_arch_get_llvm_call_info (cfg, sig);
7870 linfo->dummy_arg_pindex = -1;
7871 for (i = 0; i < sig->param_count; ++i)
7872 linfo->args [i + sig->hasthis].type = sig->params [i];
7873
7874 return linfo;
7875 }
7876
7877 static void
7878 emit_method_inner (EmitContext *ctx);
7879
7880 static void
7881 free_ctx (EmitContext *ctx)
7882 {
7883 GSList *l;
7884
7885 g_free (ctx->values);
7886 g_free (ctx->addresses);
7887 g_free (ctx->vreg_types);
7888 g_free (ctx->is_vphi);
7889 g_free (ctx->vreg_cli_types);
7890 g_free (ctx->is_dead);
7891 g_free (ctx->unreachable);
7892 g_ptr_array_free (ctx->phi_values, TRUE);
7893 g_free (ctx->bblocks);
7894 g_hash_table_destroy (ctx->region_to_handler);
7895 g_hash_table_destroy (ctx->clause_to_handler);
7896 g_hash_table_destroy (ctx->jit_callees);
7897
7898 g_ptr_array_free (ctx->callsite_list, TRUE);
7899
7900 g_free (ctx->method_name);
7901 g_ptr_array_free (ctx->bblock_list, TRUE);
7902
7903 for (l = ctx->builders; l; l = l->next) {
7904 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
7905 LLVMDisposeBuilder (builder);
7906 }
7907
7908 g_free (ctx);
7909 }
7910
7911 static gboolean
7912 is_externally_callable (EmitContext *ctx, MonoMethod *method)
7913 {
7914 if (ctx->module->llvm_only && ctx->module->static_link && (method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE || method_is_direct_callable (method)))
7915 return TRUE;
7916 return FALSE;
7917 }
7918
7919 /*
7920 * mono_llvm_emit_method:
7921 *
7922 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
7923 */
7924 void
7925 mono_llvm_emit_method (MonoCompile *cfg)
7926 {
7927 EmitContext *ctx;
7928 char *method_name;
7929 int i;
7930
7931 if (cfg->skip)
7932 return;
7933
7934 /* The code below might acquire the loader lock, so use it for global locking */
7935 mono_loader_lock ();
7936
7937 /* Used to communicate with the callbacks */
7938 mono_native_tls_set_value (current_cfg_tls_id, cfg);
7939
7940 ctx = g_new0 (EmitContext, 1);
7941 ctx->cfg = cfg;
7942 ctx->mempool = cfg->mempool;
7943
7944 /*
7945 * This maps vregs to the LLVM instruction defining them
7946 */
7947 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
7948 /*
7949 * This maps vregs for volatile variables to the LLVM instruction defining their
7950 * address.
7951 */
7952 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
7953 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
7954 ctx->is_vphi = g_new0 (gboolean, cfg->next_vreg);
7955 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
7956 ctx->phi_values = g_ptr_array_sized_new (256);
7957 /*
7958 * This signals whenever the vreg was defined by a phi node with no input vars
7959 * (i.e. all its input bblocks end with NOT_REACHABLE).
7960 */
7961 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
7962 /* Whenever the bblock is unreachable */
7963 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
7964 ctx->bblock_list = g_ptr_array_sized_new (256);
7965
7966 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
7967 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
7968 ctx->callsite_list = g_ptr_array_new ();
7969 ctx->jit_callees = g_hash_table_new (NULL, NULL);
7970 if (cfg->compile_aot) {
7971 ctx->module = &aot_module;
7972
7973 if (is_externally_callable (ctx, cfg->method))
7974 method_name = mono_aot_get_mangled_method_name (cfg->method);
7975 else
7976 method_name = mono_aot_get_method_name (cfg);
7977 cfg->llvm_method_name = g_strdup (method_name);
7978 } else {
7979 init_jit_module (cfg->domain);
7980 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
7981 method_name = mono_method_full_name (cfg->method, TRUE);
7982 }
7983 ctx->method_name = method_name;
7984
7985 if (cfg->compile_aot) {
7986 ctx->lmodule = ctx->module->lmodule;
7987 } else {
7988 ctx->lmodule = LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg->method->name));
7989 /* Reset this as it contains values from lmodule */
7990 memset (ctx->module->intrins_by_id, 0, sizeof (LLVMValueRef) * INTRINS_NUM);
7991 }
7992 ctx->llvm_only = ctx->module->llvm_only;
7993 #ifdef TARGET_WASM
7994 ctx->emit_dummy_arg = TRUE;
7995 #endif
7996
7997 emit_method_inner (ctx);
7998
7999 if (!ctx_ok (ctx)) {
8000 if (ctx->lmethod) {
8001 /* Need to add unused phi nodes as they can be referenced by other values */
8002 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
8003 LLVMBuilderRef builder;
8004
8005 builder = create_builder (ctx);
8006 LLVMPositionBuilderAtEnd (builder, phi_bb);
8007
8008 for (i = 0; i < ctx->phi_values->len; ++i) {
8009 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
8010 if (LLVMGetInstructionParent (v) == NULL)
8011 LLVMInsertIntoBuilder (builder, v);
8012 }
8013
8014 if (ctx->module->llvm_only && ctx->module->static_link) {
8015 // Keep a stub for the function since it might be called directly
8016 int nbbs = LLVMCountBasicBlocks (ctx->lmethod);
8017 LLVMBasicBlockRef *bblocks = g_new0 (LLVMBasicBlockRef, nbbs);
8018 LLVMGetBasicBlocks (ctx->lmethod, bblocks);
8019 for (int i = 0; i < nbbs; ++i)
8020 LLVMDeleteBasicBlock (bblocks [i]);
8021
8022 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (ctx->lmethod, "ENTRY");
8023 builder = create_builder (ctx);
8024 LLVMPositionBuilderAtEnd (builder, entry_bb);
8025 ctx->builder = builder;
8026
8027 LLVMTypeRef sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
8028 LLVMValueRef callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
8029 LLVMBuildCall (builder, callee, NULL, 0, "");
8030 LLVMBuildUnreachable (builder);
8031 } else {
8032 LLVMDeleteFunction (ctx->lmethod);
8033 }
8034 }
8035 }
8036
8037 free_ctx (ctx);
8038
8039 mono_native_tls_set_value (current_cfg_tls_id, NULL);
8040
8041 mono_loader_unlock ();
8042 }
8043
8044 static void
8045 emit_method_inner (EmitContext *ctx)
8046 {
8047 MonoCompile *cfg = ctx->cfg;
8048 MonoMethodSignature *sig;
8049 MonoBasicBlock *bb;
8050 LLVMTypeRef method_type;
8051 LLVMValueRef method = NULL;
8052 LLVMValueRef *values = ctx->values;
8053 int i, max_block_num, bb_index;
8054 gboolean last = FALSE;
8055 LLVMCallInfo *linfo;
8056 LLVMModuleRef lmodule = ctx->lmodule;
8057 BBInfo *bblocks;
8058 GPtrArray *bblock_list = ctx->bblock_list;
8059 MonoMethodHeader *header;
8060 MonoExceptionClause *clause;
8061 char **names;
8062 LLVMBuilderRef entry_builder = NULL;
8063 LLVMBasicBlockRef entry_bb = NULL;
8064
8065 if (cfg->gsharedvt && !cfg->llvm_only) {
8066 set_failure (ctx, "gsharedvt");
8067 return;
8068 }
8069
8070 #if 1
8071 {
8072 static int count = 0;
8073 count ++;
8074
8075 char *llvm_count_str = g_getenv ("LLVM_COUNT");
8076 if (llvm_count_str) {
8077 int lcount = atoi (llvm_count_str);
8078 g_free (llvm_count_str);
8079 if (count == lcount) {
8080 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
8081 fflush (stdout);
8082 last = TRUE;
8083 }
8084 if (count > lcount) {
8085 set_failure (ctx, "count");
8086 return;
8087 }
8088 }
8089 }
8090 #endif
8091
8092 // If we come upon one of the init_method wrappers, we need to find
8093 // the method that we have already emitted and tell LLVM that this
8094 // managed method info for the wrapper is associated with this method
8095 // we constructed ourselves from LLVM IR.
8096 //
8097 // This is necessary to unwind through the init_method, in the case that
8098 // it has to run a static cctor that throws an exception
8099 if (cfg->method->wrapper_type == MONO_WRAPPER_OTHER) {
8100 WrapperInfo *info = mono_marshal_get_wrapper_info (cfg->method);
8101 if (info->subtype == WRAPPER_SUBTYPE_AOT_INIT) {
8102 method = get_init_icall_wrapper (ctx->module, info->d.aot_init.subtype);
8103 ctx->lmethod = method;
8104 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
8105
8106 const char *init_name = mono_marshal_get_aot_init_wrapper_name (info->d.aot_init.subtype);
8107 ctx->method_name = g_strdup_printf ("%s%s", ctx->module->global_prefix, init_name);
8108 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
8109
8110 if (!cfg->llvm_only && ctx->module->external_symbols) {
8111 LLVMSetLinkage (method, LLVMExternalLinkage);
8112 LLVMSetVisibility (method, LLVMHiddenVisibility);
8113 }
8114
8115 goto after_codegen;
8116 } else if (info->subtype == WRAPPER_SUBTYPE_LLVM_FUNC) {
8117 g_assert (info->d.llvm_func.subtype == LLVM_FUNC_WRAPPER_GC_POLL);
8118
8119 if (cfg->compile_aot) {
8120 method = ctx->module->gc_poll_cold_wrapper;
8121 g_assert (method);
8122 } else {
8123 method = emit_icall_cold_wrapper (ctx->module, lmodule, MONO_JIT_ICALL_mono_threads_state_poll, FALSE);
8124 }
8125 ctx->lmethod = method;
8126 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
8127
8128 ctx->method_name = g_strdup (LLVMGetValueName (method)); //g_strdup_printf ("%s_%s", ctx->module->global_prefix, LLVMGetValueName (method));
8129 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
8130
8131 if (!cfg->llvm_only && ctx->module->external_symbols) {
8132 LLVMSetLinkage (method, LLVMExternalLinkage);
8133 LLVMSetVisibility (method, LLVMHiddenVisibility);
8134 }
8135
8136 goto after_codegen;
8137 }
8138 }
8139
8140 sig = mono_method_signature_internal (cfg->method);
8141 ctx->sig = sig;
8142
8143 linfo = get_llvm_call_info (cfg, sig);
8144 ctx->linfo = linfo;
8145 if (!ctx_ok (ctx))
8146 return;
8147
8148 if (cfg->rgctx_var)
8149 linfo->rgctx_arg = TRUE;
8150 else if (needs_extra_arg (ctx, cfg->method))
8151 linfo->dummy_arg = TRUE;
8152 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
8153 if (!ctx_ok (ctx))
8154 return;
8155
8156 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
8157 ctx->lmethod = method;
8158
8159 if (!cfg->llvm_only)
8160 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
8161
8162 /* if the method doesn't contain
8163 * (1) a call (so it's a leaf method)
8164 * (2) and no loops
8165 * we can skip the GC safepoint on method entry. */
8166 gboolean requires_safepoint;
8167 requires_safepoint = cfg->has_calls;
8168 if (!requires_safepoint) {
8169 for (bb = cfg->bb_entry->next_bb; bb; bb = bb->next_bb) {
8170 if (bb->loop_body_start || (bb->flags & BB_EXCEPTION_HANDLER)) {
8171 requires_safepoint = TRUE;
8172 }
8173 }
8174 }
8175 #ifndef MONO_LLVM_LOADED
8176 if (!cfg->llvm_only && mono_threads_are_safepoints_enabled () && requires_safepoint) {
8177 if (!cfg->compile_aot && cfg->method->wrapper_type != MONO_WRAPPER_ALLOC) {
8178 LLVMSetGC (method, "coreclr");
8179 emit_gc_safepoint_poll (ctx->module, ctx->lmodule, cfg);
8180 } else if (cfg->compile_aot) {
8181 LLVMSetGC (method, "coreclr");
8182 }
8183 }
8184 #endif
8185 LLVMSetLinkage (method, LLVMPrivateLinkage);
8186
8187 mono_llvm_add_func_attr (method, LLVM_ATTR_UW_TABLE);
8188
8189 if (cfg->compile_aot) {
8190 if (is_externally_callable (ctx, cfg->method)) {
8191 LLVMSetLinkage (method, LLVMExternalLinkage);
8192 } else {
8193 LLVMSetLinkage (method, LLVMInternalLinkage);
8194 //all methods have internal visibility when doing llvm_only
8195 if (!cfg->llvm_only && ctx->module->external_symbols) {
8196 LLVMSetLinkage (method, LLVMExternalLinkage);
8197 LLVMSetVisibility (method, LLVMHiddenVisibility);
8198 }
8199 }
8200 } else {
8201 LLVMSetLinkage (method, LLVMExternalLinkage);
8202 }
8203
8204 if (cfg->method->save_lmf && !cfg->llvm_only) {
8205 set_failure (ctx, "lmf");
8206 return;
8207 }
8208
8209 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
8210 set_failure (ctx, "pinvoke signature");
8211 return;
8212 }
8213
8214 header = cfg->header;
8215 for (i = 0; i < header->num_clauses; ++i) {
8216 clause = &header->clauses [i];
8217 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_FAULT && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
8218 set_failure (ctx, "non-finally/catch/fault clause.");
8219 return;
8220 }
8221 }
8222 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
8223 /* We can't handle inlined methods with clauses */
8224 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8225
8226 if (linfo->rgctx_arg) {
8227 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
8228 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
8229 /*
8230 * We mark the rgctx parameter with the inreg attribute, which is mapped to
8231 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
8232 * CC_X86_64_Mono in X86CallingConv.td.
8233 */
8234 if (!ctx->llvm_only)
8235 mono_llvm_add_param_attr (ctx->rgctx_arg, LLVM_ATTR_IN_REG);
8236 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
8237 } else {
8238 ctx->rgctx_arg_pindex = -1;
8239 }
8240 if (cfg->vret_addr) {
8241 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
8242 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
8243 if (linfo->ret.storage == LLVMArgVtypeByRef) {
8244 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
8245 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
8246 }
8247 }
8248
8249 if (sig->hasthis) {
8250 ctx->this_arg_pindex = linfo->this_arg_pindex;
8251 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
8252 values [cfg->args [0]->dreg] = ctx->this_arg;
8253 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
8254 }
8255 if (linfo->dummy_arg)
8256 LLVMSetValueName (LLVMGetParam (method, linfo->dummy_arg_pindex), "dummy_arg");
8257
8258 names = g_new (char *, sig->param_count);
8259 mono_method_get_param_names (cfg->method, (const char **) names);
8260
8261 /* Set parameter names/attributes */
8262 for (i = 0; i < sig->param_count; ++i) {
8263 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
8264 char *name;
8265 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
8266 int j;
8267
8268 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
8269 name = g_strdup_printf ("dummy_%d_%d", i, j);
8270 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
8271 g_free (name);
8272 }
8273
8274 if (ainfo->storage == LLVMArgVtypeInReg && ainfo->pair_storage [0] == LLVMArgNone && ainfo->pair_storage [1] == LLVMArgNone)
8275 continue;
8276
8277 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
8278 if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
8279 if (names [i] && names [i][0] != '\0')
8280 name = g_strdup_printf ("p_arg_%s", names [i]);
8281 else
8282 name = g_strdup_printf ("p_arg_%d", i);
8283 } else {
8284 if (names [i] && names [i][0] != '\0')
8285 name = g_strdup_printf ("arg_%s", names [i]);
8286 else
8287 name = g_strdup_printf ("arg_%d", i);
8288 }
8289 LLVMSetValueName (LLVMGetParam (method, pindex), name);
8290 g_free (name);
8291 if (ainfo->storage == LLVMArgVtypeByVal)
8292 mono_llvm_add_param_attr (LLVMGetParam (method, pindex), LLVM_ATTR_BY_VAL);
8293
8294 if (ainfo->storage == LLVMArgVtypeByRef || ainfo->storage == LLVMArgVtypeAddr) {
8295 /* For OP_LDADDR */
8296 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
8297 }
8298 }
8299 g_free (names);
8300
8301 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
8302 ctx->minfo = mono_debug_lookup_method (cfg->method);
8303 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
8304 }
8305
8306 max_block_num = 0;
8307 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
8308 max_block_num = MAX (max_block_num, bb->block_num);
8309 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
8310
8311 /* Add branches between non-consecutive bblocks */
8312 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8313 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
8314 bb->next_bb != bb->last_ins->inst_false_bb) {
8315
8316 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
8317 inst->opcode = OP_BR;
8318 inst->inst_target_bb = bb->last_ins->inst_false_bb;
8319 mono_bblock_add_inst (bb, inst);
8320 }
8321 }
8322
8323 /*
8324 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
8325 */
8326 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8327 MonoInst *ins;
8328 LLVMBuilderRef builder;
8329 char *dname;
8330 char dname_buf[128];
8331
8332 builder = create_builder (ctx);
8333
8334 for (ins = bb->code; ins; ins = ins->next) {
8335 switch (ins->opcode) {
8336 case OP_PHI:
8337 case OP_FPHI:
8338 case OP_VPHI:
8339 case OP_XPHI: {
8340 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
8341
8342 if (!ctx_ok (ctx))
8343 return;
8344
8345 if (ins->opcode == OP_VPHI) {
8346 /* Treat valuetype PHI nodes as operating on the address itself */
8347 g_assert (ins->klass);
8348 phi_type = LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)), 0);
8349 }
8350
8351 /*
8352 * Have to precreate these, as they can be referenced by
8353 * earlier instructions.
8354 */
8355 sprintf (dname_buf, "t%d", ins->dreg);
8356 dname = dname_buf;
8357 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
8358
8359 if (ins->opcode == OP_VPHI)
8360 ctx->addresses [ins->dreg] = values [ins->dreg];
8361
8362 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
8363
8364 /*
8365 * Set the expected type of the incoming arguments since these have
8366 * to have the same type.
8367 */
8368 for (i = 0; i < ins->inst_phi_args [0]; i++) {
8369 int sreg1 = ins->inst_phi_args [i + 1];
8370
8371 if (sreg1 != -1) {
8372 if (ins->opcode == OP_VPHI)
8373 ctx->is_vphi [sreg1] = TRUE;
8374 ctx->vreg_types [sreg1] = phi_type;
8375 }
8376 }
8377 break;
8378 }
8379 case OP_LDADDR:
8380 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
8381 break;
8382 default:
8383 break;
8384 }
8385 }
8386 }
8387
8388 /*
8389 * Create an ordering for bblocks, use the depth first order first, then
8390 * put the exception handling bblocks last.
8391 */
8392 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
8393 bb = cfg->bblocks [bb_index];
8394 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
8395 g_ptr_array_add (bblock_list, bb);
8396 bblocks [bb->block_num].added = TRUE;
8397 }
8398 }
8399
8400 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8401 if (!bblocks [bb->block_num].added)
8402 g_ptr_array_add (bblock_list, bb);
8403 }
8404
8405 /*
8406 * Second pass: generate code.
8407 */
8408 // Emit entry point
8409 entry_builder = create_builder (ctx);
8410 entry_bb = get_bb (ctx, cfg->bb_entry);
8411 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
8412 emit_entry_bb (ctx, entry_builder);
8413
8414 // Make landing pads first
8415 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
8416
8417 if (ctx->llvm_only) {
8418 size_t group_index = 0;
8419 while (group_index < cfg->header->num_clauses) {
8420 int count = 0;
8421 size_t cursor = group_index;
8422 while (cursor < cfg->header->num_clauses &&
8423 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
8424 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
8425 count++;
8426 cursor++;
8427 }
8428
8429 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
8430 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
8431 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
8432
8433 group_index = cursor;
8434 }
8435 }
8436
8437 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
8438 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
8439
8440 // Prune unreachable mono BBs.
8441 if (!(bb == cfg->bb_entry || bb->in_count > 0))
8442 continue;
8443
8444 process_bb (ctx, bb);
8445 if (!ctx_ok (ctx))
8446 return;
8447 }
8448 g_hash_table_destroy (ctx->exc_meta);
8449
8450 mono_memory_barrier ();
8451
8452 /* Add incoming phi values */
8453 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8454 GSList *l, *ins_list;
8455
8456 ins_list = bblocks [bb->block_num].phi_nodes;
8457
8458 for (l = ins_list; l; l = l->next) {
8459 PhiNode *node = (PhiNode*)l->data;
8460 MonoInst *phi = node->phi;
8461 int sreg1 = node->sreg;
8462 LLVMBasicBlockRef in_bb;
8463
8464 if (sreg1 == -1)
8465 continue;
8466
8467 in_bb = get_end_bb (ctx, node->in_bb);
8468
8469 if (ctx->unreachable [node->in_bb->block_num])
8470 continue;
8471
8472 if (phi->opcode == OP_VPHI) {
8473 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
8474 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
8475 } else {
8476 if (!values [sreg1]) {
8477 /* Can happen with values in EH clauses */
8478 set_failure (ctx, "incoming phi sreg1");
8479 return;
8480 }
8481 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
8482 set_failure (ctx, "incoming phi arg type mismatch");
8483 return;
8484 }
8485 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
8486 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
8487 }
8488 }
8489 }
8490
8491 /* Nullify empty phi instructions */
8492 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8493 GSList *l, *ins_list;
8494
8495 ins_list = bblocks [bb->block_num].phi_nodes;
8496
8497 for (l = ins_list; l; l = l->next) {
8498 PhiNode *node = (PhiNode*)l->data;
8499 MonoInst *phi = node->phi;
8500 LLVMValueRef phi_ins = values [phi->dreg];
8501
8502 if (!phi_ins)
8503 /* Already removed */
8504 continue;
8505
8506 if (LLVMCountIncoming (phi_ins) == 0) {
8507 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
8508 LLVMInstructionEraseFromParent (phi_ins);
8509 values [phi->dreg] = NULL;
8510 }
8511 }
8512 }
8513
8514 /* Create the SWITCH statements for ENDFINALLY instructions */
8515 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8516 BBInfo *info = &bblocks [bb->block_num];
8517 GSList *l;
8518 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
8519 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
8520 GSList *bb_list = info->call_handler_return_bbs;
8521
8522 GSList *bb_list_iter;
8523 i = 0;
8524 for (bb_list_iter = bb_list; bb_list_iter; bb_list_iter = g_slist_next (bb_list_iter)) {
8525 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)bb_list_iter->data);
8526 i ++;
8527 }
8528 }
8529 }
8530
8531 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
8532
8533 /* Initialize the method if needed */
8534 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
8535 // FIXME: Add more shared got entries
8536 ctx->builder = create_builder (ctx);
8537 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
8538
8539 // FIXME: beforefieldinit
8540 /*
8541 * NATIVE_TO_MANAGED methods might be called on a thread not attached to the runtime, so they are initialized when loaded
8542 * in load_method ().
8543 */
8544 gboolean needs_init = ctx->cfg->got_access_count > 0;
8545 MonoMethod *cctor = NULL;
8546 if (!needs_init && (cctor = mono_class_get_cctor (cfg->method->klass))) {
8547 /* Needs init to run the cctor */
8548 if (cfg->method->flags & METHOD_ATTRIBUTE_STATIC)
8549 needs_init = TRUE;
8550 if (cctor == cfg->method)
8551 needs_init = FALSE;
8552
8553 // If we are a constructor, we need to init so the static
8554 // constructor gets called.
8555 if (!strcmp (cfg->method->name, ".ctor"))
8556 needs_init = TRUE;
8557 }
8558 if (cfg->method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED)
8559 needs_init = FALSE;
8560 if (needs_init)
8561 emit_init_method (ctx);
8562 else
8563 LLVMBuildBr (ctx->builder, ctx->inited_bb);
8564
8565 // Was observing LLVM moving field accesses into the caller's method
8566 // body before the init call (the inlined one), leading to NULL derefs
8567 // after the init_method returns (GOT is filled out though)
8568 if (needs_init)
8569 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8570 }
8571
8572 if (mini_get_debug_options ()->llvm_disable_inlining)
8573 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8574
8575
8576 after_codegen:
8577 if (cfg->llvm_only) {
8578 g_ptr_array_add (ctx->module->cfgs, cfg);
8579
8580 /*
8581 * Add the contents of ctx->callsite_list to module->callsite_list.
8582 * We can't do this earlier, as it contains llvm instructions which can be
8583 * freed if compilation fails.
8584 * FIXME: Get rid of this when all methods can be llvm compiled.
8585 */
8586 for (int i = 0; i < ctx->callsite_list->len; ++i)
8587 g_ptr_array_add (ctx->module->callsite_list, g_ptr_array_index (ctx->callsite_list, i));
8588 }
8589
8590 if (cfg->verbose_level > 1) {
8591 g_print ("\n*** Unoptimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, TRUE));
8592 if (cfg->compile_aot) {
8593 mono_llvm_dump_value (method);
8594 } else {
8595 mono_llvm_dump_module (ctx->lmodule);
8596 }
8597 g_print ("***\n\n");
8598 }
8599
8600 if (cfg->compile_aot && !cfg->llvm_only)
8601 mark_as_used (ctx->module, method);
8602
8603 if (!cfg->llvm_only) {
8604 LLVMValueRef md_args [16];
8605 LLVMValueRef md_node;
8606 int method_index;
8607
8608 if (cfg->compile_aot)
8609 method_index = mono_aot_get_method_index (cfg->orig_method);
8610 else
8611 method_index = 1;
8612 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
8613 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
8614 md_node = LLVMMDNode (md_args, 2);
8615 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
8616 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
8617 }
8618
8619 if (cfg->compile_aot) {
8620 /* Don't generate native code, keep the LLVM IR */
8621 if (cfg->verbose_level) {
8622 char *name = mono_method_get_full_name (cfg->method);
8623 printf ("%s emitted as %s\n", name, ctx->method_name);
8624 g_free (name);
8625 }
8626
8627 //LLVMDumpValue (ctx->lmethod);
8628 //int err = LLVMVerifyFunction(ctx->lmethod, LLVMPrintMessageAction);
8629 //g_assert (err == 0);
8630 } else {
8631 //LLVMVerifyFunction (method, 0);
8632 llvm_jit_finalize_method (ctx);
8633 }
8634
8635 if (ctx->module->method_to_lmethod)
8636 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
8637
8638 if (ctx->module->idx_to_lmethod)
8639 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
8640
8641 if (ctx->llvm_only && m_class_is_valuetype (cfg->orig_method->klass) && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
8642 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
8643 }
8644
8645 /*
8646 * mono_llvm_create_vars:
8647 *
8648 * Same as mono_arch_create_vars () for LLVM.
8649 */
8650 void
8651 mono_llvm_create_vars (MonoCompile *cfg)
8652 {
8653 MonoMethodSignature *sig;
8654
8655 sig = mono_method_signature_internal (cfg->method);
8656 if (cfg->gsharedvt && cfg->llvm_only) {
8657 gboolean vretaddr = FALSE;
8658
8659 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
8660 vretaddr = TRUE;
8661 } else {
8662 MonoMethodSignature *sig = mono_method_signature_internal (cfg->method);
8663 LLVMCallInfo *linfo;
8664
8665 linfo = get_llvm_call_info (cfg, sig);
8666 vretaddr = (linfo->ret.storage == LLVMArgVtypeRetAddr || linfo->ret.storage == LLVMArgVtypeByRef || linfo->ret.storage == LLVMArgGsharedvtFixed || linfo->ret.storage == LLVMArgGsharedvtVariable || linfo->ret.storage == LLVMArgGsharedvtFixedVtype);
8667 }
8668 if (vretaddr) {
8669 /*
8670 * Creating vret_addr forces CEE_SETRET to store the result into it,
8671 * so we don't have to generate any code in our OP_SETRET case.
8672 */
8673 cfg->vret_addr = mono_compile_create_var (cfg, m_class_get_byval_arg (mono_get_intptr_class ()), OP_ARG);
8674 if (G_UNLIKELY (cfg->verbose_level > 1)) {
8675 printf ("vret_addr = ");
8676 mono_print_ins (cfg->vret_addr);
8677 }
8678 }
8679 } else {
8680 mono_arch_create_vars (cfg);
8681 }
8682 }
8683
8684 /*
8685 * mono_llvm_emit_call:
8686 *
8687 * Same as mono_arch_emit_call () for LLVM.
8688 */
8689 void
8690 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
8691 {
8692 MonoInst *in;
8693 MonoMethodSignature *sig;
8694 int i, n, stack_size;
8695 LLVMArgInfo *ainfo;
8696
8697 stack_size = 0;
8698
8699 sig = call->signature;
8700 n = sig->param_count + sig->hasthis;
8701
8702 call->cinfo = get_llvm_call_info (cfg, sig);
8703
8704 if (cfg->disable_llvm)
8705 return;
8706
8707 if (sig->call_convention == MONO_CALL_VARARG) {
8708 cfg->exception_message = g_strdup ("varargs");
8709 cfg->disable_llvm = TRUE;
8710 }
8711
8712 for (i = 0; i < n; ++i) {
8713 MonoInst *ins;
8714
8715 ainfo = call->cinfo->args + i;
8716
8717 in = call->args [i];
8718
8719 /* Simply remember the arguments */
8720 switch (ainfo->storage) {
8721 case LLVMArgNormal: {
8722 MonoType *t = (sig->hasthis && i == 0) ? m_class_get_byval_arg (mono_get_intptr_class ()) : ainfo->type;
8723 int opcode;
8724
8725 opcode = mono_type_to_regmove (cfg, t);
8726 if (opcode == OP_FMOVE) {
8727 MONO_INST_NEW (cfg, ins, OP_FMOVE);
8728 ins->dreg = mono_alloc_freg (cfg);
8729 } else if (opcode == OP_LMOVE) {
8730 MONO_INST_NEW (cfg, ins, OP_LMOVE);
8731 ins->dreg = mono_alloc_lreg (cfg);
8732 } else if (opcode == OP_RMOVE) {
8733 MONO_INST_NEW (cfg, ins, OP_RMOVE);
8734 ins->dreg = mono_alloc_freg (cfg);
8735 } else {
8736 MONO_INST_NEW (cfg, ins, OP_MOVE);
8737 ins->dreg = mono_alloc_ireg (cfg);
8738 }
8739 ins->sreg1 = in->dreg;
8740 break;
8741 }
8742 case LLVMArgVtypeByVal:
8743 case LLVMArgVtypeByRef:
8744 case LLVMArgVtypeInReg:
8745 case LLVMArgVtypeAddr:
8746 case LLVMArgVtypeAsScalar:
8747 case LLVMArgAsIArgs:
8748 case LLVMArgAsFpArgs:
8749 case LLVMArgGsharedvtVariable:
8750 case LLVMArgGsharedvtFixed:
8751 case LLVMArgGsharedvtFixedVtype:
8752 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
8753 ins->dreg = mono_alloc_ireg (cfg);
8754 ins->sreg1 = in->dreg;
8755 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
8756 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
8757 ins->inst_vtype = ainfo->type;
8758 ins->klass = mono_class_from_mono_type_internal (ainfo->type);
8759 break;
8760 default:
8761 cfg->exception_message = g_strdup ("ainfo->storage");
8762 cfg->disable_llvm = TRUE;
8763 return;
8764 }
8765
8766 if (!cfg->disable_llvm) {
8767 MONO_ADD_INS (cfg->cbb, ins);
8768 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
8769 }
8770 }
8771 }
8772
8773 static inline void
8774 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
8775 {
8776 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
8777 }
8778
8779 static inline void
8780 AddFunc1 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1)
8781 {
8782 LLVMTypeRef param_types [4];
8783
8784 param_types [0] = param_type1;
8785
8786 AddFunc (module, name, ret_type, param_types, 1);
8787 }
8788
8789 static inline void
8790 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
8791 {
8792 LLVMTypeRef param_types [4];
8793
8794 param_types [0] = param_type1;
8795 param_types [1] = param_type2;
8796
8797 AddFunc (module, name, ret_type, param_types, 2);
8798 }
8799
8800 typedef struct {
8801 IntrinsicId id;
8802 const char *name;
8803 } IntrinsicDesc;
8804
8805 static IntrinsicDesc intrinsics[] = {
8806 {INTRINS_MEMSET, "llvm.memset.p0i8.i32"},
8807 {INTRINS_MEMCPY, "llvm.memcpy.p0i8.p0i8.i32"},
8808 {INTRINS_MEMMOVE, "llvm.memmove.p0i8.p0i8.i64"},
8809 {INTRINS_SADD_OVF_I32, "llvm.sadd.with.overflow.i32"},
8810 {INTRINS_UADD_OVF_I32, "llvm.uadd.with.overflow.i32"},
8811 {INTRINS_SSUB_OVF_I32, "llvm.ssub.with.overflow.i32"},
8812 {INTRINS_USUB_OVF_I32, "llvm.usub.with.overflow.i32"},
8813 {INTRINS_SMUL_OVF_I32, "llvm.smul.with.overflow.i32"},
8814 {INTRINS_UMUL_OVF_I32, "llvm.umul.with.overflow.i32"},
8815 {INTRINS_SADD_OVF_I64, "llvm.sadd.with.overflow.i64"},
8816 {INTRINS_UADD_OVF_I64, "llvm.uadd.with.overflow.i64"},
8817 {INTRINS_SSUB_OVF_I64, "llvm.ssub.with.overflow.i64"},
8818 {INTRINS_USUB_OVF_I64, "llvm.usub.with.overflow.i64"},
8819 {INTRINS_SMUL_OVF_I64, "llvm.smul.with.overflow.i64"},
8820 {INTRINS_UMUL_OVF_I64, "llvm.umul.with.overflow.i64"},
8821 {INTRINS_SIN, "llvm.sin.f64"},
8822 {INTRINS_COS, "llvm.cos.f64"},
8823 {INTRINS_SQRT, "llvm.sqrt.f64"},
8824 {INTRINS_FLOOR, "llvm.floor.f64"},
8825 {INTRINS_FLOORF, "llvm.floor.f32"},
8826 {INTRINS_CEIL, "llvm.ceil.f64"},
8827 {INTRINS_CEILF, "llvm.ceil.f32"},
8828 {INTRINS_FMA, "llvm.fma.f64"},
8829 {INTRINS_FMAF, "llvm.fma.f32"},
8830 /* This isn't an intrinsic, instead llvm seems to special case it by name */
8831 {INTRINS_FABS, "fabs"},
8832 {INTRINS_ABSF, "llvm.fabs.f32"},
8833 {INTRINS_SINF, "llvm.sin.f32"},
8834 {INTRINS_COSF, "llvm.cos.f32"},
8835 {INTRINS_SQRTF, "llvm.sqrt.f32"},
8836 {INTRINS_POWF, "llvm.pow.f32"},
8837 {INTRINS_POW, "llvm.pow.f64"},
8838 {INTRINS_EXP, "llvm.exp.f64"},
8839 {INTRINS_EXPF, "llvm.exp.f32"},
8840 {INTRINS_LOG, "llvm.log.f64"},
8841 {INTRINS_LOG2, "llvm.log2.f64"},
8842 {INTRINS_LOG2F, "llvm.log2.f32"},
8843 {INTRINS_LOG10, "llvm.log10.f64"},
8844 {INTRINS_LOG10F, "llvm.log10.f32"},
8845 {INTRINS_TRUNC, "llvm.trunc.f64"},
8846 {INTRINS_TRUNCF, "llvm.trunc.f32"},
8847 {INTRINS_COPYSIGN, "llvm.copysign.f64"},
8848 {INTRINS_COPYSIGNF, "llvm.copysign.f32"},
8849 {INTRINS_EXPECT_I8, "llvm.expect.i8"},
8850 {INTRINS_EXPECT_I1, "llvm.expect.i1"},
8851 {INTRINS_CTPOP_I32, "llvm.ctpop.i32"},
8852 {INTRINS_CTPOP_I64, "llvm.ctpop.i64"},
8853 {INTRINS_CTLZ_I32, "llvm.ctlz.i32"},
8854 {INTRINS_CTLZ_I64, "llvm.ctlz.i64"},
8855 {INTRINS_CTTZ_I32, "llvm.cttz.i32"},
8856 {INTRINS_CTTZ_I64, "llvm.cttz.i64"},
8857 {INTRINS_BZHI_I32, "llvm.x86.bmi.bzhi.32"},
8858 {INTRINS_BZHI_I64, "llvm.x86.bmi.bzhi.64"},
8859 {INTRINS_BEXTR_I32, "llvm.x86.bmi.bextr.32"},
8860 {INTRINS_BEXTR_I64, "llvm.x86.bmi.bextr.64"},
8861 {INTRINS_PEXT_I32, "llvm.x86.bmi.pext.32"},
8862 {INTRINS_PEXT_I64, "llvm.x86.bmi.pext.64"},
8863 {INTRINS_PDEP_I32, "llvm.x86.bmi.pdep.32"},
8864 {INTRINS_PDEP_I64, "llvm.x86.bmi.pdep.64"},
8865 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8866 {INTRINS_SSE_PMOVMSKB, "llvm.x86.sse2.pmovmskb.128"},
8867 {INTRINS_SSE_PSRLI_W, "llvm.x86.sse2.psrli.w"},
8868 {INTRINS_SSE_PSRAI_W, "llvm.x86.sse2.psrai.w"},
8869 {INTRINS_SSE_PSLLI_W, "llvm.x86.sse2.pslli.w"},
8870 {INTRINS_SSE_PSRLI_D, "llvm.x86.sse2.psrli.d"},
8871 {INTRINS_SSE_PSRAI_D, "llvm.x86.sse2.psrai.d"},
8872 {INTRINS_SSE_PSLLI_D, "llvm.x86.sse2.pslli.d"},
8873 {INTRINS_SSE_PSRLI_Q, "llvm.x86.sse2.psrli.q"},
8874 {INTRINS_SSE_PSLLI_Q, "llvm.x86.sse2.pslli.q"},
8875 {INTRINS_SSE_SQRT_PD, "llvm.x86.sse2.sqrt.pd"},
8876 {INTRINS_SSE_SQRT_PS, "llvm.x86.sse.sqrt.ps"},
8877 {INTRINS_SSE_RSQRT_PS, "llvm.x86.sse.rsqrt.ps"},
8878 {INTRINS_SSE_RCP_PS, "llvm.x86.sse.rcp.ps"},
8879 {INTRINS_SSE_CVTTPD2DQ, "llvm.x86.sse2.cvttpd2dq"},
8880 {INTRINS_SSE_CVTTPS2DQ, "llvm.x86.sse2.cvttps2dq"},
8881 {INTRINS_SSE_CVTDQ2PD, "llvm.x86.sse2.cvtdq2pd"},
8882 {INTRINS_SSE_CVTDQ2PS, "llvm.x86.sse2.cvtdq2ps"},
8883 {INTRINS_SSE_CVTPD2DQ, "llvm.x86.sse2.cvtpd2dq"},
8884 {INTRINS_SSE_CVTPS2DQ, "llvm.x86.sse2.cvtps2dq"},
8885 {INTRINS_SSE_CVTPD2PS, "llvm.x86.sse2.cvtpd2ps"},
8886 {INTRINS_SSE_CVTPS2PD, "llvm.x86.sse2.cvtps2pd"},
8887 {INTRINS_SSE_CMPPD, "llvm.x86.sse2.cmp.pd"},
8888 {INTRINS_SSE_CMPPS, "llvm.x86.sse.cmp.ps"},
8889 {INTRINS_SSE_PACKSSWB, "llvm.x86.sse2.packsswb.128"},
8890 {INTRINS_SSE_PACKUSWB, "llvm.x86.sse2.packuswb.128"},
8891 {INTRINS_SSE_PACKSSDW, "llvm.x86.sse2.packssdw.128"},
8892 {INTRINS_SSE_PACKUSDW, "llvm.x86.sse41.packusdw"},
8893 {INTRINS_SSE_MINPS, "llvm.x86.sse.min.ps"},
8894 {INTRINS_SSE_MAXPS, "llvm.x86.sse.max.ps"},
8895 {INTRINS_SSE_HADDPS, "llvm.x86.sse3.hadd.ps"},
8896 {INTRINS_SSE_HSUBPS, "llvm.x86.sse3.hsub.ps"},
8897 {INTRINS_SSE_ADDSUBPS, "llvm.x86.sse3.addsub.ps"},
8898 {INTRINS_SSE_MINPD, "llvm.x86.sse2.min.pd"},
8899 {INTRINS_SSE_MAXPD, "llvm.x86.sse2.max.pd"},
8900 {INTRINS_SSE_HADDPD, "llvm.x86.sse3.hadd.pd"},
8901 {INTRINS_SSE_HSUBPD, "llvm.x86.sse3.hsub.pd"},
8902 {INTRINS_SSE_ADDSUBPD, "llvm.x86.sse3.addsub.pd"},
8903 {INTRINS_SSE_PADDSW, "llvm.x86.sse2.padds.w"},
8904 {INTRINS_SSE_PSUBSW, "llvm.x86.sse2.psubs.w"},
8905 {INTRINS_SSE_PADDUSW, "llvm.x86.sse2.paddus.w"},
8906 {INTRINS_SSE_PSUBUSW, "llvm.x86.sse2.psubus.w"},
8907 {INTRINS_SSE_PAVGW, "llvm.x86.sse2.pavg.w"},
8908 {INTRINS_SSE_PMULHW, "llvm.x86.sse2.pmulh.w"},
8909 {INTRINS_SSE_PMULHU, "llvm.x86.sse2.pmulhu.w"},
8910 {INTRINS_SE_PADDSB, "llvm.x86.sse2.padds.b"},
8911 {INTRINS_SSE_PSUBSB, "llvm.x86.sse2.psubs.b"},
8912 {INTRINS_SSE_PADDUSB, "llvm.x86.sse2.paddus.b"},
8913 {INTRINS_SSE_PSUBUSB, "llvm.x86.sse2.psubus.b"},
8914 {INTRINS_SSE_PAVGB, "llvm.x86.sse2.pavg.b"},
8915 {INTRINS_SSE_PAUSE, "llvm.x86.sse2.pause"},
8916 {INTRINS_SSE_DPPS, "llvm.x86.sse41.dpps"},
8917 {INTRINS_SSE_ROUNDSS, "llvm.x86.sse41.round.ss"},
8918 {INTRINS_SSE_ROUNDPD, "llvm.x86.sse41.round.pd"},
8919 #endif
8920 };
8921
8922 static void
8923 add_sse_binary (LLVMModuleRef module, const char *name, int type)
8924 {
8925 LLVMTypeRef ret_type = type_to_simd_type (type);
8926 AddFunc2 (module, name, ret_type, ret_type, ret_type);
8927 }
8928
8929 static void
8930 add_intrinsic (LLVMModuleRef module, int id)
8931 {
8932 const char *name;
8933 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8934 LLVMTypeRef ret_type, arg_types [16];
8935 #endif
8936
8937 name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
8938 g_assert (name);
8939
8940 switch (id) {
8941 case INTRINS_MEMSET: {
8942 #if LLVM_API_VERSION >= 900
8943 /* No alignment argument */
8944 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt1Type () };
8945
8946 AddFunc (module, name, LLVMVoidType (), params, 4);
8947 #else
8948 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
8949
8950 AddFunc (module, name, LLVMVoidType (), params, 5);
8951 #endif
8952 break;
8953 }
8954 case INTRINS_MEMCPY: {
8955 #if LLVM_API_VERSION >= 900
8956 /* No alignment argument */
8957 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt1Type () };
8958
8959 AddFunc (module, name, LLVMVoidType (), params, 4);
8960 #else
8961 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
8962
8963 AddFunc (module, name, LLVMVoidType (), params, 5);
8964 #endif
8965 break;
8966 }
8967 case INTRINS_MEMMOVE: {
8968 #if LLVM_API_VERSION >= 900
8969 /* No alignment argument */
8970 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type (), LLVMInt1Type () };
8971 AddFunc (module, name, LLVMVoidType (), params, 4);
8972 #else
8973 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type (), LLVMInt32Type (), LLVMInt1Type () };
8974 AddFunc (module, name, LLVMVoidType (), params, 5);
8975 #endif
8976 break;
8977 }
8978 case INTRINS_SADD_OVF_I32:
8979 case INTRINS_UADD_OVF_I32:
8980 case INTRINS_SSUB_OVF_I32:
8981 case INTRINS_USUB_OVF_I32:
8982 case INTRINS_SMUL_OVF_I32:
8983 case INTRINS_UMUL_OVF_I32: {
8984 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
8985 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
8986 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
8987
8988 AddFunc (module, name, ret_type, params, 2);
8989 break;
8990 }
8991 case INTRINS_SADD_OVF_I64:
8992 case INTRINS_UADD_OVF_I64:
8993 case INTRINS_SSUB_OVF_I64:
8994 case INTRINS_USUB_OVF_I64:
8995 case INTRINS_SMUL_OVF_I64:
8996 case INTRINS_UMUL_OVF_I64: {
8997 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
8998 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
8999 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
9000
9001 AddFunc (module, name, ret_type, params, 2);
9002 break;
9003 }
9004 case INTRINS_FMA: {
9005 LLVMTypeRef params [] = { LLVMDoubleType (), LLVMDoubleType (), LLVMDoubleType () };
9006
9007 AddFunc (module, name, LLVMDoubleType (), params, 3);
9008 break;
9009 }
9010 case INTRINS_FMAF: {
9011 LLVMTypeRef params [] = { LLVMFloatType (), LLVMFloatType (), LLVMFloatType () };
9012
9013 AddFunc (module, name, LLVMFloatType (), params, 3);
9014 break;
9015 }
9016 case INTRINS_EXP:
9017 case INTRINS_LOG:
9018 case INTRINS_LOG2:
9019 case INTRINS_LOG10:
9020 case INTRINS_TRUNC:
9021 case INTRINS_SIN:
9022 case INTRINS_COS:
9023 case INTRINS_SQRT:
9024 case INTRINS_FLOOR:
9025 case INTRINS_CEIL:
9026 case INTRINS_FABS: {
9027 AddFunc1 (module, name, LLVMDoubleType (), LLVMDoubleType ());
9028 break;
9029 }
9030 case INTRINS_EXPF:
9031 case INTRINS_LOG2F:
9032 case INTRINS_LOG10F:
9033 case INTRINS_TRUNCF:
9034 case INTRINS_SINF:
9035 case INTRINS_COSF:
9036 case INTRINS_SQRTF:
9037 case INTRINS_FLOORF:
9038 case INTRINS_CEILF:
9039 case INTRINS_ABSF: {
9040 AddFunc1 (module, name, LLVMFloatType (), LLVMFloatType ());
9041 break;
9042 }
9043 case INTRINS_COPYSIGNF:
9044 case INTRINS_POWF:
9045 AddFunc2 (module, name, LLVMFloatType (), LLVMFloatType (), LLVMFloatType ());
9046 break;
9047 case INTRINS_COPYSIGN:
9048 case INTRINS_POW:
9049 AddFunc2 (module, name, LLVMDoubleType (), LLVMDoubleType (), LLVMDoubleType ());
9050 break;
9051 case INTRINS_EXPECT_I8:
9052 AddFunc2 (module, name, LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
9053 break;
9054 case INTRINS_EXPECT_I1:
9055 AddFunc2 (module, name, LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type ());
9056 break;
9057 case INTRINS_CTPOP_I32:
9058 AddFunc1 (module, name, LLVMInt32Type (), LLVMInt32Type ());
9059 break;
9060 case INTRINS_CTPOP_I64:
9061 AddFunc1 (module, name, LLVMInt64Type (), LLVMInt64Type ());
9062 break;
9063 case INTRINS_CTLZ_I32:
9064 case INTRINS_CTTZ_I32:
9065 AddFunc2 (module, name, LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type ());
9066 break;
9067 case INTRINS_CTLZ_I64:
9068 case INTRINS_CTTZ_I64:
9069 AddFunc2 (module, name, LLVMInt64Type (), LLVMInt64Type (), LLVMInt1Type ());
9070 break;
9071 case INTRINS_BEXTR_I32:
9072 case INTRINS_BZHI_I32:
9073 case INTRINS_PEXT_I32:
9074 case INTRINS_PDEP_I32:
9075 AddFunc2 (module, name, LLVMInt32Type (), LLVMInt32Type (), LLVMInt32Type ());
9076 break;
9077 case INTRINS_BEXTR_I64:
9078 case INTRINS_BZHI_I64:
9079 case INTRINS_PEXT_I64:
9080 case INTRINS_PDEP_I64:
9081 AddFunc2 (module, name, LLVMInt64Type (), LLVMInt64Type (), LLVMInt64Type ());
9082 break;
9083 #if defined(TARGET_AMD64) || defined(TARGET_X86)
9084 case INTRINS_SSE_PMOVMSKB:
9085 /* pmovmskb */
9086 ret_type = LLVMInt32Type ();
9087 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
9088 AddFunc (module, name, ret_type, arg_types, 1);
9089 break;
9090 case INTRINS_SSE_PSRLI_W:
9091 case INTRINS_SSE_PSRAI_W:
9092 case INTRINS_SSE_PSLLI_W:
9093 /* shifts */
9094 ret_type = type_to_simd_type (MONO_TYPE_I2);
9095 arg_types [0] = ret_type;
9096 arg_types [1] = LLVMInt32Type ();
9097 AddFunc (module, name, ret_type, arg_types, 2);
9098 break;
9099 case INTRINS_SSE_PSRLI_D:
9100 case INTRINS_SSE_PSRAI_D:
9101 case INTRINS_SSE_PSLLI_D:
9102 ret_type = type_to_simd_type (MONO_TYPE_I4);
9103 arg_types [0] = ret_type;
9104 arg_types [1] = LLVMInt32Type ();
9105 AddFunc (module, name, ret_type, arg_types, 2);
9106 break;
9107 case INTRINS_SSE_PSRLI_Q:
9108 case INTRINS_SSE_PSLLI_Q:
9109 ret_type = type_to_simd_type (MONO_TYPE_I8);
9110 arg_types [0] = ret_type;
9111 arg_types [1] = LLVMInt32Type ();
9112 AddFunc (module, name, ret_type, arg_types, 2);
9113 break;
9114 case INTRINS_SSE_SQRT_PD:
9115 /* Unary ops */
9116 ret_type = type_to_simd_type (MONO_TYPE_R8);
9117 arg_types [0] = ret_type;
9118 AddFunc (module, name, ret_type, arg_types, 1);
9119 break;
9120 case INTRINS_SSE_SQRT_PS:
9121 ret_type = type_to_simd_type (MONO_TYPE_R4);
9122 arg_types [0] = ret_type;
9123 AddFunc (module, name, ret_type, arg_types, 1);
9124 break;
9125 case INTRINS_SSE_RSQRT_PS:
9126 ret_type = type_to_simd_type (MONO_TYPE_R4);
9127 arg_types [0] = ret_type;
9128 AddFunc (module, name, ret_type, arg_types, 1);
9129 break;
9130 case INTRINS_SSE_RCP_PS:
9131 ret_type = type_to_simd_type (MONO_TYPE_R4);
9132 arg_types [0] = ret_type;
9133 AddFunc (module, name, ret_type, arg_types, 1);
9134 break;
9135 case INTRINS_SSE_CVTTPD2DQ:
9136 ret_type = type_to_simd_type (MONO_TYPE_I4);
9137 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9138 AddFunc (module, name, ret_type, arg_types, 1);
9139 break;
9140 case INTRINS_SSE_CVTTPS2DQ:
9141 ret_type = type_to_simd_type (MONO_TYPE_I4);
9142 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9143 AddFunc (module, name, ret_type, arg_types, 1);
9144 break;
9145 case INTRINS_SSE_CVTDQ2PD:
9146 /* Conversion ops */
9147 ret_type = type_to_simd_type (MONO_TYPE_R8);
9148 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
9149 AddFunc (module, name, ret_type, arg_types, 1);
9150 break;
9151 case INTRINS_SSE_CVTDQ2PS:
9152 ret_type = type_to_simd_type (MONO_TYPE_R4);
9153 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
9154 AddFunc (module, name, ret_type, arg_types, 1);
9155 break;
9156 case INTRINS_SSE_CVTPD2DQ:
9157 ret_type = type_to_simd_type (MONO_TYPE_I4);
9158 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9159 AddFunc (module, name, ret_type, arg_types, 1);
9160 break;
9161 case INTRINS_SSE_CVTPS2DQ:
9162 ret_type = type_to_simd_type (MONO_TYPE_I4);
9163 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9164 AddFunc (module, name, ret_type, arg_types, 1);
9165 break;
9166 case INTRINS_SSE_CVTPD2PS:
9167 ret_type = type_to_simd_type (MONO_TYPE_R4);
9168 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9169 AddFunc (module, name, ret_type, arg_types, 1);
9170 break;
9171 case INTRINS_SSE_CVTPS2PD:
9172 ret_type = type_to_simd_type (MONO_TYPE_R8);
9173 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9174 AddFunc (module, name, ret_type, arg_types, 1);
9175 break;
9176 case INTRINS_SSE_CMPPD:
9177 /* cmp pd/ps */
9178 ret_type = type_to_simd_type (MONO_TYPE_R8);
9179 arg_types [0] = ret_type;
9180 arg_types [1] = ret_type;
9181 arg_types [2] = LLVMInt8Type ();
9182 AddFunc (module, name, ret_type, arg_types, 3);
9183 break;
9184 case INTRINS_SSE_CMPPS:
9185 ret_type = type_to_simd_type (MONO_TYPE_R4);
9186 arg_types [0] = ret_type;
9187 arg_types [1] = ret_type;
9188 arg_types [2] = LLVMInt8Type ();
9189 AddFunc (module, name, ret_type, arg_types, 3);
9190 break;
9191 case INTRINS_SSE_PACKSSWB:
9192 case INTRINS_SSE_PACKUSWB:
9193 case INTRINS_SSE_PACKSSDW:
9194 /* pack */
9195 ret_type = type_to_simd_type (MONO_TYPE_I1);
9196 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
9197 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
9198 AddFunc (module, name, ret_type, arg_types, 2);
9199 break;
9200 case INTRINS_SSE_PACKUSDW:
9201 ret_type = type_to_simd_type (MONO_TYPE_I2);
9202 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
9203 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
9204 AddFunc (module, name, ret_type, arg_types, 2);
9205 break;
9206 /* SSE Binary ops */
9207 case INTRINS_SSE_PADDSW:
9208 case INTRINS_SSE_PSUBSW:
9209 case INTRINS_SSE_PADDUSW:
9210 case INTRINS_SSE_PSUBUSW:
9211 case INTRINS_SSE_PAVGW:
9212 case INTRINS_SSE_PMULHW:
9213 case INTRINS_SSE_PMULHU:
9214 add_sse_binary (module, name, MONO_TYPE_I2);
9215 break;
9216 case INTRINS_SSE_MINPS:
9217 case INTRINS_SSE_MAXPS:
9218 case INTRINS_SSE_HADDPS:
9219 case INTRINS_SSE_HSUBPS:
9220 case INTRINS_SSE_ADDSUBPS:
9221 add_sse_binary (module, name, MONO_TYPE_R4);
9222 break;
9223 case INTRINS_SSE_MINPD:
9224 case INTRINS_SSE_MAXPD:
9225 case INTRINS_SSE_HADDPD:
9226 case INTRINS_SSE_HSUBPD:
9227 case INTRINS_SSE_ADDSUBPD:
9228 add_sse_binary (module, name, MONO_TYPE_R8);
9229 break;
9230 case INTRINS_SE_PADDSB:
9231 case INTRINS_SSE_PSUBSB:
9232 case INTRINS_SSE_PADDUSB:
9233 case INTRINS_SSE_PSUBUSB:
9234 case INTRINS_SSE_PAVGB:
9235 add_sse_binary (module, name, MONO_TYPE_I1);
9236 break;
9237 case INTRINS_SSE_PAUSE:
9238 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
9239 break;
9240 case INTRINS_SSE_DPPS:
9241 ret_type = type_to_simd_type (MONO_TYPE_R4);
9242 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9243 arg_types [1] = type_to_simd_type (MONO_TYPE_R4);
9244 arg_types [2] = LLVMInt8Type ();
9245 AddFunc (module, name, ret_type, arg_types, 3);
9246 break;
9247 case INTRINS_SSE_ROUNDSS:
9248 ret_type = type_to_simd_type (MONO_TYPE_R4);
9249 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9250 arg_types [1] = type_to_simd_type (MONO_TYPE_R4);
9251 arg_types [2] = LLVMInt32Type ();
9252 AddFunc (module, name, ret_type, arg_types, 3);
9253 break;
9254 case INTRINS_SSE_ROUNDPD:
9255 ret_type = type_to_simd_type (MONO_TYPE_R8);
9256 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9257 arg_types [1] = LLVMInt32Type ();
9258 AddFunc (module, name, ret_type, arg_types, 2);
9259 break;
9260 #endif /* AMD64 || X86 */
9261 default:
9262 g_assert_not_reached ();
9263 break;
9264 }
9265 }
9266
9267 static LLVMValueRef
9268 get_intrins_from_module (LLVMModuleRef lmodule, int id)
9269 {
9270 LLVMValueRef res;
9271
9272 const char *name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
9273 g_assert (name);
9274
9275 res = LLVMGetNamedFunction (lmodule, name);
9276 if (!res) {
9277 add_intrinsic (lmodule, id);
9278 res = LLVMGetNamedFunction (lmodule, name);
9279 g_assert (res);
9280 }
9281 return res;
9282 }
9283
9284 static LLVMValueRef
9285 get_intrins (EmitContext *ctx, int id)
9286 {
9287 MonoLLVMModule *module = ctx->module;
9288 LLVMValueRef res;
9289
9290 /*
9291 * Every method is emitted into its own module so
9292 * we can add intrinsics on demand.
9293 */
9294 res = module->intrins_by_id [id];
9295 if (!res) {
9296 res = get_intrins_from_module (ctx->lmodule, id);
9297 module->intrins_by_id [id] = res;
9298 }
9299 return res;
9300 }
9301
9302 static LLVMValueRef
9303 get_intrins_by_name (EmitContext *ctx, const char *name)
9304 {
9305 LLVMValueRef res;
9306
9307 /*
9308 * Every method is emitted into its own module so
9309 * we can add intrinsics on demand.
9310 */
9311 res = LLVMGetNamedFunction (ctx->lmodule, name);
9312 if (!res) {
9313 int id = -1;
9314
9315 /* No locking needed */
9316 id = GPOINTER_TO_INT (g_hash_table_lookup (intrins_name_to_id, name));
9317 id --;
9318 if (id == -1)
9319 printf ("%s\n", name);
9320 g_assert (id != -1);
9321 add_intrinsic (ctx->lmodule, id);
9322 res = LLVMGetNamedFunction (ctx->lmodule, name);
9323 g_assert (res);
9324 }
9325
9326 return res;
9327 }
9328
9329 static void
9330 add_intrinsics (LLVMModuleRef module)
9331 {
9332 int i;
9333
9334 /* Emit declarations of instrinsics */
9335 /*
9336 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
9337 * type doesn't seem to do any locking.
9338 */
9339 for (i = 0; i < INTRINS_NUM; ++i)
9340 add_intrinsic (module, i);
9341
9342 /* EH intrinsics */
9343 {
9344 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
9345
9346 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
9347 }
9348
9349 /* Load/Store intrinsics */
9350 {
9351 LLVMTypeRef arg_types [5];
9352 int i;
9353 char name [128];
9354
9355 for (i = 1; i <= 8; i *= 2) {
9356 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
9357 arg_types [1] = LLVMInt32Type ();
9358 arg_types [2] = LLVMInt1Type ();
9359 arg_types [3] = LLVMInt32Type ();
9360 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
9361 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
9362
9363 arg_types [0] = LLVMIntType (i * 8);
9364 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
9365 arg_types [2] = LLVMInt32Type ();
9366 arg_types [3] = LLVMInt1Type ();
9367 arg_types [4] = LLVMInt32Type ();
9368 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
9369 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
9370 }
9371 }
9372 }
9373
9374 static void
9375 add_types (MonoLLVMModule *module)
9376 {
9377 module->ptr_type = LLVMPointerType (TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
9378 }
9379
9380 void
9381 mono_llvm_init (void)
9382 {
9383 GHashTable *h;
9384 int i;
9385
9386 mono_native_tls_alloc (&current_cfg_tls_id, NULL);
9387
9388 h = g_hash_table_new (NULL, NULL);
9389 for (i = 0; i < INTRINS_NUM; ++i)
9390 g_hash_table_insert (h, GINT_TO_POINTER (intrinsics [i].id), (gpointer)intrinsics [i].name);
9391 intrins_id_to_name = h;
9392
9393 h = g_hash_table_new (g_str_hash, g_str_equal);
9394 for (i = 0; i < INTRINS_NUM; ++i)
9395 g_hash_table_insert (h, (gpointer)intrinsics [i].name, GINT_TO_POINTER (intrinsics [i].id + 1));
9396 intrins_name_to_id = h;
9397 }
9398
9399 void
9400 mono_llvm_cleanup (void)
9401 {
9402 MonoLLVMModule *module = &aot_module;
9403
9404 if (module->lmodule)
9405 LLVMDisposeModule (module->lmodule);
9406
9407 if (module->context)
9408 LLVMContextDispose (module->context);
9409 }
9410
9411 void
9412 mono_llvm_free_domain_info (MonoDomain *domain)
9413 {
9414 MonoJitDomainInfo *info = domain_jit_info (domain);
9415 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
9416 int i;
9417
9418 if (!module)
9419 return;
9420
9421 g_hash_table_destroy (module->llvm_types);
9422
9423 mono_llvm_dispose_ee (module->mono_ee);
9424
9425 if (module->bb_names) {
9426 for (i = 0; i < module->bb_names_len; ++i)
9427 g_free (module->bb_names [i]);
9428 g_free (module->bb_names);
9429 }
9430 //LLVMDisposeModule (module->module);
9431
9432 g_free (module);
9433
9434 info->llvm_module = NULL;
9435 }
9436
9437 void
9438 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, int initial_got_size, LLVMModuleFlags flags)
9439 {
9440 MonoLLVMModule *module = &aot_module;
9441 gboolean emit_dwarf = (flags & LLVM_MODULE_FLAG_DWARF) ? 1 : 0;
9442 #ifdef TARGET_WIN32_MSVC
9443 gboolean emit_codeview = (flags & LLVM_MODULE_FLAG_CODEVIEW) ? 1 : 0;
9444 #endif
9445 gboolean static_link = (flags & LLVM_MODULE_FLAG_STATIC) ? 1 : 0;
9446 gboolean llvm_only = (flags & LLVM_MODULE_FLAG_LLVM_ONLY) ? 1 : 0;
9447 gboolean interp = (flags & LLVM_MODULE_FLAG_INTERP) ? 1 : 0;
9448 gboolean llvm_disable_self_init = mini_get_debug_options ()->llvm_disable_self_init;
9449
9450 /* Delete previous module */
9451 g_hash_table_destroy (module->plt_entries);
9452 if (module->lmodule)
9453 LLVMDisposeModule (module->lmodule);
9454
9455 memset (module, 0, sizeof (aot_module));
9456
9457 module->lmodule = LLVMModuleCreateWithName ("aot");
9458 module->assembly = assembly;
9459 module->global_prefix = g_strdup (global_prefix);
9460 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
9461 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
9462 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
9463 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
9464 module->external_symbols = TRUE;
9465 module->emit_dwarf = emit_dwarf;
9466 module->static_link = static_link;
9467 module->llvm_only = llvm_only;
9468 module->llvm_disable_self_init = llvm_disable_self_init && !llvm_only; // llvm_only implies !llvm_disable_self_init
9469 module->interp = interp;
9470 /* The first few entries are reserved */
9471 module->max_got_offset = initial_got_size;
9472 module->context = LLVMGetGlobalContext ();
9473 module->cfgs = g_ptr_array_new ();
9474 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
9475
9476 if (llvm_only)
9477 /* clang ignores our debug info because it has an invalid version */
9478 module->emit_dwarf = FALSE;
9479
9480 add_intrinsics (module->lmodule);
9481 add_types (module);
9482
9483 #ifdef MONO_ARCH_LLVM_TARGET_LAYOUT
9484 LLVMSetDataLayout (module->lmodule, MONO_ARCH_LLVM_TARGET_LAYOUT);
9485 #else
9486 g_assert_not_reached ();
9487 #endif
9488
9489 #ifdef MONO_ARCH_LLVM_TARGET_TRIPLE
9490 LLVMSetTarget (module->lmodule, MONO_ARCH_LLVM_TARGET_TRIPLE);
9491 #endif
9492
9493 if (module->emit_dwarf) {
9494 char *dir, *build_info, *s, *cu_name;
9495
9496 module->di_builder = mono_llvm_create_di_builder (module->lmodule);
9497
9498 // FIXME:
9499 dir = g_strdup (".");
9500 build_info = mono_get_runtime_build_info ();
9501 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
9502 cu_name = g_path_get_basename (assembly->image->name);
9503 module->cu = mono_llvm_di_create_compile_unit (module->di_builder, cu_name, dir, s);
9504 g_free (dir);
9505 g_free (build_info);
9506 g_free (s);
9507 }
9508
9509 #ifdef TARGET_WIN32_MSVC
9510 if (emit_codeview) {
9511 LLVMValueRef codeview_option_args[3];
9512
9513 codeview_option_args[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9514 codeview_option_args[1] = LLVMMDString ("CodeView", 8);
9515 codeview_option_args[2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
9516
9517 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.module.flags", LLVMMDNode (codeview_option_args, G_N_ELEMENTS (codeview_option_args)));
9518 }
9519
9520 if (!static_link) {
9521 const char linker_options[] = "Linker Options";
9522 const char *default_dynamic_lib_names[] = { "/DEFAULTLIB:msvcrt",
9523 "/DEFAULTLIB:ucrt.lib",
9524 "/DEFAULTLIB:vcruntime.lib" };
9525
9526 LLVMValueRef default_lib_args[G_N_ELEMENTS (default_dynamic_lib_names)];
9527 LLVMValueRef default_lib_nodes[G_N_ELEMENTS(default_dynamic_lib_names)];
9528
9529 const char *default_lib_name = NULL;
9530 for (int i = 0; i < G_N_ELEMENTS (default_dynamic_lib_names); ++i) {
9531 const char *default_lib_name = default_dynamic_lib_names[i];
9532 default_lib_args[i] = LLVMMDString (default_lib_name, strlen (default_lib_name));
9533 default_lib_nodes[i] = LLVMMDNode (default_lib_args + i, 1);
9534 }
9535
9536 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.linker.options", LLVMMDNode (default_lib_args, G_N_ELEMENTS (default_lib_args)));
9537 }
9538 #endif
9539
9540 /* Add GOT */
9541 /*
9542 * We couldn't compute the type of the LLVM global representing the got because
9543 * its size is only known after all the methods have been emitted. So create
9544 * a dummy variable, and replace all uses it with the real got variable when
9545 * its size is known in mono_llvm_emit_aot_module ().
9546 */
9547 {
9548 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
9549
9550 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
9551 module->got_idx_to_type = g_hash_table_new (NULL, NULL);
9552 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
9553 }
9554
9555 /* Add initialization array */
9556 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
9557
9558 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
9559 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
9560
9561
9562 emit_gc_safepoint_poll (module, module->lmodule, NULL);
9563
9564 emit_llvm_code_start (module);
9565
9566 // Needs idx_to_lmethod
9567 emit_init_icall_wrappers (module);
9568
9569 /* Add a dummy personality function */
9570 if (!use_mono_personality_debug) {
9571 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
9572 LLVMSetLinkage (personality, LLVMExternalLinkage);
9573
9574 //EMCC chockes if the personality function is referenced in the 'used' array
9575 #ifndef TARGET_WASM
9576 mark_as_used (module, personality);
9577 #endif
9578 }
9579
9580 /* Add a reference to the c++ exception we throw/catch */
9581 {
9582 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
9583 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
9584 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
9585 mono_llvm_set_is_constant (module->sentinel_exception);
9586 }
9587
9588 module->llvm_types = g_hash_table_new (NULL, NULL);
9589 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
9590 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
9591 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
9592 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
9593 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
9594 module->method_to_call_info = g_hash_table_new (NULL, NULL);
9595 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
9596 module->callsite_list = g_ptr_array_new ();
9597 }
9598
9599 void
9600 mono_llvm_fixup_aot_module (void)
9601 {
9602 MonoLLVMModule *module = &aot_module;
9603 MonoMethod *method;
9604
9605 if (module->llvm_disable_self_init)
9606 return;
9607
9608 /*
9609 * Replace GOT entries for directly callable methods with the methods themselves.
9610 * It would be easier to implement this by predefining all methods before compiling
9611 * their bodies, but that couldn't handle the case when a method fails to compile
9612 * with llvm.
9613 */
9614
9615 GHashTable *specializable = g_hash_table_new (NULL, NULL);
9616
9617 GHashTable *patches_to_null = g_hash_table_new (mono_patch_info_hash, mono_patch_info_equal);
9618 for (int sindex = 0; sindex < module->callsite_list->len; ++sindex) {
9619 CallSite *site = (CallSite*)g_ptr_array_index (module->callsite_list, sindex);
9620 method = site->method;
9621 LLVMValueRef lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
9622
9623 LLVMValueRef placeholder = (LLVMValueRef)site->load;
9624 LLVMValueRef indexes [2], got_entry_addr, load;
9625 char *name;
9626
9627 if (lmethod && !(method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)) {
9628 mono_llvm_replace_uses_of (placeholder, lmethod);
9629
9630 if (mono_aot_can_specialize (method))
9631 g_hash_table_insert (specializable, lmethod, method);
9632
9633 g_hash_table_insert (patches_to_null, site->ji, site->ji);
9634 } else {
9635 int got_offset = compute_aot_got_offset (module, site->ji, site->type);
9636
9637 module->max_got_offset = MAX (module->max_got_offset, got_offset);
9638
9639 LLVMBuilderRef builder = LLVMCreateBuilder ();
9640 LLVMPositionBuilderBefore (builder, placeholder);
9641 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
9642 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
9643 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
9644
9645 name = get_aotconst_name (site->ji->type, site->ji->data.target, got_offset);
9646 load = LLVMBuildLoad (builder, got_entry_addr, "");
9647 load = LLVMBuildBitCast (builder, load, site->type, name ? name : "");
9648 LLVMReplaceAllUsesWith (placeholder, load);
9649 }
9650 g_free (site);
9651 }
9652
9653 mono_llvm_propagate_nonnull_final (specializable, module);
9654 g_hash_table_destroy (specializable);
9655
9656 for (int i = 0; i < module->cfgs->len; ++i) {
9657 /*
9658 * Nullify the patches pointing to direct calls. This is needed to
9659 * avoid allocating extra got slots, which is a perf problem and it
9660 * makes module->max_got_offset invalid.
9661 * It would be better to just store the patch_info in CallSite, but
9662 * cfg->patch_info is copied in aot-compiler.c.
9663 */
9664 MonoCompile *cfg = (MonoCompile *)g_ptr_array_index (module->cfgs, i);
9665 for (MonoJumpInfo *patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
9666 if (patch_info->type == MONO_PATCH_INFO_METHOD) {
9667 if (g_hash_table_lookup (patches_to_null, patch_info)) {
9668 patch_info->type = MONO_PATCH_INFO_NONE;
9669 /* Nullify the call to init_method () if possible */
9670 g_assert (cfg->got_access_count);
9671 cfg->got_access_count --;
9672 if (cfg->got_access_count == 0) {
9673 LLVMValueRef br = (LLVMValueRef)cfg->llvmonly_init_cond;
9674 if (br)
9675 LLVMSetSuccessor (br, 0, LLVMGetSuccessor (br, 1));
9676 }
9677 }
9678 }
9679 }
9680 }
9681
9682 g_hash_table_destroy (patches_to_null);
9683 }
9684
9685 static LLVMValueRef
9686 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
9687 {
9688 int i;
9689 LLVMValueRef res, *vals;
9690
9691 vals = g_new0 (LLVMValueRef, nvalues);
9692 for (i = 0; i < nvalues; ++i)
9693 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
9694 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
9695 g_free (vals);
9696 return res;
9697 }
9698
9699 static LLVMValueRef
9700 llvm_array_from_bytes (guint8 *values, int nvalues)
9701 {
9702 int i;
9703 LLVMValueRef res, *vals;
9704
9705 vals = g_new0 (LLVMValueRef, nvalues);
9706 for (i = 0; i < nvalues; ++i)
9707 vals [i] = LLVMConstInt (LLVMInt8Type (), values [i], FALSE);
9708 res = LLVMConstArray (LLVMInt8Type (), vals, nvalues);
9709 g_free (vals);
9710 return res;
9711 }
9712 /*
9713 * mono_llvm_emit_aot_file_info:
9714 *
9715 * Emit the MonoAotFileInfo structure.
9716 * Same as emit_aot_file_info () in aot-compiler.c.
9717 */
9718 void
9719 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
9720 {
9721 MonoLLVMModule *module = &aot_module;
9722
9723 /* Save these for later */
9724 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
9725 module->has_jitted_code = has_jitted_code;
9726 }
9727
9728 /*
9729 * mono_llvm_emit_aot_data:
9730 *
9731 * Emit the binary data DATA pointed to by symbol SYMBOL.
9732 */
9733 void
9734 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
9735 {
9736 MonoLLVMModule *module = &aot_module;
9737 LLVMTypeRef type;
9738 LLVMValueRef d;
9739
9740 type = LLVMArrayType (LLVMInt8Type (), data_len);
9741 d = LLVMAddGlobal (module->lmodule, type, symbol);
9742 LLVMSetVisibility (d, LLVMHiddenVisibility);
9743 LLVMSetLinkage (d, LLVMInternalLinkage);
9744 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
9745 LLVMSetAlignment (d, 8);
9746 mono_llvm_set_is_constant (d);
9747 }
9748
9749 /* Add a reference to a global defined in JITted code */
9750 static LLVMValueRef
9751 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
9752 {
9753 char *s;
9754 LLVMValueRef v;
9755
9756 s = g_strdup_printf ("%s%s", module->global_prefix, name);
9757 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
9758 LLVMSetVisibility (v, LLVMHiddenVisibility);
9759 g_free (s);
9760 return v;
9761 }
9762
9763 static void
9764 emit_aot_file_info (MonoLLVMModule *module)
9765 {
9766 LLVMTypeRef file_info_type;
9767 LLVMTypeRef *eltypes, eltype;
9768 LLVMValueRef info_var;
9769 LLVMValueRef *fields;
9770 int i, nfields, tindex;
9771 MonoAotFileInfo *info;
9772 LLVMModuleRef lmodule = module->lmodule;
9773
9774 info = &module->aot_info;
9775
9776 /* Create an LLVM type to represent MonoAotFileInfo */
9777 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 22 + 5;
9778 eltypes = g_new (LLVMTypeRef, nfields);
9779 tindex = 0;
9780 eltypes [tindex ++] = LLVMInt32Type ();
9781 eltypes [tindex ++] = LLVMInt32Type ();
9782 /* Symbols */
9783 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
9784 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
9785 /* Scalars */
9786 for (i = 0; i < 21; ++i)
9787 eltypes [tindex ++] = LLVMInt32Type ();
9788 /* Arrays */
9789 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
9790 for (i = 0; i < 4; ++i)
9791 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
9792 eltypes [tindex ++] = LLVMArrayType (LLVMInt8Type (), 16);
9793 g_assert (tindex == nfields);
9794 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
9795 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
9796
9797 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
9798 if (module->static_link) {
9799 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
9800 LLVMSetLinkage (info_var, LLVMInternalLinkage);
9801 }
9802
9803 #ifdef TARGET_WIN32
9804 if (!module->static_link) {
9805 LLVMSetDLLStorageClass (info_var, LLVMDLLExportStorageClass);
9806 }
9807 #endif
9808
9809 fields = g_new (LLVMValueRef, nfields);
9810 tindex = 0;
9811 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
9812 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
9813
9814 /* Symbols */
9815 /*
9816 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
9817 * for symbols defined in the .s file emitted by the aot compiler.
9818 */
9819 eltype = eltypes [tindex];
9820 if (module->llvm_only)
9821 fields [tindex ++] = LLVMConstNull (eltype);
9822 else
9823 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
9824 fields [tindex ++] = module->got_var;
9825 /* llc defines this directly */
9826 if (!module->llvm_only) {
9827 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
9828 fields [tindex ++] = LLVMConstNull (eltype);
9829 fields [tindex ++] = LLVMConstNull (eltype);
9830 } else {
9831 fields [tindex ++] = LLVMConstNull (eltype);
9832 fields [tindex ++] = module->get_method;
9833 fields [tindex ++] = module->get_unbox_tramp ? module->get_unbox_tramp : LLVMConstNull (eltype);
9834 }
9835 if (module->has_jitted_code) {
9836 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
9837 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
9838 } else {
9839 fields [tindex ++] = LLVMConstNull (eltype);
9840 fields [tindex ++] = LLVMConstNull (eltype);
9841 }
9842 if (!module->llvm_only)
9843 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
9844 else
9845 fields [tindex ++] = LLVMConstNull (eltype);
9846 if (module->llvm_only && module->unbox_tramp_indexes) {
9847 fields [tindex ++] = module->unbox_tramp_indexes;
9848 fields [tindex ++] = module->unbox_trampolines;
9849 } else {
9850 fields [tindex ++] = LLVMConstNull (eltype);
9851 fields [tindex ++] = LLVMConstNull (eltype);
9852 }
9853 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
9854 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
9855 fields [tindex ++] = LLVMConstNull (eltype);
9856 } else {
9857 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
9858 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
9859 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
9860 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
9861 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
9862 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
9863 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
9864 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
9865 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
9866 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
9867 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "weak_field_indexes");
9868 }
9869 /* Not needed (mem_end) */
9870 fields [tindex ++] = LLVMConstNull (eltype);
9871 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
9872 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
9873 if (info->trampoline_size [0]) {
9874 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
9875 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
9876 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_trampolines");
9877 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
9878 fields [tindex ++] = AddJitGlobal (module, eltype, "ftnptr_arg_trampolines");
9879 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_arbitrary_trampolines");
9880 } else {
9881 fields [tindex ++] = LLVMConstNull (eltype);
9882 fields [tindex ++] = LLVMConstNull (eltype);
9883 fields [tindex ++] = LLVMConstNull (eltype);
9884 fields [tindex ++] = LLVMConstNull (eltype);
9885 fields [tindex ++] = LLVMConstNull (eltype);
9886 fields [tindex ++] = LLVMConstNull (eltype);
9887 }
9888 if (module->static_link && !module->llvm_only)
9889 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
9890 else
9891 fields [tindex ++] = LLVMConstNull (eltype);
9892 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
9893 if (!module->llvm_only) {
9894 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
9895 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
9896 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
9897 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
9898 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
9899 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
9900 } else {
9901 fields [tindex ++] = LLVMConstNull (eltype);
9902 fields [tindex ++] = LLVMConstNull (eltype);
9903 fields [tindex ++] = LLVMConstNull (eltype);
9904 fields [tindex ++] = LLVMConstNull (eltype);
9905 fields [tindex ++] = LLVMConstNull (eltype);
9906 fields [tindex ++] = LLVMConstNull (eltype);
9907 }
9908
9909 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i) {
9910 g_assert (fields [2 + i]);
9911 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
9912 }
9913
9914 /* Scalars */
9915 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
9916 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
9917 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
9918 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
9919 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nextra_methods, FALSE);
9920 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
9921 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
9922 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
9923 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
9924 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
9925 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
9926 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
9927 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
9928 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_shift_bits, FALSE);
9929 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_mask, FALSE);
9930 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
9931 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->call_table_entry_size, FALSE);
9932 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
9933 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
9934 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_num, FALSE);
9935 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_elemsize, FALSE);
9936 /* Arrays */
9937 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
9938 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
9939 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
9940 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
9941 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
9942
9943 fields [tindex ++] = llvm_array_from_bytes (info->aotid, 16);
9944 g_assert (tindex == nfields);
9945
9946 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
9947
9948 if (module->static_link) {
9949 char *s, *p;
9950 LLVMValueRef var;
9951
9952 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
9953 /* Get rid of characters which cannot occur in symbols */
9954 p = s;
9955 for (p = s; *p; ++p) {
9956 if (!(isalnum (*p) || *p == '_'))
9957 *p = '_';
9958 }
9959 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
9960 g_free (s);
9961 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
9962 LLVMSetLinkage (var, LLVMExternalLinkage);
9963 }
9964 }
9965
9966 typedef struct {
9967 LLVMValueRef lmethod;
9968 int argument;
9969 } NonnullPropWorkItem;
9970
9971 static void
9972 mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params)
9973 {
9974 if (!ctx->module->llvm_disable_self_init && mono_aot_can_specialize (call_method)) {
9975 int num_passed = LLVMGetNumArgOperands (lcall);
9976 g_assert (num_params <= num_passed);
9977
9978 g_assert (ctx->module->method_to_call_info);
9979 GArray *call_site_union = (GArray *) g_hash_table_lookup (ctx->module->method_to_call_info, call_method);
9980
9981 if (!call_site_union) {
9982 call_site_union = g_array_sized_new (FALSE, TRUE, sizeof (gint32), num_params);
9983 int zero = 0;
9984 for (int i = 0; i < num_params; i++)
9985 g_array_insert_val (call_site_union, i, zero);
9986 }
9987
9988 for (int i = 0; i < num_params; i++) {
9989 if (mono_llvm_is_nonnull (args [i])) {
9990 g_assert (i < LLVMGetNumArgOperands (lcall));
9991 mono_llvm_set_call_nonnull_arg (lcall, i);
9992 } else {
9993 gint32 *nullable_count = &g_array_index (call_site_union, gint32, i);
9994 *nullable_count = *nullable_count + 1;
9995 }
9996 }
9997
9998 g_hash_table_insert (ctx->module->method_to_call_info, call_method, call_site_union);
9999 }
10000 }
10001
10002 static void
10003 mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module)
10004 {
10005 // When we first traverse the mini IL, we mark the things that are
10006 // nonnull (the roots). Then, for all of the methods that can be specialized, we
10007 // see if their call sites have nonnull attributes.
10008
10009 // If so, we mark the function's param. This param has uses to propagate
10010 // the attribute to. This propagation can trigger a need to mark more attributes
10011 // non-null, and so on and so forth.
10012 GSList *queue = NULL;
10013
10014 GHashTableIter iter;
10015 LLVMValueRef lmethod;
10016 MonoMethod *method;
10017 g_hash_table_iter_init (&iter, all_specializable);
10018 while (g_hash_table_iter_next (&iter, (void**)&lmethod, (void**)&method)) {
10019 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, method);
10020
10021 // Basic sanity checking
10022 if (call_site_union)
10023 g_assert (call_site_union->len == LLVMCountParams (lmethod));
10024
10025 // Add root to work queue
10026 for (int i = 0; call_site_union && i < call_site_union->len; i++) {
10027 if (g_array_index (call_site_union, gint32, i) == 0) {
10028 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
10029 item->lmethod = lmethod;
10030 item->argument = i;
10031 queue = g_slist_prepend (queue, item);
10032 }
10033 }
10034 }
10035
10036 // This is essentially reference counting, and we are propagating
10037 // the refcount decrement here. We have less work to do than we may otherwise
10038 // because we are only working with a set of subgraphs of specializable functions.
10039 //
10040 // We rely on being able to see all of the references in the graph.
10041 // This is ensured by the function mono_aot_can_specialize. Everything in
10042 // all_specializable is a function that can be specialized, and is the resulting
10043 // node in the graph after all of the subsitutions are done.
10044 //
10045 // Anything disrupting the direct calls made with self-init will break this optimization.
10046
10047 while (queue) {
10048 // Update the queue state.
10049 // Our only other per-iteration responsibility is now to free current
10050 NonnullPropWorkItem *current = (NonnullPropWorkItem *) queue->data;
10051 queue = queue->next;
10052 g_assert (current->argument < LLVMCountParams (current->lmethod));
10053
10054 // Does the actual leaf-node work here
10055 // Mark the function argument as nonnull for LLVM
10056 mono_llvm_set_func_nonnull_arg (current->lmethod, current->argument);
10057
10058 // The rest of this is for propagating forward nullability changes
10059 // to calls that use the argument that is now nullable.
10060
10061 // Get the actual LLVM value of the argument, so we can see which call instructions
10062 // used that argument
10063 LLVMValueRef caller_argument = LLVMGetParam (current->lmethod, current->argument);
10064
10065 // Iterate over the calls using the newly-non-nullable argument
10066 GSList *calls = mono_llvm_calls_using (caller_argument);
10067 for (GSList *cursor = calls; cursor != NULL; cursor = cursor->next) {
10068
10069 LLVMValueRef lcall = (LLVMValueRef) cursor->data;
10070 LLVMValueRef callee_lmethod = LLVMGetCalledValue (lcall);
10071
10072 // If this wasn't a direct call for which mono_aot_can_specialize is true,
10073 // this lookup won't find a MonoMethod.
10074 MonoMethod *callee_method = (MonoMethod *) g_hash_table_lookup (all_specializable, callee_lmethod);
10075 if (!callee_method)
10076 continue;
10077
10078 // Decrement number of nullable refs at that func's arg offset
10079 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, callee_method);
10080
10081 // It has module-local callers and is specializable, should have seen this call site
10082 // and inited this
10083 g_assert (call_site_union);
10084
10085 // The function *definition* parameter arity should always be consistent
10086 int max_params = LLVMCountParams (callee_lmethod);
10087 if (call_site_union->len != max_params) {
10088 mono_llvm_dump_value (callee_lmethod);
10089 g_assert_not_reached ();
10090 }
10091
10092 // Get the values that correspond to the parameters passed to the call
10093 // that used our argument
10094 LLVMValueRef *operands = mono_llvm_call_args (lcall);
10095 for (int call_argument = 0; call_argument < max_params; call_argument++) {
10096 // Every time we used the newly-non-nullable argument, decrement the nullable
10097 // refcount for that function.
10098 if (caller_argument == operands [call_argument]) {
10099 gint32 *nullable_count = &g_array_index (call_site_union, gint32, call_argument);
10100 g_assert (*nullable_count > 0);
10101 *nullable_count = *nullable_count - 1;
10102
10103 // If we caused that callee's parameter to become newly nullable, add to work queue
10104 if (*nullable_count == 0) {
10105 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
10106 item->lmethod = callee_lmethod;
10107 item->argument = call_argument;
10108 queue = g_slist_prepend (queue, item);
10109 }
10110 }
10111 }
10112 g_free (operands);
10113
10114 // Update nullability refcount information for the callee now
10115 g_hash_table_insert (module->method_to_call_info, callee_method, call_site_union);
10116 }
10117 g_slist_free (calls);
10118
10119 g_free (current);
10120 }
10121 }
10122
10123 /*
10124 * Emit the aot module into the LLVM bitcode file FILENAME.
10125 */
10126 void
10127 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
10128 {
10129 LLVMTypeRef got_type, inited_type;
10130 LLVMValueRef real_got, real_inited;
10131 MonoLLVMModule *module = &aot_module;
10132
10133 emit_llvm_code_end (module);
10134
10135 /*
10136 * Create the real got variable and replace all uses of the dummy variable with
10137 * the real one.
10138 */
10139 int size = module->max_got_offset + 1;
10140 LLVMTypeRef *members = g_malloc0 (sizeof (LLVMValueRef) * size);
10141 for (int i = 0; i < size; i++) {
10142 LLVMTypeRef lookup_type = NULL;
10143
10144 lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (i));
10145
10146 if (!lookup_type)
10147 lookup_type = module->ptr_type;
10148
10149 members [i] = LLVMPointerType (lookup_type, 0);
10150 }
10151
10152 got_type = LLVMStructCreateNamed (module->context, g_strdup_printf ("MONO_GOT_%s", cu_name));
10153 LLVMStructSetBody (got_type, members, size, FALSE);
10154 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
10155
10156 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
10157 if (module->external_symbols) {
10158 LLVMSetLinkage (real_got, LLVMExternalLinkage);
10159 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
10160 } else {
10161 LLVMSetLinkage (real_got, LLVMInternalLinkage);
10162 }
10163 mono_llvm_replace_uses_of (module->got_var, real_got);
10164
10165 mark_as_used (&aot_module, real_got);
10166
10167 /* Delete the dummy got so it doesn't become a global */
10168 LLVMDeleteGlobal (module->got_var);
10169 module->got_var = real_got;
10170
10171 /*
10172 * Same for the init_var
10173 */
10174 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
10175 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
10176 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
10177 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
10178 mono_llvm_replace_uses_of (module->inited_var, real_inited);
10179 LLVMDeleteGlobal (module->inited_var);
10180
10181 if (module->llvm_only) {
10182 emit_get_method (&aot_module);
10183 emit_get_unbox_tramp (&aot_module);
10184 }
10185
10186 emit_llvm_used (&aot_module);
10187 emit_dbg_info (&aot_module, filename, cu_name);
10188 emit_aot_file_info (&aot_module);
10189
10190 /* Replace PLT entries for directly callable methods with the methods themselves */
10191 {
10192 GHashTableIter iter;
10193 MonoJumpInfo *ji;
10194 LLVMValueRef callee;
10195
10196 GHashTable *specializable = g_hash_table_new (NULL, NULL);
10197
10198 g_hash_table_iter_init (&iter, module->plt_entries_ji);
10199 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
10200 if (mono_aot_is_direct_callable (ji)) {
10201 LLVMValueRef lmethod;
10202
10203 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
10204 /* The types might not match because the caller might pass an rgctx */
10205 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
10206 mono_llvm_replace_uses_of (callee, lmethod);
10207
10208 if (!module->llvm_disable_self_init && mono_aot_can_specialize (ji->data.method))
10209 g_hash_table_insert (specializable, lmethod, ji->data.method);
10210 mono_aot_mark_unused_llvm_plt_entry (ji);
10211 }
10212 }
10213 }
10214
10215 mono_llvm_propagate_nonnull_final (specializable, module);
10216
10217 g_hash_table_destroy (specializable);
10218 }
10219
10220 /* Note: You can still dump an invalid bitcode file by running `llvm-dis`
10221 * in a debugger, set a breakpoint on `LLVMVerifyModule` and fake its
10222 * result to 0 (indicating success). */
10223 LLVMWriteBitcodeToFile (module->lmodule, filename);
10224
10225 #if 1
10226 {
10227 char *verifier_err;
10228
10229 if (LLVMVerifyModule (module->lmodule, LLVMReturnStatusAction, &verifier_err)) {
10230 printf ("%s\n", verifier_err);
10231 g_assert_not_reached ();
10232 }
10233 }
10234 #endif
10235 }
10236
10237
10238 static LLVMValueRef
10239 md_string (const char *s)
10240 {
10241 return LLVMMDString (s, strlen (s));
10242 }
10243
10244 /* Debugging support */
10245
10246 static void
10247 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
10248 {
10249 LLVMModuleRef lmodule = module->lmodule;
10250 LLVMValueRef args [16], ver;
10251
10252 /*
10253 * This can only be enabled when LLVM code is emitted into a separate object
10254 * file, since the AOT compiler also emits dwarf info,
10255 * and the abbrev indexes will not be correct since llvm has added its own
10256 * abbrevs.
10257 */
10258 if (!module->emit_dwarf)
10259 return;
10260
10261 mono_llvm_di_builder_finalize (module->di_builder);
10262
10263 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
10264 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
10265 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
10266 ver = LLVMMDNode (args, 3);
10267 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
10268
10269 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
10270 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
10271 args [2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE);
10272 ver = LLVMMDNode (args, 3);
10273 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
10274 }
10275
10276 static LLVMValueRef
10277 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
10278 {
10279 MonoLLVMModule *module = ctx->module;
10280 MonoDebugMethodInfo *minfo = ctx->minfo;
10281 char *source_file, *dir, *filename;
10282 MonoSymSeqPoint *sym_seq_points;
10283 int n_seq_points;
10284
10285 if (!minfo)
10286 return NULL;
10287
10288 mono_debug_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
10289 if (!source_file)
10290 source_file = g_strdup ("<unknown>");
10291 dir = g_path_get_dirname (source_file);
10292 filename = g_path_get_basename (source_file);
10293 g_free (source_file);
10294
10295 return (LLVMValueRef)mono_llvm_di_create_function (module->di_builder, module->cu, method, cfg->method->name, name, dir, filename, n_seq_points ? sym_seq_points [0].line : 1);
10296 }
10297
10298 static void
10299 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
10300 {
10301 MonoCompile *cfg = ctx->cfg;
10302
10303 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
10304 MonoDebugSourceLocation *loc;
10305 LLVMValueRef loc_md;
10306
10307 loc = mono_debug_method_lookup_location (ctx->minfo, cil_code - cfg->header->code);
10308
10309 if (loc) {
10310 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, loc->row, loc->column);
10311 mono_llvm_di_set_location (builder, loc_md);
10312 mono_debug_free_source_location (loc);
10313 }
10314 }
10315 }
10316
10317 static void
10318 emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder)
10319 {
10320 if (ctx->minfo) {
10321 LLVMValueRef loc_md;
10322 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, 0, 0);
10323 mono_llvm_di_set_location (builder, loc_md);
10324 }
10325 }
10326
10327 /*
10328 DESIGN:
10329 - Emit LLVM IR from the mono IR using the LLVM C API.
10330 - The original arch specific code remains, so we can fall back to it if we run
10331 into something we can't handle.
10332 */
10333
10334 /*
10335 A partial list of issues:
10336 - Handling of opcodes which can throw exceptions.
10337
10338 In the mono JIT, these are implemented using code like this:
10339 method:
10340 <compare>
10341 throw_pos:
10342 b<cond> ex_label
10343 <rest of code>
10344 ex_label:
10345 push throw_pos - method
10346 call <exception trampoline>
10347
10348 The problematic part is push throw_pos - method, which cannot be represented
10349 in the LLVM IR, since it does not support label values.
10350 -> this can be implemented in AOT mode using inline asm + labels, but cannot
10351 be implemented in JIT mode ?
10352 -> a possible but slower implementation would use the normal exception
10353 throwing code but it would need to control the placement of the throw code
10354 (it needs to be exactly after the compare+branch).
10355 -> perhaps add a PC offset intrinsics ?
10356
10357 - efficient implementation of .ovf opcodes.
10358
10359 These are currently implemented as:
10360 <ins which sets the condition codes>
10361 b<cond> ex_label
10362
10363 Some overflow opcodes are now supported by LLVM SVN.
10364
10365 - exception handling, unwinding.
10366 - SSA is disabled for methods with exception handlers
10367 - How to obtain unwind info for LLVM compiled methods ?
10368 -> this is now solved by converting the unwind info generated by LLVM
10369 into our format.
10370 - LLVM uses the c++ exception handling framework, while we use our home grown
10371 code, and couldn't use the c++ one:
10372 - its not supported under VC++, other exotic platforms.
10373 - it might be impossible to support filter clauses with it.
10374
10375 - trampolines.
10376
10377 The trampolines need a predictable call sequence, since they need to disasm
10378 the calling code to obtain register numbers / offsets.
10379
10380 LLVM currently generates this code in non-JIT mode:
10381 mov -0x98(%rax),%eax
10382 callq *%rax
10383 Here, the vtable pointer is lost.
10384 -> solution: use one vtable trampoline per class.
10385
10386 - passing/receiving the IMT pointer/RGCTX.
10387 -> solution: pass them as normal arguments ?
10388
10389 - argument passing.
10390
10391 LLVM does not allow the specification of argument registers etc. This means
10392 that all calls are made according to the platform ABI.
10393
10394 - passing/receiving vtypes.
10395
10396 Vtypes passed/received in registers are handled by the front end by using
10397 a signature with scalar arguments, and loading the parts of the vtype into those
10398 arguments.
10399
10400 Vtypes passed on the stack are handled using the 'byval' attribute.
10401
10402 - ldaddr.
10403
10404 Supported though alloca, we need to emit the load/store code.
10405
10406 - types.
10407
10408 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
10409 typed registers, so we have to keep track of the precise LLVM type of each vreg.
10410 This is made easier because the IR is already in SSA form.
10411 An additional problem is that our IR is not consistent with types, i.e. i32/i64
10412 types are frequently used incorrectly.
10413 */
10414
10415 /*
10416 AOT SUPPORT:
10417 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
10418 it with the file containing the methods emitted by the JIT and the AOT data
10419 structures.
10420 */
10421
10422 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
10423 * - each bblock should end with a branch
10424 * - setting the return value, making cfg->ret non-volatile
10425 * - avoid some transformations in the JIT which make it harder for us to generate
10426 * code.
10427 * - use pointer types to help optimizations.
10428 */
10429
10430 #else /* DISABLE_JIT */
10431
10432 void
10433 mono_llvm_cleanup (void)
10434 {
10435 }
10436
10437 void
10438 mono_llvm_free_domain_info (MonoDomain *domain)
10439 {
10440 }
10441
10442 void
10443 mono_llvm_init (void)
10444 {
10445 }
10446
10447 #endif /* DISABLE_JIT */
10448
10449 #if !defined(DISABLE_JIT) && !defined(MONO_CROSS_COMPILE)
10450
10451 /* LLVM JIT support */
10452
10453 static unsigned char*
10454 alloc_cb (LLVMValueRef function, int size)
10455 {
10456 MonoCompile *cfg;
10457
10458 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10459
10460 if (cfg) {
10461 // FIXME: dynamic
10462 return (unsigned char*)mono_domain_code_reserve (cfg->domain, size);
10463 } else {
10464 return (unsigned char*)mono_domain_code_reserve (mono_domain_get (), size);
10465 }
10466 }
10467
10468 static void
10469 emitted_cb (LLVMValueRef function, void *start, void *end)
10470 {
10471 MonoCompile *cfg;
10472
10473 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10474 g_assert (cfg);
10475 cfg->code_len = (guint8*)end - (guint8*)start;
10476 }
10477
10478 static void
10479 exception_cb (void *data)
10480 {
10481 MonoCompile *cfg;
10482 MonoJitExceptionInfo *ei;
10483 guint32 ei_len, i, j, nested_len, nindex;
10484 gpointer *type_info;
10485 int this_reg, this_offset;
10486
10487 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10488 g_assert (cfg);
10489
10490 /*
10491 * data points to a DWARF FDE structure, convert it to our unwind format and
10492 * save it.
10493 * An alternative would be to save it directly, and modify our unwinder to work
10494 * with it.
10495 */
10496 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
10497 if (cfg->verbose_level > 1)
10498 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
10499
10500 /* Count nested clauses */
10501 nested_len = 0;
10502 for (i = 0; i < ei_len; ++i) {
10503 gint32 cindex1 = *(gint32*)type_info [i];
10504 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
10505
10506 for (j = 0; j < cfg->header->num_clauses; ++j) {
10507 int cindex2 = j;
10508 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
10509
10510 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
10511 nested_len ++;
10512 }
10513 }
10514 }
10515
10516 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
10517 cfg->llvm_ex_info_len = ei_len + nested_len;
10518 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
10519 /* Fill the rest of the information from the type info */
10520 for (i = 0; i < ei_len; ++i) {
10521 gint32 clause_index = *(gint32*)type_info [i];
10522 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
10523
10524 cfg->llvm_ex_info [i].flags = clause->flags;
10525 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
10526 cfg->llvm_ex_info [i].clause_index = clause_index;
10527 }
10528
10529 /*
10530 * For nested clauses, the LLVM produced exception info associates the try interval with
10531 * the innermost handler, while mono expects it to be associated with all nesting clauses.
10532 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
10533 * and everything else from the nested clause.
10534 */
10535 nindex = ei_len;
10536 for (i = 0; i < ei_len; ++i) {
10537 gint32 cindex1 = *(gint32*)type_info [i];
10538 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
10539
10540 for (j = 0; j < cfg->header->num_clauses; ++j) {
10541 int cindex2 = j;
10542 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
10543 MonoJitExceptionInfo *nesting_ei, *nested_ei;
10544
10545 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
10546 /* clause1 is the nested clause */
10547 nested_ei = &cfg->llvm_ex_info [i];
10548 nesting_ei = &cfg->llvm_ex_info [nindex];
10549 nindex ++;
10550
10551 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
10552
10553 nesting_ei->flags = clause2->flags;
10554 nesting_ei->data.catch_class = clause2->data.catch_class;
10555 nesting_ei->clause_index = cindex2;
10556 }
10557 }
10558 }
10559 g_assert (nindex == ei_len + nested_len);
10560 cfg->llvm_this_reg = this_reg;
10561 cfg->llvm_this_offset = this_offset;
10562
10563 /* type_info [i] is cfg mempool allocated, no need to free it */
10564
10565 g_free (ei);
10566 g_free (type_info);
10567 }
10568
10569 /*
10570 * decode_llvm_eh_info:
10571 *
10572 * Decode the EH table emitted by llvm in jit mode, and store
10573 * the result into cfg.
10574 */
10575 static void
10576 decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame)
10577 {
10578 MonoCompile *cfg = ctx->cfg;
10579 guint8 *cie, *fde;
10580 int fde_len;
10581 MonoLLVMFDEInfo info;
10582 MonoJitExceptionInfo *ei;
10583 guint8 *p = (guint8*)eh_frame;
10584 int version, fde_count, fde_offset;
10585 guint32 ei_len, i, nested_len;
10586 gpointer *type_info;
10587 gint32 *table;
10588 guint8 *unw_info;
10589
10590 /*
10591 * Decode the one element EH table emitted by the MonoException class
10592 * in llvm.
10593 */
10594
10595 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
10596
10597 version = *p;
10598 g_assert (version == 3);
10599 p ++;
10600 p ++;
10601 p = (guint8 *)ALIGN_PTR_TO (p, 4);
10602
10603 fde_count = *(guint32*)p;
10604 p += 4;
10605 table = (gint32*)p;
10606
10607 g_assert (fde_count <= 2);
10608
10609 /* The first entry is the real method */
10610 g_assert (table [0] == 1);
10611 fde_offset = table [1];
10612 table += fde_count * 2;
10613 /* Extra entry */
10614 cfg->code_len = table [0];
10615 fde_len = table [1] - fde_offset;
10616 table += 2;
10617
10618 fde = (guint8*)eh_frame + fde_offset;
10619 cie = (guint8*)table;
10620
10621 /* Compute lengths */
10622 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, NULL, NULL, NULL);
10623
10624 ei = (MonoJitExceptionInfo *)g_malloc0 (info.ex_info_len * sizeof (MonoJitExceptionInfo));
10625 type_info = (gpointer *)g_malloc0 (info.ex_info_len * sizeof (gpointer));
10626 unw_info = (guint8*)g_malloc0 (info.unw_info_len);
10627
10628 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, ei, type_info, unw_info);
10629
10630 cfg->encoded_unwind_ops = unw_info;
10631 cfg->encoded_unwind_ops_len = info.unw_info_len;
10632 if (cfg->verbose_level > 1)
10633 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
10634 if (info.this_reg != -1) {
10635 cfg->llvm_this_reg = info.this_reg;
10636 cfg->llvm_this_offset = info.this_offset;
10637 }
10638
10639 ei_len = info.ex_info_len;
10640
10641 // Nested clauses are currently disabled
10642 nested_len = 0;
10643
10644 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
10645 cfg->llvm_ex_info_len = ei_len + nested_len;
10646 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
10647 /* Fill the rest of the information from the type info */
10648 for (i = 0; i < ei_len; ++i) {
10649 gint32 clause_index = *(gint32*)type_info [i];
10650 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
10651
10652 cfg->llvm_ex_info [i].flags = clause->flags;
10653 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
10654 cfg->llvm_ex_info [i].clause_index = clause_index;
10655 }
10656 }
10657
10658 static void
10659 init_jit_module (MonoDomain *domain)
10660 {
10661 MonoJitDomainInfo *dinfo;
10662 MonoLLVMModule *module;
10663 char *name;
10664
10665 dinfo = domain_jit_info (domain);
10666 if (dinfo->llvm_module)
10667 return;
10668
10669 mono_loader_lock ();
10670
10671 if (dinfo->llvm_module) {
10672 mono_loader_unlock ();
10673 return;
10674 }
10675
10676 module = g_new0 (MonoLLVMModule, 1);
10677
10678 name = g_strdup_printf ("mono-%s", domain->friendly_name);
10679 module->context = LLVMGetGlobalContext ();
10680 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
10681
10682 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (alloc_cb, emitted_cb, exception_cb, &module->ee);
10683
10684 add_intrinsics (module->lmodule);
10685 add_types (module);
10686
10687 module->llvm_types = g_hash_table_new (NULL, NULL);
10688
10689 mono_memory_barrier ();
10690
10691 dinfo->llvm_module = module;
10692
10693 mono_loader_unlock ();
10694 }
10695
10696 static void
10697 llvm_jit_finalize_method (EmitContext *ctx)
10698 {
10699 MonoCompile *cfg = ctx->cfg;
10700 MonoDomain *domain = mono_domain_get ();
10701 MonoJitDomainInfo *domain_info;
10702 int nvars = g_hash_table_size (ctx->jit_callees);
10703 LLVMValueRef *callee_vars = g_new0 (LLVMValueRef, nvars);
10704 gpointer *callee_addrs = g_new0 (gpointer, nvars);
10705 GHashTableIter iter;
10706 LLVMValueRef var;
10707 MonoMethod *callee;
10708 gpointer eh_frame;
10709 int i;
10710
10711 /*
10712 * Compute the addresses of the LLVM globals pointing to the
10713 * methods called by the current method. Pass it to the trampoline
10714 * code so it can update them after their corresponding method was
10715 * compiled.
10716 */
10717 g_hash_table_iter_init (&iter, ctx->jit_callees);
10718 i = 0;
10719 while (g_hash_table_iter_next (&iter, NULL, (void**)&var))
10720 callee_vars [i ++] = var;
10721
10722 cfg->native_code = (guint8*)mono_llvm_compile_method (ctx->module->mono_ee, ctx->lmethod, nvars, callee_vars, callee_addrs, &eh_frame);
10723 mono_llvm_remove_gc_safepoint_poll (ctx->lmodule);
10724 if (cfg->verbose_level > 1) {
10725 g_print ("\n*** Optimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, TRUE));
10726 if (cfg->compile_aot) {
10727 mono_llvm_dump_value (ctx->lmethod);
10728 } else {
10729 mono_llvm_dump_module (ctx->lmodule);
10730 }
10731 g_print ("***\n\n");
10732 }
10733
10734 decode_llvm_eh_info (ctx, eh_frame);
10735
10736 mono_domain_lock (domain);
10737 domain_info = domain_jit_info (domain);
10738 if (!domain_info->llvm_jit_callees)
10739 domain_info->llvm_jit_callees = g_hash_table_new (NULL, NULL);
10740 g_hash_table_iter_init (&iter, ctx->jit_callees);
10741 i = 0;
10742 while (g_hash_table_iter_next (&iter, (void**)&callee, (void**)&var)) {
10743 GSList *addrs = (GSList*)g_hash_table_lookup (domain_info->llvm_jit_callees, callee);
10744 addrs = g_slist_prepend (addrs, callee_addrs [i]);
10745 g_hash_table_insert (domain_info->llvm_jit_callees, callee, addrs);
10746 i ++;
10747 }
10748 mono_domain_unlock (domain);
10749 }
10750
10751 #else
10752
10753 static void
10754 init_jit_module (MonoDomain *domain)
10755 {
10756 g_assert_not_reached ();
10757 }
10758
10759 static void
10760 llvm_jit_finalize_method (EmitContext *ctx)
10761 {
10762 g_assert_not_reached ();
10763 }
10764
10765 #endif
10766
10767 static MonoCPUFeatures cpu_features;
10768
10769 MonoCPUFeatures mono_llvm_get_cpu_features (void)
10770 {
10771 static const CpuFeatureAliasFlag flags_map [] = {
10772 #if defined(TARGET_X86) || defined(TARGET_AMD64)
10773 { "sse", MONO_CPU_X86_SSE },
10774 { "sse2", MONO_CPU_X86_SSE2 },
10775 { "pclmul", MONO_CPU_X86_PCLMUL },
10776 { "aes", MONO_CPU_X86_AES },
10777 { "sse2", MONO_CPU_X86_SSE2 },
10778 { "sse3", MONO_CPU_X86_SSE3 },
10779 { "ssse3", MONO_CPU_X86_SSSE3 },
10780 { "sse4.1", MONO_CPU_X86_SSE41 },
10781 { "sse4.2", MONO_CPU_X86_SSE42 },
10782 { "popcnt", MONO_CPU_X86_POPCNT },
10783 { "avx", MONO_CPU_X86_AVX },
10784 { "avx2", MONO_CPU_X86_AVX2 },
10785 { "fma", MONO_CPU_X86_FMA },
10786 { "lzcnt", MONO_CPU_X86_LZCNT },
10787 { "bmi", MONO_CPU_X86_BMI1 },
10788 { "bmi2", MONO_CPU_X86_BMI2 },
10789 #endif
10790 };
10791 if (!cpu_features)
10792 cpu_features = MONO_CPU_INITED | (MonoCPUFeatures)mono_llvm_check_cpu_features (flags_map, G_N_ELEMENTS (flags_map));
10793 return cpu_features;
10794 }
Mirror of the official Mono project git repository