?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[Sockets]: Always reset internal SAEA completion when reattempting connection in...
[mono-project.git] / mono / mini / mini-llvm.c
blob3a91bdc6535178143d6a56c4c107c52114dd7899
1 /**
2 * \file
3 * llvm "Backend" for the mono JIT
4 *
5 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
6 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
8 */
9
10 #include "config.h"
11
12 #include <mono/metadata/debug-helpers.h>
13 #include <mono/metadata/debug-internals.h>
14 #include <mono/metadata/mempool-internals.h>
15 #include <mono/metadata/environment.h>
16 #include <mono/metadata/object-internals.h>
17 #include <mono/metadata/abi-details.h>
18 #include <mono/utils/mono-tls.h>
19 #include <mono/utils/mono-dl.h>
20 #include <mono/utils/mono-time.h>
21 #include <mono/utils/freebsd-dwarf.h>
22
23 #ifndef __STDC_LIMIT_MACROS
24 #define __STDC_LIMIT_MACROS
25 #endif
26 #ifndef __STDC_CONSTANT_MACROS
27 #define __STDC_CONSTANT_MACROS
28 #endif
29
30 #include "llvm-c/BitWriter.h"
31 #include "llvm-c/Analysis.h"
32
33 #include "mini-llvm-cpp.h"
34 #include "llvm-jit.h"
35 #include "aot-compiler.h"
36 #include "mini-llvm.h"
37 #include "mini-runtime.h"
38 #include <mono/utils/mono-math.h>
39
40 #ifndef DISABLE_JIT
41
42 #if defined(TARGET_AMD64) && defined(TARGET_WIN32) && defined(HOST_WIN32) && defined(_MSC_VER)
43 #define TARGET_X86_64_WIN32_MSVC
44 #endif
45
46 #if defined(TARGET_X86_64_WIN32_MSVC)
47 #define TARGET_WIN32_MSVC
48 #endif
49
50 #if LLVM_API_VERSION < 610
51 #error "The version of the mono llvm repository is too old."
52 #endif
53
54 /*
55 * Information associated by mono with LLVM modules.
56 */
57 typedef struct {
58 LLVMModuleRef lmodule;
59 LLVMValueRef throw_icall, rethrow, throw_corlib_exception;
60 GHashTable *llvm_types;
61 LLVMValueRef got_var;
62 const char *got_symbol;
63 const char *get_method_symbol;
64 const char *get_unbox_tramp_symbol;
65 GHashTable *plt_entries;
66 GHashTable *plt_entries_ji;
67 GHashTable *method_to_lmethod;
68 GHashTable *method_to_call_info;
69 GHashTable *lvalue_to_lcalls;
70 GHashTable *direct_callables;
71 char **bb_names;
72 int bb_names_len;
73 GPtrArray *used;
74 LLVMTypeRef ptr_type;
75 GPtrArray *subprogram_mds;
76 MonoEERef *mono_ee;
77 LLVMExecutionEngineRef ee;
78 gboolean external_symbols;
79 gboolean emit_dwarf;
80 int max_got_offset;
81 LLVMValueRef personality;
82 LLVMValueRef *intrins_by_id;
83 gpointer gc_poll_cold_wrapper_compiled;
84
85 /* For AOT */
86 MonoAssembly *assembly;
87 char *global_prefix;
88 MonoAotFileInfo aot_info;
89 const char *jit_got_symbol;
90 const char *eh_frame_symbol;
91 LLVMValueRef get_method, get_unbox_tramp;
92 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
93 LLVMValueRef code_start, code_end;
94 LLVMValueRef inited_var;
95 LLVMValueRef unbox_tramp_indexes;
96 LLVMValueRef unbox_trampolines;
97 LLVMValueRef gc_poll_cold_wrapper;
98 int max_inited_idx, max_method_idx;
99 gboolean has_jitted_code;
100 gboolean static_link;
101 gboolean llvm_only;
102 gboolean llvm_disable_self_init;
103 gboolean interp;
104 GHashTable *idx_to_lmethod;
105 GHashTable *idx_to_unbox_tramp;
106 GPtrArray *callsite_list;
107 LLVMContextRef context;
108 LLVMValueRef sentinel_exception;
109 void *di_builder, *cu;
110 GHashTable *objc_selector_to_var;
111 GPtrArray *cfgs;
112 int unbox_tramp_num, unbox_tramp_elemsize;
113 GHashTable *got_idx_to_type;
114 } MonoLLVMModule;
115
116 /*
117 * Information associated by the backend with mono basic blocks.
118 */
119 typedef struct {
120 LLVMBasicBlockRef bblock, end_bblock;
121 LLVMValueRef finally_ind;
122 gboolean added, invoke_target;
123 /*
124 * If this bblock is the start of a finally clause, this is a list of bblocks it
125 * needs to branch to in ENDFINALLY.
126 */
127 GSList *call_handler_return_bbs;
128 /*
129 * If this bblock is the start of a finally clause, this is the bblock that
130 * CALL_HANDLER needs to branch to.
131 */
132 LLVMBasicBlockRef call_handler_target_bb;
133 /* The list of switch statements generated by ENDFINALLY instructions */
134 GSList *endfinally_switch_ins_list;
135 GSList *phi_nodes;
136 } BBInfo;
137
138 /*
139 * Structure containing emit state
140 */
141 typedef struct {
142 MonoMemPool *mempool;
143
144 /* Maps method names to the corresponding LLVMValueRef */
145 GHashTable *emitted_method_decls;
146
147 MonoCompile *cfg;
148 LLVMValueRef lmethod;
149 MonoLLVMModule *module;
150 LLVMModuleRef lmodule;
151 BBInfo *bblocks;
152 int sindex, default_index, ex_index;
153 LLVMBuilderRef builder;
154 LLVMValueRef *values, *addresses;
155 MonoType **vreg_cli_types;
156 LLVMCallInfo *linfo;
157 MonoMethodSignature *sig;
158 GSList *builders;
159 GHashTable *region_to_handler;
160 GHashTable *clause_to_handler;
161 LLVMBuilderRef alloca_builder;
162 LLVMValueRef last_alloca;
163 LLVMValueRef rgctx_arg;
164 LLVMValueRef this_arg;
165 LLVMTypeRef *vreg_types;
166 gboolean *is_vphi;
167 LLVMTypeRef method_type;
168 LLVMBasicBlockRef init_bb, inited_bb;
169 gboolean *is_dead;
170 gboolean *unreachable;
171 gboolean llvm_only;
172 gboolean has_got_access;
173 gboolean emit_dummy_arg;
174 int this_arg_pindex, rgctx_arg_pindex;
175 LLVMValueRef imt_rgctx_loc;
176 GHashTable *llvm_types;
177 LLVMValueRef dbg_md;
178 MonoDebugMethodInfo *minfo;
179 /* For every clause, the clauses it is nested in */
180 GSList **nested_in;
181 LLVMValueRef ex_var;
182 GHashTable *exc_meta;
183 GPtrArray *callsite_list;
184 GPtrArray *phi_values;
185 GPtrArray *bblock_list;
186 char *method_name;
187 GHashTable *jit_callees;
188 LLVMValueRef long_bb_break_var;
189 } EmitContext;
190
191 typedef struct {
192 MonoBasicBlock *bb;
193 MonoInst *phi;
194 MonoBasicBlock *in_bb;
195 int sreg;
196 } PhiNode;
197
198 /*
199 * Instruction metadata
200 * This is the same as ins_info, but LREG != IREG.
201 */
202 #ifdef MINI_OP
203 #undef MINI_OP
204 #endif
205 #ifdef MINI_OP3
206 #undef MINI_OP3
207 #endif
208 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
209 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
210 #define NONE ' '
211 #define IREG 'i'
212 #define FREG 'f'
213 #define VREG 'v'
214 #define XREG 'x'
215 #define LREG 'l'
216 /* keep in sync with the enum in mini.h */
217 const char
218 mini_llvm_ins_info[] = {
219 #include "mini-ops.h"
220 };
221 #undef MINI_OP
222 #undef MINI_OP3
223
224 #if TARGET_SIZEOF_VOID_P == 4
225 #define GET_LONG_IMM(ins) ((ins)->inst_l)
226 #else
227 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
228 #endif
229
230 #define LLVM_INS_INFO(opcode) (&mini_llvm_ins_info [((opcode) - OP_START - 1) * 4])
231
232 #if 0
233 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
234 #else
235 #define TRACE_FAILURE(msg)
236 #endif
237
238 #ifdef TARGET_X86
239 #define IS_TARGET_X86 1
240 #else
241 #define IS_TARGET_X86 0
242 #endif
243
244 #ifdef TARGET_AMD64
245 #define IS_TARGET_AMD64 1
246 #else
247 #define IS_TARGET_AMD64 0
248 #endif
249
250 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
251
252 static LLVMIntPredicate cond_to_llvm_cond [] = {
253 LLVMIntEQ,
254 LLVMIntNE,
255 LLVMIntSLE,
256 LLVMIntSGE,
257 LLVMIntSLT,
258 LLVMIntSGT,
259 LLVMIntULE,
260 LLVMIntUGE,
261 LLVMIntULT,
262 LLVMIntUGT,
263 };
264
265 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
266 LLVMRealOEQ,
267 LLVMRealUNE,
268 LLVMRealOLE,
269 LLVMRealOGE,
270 LLVMRealOLT,
271 LLVMRealOGT,
272 LLVMRealULE,
273 LLVMRealUGE,
274 LLVMRealULT,
275 LLVMRealUGT,
276 };
277
278 typedef enum {
279 INTRINS_MEMSET,
280 INTRINS_MEMCPY,
281 INTRINS_MEMMOVE,
282 INTRINS_SADD_OVF_I32,
283 INTRINS_UADD_OVF_I32,
284 INTRINS_SSUB_OVF_I32,
285 INTRINS_USUB_OVF_I32,
286 INTRINS_SMUL_OVF_I32,
287 INTRINS_UMUL_OVF_I32,
288 INTRINS_SADD_OVF_I64,
289 INTRINS_UADD_OVF_I64,
290 INTRINS_SSUB_OVF_I64,
291 INTRINS_USUB_OVF_I64,
292 INTRINS_SMUL_OVF_I64,
293 INTRINS_UMUL_OVF_I64,
294 INTRINS_SIN,
295 INTRINS_COS,
296 INTRINS_SQRT,
297 INTRINS_FLOOR,
298 INTRINS_CEIL,
299 INTRINS_FMA,
300 INTRINS_FABS,
301 INTRINS_ABSF,
302 INTRINS_SINF,
303 INTRINS_COSF,
304 INTRINS_SQRTF,
305 INTRINS_FLOORF,
306 INTRINS_CEILF,
307 INTRINS_FMAF,
308 INTRINS_POW,
309 INTRINS_POWF,
310 INTRINS_EXP,
311 INTRINS_EXPF,
312 INTRINS_LOG,
313 INTRINS_LOG2,
314 INTRINS_LOG2F,
315 INTRINS_LOG10,
316 INTRINS_LOG10F,
317 INTRINS_TRUNC,
318 INTRINS_TRUNCF,
319 INTRINS_COPYSIGN,
320 INTRINS_COPYSIGNF,
321 INTRINS_EXPECT_I8,
322 INTRINS_EXPECT_I1,
323 INTRINS_CTPOP_I32,
324 INTRINS_CTPOP_I64,
325 INTRINS_CTLZ_I32,
326 INTRINS_CTLZ_I64,
327 INTRINS_CTTZ_I32,
328 INTRINS_CTTZ_I64,
329 INTRINS_PEXT_I32,
330 INTRINS_PEXT_I64,
331 INTRINS_PDEP_I32,
332 INTRINS_PDEP_I64,
333 INTRINS_BZHI_I32,
334 INTRINS_BZHI_I64,
335 INTRINS_BEXTR_I32,
336 INTRINS_BEXTR_I64,
337 #if defined(TARGET_AMD64) || defined(TARGET_X86)
338 INTRINS_SSE_PMOVMSKB,
339 INTRINS_SSE_PSRLI_W,
340 INTRINS_SSE_PSRAI_W,
341 INTRINS_SSE_PSLLI_W,
342 INTRINS_SSE_PSRLI_D,
343 INTRINS_SSE_PSRAI_D,
344 INTRINS_SSE_PSLLI_D,
345 INTRINS_SSE_PSRLI_Q,
346 INTRINS_SSE_PSLLI_Q,
347 INTRINS_SSE_SQRT_PD,
348 INTRINS_SSE_SQRT_PS,
349 INTRINS_SSE_RSQRT_PS,
350 INTRINS_SSE_RCP_PS,
351 INTRINS_SSE_CVTTPD2DQ,
352 INTRINS_SSE_CVTTPS2DQ,
353 INTRINS_SSE_CVTDQ2PD,
354 INTRINS_SSE_CVTDQ2PS,
355 INTRINS_SSE_CVTPD2DQ,
356 INTRINS_SSE_CVTPS2DQ,
357 INTRINS_SSE_CVTPD2PS,
358 INTRINS_SSE_CVTPS2PD,
359 INTRINS_SSE_CMPPD,
360 INTRINS_SSE_CMPPS,
361 INTRINS_SSE_PACKSSWB,
362 INTRINS_SSE_PACKUSWB,
363 INTRINS_SSE_PACKSSDW,
364 INTRINS_SSE_PACKUSDW,
365 INTRINS_SSE_MINPS,
366 INTRINS_SSE_MAXPS,
367 INTRINS_SSE_HADDPS,
368 INTRINS_SSE_HSUBPS,
369 INTRINS_SSE_ADDSUBPS,
370 INTRINS_SSE_MINPD,
371 INTRINS_SSE_MAXPD,
372 INTRINS_SSE_HADDPD,
373 INTRINS_SSE_HSUBPD,
374 INTRINS_SSE_ADDSUBPD,
375 INTRINS_SSE_PADDSW,
376 INTRINS_SSE_PSUBSW,
377 INTRINS_SSE_PADDUSW,
378 INTRINS_SSE_PSUBUSW,
379 INTRINS_SSE_PAVGW,
380 INTRINS_SSE_PMULHW,
381 INTRINS_SSE_PMULHU,
382 INTRINS_SE_PADDSB,
383 INTRINS_SSE_PSUBSB,
384 INTRINS_SSE_PADDUSB,
385 INTRINS_SSE_PSUBUSB,
386 INTRINS_SSE_PAVGB,
387 INTRINS_SSE_PAUSE,
388 INTRINS_SSE_DPPS,
389 INTRINS_SSE_ROUNDSS,
390 INTRINS_SSE_ROUNDPD,
391 #endif
392 #ifdef TARGET_WASM
393 INTRINS_WASM_ANYTRUE_V16,
394 INTRINS_WASM_ANYTRUE_V8,
395 INTRINS_WASM_ANYTRUE_V4,
396 INTRINS_WASM_ANYTRUE_V2,
397 #endif
398 INTRINS_NUM
399 } IntrinsicId;
400
401 static MonoNativeTlsKey current_cfg_tls_id;
402
403 static MonoLLVMModule aot_module;
404
405 static GHashTable *intrins_id_to_name;
406 static GHashTable *intrins_name_to_id;
407
408 static void init_jit_module (MonoDomain *domain);
409
410 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
411 static void emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder);
412 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
413 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
414 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp, gboolean force_explicit);
415 static LLVMValueRef get_intrins_by_name (EmitContext *ctx, const char *name);
416 static LLVMValueRef get_intrins (EmitContext *ctx, int id);
417 static LLVMValueRef get_intrins_from_module (LLVMModuleRef lmodule, int id);
418 static void llvm_jit_finalize_method (EmitContext *ctx);
419 static void mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params);
420 static void mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module);
421
422 static inline void
423 set_failure (EmitContext *ctx, const char *message)
424 {
425 TRACE_FAILURE (reason);
426 ctx->cfg->exception_message = g_strdup (message);
427 ctx->cfg->disable_llvm = TRUE;
428 }
429
430 static LLVMValueRef
431 ConstInt32 (int v)
432 {
433 return LLVMConstInt (LLVMInt32Type (), v, FALSE);
434 }
435
436 /*
437 * IntPtrType:
438 *
439 * The LLVM type with width == TARGET_SIZEOF_VOID_P
440 */
441 static LLVMTypeRef
442 IntPtrType (void)
443 {
444 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
445 }
446
447 static LLVMTypeRef
448 ObjRefType (void)
449 {
450 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
451 }
452
453 static LLVMTypeRef
454 ThisType (void)
455 {
456 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
457 }
458
459 /*
460 * get_vtype_size:
461 *
462 * Return the size of the LLVM representation of the vtype T.
463 */
464 static guint32
465 get_vtype_size (MonoType *t)
466 {
467 int size;
468
469 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
470
471 /* LLVMArgAsIArgs depends on this since it stores whole words */
472 while (size < 2 * TARGET_SIZEOF_VOID_P && mono_is_power_of_two (size) == -1)
473 size ++;
474
475 return size;
476 }
477
478 /*
479 * simd_class_to_llvm_type:
480 *
481 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
482 */
483 static LLVMTypeRef
484 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
485 {
486 const char *klass_name = m_class_get_name (klass);
487 if (!strcmp (klass_name, "Vector2d")) {
488 return LLVMVectorType (LLVMDoubleType (), 2);
489 } else if (!strcmp (klass_name, "Vector2l")) {
490 return LLVMVectorType (LLVMInt64Type (), 2);
491 } else if (!strcmp (klass_name, "Vector2ul")) {
492 return LLVMVectorType (LLVMInt64Type (), 2);
493 } else if (!strcmp (klass_name, "Vector4i")) {
494 return LLVMVectorType (LLVMInt32Type (), 4);
495 } else if (!strcmp (klass_name, "Vector4ui")) {
496 return LLVMVectorType (LLVMInt32Type (), 4);
497 } else if (!strcmp (klass_name, "Vector4f")) {
498 return LLVMVectorType (LLVMFloatType (), 4);
499 } else if (!strcmp (klass_name, "Vector8s")) {
500 return LLVMVectorType (LLVMInt16Type (), 8);
501 } else if (!strcmp (klass_name, "Vector8us")) {
502 return LLVMVectorType (LLVMInt16Type (), 8);
503 } else if (!strcmp (klass_name, "Vector16sb")) {
504 return LLVMVectorType (LLVMInt8Type (), 16);
505 } else if (!strcmp (klass_name, "Vector16b")) {
506 return LLVMVectorType (LLVMInt8Type (), 16);
507 } else if (!strcmp (klass_name, "Vector2")) {
508 /* System.Numerics */
509 return LLVMVectorType (LLVMFloatType (), 4);
510 } else if (!strcmp (klass_name, "Vector3")) {
511 return LLVMVectorType (LLVMFloatType (), 4);
512 } else if (!strcmp (klass_name, "Vector4")) {
513 return LLVMVectorType (LLVMFloatType (), 4);
514 } else if (!strcmp (klass_name, "Vector`1") || !strcmp (klass_name, "Vector128`1") || !strcmp (klass_name, "Vector256`1")) {
515 MonoType *etype = mono_class_get_generic_class (klass)->context.class_inst->type_argv [0];
516 int size = mono_class_value_size (klass, NULL);
517 switch (etype->type) {
518 case MONO_TYPE_I1:
519 case MONO_TYPE_U1:
520 return LLVMVectorType (LLVMInt8Type (), size);
521 case MONO_TYPE_I2:
522 case MONO_TYPE_U2:
523 return LLVMVectorType (LLVMInt16Type (), size / 2);
524 case MONO_TYPE_I4:
525 case MONO_TYPE_U4:
526 return LLVMVectorType (LLVMInt32Type (), size / 4);
527 case MONO_TYPE_I8:
528 case MONO_TYPE_U8:
529 return LLVMVectorType (LLVMInt64Type (), size / 8);
530 case MONO_TYPE_R4:
531 return LLVMVectorType (LLVMFloatType (), size / 4);
532 case MONO_TYPE_R8:
533 return LLVMVectorType (LLVMDoubleType (), size / 8);
534 default:
535 g_assert_not_reached ();
536 return NULL;
537 }
538 } else {
539 printf ("%s\n", klass_name);
540 NOT_IMPLEMENTED;
541 return NULL;
542 }
543 }
544
545 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
546 static inline G_GNUC_UNUSED LLVMTypeRef
547 type_to_simd_type (int type)
548 {
549 switch (type) {
550 case MONO_TYPE_I1:
551 return LLVMVectorType (LLVMInt8Type (), 16);
552 case MONO_TYPE_I2:
553 return LLVMVectorType (LLVMInt16Type (), 8);
554 case MONO_TYPE_I4:
555 return LLVMVectorType (LLVMInt32Type (), 4);
556 case MONO_TYPE_I8:
557 return LLVMVectorType (LLVMInt64Type (), 2);
558 case MONO_TYPE_R8:
559 return LLVMVectorType (LLVMDoubleType (), 2);
560 case MONO_TYPE_R4:
561 return LLVMVectorType (LLVMFloatType (), 4);
562 default:
563 g_assert_not_reached ();
564 return NULL;
565 }
566 }
567
568 static LLVMTypeRef
569 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
570 {
571 int i, size, nfields, esize;
572 LLVMTypeRef *eltypes;
573 char *name;
574 MonoType *t;
575 LLVMTypeRef ltype;
576
577 t = m_class_get_byval_arg (klass);
578
579 if (mini_type_is_hfa (t, &nfields, &esize)) {
580 /*
581 * This is needed on arm64 where HFAs are returned in
582 * registers.
583 */
584 /* SIMD types have size 16 in mono_class_value_size () */
585 if (m_class_is_simd_type (klass))
586 nfields = 16/ esize;
587 size = nfields;
588 eltypes = g_new (LLVMTypeRef, size);
589 for (i = 0; i < size; ++i)
590 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
591 } else {
592 size = get_vtype_size (t);
593
594 eltypes = g_new (LLVMTypeRef, size);
595 for (i = 0; i < size; ++i)
596 eltypes [i] = LLVMInt8Type ();
597 }
598
599 name = mono_type_full_name (m_class_get_byval_arg (klass));
600 ltype = LLVMStructCreateNamed (module->context, name);
601 LLVMStructSetBody (ltype, eltypes, size, FALSE);
602 g_free (eltypes);
603 g_free (name);
604
605 return ltype;
606 }
607
608 /*
609 * type_to_llvm_type:
610 *
611 * Return the LLVM type corresponding to T.
612 */
613 static LLVMTypeRef
614 type_to_llvm_type (EmitContext *ctx, MonoType *t)
615 {
616 if (t->byref)
617 return ThisType ();
618
619 t = mini_get_underlying_type (t);
620
621 switch (t->type) {
622 case MONO_TYPE_VOID:
623 return LLVMVoidType ();
624 case MONO_TYPE_I1:
625 return LLVMInt8Type ();
626 case MONO_TYPE_I2:
627 return LLVMInt16Type ();
628 case MONO_TYPE_I4:
629 return LLVMInt32Type ();
630 case MONO_TYPE_U1:
631 return LLVMInt8Type ();
632 case MONO_TYPE_U2:
633 return LLVMInt16Type ();
634 case MONO_TYPE_U4:
635 return LLVMInt32Type ();
636 case MONO_TYPE_I8:
637 case MONO_TYPE_U8:
638 return LLVMInt64Type ();
639 case MONO_TYPE_R4:
640 return LLVMFloatType ();
641 case MONO_TYPE_R8:
642 return LLVMDoubleType ();
643 case MONO_TYPE_I:
644 case MONO_TYPE_U:
645 return IntPtrType ();
646 case MONO_TYPE_OBJECT:
647 return ObjRefType ();
648 case MONO_TYPE_PTR: {
649 MonoClass *klass = mono_class_from_mono_type_internal (t);
650 MonoClass *ptr_klass = m_class_get_element_class (klass);
651 MonoType *ptr_type = m_class_get_byval_arg (ptr_klass);
652 /* Handle primitive pointers */
653 switch (ptr_type->type) {
654 case MONO_TYPE_I1:
655 case MONO_TYPE_I2:
656 case MONO_TYPE_I4:
657 case MONO_TYPE_U1:
658 case MONO_TYPE_U2:
659 case MONO_TYPE_U4:
660 return LLVMPointerType (type_to_llvm_type (ctx, ptr_type), 0);
661 }
662
663 return ObjRefType ();
664 }
665 case MONO_TYPE_VAR:
666 case MONO_TYPE_MVAR:
667 /* Because of generic sharing */
668 return ObjRefType ();
669 case MONO_TYPE_GENERICINST:
670 if (!mono_type_generic_inst_is_valuetype (t))
671 return ObjRefType ();
672 /* Fall through */
673 case MONO_TYPE_VALUETYPE:
674 case MONO_TYPE_TYPEDBYREF: {
675 MonoClass *klass;
676 LLVMTypeRef ltype;
677
678 klass = mono_class_from_mono_type_internal (t);
679
680 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
681 return simd_class_to_llvm_type (ctx, klass);
682
683 if (m_class_is_enumtype (klass))
684 return type_to_llvm_type (ctx, mono_class_enum_basetype_internal (klass));
685
686 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
687 if (!ltype) {
688 ltype = create_llvm_type_for_type (ctx->module, klass);
689 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
690 }
691 return ltype;
692 }
693
694 default:
695 printf ("X: %d\n", t->type);
696 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
697 ctx->cfg->disable_llvm = TRUE;
698 return NULL;
699 }
700 }
701
702 /*
703 * type_is_unsigned:
704 *
705 * Return whenever T is an unsigned int type.
706 */
707 static gboolean
708 type_is_unsigned (EmitContext *ctx, MonoType *t)
709 {
710 t = mini_get_underlying_type (t);
711 if (t->byref)
712 return FALSE;
713 switch (t->type) {
714 case MONO_TYPE_U1:
715 case MONO_TYPE_U2:
716 case MONO_TYPE_CHAR:
717 case MONO_TYPE_U4:
718 case MONO_TYPE_U8:
719 return TRUE;
720 default:
721 return FALSE;
722 }
723 }
724
725 /*
726 * type_to_llvm_arg_type:
727 *
728 * Same as type_to_llvm_type, but treat i8/i16 as i32.
729 */
730 static LLVMTypeRef
731 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
732 {
733 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
734
735 if (ctx->cfg->llvm_only)
736 return ptype;
737
738 /*
739 * This works on all abis except arm64/ios which passes multiple
740 * arguments in one stack slot.
741 */
742 #ifndef TARGET_ARM64
743 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
744 /*
745 * LLVM generates code which only sets the lower bits, while JITted
746 * code expects all the bits to be set.
747 */
748 ptype = LLVMInt32Type ();
749 }
750 #endif
751
752 return ptype;
753 }
754
755 /*
756 * llvm_type_to_stack_type:
757 *
758 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
759 * on the IL stack.
760 */
761 static G_GNUC_UNUSED LLVMTypeRef
762 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
763 {
764 if (type == NULL)
765 return NULL;
766 if (type == LLVMInt8Type ())
767 return LLVMInt32Type ();
768 else if (type == LLVMInt16Type ())
769 return LLVMInt32Type ();
770 else if (!cfg->r4fp && type == LLVMFloatType ())
771 return LLVMDoubleType ();
772 else
773 return type;
774 }
775
776 /*
777 * regtype_to_llvm_type:
778 *
779 * Return the LLVM type corresponding to the regtype C used in instruction
780 * descriptions.
781 */
782 static LLVMTypeRef
783 regtype_to_llvm_type (char c)
784 {
785 switch (c) {
786 case 'i':
787 return LLVMInt32Type ();
788 case 'l':
789 return LLVMInt64Type ();
790 case 'f':
791 return LLVMDoubleType ();
792 default:
793 return NULL;
794 }
795 }
796
797 /*
798 * op_to_llvm_type:
799 *
800 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
801 */
802 static LLVMTypeRef
803 op_to_llvm_type (int opcode)
804 {
805 switch (opcode) {
806 case OP_ICONV_TO_I1:
807 case OP_LCONV_TO_I1:
808 return LLVMInt8Type ();
809 case OP_ICONV_TO_U1:
810 case OP_LCONV_TO_U1:
811 return LLVMInt8Type ();
812 case OP_ICONV_TO_I2:
813 case OP_LCONV_TO_I2:
814 return LLVMInt16Type ();
815 case OP_ICONV_TO_U2:
816 case OP_LCONV_TO_U2:
817 return LLVMInt16Type ();
818 case OP_ICONV_TO_I4:
819 case OP_LCONV_TO_I4:
820 return LLVMInt32Type ();
821 case OP_ICONV_TO_U4:
822 case OP_LCONV_TO_U4:
823 return LLVMInt32Type ();
824 case OP_ICONV_TO_I8:
825 return LLVMInt64Type ();
826 case OP_ICONV_TO_R4:
827 return LLVMFloatType ();
828 case OP_ICONV_TO_R8:
829 return LLVMDoubleType ();
830 case OP_ICONV_TO_U8:
831 return LLVMInt64Type ();
832 case OP_FCONV_TO_I4:
833 return LLVMInt32Type ();
834 case OP_FCONV_TO_I8:
835 return LLVMInt64Type ();
836 case OP_FCONV_TO_I1:
837 case OP_FCONV_TO_U1:
838 case OP_RCONV_TO_I1:
839 case OP_RCONV_TO_U1:
840 return LLVMInt8Type ();
841 case OP_FCONV_TO_I2:
842 case OP_FCONV_TO_U2:
843 case OP_RCONV_TO_I2:
844 case OP_RCONV_TO_U2:
845 return LLVMInt16Type ();
846 case OP_FCONV_TO_U4:
847 case OP_RCONV_TO_U4:
848 return LLVMInt32Type ();
849 case OP_FCONV_TO_U8:
850 case OP_RCONV_TO_U8:
851 return LLVMInt64Type ();
852 case OP_FCONV_TO_I:
853 case OP_FCONV_TO_U:
854 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
855 case OP_IADD_OVF:
856 case OP_IADD_OVF_UN:
857 case OP_ISUB_OVF:
858 case OP_ISUB_OVF_UN:
859 case OP_IMUL_OVF:
860 case OP_IMUL_OVF_UN:
861 return LLVMInt32Type ();
862 case OP_LADD_OVF:
863 case OP_LADD_OVF_UN:
864 case OP_LSUB_OVF:
865 case OP_LSUB_OVF_UN:
866 case OP_LMUL_OVF:
867 case OP_LMUL_OVF_UN:
868 return LLVMInt64Type ();
869 default:
870 printf ("%s\n", mono_inst_name (opcode));
871 g_assert_not_reached ();
872 return NULL;
873 }
874 }
875
876 #define CLAUSE_START(clause) ((clause)->try_offset)
877 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
878
879 /*
880 * load_store_to_llvm_type:
881 *
882 * Return the size/sign/zero extension corresponding to the load/store opcode
883 * OPCODE.
884 */
885 static LLVMTypeRef
886 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
887 {
888 *sext = FALSE;
889 *zext = FALSE;
890
891 switch (opcode) {
892 case OP_LOADI1_MEMBASE:
893 case OP_STOREI1_MEMBASE_REG:
894 case OP_STOREI1_MEMBASE_IMM:
895 case OP_ATOMIC_LOAD_I1:
896 case OP_ATOMIC_STORE_I1:
897 *size = 1;
898 *sext = TRUE;
899 return LLVMInt8Type ();
900 case OP_LOADU1_MEMBASE:
901 case OP_LOADU1_MEM:
902 case OP_ATOMIC_LOAD_U1:
903 case OP_ATOMIC_STORE_U1:
904 *size = 1;
905 *zext = TRUE;
906 return LLVMInt8Type ();
907 case OP_LOADI2_MEMBASE:
908 case OP_STOREI2_MEMBASE_REG:
909 case OP_STOREI2_MEMBASE_IMM:
910 case OP_ATOMIC_LOAD_I2:
911 case OP_ATOMIC_STORE_I2:
912 *size = 2;
913 *sext = TRUE;
914 return LLVMInt16Type ();
915 case OP_LOADU2_MEMBASE:
916 case OP_LOADU2_MEM:
917 case OP_ATOMIC_LOAD_U2:
918 case OP_ATOMIC_STORE_U2:
919 *size = 2;
920 *zext = TRUE;
921 return LLVMInt16Type ();
922 case OP_LOADI4_MEMBASE:
923 case OP_LOADU4_MEMBASE:
924 case OP_LOADI4_MEM:
925 case OP_LOADU4_MEM:
926 case OP_STOREI4_MEMBASE_REG:
927 case OP_STOREI4_MEMBASE_IMM:
928 case OP_ATOMIC_LOAD_I4:
929 case OP_ATOMIC_STORE_I4:
930 case OP_ATOMIC_LOAD_U4:
931 case OP_ATOMIC_STORE_U4:
932 *size = 4;
933 return LLVMInt32Type ();
934 case OP_LOADI8_MEMBASE:
935 case OP_LOADI8_MEM:
936 case OP_STOREI8_MEMBASE_REG:
937 case OP_STOREI8_MEMBASE_IMM:
938 case OP_ATOMIC_LOAD_I8:
939 case OP_ATOMIC_STORE_I8:
940 case OP_ATOMIC_LOAD_U8:
941 case OP_ATOMIC_STORE_U8:
942 *size = 8;
943 return LLVMInt64Type ();
944 case OP_LOADR4_MEMBASE:
945 case OP_STORER4_MEMBASE_REG:
946 case OP_ATOMIC_LOAD_R4:
947 case OP_ATOMIC_STORE_R4:
948 *size = 4;
949 return LLVMFloatType ();
950 case OP_LOADR8_MEMBASE:
951 case OP_STORER8_MEMBASE_REG:
952 case OP_ATOMIC_LOAD_R8:
953 case OP_ATOMIC_STORE_R8:
954 *size = 8;
955 return LLVMDoubleType ();
956 case OP_LOAD_MEMBASE:
957 case OP_LOAD_MEM:
958 case OP_STORE_MEMBASE_REG:
959 case OP_STORE_MEMBASE_IMM:
960 *size = TARGET_SIZEOF_VOID_P;
961 return IntPtrType ();
962 default:
963 g_assert_not_reached ();
964 return NULL;
965 }
966 }
967
968 /*
969 * ovf_op_to_intrins:
970 *
971 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
972 */
973 static const char*
974 ovf_op_to_intrins (int opcode)
975 {
976 switch (opcode) {
977 case OP_IADD_OVF:
978 return "llvm.sadd.with.overflow.i32";
979 case OP_IADD_OVF_UN:
980 return "llvm.uadd.with.overflow.i32";
981 case OP_ISUB_OVF:
982 return "llvm.ssub.with.overflow.i32";
983 case OP_ISUB_OVF_UN:
984 return "llvm.usub.with.overflow.i32";
985 case OP_IMUL_OVF:
986 return "llvm.smul.with.overflow.i32";
987 case OP_IMUL_OVF_UN:
988 return "llvm.umul.with.overflow.i32";
989 case OP_LADD_OVF:
990 return "llvm.sadd.with.overflow.i64";
991 case OP_LADD_OVF_UN:
992 return "llvm.uadd.with.overflow.i64";
993 case OP_LSUB_OVF:
994 return "llvm.ssub.with.overflow.i64";
995 case OP_LSUB_OVF_UN:
996 return "llvm.usub.with.overflow.i64";
997 case OP_LMUL_OVF:
998 return "llvm.smul.with.overflow.i64";
999 case OP_LMUL_OVF_UN:
1000 return "llvm.umul.with.overflow.i64";
1001 default:
1002 g_assert_not_reached ();
1003 return NULL;
1004 }
1005 }
1006
1007 static const char*
1008 simd_op_to_intrins (int opcode)
1009 {
1010 switch (opcode) {
1011 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1012 case OP_MINPD:
1013 return "llvm.x86.sse2.min.pd";
1014 case OP_MINPS:
1015 return "llvm.x86.sse.min.ps";
1016 case OP_MAXPD:
1017 return "llvm.x86.sse2.max.pd";
1018 case OP_MAXPS:
1019 return "llvm.x86.sse.max.ps";
1020 case OP_HADDPD:
1021 return "llvm.x86.sse3.hadd.pd";
1022 case OP_HADDPS:
1023 return "llvm.x86.sse3.hadd.ps";
1024 case OP_HSUBPD:
1025 return "llvm.x86.sse3.hsub.pd";
1026 case OP_HSUBPS:
1027 return "llvm.x86.sse3.hsub.ps";
1028 case OP_ADDSUBPS:
1029 return "llvm.x86.sse3.addsub.ps";
1030 case OP_ADDSUBPD:
1031 return "llvm.x86.sse3.addsub.pd";
1032 case OP_EXTRACT_MASK:
1033 return "llvm.x86.sse2.pmovmskb.128";
1034 case OP_PSHRW:
1035 case OP_PSHRW_REG:
1036 return "llvm.x86.sse2.psrli.w";
1037 case OP_PSHRD:
1038 case OP_PSHRD_REG:
1039 return "llvm.x86.sse2.psrli.d";
1040 case OP_PSHRQ:
1041 case OP_PSHRQ_REG:
1042 return "llvm.x86.sse2.psrli.q";
1043 case OP_PSHLW:
1044 case OP_PSHLW_REG:
1045 return "llvm.x86.sse2.pslli.w";
1046 case OP_PSHLD:
1047 case OP_PSHLD_REG:
1048 return "llvm.x86.sse2.pslli.d";
1049 case OP_PSHLQ:
1050 case OP_PSHLQ_REG:
1051 return "llvm.x86.sse2.pslli.q";
1052 case OP_PSARW:
1053 case OP_PSARW_REG:
1054 return "llvm.x86.sse2.psrai.w";
1055 case OP_PSARD:
1056 case OP_PSARD_REG:
1057 return "llvm.x86.sse2.psrai.d";
1058 case OP_PADDB_SAT:
1059 return "llvm.x86.sse2.padds.b";
1060 case OP_PADDW_SAT:
1061 return "llvm.x86.sse2.padds.w";
1062 case OP_PSUBB_SAT:
1063 return "llvm.x86.sse2.psubs.b";
1064 case OP_PSUBW_SAT:
1065 return "llvm.x86.sse2.psubs.w";
1066 case OP_PADDB_SAT_UN:
1067 return "llvm.x86.sse2.paddus.b";
1068 case OP_PADDW_SAT_UN:
1069 return "llvm.x86.sse2.paddus.w";
1070 case OP_PSUBB_SAT_UN:
1071 return "llvm.x86.sse2.psubus.b";
1072 case OP_PSUBW_SAT_UN:
1073 return "llvm.x86.sse2.psubus.w";
1074 case OP_PAVGB_UN:
1075 return "llvm.x86.sse2.pavg.b";
1076 case OP_PAVGW_UN:
1077 return "llvm.x86.sse2.pavg.w";
1078 case OP_SQRTPS:
1079 return "llvm.x86.sse.sqrt.ps";
1080 case OP_SQRTPD:
1081 return "llvm.x86.sse2.sqrt.pd";
1082 case OP_RSQRTPS:
1083 return "llvm.x86.sse.rsqrt.ps";
1084 case OP_RCPPS:
1085 return "llvm.x86.sse.rcp.ps";
1086 case OP_CVTDQ2PD:
1087 return "llvm.x86.sse2.cvtdq2pd";
1088 case OP_CVTDQ2PS:
1089 return "llvm.x86.sse2.cvtdq2ps";
1090 case OP_CVTPD2DQ:
1091 return "llvm.x86.sse2.cvtpd2dq";
1092 case OP_CVTPS2DQ:
1093 return "llvm.x86.sse2.cvtps2dq";
1094 case OP_CVTPD2PS:
1095 return "llvm.x86.sse2.cvtpd2ps";
1096 case OP_CVTPS2PD:
1097 return "llvm.x86.sse2.cvtps2pd";
1098 case OP_CVTTPD2DQ:
1099 return "llvm.x86.sse2.cvttpd2dq";
1100 case OP_CVTTPS2DQ:
1101 return "llvm.x86.sse2.cvttps2dq";
1102 case OP_PACKW:
1103 return "llvm.x86.sse2.packsswb.128";
1104 case OP_PACKD:
1105 return "llvm.x86.sse2.packssdw.128";
1106 case OP_PACKW_UN:
1107 return "llvm.x86.sse2.packuswb.128";
1108 case OP_PACKD_UN:
1109 return "llvm.x86.sse41.packusdw";
1110 case OP_PMULW_HIGH:
1111 return "llvm.x86.sse2.pmulh.w";
1112 case OP_PMULW_HIGH_UN:
1113 return "llvm.x86.sse2.pmulhu.w";
1114 case OP_DPPS:
1115 return "llvm.x86.sse41.dpps";
1116 #endif
1117 default:
1118 g_assert_not_reached ();
1119 return NULL;
1120 }
1121 }
1122
1123 static LLVMTypeRef
1124 simd_op_to_llvm_type (int opcode)
1125 {
1126 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1127 switch (opcode) {
1128 case OP_EXTRACT_R8:
1129 case OP_EXPAND_R8:
1130 return type_to_simd_type (MONO_TYPE_R8);
1131 case OP_EXTRACT_I8:
1132 case OP_EXPAND_I8:
1133 return type_to_simd_type (MONO_TYPE_I8);
1134 case OP_EXTRACT_I4:
1135 case OP_EXPAND_I4:
1136 return type_to_simd_type (MONO_TYPE_I4);
1137 case OP_EXTRACT_I2:
1138 case OP_EXTRACT_U2:
1139 case OP_EXTRACTX_U2:
1140 case OP_EXPAND_I2:
1141 return type_to_simd_type (MONO_TYPE_I2);
1142 case OP_EXTRACT_I1:
1143 case OP_EXTRACT_U1:
1144 case OP_EXPAND_I1:
1145 return type_to_simd_type (MONO_TYPE_I1);
1146 case OP_EXTRACT_R4:
1147 case OP_EXPAND_R4:
1148 return type_to_simd_type (MONO_TYPE_R4);
1149 case OP_CVTDQ2PD:
1150 case OP_CVTDQ2PS:
1151 return type_to_simd_type (MONO_TYPE_I4);
1152 case OP_CVTPD2DQ:
1153 case OP_CVTPD2PS:
1154 case OP_CVTTPD2DQ:
1155 return type_to_simd_type (MONO_TYPE_R8);
1156 case OP_CVTPS2DQ:
1157 case OP_CVTPS2PD:
1158 case OP_CVTTPS2DQ:
1159 return type_to_simd_type (MONO_TYPE_R4);
1160 case OP_EXTRACT_MASK:
1161 return type_to_simd_type (MONO_TYPE_I1);
1162 case OP_SQRTPS:
1163 case OP_RSQRTPS:
1164 case OP_RCPPS:
1165 case OP_DUPPS_LOW:
1166 case OP_DUPPS_HIGH:
1167 return type_to_simd_type (MONO_TYPE_R4);
1168 case OP_SQRTPD:
1169 case OP_DUPPD:
1170 return type_to_simd_type (MONO_TYPE_R8);
1171 default:
1172 g_assert_not_reached ();
1173 return NULL;
1174 }
1175 #else
1176 return NULL;
1177 #endif
1178 }
1179
1180 static void
1181 set_cold_cconv (LLVMValueRef func)
1182 {
1183 /*
1184 * xcode10 (watchOS) and ARM/ARM64 doesn't seem to support preserveall, it fails with:
1185 * fatal error: error in backend: Unsupported calling convention
1186 */
1187 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1188 LLVMSetFunctionCallConv (func, LLVMColdCallConv);
1189 #endif
1190 }
1191
1192 static void
1193 set_call_cold_cconv (LLVMValueRef func)
1194 {
1195 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1196 LLVMSetInstructionCallConv (func, LLVMColdCallConv);
1197 #endif
1198 }
1199
1200 /*
1201 * get_bb:
1202 *
1203 * Return the LLVM basic block corresponding to BB.
1204 */
1205 static LLVMBasicBlockRef
1206 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
1207 {
1208 char bb_name_buf [128];
1209 char *bb_name;
1210
1211 if (ctx->bblocks [bb->block_num].bblock == NULL) {
1212 if (bb->flags & BB_EXCEPTION_HANDLER) {
1213 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
1214 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
1215 bb_name = bb_name_buf;
1216 } else if (bb->block_num < 256) {
1217 if (!ctx->module->bb_names) {
1218 ctx->module->bb_names_len = 256;
1219 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
1220 }
1221 if (!ctx->module->bb_names [bb->block_num]) {
1222 char *n;
1223
1224 n = g_strdup_printf ("BB%d", bb->block_num);
1225 mono_memory_barrier ();
1226 ctx->module->bb_names [bb->block_num] = n;
1227 }
1228 bb_name = ctx->module->bb_names [bb->block_num];
1229 } else {
1230 sprintf (bb_name_buf, "BB%d", bb->block_num);
1231 bb_name = bb_name_buf;
1232 }
1233
1234 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1235 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1236 }
1237
1238 return ctx->bblocks [bb->block_num].bblock;
1239 }
1240
1241 /*
1242 * get_end_bb:
1243 *
1244 * Return the last LLVM bblock corresponding to BB.
1245 * This might not be equal to the bb returned by get_bb () since we need to generate
1246 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1247 */
1248 static LLVMBasicBlockRef
1249 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1250 {
1251 get_bb (ctx, bb);
1252 return ctx->bblocks [bb->block_num].end_bblock;
1253 }
1254
1255 static LLVMBasicBlockRef
1256 gen_bb (EmitContext *ctx, const char *prefix)
1257 {
1258 char bb_name [128];
1259
1260 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1261 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1262 }
1263
1264 /*
1265 * resolve_patch:
1266 *
1267 * Return the target of the patch identified by TYPE and TARGET.
1268 */
1269 static gpointer
1270 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1271 {
1272 MonoJumpInfo ji;
1273 ERROR_DECL (error);
1274 gpointer res;
1275
1276 memset (&ji, 0, sizeof (ji));
1277 ji.type = type;
1278 ji.data.target = target;
1279
1280 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, error);
1281 mono_error_assert_ok (error);
1282
1283 return res;
1284 }
1285
1286 /*
1287 * convert_full:
1288 *
1289 * Emit code to convert the LLVM value V to DTYPE.
1290 */
1291 static LLVMValueRef
1292 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1293 {
1294 LLVMTypeRef stype = LLVMTypeOf (v);
1295
1296 if (stype != dtype) {
1297 gboolean ext = FALSE;
1298
1299 /* Extend */
1300 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1301 ext = TRUE;
1302 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1303 ext = TRUE;
1304 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1305 ext = TRUE;
1306
1307 if (ext)
1308 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1309
1310 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1311 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1312
1313 /* Trunc */
1314 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1315 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1316 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1317 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1318 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1319 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1320 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1321 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1322
1323 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1324 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1325 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1326 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1327 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1328 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1329
1330 if (mono_arch_is_soft_float ()) {
1331 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1332 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1333 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1334 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1335 }
1336
1337 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1338 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1339
1340 mono_llvm_dump_value (v);
1341 mono_llvm_dump_value (LLVMConstNull (dtype));
1342 printf ("\n");
1343 g_assert_not_reached ();
1344 return NULL;
1345 } else {
1346 return v;
1347 }
1348 }
1349
1350 static LLVMValueRef
1351 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1352 {
1353 return convert_full (ctx, v, dtype, FALSE);
1354 }
1355
1356 static void
1357 emit_memset (EmitContext *ctx, LLVMBuilderRef builder, LLVMValueRef v, LLVMValueRef size, int alignment)
1358 {
1359 LLVMValueRef args [5];
1360 int aindex = 0;
1361
1362 args [aindex ++] = v;
1363 args [aindex ++] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
1364 args [aindex ++] = size;
1365 #if LLVM_API_VERSION < 900
1366 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), alignment, FALSE);
1367 #endif
1368 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
1369 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMSET), args, aindex, "");
1370 }
1371
1372 /*
1373 * emit_volatile_load:
1374 *
1375 * If vreg is volatile, emit a load from its address.
1376 */
1377 static LLVMValueRef
1378 emit_volatile_load (EmitContext *ctx, int vreg)
1379 {
1380 MonoType *t;
1381 LLVMValueRef v;
1382
1383 // On arm64, we pass the rgctx in a callee saved
1384 // register on arm64 (x15), and llvm might keep the value in that register
1385 // even through the register is marked as 'reserved' inside llvm.
1386
1387 v = mono_llvm_build_load (ctx->builder, ctx->addresses [vreg], "", TRUE);
1388 t = ctx->vreg_cli_types [vreg];
1389 if (t && !t->byref) {
1390 /*
1391 * Might have to zero extend since llvm doesn't have
1392 * unsigned types.
1393 */
1394 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1395 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1396 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1397 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1398 else if (t->type == MONO_TYPE_U8)
1399 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1400 }
1401
1402 return v;
1403 }
1404
1405 /*
1406 * emit_volatile_store:
1407 *
1408 * If VREG is volatile, emit a store from its value to its address.
1409 */
1410 static void
1411 emit_volatile_store (EmitContext *ctx, int vreg)
1412 {
1413 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1414
1415 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1416 g_assert (ctx->addresses [vreg]);
1417 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1418 }
1419 }
1420
1421 static LLVMTypeRef
1422 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1423 {
1424 LLVMTypeRef ret_type;
1425 LLVMTypeRef *param_types = NULL;
1426 LLVMTypeRef res;
1427 int i, pindex;
1428 MonoType *rtype;
1429
1430 ret_type = type_to_llvm_type (ctx, sig->ret);
1431 if (!ctx_ok (ctx))
1432 return NULL;
1433 rtype = mini_get_underlying_type (sig->ret);
1434
1435 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1436 pindex = 0;
1437
1438 if (sig->hasthis)
1439 param_types [pindex ++] = ThisType ();
1440 for (i = 0; i < sig->param_count; ++i)
1441 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1442
1443 if (!ctx_ok (ctx)) {
1444 g_free (param_types);
1445 return NULL;
1446 }
1447
1448 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1449 g_free (param_types);
1450
1451 return res;
1452 }
1453
1454 /*
1455 * sig_to_llvm_sig_full:
1456 *
1457 * Return the LLVM signature corresponding to the mono signature SIG using the
1458 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1459 */
1460 static LLVMTypeRef
1461 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1462 {
1463 LLVMTypeRef ret_type;
1464 LLVMTypeRef *param_types = NULL;
1465 LLVMTypeRef res;
1466 int i, j, pindex, vret_arg_pindex = 0;
1467 gboolean vretaddr = FALSE;
1468 MonoType *rtype;
1469
1470 if (!cinfo)
1471 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1472
1473 ret_type = type_to_llvm_type (ctx, sig->ret);
1474 if (!ctx_ok (ctx))
1475 return NULL;
1476 rtype = mini_get_underlying_type (sig->ret);
1477
1478 switch (cinfo->ret.storage) {
1479 case LLVMArgVtypeInReg:
1480 /* LLVM models this by returning an aggregate value */
1481 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1482 LLVMTypeRef members [2];
1483
1484 members [0] = IntPtrType ();
1485 ret_type = LLVMStructType (members, 1, FALSE);
1486 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1487 /* Empty struct */
1488 ret_type = LLVMVoidType ();
1489 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1490 LLVMTypeRef members [2];
1491
1492 members [0] = IntPtrType ();
1493 members [1] = IntPtrType ();
1494 ret_type = LLVMStructType (members, 2, FALSE);
1495 } else {
1496 g_assert_not_reached ();
1497 }
1498 break;
1499 case LLVMArgVtypeByVal:
1500 /* Vtype returned normally by val */
1501 break;
1502 case LLVMArgVtypeAsScalar: {
1503 int size = mono_class_value_size (mono_class_from_mono_type_internal (rtype), NULL);
1504 /* LLVM models this by returning an int */
1505 if (size < TARGET_SIZEOF_VOID_P) {
1506 g_assert (cinfo->ret.nslots == 1);
1507 ret_type = LLVMIntType (size * 8);
1508 } else {
1509 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1510 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (target_mgreg_t) * 8);
1511 }
1512 break;
1513 }
1514 case LLVMArgAsIArgs:
1515 ret_type = LLVMArrayType (IntPtrType (), cinfo->ret.nslots);
1516 break;
1517 case LLVMArgFpStruct: {
1518 /* Vtype returned as a fp struct */
1519 LLVMTypeRef members [16];
1520
1521 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1522 for (i = 0; i < cinfo->ret.nslots; ++i)
1523 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1524 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1525 break;
1526 }
1527 case LLVMArgVtypeByRef:
1528 /* Vtype returned using a hidden argument */
1529 ret_type = LLVMVoidType ();
1530 break;
1531 case LLVMArgVtypeRetAddr:
1532 case LLVMArgGsharedvtFixed:
1533 case LLVMArgGsharedvtFixedVtype:
1534 case LLVMArgGsharedvtVariable:
1535 vretaddr = TRUE;
1536 ret_type = LLVMVoidType ();
1537 break;
1538 default:
1539 break;
1540 }
1541
1542 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1543 pindex = 0;
1544 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1545 /*
1546 * Has to be the first argument because of the sret argument attribute
1547 * FIXME: This might conflict with passing 'this' as the first argument, but
1548 * this is only used on arm64 which has a dedicated struct return register.
1549 */
1550 cinfo->vret_arg_pindex = pindex;
1551 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1552 if (!ctx_ok (ctx)) {
1553 g_free (param_types);
1554 return NULL;
1555 }
1556 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1557 pindex ++;
1558 }
1559 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1560 cinfo->rgctx_arg_pindex = pindex;
1561 param_types [pindex] = ctx->module->ptr_type;
1562 pindex ++;
1563 }
1564 if (cinfo->imt_arg) {
1565 cinfo->imt_arg_pindex = pindex;
1566 param_types [pindex] = ctx->module->ptr_type;
1567 pindex ++;
1568 }
1569 if (vretaddr) {
1570 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1571 vret_arg_pindex = pindex;
1572 if (cinfo->vret_arg_index == 1) {
1573 /* Add the slots consumed by the first argument */
1574 LLVMArgInfo *ainfo = &cinfo->args [0];
1575 switch (ainfo->storage) {
1576 case LLVMArgVtypeInReg:
1577 for (j = 0; j < 2; ++j) {
1578 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1579 vret_arg_pindex ++;
1580 }
1581 break;
1582 default:
1583 vret_arg_pindex ++;
1584 }
1585 }
1586
1587 cinfo->vret_arg_pindex = vret_arg_pindex;
1588 }
1589
1590 if (vretaddr && vret_arg_pindex == pindex)
1591 param_types [pindex ++] = IntPtrType ();
1592 if (sig->hasthis) {
1593 cinfo->this_arg_pindex = pindex;
1594 param_types [pindex ++] = ThisType ();
1595 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1596 }
1597 if (vretaddr && vret_arg_pindex == pindex)
1598 param_types [pindex ++] = IntPtrType ();
1599 for (i = 0; i < sig->param_count; ++i) {
1600 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1601
1602 if (vretaddr && vret_arg_pindex == pindex)
1603 param_types [pindex ++] = IntPtrType ();
1604 ainfo->pindex = pindex;
1605
1606 switch (ainfo->storage) {
1607 case LLVMArgVtypeInReg:
1608 for (j = 0; j < 2; ++j) {
1609 switch (ainfo->pair_storage [j]) {
1610 case LLVMArgInIReg:
1611 param_types [pindex ++] = LLVMIntType (TARGET_SIZEOF_VOID_P * 8);
1612 break;
1613 case LLVMArgNone:
1614 break;
1615 default:
1616 g_assert_not_reached ();
1617 }
1618 }
1619 break;
1620 case LLVMArgVtypeByVal:
1621 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1622 if (!ctx_ok (ctx))
1623 break;
1624 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1625 pindex ++;
1626 break;
1627 case LLVMArgAsIArgs:
1628 if (ainfo->esize == 8)
1629 param_types [pindex] = LLVMArrayType (LLVMInt64Type (), ainfo->nslots);
1630 else
1631 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1632 pindex ++;
1633 break;
1634 case LLVMArgVtypeAddr:
1635 case LLVMArgVtypeByRef:
1636 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1637 if (!ctx_ok (ctx))
1638 break;
1639 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1640 pindex ++;
1641 break;
1642 case LLVMArgAsFpArgs: {
1643 int j;
1644
1645 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1646 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1647 param_types [pindex ++] = LLVMDoubleType ();
1648 for (j = 0; j < ainfo->nslots; ++j)
1649 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1650 break;
1651 }
1652 case LLVMArgVtypeAsScalar:
1653 g_assert_not_reached ();
1654 break;
1655 case LLVMArgGsharedvtFixed:
1656 case LLVMArgGsharedvtFixedVtype:
1657 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1658 break;
1659 case LLVMArgGsharedvtVariable:
1660 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1661 break;
1662 default:
1663 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1664 break;
1665 }
1666 }
1667 if (!ctx_ok (ctx)) {
1668 g_free (param_types);
1669 return NULL;
1670 }
1671 if (vretaddr && vret_arg_pindex == pindex)
1672 param_types [pindex ++] = IntPtrType ();
1673 if (ctx->llvm_only && cinfo->rgctx_arg) {
1674 /* Pass the rgctx as the last argument */
1675 cinfo->rgctx_arg_pindex = pindex;
1676 param_types [pindex] = ctx->module->ptr_type;
1677 pindex ++;
1678 } else if (ctx->llvm_only && cinfo->dummy_arg) {
1679 /* Pass a dummy arg last */
1680 cinfo->dummy_arg_pindex = pindex;
1681 param_types [pindex] = ctx->module->ptr_type;
1682 pindex ++;
1683 }
1684
1685 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1686 g_free (param_types);
1687
1688 return res;
1689 }
1690
1691 static LLVMTypeRef
1692 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1693 {
1694 return sig_to_llvm_sig_full (ctx, sig, NULL);
1695 }
1696
1697 /*
1698 * LLVMFunctionType1:
1699 *
1700 * Create an LLVM function type from the arguments.
1701 */
1702 static G_GNUC_UNUSED LLVMTypeRef
1703 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1704 int IsVarArg)
1705 {
1706 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1707 }
1708
1709 /*
1710 * LLVMFunctionType1:
1711 *
1712 * Create an LLVM function type from the arguments.
1713 */
1714 static G_GNUC_UNUSED LLVMTypeRef
1715 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1716 LLVMTypeRef ParamType1,
1717 int IsVarArg)
1718 {
1719 LLVMTypeRef param_types [1];
1720
1721 param_types [0] = ParamType1;
1722
1723 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1724 }
1725
1726 /*
1727 * LLVMFunctionType2:
1728 *
1729 * Create an LLVM function type from the arguments.
1730 */
1731 static G_GNUC_UNUSED LLVMTypeRef
1732 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1733 LLVMTypeRef ParamType1,
1734 LLVMTypeRef ParamType2,
1735 int IsVarArg)
1736 {
1737 LLVMTypeRef param_types [2];
1738
1739 param_types [0] = ParamType1;
1740 param_types [1] = ParamType2;
1741
1742 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1743 }
1744
1745 /*
1746 * LLVMFunctionType3:
1747 *
1748 * Create an LLVM function type from the arguments.
1749 */
1750 static G_GNUC_UNUSED LLVMTypeRef
1751 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1752 LLVMTypeRef ParamType1,
1753 LLVMTypeRef ParamType2,
1754 LLVMTypeRef ParamType3,
1755 int IsVarArg)
1756 {
1757 LLVMTypeRef param_types [3];
1758
1759 param_types [0] = ParamType1;
1760 param_types [1] = ParamType2;
1761 param_types [2] = ParamType3;
1762
1763 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1764 }
1765
1766 static G_GNUC_UNUSED LLVMTypeRef
1767 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1768 LLVMTypeRef ParamType1,
1769 LLVMTypeRef ParamType2,
1770 LLVMTypeRef ParamType3,
1771 LLVMTypeRef ParamType4,
1772 LLVMTypeRef ParamType5,
1773 int IsVarArg)
1774 {
1775 LLVMTypeRef param_types [5];
1776
1777 param_types [0] = ParamType1;
1778 param_types [1] = ParamType2;
1779 param_types [2] = ParamType3;
1780 param_types [3] = ParamType4;
1781 param_types [4] = ParamType5;
1782
1783 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1784 }
1785
1786 /*
1787 * create_builder:
1788 *
1789 * Create an LLVM builder and remember it so it can be freed later.
1790 */
1791 static LLVMBuilderRef
1792 create_builder (EmitContext *ctx)
1793 {
1794 LLVMBuilderRef builder = LLVMCreateBuilder ();
1795 if (mono_use_fast_math)
1796 mono_llvm_set_fast_math (builder);
1797
1798 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1799
1800 emit_default_dbg_loc (ctx, builder);
1801
1802 return builder;
1803 }
1804
1805 static char*
1806 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1807 {
1808 char *name;
1809
1810 switch (type) {
1811 case MONO_PATCH_INFO_JIT_ICALL_ID:
1812 name = g_strdup_printf ("jit_icall_%s", mono_find_jit_icall_info ((MonoJitICallId)(gsize)data)->name);
1813 break;
1814 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1815 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1816 name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
1817 break;
1818 }
1819 default:
1820 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1821 break;
1822 }
1823
1824 return name;
1825 }
1826
1827 static int
1828 compute_aot_got_offset (MonoLLVMModule *module, MonoJumpInfo *ji, LLVMTypeRef llvm_type)
1829 {
1830 guint32 got_offset = mono_aot_get_got_offset (ji);
1831
1832 LLVMTypeRef lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (got_offset));
1833
1834 if (!lookup_type) {
1835 lookup_type = llvm_type;
1836 } else if (llvm_type != lookup_type) {
1837 lookup_type = module->ptr_type;
1838 } else {
1839 return got_offset;
1840 }
1841
1842 g_hash_table_insert (module->got_idx_to_type, GINT_TO_POINTER (got_offset), lookup_type);
1843 return got_offset;
1844 }
1845
1846 /* Allocate a GOT slot for TYPE/DATA, and emit IR to load it */
1847 static LLVMValueRef
1848 get_aotconst_typed_module (MonoLLVMModule *module, LLVMBuilderRef builder, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1849 {
1850 guint32 got_offset;
1851 LLVMValueRef indexes [2];
1852 LLVMValueRef got_entry_addr, load;
1853 char *name = NULL;
1854
1855 MonoJumpInfo tmp_ji;
1856 tmp_ji.type = type;
1857 tmp_ji.data.target = data;
1858
1859 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1860
1861 got_offset = compute_aot_got_offset (module, ji, llvm_type);
1862 module->max_got_offset = MAX (module->max_got_offset, got_offset);
1863
1864 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1865 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1866 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
1867
1868 name = get_aotconst_name (type, data, got_offset);
1869 if (llvm_type) {
1870 load = LLVMBuildLoad (builder, got_entry_addr, "");
1871 load = LLVMBuildBitCast (builder, load, llvm_type, name);
1872 } else {
1873 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1874 }
1875 g_free (name);
1876 //set_invariant_load_flag (load);
1877
1878 return load;
1879 }
1880
1881 static LLVMValueRef
1882 get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1883 {
1884 MonoCompile *cfg;
1885 guint32 got_offset;
1886 LLVMValueRef indexes [2];
1887 LLVMValueRef got_entry_addr, load;
1888 LLVMBuilderRef builder = ctx->builder;
1889 char *name = NULL;
1890
1891 cfg = ctx->cfg;
1892
1893 MonoJumpInfo tmp_ji;
1894 tmp_ji.type = type;
1895 tmp_ji.data.target = data;
1896
1897 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1898
1899 ji->next = cfg->patch_info;
1900 cfg->patch_info = ji;
1901
1902 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, llvm_type);
1903 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1904 /*
1905 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1906 * explicitly initialize it.
1907 */
1908 if (!mono_aot_is_shared_got_offset (got_offset)) {
1909 //mono_print_ji (ji);
1910 //printf ("\n");
1911 ctx->cfg->got_access_count ++;
1912 }
1913
1914 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1915 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1916 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1917
1918 name = get_aotconst_name (type, data, got_offset);
1919 if (llvm_type) {
1920 load = LLVMBuildLoad (builder, got_entry_addr, "");
1921 load = convert (ctx, load, llvm_type);
1922 LLVMSetValueName (load, name ? name : "");
1923 } else {
1924 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1925 }
1926 g_free (name);
1927 //set_invariant_load_flag (load);
1928
1929 return load;
1930 }
1931
1932 static LLVMValueRef
1933 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1934 {
1935 return get_aotconst_typed (ctx, type, data, NULL);
1936 }
1937
1938 typedef struct {
1939 MonoJumpInfo *ji;
1940 MonoMethod *method;
1941 LLVMValueRef load;
1942 LLVMTypeRef type;
1943 } CallSite;
1944
1945 static gboolean
1946 method_is_direct_callable (MonoMethod *method)
1947 {
1948 if (method->wrapper_type == MONO_WRAPPER_ALLOC)
1949 return TRUE;
1950 if (method->string_ctor)
1951 return FALSE;
1952 if (method->wrapper_type)
1953 return FALSE;
1954 if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL))
1955 return FALSE;
1956 /* Can't enable this as the callee might fail llvm compilation */
1957 /*
1958 if (!method->is_inflated && (mono_class_get_flags (method->klass) & TYPE_ATTRIBUTE_PUBLIC) && (method->flags & METHOD_ATTRIBUTE_PUBLIC))
1959 return TRUE;
1960 */
1961 return FALSE;
1962 }
1963
1964 static LLVMValueRef
1965 get_callee_llvmonly (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1966 {
1967 LLVMValueRef callee;
1968 char *callee_name = NULL;
1969
1970 if (ctx->module->static_link && ctx->module->assembly->image != mono_get_corlib ()) {
1971 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1972 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1973 g_assert (info);
1974 if (info->func != info->wrapper) {
1975 type = MONO_PATCH_INFO_METHOD;
1976 data = mono_icall_get_wrapper_method (info);
1977 callee_name = mono_aot_get_mangled_method_name ((MonoMethod*)data);
1978 }
1979 } else if (type == MONO_PATCH_INFO_METHOD) {
1980 MonoMethod *method = (MonoMethod*)data;
1981 if (m_class_get_image (method->klass) != ctx->module->assembly->image && method_is_direct_callable (method))
1982 callee_name = mono_aot_get_mangled_method_name (method);
1983 }
1984 }
1985
1986 if (!callee_name)
1987 callee_name = mono_aot_get_direct_call_symbol (type, data);
1988 if (callee_name) {
1989 /* Directly callable */
1990 // FIXME: Locking
1991 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1992 if (!callee) {
1993 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1994
1995 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1996
1997 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1998 } else {
1999 /* LLVMTypeRef's are uniqued */
2000 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
2001 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
2002
2003 g_free (callee_name);
2004 }
2005 return callee;
2006 }
2007
2008 /*
2009 * Change references to jit icalls to the icall wrappers when in corlib, so
2010 * they can be called directly.
2011 */
2012 if (ctx->module->assembly->image == mono_get_corlib () && type == MONO_PATCH_INFO_JIT_ICALL_ID) {
2013 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
2014
2015 if (info->func != info->wrapper) {
2016 type = MONO_PATCH_INFO_METHOD;
2017 data = mono_icall_get_wrapper_method (info);
2018 }
2019 }
2020
2021 /*
2022 * Instead of emitting an indirect call through a got slot, emit a placeholder, and
2023 * replace it with a direct call or an indirect call in mono_llvm_fixup_aot_module ()
2024 * after all methods have been emitted.
2025 */
2026 if (type == MONO_PATCH_INFO_METHOD) {
2027 MonoMethod *method = (MonoMethod*)data;
2028 if (m_class_get_image (method->klass)->assembly == ctx->module->assembly) {
2029 MonoJumpInfo tmp_ji;
2030 tmp_ji.type = type;
2031 tmp_ji.data.target = data;
2032
2033 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
2034 ji->next = ctx->cfg->patch_info;
2035 ctx->cfg->patch_info = ji;
2036 LLVMTypeRef llvm_type = LLVMPointerType (llvm_sig, 0);
2037
2038 ctx->cfg->got_access_count ++;
2039
2040 CallSite *info = g_new0 (CallSite, 1);
2041 info->method = method;
2042 info->ji = ji;
2043 info->type = llvm_type;
2044
2045 /*
2046 * Emit a dummy load to represent the callee, and either replace it with
2047 * a reference to the llvm method for the callee, or from a load from the
2048 * GOT.
2049 */
2050 LLVMValueRef indexes [2];
2051 LLVMValueRef got_entry_addr, load;
2052
2053 LLVMBuilderRef builder = ctx->builder;
2054 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2055 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2056 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
2057
2058 load = LLVMBuildLoad (builder, got_entry_addr, "");
2059 load = convert (ctx, load, llvm_type);
2060 info->load = load;
2061
2062 g_ptr_array_add (ctx->callsite_list, info);
2063
2064 return load;
2065 }
2066 }
2067
2068 /*
2069 * Calls are made through the GOT.
2070 */
2071 callee = get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
2072
2073 return callee;
2074 }
2075
2076 /*
2077 * get_callee:
2078 *
2079 * Return an llvm value representing the callee given by the arguments.
2080 */
2081 static LLVMValueRef
2082 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
2083 {
2084 LLVMValueRef callee;
2085 char *callee_name;
2086 MonoJumpInfo *ji = NULL;
2087
2088 if (ctx->llvm_only)
2089 return get_callee_llvmonly (ctx, llvm_sig, type, data);
2090
2091 callee_name = mono_aot_get_plt_symbol (type, data);
2092 if (!callee_name)
2093 return NULL;
2094
2095 if (ctx->cfg->compile_aot)
2096 /* Add a patch so referenced wrappers can be compiled in full aot mode */
2097 mono_add_patch_info (ctx->cfg, 0, type, data);
2098
2099 // FIXME: Locking
2100 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
2101 if (!callee) {
2102 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
2103
2104 LLVMSetVisibility (callee, LLVMHiddenVisibility);
2105
2106 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
2107 }
2108
2109 if (ctx->cfg->compile_aot) {
2110 ji = g_new0 (MonoJumpInfo, 1);
2111 ji->type = type;
2112 ji->data.target = data;
2113
2114 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
2115 }
2116
2117 return callee;
2118 }
2119
2120 static LLVMValueRef
2121 get_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
2122 {
2123 gpointer target;
2124
2125 // This won't be patched so compile the wrapper immediately
2126 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
2127 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
2128 target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
2129 } else {
2130 target = resolve_patch (ctx->cfg, type, data);
2131 }
2132
2133 LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
2134 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
2135 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
2136 LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
2137 return callee;
2138 }
2139
2140 static int
2141 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
2142 {
2143 MonoMethodHeader *header = cfg->header;
2144 MonoExceptionClause *clause;
2145 int i;
2146
2147 /* Directly */
2148 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
2149 return (bb->region >> 8) - 1;
2150
2151 /* Indirectly */
2152 for (i = 0; i < header->num_clauses; ++i) {
2153 clause = &header->clauses [i];
2154
2155 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
2156 return i;
2157 }
2158
2159 return -1;
2160 }
2161
2162 static MonoExceptionClause *
2163 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
2164 {
2165 if (bb == cfg->bb_init)
2166 return NULL;
2167 // Since they're sorted by nesting we just need
2168 // the first one that the bb is a member of
2169 for (int i = 0; i < cfg->header->num_clauses; i++) {
2170 MonoExceptionClause *curr = &cfg->header->clauses [i];
2171
2172 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
2173 return curr;
2174 }
2175
2176 return NULL;
2177 }
2178
2179 static void
2180 set_metadata_flag (LLVMValueRef v, const char *flag_name)
2181 {
2182 LLVMValueRef md_arg;
2183 int md_kind;
2184
2185 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2186 md_arg = LLVMMDString ("mono", 4);
2187 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2188 }
2189
2190 static void
2191 set_nonnull_load_flag (LLVMValueRef v)
2192 {
2193 LLVMValueRef md_arg;
2194 int md_kind;
2195 const char *flag_name;
2196
2197 flag_name = "nonnull";
2198 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2199 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2200 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2201 }
2202
2203 static void
2204 set_invariant_load_flag (LLVMValueRef v)
2205 {
2206 LLVMValueRef md_arg;
2207 int md_kind;
2208 const char *flag_name;
2209
2210 // FIXME: Cache this
2211 flag_name = "invariant.load";
2212 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2213 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2214 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2215 }
2216
2217 /*
2218 * emit_call:
2219 *
2220 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
2221 * a try region.
2222 */
2223 static LLVMValueRef
2224 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
2225 {
2226 MonoCompile *cfg = ctx->cfg;
2227 LLVMValueRef lcall = NULL;
2228 LLVMBuilderRef builder = *builder_ref;
2229 MonoExceptionClause *clause;
2230
2231 if (ctx->llvm_only) {
2232 clause = get_most_deep_clause (cfg, ctx, bb);
2233
2234 if (clause) {
2235 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2236
2237 /*
2238 * Have to use an invoke instead of a call, branching to the
2239 * handler bblock of the clause containing this bblock.
2240 */
2241 intptr_t key = CLAUSE_END(clause);
2242
2243 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
2244
2245 // FIXME: Find the one that has the lowest end bound for the right start address
2246 // FIXME: Finally + nesting
2247
2248 if (lpad_bb) {
2249 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
2250
2251 /* Use an invoke */
2252 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
2253
2254 builder = ctx->builder = create_builder (ctx);
2255 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2256
2257 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2258 }
2259 }
2260 } else {
2261 int clause_index = get_handler_clause (cfg, bb);
2262
2263 if (clause_index != -1) {
2264 MonoMethodHeader *header = cfg->header;
2265 MonoExceptionClause *ec = &header->clauses [clause_index];
2266 MonoBasicBlock *tblock;
2267 LLVMBasicBlockRef ex_bb, noex_bb;
2268
2269 /*
2270 * Have to use an invoke instead of a call, branching to the
2271 * handler bblock of the clause containing this bblock.
2272 */
2273
2274 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY || ec->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2275
2276 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
2277 g_assert (tblock);
2278
2279 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
2280
2281 ex_bb = get_bb (ctx, tblock);
2282
2283 noex_bb = gen_bb (ctx, "NOEX_BB");
2284
2285 /* Use an invoke */
2286 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
2287
2288 builder = ctx->builder = create_builder (ctx);
2289 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2290
2291 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2292 }
2293 }
2294
2295 if (!lcall) {
2296 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
2297 ctx->builder = builder;
2298 }
2299
2300 if (builder_ref)
2301 *builder_ref = ctx->builder;
2302
2303 return lcall;
2304 }
2305
2306 static LLVMValueRef
2307 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, LLVMValueRef base, const char *name, gboolean is_faulting, gboolean is_volatile, BarrierKind barrier)
2308 {
2309 LLVMValueRef res;
2310
2311 /*
2312 * We emit volatile loads for loads which can fault, because otherwise
2313 * LLVM will generate invalid code when encountering a load from a
2314 * NULL address.
2315 */
2316 if (barrier != LLVM_BARRIER_NONE)
2317 res = mono_llvm_build_atomic_load (*builder_ref, addr, name, is_volatile, size, barrier);
2318 else
2319 res = mono_llvm_build_load (*builder_ref, addr, name, is_volatile);
2320
2321 return res;
2322 }
2323
2324 static void
2325 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, gboolean is_volatile, BarrierKind barrier)
2326 {
2327 if (barrier != LLVM_BARRIER_NONE)
2328 mono_llvm_build_aligned_store (*builder_ref, value, addr, barrier, size);
2329 else
2330 mono_llvm_build_store (*builder_ref, value, addr, is_volatile, barrier);
2331 }
2332
2333 static void
2334 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, gboolean is_volatile)
2335 {
2336 emit_store_general (ctx, bb, builder_ref, size, value, addr, base, is_faulting, is_volatile, LLVM_BARRIER_NONE);
2337 }
2338
2339 /*
2340 * emit_cond_system_exception:
2341 *
2342 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2343 * Might set the ctx exception.
2344 */
2345 static void
2346 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp, gboolean force_explicit)
2347 {
2348 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2349 LLVMBuilderRef builder;
2350 MonoClass *exc_class;
2351 LLVMValueRef args [2];
2352 LLVMValueRef callee;
2353 gboolean no_pc = FALSE;
2354 static MonoClass *exc_classes [MONO_EXC_INTRINS_NUM];
2355
2356 if (IS_TARGET_AMD64)
2357 /* Some platforms don't require the pc argument */
2358 no_pc = TRUE;
2359
2360 int exc_id = mini_exception_id_by_name (exc_type);
2361 if (!exc_classes [exc_id])
2362 exc_classes [exc_id] = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
2363 exc_class = exc_classes [exc_id];
2364
2365 ex_bb = gen_bb (ctx, "EX_BB");
2366 if (ctx->llvm_only)
2367 ex2_bb = gen_bb (ctx, "EX2_BB");
2368 noex_bb = gen_bb (ctx, "NOEX_BB");
2369
2370 LLVMValueRef branch = LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2371 if (exc_id == MONO_EXC_NULL_REF && !ctx->cfg->disable_llvm_implicit_null_checks && !force_explicit) {
2372 mono_llvm_set_implicit_branch (ctx->builder, branch);
2373 }
2374
2375 /* Emit exception throwing code */
2376 ctx->builder = builder = create_builder (ctx);
2377 LLVMPositionBuilderAtEnd (builder, ex_bb);
2378
2379 if (ctx->cfg->llvm_only) {
2380 LLVMBuildBr (builder, ex2_bb);
2381
2382 ctx->builder = builder = create_builder (ctx);
2383 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2384
2385 if (exc_id == MONO_EXC_NULL_REF) {
2386 static LLVMTypeRef sig;
2387
2388 if (!sig)
2389 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
2390 /* Can't cache this */
2391 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
2392 emit_call (ctx, bb, &builder, callee, NULL, 0);
2393 } else {
2394 static LLVMTypeRef sig;
2395
2396 if (!sig)
2397 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2398 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_throw_corlib_exception));
2399 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2400 emit_call (ctx, bb, &builder, callee, args, 1);
2401 }
2402
2403 LLVMBuildUnreachable (builder);
2404
2405 ctx->builder = builder = create_builder (ctx);
2406 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2407
2408 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2409
2410 ctx->ex_index ++;
2411 return;
2412 }
2413
2414 callee = ctx->module->throw_corlib_exception;
2415 if (!callee) {
2416 LLVMTypeRef sig;
2417
2418 if (no_pc)
2419 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2420 else
2421 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2422
2423 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_throw_corlib_exception_abs_trampoline;
2424
2425 if (ctx->cfg->compile_aot) {
2426 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2427 } else {
2428 /*
2429 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2430 * - On x86, LLVM generated code doesn't push the arguments
2431 * - The trampoline takes the throw address as an arguments, not a pc offset.
2432 */
2433 callee = get_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2434
2435 /*
2436 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2437 * added by get_jit_callee ().
2438 */
2439 ex2_bb = gen_bb (ctx, "EX2_BB");
2440 LLVMBuildBr (builder, ex2_bb);
2441 ex_bb = ex2_bb;
2442
2443 ctx->builder = builder = create_builder (ctx);
2444 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2445 }
2446 }
2447
2448 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2449
2450 /*
2451 * The LLVM mono branch contains changes so a block address can be passed as an
2452 * argument to a call.
2453 */
2454 if (no_pc) {
2455 emit_call (ctx, bb, &builder, callee, args, 1);
2456 } else {
2457 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2458 emit_call (ctx, bb, &builder, callee, args, 2);
2459 }
2460
2461 LLVMBuildUnreachable (builder);
2462
2463 ctx->builder = builder = create_builder (ctx);
2464 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2465
2466 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2467
2468 ctx->ex_index ++;
2469 return;
2470 }
2471
2472 /*
2473 * emit_args_to_vtype:
2474 *
2475 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2476 */
2477 static void
2478 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2479 {
2480 int j, size, nslots;
2481 MonoClass *klass;
2482
2483 t = mini_get_underlying_type (t);
2484 klass = mono_class_from_mono_type_internal (t);
2485 size = mono_class_value_size (klass, NULL);
2486
2487 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
2488 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2489
2490 if (ainfo->storage == LLVMArgAsFpArgs)
2491 nslots = ainfo->nslots;
2492 else
2493 nslots = 2;
2494
2495 for (j = 0; j < nslots; ++j) {
2496 LLVMValueRef index [2], addr, daddr;
2497 int part_size = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2498 LLVMTypeRef part_type;
2499
2500 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2501 part_size ++;
2502
2503 if (ainfo->pair_storage [j] == LLVMArgNone)
2504 continue;
2505
2506 switch (ainfo->pair_storage [j]) {
2507 case LLVMArgInIReg: {
2508 part_type = LLVMIntType (part_size * 8);
2509 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass)) {
2510 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2511 addr = LLVMBuildGEP (builder, address, index, 1, "");
2512 } else {
2513 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2514 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2515 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2516 }
2517 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2518 break;
2519 }
2520 case LLVMArgInFPReg: {
2521 LLVMTypeRef arg_type;
2522
2523 if (ainfo->esize == 8)
2524 arg_type = LLVMDoubleType ();
2525 else
2526 arg_type = LLVMFloatType ();
2527
2528 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2529 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2530 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2531 LLVMBuildStore (builder, args [j], addr);
2532 break;
2533 }
2534 case LLVMArgNone:
2535 break;
2536 default:
2537 g_assert_not_reached ();
2538 }
2539
2540 size -= TARGET_SIZEOF_VOID_P;
2541 }
2542 }
2543
2544 /*
2545 * emit_vtype_to_args:
2546 *
2547 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2548 * into ARGS, and the number of arguments into NARGS.
2549 */
2550 static void
2551 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2552 {
2553 int pindex = 0;
2554 int j, size, nslots;
2555 LLVMTypeRef arg_type;
2556
2557 t = mini_get_underlying_type (t);
2558 size = get_vtype_size (t);
2559
2560 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t)))
2561 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2562
2563 if (ainfo->storage == LLVMArgAsFpArgs)
2564 nslots = ainfo->nslots;
2565 else
2566 nslots = 2;
2567 for (j = 0; j < nslots; ++j) {
2568 LLVMValueRef index [2], addr, daddr;
2569 int partsize = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2570
2571 if (ainfo->pair_storage [j] == LLVMArgNone)
2572 continue;
2573
2574 switch (ainfo->pair_storage [j]) {
2575 case LLVMArgInIReg:
2576 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
2577 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2578 addr = LLVMBuildGEP (builder, address, index, 1, "");
2579 } else {
2580 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2581 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2582 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2583 }
2584 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2585 break;
2586 case LLVMArgInFPReg:
2587 if (ainfo->esize == 8)
2588 arg_type = LLVMDoubleType ();
2589 else
2590 arg_type = LLVMFloatType ();
2591 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2592 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2593 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2594 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2595 break;
2596 case LLVMArgNone:
2597 break;
2598 default:
2599 g_assert_not_reached ();
2600 }
2601 size -= TARGET_SIZEOF_VOID_P;
2602 }
2603
2604 *nargs = pindex;
2605 }
2606
2607 static LLVMValueRef
2608 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2609 {
2610 /*
2611 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2612 * get executed every time control reaches them.
2613 */
2614 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2615
2616 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2617 return ctx->last_alloca;
2618 }
2619
2620 static LLVMValueRef
2621 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2622 {
2623 return build_alloca_llvm_type_name (ctx, t, align, "");
2624 }
2625
2626 static LLVMValueRef
2627 build_alloca (EmitContext *ctx, MonoType *t)
2628 {
2629 MonoClass *k = mono_class_from_mono_type_internal (t);
2630 int align;
2631
2632 g_assert (!mini_is_gsharedvt_variable_type (t));
2633
2634 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2635 align = mono_class_value_size (k, NULL);
2636 else
2637 align = mono_class_min_align (k);
2638
2639 /* Sometimes align is not a power of 2 */
2640 while (mono_is_power_of_two (align) == -1)
2641 align ++;
2642
2643 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2644 }
2645
2646 static LLVMValueRef
2647 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2648 {
2649 /*
2650 * gsharedvt local.
2651 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2652 */
2653 MonoCompile *cfg = ctx->cfg;
2654 LLVMBuilderRef builder = ctx->builder;
2655 LLVMValueRef offset, offset_var;
2656 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2657 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2658 LLVMValueRef ptr;
2659 char *name;
2660
2661 g_assert (info_var);
2662 g_assert (locals_var);
2663
2664 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2665
2666 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * TARGET_SIZEOF_VOID_P), FALSE);
2667 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2668
2669 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2670 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2671
2672 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2673 }
2674
2675 /*
2676 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2677 */
2678 static void
2679 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2680 {
2681 if (!module->used)
2682 module->used = g_ptr_array_sized_new (16);
2683 g_ptr_array_add (module->used, global);
2684 }
2685
2686 static void
2687 emit_llvm_used (MonoLLVMModule *module)
2688 {
2689 LLVMModuleRef lmodule = module->lmodule;
2690 LLVMTypeRef used_type;
2691 LLVMValueRef used, *used_elem;
2692 int i;
2693
2694 if (!module->used)
2695 return;
2696
2697 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2698 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2699 used_elem = g_new0 (LLVMValueRef, module->used->len);
2700 for (i = 0; i < module->used->len; ++i)
2701 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2702 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2703 LLVMSetLinkage (used, LLVMAppendingLinkage);
2704 LLVMSetSection (used, "llvm.metadata");
2705 }
2706
2707 /*
2708 * emit_get_method:
2709 *
2710 * Emit a function mapping method indexes to their code
2711 */
2712 static void
2713 emit_get_method (MonoLLVMModule *module)
2714 {
2715 LLVMModuleRef lmodule = module->lmodule;
2716 LLVMValueRef func, switch_ins, m;
2717 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2718 LLVMBasicBlockRef *bbs = NULL;
2719 LLVMTypeRef rtype;
2720 LLVMBuilderRef builder = LLVMCreateBuilder ();
2721 LLVMValueRef table = NULL;
2722 char *name;
2723 int i;
2724 gboolean emit_table = FALSE;
2725
2726 #ifdef TARGET_WASM
2727 /*
2728 * Emit a table of functions instead of a switch statement,
2729 * its very efficient on wasm. This might be usable on
2730 * other platforms too.
2731 */
2732 emit_table = TRUE;
2733 #endif
2734
2735 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2736
2737 if (emit_table) {
2738 LLVMTypeRef table_type;
2739 LLVMValueRef *table_elems;
2740 char *table_name;
2741
2742 int table_len = module->max_method_idx + 1;
2743 table_type = LLVMArrayType (rtype, table_len);
2744 table_name = g_strdup_printf ("%s_method_table", module->assembly->aname.name);
2745 table = LLVMAddGlobal (lmodule, table_type, table_name);
2746 table_elems = g_new0 (LLVMValueRef, table_len);
2747 for (i = 0; i < table_len; ++i) {
2748 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2749 if (m)
2750 table_elems [i] = LLVMBuildBitCast (builder, m, rtype, "");
2751 else
2752 table_elems [i] = LLVMConstNull (rtype);
2753 }
2754 LLVMSetInitializer (table, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), table_elems, table_len));
2755 }
2756
2757 /*
2758 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2759 * but generating code seems safer.
2760 */
2761 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2762 LLVMSetLinkage (func, LLVMExternalLinkage);
2763 LLVMSetVisibility (func, LLVMHiddenVisibility);
2764 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2765 module->get_method = func;
2766
2767 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2768
2769 /*
2770 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2771 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2772 * then we will have to find another solution.
2773 */
2774
2775 name = g_strdup_printf ("BB_CODE_START");
2776 code_start_bb = LLVMAppendBasicBlock (func, name);
2777 g_free (name);
2778 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2779 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2780
2781 name = g_strdup_printf ("BB_CODE_END");
2782 code_end_bb = LLVMAppendBasicBlock (func, name);
2783 g_free (name);
2784 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2785 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2786
2787 if (emit_table) {
2788 /*
2789 * switch (index) {
2790 * case -1: return code_start;
2791 * case -2: return code_end;
2792 * default: return method_table [index];
2793 */
2794 LLVMBasicBlockRef default_bb = LLVMAppendBasicBlock (func, "DEFAULT");
2795 LLVMPositionBuilderAtEnd (builder, default_bb);
2796 LLVMValueRef base = table;
2797 LLVMValueRef indexes [2];
2798 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2799 indexes [1] = LLVMGetParam (func, 0);
2800 LLVMValueRef addr = LLVMBuildGEP (builder, base, indexes, 2, "");
2801 LLVMValueRef res = mono_llvm_build_load (builder, addr, "", FALSE);
2802 LLVMBuildRet (builder, res);
2803
2804 LLVMPositionBuilderAtEnd (builder, entry_bb);
2805
2806 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), default_bb, 0);
2807 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2808 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2809 } else {
2810 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2811 for (i = 0; i < module->max_method_idx + 1; ++i) {
2812 name = g_strdup_printf ("BB_%d", i);
2813 bb = LLVMAppendBasicBlock (func, name);
2814 g_free (name);
2815 bbs [i] = bb;
2816
2817 LLVMPositionBuilderAtEnd (builder, bb);
2818
2819 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2820 if (m)
2821 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2822 else
2823 LLVMBuildRet (builder, LLVMConstNull (rtype));
2824 }
2825
2826 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2827 LLVMPositionBuilderAtEnd (builder, fail_bb);
2828 LLVMBuildRet (builder, LLVMConstNull (rtype));
2829
2830 LLVMPositionBuilderAtEnd (builder, entry_bb);
2831
2832 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2833 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2834 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2835 for (i = 0; i < module->max_method_idx + 1; ++i) {
2836 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2837 }
2838 }
2839
2840 mark_as_used (module, func);
2841
2842 LLVMDisposeBuilder (builder);
2843 }
2844
2845 /*
2846 * emit_get_unbox_tramp:
2847 *
2848 * Emit a function mapping method indexes to their unbox trampoline
2849 */
2850 static void
2851 emit_get_unbox_tramp (MonoLLVMModule *module)
2852 {
2853 LLVMModuleRef lmodule = module->lmodule;
2854 LLVMValueRef func, switch_ins, m;
2855 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2856 LLVMBasicBlockRef *bbs;
2857 LLVMTypeRef rtype;
2858 LLVMBuilderRef builder = LLVMCreateBuilder ();
2859 char *name;
2860 int i;
2861 gboolean emit_table = FALSE;
2862
2863 /* Similar to emit_get_method () */
2864
2865 #ifndef TARGET_WATCHOS
2866 emit_table = TRUE;
2867 #endif
2868
2869 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2870
2871 if (emit_table) {
2872 // About 10% of methods have an unbox tramp, so emit a table of indexes for them
2873 // that the runtime can search using a binary search
2874 int len = 0;
2875 for (i = 0; i < module->max_method_idx + 1; ++i) {
2876 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2877 if (m)
2878 len ++;
2879 }
2880
2881 LLVMTypeRef table_type, elemtype;
2882 LLVMValueRef *table_elems;
2883 LLVMValueRef table;
2884 char *table_name;
2885 int table_len;
2886 int elemsize;
2887
2888 table_len = len;
2889 elemsize = module->max_method_idx < 65000 ? 2 : 4;
2890
2891 // The index table
2892 elemtype = elemsize == 2 ? LLVMInt16Type () : LLVMInt32Type ();
2893 table_type = LLVMArrayType (elemtype, table_len);
2894 table_name = g_strdup_printf ("%s_unbox_tramp_indexes", module->assembly->aname.name);
2895 table = LLVMAddGlobal (lmodule, table_type, table_name);
2896 table_elems = g_new0 (LLVMValueRef, table_len);
2897 int idx = 0;
2898 for (i = 0; i < module->max_method_idx + 1; ++i) {
2899 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2900 if (m)
2901 table_elems [idx ++] = LLVMConstInt (elemtype, i, FALSE);
2902 }
2903 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2904 module->unbox_tramp_indexes = table;
2905
2906 // The trampoline table
2907 elemtype = rtype;
2908 table_type = LLVMArrayType (elemtype, table_len);
2909 table_name = g_strdup_printf ("%s_unbox_trampolines", module->assembly->aname.name);
2910 table = LLVMAddGlobal (lmodule, table_type, table_name);
2911 table_elems = g_new0 (LLVMValueRef, table_len);
2912 idx = 0;
2913 for (i = 0; i < module->max_method_idx + 1; ++i) {
2914 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2915 if (m)
2916 table_elems [idx ++] = LLVMBuildBitCast (builder, m, rtype, "");
2917 }
2918 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2919 module->unbox_trampolines = table;
2920
2921 module->unbox_tramp_num = table_len;
2922 module->unbox_tramp_elemsize = elemsize;
2923 return;
2924 }
2925
2926 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2927 LLVMSetLinkage (func, LLVMExternalLinkage);
2928 LLVMSetVisibility (func, LLVMHiddenVisibility);
2929 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2930 module->get_unbox_tramp = func;
2931
2932 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2933
2934 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2935 for (i = 0; i < module->max_method_idx + 1; ++i) {
2936 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2937 if (!m)
2938 continue;
2939
2940 name = g_strdup_printf ("BB_%d", i);
2941 bb = LLVMAppendBasicBlock (func, name);
2942 g_free (name);
2943 bbs [i] = bb;
2944
2945 LLVMPositionBuilderAtEnd (builder, bb);
2946
2947 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2948 }
2949
2950 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2951 LLVMPositionBuilderAtEnd (builder, fail_bb);
2952 LLVMBuildRet (builder, LLVMConstNull (rtype));
2953
2954 LLVMPositionBuilderAtEnd (builder, entry_bb);
2955
2956 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2957 for (i = 0; i < module->max_method_idx + 1; ++i) {
2958 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2959 if (!m)
2960 continue;
2961
2962 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2963 }
2964
2965 mark_as_used (module, func);
2966 LLVMDisposeBuilder (builder);
2967 }
2968
2969 /* Add a function to mark the beginning of LLVM code */
2970 static void
2971 emit_llvm_code_start (MonoLLVMModule *module)
2972 {
2973 LLVMModuleRef lmodule = module->lmodule;
2974 LLVMValueRef func;
2975 LLVMBasicBlockRef entry_bb;
2976 LLVMBuilderRef builder;
2977
2978 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2979 LLVMSetLinkage (func, LLVMInternalLinkage);
2980 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2981 module->code_start = func;
2982 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2983 builder = LLVMCreateBuilder ();
2984 LLVMPositionBuilderAtEnd (builder, entry_bb);
2985 LLVMBuildRetVoid (builder);
2986 LLVMDisposeBuilder (builder);
2987 }
2988
2989 static LLVMValueRef
2990 emit_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
2991 {
2992 LLVMModuleRef lmodule = module->lmodule;
2993 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
2994 LLVMBasicBlockRef entry_bb;
2995 LLVMBuilderRef builder;
2996 LLVMTypeRef sig;
2997 MonoJumpInfo *ji;
2998 int got_offset;
2999 const char *wrapper_name = mono_marshal_get_aot_init_wrapper_name (subtype);
3000 char *name = g_strdup_printf ("%s%s", module->global_prefix, wrapper_name);
3001 MonoJitICallId icall_id = MONO_JIT_ICALL_ZeroIsReserved;
3002
3003 switch (subtype) {
3004 case AOT_INIT_METHOD:
3005 icall_id = MONO_JIT_ICALL_mini_llvm_init_method;
3006 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
3007 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
3008 break;
3009 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3010 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_mrgctx; // Deliberate fall-through
3011 case AOT_INIT_METHOD_GSHARED_VTABLE:
3012 /* mrgctx/vtable */
3013 if (!icall_id)
3014 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_vtable;
3015 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
3016 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
3017 break;
3018 case AOT_INIT_METHOD_GSHARED_THIS:
3019 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_this;
3020 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
3021 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
3022 break;
3023 default:
3024 g_assert_not_reached ();
3025 }
3026
3027 g_assert (icall_id);
3028 LLVMSetLinkage (func, LLVMInternalLinkage);
3029
3030 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3031
3032 set_cold_cconv (func);
3033
3034 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3035 builder = LLVMCreateBuilder ();
3036 LLVMPositionBuilderAtEnd (builder, entry_bb);
3037
3038 /* get_aotconst */
3039 ji = g_new0 (MonoJumpInfo, 1);
3040 ji->type = MONO_PATCH_INFO_AOT_MODULE;
3041 ji = mono_aot_patch_info_dup (ji);
3042 got_offset = compute_aot_got_offset (module, ji, IntPtrType ());
3043 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3044 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3045 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3046 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3047 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
3048 args [1] = LLVMGetParam (func, 0);
3049 if (subtype)
3050 args [2] = LLVMGetParam (func, 1);
3051
3052 ji = g_new0 (MonoJumpInfo, 1);
3053 ji->type = MONO_PATCH_INFO_JIT_ICALL_ID;
3054 ji->data.jit_icall_id = icall_id;
3055 ji = mono_aot_patch_info_dup (ji);
3056 got_offset = compute_aot_got_offset (module, ji, sig);
3057 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3058 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3059 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3060 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3061 callee = LLVMBuildLoad (builder, got_entry_addr, "");
3062 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
3063 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
3064
3065 // Set the inited flag
3066 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3067 indexes [1] = LLVMGetParam (func, 0);
3068 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
3069
3070 LLVMBuildRetVoid (builder);
3071
3072 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3073 LLVMDisposeBuilder (builder);
3074 return func;
3075 }
3076
3077 /* Emit a wrapper around the parameterless JIT icall ICALL_ID with a cold calling convention */
3078 static LLVMValueRef
3079 emit_icall_cold_wrapper (MonoLLVMModule *module, LLVMModuleRef lmodule, MonoJitICallId icall_id, gboolean aot)
3080 {
3081 LLVMValueRef func, callee;
3082 LLVMBasicBlockRef entry_bb;
3083 LLVMBuilderRef builder;
3084 LLVMTypeRef sig;
3085 char *name;
3086
3087 name = g_strdup_printf ("%s_icall_cold_wrapper_%d", module->global_prefix, icall_id);
3088
3089 func = LLVMAddFunction (lmodule, name, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3090 sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3091 LLVMSetLinkage (func, LLVMInternalLinkage);
3092 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3093 set_cold_cconv (func);
3094
3095 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3096 builder = LLVMCreateBuilder ();
3097 LLVMPositionBuilderAtEnd (builder, entry_bb);
3098
3099 if (aot) {
3100 callee = get_aotconst_typed_module (module, builder, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id), LLVMPointerType (sig, 0));
3101 } else {
3102 MonoJitICallInfo * const info = mono_find_jit_icall_info (icall_id);
3103 gpointer target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
3104
3105 LLVMValueRef tramp_var = LLVMAddGlobal (lmodule, LLVMPointerType (sig, 0), name);
3106 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (sig, 0)));
3107 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3108 callee = LLVMBuildLoad (builder, tramp_var, "");
3109 }
3110 LLVMBuildCall (builder, callee, NULL, 0, "");
3111
3112 LLVMBuildRetVoid (builder);
3113
3114 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3115 LLVMDisposeBuilder (builder);
3116 return func;
3117 }
3118
3119 /*
3120 * Emit wrappers around the C icalls used to initialize llvm methods, to
3121 * make the calling code smaller and to enable usage of the llvm
3122 * cold calling convention.
3123 */
3124 static void
3125 emit_init_icall_wrappers (MonoLLVMModule *module)
3126 {
3127 module->init_method = emit_init_icall_wrapper (module, AOT_INIT_METHOD);
3128 module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_MRGCTX);
3129 module->init_method_gshared_this = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_THIS);
3130 module->init_method_gshared_vtable = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_VTABLE);
3131 }
3132
3133 static LLVMValueRef
3134 get_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
3135 {
3136 switch (subtype) {
3137 case AOT_INIT_METHOD:
3138 return module->init_method;
3139 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3140 return module->init_method_gshared_mrgctx;
3141 case AOT_INIT_METHOD_GSHARED_THIS:
3142 return module->init_method_gshared_this;
3143 case AOT_INIT_METHOD_GSHARED_VTABLE:
3144 return module->init_method_gshared_vtable;
3145 default:
3146 g_assert_not_reached ();
3147 }
3148 }
3149
3150 static void
3151 emit_gc_safepoint_poll (MonoLLVMModule *module, LLVMModuleRef lmodule, MonoCompile *cfg)
3152 {
3153 gboolean is_aot = cfg == NULL || cfg->compile_aot;
3154 LLVMValueRef func = mono_llvm_get_or_insert_gc_safepoint_poll (lmodule);
3155 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3156 if (is_aot) {
3157 LLVMSetLinkage (func, LLVMWeakODRLinkage);
3158 } else {
3159 mono_llvm_add_func_attr (func, LLVM_ATTR_OPTIMIZE_NONE); // no need to waste time here, the function is already optimized and will be inlined.
3160 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE); // optnone attribute requires noinline (but it will be inlined anyway)
3161 if (!module->gc_poll_cold_wrapper_compiled) {
3162 ERROR_DECL (error);
3163 /* Compiling a method here is a bit ugly, but it works */
3164 MonoMethod *wrapper = mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL);
3165 module->gc_poll_cold_wrapper_compiled = mono_jit_compile_method (wrapper, error);
3166 mono_error_assert_ok (error);
3167 }
3168 }
3169 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.entry");
3170 LLVMBasicBlockRef poll_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.poll");
3171 LLVMBasicBlockRef exit_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.exit");
3172 LLVMTypeRef ptr_type = LLVMPointerType (IntPtrType (), 0);
3173 LLVMBuilderRef builder = LLVMCreateBuilder ();
3174
3175 /* entry: */
3176 LLVMPositionBuilderAtEnd (builder, entry_bb);
3177 LLVMValueRef poll_val_ptr;
3178 if (is_aot) {
3179 poll_val_ptr = get_aotconst_typed_module (module, builder, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, NULL, ptr_type);
3180 } else {
3181 LLVMValueRef poll_val_int = LLVMConstInt (IntPtrType (), (guint64) &mono_polling_required, FALSE);
3182 poll_val_ptr = LLVMBuildIntToPtr (builder, poll_val_int, ptr_type, "");
3183 }
3184 LLVMValueRef poll_val_ptr_load = LLVMBuildLoad (builder, poll_val_ptr, ""); // probably needs to be volatile
3185 LLVMValueRef poll_val = LLVMBuildPtrToInt (builder, poll_val_ptr_load, IntPtrType (), "");
3186 LLVMValueRef poll_val_zero = LLVMConstNull (LLVMTypeOf (poll_val));
3187 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, poll_val, poll_val_zero, "");
3188 mono_llvm_build_weighted_branch (builder, cmp, exit_bb, poll_bb, 1000 /* weight for exit_bb */, 1 /* weight for poll_bb */);
3189
3190 /* poll: */
3191 LLVMPositionBuilderAtEnd (builder, poll_bb);
3192 LLVMValueRef call;
3193 if (is_aot) {
3194 LLVMValueRef icall_wrapper = emit_icall_cold_wrapper (module, lmodule, MONO_JIT_ICALL_mono_threads_state_poll, TRUE);
3195 module->gc_poll_cold_wrapper = icall_wrapper;
3196 call = LLVMBuildCall (builder, icall_wrapper, NULL, 0, "");
3197 } else {
3198 // in JIT mode we have to emit @gc.safepoint_poll function for each method (module)
3199 // this function calls gc_poll_cold_wrapper_compiled via a global variable.
3200 // @gc.safepoint_poll will be inlined and can be deleted after -place-safepoints pass.
3201 LLVMTypeRef poll_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3202 LLVMTypeRef poll_sig_ptr = LLVMPointerType (poll_sig, 0);
3203 gpointer target = resolve_patch (cfg, MONO_PATCH_INFO_ABS, module->gc_poll_cold_wrapper_compiled);
3204 LLVMValueRef tramp_var = LLVMAddGlobal (lmodule, poll_sig_ptr, "mono_threads_state_poll");
3205 LLVMValueRef target_val = LLVMConstInt (LLVMInt64Type (), (guint64) target, FALSE);
3206 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (target_val, poll_sig_ptr));
3207 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3208 LLVMValueRef callee = LLVMBuildLoad (builder, tramp_var, "");
3209 call = LLVMBuildCall (builder, callee, NULL, 0, "");
3210 }
3211 set_call_cold_cconv (call);
3212 LLVMBuildBr (builder, exit_bb);
3213
3214 /* exit: */
3215 LLVMPositionBuilderAtEnd (builder, exit_bb);
3216 LLVMBuildRetVoid (builder);
3217 LLVMDisposeBuilder (builder);
3218 }
3219
3220 static void
3221 emit_llvm_code_end (MonoLLVMModule *module)
3222 {
3223 LLVMModuleRef lmodule = module->lmodule;
3224 LLVMValueRef func;
3225 LLVMBasicBlockRef entry_bb;
3226 LLVMBuilderRef builder;
3227
3228 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3229 LLVMSetLinkage (func, LLVMInternalLinkage);
3230 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3231 module->code_end = func;
3232 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3233 builder = LLVMCreateBuilder ();
3234 LLVMPositionBuilderAtEnd (builder, entry_bb);
3235 LLVMBuildRetVoid (builder);
3236 LLVMDisposeBuilder (builder);
3237 }
3238
3239 static void
3240 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
3241 {
3242 gboolean need_div_check = ctx->cfg->backend->need_div_check;
3243
3244 if (bb->region)
3245 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
3246 need_div_check = TRUE;
3247
3248 if (!need_div_check)
3249 return;
3250
3251 switch (ins->opcode) {
3252 case OP_IDIV:
3253 case OP_LDIV:
3254 case OP_IREM:
3255 case OP_LREM:
3256 case OP_IDIV_UN:
3257 case OP_LDIV_UN:
3258 case OP_IREM_UN:
3259 case OP_LREM_UN:
3260 case OP_IDIV_IMM:
3261 case OP_LDIV_IMM:
3262 case OP_IREM_IMM:
3263 case OP_LREM_IMM:
3264 case OP_IDIV_UN_IMM:
3265 case OP_LDIV_UN_IMM:
3266 case OP_IREM_UN_IMM:
3267 case OP_LREM_UN_IMM: {
3268 LLVMValueRef cmp;
3269 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
3270 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
3271
3272 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
3273 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp, FALSE);
3274 if (!ctx_ok (ctx))
3275 break;
3276 builder = ctx->builder;
3277
3278 /* b == -1 && a == 0x80000000 */
3279 if (is_signed) {
3280 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
3281 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
3282 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
3283
3284 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
3285 emit_cond_system_exception (ctx, bb, "OverflowException", cmp, FALSE);
3286 if (!ctx_ok (ctx))
3287 break;
3288 builder = ctx->builder;
3289 }
3290 break;
3291 }
3292 default:
3293 break;
3294 }
3295 }
3296
3297 /*
3298 * emit_init_method:
3299 *
3300 * Emit code to initialize the GOT slots used by the method.
3301 */
3302 static void
3303 emit_init_method (EmitContext *ctx)
3304 {
3305 LLVMValueRef indexes [16], args [16], callee;
3306 LLVMValueRef inited_var, cmp, call;
3307 LLVMBasicBlockRef inited_bb, notinited_bb;
3308 LLVMBuilderRef builder = ctx->builder;
3309 MonoCompile *cfg = ctx->cfg;
3310
3311 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
3312
3313 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3314 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
3315 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
3316
3317 //WASM doesn't support the "llvm.expect.i8" intrinsic
3318 #ifndef TARGET_WASM
3319 args [0] = inited_var;
3320 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
3321 inited_var = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I8), args, 2, "");
3322 #endif
3323
3324 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
3325
3326 inited_bb = ctx->inited_bb;
3327 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
3328
3329 ctx->cfg->llvmonly_init_cond = LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
3330
3331 builder = ctx->builder = create_builder (ctx);
3332 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
3333
3334 // FIXME: Cache
3335 if (ctx->rgctx_arg && ((cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) ||
3336 mini_method_is_default_method (cfg->method))) {
3337 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3338 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3339 callee = ctx->module->init_method_gshared_mrgctx;
3340 call = LLVMBuildCall (builder, callee, args, 2, "");
3341 } else if (ctx->rgctx_arg) {
3342 /* A vtable is passed as the rgctx argument */
3343 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3344 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3345 callee = ctx->module->init_method_gshared_vtable;
3346 call = LLVMBuildCall (builder, callee, args, 2, "");
3347 } else if (cfg->gshared) {
3348 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3349 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
3350 callee = ctx->module->init_method_gshared_this;
3351 call = LLVMBuildCall (builder, callee, args, 2, "");
3352 } else {
3353 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3354 callee = ctx->module->init_method;
3355 call = LLVMBuildCall (builder, callee, args, 1, "");
3356 }
3357
3358 /*
3359 * This enables llvm to keep arguments in their original registers/
3360 * scratch registers, since the call will not clobber them.
3361 */
3362 set_call_cold_cconv (call);
3363
3364 LLVMBuildBr (builder, inited_bb);
3365 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
3366
3367 builder = ctx->builder = create_builder (ctx);
3368 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
3369 }
3370
3371 static void
3372 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
3373 {
3374 /*
3375 * Emit unbox trampoline using a tailcall
3376 */
3377 LLVMValueRef tramp, call, *args;
3378 LLVMBuilderRef builder;
3379 LLVMBasicBlockRef lbb;
3380 LLVMCallInfo *linfo;
3381 char *tramp_name;
3382 int i, nargs;
3383
3384 tramp_name = g_strdup_printf ("ut_%s", method_name);
3385 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
3386 LLVMSetLinkage (tramp, LLVMInternalLinkage);
3387 mono_llvm_add_func_attr (tramp, LLVM_ATTR_OPTIMIZE_FOR_SIZE);
3388 //mono_llvm_add_func_attr (tramp, LLVM_ATTR_NO_UNWIND);
3389 linfo = ctx->linfo;
3390 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
3391 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3392 mono_llvm_add_param_attr (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVM_ATTR_IN_REG);
3393 if (ctx->cfg->vret_addr) {
3394 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
3395 if (linfo->ret.storage == LLVMArgVtypeByRef) {
3396 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
3397 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
3398 }
3399 }
3400
3401 lbb = LLVMAppendBasicBlock (tramp, "");
3402 builder = LLVMCreateBuilder ();
3403 LLVMPositionBuilderAtEnd (builder, lbb);
3404
3405 nargs = LLVMCountParamTypes (method_type);
3406 args = g_new0 (LLVMValueRef, nargs);
3407 for (i = 0; i < nargs; ++i) {
3408 args [i] = LLVMGetParam (tramp, i);
3409 if (i == ctx->this_arg_pindex) {
3410 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
3411
3412 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
3413 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), MONO_ABI_SIZEOF (MonoObject), FALSE), "");
3414 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
3415 }
3416 }
3417 call = LLVMBuildCall (builder, method, args, nargs, "");
3418 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3419 mono_llvm_add_instr_attr (call, 1 + ctx->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
3420 if (linfo->ret.storage == LLVMArgVtypeByRef)
3421 mono_llvm_add_instr_attr (call, 1 + linfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
3422
3423 // FIXME: This causes assertions in clang
3424 //mono_llvm_set_must_tailcall (call);
3425 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
3426 LLVMBuildRetVoid (builder);
3427 else
3428 LLVMBuildRet (builder, call);
3429
3430 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
3431 LLVMDisposeBuilder (builder);
3432 }
3433
3434 /*
3435 * emit_entry_bb:
3436 *
3437 * Emit code to load/convert arguments.
3438 */
3439 static void
3440 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
3441 {
3442 int i, j, pindex;
3443 MonoCompile *cfg = ctx->cfg;
3444 MonoMethodSignature *sig = ctx->sig;
3445 LLVMCallInfo *linfo = ctx->linfo;
3446 MonoBasicBlock *bb;
3447 char **names;
3448
3449 LLVMBuilderRef old_builder = ctx->builder;
3450 ctx->builder = builder;
3451
3452 ctx->alloca_builder = create_builder (ctx);
3453
3454 /*
3455 * Handle indirect/volatile variables by allocating memory for them
3456 * using 'alloca', and storing their address in a temporary.
3457 */
3458 for (i = 0; i < cfg->num_varinfo; ++i) {
3459 MonoInst *var = cfg->varinfo [i];
3460 LLVMTypeRef vtype;
3461
3462 if ((var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET))
3463 continue;
3464
3465 #ifdef TARGET_WASM
3466 // For GC stack scanning to work, have to spill all reference variables to the stack
3467 // Some ref variables have type intptr
3468 if (MONO_TYPE_IS_REFERENCE (var->inst_vtype) || var->inst_vtype->type == MONO_TYPE_I)
3469 var->flags |= MONO_INST_INDIRECT;
3470 #endif
3471
3472 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
3473 vtype = type_to_llvm_type (ctx, var->inst_vtype);
3474 if (!ctx_ok (ctx))
3475 return;
3476 /* Could be already created by an OP_VPHI */
3477 if (!ctx->addresses [var->dreg]) {
3478 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
3479 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
3480 }
3481 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
3482 }
3483 }
3484
3485 names = g_new (char *, sig->param_count);
3486 mono_method_get_param_names (cfg->method, (const char **) names);
3487
3488 for (i = 0; i < sig->param_count; ++i) {
3489 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
3490 int reg = cfg->args [i + sig->hasthis]->dreg;
3491 char *name;
3492
3493 pindex = ainfo->pindex;
3494
3495 switch (ainfo->storage) {
3496 case LLVMArgVtypeInReg:
3497 case LLVMArgAsFpArgs: {
3498 LLVMValueRef args [8];
3499 int j;
3500
3501 pindex += ainfo->ndummy_fpargs;
3502
3503 /* The argument is received as a set of int/fp arguments, store them into the real argument */
3504 memset (args, 0, sizeof (args));
3505 if (ainfo->storage == LLVMArgVtypeInReg) {
3506 args [0] = LLVMGetParam (ctx->lmethod, pindex);
3507 if (ainfo->pair_storage [1] != LLVMArgNone)
3508 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
3509 } else {
3510 g_assert (ainfo->nslots <= 8);
3511 for (j = 0; j < ainfo->nslots; ++j)
3512 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
3513 }
3514 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
3515
3516 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
3517 break;
3518 }
3519 case LLVMArgVtypeByVal: {
3520 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3521 break;
3522 }
3523 case LLVMArgVtypeAddr:
3524 case LLVMArgVtypeByRef: {
3525 /* The argument is passed by ref */
3526 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3527 break;
3528 }
3529 case LLVMArgAsIArgs: {
3530 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3531 int size;
3532 MonoType *t = mini_get_underlying_type (ainfo->type);
3533
3534 /* The argument is received as an array of ints, store it into the real argument */
3535 ctx->addresses [reg] = build_alloca (ctx, t);
3536
3537 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
3538 if (size == 0) {
3539 } else if (size < TARGET_SIZEOF_VOID_P) {
3540 /* The upper bits of the registers might not be valid */
3541 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
3542 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
3543 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
3544 } else {
3545 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
3546 }
3547 break;
3548 }
3549 case LLVMArgVtypeAsScalar:
3550 g_assert_not_reached ();
3551 break;
3552 case LLVMArgGsharedvtFixed: {
3553 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
3554 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3555
3556 if (names [i])
3557 name = g_strdup_printf ("arg_%s", names [i]);
3558 else
3559 name = g_strdup_printf ("arg_%d", i);
3560
3561 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
3562 break;
3563 }
3564 case LLVMArgGsharedvtFixedVtype: {
3565 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3566
3567 if (names [i])
3568 name = g_strdup_printf ("vtype_arg_%s", names [i]);
3569 else
3570 name = g_strdup_printf ("vtype_arg_%d", i);
3571
3572 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3573 g_assert (ctx->addresses [reg]);
3574 LLVMSetValueName (ctx->addresses [reg], name);
3575 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
3576 break;
3577 }
3578 case LLVMArgGsharedvtVariable:
3579 /* The IR treats these as variables with addresses */
3580 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3581 break;
3582 default: {
3583 LLVMTypeRef t;
3584 /* Needed to avoid phi argument mismatch errors since operations on pointers produce i32/i64 */
3585 if (ainfo->type->byref)
3586 t = IntPtrType ();
3587 else
3588 t = type_to_llvm_type (ctx, ainfo->type);
3589 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, t), type_is_unsigned (ctx, ainfo->type));
3590 break;
3591 }
3592 }
3593
3594 switch (ainfo->storage) {
3595 case LLVMArgVtypeInReg:
3596 case LLVMArgVtypeByVal:
3597 #ifdef ENABLE_NETCORE
3598 // FIXME: Enabling this fails on windows
3599 case LLVMArgVtypeAddr:
3600 case LLVMArgVtypeByRef:
3601 #endif
3602 {
3603 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type)))
3604 /* Treat these as normal values */
3605 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3606 break;
3607 }
3608 default:
3609 break;
3610 }
3611 }
3612 g_free (names);
3613
3614 if (cfg->vret_addr)
3615 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3616 if (sig->hasthis)
3617 emit_volatile_store (ctx, cfg->args [0]->dreg);
3618 for (i = 0; i < sig->param_count; ++i)
3619 if (!mini_type_is_vtype (sig->params [i]))
3620 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3621
3622 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
3623 LLVMValueRef this_alloc;
3624
3625 /*
3626 * The exception handling code needs the location where the this argument was
3627 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3628 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3629 * location into the LSDA.
3630 */
3631 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3632 /* This volatile store will keep the alloca alive */
3633 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3634
3635 set_metadata_flag (this_alloc, "mono.this");
3636 }
3637
3638 if (cfg->rgctx_var) {
3639 LLVMValueRef rgctx_alloc, store;
3640
3641 /*
3642 * We handle the rgctx arg similarly to the this pointer.
3643 */
3644 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3645 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3646 /* This volatile store will keep the alloca alive */
3647 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3648
3649 set_metadata_flag (rgctx_alloc, "mono.this");
3650 }
3651
3652 /* Initialize the method if needed */
3653 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
3654 /* Emit a location for the initialization code */
3655 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3656 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3657
3658 LLVMBuildBr (ctx->builder, ctx->init_bb);
3659 builder = ctx->builder = create_builder (ctx);
3660 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3661 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3662 }
3663
3664 /* Compute nesting between clauses */
3665 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3666 for (i = 0; i < cfg->header->num_clauses; ++i) {
3667 for (j = 0; j < cfg->header->num_clauses; ++j) {
3668 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3669 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3670
3671 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3672 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3673 }
3674 }
3675
3676 /*
3677 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3678 * it needs to continue normally, or return back to the exception handling system.
3679 */
3680 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3681 int clause_index;
3682 char name [128];
3683
3684 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3685 continue;
3686
3687 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3688 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
3689 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
3690
3691 if (bb->in_scount == 0) {
3692 LLVMValueRef val;
3693
3694 sprintf (name, "finally_ind_bb%d", bb->block_num);
3695 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3696 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3697
3698 ctx->bblocks [bb->block_num].finally_ind = val;
3699 } else {
3700 /* Create a variable to hold the exception var */
3701 if (!ctx->ex_var)
3702 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3703 }
3704
3705 /*
3706 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
3707 * LLVM bblock containing a landing pad causes problems for the
3708 * LLVM optimizer passes.
3709 */
3710 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
3711 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
3712 }
3713 ctx->builder = old_builder;
3714 }
3715
3716 static gboolean
3717 needs_extra_arg (EmitContext *ctx, MonoMethod *method)
3718 {
3719 WrapperInfo *info = NULL;
3720
3721 /*
3722 * When targeting wasm, the caller and callee signature has to match exactly. This means
3723 * that every method which can be called indirectly need an extra arg since the caller
3724 * will call it through an ftnptr and will pass an extra arg.
3725 */
3726 if (!ctx->cfg->llvm_only || !ctx->emit_dummy_arg)
3727 return FALSE;
3728 if (method->wrapper_type)
3729 info = mono_marshal_get_wrapper_info (method);
3730
3731 switch (method->wrapper_type) {
3732 case MONO_WRAPPER_OTHER:
3733 if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
3734 /* Already have an explicit extra arg */
3735 return FALSE;
3736 break;
3737 case MONO_WRAPPER_MANAGED_TO_NATIVE:
3738 if (strstr (method->name, "icall_wrapper"))
3739 /* These are JIT icall wrappers which are only called from JITted code directly */
3740 return FALSE;
3741 /* Normal icalls can be virtual methods which need an extra arg */
3742 break;
3743 case MONO_WRAPPER_RUNTIME_INVOKE:
3744 case MONO_WRAPPER_ALLOC:
3745 case MONO_WRAPPER_CASTCLASS:
3746 case MONO_WRAPPER_WRITE_BARRIER:
3747 case MONO_WRAPPER_NATIVE_TO_MANAGED:
3748 return FALSE;
3749 case MONO_WRAPPER_STELEMREF:
3750 if (info->subtype != WRAPPER_SUBTYPE_VIRTUAL_STELEMREF)
3751 return FALSE;
3752 break;
3753 case MONO_WRAPPER_MANAGED_TO_MANAGED:
3754 if (info->subtype == WRAPPER_SUBTYPE_STRING_CTOR)
3755 return FALSE;
3756 break;
3757 default:
3758 break;
3759 }
3760 if (method->string_ctor)
3761 return FALSE;
3762
3763 /* These are called from gsharedvt code with an indirect call which doesn't pass an extra arg */
3764 if (method->klass == mono_get_string_class () && (strstr (method->name, "memcpy") || strstr (method->name, "bzero")))
3765 return FALSE;
3766 return TRUE;
3767 }
3768
3769 static inline gboolean
3770 is_supported_callconv (EmitContext *ctx, MonoCallInst *call)
3771 {
3772 #if defined(TARGET_WIN32) && defined(TARGET_AMD64)
3773 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) ||
3774 (call->signature->call_convention == MONO_CALL_C) ||
3775 (call->signature->call_convention == MONO_CALL_STDCALL);
3776 #else
3777 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) || ((call->signature->call_convention == MONO_CALL_C) && ctx->llvm_only);
3778 #endif
3779 return result;
3780 }
3781
3782 static void
3783 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3784 {
3785 MonoCompile *cfg = ctx->cfg;
3786 LLVMValueRef *values = ctx->values;
3787 LLVMValueRef *addresses = ctx->addresses;
3788 MonoCallInst *call = (MonoCallInst*)ins;
3789 MonoMethodSignature *sig = call->signature;
3790 LLVMValueRef callee = NULL, lcall;
3791 LLVMValueRef *args;
3792 LLVMCallInfo *cinfo;
3793 GSList *l;
3794 int i, len, nargs;
3795 gboolean vretaddr;
3796 LLVMTypeRef llvm_sig;
3797 gpointer target;
3798 gboolean is_virtual, calli;
3799 LLVMBuilderRef builder = *builder_ref;
3800
3801 /* If both imt and rgctx arg are required, only pass the imt arg, the rgctx trampoline will pass the rgctx */
3802 if (call->imt_arg_reg)
3803 call->rgctx_arg_reg = 0;
3804
3805 if (!is_supported_callconv (ctx, call)) {
3806 set_failure (ctx, "non-default callconv");
3807 return;
3808 }
3809
3810 cinfo = call->cinfo;
3811 g_assert (cinfo);
3812 if (call->rgctx_arg_reg)
3813 cinfo->rgctx_arg = TRUE;
3814 if (call->imt_arg_reg)
3815 cinfo->imt_arg = TRUE;
3816 if (!call->rgctx_arg_reg && call->method && needs_extra_arg (ctx, call->method))
3817 cinfo->dummy_arg = TRUE;
3818
3819 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3820
3821 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3822 if (!ctx_ok (ctx))
3823 return;
3824
3825 int const opcode = ins->opcode;
3826
3827 is_virtual = opcode == OP_VOIDCALL_MEMBASE || opcode == OP_CALL_MEMBASE
3828 || opcode == OP_VCALL_MEMBASE || opcode == OP_LCALL_MEMBASE
3829 || opcode == OP_FCALL_MEMBASE || opcode == OP_RCALL_MEMBASE
3830 || opcode == OP_TAILCALL_MEMBASE;
3831 calli = !call->fptr_is_patch && (opcode == OP_VOIDCALL_REG || opcode == OP_CALL_REG
3832 || opcode == OP_VCALL_REG || opcode == OP_LCALL_REG || opcode == OP_FCALL_REG
3833 || opcode == OP_RCALL_REG || opcode == OP_TAILCALL_REG);
3834
3835 /* FIXME: Avoid creating duplicate methods */
3836
3837 if (ins->flags & MONO_INST_HAS_METHOD) {
3838 if (is_virtual) {
3839 callee = NULL;
3840 } else {
3841 if (cfg->compile_aot) {
3842 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3843 if (!callee) {
3844 set_failure (ctx, "can't encode patch");
3845 return;
3846 }
3847 } else if (cfg->method == call->method) {
3848 callee = ctx->lmethod;
3849 } else {
3850 ERROR_DECL (error);
3851 static int tramp_index;
3852 char *name;
3853
3854 name = g_strdup_printf ("[tramp_%d] %s", tramp_index, mono_method_full_name (call->method, TRUE));
3855 tramp_index ++;
3856
3857 /*
3858 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3859 * Make all calls through a global. The address of the global will be saved in
3860 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3861 * compiled.
3862 */
3863 LLVMValueRef tramp_var = (LLVMValueRef)g_hash_table_lookup (ctx->jit_callees, call->method);
3864 if (!tramp_var) {
3865 target =
3866 mono_create_jit_trampoline (mono_domain_get (),
3867 call->method, error);
3868 if (!is_ok (error)) {
3869 set_failure (ctx, mono_error_get_message (error));
3870 mono_error_cleanup (error);
3871 return;
3872 }
3873
3874 tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
3875 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
3876 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3877 g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
3878 }
3879 callee = LLVMBuildLoad (builder, tramp_var, "");
3880 }
3881 }
3882
3883 if (!cfg->llvm_only && call->method && strstr (m_class_get_name (call->method->klass), "AsyncVoidMethodBuilder")) {
3884 /* LLVM miscompiles async methods */
3885 set_failure (ctx, "#13734");
3886 return;
3887 }
3888 } else if (calli) {
3889 } else {
3890 const MonoJitICallId jit_icall_id = call->jit_icall_id;
3891
3892 if (jit_icall_id) {
3893 if (cfg->compile_aot) {
3894 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3895 if (!callee) {
3896 set_failure (ctx, "can't encode patch");
3897 return;
3898 }
3899 } else {
3900 callee = get_jit_callee (ctx, "", llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3901 }
3902 } else {
3903 if (cfg->compile_aot) {
3904 callee = NULL;
3905 if (cfg->abs_patches) {
3906 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3907 if (abs_ji) {
3908 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3909 if (!callee) {
3910 set_failure (ctx, "can't encode patch");
3911 return;
3912 }
3913 }
3914 }
3915 if (!callee) {
3916 set_failure (ctx, "aot");
3917 return;
3918 }
3919 } else {
3920 if (cfg->abs_patches) {
3921 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3922 if (abs_ji) {
3923 ERROR_DECL (error);
3924
3925 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, error);
3926 mono_error_assert_ok (error);
3927 callee = get_jit_callee (ctx, "", llvm_sig, abs_ji->type, abs_ji->data.target);
3928 } else {
3929 g_assert_not_reached ();
3930 }
3931 } else {
3932 g_assert_not_reached ();
3933 }
3934 }
3935 }
3936 }
3937
3938 if (is_virtual) {
3939 int size = TARGET_SIZEOF_VOID_P;
3940 LLVMValueRef index;
3941
3942 g_assert (ins->inst_offset % size == 0);
3943 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3944
3945 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3946 } else if (calli) {
3947 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3948 } else {
3949 if (ins->flags & MONO_INST_HAS_METHOD) {
3950 }
3951 }
3952
3953 /*
3954 * Collect and convert arguments
3955 */
3956 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg + call->cinfo->dummy_arg + 1;
3957 len = sizeof (LLVMValueRef) * nargs;
3958 args = g_newa (LLVMValueRef, nargs);
3959 memset (args, 0, len);
3960 l = call->out_ireg_args;
3961
3962 if (call->rgctx_arg_reg) {
3963 g_assert (values [call->rgctx_arg_reg]);
3964 g_assert (cinfo->rgctx_arg_pindex < nargs);
3965 /*
3966 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3967 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3968 * it using a volatile load.
3969 */
3970 #ifdef TARGET_ARM
3971 if (!ctx->imt_rgctx_loc)
3972 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3973 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3974 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3975 #else
3976 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3977 #endif
3978 }
3979 if (call->imt_arg_reg) {
3980 g_assert (!ctx->llvm_only);
3981 g_assert (values [call->imt_arg_reg]);
3982 g_assert (cinfo->imt_arg_pindex < nargs);
3983 #ifdef TARGET_ARM
3984 if (!ctx->imt_rgctx_loc)
3985 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3986 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3987 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3988 #else
3989 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3990 #endif
3991 }
3992 switch (cinfo->ret.storage) {
3993 case LLVMArgGsharedvtVariable: {
3994 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
3995
3996 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
3997 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
3998 } else {
3999 g_assert (addresses [call->inst.dreg]);
4000 args [cinfo->vret_arg_pindex] = convert (ctx, addresses [call->inst.dreg], IntPtrType ());
4001 }
4002 break;
4003 }
4004 default:
4005 if (vretaddr) {
4006 if (!addresses [call->inst.dreg])
4007 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4008 g_assert (cinfo->vret_arg_pindex < nargs);
4009 if (cinfo->ret.storage == LLVMArgVtypeByRef)
4010 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
4011 else
4012 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
4013 }
4014 break;
4015 }
4016
4017 /*
4018 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
4019 * use the real callee for argument type conversion.
4020 */
4021 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
4022 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
4023 LLVMGetParamTypes (callee_type, param_types);
4024
4025 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
4026 guint32 regpair;
4027 int reg, pindex;
4028 LLVMArgInfo *ainfo = &call->cinfo->args [i];
4029
4030 pindex = ainfo->pindex;
4031
4032 regpair = (guint32)(gssize)(l->data);
4033 reg = regpair & 0xffffff;
4034 args [pindex] = values [reg];
4035 switch (ainfo->storage) {
4036 case LLVMArgVtypeInReg:
4037 case LLVMArgAsFpArgs: {
4038 guint32 nargs;
4039 int j;
4040
4041 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
4042 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
4043 pindex += ainfo->ndummy_fpargs;
4044
4045 g_assert (addresses [reg]);
4046 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
4047 pindex += nargs;
4048
4049 // FIXME: alignment
4050 // FIXME: Get rid of the VMOVE
4051 break;
4052 }
4053 case LLVMArgVtypeByVal:
4054 g_assert (addresses [reg]);
4055 args [pindex] = addresses [reg];
4056 break;
4057 case LLVMArgVtypeAddr :
4058 case LLVMArgVtypeByRef: {
4059 g_assert (addresses [reg]);
4060 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4061 break;
4062 }
4063 case LLVMArgAsIArgs:
4064 g_assert (addresses [reg]);
4065 if (ainfo->esize == 8)
4066 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (LLVMInt64Type (), ainfo->nslots), 0)), "");
4067 else
4068 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
4069 break;
4070 case LLVMArgVtypeAsScalar:
4071 g_assert_not_reached ();
4072 break;
4073 case LLVMArgGsharedvtFixed:
4074 case LLVMArgGsharedvtFixedVtype:
4075 g_assert (addresses [reg]);
4076 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4077 break;
4078 case LLVMArgGsharedvtVariable:
4079 g_assert (addresses [reg]);
4080 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
4081 break;
4082 default:
4083 g_assert (args [pindex]);
4084 if (i == 0 && sig->hasthis)
4085 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
4086 else
4087 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
4088 break;
4089 }
4090 g_assert (pindex <= nargs);
4091
4092 l = l->next;
4093 }
4094
4095
4096 if (call->cinfo->dummy_arg) {
4097 g_assert (call->cinfo->dummy_arg_pindex < nargs);
4098 args [call->cinfo->dummy_arg_pindex] = LLVMConstNull (ctx->module->ptr_type);
4099 }
4100
4101 // FIXME: Align call sites
4102
4103 /*
4104 * Emit the call
4105 */
4106 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
4107
4108 mono_llvm_nonnull_state_update (ctx, lcall, call->method, args, LLVMCountParamTypes (llvm_sig));
4109
4110 // If we just allocated an object, it's not null.
4111 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC) {
4112 mono_llvm_set_call_nonnull_ret (lcall);
4113 }
4114
4115 if (ins->opcode != OP_TAILCALL && ins->opcode != OP_TAILCALL_MEMBASE && LLVMGetInstructionOpcode (lcall) == LLVMCall)
4116 mono_llvm_set_call_notailcall (lcall);
4117
4118 // Add original method name we are currently emitting as a custom string metadata (the only way to leave comments in LLVM IR)
4119 if (mono_debug_enabled () && call && call->method)
4120 mono_llvm_add_string_metadata (lcall, "managed_name", mono_method_full_name (call->method, TRUE));
4121
4122 // As per the LLVM docs, a function has a noalias return value if and only if
4123 // it is an allocation function. This is an allocation function.
4124 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC) {
4125 mono_llvm_set_call_noalias_ret (lcall);
4126 // All objects are expected to be 8-byte aligned (SGEN_ALLOC_ALIGN)
4127 mono_llvm_set_alignment_ret (lcall, 8);
4128 }
4129
4130 /*
4131 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
4132 */
4133 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
4134 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
4135 #endif
4136 /* The two can't be used together, so use only one LLVM calling conv to pass them */
4137 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
4138 if (!sig->pinvoke && !cfg->llvm_only)
4139 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
4140
4141 if (cinfo->ret.storage == LLVMArgVtypeByRef)
4142 mono_llvm_add_instr_attr (lcall, 1 + cinfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
4143 if (!ctx->llvm_only && call->rgctx_arg_reg)
4144 mono_llvm_add_instr_attr (lcall, 1 + cinfo->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
4145 if (call->imt_arg_reg)
4146 mono_llvm_add_instr_attr (lcall, 1 + cinfo->imt_arg_pindex, LLVM_ATTR_IN_REG);
4147
4148 /* Add byval attributes if needed */
4149 for (i = 0; i < sig->param_count; ++i) {
4150 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
4151
4152 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
4153 mono_llvm_add_instr_attr (lcall, 1 + ainfo->pindex, LLVM_ATTR_BY_VAL);
4154 }
4155
4156 /*
4157 * Convert the result
4158 */
4159 switch (cinfo->ret.storage) {
4160 case LLVMArgVtypeInReg: {
4161 LLVMValueRef regs [2];
4162
4163 if (LLVMTypeOf (lcall) == LLVMVoidType ())
4164 /* Empty struct */
4165 break;
4166
4167 if (!addresses [ins->dreg])
4168 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
4169
4170 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
4171 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
4172 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
4173 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
4174 break;
4175 }
4176 case LLVMArgVtypeByVal:
4177 if (!addresses [call->inst.dreg])
4178 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4179 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
4180 break;
4181 case LLVMArgAsIArgs:
4182 case LLVMArgFpStruct:
4183 if (!addresses [call->inst.dreg])
4184 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4185 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4186 break;
4187 case LLVMArgVtypeAsScalar:
4188 if (!addresses [call->inst.dreg])
4189 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4190 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4191 break;
4192 case LLVMArgVtypeRetAddr:
4193 case LLVMArgVtypeByRef:
4194 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret))) {
4195 /* Some opcodes like STOREX_MEMBASE access these by value */
4196 g_assert (addresses [call->inst.dreg]);
4197 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4198 }
4199 break;
4200 case LLVMArgGsharedvtVariable:
4201 break;
4202 case LLVMArgGsharedvtFixed:
4203 case LLVMArgGsharedvtFixedVtype:
4204 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4205 break;
4206 default:
4207 if (sig->ret->type != MONO_TYPE_VOID)
4208 /* If the method returns an unsigned value, need to zext it */
4209 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
4210 break;
4211 }
4212
4213 *builder_ref = ctx->builder;
4214 }
4215
4216 static void
4217 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4218 {
4219 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception : MONO_JIT_ICALL_mono_llvm_throw_exception;
4220 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
4221
4222 LLVMTypeRef exc_type = type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_exception_class ()));
4223
4224 if (!callee) {
4225 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
4226
4227 g_assert (ctx->cfg->compile_aot);
4228 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (icall_id));
4229 }
4230
4231 LLVMValueRef args [2];
4232
4233 args [0] = convert (ctx, exc, exc_type);
4234 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
4235
4236 LLVMBuildUnreachable (ctx->builder);
4237
4238 ctx->builder = create_builder (ctx);
4239 }
4240
4241 static void
4242 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4243 {
4244 MonoMethodSignature *throw_sig;
4245
4246 LLVMValueRef * const pcallee = rethrow ? &ctx->module->rethrow : &ctx->module->throw_icall;
4247 LLVMValueRef callee = *pcallee;
4248 char const * const icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4249 #ifndef TARGET_X86
4250 const
4251 #endif
4252 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_arch_rethrow_exception : MONO_JIT_ICALL_mono_arch_throw_exception;
4253
4254 if (!callee) {
4255 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
4256 throw_sig->ret = m_class_get_byval_arg (mono_get_void_class ());
4257 throw_sig->params [0] = m_class_get_byval_arg (mono_get_object_class ());
4258 if (ctx->cfg->compile_aot) {
4259 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4260 } else {
4261 #ifdef TARGET_X86
4262 /*
4263 * LLVM doesn't push the exception argument, so we need a different
4264 * trampoline.
4265 */
4266 icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception_trampoline : MONO_JIT_ICALL_mono_llvm_throw_exception_trampoline;
4267 #endif
4268 callee = get_jit_callee (ctx, icall_name, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4269 }
4270
4271 mono_memory_barrier ();
4272 }
4273 LLVMValueRef arg;
4274 arg = convert (ctx, exc, type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_object_class ())));
4275 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
4276 }
4277
4278 static void
4279 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
4280 {
4281 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_resume_exception;
4282 LLVMValueRef callee;
4283
4284 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
4285
4286 g_assert (ctx->cfg->compile_aot);
4287 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4288
4289 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
4290
4291 LLVMBuildUnreachable (ctx->builder);
4292
4293 ctx->builder = create_builder (ctx);
4294 }
4295
4296 static LLVMValueRef
4297 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4298 {
4299 const char *icall_name = "mono_llvm_clear_exception";
4300 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_clear_exception;
4301
4302 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
4303 LLVMValueRef callee = NULL;
4304
4305 if (!callee) {
4306 if (ctx->cfg->compile_aot) {
4307 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4308 } else {
4309 // FIXME: This is broken.
4310 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4311 }
4312 }
4313
4314 g_assert (builder && callee);
4315
4316 return LLVMBuildCall (builder, callee, NULL, 0, "");
4317 }
4318
4319 static LLVMValueRef
4320 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4321 {
4322 const char *icall_name = "mono_llvm_load_exception";
4323 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_load_exception;
4324
4325 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
4326 LLVMValueRef callee = NULL;
4327
4328 if (!callee) {
4329 if (ctx->cfg->compile_aot) {
4330 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4331 } else {
4332 // FIXME: This is broken.
4333 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4334 }
4335 }
4336
4337 g_assert (builder && callee);
4338
4339 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
4340 }
4341
4342
4343 static LLVMValueRef
4344 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
4345 {
4346 const char *icall_name = "mono_llvm_match_exception";
4347 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_match_exception;
4348
4349 ctx->builder = builder;
4350
4351 LLVMValueRef args[5];
4352 const int num_args = G_N_ELEMENTS (args);
4353
4354 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
4355 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
4356 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
4357 if (ctx->cfg->rgctx_var) {
4358 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
4359 g_assert (rgctx_alloc);
4360 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
4361 } else {
4362 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
4363 }
4364 if (ctx->this_arg)
4365 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
4366 else
4367 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
4368
4369 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
4370 LLVMValueRef callee;
4371 g_assert (ctx->cfg->compile_aot);
4372 ctx->builder = builder;
4373 // get_callee expects ctx->builder to be the emitting builder
4374 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4375
4376 g_assert (builder && callee);
4377
4378 g_assert (ctx->ex_var);
4379
4380 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
4381 }
4382
4383 // FIXME: This won't work because the code-finding makes this
4384 // not a constant.
4385 /*#define MONO_PERSONALITY_DEBUG*/
4386
4387 #ifdef MONO_PERSONALITY_DEBUG
4388 static const gboolean use_mono_personality_debug = TRUE;
4389 static const char *default_personality_name = "mono_debug_personality";
4390 #else
4391 static const gboolean use_mono_personality_debug = FALSE;
4392 static const char *default_personality_name = "__gxx_personality_v0";
4393 #endif
4394
4395 static LLVMTypeRef
4396 default_cpp_lpad_exc_signature (void)
4397 {
4398 static gboolean inited = FALSE;
4399 static LLVMTypeRef sig;
4400
4401 if (!sig) {
4402 LLVMTypeRef signature [2];
4403 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
4404 signature [1] = LLVMInt32Type ();
4405 sig = LLVMStructType (signature, 2, FALSE);
4406 inited = TRUE;
4407 }
4408
4409 return sig;
4410 }
4411
4412 static LLVMValueRef
4413 get_mono_personality (EmitContext *ctx)
4414 {
4415 LLVMValueRef personality = NULL;
4416 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4417
4418 g_assert (ctx->cfg->compile_aot);
4419 if (!use_mono_personality_debug) {
4420 personality = get_intrins_by_name (ctx, default_personality_name);
4421 } else {
4422 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_debug_personality));
4423 }
4424
4425 g_assert (personality);
4426 return personality;
4427 }
4428
4429 static LLVMBasicBlockRef
4430 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
4431 {
4432 MonoCompile *cfg = ctx->cfg;
4433 LLVMBuilderRef old_builder = ctx->builder;
4434 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
4435
4436 LLVMBuilderRef lpadBuilder = create_builder (ctx);
4437 ctx->builder = lpadBuilder;
4438
4439 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
4440 g_assert (handler_bb);
4441
4442 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4443 LLVMValueRef personality = get_mono_personality (ctx);
4444 g_assert (personality);
4445
4446 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
4447 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
4448 g_free (bb_name);
4449 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4450 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
4451 g_assert (landing_pad);
4452
4453 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
4454 LLVMAddClause (landing_pad, cast);
4455
4456 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
4457 LLVMBuilderRef resume_builder = create_builder (ctx);
4458 ctx->builder = resume_builder;
4459 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
4460
4461 emit_resume_eh (ctx, handler_bb);
4462
4463 // Build match
4464 ctx->builder = lpadBuilder;
4465 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4466
4467 gboolean finally_only = TRUE;
4468
4469 MonoExceptionClause *group_cursor = group_start;
4470
4471 for (int i = 0; i < group_size; i ++) {
4472 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY || group_cursor->flags & MONO_EXCEPTION_CLAUSE_FAULT))
4473 finally_only = FALSE;
4474
4475 group_cursor++;
4476 }
4477
4478 // FIXME:
4479 // Handle landing pad inlining
4480
4481 if (!finally_only) {
4482 // So at each level of the exception stack we will match the exception again.
4483 // During that match, we need to compare against the handler types for the current
4484 // protected region. We send the try start and end so that we can only check against
4485 // handlers for this lexical protected region.
4486 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
4487
4488 // if returns -1, resume
4489 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
4490
4491 // else move to that target bb
4492 for (int i = 0; i < group_size; i++) {
4493 MonoExceptionClause *clause = group_start + i;
4494 int clause_index = clause - cfg->header->clauses;
4495 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4496 g_assert (handler_bb);
4497 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4498 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4499 }
4500 } else {
4501 int clause_index = group_start - cfg->header->clauses;
4502 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4503 g_assert (finally_bb);
4504
4505 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
4506 }
4507
4508 ctx->builder = old_builder;
4509
4510 return lpad_bb;
4511 }
4512
4513
4514 static void
4515 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
4516 {
4517 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
4518 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
4519
4520 // Make exception available to catch blocks
4521 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags & MONO_EXCEPTION_CLAUSE_FAULT)) {
4522 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
4523
4524 g_assert (ctx->ex_var);
4525 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
4526
4527 if (bb->in_scount == 1) {
4528 MonoInst *exvar = bb->in_stack [0];
4529 g_assert (!ctx->values [exvar->dreg]);
4530 g_assert (ctx->ex_var);
4531 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
4532 emit_volatile_store (ctx, exvar->dreg);
4533 }
4534
4535 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
4536 }
4537
4538 LLVMBuilderRef handler_builder = create_builder (ctx);
4539 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
4540 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
4541
4542 // Make the handler code end with a jump to cbb
4543 LLVMBuildBr (handler_builder, cbb);
4544 }
4545
4546 static void
4547 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
4548 {
4549 MonoCompile *cfg = ctx->cfg;
4550 LLVMValueRef *values = ctx->values;
4551 LLVMModuleRef lmodule = ctx->lmodule;
4552 BBInfo *bblocks = ctx->bblocks;
4553 LLVMTypeRef i8ptr;
4554 LLVMValueRef personality;
4555 LLVMValueRef landing_pad;
4556 LLVMBasicBlockRef target_bb;
4557 MonoInst *exvar;
4558 static int ti_generator;
4559 char ti_name [128];
4560 LLVMValueRef type_info;
4561 int clause_index;
4562 GSList *l;
4563
4564 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4565
4566 if (cfg->compile_aot) {
4567 /* Use a dummy personality function */
4568 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
4569 g_assert (personality);
4570 } else {
4571 /* Can't cache this as each method is in its own llvm module */
4572 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4573 personality = LLVMAddFunction (ctx->lmodule, "mono_personality", personality_type);
4574 mono_llvm_add_func_attr (personality, LLVM_ATTR_NO_UNWIND);
4575 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (personality, "ENTRY");
4576 LLVMBuilderRef builder2 = LLVMCreateBuilder ();
4577 LLVMPositionBuilderAtEnd (builder2, entry_bb);
4578 LLVMBuildRet (builder2, LLVMConstInt (LLVMInt32Type (), 0, FALSE));
4579 LLVMDisposeBuilder (builder2);
4580 }
4581
4582 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
4583
4584 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
4585
4586 /*
4587 * Create the type info
4588 */
4589 sprintf (ti_name, "type_info_%d", ti_generator);
4590 ti_generator ++;
4591
4592 if (cfg->compile_aot) {
4593 /* decode_eh_frame () in aot-runtime.c will decode this */
4594 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4595 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4596
4597 /*
4598 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4599 */
4600 LLVMSetLinkage (type_info, LLVMInternalLinkage);
4601 } else {
4602 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4603 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4604 }
4605
4606 {
4607 LLVMTypeRef members [2], ret_type;
4608
4609 members [0] = i8ptr;
4610 members [1] = LLVMInt32Type ();
4611 ret_type = LLVMStructType (members, 2, FALSE);
4612
4613 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
4614 LLVMAddClause (landing_pad, type_info);
4615
4616 /* Store the exception into the exvar */
4617 if (ctx->ex_var)
4618 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
4619 }
4620
4621 /*
4622 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4623 * code expects control to be transferred to this landing pad even in the
4624 * presence of nested clauses. The landing pad needs to branch to the landing
4625 * pads belonging to nested clauses based on the selector value returned by
4626 * the landing pad instruction, which is passed to the landing pad in a
4627 * register by the EH code.
4628 */
4629 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4630 g_assert (target_bb);
4631
4632 /*
4633 * Branch to the correct landing pad
4634 */
4635 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
4636 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
4637
4638 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
4639 int nesting_clause_index = GPOINTER_TO_INT (l->data);
4640 MonoBasicBlock *handler_bb;
4641
4642 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
4643 g_assert (handler_bb);
4644
4645 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4646 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4647 }
4648
4649 /* Start a new bblock which CALL_HANDLER can branch to */
4650 ctx->builder = builder = create_builder (ctx);
4651 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
4652
4653 ctx->bblocks [bb->block_num].end_bblock = target_bb;
4654
4655 /* Store the exception into the IL level exvar */
4656 if (bb->in_scount == 1) {
4657 g_assert (bb->in_scount == 1);
4658 exvar = bb->in_stack [0];
4659
4660 // FIXME: This is shared with filter clauses ?
4661 g_assert (!values [exvar->dreg]);
4662
4663 g_assert (ctx->ex_var);
4664 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
4665 emit_volatile_store (ctx, exvar->dreg);
4666 }
4667
4668 /* Make normal branches to the start of the clause branch to the new bblock */
4669 bblocks [bb->block_num].bblock = target_bb;
4670 }
4671
4672 //Wasm requires us to canonicalize NaNs.
4673 static LLVMValueRef
4674 get_double_const (MonoCompile *cfg, double val)
4675 {
4676 #ifdef TARGET_WASM
4677 if (mono_isnan (val))
4678 *(gint64 *)&val = 0x7FF8000000000000ll;
4679 #endif
4680 return LLVMConstReal (LLVMDoubleType (), val);
4681 }
4682
4683 static LLVMValueRef
4684 get_float_const (MonoCompile *cfg, float val)
4685 {
4686 #ifdef TARGET_WASM
4687 if (mono_isnan (val))
4688 *(int *)&val = 0x7FC00000;
4689 #endif
4690 if (cfg->r4fp)
4691 return LLVMConstReal (LLVMFloatType (), val);
4692 else
4693 return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val), LLVMDoubleType ());
4694 }
4695
4696 static void
4697 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4698 {
4699 MonoCompile *cfg = ctx->cfg;
4700 MonoMethodSignature *sig = ctx->sig;
4701 LLVMValueRef method = ctx->lmethod;
4702 LLVMValueRef *values = ctx->values;
4703 LLVMValueRef *addresses = ctx->addresses;
4704 LLVMCallInfo *linfo = ctx->linfo;
4705 BBInfo *bblocks = ctx->bblocks;
4706 MonoInst *ins;
4707 LLVMBasicBlockRef cbb;
4708 LLVMBuilderRef builder, starting_builder;
4709 gboolean has_terminator;
4710 LLVMValueRef v;
4711 LLVMValueRef lhs, rhs;
4712 int nins = 0;
4713
4714 cbb = get_end_bb (ctx, bb);
4715
4716 builder = create_builder (ctx);
4717 ctx->builder = builder;
4718 LLVMPositionBuilderAtEnd (builder, cbb);
4719
4720 if (!ctx_ok (ctx))
4721 return;
4722
4723 if (bb->flags & BB_EXCEPTION_HANDLER) {
4724 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4725 set_failure (ctx, "handler without invokes");
4726 return;
4727 }
4728
4729 if (ctx->llvm_only)
4730 emit_llvmonly_handler_start (ctx, bb, cbb);
4731 else
4732 emit_handler_start (ctx, bb, builder);
4733 if (!ctx_ok (ctx))
4734 return;
4735 builder = ctx->builder;
4736 }
4737
4738 /* Handle PHI nodes first */
4739 /* They should be grouped at the start of the bb */
4740 for (ins = bb->code; ins; ins = ins->next) {
4741 emit_dbg_loc (ctx, builder, ins->cil_code);
4742
4743 if (ins->opcode == OP_NOP)
4744 continue;
4745 if (!MONO_IS_PHI (ins))
4746 break;
4747
4748 int i;
4749 gboolean empty = TRUE;
4750
4751 /* Check that all input bblocks really branch to us */
4752 for (i = 0; i < bb->in_count; ++i) {
4753 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4754 ins->inst_phi_args [i + 1] = -1;
4755 else
4756 empty = FALSE;
4757 }
4758
4759 if (empty) {
4760 /* LLVM doesn't like phi instructions with zero operands */
4761 ctx->is_dead [ins->dreg] = TRUE;
4762 continue;
4763 }
4764
4765 /* Created earlier, insert it now */
4766 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4767
4768 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4769 int sreg1 = ins->inst_phi_args [i + 1];
4770 int count, j;
4771
4772 /*
4773 * Count the number of times the incoming bblock branches to us,
4774 * since llvm requires a separate entry for each.
4775 */
4776 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4777 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4778
4779 count = 0;
4780 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4781 if (switch_ins->inst_many_bb [j] == bb)
4782 count ++;
4783 }
4784 } else {
4785 count = 1;
4786 }
4787
4788 /* Remember for later */
4789 for (j = 0; j < count; ++j) {
4790 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4791 node->bb = bb;
4792 node->phi = ins;
4793 node->in_bb = bb->in_bb [i];
4794 node->sreg = sreg1;
4795 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4796 }
4797 }
4798 }
4799 // Add volatile stores for PHI nodes
4800 // These need to be emitted after the PHI nodes
4801 for (ins = bb->code; ins; ins = ins->next) {
4802 const char *spec = LLVM_INS_INFO (ins->opcode);
4803
4804 if (ins->opcode == OP_NOP)
4805 continue;
4806 if (!MONO_IS_PHI (ins))
4807 break;
4808
4809 if (spec [MONO_INST_DEST] != 'v')
4810 emit_volatile_store (ctx, ins->dreg);
4811 }
4812
4813 has_terminator = FALSE;
4814 starting_builder = builder;
4815 for (ins = bb->code; ins; ins = ins->next) {
4816 const char *spec = LLVM_INS_INFO (ins->opcode);
4817 char *dname = NULL;
4818 char dname_buf [128];
4819
4820 emit_dbg_loc (ctx, builder, ins->cil_code);
4821
4822 nins ++;
4823 if (nins > 1000) {
4824 /*
4825 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4826 * Start a new bblock.
4827 * Prevent the bblocks to be merged by doing a volatile load + cond branch
4828 * from localloc-ed memory.
4829 */
4830 if (!cfg->llvm_only)
4831 ;//set_failure (ctx, "basic block too long");
4832
4833 if (!ctx->long_bb_break_var) {
4834 ctx->long_bb_break_var = build_alloca_llvm_type_name (ctx, LLVMInt32Type (), 0, "long_bb_break");
4835 mono_llvm_build_store (ctx->alloca_builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4836 }
4837
4838 cbb = gen_bb (ctx, "CONT_LONG_BB");
4839 LLVMBasicBlockRef dummy_bb = gen_bb (ctx, "CONT_LONG_BB_DUMMY");
4840
4841 LLVMValueRef load = mono_llvm_build_load (builder, ctx->long_bb_break_var, "", TRUE);
4842 /*
4843 * The long_bb_break_var is initialized to 0 in the prolog, so this branch will always go to 'cbb'
4844 * but llvm doesn't know that, so the branch is not going to be eliminated.
4845 */
4846 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, load, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4847
4848 LLVMBuildCondBr (builder, cmp, cbb, dummy_bb);
4849
4850 /* Emit a dummy false bblock which does nothing but contains a volatile store so it cannot be eliminated */
4851 ctx->builder = builder = create_builder (ctx);
4852 LLVMPositionBuilderAtEnd (builder, dummy_bb);
4853 mono_llvm_build_store (builder, LLVMConstInt (LLVMInt32Type (), 1, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4854 LLVMBuildBr (builder, cbb);
4855
4856 ctx->builder = builder = create_builder (ctx);
4857 LLVMPositionBuilderAtEnd (builder, cbb);
4858 ctx->bblocks [bb->block_num].end_bblock = cbb;
4859 nins = 0;
4860
4861 emit_dbg_loc (ctx, builder, ins->cil_code);
4862 }
4863
4864 if (has_terminator)
4865 /* There could be instructions after a terminator, skip them */
4866 break;
4867
4868 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4869 sprintf (dname_buf, "t%d", ins->dreg);
4870 dname = dname_buf;
4871 }
4872
4873 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4874 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4875
4876 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4877 lhs = emit_volatile_load (ctx, ins->sreg1);
4878 } else {
4879 /* It is ok for SETRET to have an uninitialized argument */
4880 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4881 set_failure (ctx, "sreg1");
4882 return;
4883 }
4884 lhs = values [ins->sreg1];
4885 }
4886 } else {
4887 lhs = NULL;
4888 }
4889
4890 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4891 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4892 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4893 rhs = emit_volatile_load (ctx, ins->sreg2);
4894 } else {
4895 if (!values [ins->sreg2]) {
4896 set_failure (ctx, "sreg2");
4897 return;
4898 }
4899 rhs = values [ins->sreg2];
4900 }
4901 } else {
4902 rhs = NULL;
4903 }
4904
4905 //mono_print_ins (ins);
4906 gboolean skip_volatile_store = FALSE;
4907 switch (ins->opcode) {
4908 case OP_NOP:
4909 case OP_NOT_NULL:
4910 case OP_LIVERANGE_START:
4911 case OP_LIVERANGE_END:
4912 break;
4913 case OP_ICONST:
4914 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
4915 break;
4916 case OP_I8CONST:
4917 #if TARGET_SIZEOF_VOID_P == 4
4918 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4919 #else
4920 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
4921 #endif
4922 break;
4923 case OP_R8CONST:
4924 values [ins->dreg] = get_double_const (cfg, *(double*)ins->inst_p0);
4925 break;
4926 case OP_R4CONST:
4927 values [ins->dreg] = get_float_const (cfg, *(float*)ins->inst_p0);
4928 break;
4929 case OP_DUMMY_ICONST:
4930 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4931 break;
4932 case OP_DUMMY_I8CONST:
4933 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
4934 break;
4935 case OP_DUMMY_R8CONST:
4936 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
4937 break;
4938 case OP_BR: {
4939 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
4940 LLVMBuildBr (builder, target_bb);
4941 has_terminator = TRUE;
4942 break;
4943 }
4944 case OP_SWITCH: {
4945 int i;
4946 LLVMValueRef v;
4947 char bb_name [128];
4948 LLVMBasicBlockRef new_bb;
4949 LLVMBuilderRef new_builder;
4950
4951 // The default branch is already handled
4952 // FIXME: Handle it here
4953
4954 /* Start new bblock */
4955 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
4956 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
4957
4958 lhs = convert (ctx, lhs, LLVMInt32Type ());
4959 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
4960 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
4961 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
4962
4963 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
4964 }
4965
4966 new_builder = create_builder (ctx);
4967 LLVMPositionBuilderAtEnd (new_builder, new_bb);
4968 LLVMBuildUnreachable (new_builder);
4969
4970 has_terminator = TRUE;
4971 g_assert (!ins->next);
4972
4973 break;
4974 }
4975
4976 case OP_SETRET:
4977 switch (linfo->ret.storage) {
4978 case LLVMArgVtypeInReg: {
4979 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4980 LLVMValueRef val, addr, retval;
4981 int i;
4982
4983 retval = LLVMGetUndef (ret_type);
4984
4985 if (!addresses [ins->sreg1]) {
4986 /*
4987 * The return type is an LLVM vector type, have to convert between it and the
4988 * real return type which is a struct type.
4989 */
4990 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret)));
4991 /* Convert to 2xi64 first */
4992 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
4993
4994 for (i = 0; i < 2; ++i) {
4995 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4996 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
4997 } else {
4998 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4999 }
5000 }
5001 } else {
5002 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
5003 for (i = 0; i < 2; ++i) {
5004 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
5005 LLVMValueRef indexes [2], part_addr;
5006
5007 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5008 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
5009 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
5010
5011 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
5012 } else {
5013 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
5014 }
5015 }
5016 }
5017 LLVMBuildRet (builder, retval);
5018 break;
5019 }
5020 case LLVMArgVtypeAsScalar: {
5021 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5022 LLVMValueRef retval;
5023
5024 g_assert (addresses [ins->sreg1]);
5025
5026 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
5027 LLVMBuildRet (builder, retval);
5028 break;
5029 }
5030 case LLVMArgVtypeByVal: {
5031 LLVMValueRef retval;
5032
5033 g_assert (addresses [ins->sreg1]);
5034 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
5035 LLVMBuildRet (builder, retval);
5036 break;
5037 }
5038 case LLVMArgVtypeByRef: {
5039 LLVMBuildRetVoid (builder);
5040 break;
5041 }
5042 case LLVMArgGsharedvtFixed: {
5043 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
5044 /* The return value is in lhs, need to store to the vret argument */
5045 /* sreg1 might not be set */
5046 if (lhs) {
5047 g_assert (cfg->vret_addr);
5048 g_assert (values [cfg->vret_addr->dreg]);
5049 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
5050 }
5051 LLVMBuildRetVoid (builder);
5052 break;
5053 }
5054 case LLVMArgGsharedvtFixedVtype: {
5055 /* Already set */
5056 LLVMBuildRetVoid (builder);
5057 break;
5058 }
5059 case LLVMArgGsharedvtVariable: {
5060 /* Already set */
5061 LLVMBuildRetVoid (builder);
5062 break;
5063 }
5064 case LLVMArgVtypeRetAddr: {
5065 LLVMBuildRetVoid (builder);
5066 break;
5067 }
5068 case LLVMArgAsIArgs:
5069 case LLVMArgFpStruct: {
5070 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5071 LLVMValueRef retval;
5072
5073 g_assert (addresses [ins->sreg1]);
5074 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
5075 LLVMBuildRet (builder, retval);
5076 break;
5077 }
5078 case LLVMArgNone:
5079 case LLVMArgNormal: {
5080 if (!lhs || ctx->is_dead [ins->sreg1]) {
5081 /*
5082 * The method did not set its return value, probably because it
5083 * ends with a throw.
5084 */
5085 if (cfg->vret_addr)
5086 LLVMBuildRetVoid (builder);
5087 else
5088 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
5089 } else {
5090 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
5091 }
5092 has_terminator = TRUE;
5093 break;
5094 }
5095 default:
5096 g_assert_not_reached ();
5097 break;
5098 }
5099 break;
5100 case OP_ICOMPARE:
5101 case OP_FCOMPARE:
5102 case OP_RCOMPARE:
5103 case OP_LCOMPARE:
5104 case OP_COMPARE:
5105 case OP_ICOMPARE_IMM:
5106 case OP_LCOMPARE_IMM:
5107 case OP_COMPARE_IMM: {
5108 CompRelation rel;
5109 LLVMValueRef cmp, args [16];
5110 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
5111 gboolean unlikely = FALSE;
5112
5113 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5114 if (ins->next->inst_false_bb->out_of_line)
5115 likely = TRUE;
5116 else if (ins->next->inst_true_bb->out_of_line)
5117 unlikely = TRUE;
5118 }
5119
5120 if (ins->next->opcode == OP_NOP)
5121 break;
5122
5123 if (ins->next->opcode == OP_BR)
5124 /* The comparison result is not needed */
5125 continue;
5126
5127 rel = mono_opcode_to_cond (ins->next->opcode);
5128
5129 if (ins->opcode == OP_ICOMPARE_IMM) {
5130 lhs = convert (ctx, lhs, LLVMInt32Type ());
5131 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5132 }
5133 if (ins->opcode == OP_LCOMPARE_IMM) {
5134 lhs = convert (ctx, lhs, LLVMInt64Type ());
5135 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5136 }
5137 if (ins->opcode == OP_LCOMPARE) {
5138 lhs = convert (ctx, lhs, LLVMInt64Type ());
5139 rhs = convert (ctx, rhs, LLVMInt64Type ());
5140 }
5141 if (ins->opcode == OP_ICOMPARE) {
5142 lhs = convert (ctx, lhs, LLVMInt32Type ());
5143 rhs = convert (ctx, rhs, LLVMInt32Type ());
5144 }
5145
5146 if (lhs && rhs) {
5147 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5148 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
5149 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
5150 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
5151 }
5152
5153 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
5154 if (ins->opcode == OP_FCOMPARE) {
5155 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5156 } else if (ins->opcode == OP_RCOMPARE) {
5157 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5158 } else if (ins->opcode == OP_COMPARE_IMM) {
5159 LLVMIntPredicate llvm_pred = cond_to_llvm_cond [rel];
5160 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0) {
5161 // We are emitting a NULL check for a pointer
5162 gboolean nonnull = mono_llvm_is_nonnull (lhs);
5163
5164 if (nonnull && llvm_pred == LLVMIntEQ)
5165 cmp = LLVMConstInt (LLVMInt1Type (), FALSE, FALSE);
5166 else if (nonnull && llvm_pred == LLVMIntNE)
5167 cmp = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
5168 else
5169 cmp = LLVMBuildICmp (builder, llvm_pred, lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
5170
5171 } else {
5172 cmp = LLVMBuildICmp (builder, llvm_pred, convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
5173 }
5174 } else if (ins->opcode == OP_LCOMPARE_IMM) {
5175 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5176 }
5177 else if (ins->opcode == OP_COMPARE) {
5178 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
5179 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5180 else
5181 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
5182 } else
5183 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5184
5185 if (likely || unlikely) {
5186 args [0] = cmp;
5187 args [1] = LLVMConstInt (LLVMInt1Type (), likely ? 1 : 0, FALSE);
5188 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
5189 }
5190
5191 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5192 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
5193 /*
5194 * If the target bb contains PHI instructions, LLVM requires
5195 * two PHI entries for this bblock, while we only generate one.
5196 * So convert this to an unconditional bblock. (bxc #171).
5197 */
5198 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
5199 } else {
5200 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
5201 }
5202 has_terminator = TRUE;
5203 } else if (MONO_IS_SETCC (ins->next)) {
5204 sprintf (dname_buf, "t%d", ins->next->dreg);
5205 dname = dname_buf;
5206 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5207
5208 /* Add stores for volatile variables */
5209 emit_volatile_store (ctx, ins->next->dreg);
5210 } else if (MONO_IS_COND_EXC (ins->next)) {
5211 gboolean force_explicit_branch = FALSE;
5212 if (bb->region != -1) {
5213 /* Don't tag null check branches in exception-handling
5214 * regions with `make.implicit`.
5215 */
5216 force_explicit_branch = TRUE;
5217 }
5218 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp, force_explicit_branch);
5219 if (!ctx_ok (ctx))
5220 break;
5221 builder = ctx->builder;
5222 } else {
5223 set_failure (ctx, "next");
5224 break;
5225 }
5226
5227 ins = ins->next;
5228 break;
5229 }
5230 case OP_FCEQ:
5231 case OP_FCNEQ:
5232 case OP_FCLT:
5233 case OP_FCLT_UN:
5234 case OP_FCGT:
5235 case OP_FCGT_UN:
5236 case OP_FCGE:
5237 case OP_FCLE: {
5238 CompRelation rel;
5239 LLVMValueRef cmp;
5240
5241 rel = mono_opcode_to_cond (ins->opcode);
5242
5243 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5244 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5245 break;
5246 }
5247 case OP_RCEQ:
5248 case OP_RCNEQ:
5249 case OP_RCLT:
5250 case OP_RCLT_UN:
5251 case OP_RCGT:
5252 case OP_RCGT_UN: {
5253 CompRelation rel;
5254 LLVMValueRef cmp;
5255
5256 rel = mono_opcode_to_cond (ins->opcode);
5257
5258 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5259 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5260 break;
5261 }
5262 case OP_PHI:
5263 case OP_FPHI:
5264 case OP_VPHI:
5265 case OP_XPHI: {
5266 // Handled above
5267 skip_volatile_store = TRUE;
5268 break;
5269 }
5270 case OP_MOVE:
5271 case OP_LMOVE:
5272 case OP_XMOVE:
5273 case OP_SETFRET:
5274 g_assert (lhs);
5275 values [ins->dreg] = lhs;
5276 break;
5277 case OP_FMOVE:
5278 case OP_RMOVE: {
5279 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
5280
5281 g_assert (lhs);
5282 values [ins->dreg] = lhs;
5283
5284 if (var && m_class_get_byval_arg (var->klass)->type == MONO_TYPE_R4) {
5285 /*
5286 * This is added by the spilling pass in case of the JIT,
5287 * but we have to do it ourselves.
5288 */
5289 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
5290 }
5291 break;
5292 }
5293 case OP_MOVE_F_TO_I4: {
5294 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
5295 break;
5296 }
5297 case OP_MOVE_I4_TO_F: {
5298 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
5299 break;
5300 }
5301 case OP_MOVE_F_TO_I8: {
5302 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
5303 break;
5304 }
5305 case OP_MOVE_I8_TO_F: {
5306 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
5307 break;
5308 }
5309 case OP_IADD:
5310 case OP_ISUB:
5311 case OP_IAND:
5312 case OP_IMUL:
5313 case OP_IDIV:
5314 case OP_IDIV_UN:
5315 case OP_IREM:
5316 case OP_IREM_UN:
5317 case OP_IOR:
5318 case OP_IXOR:
5319 case OP_ISHL:
5320 case OP_ISHR:
5321 case OP_ISHR_UN:
5322 case OP_FADD:
5323 case OP_FSUB:
5324 case OP_FMUL:
5325 case OP_FDIV:
5326 case OP_LADD:
5327 case OP_LSUB:
5328 case OP_LMUL:
5329 case OP_LDIV:
5330 case OP_LDIV_UN:
5331 case OP_LREM:
5332 case OP_LREM_UN:
5333 case OP_LAND:
5334 case OP_LOR:
5335 case OP_LXOR:
5336 case OP_LSHL:
5337 case OP_LSHR:
5338 case OP_LSHR_UN:
5339 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5340 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5341
5342 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
5343 if (!ctx_ok (ctx))
5344 break;
5345 builder = ctx->builder;
5346
5347 switch (ins->opcode) {
5348 case OP_IADD:
5349 case OP_LADD:
5350 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
5351 break;
5352 case OP_ISUB:
5353 case OP_LSUB:
5354 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
5355 break;
5356 case OP_IMUL:
5357 case OP_LMUL:
5358 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
5359 break;
5360 case OP_IREM:
5361 case OP_LREM:
5362 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
5363 break;
5364 case OP_IREM_UN:
5365 case OP_LREM_UN:
5366 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
5367 break;
5368 case OP_IDIV:
5369 case OP_LDIV:
5370 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
5371 break;
5372 case OP_IDIV_UN:
5373 case OP_LDIV_UN:
5374 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
5375 break;
5376 case OP_FDIV:
5377 case OP_RDIV:
5378 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5379 break;
5380 case OP_IAND:
5381 case OP_LAND:
5382 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
5383 break;
5384 case OP_IOR:
5385 case OP_LOR:
5386 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
5387 break;
5388 case OP_IXOR:
5389 case OP_LXOR:
5390 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
5391 break;
5392 case OP_ISHL:
5393 case OP_LSHL:
5394 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
5395 break;
5396 case OP_ISHR:
5397 case OP_LSHR:
5398 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
5399 break;
5400 case OP_ISHR_UN:
5401 case OP_LSHR_UN:
5402 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
5403 break;
5404
5405 case OP_FADD:
5406 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5407 break;
5408 case OP_FSUB:
5409 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5410 break;
5411 case OP_FMUL:
5412 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5413 break;
5414
5415 default:
5416 g_assert_not_reached ();
5417 }
5418 break;
5419 case OP_RADD:
5420 case OP_RSUB:
5421 case OP_RMUL:
5422 case OP_RDIV: {
5423 lhs = convert (ctx, lhs, LLVMFloatType ());
5424 rhs = convert (ctx, rhs, LLVMFloatType ());
5425 switch (ins->opcode) {
5426 case OP_RADD:
5427 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5428 break;
5429 case OP_RSUB:
5430 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5431 break;
5432 case OP_RMUL:
5433 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5434 break;
5435 case OP_RDIV:
5436 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5437 break;
5438 default:
5439 g_assert_not_reached ();
5440 break;
5441 }
5442 break;
5443 }
5444 case OP_IADD_IMM:
5445 case OP_ISUB_IMM:
5446 case OP_IMUL_IMM:
5447 case OP_IREM_IMM:
5448 case OP_IREM_UN_IMM:
5449 case OP_IDIV_IMM:
5450 case OP_IDIV_UN_IMM:
5451 case OP_IAND_IMM:
5452 case OP_IOR_IMM:
5453 case OP_IXOR_IMM:
5454 case OP_ISHL_IMM:
5455 case OP_ISHR_IMM:
5456 case OP_ISHR_UN_IMM:
5457 case OP_LADD_IMM:
5458 case OP_LSUB_IMM:
5459 case OP_LMUL_IMM:
5460 case OP_LREM_IMM:
5461 case OP_LAND_IMM:
5462 case OP_LOR_IMM:
5463 case OP_LXOR_IMM:
5464 case OP_LSHL_IMM:
5465 case OP_LSHR_IMM:
5466 case OP_LSHR_UN_IMM:
5467 case OP_ADD_IMM:
5468 case OP_AND_IMM:
5469 case OP_MUL_IMM:
5470 case OP_SHL_IMM:
5471 case OP_SHR_IMM:
5472 case OP_SHR_UN_IMM: {
5473 LLVMValueRef imm;
5474
5475 if (spec [MONO_INST_SRC1] == 'l') {
5476 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5477 } else {
5478 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5479 }
5480
5481 emit_div_check (ctx, builder, bb, ins, lhs, imm);
5482 if (!ctx_ok (ctx))
5483 break;
5484 builder = ctx->builder;
5485
5486 #if TARGET_SIZEOF_VOID_P == 4
5487 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
5488 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5489 #endif
5490
5491 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5492 lhs = convert (ctx, lhs, IntPtrType ());
5493 imm = convert (ctx, imm, LLVMTypeOf (lhs));
5494 switch (ins->opcode) {
5495 case OP_IADD_IMM:
5496 case OP_LADD_IMM:
5497 case OP_ADD_IMM:
5498 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
5499 break;
5500 case OP_ISUB_IMM:
5501 case OP_LSUB_IMM:
5502 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
5503 break;
5504 case OP_IMUL_IMM:
5505 case OP_MUL_IMM:
5506 case OP_LMUL_IMM:
5507 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
5508 break;
5509 case OP_IDIV_IMM:
5510 case OP_LDIV_IMM:
5511 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
5512 break;
5513 case OP_IDIV_UN_IMM:
5514 case OP_LDIV_UN_IMM:
5515 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
5516 break;
5517 case OP_IREM_IMM:
5518 case OP_LREM_IMM:
5519 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
5520 break;
5521 case OP_IREM_UN_IMM:
5522 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
5523 break;
5524 case OP_IAND_IMM:
5525 case OP_LAND_IMM:
5526 case OP_AND_IMM:
5527 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
5528 break;
5529 case OP_IOR_IMM:
5530 case OP_LOR_IMM:
5531 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
5532 break;
5533 case OP_IXOR_IMM:
5534 case OP_LXOR_IMM:
5535 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
5536 break;
5537 case OP_ISHL_IMM:
5538 case OP_LSHL_IMM:
5539 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5540 break;
5541 case OP_SHL_IMM:
5542 if (TARGET_SIZEOF_VOID_P == 8) {
5543 /* The IL is not regular */
5544 lhs = convert (ctx, lhs, LLVMInt64Type ());
5545 imm = convert (ctx, imm, LLVMInt64Type ());
5546 }
5547 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5548 break;
5549 case OP_ISHR_IMM:
5550 case OP_LSHR_IMM:
5551 case OP_SHR_IMM:
5552 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
5553 break;
5554 case OP_ISHR_UN_IMM:
5555 /* This is used to implement conv.u4, so the lhs could be an i8 */
5556 lhs = convert (ctx, lhs, LLVMInt32Type ());
5557 imm = convert (ctx, imm, LLVMInt32Type ());
5558 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5559 break;
5560 case OP_LSHR_UN_IMM:
5561 case OP_SHR_UN_IMM:
5562 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5563 break;
5564 default:
5565 g_assert_not_reached ();
5566 }
5567 break;
5568 }
5569 case OP_INEG:
5570 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5571 break;
5572 case OP_LNEG:
5573 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5574 lhs = convert (ctx, lhs, LLVMInt64Type ());
5575 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
5576 break;
5577 case OP_FNEG:
5578 lhs = convert (ctx, lhs, LLVMDoubleType ());
5579 values [ins->dreg] = LLVMBuildFNeg (builder, lhs, dname);
5580 break;
5581 case OP_RNEG:
5582 lhs = convert (ctx, lhs, LLVMFloatType ());
5583 values [ins->dreg] = LLVMBuildFNeg (builder, lhs, dname);
5584 break;
5585 case OP_INOT: {
5586 guint32 v = 0xffffffff;
5587 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5588 break;
5589 }
5590 case OP_LNOT: {
5591 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5592 lhs = convert (ctx, lhs, LLVMInt64Type ());
5593 guint64 v = 0xffffffffffffffffLL;
5594 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
5595 break;
5596 }
5597 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5598 case OP_X86_LEA: {
5599 LLVMValueRef v1, v2;
5600
5601 rhs = LLVMBuildSExt (builder, convert (ctx, rhs, LLVMInt32Type ()), LLVMInt64Type (), "");
5602
5603 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ((unsigned long long)1 << ins->backend.shift_amount), FALSE), "");
5604 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
5605 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
5606 break;
5607 }
5608 #endif
5609
5610 case OP_ICONV_TO_I1:
5611 case OP_ICONV_TO_I2:
5612 case OP_ICONV_TO_I4:
5613 case OP_ICONV_TO_U1:
5614 case OP_ICONV_TO_U2:
5615 case OP_ICONV_TO_U4:
5616 case OP_LCONV_TO_I1:
5617 case OP_LCONV_TO_I2:
5618 case OP_LCONV_TO_U1:
5619 case OP_LCONV_TO_U2:
5620 case OP_LCONV_TO_U4: {
5621 gboolean sign;
5622
5623 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
5624
5625 /* Have to do two casts since our vregs have type int */
5626 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
5627 if (sign)
5628 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
5629 else
5630 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
5631 break;
5632 }
5633 case OP_ICONV_TO_I8:
5634 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
5635 break;
5636 case OP_ICONV_TO_U8:
5637 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
5638 break;
5639 case OP_FCONV_TO_I4:
5640 case OP_RCONV_TO_I4:
5641 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
5642 break;
5643 case OP_FCONV_TO_I1:
5644 case OP_RCONV_TO_I1:
5645 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
5646 break;
5647 case OP_FCONV_TO_U1:
5648 case OP_RCONV_TO_U1:
5649 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
5650 break;
5651 case OP_FCONV_TO_I2:
5652 case OP_RCONV_TO_I2:
5653 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5654 break;
5655 case OP_FCONV_TO_U2:
5656 case OP_RCONV_TO_U2:
5657 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5658 break;
5659 case OP_FCONV_TO_U4:
5660 case OP_RCONV_TO_U4:
5661 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
5662 break;
5663 case OP_FCONV_TO_U8:
5664 case OP_RCONV_TO_U8:
5665 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt64Type (), dname);
5666 break;
5667 case OP_FCONV_TO_I8:
5668 case OP_RCONV_TO_I8:
5669 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
5670 break;
5671 case OP_FCONV_TO_I:
5672 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
5673 break;
5674 case OP_ICONV_TO_R8:
5675 case OP_LCONV_TO_R8:
5676 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
5677 break;
5678 case OP_ICONV_TO_R_UN:
5679 case OP_LCONV_TO_R_UN:
5680 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
5681 break;
5682 #if TARGET_SIZEOF_VOID_P == 4
5683 case OP_LCONV_TO_U:
5684 #endif
5685 case OP_LCONV_TO_I4:
5686 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5687 break;
5688 case OP_ICONV_TO_R4:
5689 case OP_LCONV_TO_R4:
5690 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
5691 if (cfg->r4fp)
5692 values [ins->dreg] = v;
5693 else
5694 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5695 break;
5696 case OP_FCONV_TO_R4:
5697 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
5698 if (cfg->r4fp)
5699 values [ins->dreg] = v;
5700 else
5701 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5702 break;
5703 case OP_RCONV_TO_R8:
5704 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
5705 break;
5706 case OP_RCONV_TO_R4:
5707 values [ins->dreg] = lhs;
5708 break;
5709 case OP_SEXT_I4:
5710 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5711 break;
5712 case OP_ZEXT_I4:
5713 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5714 break;
5715 case OP_TRUNC_I4:
5716 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5717 break;
5718 case OP_LOCALLOC_IMM: {
5719 LLVMValueRef v;
5720
5721 guint32 size = ins->inst_imm;
5722 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5723
5724 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
5725
5726 if (ins->flags & MONO_INST_INIT)
5727 emit_memset (ctx, builder, v, ConstInt32 (size), MONO_ARCH_FRAME_ALIGNMENT);
5728
5729 values [ins->dreg] = v;
5730 break;
5731 }
5732 case OP_LOCALLOC: {
5733 LLVMValueRef v, size;
5734
5735 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
5736
5737 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
5738
5739 if (ins->flags & MONO_INST_INIT)
5740 emit_memset (ctx, builder, v, size, MONO_ARCH_FRAME_ALIGNMENT);
5741 values [ins->dreg] = v;
5742 break;
5743 }
5744
5745 case OP_LOADI1_MEMBASE:
5746 case OP_LOADU1_MEMBASE:
5747 case OP_LOADI2_MEMBASE:
5748 case OP_LOADU2_MEMBASE:
5749 case OP_LOADI4_MEMBASE:
5750 case OP_LOADU4_MEMBASE:
5751 case OP_LOADI8_MEMBASE:
5752 case OP_LOADR4_MEMBASE:
5753 case OP_LOADR8_MEMBASE:
5754 case OP_LOAD_MEMBASE:
5755 case OP_LOADI8_MEM:
5756 case OP_LOADU1_MEM:
5757 case OP_LOADU2_MEM:
5758 case OP_LOADI4_MEM:
5759 case OP_LOADU4_MEM:
5760 case OP_LOAD_MEM: {
5761 int size = 8;
5762 LLVMValueRef base, index, addr;
5763 LLVMTypeRef t;
5764 gboolean sext = FALSE, zext = FALSE;
5765 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
5766 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
5767 gboolean is_unaligned = (ins->flags & MONO_INST_UNALIGNED) != 0;
5768
5769 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5770
5771 if (sext || zext)
5772 dname = (char*)"";
5773
5774 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5775 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5776 base = addr;
5777 } else {
5778 /* _MEMBASE */
5779 base = lhs;
5780
5781 if (ins->inst_offset == 0) {
5782 LLVMValueRef gep_base, gep_offset;
5783 if (mono_llvm_can_be_gep (base, &gep_base, &gep_offset)) {
5784 addr = LLVMBuildGEP (builder, convert (ctx, gep_base, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset, 1, "");
5785 } else {
5786 addr = base;
5787 }
5788 } else if (ins->inst_offset % size != 0) {
5789 /* Unaligned load */
5790 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5791 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5792 } else {
5793 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5794 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5795 }
5796 }
5797
5798 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5799
5800 if (is_unaligned)
5801 values [ins->dreg] = mono_llvm_build_aligned_load (builder, addr, dname, is_volatile, 1);
5802 else
5803 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, base, dname, is_faulting, is_volatile, LLVM_BARRIER_NONE);
5804
5805 if (!(is_faulting || is_volatile) && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5806 /*
5807 * These will signal LLVM that these loads do not alias any stores, and
5808 * they can't fail, allowing them to be hoisted out of loops.
5809 */
5810 set_invariant_load_flag (values [ins->dreg]);
5811 }
5812
5813 if (sext)
5814 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5815 else if (zext)
5816 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5817 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5818 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5819 break;
5820 }
5821
5822 case OP_STOREI1_MEMBASE_REG:
5823 case OP_STOREI2_MEMBASE_REG:
5824 case OP_STOREI4_MEMBASE_REG:
5825 case OP_STOREI8_MEMBASE_REG:
5826 case OP_STORER4_MEMBASE_REG:
5827 case OP_STORER8_MEMBASE_REG:
5828 case OP_STORE_MEMBASE_REG: {
5829 int size = 8;
5830 LLVMValueRef index, addr, base;
5831 LLVMTypeRef t;
5832 gboolean sext = FALSE, zext = FALSE;
5833 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
5834 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
5835 gboolean is_unaligned = (ins->flags & MONO_INST_UNALIGNED) != 0;
5836
5837 if (!values [ins->inst_destbasereg]) {
5838 set_failure (ctx, "inst_destbasereg");
5839 break;
5840 }
5841
5842 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5843
5844 base = values [ins->inst_destbasereg];
5845 LLVMValueRef gep_base, gep_offset;
5846 if (ins->inst_offset == 0 && mono_llvm_can_be_gep (base, &gep_base, &gep_offset)) {
5847 addr = LLVMBuildGEP (builder, convert (ctx, gep_base, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset, 1, "");
5848 } else if (ins->inst_offset % size != 0) {
5849 /* Unaligned store */
5850 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5851 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5852 } else {
5853 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5854 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5855 }
5856 if (is_volatile && LLVMGetInstructionOpcode (base) == LLVMAlloca && !(ins->flags & MONO_INST_VOLATILE))
5857 /* Storing to an alloca cannot fail */
5858 is_volatile = FALSE;
5859 LLVMValueRef srcval = convert (ctx, values [ins->sreg1], t);
5860 LLVMValueRef ptrdst = convert (ctx, addr, LLVMPointerType (t, 0));
5861
5862 if (is_unaligned)
5863 mono_llvm_build_aligned_store (builder, srcval, ptrdst, is_volatile, 1);
5864 else
5865 emit_store (ctx, bb, &builder, size, srcval, ptrdst, base, is_faulting, is_volatile);
5866 break;
5867 }
5868
5869 case OP_STOREI1_MEMBASE_IMM:
5870 case OP_STOREI2_MEMBASE_IMM:
5871 case OP_STOREI4_MEMBASE_IMM:
5872 case OP_STOREI8_MEMBASE_IMM:
5873 case OP_STORE_MEMBASE_IMM: {
5874 int size = 8;
5875 LLVMValueRef index, addr, base;
5876 LLVMTypeRef t;
5877 gboolean sext = FALSE, zext = FALSE;
5878 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
5879 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
5880 gboolean is_unaligned = (ins->flags & MONO_INST_UNALIGNED) != 0;
5881
5882 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5883
5884 base = values [ins->inst_destbasereg];
5885 LLVMValueRef gep_base, gep_offset;
5886 if (ins->inst_offset == 0 && mono_llvm_can_be_gep (base, &gep_base, &gep_offset)) {
5887 addr = LLVMBuildGEP (builder, convert (ctx, gep_base, LLVMPointerType (LLVMInt8Type (), 0)), &gep_offset, 1, "");
5888 } else if (ins->inst_offset % size != 0) {
5889 /* Unaligned store */
5890 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5891 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5892 } else {
5893 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5894 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5895 }
5896 LLVMValueRef srcval = convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t);
5897 LLVMValueRef ptrdst = convert (ctx, addr, LLVMPointerType (t, 0));
5898 if (is_unaligned)
5899 mono_llvm_build_aligned_store (builder, srcval, ptrdst, is_volatile, 1);
5900 else
5901 emit_store (ctx, bb, &builder, size, srcval, ptrdst, base, is_faulting, is_volatile);
5902 break;
5903 }
5904
5905 case OP_CHECK_THIS:
5906 emit_load (ctx, bb, &builder, TARGET_SIZEOF_VOID_P, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), lhs, "", TRUE, FALSE, LLVM_BARRIER_NONE);
5907 break;
5908 case OP_OUTARG_VTRETADDR:
5909 break;
5910 case OP_VOIDCALL:
5911 case OP_CALL:
5912 case OP_LCALL:
5913 case OP_FCALL:
5914 case OP_RCALL:
5915 case OP_VCALL:
5916 case OP_VOIDCALL_MEMBASE:
5917 case OP_CALL_MEMBASE:
5918 case OP_LCALL_MEMBASE:
5919 case OP_FCALL_MEMBASE:
5920 case OP_RCALL_MEMBASE:
5921 case OP_VCALL_MEMBASE:
5922 case OP_VOIDCALL_REG:
5923 case OP_CALL_REG:
5924 case OP_LCALL_REG:
5925 case OP_FCALL_REG:
5926 case OP_RCALL_REG:
5927 case OP_VCALL_REG: {
5928 process_call (ctx, bb, &builder, ins);
5929 break;
5930 }
5931 case OP_AOTCONST: {
5932 guint32 got_offset;
5933 LLVMValueRef indexes [2];
5934 MonoJumpInfo *tmp_ji, *ji;
5935 LLVMValueRef got_entry_addr;
5936 char *name;
5937
5938 /*
5939 * FIXME: Can't allocate from the cfg mempool since that is freed if
5940 * the LLVM compile fails.
5941 */
5942 tmp_ji = g_new0 (MonoJumpInfo, 1);
5943 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
5944 tmp_ji->data.target = ins->inst_p0;
5945
5946 ji = mono_aot_patch_info_dup (tmp_ji);
5947 g_free (tmp_ji);
5948
5949 if (ji->type == MONO_PATCH_INFO_ICALL_ADDR) {
5950 char *symbol = mono_aot_get_direct_call_symbol (MONO_PATCH_INFO_ICALL_ADDR_CALL, ji->data.target);
5951 if (symbol) {
5952 /*
5953 * Avoid emitting a got entry for these since the method is directly called, and it might not be
5954 * resolvable at runtime using dlsym ().
5955 */
5956 g_free (symbol);
5957 values [ins->dreg] = LLVMConstInt (IntPtrType (), 0, FALSE);
5958 break;
5959 }
5960 }
5961
5962 ji->next = cfg->patch_info;
5963 cfg->patch_info = ji;
5964
5965 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
5966 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, NULL);
5967
5968 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
5969 if (!mono_aot_is_shared_got_offset (got_offset)) {
5970 //mono_print_ji (ji);
5971 //printf ("\n");
5972 ctx->cfg->got_access_count ++;
5973 }
5974
5975 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5976 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
5977 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
5978
5979 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
5980 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
5981 g_free (name);
5982 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5983 if (!cfg->llvm_only || mono_aot_is_shared_got_offset (got_offset)) {
5984 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5985 set_invariant_load_flag (values [ins->dreg]);
5986 }
5987
5988 if (ji->type == MONO_PATCH_INFO_LDSTR)
5989 set_nonnull_load_flag (values [ins->dreg]);
5990 break;
5991 }
5992 case OP_MEMMOVE: {
5993 int argn = 0;
5994 LLVMValueRef args [5];
5995 args [argn++] = convert (ctx, values [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0));
5996 args [argn++] = convert (ctx, values [ins->sreg2], LLVMPointerType (LLVMInt8Type (), 0));
5997 args [argn++] = convert (ctx, values [ins->sreg3], LLVMInt64Type ());
5998 #if LLVM_API_VERSION < 900
5999 args [argn++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE); // alignment
6000 #endif
6001 args [argn++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); // is_volatile
6002
6003 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMMOVE), args, argn, "");
6004 break;
6005 }
6006 case OP_NOT_REACHED:
6007 LLVMBuildUnreachable (builder);
6008 has_terminator = TRUE;
6009 g_assert (bb->block_num < cfg->max_block_num);
6010 ctx->unreachable [bb->block_num] = TRUE;
6011 /* Might have instructions after this */
6012 while (ins->next) {
6013 MonoInst *next = ins->next;
6014 /*
6015 * FIXME: If later code uses the regs defined by these instructions,
6016 * compilation will fail.
6017 */
6018 const char *spec = INS_INFO (next->opcode);
6019 if (spec [MONO_INST_DEST] == 'i' && !MONO_IS_STORE_MEMBASE (next))
6020 ctx->values [next->dreg] = LLVMConstNull (LLVMInt32Type ());
6021 MONO_DELETE_INS (bb, next);
6022 }
6023 break;
6024 case OP_LDADDR: {
6025 MonoInst *var = ins->inst_i0;
6026
6027 if (var->opcode == OP_VTARG_ADDR) {
6028 /* The variable contains the vtype address */
6029 values [ins->dreg] = values [var->dreg];
6030 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
6031 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
6032 } else {
6033 values [ins->dreg] = addresses [var->dreg];
6034 }
6035 break;
6036 }
6037 case OP_SIN: {
6038 LLVMValueRef args [1];
6039
6040 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6041 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SIN), args, 1, dname);
6042 break;
6043 }
6044 case OP_SINF: {
6045 LLVMValueRef args [1];
6046
6047 args [0] = convert (ctx, lhs, LLVMFloatType ());
6048 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SINF), args, 1, dname);
6049 break;
6050 }
6051 case OP_EXP: {
6052 LLVMValueRef args [1];
6053
6054 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6055 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_EXP), args, 1, dname);
6056 break;
6057 }
6058 case OP_EXPF: {
6059 LLVMValueRef args [1];
6060
6061 args [0] = convert (ctx, lhs, LLVMFloatType ());
6062 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_EXPF), args, 1, dname);
6063 break;
6064 }
6065 case OP_LOG2: {
6066 LLVMValueRef args [1];
6067
6068 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6069 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG2), args, 1, dname);
6070 break;
6071 }
6072 case OP_LOG2F: {
6073 LLVMValueRef args [1];
6074
6075 args [0] = convert (ctx, lhs, LLVMFloatType ());
6076 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG2F), args, 1, dname);
6077 break;
6078 }
6079 case OP_LOG10: {
6080 LLVMValueRef args [1];
6081
6082 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6083 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG10), args, 1, dname);
6084 break;
6085 }
6086 case OP_LOG10F: {
6087 LLVMValueRef args [1];
6088
6089 args [0] = convert (ctx, lhs, LLVMFloatType ());
6090 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG10F), args, 1, dname);
6091 break;
6092 }
6093 case OP_LOG: {
6094 LLVMValueRef args [1];
6095
6096 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6097 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_LOG), args, 1, dname);
6098 break;
6099 }
6100 case OP_TRUNC: {
6101 LLVMValueRef args [1];
6102
6103 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6104 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_TRUNC), args, 1, dname);
6105 break;
6106 }
6107 case OP_TRUNCF: {
6108 LLVMValueRef args [1];
6109
6110 args [0] = convert (ctx, lhs, LLVMFloatType ());
6111 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_TRUNCF), args, 1, dname);
6112 break;
6113 }
6114 case OP_COS: {
6115 LLVMValueRef args [1];
6116
6117 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6118 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COS), args, 1, dname);
6119 break;
6120 }
6121 case OP_COSF: {
6122 LLVMValueRef args [1];
6123
6124 args [0] = convert (ctx, lhs, LLVMFloatType ());
6125 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COSF), args, 1, dname);
6126 break;
6127 }
6128 case OP_SQRT: {
6129 LLVMValueRef args [1];
6130
6131 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6132 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRT), args, 1, dname);
6133 break;
6134 }
6135 case OP_SQRTF: {
6136 LLVMValueRef args [1];
6137
6138 args [0] = convert (ctx, lhs, LLVMFloatType ());
6139 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRTF), args, 1, dname);
6140 break;
6141 }
6142 case OP_FLOOR: {
6143 LLVMValueRef args [1];
6144
6145 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6146 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FLOOR), args, 1, dname);
6147 break;
6148 }
6149 case OP_FLOORF: {
6150 LLVMValueRef args [1];
6151
6152 args [0] = convert (ctx, lhs, LLVMFloatType ());
6153 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FLOORF), args, 1, dname);
6154 break;
6155 }
6156 case OP_CEIL: {
6157 LLVMValueRef args [1];
6158
6159 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6160 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CEIL), args, 1, dname);
6161 break;
6162 }
6163 case OP_CEILF: {
6164 LLVMValueRef args [1];
6165
6166 args [0] = convert (ctx, lhs, LLVMFloatType ());
6167 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CEILF), args, 1, dname);
6168 break;
6169 }
6170 case OP_FMA: {
6171 LLVMValueRef args [3];
6172
6173 args [0] = convert (ctx, values [ins->sreg1], LLVMDoubleType ());
6174 args [1] = convert (ctx, values [ins->sreg2], LLVMDoubleType ());
6175 args [2] = convert (ctx, values [ins->sreg3], LLVMDoubleType ());
6176
6177 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FMA), args, 3, dname);
6178 break;
6179 }
6180 case OP_FMAF: {
6181 LLVMValueRef args [3];
6182
6183 args [0] = convert (ctx, values [ins->sreg1], LLVMFloatType ());
6184 args [1] = convert (ctx, values [ins->sreg2], LLVMFloatType ());
6185 args [2] = convert (ctx, values [ins->sreg3], LLVMFloatType ());
6186
6187 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FMAF), args, 3, dname);
6188 break;
6189 }
6190 case OP_ABS: {
6191 LLVMValueRef args [1];
6192
6193 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6194 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
6195 break;
6196 }
6197 case OP_ABSF: {
6198 LLVMValueRef args [1];
6199
6200 #ifdef TARGET_AMD64
6201 args [0] = convert (ctx, lhs, LLVMFloatType ());
6202 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_ABSF), args, 1, dname);
6203 #else
6204 /* llvm.fabs not supported on all platforms */
6205 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6206 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
6207 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
6208 #endif
6209 break;
6210 }
6211 case OP_RPOW: {
6212 LLVMValueRef args [2];
6213
6214 args [0] = convert (ctx, lhs, LLVMFloatType ());
6215 args [1] = convert (ctx, rhs, LLVMFloatType ());
6216 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_POWF), args, 2, dname);
6217 break;
6218 }
6219 case OP_FPOW: {
6220 LLVMValueRef args [2];
6221
6222 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6223 args [1] = convert (ctx, rhs, LLVMDoubleType ());
6224 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_POW), args, 2, dname);
6225 break;
6226 }
6227 case OP_FCOPYSIGN: {
6228 LLVMValueRef args [2];
6229
6230 args [0] = convert (ctx, lhs, LLVMDoubleType ());
6231 args [1] = convert (ctx, rhs, LLVMDoubleType ());
6232 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COPYSIGN), args, 2, dname);
6233 break;
6234 }
6235 case OP_RCOPYSIGN: {
6236 LLVMValueRef args [2];
6237
6238 args [0] = convert (ctx, lhs, LLVMFloatType ());
6239 args [1] = convert (ctx, rhs, LLVMFloatType ());
6240 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COPYSIGNF), args, 2, dname);
6241 break;
6242 }
6243
6244 case OP_IMIN:
6245 case OP_LMIN:
6246 case OP_IMAX:
6247 case OP_LMAX:
6248 case OP_IMIN_UN:
6249 case OP_LMIN_UN:
6250 case OP_IMAX_UN:
6251 case OP_LMAX_UN:
6252 case OP_FMIN:
6253 case OP_FMAX:
6254 case OP_RMIN:
6255 case OP_RMAX: {
6256 LLVMValueRef v;
6257
6258 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6259 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6260
6261 switch (ins->opcode) {
6262 case OP_IMIN:
6263 case OP_LMIN:
6264 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
6265 break;
6266 case OP_IMAX:
6267 case OP_LMAX:
6268 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
6269 break;
6270 case OP_IMIN_UN:
6271 case OP_LMIN_UN:
6272 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
6273 break;
6274 case OP_IMAX_UN:
6275 case OP_LMAX_UN:
6276 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
6277 break;
6278 case OP_FMAX:
6279 case OP_RMAX:
6280 v = LLVMBuildFCmp (builder, LLVMRealUGE, lhs, rhs, "");
6281 break;
6282 case OP_FMIN:
6283 case OP_RMIN:
6284 v = LLVMBuildFCmp (builder, LLVMRealULE, lhs, rhs, "");
6285 break;
6286 default:
6287 g_assert_not_reached ();
6288 break;
6289 }
6290 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
6291 break;
6292 }
6293
6294 /*
6295 * See the ARM64 comment in mono/utils/atomic.h for an explanation of why this
6296 * hack is necessary (for now).
6297 */
6298 #ifdef TARGET_ARM64
6299 #define ARM64_ATOMIC_FENCE_FIX mono_llvm_build_fence (builder, LLVM_BARRIER_SEQ)
6300 #else
6301 #define ARM64_ATOMIC_FENCE_FIX
6302 #endif
6303
6304 case OP_ATOMIC_EXCHANGE_I4:
6305 case OP_ATOMIC_EXCHANGE_I8: {
6306 LLVMValueRef args [2];
6307 LLVMTypeRef t;
6308
6309 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
6310 t = LLVMInt32Type ();
6311 else
6312 t = LLVMInt64Type ();
6313
6314 g_assert (ins->inst_offset == 0);
6315
6316 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6317 args [1] = convert (ctx, rhs, t);
6318
6319 ARM64_ATOMIC_FENCE_FIX;
6320 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
6321 ARM64_ATOMIC_FENCE_FIX;
6322 break;
6323 }
6324 case OP_ATOMIC_ADD_I4:
6325 case OP_ATOMIC_ADD_I8: {
6326 LLVMValueRef args [2];
6327 LLVMTypeRef t;
6328
6329 if (ins->opcode == OP_ATOMIC_ADD_I4)
6330 t = LLVMInt32Type ();
6331 else
6332 t = LLVMInt64Type ();
6333
6334 g_assert (ins->inst_offset == 0);
6335
6336 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6337 args [1] = convert (ctx, rhs, t);
6338 ARM64_ATOMIC_FENCE_FIX;
6339 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
6340 ARM64_ATOMIC_FENCE_FIX;
6341 break;
6342 }
6343 case OP_ATOMIC_CAS_I4:
6344 case OP_ATOMIC_CAS_I8: {
6345 LLVMValueRef args [3], val;
6346 LLVMTypeRef t;
6347
6348 if (ins->opcode == OP_ATOMIC_CAS_I4)
6349 t = LLVMInt32Type ();
6350 else
6351 t = LLVMInt64Type ();
6352
6353 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6354 /* comparand */
6355 args [1] = convert (ctx, values [ins->sreg3], t);
6356 /* new value */
6357 args [2] = convert (ctx, values [ins->sreg2], t);
6358 ARM64_ATOMIC_FENCE_FIX;
6359 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
6360 ARM64_ATOMIC_FENCE_FIX;
6361 /* cmpxchg returns a pair */
6362 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
6363 break;
6364 }
6365 case OP_MEMORY_BARRIER: {
6366 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
6367 break;
6368 }
6369 case OP_ATOMIC_LOAD_I1:
6370 case OP_ATOMIC_LOAD_I2:
6371 case OP_ATOMIC_LOAD_I4:
6372 case OP_ATOMIC_LOAD_I8:
6373 case OP_ATOMIC_LOAD_U1:
6374 case OP_ATOMIC_LOAD_U2:
6375 case OP_ATOMIC_LOAD_U4:
6376 case OP_ATOMIC_LOAD_U8:
6377 case OP_ATOMIC_LOAD_R4:
6378 case OP_ATOMIC_LOAD_R8: {
6379 int size;
6380 gboolean sext, zext;
6381 LLVMTypeRef t;
6382 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
6383 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
6384 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6385 LLVMValueRef index, addr;
6386
6387 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6388
6389 if (sext || zext)
6390 dname = (char *)"";
6391
6392 if (ins->inst_offset != 0) {
6393 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6394 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
6395 } else {
6396 addr = lhs;
6397 }
6398
6399 addr = convert (ctx, addr, LLVMPointerType (t, 0));
6400
6401 ARM64_ATOMIC_FENCE_FIX;
6402 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, lhs, dname, is_faulting, is_volatile, barrier);
6403 ARM64_ATOMIC_FENCE_FIX;
6404
6405 if (sext)
6406 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6407 else if (zext)
6408 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6409 break;
6410 }
6411 case OP_ATOMIC_STORE_I1:
6412 case OP_ATOMIC_STORE_I2:
6413 case OP_ATOMIC_STORE_I4:
6414 case OP_ATOMIC_STORE_I8:
6415 case OP_ATOMIC_STORE_U1:
6416 case OP_ATOMIC_STORE_U2:
6417 case OP_ATOMIC_STORE_U4:
6418 case OP_ATOMIC_STORE_U8:
6419 case OP_ATOMIC_STORE_R4:
6420 case OP_ATOMIC_STORE_R8: {
6421 int size;
6422 gboolean sext, zext;
6423 LLVMTypeRef t;
6424 gboolean is_faulting = (ins->flags & MONO_INST_FAULT) != 0;
6425 gboolean is_volatile = (ins->flags & MONO_INST_VOLATILE) != 0;
6426 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6427 LLVMValueRef index, addr, value, base;
6428
6429 if (!values [ins->inst_destbasereg]) {
6430 set_failure (ctx, "inst_destbasereg");
6431 break;
6432 }
6433
6434 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6435
6436 base = values [ins->inst_destbasereg];
6437 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6438 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
6439 value = convert (ctx, values [ins->sreg1], t);
6440
6441 ARM64_ATOMIC_FENCE_FIX;
6442 emit_store_general (ctx, bb, &builder, size, value, addr, base, is_faulting, is_volatile, barrier);
6443 ARM64_ATOMIC_FENCE_FIX;
6444 break;
6445 }
6446 case OP_RELAXED_NOP: {
6447 #if defined(TARGET_AMD64) || defined(TARGET_X86)
6448 emit_call (ctx, bb, &builder, get_intrins_by_name (ctx, "llvm.x86.sse2.pause"), NULL, 0);
6449 break;
6450 #else
6451 break;
6452 #endif
6453 }
6454 case OP_TLS_GET: {
6455 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
6456 #ifdef TARGET_AMD64
6457 // 257 == FS segment register
6458 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
6459 #else
6460 // 256 == GS segment register
6461 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6462 #endif
6463 // FIXME: XEN
6464 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
6465 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
6466 /* See mono_amd64_emit_tls_get () */
6467 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
6468
6469 // 256 == GS segment register
6470 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6471 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
6472 #else
6473 set_failure (ctx, "opcode tls-get");
6474 break;
6475 #endif
6476
6477 break;
6478 }
6479 case OP_GC_SAFE_POINT: {
6480 LLVMValueRef val, cmp, callee, call;
6481 LLVMBasicBlockRef poll_bb, cont_bb;
6482 LLVMValueRef args [2];
6483 static LLVMTypeRef sig;
6484 const char *icall_name = "mono_threads_state_poll";
6485
6486 /*
6487 * Create the cold wrapper around the icall, along with a managed method for it so
6488 * unwinding works.
6489 */
6490 if (!cfg->compile_aot && !ctx->module->gc_poll_cold_wrapper_compiled) {
6491 ERROR_DECL (error);
6492 /* Compiling a method here is a bit ugly, but it works */
6493 MonoMethod *wrapper = mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL);
6494 ctx->module->gc_poll_cold_wrapper_compiled = mono_jit_compile_method (wrapper, error);
6495 mono_error_assert_ok (error);
6496 }
6497
6498 if (!sig)
6499 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
6500
6501 /*
6502 * if (!*sreg1)
6503 * mono_threads_state_poll ();
6504 */
6505 val = mono_llvm_build_load (builder, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
6506 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
6507 poll_bb = gen_bb (ctx, "POLL_BB");
6508 cont_bb = gen_bb (ctx, "CONT_BB");
6509
6510 args [0] = cmp;
6511 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
6512 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
6513
6514 mono_llvm_build_weighted_branch (builder, cmp, cont_bb, poll_bb, 64, 4);
6515
6516 ctx->builder = builder = create_builder (ctx);
6517 LLVMPositionBuilderAtEnd (builder, poll_bb);
6518
6519 if (ctx->cfg->compile_aot) {
6520 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_threads_state_poll));
6521 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6522 } else {
6523 callee = get_jit_callee (ctx, icall_name, sig, MONO_PATCH_INFO_ABS, ctx->module->gc_poll_cold_wrapper_compiled);
6524 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6525 set_call_cold_cconv (call);
6526 }
6527 LLVMBuildBr (builder, cont_bb);
6528
6529 ctx->builder = builder = create_builder (ctx);
6530 LLVMPositionBuilderAtEnd (builder, cont_bb);
6531 ctx->bblocks [bb->block_num].end_bblock = cont_bb;
6532 break;
6533 }
6534
6535 /*
6536 * Overflow opcodes.
6537 */
6538 case OP_IADD_OVF:
6539 case OP_IADD_OVF_UN:
6540 case OP_ISUB_OVF:
6541 case OP_ISUB_OVF_UN:
6542 case OP_IMUL_OVF:
6543 case OP_IMUL_OVF_UN:
6544 case OP_LADD_OVF:
6545 case OP_LADD_OVF_UN:
6546 case OP_LSUB_OVF:
6547 case OP_LSUB_OVF_UN:
6548 case OP_LMUL_OVF:
6549 case OP_LMUL_OVF_UN:
6550 {
6551 LLVMValueRef args [2], val, ovf, func;
6552
6553 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
6554 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
6555 func = get_intrins_by_name (ctx, ovf_op_to_intrins (ins->opcode));
6556 g_assert (func);
6557 val = LLVMBuildCall (builder, func, args, 2, "");
6558 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
6559 ovf = LLVMBuildExtractValue (builder, val, 1, "");
6560 emit_cond_system_exception (ctx, bb, "OverflowException", ovf, FALSE);
6561 if (!ctx_ok (ctx))
6562 break;
6563 builder = ctx->builder;
6564 break;
6565 }
6566
6567 /*
6568 * Valuetypes.
6569 * We currently model them using arrays. Promotion to local vregs is
6570 * disabled for them in mono_handle_global_vregs () in the LLVM case,
6571 * so we always have an entry in cfg->varinfo for them.
6572 * FIXME: Is this needed ?
6573 */
6574 case OP_VZERO: {
6575 MonoClass *klass = ins->klass;
6576
6577 if (!klass) {
6578 // FIXME:
6579 set_failure (ctx, "!klass");
6580 break;
6581 }
6582
6583 if (!addresses [ins->dreg])
6584 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6585 LLVMValueRef ptr = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6586 emit_memset (ctx, builder, ptr, ConstInt32 (mono_class_value_size (klass, NULL)), 0);
6587 break;
6588 }
6589 case OP_DUMMY_VZERO:
6590 break;
6591
6592 case OP_STOREV_MEMBASE:
6593 case OP_LOADV_MEMBASE:
6594 case OP_VMOVE: {
6595 MonoClass *klass = ins->klass;
6596 LLVMValueRef src = NULL, dst, args [5];
6597 gboolean done = FALSE;
6598
6599 if (!klass) {
6600 // FIXME:
6601 set_failure (ctx, "!klass");
6602 break;
6603 }
6604
6605 if (mini_is_gsharedvt_klass (klass)) {
6606 // FIXME:
6607 set_failure (ctx, "gsharedvt");
6608 break;
6609 }
6610
6611 switch (ins->opcode) {
6612 case OP_STOREV_MEMBASE:
6613 if (cfg->gen_write_barriers && m_class_has_references (klass) && ins->inst_destbasereg != cfg->frame_reg &&
6614 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
6615 /* Decomposed earlier */
6616 g_assert_not_reached ();
6617 break;
6618 }
6619 if (!addresses [ins->sreg1]) {
6620 /* SIMD */
6621 g_assert (values [ins->sreg1]);
6622 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (klass)), 0));
6623 LLVMBuildStore (builder, values [ins->sreg1], dst);
6624 done = TRUE;
6625 } else {
6626 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6627 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6628 }
6629 break;
6630 case OP_LOADV_MEMBASE:
6631 if (!addresses [ins->dreg])
6632 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6633 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6634 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6635 break;
6636 case OP_VMOVE:
6637 if (!addresses [ins->sreg1])
6638 addresses [ins->sreg1] = build_alloca (ctx, m_class_get_byval_arg (klass));
6639 if (!addresses [ins->dreg])
6640 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6641 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6642 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6643 break;
6644 default:
6645 g_assert_not_reached ();
6646 }
6647 if (!ctx_ok (ctx))
6648 break;
6649
6650 if (done)
6651 break;
6652
6653 int aindex = 0;
6654 args [aindex ++] = dst;
6655 args [aindex ++] = src;
6656 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
6657 #if LLVM_API_VERSION < 900
6658 // FIXME: Alignment
6659 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6660 #endif
6661 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6662 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMCPY), args, aindex, "");
6663 break;
6664 }
6665 case OP_LLVM_OUTARG_VT: {
6666 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
6667 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
6668
6669 if (ainfo->storage == LLVMArgGsharedvtVariable) {
6670 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
6671
6672 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
6673 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
6674 } else {
6675 g_assert (addresses [ins->sreg1]);
6676 addresses [ins->dreg] = addresses [ins->sreg1];
6677 }
6678 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
6679 if (!addresses [ins->sreg1]) {
6680 addresses [ins->sreg1] = build_alloca (ctx, t);
6681 g_assert (values [ins->sreg1]);
6682 }
6683 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
6684 addresses [ins->dreg] = addresses [ins->sreg1];
6685 } else {
6686 if (!addresses [ins->sreg1]) {
6687 addresses [ins->sreg1] = build_alloca (ctx, t);
6688 g_assert (values [ins->sreg1]);
6689 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
6690 addresses [ins->dreg] = addresses [ins->sreg1];
6691 } else if (ainfo->storage == LLVMArgVtypeAddr || values [ins->sreg1] == addresses [ins->sreg1]) {
6692 /* LLVMArgVtypeByRef/LLVMArgVtypeAddr, have to make a copy */
6693 addresses [ins->dreg] = build_alloca (ctx, t);
6694 LLVMValueRef v = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
6695 LLVMBuildStore (builder, convert (ctx, v, type_to_llvm_type (ctx, t)), addresses [ins->dreg]);
6696 } else {
6697 addresses [ins->dreg] = addresses [ins->sreg1];
6698 }
6699 }
6700 break;
6701 }
6702 case OP_OBJC_GET_SELECTOR: {
6703 const char *name = (const char*)ins->inst_p0;
6704 LLVMValueRef var;
6705
6706 if (!ctx->module->objc_selector_to_var) {
6707 ctx->module->objc_selector_to_var = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
6708
6709 LLVMValueRef info_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), 8), "@OBJC_IMAGE_INFO");
6710 int32_t objc_imageinfo [] = { 0, 16 };
6711 LLVMSetInitializer (info_var, mono_llvm_create_constant_data_array ((uint8_t *) &objc_imageinfo, 8));
6712 LLVMSetLinkage (info_var, LLVMPrivateLinkage);
6713 LLVMSetExternallyInitialized (info_var, TRUE);
6714 LLVMSetSection (info_var, "__DATA, __objc_imageinfo,regular,no_dead_strip");
6715 LLVMSetAlignment (info_var, sizeof (target_mgreg_t));
6716 mark_as_used (ctx->module, info_var);
6717 }
6718
6719 var = (LLVMValueRef)g_hash_table_lookup (ctx->module->objc_selector_to_var, name);
6720 if (!var) {
6721 LLVMValueRef indexes [16];
6722
6723 LLVMValueRef name_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), strlen (name) + 1), "@OBJC_METH_VAR_NAME_");
6724 LLVMSetInitializer (name_var, mono_llvm_create_constant_data_array ((const uint8_t*)name, strlen (name) + 1));
6725 LLVMSetLinkage (name_var, LLVMPrivateLinkage);
6726 LLVMSetSection (name_var, "__TEXT,__objc_methname,cstring_literals");
6727 mark_as_used (ctx->module, name_var);
6728
6729 LLVMValueRef ref_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (LLVMInt8Type (), 0), "@OBJC_SELECTOR_REFERENCES_");
6730
6731 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6732 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6733 LLVMSetInitializer (ref_var, LLVMConstGEP (name_var, indexes, 2));
6734 LLVMSetLinkage (ref_var, LLVMPrivateLinkage);
6735 LLVMSetExternallyInitialized (ref_var, TRUE);
6736 LLVMSetSection (ref_var, "__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
6737 LLVMSetAlignment (ref_var, sizeof (target_mgreg_t));
6738 mark_as_used (ctx->module, ref_var);
6739
6740 g_hash_table_insert (ctx->module->objc_selector_to_var, g_strdup (name), ref_var);
6741 var = ref_var;
6742 }
6743
6744 values [ins->dreg] = LLVMBuildLoad (builder, var, "");
6745 break;
6746 }
6747
6748 /*
6749 * SIMD
6750 */
6751 #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM)
6752 case OP_XZERO: {
6753 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
6754 break;
6755 }
6756 case OP_LOADX_MEMBASE: {
6757 LLVMTypeRef t = type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass));
6758 LLVMValueRef src;
6759
6760 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6761 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
6762 break;
6763 }
6764 case OP_STOREX_MEMBASE: {
6765 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
6766 LLVMValueRef dest;
6767
6768 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6769 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
6770 break;
6771 }
6772 case OP_PADDB:
6773 case OP_PADDW:
6774 case OP_PADDD:
6775 case OP_PADDQ:
6776 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
6777 break;
6778 case OP_ADDPD:
6779 case OP_ADDPS:
6780 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
6781 break;
6782 case OP_PSUBB:
6783 case OP_PSUBW:
6784 case OP_PSUBD:
6785 case OP_PSUBQ:
6786 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
6787 break;
6788 case OP_SUBPD:
6789 case OP_SUBPS:
6790 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
6791 break;
6792 case OP_MULPD:
6793 case OP_MULPS:
6794 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
6795 break;
6796 case OP_DIVPD:
6797 case OP_DIVPS:
6798 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
6799 break;
6800 case OP_PAND:
6801 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
6802 break;
6803 case OP_POR:
6804 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
6805 break;
6806 case OP_PXOR:
6807 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
6808 break;
6809 case OP_PMULW:
6810 case OP_PMULD:
6811 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
6812 break;
6813 case OP_ANDPS:
6814 case OP_ANDNPS:
6815 case OP_ORPS:
6816 case OP_XORPS:
6817 case OP_ANDPD:
6818 case OP_ANDNPD:
6819 case OP_ORPD:
6820 case OP_XORPD: {
6821 LLVMTypeRef t, rt;
6822 LLVMValueRef v = NULL;
6823
6824 switch (ins->opcode) {
6825 case OP_ANDPS:
6826 case OP_ANDNPS:
6827 case OP_ORPS:
6828 case OP_XORPS:
6829 t = LLVMVectorType (LLVMInt32Type (), 4);
6830 rt = LLVMVectorType (LLVMFloatType (), 4);
6831 break;
6832 case OP_ANDPD:
6833 case OP_ANDNPD:
6834 case OP_ORPD:
6835 case OP_XORPD:
6836 t = LLVMVectorType (LLVMInt64Type (), 2);
6837 rt = LLVMVectorType (LLVMDoubleType (), 2);
6838 break;
6839 default:
6840 t = LLVMInt32Type ();
6841 rt = LLVMInt32Type ();
6842 g_assert_not_reached ();
6843 }
6844
6845 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6846 rhs = LLVMBuildBitCast (builder, rhs, t, "");
6847 switch (ins->opcode) {
6848 case OP_ANDPS:
6849 case OP_ANDPD:
6850 v = LLVMBuildAnd (builder, lhs, rhs, "");
6851 break;
6852 case OP_ORPS:
6853 case OP_ORPD:
6854 v = LLVMBuildOr (builder, lhs, rhs, "");
6855 break;
6856 case OP_XORPS:
6857 case OP_XORPD:
6858 v = LLVMBuildXor (builder, lhs, rhs, "");
6859 break;
6860 case OP_ANDNPS:
6861 case OP_ANDNPD:
6862 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
6863 break;
6864 }
6865 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
6866 break;
6867 }
6868 case OP_PMIND_UN:
6869 case OP_PMINW_UN:
6870 case OP_PMINB_UN: {
6871 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntULT, lhs, rhs, "");
6872 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6873 break;
6874 }
6875 case OP_PMAXD_UN:
6876 case OP_PMAXW_UN:
6877 case OP_PMAXB_UN: {
6878 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntUGT, lhs, rhs, "");
6879 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6880 break;
6881 }
6882 case OP_PMINW: {
6883 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntSLT, lhs, rhs, "");
6884 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6885 break;
6886 }
6887 case OP_MINPD:
6888 case OP_MINPS:
6889 case OP_MAXPD:
6890 case OP_MAXPS:
6891 case OP_ADDSUBPD:
6892 case OP_ADDSUBPS:
6893 case OP_HADDPD:
6894 case OP_HADDPS:
6895 case OP_HSUBPD:
6896 case OP_HSUBPS:
6897 case OP_PADDB_SAT:
6898 case OP_PADDW_SAT:
6899 case OP_PSUBB_SAT:
6900 case OP_PSUBW_SAT:
6901 case OP_PADDB_SAT_UN:
6902 case OP_PADDW_SAT_UN:
6903 case OP_PSUBB_SAT_UN:
6904 case OP_PSUBW_SAT_UN:
6905 case OP_PACKW:
6906 case OP_PACKD:
6907 case OP_PACKW_UN:
6908 case OP_PACKD_UN:
6909 case OP_PMULW_HIGH:
6910 case OP_PMULW_HIGH_UN: {
6911 LLVMValueRef args [2];
6912
6913 args [0] = lhs;
6914 args [1] = rhs;
6915
6916 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6917 break;
6918 }
6919 case OP_PAVGB_UN:
6920 case OP_PAVGW_UN: {
6921 LLVMValueRef ones_vec;
6922 LLVMValueRef ones [32];
6923 int vector_size = LLVMGetVectorSize (LLVMTypeOf (lhs));
6924 LLVMTypeRef ext_elem_type = vector_size == 16 ? LLVMInt16Type () : LLVMInt32Type ();
6925
6926 for (int i = 0; i < 32; ++i)
6927 ones [i] = LLVMConstInt (ext_elem_type, 1, FALSE);
6928 ones_vec = LLVMConstVector (ones, vector_size);
6929
6930 LLVMValueRef val;
6931 LLVMTypeRef ext_type = LLVMVectorType (ext_elem_type, vector_size);
6932
6933 /* Have to increase the vector element size to prevent overflows */
6934 /* res = trunc ((zext (lhs) + zext (rhs) + 1) >> 1) */
6935 val = LLVMBuildAdd (builder, LLVMBuildZExt (builder, lhs, ext_type, ""), LLVMBuildZExt (builder, rhs, ext_type, ""), "");
6936 val = LLVMBuildAdd (builder, val, ones_vec, "");
6937 val = LLVMBuildLShr (builder, val, ones_vec, "");
6938 values [ins->dreg] = LLVMBuildTrunc (builder, val, LLVMTypeOf (lhs), "");
6939 break;
6940 }
6941 case OP_PCMPEQB:
6942 case OP_PCMPEQW:
6943 case OP_PCMPEQD:
6944 case OP_PCMPEQQ:
6945 case OP_PCMPGTB: {
6946 LLVMValueRef pcmp;
6947 LLVMTypeRef retType;
6948 LLVMIntPredicate cmpOp;
6949
6950 if (ins->opcode == OP_PCMPGTB)
6951 cmpOp = LLVMIntSGT;
6952 else
6953 cmpOp = LLVMIntEQ;
6954
6955 if (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)) {
6956 pcmp = LLVMBuildICmp (builder, cmpOp, lhs, rhs, "");
6957 retType = LLVMTypeOf (lhs);
6958 } else {
6959 LLVMTypeRef flatType = LLVMVectorType (LLVMInt8Type (), 16);
6960 LLVMValueRef flatRHS = convert (ctx, rhs, flatType);
6961 LLVMValueRef flatLHS = convert (ctx, lhs, flatType);
6962
6963 pcmp = LLVMBuildICmp (builder, cmpOp, flatLHS, flatRHS, "");
6964 retType = flatType;
6965 }
6966
6967 values [ins->dreg] = LLVMBuildSExt (builder, pcmp, retType, "");
6968 break;
6969 }
6970 case OP_EXTRACT_R4:
6971 case OP_EXTRACT_R8:
6972 case OP_EXTRACT_I8:
6973 case OP_EXTRACT_I4:
6974 case OP_EXTRACT_I2:
6975 case OP_EXTRACT_U2:
6976 case OP_EXTRACTX_U2:
6977 case OP_EXTRACT_I1:
6978 case OP_EXTRACT_U1: {
6979 LLVMTypeRef t;
6980 gboolean zext = FALSE;
6981
6982 t = simd_op_to_llvm_type (ins->opcode);
6983
6984 switch (ins->opcode) {
6985 case OP_EXTRACT_R4:
6986 case OP_EXTRACT_R8:
6987 case OP_EXTRACT_I8:
6988 case OP_EXTRACT_I4:
6989 case OP_EXTRACT_I2:
6990 case OP_EXTRACT_I1:
6991 break;
6992 case OP_EXTRACT_U2:
6993 case OP_EXTRACTX_U2:
6994 case OP_EXTRACT_U1:
6995 zext = TRUE;
6996 break;
6997 default:
6998 t = LLVMInt32Type ();
6999 g_assert_not_reached ();
7000 }
7001
7002 lhs = LLVMBuildBitCast (builder, lhs, t, "");
7003 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
7004 if (zext)
7005 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
7006 break;
7007 }
7008
7009 case OP_EXPAND_I1:
7010 case OP_EXPAND_I2:
7011 case OP_EXPAND_I4:
7012 case OP_EXPAND_I8:
7013 case OP_EXPAND_R4:
7014 case OP_EXPAND_R8: {
7015 LLVMTypeRef t;
7016 LLVMValueRef mask [32], v;
7017 int i;
7018
7019 #ifdef ENABLE_NETCORE
7020 t = simd_class_to_llvm_type (ctx, ins->klass);
7021 #else
7022 t = simd_op_to_llvm_type (ins->opcode);
7023 #endif
7024 for (i = 0; i < 32; ++i)
7025 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7026
7027 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
7028
7029 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7030 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
7031 break;
7032 }
7033
7034 case OP_INSERT_I1:
7035 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7036 break;
7037 case OP_INSERT_I2:
7038 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7039 break;
7040 case OP_INSERT_I4:
7041 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7042 break;
7043 case OP_INSERT_I8:
7044 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7045 break;
7046 case OP_INSERT_R4:
7047 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7048 break;
7049 case OP_INSERT_R8:
7050 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
7051 break;
7052
7053 case OP_CVTDQ2PD: {
7054 LLVMValueRef indexes [16];
7055
7056 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7057 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7058 LLVMValueRef mask = LLVMConstVector (indexes, 2);
7059 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
7060 values [ins->dreg] = LLVMBuildSIToFP (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
7061 break;
7062 }
7063 case OP_CVTPS2PD: {
7064 LLVMValueRef indexes [16];
7065
7066 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7067 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7068 LLVMValueRef mask = LLVMConstVector (indexes, 2);
7069 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
7070 values [ins->dreg] = LLVMBuildFPExt (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
7071 break;
7072 }
7073 case OP_CVTTPS2DQ:
7074 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMVectorType (LLVMInt32Type (), 4), dname);
7075 break;
7076
7077 case OP_CVTDQ2PS:
7078 case OP_CVTPD2DQ:
7079 case OP_CVTPS2DQ:
7080 case OP_CVTPD2PS:
7081 case OP_CVTTPD2DQ:
7082 case OP_EXTRACT_MASK:
7083 case OP_SQRTPS:
7084 case OP_SQRTPD:
7085 case OP_RSQRTPS:
7086 case OP_RCPPS: {
7087 LLVMValueRef v;
7088
7089 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
7090
7091 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
7092 break;
7093 }
7094 case OP_COMPPS:
7095 case OP_COMPPD: {
7096 LLVMRealPredicate op;
7097
7098 switch (ins->inst_c0) {
7099 case SIMD_COMP_EQ:
7100 op = LLVMRealOEQ;
7101 break;
7102 case SIMD_COMP_LT:
7103 op = LLVMRealOLT;
7104 break;
7105 case SIMD_COMP_LE:
7106 op = LLVMRealOLE;
7107 break;
7108 case SIMD_COMP_UNORD:
7109 op = LLVMRealUNO;
7110 break;
7111 case SIMD_COMP_NEQ:
7112 op = LLVMRealUNE;
7113 break;
7114 case SIMD_COMP_NLT:
7115 op = LLVMRealUGE;
7116 break;
7117 case SIMD_COMP_NLE:
7118 op = LLVMRealUGT;
7119 break;
7120 case SIMD_COMP_ORD:
7121 op = LLVMRealORD;
7122 break;
7123 default:
7124 g_assert_not_reached ();
7125 }
7126
7127 LLVMValueRef cmp = LLVMBuildFCmp (builder, op, lhs, rhs, "");
7128 if (ins->opcode == OP_COMPPD)
7129 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), 2), ""), LLVMTypeOf (lhs), "");
7130 else
7131 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), 4), ""), LLVMTypeOf (lhs), "");
7132 break;
7133 }
7134 case OP_ICONV_TO_X:
7135 /* This is only used for implementing shifts by non-immediate */
7136 values [ins->dreg] = lhs;
7137 break;
7138
7139 case OP_PSHRW:
7140 case OP_PSHRD:
7141 case OP_PSHRQ:
7142 case OP_PSARW:
7143 case OP_PSARD:
7144 case OP_PSHLW:
7145 case OP_PSHLD:
7146 case OP_PSHLQ: {
7147 LLVMValueRef args [3];
7148
7149 args [0] = lhs;
7150 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
7151
7152 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
7153 break;
7154 }
7155
7156 case OP_PSHRW_REG:
7157 case OP_PSHRD_REG:
7158 case OP_PSHRQ_REG:
7159 case OP_PSARW_REG:
7160 case OP_PSARD_REG:
7161 case OP_PSHLW_REG:
7162 case OP_PSHLD_REG:
7163 case OP_PSHLQ_REG: {
7164 LLVMValueRef args [3];
7165
7166 args [0] = lhs;
7167 args [1] = values [ins->sreg2];
7168
7169 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
7170 break;
7171 }
7172
7173 case OP_SHUFPS:
7174 case OP_SHUFPD:
7175 case OP_PSHUFLED:
7176 case OP_PSHUFLEW_LOW:
7177 case OP_PSHUFLEW_HIGH: {
7178 int mask [16];
7179 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
7180 int i, mask_size = 0;
7181 int imask = ins->inst_c0;
7182
7183 /* Convert the x86 shuffle mask to LLVM's */
7184 switch (ins->opcode) {
7185 case OP_SHUFPS:
7186 mask_size = 4;
7187 mask [0] = ((imask >> 0) & 3);
7188 mask [1] = ((imask >> 2) & 3);
7189 mask [2] = ((imask >> 4) & 3) + 4;
7190 mask [3] = ((imask >> 6) & 3) + 4;
7191 v1 = values [ins->sreg1];
7192 v2 = values [ins->sreg2];
7193 break;
7194 case OP_SHUFPD:
7195 mask_size = 2;
7196 mask [0] = ((imask >> 0) & 1);
7197 mask [1] = ((imask >> 1) & 1) + 2;
7198 v1 = values [ins->sreg1];
7199 v2 = values [ins->sreg2];
7200 break;
7201 case OP_PSHUFLEW_LOW:
7202 mask_size = 8;
7203 mask [0] = ((imask >> 0) & 3);
7204 mask [1] = ((imask >> 2) & 3);
7205 mask [2] = ((imask >> 4) & 3);
7206 mask [3] = ((imask >> 6) & 3);
7207 mask [4] = 4 + 0;
7208 mask [5] = 4 + 1;
7209 mask [6] = 4 + 2;
7210 mask [7] = 4 + 3;
7211 v1 = values [ins->sreg1];
7212 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7213 break;
7214 case OP_PSHUFLEW_HIGH:
7215 mask_size = 8;
7216 mask [0] = 0;
7217 mask [1] = 1;
7218 mask [2] = 2;
7219 mask [3] = 3;
7220 mask [4] = 4 + ((imask >> 0) & 3);
7221 mask [5] = 4 + ((imask >> 2) & 3);
7222 mask [6] = 4 + ((imask >> 4) & 3);
7223 mask [7] = 4 + ((imask >> 6) & 3);
7224 v1 = values [ins->sreg1];
7225 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7226 break;
7227 case OP_PSHUFLED:
7228 mask_size = 4;
7229 mask [0] = ((imask >> 0) & 3);
7230 mask [1] = ((imask >> 2) & 3);
7231 mask [2] = ((imask >> 4) & 3);
7232 mask [3] = ((imask >> 6) & 3);
7233 v1 = values [ins->sreg1];
7234 v2 = LLVMGetUndef (LLVMTypeOf (v1));
7235 break;
7236 default:
7237 g_assert_not_reached ();
7238 }
7239 for (i = 0; i < mask_size; ++i)
7240 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
7241
7242 values [ins->dreg] =
7243 LLVMBuildShuffleVector (builder, v1, v2,
7244 LLVMConstVector (mask_values, mask_size), dname);
7245 break;
7246 }
7247
7248 case OP_UNPACK_LOWB:
7249 case OP_UNPACK_LOWW:
7250 case OP_UNPACK_LOWD:
7251 case OP_UNPACK_LOWQ:
7252 case OP_UNPACK_LOWPS:
7253 case OP_UNPACK_LOWPD:
7254 case OP_UNPACK_HIGHB:
7255 case OP_UNPACK_HIGHW:
7256 case OP_UNPACK_HIGHD:
7257 case OP_UNPACK_HIGHQ:
7258 case OP_UNPACK_HIGHPS:
7259 case OP_UNPACK_HIGHPD: {
7260 int mask [16];
7261 LLVMValueRef mask_values [16];
7262 int i, mask_size = 0;
7263 gboolean low = FALSE;
7264
7265 switch (ins->opcode) {
7266 case OP_UNPACK_LOWB:
7267 mask_size = 16;
7268 low = TRUE;
7269 break;
7270 case OP_UNPACK_LOWW:
7271 mask_size = 8;
7272 low = TRUE;
7273 break;
7274 case OP_UNPACK_LOWD:
7275 case OP_UNPACK_LOWPS:
7276 mask_size = 4;
7277 low = TRUE;
7278 break;
7279 case OP_UNPACK_LOWQ:
7280 case OP_UNPACK_LOWPD:
7281 mask_size = 2;
7282 low = TRUE;
7283 break;
7284 case OP_UNPACK_HIGHB:
7285 mask_size = 16;
7286 break;
7287 case OP_UNPACK_HIGHW:
7288 mask_size = 8;
7289 break;
7290 case OP_UNPACK_HIGHD:
7291 case OP_UNPACK_HIGHPS:
7292 mask_size = 4;
7293 break;
7294 case OP_UNPACK_HIGHQ:
7295 case OP_UNPACK_HIGHPD:
7296 mask_size = 2;
7297 break;
7298 default:
7299 g_assert_not_reached ();
7300 }
7301
7302 if (low) {
7303 for (i = 0; i < (mask_size / 2); ++i) {
7304 mask [(i * 2)] = i;
7305 mask [(i * 2) + 1] = mask_size + i;
7306 }
7307 } else {
7308 for (i = 0; i < (mask_size / 2); ++i) {
7309 mask [(i * 2)] = (mask_size / 2) + i;
7310 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
7311 }
7312 }
7313
7314 for (i = 0; i < mask_size; ++i)
7315 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
7316
7317 values [ins->dreg] =
7318 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
7319 LLVMConstVector (mask_values, mask_size), dname);
7320 break;
7321 }
7322
7323 case OP_DUPPD: {
7324 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7325 LLVMValueRef v, val;
7326
7327 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7328 val = LLVMConstNull (t);
7329 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7330 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
7331
7332 values [ins->dreg] = val;
7333 break;
7334 }
7335 case OP_DUPPS_LOW:
7336 case OP_DUPPS_HIGH: {
7337 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7338 LLVMValueRef v1, v2, val;
7339
7340
7341 if (ins->opcode == OP_DUPPS_LOW) {
7342 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7343 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7344 } else {
7345 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7346 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7347 }
7348 val = LLVMConstNull (t);
7349 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7350 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7351 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7352 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7353
7354 values [ins->dreg] = val;
7355 break;
7356 }
7357
7358 case OP_DPPS: {
7359 LLVMValueRef args [3];
7360
7361 args [0] = lhs;
7362 args [1] = rhs;
7363 /* 0xf1 == multiply all 4 elements, add them together, and store the result to the lowest element */
7364 args [2] = LLVMConstInt (LLVMInt8Type (), 0xf1, FALSE);
7365
7366 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 3, dname);
7367 break;
7368 }
7369
7370 case OP_FCONV_TO_R8_X: {
7371 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (type_to_simd_type (MONO_TYPE_R8)), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7372 break;
7373 }
7374
7375 case OP_FCONV_TO_R4_X: {
7376 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (type_to_simd_type (MONO_TYPE_R4)), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7377 break;
7378 }
7379
7380 #if defined(TARGET_X86) || defined(TARGET_AMD64)
7381 case OP_SSE41_ROUNDSS: {
7382 LLVMValueRef args [3];
7383
7384 args [0] = lhs;
7385 args [1] = lhs;
7386 args [2] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
7387
7388 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SSE_ROUNDSS), args, 3, dname);
7389 break;
7390 }
7391
7392 case OP_SSE41_ROUNDPD: {
7393 LLVMValueRef args [3];
7394
7395 args [0] = lhs;
7396 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
7397
7398 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SSE_ROUNDPD), args, 2, dname);
7399 break;
7400 }
7401 #endif
7402
7403 #ifdef ENABLE_NETCORE
7404 case OP_XCAST: {
7405 LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass);
7406
7407 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, "");
7408 break;
7409 }
7410 case OP_XCOMPARE_FP: {
7411 LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0];
7412 LLVMValueRef cmp = LLVMBuildFCmp (builder, pred, lhs, rhs, "");
7413 int nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
7414 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
7415 if (ins->inst_c1 == MONO_TYPE_R8)
7416 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), nelems), ""), LLVMTypeOf (lhs), "");
7417 else
7418 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), nelems), ""), LLVMTypeOf (lhs), "");
7419 break;
7420 }
7421 case OP_XCOMPARE: {
7422 LLVMIntPredicate pred = cond_to_llvm_cond [ins->inst_c0];
7423 LLVMValueRef cmp = LLVMBuildICmp (builder, pred, lhs, rhs, "");
7424 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
7425 values [ins->dreg] = LLVMBuildSExt (builder, cmp, LLVMTypeOf (lhs), "");
7426 break;
7427 }
7428 case OP_XEQUAL: {
7429 LLVMTypeRef t;
7430 LLVMValueRef cmp, mask [32], shuffle;
7431 int nelems;
7432
7433 #ifdef TARGET_WASM
7434 /* The wasm code generator doesn't understand the shuffle/and code sequence below */
7435 LLVMValueRef val;
7436 if (LLVMIsNull (lhs) || LLVMIsNull (rhs)) {
7437 val = LLVMIsNull (lhs) ? rhs : lhs;
7438 nelems = LLVMGetVectorSize (LLVMTypeOf (lhs));
7439
7440 IntrinsicId intrins = (IntrinsicId)0;
7441 switch (nelems) {
7442 case 16:
7443 intrins = INTRINS_WASM_ANYTRUE_V16;
7444 break;
7445 case 8:
7446 intrins = INTRINS_WASM_ANYTRUE_V8;
7447 break;
7448 case 4:
7449 intrins = INTRINS_WASM_ANYTRUE_V4;
7450 break;
7451 case 2:
7452 intrins = INTRINS_WASM_ANYTRUE_V2;
7453 break;
7454 default:
7455 g_assert_not_reached ();
7456 }
7457 /* res = !wasm.anytrue (val) */
7458 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, intrins), &val, 1, "");
7459 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, values [ins->dreg], LLVMConstInt (LLVMInt32Type (), 0, FALSE), ""), LLVMInt32Type (), dname);
7460 break;
7461 }
7462 #endif
7463 LLVMTypeRef srcelemt = LLVMGetElementType (LLVMTypeOf (lhs));
7464
7465 //%c = icmp sgt <16 x i8> %a0, %a1
7466 if (srcelemt == LLVMDoubleType () || srcelemt == LLVMFloatType ())
7467 cmp = LLVMBuildFCmp (builder, LLVMRealOEQ, lhs, rhs, "");
7468 else
7469 cmp = LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, "");
7470 nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
7471
7472 LLVMTypeRef elemt;
7473 if (srcelemt == LLVMDoubleType ())
7474 elemt = LLVMInt64Type ();
7475 else if (srcelemt == LLVMFloatType ())
7476 elemt = LLVMInt32Type ();
7477 else
7478 elemt = srcelemt;
7479
7480 t = LLVMVectorType (elemt, nelems);
7481 cmp = LLVMBuildSExt (builder, cmp, t, "");
7482 // cmp is a <nelems x elemt> vector, each element is either 0xff... or 0
7483 int half = nelems / 2;
7484 while (half >= 1) {
7485 // AND the top and bottom halfes into the bottom half
7486 for (int i = 0; i < half; ++i)
7487 mask [i] = LLVMConstInt (LLVMInt32Type (), half + i, FALSE);
7488 for (int i = half; i < nelems; ++i)
7489 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7490 shuffle = LLVMBuildShuffleVector (builder, cmp, LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
7491 cmp = LLVMBuildAnd (builder, cmp, shuffle, "");
7492 half = half / 2;
7493 }
7494 // Extract [0]
7495 LLVMValueRef first_elem = LLVMBuildExtractElement (builder, cmp, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7496 // convert to 0/1
7497 LLVMValueRef cmp_zero = LLVMBuildICmp (builder, LLVMIntNE, first_elem, LLVMConstInt (elemt, 0, FALSE), "");
7498 values [ins->dreg] = LLVMBuildZExt (builder, cmp_zero, LLVMInt8Type (), "");
7499 break;
7500 }
7501 case OP_XBINOP: {
7502 switch (ins->inst_c0) {
7503 case OP_IADD:
7504 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
7505 break;
7506 case OP_ISUB:
7507 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
7508 break;
7509 case OP_IAND:
7510 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
7511 break;
7512 case OP_IOR:
7513 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
7514 break;
7515 case OP_IXOR:
7516 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
7517 break;
7518 case OP_FADD:
7519 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
7520 break;
7521 case OP_FSUB:
7522 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
7523 break;
7524 case OP_FMUL:
7525 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
7526 break;
7527 case OP_FDIV:
7528 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
7529 break;
7530 case OP_FMAX:
7531 case OP_FMIN: {
7532 #if defined(TARGET_X86) || defined(TARGET_AMD64)
7533 LLVMValueRef args [] = { lhs, rhs };
7534
7535 gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4;
7536 if (ins->inst_c0 == OP_FMAX)
7537 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD), args, 2, dname);
7538 else
7539 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD), args, 2, dname);
7540 #else
7541 NOT_IMPLEMENTED;
7542 #endif
7543 break;
7544 }
7545 case OP_IMAX: {
7546 gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8;
7547 LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, lhs, rhs, "");
7548 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
7549 break;
7550 }
7551 case OP_IMIN: {
7552 gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8;
7553 LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, lhs, rhs, "");
7554 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
7555 }
7556 break;
7557
7558 default:
7559 g_assert_not_reached ();
7560 }
7561 break;
7562 }
7563 case OP_XEXTRACT_I32:
7564 case OP_XEXTRACT_I64:
7565 case OP_XEXTRACT_R8:
7566 case OP_XEXTRACT_R4: {
7567 LLVMBasicBlockRef bbs [64];
7568 LLVMValueRef switch_ins;
7569 LLVMValueRef phi_values [64];
7570 int nelems = LLVMGetVectorSize (LLVMTypeOf (lhs));
7571 int i;
7572
7573 g_assert (nelems <= 64);
7574 for (i = 0; i < nelems; ++i)
7575 bbs [i] = gen_bb (ctx, "XEXTRACT_CASE_BB");
7576 cbb = gen_bb (ctx, "XEXTRACT_COND_BB");
7577
7578 switch_ins = LLVMBuildSwitch (builder, rhs, bbs [0], 0);
7579 for (i = 0; i < nelems; ++i) {
7580 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
7581 LLVMPositionBuilderAtEnd (builder, bbs [i]);
7582 phi_values [i] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), i, FALSE), "");
7583 LLVMBuildBr (builder, cbb);
7584 }
7585
7586 LLVMPositionBuilderAtEnd (builder, cbb);
7587 values [ins->dreg] = LLVMBuildPhi (builder, LLVMTypeOf (phi_values [0]), "");
7588 LLVMAddIncoming (values [ins->dreg], phi_values, bbs, nelems);
7589
7590 MonoTypeEnum type = (MonoTypeEnum)ins->inst_c0;
7591 switch (type) {
7592 case MONO_TYPE_U1:
7593 case MONO_TYPE_U2:
7594 values [ins->dreg] = LLVMBuildZExt (ctx->builder, values [ins->dreg], LLVMInt32Type (), "");
7595 break;
7596 default:
7597 break;
7598 }
7599 ctx->bblocks [bb->block_num].end_bblock = cbb;
7600 break;
7601 }
7602 case OP_POPCNT32:
7603 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CTPOP_I32), &lhs, 1, "");
7604 break;
7605 case OP_POPCNT64:
7606 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CTPOP_I64), &lhs, 1, "");
7607 break;
7608 case OP_LZCNT32:
7609 case OP_LZCNT64: {
7610 LLVMValueRef args [2];
7611 args [0] = lhs;
7612 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
7613 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_LZCNT32 ? INTRINS_CTLZ_I32 : INTRINS_CTLZ_I64), args, 2, "");
7614 break;
7615 }
7616 case OP_CTTZ32:
7617 case OP_CTTZ64: {
7618 LLVMValueRef args [2];
7619 args [0] = lhs;
7620 args [1] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
7621 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_CTTZ32 ? INTRINS_CTTZ_I32 : INTRINS_CTTZ_I64), args, 2, "");
7622 break;
7623 }
7624 case OP_BEXTR32:
7625 case OP_BEXTR64: {
7626 LLVMValueRef args [2];
7627 args [0] = lhs;
7628 args [1] = convert (ctx, rhs, ins->opcode == OP_BEXTR32 ? LLVMInt32Type () : LLVMInt64Type ()); // cast ushort to u32/u64
7629 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_BEXTR32 ? INTRINS_BEXTR_I32 : INTRINS_BEXTR_I64), args, 2, "");
7630 break;
7631 }
7632 case OP_BZHI32:
7633 case OP_BZHI64: {
7634 LLVMValueRef args [2];
7635 args [0] = lhs;
7636 args [1] = rhs;
7637 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_BZHI32 ? INTRINS_BZHI_I32 : INTRINS_BZHI_I64), args, 2, "");
7638 break;
7639 }
7640 case OP_MULX_H32:
7641 case OP_MULX_H64:
7642 case OP_MULX_HL32:
7643 case OP_MULX_HL64: {
7644 gboolean is_64 = ins->opcode == OP_MULX_H64 || ins->opcode == OP_MULX_HL64;
7645 gboolean only_high = ins->opcode == OP_MULX_H32 || ins->opcode == OP_MULX_H64;
7646 LLVMValueRef lx = LLVMBuildZExt (ctx->builder, lhs, LLVMInt128Type (), "");
7647 LLVMValueRef rx = LLVMBuildZExt (ctx->builder, rhs, LLVMInt128Type (), "");
7648 LLVMValueRef mulx = LLVMBuildMul (ctx->builder, lx, rx, "");
7649 if (!only_high) {
7650 LLVMValueRef lowx = LLVMBuildTrunc (ctx->builder, mulx, is_64 ? LLVMInt64Type () : LLVMInt32Type (), "");
7651 LLVMBuildStore (ctx->builder, lowx, values [ins->sreg3]);
7652 }
7653 LLVMValueRef shift = LLVMConstInt (LLVMInt128Type (), is_64 ? 64 : 32, FALSE);
7654 LLVMValueRef highx = LLVMBuildLShr (ctx->builder, mulx, shift, "");
7655 values [ins->dreg] = LLVMBuildTrunc (ctx->builder, highx, is_64 ? LLVMInt64Type () : LLVMInt32Type (), "");
7656 break;
7657 }
7658 case OP_PEXT32:
7659 case OP_PEXT64: {
7660 LLVMValueRef args [2];
7661 args [0] = lhs;
7662 args [1] = rhs;
7663 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_PEXT32 ? INTRINS_PEXT_I32 : INTRINS_PEXT_I64), args, 2, "");
7664 break;
7665 }
7666 case OP_PDEP32:
7667 case OP_PDEP64: {
7668 LLVMValueRef args [2];
7669 args [0] = lhs;
7670 args [1] = rhs;
7671 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_PDEP32 ? INTRINS_PDEP_I32 : INTRINS_PDEP_I64), args, 2, "");
7672 break;
7673 }
7674 #endif /* ENABLE_NETCORE */
7675 #endif /* SIMD */
7676
7677 case OP_DUMMY_USE:
7678 break;
7679
7680 /*
7681 * EXCEPTION HANDLING
7682 */
7683 case OP_IMPLICIT_EXCEPTION:
7684 /* This marks a place where an implicit exception can happen */
7685 if (bb->region != -1)
7686 set_failure (ctx, "implicit-exception");
7687 break;
7688 case OP_THROW:
7689 case OP_RETHROW: {
7690 gboolean rethrow = (ins->opcode == OP_RETHROW);
7691 if (ctx->llvm_only) {
7692 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
7693 has_terminator = TRUE;
7694 ctx->unreachable [bb->block_num] = TRUE;
7695 } else {
7696 emit_throw (ctx, bb, rethrow, lhs);
7697 builder = ctx->builder;
7698 }
7699 break;
7700 }
7701 case OP_CALL_HANDLER: {
7702 /*
7703 * We don't 'call' handlers, but instead simply branch to them.
7704 * The code generated by ENDFINALLY will branch back to us.
7705 */
7706 LLVMBasicBlockRef noex_bb;
7707 GSList *bb_list;
7708 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
7709
7710 bb_list = info->call_handler_return_bbs;
7711
7712 /*
7713 * Set the indicator variable for the finally clause.
7714 */
7715 lhs = info->finally_ind;
7716 g_assert (lhs);
7717 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
7718
7719 /* Branch to the finally clause */
7720 LLVMBuildBr (builder, info->call_handler_target_bb);
7721
7722 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
7723 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
7724
7725 builder = ctx->builder = create_builder (ctx);
7726 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
7727
7728 bblocks [bb->block_num].end_bblock = noex_bb;
7729 break;
7730 }
7731 case OP_START_HANDLER: {
7732 break;
7733 }
7734 case OP_ENDFINALLY: {
7735 LLVMBasicBlockRef resume_bb;
7736 MonoBasicBlock *handler_bb;
7737 LLVMValueRef val, switch_ins, callee;
7738 GSList *bb_list;
7739 BBInfo *info;
7740 gboolean is_fault = MONO_REGION_FLAGS (bb->region) == MONO_EXCEPTION_CLAUSE_FAULT;
7741
7742 /*
7743 * Fault clauses are like finally clauses, but they are only called if an exception is thrown.
7744 */
7745 if (!is_fault) {
7746 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
7747 g_assert (handler_bb);
7748 info = &bblocks [handler_bb->block_num];
7749 lhs = info->finally_ind;
7750 g_assert (lhs);
7751
7752 bb_list = info->call_handler_return_bbs;
7753
7754 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
7755
7756 /* Load the finally variable */
7757 val = LLVMBuildLoad (builder, lhs, "");
7758
7759 /* Reset the variable */
7760 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
7761
7762 /* Branch to either resume_bb, or to the bblocks in bb_list */
7763 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
7764 /*
7765 * The other targets are added at the end to handle OP_CALL_HANDLER
7766 * opcodes processed later.
7767 */
7768 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
7769
7770 builder = ctx->builder = create_builder (ctx);
7771 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
7772 }
7773
7774 if (ctx->llvm_only) {
7775 emit_resume_eh (ctx, bb);
7776 } else {
7777 LLVMTypeRef icall_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
7778 if (ctx->cfg->compile_aot) {
7779 callee = get_callee (ctx, icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7780 } else {
7781 callee = get_jit_callee (ctx, "llvm_resume_unwind_trampoline", icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7782 }
7783 LLVMBuildCall (builder, callee, NULL, 0, "");
7784 LLVMBuildUnreachable (builder);
7785 }
7786
7787 has_terminator = TRUE;
7788 break;
7789 }
7790 case OP_IL_SEQ_POINT:
7791 break;
7792 default: {
7793 char reason [128];
7794
7795 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
7796 set_failure (ctx, reason);
7797 break;
7798 }
7799 }
7800
7801 if (!ctx_ok (ctx))
7802 break;
7803
7804 /* Convert the value to the type required by phi nodes */
7805 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
7806 if (ctx->is_vphi [ins->dreg])
7807 /* vtypes */
7808 values [ins->dreg] = addresses [ins->dreg];
7809 else
7810 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
7811 }
7812
7813 /* Add stores for volatile variables */
7814 if (!skip_volatile_store && spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
7815 emit_volatile_store (ctx, ins->dreg);
7816 }
7817
7818 if (!ctx_ok (ctx))
7819 return;
7820
7821 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
7822 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
7823 }
7824
7825 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
7826 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
7827 LLVMBuildRetVoid (builder);
7828 }
7829
7830 if (bb == cfg->bb_entry)
7831 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
7832 }
7833
7834 /*
7835 * mono_llvm_check_method_supported:
7836 *
7837 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
7838 * compiling a method twice.
7839 */
7840 void
7841 mono_llvm_check_method_supported (MonoCompile *cfg)
7842 {
7843 int i, j;
7844
7845 #ifdef TARGET_WASM
7846 if (mono_method_signature_internal (cfg->method)->call_convention == MONO_CALL_VARARG) {
7847 cfg->exception_message = g_strdup ("vararg callconv");
7848 cfg->disable_llvm = TRUE;
7849 return;
7850 }
7851 #endif
7852
7853 if (cfg->llvm_only)
7854 return;
7855
7856 if (cfg->method->save_lmf) {
7857 cfg->exception_message = g_strdup ("lmf");
7858 cfg->disable_llvm = TRUE;
7859 }
7860 if (cfg->disable_llvm)
7861 return;
7862
7863 /*
7864 * Nested clauses where one of the clauses is a finally clause is
7865 * not supported, because LLVM can't figure out the control flow,
7866 * probably because we resume exception handling by calling our
7867 * own function instead of using the 'resume' llvm instruction.
7868 */
7869 for (i = 0; i < cfg->header->num_clauses; ++i) {
7870 for (j = 0; j < cfg->header->num_clauses; ++j) {
7871 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
7872 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
7873
7874 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
7875 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7876 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
7877 cfg->exception_message = g_strdup ("nested clauses");
7878 cfg->disable_llvm = TRUE;
7879 break;
7880 }
7881 }
7882 }
7883 if (cfg->disable_llvm)
7884 return;
7885
7886 /* FIXME: */
7887 if (cfg->method->dynamic) {
7888 cfg->exception_message = g_strdup ("dynamic.");
7889 cfg->disable_llvm = TRUE;
7890 }
7891 if (cfg->disable_llvm)
7892 return;
7893 }
7894
7895 static LLVMCallInfo*
7896 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
7897 {
7898 LLVMCallInfo *linfo;
7899 int i;
7900
7901 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
7902 int i, n, pindex;
7903
7904 /*
7905 * Gsharedvt methods have the following calling convention:
7906 * - all arguments are passed by ref, even non generic ones
7907 * - the return value is returned by ref too, using a vret
7908 * argument passed after 'this'.
7909 */
7910 n = sig->param_count + sig->hasthis;
7911 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
7912
7913 pindex = 0;
7914 if (sig->hasthis)
7915 linfo->args [pindex ++].storage = LLVMArgNormal;
7916
7917 if (sig->ret->type != MONO_TYPE_VOID) {
7918 if (mini_is_gsharedvt_variable_type (sig->ret))
7919 linfo->ret.storage = LLVMArgGsharedvtVariable;
7920 else if (mini_type_is_vtype (sig->ret))
7921 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
7922 else
7923 linfo->ret.storage = LLVMArgGsharedvtFixed;
7924 linfo->vret_arg_index = pindex;
7925 } else {
7926 linfo->ret.storage = LLVMArgNone;
7927 }
7928
7929 for (i = 0; i < sig->param_count; ++i) {
7930 if (sig->params [i]->byref)
7931 linfo->args [pindex].storage = LLVMArgNormal;
7932 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
7933 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
7934 else if (mini_type_is_vtype (sig->params [i]))
7935 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
7936 else
7937 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
7938 linfo->args [pindex].type = sig->params [i];
7939 pindex ++;
7940 }
7941 return linfo;
7942 }
7943
7944 linfo = mono_arch_get_llvm_call_info (cfg, sig);
7945 linfo->dummy_arg_pindex = -1;
7946 for (i = 0; i < sig->param_count; ++i)
7947 linfo->args [i + sig->hasthis].type = sig->params [i];
7948
7949 return linfo;
7950 }
7951
7952 static void
7953 emit_method_inner (EmitContext *ctx);
7954
7955 static void
7956 free_ctx (EmitContext *ctx)
7957 {
7958 GSList *l;
7959
7960 g_free (ctx->values);
7961 g_free (ctx->addresses);
7962 g_free (ctx->vreg_types);
7963 g_free (ctx->is_vphi);
7964 g_free (ctx->vreg_cli_types);
7965 g_free (ctx->is_dead);
7966 g_free (ctx->unreachable);
7967 g_ptr_array_free (ctx->phi_values, TRUE);
7968 g_free (ctx->bblocks);
7969 g_hash_table_destroy (ctx->region_to_handler);
7970 g_hash_table_destroy (ctx->clause_to_handler);
7971 g_hash_table_destroy (ctx->jit_callees);
7972
7973 g_ptr_array_free (ctx->callsite_list, TRUE);
7974
7975 g_free (ctx->method_name);
7976 g_ptr_array_free (ctx->bblock_list, TRUE);
7977
7978 for (l = ctx->builders; l; l = l->next) {
7979 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
7980 LLVMDisposeBuilder (builder);
7981 }
7982
7983 g_free (ctx);
7984 }
7985
7986 static gboolean
7987 is_externally_callable (EmitContext *ctx, MonoMethod *method)
7988 {
7989 if (ctx->module->llvm_only && ctx->module->static_link && (method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE || method_is_direct_callable (method)))
7990 return TRUE;
7991 return FALSE;
7992 }
7993
7994 /*
7995 * mono_llvm_emit_method:
7996 *
7997 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
7998 */
7999 void
8000 mono_llvm_emit_method (MonoCompile *cfg)
8001 {
8002 EmitContext *ctx;
8003 char *method_name;
8004 int i;
8005
8006 if (cfg->skip)
8007 return;
8008
8009 /* The code below might acquire the loader lock, so use it for global locking */
8010 mono_loader_lock ();
8011
8012 /* Used to communicate with the callbacks */
8013 mono_native_tls_set_value (current_cfg_tls_id, cfg);
8014
8015 ctx = g_new0 (EmitContext, 1);
8016 ctx->cfg = cfg;
8017 ctx->mempool = cfg->mempool;
8018
8019 /*
8020 * This maps vregs to the LLVM instruction defining them
8021 */
8022 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
8023 /*
8024 * This maps vregs for volatile variables to the LLVM instruction defining their
8025 * address.
8026 */
8027 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
8028 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
8029 ctx->is_vphi = g_new0 (gboolean, cfg->next_vreg);
8030 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
8031 ctx->phi_values = g_ptr_array_sized_new (256);
8032 /*
8033 * This signals whenever the vreg was defined by a phi node with no input vars
8034 * (i.e. all its input bblocks end with NOT_REACHABLE).
8035 */
8036 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
8037 /* Whenever the bblock is unreachable */
8038 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
8039 ctx->bblock_list = g_ptr_array_sized_new (256);
8040
8041 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
8042 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
8043 ctx->callsite_list = g_ptr_array_new ();
8044 ctx->jit_callees = g_hash_table_new (NULL, NULL);
8045 if (cfg->compile_aot) {
8046 ctx->module = &aot_module;
8047
8048 if (is_externally_callable (ctx, cfg->method))
8049 method_name = mono_aot_get_mangled_method_name (cfg->method);
8050 else
8051 method_name = mono_aot_get_method_name (cfg);
8052 cfg->llvm_method_name = g_strdup (method_name);
8053 } else {
8054 init_jit_module (cfg->domain);
8055 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
8056 method_name = mono_method_full_name (cfg->method, TRUE);
8057 }
8058 ctx->method_name = method_name;
8059
8060 if (cfg->compile_aot) {
8061 ctx->lmodule = ctx->module->lmodule;
8062 } else {
8063 ctx->lmodule = LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg->method->name));
8064 /* Reset this as it contains values from lmodule */
8065 memset (ctx->module->intrins_by_id, 0, sizeof (LLVMValueRef) * INTRINS_NUM);
8066 }
8067 ctx->llvm_only = ctx->module->llvm_only;
8068 #ifdef TARGET_WASM
8069 ctx->emit_dummy_arg = TRUE;
8070 #endif
8071
8072 emit_method_inner (ctx);
8073
8074 if (!ctx_ok (ctx)) {
8075 if (ctx->lmethod) {
8076 /* Need to add unused phi nodes as they can be referenced by other values */
8077 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
8078 LLVMBuilderRef builder;
8079
8080 builder = create_builder (ctx);
8081 LLVMPositionBuilderAtEnd (builder, phi_bb);
8082
8083 for (i = 0; i < ctx->phi_values->len; ++i) {
8084 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
8085 if (LLVMGetInstructionParent (v) == NULL)
8086 LLVMInsertIntoBuilder (builder, v);
8087 }
8088
8089 if (ctx->module->llvm_only && ctx->module->static_link) {
8090 // Keep a stub for the function since it might be called directly
8091 int nbbs = LLVMCountBasicBlocks (ctx->lmethod);
8092 LLVMBasicBlockRef *bblocks = g_new0 (LLVMBasicBlockRef, nbbs);
8093 LLVMGetBasicBlocks (ctx->lmethod, bblocks);
8094 for (int i = 0; i < nbbs; ++i)
8095 LLVMDeleteBasicBlock (bblocks [i]);
8096
8097 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (ctx->lmethod, "ENTRY");
8098 builder = create_builder (ctx);
8099 LLVMPositionBuilderAtEnd (builder, entry_bb);
8100 ctx->builder = builder;
8101
8102 LLVMTypeRef sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
8103 LLVMValueRef callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
8104 LLVMBuildCall (builder, callee, NULL, 0, "");
8105 LLVMBuildUnreachable (builder);
8106 } else {
8107 LLVMDeleteFunction (ctx->lmethod);
8108 }
8109 }
8110 }
8111
8112 free_ctx (ctx);
8113
8114 mono_native_tls_set_value (current_cfg_tls_id, NULL);
8115
8116 mono_loader_unlock ();
8117 }
8118
8119 static void
8120 emit_method_inner (EmitContext *ctx)
8121 {
8122 MonoCompile *cfg = ctx->cfg;
8123 MonoMethodSignature *sig;
8124 MonoBasicBlock *bb;
8125 LLVMTypeRef method_type;
8126 LLVMValueRef method = NULL;
8127 LLVMValueRef *values = ctx->values;
8128 int i, max_block_num, bb_index;
8129 gboolean last = FALSE;
8130 LLVMCallInfo *linfo;
8131 LLVMModuleRef lmodule = ctx->lmodule;
8132 BBInfo *bblocks;
8133 GPtrArray *bblock_list = ctx->bblock_list;
8134 MonoMethodHeader *header;
8135 MonoExceptionClause *clause;
8136 char **names;
8137 LLVMBuilderRef entry_builder = NULL;
8138 LLVMBasicBlockRef entry_bb = NULL;
8139
8140 if (cfg->gsharedvt && !cfg->llvm_only) {
8141 set_failure (ctx, "gsharedvt");
8142 return;
8143 }
8144
8145 #if 1
8146 {
8147 static int count = 0;
8148 count ++;
8149
8150 char *llvm_count_str = g_getenv ("LLVM_COUNT");
8151 if (llvm_count_str) {
8152 int lcount = atoi (llvm_count_str);
8153 g_free (llvm_count_str);
8154 if (count == lcount) {
8155 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
8156 fflush (stdout);
8157 last = TRUE;
8158 }
8159 if (count > lcount) {
8160 set_failure (ctx, "count");
8161 return;
8162 }
8163 }
8164 }
8165 #endif
8166
8167 // If we come upon one of the init_method wrappers, we need to find
8168 // the method that we have already emitted and tell LLVM that this
8169 // managed method info for the wrapper is associated with this method
8170 // we constructed ourselves from LLVM IR.
8171 //
8172 // This is necessary to unwind through the init_method, in the case that
8173 // it has to run a static cctor that throws an exception
8174 if (cfg->method->wrapper_type == MONO_WRAPPER_OTHER) {
8175 WrapperInfo *info = mono_marshal_get_wrapper_info (cfg->method);
8176 if (info->subtype == WRAPPER_SUBTYPE_AOT_INIT) {
8177 method = get_init_icall_wrapper (ctx->module, info->d.aot_init.subtype);
8178 ctx->lmethod = method;
8179 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
8180
8181 const char *init_name = mono_marshal_get_aot_init_wrapper_name (info->d.aot_init.subtype);
8182 ctx->method_name = g_strdup_printf ("%s%s", ctx->module->global_prefix, init_name);
8183 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
8184
8185 if (!cfg->llvm_only && ctx->module->external_symbols) {
8186 LLVMSetLinkage (method, LLVMExternalLinkage);
8187 LLVMSetVisibility (method, LLVMHiddenVisibility);
8188 }
8189
8190 goto after_codegen;
8191 } else if (info->subtype == WRAPPER_SUBTYPE_LLVM_FUNC) {
8192 g_assert (info->d.llvm_func.subtype == LLVM_FUNC_WRAPPER_GC_POLL);
8193
8194 if (cfg->compile_aot) {
8195 method = ctx->module->gc_poll_cold_wrapper;
8196 g_assert (method);
8197 } else {
8198 method = emit_icall_cold_wrapper (ctx->module, lmodule, MONO_JIT_ICALL_mono_threads_state_poll, FALSE);
8199 }
8200 ctx->lmethod = method;
8201 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
8202
8203 ctx->method_name = g_strdup (LLVMGetValueName (method)); //g_strdup_printf ("%s_%s", ctx->module->global_prefix, LLVMGetValueName (method));
8204 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
8205
8206 if (!cfg->llvm_only && ctx->module->external_symbols) {
8207 LLVMSetLinkage (method, LLVMExternalLinkage);
8208 LLVMSetVisibility (method, LLVMHiddenVisibility);
8209 }
8210
8211 goto after_codegen;
8212 }
8213 }
8214
8215 sig = mono_method_signature_internal (cfg->method);
8216 ctx->sig = sig;
8217
8218 linfo = get_llvm_call_info (cfg, sig);
8219 ctx->linfo = linfo;
8220 if (!ctx_ok (ctx))
8221 return;
8222
8223 if (cfg->rgctx_var)
8224 linfo->rgctx_arg = TRUE;
8225 else if (needs_extra_arg (ctx, cfg->method))
8226 linfo->dummy_arg = TRUE;
8227 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
8228 if (!ctx_ok (ctx))
8229 return;
8230
8231 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
8232 ctx->lmethod = method;
8233
8234 if (!cfg->llvm_only)
8235 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
8236
8237 /* if the method doesn't contain
8238 * (1) a call (so it's a leaf method)
8239 * (2) and no loops
8240 * we can skip the GC safepoint on method entry. */
8241 gboolean requires_safepoint;
8242 requires_safepoint = cfg->has_calls;
8243 if (!requires_safepoint) {
8244 for (bb = cfg->bb_entry->next_bb; bb; bb = bb->next_bb) {
8245 if (bb->loop_body_start || (bb->flags & BB_EXCEPTION_HANDLER)) {
8246 requires_safepoint = TRUE;
8247 }
8248 }
8249 }
8250 #ifndef MONO_LLVM_LOADED
8251 if (!cfg->llvm_only && mono_threads_are_safepoints_enabled () && requires_safepoint) {
8252 if (!cfg->compile_aot && cfg->method->wrapper_type != MONO_WRAPPER_ALLOC) {
8253 LLVMSetGC (method, "coreclr");
8254 emit_gc_safepoint_poll (ctx->module, ctx->lmodule, cfg);
8255 } else if (cfg->compile_aot) {
8256 LLVMSetGC (method, "coreclr");
8257 }
8258 }
8259 #endif
8260 LLVMSetLinkage (method, LLVMPrivateLinkage);
8261
8262 mono_llvm_add_func_attr (method, LLVM_ATTR_UW_TABLE);
8263
8264 if (cfg->compile_aot) {
8265 if (is_externally_callable (ctx, cfg->method)) {
8266 LLVMSetLinkage (method, LLVMExternalLinkage);
8267 } else {
8268 LLVMSetLinkage (method, LLVMInternalLinkage);
8269 //all methods have internal visibility when doing llvm_only
8270 if (!cfg->llvm_only && ctx->module->external_symbols) {
8271 LLVMSetLinkage (method, LLVMExternalLinkage);
8272 LLVMSetVisibility (method, LLVMHiddenVisibility);
8273 }
8274 }
8275 } else {
8276 LLVMSetLinkage (method, LLVMExternalLinkage);
8277 }
8278
8279 if (cfg->method->save_lmf && !cfg->llvm_only) {
8280 set_failure (ctx, "lmf");
8281 return;
8282 }
8283
8284 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
8285 set_failure (ctx, "pinvoke signature");
8286 return;
8287 }
8288
8289 header = cfg->header;
8290 for (i = 0; i < header->num_clauses; ++i) {
8291 clause = &header->clauses [i];
8292 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_FAULT && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
8293 set_failure (ctx, "non-finally/catch/fault clause.");
8294 return;
8295 }
8296 }
8297 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
8298 /* We can't handle inlined methods with clauses */
8299 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8300
8301 if (linfo->rgctx_arg) {
8302 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
8303 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
8304 /*
8305 * We mark the rgctx parameter with the inreg attribute, which is mapped to
8306 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
8307 * CC_X86_64_Mono in X86CallingConv.td.
8308 */
8309 if (!ctx->llvm_only)
8310 mono_llvm_add_param_attr (ctx->rgctx_arg, LLVM_ATTR_IN_REG);
8311 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
8312 } else {
8313 ctx->rgctx_arg_pindex = -1;
8314 }
8315 if (cfg->vret_addr) {
8316 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
8317 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
8318 if (linfo->ret.storage == LLVMArgVtypeByRef) {
8319 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
8320 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
8321 }
8322 }
8323
8324 if (sig->hasthis) {
8325 ctx->this_arg_pindex = linfo->this_arg_pindex;
8326 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
8327 values [cfg->args [0]->dreg] = ctx->this_arg;
8328 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
8329 }
8330 if (linfo->dummy_arg)
8331 LLVMSetValueName (LLVMGetParam (method, linfo->dummy_arg_pindex), "dummy_arg");
8332
8333 names = g_new (char *, sig->param_count);
8334 mono_method_get_param_names (cfg->method, (const char **) names);
8335
8336 /* Set parameter names/attributes */
8337 for (i = 0; i < sig->param_count; ++i) {
8338 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
8339 char *name;
8340 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
8341 int j;
8342
8343 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
8344 name = g_strdup_printf ("dummy_%d_%d", i, j);
8345 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
8346 g_free (name);
8347 }
8348
8349 if (ainfo->storage == LLVMArgVtypeInReg && ainfo->pair_storage [0] == LLVMArgNone && ainfo->pair_storage [1] == LLVMArgNone)
8350 continue;
8351
8352 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
8353 if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
8354 if (names [i] && names [i][0] != '\0')
8355 name = g_strdup_printf ("p_arg_%s", names [i]);
8356 else
8357 name = g_strdup_printf ("p_arg_%d", i);
8358 } else {
8359 if (names [i] && names [i][0] != '\0')
8360 name = g_strdup_printf ("arg_%s", names [i]);
8361 else
8362 name = g_strdup_printf ("arg_%d", i);
8363 }
8364 LLVMSetValueName (LLVMGetParam (method, pindex), name);
8365 g_free (name);
8366 if (ainfo->storage == LLVMArgVtypeByVal)
8367 mono_llvm_add_param_attr (LLVMGetParam (method, pindex), LLVM_ATTR_BY_VAL);
8368
8369 if (ainfo->storage == LLVMArgVtypeByRef || ainfo->storage == LLVMArgVtypeAddr) {
8370 /* For OP_LDADDR */
8371 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
8372 }
8373 }
8374 g_free (names);
8375
8376 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
8377 ctx->minfo = mono_debug_lookup_method (cfg->method);
8378 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
8379 }
8380
8381 max_block_num = 0;
8382 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
8383 max_block_num = MAX (max_block_num, bb->block_num);
8384 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
8385
8386 /* Add branches between non-consecutive bblocks */
8387 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8388 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
8389 bb->next_bb != bb->last_ins->inst_false_bb) {
8390
8391 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
8392 inst->opcode = OP_BR;
8393 inst->inst_target_bb = bb->last_ins->inst_false_bb;
8394 mono_bblock_add_inst (bb, inst);
8395 }
8396 }
8397
8398 /*
8399 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
8400 */
8401 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8402 MonoInst *ins;
8403 LLVMBuilderRef builder;
8404 char *dname;
8405 char dname_buf[128];
8406
8407 builder = create_builder (ctx);
8408
8409 for (ins = bb->code; ins; ins = ins->next) {
8410 switch (ins->opcode) {
8411 case OP_PHI:
8412 case OP_FPHI:
8413 case OP_VPHI:
8414 case OP_XPHI: {
8415 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
8416
8417 if (!ctx_ok (ctx))
8418 return;
8419
8420 if (ins->opcode == OP_VPHI) {
8421 /* Treat valuetype PHI nodes as operating on the address itself */
8422 g_assert (ins->klass);
8423 phi_type = LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)), 0);
8424 }
8425
8426 /*
8427 * Have to precreate these, as they can be referenced by
8428 * earlier instructions.
8429 */
8430 sprintf (dname_buf, "t%d", ins->dreg);
8431 dname = dname_buf;
8432 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
8433
8434 if (ins->opcode == OP_VPHI)
8435 ctx->addresses [ins->dreg] = values [ins->dreg];
8436
8437 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
8438
8439 /*
8440 * Set the expected type of the incoming arguments since these have
8441 * to have the same type.
8442 */
8443 for (i = 0; i < ins->inst_phi_args [0]; i++) {
8444 int sreg1 = ins->inst_phi_args [i + 1];
8445
8446 if (sreg1 != -1) {
8447 if (ins->opcode == OP_VPHI)
8448 ctx->is_vphi [sreg1] = TRUE;
8449 ctx->vreg_types [sreg1] = phi_type;
8450 }
8451 }
8452 break;
8453 }
8454 case OP_LDADDR:
8455 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
8456 break;
8457 default:
8458 break;
8459 }
8460 }
8461 }
8462
8463 /*
8464 * Create an ordering for bblocks, use the depth first order first, then
8465 * put the exception handling bblocks last.
8466 */
8467 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
8468 bb = cfg->bblocks [bb_index];
8469 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
8470 g_ptr_array_add (bblock_list, bb);
8471 bblocks [bb->block_num].added = TRUE;
8472 }
8473 }
8474
8475 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8476 if (!bblocks [bb->block_num].added)
8477 g_ptr_array_add (bblock_list, bb);
8478 }
8479
8480 /*
8481 * Second pass: generate code.
8482 */
8483 // Emit entry point
8484 entry_builder = create_builder (ctx);
8485 entry_bb = get_bb (ctx, cfg->bb_entry);
8486 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
8487 emit_entry_bb (ctx, entry_builder);
8488
8489 // Make landing pads first
8490 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
8491
8492 if (ctx->llvm_only) {
8493 size_t group_index = 0;
8494 while (group_index < cfg->header->num_clauses) {
8495 int count = 0;
8496 size_t cursor = group_index;
8497 while (cursor < cfg->header->num_clauses &&
8498 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
8499 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
8500 count++;
8501 cursor++;
8502 }
8503
8504 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
8505 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
8506 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
8507
8508 group_index = cursor;
8509 }
8510 }
8511
8512 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
8513 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
8514
8515 // Prune unreachable mono BBs.
8516 if (!(bb == cfg->bb_entry || bb->in_count > 0))
8517 continue;
8518
8519 process_bb (ctx, bb);
8520 if (!ctx_ok (ctx))
8521 return;
8522 }
8523 g_hash_table_destroy (ctx->exc_meta);
8524
8525 mono_memory_barrier ();
8526
8527 /* Add incoming phi values */
8528 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8529 GSList *l, *ins_list;
8530
8531 ins_list = bblocks [bb->block_num].phi_nodes;
8532
8533 for (l = ins_list; l; l = l->next) {
8534 PhiNode *node = (PhiNode*)l->data;
8535 MonoInst *phi = node->phi;
8536 int sreg1 = node->sreg;
8537 LLVMBasicBlockRef in_bb;
8538
8539 if (sreg1 == -1)
8540 continue;
8541
8542 in_bb = get_end_bb (ctx, node->in_bb);
8543
8544 if (ctx->unreachable [node->in_bb->block_num])
8545 continue;
8546
8547 if (phi->opcode == OP_VPHI) {
8548 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
8549 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
8550 } else {
8551 if (!values [sreg1]) {
8552 /* Can happen with values in EH clauses */
8553 set_failure (ctx, "incoming phi sreg1");
8554 return;
8555 }
8556 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
8557 set_failure (ctx, "incoming phi arg type mismatch");
8558 return;
8559 }
8560 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
8561 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
8562 }
8563 }
8564 }
8565
8566 /* Nullify empty phi instructions */
8567 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8568 GSList *l, *ins_list;
8569
8570 ins_list = bblocks [bb->block_num].phi_nodes;
8571
8572 for (l = ins_list; l; l = l->next) {
8573 PhiNode *node = (PhiNode*)l->data;
8574 MonoInst *phi = node->phi;
8575 LLVMValueRef phi_ins = values [phi->dreg];
8576
8577 if (!phi_ins)
8578 /* Already removed */
8579 continue;
8580
8581 if (LLVMCountIncoming (phi_ins) == 0) {
8582 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
8583 LLVMInstructionEraseFromParent (phi_ins);
8584 values [phi->dreg] = NULL;
8585 }
8586 }
8587 }
8588
8589 /* Create the SWITCH statements for ENDFINALLY instructions */
8590 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8591 BBInfo *info = &bblocks [bb->block_num];
8592 GSList *l;
8593 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
8594 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
8595 GSList *bb_list = info->call_handler_return_bbs;
8596
8597 GSList *bb_list_iter;
8598 i = 0;
8599 for (bb_list_iter = bb_list; bb_list_iter; bb_list_iter = g_slist_next (bb_list_iter)) {
8600 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)bb_list_iter->data);
8601 i ++;
8602 }
8603 }
8604 }
8605
8606 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
8607
8608 /* Initialize the method if needed */
8609 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
8610 // FIXME: Add more shared got entries
8611 ctx->builder = create_builder (ctx);
8612 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
8613
8614 // FIXME: beforefieldinit
8615 /*
8616 * NATIVE_TO_MANAGED methods might be called on a thread not attached to the runtime, so they are initialized when loaded
8617 * in load_method ().
8618 */
8619 gboolean needs_init = ctx->cfg->got_access_count > 0;
8620 MonoMethod *cctor = NULL;
8621 if (!needs_init && (cctor = mono_class_get_cctor (cfg->method->klass))) {
8622 /* Needs init to run the cctor */
8623 if (cfg->method->flags & METHOD_ATTRIBUTE_STATIC)
8624 needs_init = TRUE;
8625 if (cctor == cfg->method)
8626 needs_init = FALSE;
8627
8628 // If we are a constructor, we need to init so the static
8629 // constructor gets called.
8630 if (!strcmp (cfg->method->name, ".ctor"))
8631 needs_init = TRUE;
8632 }
8633 if (cfg->method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED)
8634 needs_init = FALSE;
8635 if (needs_init)
8636 emit_init_method (ctx);
8637 else
8638 LLVMBuildBr (ctx->builder, ctx->inited_bb);
8639
8640 // Was observing LLVM moving field accesses into the caller's method
8641 // body before the init call (the inlined one), leading to NULL derefs
8642 // after the init_method returns (GOT is filled out though)
8643 if (needs_init)
8644 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8645 }
8646
8647 if (mini_get_debug_options ()->llvm_disable_inlining)
8648 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8649
8650
8651 after_codegen:
8652 if (cfg->llvm_only) {
8653 g_ptr_array_add (ctx->module->cfgs, cfg);
8654
8655 /*
8656 * Add the contents of ctx->callsite_list to module->callsite_list.
8657 * We can't do this earlier, as it contains llvm instructions which can be
8658 * freed if compilation fails.
8659 * FIXME: Get rid of this when all methods can be llvm compiled.
8660 */
8661 for (int i = 0; i < ctx->callsite_list->len; ++i)
8662 g_ptr_array_add (ctx->module->callsite_list, g_ptr_array_index (ctx->callsite_list, i));
8663 }
8664
8665 if (cfg->verbose_level > 1) {
8666 g_print ("\n*** Unoptimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, TRUE));
8667 if (cfg->compile_aot) {
8668 mono_llvm_dump_value (method);
8669 } else {
8670 mono_llvm_dump_module (ctx->lmodule);
8671 }
8672 g_print ("***\n\n");
8673 }
8674
8675 if (cfg->compile_aot && !cfg->llvm_only)
8676 mark_as_used (ctx->module, method);
8677
8678 if (!cfg->llvm_only) {
8679 LLVMValueRef md_args [16];
8680 LLVMValueRef md_node;
8681 int method_index;
8682
8683 if (cfg->compile_aot)
8684 method_index = mono_aot_get_method_index (cfg->orig_method);
8685 else
8686 method_index = 1;
8687 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
8688 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
8689 md_node = LLVMMDNode (md_args, 2);
8690 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
8691 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
8692 }
8693
8694 if (cfg->compile_aot) {
8695 /* Don't generate native code, keep the LLVM IR */
8696 if (cfg->verbose_level) {
8697 char *name = mono_method_get_full_name (cfg->method);
8698 printf ("%s emitted as %s\n", name, ctx->method_name);
8699 g_free (name);
8700 }
8701
8702 //LLVMDumpValue (ctx->lmethod);
8703 //int err = LLVMVerifyFunction(ctx->lmethod, LLVMPrintMessageAction);
8704 //g_assert (err == 0);
8705 } else {
8706 //LLVMVerifyFunction (method, 0);
8707 llvm_jit_finalize_method (ctx);
8708 }
8709
8710 if (ctx->module->method_to_lmethod)
8711 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
8712
8713 if (ctx->module->idx_to_lmethod)
8714 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
8715
8716 if (ctx->llvm_only && m_class_is_valuetype (cfg->orig_method->klass) && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
8717 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
8718 }
8719
8720 /*
8721 * mono_llvm_create_vars:
8722 *
8723 * Same as mono_arch_create_vars () for LLVM.
8724 */
8725 void
8726 mono_llvm_create_vars (MonoCompile *cfg)
8727 {
8728 MonoMethodSignature *sig;
8729
8730 sig = mono_method_signature_internal (cfg->method);
8731 if (cfg->gsharedvt && cfg->llvm_only) {
8732 gboolean vretaddr = FALSE;
8733
8734 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
8735 vretaddr = TRUE;
8736 } else {
8737 MonoMethodSignature *sig = mono_method_signature_internal (cfg->method);
8738 LLVMCallInfo *linfo;
8739
8740 linfo = get_llvm_call_info (cfg, sig);
8741 vretaddr = (linfo->ret.storage == LLVMArgVtypeRetAddr || linfo->ret.storage == LLVMArgVtypeByRef || linfo->ret.storage == LLVMArgGsharedvtFixed || linfo->ret.storage == LLVMArgGsharedvtVariable || linfo->ret.storage == LLVMArgGsharedvtFixedVtype);
8742 }
8743 if (vretaddr) {
8744 /*
8745 * Creating vret_addr forces CEE_SETRET to store the result into it,
8746 * so we don't have to generate any code in our OP_SETRET case.
8747 */
8748 cfg->vret_addr = mono_compile_create_var (cfg, m_class_get_byval_arg (mono_get_intptr_class ()), OP_ARG);
8749 if (G_UNLIKELY (cfg->verbose_level > 1)) {
8750 printf ("vret_addr = ");
8751 mono_print_ins (cfg->vret_addr);
8752 }
8753 }
8754 } else {
8755 mono_arch_create_vars (cfg);
8756 }
8757 }
8758
8759 /*
8760 * mono_llvm_emit_call:
8761 *
8762 * Same as mono_arch_emit_call () for LLVM.
8763 */
8764 void
8765 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
8766 {
8767 MonoInst *in;
8768 MonoMethodSignature *sig;
8769 int i, n, stack_size;
8770 LLVMArgInfo *ainfo;
8771
8772 stack_size = 0;
8773
8774 sig = call->signature;
8775 n = sig->param_count + sig->hasthis;
8776
8777 call->cinfo = get_llvm_call_info (cfg, sig);
8778
8779 if (cfg->disable_llvm)
8780 return;
8781
8782 if (sig->call_convention == MONO_CALL_VARARG) {
8783 cfg->exception_message = g_strdup ("varargs");
8784 cfg->disable_llvm = TRUE;
8785 }
8786
8787 for (i = 0; i < n; ++i) {
8788 MonoInst *ins;
8789
8790 ainfo = call->cinfo->args + i;
8791
8792 in = call->args [i];
8793
8794 /* Simply remember the arguments */
8795 switch (ainfo->storage) {
8796 case LLVMArgNormal: {
8797 MonoType *t = (sig->hasthis && i == 0) ? m_class_get_byval_arg (mono_get_intptr_class ()) : ainfo->type;
8798 int opcode;
8799
8800 opcode = mono_type_to_regmove (cfg, t);
8801 if (opcode == OP_FMOVE) {
8802 MONO_INST_NEW (cfg, ins, OP_FMOVE);
8803 ins->dreg = mono_alloc_freg (cfg);
8804 } else if (opcode == OP_LMOVE) {
8805 MONO_INST_NEW (cfg, ins, OP_LMOVE);
8806 ins->dreg = mono_alloc_lreg (cfg);
8807 } else if (opcode == OP_RMOVE) {
8808 MONO_INST_NEW (cfg, ins, OP_RMOVE);
8809 ins->dreg = mono_alloc_freg (cfg);
8810 } else {
8811 MONO_INST_NEW (cfg, ins, OP_MOVE);
8812 ins->dreg = mono_alloc_ireg (cfg);
8813 }
8814 ins->sreg1 = in->dreg;
8815 break;
8816 }
8817 case LLVMArgVtypeByVal:
8818 case LLVMArgVtypeByRef:
8819 case LLVMArgVtypeInReg:
8820 case LLVMArgVtypeAddr:
8821 case LLVMArgVtypeAsScalar:
8822 case LLVMArgAsIArgs:
8823 case LLVMArgAsFpArgs:
8824 case LLVMArgGsharedvtVariable:
8825 case LLVMArgGsharedvtFixed:
8826 case LLVMArgGsharedvtFixedVtype:
8827 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
8828 ins->dreg = mono_alloc_ireg (cfg);
8829 ins->sreg1 = in->dreg;
8830 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
8831 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
8832 ins->inst_vtype = ainfo->type;
8833 ins->klass = mono_class_from_mono_type_internal (ainfo->type);
8834 break;
8835 default:
8836 cfg->exception_message = g_strdup ("ainfo->storage");
8837 cfg->disable_llvm = TRUE;
8838 return;
8839 }
8840
8841 if (!cfg->disable_llvm) {
8842 MONO_ADD_INS (cfg->cbb, ins);
8843 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
8844 }
8845 }
8846 }
8847
8848 static inline void
8849 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
8850 {
8851 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
8852 }
8853
8854 static inline void
8855 AddFunc1 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1)
8856 {
8857 LLVMTypeRef param_types [4];
8858
8859 param_types [0] = param_type1;
8860
8861 AddFunc (module, name, ret_type, param_types, 1);
8862 }
8863
8864 static inline void
8865 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
8866 {
8867 LLVMTypeRef param_types [4];
8868
8869 param_types [0] = param_type1;
8870 param_types [1] = param_type2;
8871
8872 AddFunc (module, name, ret_type, param_types, 2);
8873 }
8874
8875 typedef struct {
8876 IntrinsicId id;
8877 const char *name;
8878 } IntrinsicDesc;
8879
8880 static IntrinsicDesc intrinsics[] = {
8881 {INTRINS_MEMSET, "llvm.memset.p0i8.i32"},
8882 {INTRINS_MEMCPY, "llvm.memcpy.p0i8.p0i8.i32"},
8883 {INTRINS_MEMMOVE, "llvm.memmove.p0i8.p0i8.i64"},
8884 {INTRINS_SADD_OVF_I32, "llvm.sadd.with.overflow.i32"},
8885 {INTRINS_UADD_OVF_I32, "llvm.uadd.with.overflow.i32"},
8886 {INTRINS_SSUB_OVF_I32, "llvm.ssub.with.overflow.i32"},
8887 {INTRINS_USUB_OVF_I32, "llvm.usub.with.overflow.i32"},
8888 {INTRINS_SMUL_OVF_I32, "llvm.smul.with.overflow.i32"},
8889 {INTRINS_UMUL_OVF_I32, "llvm.umul.with.overflow.i32"},
8890 {INTRINS_SADD_OVF_I64, "llvm.sadd.with.overflow.i64"},
8891 {INTRINS_UADD_OVF_I64, "llvm.uadd.with.overflow.i64"},
8892 {INTRINS_SSUB_OVF_I64, "llvm.ssub.with.overflow.i64"},
8893 {INTRINS_USUB_OVF_I64, "llvm.usub.with.overflow.i64"},
8894 {INTRINS_SMUL_OVF_I64, "llvm.smul.with.overflow.i64"},
8895 {INTRINS_UMUL_OVF_I64, "llvm.umul.with.overflow.i64"},
8896 {INTRINS_SIN, "llvm.sin.f64"},
8897 {INTRINS_COS, "llvm.cos.f64"},
8898 {INTRINS_SQRT, "llvm.sqrt.f64"},
8899 {INTRINS_FLOOR, "llvm.floor.f64"},
8900 {INTRINS_FLOORF, "llvm.floor.f32"},
8901 {INTRINS_CEIL, "llvm.ceil.f64"},
8902 {INTRINS_CEILF, "llvm.ceil.f32"},
8903 {INTRINS_FMA, "llvm.fma.f64"},
8904 {INTRINS_FMAF, "llvm.fma.f32"},
8905 /* This isn't an intrinsic, instead llvm seems to special case it by name */
8906 {INTRINS_FABS, "fabs"},
8907 {INTRINS_ABSF, "llvm.fabs.f32"},
8908 {INTRINS_SINF, "llvm.sin.f32"},
8909 {INTRINS_COSF, "llvm.cos.f32"},
8910 {INTRINS_SQRTF, "llvm.sqrt.f32"},
8911 {INTRINS_POWF, "llvm.pow.f32"},
8912 {INTRINS_POW, "llvm.pow.f64"},
8913 {INTRINS_EXP, "llvm.exp.f64"},
8914 {INTRINS_EXPF, "llvm.exp.f32"},
8915 {INTRINS_LOG, "llvm.log.f64"},
8916 {INTRINS_LOG2, "llvm.log2.f64"},
8917 {INTRINS_LOG2F, "llvm.log2.f32"},
8918 {INTRINS_LOG10, "llvm.log10.f64"},
8919 {INTRINS_LOG10F, "llvm.log10.f32"},
8920 {INTRINS_TRUNC, "llvm.trunc.f64"},
8921 {INTRINS_TRUNCF, "llvm.trunc.f32"},
8922 {INTRINS_COPYSIGN, "llvm.copysign.f64"},
8923 {INTRINS_COPYSIGNF, "llvm.copysign.f32"},
8924 {INTRINS_EXPECT_I8, "llvm.expect.i8"},
8925 {INTRINS_EXPECT_I1, "llvm.expect.i1"},
8926 {INTRINS_CTPOP_I32, "llvm.ctpop.i32"},
8927 {INTRINS_CTPOP_I64, "llvm.ctpop.i64"},
8928 {INTRINS_CTLZ_I32, "llvm.ctlz.i32"},
8929 {INTRINS_CTLZ_I64, "llvm.ctlz.i64"},
8930 {INTRINS_CTTZ_I32, "llvm.cttz.i32"},
8931 {INTRINS_CTTZ_I64, "llvm.cttz.i64"},
8932 {INTRINS_BZHI_I32, "llvm.x86.bmi.bzhi.32"},
8933 {INTRINS_BZHI_I64, "llvm.x86.bmi.bzhi.64"},
8934 {INTRINS_BEXTR_I32, "llvm.x86.bmi.bextr.32"},
8935 {INTRINS_BEXTR_I64, "llvm.x86.bmi.bextr.64"},
8936 {INTRINS_PEXT_I32, "llvm.x86.bmi.pext.32"},
8937 {INTRINS_PEXT_I64, "llvm.x86.bmi.pext.64"},
8938 {INTRINS_PDEP_I32, "llvm.x86.bmi.pdep.32"},
8939 {INTRINS_PDEP_I64, "llvm.x86.bmi.pdep.64"},
8940 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8941 {INTRINS_SSE_PMOVMSKB, "llvm.x86.sse2.pmovmskb.128"},
8942 {INTRINS_SSE_PSRLI_W, "llvm.x86.sse2.psrli.w"},
8943 {INTRINS_SSE_PSRAI_W, "llvm.x86.sse2.psrai.w"},
8944 {INTRINS_SSE_PSLLI_W, "llvm.x86.sse2.pslli.w"},
8945 {INTRINS_SSE_PSRLI_D, "llvm.x86.sse2.psrli.d"},
8946 {INTRINS_SSE_PSRAI_D, "llvm.x86.sse2.psrai.d"},
8947 {INTRINS_SSE_PSLLI_D, "llvm.x86.sse2.pslli.d"},
8948 {INTRINS_SSE_PSRLI_Q, "llvm.x86.sse2.psrli.q"},
8949 {INTRINS_SSE_PSLLI_Q, "llvm.x86.sse2.pslli.q"},
8950 {INTRINS_SSE_SQRT_PD, "llvm.x86.sse2.sqrt.pd"},
8951 {INTRINS_SSE_SQRT_PS, "llvm.x86.sse.sqrt.ps"},
8952 {INTRINS_SSE_RSQRT_PS, "llvm.x86.sse.rsqrt.ps"},
8953 {INTRINS_SSE_RCP_PS, "llvm.x86.sse.rcp.ps"},
8954 {INTRINS_SSE_CVTTPD2DQ, "llvm.x86.sse2.cvttpd2dq"},
8955 {INTRINS_SSE_CVTTPS2DQ, "llvm.x86.sse2.cvttps2dq"},
8956 {INTRINS_SSE_CVTDQ2PD, "llvm.x86.sse2.cvtdq2pd"},
8957 {INTRINS_SSE_CVTDQ2PS, "llvm.x86.sse2.cvtdq2ps"},
8958 {INTRINS_SSE_CVTPD2DQ, "llvm.x86.sse2.cvtpd2dq"},
8959 {INTRINS_SSE_CVTPS2DQ, "llvm.x86.sse2.cvtps2dq"},
8960 {INTRINS_SSE_CVTPD2PS, "llvm.x86.sse2.cvtpd2ps"},
8961 {INTRINS_SSE_CVTPS2PD, "llvm.x86.sse2.cvtps2pd"},
8962 {INTRINS_SSE_CMPPD, "llvm.x86.sse2.cmp.pd"},
8963 {INTRINS_SSE_CMPPS, "llvm.x86.sse.cmp.ps"},
8964 {INTRINS_SSE_PACKSSWB, "llvm.x86.sse2.packsswb.128"},
8965 {INTRINS_SSE_PACKUSWB, "llvm.x86.sse2.packuswb.128"},
8966 {INTRINS_SSE_PACKSSDW, "llvm.x86.sse2.packssdw.128"},
8967 {INTRINS_SSE_PACKUSDW, "llvm.x86.sse41.packusdw"},
8968 {INTRINS_SSE_MINPS, "llvm.x86.sse.min.ps"},
8969 {INTRINS_SSE_MAXPS, "llvm.x86.sse.max.ps"},
8970 {INTRINS_SSE_HADDPS, "llvm.x86.sse3.hadd.ps"},
8971 {INTRINS_SSE_HSUBPS, "llvm.x86.sse3.hsub.ps"},
8972 {INTRINS_SSE_ADDSUBPS, "llvm.x86.sse3.addsub.ps"},
8973 {INTRINS_SSE_MINPD, "llvm.x86.sse2.min.pd"},
8974 {INTRINS_SSE_MAXPD, "llvm.x86.sse2.max.pd"},
8975 {INTRINS_SSE_HADDPD, "llvm.x86.sse3.hadd.pd"},
8976 {INTRINS_SSE_HSUBPD, "llvm.x86.sse3.hsub.pd"},
8977 {INTRINS_SSE_ADDSUBPD, "llvm.x86.sse3.addsub.pd"},
8978 {INTRINS_SSE_PADDSW, "llvm.x86.sse2.padds.w"},
8979 {INTRINS_SSE_PSUBSW, "llvm.x86.sse2.psubs.w"},
8980 {INTRINS_SSE_PADDUSW, "llvm.x86.sse2.paddus.w"},
8981 {INTRINS_SSE_PSUBUSW, "llvm.x86.sse2.psubus.w"},
8982 {INTRINS_SSE_PAVGW, "llvm.x86.sse2.pavg.w"},
8983 {INTRINS_SSE_PMULHW, "llvm.x86.sse2.pmulh.w"},
8984 {INTRINS_SSE_PMULHU, "llvm.x86.sse2.pmulhu.w"},
8985 {INTRINS_SE_PADDSB, "llvm.x86.sse2.padds.b"},
8986 {INTRINS_SSE_PSUBSB, "llvm.x86.sse2.psubs.b"},
8987 {INTRINS_SSE_PADDUSB, "llvm.x86.sse2.paddus.b"},
8988 {INTRINS_SSE_PSUBUSB, "llvm.x86.sse2.psubus.b"},
8989 {INTRINS_SSE_PAVGB, "llvm.x86.sse2.pavg.b"},
8990 {INTRINS_SSE_PAUSE, "llvm.x86.sse2.pause"},
8991 {INTRINS_SSE_DPPS, "llvm.x86.sse41.dpps"},
8992 {INTRINS_SSE_ROUNDSS, "llvm.x86.sse41.round.ss"},
8993 {INTRINS_SSE_ROUNDPD, "llvm.x86.sse41.round.pd"},
8994 #endif
8995 #ifdef TARGET_WASM
8996 {INTRINS_WASM_ANYTRUE_V16, "llvm.wasm.anytrue.v16i8"},
8997 {INTRINS_WASM_ANYTRUE_V8, "llvm.wasm.anytrue.v8i16"},
8998 {INTRINS_WASM_ANYTRUE_V4, "llvm.wasm.anytrue.v4i32"},
8999 {INTRINS_WASM_ANYTRUE_V2, "llvm.wasm.anytrue.v2i64"},
9000 #endif
9001 };
9002
9003 static void
9004 add_sse_binary (LLVMModuleRef module, const char *name, int type)
9005 {
9006 LLVMTypeRef ret_type = type_to_simd_type (type);
9007 AddFunc2 (module, name, ret_type, ret_type, ret_type);
9008 }
9009
9010 static void
9011 add_intrinsic (LLVMModuleRef module, int id)
9012 {
9013 const char *name;
9014 #if defined(TARGET_AMD64) || defined(TARGET_X86)
9015 LLVMTypeRef ret_type, arg_types [16];
9016 #endif
9017
9018 name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
9019 g_assert (name);
9020
9021 switch (id) {
9022 case INTRINS_MEMSET: {
9023 #if LLVM_API_VERSION >= 900
9024 /* No alignment argument */
9025 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt1Type () };
9026
9027 AddFunc (module, name, LLVMVoidType (), params, 4);
9028 #else
9029 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
9030
9031 AddFunc (module, name, LLVMVoidType (), params, 5);
9032 #endif
9033 break;
9034 }
9035 case INTRINS_MEMCPY: {
9036 #if LLVM_API_VERSION >= 900
9037 /* No alignment argument */
9038 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt1Type () };
9039
9040 AddFunc (module, name, LLVMVoidType (), params, 4);
9041 #else
9042 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
9043
9044 AddFunc (module, name, LLVMVoidType (), params, 5);
9045 #endif
9046 break;
9047 }
9048 case INTRINS_MEMMOVE: {
9049 #if LLVM_API_VERSION >= 900
9050 /* No alignment argument */
9051 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type (), LLVMInt1Type () };
9052 AddFunc (module, name, LLVMVoidType (), params, 4);
9053 #else
9054 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type (), LLVMInt32Type (), LLVMInt1Type () };
9055 AddFunc (module, name, LLVMVoidType (), params, 5);
9056 #endif
9057 break;
9058 }
9059 case INTRINS_SADD_OVF_I32:
9060 case INTRINS_UADD_OVF_I32:
9061 case INTRINS_SSUB_OVF_I32:
9062 case INTRINS_USUB_OVF_I32:
9063 case INTRINS_SMUL_OVF_I32:
9064 case INTRINS_UMUL_OVF_I32: {
9065 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
9066 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
9067 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
9068
9069 AddFunc (module, name, ret_type, params, 2);
9070 break;
9071 }
9072 case INTRINS_SADD_OVF_I64:
9073 case INTRINS_UADD_OVF_I64:
9074 case INTRINS_SSUB_OVF_I64:
9075 case INTRINS_USUB_OVF_I64:
9076 case INTRINS_SMUL_OVF_I64:
9077 case INTRINS_UMUL_OVF_I64: {
9078 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
9079 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
9080 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
9081
9082 AddFunc (module, name, ret_type, params, 2);
9083 break;
9084 }
9085 case INTRINS_FMA: {
9086 LLVMTypeRef params [] = { LLVMDoubleType (), LLVMDoubleType (), LLVMDoubleType () };
9087
9088 AddFunc (module, name, LLVMDoubleType (), params, 3);
9089 break;
9090 }
9091 case INTRINS_FMAF: {
9092 LLVMTypeRef params [] = { LLVMFloatType (), LLVMFloatType (), LLVMFloatType () };
9093
9094 AddFunc (module, name, LLVMFloatType (), params, 3);
9095 break;
9096 }
9097 case INTRINS_EXP:
9098 case INTRINS_LOG:
9099 case INTRINS_LOG2:
9100 case INTRINS_LOG10:
9101 case INTRINS_TRUNC:
9102 case INTRINS_SIN:
9103 case INTRINS_COS:
9104 case INTRINS_SQRT:
9105 case INTRINS_FLOOR:
9106 case INTRINS_CEIL:
9107 case INTRINS_FABS: {
9108 AddFunc1 (module, name, LLVMDoubleType (), LLVMDoubleType ());
9109 break;
9110 }
9111 case INTRINS_EXPF:
9112 case INTRINS_LOG2F:
9113 case INTRINS_LOG10F:
9114 case INTRINS_TRUNCF:
9115 case INTRINS_SINF:
9116 case INTRINS_COSF:
9117 case INTRINS_SQRTF:
9118 case INTRINS_FLOORF:
9119 case INTRINS_CEILF:
9120 case INTRINS_ABSF: {
9121 AddFunc1 (module, name, LLVMFloatType (), LLVMFloatType ());
9122 break;
9123 }
9124 case INTRINS_COPYSIGNF:
9125 case INTRINS_POWF:
9126 AddFunc2 (module, name, LLVMFloatType (), LLVMFloatType (), LLVMFloatType ());
9127 break;
9128 case INTRINS_COPYSIGN:
9129 case INTRINS_POW:
9130 AddFunc2 (module, name, LLVMDoubleType (), LLVMDoubleType (), LLVMDoubleType ());
9131 break;
9132 case INTRINS_EXPECT_I8:
9133 AddFunc2 (module, name, LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
9134 break;
9135 case INTRINS_EXPECT_I1:
9136 AddFunc2 (module, name, LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type ());
9137 break;
9138 case INTRINS_CTPOP_I32:
9139 AddFunc1 (module, name, LLVMInt32Type (), LLVMInt32Type ());
9140 break;
9141 case INTRINS_CTPOP_I64:
9142 AddFunc1 (module, name, LLVMInt64Type (), LLVMInt64Type ());
9143 break;
9144 case INTRINS_CTLZ_I32:
9145 case INTRINS_CTTZ_I32:
9146 AddFunc2 (module, name, LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type ());
9147 break;
9148 case INTRINS_CTLZ_I64:
9149 case INTRINS_CTTZ_I64:
9150 AddFunc2 (module, name, LLVMInt64Type (), LLVMInt64Type (), LLVMInt1Type ());
9151 break;
9152 case INTRINS_BEXTR_I32:
9153 case INTRINS_BZHI_I32:
9154 case INTRINS_PEXT_I32:
9155 case INTRINS_PDEP_I32:
9156 AddFunc2 (module, name, LLVMInt32Type (), LLVMInt32Type (), LLVMInt32Type ());
9157 break;
9158 case INTRINS_BEXTR_I64:
9159 case INTRINS_BZHI_I64:
9160 case INTRINS_PEXT_I64:
9161 case INTRINS_PDEP_I64:
9162 AddFunc2 (module, name, LLVMInt64Type (), LLVMInt64Type (), LLVMInt64Type ());
9163 break;
9164 #if defined(TARGET_AMD64) || defined(TARGET_X86)
9165 case INTRINS_SSE_PMOVMSKB:
9166 /* pmovmskb */
9167 ret_type = LLVMInt32Type ();
9168 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
9169 AddFunc (module, name, ret_type, arg_types, 1);
9170 break;
9171 case INTRINS_SSE_PSRLI_W:
9172 case INTRINS_SSE_PSRAI_W:
9173 case INTRINS_SSE_PSLLI_W:
9174 /* shifts */
9175 ret_type = type_to_simd_type (MONO_TYPE_I2);
9176 arg_types [0] = ret_type;
9177 arg_types [1] = LLVMInt32Type ();
9178 AddFunc (module, name, ret_type, arg_types, 2);
9179 break;
9180 case INTRINS_SSE_PSRLI_D:
9181 case INTRINS_SSE_PSRAI_D:
9182 case INTRINS_SSE_PSLLI_D:
9183 ret_type = type_to_simd_type (MONO_TYPE_I4);
9184 arg_types [0] = ret_type;
9185 arg_types [1] = LLVMInt32Type ();
9186 AddFunc (module, name, ret_type, arg_types, 2);
9187 break;
9188 case INTRINS_SSE_PSRLI_Q:
9189 case INTRINS_SSE_PSLLI_Q:
9190 ret_type = type_to_simd_type (MONO_TYPE_I8);
9191 arg_types [0] = ret_type;
9192 arg_types [1] = LLVMInt32Type ();
9193 AddFunc (module, name, ret_type, arg_types, 2);
9194 break;
9195 case INTRINS_SSE_SQRT_PD:
9196 /* Unary ops */
9197 ret_type = type_to_simd_type (MONO_TYPE_R8);
9198 arg_types [0] = ret_type;
9199 AddFunc (module, name, ret_type, arg_types, 1);
9200 break;
9201 case INTRINS_SSE_SQRT_PS:
9202 ret_type = type_to_simd_type (MONO_TYPE_R4);
9203 arg_types [0] = ret_type;
9204 AddFunc (module, name, ret_type, arg_types, 1);
9205 break;
9206 case INTRINS_SSE_RSQRT_PS:
9207 ret_type = type_to_simd_type (MONO_TYPE_R4);
9208 arg_types [0] = ret_type;
9209 AddFunc (module, name, ret_type, arg_types, 1);
9210 break;
9211 case INTRINS_SSE_RCP_PS:
9212 ret_type = type_to_simd_type (MONO_TYPE_R4);
9213 arg_types [0] = ret_type;
9214 AddFunc (module, name, ret_type, arg_types, 1);
9215 break;
9216 case INTRINS_SSE_CVTTPD2DQ:
9217 ret_type = type_to_simd_type (MONO_TYPE_I4);
9218 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9219 AddFunc (module, name, ret_type, arg_types, 1);
9220 break;
9221 case INTRINS_SSE_CVTTPS2DQ:
9222 ret_type = type_to_simd_type (MONO_TYPE_I4);
9223 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9224 AddFunc (module, name, ret_type, arg_types, 1);
9225 break;
9226 case INTRINS_SSE_CVTDQ2PD:
9227 /* Conversion ops */
9228 ret_type = type_to_simd_type (MONO_TYPE_R8);
9229 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
9230 AddFunc (module, name, ret_type, arg_types, 1);
9231 break;
9232 case INTRINS_SSE_CVTDQ2PS:
9233 ret_type = type_to_simd_type (MONO_TYPE_R4);
9234 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
9235 AddFunc (module, name, ret_type, arg_types, 1);
9236 break;
9237 case INTRINS_SSE_CVTPD2DQ:
9238 ret_type = type_to_simd_type (MONO_TYPE_I4);
9239 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9240 AddFunc (module, name, ret_type, arg_types, 1);
9241 break;
9242 case INTRINS_SSE_CVTPS2DQ:
9243 ret_type = type_to_simd_type (MONO_TYPE_I4);
9244 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9245 AddFunc (module, name, ret_type, arg_types, 1);
9246 break;
9247 case INTRINS_SSE_CVTPD2PS:
9248 ret_type = type_to_simd_type (MONO_TYPE_R4);
9249 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9250 AddFunc (module, name, ret_type, arg_types, 1);
9251 break;
9252 case INTRINS_SSE_CVTPS2PD:
9253 ret_type = type_to_simd_type (MONO_TYPE_R8);
9254 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9255 AddFunc (module, name, ret_type, arg_types, 1);
9256 break;
9257 case INTRINS_SSE_CMPPD:
9258 /* cmp pd/ps */
9259 ret_type = type_to_simd_type (MONO_TYPE_R8);
9260 arg_types [0] = ret_type;
9261 arg_types [1] = ret_type;
9262 arg_types [2] = LLVMInt8Type ();
9263 AddFunc (module, name, ret_type, arg_types, 3);
9264 break;
9265 case INTRINS_SSE_CMPPS:
9266 ret_type = type_to_simd_type (MONO_TYPE_R4);
9267 arg_types [0] = ret_type;
9268 arg_types [1] = ret_type;
9269 arg_types [2] = LLVMInt8Type ();
9270 AddFunc (module, name, ret_type, arg_types, 3);
9271 break;
9272 case INTRINS_SSE_PACKSSWB:
9273 case INTRINS_SSE_PACKUSWB:
9274 case INTRINS_SSE_PACKSSDW:
9275 /* pack */
9276 ret_type = type_to_simd_type (MONO_TYPE_I1);
9277 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
9278 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
9279 AddFunc (module, name, ret_type, arg_types, 2);
9280 break;
9281 case INTRINS_SSE_PACKUSDW:
9282 ret_type = type_to_simd_type (MONO_TYPE_I2);
9283 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
9284 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
9285 AddFunc (module, name, ret_type, arg_types, 2);
9286 break;
9287 /* SSE Binary ops */
9288 case INTRINS_SSE_PADDSW:
9289 case INTRINS_SSE_PSUBSW:
9290 case INTRINS_SSE_PADDUSW:
9291 case INTRINS_SSE_PSUBUSW:
9292 case INTRINS_SSE_PAVGW:
9293 case INTRINS_SSE_PMULHW:
9294 case INTRINS_SSE_PMULHU:
9295 add_sse_binary (module, name, MONO_TYPE_I2);
9296 break;
9297 case INTRINS_SSE_MINPS:
9298 case INTRINS_SSE_MAXPS:
9299 case INTRINS_SSE_HADDPS:
9300 case INTRINS_SSE_HSUBPS:
9301 case INTRINS_SSE_ADDSUBPS:
9302 add_sse_binary (module, name, MONO_TYPE_R4);
9303 break;
9304 case INTRINS_SSE_MINPD:
9305 case INTRINS_SSE_MAXPD:
9306 case INTRINS_SSE_HADDPD:
9307 case INTRINS_SSE_HSUBPD:
9308 case INTRINS_SSE_ADDSUBPD:
9309 add_sse_binary (module, name, MONO_TYPE_R8);
9310 break;
9311 case INTRINS_SE_PADDSB:
9312 case INTRINS_SSE_PSUBSB:
9313 case INTRINS_SSE_PADDUSB:
9314 case INTRINS_SSE_PSUBUSB:
9315 case INTRINS_SSE_PAVGB:
9316 add_sse_binary (module, name, MONO_TYPE_I1);
9317 break;
9318 case INTRINS_SSE_PAUSE:
9319 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
9320 break;
9321 case INTRINS_SSE_DPPS:
9322 ret_type = type_to_simd_type (MONO_TYPE_R4);
9323 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9324 arg_types [1] = type_to_simd_type (MONO_TYPE_R4);
9325 arg_types [2] = LLVMInt8Type ();
9326 AddFunc (module, name, ret_type, arg_types, 3);
9327 break;
9328 case INTRINS_SSE_ROUNDSS:
9329 ret_type = type_to_simd_type (MONO_TYPE_R4);
9330 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
9331 arg_types [1] = type_to_simd_type (MONO_TYPE_R4);
9332 arg_types [2] = LLVMInt32Type ();
9333 AddFunc (module, name, ret_type, arg_types, 3);
9334 break;
9335 case INTRINS_SSE_ROUNDPD:
9336 ret_type = type_to_simd_type (MONO_TYPE_R8);
9337 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
9338 arg_types [1] = LLVMInt32Type ();
9339 AddFunc (module, name, ret_type, arg_types, 2);
9340 break;
9341 #endif /* AMD64 || X86 */
9342 #ifdef TARGET_WASM
9343 case INTRINS_WASM_ANYTRUE_V16:
9344 AddFunc1 (module, name, LLVMInt32Type (), type_to_simd_type (MONO_TYPE_I1));
9345 break;
9346 case INTRINS_WASM_ANYTRUE_V8:
9347 AddFunc1 (module, name, LLVMInt32Type (), type_to_simd_type (MONO_TYPE_I2));
9348 break;
9349 case INTRINS_WASM_ANYTRUE_V4:
9350 AddFunc1 (module, name, LLVMInt32Type (), type_to_simd_type (MONO_TYPE_I4));
9351 break;
9352 case INTRINS_WASM_ANYTRUE_V2:
9353 AddFunc1 (module, name, LLVMInt32Type (), type_to_simd_type (MONO_TYPE_I8));
9354 break;
9355 #endif
9356 default:
9357 g_assert_not_reached ();
9358 break;
9359 }
9360 }
9361
9362 static LLVMValueRef
9363 get_intrins_from_module (LLVMModuleRef lmodule, int id)
9364 {
9365 LLVMValueRef res;
9366
9367 const char *name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
9368 g_assert (name);
9369
9370 res = LLVMGetNamedFunction (lmodule, name);
9371 if (!res) {
9372 add_intrinsic (lmodule, id);
9373 res = LLVMGetNamedFunction (lmodule, name);
9374 g_assert (res);
9375 }
9376 return res;
9377 }
9378
9379 static LLVMValueRef
9380 get_intrins (EmitContext *ctx, int id)
9381 {
9382 MonoLLVMModule *module = ctx->module;
9383 LLVMValueRef res;
9384
9385 /*
9386 * Every method is emitted into its own module so
9387 * we can add intrinsics on demand.
9388 */
9389 res = module->intrins_by_id [id];
9390 if (!res) {
9391 res = get_intrins_from_module (ctx->lmodule, id);
9392 module->intrins_by_id [id] = res;
9393 }
9394 return res;
9395 }
9396
9397 static LLVMValueRef
9398 get_intrins_by_name (EmitContext *ctx, const char *name)
9399 {
9400 LLVMValueRef res;
9401
9402 /*
9403 * Every method is emitted into its own module so
9404 * we can add intrinsics on demand.
9405 */
9406 res = LLVMGetNamedFunction (ctx->lmodule, name);
9407 if (!res) {
9408 int id = -1;
9409
9410 /* No locking needed */
9411 id = GPOINTER_TO_INT (g_hash_table_lookup (intrins_name_to_id, name));
9412 id --;
9413 if (id == -1)
9414 printf ("%s\n", name);
9415 g_assert (id != -1);
9416 add_intrinsic (ctx->lmodule, id);
9417 res = LLVMGetNamedFunction (ctx->lmodule, name);
9418 g_assert (res);
9419 }
9420
9421 return res;
9422 }
9423
9424 static void
9425 add_intrinsics (LLVMModuleRef module)
9426 {
9427 int i;
9428
9429 /* Emit declarations of instrinsics */
9430 /*
9431 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
9432 * type doesn't seem to do any locking.
9433 */
9434 for (i = 0; i < INTRINS_NUM; ++i)
9435 add_intrinsic (module, i);
9436
9437 /* EH intrinsics */
9438 {
9439 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
9440
9441 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
9442 }
9443
9444 /* Load/Store intrinsics */
9445 {
9446 LLVMTypeRef arg_types [5];
9447 int i;
9448 char name [128];
9449
9450 for (i = 1; i <= 8; i *= 2) {
9451 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
9452 arg_types [1] = LLVMInt32Type ();
9453 arg_types [2] = LLVMInt1Type ();
9454 arg_types [3] = LLVMInt32Type ();
9455 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
9456 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
9457
9458 arg_types [0] = LLVMIntType (i * 8);
9459 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
9460 arg_types [2] = LLVMInt32Type ();
9461 arg_types [3] = LLVMInt1Type ();
9462 arg_types [4] = LLVMInt32Type ();
9463 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
9464 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
9465 }
9466 }
9467 }
9468
9469 static void
9470 add_types (MonoLLVMModule *module)
9471 {
9472 module->ptr_type = LLVMPointerType (TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
9473 }
9474
9475 void
9476 mono_llvm_init (void)
9477 {
9478 GHashTable *h;
9479 int i;
9480
9481 mono_native_tls_alloc (&current_cfg_tls_id, NULL);
9482
9483 h = g_hash_table_new (NULL, NULL);
9484 for (i = 0; i < INTRINS_NUM; ++i)
9485 g_hash_table_insert (h, GINT_TO_POINTER (intrinsics [i].id), (gpointer)intrinsics [i].name);
9486 intrins_id_to_name = h;
9487
9488 h = g_hash_table_new (g_str_hash, g_str_equal);
9489 for (i = 0; i < INTRINS_NUM; ++i)
9490 g_hash_table_insert (h, (gpointer)intrinsics [i].name, GINT_TO_POINTER (intrinsics [i].id + 1));
9491 intrins_name_to_id = h;
9492 }
9493
9494 void
9495 mono_llvm_cleanup (void)
9496 {
9497 MonoLLVMModule *module = &aot_module;
9498
9499 if (module->lmodule)
9500 LLVMDisposeModule (module->lmodule);
9501
9502 if (module->context)
9503 LLVMContextDispose (module->context);
9504 }
9505
9506 void
9507 mono_llvm_free_domain_info (MonoDomain *domain)
9508 {
9509 MonoJitDomainInfo *info = domain_jit_info (domain);
9510 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
9511 int i;
9512
9513 if (!module)
9514 return;
9515
9516 g_hash_table_destroy (module->llvm_types);
9517
9518 mono_llvm_dispose_ee (module->mono_ee);
9519
9520 if (module->bb_names) {
9521 for (i = 0; i < module->bb_names_len; ++i)
9522 g_free (module->bb_names [i]);
9523 g_free (module->bb_names);
9524 }
9525 //LLVMDisposeModule (module->module);
9526
9527 g_free (module);
9528
9529 info->llvm_module = NULL;
9530 }
9531
9532 void
9533 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, int initial_got_size, LLVMModuleFlags flags)
9534 {
9535 MonoLLVMModule *module = &aot_module;
9536 gboolean emit_dwarf = (flags & LLVM_MODULE_FLAG_DWARF) ? 1 : 0;
9537 #ifdef TARGET_WIN32_MSVC
9538 gboolean emit_codeview = (flags & LLVM_MODULE_FLAG_CODEVIEW) ? 1 : 0;
9539 #endif
9540 gboolean static_link = (flags & LLVM_MODULE_FLAG_STATIC) ? 1 : 0;
9541 gboolean llvm_only = (flags & LLVM_MODULE_FLAG_LLVM_ONLY) ? 1 : 0;
9542 gboolean interp = (flags & LLVM_MODULE_FLAG_INTERP) ? 1 : 0;
9543 gboolean llvm_disable_self_init = mini_get_debug_options ()->llvm_disable_self_init;
9544
9545 /* Delete previous module */
9546 g_hash_table_destroy (module->plt_entries);
9547 if (module->lmodule)
9548 LLVMDisposeModule (module->lmodule);
9549
9550 memset (module, 0, sizeof (aot_module));
9551
9552 module->lmodule = LLVMModuleCreateWithName ("aot");
9553 module->assembly = assembly;
9554 module->global_prefix = g_strdup (global_prefix);
9555 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
9556 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
9557 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
9558 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
9559 module->external_symbols = TRUE;
9560 module->emit_dwarf = emit_dwarf;
9561 module->static_link = static_link;
9562 module->llvm_only = llvm_only;
9563 module->llvm_disable_self_init = llvm_disable_self_init && !llvm_only; // llvm_only implies !llvm_disable_self_init
9564 module->interp = interp;
9565 /* The first few entries are reserved */
9566 module->max_got_offset = initial_got_size;
9567 module->context = LLVMGetGlobalContext ();
9568 module->cfgs = g_ptr_array_new ();
9569 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
9570
9571 if (llvm_only)
9572 /* clang ignores our debug info because it has an invalid version */
9573 module->emit_dwarf = FALSE;
9574
9575 add_intrinsics (module->lmodule);
9576 add_types (module);
9577
9578 #ifdef MONO_ARCH_LLVM_TARGET_LAYOUT
9579 LLVMSetDataLayout (module->lmodule, MONO_ARCH_LLVM_TARGET_LAYOUT);
9580 #else
9581 g_assert_not_reached ();
9582 #endif
9583
9584 #ifdef MONO_ARCH_LLVM_TARGET_TRIPLE
9585 LLVMSetTarget (module->lmodule, MONO_ARCH_LLVM_TARGET_TRIPLE);
9586 #endif
9587
9588 if (module->emit_dwarf) {
9589 char *dir, *build_info, *s, *cu_name;
9590
9591 module->di_builder = mono_llvm_create_di_builder (module->lmodule);
9592
9593 // FIXME:
9594 dir = g_strdup (".");
9595 build_info = mono_get_runtime_build_info ();
9596 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
9597 cu_name = g_path_get_basename (assembly->image->name);
9598 module->cu = mono_llvm_di_create_compile_unit (module->di_builder, cu_name, dir, s);
9599 g_free (dir);
9600 g_free (build_info);
9601 g_free (s);
9602 }
9603
9604 #ifdef TARGET_WIN32_MSVC
9605 if (emit_codeview) {
9606 LLVMValueRef codeview_option_args[3];
9607
9608 codeview_option_args[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9609 codeview_option_args[1] = LLVMMDString ("CodeView", 8);
9610 codeview_option_args[2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
9611
9612 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.module.flags", LLVMMDNode (codeview_option_args, G_N_ELEMENTS (codeview_option_args)));
9613 }
9614
9615 if (!static_link) {
9616 const char linker_options[] = "Linker Options";
9617 const char *default_dynamic_lib_names[] = { "/DEFAULTLIB:msvcrt",
9618 "/DEFAULTLIB:ucrt.lib",
9619 "/DEFAULTLIB:vcruntime.lib" };
9620
9621 LLVMValueRef default_lib_args[G_N_ELEMENTS (default_dynamic_lib_names)];
9622 LLVMValueRef default_lib_nodes[G_N_ELEMENTS(default_dynamic_lib_names)];
9623
9624 const char *default_lib_name = NULL;
9625 for (int i = 0; i < G_N_ELEMENTS (default_dynamic_lib_names); ++i) {
9626 const char *default_lib_name = default_dynamic_lib_names[i];
9627 default_lib_args[i] = LLVMMDString (default_lib_name, strlen (default_lib_name));
9628 default_lib_nodes[i] = LLVMMDNode (default_lib_args + i, 1);
9629 }
9630
9631 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.linker.options", LLVMMDNode (default_lib_args, G_N_ELEMENTS (default_lib_args)));
9632 }
9633 #endif
9634
9635 /* Add GOT */
9636 /*
9637 * We couldn't compute the type of the LLVM global representing the got because
9638 * its size is only known after all the methods have been emitted. So create
9639 * a dummy variable, and replace all uses it with the real got variable when
9640 * its size is known in mono_llvm_emit_aot_module ().
9641 */
9642 {
9643 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
9644
9645 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
9646 module->got_idx_to_type = g_hash_table_new (NULL, NULL);
9647 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
9648 }
9649
9650 /* Add initialization array */
9651 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
9652
9653 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
9654 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
9655
9656
9657 emit_gc_safepoint_poll (module, module->lmodule, NULL);
9658
9659 emit_llvm_code_start (module);
9660
9661 // Needs idx_to_lmethod
9662 emit_init_icall_wrappers (module);
9663
9664 /* Add a dummy personality function */
9665 if (!use_mono_personality_debug) {
9666 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
9667 LLVMSetLinkage (personality, LLVMExternalLinkage);
9668
9669 //EMCC chockes if the personality function is referenced in the 'used' array
9670 #ifndef TARGET_WASM
9671 mark_as_used (module, personality);
9672 #endif
9673 }
9674
9675 /* Add a reference to the c++ exception we throw/catch */
9676 {
9677 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
9678 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
9679 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
9680 mono_llvm_set_is_constant (module->sentinel_exception);
9681 }
9682
9683 module->llvm_types = g_hash_table_new (NULL, NULL);
9684 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
9685 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
9686 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
9687 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
9688 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
9689 module->method_to_call_info = g_hash_table_new (NULL, NULL);
9690 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
9691 module->callsite_list = g_ptr_array_new ();
9692 }
9693
9694 void
9695 mono_llvm_fixup_aot_module (void)
9696 {
9697 MonoLLVMModule *module = &aot_module;
9698 MonoMethod *method;
9699
9700 if (module->llvm_disable_self_init)
9701 return;
9702
9703 /*
9704 * Replace GOT entries for directly callable methods with the methods themselves.
9705 * It would be easier to implement this by predefining all methods before compiling
9706 * their bodies, but that couldn't handle the case when a method fails to compile
9707 * with llvm.
9708 */
9709
9710 GHashTable *specializable = g_hash_table_new (NULL, NULL);
9711
9712 GHashTable *patches_to_null = g_hash_table_new (mono_patch_info_hash, mono_patch_info_equal);
9713 for (int sindex = 0; sindex < module->callsite_list->len; ++sindex) {
9714 CallSite *site = (CallSite*)g_ptr_array_index (module->callsite_list, sindex);
9715 method = site->method;
9716 LLVMValueRef lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
9717
9718 LLVMValueRef placeholder = (LLVMValueRef)site->load;
9719 LLVMValueRef indexes [2], got_entry_addr, load;
9720 char *name;
9721
9722 if (lmethod && !(method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)) {
9723 mono_llvm_replace_uses_of (placeholder, lmethod);
9724
9725 if (mono_aot_can_specialize (method))
9726 g_hash_table_insert (specializable, lmethod, method);
9727
9728 g_hash_table_insert (patches_to_null, site->ji, site->ji);
9729 } else {
9730 int got_offset = compute_aot_got_offset (module, site->ji, site->type);
9731
9732 module->max_got_offset = MAX (module->max_got_offset, got_offset);
9733
9734 LLVMBuilderRef builder = LLVMCreateBuilder ();
9735 LLVMPositionBuilderBefore (builder, placeholder);
9736 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
9737 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
9738 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
9739
9740 name = get_aotconst_name (site->ji->type, site->ji->data.target, got_offset);
9741 load = LLVMBuildLoad (builder, got_entry_addr, "");
9742 load = LLVMBuildBitCast (builder, load, site->type, name ? name : "");
9743 LLVMReplaceAllUsesWith (placeholder, load);
9744 }
9745 g_free (site);
9746 }
9747
9748 mono_llvm_propagate_nonnull_final (specializable, module);
9749 g_hash_table_destroy (specializable);
9750
9751 for (int i = 0; i < module->cfgs->len; ++i) {
9752 /*
9753 * Nullify the patches pointing to direct calls. This is needed to
9754 * avoid allocating extra got slots, which is a perf problem and it
9755 * makes module->max_got_offset invalid.
9756 * It would be better to just store the patch_info in CallSite, but
9757 * cfg->patch_info is copied in aot-compiler.c.
9758 */
9759 MonoCompile *cfg = (MonoCompile *)g_ptr_array_index (module->cfgs, i);
9760 for (MonoJumpInfo *patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
9761 if (patch_info->type == MONO_PATCH_INFO_METHOD) {
9762 if (g_hash_table_lookup (patches_to_null, patch_info)) {
9763 patch_info->type = MONO_PATCH_INFO_NONE;
9764 /* Nullify the call to init_method () if possible */
9765 g_assert (cfg->got_access_count);
9766 cfg->got_access_count --;
9767 if (cfg->got_access_count == 0) {
9768 LLVMValueRef br = (LLVMValueRef)cfg->llvmonly_init_cond;
9769 if (br)
9770 LLVMSetSuccessor (br, 0, LLVMGetSuccessor (br, 1));
9771 }
9772 }
9773 }
9774 }
9775 }
9776
9777 g_hash_table_destroy (patches_to_null);
9778 }
9779
9780 static LLVMValueRef
9781 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
9782 {
9783 int i;
9784 LLVMValueRef res, *vals;
9785
9786 vals = g_new0 (LLVMValueRef, nvalues);
9787 for (i = 0; i < nvalues; ++i)
9788 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
9789 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
9790 g_free (vals);
9791 return res;
9792 }
9793
9794 static LLVMValueRef
9795 llvm_array_from_bytes (guint8 *values, int nvalues)
9796 {
9797 int i;
9798 LLVMValueRef res, *vals;
9799
9800 vals = g_new0 (LLVMValueRef, nvalues);
9801 for (i = 0; i < nvalues; ++i)
9802 vals [i] = LLVMConstInt (LLVMInt8Type (), values [i], FALSE);
9803 res = LLVMConstArray (LLVMInt8Type (), vals, nvalues);
9804 g_free (vals);
9805 return res;
9806 }
9807 /*
9808 * mono_llvm_emit_aot_file_info:
9809 *
9810 * Emit the MonoAotFileInfo structure.
9811 * Same as emit_aot_file_info () in aot-compiler.c.
9812 */
9813 void
9814 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
9815 {
9816 MonoLLVMModule *module = &aot_module;
9817
9818 /* Save these for later */
9819 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
9820 module->has_jitted_code = has_jitted_code;
9821 }
9822
9823 /*
9824 * mono_llvm_emit_aot_data:
9825 *
9826 * Emit the binary data DATA pointed to by symbol SYMBOL.
9827 */
9828 void
9829 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
9830 {
9831 MonoLLVMModule *module = &aot_module;
9832 LLVMTypeRef type;
9833 LLVMValueRef d;
9834
9835 type = LLVMArrayType (LLVMInt8Type (), data_len);
9836 d = LLVMAddGlobal (module->lmodule, type, symbol);
9837 LLVMSetVisibility (d, LLVMHiddenVisibility);
9838 LLVMSetLinkage (d, LLVMInternalLinkage);
9839 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
9840 LLVMSetAlignment (d, 8);
9841 mono_llvm_set_is_constant (d);
9842 }
9843
9844 /* Add a reference to a global defined in JITted code */
9845 static LLVMValueRef
9846 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
9847 {
9848 char *s;
9849 LLVMValueRef v;
9850
9851 s = g_strdup_printf ("%s%s", module->global_prefix, name);
9852 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
9853 LLVMSetVisibility (v, LLVMHiddenVisibility);
9854 g_free (s);
9855 return v;
9856 }
9857
9858 static void
9859 emit_aot_file_info (MonoLLVMModule *module)
9860 {
9861 LLVMTypeRef file_info_type;
9862 LLVMTypeRef *eltypes, eltype;
9863 LLVMValueRef info_var;
9864 LLVMValueRef *fields;
9865 int i, nfields, tindex;
9866 MonoAotFileInfo *info;
9867 LLVMModuleRef lmodule = module->lmodule;
9868
9869 info = &module->aot_info;
9870
9871 /* Create an LLVM type to represent MonoAotFileInfo */
9872 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 22 + 5;
9873 eltypes = g_new (LLVMTypeRef, nfields);
9874 tindex = 0;
9875 eltypes [tindex ++] = LLVMInt32Type ();
9876 eltypes [tindex ++] = LLVMInt32Type ();
9877 /* Symbols */
9878 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
9879 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
9880 /* Scalars */
9881 for (i = 0; i < 21; ++i)
9882 eltypes [tindex ++] = LLVMInt32Type ();
9883 /* Arrays */
9884 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
9885 for (i = 0; i < 4; ++i)
9886 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
9887 eltypes [tindex ++] = LLVMArrayType (LLVMInt8Type (), 16);
9888 g_assert (tindex == nfields);
9889 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
9890 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
9891
9892 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
9893 if (module->static_link) {
9894 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
9895 LLVMSetLinkage (info_var, LLVMInternalLinkage);
9896 }
9897
9898 #ifdef TARGET_WIN32
9899 if (!module->static_link) {
9900 LLVMSetDLLStorageClass (info_var, LLVMDLLExportStorageClass);
9901 }
9902 #endif
9903
9904 fields = g_new (LLVMValueRef, nfields);
9905 tindex = 0;
9906 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
9907 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
9908
9909 /* Symbols */
9910 /*
9911 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
9912 * for symbols defined in the .s file emitted by the aot compiler.
9913 */
9914 eltype = eltypes [tindex];
9915 if (module->llvm_only)
9916 fields [tindex ++] = LLVMConstNull (eltype);
9917 else
9918 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
9919 fields [tindex ++] = module->got_var;
9920 /* llc defines this directly */
9921 if (!module->llvm_only) {
9922 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
9923 fields [tindex ++] = LLVMConstNull (eltype);
9924 fields [tindex ++] = LLVMConstNull (eltype);
9925 } else {
9926 fields [tindex ++] = LLVMConstNull (eltype);
9927 fields [tindex ++] = module->get_method;
9928 fields [tindex ++] = module->get_unbox_tramp ? module->get_unbox_tramp : LLVMConstNull (eltype);
9929 }
9930 if (module->has_jitted_code) {
9931 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
9932 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
9933 } else {
9934 fields [tindex ++] = LLVMConstNull (eltype);
9935 fields [tindex ++] = LLVMConstNull (eltype);
9936 }
9937 if (!module->llvm_only)
9938 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
9939 else
9940 fields [tindex ++] = LLVMConstNull (eltype);
9941 if (module->llvm_only && module->unbox_tramp_indexes) {
9942 fields [tindex ++] = module->unbox_tramp_indexes;
9943 fields [tindex ++] = module->unbox_trampolines;
9944 } else {
9945 fields [tindex ++] = LLVMConstNull (eltype);
9946 fields [tindex ++] = LLVMConstNull (eltype);
9947 }
9948 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
9949 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
9950 fields [tindex ++] = LLVMConstNull (eltype);
9951 } else {
9952 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
9953 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
9954 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
9955 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
9956 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
9957 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
9958 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
9959 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
9960 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
9961 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
9962 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "weak_field_indexes");
9963 }
9964 /* Not needed (mem_end) */
9965 fields [tindex ++] = LLVMConstNull (eltype);
9966 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
9967 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
9968 if (info->trampoline_size [0]) {
9969 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
9970 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
9971 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_trampolines");
9972 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
9973 fields [tindex ++] = AddJitGlobal (module, eltype, "ftnptr_arg_trampolines");
9974 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_arbitrary_trampolines");
9975 } else {
9976 fields [tindex ++] = LLVMConstNull (eltype);
9977 fields [tindex ++] = LLVMConstNull (eltype);
9978 fields [tindex ++] = LLVMConstNull (eltype);
9979 fields [tindex ++] = LLVMConstNull (eltype);
9980 fields [tindex ++] = LLVMConstNull (eltype);
9981 fields [tindex ++] = LLVMConstNull (eltype);
9982 }
9983 if (module->static_link && !module->llvm_only)
9984 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
9985 else
9986 fields [tindex ++] = LLVMConstNull (eltype);
9987 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
9988 if (!module->llvm_only) {
9989 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
9990 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
9991 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
9992 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
9993 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
9994 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
9995 } else {
9996 fields [tindex ++] = LLVMConstNull (eltype);
9997 fields [tindex ++] = LLVMConstNull (eltype);
9998 fields [tindex ++] = LLVMConstNull (eltype);
9999 fields [tindex ++] = LLVMConstNull (eltype);
10000 fields [tindex ++] = LLVMConstNull (eltype);
10001 fields [tindex ++] = LLVMConstNull (eltype);
10002 }
10003
10004 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i) {
10005 g_assert (fields [2 + i]);
10006 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
10007 }
10008
10009 /* Scalars */
10010 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
10011 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
10012 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
10013 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
10014 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nextra_methods, FALSE);
10015 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
10016 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
10017 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
10018 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
10019 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
10020 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
10021 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
10022 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
10023 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_shift_bits, FALSE);
10024 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_mask, FALSE);
10025 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
10026 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->call_table_entry_size, FALSE);
10027 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
10028 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
10029 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_num, FALSE);
10030 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_elemsize, FALSE);
10031 /* Arrays */
10032 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
10033 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
10034 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
10035 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
10036 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
10037
10038 fields [tindex ++] = llvm_array_from_bytes (info->aotid, 16);
10039 g_assert (tindex == nfields);
10040
10041 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
10042
10043 if (module->static_link) {
10044 char *s, *p;
10045 LLVMValueRef var;
10046
10047 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
10048 /* Get rid of characters which cannot occur in symbols */
10049 p = s;
10050 for (p = s; *p; ++p) {
10051 if (!(isalnum (*p) || *p == '_'))
10052 *p = '_';
10053 }
10054 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
10055 g_free (s);
10056 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
10057 LLVMSetLinkage (var, LLVMExternalLinkage);
10058 }
10059 }
10060
10061 typedef struct {
10062 LLVMValueRef lmethod;
10063 int argument;
10064 } NonnullPropWorkItem;
10065
10066 static void
10067 mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params)
10068 {
10069 if (!ctx->module->llvm_disable_self_init && mono_aot_can_specialize (call_method)) {
10070 int num_passed = LLVMGetNumArgOperands (lcall);
10071 g_assert (num_params <= num_passed);
10072
10073 g_assert (ctx->module->method_to_call_info);
10074 GArray *call_site_union = (GArray *) g_hash_table_lookup (ctx->module->method_to_call_info, call_method);
10075
10076 if (!call_site_union) {
10077 call_site_union = g_array_sized_new (FALSE, TRUE, sizeof (gint32), num_params);
10078 int zero = 0;
10079 for (int i = 0; i < num_params; i++)
10080 g_array_insert_val (call_site_union, i, zero);
10081 }
10082
10083 for (int i = 0; i < num_params; i++) {
10084 if (mono_llvm_is_nonnull (args [i])) {
10085 g_assert (i < LLVMGetNumArgOperands (lcall));
10086 mono_llvm_set_call_nonnull_arg (lcall, i);
10087 } else {
10088 gint32 *nullable_count = &g_array_index (call_site_union, gint32, i);
10089 *nullable_count = *nullable_count + 1;
10090 }
10091 }
10092
10093 g_hash_table_insert (ctx->module->method_to_call_info, call_method, call_site_union);
10094 }
10095 }
10096
10097 static void
10098 mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module)
10099 {
10100 // When we first traverse the mini IL, we mark the things that are
10101 // nonnull (the roots). Then, for all of the methods that can be specialized, we
10102 // see if their call sites have nonnull attributes.
10103
10104 // If so, we mark the function's param. This param has uses to propagate
10105 // the attribute to. This propagation can trigger a need to mark more attributes
10106 // non-null, and so on and so forth.
10107 GSList *queue = NULL;
10108
10109 GHashTableIter iter;
10110 LLVMValueRef lmethod;
10111 MonoMethod *method;
10112 g_hash_table_iter_init (&iter, all_specializable);
10113 while (g_hash_table_iter_next (&iter, (void**)&lmethod, (void**)&method)) {
10114 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, method);
10115
10116 // Basic sanity checking
10117 if (call_site_union)
10118 g_assert (call_site_union->len == LLVMCountParams (lmethod));
10119
10120 // Add root to work queue
10121 for (int i = 0; call_site_union && i < call_site_union->len; i++) {
10122 if (g_array_index (call_site_union, gint32, i) == 0) {
10123 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
10124 item->lmethod = lmethod;
10125 item->argument = i;
10126 queue = g_slist_prepend (queue, item);
10127 }
10128 }
10129 }
10130
10131 // This is essentially reference counting, and we are propagating
10132 // the refcount decrement here. We have less work to do than we may otherwise
10133 // because we are only working with a set of subgraphs of specializable functions.
10134 //
10135 // We rely on being able to see all of the references in the graph.
10136 // This is ensured by the function mono_aot_can_specialize. Everything in
10137 // all_specializable is a function that can be specialized, and is the resulting
10138 // node in the graph after all of the subsitutions are done.
10139 //
10140 // Anything disrupting the direct calls made with self-init will break this optimization.
10141
10142 while (queue) {
10143 // Update the queue state.
10144 // Our only other per-iteration responsibility is now to free current
10145 NonnullPropWorkItem *current = (NonnullPropWorkItem *) queue->data;
10146 queue = queue->next;
10147 g_assert (current->argument < LLVMCountParams (current->lmethod));
10148
10149 // Does the actual leaf-node work here
10150 // Mark the function argument as nonnull for LLVM
10151 mono_llvm_set_func_nonnull_arg (current->lmethod, current->argument);
10152
10153 // The rest of this is for propagating forward nullability changes
10154 // to calls that use the argument that is now nullable.
10155
10156 // Get the actual LLVM value of the argument, so we can see which call instructions
10157 // used that argument
10158 LLVMValueRef caller_argument = LLVMGetParam (current->lmethod, current->argument);
10159
10160 // Iterate over the calls using the newly-non-nullable argument
10161 GSList *calls = mono_llvm_calls_using (caller_argument);
10162 for (GSList *cursor = calls; cursor != NULL; cursor = cursor->next) {
10163
10164 LLVMValueRef lcall = (LLVMValueRef) cursor->data;
10165 LLVMValueRef callee_lmethod = LLVMGetCalledValue (lcall);
10166
10167 // If this wasn't a direct call for which mono_aot_can_specialize is true,
10168 // this lookup won't find a MonoMethod.
10169 MonoMethod *callee_method = (MonoMethod *) g_hash_table_lookup (all_specializable, callee_lmethod);
10170 if (!callee_method)
10171 continue;
10172
10173 // Decrement number of nullable refs at that func's arg offset
10174 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, callee_method);
10175
10176 // It has module-local callers and is specializable, should have seen this call site
10177 // and inited this
10178 g_assert (call_site_union);
10179
10180 // The function *definition* parameter arity should always be consistent
10181 int max_params = LLVMCountParams (callee_lmethod);
10182 if (call_site_union->len != max_params) {
10183 mono_llvm_dump_value (callee_lmethod);
10184 g_assert_not_reached ();
10185 }
10186
10187 // Get the values that correspond to the parameters passed to the call
10188 // that used our argument
10189 LLVMValueRef *operands = mono_llvm_call_args (lcall);
10190 for (int call_argument = 0; call_argument < max_params; call_argument++) {
10191 // Every time we used the newly-non-nullable argument, decrement the nullable
10192 // refcount for that function.
10193 if (caller_argument == operands [call_argument]) {
10194 gint32 *nullable_count = &g_array_index (call_site_union, gint32, call_argument);
10195 g_assert (*nullable_count > 0);
10196 *nullable_count = *nullable_count - 1;
10197
10198 // If we caused that callee's parameter to become newly nullable, add to work queue
10199 if (*nullable_count == 0) {
10200 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
10201 item->lmethod = callee_lmethod;
10202 item->argument = call_argument;
10203 queue = g_slist_prepend (queue, item);
10204 }
10205 }
10206 }
10207 g_free (operands);
10208
10209 // Update nullability refcount information for the callee now
10210 g_hash_table_insert (module->method_to_call_info, callee_method, call_site_union);
10211 }
10212 g_slist_free (calls);
10213
10214 g_free (current);
10215 }
10216 }
10217
10218 /*
10219 * Emit the aot module into the LLVM bitcode file FILENAME.
10220 */
10221 void
10222 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
10223 {
10224 LLVMTypeRef got_type, inited_type;
10225 LLVMValueRef real_got, real_inited;
10226 MonoLLVMModule *module = &aot_module;
10227
10228 emit_llvm_code_end (module);
10229
10230 /*
10231 * Create the real got variable and replace all uses of the dummy variable with
10232 * the real one.
10233 */
10234 int size = module->max_got_offset + 1;
10235 LLVMTypeRef *members = g_malloc0 (sizeof (LLVMValueRef) * size);
10236 for (int i = 0; i < size; i++) {
10237 LLVMTypeRef lookup_type = NULL;
10238
10239 lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (i));
10240
10241 if (!lookup_type)
10242 lookup_type = module->ptr_type;
10243
10244 members [i] = LLVMPointerType (lookup_type, 0);
10245 }
10246
10247 got_type = LLVMStructCreateNamed (module->context, g_strdup_printf ("MONO_GOT_%s", cu_name));
10248 LLVMStructSetBody (got_type, members, size, FALSE);
10249 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
10250
10251 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
10252 if (module->external_symbols) {
10253 LLVMSetLinkage (real_got, LLVMExternalLinkage);
10254 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
10255 } else {
10256 LLVMSetLinkage (real_got, LLVMInternalLinkage);
10257 }
10258 mono_llvm_replace_uses_of (module->got_var, real_got);
10259
10260 mark_as_used (&aot_module, real_got);
10261
10262 /* Delete the dummy got so it doesn't become a global */
10263 LLVMDeleteGlobal (module->got_var);
10264 module->got_var = real_got;
10265
10266 /*
10267 * Same for the init_var
10268 */
10269 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
10270 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
10271 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
10272 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
10273 mono_llvm_replace_uses_of (module->inited_var, real_inited);
10274 LLVMDeleteGlobal (module->inited_var);
10275
10276 if (module->llvm_only) {
10277 emit_get_method (&aot_module);
10278 emit_get_unbox_tramp (&aot_module);
10279 }
10280
10281 emit_llvm_used (&aot_module);
10282 emit_dbg_info (&aot_module, filename, cu_name);
10283 emit_aot_file_info (&aot_module);
10284
10285 /* Replace PLT entries for directly callable methods with the methods themselves */
10286 {
10287 GHashTableIter iter;
10288 MonoJumpInfo *ji;
10289 LLVMValueRef callee;
10290
10291 GHashTable *specializable = g_hash_table_new (NULL, NULL);
10292
10293 g_hash_table_iter_init (&iter, module->plt_entries_ji);
10294 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
10295 if (mono_aot_is_direct_callable (ji)) {
10296 LLVMValueRef lmethod;
10297
10298 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
10299 /* The types might not match because the caller might pass an rgctx */
10300 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
10301 mono_llvm_replace_uses_of (callee, lmethod);
10302
10303 if (!module->llvm_disable_self_init && mono_aot_can_specialize (ji->data.method))
10304 g_hash_table_insert (specializable, lmethod, ji->data.method);
10305 mono_aot_mark_unused_llvm_plt_entry (ji);
10306 }
10307 }
10308 }
10309
10310 mono_llvm_propagate_nonnull_final (specializable, module);
10311
10312 g_hash_table_destroy (specializable);
10313 }
10314
10315 /* Note: You can still dump an invalid bitcode file by running `llvm-dis`
10316 * in a debugger, set a breakpoint on `LLVMVerifyModule` and fake its
10317 * result to 0 (indicating success). */
10318 LLVMWriteBitcodeToFile (module->lmodule, filename);
10319
10320 #if 1
10321 {
10322 char *verifier_err;
10323
10324 if (LLVMVerifyModule (module->lmodule, LLVMReturnStatusAction, &verifier_err)) {
10325 printf ("%s\n", verifier_err);
10326 g_assert_not_reached ();
10327 }
10328 }
10329 #endif
10330 }
10331
10332
10333 static LLVMValueRef
10334 md_string (const char *s)
10335 {
10336 return LLVMMDString (s, strlen (s));
10337 }
10338
10339 /* Debugging support */
10340
10341 static void
10342 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
10343 {
10344 LLVMModuleRef lmodule = module->lmodule;
10345 LLVMValueRef args [16], ver;
10346
10347 /*
10348 * This can only be enabled when LLVM code is emitted into a separate object
10349 * file, since the AOT compiler also emits dwarf info,
10350 * and the abbrev indexes will not be correct since llvm has added its own
10351 * abbrevs.
10352 */
10353 if (!module->emit_dwarf)
10354 return;
10355
10356 mono_llvm_di_builder_finalize (module->di_builder);
10357
10358 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
10359 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
10360 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
10361 ver = LLVMMDNode (args, 3);
10362 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
10363
10364 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
10365 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
10366 args [2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE);
10367 ver = LLVMMDNode (args, 3);
10368 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
10369 }
10370
10371 static LLVMValueRef
10372 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
10373 {
10374 MonoLLVMModule *module = ctx->module;
10375 MonoDebugMethodInfo *minfo = ctx->minfo;
10376 char *source_file, *dir, *filename;
10377 MonoSymSeqPoint *sym_seq_points;
10378 int n_seq_points;
10379
10380 if (!minfo)
10381 return NULL;
10382
10383 mono_debug_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
10384 if (!source_file)
10385 source_file = g_strdup ("<unknown>");
10386 dir = g_path_get_dirname (source_file);
10387 filename = g_path_get_basename (source_file);
10388 g_free (source_file);
10389
10390 return (LLVMValueRef)mono_llvm_di_create_function (module->di_builder, module->cu, method, cfg->method->name, name, dir, filename, n_seq_points ? sym_seq_points [0].line : 1);
10391 }
10392
10393 static void
10394 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
10395 {
10396 MonoCompile *cfg = ctx->cfg;
10397
10398 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
10399 MonoDebugSourceLocation *loc;
10400 LLVMValueRef loc_md;
10401
10402 loc = mono_debug_method_lookup_location (ctx->minfo, cil_code - cfg->header->code);
10403
10404 if (loc) {
10405 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, loc->row, loc->column);
10406 mono_llvm_di_set_location (builder, loc_md);
10407 mono_debug_free_source_location (loc);
10408 }
10409 }
10410 }
10411
10412 static void
10413 emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder)
10414 {
10415 if (ctx->minfo) {
10416 LLVMValueRef loc_md;
10417 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, 0, 0);
10418 mono_llvm_di_set_location (builder, loc_md);
10419 }
10420 }
10421
10422 /*
10423 DESIGN:
10424 - Emit LLVM IR from the mono IR using the LLVM C API.
10425 - The original arch specific code remains, so we can fall back to it if we run
10426 into something we can't handle.
10427 */
10428
10429 /*
10430 A partial list of issues:
10431 - Handling of opcodes which can throw exceptions.
10432
10433 In the mono JIT, these are implemented using code like this:
10434 method:
10435 <compare>
10436 throw_pos:
10437 b<cond> ex_label
10438 <rest of code>
10439 ex_label:
10440 push throw_pos - method
10441 call <exception trampoline>
10442
10443 The problematic part is push throw_pos - method, which cannot be represented
10444 in the LLVM IR, since it does not support label values.
10445 -> this can be implemented in AOT mode using inline asm + labels, but cannot
10446 be implemented in JIT mode ?
10447 -> a possible but slower implementation would use the normal exception
10448 throwing code but it would need to control the placement of the throw code
10449 (it needs to be exactly after the compare+branch).
10450 -> perhaps add a PC offset intrinsics ?
10451
10452 - efficient implementation of .ovf opcodes.
10453
10454 These are currently implemented as:
10455 <ins which sets the condition codes>
10456 b<cond> ex_label
10457
10458 Some overflow opcodes are now supported by LLVM SVN.
10459
10460 - exception handling, unwinding.
10461 - SSA is disabled for methods with exception handlers
10462 - How to obtain unwind info for LLVM compiled methods ?
10463 -> this is now solved by converting the unwind info generated by LLVM
10464 into our format.
10465 - LLVM uses the c++ exception handling framework, while we use our home grown
10466 code, and couldn't use the c++ one:
10467 - its not supported under VC++, other exotic platforms.
10468 - it might be impossible to support filter clauses with it.
10469
10470 - trampolines.
10471
10472 The trampolines need a predictable call sequence, since they need to disasm
10473 the calling code to obtain register numbers / offsets.
10474
10475 LLVM currently generates this code in non-JIT mode:
10476 mov -0x98(%rax),%eax
10477 callq *%rax
10478 Here, the vtable pointer is lost.
10479 -> solution: use one vtable trampoline per class.
10480
10481 - passing/receiving the IMT pointer/RGCTX.
10482 -> solution: pass them as normal arguments ?
10483
10484 - argument passing.
10485
10486 LLVM does not allow the specification of argument registers etc. This means
10487 that all calls are made according to the platform ABI.
10488
10489 - passing/receiving vtypes.
10490
10491 Vtypes passed/received in registers are handled by the front end by using
10492 a signature with scalar arguments, and loading the parts of the vtype into those
10493 arguments.
10494
10495 Vtypes passed on the stack are handled using the 'byval' attribute.
10496
10497 - ldaddr.
10498
10499 Supported though alloca, we need to emit the load/store code.
10500
10501 - types.
10502
10503 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
10504 typed registers, so we have to keep track of the precise LLVM type of each vreg.
10505 This is made easier because the IR is already in SSA form.
10506 An additional problem is that our IR is not consistent with types, i.e. i32/i64
10507 types are frequently used incorrectly.
10508 */
10509
10510 /*
10511 AOT SUPPORT:
10512 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
10513 it with the file containing the methods emitted by the JIT and the AOT data
10514 structures.
10515 */
10516
10517 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
10518 * - each bblock should end with a branch
10519 * - setting the return value, making cfg->ret non-volatile
10520 * - avoid some transformations in the JIT which make it harder for us to generate
10521 * code.
10522 * - use pointer types to help optimizations.
10523 */
10524
10525 #else /* DISABLE_JIT */
10526
10527 void
10528 mono_llvm_cleanup (void)
10529 {
10530 }
10531
10532 void
10533 mono_llvm_free_domain_info (MonoDomain *domain)
10534 {
10535 }
10536
10537 void
10538 mono_llvm_init (void)
10539 {
10540 }
10541
10542 #endif /* DISABLE_JIT */
10543
10544 #if !defined(DISABLE_JIT) && !defined(MONO_CROSS_COMPILE)
10545
10546 /* LLVM JIT support */
10547
10548 static unsigned char*
10549 alloc_cb (LLVMValueRef function, int size)
10550 {
10551 MonoCompile *cfg;
10552
10553 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10554
10555 if (cfg) {
10556 // FIXME: dynamic
10557 return (unsigned char*)mono_domain_code_reserve (cfg->domain, size);
10558 } else {
10559 return (unsigned char*)mono_domain_code_reserve (mono_domain_get (), size);
10560 }
10561 }
10562
10563 static void
10564 emitted_cb (LLVMValueRef function, void *start, void *end)
10565 {
10566 MonoCompile *cfg;
10567
10568 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10569 g_assert (cfg);
10570 cfg->code_len = (guint8*)end - (guint8*)start;
10571 }
10572
10573 static void
10574 exception_cb (void *data)
10575 {
10576 MonoCompile *cfg;
10577 MonoJitExceptionInfo *ei;
10578 guint32 ei_len, i, j, nested_len, nindex;
10579 gpointer *type_info;
10580 int this_reg, this_offset;
10581
10582 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10583 g_assert (cfg);
10584
10585 /*
10586 * data points to a DWARF FDE structure, convert it to our unwind format and
10587 * save it.
10588 * An alternative would be to save it directly, and modify our unwinder to work
10589 * with it.
10590 */
10591 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
10592 if (cfg->verbose_level > 1)
10593 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
10594
10595 /* Count nested clauses */
10596 nested_len = 0;
10597 for (i = 0; i < ei_len; ++i) {
10598 gint32 cindex1 = *(gint32*)type_info [i];
10599 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
10600
10601 for (j = 0; j < cfg->header->num_clauses; ++j) {
10602 int cindex2 = j;
10603 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
10604
10605 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
10606 nested_len ++;
10607 }
10608 }
10609 }
10610
10611 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
10612 cfg->llvm_ex_info_len = ei_len + nested_len;
10613 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
10614 /* Fill the rest of the information from the type info */
10615 for (i = 0; i < ei_len; ++i) {
10616 gint32 clause_index = *(gint32*)type_info [i];
10617 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
10618
10619 cfg->llvm_ex_info [i].flags = clause->flags;
10620 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
10621 cfg->llvm_ex_info [i].clause_index = clause_index;
10622 }
10623
10624 /*
10625 * For nested clauses, the LLVM produced exception info associates the try interval with
10626 * the innermost handler, while mono expects it to be associated with all nesting clauses.
10627 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
10628 * and everything else from the nested clause.
10629 */
10630 nindex = ei_len;
10631 for (i = 0; i < ei_len; ++i) {
10632 gint32 cindex1 = *(gint32*)type_info [i];
10633 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
10634
10635 for (j = 0; j < cfg->header->num_clauses; ++j) {
10636 int cindex2 = j;
10637 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
10638 MonoJitExceptionInfo *nesting_ei, *nested_ei;
10639
10640 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
10641 /* clause1 is the nested clause */
10642 nested_ei = &cfg->llvm_ex_info [i];
10643 nesting_ei = &cfg->llvm_ex_info [nindex];
10644 nindex ++;
10645
10646 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
10647
10648 nesting_ei->flags = clause2->flags;
10649 nesting_ei->data.catch_class = clause2->data.catch_class;
10650 nesting_ei->clause_index = cindex2;
10651 }
10652 }
10653 }
10654 g_assert (nindex == ei_len + nested_len);
10655 cfg->llvm_this_reg = this_reg;
10656 cfg->llvm_this_offset = this_offset;
10657
10658 /* type_info [i] is cfg mempool allocated, no need to free it */
10659
10660 g_free (ei);
10661 g_free (type_info);
10662 }
10663
10664 /*
10665 * decode_llvm_eh_info:
10666 *
10667 * Decode the EH table emitted by llvm in jit mode, and store
10668 * the result into cfg.
10669 */
10670 static void
10671 decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame)
10672 {
10673 MonoCompile *cfg = ctx->cfg;
10674 guint8 *cie, *fde;
10675 int fde_len;
10676 MonoLLVMFDEInfo info;
10677 MonoJitExceptionInfo *ei;
10678 guint8 *p = (guint8*)eh_frame;
10679 int version, fde_count, fde_offset;
10680 guint32 ei_len, i, nested_len;
10681 gpointer *type_info;
10682 gint32 *table;
10683 guint8 *unw_info;
10684
10685 /*
10686 * Decode the one element EH table emitted by the MonoException class
10687 * in llvm.
10688 */
10689
10690 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
10691
10692 version = *p;
10693 g_assert (version == 3);
10694 p ++;
10695 p ++;
10696 p = (guint8 *)ALIGN_PTR_TO (p, 4);
10697
10698 fde_count = *(guint32*)p;
10699 p += 4;
10700 table = (gint32*)p;
10701
10702 g_assert (fde_count <= 2);
10703
10704 /* The first entry is the real method */
10705 g_assert (table [0] == 1);
10706 fde_offset = table [1];
10707 table += fde_count * 2;
10708 /* Extra entry */
10709 cfg->code_len = table [0];
10710 fde_len = table [1] - fde_offset;
10711 table += 2;
10712
10713 fde = (guint8*)eh_frame + fde_offset;
10714 cie = (guint8*)table;
10715
10716 /* Compute lengths */
10717 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, NULL, NULL, NULL);
10718
10719 ei = (MonoJitExceptionInfo *)g_malloc0 (info.ex_info_len * sizeof (MonoJitExceptionInfo));
10720 type_info = (gpointer *)g_malloc0 (info.ex_info_len * sizeof (gpointer));
10721 unw_info = (guint8*)g_malloc0 (info.unw_info_len);
10722
10723 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, ei, type_info, unw_info);
10724
10725 cfg->encoded_unwind_ops = unw_info;
10726 cfg->encoded_unwind_ops_len = info.unw_info_len;
10727 if (cfg->verbose_level > 1)
10728 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
10729 if (info.this_reg != -1) {
10730 cfg->llvm_this_reg = info.this_reg;
10731 cfg->llvm_this_offset = info.this_offset;
10732 }
10733
10734 ei_len = info.ex_info_len;
10735
10736 // Nested clauses are currently disabled
10737 nested_len = 0;
10738
10739 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
10740 cfg->llvm_ex_info_len = ei_len + nested_len;
10741 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
10742 /* Fill the rest of the information from the type info */
10743 for (i = 0; i < ei_len; ++i) {
10744 gint32 clause_index = *(gint32*)type_info [i];
10745 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
10746
10747 cfg->llvm_ex_info [i].flags = clause->flags;
10748 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
10749 cfg->llvm_ex_info [i].clause_index = clause_index;
10750 }
10751 }
10752
10753 static void
10754 init_jit_module (MonoDomain *domain)
10755 {
10756 MonoJitDomainInfo *dinfo;
10757 MonoLLVMModule *module;
10758 char *name;
10759
10760 dinfo = domain_jit_info (domain);
10761 if (dinfo->llvm_module)
10762 return;
10763
10764 mono_loader_lock ();
10765
10766 if (dinfo->llvm_module) {
10767 mono_loader_unlock ();
10768 return;
10769 }
10770
10771 module = g_new0 (MonoLLVMModule, 1);
10772
10773 name = g_strdup_printf ("mono-%s", domain->friendly_name);
10774 module->context = LLVMGetGlobalContext ();
10775 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
10776
10777 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (alloc_cb, emitted_cb, exception_cb, &module->ee);
10778
10779 add_intrinsics (module->lmodule);
10780 add_types (module);
10781
10782 module->llvm_types = g_hash_table_new (NULL, NULL);
10783
10784 mono_memory_barrier ();
10785
10786 dinfo->llvm_module = module;
10787
10788 mono_loader_unlock ();
10789 }
10790
10791 static void
10792 llvm_jit_finalize_method (EmitContext *ctx)
10793 {
10794 MonoCompile *cfg = ctx->cfg;
10795 MonoDomain *domain = mono_domain_get ();
10796 MonoJitDomainInfo *domain_info;
10797 int nvars = g_hash_table_size (ctx->jit_callees);
10798 LLVMValueRef *callee_vars = g_new0 (LLVMValueRef, nvars);
10799 gpointer *callee_addrs = g_new0 (gpointer, nvars);
10800 GHashTableIter iter;
10801 LLVMValueRef var;
10802 MonoMethod *callee;
10803 gpointer eh_frame;
10804 int i;
10805
10806 /*
10807 * Compute the addresses of the LLVM globals pointing to the
10808 * methods called by the current method. Pass it to the trampoline
10809 * code so it can update them after their corresponding method was
10810 * compiled.
10811 */
10812 g_hash_table_iter_init (&iter, ctx->jit_callees);
10813 i = 0;
10814 while (g_hash_table_iter_next (&iter, NULL, (void**)&var))
10815 callee_vars [i ++] = var;
10816
10817 cfg->native_code = (guint8*)mono_llvm_compile_method (ctx->module->mono_ee, ctx->lmethod, nvars, callee_vars, callee_addrs, &eh_frame);
10818 mono_llvm_remove_gc_safepoint_poll (ctx->lmodule);
10819 if (cfg->verbose_level > 1) {
10820 g_print ("\n*** Optimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, TRUE));
10821 if (cfg->compile_aot) {
10822 mono_llvm_dump_value (ctx->lmethod);
10823 } else {
10824 mono_llvm_dump_module (ctx->lmodule);
10825 }
10826 g_print ("***\n\n");
10827 }
10828
10829 decode_llvm_eh_info (ctx, eh_frame);
10830
10831 mono_domain_lock (domain);
10832 domain_info = domain_jit_info (domain);
10833 if (!domain_info->llvm_jit_callees)
10834 domain_info->llvm_jit_callees = g_hash_table_new (NULL, NULL);
10835 g_hash_table_iter_init (&iter, ctx->jit_callees);
10836 i = 0;
10837 while (g_hash_table_iter_next (&iter, (void**)&callee, (void**)&var)) {
10838 GSList *addrs = (GSList*)g_hash_table_lookup (domain_info->llvm_jit_callees, callee);
10839 addrs = g_slist_prepend (addrs, callee_addrs [i]);
10840 g_hash_table_insert (domain_info->llvm_jit_callees, callee, addrs);
10841 i ++;
10842 }
10843 mono_domain_unlock (domain);
10844 }
10845
10846 #else
10847
10848 static void
10849 init_jit_module (MonoDomain *domain)
10850 {
10851 g_assert_not_reached ();
10852 }
10853
10854 static void
10855 llvm_jit_finalize_method (EmitContext *ctx)
10856 {
10857 g_assert_not_reached ();
10858 }
10859
10860 #endif
10861
10862 static MonoCPUFeatures cpu_features;
10863
10864 MonoCPUFeatures mono_llvm_get_cpu_features (void)
10865 {
10866 static const CpuFeatureAliasFlag flags_map [] = {
10867 #if defined(TARGET_X86) || defined(TARGET_AMD64)
10868 { "sse", MONO_CPU_X86_SSE },
10869 { "sse2", MONO_CPU_X86_SSE2 },
10870 { "pclmul", MONO_CPU_X86_PCLMUL },
10871 { "aes", MONO_CPU_X86_AES },
10872 { "sse2", MONO_CPU_X86_SSE2 },
10873 { "sse3", MONO_CPU_X86_SSE3 },
10874 { "ssse3", MONO_CPU_X86_SSSE3 },
10875 { "sse4.1", MONO_CPU_X86_SSE41 },
10876 { "sse4.2", MONO_CPU_X86_SSE42 },
10877 { "popcnt", MONO_CPU_X86_POPCNT },
10878 { "avx", MONO_CPU_X86_AVX },
10879 { "avx2", MONO_CPU_X86_AVX2 },
10880 { "fma", MONO_CPU_X86_FMA },
10881 { "lzcnt", MONO_CPU_X86_LZCNT },
10882 { "bmi", MONO_CPU_X86_BMI1 },
10883 { "bmi2", MONO_CPU_X86_BMI2 },
10884 #endif
10885 };
10886 if (!cpu_features)
10887 cpu_features = MONO_CPU_INITED | (MonoCPUFeatures)mono_llvm_check_cpu_features (flags_map, G_N_ELEMENTS (flags_map));
10888 return cpu_features;
10889 }
Mirror of the official Mono project git repository