?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[wasm] Add normal GC support. (#15577)
[mono-project.git] / mono / mini / mini-llvm.c
blobf91a2462d42a2940d0b526ffbaeaa93b8695cff2
1 /**
2 * \file
3 * llvm "Backend" for the mono JIT
4 *
5 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
6 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
8 */
9
10 #include "config.h"
11
12 #include <mono/metadata/debug-helpers.h>
13 #include <mono/metadata/debug-internals.h>
14 #include <mono/metadata/mempool-internals.h>
15 #include <mono/metadata/environment.h>
16 #include <mono/metadata/object-internals.h>
17 #include <mono/metadata/abi-details.h>
18 #include <mono/utils/mono-tls.h>
19 #include <mono/utils/mono-dl.h>
20 #include <mono/utils/mono-time.h>
21 #include <mono/utils/freebsd-dwarf.h>
22
23 #ifndef __STDC_LIMIT_MACROS
24 #define __STDC_LIMIT_MACROS
25 #endif
26 #ifndef __STDC_CONSTANT_MACROS
27 #define __STDC_CONSTANT_MACROS
28 #endif
29
30 #include "llvm-c/BitWriter.h"
31 #include "llvm-c/Analysis.h"
32
33 #include "mini-llvm-cpp.h"
34 #include "llvm-jit.h"
35 #include "aot-compiler.h"
36 #include "mini-llvm.h"
37 #include "mini-runtime.h"
38 #include <mono/utils/mono-math.h>
39
40 #ifndef DISABLE_JIT
41
42 #if defined(TARGET_AMD64) && defined(TARGET_WIN32) && defined(HOST_WIN32) && defined(_MSC_VER)
43 #define TARGET_X86_64_WIN32_MSVC
44 #endif
45
46 #if defined(TARGET_X86_64_WIN32_MSVC)
47 #define TARGET_WIN32_MSVC
48 #endif
49
50 #if LLVM_API_VERSION < 610
51 #error "The version of the mono llvm repository is too old."
52 #endif
53
54 /*
55 * Information associated by mono with LLVM modules.
56 */
57 typedef struct {
58 LLVMModuleRef lmodule;
59 LLVMValueRef throw_icall, rethrow, throw_corlib_exception;
60 GHashTable *llvm_types;
61 LLVMValueRef got_var;
62 const char *got_symbol;
63 const char *get_method_symbol;
64 const char *get_unbox_tramp_symbol;
65 GHashTable *plt_entries;
66 GHashTable *plt_entries_ji;
67 GHashTable *method_to_lmethod;
68 GHashTable *method_to_call_info;
69 GHashTable *lvalue_to_lcalls;
70 GHashTable *direct_callables;
71 char **bb_names;
72 int bb_names_len;
73 GPtrArray *used;
74 LLVMTypeRef ptr_type;
75 GPtrArray *subprogram_mds;
76 MonoEERef *mono_ee;
77 LLVMExecutionEngineRef ee;
78 gboolean external_symbols;
79 gboolean emit_dwarf;
80 int max_got_offset;
81 LLVMValueRef personality;
82 LLVMValueRef *intrins_by_id;
83
84 /* For AOT */
85 MonoAssembly *assembly;
86 char *global_prefix;
87 MonoAotFileInfo aot_info;
88 const char *jit_got_symbol;
89 const char *eh_frame_symbol;
90 LLVMValueRef get_method, get_unbox_tramp;
91 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
92 LLVMValueRef code_start, code_end;
93 LLVMValueRef inited_var;
94 LLVMValueRef unbox_tramp_indexes;
95 LLVMValueRef unbox_trampolines;
96 int max_inited_idx, max_method_idx;
97 gboolean has_jitted_code;
98 gboolean static_link;
99 gboolean llvm_only;
100 gboolean llvm_disable_self_init;
101 gboolean interp;
102 GHashTable *idx_to_lmethod;
103 GHashTable *idx_to_unbox_tramp;
104 GPtrArray *callsite_list;
105 LLVMContextRef context;
106 LLVMValueRef sentinel_exception;
107 void *di_builder, *cu;
108 GHashTable *objc_selector_to_var;
109 GPtrArray *cfgs;
110 int unbox_tramp_num, unbox_tramp_elemsize;
111 GHashTable *got_idx_to_type;
112 } MonoLLVMModule;
113
114 /*
115 * Information associated by the backend with mono basic blocks.
116 */
117 typedef struct {
118 LLVMBasicBlockRef bblock, end_bblock;
119 LLVMValueRef finally_ind;
120 gboolean added, invoke_target;
121 /*
122 * If this bblock is the start of a finally clause, this is a list of bblocks it
123 * needs to branch to in ENDFINALLY.
124 */
125 GSList *call_handler_return_bbs;
126 /*
127 * If this bblock is the start of a finally clause, this is the bblock that
128 * CALL_HANDLER needs to branch to.
129 */
130 LLVMBasicBlockRef call_handler_target_bb;
131 /* The list of switch statements generated by ENDFINALLY instructions */
132 GSList *endfinally_switch_ins_list;
133 GSList *phi_nodes;
134 } BBInfo;
135
136 /*
137 * Structure containing emit state
138 */
139 typedef struct {
140 MonoMemPool *mempool;
141
142 /* Maps method names to the corresponding LLVMValueRef */
143 GHashTable *emitted_method_decls;
144
145 MonoCompile *cfg;
146 LLVMValueRef lmethod;
147 MonoLLVMModule *module;
148 LLVMModuleRef lmodule;
149 BBInfo *bblocks;
150 int sindex, default_index, ex_index;
151 LLVMBuilderRef builder;
152 LLVMValueRef *values, *addresses;
153 MonoType **vreg_cli_types;
154 LLVMCallInfo *linfo;
155 MonoMethodSignature *sig;
156 GSList *builders;
157 GHashTable *region_to_handler;
158 GHashTable *clause_to_handler;
159 LLVMBuilderRef alloca_builder;
160 LLVMValueRef last_alloca;
161 LLVMValueRef rgctx_arg;
162 LLVMValueRef this_arg;
163 LLVMTypeRef *vreg_types;
164 gboolean *is_vphi;
165 LLVMTypeRef method_type;
166 LLVMBasicBlockRef init_bb, inited_bb;
167 gboolean *is_dead;
168 gboolean *unreachable;
169 gboolean llvm_only;
170 gboolean has_got_access;
171 gboolean emit_dummy_arg;
172 int this_arg_pindex, rgctx_arg_pindex;
173 LLVMValueRef imt_rgctx_loc;
174 GHashTable *llvm_types;
175 LLVMValueRef dbg_md;
176 MonoDebugMethodInfo *minfo;
177 /* For every clause, the clauses it is nested in */
178 GSList **nested_in;
179 LLVMValueRef ex_var;
180 GHashTable *exc_meta;
181 GPtrArray *callsite_list;
182 GPtrArray *phi_values;
183 GPtrArray *bblock_list;
184 char *method_name;
185 GHashTable *jit_callees;
186 LLVMValueRef long_bb_break_var;
187 } EmitContext;
188
189 typedef struct {
190 MonoBasicBlock *bb;
191 MonoInst *phi;
192 MonoBasicBlock *in_bb;
193 int sreg;
194 } PhiNode;
195
196 /*
197 * Instruction metadata
198 * This is the same as ins_info, but LREG != IREG.
199 */
200 #ifdef MINI_OP
201 #undef MINI_OP
202 #endif
203 #ifdef MINI_OP3
204 #undef MINI_OP3
205 #endif
206 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
207 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
208 #define NONE ' '
209 #define IREG 'i'
210 #define FREG 'f'
211 #define VREG 'v'
212 #define XREG 'x'
213 #define LREG 'l'
214 /* keep in sync with the enum in mini.h */
215 const char
216 mini_llvm_ins_info[] = {
217 #include "mini-ops.h"
218 };
219 #undef MINI_OP
220 #undef MINI_OP3
221
222 #if TARGET_SIZEOF_VOID_P == 4
223 #define GET_LONG_IMM(ins) ((ins)->inst_l)
224 #else
225 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
226 #endif
227
228 #define LLVM_INS_INFO(opcode) (&mini_llvm_ins_info [((opcode) - OP_START - 1) * 4])
229
230 #if 0
231 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
232 #else
233 #define TRACE_FAILURE(msg)
234 #endif
235
236 #ifdef TARGET_X86
237 #define IS_TARGET_X86 1
238 #else
239 #define IS_TARGET_X86 0
240 #endif
241
242 #ifdef TARGET_AMD64
243 #define IS_TARGET_AMD64 1
244 #else
245 #define IS_TARGET_AMD64 0
246 #endif
247
248 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
249
250 static LLVMIntPredicate cond_to_llvm_cond [] = {
251 LLVMIntEQ,
252 LLVMIntNE,
253 LLVMIntSLE,
254 LLVMIntSGE,
255 LLVMIntSLT,
256 LLVMIntSGT,
257 LLVMIntULE,
258 LLVMIntUGE,
259 LLVMIntULT,
260 LLVMIntUGT,
261 };
262
263 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
264 LLVMRealOEQ,
265 LLVMRealUNE,
266 LLVMRealOLE,
267 LLVMRealOGE,
268 LLVMRealOLT,
269 LLVMRealOGT,
270 LLVMRealULE,
271 LLVMRealUGE,
272 LLVMRealULT,
273 LLVMRealUGT,
274 };
275
276 typedef enum {
277 INTRINS_MEMSET,
278 INTRINS_MEMCPY,
279 INTRINS_SADD_OVF_I32,
280 INTRINS_UADD_OVF_I32,
281 INTRINS_SSUB_OVF_I32,
282 INTRINS_USUB_OVF_I32,
283 INTRINS_SMUL_OVF_I32,
284 INTRINS_UMUL_OVF_I32,
285 INTRINS_SADD_OVF_I64,
286 INTRINS_UADD_OVF_I64,
287 INTRINS_SSUB_OVF_I64,
288 INTRINS_USUB_OVF_I64,
289 INTRINS_SMUL_OVF_I64,
290 INTRINS_UMUL_OVF_I64,
291 INTRINS_SIN,
292 INTRINS_COS,
293 INTRINS_SQRT,
294 INTRINS_FABS,
295 INTRINS_ABSF,
296 INTRINS_SINF,
297 INTRINS_COSF,
298 INTRINS_SQRTF,
299 INTRINS_POWF,
300 INTRINS_EXPECT_I8,
301 INTRINS_EXPECT_I1,
302 #if defined(TARGET_AMD64) || defined(TARGET_X86)
303 INTRINS_SSE_PMOVMSKB,
304 INTRINS_SSE_PSRLI_W,
305 INTRINS_SSE_PSRAI_W,
306 INTRINS_SSE_PSLLI_W,
307 INTRINS_SSE_PSRLI_D,
308 INTRINS_SSE_PSRAI_D,
309 INTRINS_SSE_PSLLI_D,
310 INTRINS_SSE_PSRLI_Q,
311 INTRINS_SSE_PSLLI_Q,
312 INTRINS_SSE_SQRT_PD,
313 INTRINS_SSE_SQRT_PS,
314 INTRINS_SSE_RSQRT_PS,
315 INTRINS_SSE_RCP_PS,
316 INTRINS_SSE_CVTTPD2DQ,
317 INTRINS_SSE_CVTTPS2DQ,
318 INTRINS_SSE_CVTDQ2PD,
319 INTRINS_SSE_CVTDQ2PS,
320 INTRINS_SSE_CVTPD2DQ,
321 INTRINS_SSE_CVTPS2DQ,
322 INTRINS_SSE_CVTPD2PS,
323 INTRINS_SSE_CVTPS2PD,
324 INTRINS_SSE_CMPPD,
325 INTRINS_SSE_CMPPS,
326 INTRINS_SSE_PACKSSWB,
327 INTRINS_SSE_PACKUSWB,
328 INTRINS_SSE_PACKSSDW,
329 INTRINS_SSE_PACKUSDW,
330 INTRINS_SSE_MINPS,
331 INTRINS_SSE_MAXPS,
332 INTRINS_SSE_HADDPS,
333 INTRINS_SSE_HSUBPS,
334 INTRINS_SSE_ADDSUBPS,
335 INTRINS_SSE_MINPD,
336 INTRINS_SSE_MAXPD,
337 INTRINS_SSE_HADDPD,
338 INTRINS_SSE_HSUBPD,
339 INTRINS_SSE_ADDSUBPD,
340 INTRINS_SSE_PADDSW,
341 INTRINS_SSE_PSUBSW,
342 INTRINS_SSE_PADDUSW,
343 INTRINS_SSE_PSUBUSW,
344 INTRINS_SSE_PAVGW,
345 INTRINS_SSE_PMULHW,
346 INTRINS_SSE_PMULHU,
347 INTRINS_SE_PADDSB,
348 INTRINS_SSE_PSUBSB,
349 INTRINS_SSE_PADDUSB,
350 INTRINS_SSE_PSUBUSB,
351 INTRINS_SSE_PAVGB,
352 INTRINS_SSE_PAUSE,
353 INTRINS_SSE_DPPS,
354 INTRINS_SSE_ROUNDPD,
355 #endif
356 INTRINS_NUM
357 } IntrinsicId;
358
359 static MonoNativeTlsKey current_cfg_tls_id;
360
361 static MonoLLVMModule aot_module;
362
363 static GHashTable *intrins_id_to_name;
364 static GHashTable *intrins_name_to_id;
365
366 static void init_jit_module (MonoDomain *domain);
367
368 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
369 static void emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder);
370 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
371 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
372 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp);
373 static LLVMValueRef get_intrins_by_name (EmitContext *ctx, const char *name);
374 static LLVMValueRef get_intrins (EmitContext *ctx, int id);
375 static void llvm_jit_finalize_method (EmitContext *ctx);
376 static void mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params);
377 static void mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module);
378
379 static inline void
380 set_failure (EmitContext *ctx, const char *message)
381 {
382 TRACE_FAILURE (reason);
383 ctx->cfg->exception_message = g_strdup (message);
384 ctx->cfg->disable_llvm = TRUE;
385 }
386
387 /*
388 * IntPtrType:
389 *
390 * The LLVM type with width == TARGET_SIZEOF_VOID_P
391 */
392 static LLVMTypeRef
393 IntPtrType (void)
394 {
395 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
396 }
397
398 static LLVMTypeRef
399 ObjRefType (void)
400 {
401 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
402 }
403
404 static LLVMTypeRef
405 ThisType (void)
406 {
407 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
408 }
409
410 /*
411 * get_vtype_size:
412 *
413 * Return the size of the LLVM representation of the vtype T.
414 */
415 static guint32
416 get_vtype_size (MonoType *t)
417 {
418 int size;
419
420 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
421
422 /* LLVMArgAsIArgs depends on this since it stores whole words */
423 while (size < 2 * TARGET_SIZEOF_VOID_P && mono_is_power_of_two (size) == -1)
424 size ++;
425
426 return size;
427 }
428
429 /*
430 * simd_class_to_llvm_type:
431 *
432 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
433 */
434 static LLVMTypeRef
435 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
436 {
437 const char *klass_name = m_class_get_name (klass);
438 if (!strcmp (klass_name, "Vector2d")) {
439 return LLVMVectorType (LLVMDoubleType (), 2);
440 } else if (!strcmp (klass_name, "Vector2l")) {
441 return LLVMVectorType (LLVMInt64Type (), 2);
442 } else if (!strcmp (klass_name, "Vector2ul")) {
443 return LLVMVectorType (LLVMInt64Type (), 2);
444 } else if (!strcmp (klass_name, "Vector4i")) {
445 return LLVMVectorType (LLVMInt32Type (), 4);
446 } else if (!strcmp (klass_name, "Vector4ui")) {
447 return LLVMVectorType (LLVMInt32Type (), 4);
448 } else if (!strcmp (klass_name, "Vector4f")) {
449 return LLVMVectorType (LLVMFloatType (), 4);
450 } else if (!strcmp (klass_name, "Vector8s")) {
451 return LLVMVectorType (LLVMInt16Type (), 8);
452 } else if (!strcmp (klass_name, "Vector8us")) {
453 return LLVMVectorType (LLVMInt16Type (), 8);
454 } else if (!strcmp (klass_name, "Vector16sb")) {
455 return LLVMVectorType (LLVMInt8Type (), 16);
456 } else if (!strcmp (klass_name, "Vector16b")) {
457 return LLVMVectorType (LLVMInt8Type (), 16);
458 } else if (!strcmp (klass_name, "Vector2")) {
459 /* System.Numerics */
460 return LLVMVectorType (LLVMFloatType (), 4);
461 } else if (!strcmp (klass_name, "Vector3")) {
462 return LLVMVectorType (LLVMFloatType (), 4);
463 } else if (!strcmp (klass_name, "Vector4")) {
464 return LLVMVectorType (LLVMFloatType (), 4);
465 } else if (!strcmp (klass_name, "Vector`1")) {
466 MonoType *etype = mono_class_get_generic_class (klass)->context.class_inst->type_argv [0];
467 switch (etype->type) {
468 case MONO_TYPE_I1:
469 case MONO_TYPE_U1:
470 return LLVMVectorType (LLVMInt8Type (), 16);
471 case MONO_TYPE_I2:
472 case MONO_TYPE_U2:
473 return LLVMVectorType (LLVMInt16Type (), 8);
474 case MONO_TYPE_I4:
475 case MONO_TYPE_U4:
476 return LLVMVectorType (LLVMInt32Type (), 4);
477 case MONO_TYPE_I8:
478 case MONO_TYPE_U8:
479 return LLVMVectorType (LLVMInt64Type (), 2);
480 case MONO_TYPE_R4:
481 return LLVMVectorType (LLVMFloatType (), 4);
482 case MONO_TYPE_R8:
483 return LLVMVectorType (LLVMDoubleType (), 2);
484 default:
485 g_assert_not_reached ();
486 return NULL;
487 }
488 } else {
489 printf ("%s\n", klass_name);
490 NOT_IMPLEMENTED;
491 return NULL;
492 }
493 }
494
495 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
496 static inline G_GNUC_UNUSED LLVMTypeRef
497 type_to_simd_type (int type)
498 {
499 switch (type) {
500 case MONO_TYPE_I1:
501 return LLVMVectorType (LLVMInt8Type (), 16);
502 case MONO_TYPE_I2:
503 return LLVMVectorType (LLVMInt16Type (), 8);
504 case MONO_TYPE_I4:
505 return LLVMVectorType (LLVMInt32Type (), 4);
506 case MONO_TYPE_I8:
507 return LLVMVectorType (LLVMInt64Type (), 2);
508 case MONO_TYPE_R8:
509 return LLVMVectorType (LLVMDoubleType (), 2);
510 case MONO_TYPE_R4:
511 return LLVMVectorType (LLVMFloatType (), 4);
512 default:
513 g_assert_not_reached ();
514 return NULL;
515 }
516 }
517
518 static LLVMTypeRef
519 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
520 {
521 int i, size, nfields, esize;
522 LLVMTypeRef *eltypes;
523 char *name;
524 MonoType *t;
525 LLVMTypeRef ltype;
526
527 t = m_class_get_byval_arg (klass);
528
529 if (mini_type_is_hfa (t, &nfields, &esize)) {
530 /*
531 * This is needed on arm64 where HFAs are returned in
532 * registers.
533 */
534 /* SIMD types have size 16 in mono_class_value_size () */
535 if (m_class_is_simd_type (klass))
536 nfields = 16/ esize;
537 size = nfields;
538 eltypes = g_new (LLVMTypeRef, size);
539 for (i = 0; i < size; ++i)
540 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
541 } else {
542 size = get_vtype_size (t);
543
544 eltypes = g_new (LLVMTypeRef, size);
545 for (i = 0; i < size; ++i)
546 eltypes [i] = LLVMInt8Type ();
547 }
548
549 name = mono_type_full_name (m_class_get_byval_arg (klass));
550 ltype = LLVMStructCreateNamed (module->context, name);
551 LLVMStructSetBody (ltype, eltypes, size, FALSE);
552 g_free (eltypes);
553 g_free (name);
554
555 return ltype;
556 }
557
558 /*
559 * type_to_llvm_type:
560 *
561 * Return the LLVM type corresponding to T.
562 */
563 static LLVMTypeRef
564 type_to_llvm_type (EmitContext *ctx, MonoType *t)
565 {
566 if (t->byref)
567 return ThisType ();
568
569 t = mini_get_underlying_type (t);
570
571 switch (t->type) {
572 case MONO_TYPE_VOID:
573 return LLVMVoidType ();
574 case MONO_TYPE_I1:
575 return LLVMInt8Type ();
576 case MONO_TYPE_I2:
577 return LLVMInt16Type ();
578 case MONO_TYPE_I4:
579 return LLVMInt32Type ();
580 case MONO_TYPE_U1:
581 return LLVMInt8Type ();
582 case MONO_TYPE_U2:
583 return LLVMInt16Type ();
584 case MONO_TYPE_U4:
585 return LLVMInt32Type ();
586 case MONO_TYPE_I8:
587 case MONO_TYPE_U8:
588 return LLVMInt64Type ();
589 case MONO_TYPE_R4:
590 return LLVMFloatType ();
591 case MONO_TYPE_R8:
592 return LLVMDoubleType ();
593 case MONO_TYPE_I:
594 case MONO_TYPE_U:
595 return IntPtrType ();
596 case MONO_TYPE_OBJECT:
597 case MONO_TYPE_PTR:
598 return ObjRefType ();
599 case MONO_TYPE_VAR:
600 case MONO_TYPE_MVAR:
601 /* Because of generic sharing */
602 return ObjRefType ();
603 case MONO_TYPE_GENERICINST:
604 if (!mono_type_generic_inst_is_valuetype (t))
605 return ObjRefType ();
606 /* Fall through */
607 case MONO_TYPE_VALUETYPE:
608 case MONO_TYPE_TYPEDBYREF: {
609 MonoClass *klass;
610 LLVMTypeRef ltype;
611
612 klass = mono_class_from_mono_type_internal (t);
613
614 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
615 return simd_class_to_llvm_type (ctx, klass);
616
617 if (m_class_is_enumtype (klass))
618 return type_to_llvm_type (ctx, mono_class_enum_basetype_internal (klass));
619
620 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
621 if (!ltype) {
622 ltype = create_llvm_type_for_type (ctx->module, klass);
623 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
624 }
625 return ltype;
626 }
627
628 default:
629 printf ("X: %d\n", t->type);
630 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
631 ctx->cfg->disable_llvm = TRUE;
632 return NULL;
633 }
634 }
635
636 /*
637 * type_is_unsigned:
638 *
639 * Return whenever T is an unsigned int type.
640 */
641 static gboolean
642 type_is_unsigned (EmitContext *ctx, MonoType *t)
643 {
644 t = mini_get_underlying_type (t);
645 if (t->byref)
646 return FALSE;
647 switch (t->type) {
648 case MONO_TYPE_U1:
649 case MONO_TYPE_U2:
650 case MONO_TYPE_CHAR:
651 case MONO_TYPE_U4:
652 case MONO_TYPE_U8:
653 return TRUE;
654 default:
655 return FALSE;
656 }
657 }
658
659 /*
660 * type_to_llvm_arg_type:
661 *
662 * Same as type_to_llvm_type, but treat i8/i16 as i32.
663 */
664 static LLVMTypeRef
665 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
666 {
667 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
668
669 if (ctx->cfg->llvm_only)
670 return ptype;
671
672 /*
673 * This works on all abis except arm64/ios which passes multiple
674 * arguments in one stack slot.
675 */
676 #ifndef TARGET_ARM64
677 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
678 /*
679 * LLVM generates code which only sets the lower bits, while JITted
680 * code expects all the bits to be set.
681 */
682 ptype = LLVMInt32Type ();
683 }
684 #endif
685
686 return ptype;
687 }
688
689 /*
690 * llvm_type_to_stack_type:
691 *
692 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
693 * on the IL stack.
694 */
695 static G_GNUC_UNUSED LLVMTypeRef
696 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
697 {
698 if (type == NULL)
699 return NULL;
700 if (type == LLVMInt8Type ())
701 return LLVMInt32Type ();
702 else if (type == LLVMInt16Type ())
703 return LLVMInt32Type ();
704 else if (!cfg->r4fp && type == LLVMFloatType ())
705 return LLVMDoubleType ();
706 else
707 return type;
708 }
709
710 /*
711 * regtype_to_llvm_type:
712 *
713 * Return the LLVM type corresponding to the regtype C used in instruction
714 * descriptions.
715 */
716 static LLVMTypeRef
717 regtype_to_llvm_type (char c)
718 {
719 switch (c) {
720 case 'i':
721 return LLVMInt32Type ();
722 case 'l':
723 return LLVMInt64Type ();
724 case 'f':
725 return LLVMDoubleType ();
726 default:
727 return NULL;
728 }
729 }
730
731 /*
732 * op_to_llvm_type:
733 *
734 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
735 */
736 static LLVMTypeRef
737 op_to_llvm_type (int opcode)
738 {
739 switch (opcode) {
740 case OP_ICONV_TO_I1:
741 case OP_LCONV_TO_I1:
742 return LLVMInt8Type ();
743 case OP_ICONV_TO_U1:
744 case OP_LCONV_TO_U1:
745 return LLVMInt8Type ();
746 case OP_ICONV_TO_I2:
747 case OP_LCONV_TO_I2:
748 return LLVMInt16Type ();
749 case OP_ICONV_TO_U2:
750 case OP_LCONV_TO_U2:
751 return LLVMInt16Type ();
752 case OP_ICONV_TO_I4:
753 case OP_LCONV_TO_I4:
754 return LLVMInt32Type ();
755 case OP_ICONV_TO_U4:
756 case OP_LCONV_TO_U4:
757 return LLVMInt32Type ();
758 case OP_ICONV_TO_I8:
759 return LLVMInt64Type ();
760 case OP_ICONV_TO_R4:
761 return LLVMFloatType ();
762 case OP_ICONV_TO_R8:
763 return LLVMDoubleType ();
764 case OP_ICONV_TO_U8:
765 return LLVMInt64Type ();
766 case OP_FCONV_TO_I4:
767 return LLVMInt32Type ();
768 case OP_FCONV_TO_I8:
769 return LLVMInt64Type ();
770 case OP_FCONV_TO_I1:
771 case OP_FCONV_TO_U1:
772 case OP_RCONV_TO_I1:
773 case OP_RCONV_TO_U1:
774 return LLVMInt8Type ();
775 case OP_FCONV_TO_I2:
776 case OP_FCONV_TO_U2:
777 case OP_RCONV_TO_I2:
778 case OP_RCONV_TO_U2:
779 return LLVMInt16Type ();
780 case OP_RCONV_TO_U4:
781 return LLVMInt32Type ();
782 case OP_FCONV_TO_I:
783 case OP_FCONV_TO_U:
784 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
785 case OP_IADD_OVF:
786 case OP_IADD_OVF_UN:
787 case OP_ISUB_OVF:
788 case OP_ISUB_OVF_UN:
789 case OP_IMUL_OVF:
790 case OP_IMUL_OVF_UN:
791 return LLVMInt32Type ();
792 case OP_LADD_OVF:
793 case OP_LADD_OVF_UN:
794 case OP_LSUB_OVF:
795 case OP_LSUB_OVF_UN:
796 case OP_LMUL_OVF:
797 case OP_LMUL_OVF_UN:
798 return LLVMInt64Type ();
799 default:
800 printf ("%s\n", mono_inst_name (opcode));
801 g_assert_not_reached ();
802 return NULL;
803 }
804 }
805
806 #define CLAUSE_START(clause) ((clause)->try_offset)
807 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
808
809 /*
810 * load_store_to_llvm_type:
811 *
812 * Return the size/sign/zero extension corresponding to the load/store opcode
813 * OPCODE.
814 */
815 static LLVMTypeRef
816 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
817 {
818 *sext = FALSE;
819 *zext = FALSE;
820
821 switch (opcode) {
822 case OP_LOADI1_MEMBASE:
823 case OP_STOREI1_MEMBASE_REG:
824 case OP_STOREI1_MEMBASE_IMM:
825 case OP_ATOMIC_LOAD_I1:
826 case OP_ATOMIC_STORE_I1:
827 *size = 1;
828 *sext = TRUE;
829 return LLVMInt8Type ();
830 case OP_LOADU1_MEMBASE:
831 case OP_LOADU1_MEM:
832 case OP_ATOMIC_LOAD_U1:
833 case OP_ATOMIC_STORE_U1:
834 *size = 1;
835 *zext = TRUE;
836 return LLVMInt8Type ();
837 case OP_LOADI2_MEMBASE:
838 case OP_STOREI2_MEMBASE_REG:
839 case OP_STOREI2_MEMBASE_IMM:
840 case OP_ATOMIC_LOAD_I2:
841 case OP_ATOMIC_STORE_I2:
842 *size = 2;
843 *sext = TRUE;
844 return LLVMInt16Type ();
845 case OP_LOADU2_MEMBASE:
846 case OP_LOADU2_MEM:
847 case OP_ATOMIC_LOAD_U2:
848 case OP_ATOMIC_STORE_U2:
849 *size = 2;
850 *zext = TRUE;
851 return LLVMInt16Type ();
852 case OP_LOADI4_MEMBASE:
853 case OP_LOADU4_MEMBASE:
854 case OP_LOADI4_MEM:
855 case OP_LOADU4_MEM:
856 case OP_STOREI4_MEMBASE_REG:
857 case OP_STOREI4_MEMBASE_IMM:
858 case OP_ATOMIC_LOAD_I4:
859 case OP_ATOMIC_STORE_I4:
860 case OP_ATOMIC_LOAD_U4:
861 case OP_ATOMIC_STORE_U4:
862 *size = 4;
863 return LLVMInt32Type ();
864 case OP_LOADI8_MEMBASE:
865 case OP_LOADI8_MEM:
866 case OP_STOREI8_MEMBASE_REG:
867 case OP_STOREI8_MEMBASE_IMM:
868 case OP_ATOMIC_LOAD_I8:
869 case OP_ATOMIC_STORE_I8:
870 case OP_ATOMIC_LOAD_U8:
871 case OP_ATOMIC_STORE_U8:
872 *size = 8;
873 return LLVMInt64Type ();
874 case OP_LOADR4_MEMBASE:
875 case OP_STORER4_MEMBASE_REG:
876 case OP_ATOMIC_LOAD_R4:
877 case OP_ATOMIC_STORE_R4:
878 *size = 4;
879 return LLVMFloatType ();
880 case OP_LOADR8_MEMBASE:
881 case OP_STORER8_MEMBASE_REG:
882 case OP_ATOMIC_LOAD_R8:
883 case OP_ATOMIC_STORE_R8:
884 *size = 8;
885 return LLVMDoubleType ();
886 case OP_LOAD_MEMBASE:
887 case OP_LOAD_MEM:
888 case OP_STORE_MEMBASE_REG:
889 case OP_STORE_MEMBASE_IMM:
890 *size = TARGET_SIZEOF_VOID_P;
891 return IntPtrType ();
892 default:
893 g_assert_not_reached ();
894 return NULL;
895 }
896 }
897
898 /*
899 * ovf_op_to_intrins:
900 *
901 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
902 */
903 static const char*
904 ovf_op_to_intrins (int opcode)
905 {
906 switch (opcode) {
907 case OP_IADD_OVF:
908 return "llvm.sadd.with.overflow.i32";
909 case OP_IADD_OVF_UN:
910 return "llvm.uadd.with.overflow.i32";
911 case OP_ISUB_OVF:
912 return "llvm.ssub.with.overflow.i32";
913 case OP_ISUB_OVF_UN:
914 return "llvm.usub.with.overflow.i32";
915 case OP_IMUL_OVF:
916 return "llvm.smul.with.overflow.i32";
917 case OP_IMUL_OVF_UN:
918 return "llvm.umul.with.overflow.i32";
919 case OP_LADD_OVF:
920 return "llvm.sadd.with.overflow.i64";
921 case OP_LADD_OVF_UN:
922 return "llvm.uadd.with.overflow.i64";
923 case OP_LSUB_OVF:
924 return "llvm.ssub.with.overflow.i64";
925 case OP_LSUB_OVF_UN:
926 return "llvm.usub.with.overflow.i64";
927 case OP_LMUL_OVF:
928 return "llvm.smul.with.overflow.i64";
929 case OP_LMUL_OVF_UN:
930 return "llvm.umul.with.overflow.i64";
931 default:
932 g_assert_not_reached ();
933 return NULL;
934 }
935 }
936
937 static const char*
938 simd_op_to_intrins (int opcode)
939 {
940 switch (opcode) {
941 #if defined(TARGET_X86) || defined(TARGET_AMD64)
942 case OP_MINPD:
943 return "llvm.x86.sse2.min.pd";
944 case OP_MINPS:
945 return "llvm.x86.sse.min.ps";
946 case OP_MAXPD:
947 return "llvm.x86.sse2.max.pd";
948 case OP_MAXPS:
949 return "llvm.x86.sse.max.ps";
950 case OP_HADDPD:
951 return "llvm.x86.sse3.hadd.pd";
952 case OP_HADDPS:
953 return "llvm.x86.sse3.hadd.ps";
954 case OP_HSUBPD:
955 return "llvm.x86.sse3.hsub.pd";
956 case OP_HSUBPS:
957 return "llvm.x86.sse3.hsub.ps";
958 case OP_ADDSUBPS:
959 return "llvm.x86.sse3.addsub.ps";
960 case OP_ADDSUBPD:
961 return "llvm.x86.sse3.addsub.pd";
962 case OP_EXTRACT_MASK:
963 return "llvm.x86.sse2.pmovmskb.128";
964 case OP_PSHRW:
965 case OP_PSHRW_REG:
966 return "llvm.x86.sse2.psrli.w";
967 case OP_PSHRD:
968 case OP_PSHRD_REG:
969 return "llvm.x86.sse2.psrli.d";
970 case OP_PSHRQ:
971 case OP_PSHRQ_REG:
972 return "llvm.x86.sse2.psrli.q";
973 case OP_PSHLW:
974 case OP_PSHLW_REG:
975 return "llvm.x86.sse2.pslli.w";
976 case OP_PSHLD:
977 case OP_PSHLD_REG:
978 return "llvm.x86.sse2.pslli.d";
979 case OP_PSHLQ:
980 case OP_PSHLQ_REG:
981 return "llvm.x86.sse2.pslli.q";
982 case OP_PSARW:
983 case OP_PSARW_REG:
984 return "llvm.x86.sse2.psrai.w";
985 case OP_PSARD:
986 case OP_PSARD_REG:
987 return "llvm.x86.sse2.psrai.d";
988 case OP_PADDB_SAT:
989 return "llvm.x86.sse2.padds.b";
990 case OP_PADDW_SAT:
991 return "llvm.x86.sse2.padds.w";
992 case OP_PSUBB_SAT:
993 return "llvm.x86.sse2.psubs.b";
994 case OP_PSUBW_SAT:
995 return "llvm.x86.sse2.psubs.w";
996 case OP_PADDB_SAT_UN:
997 return "llvm.x86.sse2.paddus.b";
998 case OP_PADDW_SAT_UN:
999 return "llvm.x86.sse2.paddus.w";
1000 case OP_PSUBB_SAT_UN:
1001 return "llvm.x86.sse2.psubus.b";
1002 case OP_PSUBW_SAT_UN:
1003 return "llvm.x86.sse2.psubus.w";
1004 case OP_PAVGB_UN:
1005 return "llvm.x86.sse2.pavg.b";
1006 case OP_PAVGW_UN:
1007 return "llvm.x86.sse2.pavg.w";
1008 case OP_SQRTPS:
1009 return "llvm.x86.sse.sqrt.ps";
1010 case OP_SQRTPD:
1011 return "llvm.x86.sse2.sqrt.pd";
1012 case OP_RSQRTPS:
1013 return "llvm.x86.sse.rsqrt.ps";
1014 case OP_RCPPS:
1015 return "llvm.x86.sse.rcp.ps";
1016 case OP_CVTDQ2PD:
1017 return "llvm.x86.sse2.cvtdq2pd";
1018 case OP_CVTDQ2PS:
1019 return "llvm.x86.sse2.cvtdq2ps";
1020 case OP_CVTPD2DQ:
1021 return "llvm.x86.sse2.cvtpd2dq";
1022 case OP_CVTPS2DQ:
1023 return "llvm.x86.sse2.cvtps2dq";
1024 case OP_CVTPD2PS:
1025 return "llvm.x86.sse2.cvtpd2ps";
1026 case OP_CVTPS2PD:
1027 return "llvm.x86.sse2.cvtps2pd";
1028 case OP_CVTTPD2DQ:
1029 return "llvm.x86.sse2.cvttpd2dq";
1030 case OP_CVTTPS2DQ:
1031 return "llvm.x86.sse2.cvttps2dq";
1032 case OP_PACKW:
1033 return "llvm.x86.sse2.packsswb.128";
1034 case OP_PACKD:
1035 return "llvm.x86.sse2.packssdw.128";
1036 case OP_PACKW_UN:
1037 return "llvm.x86.sse2.packuswb.128";
1038 case OP_PACKD_UN:
1039 return "llvm.x86.sse41.packusdw";
1040 case OP_PMULW_HIGH:
1041 return "llvm.x86.sse2.pmulh.w";
1042 case OP_PMULW_HIGH_UN:
1043 return "llvm.x86.sse2.pmulhu.w";
1044 case OP_DPPS:
1045 return "llvm.x86.sse41.dpps";
1046 #endif
1047 default:
1048 g_assert_not_reached ();
1049 return NULL;
1050 }
1051 }
1052
1053 static LLVMTypeRef
1054 simd_op_to_llvm_type (int opcode)
1055 {
1056 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1057 switch (opcode) {
1058 case OP_EXTRACT_R8:
1059 case OP_EXPAND_R8:
1060 return type_to_simd_type (MONO_TYPE_R8);
1061 case OP_EXTRACT_I8:
1062 case OP_EXPAND_I8:
1063 return type_to_simd_type (MONO_TYPE_I8);
1064 case OP_EXTRACT_I4:
1065 case OP_EXPAND_I4:
1066 return type_to_simd_type (MONO_TYPE_I4);
1067 case OP_EXTRACT_I2:
1068 case OP_EXTRACT_U2:
1069 case OP_EXTRACTX_U2:
1070 case OP_EXPAND_I2:
1071 return type_to_simd_type (MONO_TYPE_I2);
1072 case OP_EXTRACT_I1:
1073 case OP_EXTRACT_U1:
1074 case OP_EXPAND_I1:
1075 return type_to_simd_type (MONO_TYPE_I1);
1076 case OP_EXPAND_R4:
1077 return type_to_simd_type (MONO_TYPE_R4);
1078 case OP_CVTDQ2PD:
1079 case OP_CVTDQ2PS:
1080 return type_to_simd_type (MONO_TYPE_I4);
1081 case OP_CVTPD2DQ:
1082 case OP_CVTPD2PS:
1083 case OP_CVTTPD2DQ:
1084 return type_to_simd_type (MONO_TYPE_R8);
1085 case OP_CVTPS2DQ:
1086 case OP_CVTPS2PD:
1087 case OP_CVTTPS2DQ:
1088 return type_to_simd_type (MONO_TYPE_R4);
1089 case OP_EXTRACT_MASK:
1090 return type_to_simd_type (MONO_TYPE_I1);
1091 case OP_SQRTPS:
1092 case OP_RSQRTPS:
1093 case OP_RCPPS:
1094 case OP_DUPPS_LOW:
1095 case OP_DUPPS_HIGH:
1096 return type_to_simd_type (MONO_TYPE_R4);
1097 case OP_SQRTPD:
1098 case OP_DUPPD:
1099 return type_to_simd_type (MONO_TYPE_R8);
1100 default:
1101 g_assert_not_reached ();
1102 return NULL;
1103 }
1104 #else
1105 return NULL;
1106 #endif
1107 }
1108
1109 static void
1110 set_preserveall_cc (LLVMValueRef func)
1111 {
1112 /*
1113 * xcode10 (watchOS) and ARM/ARM64 doesn't seem to support preserveall, it fails with:
1114 * fatal error: error in backend: Unsupported calling convention
1115 */
1116 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1117 mono_llvm_set_preserveall_cc (func);
1118 #endif
1119 }
1120
1121 static void
1122 set_call_preserveall_cc (LLVMValueRef func)
1123 {
1124 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1125 mono_llvm_set_call_preserveall_cc (func);
1126 #endif
1127 }
1128
1129 /*
1130 * get_bb:
1131 *
1132 * Return the LLVM basic block corresponding to BB.
1133 */
1134 static LLVMBasicBlockRef
1135 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
1136 {
1137 char bb_name_buf [128];
1138 char *bb_name;
1139
1140 if (ctx->bblocks [bb->block_num].bblock == NULL) {
1141 if (bb->flags & BB_EXCEPTION_HANDLER) {
1142 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
1143 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
1144 bb_name = bb_name_buf;
1145 } else if (bb->block_num < 256) {
1146 if (!ctx->module->bb_names) {
1147 ctx->module->bb_names_len = 256;
1148 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
1149 }
1150 if (!ctx->module->bb_names [bb->block_num]) {
1151 char *n;
1152
1153 n = g_strdup_printf ("BB%d", bb->block_num);
1154 mono_memory_barrier ();
1155 ctx->module->bb_names [bb->block_num] = n;
1156 }
1157 bb_name = ctx->module->bb_names [bb->block_num];
1158 } else {
1159 sprintf (bb_name_buf, "BB%d", bb->block_num);
1160 bb_name = bb_name_buf;
1161 }
1162
1163 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1164 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1165 }
1166
1167 return ctx->bblocks [bb->block_num].bblock;
1168 }
1169
1170 /*
1171 * get_end_bb:
1172 *
1173 * Return the last LLVM bblock corresponding to BB.
1174 * This might not be equal to the bb returned by get_bb () since we need to generate
1175 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1176 */
1177 static LLVMBasicBlockRef
1178 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1179 {
1180 get_bb (ctx, bb);
1181 return ctx->bblocks [bb->block_num].end_bblock;
1182 }
1183
1184 static LLVMBasicBlockRef
1185 gen_bb (EmitContext *ctx, const char *prefix)
1186 {
1187 char bb_name [128];
1188
1189 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1190 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1191 }
1192
1193 /*
1194 * resolve_patch:
1195 *
1196 * Return the target of the patch identified by TYPE and TARGET.
1197 */
1198 static gpointer
1199 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1200 {
1201 MonoJumpInfo ji;
1202 ERROR_DECL (error);
1203 gpointer res;
1204
1205 memset (&ji, 0, sizeof (ji));
1206 ji.type = type;
1207 ji.data.target = target;
1208
1209 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, error);
1210 mono_error_assert_ok (error);
1211
1212 return res;
1213 }
1214
1215 /*
1216 * convert_full:
1217 *
1218 * Emit code to convert the LLVM value V to DTYPE.
1219 */
1220 static LLVMValueRef
1221 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1222 {
1223 LLVMTypeRef stype = LLVMTypeOf (v);
1224
1225 if (stype != dtype) {
1226 gboolean ext = FALSE;
1227
1228 /* Extend */
1229 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1230 ext = TRUE;
1231 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1232 ext = TRUE;
1233 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1234 ext = TRUE;
1235
1236 if (ext)
1237 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1238
1239 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1240 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1241
1242 /* Trunc */
1243 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1244 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1245 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1246 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1247 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1248 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1249 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1250 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1251
1252 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1253 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1254 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1255 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1256 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1257 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1258
1259 if (mono_arch_is_soft_float ()) {
1260 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1261 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1262 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1263 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1264 }
1265
1266 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1267 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1268
1269 LLVMDumpValue (v);
1270 LLVMDumpValue (LLVMConstNull (dtype));
1271 g_assert_not_reached ();
1272 return NULL;
1273 } else {
1274 return v;
1275 }
1276 }
1277
1278 static LLVMValueRef
1279 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1280 {
1281 return convert_full (ctx, v, dtype, FALSE);
1282 }
1283
1284 /*
1285 * emit_volatile_load:
1286 *
1287 * If vreg is volatile, emit a load from its address.
1288 */
1289 static LLVMValueRef
1290 emit_volatile_load (EmitContext *ctx, int vreg)
1291 {
1292 MonoType *t;
1293 LLVMValueRef v;
1294
1295 // On arm64, we pass the rgctx in a callee saved
1296 // register on arm64 (x15), and llvm might keep the value in that register
1297 // even through the register is marked as 'reserved' inside llvm.
1298
1299 v = mono_llvm_build_load (ctx->builder, ctx->addresses [vreg], "", TRUE);
1300 t = ctx->vreg_cli_types [vreg];
1301 if (t && !t->byref) {
1302 /*
1303 * Might have to zero extend since llvm doesn't have
1304 * unsigned types.
1305 */
1306 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1307 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1308 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1309 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1310 else if (t->type == MONO_TYPE_U8)
1311 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1312 }
1313
1314 return v;
1315 }
1316
1317 /*
1318 * emit_volatile_store:
1319 *
1320 * If VREG is volatile, emit a store from its value to its address.
1321 */
1322 static void
1323 emit_volatile_store (EmitContext *ctx, int vreg)
1324 {
1325 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1326
1327 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1328 g_assert (ctx->addresses [vreg]);
1329 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1330 }
1331 }
1332
1333 static LLVMTypeRef
1334 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1335 {
1336 LLVMTypeRef ret_type;
1337 LLVMTypeRef *param_types = NULL;
1338 LLVMTypeRef res;
1339 int i, pindex;
1340 MonoType *rtype;
1341
1342 ret_type = type_to_llvm_type (ctx, sig->ret);
1343 if (!ctx_ok (ctx))
1344 return NULL;
1345 rtype = mini_get_underlying_type (sig->ret);
1346
1347 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1348 pindex = 0;
1349
1350 if (sig->hasthis)
1351 param_types [pindex ++] = ThisType ();
1352 for (i = 0; i < sig->param_count; ++i)
1353 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1354
1355 if (!ctx_ok (ctx)) {
1356 g_free (param_types);
1357 return NULL;
1358 }
1359
1360 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1361 g_free (param_types);
1362
1363 return res;
1364 }
1365
1366 /*
1367 * sig_to_llvm_sig_full:
1368 *
1369 * Return the LLVM signature corresponding to the mono signature SIG using the
1370 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1371 */
1372 static LLVMTypeRef
1373 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1374 {
1375 LLVMTypeRef ret_type;
1376 LLVMTypeRef *param_types = NULL;
1377 LLVMTypeRef res;
1378 int i, j, pindex, vret_arg_pindex = 0;
1379 gboolean vretaddr = FALSE;
1380 MonoType *rtype;
1381
1382 if (!cinfo)
1383 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1384
1385 ret_type = type_to_llvm_type (ctx, sig->ret);
1386 if (!ctx_ok (ctx))
1387 return NULL;
1388 rtype = mini_get_underlying_type (sig->ret);
1389
1390 switch (cinfo->ret.storage) {
1391 case LLVMArgVtypeInReg:
1392 /* LLVM models this by returning an aggregate value */
1393 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1394 LLVMTypeRef members [2];
1395
1396 members [0] = IntPtrType ();
1397 ret_type = LLVMStructType (members, 1, FALSE);
1398 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1399 /* Empty struct */
1400 ret_type = LLVMVoidType ();
1401 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1402 LLVMTypeRef members [2];
1403
1404 members [0] = IntPtrType ();
1405 members [1] = IntPtrType ();
1406 ret_type = LLVMStructType (members, 2, FALSE);
1407 } else {
1408 g_assert_not_reached ();
1409 }
1410 break;
1411 case LLVMArgVtypeByVal:
1412 /* Vtype returned normally by val */
1413 break;
1414 case LLVMArgVtypeAsScalar: {
1415 int size = mono_class_value_size (mono_class_from_mono_type_internal (rtype), NULL);
1416 /* LLVM models this by returning an int */
1417 if (size < TARGET_SIZEOF_VOID_P) {
1418 g_assert (cinfo->ret.nslots == 1);
1419 ret_type = LLVMIntType (size * 8);
1420 } else {
1421 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1422 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (target_mgreg_t) * 8);
1423 }
1424 break;
1425 }
1426 case LLVMArgAsIArgs:
1427 ret_type = LLVMArrayType (IntPtrType (), cinfo->ret.nslots);
1428 break;
1429 case LLVMArgFpStruct: {
1430 /* Vtype returned as a fp struct */
1431 LLVMTypeRef members [16];
1432
1433 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1434 for (i = 0; i < cinfo->ret.nslots; ++i)
1435 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1436 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1437 break;
1438 }
1439 case LLVMArgVtypeByRef:
1440 /* Vtype returned using a hidden argument */
1441 ret_type = LLVMVoidType ();
1442 break;
1443 case LLVMArgVtypeRetAddr:
1444 case LLVMArgGsharedvtFixed:
1445 case LLVMArgGsharedvtFixedVtype:
1446 case LLVMArgGsharedvtVariable:
1447 vretaddr = TRUE;
1448 ret_type = LLVMVoidType ();
1449 break;
1450 default:
1451 break;
1452 }
1453
1454 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1455 pindex = 0;
1456 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1457 /*
1458 * Has to be the first argument because of the sret argument attribute
1459 * FIXME: This might conflict with passing 'this' as the first argument, but
1460 * this is only used on arm64 which has a dedicated struct return register.
1461 */
1462 cinfo->vret_arg_pindex = pindex;
1463 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1464 if (!ctx_ok (ctx)) {
1465 g_free (param_types);
1466 return NULL;
1467 }
1468 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1469 pindex ++;
1470 }
1471 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1472 cinfo->rgctx_arg_pindex = pindex;
1473 param_types [pindex] = ctx->module->ptr_type;
1474 pindex ++;
1475 }
1476 if (cinfo->imt_arg) {
1477 cinfo->imt_arg_pindex = pindex;
1478 param_types [pindex] = ctx->module->ptr_type;
1479 pindex ++;
1480 }
1481 if (vretaddr) {
1482 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1483 vret_arg_pindex = pindex;
1484 if (cinfo->vret_arg_index == 1) {
1485 /* Add the slots consumed by the first argument */
1486 LLVMArgInfo *ainfo = &cinfo->args [0];
1487 switch (ainfo->storage) {
1488 case LLVMArgVtypeInReg:
1489 for (j = 0; j < 2; ++j) {
1490 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1491 vret_arg_pindex ++;
1492 }
1493 break;
1494 default:
1495 vret_arg_pindex ++;
1496 }
1497 }
1498
1499 cinfo->vret_arg_pindex = vret_arg_pindex;
1500 }
1501
1502 if (vretaddr && vret_arg_pindex == pindex)
1503 param_types [pindex ++] = IntPtrType ();
1504 if (sig->hasthis) {
1505 cinfo->this_arg_pindex = pindex;
1506 param_types [pindex ++] = ThisType ();
1507 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1508 }
1509 if (vretaddr && vret_arg_pindex == pindex)
1510 param_types [pindex ++] = IntPtrType ();
1511 for (i = 0; i < sig->param_count; ++i) {
1512 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1513
1514 if (vretaddr && vret_arg_pindex == pindex)
1515 param_types [pindex ++] = IntPtrType ();
1516 ainfo->pindex = pindex;
1517
1518 switch (ainfo->storage) {
1519 case LLVMArgVtypeInReg:
1520 for (j = 0; j < 2; ++j) {
1521 switch (ainfo->pair_storage [j]) {
1522 case LLVMArgInIReg:
1523 param_types [pindex ++] = LLVMIntType (TARGET_SIZEOF_VOID_P * 8);
1524 break;
1525 case LLVMArgNone:
1526 break;
1527 default:
1528 g_assert_not_reached ();
1529 }
1530 }
1531 break;
1532 case LLVMArgVtypeByVal:
1533 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1534 if (!ctx_ok (ctx))
1535 break;
1536 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1537 pindex ++;
1538 break;
1539 case LLVMArgAsIArgs:
1540 if (ainfo->esize == 8)
1541 param_types [pindex] = LLVMArrayType (LLVMInt64Type (), ainfo->nslots);
1542 else
1543 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1544 pindex ++;
1545 break;
1546 case LLVMArgVtypeByRef:
1547 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1548 if (!ctx_ok (ctx))
1549 break;
1550 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1551 pindex ++;
1552 break;
1553 case LLVMArgAsFpArgs: {
1554 int j;
1555
1556 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1557 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1558 param_types [pindex ++] = LLVMDoubleType ();
1559 for (j = 0; j < ainfo->nslots; ++j)
1560 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1561 break;
1562 }
1563 case LLVMArgVtypeAsScalar:
1564 g_assert_not_reached ();
1565 break;
1566 case LLVMArgGsharedvtFixed:
1567 case LLVMArgGsharedvtFixedVtype:
1568 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1569 break;
1570 case LLVMArgGsharedvtVariable:
1571 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1572 break;
1573 default:
1574 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1575 break;
1576 }
1577 }
1578 if (!ctx_ok (ctx)) {
1579 g_free (param_types);
1580 return NULL;
1581 }
1582 if (vretaddr && vret_arg_pindex == pindex)
1583 param_types [pindex ++] = IntPtrType ();
1584 if (ctx->llvm_only && cinfo->rgctx_arg) {
1585 /* Pass the rgctx as the last argument */
1586 cinfo->rgctx_arg_pindex = pindex;
1587 param_types [pindex] = ctx->module->ptr_type;
1588 pindex ++;
1589 } else if (ctx->llvm_only && cinfo->dummy_arg) {
1590 /* Pass a dummy arg last */
1591 cinfo->dummy_arg_pindex = pindex;
1592 param_types [pindex] = ctx->module->ptr_type;
1593 pindex ++;
1594 }
1595
1596 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1597 g_free (param_types);
1598
1599 return res;
1600 }
1601
1602 static LLVMTypeRef
1603 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1604 {
1605 return sig_to_llvm_sig_full (ctx, sig, NULL);
1606 }
1607
1608 /*
1609 * LLVMFunctionType1:
1610 *
1611 * Create an LLVM function type from the arguments.
1612 */
1613 static G_GNUC_UNUSED LLVMTypeRef
1614 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1615 int IsVarArg)
1616 {
1617 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1618 }
1619
1620 /*
1621 * LLVMFunctionType1:
1622 *
1623 * Create an LLVM function type from the arguments.
1624 */
1625 static G_GNUC_UNUSED LLVMTypeRef
1626 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1627 LLVMTypeRef ParamType1,
1628 int IsVarArg)
1629 {
1630 LLVMTypeRef param_types [1];
1631
1632 param_types [0] = ParamType1;
1633
1634 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1635 }
1636
1637 /*
1638 * LLVMFunctionType2:
1639 *
1640 * Create an LLVM function type from the arguments.
1641 */
1642 static G_GNUC_UNUSED LLVMTypeRef
1643 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1644 LLVMTypeRef ParamType1,
1645 LLVMTypeRef ParamType2,
1646 int IsVarArg)
1647 {
1648 LLVMTypeRef param_types [2];
1649
1650 param_types [0] = ParamType1;
1651 param_types [1] = ParamType2;
1652
1653 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1654 }
1655
1656 /*
1657 * LLVMFunctionType3:
1658 *
1659 * Create an LLVM function type from the arguments.
1660 */
1661 static G_GNUC_UNUSED LLVMTypeRef
1662 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1663 LLVMTypeRef ParamType1,
1664 LLVMTypeRef ParamType2,
1665 LLVMTypeRef ParamType3,
1666 int IsVarArg)
1667 {
1668 LLVMTypeRef param_types [3];
1669
1670 param_types [0] = ParamType1;
1671 param_types [1] = ParamType2;
1672 param_types [2] = ParamType3;
1673
1674 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1675 }
1676
1677 static G_GNUC_UNUSED LLVMTypeRef
1678 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1679 LLVMTypeRef ParamType1,
1680 LLVMTypeRef ParamType2,
1681 LLVMTypeRef ParamType3,
1682 LLVMTypeRef ParamType4,
1683 LLVMTypeRef ParamType5,
1684 int IsVarArg)
1685 {
1686 LLVMTypeRef param_types [5];
1687
1688 param_types [0] = ParamType1;
1689 param_types [1] = ParamType2;
1690 param_types [2] = ParamType3;
1691 param_types [3] = ParamType4;
1692 param_types [4] = ParamType5;
1693
1694 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1695 }
1696
1697 /*
1698 * create_builder:
1699 *
1700 * Create an LLVM builder and remember it so it can be freed later.
1701 */
1702 static LLVMBuilderRef
1703 create_builder (EmitContext *ctx)
1704 {
1705 LLVMBuilderRef builder = LLVMCreateBuilder ();
1706
1707 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1708
1709 emit_default_dbg_loc (ctx, builder);
1710
1711 return builder;
1712 }
1713
1714 static char*
1715 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1716 {
1717 char *name;
1718
1719 switch (type) {
1720 case MONO_PATCH_INFO_JIT_ICALL_ID:
1721 name = g_strdup_printf ("jit_icall_%s", mono_find_jit_icall_info ((MonoJitICallId)(gsize)data)->name);
1722 break;
1723 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1724 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1725 name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
1726 break;
1727 }
1728 default:
1729 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1730 break;
1731 }
1732
1733 return name;
1734 }
1735
1736 static int
1737 compute_aot_got_offset (MonoLLVMModule *module, MonoJumpInfo *ji, LLVMTypeRef llvm_type)
1738 {
1739 guint32 got_offset = mono_aot_get_got_offset (ji);
1740
1741 LLVMTypeRef lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (got_offset));
1742
1743 if (!lookup_type) {
1744 lookup_type = llvm_type;
1745 } else if (llvm_type != lookup_type) {
1746 lookup_type = module->ptr_type;
1747 } else {
1748 return got_offset;
1749 }
1750
1751 g_hash_table_insert (module->got_idx_to_type, GINT_TO_POINTER (got_offset), lookup_type);
1752 return got_offset;
1753 }
1754
1755 static LLVMValueRef
1756 get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1757 {
1758 MonoCompile *cfg;
1759 guint32 got_offset;
1760 LLVMValueRef indexes [2];
1761 LLVMValueRef got_entry_addr, load;
1762 LLVMBuilderRef builder = ctx->builder;
1763 char *name = NULL;
1764
1765 cfg = ctx->cfg;
1766
1767 MonoJumpInfo tmp_ji;
1768 tmp_ji.type = type;
1769 tmp_ji.data.target = data;
1770
1771 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1772
1773 ji->next = cfg->patch_info;
1774 cfg->patch_info = ji;
1775
1776 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, llvm_type);
1777 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1778 /*
1779 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1780 * explicitly initialize it.
1781 */
1782 if (!mono_aot_is_shared_got_offset (got_offset)) {
1783 //mono_print_ji (ji);
1784 //printf ("\n");
1785 ctx->cfg->got_access_count ++;
1786 }
1787
1788 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1789 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1790 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1791
1792 name = get_aotconst_name (type, data, got_offset);
1793 if (llvm_type) {
1794 load = LLVMBuildLoad (builder, got_entry_addr, "");
1795 load = convert (ctx, load, llvm_type);
1796 LLVMSetValueName (load, name ? name : "");
1797 } else {
1798 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1799 }
1800 g_free (name);
1801 //set_invariant_load_flag (load);
1802
1803 return load;
1804 }
1805
1806 static LLVMValueRef
1807 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1808 {
1809 return get_aotconst_typed (ctx, type, data, NULL);
1810 }
1811
1812 typedef struct {
1813 MonoJumpInfo *ji;
1814 MonoMethod *method;
1815 LLVMValueRef load;
1816 LLVMTypeRef type;
1817 } CallSite;
1818
1819 static gboolean
1820 method_is_direct_callable (MonoMethod *method)
1821 {
1822 if (method->wrapper_type == MONO_WRAPPER_ALLOC)
1823 return TRUE;
1824 if (method->string_ctor)
1825 return FALSE;
1826 if (method->wrapper_type)
1827 return FALSE;
1828 if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL))
1829 return FALSE;
1830 /* Can't enable this as the callee might fail llvm compilation */
1831 /*
1832 if (!method->is_inflated && (mono_class_get_flags (method->klass) & TYPE_ATTRIBUTE_PUBLIC) && (method->flags & METHOD_ATTRIBUTE_PUBLIC))
1833 return TRUE;
1834 */
1835 return FALSE;
1836 }
1837
1838 static LLVMValueRef
1839 get_callee_llvmonly (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1840 {
1841 LLVMValueRef callee;
1842 char *callee_name = NULL;
1843
1844 if (ctx->module->static_link && ctx->module->assembly->image != mono_get_corlib ()) {
1845 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1846 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1847 g_assert (info);
1848 if (info->func != info->wrapper) {
1849 type = MONO_PATCH_INFO_METHOD;
1850 data = mono_icall_get_wrapper_method (info);
1851 callee_name = mono_aot_get_mangled_method_name ((MonoMethod*)data);
1852 }
1853 } else if (type == MONO_PATCH_INFO_METHOD) {
1854 MonoMethod *method = (MonoMethod*)data;
1855 if (m_class_get_image (method->klass) != ctx->module->assembly->image && method_is_direct_callable (method))
1856 callee_name = mono_aot_get_mangled_method_name (method);
1857 }
1858 }
1859
1860 if (!callee_name)
1861 callee_name = mono_aot_get_direct_call_symbol (type, data);
1862 if (callee_name) {
1863 /* Directly callable */
1864 // FIXME: Locking
1865 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1866 if (!callee) {
1867 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1868
1869 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1870
1871 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1872 } else {
1873 /* LLVMTypeRef's are uniqued */
1874 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
1875 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
1876
1877 g_free (callee_name);
1878 }
1879 return callee;
1880 }
1881
1882 /*
1883 * Change references to jit icalls to the icall wrappers when in corlib, so
1884 * they can be called directly.
1885 */
1886 if (ctx->module->assembly->image == mono_get_corlib () && type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1887 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1888
1889 if (info->func != info->wrapper) {
1890 type = MONO_PATCH_INFO_METHOD;
1891 data = mono_icall_get_wrapper_method (info);
1892 }
1893 }
1894
1895 /*
1896 * Instead of emitting an indirect call through a got slot, emit a placeholder, and
1897 * replace it with a direct call or an indirect call in mono_llvm_fixup_aot_module ()
1898 * after all methods have been emitted.
1899 */
1900 if (type == MONO_PATCH_INFO_METHOD) {
1901 MonoMethod *method = (MonoMethod*)data;
1902 if (m_class_get_image (method->klass)->assembly == ctx->module->assembly) {
1903 MonoJumpInfo tmp_ji;
1904 tmp_ji.type = type;
1905 tmp_ji.data.target = data;
1906
1907 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1908 ji->next = ctx->cfg->patch_info;
1909 ctx->cfg->patch_info = ji;
1910 LLVMTypeRef llvm_type = LLVMPointerType (llvm_sig, 0);
1911
1912 ctx->cfg->got_access_count ++;
1913
1914 CallSite *info = g_new0 (CallSite, 1);
1915 info->method = method;
1916 info->ji = ji;
1917 info->type = llvm_type;
1918
1919 /*
1920 * Emit a dummy load to represent the callee, and either replace it with
1921 * a reference to the llvm method for the callee, or from a load from the
1922 * GOT.
1923 */
1924 LLVMValueRef indexes [2];
1925 LLVMValueRef got_entry_addr, load;
1926
1927 LLVMBuilderRef builder = ctx->builder;
1928 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1929 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1930 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1931
1932 load = LLVMBuildLoad (builder, got_entry_addr, "");
1933 load = convert (ctx, load, llvm_type);
1934 info->load = load;
1935
1936 g_ptr_array_add (ctx->callsite_list, info);
1937
1938 return load;
1939 }
1940 }
1941
1942 /*
1943 * Calls are made through the GOT.
1944 */
1945 callee = get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
1946
1947 return callee;
1948 }
1949
1950 /*
1951 * get_callee:
1952 *
1953 * Return an llvm value representing the callee given by the arguments.
1954 */
1955 static LLVMValueRef
1956 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1957 {
1958 LLVMValueRef callee;
1959 char *callee_name;
1960 MonoJumpInfo *ji = NULL;
1961
1962 if (ctx->llvm_only)
1963 return get_callee_llvmonly (ctx, llvm_sig, type, data);
1964
1965 callee_name = mono_aot_get_plt_symbol (type, data);
1966 if (!callee_name)
1967 return NULL;
1968
1969 if (ctx->cfg->compile_aot)
1970 /* Add a patch so referenced wrappers can be compiled in full aot mode */
1971 mono_add_patch_info (ctx->cfg, 0, type, data);
1972
1973 // FIXME: Locking
1974 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
1975 if (!callee) {
1976 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1977
1978 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1979
1980 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
1981 }
1982
1983 if (ctx->cfg->compile_aot) {
1984 ji = g_new0 (MonoJumpInfo, 1);
1985 ji->type = type;
1986 ji->data.target = data;
1987
1988 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
1989 }
1990
1991 return callee;
1992 }
1993
1994 static LLVMValueRef
1995 get_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1996 {
1997 gpointer target;
1998
1999 // This won't be patched so compile the wrapper immediately
2000 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
2001 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
2002 target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
2003 } else {
2004 target = resolve_patch (ctx->cfg, type, data);
2005 }
2006
2007 LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
2008 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
2009 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
2010 LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
2011 return callee;
2012 }
2013
2014 static int
2015 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
2016 {
2017 MonoMethodHeader *header = cfg->header;
2018 MonoExceptionClause *clause;
2019 int i;
2020
2021 /* Directly */
2022 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
2023 return (bb->region >> 8) - 1;
2024
2025 /* Indirectly */
2026 for (i = 0; i < header->num_clauses; ++i) {
2027 clause = &header->clauses [i];
2028
2029 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
2030 return i;
2031 }
2032
2033 return -1;
2034 }
2035
2036 static MonoExceptionClause *
2037 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
2038 {
2039 if (bb == cfg->bb_init)
2040 return NULL;
2041 // Since they're sorted by nesting we just need
2042 // the first one that the bb is a member of
2043 for (int i = 0; i < cfg->header->num_clauses; i++) {
2044 MonoExceptionClause *curr = &cfg->header->clauses [i];
2045
2046 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
2047 return curr;
2048 }
2049
2050 return NULL;
2051 }
2052
2053 static void
2054 set_metadata_flag (LLVMValueRef v, const char *flag_name)
2055 {
2056 LLVMValueRef md_arg;
2057 int md_kind;
2058
2059 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2060 md_arg = LLVMMDString ("mono", 4);
2061 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2062 }
2063
2064 static void
2065 set_nonnull_load_flag (LLVMValueRef v)
2066 {
2067 LLVMValueRef md_arg;
2068 int md_kind;
2069 const char *flag_name;
2070
2071 flag_name = "nonnull";
2072 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2073 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2074 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2075 }
2076
2077 static void
2078 set_invariant_load_flag (LLVMValueRef v)
2079 {
2080 LLVMValueRef md_arg;
2081 int md_kind;
2082 const char *flag_name;
2083
2084 // FIXME: Cache this
2085 flag_name = "invariant.load";
2086 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2087 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2088 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2089 }
2090
2091 /*
2092 * emit_call:
2093 *
2094 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
2095 * a try region.
2096 */
2097 static LLVMValueRef
2098 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
2099 {
2100 MonoCompile *cfg = ctx->cfg;
2101 LLVMValueRef lcall = NULL;
2102 LLVMBuilderRef builder = *builder_ref;
2103 MonoExceptionClause *clause;
2104
2105 if (ctx->llvm_only) {
2106 clause = get_most_deep_clause (cfg, ctx, bb);
2107
2108 if (clause) {
2109 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2110
2111 /*
2112 * Have to use an invoke instead of a call, branching to the
2113 * handler bblock of the clause containing this bblock.
2114 */
2115 intptr_t key = CLAUSE_END(clause);
2116
2117 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
2118
2119 // FIXME: Find the one that has the lowest end bound for the right start address
2120 // FIXME: Finally + nesting
2121
2122 if (lpad_bb) {
2123 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
2124
2125 /* Use an invoke */
2126 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
2127
2128 builder = ctx->builder = create_builder (ctx);
2129 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2130
2131 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2132 }
2133 }
2134 } else {
2135 int clause_index = get_handler_clause (cfg, bb);
2136
2137 if (clause_index != -1) {
2138 MonoMethodHeader *header = cfg->header;
2139 MonoExceptionClause *ec = &header->clauses [clause_index];
2140 MonoBasicBlock *tblock;
2141 LLVMBasicBlockRef ex_bb, noex_bb;
2142
2143 /*
2144 * Have to use an invoke instead of a call, branching to the
2145 * handler bblock of the clause containing this bblock.
2146 */
2147
2148 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY || ec->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2149
2150 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
2151 g_assert (tblock);
2152
2153 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
2154
2155 ex_bb = get_bb (ctx, tblock);
2156
2157 noex_bb = gen_bb (ctx, "NOEX_BB");
2158
2159 /* Use an invoke */
2160 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
2161
2162 builder = ctx->builder = create_builder (ctx);
2163 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2164
2165 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2166 }
2167 }
2168
2169 if (!lcall) {
2170 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
2171 ctx->builder = builder;
2172 }
2173
2174 if (builder_ref)
2175 *builder_ref = ctx->builder;
2176
2177 return lcall;
2178 }
2179
2180 static LLVMValueRef
2181 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, LLVMValueRef base, const char *name, gboolean is_faulting, BarrierKind barrier)
2182 {
2183 LLVMValueRef res;
2184
2185 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
2186 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
2187 LLVMValueRef cmp;
2188
2189 cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
2190 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
2191 *builder_ref = ctx->builder;
2192 }
2193
2194 /*
2195 * We emit volatile loads for loads which can fault, because otherwise
2196 * LLVM will generate invalid code when encountering a load from a
2197 * NULL address.
2198 */
2199 if (barrier != LLVM_BARRIER_NONE)
2200 res = mono_llvm_build_atomic_load (*builder_ref, addr, name, is_faulting, size, barrier);
2201 else
2202 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting);
2203
2204 /* Mark it with a custom metadata */
2205 /*
2206 if (is_faulting)
2207 set_metadata_flag (res, "mono.faulting.load");
2208 */
2209
2210 return res;
2211 }
2212
2213 static void
2214 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, BarrierKind barrier)
2215 {
2216 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
2217 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
2218 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
2219 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
2220 *builder_ref = ctx->builder;
2221 }
2222
2223 if (barrier != LLVM_BARRIER_NONE)
2224 mono_llvm_build_aligned_store (*builder_ref, value, addr, barrier, size);
2225 else
2226 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
2227 }
2228
2229 static void
2230 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting)
2231 {
2232 emit_store_general (ctx, bb, builder_ref, size, value, addr, base, is_faulting, LLVM_BARRIER_NONE);
2233 }
2234
2235 /*
2236 * emit_cond_system_exception:
2237 *
2238 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2239 * Might set the ctx exception.
2240 */
2241 static void
2242 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
2243 {
2244 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2245 LLVMBuilderRef builder;
2246 MonoClass *exc_class;
2247 LLVMValueRef args [2];
2248 LLVMValueRef callee;
2249 gboolean no_pc = FALSE;
2250 static MonoClass *exc_classes [MONO_EXC_INTRINS_NUM];
2251
2252 if (IS_TARGET_AMD64)
2253 /* Some platforms don't require the pc argument */
2254 no_pc = TRUE;
2255
2256 int exc_id = mini_exception_id_by_name (exc_type);
2257 if (!exc_classes [exc_id])
2258 exc_classes [exc_id] = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
2259 exc_class = exc_classes [exc_id];
2260
2261 ex_bb = gen_bb (ctx, "EX_BB");
2262 if (ctx->llvm_only)
2263 ex2_bb = gen_bb (ctx, "EX2_BB");
2264 noex_bb = gen_bb (ctx, "NOEX_BB");
2265
2266 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2267
2268 /* Emit exception throwing code */
2269 ctx->builder = builder = create_builder (ctx);
2270 LLVMPositionBuilderAtEnd (builder, ex_bb);
2271
2272 if (ctx->cfg->llvm_only) {
2273 LLVMBuildBr (builder, ex2_bb);
2274
2275 ctx->builder = builder = create_builder (ctx);
2276 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2277
2278 if (exc_id == MONO_EXC_NULL_REF) {
2279 static LLVMTypeRef sig;
2280
2281 if (!sig)
2282 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
2283 /* Can't cache this */
2284 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
2285 emit_call (ctx, bb, &builder, callee, NULL, 0);
2286 } else {
2287 static LLVMTypeRef sig;
2288
2289 if (!sig)
2290 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2291 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_throw_corlib_exception));
2292 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2293 emit_call (ctx, bb, &builder, callee, args, 1);
2294 }
2295
2296 LLVMBuildUnreachable (builder);
2297
2298 ctx->builder = builder = create_builder (ctx);
2299 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2300
2301 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2302
2303 ctx->ex_index ++;
2304 return;
2305 }
2306
2307 callee = ctx->module->throw_corlib_exception;
2308 if (!callee) {
2309 LLVMTypeRef sig;
2310
2311 if (no_pc)
2312 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2313 else
2314 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2315
2316 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_throw_corlib_exception_abs_trampoline;
2317
2318 if (ctx->cfg->compile_aot) {
2319 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2320 } else {
2321 /*
2322 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2323 * - On x86, LLVM generated code doesn't push the arguments
2324 * - The trampoline takes the throw address as an arguments, not a pc offset.
2325 */
2326 callee = get_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2327
2328 /*
2329 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2330 * added by get_jit_callee ().
2331 */
2332 ex2_bb = gen_bb (ctx, "EX2_BB");
2333 LLVMBuildBr (builder, ex2_bb);
2334 ex_bb = ex2_bb;
2335
2336 ctx->builder = builder = create_builder (ctx);
2337 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2338 }
2339 }
2340
2341 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2342
2343 /*
2344 * The LLVM mono branch contains changes so a block address can be passed as an
2345 * argument to a call.
2346 */
2347 if (no_pc) {
2348 emit_call (ctx, bb, &builder, callee, args, 1);
2349 } else {
2350 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2351 emit_call (ctx, bb, &builder, callee, args, 2);
2352 }
2353
2354 LLVMBuildUnreachable (builder);
2355
2356 ctx->builder = builder = create_builder (ctx);
2357 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2358
2359 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2360
2361 ctx->ex_index ++;
2362 return;
2363 }
2364
2365 /*
2366 * emit_args_to_vtype:
2367 *
2368 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2369 */
2370 static void
2371 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2372 {
2373 int j, size, nslots;
2374 MonoClass *klass;
2375
2376 t = mini_get_underlying_type (t);
2377 klass = mono_class_from_mono_type_internal (t);
2378 size = mono_class_value_size (klass, NULL);
2379
2380 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
2381 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2382
2383 if (ainfo->storage == LLVMArgAsFpArgs)
2384 nslots = ainfo->nslots;
2385 else
2386 nslots = 2;
2387
2388 for (j = 0; j < nslots; ++j) {
2389 LLVMValueRef index [2], addr, daddr;
2390 int part_size = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2391 LLVMTypeRef part_type;
2392
2393 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2394 part_size ++;
2395
2396 if (ainfo->pair_storage [j] == LLVMArgNone)
2397 continue;
2398
2399 switch (ainfo->pair_storage [j]) {
2400 case LLVMArgInIReg: {
2401 part_type = LLVMIntType (part_size * 8);
2402 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass)) {
2403 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2404 addr = LLVMBuildGEP (builder, address, index, 1, "");
2405 } else {
2406 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2407 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2408 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2409 }
2410 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2411 break;
2412 }
2413 case LLVMArgInFPReg: {
2414 LLVMTypeRef arg_type;
2415
2416 if (ainfo->esize == 8)
2417 arg_type = LLVMDoubleType ();
2418 else
2419 arg_type = LLVMFloatType ();
2420
2421 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2422 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2423 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2424 LLVMBuildStore (builder, args [j], addr);
2425 break;
2426 }
2427 case LLVMArgNone:
2428 break;
2429 default:
2430 g_assert_not_reached ();
2431 }
2432
2433 size -= TARGET_SIZEOF_VOID_P;
2434 }
2435 }
2436
2437 /*
2438 * emit_vtype_to_args:
2439 *
2440 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2441 * into ARGS, and the number of arguments into NARGS.
2442 */
2443 static void
2444 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2445 {
2446 int pindex = 0;
2447 int j, size, nslots;
2448 LLVMTypeRef arg_type;
2449
2450 t = mini_get_underlying_type (t);
2451 size = get_vtype_size (t);
2452
2453 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t)))
2454 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2455
2456 if (ainfo->storage == LLVMArgAsFpArgs)
2457 nslots = ainfo->nslots;
2458 else
2459 nslots = 2;
2460 for (j = 0; j < nslots; ++j) {
2461 LLVMValueRef index [2], addr, daddr;
2462 int partsize = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2463
2464 if (ainfo->pair_storage [j] == LLVMArgNone)
2465 continue;
2466
2467 switch (ainfo->pair_storage [j]) {
2468 case LLVMArgInIReg:
2469 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
2470 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2471 addr = LLVMBuildGEP (builder, address, index, 1, "");
2472 } else {
2473 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2474 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2475 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2476 }
2477 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2478 break;
2479 case LLVMArgInFPReg:
2480 if (ainfo->esize == 8)
2481 arg_type = LLVMDoubleType ();
2482 else
2483 arg_type = LLVMFloatType ();
2484 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2485 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2486 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2487 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2488 break;
2489 case LLVMArgNone:
2490 break;
2491 default:
2492 g_assert_not_reached ();
2493 }
2494 size -= TARGET_SIZEOF_VOID_P;
2495 }
2496
2497 *nargs = pindex;
2498 }
2499
2500 static LLVMValueRef
2501 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2502 {
2503 /*
2504 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2505 * get executed every time control reaches them.
2506 */
2507 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2508
2509 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2510 return ctx->last_alloca;
2511 }
2512
2513 static LLVMValueRef
2514 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2515 {
2516 return build_alloca_llvm_type_name (ctx, t, align, "");
2517 }
2518
2519 static LLVMValueRef
2520 build_alloca (EmitContext *ctx, MonoType *t)
2521 {
2522 MonoClass *k = mono_class_from_mono_type_internal (t);
2523 int align;
2524
2525 g_assert (!mini_is_gsharedvt_variable_type (t));
2526
2527 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2528 align = 16;
2529 else
2530 align = mono_class_min_align (k);
2531
2532 /* Sometimes align is not a power of 2 */
2533 while (mono_is_power_of_two (align) == -1)
2534 align ++;
2535
2536 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2537 }
2538
2539 static LLVMValueRef
2540 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2541 {
2542 /*
2543 * gsharedvt local.
2544 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2545 */
2546 MonoCompile *cfg = ctx->cfg;
2547 LLVMBuilderRef builder = ctx->builder;
2548 LLVMValueRef offset, offset_var;
2549 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2550 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2551 LLVMValueRef ptr;
2552 char *name;
2553
2554 g_assert (info_var);
2555 g_assert (locals_var);
2556
2557 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2558
2559 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * TARGET_SIZEOF_VOID_P), FALSE);
2560 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2561
2562 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2563 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2564
2565 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2566 }
2567
2568 /*
2569 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2570 */
2571 static void
2572 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2573 {
2574 if (!module->used)
2575 module->used = g_ptr_array_sized_new (16);
2576 g_ptr_array_add (module->used, global);
2577 }
2578
2579 static void
2580 emit_llvm_used (MonoLLVMModule *module)
2581 {
2582 LLVMModuleRef lmodule = module->lmodule;
2583 LLVMTypeRef used_type;
2584 LLVMValueRef used, *used_elem;
2585 int i;
2586
2587 if (!module->used)
2588 return;
2589
2590 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2591 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2592 used_elem = g_new0 (LLVMValueRef, module->used->len);
2593 for (i = 0; i < module->used->len; ++i)
2594 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2595 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2596 LLVMSetLinkage (used, LLVMAppendingLinkage);
2597 LLVMSetSection (used, "llvm.metadata");
2598 }
2599
2600 /*
2601 * emit_get_method:
2602 *
2603 * Emit a function mapping method indexes to their code
2604 */
2605 static void
2606 emit_get_method (MonoLLVMModule *module)
2607 {
2608 LLVMModuleRef lmodule = module->lmodule;
2609 LLVMValueRef func, switch_ins, m;
2610 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2611 LLVMBasicBlockRef *bbs = NULL;
2612 LLVMTypeRef rtype;
2613 LLVMBuilderRef builder = LLVMCreateBuilder ();
2614 LLVMValueRef table = NULL;
2615 char *name;
2616 int i;
2617 gboolean emit_table = FALSE;
2618
2619 #ifdef TARGET_WASM
2620 /*
2621 * Emit a table of functions instead of a switch statement,
2622 * its very efficient on wasm. This might be usable on
2623 * other platforms too.
2624 */
2625 emit_table = TRUE;
2626 #endif
2627
2628 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2629
2630 if (emit_table) {
2631 LLVMTypeRef table_type;
2632 LLVMValueRef *table_elems;
2633 char *table_name;
2634
2635 int table_len = module->max_method_idx + 1;
2636 table_type = LLVMArrayType (rtype, table_len);
2637 table_name = g_strdup_printf ("%s_method_table", module->assembly->aname.name);
2638 table = LLVMAddGlobal (lmodule, table_type, table_name);
2639 table_elems = g_new0 (LLVMValueRef, table_len);
2640 for (i = 0; i < table_len; ++i) {
2641 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2642 if (m)
2643 table_elems [i] = LLVMBuildBitCast (builder, m, rtype, "");
2644 else
2645 table_elems [i] = LLVMConstNull (rtype);
2646 }
2647 LLVMSetInitializer (table, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), table_elems, table_len));
2648 }
2649
2650 /*
2651 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2652 * but generating code seems safer.
2653 */
2654 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2655 LLVMSetLinkage (func, LLVMExternalLinkage);
2656 LLVMSetVisibility (func, LLVMHiddenVisibility);
2657 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2658 module->get_method = func;
2659
2660 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2661
2662 /*
2663 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2664 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2665 * then we will have to find another solution.
2666 */
2667
2668 name = g_strdup_printf ("BB_CODE_START");
2669 code_start_bb = LLVMAppendBasicBlock (func, name);
2670 g_free (name);
2671 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2672 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2673
2674 name = g_strdup_printf ("BB_CODE_END");
2675 code_end_bb = LLVMAppendBasicBlock (func, name);
2676 g_free (name);
2677 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2678 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2679
2680 if (emit_table) {
2681 /*
2682 * switch (index) {
2683 * case -1: return code_start;
2684 * case -2: return code_end;
2685 * default: return method_table [index];
2686 */
2687 LLVMBasicBlockRef default_bb = LLVMAppendBasicBlock (func, "DEFAULT");
2688 LLVMPositionBuilderAtEnd (builder, default_bb);
2689 LLVMValueRef base = table;
2690 LLVMValueRef indexes [2];
2691 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2692 indexes [1] = LLVMGetParam (func, 0);
2693 LLVMValueRef addr = LLVMBuildGEP (builder, base, indexes, 2, "");
2694 LLVMValueRef res = mono_llvm_build_load (builder, addr, "", FALSE);
2695 LLVMBuildRet (builder, res);
2696
2697 LLVMPositionBuilderAtEnd (builder, entry_bb);
2698
2699 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), default_bb, 0);
2700 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2701 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2702 } else {
2703 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2704 for (i = 0; i < module->max_method_idx + 1; ++i) {
2705 name = g_strdup_printf ("BB_%d", i);
2706 bb = LLVMAppendBasicBlock (func, name);
2707 g_free (name);
2708 bbs [i] = bb;
2709
2710 LLVMPositionBuilderAtEnd (builder, bb);
2711
2712 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2713 if (m)
2714 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2715 else
2716 LLVMBuildRet (builder, LLVMConstNull (rtype));
2717 }
2718
2719 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2720 LLVMPositionBuilderAtEnd (builder, fail_bb);
2721 LLVMBuildRet (builder, LLVMConstNull (rtype));
2722
2723 LLVMPositionBuilderAtEnd (builder, entry_bb);
2724
2725 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2726 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2727 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2728 for (i = 0; i < module->max_method_idx + 1; ++i) {
2729 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2730 }
2731 }
2732
2733 mark_as_used (module, func);
2734
2735 LLVMDisposeBuilder (builder);
2736 }
2737
2738 /*
2739 * emit_get_unbox_tramp:
2740 *
2741 * Emit a function mapping method indexes to their unbox trampoline
2742 */
2743 static void
2744 emit_get_unbox_tramp (MonoLLVMModule *module)
2745 {
2746 LLVMModuleRef lmodule = module->lmodule;
2747 LLVMValueRef func, switch_ins, m;
2748 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2749 LLVMBasicBlockRef *bbs;
2750 LLVMTypeRef rtype;
2751 LLVMBuilderRef builder = LLVMCreateBuilder ();
2752 char *name;
2753 int i;
2754 gboolean emit_table = FALSE;
2755
2756 /* Similar to emit_get_method () */
2757
2758 #ifndef TARGET_WATCHOS
2759 emit_table = TRUE;
2760 #endif
2761
2762 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2763
2764 if (emit_table) {
2765 // About 10% of methods have an unbox tramp, so emit a table of indexes for them
2766 // that the runtime can search using a binary search
2767 int len = 0;
2768 for (i = 0; i < module->max_method_idx + 1; ++i) {
2769 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2770 if (m)
2771 len ++;
2772 }
2773
2774 LLVMTypeRef table_type, elemtype;
2775 LLVMValueRef *table_elems;
2776 LLVMValueRef table;
2777 char *table_name;
2778 int table_len;
2779 int elemsize;
2780
2781 table_len = len;
2782 elemsize = module->max_method_idx < 65000 ? 2 : 4;
2783
2784 // The index table
2785 elemtype = elemsize == 2 ? LLVMInt16Type () : LLVMInt32Type ();
2786 table_type = LLVMArrayType (elemtype, table_len);
2787 table_name = g_strdup_printf ("%s_unbox_tramp_indexes", module->assembly->aname.name);
2788 table = LLVMAddGlobal (lmodule, table_type, table_name);
2789 table_elems = g_new0 (LLVMValueRef, table_len);
2790 int idx = 0;
2791 for (i = 0; i < module->max_method_idx + 1; ++i) {
2792 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2793 if (m)
2794 table_elems [idx ++] = LLVMConstInt (elemtype, i, FALSE);
2795 }
2796 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2797 module->unbox_tramp_indexes = table;
2798
2799 // The trampoline table
2800 elemtype = rtype;
2801 table_type = LLVMArrayType (elemtype, table_len);
2802 table_name = g_strdup_printf ("%s_unbox_trampolines", module->assembly->aname.name);
2803 table = LLVMAddGlobal (lmodule, table_type, table_name);
2804 table_elems = g_new0 (LLVMValueRef, table_len);
2805 idx = 0;
2806 for (i = 0; i < module->max_method_idx + 1; ++i) {
2807 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2808 if (m)
2809 table_elems [idx ++] = LLVMBuildBitCast (builder, m, rtype, "");
2810 }
2811 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2812 module->unbox_trampolines = table;
2813
2814 module->unbox_tramp_num = table_len;
2815 module->unbox_tramp_elemsize = elemsize;
2816 return;
2817 }
2818
2819 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2820 LLVMSetLinkage (func, LLVMExternalLinkage);
2821 LLVMSetVisibility (func, LLVMHiddenVisibility);
2822 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2823 module->get_unbox_tramp = func;
2824
2825 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2826
2827 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2828 for (i = 0; i < module->max_method_idx + 1; ++i) {
2829 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2830 if (!m)
2831 continue;
2832
2833 name = g_strdup_printf ("BB_%d", i);
2834 bb = LLVMAppendBasicBlock (func, name);
2835 g_free (name);
2836 bbs [i] = bb;
2837
2838 LLVMPositionBuilderAtEnd (builder, bb);
2839
2840 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2841 }
2842
2843 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2844 LLVMPositionBuilderAtEnd (builder, fail_bb);
2845 LLVMBuildRet (builder, LLVMConstNull (rtype));
2846
2847 LLVMPositionBuilderAtEnd (builder, entry_bb);
2848
2849 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2850 for (i = 0; i < module->max_method_idx + 1; ++i) {
2851 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2852 if (!m)
2853 continue;
2854
2855 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2856 }
2857
2858 mark_as_used (module, func);
2859 LLVMDisposeBuilder (builder);
2860 }
2861
2862 /* Add a function to mark the beginning of LLVM code */
2863 static void
2864 emit_llvm_code_start (MonoLLVMModule *module)
2865 {
2866 LLVMModuleRef lmodule = module->lmodule;
2867 LLVMValueRef func;
2868 LLVMBasicBlockRef entry_bb;
2869 LLVMBuilderRef builder;
2870
2871 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2872 LLVMSetLinkage (func, LLVMInternalLinkage);
2873 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2874 module->code_start = func;
2875 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2876 builder = LLVMCreateBuilder ();
2877 LLVMPositionBuilderAtEnd (builder, entry_bb);
2878 LLVMBuildRetVoid (builder);
2879 LLVMDisposeBuilder (builder);
2880 }
2881
2882 static LLVMValueRef
2883 emit_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
2884 {
2885 LLVMModuleRef lmodule = module->lmodule;
2886 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
2887 LLVMBasicBlockRef entry_bb;
2888 LLVMBuilderRef builder;
2889 LLVMTypeRef sig;
2890 MonoJumpInfo *ji;
2891 int got_offset;
2892 const char *wrapper_name = mono_marshal_get_aot_init_wrapper_name (subtype);
2893 char *name = g_strdup_printf ("%s%s", module->global_prefix, wrapper_name);
2894 MonoJitICallId icall_id = MONO_JIT_ICALL_ZeroIsReserved;
2895
2896 switch (subtype) {
2897 case AOT_INIT_METHOD:
2898 icall_id = MONO_JIT_ICALL_mini_llvm_init_method;
2899 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
2900 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
2901 break;
2902 case AOT_INIT_METHOD_GSHARED_MRGCTX:
2903 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_mrgctx; // Deliberate fall-through
2904 case AOT_INIT_METHOD_GSHARED_VTABLE:
2905 /* mrgctx/vtable */
2906 if (!icall_id)
2907 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_vtable;
2908 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
2909 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
2910 break;
2911 case AOT_INIT_METHOD_GSHARED_THIS:
2912 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_this;
2913 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
2914 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
2915 break;
2916 default:
2917 g_assert_not_reached ();
2918 }
2919
2920 g_assert (icall_id);
2921 LLVMSetLinkage (func, LLVMInternalLinkage);
2922
2923 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
2924
2925 // FIXME? Using this with mono debug info causes unwind.c to explode when
2926 // parsing some of these registers saved by this call. Can't unwind through it.
2927 // Not an issue with llvmonly because it doesn't use that DWARF
2928 if (module->llvm_only)
2929 set_preserveall_cc (func);
2930 else
2931 LLVMSetFunctionCallConv (func, LLVMMono1CallConv);
2932
2933 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2934 builder = LLVMCreateBuilder ();
2935 LLVMPositionBuilderAtEnd (builder, entry_bb);
2936
2937 /* get_aotconst */
2938 ji = g_new0 (MonoJumpInfo, 1);
2939 ji->type = MONO_PATCH_INFO_AOT_MODULE;
2940 ji = mono_aot_patch_info_dup (ji);
2941 got_offset = compute_aot_got_offset (module, ji, IntPtrType ());
2942 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2943 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2944 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2945 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2946 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
2947 args [1] = LLVMGetParam (func, 0);
2948 if (subtype)
2949 args [2] = LLVMGetParam (func, 1);
2950
2951 ji = g_new0 (MonoJumpInfo, 1);
2952 ji->type = MONO_PATCH_INFO_JIT_ICALL_ID;
2953 ji->data.jit_icall_id = icall_id;
2954 ji = mono_aot_patch_info_dup (ji);
2955 got_offset = compute_aot_got_offset (module, ji, sig);
2956 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2957 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2958 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2959 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2960 callee = LLVMBuildLoad (builder, got_entry_addr, "");
2961 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
2962 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
2963
2964 // Set the inited flag
2965 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2966 indexes [1] = LLVMGetParam (func, 0);
2967 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
2968
2969 LLVMBuildRetVoid (builder);
2970
2971 LLVMVerifyFunction(func, LLVMAbortProcessAction);
2972 LLVMDisposeBuilder (builder);
2973 return func;
2974 }
2975
2976 /*
2977 * Emit wrappers around the C icalls used to initialize llvm methods, to
2978 * make the calling code smaller and to enable usage of the llvm
2979 * PreserveAll calling convention.
2980 */
2981 static void
2982 emit_init_icall_wrappers (MonoLLVMModule *module)
2983 {
2984 module->init_method = emit_init_icall_wrapper (module, AOT_INIT_METHOD);
2985 module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_MRGCTX);
2986 module->init_method_gshared_this = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_THIS);
2987 module->init_method_gshared_vtable = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_VTABLE);
2988 }
2989
2990 static LLVMValueRef
2991 get_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
2992 {
2993 switch (subtype) {
2994 case AOT_INIT_METHOD:
2995 return module->init_method;
2996 case AOT_INIT_METHOD_GSHARED_MRGCTX:
2997 return module->init_method_gshared_mrgctx;
2998 case AOT_INIT_METHOD_GSHARED_THIS:
2999 return module->init_method_gshared_this;
3000 case AOT_INIT_METHOD_GSHARED_VTABLE:
3001 return module->init_method_gshared_vtable;
3002 default:
3003 g_assert_not_reached ();
3004 }
3005 }
3006
3007 static void
3008 emit_gc_safepoint_poll (MonoLLVMModule *module)
3009 {
3010 LLVMModuleRef lmodule = module->lmodule;
3011 LLVMValueRef func, indexes [2], got_entry_addr, flag_addr, val_ptr, callee, val, cmp;
3012 LLVMBasicBlockRef entry_bb, poll_bb, exit_bb;
3013 LLVMBuilderRef builder;
3014 LLVMTypeRef sig;
3015 MonoJumpInfo *ji;
3016 int got_offset;
3017
3018 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3019 func = mono_llvm_get_or_insert_gc_safepoint_poll (lmodule);
3020 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3021 LLVMSetLinkage (func, LLVMWeakODRLinkage);
3022 // set_preserveall_cc (func);
3023
3024 entry_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.entry");
3025 poll_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.poll");
3026 exit_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.exit");
3027
3028 builder = LLVMCreateBuilder ();
3029
3030 /* entry: */
3031 LLVMPositionBuilderAtEnd (builder, entry_bb);
3032
3033 /* get_aotconst */
3034 ji = g_new0 (MonoJumpInfo, 1);
3035 ji->type = MONO_PATCH_INFO_GC_SAFE_POINT_FLAG;
3036 ji = mono_aot_patch_info_dup (ji);
3037 got_offset = compute_aot_got_offset (module, ji, IntPtrType ());
3038 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3039 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3040 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3041 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3042 flag_addr = LLVMBuildLoad (builder, got_entry_addr, "");
3043 val_ptr = LLVMBuildLoad (builder, flag_addr, "");
3044 val = LLVMBuildPtrToInt (builder, val_ptr, IntPtrType (), "");
3045 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
3046 LLVMBuildCondBr (builder, cmp, exit_bb, poll_bb);
3047
3048 /* poll: */
3049 LLVMPositionBuilderAtEnd(builder, poll_bb);
3050
3051 ji = g_new0 (MonoJumpInfo, 1);
3052 ji->type = MONO_PATCH_INFO_JIT_ICALL_ID;
3053 ji->data.jit_icall_id = MONO_JIT_ICALL_mono_threads_state_poll;
3054 ji = mono_aot_patch_info_dup (ji);
3055 got_offset = compute_aot_got_offset (module, ji, sig);
3056 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3057 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3058 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3059 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3060 callee = LLVMBuildLoad (builder, got_entry_addr, "");
3061 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
3062 LLVMBuildCall (builder, callee, NULL, 0, "");
3063 LLVMBuildBr(builder, exit_bb);
3064
3065 /* exit: */
3066 LLVMPositionBuilderAtEnd(builder, exit_bb);
3067
3068 LLVMBuildRetVoid (builder);
3069
3070 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3071 LLVMDisposeBuilder (builder);
3072 }
3073
3074 static void
3075 emit_llvm_code_end (MonoLLVMModule *module)
3076 {
3077 LLVMModuleRef lmodule = module->lmodule;
3078 LLVMValueRef func;
3079 LLVMBasicBlockRef entry_bb;
3080 LLVMBuilderRef builder;
3081
3082 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3083 LLVMSetLinkage (func, LLVMInternalLinkage);
3084 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3085 module->code_end = func;
3086 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3087 builder = LLVMCreateBuilder ();
3088 LLVMPositionBuilderAtEnd (builder, entry_bb);
3089 LLVMBuildRetVoid (builder);
3090 LLVMDisposeBuilder (builder);
3091 }
3092
3093 static void
3094 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
3095 {
3096 gboolean need_div_check = ctx->cfg->backend->need_div_check;
3097
3098 if (bb->region)
3099 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
3100 need_div_check = TRUE;
3101
3102 if (!need_div_check)
3103 return;
3104
3105 switch (ins->opcode) {
3106 case OP_IDIV:
3107 case OP_LDIV:
3108 case OP_IREM:
3109 case OP_LREM:
3110 case OP_IDIV_UN:
3111 case OP_LDIV_UN:
3112 case OP_IREM_UN:
3113 case OP_LREM_UN:
3114 case OP_IDIV_IMM:
3115 case OP_LDIV_IMM:
3116 case OP_IREM_IMM:
3117 case OP_LREM_IMM:
3118 case OP_IDIV_UN_IMM:
3119 case OP_LDIV_UN_IMM:
3120 case OP_IREM_UN_IMM:
3121 case OP_LREM_UN_IMM: {
3122 LLVMValueRef cmp;
3123 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
3124 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
3125
3126 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
3127 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
3128 if (!ctx_ok (ctx))
3129 break;
3130 builder = ctx->builder;
3131
3132 /* b == -1 && a == 0x80000000 */
3133 if (is_signed) {
3134 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
3135 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
3136 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
3137
3138 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
3139 emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
3140 if (!ctx_ok (ctx))
3141 break;
3142 builder = ctx->builder;
3143 }
3144 break;
3145 }
3146 default:
3147 break;
3148 }
3149 }
3150
3151 /*
3152 * emit_init_method:
3153 *
3154 * Emit code to initialize the GOT slots used by the method.
3155 */
3156 static void
3157 emit_init_method (EmitContext *ctx)
3158 {
3159 LLVMValueRef indexes [16], args [16], callee;
3160 LLVMValueRef inited_var, cmp, call;
3161 LLVMBasicBlockRef inited_bb, notinited_bb;
3162 LLVMBuilderRef builder = ctx->builder;
3163 MonoCompile *cfg = ctx->cfg;
3164
3165 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
3166
3167 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3168 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
3169 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
3170
3171 //WASM doesn't support the "llvm.expect.i8" intrinsic
3172 #ifndef TARGET_WASM
3173 args [0] = inited_var;
3174 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
3175 inited_var = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I8), args, 2, "");
3176 #endif
3177
3178 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
3179
3180 inited_bb = ctx->inited_bb;
3181 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
3182
3183 ctx->cfg->llvmonly_init_cond = LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
3184
3185 builder = ctx->builder = create_builder (ctx);
3186 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
3187
3188 // FIXME: Cache
3189 if (ctx->rgctx_arg && ((cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) ||
3190 mini_method_is_default_method (cfg->method))) {
3191 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3192 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3193 callee = ctx->module->init_method_gshared_mrgctx;
3194 call = LLVMBuildCall (builder, callee, args, 2, "");
3195 } else if (ctx->rgctx_arg) {
3196 /* A vtable is passed as the rgctx argument */
3197 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3198 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3199 callee = ctx->module->init_method_gshared_vtable;
3200 call = LLVMBuildCall (builder, callee, args, 2, "");
3201 } else if (cfg->gshared) {
3202 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3203 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
3204 callee = ctx->module->init_method_gshared_this;
3205 call = LLVMBuildCall (builder, callee, args, 2, "");
3206 } else {
3207 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3208 callee = ctx->module->init_method;
3209 call = LLVMBuildCall (builder, callee, args, 1, "");
3210 }
3211
3212 /*
3213 * This enables llvm to keep arguments in their original registers/
3214 * scratch registers, since the call will not clobber them.
3215 */
3216
3217 if (ctx->llvm_only)
3218 set_call_preserveall_cc (call);
3219 else
3220 LLVMSetInstructionCallConv (call, LLVMMono1CallConv);
3221
3222 LLVMBuildBr (builder, inited_bb);
3223 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
3224
3225 builder = ctx->builder = create_builder (ctx);
3226 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
3227 }
3228
3229 static void
3230 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
3231 {
3232 /*
3233 * Emit unbox trampoline using a tailcall
3234 */
3235 LLVMValueRef tramp, call, *args;
3236 LLVMBuilderRef builder;
3237 LLVMBasicBlockRef lbb;
3238 LLVMCallInfo *linfo;
3239 char *tramp_name;
3240 int i, nargs;
3241
3242 tramp_name = g_strdup_printf ("ut_%s", method_name);
3243 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
3244 LLVMSetLinkage (tramp, LLVMInternalLinkage);
3245 mono_llvm_add_func_attr (tramp, LLVM_ATTR_OPTIMIZE_FOR_SIZE);
3246 //mono_llvm_add_func_attr (tramp, LLVM_ATTR_NO_UNWIND);
3247 linfo = ctx->linfo;
3248 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
3249 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3250 mono_llvm_add_param_attr (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVM_ATTR_IN_REG);
3251 if (ctx->cfg->vret_addr) {
3252 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
3253 if (linfo->ret.storage == LLVMArgVtypeByRef) {
3254 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
3255 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
3256 }
3257 }
3258
3259 lbb = LLVMAppendBasicBlock (tramp, "");
3260 builder = LLVMCreateBuilder ();
3261 LLVMPositionBuilderAtEnd (builder, lbb);
3262
3263 nargs = LLVMCountParamTypes (method_type);
3264 args = g_new0 (LLVMValueRef, nargs);
3265 for (i = 0; i < nargs; ++i) {
3266 args [i] = LLVMGetParam (tramp, i);
3267 if (i == ctx->this_arg_pindex) {
3268 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
3269
3270 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
3271 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), MONO_ABI_SIZEOF (MonoObject), FALSE), "");
3272 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
3273 }
3274 }
3275 call = LLVMBuildCall (builder, method, args, nargs, "");
3276 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3277 mono_llvm_add_instr_attr (call, 1 + ctx->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
3278 if (linfo->ret.storage == LLVMArgVtypeByRef)
3279 mono_llvm_add_instr_attr (call, 1 + linfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
3280
3281 // FIXME: This causes assertions in clang
3282 //mono_llvm_set_must_tailcall (call);
3283 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
3284 LLVMBuildRetVoid (builder);
3285 else
3286 LLVMBuildRet (builder, call);
3287
3288 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
3289 LLVMDisposeBuilder (builder);
3290 }
3291
3292 /*
3293 * emit_entry_bb:
3294 *
3295 * Emit code to load/convert arguments.
3296 */
3297 static void
3298 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
3299 {
3300 int i, j, pindex;
3301 MonoCompile *cfg = ctx->cfg;
3302 MonoMethodSignature *sig = ctx->sig;
3303 LLVMCallInfo *linfo = ctx->linfo;
3304 MonoBasicBlock *bb;
3305 char **names;
3306
3307 LLVMBuilderRef old_builder = ctx->builder;
3308 ctx->builder = builder;
3309
3310 ctx->alloca_builder = create_builder (ctx);
3311
3312 /*
3313 * Handle indirect/volatile variables by allocating memory for them
3314 * using 'alloca', and storing their address in a temporary.
3315 */
3316 for (i = 0; i < cfg->num_varinfo; ++i) {
3317 MonoInst *var = cfg->varinfo [i];
3318 LLVMTypeRef vtype;
3319
3320 if ((var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET))
3321 continue;
3322
3323 #ifdef TARGET_WASM
3324 // For GC stack scanning to work, have to spill all reference variables to the stack
3325 // Some ref variables have type intptr
3326 if (MONO_TYPE_IS_REFERENCE (var->inst_vtype) || var->inst_vtype->type == MONO_TYPE_I)
3327 var->flags |= MONO_INST_INDIRECT;
3328 #endif
3329
3330 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
3331 vtype = type_to_llvm_type (ctx, var->inst_vtype);
3332 if (!ctx_ok (ctx))
3333 return;
3334 /* Could be already created by an OP_VPHI */
3335 if (!ctx->addresses [var->dreg]) {
3336 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
3337 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
3338 }
3339 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
3340 }
3341 }
3342
3343 names = g_new (char *, sig->param_count);
3344 mono_method_get_param_names (cfg->method, (const char **) names);
3345
3346 for (i = 0; i < sig->param_count; ++i) {
3347 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
3348 int reg = cfg->args [i + sig->hasthis]->dreg;
3349 char *name;
3350
3351 pindex = ainfo->pindex;
3352
3353 switch (ainfo->storage) {
3354 case LLVMArgVtypeInReg:
3355 case LLVMArgAsFpArgs: {
3356 LLVMValueRef args [8];
3357 int j;
3358
3359 pindex += ainfo->ndummy_fpargs;
3360
3361 /* The argument is received as a set of int/fp arguments, store them into the real argument */
3362 memset (args, 0, sizeof (args));
3363 if (ainfo->storage == LLVMArgVtypeInReg) {
3364 args [0] = LLVMGetParam (ctx->lmethod, pindex);
3365 if (ainfo->pair_storage [1] != LLVMArgNone)
3366 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
3367 } else {
3368 g_assert (ainfo->nslots <= 8);
3369 for (j = 0; j < ainfo->nslots; ++j)
3370 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
3371 }
3372 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
3373
3374 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
3375
3376 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
3377 /* Treat these as normal values */
3378 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3379 }
3380 break;
3381 }
3382 case LLVMArgVtypeByVal: {
3383 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3384
3385 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
3386 /* Treat these as normal values */
3387 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3388 }
3389 break;
3390 }
3391 case LLVMArgVtypeByRef: {
3392 /* The argument is passed by ref */
3393 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3394 break;
3395 }
3396 case LLVMArgAsIArgs: {
3397 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3398 int size;
3399 MonoType *t = mini_get_underlying_type (ainfo->type);
3400
3401 /* The argument is received as an array of ints, store it into the real argument */
3402 ctx->addresses [reg] = build_alloca (ctx, t);
3403
3404 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
3405 if (size == 0) {
3406 } else if (size < TARGET_SIZEOF_VOID_P) {
3407 /* The upper bits of the registers might not be valid */
3408 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
3409 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
3410 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
3411 } else {
3412 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
3413 }
3414 break;
3415 }
3416 case LLVMArgVtypeAsScalar:
3417 g_assert_not_reached ();
3418 break;
3419 case LLVMArgGsharedvtFixed: {
3420 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
3421 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3422
3423 if (names [i])
3424 name = g_strdup_printf ("arg_%s", names [i]);
3425 else
3426 name = g_strdup_printf ("arg_%d", i);
3427
3428 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
3429 break;
3430 }
3431 case LLVMArgGsharedvtFixedVtype: {
3432 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3433
3434 if (names [i])
3435 name = g_strdup_printf ("vtype_arg_%s", names [i]);
3436 else
3437 name = g_strdup_printf ("vtype_arg_%d", i);
3438
3439 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3440 g_assert (ctx->addresses [reg]);
3441 LLVMSetValueName (ctx->addresses [reg], name);
3442 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
3443 break;
3444 }
3445 case LLVMArgGsharedvtVariable:
3446 /* The IR treats these as variables with addresses */
3447 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3448 break;
3449 default:
3450 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, ainfo->type)), type_is_unsigned (ctx, ainfo->type));
3451 break;
3452 }
3453 }
3454 g_free (names);
3455
3456 if (cfg->vret_addr)
3457 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3458 if (sig->hasthis)
3459 emit_volatile_store (ctx, cfg->args [0]->dreg);
3460 for (i = 0; i < sig->param_count; ++i)
3461 if (!mini_type_is_vtype (sig->params [i]))
3462 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3463
3464 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
3465 LLVMValueRef this_alloc;
3466
3467 /*
3468 * The exception handling code needs the location where the this argument was
3469 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3470 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3471 * location into the LSDA.
3472 */
3473 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3474 /* This volatile store will keep the alloca alive */
3475 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3476
3477 set_metadata_flag (this_alloc, "mono.this");
3478 }
3479
3480 if (cfg->rgctx_var) {
3481 LLVMValueRef rgctx_alloc, store;
3482
3483 /*
3484 * We handle the rgctx arg similarly to the this pointer.
3485 */
3486 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3487 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3488 /* This volatile store will keep the alloca alive */
3489 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3490
3491 set_metadata_flag (rgctx_alloc, "mono.this");
3492 }
3493
3494 /* Initialize the method if needed */
3495 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
3496 /* Emit a location for the initialization code */
3497 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3498 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3499
3500 LLVMBuildBr (ctx->builder, ctx->init_bb);
3501 builder = ctx->builder = create_builder (ctx);
3502 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3503 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3504 }
3505
3506 /* Compute nesting between clauses */
3507 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3508 for (i = 0; i < cfg->header->num_clauses; ++i) {
3509 for (j = 0; j < cfg->header->num_clauses; ++j) {
3510 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3511 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3512
3513 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3514 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3515 }
3516 }
3517
3518 /*
3519 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3520 * it needs to continue normally, or return back to the exception handling system.
3521 */
3522 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3523 int clause_index;
3524 char name [128];
3525
3526 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3527 continue;
3528
3529 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3530 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
3531 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
3532
3533 if (bb->in_scount == 0) {
3534 LLVMValueRef val;
3535
3536 sprintf (name, "finally_ind_bb%d", bb->block_num);
3537 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3538 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3539
3540 ctx->bblocks [bb->block_num].finally_ind = val;
3541 } else {
3542 /* Create a variable to hold the exception var */
3543 if (!ctx->ex_var)
3544 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3545 }
3546
3547 /*
3548 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
3549 * LLVM bblock containing a landing pad causes problems for the
3550 * LLVM optimizer passes.
3551 */
3552 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
3553 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
3554 }
3555 ctx->builder = old_builder;
3556 }
3557
3558 static gboolean
3559 needs_extra_arg (EmitContext *ctx, MonoMethod *method)
3560 {
3561 WrapperInfo *info = NULL;
3562
3563 /*
3564 * When targeting wasm, the caller and callee signature has to match exactly. This means
3565 * that every method which can be called indirectly need an extra arg since the caller
3566 * will call it through an ftnptr and will pass an extra arg.
3567 */
3568 if (!ctx->cfg->llvm_only || !ctx->emit_dummy_arg)
3569 return FALSE;
3570 if (method->wrapper_type)
3571 info = mono_marshal_get_wrapper_info (method);
3572
3573 switch (method->wrapper_type) {
3574 case MONO_WRAPPER_OTHER:
3575 if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
3576 /* Already have an explicit extra arg */
3577 return FALSE;
3578 break;
3579 case MONO_WRAPPER_MANAGED_TO_NATIVE:
3580 if (strstr (method->name, "icall_wrapper"))
3581 /* These are JIT icall wrappers which are only called from JITted code directly */
3582 return FALSE;
3583 /* Normal icalls can be virtual methods which need an extra arg */
3584 break;
3585 case MONO_WRAPPER_RUNTIME_INVOKE:
3586 case MONO_WRAPPER_ALLOC:
3587 case MONO_WRAPPER_CASTCLASS:
3588 case MONO_WRAPPER_WRITE_BARRIER:
3589 return FALSE;
3590 case MONO_WRAPPER_STELEMREF:
3591 if (info->subtype != WRAPPER_SUBTYPE_VIRTUAL_STELEMREF)
3592 return FALSE;
3593 break;
3594 case MONO_WRAPPER_MANAGED_TO_MANAGED:
3595 if (info->subtype == WRAPPER_SUBTYPE_STRING_CTOR)
3596 return FALSE;
3597 break;
3598 default:
3599 break;
3600 }
3601 if (method->string_ctor)
3602 return FALSE;
3603
3604 /* These are called from gsharedvt code with an indirect call which doesn't pass an extra arg */
3605 if (method->klass == mono_get_string_class () && (strstr (method->name, "memcpy") || strstr (method->name, "bzero")))
3606 return FALSE;
3607 return TRUE;
3608 }
3609
3610 static inline gboolean
3611 is_supported_callconv (EmitContext *ctx, MonoCallInst *call)
3612 {
3613 #if defined(TARGET_WIN32) && defined(TARGET_AMD64)
3614 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) ||
3615 (call->signature->call_convention == MONO_CALL_C) ||
3616 (call->signature->call_convention == MONO_CALL_STDCALL);
3617 #else
3618 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) || ((call->signature->call_convention == MONO_CALL_C) && ctx->llvm_only);
3619 #endif
3620 return result;
3621 }
3622
3623 static void
3624 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3625 {
3626 MonoCompile *cfg = ctx->cfg;
3627 LLVMValueRef *values = ctx->values;
3628 LLVMValueRef *addresses = ctx->addresses;
3629 MonoCallInst *call = (MonoCallInst*)ins;
3630 MonoMethodSignature *sig = call->signature;
3631 LLVMValueRef callee = NULL, lcall;
3632 LLVMValueRef *args;
3633 LLVMCallInfo *cinfo;
3634 GSList *l;
3635 int i, len, nargs;
3636 gboolean vretaddr;
3637 LLVMTypeRef llvm_sig;
3638 gpointer target;
3639 gboolean is_virtual, calli, preserveall;
3640 LLVMBuilderRef builder = *builder_ref;
3641
3642 /* If both imt and rgctx arg are required, only pass the imt arg, the rgctx trampoline will pass the rgctx */
3643 if (call->imt_arg_reg)
3644 call->rgctx_arg_reg = 0;
3645
3646 if (!is_supported_callconv (ctx, call)) {
3647 set_failure (ctx, "non-default callconv");
3648 return;
3649 }
3650
3651 cinfo = call->cinfo;
3652 g_assert (cinfo);
3653 if (call->rgctx_arg_reg)
3654 cinfo->rgctx_arg = TRUE;
3655 if (call->imt_arg_reg)
3656 cinfo->imt_arg = TRUE;
3657 if (!call->rgctx_arg_reg && call->method && needs_extra_arg (ctx, call->method))
3658 cinfo->dummy_arg = TRUE;
3659
3660 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3661
3662 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3663 if (!ctx_ok (ctx))
3664 return;
3665
3666 int const opcode = ins->opcode;
3667
3668 is_virtual = opcode == OP_VOIDCALL_MEMBASE || opcode == OP_CALL_MEMBASE
3669 || opcode == OP_VCALL_MEMBASE || opcode == OP_LCALL_MEMBASE
3670 || opcode == OP_FCALL_MEMBASE || opcode == OP_RCALL_MEMBASE
3671 || opcode == OP_TAILCALL_MEMBASE;
3672 calli = !call->fptr_is_patch && (opcode == OP_VOIDCALL_REG || opcode == OP_CALL_REG
3673 || opcode == OP_VCALL_REG || opcode == OP_LCALL_REG || opcode == OP_FCALL_REG
3674 || opcode == OP_RCALL_REG || opcode == OP_TAILCALL_REG);
3675
3676 /* Unused */
3677 preserveall = FALSE;
3678
3679 /* FIXME: Avoid creating duplicate methods */
3680
3681 if (ins->flags & MONO_INST_HAS_METHOD) {
3682 if (is_virtual) {
3683 callee = NULL;
3684 } else {
3685 if (cfg->compile_aot) {
3686 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3687 if (!callee) {
3688 set_failure (ctx, "can't encode patch");
3689 return;
3690 }
3691 } else {
3692 ERROR_DECL (error);
3693 static int tramp_index;
3694 char *name;
3695
3696 name = g_strdup_printf ("tramp_%d", tramp_index);
3697 tramp_index ++;
3698
3699 /*
3700 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3701 * Make all calls through a global. The address of the global will be saved in
3702 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3703 * compiled.
3704 */
3705 LLVMValueRef tramp_var = (LLVMValueRef)g_hash_table_lookup (ctx->jit_callees, call->method);
3706 if (!tramp_var) {
3707 target =
3708 mono_create_jit_trampoline (mono_domain_get (),
3709 call->method, error);
3710 if (!is_ok (error)) {
3711 set_failure (ctx, mono_error_get_message (error));
3712 mono_error_cleanup (error);
3713 return;
3714 }
3715
3716 tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
3717 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
3718 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3719 g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
3720 }
3721 callee = LLVMBuildLoad (builder, tramp_var, "");
3722 }
3723 }
3724
3725 if (!cfg->llvm_only && call->method && strstr (m_class_get_name (call->method->klass), "AsyncVoidMethodBuilder")) {
3726 /* LLVM miscompiles async methods */
3727 set_failure (ctx, "#13734");
3728 return;
3729 }
3730 } else if (calli) {
3731 } else {
3732 const MonoJitICallId jit_icall_id = call->jit_icall_id;
3733
3734 if (jit_icall_id) {
3735 if (cfg->compile_aot) {
3736 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3737 if (!callee) {
3738 set_failure (ctx, "can't encode patch");
3739 return;
3740 }
3741 } else {
3742 callee = get_jit_callee (ctx, "", llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3743 }
3744 } else {
3745 if (cfg->compile_aot) {
3746 callee = NULL;
3747 if (cfg->abs_patches) {
3748 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3749 if (abs_ji) {
3750 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3751 if (!callee) {
3752 set_failure (ctx, "can't encode patch");
3753 return;
3754 }
3755 }
3756 }
3757 if (!callee) {
3758 set_failure (ctx, "aot");
3759 return;
3760 }
3761 } else {
3762 if (cfg->abs_patches) {
3763 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3764 if (abs_ji) {
3765 ERROR_DECL (error);
3766
3767 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, error);
3768 mono_error_assert_ok (error);
3769 callee = get_jit_callee (ctx, "", llvm_sig, abs_ji->type, abs_ji->data.target);
3770 } else {
3771 g_assert_not_reached ();
3772 }
3773 } else {
3774 g_assert_not_reached ();
3775 }
3776 }
3777 }
3778 }
3779
3780 if (is_virtual) {
3781 int size = TARGET_SIZEOF_VOID_P;
3782 LLVMValueRef index;
3783
3784 g_assert (ins->inst_offset % size == 0);
3785 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3786
3787 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3788 } else if (calli) {
3789 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3790 } else {
3791 if (ins->flags & MONO_INST_HAS_METHOD) {
3792 }
3793 }
3794
3795 /*
3796 * Collect and convert arguments
3797 */
3798 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg + call->cinfo->dummy_arg + 1;
3799 len = sizeof (LLVMValueRef) * nargs;
3800 args = g_newa (LLVMValueRef, nargs);
3801 memset (args, 0, len);
3802 l = call->out_ireg_args;
3803
3804 if (call->rgctx_arg_reg) {
3805 g_assert (values [call->rgctx_arg_reg]);
3806 g_assert (cinfo->rgctx_arg_pindex < nargs);
3807 /*
3808 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3809 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3810 * it using a volatile load.
3811 */
3812 #ifdef TARGET_ARM
3813 if (!ctx->imt_rgctx_loc)
3814 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3815 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3816 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3817 #else
3818 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3819 #endif
3820 }
3821 if (call->imt_arg_reg) {
3822 g_assert (!ctx->llvm_only);
3823 g_assert (values [call->imt_arg_reg]);
3824 g_assert (cinfo->imt_arg_pindex < nargs);
3825 #ifdef TARGET_ARM
3826 if (!ctx->imt_rgctx_loc)
3827 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3828 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3829 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3830 #else
3831 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3832 #endif
3833 }
3834 switch (cinfo->ret.storage) {
3835 case LLVMArgGsharedvtVariable: {
3836 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
3837
3838 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
3839 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
3840 } else {
3841 g_assert (addresses [call->inst.dreg]);
3842 args [cinfo->vret_arg_pindex] = convert (ctx, addresses [call->inst.dreg], IntPtrType ());
3843 }
3844 break;
3845 }
3846 default:
3847 if (vretaddr) {
3848 if (!addresses [call->inst.dreg])
3849 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3850 g_assert (cinfo->vret_arg_pindex < nargs);
3851 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3852 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3853 else
3854 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
3855 }
3856 break;
3857 }
3858
3859 /*
3860 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
3861 * use the real callee for argument type conversion.
3862 */
3863 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
3864 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
3865 LLVMGetParamTypes (callee_type, param_types);
3866
3867 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
3868 guint32 regpair;
3869 int reg, pindex;
3870 LLVMArgInfo *ainfo = &call->cinfo->args [i];
3871
3872 pindex = ainfo->pindex;
3873
3874 regpair = (guint32)(gssize)(l->data);
3875 reg = regpair & 0xffffff;
3876 args [pindex] = values [reg];
3877 switch (ainfo->storage) {
3878 case LLVMArgVtypeInReg:
3879 case LLVMArgAsFpArgs: {
3880 guint32 nargs;
3881 int j;
3882
3883 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
3884 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
3885 pindex += ainfo->ndummy_fpargs;
3886
3887 g_assert (addresses [reg]);
3888 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
3889 pindex += nargs;
3890
3891 // FIXME: alignment
3892 // FIXME: Get rid of the VMOVE
3893 break;
3894 }
3895 case LLVMArgVtypeByVal:
3896 g_assert (addresses [reg]);
3897 args [pindex] = addresses [reg];
3898 break;
3899 case LLVMArgVtypeByRef: {
3900 g_assert (addresses [reg]);
3901 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3902 break;
3903 }
3904 case LLVMArgAsIArgs:
3905 g_assert (addresses [reg]);
3906 if (ainfo->esize == 8)
3907 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (LLVMInt64Type (), ainfo->nslots), 0)), "");
3908 else
3909 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
3910 break;
3911 case LLVMArgVtypeAsScalar:
3912 g_assert_not_reached ();
3913 break;
3914 case LLVMArgGsharedvtFixed:
3915 case LLVMArgGsharedvtFixedVtype:
3916 g_assert (addresses [reg]);
3917 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3918 break;
3919 case LLVMArgGsharedvtVariable:
3920 g_assert (addresses [reg]);
3921 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
3922 break;
3923 default:
3924 g_assert (args [pindex]);
3925 if (i == 0 && sig->hasthis)
3926 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
3927 else
3928 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
3929 break;
3930 }
3931 g_assert (pindex <= nargs);
3932
3933 l = l->next;
3934 }
3935
3936
3937 if (call->cinfo->dummy_arg) {
3938 g_assert (call->cinfo->dummy_arg_pindex < nargs);
3939 args [call->cinfo->dummy_arg_pindex] = LLVMConstNull (ctx->module->ptr_type);
3940 }
3941
3942 // FIXME: Align call sites
3943
3944 /*
3945 * Emit the call
3946 */
3947 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
3948
3949 mono_llvm_nonnull_state_update (ctx, lcall, call->method, args, LLVMCountParamTypes (llvm_sig));
3950
3951 // If we just allocated an object, it's not null.
3952 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC) {
3953 mono_llvm_set_call_nonnull_ret (lcall);
3954 }
3955
3956 if (ins->opcode != OP_TAILCALL && ins->opcode != OP_TAILCALL_MEMBASE && LLVMGetInstructionOpcode (lcall) == LLVMCall)
3957 mono_llvm_set_call_notailcall (lcall);
3958
3959 // As per the LLVM docs, a function has a noalias return value if and only if
3960 // it is an allocation function. This is an allocation function.
3961 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC)
3962 mono_llvm_set_call_noalias_ret (lcall);
3963
3964 /*
3965 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
3966 */
3967 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
3968 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
3969 #endif
3970 /* The two can't be used together, so use only one LLVM calling conv to pass them */
3971 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
3972 if (!sig->pinvoke && !cfg->llvm_only)
3973 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
3974 if (preserveall)
3975 set_call_preserveall_cc (lcall);
3976
3977 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3978 mono_llvm_add_instr_attr (lcall, 1 + cinfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
3979 if (!ctx->llvm_only && call->rgctx_arg_reg)
3980 mono_llvm_add_instr_attr (lcall, 1 + cinfo->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
3981 if (call->imt_arg_reg)
3982 mono_llvm_add_instr_attr (lcall, 1 + cinfo->imt_arg_pindex, LLVM_ATTR_IN_REG);
3983
3984 /* Add byval attributes if needed */
3985 for (i = 0; i < sig->param_count; ++i) {
3986 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
3987
3988 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
3989 mono_llvm_add_instr_attr (lcall, 1 + ainfo->pindex, LLVM_ATTR_BY_VAL);
3990 }
3991
3992 /*
3993 * Convert the result
3994 */
3995 switch (cinfo->ret.storage) {
3996 case LLVMArgVtypeInReg: {
3997 LLVMValueRef regs [2];
3998
3999 if (LLVMTypeOf (lcall) == LLVMVoidType ())
4000 /* Empty struct */
4001 break;
4002
4003 if (!addresses [ins->dreg])
4004 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
4005
4006 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
4007 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
4008 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
4009 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
4010 break;
4011 }
4012 case LLVMArgVtypeByVal:
4013 if (!addresses [call->inst.dreg])
4014 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4015 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
4016 break;
4017 case LLVMArgAsIArgs:
4018 case LLVMArgFpStruct:
4019 if (!addresses [call->inst.dreg])
4020 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4021 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4022 break;
4023 case LLVMArgVtypeAsScalar:
4024 if (!addresses [call->inst.dreg])
4025 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4026 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4027 break;
4028 case LLVMArgVtypeRetAddr:
4029 case LLVMArgVtypeByRef:
4030 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret))) {
4031 /* Some opcodes like STOREX_MEMBASE access these by value */
4032 g_assert (addresses [call->inst.dreg]);
4033 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4034 }
4035 break;
4036 case LLVMArgGsharedvtVariable:
4037 break;
4038 case LLVMArgGsharedvtFixed:
4039 case LLVMArgGsharedvtFixedVtype:
4040 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4041 break;
4042 default:
4043 if (sig->ret->type != MONO_TYPE_VOID)
4044 /* If the method returns an unsigned value, need to zext it */
4045 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
4046 break;
4047 }
4048
4049 *builder_ref = ctx->builder;
4050 }
4051
4052 static void
4053 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4054 {
4055 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception : MONO_JIT_ICALL_mono_llvm_throw_exception;
4056 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
4057
4058 LLVMTypeRef exc_type = type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_exception_class ()));
4059
4060 if (!callee) {
4061 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
4062
4063 g_assert (ctx->cfg->compile_aot);
4064 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (icall_id));
4065 }
4066
4067 LLVMValueRef args [2];
4068
4069 args [0] = convert (ctx, exc, exc_type);
4070 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
4071
4072 LLVMBuildUnreachable (ctx->builder);
4073
4074 ctx->builder = create_builder (ctx);
4075 }
4076
4077 static void
4078 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4079 {
4080 MonoMethodSignature *throw_sig;
4081
4082 LLVMValueRef * const pcallee = rethrow ? &ctx->module->rethrow : &ctx->module->throw_icall;
4083 LLVMValueRef callee = *pcallee;
4084 char const * const icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4085 #ifndef TARGET_X86
4086 const
4087 #endif
4088 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_arch_rethrow_exception : MONO_JIT_ICALL_mono_arch_throw_exception;
4089
4090 if (!callee) {
4091 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
4092 throw_sig->ret = m_class_get_byval_arg (mono_get_void_class ());
4093 throw_sig->params [0] = m_class_get_byval_arg (mono_get_object_class ());
4094 if (ctx->cfg->compile_aot) {
4095 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4096 } else {
4097 #ifdef TARGET_X86
4098 /*
4099 * LLVM doesn't push the exception argument, so we need a different
4100 * trampoline.
4101 */
4102 icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception_trampoline : MONO_JIT_ICALL_mono_llvm_throw_exception_trampoline;
4103 #endif
4104 callee = get_jit_callee (ctx, icall_name, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4105 }
4106
4107 mono_memory_barrier ();
4108 }
4109 LLVMValueRef arg;
4110 arg = convert (ctx, exc, type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_object_class ())));
4111 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
4112 }
4113
4114 static void
4115 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
4116 {
4117 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_resume_exception;
4118 LLVMValueRef callee;
4119
4120 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
4121
4122 g_assert (ctx->cfg->compile_aot);
4123 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4124
4125 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
4126
4127 LLVMBuildUnreachable (ctx->builder);
4128
4129 ctx->builder = create_builder (ctx);
4130 }
4131
4132 static LLVMValueRef
4133 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4134 {
4135 const char *icall_name = "mono_llvm_clear_exception";
4136 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_clear_exception;
4137
4138 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
4139 LLVMValueRef callee = NULL;
4140
4141 if (!callee) {
4142 if (ctx->cfg->compile_aot) {
4143 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4144 } else {
4145 // FIXME: This is broken.
4146 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4147 }
4148 }
4149
4150 g_assert (builder && callee);
4151
4152 return LLVMBuildCall (builder, callee, NULL, 0, "");
4153 }
4154
4155 static LLVMValueRef
4156 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4157 {
4158 const char *icall_name = "mono_llvm_load_exception";
4159 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_load_exception;
4160
4161 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
4162 LLVMValueRef callee = NULL;
4163
4164 if (!callee) {
4165 if (ctx->cfg->compile_aot) {
4166 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4167 } else {
4168 // FIXME: This is broken.
4169 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4170 }
4171 }
4172
4173 g_assert (builder && callee);
4174
4175 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
4176 }
4177
4178
4179 static LLVMValueRef
4180 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
4181 {
4182 const char *icall_name = "mono_llvm_match_exception";
4183 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_match_exception;
4184
4185 ctx->builder = builder;
4186
4187 LLVMValueRef args[5];
4188 const int num_args = G_N_ELEMENTS (args);
4189
4190 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
4191 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
4192 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
4193 if (ctx->cfg->rgctx_var) {
4194 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
4195 g_assert (rgctx_alloc);
4196 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
4197 } else {
4198 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
4199 }
4200 if (ctx->this_arg)
4201 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
4202 else
4203 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
4204
4205 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
4206 LLVMValueRef callee;
4207 g_assert (ctx->cfg->compile_aot);
4208 ctx->builder = builder;
4209 // get_callee expects ctx->builder to be the emitting builder
4210 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4211
4212 g_assert (builder && callee);
4213
4214 g_assert (ctx->ex_var);
4215
4216 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
4217 }
4218
4219 // FIXME: This won't work because the code-finding makes this
4220 // not a constant.
4221 /*#define MONO_PERSONALITY_DEBUG*/
4222
4223 #ifdef MONO_PERSONALITY_DEBUG
4224 static const gboolean use_mono_personality_debug = TRUE;
4225 static const char *default_personality_name = "mono_debug_personality";
4226 #else
4227 static const gboolean use_mono_personality_debug = FALSE;
4228 static const char *default_personality_name = "__gxx_personality_v0";
4229 #endif
4230
4231 static LLVMTypeRef
4232 default_cpp_lpad_exc_signature (void)
4233 {
4234 static gboolean inited = FALSE;
4235 static LLVMTypeRef sig;
4236
4237 if (!sig) {
4238 LLVMTypeRef signature [2];
4239 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
4240 signature [1] = LLVMInt32Type ();
4241 sig = LLVMStructType (signature, 2, FALSE);
4242 inited = TRUE;
4243 }
4244
4245 return sig;
4246 }
4247
4248 static LLVMValueRef
4249 get_mono_personality (EmitContext *ctx)
4250 {
4251 LLVMValueRef personality = NULL;
4252 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4253
4254 g_assert (ctx->cfg->compile_aot);
4255 if (!use_mono_personality_debug) {
4256 personality = get_intrins_by_name (ctx, default_personality_name);
4257 } else {
4258 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_debug_personality));
4259 }
4260
4261 g_assert (personality);
4262 return personality;
4263 }
4264
4265 static LLVMBasicBlockRef
4266 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
4267 {
4268 MonoCompile *cfg = ctx->cfg;
4269 LLVMBuilderRef old_builder = ctx->builder;
4270 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
4271
4272 LLVMBuilderRef lpadBuilder = create_builder (ctx);
4273 ctx->builder = lpadBuilder;
4274
4275 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
4276 g_assert (handler_bb);
4277
4278 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4279 LLVMValueRef personality = get_mono_personality (ctx);
4280 g_assert (personality);
4281
4282 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
4283 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
4284 g_free (bb_name);
4285 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4286 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
4287 g_assert (landing_pad);
4288
4289 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
4290 LLVMAddClause (landing_pad, cast);
4291
4292 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
4293 LLVMBuilderRef resume_builder = create_builder (ctx);
4294 ctx->builder = resume_builder;
4295 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
4296
4297 emit_resume_eh (ctx, handler_bb);
4298
4299 // Build match
4300 ctx->builder = lpadBuilder;
4301 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4302
4303 gboolean finally_only = TRUE;
4304
4305 MonoExceptionClause *group_cursor = group_start;
4306
4307 for (int i = 0; i < group_size; i ++) {
4308 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY || group_cursor->flags & MONO_EXCEPTION_CLAUSE_FAULT))
4309 finally_only = FALSE;
4310
4311 group_cursor++;
4312 }
4313
4314 // FIXME:
4315 // Handle landing pad inlining
4316
4317 if (!finally_only) {
4318 // So at each level of the exception stack we will match the exception again.
4319 // During that match, we need to compare against the handler types for the current
4320 // protected region. We send the try start and end so that we can only check against
4321 // handlers for this lexical protected region.
4322 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
4323
4324 // if returns -1, resume
4325 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
4326
4327 // else move to that target bb
4328 for (int i = 0; i < group_size; i++) {
4329 MonoExceptionClause *clause = group_start + i;
4330 int clause_index = clause - cfg->header->clauses;
4331 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4332 g_assert (handler_bb);
4333 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4334 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4335 }
4336 } else {
4337 int clause_index = group_start - cfg->header->clauses;
4338 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4339 g_assert (finally_bb);
4340
4341 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
4342 }
4343
4344 ctx->builder = old_builder;
4345
4346 return lpad_bb;
4347 }
4348
4349
4350 static void
4351 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
4352 {
4353 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
4354 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
4355
4356 // Make exception available to catch blocks
4357 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags & MONO_EXCEPTION_CLAUSE_FAULT)) {
4358 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
4359
4360 g_assert (ctx->ex_var);
4361 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
4362
4363 if (bb->in_scount == 1) {
4364 MonoInst *exvar = bb->in_stack [0];
4365 g_assert (!ctx->values [exvar->dreg]);
4366 g_assert (ctx->ex_var);
4367 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
4368 emit_volatile_store (ctx, exvar->dreg);
4369 }
4370
4371 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
4372 }
4373
4374 LLVMBuilderRef handler_builder = create_builder (ctx);
4375 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
4376 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
4377
4378 // Make the handler code end with a jump to cbb
4379 LLVMBuildBr (handler_builder, cbb);
4380 }
4381
4382 static void
4383 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
4384 {
4385 MonoCompile *cfg = ctx->cfg;
4386 LLVMValueRef *values = ctx->values;
4387 LLVMModuleRef lmodule = ctx->lmodule;
4388 BBInfo *bblocks = ctx->bblocks;
4389 LLVMTypeRef i8ptr;
4390 LLVMValueRef personality;
4391 LLVMValueRef landing_pad;
4392 LLVMBasicBlockRef target_bb;
4393 MonoInst *exvar;
4394 static int ti_generator;
4395 char ti_name [128];
4396 LLVMValueRef type_info;
4397 int clause_index;
4398 GSList *l;
4399
4400 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4401
4402 if (cfg->compile_aot) {
4403 /* Use a dummy personality function */
4404 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
4405 g_assert (personality);
4406 } else {
4407 /* Can't cache this as each method is in its own llvm module */
4408 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4409 personality = LLVMAddFunction (ctx->lmodule, "mono_personality", personality_type);
4410 mono_llvm_add_func_attr (personality, LLVM_ATTR_NO_UNWIND);
4411 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (personality, "ENTRY");
4412 LLVMBuilderRef builder2 = LLVMCreateBuilder ();
4413 LLVMPositionBuilderAtEnd (builder2, entry_bb);
4414 LLVMBuildRet (builder2, LLVMConstInt (LLVMInt32Type (), 0, FALSE));
4415 LLVMDisposeBuilder (builder2);
4416 }
4417
4418 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
4419
4420 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
4421
4422 /*
4423 * Create the type info
4424 */
4425 sprintf (ti_name, "type_info_%d", ti_generator);
4426 ti_generator ++;
4427
4428 if (cfg->compile_aot) {
4429 /* decode_eh_frame () in aot-runtime.c will decode this */
4430 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4431 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4432
4433 /*
4434 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4435 */
4436 LLVMSetLinkage (type_info, LLVMInternalLinkage);
4437 } else {
4438 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4439 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4440 }
4441
4442 {
4443 LLVMTypeRef members [2], ret_type;
4444
4445 members [0] = i8ptr;
4446 members [1] = LLVMInt32Type ();
4447 ret_type = LLVMStructType (members, 2, FALSE);
4448
4449 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
4450 LLVMAddClause (landing_pad, type_info);
4451
4452 /* Store the exception into the exvar */
4453 if (ctx->ex_var)
4454 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
4455 }
4456
4457 /*
4458 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4459 * code expects control to be transferred to this landing pad even in the
4460 * presence of nested clauses. The landing pad needs to branch to the landing
4461 * pads belonging to nested clauses based on the selector value returned by
4462 * the landing pad instruction, which is passed to the landing pad in a
4463 * register by the EH code.
4464 */
4465 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4466 g_assert (target_bb);
4467
4468 /*
4469 * Branch to the correct landing pad
4470 */
4471 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
4472 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
4473
4474 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
4475 int nesting_clause_index = GPOINTER_TO_INT (l->data);
4476 MonoBasicBlock *handler_bb;
4477
4478 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
4479 g_assert (handler_bb);
4480
4481 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4482 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4483 }
4484
4485 /* Start a new bblock which CALL_HANDLER can branch to */
4486 ctx->builder = builder = create_builder (ctx);
4487 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
4488
4489 ctx->bblocks [bb->block_num].end_bblock = target_bb;
4490
4491 /* Store the exception into the IL level exvar */
4492 if (bb->in_scount == 1) {
4493 g_assert (bb->in_scount == 1);
4494 exvar = bb->in_stack [0];
4495
4496 // FIXME: This is shared with filter clauses ?
4497 g_assert (!values [exvar->dreg]);
4498
4499 g_assert (ctx->ex_var);
4500 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
4501 emit_volatile_store (ctx, exvar->dreg);
4502 }
4503
4504 /* Make normal branches to the start of the clause branch to the new bblock */
4505 bblocks [bb->block_num].bblock = target_bb;
4506 }
4507
4508 //Wasm requires us to canonicalize NaNs.
4509 static LLVMValueRef
4510 get_double_const (MonoCompile *cfg, double val)
4511 {
4512 #ifdef TARGET_WASM
4513 if (mono_isnan (val))
4514 *(gint64 *)&val = 0x7FF8000000000000ll;
4515 #endif
4516 return LLVMConstReal (LLVMDoubleType (), val);
4517 }
4518
4519 static LLVMValueRef
4520 get_float_const (MonoCompile *cfg, float val)
4521 {
4522 #ifdef TARGET_WASM
4523 if (mono_isnan (val))
4524 *(int *)&val = 0x7FC00000;
4525 #endif
4526 if (cfg->r4fp)
4527 return LLVMConstReal (LLVMFloatType (), val);
4528 else
4529 return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val), LLVMDoubleType ());
4530 }
4531
4532 static void
4533 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4534 {
4535 MonoCompile *cfg = ctx->cfg;
4536 MonoMethodSignature *sig = ctx->sig;
4537 LLVMValueRef method = ctx->lmethod;
4538 LLVMValueRef *values = ctx->values;
4539 LLVMValueRef *addresses = ctx->addresses;
4540 LLVMCallInfo *linfo = ctx->linfo;
4541 BBInfo *bblocks = ctx->bblocks;
4542 MonoInst *ins;
4543 LLVMBasicBlockRef cbb;
4544 LLVMBuilderRef builder, starting_builder;
4545 gboolean has_terminator;
4546 LLVMValueRef v;
4547 LLVMValueRef lhs, rhs;
4548 int nins = 0;
4549
4550 cbb = get_end_bb (ctx, bb);
4551
4552 builder = create_builder (ctx);
4553 ctx->builder = builder;
4554 LLVMPositionBuilderAtEnd (builder, cbb);
4555
4556 if (!ctx_ok (ctx))
4557 return;
4558
4559 if (bb->flags & BB_EXCEPTION_HANDLER) {
4560 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4561 set_failure (ctx, "handler without invokes");
4562 return;
4563 }
4564
4565 if (ctx->llvm_only)
4566 emit_llvmonly_handler_start (ctx, bb, cbb);
4567 else
4568 emit_handler_start (ctx, bb, builder);
4569 if (!ctx_ok (ctx))
4570 return;
4571 builder = ctx->builder;
4572 }
4573
4574 /* Handle PHI nodes first */
4575 /* They should be grouped at the start of the bb */
4576 for (ins = bb->code; ins; ins = ins->next) {
4577 emit_dbg_loc (ctx, builder, ins->cil_code);
4578
4579 if (ins->opcode == OP_NOP)
4580 continue;
4581 if (!MONO_IS_PHI (ins))
4582 break;
4583
4584 int i;
4585 gboolean empty = TRUE;
4586
4587 /* Check that all input bblocks really branch to us */
4588 for (i = 0; i < bb->in_count; ++i) {
4589 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4590 ins->inst_phi_args [i + 1] = -1;
4591 else
4592 empty = FALSE;
4593 }
4594
4595 if (empty) {
4596 /* LLVM doesn't like phi instructions with zero operands */
4597 ctx->is_dead [ins->dreg] = TRUE;
4598 continue;
4599 }
4600
4601 /* Created earlier, insert it now */
4602 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4603
4604 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4605 int sreg1 = ins->inst_phi_args [i + 1];
4606 int count, j;
4607
4608 /*
4609 * Count the number of times the incoming bblock branches to us,
4610 * since llvm requires a separate entry for each.
4611 */
4612 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4613 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4614
4615 count = 0;
4616 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4617 if (switch_ins->inst_many_bb [j] == bb)
4618 count ++;
4619 }
4620 } else {
4621 count = 1;
4622 }
4623
4624 /* Remember for later */
4625 for (j = 0; j < count; ++j) {
4626 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4627 node->bb = bb;
4628 node->phi = ins;
4629 node->in_bb = bb->in_bb [i];
4630 node->sreg = sreg1;
4631 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4632 }
4633 }
4634 }
4635 // Add volatile stores for PHI nodes
4636 // These need to be emitted after the PHI nodes
4637 for (ins = bb->code; ins; ins = ins->next) {
4638 const char *spec = LLVM_INS_INFO (ins->opcode);
4639
4640 if (ins->opcode == OP_NOP)
4641 continue;
4642 if (!MONO_IS_PHI (ins))
4643 break;
4644
4645 if (spec [MONO_INST_DEST] != 'v')
4646 emit_volatile_store (ctx, ins->dreg);
4647 }
4648
4649 has_terminator = FALSE;
4650 starting_builder = builder;
4651 for (ins = bb->code; ins; ins = ins->next) {
4652 const char *spec = LLVM_INS_INFO (ins->opcode);
4653 char *dname = NULL;
4654 char dname_buf [128];
4655
4656 emit_dbg_loc (ctx, builder, ins->cil_code);
4657
4658 nins ++;
4659 if (nins > 1000) {
4660 /*
4661 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4662 * Start a new bblock.
4663 * Prevent the bblocks to be merged by doing a volatile load + cond branch
4664 * from localloc-ed memory.
4665 */
4666 if (!cfg->llvm_only)
4667 ;//set_failure (ctx, "basic block too long");
4668
4669 if (!ctx->long_bb_break_var) {
4670 ctx->long_bb_break_var = build_alloca_llvm_type_name (ctx, LLVMInt32Type (), 0, "long_bb_break");
4671 mono_llvm_build_store (ctx->alloca_builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4672 }
4673
4674 cbb = gen_bb (ctx, "CONT_LONG_BB");
4675 LLVMBasicBlockRef dummy_bb = gen_bb (ctx, "CONT_LONG_BB_DUMMY");
4676
4677 LLVMValueRef load = mono_llvm_build_load (builder, ctx->long_bb_break_var, "", TRUE);
4678 /*
4679 * The long_bb_break_var is initialized to 0 in the prolog, so this branch will always go to 'cbb'
4680 * but llvm doesn't know that, so the branch is not going to be eliminated.
4681 */
4682 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, load, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4683
4684 LLVMBuildCondBr (builder, cmp, cbb, dummy_bb);
4685
4686 /* Emit a dummy false bblock which does nothing but contains a volatile store so it cannot be eliminated */
4687 ctx->builder = builder = create_builder (ctx);
4688 LLVMPositionBuilderAtEnd (builder, dummy_bb);
4689 mono_llvm_build_store (builder, LLVMConstInt (LLVMInt32Type (), 1, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4690 LLVMBuildBr (builder, cbb);
4691
4692 ctx->builder = builder = create_builder (ctx);
4693 LLVMPositionBuilderAtEnd (builder, cbb);
4694 ctx->bblocks [bb->block_num].end_bblock = cbb;
4695 nins = 0;
4696
4697 emit_dbg_loc (ctx, builder, ins->cil_code);
4698 }
4699
4700 if (has_terminator)
4701 /* There could be instructions after a terminator, skip them */
4702 break;
4703
4704 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4705 sprintf (dname_buf, "t%d", ins->dreg);
4706 dname = dname_buf;
4707 }
4708
4709 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4710 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4711
4712 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4713 lhs = emit_volatile_load (ctx, ins->sreg1);
4714 } else {
4715 /* It is ok for SETRET to have an uninitialized argument */
4716 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4717 set_failure (ctx, "sreg1");
4718 return;
4719 }
4720 lhs = values [ins->sreg1];
4721 }
4722 } else {
4723 lhs = NULL;
4724 }
4725
4726 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4727 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4728 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4729 rhs = emit_volatile_load (ctx, ins->sreg2);
4730 } else {
4731 if (!values [ins->sreg2]) {
4732 set_failure (ctx, "sreg2");
4733 return;
4734 }
4735 rhs = values [ins->sreg2];
4736 }
4737 } else {
4738 rhs = NULL;
4739 }
4740
4741 //mono_print_ins (ins);
4742 gboolean skip_volatile_store = FALSE;
4743 switch (ins->opcode) {
4744 case OP_NOP:
4745 case OP_NOT_NULL:
4746 case OP_LIVERANGE_START:
4747 case OP_LIVERANGE_END:
4748 break;
4749 case OP_ICONST:
4750 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
4751 break;
4752 case OP_I8CONST:
4753 #if TARGET_SIZEOF_VOID_P == 4
4754 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4755 #else
4756 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
4757 #endif
4758 break;
4759 case OP_R8CONST:
4760 values [ins->dreg] = get_double_const (cfg, *(double*)ins->inst_p0);
4761 break;
4762 case OP_R4CONST:
4763 values [ins->dreg] = get_float_const (cfg, *(float*)ins->inst_p0);
4764 break;
4765 case OP_DUMMY_ICONST:
4766 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4767 break;
4768 case OP_DUMMY_I8CONST:
4769 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
4770 break;
4771 case OP_DUMMY_R8CONST:
4772 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
4773 break;
4774 case OP_BR: {
4775 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
4776 LLVMBuildBr (builder, target_bb);
4777 has_terminator = TRUE;
4778 break;
4779 }
4780 case OP_SWITCH: {
4781 int i;
4782 LLVMValueRef v;
4783 char bb_name [128];
4784 LLVMBasicBlockRef new_bb;
4785 LLVMBuilderRef new_builder;
4786
4787 // The default branch is already handled
4788 // FIXME: Handle it here
4789
4790 /* Start new bblock */
4791 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
4792 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
4793
4794 lhs = convert (ctx, lhs, LLVMInt32Type ());
4795 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
4796 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
4797 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
4798
4799 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
4800 }
4801
4802 new_builder = create_builder (ctx);
4803 LLVMPositionBuilderAtEnd (new_builder, new_bb);
4804 LLVMBuildUnreachable (new_builder);
4805
4806 has_terminator = TRUE;
4807 g_assert (!ins->next);
4808
4809 break;
4810 }
4811
4812 case OP_SETRET:
4813 switch (linfo->ret.storage) {
4814 case LLVMArgVtypeInReg: {
4815 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4816 LLVMValueRef val, addr, retval;
4817 int i;
4818
4819 retval = LLVMGetUndef (ret_type);
4820
4821 if (!addresses [ins->sreg1]) {
4822 /*
4823 * The return type is an LLVM vector type, have to convert between it and the
4824 * real return type which is a struct type.
4825 */
4826 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret)));
4827 /* Convert to 2xi64 first */
4828 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
4829
4830 for (i = 0; i < 2; ++i) {
4831 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4832 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
4833 } else {
4834 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4835 }
4836 }
4837 } else {
4838 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
4839 for (i = 0; i < 2; ++i) {
4840 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4841 LLVMValueRef indexes [2], part_addr;
4842
4843 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4844 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
4845 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
4846
4847 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
4848 } else {
4849 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4850 }
4851 }
4852 }
4853 LLVMBuildRet (builder, retval);
4854 break;
4855 }
4856 case LLVMArgVtypeAsScalar: {
4857 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4858 LLVMValueRef retval;
4859
4860 g_assert (addresses [ins->sreg1]);
4861
4862 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
4863 LLVMBuildRet (builder, retval);
4864 break;
4865 }
4866 case LLVMArgVtypeByVal: {
4867 LLVMValueRef retval;
4868
4869 g_assert (addresses [ins->sreg1]);
4870 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
4871 LLVMBuildRet (builder, retval);
4872 break;
4873 }
4874 case LLVMArgVtypeByRef: {
4875 LLVMBuildRetVoid (builder);
4876 break;
4877 }
4878 case LLVMArgGsharedvtFixed: {
4879 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
4880 /* The return value is in lhs, need to store to the vret argument */
4881 /* sreg1 might not be set */
4882 if (lhs) {
4883 g_assert (cfg->vret_addr);
4884 g_assert (values [cfg->vret_addr->dreg]);
4885 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
4886 }
4887 LLVMBuildRetVoid (builder);
4888 break;
4889 }
4890 case LLVMArgGsharedvtFixedVtype: {
4891 /* Already set */
4892 LLVMBuildRetVoid (builder);
4893 break;
4894 }
4895 case LLVMArgGsharedvtVariable: {
4896 /* Already set */
4897 LLVMBuildRetVoid (builder);
4898 break;
4899 }
4900 case LLVMArgVtypeRetAddr: {
4901 LLVMBuildRetVoid (builder);
4902 break;
4903 }
4904 case LLVMArgAsIArgs:
4905 case LLVMArgFpStruct: {
4906 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4907 LLVMValueRef retval;
4908
4909 g_assert (addresses [ins->sreg1]);
4910 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
4911 LLVMBuildRet (builder, retval);
4912 break;
4913 }
4914 case LLVMArgNone:
4915 case LLVMArgNormal: {
4916 if (!lhs || ctx->is_dead [ins->sreg1]) {
4917 /*
4918 * The method did not set its return value, probably because it
4919 * ends with a throw.
4920 */
4921 if (cfg->vret_addr)
4922 LLVMBuildRetVoid (builder);
4923 else
4924 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
4925 } else {
4926 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
4927 }
4928 has_terminator = TRUE;
4929 break;
4930 }
4931 default:
4932 g_assert_not_reached ();
4933 break;
4934 }
4935 break;
4936 case OP_ICOMPARE:
4937 case OP_FCOMPARE:
4938 case OP_RCOMPARE:
4939 case OP_LCOMPARE:
4940 case OP_COMPARE:
4941 case OP_ICOMPARE_IMM:
4942 case OP_LCOMPARE_IMM:
4943 case OP_COMPARE_IMM: {
4944 CompRelation rel;
4945 LLVMValueRef cmp, args [16];
4946 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
4947 gboolean unlikely = FALSE;
4948
4949 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
4950 if (ins->next->inst_false_bb->out_of_line)
4951 likely = TRUE;
4952 else if (ins->next->inst_true_bb->out_of_line)
4953 unlikely = TRUE;
4954 }
4955
4956 if (ins->next->opcode == OP_NOP)
4957 break;
4958
4959 if (ins->next->opcode == OP_BR)
4960 /* The comparison result is not needed */
4961 continue;
4962
4963 rel = mono_opcode_to_cond (ins->next->opcode);
4964
4965 if (ins->opcode == OP_ICOMPARE_IMM) {
4966 lhs = convert (ctx, lhs, LLVMInt32Type ());
4967 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4968 }
4969 if (ins->opcode == OP_LCOMPARE_IMM) {
4970 lhs = convert (ctx, lhs, LLVMInt64Type ());
4971 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4972 }
4973 if (ins->opcode == OP_LCOMPARE) {
4974 lhs = convert (ctx, lhs, LLVMInt64Type ());
4975 rhs = convert (ctx, rhs, LLVMInt64Type ());
4976 }
4977 if (ins->opcode == OP_ICOMPARE) {
4978 lhs = convert (ctx, lhs, LLVMInt32Type ());
4979 rhs = convert (ctx, rhs, LLVMInt32Type ());
4980 }
4981
4982 if (lhs && rhs) {
4983 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4984 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
4985 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
4986 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
4987 }
4988
4989 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
4990 if (ins->opcode == OP_FCOMPARE) {
4991 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4992 } else if (ins->opcode == OP_RCOMPARE) {
4993 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4994 } else if (ins->opcode == OP_COMPARE_IMM) {
4995 LLVMIntPredicate llvm_pred = cond_to_llvm_cond [rel];
4996 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0) {
4997 // We are emitting a NULL check for a pointer
4998 gboolean nonnull = mono_llvm_is_nonnull (lhs);
4999
5000 if (nonnull && llvm_pred == LLVMIntEQ)
5001 cmp = LLVMConstInt (LLVMInt1Type (), FALSE, FALSE);
5002 else if (nonnull && llvm_pred == LLVMIntNE)
5003 cmp = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
5004 else
5005 cmp = LLVMBuildICmp (builder, llvm_pred, lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
5006
5007 } else {
5008 cmp = LLVMBuildICmp (builder, llvm_pred, convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
5009 }
5010 } else if (ins->opcode == OP_LCOMPARE_IMM) {
5011 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5012 }
5013 else if (ins->opcode == OP_COMPARE) {
5014 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
5015 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5016 else
5017 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
5018 } else
5019 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5020
5021 if (likely || unlikely) {
5022 args [0] = cmp;
5023 args [1] = LLVMConstInt (LLVMInt1Type (), likely ? 1 : 0, FALSE);
5024 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
5025 }
5026
5027 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5028 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
5029 /*
5030 * If the target bb contains PHI instructions, LLVM requires
5031 * two PHI entries for this bblock, while we only generate one.
5032 * So convert this to an unconditional bblock. (bxc #171).
5033 */
5034 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
5035 } else {
5036 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
5037 }
5038 has_terminator = TRUE;
5039 } else if (MONO_IS_SETCC (ins->next)) {
5040 sprintf (dname_buf, "t%d", ins->next->dreg);
5041 dname = dname_buf;
5042 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5043
5044 /* Add stores for volatile variables */
5045 emit_volatile_store (ctx, ins->next->dreg);
5046 } else if (MONO_IS_COND_EXC (ins->next)) {
5047 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp);
5048 if (!ctx_ok (ctx))
5049 break;
5050 builder = ctx->builder;
5051 } else {
5052 set_failure (ctx, "next");
5053 break;
5054 }
5055
5056 ins = ins->next;
5057 break;
5058 }
5059 case OP_FCEQ:
5060 case OP_FCNEQ:
5061 case OP_FCLT:
5062 case OP_FCLT_UN:
5063 case OP_FCGT:
5064 case OP_FCGT_UN:
5065 case OP_FCGE:
5066 case OP_FCLE: {
5067 CompRelation rel;
5068 LLVMValueRef cmp;
5069
5070 rel = mono_opcode_to_cond (ins->opcode);
5071
5072 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5073 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5074 break;
5075 }
5076 case OP_RCEQ:
5077 case OP_RCNEQ:
5078 case OP_RCLT:
5079 case OP_RCLT_UN:
5080 case OP_RCGT:
5081 case OP_RCGT_UN: {
5082 CompRelation rel;
5083 LLVMValueRef cmp;
5084
5085 rel = mono_opcode_to_cond (ins->opcode);
5086
5087 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5088 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5089 break;
5090 }
5091 case OP_PHI:
5092 case OP_FPHI:
5093 case OP_VPHI:
5094 case OP_XPHI: {
5095 // Handled above
5096 skip_volatile_store = TRUE;
5097 break;
5098 }
5099 case OP_MOVE:
5100 case OP_LMOVE:
5101 case OP_XMOVE:
5102 case OP_SETFRET:
5103 g_assert (lhs);
5104 values [ins->dreg] = lhs;
5105 break;
5106 case OP_FMOVE:
5107 case OP_RMOVE: {
5108 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
5109
5110 g_assert (lhs);
5111 values [ins->dreg] = lhs;
5112
5113 if (var && m_class_get_byval_arg (var->klass)->type == MONO_TYPE_R4) {
5114 /*
5115 * This is added by the spilling pass in case of the JIT,
5116 * but we have to do it ourselves.
5117 */
5118 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
5119 }
5120 break;
5121 }
5122 case OP_MOVE_F_TO_I4: {
5123 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
5124 break;
5125 }
5126 case OP_MOVE_I4_TO_F: {
5127 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
5128 break;
5129 }
5130 case OP_MOVE_F_TO_I8: {
5131 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
5132 break;
5133 }
5134 case OP_MOVE_I8_TO_F: {
5135 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
5136 break;
5137 }
5138 case OP_IADD:
5139 case OP_ISUB:
5140 case OP_IAND:
5141 case OP_IMUL:
5142 case OP_IDIV:
5143 case OP_IDIV_UN:
5144 case OP_IREM:
5145 case OP_IREM_UN:
5146 case OP_IOR:
5147 case OP_IXOR:
5148 case OP_ISHL:
5149 case OP_ISHR:
5150 case OP_ISHR_UN:
5151 case OP_FADD:
5152 case OP_FSUB:
5153 case OP_FMUL:
5154 case OP_FDIV:
5155 case OP_LADD:
5156 case OP_LSUB:
5157 case OP_LMUL:
5158 case OP_LDIV:
5159 case OP_LDIV_UN:
5160 case OP_LREM:
5161 case OP_LREM_UN:
5162 case OP_LAND:
5163 case OP_LOR:
5164 case OP_LXOR:
5165 case OP_LSHL:
5166 case OP_LSHR:
5167 case OP_LSHR_UN:
5168 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5169 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5170
5171 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
5172 if (!ctx_ok (ctx))
5173 break;
5174 builder = ctx->builder;
5175
5176 switch (ins->opcode) {
5177 case OP_IADD:
5178 case OP_LADD:
5179 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
5180 break;
5181 case OP_ISUB:
5182 case OP_LSUB:
5183 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
5184 break;
5185 case OP_IMUL:
5186 case OP_LMUL:
5187 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
5188 break;
5189 case OP_IREM:
5190 case OP_LREM:
5191 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
5192 break;
5193 case OP_IREM_UN:
5194 case OP_LREM_UN:
5195 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
5196 break;
5197 case OP_IDIV:
5198 case OP_LDIV:
5199 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
5200 break;
5201 case OP_IDIV_UN:
5202 case OP_LDIV_UN:
5203 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
5204 break;
5205 case OP_FDIV:
5206 case OP_RDIV:
5207 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5208 break;
5209 case OP_IAND:
5210 case OP_LAND:
5211 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
5212 break;
5213 case OP_IOR:
5214 case OP_LOR:
5215 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
5216 break;
5217 case OP_IXOR:
5218 case OP_LXOR:
5219 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
5220 break;
5221 case OP_ISHL:
5222 case OP_LSHL:
5223 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
5224 break;
5225 case OP_ISHR:
5226 case OP_LSHR:
5227 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
5228 break;
5229 case OP_ISHR_UN:
5230 case OP_LSHR_UN:
5231 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
5232 break;
5233
5234 case OP_FADD:
5235 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5236 break;
5237 case OP_FSUB:
5238 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5239 break;
5240 case OP_FMUL:
5241 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5242 break;
5243
5244 default:
5245 g_assert_not_reached ();
5246 }
5247 break;
5248 case OP_RADD:
5249 case OP_RSUB:
5250 case OP_RMUL:
5251 case OP_RDIV: {
5252 lhs = convert (ctx, lhs, LLVMFloatType ());
5253 rhs = convert (ctx, rhs, LLVMFloatType ());
5254 switch (ins->opcode) {
5255 case OP_RADD:
5256 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5257 break;
5258 case OP_RSUB:
5259 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5260 break;
5261 case OP_RMUL:
5262 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5263 break;
5264 case OP_RDIV:
5265 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5266 break;
5267 default:
5268 g_assert_not_reached ();
5269 break;
5270 }
5271 break;
5272 }
5273 case OP_IADD_IMM:
5274 case OP_ISUB_IMM:
5275 case OP_IMUL_IMM:
5276 case OP_IREM_IMM:
5277 case OP_IREM_UN_IMM:
5278 case OP_IDIV_IMM:
5279 case OP_IDIV_UN_IMM:
5280 case OP_IAND_IMM:
5281 case OP_IOR_IMM:
5282 case OP_IXOR_IMM:
5283 case OP_ISHL_IMM:
5284 case OP_ISHR_IMM:
5285 case OP_ISHR_UN_IMM:
5286 case OP_LADD_IMM:
5287 case OP_LSUB_IMM:
5288 case OP_LMUL_IMM:
5289 case OP_LREM_IMM:
5290 case OP_LAND_IMM:
5291 case OP_LOR_IMM:
5292 case OP_LXOR_IMM:
5293 case OP_LSHL_IMM:
5294 case OP_LSHR_IMM:
5295 case OP_LSHR_UN_IMM:
5296 case OP_ADD_IMM:
5297 case OP_AND_IMM:
5298 case OP_MUL_IMM:
5299 case OP_SHL_IMM:
5300 case OP_SHR_IMM:
5301 case OP_SHR_UN_IMM: {
5302 LLVMValueRef imm;
5303
5304 if (spec [MONO_INST_SRC1] == 'l') {
5305 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5306 } else {
5307 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5308 }
5309
5310 emit_div_check (ctx, builder, bb, ins, lhs, imm);
5311 if (!ctx_ok (ctx))
5312 break;
5313 builder = ctx->builder;
5314
5315 #if TARGET_SIZEOF_VOID_P == 4
5316 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
5317 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5318 #endif
5319
5320 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5321 lhs = convert (ctx, lhs, IntPtrType ());
5322 imm = convert (ctx, imm, LLVMTypeOf (lhs));
5323 switch (ins->opcode) {
5324 case OP_IADD_IMM:
5325 case OP_LADD_IMM:
5326 case OP_ADD_IMM:
5327 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
5328 break;
5329 case OP_ISUB_IMM:
5330 case OP_LSUB_IMM:
5331 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
5332 break;
5333 case OP_IMUL_IMM:
5334 case OP_MUL_IMM:
5335 case OP_LMUL_IMM:
5336 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
5337 break;
5338 case OP_IDIV_IMM:
5339 case OP_LDIV_IMM:
5340 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
5341 break;
5342 case OP_IDIV_UN_IMM:
5343 case OP_LDIV_UN_IMM:
5344 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
5345 break;
5346 case OP_IREM_IMM:
5347 case OP_LREM_IMM:
5348 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
5349 break;
5350 case OP_IREM_UN_IMM:
5351 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
5352 break;
5353 case OP_IAND_IMM:
5354 case OP_LAND_IMM:
5355 case OP_AND_IMM:
5356 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
5357 break;
5358 case OP_IOR_IMM:
5359 case OP_LOR_IMM:
5360 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
5361 break;
5362 case OP_IXOR_IMM:
5363 case OP_LXOR_IMM:
5364 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
5365 break;
5366 case OP_ISHL_IMM:
5367 case OP_LSHL_IMM:
5368 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5369 break;
5370 case OP_SHL_IMM:
5371 if (TARGET_SIZEOF_VOID_P == 8) {
5372 /* The IL is not regular */
5373 lhs = convert (ctx, lhs, LLVMInt64Type ());
5374 imm = convert (ctx, imm, LLVMInt64Type ());
5375 }
5376 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5377 break;
5378 case OP_ISHR_IMM:
5379 case OP_LSHR_IMM:
5380 case OP_SHR_IMM:
5381 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
5382 break;
5383 case OP_ISHR_UN_IMM:
5384 /* This is used to implement conv.u4, so the lhs could be an i8 */
5385 lhs = convert (ctx, lhs, LLVMInt32Type ());
5386 imm = convert (ctx, imm, LLVMInt32Type ());
5387 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5388 break;
5389 case OP_LSHR_UN_IMM:
5390 case OP_SHR_UN_IMM:
5391 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5392 break;
5393 default:
5394 g_assert_not_reached ();
5395 }
5396 break;
5397 }
5398 case OP_INEG:
5399 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5400 break;
5401 case OP_LNEG:
5402 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5403 lhs = convert (ctx, lhs, LLVMInt64Type ());
5404 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
5405 break;
5406 case OP_FNEG:
5407 lhs = convert (ctx, lhs, LLVMDoubleType ());
5408 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
5409 break;
5410 case OP_RNEG:
5411 lhs = convert (ctx, lhs, LLVMFloatType ());
5412 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
5413 break;
5414 case OP_INOT: {
5415 guint32 v = 0xffffffff;
5416 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5417 break;
5418 }
5419 case OP_LNOT: {
5420 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5421 lhs = convert (ctx, lhs, LLVMInt64Type ());
5422 guint64 v = 0xffffffffffffffffLL;
5423 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
5424 break;
5425 }
5426 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5427 case OP_X86_LEA: {
5428 LLVMValueRef v1, v2;
5429
5430 rhs = LLVMBuildSExt (builder, convert (ctx, rhs, LLVMInt32Type ()), LLVMInt64Type (), "");
5431
5432 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ((unsigned long long)1 << ins->backend.shift_amount), FALSE), "");
5433 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
5434 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
5435 break;
5436 }
5437 #endif
5438
5439 case OP_ICONV_TO_I1:
5440 case OP_ICONV_TO_I2:
5441 case OP_ICONV_TO_I4:
5442 case OP_ICONV_TO_U1:
5443 case OP_ICONV_TO_U2:
5444 case OP_ICONV_TO_U4:
5445 case OP_LCONV_TO_I1:
5446 case OP_LCONV_TO_I2:
5447 case OP_LCONV_TO_U1:
5448 case OP_LCONV_TO_U2:
5449 case OP_LCONV_TO_U4: {
5450 gboolean sign;
5451
5452 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
5453
5454 /* Have to do two casts since our vregs have type int */
5455 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
5456 if (sign)
5457 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
5458 else
5459 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
5460 break;
5461 }
5462 case OP_ICONV_TO_I8:
5463 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
5464 break;
5465 case OP_ICONV_TO_U8:
5466 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
5467 break;
5468 case OP_FCONV_TO_I4:
5469 case OP_RCONV_TO_I4:
5470 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
5471 break;
5472 case OP_FCONV_TO_I1:
5473 case OP_RCONV_TO_I1:
5474 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
5475 break;
5476 case OP_FCONV_TO_U1:
5477 case OP_RCONV_TO_U1:
5478 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
5479 break;
5480 case OP_FCONV_TO_I2:
5481 case OP_RCONV_TO_I2:
5482 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5483 break;
5484 case OP_FCONV_TO_U2:
5485 case OP_RCONV_TO_U2:
5486 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5487 break;
5488 case OP_RCONV_TO_U4:
5489 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
5490 break;
5491 case OP_FCONV_TO_I8:
5492 case OP_RCONV_TO_I8:
5493 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
5494 break;
5495 case OP_FCONV_TO_I:
5496 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
5497 break;
5498 case OP_ICONV_TO_R8:
5499 case OP_LCONV_TO_R8:
5500 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
5501 break;
5502 case OP_ICONV_TO_R_UN:
5503 case OP_LCONV_TO_R_UN:
5504 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
5505 break;
5506 #if TARGET_SIZEOF_VOID_P == 4
5507 case OP_LCONV_TO_U:
5508 #endif
5509 case OP_LCONV_TO_I4:
5510 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5511 break;
5512 case OP_ICONV_TO_R4:
5513 case OP_LCONV_TO_R4:
5514 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
5515 if (cfg->r4fp)
5516 values [ins->dreg] = v;
5517 else
5518 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5519 break;
5520 case OP_FCONV_TO_R4:
5521 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
5522 if (cfg->r4fp)
5523 values [ins->dreg] = v;
5524 else
5525 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5526 break;
5527 case OP_RCONV_TO_R8:
5528 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
5529 break;
5530 case OP_RCONV_TO_R4:
5531 values [ins->dreg] = lhs;
5532 break;
5533 case OP_SEXT_I4:
5534 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5535 break;
5536 case OP_ZEXT_I4:
5537 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5538 break;
5539 case OP_TRUNC_I4:
5540 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5541 break;
5542 case OP_LOCALLOC_IMM: {
5543 LLVMValueRef v;
5544
5545 guint32 size = ins->inst_imm;
5546 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5547
5548 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
5549
5550 if (ins->flags & MONO_INST_INIT) {
5551 LLVMValueRef args [5];
5552
5553 args [0] = v;
5554 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5555 args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
5556 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
5557 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5558 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMSET), args, 5, "");
5559 }
5560
5561 values [ins->dreg] = v;
5562 break;
5563 }
5564 case OP_LOCALLOC: {
5565 LLVMValueRef v, size;
5566
5567 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
5568
5569 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
5570
5571 if (ins->flags & MONO_INST_INIT) {
5572 LLVMValueRef args [5];
5573
5574 args [0] = v;
5575 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5576 args [2] = size;
5577 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
5578 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5579 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMSET), args, 5, "");
5580 }
5581 values [ins->dreg] = v;
5582 break;
5583 }
5584
5585 case OP_LOADI1_MEMBASE:
5586 case OP_LOADU1_MEMBASE:
5587 case OP_LOADI2_MEMBASE:
5588 case OP_LOADU2_MEMBASE:
5589 case OP_LOADI4_MEMBASE:
5590 case OP_LOADU4_MEMBASE:
5591 case OP_LOADI8_MEMBASE:
5592 case OP_LOADR4_MEMBASE:
5593 case OP_LOADR8_MEMBASE:
5594 case OP_LOAD_MEMBASE:
5595 case OP_LOADI8_MEM:
5596 case OP_LOADU1_MEM:
5597 case OP_LOADU2_MEM:
5598 case OP_LOADI4_MEM:
5599 case OP_LOADU4_MEM:
5600 case OP_LOAD_MEM: {
5601 int size = 8;
5602 LLVMValueRef base, index, addr;
5603 LLVMTypeRef t;
5604 gboolean sext = FALSE, zext = FALSE;
5605 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5606
5607 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5608
5609 if (sext || zext)
5610 dname = (char*)"";
5611
5612 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5613 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5614 base = addr;
5615 } else {
5616 /* _MEMBASE */
5617 base = lhs;
5618
5619 if (ins->inst_offset == 0) {
5620 addr = base;
5621 } else if (ins->inst_offset % size != 0) {
5622 /* Unaligned load */
5623 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5624 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5625 } else {
5626 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5627 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5628 }
5629 }
5630
5631 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5632
5633 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, base, dname, is_volatile, LLVM_BARRIER_NONE);
5634
5635 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5636 /*
5637 * These will signal LLVM that these loads do not alias any stores, and
5638 * they can't fail, allowing them to be hoisted out of loops.
5639 */
5640 set_invariant_load_flag (values [ins->dreg]);
5641 }
5642
5643 if (sext)
5644 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5645 else if (zext)
5646 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5647 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5648 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5649 break;
5650 }
5651
5652 case OP_STOREI1_MEMBASE_REG:
5653 case OP_STOREI2_MEMBASE_REG:
5654 case OP_STOREI4_MEMBASE_REG:
5655 case OP_STOREI8_MEMBASE_REG:
5656 case OP_STORER4_MEMBASE_REG:
5657 case OP_STORER8_MEMBASE_REG:
5658 case OP_STORE_MEMBASE_REG: {
5659 int size = 8;
5660 LLVMValueRef index, addr, base;
5661 LLVMTypeRef t;
5662 gboolean sext = FALSE, zext = FALSE;
5663 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5664
5665 if (!values [ins->inst_destbasereg]) {
5666 set_failure (ctx, "inst_destbasereg");
5667 break;
5668 }
5669
5670 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5671
5672 base = values [ins->inst_destbasereg];
5673 if (ins->inst_offset % size != 0) {
5674 /* Unaligned store */
5675 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5676 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5677 } else {
5678 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5679 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5680 }
5681 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5682 break;
5683 }
5684
5685 case OP_STOREI1_MEMBASE_IMM:
5686 case OP_STOREI2_MEMBASE_IMM:
5687 case OP_STOREI4_MEMBASE_IMM:
5688 case OP_STOREI8_MEMBASE_IMM:
5689 case OP_STORE_MEMBASE_IMM: {
5690 int size = 8;
5691 LLVMValueRef index, addr, base;
5692 LLVMTypeRef t;
5693 gboolean sext = FALSE, zext = FALSE;
5694 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5695
5696 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5697
5698 base = values [ins->inst_destbasereg];
5699 if (ins->inst_offset % size != 0) {
5700 /* Unaligned store */
5701 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5702 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5703 } else {
5704 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5705 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5706 }
5707 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5708 break;
5709 }
5710
5711 case OP_CHECK_THIS:
5712 emit_load (ctx, bb, &builder, TARGET_SIZEOF_VOID_P, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), lhs, "", TRUE, LLVM_BARRIER_NONE);
5713 break;
5714 case OP_OUTARG_VTRETADDR:
5715 break;
5716 case OP_VOIDCALL:
5717 case OP_CALL:
5718 case OP_LCALL:
5719 case OP_FCALL:
5720 case OP_RCALL:
5721 case OP_VCALL:
5722 case OP_VOIDCALL_MEMBASE:
5723 case OP_CALL_MEMBASE:
5724 case OP_LCALL_MEMBASE:
5725 case OP_FCALL_MEMBASE:
5726 case OP_RCALL_MEMBASE:
5727 case OP_VCALL_MEMBASE:
5728 case OP_VOIDCALL_REG:
5729 case OP_CALL_REG:
5730 case OP_LCALL_REG:
5731 case OP_FCALL_REG:
5732 case OP_RCALL_REG:
5733 case OP_VCALL_REG: {
5734 process_call (ctx, bb, &builder, ins);
5735 break;
5736 }
5737 case OP_AOTCONST: {
5738 guint32 got_offset;
5739 LLVMValueRef indexes [2];
5740 MonoJumpInfo *tmp_ji, *ji;
5741 LLVMValueRef got_entry_addr;
5742 char *name;
5743
5744 /*
5745 * FIXME: Can't allocate from the cfg mempool since that is freed if
5746 * the LLVM compile fails.
5747 */
5748 tmp_ji = g_new0 (MonoJumpInfo, 1);
5749 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
5750 tmp_ji->data.target = ins->inst_p0;
5751
5752 ji = mono_aot_patch_info_dup (tmp_ji);
5753 g_free (tmp_ji);
5754
5755 if (ji->type == MONO_PATCH_INFO_ICALL_ADDR) {
5756 char *symbol = mono_aot_get_direct_call_symbol (MONO_PATCH_INFO_ICALL_ADDR_CALL, ji->data.target);
5757 if (symbol) {
5758 /*
5759 * Avoid emitting a got entry for these since the method is directly called, and it might not be
5760 * resolvable at runtime using dlsym ().
5761 */
5762 g_free (symbol);
5763 values [ins->dreg] = LLVMConstInt (IntPtrType (), 0, FALSE);
5764 break;
5765 }
5766 }
5767
5768 ji->next = cfg->patch_info;
5769 cfg->patch_info = ji;
5770
5771 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
5772 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, NULL);
5773
5774 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
5775 if (!mono_aot_is_shared_got_offset (got_offset)) {
5776 //mono_print_ji (ji);
5777 //printf ("\n");
5778 ctx->cfg->got_access_count ++;
5779 }
5780
5781 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5782 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
5783 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
5784
5785 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
5786 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
5787 g_free (name);
5788 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5789 if (!cfg->llvm_only || mono_aot_is_shared_got_offset (got_offset)) {
5790 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5791 set_invariant_load_flag (values [ins->dreg]);
5792 }
5793
5794 if (ji->type == MONO_PATCH_INFO_LDSTR)
5795 set_nonnull_load_flag (values [ins->dreg]);
5796 break;
5797 }
5798 case OP_NOT_REACHED:
5799 LLVMBuildUnreachable (builder);
5800 has_terminator = TRUE;
5801 g_assert (bb->block_num < cfg->max_block_num);
5802 ctx->unreachable [bb->block_num] = TRUE;
5803 /* Might have instructions after this */
5804 while (ins->next) {
5805 MonoInst *next = ins->next;
5806 /*
5807 * FIXME: If later code uses the regs defined by these instructions,
5808 * compilation will fail.
5809 */
5810 MONO_DELETE_INS (bb, next);
5811 }
5812 break;
5813 case OP_LDADDR: {
5814 MonoInst *var = ins->inst_i0;
5815
5816 if (var->opcode == OP_VTARG_ADDR) {
5817 /* The variable contains the vtype address */
5818 values [ins->dreg] = values [var->dreg];
5819 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
5820 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
5821 } else {
5822 values [ins->dreg] = addresses [var->dreg];
5823 }
5824 break;
5825 }
5826 case OP_SIN: {
5827 LLVMValueRef args [1];
5828
5829 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5830 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SIN), args, 1, dname);
5831 break;
5832 }
5833 case OP_SINF: {
5834 LLVMValueRef args [1];
5835
5836 args [0] = convert (ctx, lhs, LLVMFloatType ());
5837 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SINF), args, 1, dname);
5838 break;
5839 }
5840 case OP_COS: {
5841 LLVMValueRef args [1];
5842
5843 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5844 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COS), args, 1, dname);
5845 break;
5846 }
5847 case OP_COSF: {
5848 LLVMValueRef args [1];
5849
5850 args [0] = convert (ctx, lhs, LLVMFloatType ());
5851 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COSF), args, 1, dname);
5852 break;
5853 }
5854 case OP_SQRT: {
5855 LLVMValueRef args [1];
5856
5857 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5858 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRT), args, 1, dname);
5859 break;
5860 }
5861 case OP_SQRTF: {
5862 LLVMValueRef args [1];
5863
5864 args [0] = convert (ctx, lhs, LLVMFloatType ());
5865 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRTF), args, 1, dname);
5866 break;
5867 }
5868 case OP_ABS: {
5869 LLVMValueRef args [1];
5870
5871 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5872 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
5873 break;
5874 }
5875 case OP_ABSF: {
5876 LLVMValueRef args [1];
5877
5878 #ifdef TARGET_AMD64
5879 args [0] = convert (ctx, lhs, LLVMFloatType ());
5880 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_ABSF), args, 1, dname);
5881 #else
5882 /* llvm.fabs not supported on all platforms */
5883 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5884 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
5885 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
5886 #endif
5887 break;
5888 }
5889 case OP_RPOW: {
5890 LLVMValueRef args [2];
5891
5892 args [0] = convert (ctx, lhs, LLVMFloatType ());
5893 args [1] = convert (ctx, rhs, LLVMFloatType ());
5894 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_POWF), args, 2, dname);
5895 break;
5896 }
5897
5898 case OP_IMIN:
5899 case OP_LMIN:
5900 case OP_IMAX:
5901 case OP_LMAX:
5902 case OP_IMIN_UN:
5903 case OP_LMIN_UN:
5904 case OP_IMAX_UN:
5905 case OP_LMAX_UN:
5906 case OP_RMAX: {
5907 LLVMValueRef v;
5908
5909 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5910 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5911
5912 switch (ins->opcode) {
5913 case OP_IMIN:
5914 case OP_LMIN:
5915 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
5916 break;
5917 case OP_IMAX:
5918 case OP_LMAX:
5919 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
5920 break;
5921 case OP_IMIN_UN:
5922 case OP_LMIN_UN:
5923 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
5924 break;
5925 case OP_IMAX_UN:
5926 case OP_LMAX_UN:
5927 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
5928 break;
5929 case OP_RMAX:
5930 v = LLVMBuildFCmp (builder, LLVMRealUGE, lhs, rhs, "");
5931 break;
5932 default:
5933 g_assert_not_reached ();
5934 break;
5935 }
5936 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
5937 break;
5938 }
5939
5940 /*
5941 * See the ARM64 comment in mono/utils/atomic.h for an explanation of why this
5942 * hack is necessary (for now).
5943 */
5944 #ifdef TARGET_ARM64
5945 #define ARM64_ATOMIC_FENCE_FIX mono_llvm_build_fence (builder, LLVM_BARRIER_SEQ)
5946 #else
5947 #define ARM64_ATOMIC_FENCE_FIX
5948 #endif
5949
5950 case OP_ATOMIC_EXCHANGE_I4:
5951 case OP_ATOMIC_EXCHANGE_I8: {
5952 LLVMValueRef args [2];
5953 LLVMTypeRef t;
5954
5955 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
5956 t = LLVMInt32Type ();
5957 else
5958 t = LLVMInt64Type ();
5959
5960 g_assert (ins->inst_offset == 0);
5961
5962 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5963 args [1] = convert (ctx, rhs, t);
5964
5965 ARM64_ATOMIC_FENCE_FIX;
5966 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
5967 ARM64_ATOMIC_FENCE_FIX;
5968 break;
5969 }
5970 case OP_ATOMIC_ADD_I4:
5971 case OP_ATOMIC_ADD_I8: {
5972 LLVMValueRef args [2];
5973 LLVMTypeRef t;
5974
5975 if (ins->opcode == OP_ATOMIC_ADD_I4)
5976 t = LLVMInt32Type ();
5977 else
5978 t = LLVMInt64Type ();
5979
5980 g_assert (ins->inst_offset == 0);
5981
5982 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5983 args [1] = convert (ctx, rhs, t);
5984 ARM64_ATOMIC_FENCE_FIX;
5985 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
5986 ARM64_ATOMIC_FENCE_FIX;
5987 break;
5988 }
5989 case OP_ATOMIC_CAS_I4:
5990 case OP_ATOMIC_CAS_I8: {
5991 LLVMValueRef args [3], val;
5992 LLVMTypeRef t;
5993
5994 if (ins->opcode == OP_ATOMIC_CAS_I4)
5995 t = LLVMInt32Type ();
5996 else
5997 t = LLVMInt64Type ();
5998
5999 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6000 /* comparand */
6001 args [1] = convert (ctx, values [ins->sreg3], t);
6002 /* new value */
6003 args [2] = convert (ctx, values [ins->sreg2], t);
6004 ARM64_ATOMIC_FENCE_FIX;
6005 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
6006 ARM64_ATOMIC_FENCE_FIX;
6007 /* cmpxchg returns a pair */
6008 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
6009 break;
6010 }
6011 case OP_MEMORY_BARRIER: {
6012 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
6013 break;
6014 }
6015 case OP_ATOMIC_LOAD_I1:
6016 case OP_ATOMIC_LOAD_I2:
6017 case OP_ATOMIC_LOAD_I4:
6018 case OP_ATOMIC_LOAD_I8:
6019 case OP_ATOMIC_LOAD_U1:
6020 case OP_ATOMIC_LOAD_U2:
6021 case OP_ATOMIC_LOAD_U4:
6022 case OP_ATOMIC_LOAD_U8:
6023 case OP_ATOMIC_LOAD_R4:
6024 case OP_ATOMIC_LOAD_R8: {
6025 int size;
6026 gboolean sext, zext;
6027 LLVMTypeRef t;
6028 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
6029 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6030 LLVMValueRef index, addr;
6031
6032 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6033
6034 if (sext || zext)
6035 dname = (char *)"";
6036
6037 if (ins->inst_offset != 0) {
6038 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6039 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
6040 } else {
6041 addr = lhs;
6042 }
6043
6044 addr = convert (ctx, addr, LLVMPointerType (t, 0));
6045
6046 ARM64_ATOMIC_FENCE_FIX;
6047 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, lhs, dname, is_volatile, barrier);
6048 ARM64_ATOMIC_FENCE_FIX;
6049
6050 if (sext)
6051 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6052 else if (zext)
6053 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6054 break;
6055 }
6056 case OP_ATOMIC_STORE_I1:
6057 case OP_ATOMIC_STORE_I2:
6058 case OP_ATOMIC_STORE_I4:
6059 case OP_ATOMIC_STORE_I8:
6060 case OP_ATOMIC_STORE_U1:
6061 case OP_ATOMIC_STORE_U2:
6062 case OP_ATOMIC_STORE_U4:
6063 case OP_ATOMIC_STORE_U8:
6064 case OP_ATOMIC_STORE_R4:
6065 case OP_ATOMIC_STORE_R8: {
6066 int size;
6067 gboolean sext, zext;
6068 LLVMTypeRef t;
6069 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
6070 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6071 LLVMValueRef index, addr, value, base;
6072
6073 if (!values [ins->inst_destbasereg]) {
6074 set_failure (ctx, "inst_destbasereg");
6075 break;
6076 }
6077
6078 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6079
6080 base = values [ins->inst_destbasereg];
6081 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6082 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
6083 value = convert (ctx, values [ins->sreg1], t);
6084
6085 ARM64_ATOMIC_FENCE_FIX;
6086 emit_store_general (ctx, bb, &builder, size, value, addr, base, is_volatile, barrier);
6087 ARM64_ATOMIC_FENCE_FIX;
6088 break;
6089 }
6090 case OP_RELAXED_NOP: {
6091 #if defined(TARGET_AMD64) || defined(TARGET_X86)
6092 emit_call (ctx, bb, &builder, get_intrins_by_name (ctx, "llvm.x86.sse2.pause"), NULL, 0);
6093 break;
6094 #else
6095 break;
6096 #endif
6097 }
6098 case OP_TLS_GET: {
6099 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
6100 #ifdef TARGET_AMD64
6101 // 257 == FS segment register
6102 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
6103 #else
6104 // 256 == GS segment register
6105 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6106 #endif
6107 // FIXME: XEN
6108 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
6109 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
6110 /* See mono_amd64_emit_tls_get () */
6111 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
6112
6113 // 256 == GS segment register
6114 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6115 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
6116 #else
6117 set_failure (ctx, "opcode tls-get");
6118 break;
6119 #endif
6120
6121 break;
6122 }
6123 case OP_GC_SAFE_POINT: {
6124 LLVMValueRef val, cmp, callee;
6125 LLVMBasicBlockRef poll_bb, cont_bb;
6126 LLVMValueRef args [2];
6127 static LLVMTypeRef sig;
6128 MonoJitICallId const icall_id = MONO_JIT_ICALL_mono_threads_state_poll;
6129 const char *icall_name = "mono_threads_state_poll";
6130
6131 if (!sig)
6132 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
6133
6134 /*
6135 * if (!*sreg1)
6136 * mono_threads_state_poll ();
6137 * FIXME: Use a preserveall wrapper
6138 */
6139 val = mono_llvm_build_load (builder, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
6140 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
6141 poll_bb = gen_bb (ctx, "POLL_BB");
6142 cont_bb = gen_bb (ctx, "CONT_BB");
6143
6144 args [0] = cmp;
6145 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
6146 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
6147
6148 LLVMBuildCondBr (builder, cmp, cont_bb, poll_bb);
6149
6150 ctx->builder = builder = create_builder (ctx);
6151 LLVMPositionBuilderAtEnd (builder, poll_bb);
6152
6153 if (ctx->cfg->compile_aot) {
6154 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
6155 } else {
6156 callee = get_jit_callee (ctx, icall_name, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
6157 }
6158 LLVMBuildCall (builder, callee, NULL, 0, "");
6159 LLVMBuildBr (builder, cont_bb);
6160
6161 ctx->builder = builder = create_builder (ctx);
6162 LLVMPositionBuilderAtEnd (builder, cont_bb);
6163 ctx->bblocks [bb->block_num].end_bblock = cont_bb;
6164 break;
6165 }
6166
6167 /*
6168 * Overflow opcodes.
6169 */
6170 case OP_IADD_OVF:
6171 case OP_IADD_OVF_UN:
6172 case OP_ISUB_OVF:
6173 case OP_ISUB_OVF_UN:
6174 case OP_IMUL_OVF:
6175 case OP_IMUL_OVF_UN:
6176 case OP_LADD_OVF:
6177 case OP_LADD_OVF_UN:
6178 case OP_LSUB_OVF:
6179 case OP_LSUB_OVF_UN:
6180 case OP_LMUL_OVF:
6181 case OP_LMUL_OVF_UN:
6182 {
6183 LLVMValueRef args [2], val, ovf, func;
6184
6185 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
6186 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
6187 func = get_intrins_by_name (ctx, ovf_op_to_intrins (ins->opcode));
6188 g_assert (func);
6189 val = LLVMBuildCall (builder, func, args, 2, "");
6190 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
6191 ovf = LLVMBuildExtractValue (builder, val, 1, "");
6192 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
6193 if (!ctx_ok (ctx))
6194 break;
6195 builder = ctx->builder;
6196 break;
6197 }
6198
6199 /*
6200 * Valuetypes.
6201 * We currently model them using arrays. Promotion to local vregs is
6202 * disabled for them in mono_handle_global_vregs () in the LLVM case,
6203 * so we always have an entry in cfg->varinfo for them.
6204 * FIXME: Is this needed ?
6205 */
6206 case OP_VZERO: {
6207 MonoClass *klass = ins->klass;
6208 LLVMValueRef args [5];
6209
6210 if (!klass) {
6211 // FIXME:
6212 set_failure (ctx, "!klass");
6213 break;
6214 }
6215
6216 if (!addresses [ins->dreg])
6217 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6218 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6219 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
6220 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
6221 // FIXME: Alignment
6222 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6223 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6224 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMSET), args, 5, "");
6225 break;
6226 }
6227 case OP_DUMMY_VZERO:
6228 break;
6229
6230 case OP_STOREV_MEMBASE:
6231 case OP_LOADV_MEMBASE:
6232 case OP_VMOVE: {
6233 MonoClass *klass = ins->klass;
6234 LLVMValueRef src = NULL, dst, args [5];
6235 gboolean done = FALSE;
6236
6237 if (!klass) {
6238 // FIXME:
6239 set_failure (ctx, "!klass");
6240 break;
6241 }
6242
6243 if (mini_is_gsharedvt_klass (klass)) {
6244 // FIXME:
6245 set_failure (ctx, "gsharedvt");
6246 break;
6247 }
6248
6249 switch (ins->opcode) {
6250 case OP_STOREV_MEMBASE:
6251 if (cfg->gen_write_barriers && m_class_has_references (klass) && ins->inst_destbasereg != cfg->frame_reg &&
6252 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
6253 /* Decomposed earlier */
6254 g_assert_not_reached ();
6255 break;
6256 }
6257 if (!addresses [ins->sreg1]) {
6258 /* SIMD */
6259 g_assert (values [ins->sreg1]);
6260 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (klass)), 0));
6261 LLVMBuildStore (builder, values [ins->sreg1], dst);
6262 done = TRUE;
6263 } else {
6264 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6265 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6266 }
6267 break;
6268 case OP_LOADV_MEMBASE:
6269 if (!addresses [ins->dreg])
6270 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6271 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6272 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6273 break;
6274 case OP_VMOVE:
6275 if (!addresses [ins->sreg1])
6276 addresses [ins->sreg1] = build_alloca (ctx, m_class_get_byval_arg (klass));
6277 if (!addresses [ins->dreg])
6278 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6279 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6280 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6281 break;
6282 default:
6283 g_assert_not_reached ();
6284 }
6285 if (!ctx_ok (ctx))
6286 break;
6287
6288 if (done)
6289 break;
6290
6291 args [0] = dst;
6292 args [1] = src;
6293 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
6294 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6295 // FIXME: Alignment
6296 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6297 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6298 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMCPY), args, 5, "");
6299 break;
6300 }
6301 case OP_LLVM_OUTARG_VT: {
6302 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
6303 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
6304
6305 if (ainfo->storage == LLVMArgGsharedvtVariable) {
6306 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
6307
6308 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
6309 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
6310 } else {
6311 g_assert (addresses [ins->sreg1]);
6312 addresses [ins->dreg] = addresses [ins->sreg1];
6313 }
6314 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
6315 if (!addresses [ins->sreg1]) {
6316 addresses [ins->sreg1] = build_alloca (ctx, t);
6317 g_assert (values [ins->sreg1]);
6318 }
6319 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
6320 addresses [ins->dreg] = addresses [ins->sreg1];
6321 } else {
6322 if (!addresses [ins->sreg1]) {
6323 addresses [ins->sreg1] = build_alloca (ctx, t);
6324 g_assert (values [ins->sreg1]);
6325 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
6326 addresses [ins->dreg] = addresses [ins->sreg1];
6327 } else if (values [ins->sreg1] == addresses [ins->sreg1]) {
6328 /* LLVMArgVtypeByRef, have to make a copy */
6329 addresses [ins->dreg] = build_alloca (ctx, t);
6330 LLVMValueRef v = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
6331 LLVMBuildStore (builder, convert (ctx, v, type_to_llvm_type (ctx, t)), addresses [ins->dreg]);
6332 } else {
6333 addresses [ins->dreg] = addresses [ins->sreg1];
6334 }
6335 }
6336 break;
6337 }
6338 case OP_OBJC_GET_SELECTOR: {
6339 const char *name = (const char*)ins->inst_p0;
6340 LLVMValueRef var;
6341
6342 if (!ctx->module->objc_selector_to_var) {
6343 ctx->module->objc_selector_to_var = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
6344
6345 LLVMValueRef info_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), 8), "@OBJC_IMAGE_INFO");
6346 int32_t objc_imageinfo [] = { 0, 16 };
6347 LLVMSetInitializer (info_var, mono_llvm_create_constant_data_array ((uint8_t *) &objc_imageinfo, 8));
6348 LLVMSetLinkage (info_var, LLVMPrivateLinkage);
6349 LLVMSetExternallyInitialized (info_var, TRUE);
6350 LLVMSetSection (info_var, "__DATA, __objc_imageinfo,regular,no_dead_strip");
6351 LLVMSetAlignment (info_var, sizeof (target_mgreg_t));
6352 mark_as_used (ctx->module, info_var);
6353 }
6354
6355 var = (LLVMValueRef)g_hash_table_lookup (ctx->module->objc_selector_to_var, name);
6356 if (!var) {
6357 LLVMValueRef indexes [16];
6358
6359 LLVMValueRef name_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), strlen (name) + 1), "@OBJC_METH_VAR_NAME_");
6360 LLVMSetInitializer (name_var, mono_llvm_create_constant_data_array ((const uint8_t*)name, strlen (name) + 1));
6361 LLVMSetLinkage (name_var, LLVMPrivateLinkage);
6362 LLVMSetSection (name_var, "__TEXT,__objc_methname,cstring_literals");
6363 mark_as_used (ctx->module, name_var);
6364
6365 LLVMValueRef ref_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (LLVMInt8Type (), 0), "@OBJC_SELECTOR_REFERENCES_");
6366
6367 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6368 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6369 LLVMSetInitializer (ref_var, LLVMConstGEP (name_var, indexes, 2));
6370 LLVMSetLinkage (ref_var, LLVMPrivateLinkage);
6371 LLVMSetExternallyInitialized (ref_var, TRUE);
6372 LLVMSetSection (ref_var, "__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
6373 LLVMSetAlignment (ref_var, sizeof (target_mgreg_t));
6374 mark_as_used (ctx->module, ref_var);
6375
6376 g_hash_table_insert (ctx->module->objc_selector_to_var, g_strdup (name), ref_var);
6377 var = ref_var;
6378 }
6379
6380 values [ins->dreg] = LLVMBuildLoad (builder, var, "");
6381 break;
6382 }
6383
6384 /*
6385 * SIMD
6386 */
6387 #if defined(TARGET_X86) || defined(TARGET_AMD64)
6388 case OP_XZERO: {
6389 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
6390 break;
6391 }
6392 case OP_LOADX_MEMBASE: {
6393 LLVMTypeRef t = type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass));
6394 LLVMValueRef src;
6395
6396 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6397 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
6398 break;
6399 }
6400 case OP_STOREX_MEMBASE: {
6401 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
6402 LLVMValueRef dest;
6403
6404 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6405 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
6406 break;
6407 }
6408 case OP_PADDB:
6409 case OP_PADDW:
6410 case OP_PADDD:
6411 case OP_PADDQ:
6412 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
6413 break;
6414 case OP_ADDPD:
6415 case OP_ADDPS:
6416 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
6417 break;
6418 case OP_PSUBB:
6419 case OP_PSUBW:
6420 case OP_PSUBD:
6421 case OP_PSUBQ:
6422 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
6423 break;
6424 case OP_SUBPD:
6425 case OP_SUBPS:
6426 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
6427 break;
6428 case OP_MULPD:
6429 case OP_MULPS:
6430 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
6431 break;
6432 case OP_DIVPD:
6433 case OP_DIVPS:
6434 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
6435 break;
6436 case OP_PAND:
6437 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
6438 break;
6439 case OP_POR:
6440 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
6441 break;
6442 case OP_PXOR:
6443 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
6444 break;
6445 case OP_PMULW:
6446 case OP_PMULD:
6447 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
6448 break;
6449 case OP_ANDPS:
6450 case OP_ANDNPS:
6451 case OP_ORPS:
6452 case OP_XORPS:
6453 case OP_ANDPD:
6454 case OP_ANDNPD:
6455 case OP_ORPD:
6456 case OP_XORPD: {
6457 LLVMTypeRef t, rt;
6458 LLVMValueRef v = NULL;
6459
6460 switch (ins->opcode) {
6461 case OP_ANDPS:
6462 case OP_ANDNPS:
6463 case OP_ORPS:
6464 case OP_XORPS:
6465 t = LLVMVectorType (LLVMInt32Type (), 4);
6466 rt = LLVMVectorType (LLVMFloatType (), 4);
6467 break;
6468 case OP_ANDPD:
6469 case OP_ANDNPD:
6470 case OP_ORPD:
6471 case OP_XORPD:
6472 t = LLVMVectorType (LLVMInt64Type (), 2);
6473 rt = LLVMVectorType (LLVMDoubleType (), 2);
6474 break;
6475 default:
6476 t = LLVMInt32Type ();
6477 rt = LLVMInt32Type ();
6478 g_assert_not_reached ();
6479 }
6480
6481 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6482 rhs = LLVMBuildBitCast (builder, rhs, t, "");
6483 switch (ins->opcode) {
6484 case OP_ANDPS:
6485 case OP_ANDPD:
6486 v = LLVMBuildAnd (builder, lhs, rhs, "");
6487 break;
6488 case OP_ORPS:
6489 case OP_ORPD:
6490 v = LLVMBuildOr (builder, lhs, rhs, "");
6491 break;
6492 case OP_XORPS:
6493 case OP_XORPD:
6494 v = LLVMBuildXor (builder, lhs, rhs, "");
6495 break;
6496 case OP_ANDNPS:
6497 case OP_ANDNPD:
6498 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
6499 break;
6500 }
6501 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
6502 break;
6503 }
6504 case OP_PMIND_UN:
6505 case OP_PMINW_UN:
6506 case OP_PMINB_UN: {
6507 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntULT, lhs, rhs, "");
6508 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6509 break;
6510 }
6511 case OP_PMAXD_UN:
6512 case OP_PMAXW_UN:
6513 case OP_PMAXB_UN: {
6514 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntUGT, lhs, rhs, "");
6515 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6516 break;
6517 }
6518 case OP_PMINW: {
6519 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntSLT, lhs, rhs, "");
6520 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6521 break;
6522 }
6523 case OP_MINPD:
6524 case OP_MINPS:
6525 case OP_MAXPD:
6526 case OP_MAXPS:
6527 case OP_ADDSUBPD:
6528 case OP_ADDSUBPS:
6529 case OP_HADDPD:
6530 case OP_HADDPS:
6531 case OP_HSUBPD:
6532 case OP_HSUBPS:
6533 case OP_PADDB_SAT:
6534 case OP_PADDW_SAT:
6535 case OP_PSUBB_SAT:
6536 case OP_PSUBW_SAT:
6537 case OP_PADDB_SAT_UN:
6538 case OP_PADDW_SAT_UN:
6539 case OP_PSUBB_SAT_UN:
6540 case OP_PSUBW_SAT_UN:
6541 case OP_PACKW:
6542 case OP_PACKD:
6543 case OP_PACKW_UN:
6544 case OP_PACKD_UN:
6545 case OP_PMULW_HIGH:
6546 case OP_PMULW_HIGH_UN: {
6547 LLVMValueRef args [2];
6548
6549 args [0] = lhs;
6550 args [1] = rhs;
6551
6552 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6553 break;
6554 }
6555 case OP_PAVGB_UN:
6556 case OP_PAVGW_UN: {
6557 LLVMValueRef ones_vec;
6558 LLVMValueRef ones [32];
6559 int vector_size = LLVMGetVectorSize (LLVMTypeOf (lhs));
6560 LLVMTypeRef ext_elem_type = vector_size == 16 ? LLVMInt16Type () : LLVMInt32Type ();
6561
6562 for (int i = 0; i < 32; ++i)
6563 ones [i] = LLVMConstInt (ext_elem_type, 1, FALSE);
6564 ones_vec = LLVMConstVector (ones, vector_size);
6565
6566 LLVMValueRef val;
6567 LLVMTypeRef ext_type = LLVMVectorType (ext_elem_type, vector_size);
6568
6569 /* Have to increase the vector element size to prevent overflows */
6570 /* res = trunc ((zext (lhs) + zext (rhs) + 1) >> 1) */
6571 val = LLVMBuildAdd (builder, LLVMBuildZExt (builder, lhs, ext_type, ""), LLVMBuildZExt (builder, rhs, ext_type, ""), "");
6572 val = LLVMBuildAdd (builder, val, ones_vec, "");
6573 val = LLVMBuildLShr (builder, val, ones_vec, "");
6574 values [ins->dreg] = LLVMBuildTrunc (builder, val, LLVMTypeOf (lhs), "");
6575 break;
6576 }
6577 case OP_PCMPEQB:
6578 case OP_PCMPEQW:
6579 case OP_PCMPEQD:
6580 case OP_PCMPEQQ:
6581 case OP_PCMPGTB: {
6582 LLVMValueRef pcmp;
6583 LLVMTypeRef retType;
6584 int cmpOp;
6585
6586 if (ins->opcode == OP_PCMPGTB)
6587 cmpOp = LLVMIntSGT;
6588 else
6589 cmpOp = LLVMIntEQ;
6590
6591 if (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)) {
6592 pcmp = LLVMBuildICmp (builder, cmpOp, lhs, rhs, "");
6593 retType = LLVMTypeOf (lhs);
6594 } else {
6595 LLVMTypeRef flatType = LLVMVectorType (LLVMInt8Type (), 16);
6596 LLVMValueRef flatRHS = convert (ctx, rhs, flatType);
6597 LLVMValueRef flatLHS = convert (ctx, lhs, flatType);
6598
6599 pcmp = LLVMBuildICmp (builder, cmpOp, flatLHS, flatRHS, "");
6600 retType = flatType;
6601 }
6602
6603 values [ins->dreg] = LLVMBuildSExt (builder, pcmp, retType, "");
6604 break;
6605 }
6606 case OP_EXTRACT_R8:
6607 case OP_EXTRACT_I8:
6608 case OP_EXTRACT_I4:
6609 case OP_EXTRACT_I2:
6610 case OP_EXTRACT_U2:
6611 case OP_EXTRACTX_U2:
6612 case OP_EXTRACT_I1:
6613 case OP_EXTRACT_U1: {
6614 LLVMTypeRef t;
6615 gboolean zext = FALSE;
6616
6617 t = simd_op_to_llvm_type (ins->opcode);
6618
6619 switch (ins->opcode) {
6620 case OP_EXTRACT_R8:
6621 case OP_EXTRACT_I8:
6622 case OP_EXTRACT_I4:
6623 case OP_EXTRACT_I2:
6624 case OP_EXTRACT_I1:
6625 break;
6626 case OP_EXTRACT_U2:
6627 case OP_EXTRACTX_U2:
6628 case OP_EXTRACT_U1:
6629 zext = TRUE;
6630 break;
6631 default:
6632 t = LLVMInt32Type ();
6633 g_assert_not_reached ();
6634 }
6635
6636 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6637 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
6638 if (zext)
6639 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
6640 break;
6641 }
6642
6643 case OP_EXPAND_I1:
6644 case OP_EXPAND_I2:
6645 case OP_EXPAND_I4:
6646 case OP_EXPAND_I8:
6647 case OP_EXPAND_R4:
6648 case OP_EXPAND_R8: {
6649 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6650 LLVMValueRef mask [16], v;
6651 int i;
6652
6653 for (i = 0; i < 16; ++i)
6654 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6655
6656 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
6657
6658 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6659 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
6660 break;
6661 }
6662
6663 case OP_INSERT_I1:
6664 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6665 break;
6666 case OP_INSERT_I2:
6667 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6668 break;
6669 case OP_INSERT_I4:
6670 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6671 break;
6672 case OP_INSERT_I8:
6673 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6674 break;
6675 case OP_INSERT_R4:
6676 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6677 break;
6678 case OP_INSERT_R8:
6679 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6680 break;
6681
6682 case OP_CVTDQ2PD: {
6683 LLVMValueRef indexes [16];
6684
6685 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6686 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6687 LLVMValueRef mask = LLVMConstVector (indexes, 2);
6688 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
6689 values [ins->dreg] = LLVMBuildSIToFP (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
6690 break;
6691 }
6692 case OP_CVTPS2PD: {
6693 LLVMValueRef indexes [16];
6694
6695 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6696 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6697 LLVMValueRef mask = LLVMConstVector (indexes, 2);
6698 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
6699 values [ins->dreg] = LLVMBuildFPExt (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
6700 break;
6701 }
6702 case OP_CVTTPS2DQ:
6703 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMVectorType (LLVMInt32Type (), 4), dname);
6704 break;
6705
6706 case OP_CVTDQ2PS:
6707 case OP_CVTPD2DQ:
6708 case OP_CVTPS2DQ:
6709 case OP_CVTPD2PS:
6710 case OP_CVTTPD2DQ:
6711 case OP_EXTRACT_MASK:
6712 case OP_SQRTPS:
6713 case OP_SQRTPD:
6714 case OP_RSQRTPS:
6715 case OP_RCPPS: {
6716 LLVMValueRef v;
6717
6718 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
6719
6720 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
6721 break;
6722 }
6723 case OP_COMPPS:
6724 case OP_COMPPD: {
6725 LLVMRealPredicate op;
6726
6727 switch (ins->inst_c0) {
6728 case SIMD_COMP_EQ:
6729 op = LLVMRealOEQ;
6730 break;
6731 case SIMD_COMP_LT:
6732 op = LLVMRealOLT;
6733 break;
6734 case SIMD_COMP_LE:
6735 op = LLVMRealOLE;
6736 break;
6737 case SIMD_COMP_UNORD:
6738 op = LLVMRealUNO;
6739 break;
6740 case SIMD_COMP_NEQ:
6741 op = LLVMRealUNE;
6742 break;
6743 case SIMD_COMP_NLT:
6744 op = LLVMRealUGE;
6745 break;
6746 case SIMD_COMP_NLE:
6747 op = LLVMRealUGT;
6748 break;
6749 case SIMD_COMP_ORD:
6750 op = LLVMRealORD;
6751 break;
6752 default:
6753 g_assert_not_reached ();
6754 }
6755
6756 LLVMValueRef cmp = LLVMBuildFCmp (builder, op, lhs, rhs, "");
6757 if (ins->opcode == OP_COMPPD)
6758 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), 2), ""), LLVMTypeOf (lhs), "");
6759 else
6760 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), 4), ""), LLVMTypeOf (lhs), "");
6761 break;
6762 }
6763 case OP_ICONV_TO_X:
6764 /* This is only used for implementing shifts by non-immediate */
6765 values [ins->dreg] = lhs;
6766 break;
6767
6768 case OP_PSHRW:
6769 case OP_PSHRD:
6770 case OP_PSHRQ:
6771 case OP_PSARW:
6772 case OP_PSARD:
6773 case OP_PSHLW:
6774 case OP_PSHLD:
6775 case OP_PSHLQ: {
6776 LLVMValueRef args [3];
6777
6778 args [0] = lhs;
6779 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
6780
6781 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6782 break;
6783 }
6784
6785 case OP_PSHRW_REG:
6786 case OP_PSHRD_REG:
6787 case OP_PSHRQ_REG:
6788 case OP_PSARW_REG:
6789 case OP_PSARD_REG:
6790 case OP_PSHLW_REG:
6791 case OP_PSHLD_REG:
6792 case OP_PSHLQ_REG: {
6793 LLVMValueRef args [3];
6794
6795 args [0] = lhs;
6796 args [1] = values [ins->sreg2];
6797
6798 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6799 break;
6800 }
6801
6802 case OP_SHUFPS:
6803 case OP_SHUFPD:
6804 case OP_PSHUFLED:
6805 case OP_PSHUFLEW_LOW:
6806 case OP_PSHUFLEW_HIGH: {
6807 int mask [16];
6808 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
6809 int i, mask_size = 0;
6810 int imask = ins->inst_c0;
6811
6812 /* Convert the x86 shuffle mask to LLVM's */
6813 switch (ins->opcode) {
6814 case OP_SHUFPS:
6815 mask_size = 4;
6816 mask [0] = ((imask >> 0) & 3);
6817 mask [1] = ((imask >> 2) & 3);
6818 mask [2] = ((imask >> 4) & 3) + 4;
6819 mask [3] = ((imask >> 6) & 3) + 4;
6820 v1 = values [ins->sreg1];
6821 v2 = values [ins->sreg2];
6822 break;
6823 case OP_SHUFPD:
6824 mask_size = 2;
6825 mask [0] = ((imask >> 0) & 1);
6826 mask [1] = ((imask >> 1) & 1) + 2;
6827 v1 = values [ins->sreg1];
6828 v2 = values [ins->sreg2];
6829 break;
6830 case OP_PSHUFLEW_LOW:
6831 mask_size = 8;
6832 mask [0] = ((imask >> 0) & 3);
6833 mask [1] = ((imask >> 2) & 3);
6834 mask [2] = ((imask >> 4) & 3);
6835 mask [3] = ((imask >> 6) & 3);
6836 mask [4] = 4 + 0;
6837 mask [5] = 4 + 1;
6838 mask [6] = 4 + 2;
6839 mask [7] = 4 + 3;
6840 v1 = values [ins->sreg1];
6841 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6842 break;
6843 case OP_PSHUFLEW_HIGH:
6844 mask_size = 8;
6845 mask [0] = 0;
6846 mask [1] = 1;
6847 mask [2] = 2;
6848 mask [3] = 3;
6849 mask [4] = 4 + ((imask >> 0) & 3);
6850 mask [5] = 4 + ((imask >> 2) & 3);
6851 mask [6] = 4 + ((imask >> 4) & 3);
6852 mask [7] = 4 + ((imask >> 6) & 3);
6853 v1 = values [ins->sreg1];
6854 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6855 break;
6856 case OP_PSHUFLED:
6857 mask_size = 4;
6858 mask [0] = ((imask >> 0) & 3);
6859 mask [1] = ((imask >> 2) & 3);
6860 mask [2] = ((imask >> 4) & 3);
6861 mask [3] = ((imask >> 6) & 3);
6862 v1 = values [ins->sreg1];
6863 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6864 break;
6865 default:
6866 g_assert_not_reached ();
6867 }
6868 for (i = 0; i < mask_size; ++i)
6869 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6870
6871 values [ins->dreg] =
6872 LLVMBuildShuffleVector (builder, v1, v2,
6873 LLVMConstVector (mask_values, mask_size), dname);
6874 break;
6875 }
6876
6877 case OP_UNPACK_LOWB:
6878 case OP_UNPACK_LOWW:
6879 case OP_UNPACK_LOWD:
6880 case OP_UNPACK_LOWQ:
6881 case OP_UNPACK_LOWPS:
6882 case OP_UNPACK_LOWPD:
6883 case OP_UNPACK_HIGHB:
6884 case OP_UNPACK_HIGHW:
6885 case OP_UNPACK_HIGHD:
6886 case OP_UNPACK_HIGHQ:
6887 case OP_UNPACK_HIGHPS:
6888 case OP_UNPACK_HIGHPD: {
6889 int mask [16];
6890 LLVMValueRef mask_values [16];
6891 int i, mask_size = 0;
6892 gboolean low = FALSE;
6893
6894 switch (ins->opcode) {
6895 case OP_UNPACK_LOWB:
6896 mask_size = 16;
6897 low = TRUE;
6898 break;
6899 case OP_UNPACK_LOWW:
6900 mask_size = 8;
6901 low = TRUE;
6902 break;
6903 case OP_UNPACK_LOWD:
6904 case OP_UNPACK_LOWPS:
6905 mask_size = 4;
6906 low = TRUE;
6907 break;
6908 case OP_UNPACK_LOWQ:
6909 case OP_UNPACK_LOWPD:
6910 mask_size = 2;
6911 low = TRUE;
6912 break;
6913 case OP_UNPACK_HIGHB:
6914 mask_size = 16;
6915 break;
6916 case OP_UNPACK_HIGHW:
6917 mask_size = 8;
6918 break;
6919 case OP_UNPACK_HIGHD:
6920 case OP_UNPACK_HIGHPS:
6921 mask_size = 4;
6922 break;
6923 case OP_UNPACK_HIGHQ:
6924 case OP_UNPACK_HIGHPD:
6925 mask_size = 2;
6926 break;
6927 default:
6928 g_assert_not_reached ();
6929 }
6930
6931 if (low) {
6932 for (i = 0; i < (mask_size / 2); ++i) {
6933 mask [(i * 2)] = i;
6934 mask [(i * 2) + 1] = mask_size + i;
6935 }
6936 } else {
6937 for (i = 0; i < (mask_size / 2); ++i) {
6938 mask [(i * 2)] = (mask_size / 2) + i;
6939 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
6940 }
6941 }
6942
6943 for (i = 0; i < mask_size; ++i)
6944 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6945
6946 values [ins->dreg] =
6947 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
6948 LLVMConstVector (mask_values, mask_size), dname);
6949 break;
6950 }
6951
6952 case OP_DUPPD: {
6953 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6954 LLVMValueRef v, val;
6955
6956 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6957 val = LLVMConstNull (t);
6958 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6959 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
6960
6961 values [ins->dreg] = val;
6962 break;
6963 }
6964 case OP_DUPPS_LOW:
6965 case OP_DUPPS_HIGH: {
6966 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6967 LLVMValueRef v1, v2, val;
6968
6969
6970 if (ins->opcode == OP_DUPPS_LOW) {
6971 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6972 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6973 } else {
6974 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6975 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6976 }
6977 val = LLVMConstNull (t);
6978 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6979 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6980 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6981 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6982
6983 values [ins->dreg] = val;
6984 break;
6985 }
6986
6987 case OP_DPPS: {
6988 LLVMValueRef args [3];
6989
6990 args [0] = lhs;
6991 args [1] = rhs;
6992 /* 0xf1 == multiply all 4 elements, add them together, and store the result to the lowest element */
6993 args [2] = LLVMConstInt (LLVMInt8Type (), 0xf1, FALSE);
6994
6995 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 3, dname);
6996 break;
6997 }
6998
6999 case OP_FCONV_TO_R8_X: {
7000 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (type_to_simd_type (MONO_TYPE_R8)), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7001 break;
7002 }
7003
7004 case OP_SSE41_ROUNDPD: {
7005 LLVMValueRef args [3];
7006
7007 args [0] = lhs;
7008 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
7009
7010 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, "llvm.x86.sse41.round.pd"), args, 2, dname);
7011 break;
7012 }
7013
7014 #endif /* SIMD */
7015
7016 case OP_DUMMY_USE:
7017 break;
7018
7019 /*
7020 * EXCEPTION HANDLING
7021 */
7022 case OP_IMPLICIT_EXCEPTION:
7023 /* This marks a place where an implicit exception can happen */
7024 if (bb->region != -1)
7025 set_failure (ctx, "implicit-exception");
7026 break;
7027 case OP_THROW:
7028 case OP_RETHROW: {
7029 gboolean rethrow = (ins->opcode == OP_RETHROW);
7030 if (ctx->llvm_only) {
7031 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
7032 has_terminator = TRUE;
7033 ctx->unreachable [bb->block_num] = TRUE;
7034 } else {
7035 emit_throw (ctx, bb, rethrow, lhs);
7036 builder = ctx->builder;
7037 }
7038 break;
7039 }
7040 case OP_CALL_HANDLER: {
7041 /*
7042 * We don't 'call' handlers, but instead simply branch to them.
7043 * The code generated by ENDFINALLY will branch back to us.
7044 */
7045 LLVMBasicBlockRef noex_bb;
7046 GSList *bb_list;
7047 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
7048
7049 bb_list = info->call_handler_return_bbs;
7050
7051 /*
7052 * Set the indicator variable for the finally clause.
7053 */
7054 lhs = info->finally_ind;
7055 g_assert (lhs);
7056 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
7057
7058 /* Branch to the finally clause */
7059 LLVMBuildBr (builder, info->call_handler_target_bb);
7060
7061 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
7062 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
7063
7064 builder = ctx->builder = create_builder (ctx);
7065 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
7066
7067 bblocks [bb->block_num].end_bblock = noex_bb;
7068 break;
7069 }
7070 case OP_START_HANDLER: {
7071 break;
7072 }
7073 case OP_ENDFINALLY: {
7074 LLVMBasicBlockRef resume_bb;
7075 MonoBasicBlock *handler_bb;
7076 LLVMValueRef val, switch_ins, callee;
7077 GSList *bb_list;
7078 BBInfo *info;
7079 gboolean is_fault = MONO_REGION_FLAGS (bb->region) == MONO_EXCEPTION_CLAUSE_FAULT;
7080
7081 /*
7082 * Fault clauses are like finally clauses, but they are only called if an exception is thrown.
7083 */
7084 if (!is_fault) {
7085 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
7086 g_assert (handler_bb);
7087 info = &bblocks [handler_bb->block_num];
7088 lhs = info->finally_ind;
7089 g_assert (lhs);
7090
7091 bb_list = info->call_handler_return_bbs;
7092
7093 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
7094
7095 /* Load the finally variable */
7096 val = LLVMBuildLoad (builder, lhs, "");
7097
7098 /* Reset the variable */
7099 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
7100
7101 /* Branch to either resume_bb, or to the bblocks in bb_list */
7102 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
7103 /*
7104 * The other targets are added at the end to handle OP_CALL_HANDLER
7105 * opcodes processed later.
7106 */
7107 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
7108
7109 builder = ctx->builder = create_builder (ctx);
7110 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
7111 }
7112
7113 if (ctx->llvm_only) {
7114 emit_resume_eh (ctx, bb);
7115 } else {
7116 LLVMTypeRef icall_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
7117 if (ctx->cfg->compile_aot) {
7118 callee = get_callee (ctx, icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7119 } else {
7120 callee = get_jit_callee (ctx, "llvm_resume_unwind_trampoline", icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7121 }
7122 LLVMBuildCall (builder, callee, NULL, 0, "");
7123 LLVMBuildUnreachable (builder);
7124 }
7125
7126 has_terminator = TRUE;
7127 break;
7128 }
7129 case OP_IL_SEQ_POINT:
7130 break;
7131 default: {
7132 char reason [128];
7133
7134 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
7135 set_failure (ctx, reason);
7136 break;
7137 }
7138 }
7139
7140 if (!ctx_ok (ctx))
7141 break;
7142
7143 /* Convert the value to the type required by phi nodes */
7144 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
7145 if (ctx->is_vphi [ins->dreg])
7146 /* vtypes */
7147 values [ins->dreg] = addresses [ins->dreg];
7148 else
7149 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
7150 }
7151
7152 /* Add stores for volatile variables */
7153 if (!skip_volatile_store && spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
7154 emit_volatile_store (ctx, ins->dreg);
7155 }
7156
7157 if (!ctx_ok (ctx))
7158 return;
7159
7160 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
7161 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
7162 }
7163
7164 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
7165 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
7166 LLVMBuildRetVoid (builder);
7167 }
7168
7169 if (bb == cfg->bb_entry)
7170 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
7171 }
7172
7173 /*
7174 * mono_llvm_check_method_supported:
7175 *
7176 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
7177 * compiling a method twice.
7178 */
7179 void
7180 mono_llvm_check_method_supported (MonoCompile *cfg)
7181 {
7182 int i, j;
7183
7184 #ifdef TARGET_WASM
7185 if (mono_method_signature_internal (cfg->method)->call_convention == MONO_CALL_VARARG) {
7186 cfg->exception_message = g_strdup ("vararg callconv");
7187 cfg->disable_llvm = TRUE;
7188 return;
7189 }
7190 #endif
7191
7192 if (cfg->llvm_only)
7193 return;
7194
7195 if (cfg->method->save_lmf) {
7196 cfg->exception_message = g_strdup ("lmf");
7197 cfg->disable_llvm = TRUE;
7198 }
7199 if (cfg->disable_llvm)
7200 return;
7201
7202 /*
7203 * Nested clauses where one of the clauses is a finally clause is
7204 * not supported, because LLVM can't figure out the control flow,
7205 * probably because we resume exception handling by calling our
7206 * own function instead of using the 'resume' llvm instruction.
7207 */
7208 for (i = 0; i < cfg->header->num_clauses; ++i) {
7209 for (j = 0; j < cfg->header->num_clauses; ++j) {
7210 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
7211 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
7212
7213 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
7214 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7215 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
7216 cfg->exception_message = g_strdup ("nested clauses");
7217 cfg->disable_llvm = TRUE;
7218 break;
7219 }
7220 }
7221 }
7222 if (cfg->disable_llvm)
7223 return;
7224
7225 /* FIXME: */
7226 if (cfg->method->dynamic) {
7227 cfg->exception_message = g_strdup ("dynamic.");
7228 cfg->disable_llvm = TRUE;
7229 }
7230 if (cfg->disable_llvm)
7231 return;
7232 }
7233
7234 static LLVMCallInfo*
7235 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
7236 {
7237 LLVMCallInfo *linfo;
7238 int i;
7239
7240 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
7241 int i, n, pindex;
7242
7243 /*
7244 * Gsharedvt methods have the following calling convention:
7245 * - all arguments are passed by ref, even non generic ones
7246 * - the return value is returned by ref too, using a vret
7247 * argument passed after 'this'.
7248 */
7249 n = sig->param_count + sig->hasthis;
7250 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
7251
7252 pindex = 0;
7253 if (sig->hasthis)
7254 linfo->args [pindex ++].storage = LLVMArgNormal;
7255
7256 if (sig->ret->type != MONO_TYPE_VOID) {
7257 if (mini_is_gsharedvt_variable_type (sig->ret))
7258 linfo->ret.storage = LLVMArgGsharedvtVariable;
7259 else if (mini_type_is_vtype (sig->ret))
7260 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
7261 else
7262 linfo->ret.storage = LLVMArgGsharedvtFixed;
7263 linfo->vret_arg_index = pindex;
7264 } else {
7265 linfo->ret.storage = LLVMArgNone;
7266 }
7267
7268 for (i = 0; i < sig->param_count; ++i) {
7269 if (sig->params [i]->byref)
7270 linfo->args [pindex].storage = LLVMArgNormal;
7271 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
7272 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
7273 else if (mini_type_is_vtype (sig->params [i]))
7274 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
7275 else
7276 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
7277 linfo->args [pindex].type = sig->params [i];
7278 pindex ++;
7279 }
7280 return linfo;
7281 }
7282
7283 linfo = mono_arch_get_llvm_call_info (cfg, sig);
7284 linfo->dummy_arg_pindex = -1;
7285 for (i = 0; i < sig->param_count; ++i)
7286 linfo->args [i + sig->hasthis].type = sig->params [i];
7287
7288 return linfo;
7289 }
7290
7291 static void
7292 emit_method_inner (EmitContext *ctx);
7293
7294 static void
7295 free_ctx (EmitContext *ctx)
7296 {
7297 GSList *l;
7298
7299 g_free (ctx->values);
7300 g_free (ctx->addresses);
7301 g_free (ctx->vreg_types);
7302 g_free (ctx->is_vphi);
7303 g_free (ctx->vreg_cli_types);
7304 g_free (ctx->is_dead);
7305 g_free (ctx->unreachable);
7306 g_ptr_array_free (ctx->phi_values, TRUE);
7307 g_free (ctx->bblocks);
7308 g_hash_table_destroy (ctx->region_to_handler);
7309 g_hash_table_destroy (ctx->clause_to_handler);
7310 g_hash_table_destroy (ctx->jit_callees);
7311
7312 g_ptr_array_free (ctx->callsite_list, TRUE);
7313
7314 g_free (ctx->method_name);
7315 g_ptr_array_free (ctx->bblock_list, TRUE);
7316
7317 for (l = ctx->builders; l; l = l->next) {
7318 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
7319 LLVMDisposeBuilder (builder);
7320 }
7321
7322 g_free (ctx);
7323 }
7324
7325 static gboolean
7326 is_externally_callable (EmitContext *ctx, MonoMethod *method)
7327 {
7328 if (ctx->module->llvm_only && ctx->module->static_link && (method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE || method_is_direct_callable (method)))
7329 return TRUE;
7330 return FALSE;
7331 }
7332
7333 /*
7334 * mono_llvm_emit_method:
7335 *
7336 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
7337 */
7338 void
7339 mono_llvm_emit_method (MonoCompile *cfg)
7340 {
7341 EmitContext *ctx;
7342 char *method_name;
7343 int i;
7344
7345 if (cfg->skip)
7346 return;
7347
7348 /* The code below might acquire the loader lock, so use it for global locking */
7349 mono_loader_lock ();
7350
7351 /* Used to communicate with the callbacks */
7352 mono_native_tls_set_value (current_cfg_tls_id, cfg);
7353
7354 ctx = g_new0 (EmitContext, 1);
7355 ctx->cfg = cfg;
7356 ctx->mempool = cfg->mempool;
7357
7358 /*
7359 * This maps vregs to the LLVM instruction defining them
7360 */
7361 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
7362 /*
7363 * This maps vregs for volatile variables to the LLVM instruction defining their
7364 * address.
7365 */
7366 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
7367 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
7368 ctx->is_vphi = g_new0 (gboolean, cfg->next_vreg);
7369 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
7370 ctx->phi_values = g_ptr_array_sized_new (256);
7371 /*
7372 * This signals whenever the vreg was defined by a phi node with no input vars
7373 * (i.e. all its input bblocks end with NOT_REACHABLE).
7374 */
7375 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
7376 /* Whenever the bblock is unreachable */
7377 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
7378 ctx->bblock_list = g_ptr_array_sized_new (256);
7379
7380 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
7381 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
7382 ctx->callsite_list = g_ptr_array_new ();
7383 ctx->jit_callees = g_hash_table_new (NULL, NULL);
7384 if (cfg->compile_aot) {
7385 ctx->module = &aot_module;
7386
7387 if (is_externally_callable (ctx, cfg->method))
7388 method_name = mono_aot_get_mangled_method_name (cfg->method);
7389 else
7390 method_name = mono_aot_get_method_name (cfg);
7391 cfg->llvm_method_name = g_strdup (method_name);
7392 } else {
7393 init_jit_module (cfg->domain);
7394 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
7395 method_name = mono_method_full_name (cfg->method, TRUE);
7396 }
7397 ctx->method_name = method_name;
7398
7399 if (cfg->compile_aot)
7400 ctx->lmodule = ctx->module->lmodule;
7401 else
7402 ctx->lmodule = LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg->method->name));
7403 ctx->llvm_only = ctx->module->llvm_only;
7404 #ifdef TARGET_WASM
7405 ctx->emit_dummy_arg = TRUE;
7406 #endif
7407
7408 emit_method_inner (ctx);
7409
7410 if (!ctx_ok (ctx)) {
7411 if (ctx->lmethod) {
7412 /* Need to add unused phi nodes as they can be referenced by other values */
7413 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
7414 LLVMBuilderRef builder;
7415
7416 builder = create_builder (ctx);
7417 LLVMPositionBuilderAtEnd (builder, phi_bb);
7418
7419 for (i = 0; i < ctx->phi_values->len; ++i) {
7420 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
7421 if (LLVMGetInstructionParent (v) == NULL)
7422 LLVMInsertIntoBuilder (builder, v);
7423 }
7424
7425 if (ctx->module->llvm_only && ctx->module->static_link) {
7426 // Keep a stub for the function since it might be called directly
7427 int nbbs = LLVMCountBasicBlocks (ctx->lmethod);
7428 LLVMBasicBlockRef *bblocks = g_new0 (LLVMBasicBlockRef, nbbs);
7429 LLVMGetBasicBlocks (ctx->lmethod, bblocks);
7430 for (int i = 0; i < nbbs; ++i)
7431 LLVMDeleteBasicBlock (bblocks [i]);
7432
7433 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (ctx->lmethod, "ENTRY");
7434 builder = create_builder (ctx);
7435 LLVMPositionBuilderAtEnd (builder, entry_bb);
7436 ctx->builder = builder;
7437
7438 LLVMTypeRef sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
7439 LLVMValueRef callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
7440 LLVMBuildCall (builder, callee, NULL, 0, "");
7441 LLVMBuildUnreachable (builder);
7442 } else {
7443 LLVMDeleteFunction (ctx->lmethod);
7444 }
7445 }
7446 }
7447
7448 free_ctx (ctx);
7449
7450 mono_native_tls_set_value (current_cfg_tls_id, NULL);
7451
7452 mono_loader_unlock ();
7453 }
7454
7455 static void
7456 emit_method_inner (EmitContext *ctx)
7457 {
7458 MonoCompile *cfg = ctx->cfg;
7459 MonoMethodSignature *sig;
7460 MonoBasicBlock *bb;
7461 LLVMTypeRef method_type;
7462 LLVMValueRef method = NULL;
7463 LLVMValueRef *values = ctx->values;
7464 int i, max_block_num, bb_index;
7465 gboolean last = FALSE;
7466 LLVMCallInfo *linfo;
7467 LLVMModuleRef lmodule = ctx->lmodule;
7468 BBInfo *bblocks;
7469 GPtrArray *bblock_list = ctx->bblock_list;
7470 MonoMethodHeader *header;
7471 MonoExceptionClause *clause;
7472 char **names;
7473 LLVMBuilderRef entry_builder = NULL;
7474 LLVMBasicBlockRef entry_bb = NULL;
7475
7476 if (cfg->gsharedvt && !cfg->llvm_only) {
7477 set_failure (ctx, "gsharedvt");
7478 return;
7479 }
7480
7481 #if 1
7482 {
7483 static int count = 0;
7484 count ++;
7485
7486 char *llvm_count_str = g_getenv ("LLVM_COUNT");
7487 if (llvm_count_str) {
7488 int lcount = atoi (llvm_count_str);
7489 g_free (llvm_count_str);
7490 if (count == lcount) {
7491 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
7492 fflush (stdout);
7493 last = TRUE;
7494 }
7495 if (count > lcount) {
7496 set_failure (ctx, "count");
7497 return;
7498 }
7499 }
7500 }
7501 #endif
7502
7503 // If we come upon one of the init_method wrappers, we need to find
7504 // the method that we have already emitted and tell LLVM that this
7505 // managed method info for the wrapper is associated with this method
7506 // we constructed ourselves from LLVM IR.
7507 //
7508 // This is necessary to unwind through the init_method, in the case that
7509 // it has to run a static cctor that throws an exception
7510 if (cfg->method->wrapper_type == MONO_WRAPPER_OTHER) {
7511 WrapperInfo *info = mono_marshal_get_wrapper_info (cfg->method);
7512 if (info->subtype == WRAPPER_SUBTYPE_AOT_INIT) {
7513 method = get_init_icall_wrapper (ctx->module, info->d.aot_init.subtype);
7514 ctx->lmethod = method;
7515 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
7516
7517 const char *init_name = mono_marshal_get_aot_init_wrapper_name (info->d.aot_init.subtype);
7518 ctx->method_name = g_strdup_printf ("%s%s", ctx->module->global_prefix, init_name);
7519 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
7520
7521 if (!cfg->llvm_only && ctx->module->external_symbols) {
7522 LLVMSetLinkage (method, LLVMExternalLinkage);
7523 LLVMSetVisibility (method, LLVMHiddenVisibility);
7524 }
7525
7526 goto after_codegen;
7527 }
7528 }
7529
7530 sig = mono_method_signature_internal (cfg->method);
7531 ctx->sig = sig;
7532
7533 linfo = get_llvm_call_info (cfg, sig);
7534 ctx->linfo = linfo;
7535 if (!ctx_ok (ctx))
7536 return;
7537
7538 if (cfg->rgctx_var)
7539 linfo->rgctx_arg = TRUE;
7540 else if (needs_extra_arg (ctx, cfg->method))
7541 linfo->dummy_arg = TRUE;
7542 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
7543 if (!ctx_ok (ctx))
7544 return;
7545
7546 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
7547 ctx->lmethod = method;
7548
7549 if (!cfg->llvm_only)
7550 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
7551
7552 /* if the method doesn't contain
7553 * (1) a call (so it's a leaf method)
7554 * (2) and no loops
7555 * we can skip the GC safepoint on method entry. */
7556 gboolean requires_safepoint;
7557 requires_safepoint = cfg->has_calls;
7558 if (!requires_safepoint) {
7559 for (bb = cfg->bb_entry->next_bb; bb; bb = bb->next_bb) {
7560 if (bb->loop_body_start || (bb->flags & BB_EXCEPTION_HANDLER)) {
7561 requires_safepoint = TRUE;
7562 }
7563 }
7564 }
7565
7566 if (!cfg->llvm_only && cfg->compile_aot && mono_threads_are_safepoints_enabled () && requires_safepoint)
7567 LLVMSetGC (method, "coreclr");
7568 LLVMSetLinkage (method, LLVMPrivateLinkage);
7569
7570 mono_llvm_add_func_attr (method, LLVM_ATTR_UW_TABLE);
7571
7572 if (cfg->compile_aot) {
7573 if (is_externally_callable (ctx, cfg->method)) {
7574 LLVMSetLinkage (method, LLVMExternalLinkage);
7575 } else {
7576 LLVMSetLinkage (method, LLVMInternalLinkage);
7577 //all methods have internal visibility when doing llvm_only
7578 if (!cfg->llvm_only && ctx->module->external_symbols) {
7579 LLVMSetLinkage (method, LLVMExternalLinkage);
7580 LLVMSetVisibility (method, LLVMHiddenVisibility);
7581 }
7582 }
7583 } else {
7584 LLVMSetLinkage (method, LLVMExternalLinkage);
7585 }
7586
7587 if (cfg->method->save_lmf && !cfg->llvm_only) {
7588 set_failure (ctx, "lmf");
7589 return;
7590 }
7591
7592 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
7593 set_failure (ctx, "pinvoke signature");
7594 return;
7595 }
7596
7597 header = cfg->header;
7598 for (i = 0; i < header->num_clauses; ++i) {
7599 clause = &header->clauses [i];
7600 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_FAULT && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
7601 set_failure (ctx, "non-finally/catch/fault clause.");
7602 return;
7603 }
7604 }
7605 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
7606 /* We can't handle inlined methods with clauses */
7607 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
7608
7609 if (linfo->rgctx_arg) {
7610 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
7611 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
7612 /*
7613 * We mark the rgctx parameter with the inreg attribute, which is mapped to
7614 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
7615 * CC_X86_64_Mono in X86CallingConv.td.
7616 */
7617 if (!ctx->llvm_only)
7618 mono_llvm_add_param_attr (ctx->rgctx_arg, LLVM_ATTR_IN_REG);
7619 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
7620 } else {
7621 ctx->rgctx_arg_pindex = -1;
7622 }
7623 if (cfg->vret_addr) {
7624 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
7625 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
7626 if (linfo->ret.storage == LLVMArgVtypeByRef) {
7627 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
7628 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
7629 }
7630 }
7631
7632 if (sig->hasthis) {
7633 ctx->this_arg_pindex = linfo->this_arg_pindex;
7634 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
7635 values [cfg->args [0]->dreg] = ctx->this_arg;
7636 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
7637 }
7638 if (linfo->dummy_arg)
7639 LLVMSetValueName (LLVMGetParam (method, linfo->dummy_arg_pindex), "dummy_arg");
7640
7641 names = g_new (char *, sig->param_count);
7642 mono_method_get_param_names (cfg->method, (const char **) names);
7643
7644 /* Set parameter names/attributes */
7645 for (i = 0; i < sig->param_count; ++i) {
7646 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
7647 char *name;
7648 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
7649 int j;
7650
7651 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
7652 name = g_strdup_printf ("dummy_%d_%d", i, j);
7653 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
7654 g_free (name);
7655 }
7656
7657 if (ainfo->storage == LLVMArgVtypeInReg && ainfo->pair_storage [0] == LLVMArgNone && ainfo->pair_storage [1] == LLVMArgNone)
7658 continue;
7659
7660 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
7661 if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
7662 if (names [i] && names [i][0] != '\0')
7663 name = g_strdup_printf ("p_arg_%s", names [i]);
7664 else
7665 name = g_strdup_printf ("p_arg_%d", i);
7666 } else {
7667 if (names [i] && names [i][0] != '\0')
7668 name = g_strdup_printf ("arg_%s", names [i]);
7669 else
7670 name = g_strdup_printf ("arg_%d", i);
7671 }
7672 LLVMSetValueName (LLVMGetParam (method, pindex), name);
7673 g_free (name);
7674 if (ainfo->storage == LLVMArgVtypeByVal)
7675 mono_llvm_add_param_attr (LLVMGetParam (method, pindex), LLVM_ATTR_BY_VAL);
7676
7677 if (ainfo->storage == LLVMArgVtypeByRef) {
7678 /* For OP_LDADDR */
7679 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
7680 }
7681 }
7682 g_free (names);
7683
7684 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
7685 ctx->minfo = mono_debug_lookup_method (cfg->method);
7686 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
7687 }
7688
7689 max_block_num = 0;
7690 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
7691 max_block_num = MAX (max_block_num, bb->block_num);
7692 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
7693
7694 /* Add branches between non-consecutive bblocks */
7695 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7696 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
7697 bb->next_bb != bb->last_ins->inst_false_bb) {
7698
7699 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
7700 inst->opcode = OP_BR;
7701 inst->inst_target_bb = bb->last_ins->inst_false_bb;
7702 mono_bblock_add_inst (bb, inst);
7703 }
7704 }
7705
7706 /*
7707 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
7708 */
7709 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7710 MonoInst *ins;
7711 LLVMBuilderRef builder;
7712 char *dname;
7713 char dname_buf[128];
7714
7715 builder = create_builder (ctx);
7716
7717 for (ins = bb->code; ins; ins = ins->next) {
7718 switch (ins->opcode) {
7719 case OP_PHI:
7720 case OP_FPHI:
7721 case OP_VPHI:
7722 case OP_XPHI: {
7723 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
7724
7725 if (!ctx_ok (ctx))
7726 return;
7727
7728 if (ins->opcode == OP_VPHI) {
7729 /* Treat valuetype PHI nodes as operating on the address itself */
7730 g_assert (ins->klass);
7731 phi_type = LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)), 0);
7732 }
7733
7734 /*
7735 * Have to precreate these, as they can be referenced by
7736 * earlier instructions.
7737 */
7738 sprintf (dname_buf, "t%d", ins->dreg);
7739 dname = dname_buf;
7740 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
7741
7742 if (ins->opcode == OP_VPHI)
7743 ctx->addresses [ins->dreg] = values [ins->dreg];
7744
7745 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
7746
7747 /*
7748 * Set the expected type of the incoming arguments since these have
7749 * to have the same type.
7750 */
7751 for (i = 0; i < ins->inst_phi_args [0]; i++) {
7752 int sreg1 = ins->inst_phi_args [i + 1];
7753
7754 if (sreg1 != -1) {
7755 if (ins->opcode == OP_VPHI)
7756 ctx->is_vphi [sreg1] = TRUE;
7757 ctx->vreg_types [sreg1] = phi_type;
7758 }
7759 }
7760 break;
7761 }
7762 case OP_LDADDR:
7763 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
7764 break;
7765 default:
7766 break;
7767 }
7768 }
7769 }
7770
7771 /*
7772 * Create an ordering for bblocks, use the depth first order first, then
7773 * put the exception handling bblocks last.
7774 */
7775 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
7776 bb = cfg->bblocks [bb_index];
7777 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
7778 g_ptr_array_add (bblock_list, bb);
7779 bblocks [bb->block_num].added = TRUE;
7780 }
7781 }
7782
7783 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7784 if (!bblocks [bb->block_num].added)
7785 g_ptr_array_add (bblock_list, bb);
7786 }
7787
7788 /*
7789 * Second pass: generate code.
7790 */
7791 // Emit entry point
7792 entry_builder = create_builder (ctx);
7793 entry_bb = get_bb (ctx, cfg->bb_entry);
7794 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
7795 emit_entry_bb (ctx, entry_builder);
7796
7797 // Make landing pads first
7798 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
7799
7800 if (ctx->llvm_only) {
7801 size_t group_index = 0;
7802 while (group_index < cfg->header->num_clauses) {
7803 int count = 0;
7804 size_t cursor = group_index;
7805 while (cursor < cfg->header->num_clauses &&
7806 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
7807 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
7808 count++;
7809 cursor++;
7810 }
7811
7812 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
7813 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
7814 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
7815
7816 group_index = cursor;
7817 }
7818 }
7819
7820 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
7821 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
7822
7823 // Prune unreachable mono BBs.
7824 if (!(bb == cfg->bb_entry || bb->in_count > 0))
7825 continue;
7826
7827 process_bb (ctx, bb);
7828 if (!ctx_ok (ctx))
7829 return;
7830 }
7831 g_hash_table_destroy (ctx->exc_meta);
7832
7833 mono_memory_barrier ();
7834
7835 /* Add incoming phi values */
7836 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7837 GSList *l, *ins_list;
7838
7839 ins_list = bblocks [bb->block_num].phi_nodes;
7840
7841 for (l = ins_list; l; l = l->next) {
7842 PhiNode *node = (PhiNode*)l->data;
7843 MonoInst *phi = node->phi;
7844 int sreg1 = node->sreg;
7845 LLVMBasicBlockRef in_bb;
7846
7847 if (sreg1 == -1)
7848 continue;
7849
7850 in_bb = get_end_bb (ctx, node->in_bb);
7851
7852 if (ctx->unreachable [node->in_bb->block_num])
7853 continue;
7854
7855 if (phi->opcode == OP_VPHI) {
7856 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
7857 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
7858 } else {
7859 if (!values [sreg1]) {
7860 /* Can happen with values in EH clauses */
7861 set_failure (ctx, "incoming phi sreg1");
7862 return;
7863 }
7864 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
7865 set_failure (ctx, "incoming phi arg type mismatch");
7866 return;
7867 }
7868 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
7869 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
7870 }
7871 }
7872 }
7873
7874 /* Nullify empty phi instructions */
7875 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7876 GSList *l, *ins_list;
7877
7878 ins_list = bblocks [bb->block_num].phi_nodes;
7879
7880 for (l = ins_list; l; l = l->next) {
7881 PhiNode *node = (PhiNode*)l->data;
7882 MonoInst *phi = node->phi;
7883 LLVMValueRef phi_ins = values [phi->dreg];
7884
7885 if (!phi_ins)
7886 /* Already removed */
7887 continue;
7888
7889 if (LLVMCountIncoming (phi_ins) == 0) {
7890 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
7891 LLVMInstructionEraseFromParent (phi_ins);
7892 values [phi->dreg] = NULL;
7893 }
7894 }
7895 }
7896
7897 /* Create the SWITCH statements for ENDFINALLY instructions */
7898 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7899 BBInfo *info = &bblocks [bb->block_num];
7900 GSList *l;
7901 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
7902 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
7903 GSList *bb_list = info->call_handler_return_bbs;
7904
7905 GSList *bb_list_iter;
7906 i = 0;
7907 for (bb_list_iter = bb_list; bb_list_iter; bb_list_iter = g_slist_next (bb_list_iter)) {
7908 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)bb_list_iter->data);
7909 i ++;
7910 }
7911 }
7912 }
7913
7914 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
7915
7916 /* Initialize the method if needed */
7917 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
7918 // FIXME: Add more shared got entries
7919 ctx->builder = create_builder (ctx);
7920 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
7921
7922 // FIXME: beforefieldinit
7923 /*
7924 * NATIVE_TO_MANAGED methods might be called on a thread not attached to the runtime, so they are initialized when loaded
7925 * in load_method ().
7926 */
7927 gboolean needs_init = ctx->cfg->got_access_count > 0;
7928 MonoMethod *cctor = NULL;
7929 if (!needs_init && (cctor = mono_class_get_cctor (cfg->method->klass))) {
7930 /* Needs init to run the cctor */
7931 if (cfg->method->flags & METHOD_ATTRIBUTE_STATIC)
7932 needs_init = TRUE;
7933 if (cctor == cfg->method)
7934 needs_init = FALSE;
7935
7936 // If we are a constructor, we need to init so the static
7937 // constructor gets called.
7938 if (!strcmp (cfg->method->name, ".ctor"))
7939 needs_init = TRUE;
7940 }
7941 if (cfg->method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED)
7942 needs_init = FALSE;
7943 if (needs_init)
7944 emit_init_method (ctx);
7945 else
7946 LLVMBuildBr (ctx->builder, ctx->inited_bb);
7947
7948 // Was observing LLVM moving field accesses into the caller's method
7949 // body before the init call (the inlined one), leading to NULL derefs
7950 // after the init_method returns (GOT is filled out though)
7951 if (needs_init)
7952 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
7953 }
7954
7955 if (mini_get_debug_options ()->llvm_disable_inlining)
7956 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
7957
7958
7959 after_codegen:
7960 if (cfg->llvm_only) {
7961 g_ptr_array_add (ctx->module->cfgs, cfg);
7962
7963 /*
7964 * Add the contents of ctx->callsite_list to module->callsite_list.
7965 * We can't do this earlier, as it contains llvm instructions which can be
7966 * freed if compilation fails.
7967 * FIXME: Get rid of this when all methods can be llvm compiled.
7968 */
7969 for (int i = 0; i < ctx->callsite_list->len; ++i)
7970 g_ptr_array_add (ctx->module->callsite_list, g_ptr_array_index (ctx->callsite_list, i));
7971 }
7972
7973 if (cfg->verbose_level > 1)
7974 mono_llvm_dump_value (method);
7975
7976 if (cfg->compile_aot && !cfg->llvm_only)
7977 mark_as_used (ctx->module, method);
7978
7979 if (!cfg->llvm_only) {
7980 LLVMValueRef md_args [16];
7981 LLVMValueRef md_node;
7982 int method_index;
7983
7984 if (cfg->compile_aot)
7985 method_index = mono_aot_get_method_index (cfg->orig_method);
7986 else
7987 method_index = 1;
7988 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
7989 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
7990 md_node = LLVMMDNode (md_args, 2);
7991 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
7992 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
7993 }
7994
7995 if (cfg->compile_aot) {
7996 /* Don't generate native code, keep the LLVM IR */
7997 if (cfg->verbose_level) {
7998 char *name = mono_method_get_full_name (cfg->method);
7999 printf ("%s emitted as %s\n", name, ctx->method_name);
8000 g_free (name);
8001 }
8002
8003 //LLVMDumpValue (ctx->lmethod);
8004 //int err = LLVMVerifyFunction(ctx->lmethod, LLVMPrintMessageAction);
8005 //g_assert (err == 0);
8006 } else {
8007 //LLVMVerifyFunction (method, 0);
8008 llvm_jit_finalize_method (ctx);
8009 }
8010
8011 if (ctx->module->method_to_lmethod)
8012 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
8013
8014 if (ctx->module->idx_to_lmethod)
8015 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
8016
8017 if (ctx->llvm_only && m_class_is_valuetype (cfg->orig_method->klass) && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
8018 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
8019 }
8020
8021 /*
8022 * mono_llvm_create_vars:
8023 *
8024 * Same as mono_arch_create_vars () for LLVM.
8025 */
8026 void
8027 mono_llvm_create_vars (MonoCompile *cfg)
8028 {
8029 MonoMethodSignature *sig;
8030
8031 sig = mono_method_signature_internal (cfg->method);
8032 if (cfg->gsharedvt && cfg->llvm_only) {
8033 gboolean vretaddr = FALSE;
8034
8035 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
8036 vretaddr = TRUE;
8037 } else {
8038 MonoMethodSignature *sig = mono_method_signature_internal (cfg->method);
8039 LLVMCallInfo *linfo;
8040
8041 linfo = get_llvm_call_info (cfg, sig);
8042 vretaddr = (linfo->ret.storage == LLVMArgVtypeRetAddr || linfo->ret.storage == LLVMArgVtypeByRef || linfo->ret.storage == LLVMArgGsharedvtFixed || linfo->ret.storage == LLVMArgGsharedvtVariable || linfo->ret.storage == LLVMArgGsharedvtFixedVtype);
8043 }
8044 if (vretaddr) {
8045 /*
8046 * Creating vret_addr forces CEE_SETRET to store the result into it,
8047 * so we don't have to generate any code in our OP_SETRET case.
8048 */
8049 cfg->vret_addr = mono_compile_create_var (cfg, m_class_get_byval_arg (mono_get_intptr_class ()), OP_ARG);
8050 if (G_UNLIKELY (cfg->verbose_level > 1)) {
8051 printf ("vret_addr = ");
8052 mono_print_ins (cfg->vret_addr);
8053 }
8054 }
8055 } else {
8056 mono_arch_create_vars (cfg);
8057 }
8058 }
8059
8060 /*
8061 * mono_llvm_emit_call:
8062 *
8063 * Same as mono_arch_emit_call () for LLVM.
8064 */
8065 void
8066 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
8067 {
8068 MonoInst *in;
8069 MonoMethodSignature *sig;
8070 int i, n, stack_size;
8071 LLVMArgInfo *ainfo;
8072
8073 stack_size = 0;
8074
8075 sig = call->signature;
8076 n = sig->param_count + sig->hasthis;
8077
8078 call->cinfo = get_llvm_call_info (cfg, sig);
8079
8080 if (cfg->disable_llvm)
8081 return;
8082
8083 if (sig->call_convention == MONO_CALL_VARARG) {
8084 cfg->exception_message = g_strdup ("varargs");
8085 cfg->disable_llvm = TRUE;
8086 }
8087
8088 for (i = 0; i < n; ++i) {
8089 MonoInst *ins;
8090
8091 ainfo = call->cinfo->args + i;
8092
8093 in = call->args [i];
8094
8095 /* Simply remember the arguments */
8096 switch (ainfo->storage) {
8097 case LLVMArgNormal: {
8098 MonoType *t = (sig->hasthis && i == 0) ? m_class_get_byval_arg (mono_get_intptr_class ()) : ainfo->type;
8099 int opcode;
8100
8101 opcode = mono_type_to_regmove (cfg, t);
8102 if (opcode == OP_FMOVE) {
8103 MONO_INST_NEW (cfg, ins, OP_FMOVE);
8104 ins->dreg = mono_alloc_freg (cfg);
8105 } else if (opcode == OP_LMOVE) {
8106 MONO_INST_NEW (cfg, ins, OP_LMOVE);
8107 ins->dreg = mono_alloc_lreg (cfg);
8108 } else if (opcode == OP_RMOVE) {
8109 MONO_INST_NEW (cfg, ins, OP_RMOVE);
8110 ins->dreg = mono_alloc_freg (cfg);
8111 } else {
8112 MONO_INST_NEW (cfg, ins, OP_MOVE);
8113 ins->dreg = mono_alloc_ireg (cfg);
8114 }
8115 ins->sreg1 = in->dreg;
8116 break;
8117 }
8118 case LLVMArgVtypeByVal:
8119 case LLVMArgVtypeByRef:
8120 case LLVMArgVtypeInReg:
8121 case LLVMArgVtypeAsScalar:
8122 case LLVMArgAsIArgs:
8123 case LLVMArgAsFpArgs:
8124 case LLVMArgGsharedvtVariable:
8125 case LLVMArgGsharedvtFixed:
8126 case LLVMArgGsharedvtFixedVtype:
8127 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
8128 ins->dreg = mono_alloc_ireg (cfg);
8129 ins->sreg1 = in->dreg;
8130 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
8131 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
8132 ins->inst_vtype = ainfo->type;
8133 ins->klass = mono_class_from_mono_type_internal (ainfo->type);
8134 break;
8135 default:
8136 cfg->exception_message = g_strdup ("ainfo->storage");
8137 cfg->disable_llvm = TRUE;
8138 return;
8139 }
8140
8141 if (!cfg->disable_llvm) {
8142 MONO_ADD_INS (cfg->cbb, ins);
8143 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
8144 }
8145 }
8146 }
8147
8148 static inline void
8149 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
8150 {
8151 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
8152 }
8153
8154 static inline void
8155 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
8156 {
8157 LLVMTypeRef param_types [4];
8158
8159 param_types [0] = param_type1;
8160 param_types [1] = param_type2;
8161
8162 AddFunc (module, name, ret_type, param_types, 2);
8163 }
8164
8165 typedef struct {
8166 IntrinsicId id;
8167 const char *name;
8168 } IntrinsicDesc;
8169
8170 static IntrinsicDesc intrinsics[] = {
8171 {INTRINS_MEMSET, "llvm.memset.p0i8.i32"},
8172 {INTRINS_MEMCPY, "llvm.memcpy.p0i8.p0i8.i32"},
8173 {INTRINS_SADD_OVF_I32, "llvm.sadd.with.overflow.i32"},
8174 {INTRINS_UADD_OVF_I32, "llvm.uadd.with.overflow.i32"},
8175 {INTRINS_SSUB_OVF_I32, "llvm.ssub.with.overflow.i32"},
8176 {INTRINS_USUB_OVF_I32, "llvm.usub.with.overflow.i32"},
8177 {INTRINS_SMUL_OVF_I32, "llvm.smul.with.overflow.i32"},
8178 {INTRINS_UMUL_OVF_I32, "llvm.umul.with.overflow.i32"},
8179 {INTRINS_SADD_OVF_I64, "llvm.sadd.with.overflow.i64"},
8180 {INTRINS_UADD_OVF_I64, "llvm.uadd.with.overflow.i64"},
8181 {INTRINS_SSUB_OVF_I64, "llvm.ssub.with.overflow.i64"},
8182 {INTRINS_USUB_OVF_I64, "llvm.usub.with.overflow.i64"},
8183 {INTRINS_SMUL_OVF_I64, "llvm.smul.with.overflow.i64"},
8184 {INTRINS_UMUL_OVF_I64, "llvm.umul.with.overflow.i64"},
8185 {INTRINS_SIN, "llvm.sin.f64"},
8186 {INTRINS_COS, "llvm.cos.f64"},
8187 {INTRINS_SQRT, "llvm.sqrt.f64"},
8188 /* This isn't an intrinsic, instead llvm seems to special case it by name */
8189 {INTRINS_FABS, "fabs"},
8190 {INTRINS_ABSF, "llvm.fabs.f32"},
8191 {INTRINS_SINF, "llvm.sin.f32"},
8192 {INTRINS_COSF, "llvm.cos.f32"},
8193 {INTRINS_SQRTF, "llvm.sqrt.f32"},
8194 {INTRINS_POWF, "llvm.pow.f32"},
8195 {INTRINS_EXPECT_I8, "llvm.expect.i8"},
8196 {INTRINS_EXPECT_I1, "llvm.expect.i1"},
8197 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8198 {INTRINS_SSE_PMOVMSKB, "llvm.x86.sse2.pmovmskb.128"},
8199 {INTRINS_SSE_PSRLI_W, "llvm.x86.sse2.psrli.w"},
8200 {INTRINS_SSE_PSRAI_W, "llvm.x86.sse2.psrai.w"},
8201 {INTRINS_SSE_PSLLI_W, "llvm.x86.sse2.pslli.w"},
8202 {INTRINS_SSE_PSRLI_D, "llvm.x86.sse2.psrli.d"},
8203 {INTRINS_SSE_PSRAI_D, "llvm.x86.sse2.psrai.d"},
8204 {INTRINS_SSE_PSLLI_D, "llvm.x86.sse2.pslli.d"},
8205 {INTRINS_SSE_PSRLI_Q, "llvm.x86.sse2.psrli.q"},
8206 {INTRINS_SSE_PSLLI_Q, "llvm.x86.sse2.pslli.q"},
8207 {INTRINS_SSE_SQRT_PD, "llvm.x86.sse2.sqrt.pd"},
8208 {INTRINS_SSE_SQRT_PS, "llvm.x86.sse.sqrt.ps"},
8209 {INTRINS_SSE_RSQRT_PS, "llvm.x86.sse.rsqrt.ps"},
8210 {INTRINS_SSE_RCP_PS, "llvm.x86.sse.rcp.ps"},
8211 {INTRINS_SSE_CVTTPD2DQ, "llvm.x86.sse2.cvttpd2dq"},
8212 {INTRINS_SSE_CVTTPS2DQ, "llvm.x86.sse2.cvttps2dq"},
8213 {INTRINS_SSE_CVTDQ2PD, "llvm.x86.sse2.cvtdq2pd"},
8214 {INTRINS_SSE_CVTDQ2PS, "llvm.x86.sse2.cvtdq2ps"},
8215 {INTRINS_SSE_CVTPD2DQ, "llvm.x86.sse2.cvtpd2dq"},
8216 {INTRINS_SSE_CVTPS2DQ, "llvm.x86.sse2.cvtps2dq"},
8217 {INTRINS_SSE_CVTPD2PS, "llvm.x86.sse2.cvtpd2ps"},
8218 {INTRINS_SSE_CVTPS2PD, "llvm.x86.sse2.cvtps2pd"},
8219 {INTRINS_SSE_CMPPD, "llvm.x86.sse2.cmp.pd"},
8220 {INTRINS_SSE_CMPPS, "llvm.x86.sse.cmp.ps"},
8221 {INTRINS_SSE_PACKSSWB, "llvm.x86.sse2.packsswb.128"},
8222 {INTRINS_SSE_PACKUSWB, "llvm.x86.sse2.packuswb.128"},
8223 {INTRINS_SSE_PACKSSDW, "llvm.x86.sse2.packssdw.128"},
8224 {INTRINS_SSE_PACKUSDW, "llvm.x86.sse41.packusdw"},
8225 {INTRINS_SSE_MINPS, "llvm.x86.sse.min.ps"},
8226 {INTRINS_SSE_MAXPS, "llvm.x86.sse.max.ps"},
8227 {INTRINS_SSE_HADDPS, "llvm.x86.sse3.hadd.ps"},
8228 {INTRINS_SSE_HSUBPS, "llvm.x86.sse3.hsub.ps"},
8229 {INTRINS_SSE_ADDSUBPS, "llvm.x86.sse3.addsub.ps"},
8230 {INTRINS_SSE_MINPD, "llvm.x86.sse2.min.pd"},
8231 {INTRINS_SSE_MAXPD, "llvm.x86.sse2.max.pd"},
8232 {INTRINS_SSE_HADDPD, "llvm.x86.sse3.hadd.pd"},
8233 {INTRINS_SSE_HSUBPD, "llvm.x86.sse3.hsub.pd"},
8234 {INTRINS_SSE_ADDSUBPD, "llvm.x86.sse3.addsub.pd"},
8235 {INTRINS_SSE_PADDSW, "llvm.x86.sse2.padds.w"},
8236 {INTRINS_SSE_PSUBSW, "llvm.x86.sse2.psubs.w"},
8237 {INTRINS_SSE_PADDUSW, "llvm.x86.sse2.paddus.w"},
8238 {INTRINS_SSE_PSUBUSW, "llvm.x86.sse2.psubus.w"},
8239 {INTRINS_SSE_PAVGW, "llvm.x86.sse2.pavg.w"},
8240 {INTRINS_SSE_PMULHW, "llvm.x86.sse2.pmulh.w"},
8241 {INTRINS_SSE_PMULHU, "llvm.x86.sse2.pmulhu.w"},
8242 {INTRINS_SE_PADDSB, "llvm.x86.sse2.padds.b"},
8243 {INTRINS_SSE_PSUBSB, "llvm.x86.sse2.psubs.b"},
8244 {INTRINS_SSE_PADDUSB, "llvm.x86.sse2.paddus.b"},
8245 {INTRINS_SSE_PSUBUSB, "llvm.x86.sse2.psubus.b"},
8246 {INTRINS_SSE_PAVGB, "llvm.x86.sse2.pavg.b"},
8247 {INTRINS_SSE_PAUSE, "llvm.x86.sse2.pause"},
8248 {INTRINS_SSE_DPPS, "llvm.x86.sse41.dpps"},
8249 {INTRINS_SSE_ROUNDPD, "llvm.x86.sse41.round.pd"}
8250 #endif
8251 };
8252
8253 static void
8254 add_sse_binary (LLVMModuleRef module, const char *name, int type)
8255 {
8256 LLVMTypeRef ret_type = type_to_simd_type (type);
8257 AddFunc2 (module, name, ret_type, ret_type, ret_type);
8258 }
8259
8260 static void
8261 add_intrinsic (LLVMModuleRef module, int id)
8262 {
8263 const char *name;
8264 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8265 LLVMTypeRef ret_type, arg_types [16];
8266 #endif
8267
8268 name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
8269 g_assert (name);
8270
8271 switch (id) {
8272 case INTRINS_MEMSET: {
8273 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
8274
8275 AddFunc (module, name, LLVMVoidType (), params, 5);
8276 break;
8277 }
8278 case INTRINS_MEMCPY: {
8279 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
8280
8281 AddFunc (module, name, LLVMVoidType (), params, 5);
8282 break;
8283 }
8284 case INTRINS_SADD_OVF_I32:
8285 case INTRINS_UADD_OVF_I32:
8286 case INTRINS_SSUB_OVF_I32:
8287 case INTRINS_USUB_OVF_I32:
8288 case INTRINS_SMUL_OVF_I32:
8289 case INTRINS_UMUL_OVF_I32: {
8290 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
8291 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
8292 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
8293
8294 AddFunc (module, name, ret_type, params, 2);
8295 break;
8296 }
8297 case INTRINS_SADD_OVF_I64:
8298 case INTRINS_UADD_OVF_I64:
8299 case INTRINS_SSUB_OVF_I64:
8300 case INTRINS_USUB_OVF_I64:
8301 case INTRINS_SMUL_OVF_I64:
8302 case INTRINS_UMUL_OVF_I64: {
8303 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
8304 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
8305 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
8306
8307 AddFunc (module, name, ret_type, params, 2);
8308 break;
8309 }
8310 case INTRINS_SIN:
8311 case INTRINS_COS:
8312 case INTRINS_SQRT:
8313 case INTRINS_FABS: {
8314 LLVMTypeRef params [] = { LLVMDoubleType () };
8315
8316 AddFunc (module, name, LLVMDoubleType (), params, 1);
8317 break;
8318 }
8319 case INTRINS_SINF:
8320 case INTRINS_COSF:
8321 case INTRINS_SQRTF:
8322 case INTRINS_ABSF: {
8323 LLVMTypeRef params [] = { LLVMFloatType () };
8324
8325 AddFunc (module, name, LLVMFloatType (), params, 1);
8326 break;
8327 }
8328 case INTRINS_POWF: {
8329 LLVMTypeRef params [] = { LLVMFloatType (), LLVMFloatType () };
8330
8331 AddFunc (module, name, LLVMFloatType (), params, 2);
8332 break;
8333 }
8334 case INTRINS_EXPECT_I8:
8335 AddFunc2 (module, name, LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
8336 break;
8337 case INTRINS_EXPECT_I1:
8338 AddFunc2 (module, name, LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type ());
8339 break;
8340 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8341 case INTRINS_SSE_PMOVMSKB:
8342 /* pmovmskb */
8343 ret_type = LLVMInt32Type ();
8344 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
8345 AddFunc (module, name, ret_type, arg_types, 1);
8346 break;
8347 case INTRINS_SSE_PSRLI_W:
8348 case INTRINS_SSE_PSRAI_W:
8349 case INTRINS_SSE_PSLLI_W:
8350 /* shifts */
8351 ret_type = type_to_simd_type (MONO_TYPE_I2);
8352 arg_types [0] = ret_type;
8353 arg_types [1] = LLVMInt32Type ();
8354 AddFunc (module, name, ret_type, arg_types, 2);
8355 break;
8356 case INTRINS_SSE_PSRLI_D:
8357 case INTRINS_SSE_PSRAI_D:
8358 case INTRINS_SSE_PSLLI_D:
8359 ret_type = type_to_simd_type (MONO_TYPE_I4);
8360 arg_types [0] = ret_type;
8361 arg_types [1] = LLVMInt32Type ();
8362 AddFunc (module, name, ret_type, arg_types, 2);
8363 break;
8364 case INTRINS_SSE_PSRLI_Q:
8365 case INTRINS_SSE_PSLLI_Q:
8366 ret_type = type_to_simd_type (MONO_TYPE_I8);
8367 arg_types [0] = ret_type;
8368 arg_types [1] = LLVMInt32Type ();
8369 AddFunc (module, name, ret_type, arg_types, 2);
8370 break;
8371 case INTRINS_SSE_SQRT_PD:
8372 /* Unary ops */
8373 ret_type = type_to_simd_type (MONO_TYPE_R8);
8374 arg_types [0] = ret_type;
8375 AddFunc (module, name, ret_type, arg_types, 1);
8376 break;
8377 case INTRINS_SSE_SQRT_PS:
8378 ret_type = type_to_simd_type (MONO_TYPE_R4);
8379 arg_types [0] = ret_type;
8380 AddFunc (module, name, ret_type, arg_types, 1);
8381 break;
8382 case INTRINS_SSE_RSQRT_PS:
8383 ret_type = type_to_simd_type (MONO_TYPE_R4);
8384 arg_types [0] = ret_type;
8385 AddFunc (module, name, ret_type, arg_types, 1);
8386 break;
8387 case INTRINS_SSE_RCP_PS:
8388 ret_type = type_to_simd_type (MONO_TYPE_R4);
8389 arg_types [0] = ret_type;
8390 AddFunc (module, name, ret_type, arg_types, 1);
8391 break;
8392 case INTRINS_SSE_CVTTPD2DQ:
8393 ret_type = type_to_simd_type (MONO_TYPE_I4);
8394 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8395 AddFunc (module, name, ret_type, arg_types, 1);
8396 break;
8397 case INTRINS_SSE_CVTTPS2DQ:
8398 ret_type = type_to_simd_type (MONO_TYPE_I4);
8399 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8400 AddFunc (module, name, ret_type, arg_types, 1);
8401 break;
8402 case INTRINS_SSE_CVTDQ2PD:
8403 /* Conversion ops */
8404 ret_type = type_to_simd_type (MONO_TYPE_R8);
8405 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8406 AddFunc (module, name, ret_type, arg_types, 1);
8407 break;
8408 case INTRINS_SSE_CVTDQ2PS:
8409 ret_type = type_to_simd_type (MONO_TYPE_R4);
8410 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8411 AddFunc (module, name, ret_type, arg_types, 1);
8412 break;
8413 case INTRINS_SSE_CVTPD2DQ:
8414 ret_type = type_to_simd_type (MONO_TYPE_I4);
8415 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8416 AddFunc (module, name, ret_type, arg_types, 1);
8417 break;
8418 case INTRINS_SSE_CVTPS2DQ:
8419 ret_type = type_to_simd_type (MONO_TYPE_I4);
8420 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8421 AddFunc (module, name, ret_type, arg_types, 1);
8422 break;
8423 case INTRINS_SSE_CVTPD2PS:
8424 ret_type = type_to_simd_type (MONO_TYPE_R4);
8425 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8426 AddFunc (module, name, ret_type, arg_types, 1);
8427 break;
8428 case INTRINS_SSE_CVTPS2PD:
8429 ret_type = type_to_simd_type (MONO_TYPE_R8);
8430 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8431 AddFunc (module, name, ret_type, arg_types, 1);
8432 break;
8433 case INTRINS_SSE_CMPPD:
8434 /* cmp pd/ps */
8435 ret_type = type_to_simd_type (MONO_TYPE_R8);
8436 arg_types [0] = ret_type;
8437 arg_types [1] = ret_type;
8438 arg_types [2] = LLVMInt8Type ();
8439 AddFunc (module, name, ret_type, arg_types, 3);
8440 break;
8441 case INTRINS_SSE_CMPPS:
8442 ret_type = type_to_simd_type (MONO_TYPE_R4);
8443 arg_types [0] = ret_type;
8444 arg_types [1] = ret_type;
8445 arg_types [2] = LLVMInt8Type ();
8446 AddFunc (module, name, ret_type, arg_types, 3);
8447 break;
8448 case INTRINS_SSE_PACKSSWB:
8449 case INTRINS_SSE_PACKUSWB:
8450 case INTRINS_SSE_PACKSSDW:
8451 /* pack */
8452 ret_type = type_to_simd_type (MONO_TYPE_I1);
8453 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
8454 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
8455 AddFunc (module, name, ret_type, arg_types, 2);
8456 break;
8457 case INTRINS_SSE_PACKUSDW:
8458 ret_type = type_to_simd_type (MONO_TYPE_I2);
8459 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8460 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
8461 AddFunc (module, name, ret_type, arg_types, 2);
8462 break;
8463 /* SSE Binary ops */
8464 case INTRINS_SSE_PADDSW:
8465 case INTRINS_SSE_PSUBSW:
8466 case INTRINS_SSE_PADDUSW:
8467 case INTRINS_SSE_PSUBUSW:
8468 case INTRINS_SSE_PAVGW:
8469 case INTRINS_SSE_PMULHW:
8470 case INTRINS_SSE_PMULHU:
8471 add_sse_binary (module, name, MONO_TYPE_I2);
8472 break;
8473 case INTRINS_SSE_MINPS:
8474 case INTRINS_SSE_MAXPS:
8475 case INTRINS_SSE_HADDPS:
8476 case INTRINS_SSE_HSUBPS:
8477 case INTRINS_SSE_ADDSUBPS:
8478 add_sse_binary (module, name, MONO_TYPE_R4);
8479 break;
8480 case INTRINS_SSE_MINPD:
8481 case INTRINS_SSE_MAXPD:
8482 case INTRINS_SSE_HADDPD:
8483 case INTRINS_SSE_HSUBPD:
8484 case INTRINS_SSE_ADDSUBPD:
8485 add_sse_binary (module, name, MONO_TYPE_R8);
8486 break;
8487 case INTRINS_SE_PADDSB:
8488 case INTRINS_SSE_PSUBSB:
8489 case INTRINS_SSE_PADDUSB:
8490 case INTRINS_SSE_PSUBUSB:
8491 case INTRINS_SSE_PAVGB:
8492 add_sse_binary (module, name, MONO_TYPE_I1);
8493 break;
8494 case INTRINS_SSE_PAUSE:
8495 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
8496 break;
8497 case INTRINS_SSE_DPPS:
8498 ret_type = type_to_simd_type (MONO_TYPE_R4);
8499 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8500 arg_types [1] = type_to_simd_type (MONO_TYPE_R4);
8501 arg_types [2] = LLVMInt8Type ();
8502 AddFunc (module, name, ret_type, arg_types, 3);
8503 break;
8504 case INTRINS_SSE_ROUNDPD:
8505 ret_type = type_to_simd_type (MONO_TYPE_R8);
8506 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8507 arg_types [1] = LLVMInt32Type ();
8508 AddFunc (module, name, ret_type, arg_types, 2);
8509 break;
8510 #endif /* AMD64 || X86 */
8511 default:
8512 g_assert_not_reached ();
8513 break;
8514 }
8515 }
8516
8517 static LLVMValueRef
8518 get_intrins (EmitContext *ctx, int id)
8519 {
8520 const char *name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
8521 g_assert (name);
8522
8523 LLVMValueRef res;
8524
8525 /*
8526 * Every method is emitted into its own module so
8527 * we can add intrinsics on demand.
8528 */
8529 res = ctx->module->intrins_by_id [id];
8530 if (!res) {
8531 res = LLVMGetNamedFunction (ctx->lmodule, name);
8532 if (!res) {
8533 add_intrinsic (ctx->lmodule, id);
8534 res = LLVMGetNamedFunction (ctx->lmodule, name);
8535 g_assert (res);
8536 }
8537 ctx->module->intrins_by_id [id] = res;
8538 }
8539 return res;
8540 }
8541
8542 static LLVMValueRef
8543 get_intrins_by_name (EmitContext *ctx, const char *name)
8544 {
8545 LLVMValueRef res;
8546
8547 /*
8548 * Every method is emitted into its own module so
8549 * we can add intrinsics on demand.
8550 */
8551 res = LLVMGetNamedFunction (ctx->lmodule, name);
8552 if (!res) {
8553 int id = -1;
8554
8555 /* No locking needed */
8556 id = GPOINTER_TO_INT (g_hash_table_lookup (intrins_name_to_id, name));
8557 id --;
8558 if (id == -1)
8559 printf ("%s\n", name);
8560 g_assert (id != -1);
8561 add_intrinsic (ctx->lmodule, id);
8562 res = LLVMGetNamedFunction (ctx->lmodule, name);
8563 g_assert (res);
8564 }
8565
8566 return res;
8567 }
8568
8569 static void
8570 add_intrinsics (LLVMModuleRef module)
8571 {
8572 int i;
8573
8574 /* Emit declarations of instrinsics */
8575 /*
8576 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
8577 * type doesn't seem to do any locking.
8578 */
8579 for (i = 0; i < INTRINS_NUM; ++i)
8580 add_intrinsic (module, i);
8581
8582 /* EH intrinsics */
8583 {
8584 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
8585
8586 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
8587 }
8588
8589 /* Load/Store intrinsics */
8590 {
8591 LLVMTypeRef arg_types [5];
8592 int i;
8593 char name [128];
8594
8595 for (i = 1; i <= 8; i *= 2) {
8596 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
8597 arg_types [1] = LLVMInt32Type ();
8598 arg_types [2] = LLVMInt1Type ();
8599 arg_types [3] = LLVMInt32Type ();
8600 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
8601 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
8602
8603 arg_types [0] = LLVMIntType (i * 8);
8604 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
8605 arg_types [2] = LLVMInt32Type ();
8606 arg_types [3] = LLVMInt1Type ();
8607 arg_types [4] = LLVMInt32Type ();
8608 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
8609 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
8610 }
8611 }
8612 }
8613
8614 static void
8615 add_types (MonoLLVMModule *module)
8616 {
8617 module->ptr_type = LLVMPointerType (TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
8618 }
8619
8620 void
8621 mono_llvm_init (void)
8622 {
8623 GHashTable *h;
8624 int i;
8625
8626 mono_native_tls_alloc (&current_cfg_tls_id, NULL);
8627
8628 h = g_hash_table_new (NULL, NULL);
8629 for (i = 0; i < INTRINS_NUM; ++i)
8630 g_hash_table_insert (h, GINT_TO_POINTER (intrinsics [i].id), (gpointer)intrinsics [i].name);
8631 intrins_id_to_name = h;
8632
8633 h = g_hash_table_new (g_str_hash, g_str_equal);
8634 for (i = 0; i < INTRINS_NUM; ++i)
8635 g_hash_table_insert (h, (gpointer)intrinsics [i].name, GINT_TO_POINTER (intrinsics [i].id + 1));
8636 intrins_name_to_id = h;
8637 }
8638
8639 void
8640 mono_llvm_cleanup (void)
8641 {
8642 MonoLLVMModule *module = &aot_module;
8643
8644 if (module->lmodule)
8645 LLVMDisposeModule (module->lmodule);
8646
8647 if (module->context)
8648 LLVMContextDispose (module->context);
8649 }
8650
8651 void
8652 mono_llvm_free_domain_info (MonoDomain *domain)
8653 {
8654 MonoJitDomainInfo *info = domain_jit_info (domain);
8655 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
8656 int i;
8657
8658 if (!module)
8659 return;
8660
8661 g_hash_table_destroy (module->llvm_types);
8662
8663 mono_llvm_dispose_ee (module->mono_ee);
8664
8665 if (module->bb_names) {
8666 for (i = 0; i < module->bb_names_len; ++i)
8667 g_free (module->bb_names [i]);
8668 g_free (module->bb_names);
8669 }
8670 //LLVMDisposeModule (module->module);
8671
8672 g_free (module);
8673
8674 info->llvm_module = NULL;
8675 }
8676
8677 void
8678 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, int initial_got_size, LLVMModuleFlags flags)
8679 {
8680 MonoLLVMModule *module = &aot_module;
8681 gboolean emit_dwarf = (flags & LLVM_MODULE_FLAG_DWARF) ? 1 : 0;
8682 #ifdef TARGET_WIN32_MSVC
8683 gboolean emit_codeview = (flags & LLVM_MODULE_FLAG_CODEVIEW) ? 1 : 0;
8684 #endif
8685 gboolean static_link = (flags & LLVM_MODULE_FLAG_STATIC) ? 1 : 0;
8686 gboolean llvm_only = (flags & LLVM_MODULE_FLAG_LLVM_ONLY) ? 1 : 0;
8687 gboolean interp = (flags & LLVM_MODULE_FLAG_INTERP) ? 1 : 0;
8688 gboolean llvm_disable_self_init = mini_get_debug_options ()->llvm_disable_self_init;
8689
8690 /* Delete previous module */
8691 g_hash_table_destroy (module->plt_entries);
8692 if (module->lmodule)
8693 LLVMDisposeModule (module->lmodule);
8694
8695 memset (module, 0, sizeof (aot_module));
8696
8697 module->lmodule = LLVMModuleCreateWithName ("aot");
8698 module->assembly = assembly;
8699 module->global_prefix = g_strdup (global_prefix);
8700 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
8701 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
8702 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
8703 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
8704 module->external_symbols = TRUE;
8705 module->emit_dwarf = emit_dwarf;
8706 module->static_link = static_link;
8707 module->llvm_only = llvm_only;
8708 module->llvm_disable_self_init = llvm_disable_self_init && !llvm_only; // llvm_only implies !llvm_disable_self_init
8709 module->interp = interp;
8710 /* The first few entries are reserved */
8711 module->max_got_offset = initial_got_size;
8712 module->context = LLVMGetGlobalContext ();
8713 module->cfgs = g_ptr_array_new ();
8714 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
8715
8716 if (llvm_only)
8717 /* clang ignores our debug info because it has an invalid version */
8718 module->emit_dwarf = FALSE;
8719
8720 add_intrinsics (module->lmodule);
8721 add_types (module);
8722
8723 #ifdef MONO_ARCH_LLVM_TARGET_LAYOUT
8724 LLVMSetDataLayout (module->lmodule, MONO_ARCH_LLVM_TARGET_LAYOUT);
8725 #else
8726 g_assert_not_reached ();
8727 #endif
8728
8729 #ifdef MONO_ARCH_LLVM_TARGET_TRIPLE
8730 LLVMSetTarget (module->lmodule, MONO_ARCH_LLVM_TARGET_TRIPLE);
8731 #endif
8732
8733 if (module->emit_dwarf) {
8734 char *dir, *build_info, *s, *cu_name;
8735
8736 module->di_builder = mono_llvm_create_di_builder (module->lmodule);
8737
8738 // FIXME:
8739 dir = g_strdup (".");
8740 build_info = mono_get_runtime_build_info ();
8741 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
8742 cu_name = g_path_get_basename (assembly->image->name);
8743 module->cu = mono_llvm_di_create_compile_unit (module->di_builder, cu_name, dir, s);
8744 g_free (dir);
8745 g_free (build_info);
8746 g_free (s);
8747 }
8748
8749 #ifdef TARGET_WIN32_MSVC
8750 if (emit_codeview) {
8751 LLVMValueRef codeview_option_args[3];
8752
8753 codeview_option_args[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8754 codeview_option_args[1] = LLVMMDString ("CodeView", 8);
8755 codeview_option_args[2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8756
8757 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.module.flags", LLVMMDNode (codeview_option_args, G_N_ELEMENTS (codeview_option_args)));
8758 }
8759
8760 if (!static_link) {
8761 const char linker_options[] = "Linker Options";
8762 const char *default_dynamic_lib_names[] = { "/DEFAULTLIB:msvcrt",
8763 "/DEFAULTLIB:ucrt.lib",
8764 "/DEFAULTLIB:vcruntime.lib" };
8765
8766 LLVMValueRef default_lib_args[G_N_ELEMENTS (default_dynamic_lib_names)];
8767 LLVMValueRef default_lib_nodes[G_N_ELEMENTS(default_dynamic_lib_names)];
8768
8769 const char *default_lib_name = NULL;
8770 for (int i = 0; i < G_N_ELEMENTS (default_dynamic_lib_names); ++i) {
8771 const char *default_lib_name = default_dynamic_lib_names[i];
8772 default_lib_args[i] = LLVMMDString (default_lib_name, strlen (default_lib_name));
8773 default_lib_nodes[i] = LLVMMDNode (default_lib_args + i, 1);
8774 }
8775
8776 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.linker.options", LLVMMDNode (default_lib_args, G_N_ELEMENTS (default_lib_args)));
8777 }
8778 #endif
8779
8780 /* Add GOT */
8781 /*
8782 * We couldn't compute the type of the LLVM global representing the got because
8783 * its size is only known after all the methods have been emitted. So create
8784 * a dummy variable, and replace all uses it with the real got variable when
8785 * its size is known in mono_llvm_emit_aot_module ().
8786 */
8787 {
8788 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
8789
8790 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
8791 module->got_idx_to_type = g_hash_table_new (NULL, NULL);
8792 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
8793 }
8794
8795 /* Add initialization array */
8796 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
8797
8798 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
8799 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
8800
8801
8802 emit_gc_safepoint_poll (module);
8803
8804 emit_llvm_code_start (module);
8805
8806 // Needs idx_to_lmethod
8807 emit_init_icall_wrappers (module);
8808
8809 /* Add a dummy personality function */
8810 if (!use_mono_personality_debug) {
8811 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
8812 LLVMSetLinkage (personality, LLVMExternalLinkage);
8813
8814 //EMCC chockes if the personality function is referenced in the 'used' array
8815 #ifndef TARGET_WASM
8816 mark_as_used (module, personality);
8817 #endif
8818 }
8819
8820 /* Add a reference to the c++ exception we throw/catch */
8821 {
8822 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
8823 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
8824 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
8825 mono_llvm_set_is_constant (module->sentinel_exception);
8826 }
8827
8828 module->llvm_types = g_hash_table_new (NULL, NULL);
8829 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
8830 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
8831 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
8832 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
8833 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
8834 module->method_to_call_info = g_hash_table_new (NULL, NULL);
8835 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
8836 module->callsite_list = g_ptr_array_new ();
8837 }
8838
8839 void
8840 mono_llvm_fixup_aot_module (void)
8841 {
8842 MonoLLVMModule *module = &aot_module;
8843 MonoMethod *method;
8844
8845 if (module->llvm_disable_self_init)
8846 return;
8847
8848 /*
8849 * Replace GOT entries for directly callable methods with the methods themselves.
8850 * It would be easier to implement this by predefining all methods before compiling
8851 * their bodies, but that couldn't handle the case when a method fails to compile
8852 * with llvm.
8853 */
8854
8855 GHashTable *specializable = g_hash_table_new (NULL, NULL);
8856
8857 GHashTable *patches_to_null = g_hash_table_new (mono_patch_info_hash, mono_patch_info_equal);
8858 for (int sindex = 0; sindex < module->callsite_list->len; ++sindex) {
8859 CallSite *site = (CallSite*)g_ptr_array_index (module->callsite_list, sindex);
8860 method = site->method;
8861 LLVMValueRef lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
8862
8863 LLVMValueRef placeholder = (LLVMValueRef)site->load;
8864 LLVMValueRef indexes [2], got_entry_addr, load;
8865 char *name;
8866
8867 if (lmethod && !(method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)) {
8868 mono_llvm_replace_uses_of (placeholder, lmethod);
8869
8870 if (mono_aot_can_specialize (method))
8871 g_hash_table_insert (specializable, lmethod, method);
8872
8873 g_hash_table_insert (patches_to_null, site->ji, site->ji);
8874 } else {
8875 int got_offset = compute_aot_got_offset (module, site->ji, site->type);
8876
8877 module->max_got_offset = MAX (module->max_got_offset, got_offset);
8878
8879 LLVMBuilderRef builder = LLVMCreateBuilder ();
8880 LLVMPositionBuilderBefore (builder, placeholder);
8881 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8882 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
8883 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
8884
8885 name = get_aotconst_name (site->ji->type, site->ji->data.target, got_offset);
8886 load = LLVMBuildLoad (builder, got_entry_addr, "");
8887 load = LLVMBuildBitCast (builder, load, site->type, name ? name : "");
8888 LLVMReplaceAllUsesWith (placeholder, load);
8889 }
8890 g_free (site);
8891 }
8892
8893 mono_llvm_propagate_nonnull_final (specializable, module);
8894 g_hash_table_destroy (specializable);
8895
8896 for (int i = 0; i < module->cfgs->len; ++i) {
8897 /*
8898 * Nullify the patches pointing to direct calls. This is needed to
8899 * avoid allocating extra got slots, which is a perf problem and it
8900 * makes module->max_got_offset invalid.
8901 * It would be better to just store the patch_info in CallSite, but
8902 * cfg->patch_info is copied in aot-compiler.c.
8903 */
8904 MonoCompile *cfg = (MonoCompile *)g_ptr_array_index (module->cfgs, i);
8905 for (MonoJumpInfo *patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
8906 if (patch_info->type == MONO_PATCH_INFO_METHOD) {
8907 if (g_hash_table_lookup (patches_to_null, patch_info)) {
8908 patch_info->type = MONO_PATCH_INFO_NONE;
8909 /* Nullify the call to init_method () if possible */
8910 g_assert (cfg->got_access_count);
8911 cfg->got_access_count --;
8912 if (cfg->got_access_count == 0) {
8913 LLVMValueRef br = (LLVMValueRef)cfg->llvmonly_init_cond;
8914 if (br)
8915 LLVMSetSuccessor (br, 0, LLVMGetSuccessor (br, 1));
8916 }
8917 }
8918 }
8919 }
8920 }
8921
8922 g_hash_table_destroy (patches_to_null);
8923 }
8924
8925 static LLVMValueRef
8926 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
8927 {
8928 int i;
8929 LLVMValueRef res, *vals;
8930
8931 vals = g_new0 (LLVMValueRef, nvalues);
8932 for (i = 0; i < nvalues; ++i)
8933 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
8934 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
8935 g_free (vals);
8936 return res;
8937 }
8938
8939 static LLVMValueRef
8940 llvm_array_from_bytes (guint8 *values, int nvalues)
8941 {
8942 int i;
8943 LLVMValueRef res, *vals;
8944
8945 vals = g_new0 (LLVMValueRef, nvalues);
8946 for (i = 0; i < nvalues; ++i)
8947 vals [i] = LLVMConstInt (LLVMInt8Type (), values [i], FALSE);
8948 res = LLVMConstArray (LLVMInt8Type (), vals, nvalues);
8949 g_free (vals);
8950 return res;
8951 }
8952 /*
8953 * mono_llvm_emit_aot_file_info:
8954 *
8955 * Emit the MonoAotFileInfo structure.
8956 * Same as emit_aot_file_info () in aot-compiler.c.
8957 */
8958 void
8959 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
8960 {
8961 MonoLLVMModule *module = &aot_module;
8962
8963 /* Save these for later */
8964 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
8965 module->has_jitted_code = has_jitted_code;
8966 }
8967
8968 /*
8969 * mono_llvm_emit_aot_data:
8970 *
8971 * Emit the binary data DATA pointed to by symbol SYMBOL.
8972 */
8973 void
8974 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
8975 {
8976 MonoLLVMModule *module = &aot_module;
8977 LLVMTypeRef type;
8978 LLVMValueRef d;
8979
8980 type = LLVMArrayType (LLVMInt8Type (), data_len);
8981 d = LLVMAddGlobal (module->lmodule, type, symbol);
8982 LLVMSetVisibility (d, LLVMHiddenVisibility);
8983 LLVMSetLinkage (d, LLVMInternalLinkage);
8984 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
8985 LLVMSetAlignment (d, 8);
8986 mono_llvm_set_is_constant (d);
8987 }
8988
8989 /* Add a reference to a global defined in JITted code */
8990 static LLVMValueRef
8991 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
8992 {
8993 char *s;
8994 LLVMValueRef v;
8995
8996 s = g_strdup_printf ("%s%s", module->global_prefix, name);
8997 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
8998 LLVMSetVisibility (v, LLVMHiddenVisibility);
8999 g_free (s);
9000 return v;
9001 }
9002
9003 static void
9004 emit_aot_file_info (MonoLLVMModule *module)
9005 {
9006 LLVMTypeRef file_info_type;
9007 LLVMTypeRef *eltypes, eltype;
9008 LLVMValueRef info_var;
9009 LLVMValueRef *fields;
9010 int i, nfields, tindex;
9011 MonoAotFileInfo *info;
9012 LLVMModuleRef lmodule = module->lmodule;
9013
9014 info = &module->aot_info;
9015
9016 /* Create an LLVM type to represent MonoAotFileInfo */
9017 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 21 + 5;
9018 eltypes = g_new (LLVMTypeRef, nfields);
9019 tindex = 0;
9020 eltypes [tindex ++] = LLVMInt32Type ();
9021 eltypes [tindex ++] = LLVMInt32Type ();
9022 /* Symbols */
9023 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
9024 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
9025 /* Scalars */
9026 for (i = 0; i < 20; ++i)
9027 eltypes [tindex ++] = LLVMInt32Type ();
9028 /* Arrays */
9029 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
9030 for (i = 0; i < 4; ++i)
9031 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
9032 eltypes [tindex ++] = LLVMArrayType (LLVMInt8Type (), 16);
9033 g_assert (tindex == nfields);
9034 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
9035 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
9036
9037 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
9038 if (module->static_link) {
9039 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
9040 LLVMSetLinkage (info_var, LLVMInternalLinkage);
9041 }
9042
9043 #ifdef TARGET_WIN32
9044 if (!module->static_link) {
9045 LLVMSetDLLStorageClass (info_var, LLVMDLLExportStorageClass);
9046 }
9047 #endif
9048
9049 fields = g_new (LLVMValueRef, nfields);
9050 tindex = 0;
9051 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
9052 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
9053
9054 /* Symbols */
9055 /*
9056 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
9057 * for symbols defined in the .s file emitted by the aot compiler.
9058 */
9059 eltype = eltypes [tindex];
9060 if (module->llvm_only)
9061 fields [tindex ++] = LLVMConstNull (eltype);
9062 else
9063 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
9064 fields [tindex ++] = module->got_var;
9065 /* llc defines this directly */
9066 if (!module->llvm_only) {
9067 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
9068 fields [tindex ++] = LLVMConstNull (eltype);
9069 fields [tindex ++] = LLVMConstNull (eltype);
9070 } else {
9071 fields [tindex ++] = LLVMConstNull (eltype);
9072 fields [tindex ++] = module->get_method;
9073 fields [tindex ++] = module->get_unbox_tramp ? module->get_unbox_tramp : LLVMConstNull (eltype);
9074 }
9075 if (module->has_jitted_code) {
9076 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
9077 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
9078 } else {
9079 fields [tindex ++] = LLVMConstNull (eltype);
9080 fields [tindex ++] = LLVMConstNull (eltype);
9081 }
9082 if (!module->llvm_only)
9083 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
9084 else
9085 fields [tindex ++] = LLVMConstNull (eltype);
9086 if (module->llvm_only && module->unbox_tramp_indexes) {
9087 fields [tindex ++] = module->unbox_tramp_indexes;
9088 fields [tindex ++] = module->unbox_trampolines;
9089 } else {
9090 fields [tindex ++] = LLVMConstNull (eltype);
9091 fields [tindex ++] = LLVMConstNull (eltype);
9092 }
9093 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
9094 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
9095 fields [tindex ++] = LLVMConstNull (eltype);
9096 } else {
9097 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
9098 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
9099 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
9100 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
9101 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
9102 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
9103 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
9104 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
9105 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
9106 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
9107 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "weak_field_indexes");
9108 }
9109 /* Not needed (mem_end) */
9110 fields [tindex ++] = LLVMConstNull (eltype);
9111 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
9112 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
9113 if (info->trampoline_size [0]) {
9114 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
9115 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
9116 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_trampolines");
9117 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
9118 fields [tindex ++] = AddJitGlobal (module, eltype, "ftnptr_arg_trampolines");
9119 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_arbitrary_trampolines");
9120 } else {
9121 fields [tindex ++] = LLVMConstNull (eltype);
9122 fields [tindex ++] = LLVMConstNull (eltype);
9123 fields [tindex ++] = LLVMConstNull (eltype);
9124 fields [tindex ++] = LLVMConstNull (eltype);
9125 fields [tindex ++] = LLVMConstNull (eltype);
9126 fields [tindex ++] = LLVMConstNull (eltype);
9127 }
9128 if (module->static_link && !module->llvm_only)
9129 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
9130 else
9131 fields [tindex ++] = LLVMConstNull (eltype);
9132 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
9133 if (!module->llvm_only) {
9134 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
9135 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
9136 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
9137 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
9138 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
9139 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
9140 } else {
9141 fields [tindex ++] = LLVMConstNull (eltype);
9142 fields [tindex ++] = LLVMConstNull (eltype);
9143 fields [tindex ++] = LLVMConstNull (eltype);
9144 fields [tindex ++] = LLVMConstNull (eltype);
9145 fields [tindex ++] = LLVMConstNull (eltype);
9146 fields [tindex ++] = LLVMConstNull (eltype);
9147 }
9148
9149 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i) {
9150 g_assert (fields [2 + i]);
9151 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
9152 }
9153
9154 /* Scalars */
9155 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
9156 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
9157 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
9158 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
9159 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nextra_methods, FALSE);
9160 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
9161 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
9162 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
9163 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
9164 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
9165 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
9166 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
9167 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
9168 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_shift_bits, FALSE);
9169 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_mask, FALSE);
9170 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
9171 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
9172 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
9173 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_num, FALSE);
9174 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_elemsize, FALSE);
9175 /* Arrays */
9176 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
9177 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
9178 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
9179 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
9180 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
9181
9182 fields [tindex ++] = llvm_array_from_bytes (info->aotid, 16);
9183 g_assert (tindex == nfields);
9184
9185 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
9186
9187 if (module->static_link) {
9188 char *s, *p;
9189 LLVMValueRef var;
9190
9191 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
9192 /* Get rid of characters which cannot occur in symbols */
9193 p = s;
9194 for (p = s; *p; ++p) {
9195 if (!(isalnum (*p) || *p == '_'))
9196 *p = '_';
9197 }
9198 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
9199 g_free (s);
9200 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
9201 LLVMSetLinkage (var, LLVMExternalLinkage);
9202 }
9203 }
9204
9205 typedef struct {
9206 LLVMValueRef lmethod;
9207 int argument;
9208 } NonnullPropWorkItem;
9209
9210 static void
9211 mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params)
9212 {
9213 if (!ctx->module->llvm_disable_self_init && mono_aot_can_specialize (call_method)) {
9214 int num_passed = LLVMGetNumArgOperands (lcall);
9215 g_assert (num_params <= num_passed);
9216
9217 g_assert (ctx->module->method_to_call_info);
9218 GArray *call_site_union = (GArray *) g_hash_table_lookup (ctx->module->method_to_call_info, call_method);
9219
9220 if (!call_site_union) {
9221 call_site_union = g_array_sized_new (FALSE, TRUE, sizeof (gint32), num_params);
9222 int zero = 0;
9223 for (int i = 0; i < num_params; i++)
9224 g_array_insert_val (call_site_union, i, zero);
9225 }
9226
9227 for (int i = 0; i < num_params; i++) {
9228 if (mono_llvm_is_nonnull (args [i])) {
9229 g_assert (i < LLVMGetNumArgOperands (lcall));
9230 mono_llvm_set_call_nonnull_arg (lcall, i);
9231 } else {
9232 gint32 *nullable_count = &g_array_index (call_site_union, gint32, i);
9233 *nullable_count = *nullable_count + 1;
9234 }
9235 }
9236
9237 g_hash_table_insert (ctx->module->method_to_call_info, call_method, call_site_union);
9238 }
9239 }
9240
9241 static void
9242 mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module)
9243 {
9244 // When we first traverse the mini IL, we mark the things that are
9245 // nonnull (the roots). Then, for all of the methods that can be specialized, we
9246 // see if their call sites have nonnull attributes.
9247
9248 // If so, we mark the function's param. This param has uses to propagate
9249 // the attribute to. This propagation can trigger a need to mark more attributes
9250 // non-null, and so on and so forth.
9251 GSList *queue = NULL;
9252
9253 GHashTableIter iter;
9254 LLVMValueRef lmethod;
9255 MonoMethod *method;
9256 g_hash_table_iter_init (&iter, all_specializable);
9257 while (g_hash_table_iter_next (&iter, (void**)&lmethod, (void**)&method)) {
9258 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, method);
9259
9260 // Basic sanity checking
9261 if (call_site_union)
9262 g_assert (call_site_union->len == LLVMCountParams (lmethod));
9263
9264 // Add root to work queue
9265 for (int i = 0; call_site_union && i < call_site_union->len; i++) {
9266 if (g_array_index (call_site_union, gint32, i) == 0) {
9267 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
9268 item->lmethod = lmethod;
9269 item->argument = i;
9270 queue = g_slist_prepend (queue, item);
9271 }
9272 }
9273 }
9274
9275 // This is essentially reference counting, and we are propagating
9276 // the refcount decrement here. We have less work to do than we may otherwise
9277 // because we are only working with a set of subgraphs of specializable functions.
9278 //
9279 // We rely on being able to see all of the references in the graph.
9280 // This is ensured by the function mono_aot_can_specialize. Everything in
9281 // all_specializable is a function that can be specialized, and is the resulting
9282 // node in the graph after all of the subsitutions are done.
9283 //
9284 // Anything disrupting the direct calls made with self-init will break this optimization.
9285
9286 while (queue) {
9287 // Update the queue state.
9288 // Our only other per-iteration responsibility is now to free current
9289 NonnullPropWorkItem *current = (NonnullPropWorkItem *) queue->data;
9290 queue = queue->next;
9291 g_assert (current->argument < LLVMCountParams (current->lmethod));
9292
9293 // Does the actual leaf-node work here
9294 // Mark the function argument as nonnull for LLVM
9295 mono_llvm_set_func_nonnull_arg (current->lmethod, current->argument);
9296
9297 // The rest of this is for propagating forward nullability changes
9298 // to calls that use the argument that is now nullable.
9299
9300 // Get the actual LLVM value of the argument, so we can see which call instructions
9301 // used that argument
9302 LLVMValueRef caller_argument = LLVMGetParam (current->lmethod, current->argument);
9303
9304 // Iterate over the calls using the newly-non-nullable argument
9305 GSList *calls = mono_llvm_calls_using (caller_argument);
9306 for (GSList *cursor = calls; cursor != NULL; cursor = cursor->next) {
9307
9308 LLVMValueRef lcall = (LLVMValueRef) cursor->data;
9309 LLVMValueRef callee_lmethod = LLVMGetCalledValue (lcall);
9310
9311 // If this wasn't a direct call for which mono_aot_can_specialize is true,
9312 // this lookup won't find a MonoMethod.
9313 MonoMethod *callee_method = (MonoMethod *) g_hash_table_lookup (all_specializable, callee_lmethod);
9314 if (!callee_method)
9315 continue;
9316
9317 // Decrement number of nullable refs at that func's arg offset
9318 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, callee_method);
9319
9320 // It has module-local callers and is specializable, should have seen this call site
9321 // and inited this
9322 g_assert (call_site_union);
9323
9324 // The function *definition* parameter arity should always be consistent
9325 int max_params = LLVMCountParams (callee_lmethod);
9326 if (call_site_union->len != max_params) {
9327 mono_llvm_dump_value (callee_lmethod);
9328 g_assert_not_reached ();
9329 }
9330
9331 // Get the values that correspond to the parameters passed to the call
9332 // that used our argument
9333 LLVMValueRef *operands = mono_llvm_call_args (lcall);
9334 for (int call_argument = 0; call_argument < max_params; call_argument++) {
9335 // Every time we used the newly-non-nullable argument, decrement the nullable
9336 // refcount for that function.
9337 if (caller_argument == operands [call_argument]) {
9338 gint32 *nullable_count = &g_array_index (call_site_union, gint32, call_argument);
9339 g_assert (*nullable_count > 0);
9340 *nullable_count = *nullable_count - 1;
9341
9342 // If we caused that callee's parameter to become newly nullable, add to work queue
9343 if (*nullable_count == 0) {
9344 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
9345 item->lmethod = callee_lmethod;
9346 item->argument = call_argument;
9347 queue = g_slist_prepend (queue, item);
9348 }
9349 }
9350 }
9351 g_free (operands);
9352
9353 // Update nullability refcount information for the callee now
9354 g_hash_table_insert (module->method_to_call_info, callee_method, call_site_union);
9355 }
9356 g_slist_free (calls);
9357
9358 g_free (current);
9359 }
9360 }
9361
9362 /*
9363 * Emit the aot module into the LLVM bitcode file FILENAME.
9364 */
9365 void
9366 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
9367 {
9368 LLVMTypeRef got_type, inited_type;
9369 LLVMValueRef real_got, real_inited;
9370 MonoLLVMModule *module = &aot_module;
9371
9372 emit_llvm_code_end (module);
9373
9374 /*
9375 * Create the real got variable and replace all uses of the dummy variable with
9376 * the real one.
9377 */
9378 int size = module->max_got_offset + 1;
9379 LLVMTypeRef *members = g_malloc0 (sizeof (LLVMValueRef) * size);
9380 for (int i = 0; i < size; i++) {
9381 LLVMTypeRef lookup_type = NULL;
9382
9383 lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (i));
9384
9385 if (!lookup_type)
9386 lookup_type = module->ptr_type;
9387
9388 members [i] = LLVMPointerType (lookup_type, 0);
9389 }
9390
9391 got_type = LLVMStructCreateNamed (module->context, g_strdup_printf ("MONO_GOT_%s", cu_name));
9392 LLVMStructSetBody (got_type, members, size, FALSE);
9393 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
9394
9395 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
9396 if (module->external_symbols) {
9397 LLVMSetLinkage (real_got, LLVMExternalLinkage);
9398 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
9399 } else {
9400 LLVMSetLinkage (real_got, LLVMInternalLinkage);
9401 }
9402 mono_llvm_replace_uses_of (module->got_var, real_got);
9403
9404 mark_as_used (&aot_module, real_got);
9405
9406 /* Delete the dummy got so it doesn't become a global */
9407 LLVMDeleteGlobal (module->got_var);
9408 module->got_var = real_got;
9409
9410 /*
9411 * Same for the init_var
9412 */
9413 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
9414 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
9415 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
9416 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
9417 mono_llvm_replace_uses_of (module->inited_var, real_inited);
9418 LLVMDeleteGlobal (module->inited_var);
9419
9420 if (module->llvm_only) {
9421 emit_get_method (&aot_module);
9422 emit_get_unbox_tramp (&aot_module);
9423 }
9424
9425 emit_llvm_used (&aot_module);
9426 emit_dbg_info (&aot_module, filename, cu_name);
9427 emit_aot_file_info (&aot_module);
9428
9429 /* Replace PLT entries for directly callable methods with the methods themselves */
9430 {
9431 GHashTableIter iter;
9432 MonoJumpInfo *ji;
9433 LLVMValueRef callee;
9434
9435 GHashTable *specializable = g_hash_table_new (NULL, NULL);
9436
9437 g_hash_table_iter_init (&iter, module->plt_entries_ji);
9438 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
9439 if (mono_aot_is_direct_callable (ji)) {
9440 LLVMValueRef lmethod;
9441
9442 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
9443 /* The types might not match because the caller might pass an rgctx */
9444 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
9445 mono_llvm_replace_uses_of (callee, lmethod);
9446
9447 if (!module->llvm_disable_self_init && mono_aot_can_specialize (ji->data.method))
9448 g_hash_table_insert (specializable, lmethod, ji->data.method);
9449 mono_aot_mark_unused_llvm_plt_entry (ji);
9450 }
9451 }
9452 }
9453
9454 mono_llvm_propagate_nonnull_final (specializable, module);
9455
9456 g_hash_table_destroy (specializable);
9457 }
9458
9459 /* Note: You can still dump an invalid bitcode file by running `llvm-dis`
9460 * in a debugger, set a breakpoint on `LLVMVerifyModule` and fake its
9461 * result to 0 (indicating success). */
9462 LLVMWriteBitcodeToFile (module->lmodule, filename);
9463
9464 #if 1
9465 {
9466 char *verifier_err;
9467
9468 if (LLVMVerifyModule (module->lmodule, LLVMReturnStatusAction, &verifier_err)) {
9469 printf ("%s\n", verifier_err);
9470 g_assert_not_reached ();
9471 }
9472 }
9473 #endif
9474 }
9475
9476
9477 static LLVMValueRef
9478 md_string (const char *s)
9479 {
9480 return LLVMMDString (s, strlen (s));
9481 }
9482
9483 /* Debugging support */
9484
9485 static void
9486 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
9487 {
9488 LLVMModuleRef lmodule = module->lmodule;
9489 LLVMValueRef args [16], ver;
9490
9491 /*
9492 * This can only be enabled when LLVM code is emitted into a separate object
9493 * file, since the AOT compiler also emits dwarf info,
9494 * and the abbrev indexes will not be correct since llvm has added its own
9495 * abbrevs.
9496 */
9497 if (!module->emit_dwarf)
9498 return;
9499
9500 mono_llvm_di_builder_finalize (module->di_builder);
9501
9502 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9503 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
9504 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9505 ver = LLVMMDNode (args, 3);
9506 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
9507
9508 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9509 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
9510 args [2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE);
9511 ver = LLVMMDNode (args, 3);
9512 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
9513 }
9514
9515 static LLVMValueRef
9516 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
9517 {
9518 MonoLLVMModule *module = ctx->module;
9519 MonoDebugMethodInfo *minfo = ctx->minfo;
9520 char *source_file, *dir, *filename;
9521 MonoSymSeqPoint *sym_seq_points;
9522 int n_seq_points;
9523
9524 if (!minfo)
9525 return NULL;
9526
9527 mono_debug_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
9528 if (!source_file)
9529 source_file = g_strdup ("<unknown>");
9530 dir = g_path_get_dirname (source_file);
9531 filename = g_path_get_basename (source_file);
9532
9533 return (LLVMValueRef)mono_llvm_di_create_function (module->di_builder, module->cu, method, cfg->method->name, name, dir, filename, n_seq_points ? sym_seq_points [0].line : 1);
9534 }
9535
9536 static void
9537 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
9538 {
9539 MonoCompile *cfg = ctx->cfg;
9540
9541 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
9542 MonoDebugSourceLocation *loc;
9543 LLVMValueRef loc_md;
9544
9545 loc = mono_debug_method_lookup_location (ctx->minfo, cil_code - cfg->header->code);
9546
9547 if (loc) {
9548 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, loc->row, loc->column);
9549 mono_llvm_di_set_location (builder, loc_md);
9550 mono_debug_free_source_location (loc);
9551 }
9552 }
9553 }
9554
9555 static void
9556 emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder)
9557 {
9558 if (ctx->minfo) {
9559 LLVMValueRef loc_md;
9560 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, 0, 0);
9561 mono_llvm_di_set_location (builder, loc_md);
9562 }
9563 }
9564
9565 /*
9566 DESIGN:
9567 - Emit LLVM IR from the mono IR using the LLVM C API.
9568 - The original arch specific code remains, so we can fall back to it if we run
9569 into something we can't handle.
9570 */
9571
9572 /*
9573 A partial list of issues:
9574 - Handling of opcodes which can throw exceptions.
9575
9576 In the mono JIT, these are implemented using code like this:
9577 method:
9578 <compare>
9579 throw_pos:
9580 b<cond> ex_label
9581 <rest of code>
9582 ex_label:
9583 push throw_pos - method
9584 call <exception trampoline>
9585
9586 The problematic part is push throw_pos - method, which cannot be represented
9587 in the LLVM IR, since it does not support label values.
9588 -> this can be implemented in AOT mode using inline asm + labels, but cannot
9589 be implemented in JIT mode ?
9590 -> a possible but slower implementation would use the normal exception
9591 throwing code but it would need to control the placement of the throw code
9592 (it needs to be exactly after the compare+branch).
9593 -> perhaps add a PC offset intrinsics ?
9594
9595 - efficient implementation of .ovf opcodes.
9596
9597 These are currently implemented as:
9598 <ins which sets the condition codes>
9599 b<cond> ex_label
9600
9601 Some overflow opcodes are now supported by LLVM SVN.
9602
9603 - exception handling, unwinding.
9604 - SSA is disabled for methods with exception handlers
9605 - How to obtain unwind info for LLVM compiled methods ?
9606 -> this is now solved by converting the unwind info generated by LLVM
9607 into our format.
9608 - LLVM uses the c++ exception handling framework, while we use our home grown
9609 code, and couldn't use the c++ one:
9610 - its not supported under VC++, other exotic platforms.
9611 - it might be impossible to support filter clauses with it.
9612
9613 - trampolines.
9614
9615 The trampolines need a predictable call sequence, since they need to disasm
9616 the calling code to obtain register numbers / offsets.
9617
9618 LLVM currently generates this code in non-JIT mode:
9619 mov -0x98(%rax),%eax
9620 callq *%rax
9621 Here, the vtable pointer is lost.
9622 -> solution: use one vtable trampoline per class.
9623
9624 - passing/receiving the IMT pointer/RGCTX.
9625 -> solution: pass them as normal arguments ?
9626
9627 - argument passing.
9628
9629 LLVM does not allow the specification of argument registers etc. This means
9630 that all calls are made according to the platform ABI.
9631
9632 - passing/receiving vtypes.
9633
9634 Vtypes passed/received in registers are handled by the front end by using
9635 a signature with scalar arguments, and loading the parts of the vtype into those
9636 arguments.
9637
9638 Vtypes passed on the stack are handled using the 'byval' attribute.
9639
9640 - ldaddr.
9641
9642 Supported though alloca, we need to emit the load/store code.
9643
9644 - types.
9645
9646 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
9647 typed registers, so we have to keep track of the precise LLVM type of each vreg.
9648 This is made easier because the IR is already in SSA form.
9649 An additional problem is that our IR is not consistent with types, i.e. i32/i64
9650 types are frequently used incorrectly.
9651 */
9652
9653 /*
9654 AOT SUPPORT:
9655 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
9656 it with the file containing the methods emitted by the JIT and the AOT data
9657 structures.
9658 */
9659
9660 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
9661 * - each bblock should end with a branch
9662 * - setting the return value, making cfg->ret non-volatile
9663 * - avoid some transformations in the JIT which make it harder for us to generate
9664 * code.
9665 * - use pointer types to help optimizations.
9666 */
9667
9668 #else /* DISABLE_JIT */
9669
9670 void
9671 mono_llvm_cleanup (void)
9672 {
9673 }
9674
9675 void
9676 mono_llvm_free_domain_info (MonoDomain *domain)
9677 {
9678 }
9679
9680 void
9681 mono_llvm_init (void)
9682 {
9683 }
9684
9685 #endif /* DISABLE_JIT */
9686
9687 #if !defined(DISABLE_JIT) && !defined(MONO_CROSS_COMPILE)
9688
9689 /* LLVM JIT support */
9690
9691 static unsigned char*
9692 alloc_cb (LLVMValueRef function, int size)
9693 {
9694 MonoCompile *cfg;
9695
9696 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
9697
9698 if (cfg) {
9699 // FIXME: dynamic
9700 return (unsigned char*)mono_domain_code_reserve (cfg->domain, size);
9701 } else {
9702 return (unsigned char*)mono_domain_code_reserve (mono_domain_get (), size);
9703 }
9704 }
9705
9706 static void
9707 emitted_cb (LLVMValueRef function, void *start, void *end)
9708 {
9709 MonoCompile *cfg;
9710
9711 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
9712 g_assert (cfg);
9713 cfg->code_len = (guint8*)end - (guint8*)start;
9714 }
9715
9716 static void
9717 exception_cb (void *data)
9718 {
9719 MonoCompile *cfg;
9720 MonoJitExceptionInfo *ei;
9721 guint32 ei_len, i, j, nested_len, nindex;
9722 gpointer *type_info;
9723 int this_reg, this_offset;
9724
9725 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
9726 g_assert (cfg);
9727
9728 /*
9729 * data points to a DWARF FDE structure, convert it to our unwind format and
9730 * save it.
9731 * An alternative would be to save it directly, and modify our unwinder to work
9732 * with it.
9733 */
9734 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
9735 if (cfg->verbose_level > 1)
9736 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
9737
9738 /* Count nested clauses */
9739 nested_len = 0;
9740 for (i = 0; i < ei_len; ++i) {
9741 gint32 cindex1 = *(gint32*)type_info [i];
9742 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
9743
9744 for (j = 0; j < cfg->header->num_clauses; ++j) {
9745 int cindex2 = j;
9746 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
9747
9748 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
9749 nested_len ++;
9750 }
9751 }
9752 }
9753
9754 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
9755 cfg->llvm_ex_info_len = ei_len + nested_len;
9756 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
9757 /* Fill the rest of the information from the type info */
9758 for (i = 0; i < ei_len; ++i) {
9759 gint32 clause_index = *(gint32*)type_info [i];
9760 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
9761
9762 cfg->llvm_ex_info [i].flags = clause->flags;
9763 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
9764 cfg->llvm_ex_info [i].clause_index = clause_index;
9765 }
9766
9767 /*
9768 * For nested clauses, the LLVM produced exception info associates the try interval with
9769 * the innermost handler, while mono expects it to be associated with all nesting clauses.
9770 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
9771 * and everything else from the nested clause.
9772 */
9773 nindex = ei_len;
9774 for (i = 0; i < ei_len; ++i) {
9775 gint32 cindex1 = *(gint32*)type_info [i];
9776 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
9777
9778 for (j = 0; j < cfg->header->num_clauses; ++j) {
9779 int cindex2 = j;
9780 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
9781 MonoJitExceptionInfo *nesting_ei, *nested_ei;
9782
9783 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
9784 /* clause1 is the nested clause */
9785 nested_ei = &cfg->llvm_ex_info [i];
9786 nesting_ei = &cfg->llvm_ex_info [nindex];
9787 nindex ++;
9788
9789 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
9790
9791 nesting_ei->flags = clause2->flags;
9792 nesting_ei->data.catch_class = clause2->data.catch_class;
9793 nesting_ei->clause_index = cindex2;
9794 }
9795 }
9796 }
9797 g_assert (nindex == ei_len + nested_len);
9798 cfg->llvm_this_reg = this_reg;
9799 cfg->llvm_this_offset = this_offset;
9800
9801 /* type_info [i] is cfg mempool allocated, no need to free it */
9802
9803 g_free (ei);
9804 g_free (type_info);
9805 }
9806
9807 /*
9808 * decode_llvm_eh_info:
9809 *
9810 * Decode the EH table emitted by llvm in jit mode, and store
9811 * the result into cfg.
9812 */
9813 static void
9814 decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame)
9815 {
9816 MonoCompile *cfg = ctx->cfg;
9817 guint8 *cie, *fde;
9818 int fde_len;
9819 MonoLLVMFDEInfo info;
9820 MonoJitExceptionInfo *ei;
9821 guint8 *p = (guint8*)eh_frame;
9822 int version, fde_count, fde_offset;
9823 guint32 ei_len, i, nested_len;
9824 gpointer *type_info;
9825 gint32 *table;
9826 guint8 *unw_info;
9827
9828 /*
9829 * Decode the one element EH table emitted by the MonoException class
9830 * in llvm.
9831 */
9832
9833 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
9834
9835 version = *p;
9836 g_assert (version == 3);
9837 p ++;
9838 p ++;
9839 p = (guint8 *)ALIGN_PTR_TO (p, 4);
9840
9841 fde_count = *(guint32*)p;
9842 p += 4;
9843 table = (gint32*)p;
9844
9845 g_assert (fde_count <= 2);
9846
9847 /* The first entry is the real method */
9848 g_assert (table [0] == 1);
9849 fde_offset = table [1];
9850 table += fde_count * 2;
9851 /* Extra entry */
9852 cfg->code_len = table [0];
9853 fde_len = table [1] - fde_offset;
9854 table += 2;
9855
9856 fde = (guint8*)eh_frame + fde_offset;
9857 cie = (guint8*)table;
9858
9859 /* Compute lengths */
9860 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, NULL, NULL, NULL);
9861
9862 ei = (MonoJitExceptionInfo *)g_malloc0 (info.ex_info_len * sizeof (MonoJitExceptionInfo));
9863 type_info = (gpointer *)g_malloc0 (info.ex_info_len * sizeof (gpointer));
9864 unw_info = (guint8*)g_malloc0 (info.unw_info_len);
9865
9866 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, ei, type_info, unw_info);
9867
9868 cfg->encoded_unwind_ops = unw_info;
9869 cfg->encoded_unwind_ops_len = info.unw_info_len;
9870 if (cfg->verbose_level > 1)
9871 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
9872 if (info.this_reg != -1) {
9873 cfg->llvm_this_reg = info.this_reg;
9874 cfg->llvm_this_offset = info.this_offset;
9875 }
9876
9877 ei_len = info.ex_info_len;
9878
9879 // Nested clauses are currently disabled
9880 nested_len = 0;
9881
9882 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
9883 cfg->llvm_ex_info_len = ei_len + nested_len;
9884 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
9885 /* Fill the rest of the information from the type info */
9886 for (i = 0; i < ei_len; ++i) {
9887 gint32 clause_index = *(gint32*)type_info [i];
9888 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
9889
9890 cfg->llvm_ex_info [i].flags = clause->flags;
9891 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
9892 cfg->llvm_ex_info [i].clause_index = clause_index;
9893 }
9894 }
9895
9896 static void
9897 init_jit_module (MonoDomain *domain)
9898 {
9899 MonoJitDomainInfo *dinfo;
9900 MonoLLVMModule *module;
9901 char *name;
9902
9903 dinfo = domain_jit_info (domain);
9904 if (dinfo->llvm_module)
9905 return;
9906
9907 mono_loader_lock ();
9908
9909 if (dinfo->llvm_module) {
9910 mono_loader_unlock ();
9911 return;
9912 }
9913
9914 module = g_new0 (MonoLLVMModule, 1);
9915
9916 name = g_strdup_printf ("mono-%s", domain->friendly_name);
9917 module->lmodule = LLVMModuleCreateWithName (name);
9918 module->context = LLVMGetGlobalContext ();
9919 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
9920
9921 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (module->lmodule), alloc_cb, emitted_cb, exception_cb, &module->ee);
9922
9923 add_intrinsics (module->lmodule);
9924 add_types (module);
9925
9926 module->llvm_types = g_hash_table_new (NULL, NULL);
9927
9928 mono_memory_barrier ();
9929
9930 dinfo->llvm_module = module;
9931
9932 mono_loader_unlock ();
9933 }
9934
9935 static void
9936 llvm_jit_finalize_method (EmitContext *ctx)
9937 {
9938 MonoCompile *cfg = ctx->cfg;
9939 MonoDomain *domain = mono_domain_get ();
9940 MonoJitDomainInfo *domain_info;
9941 int nvars = g_hash_table_size (ctx->jit_callees);
9942 LLVMValueRef *callee_vars = g_new0 (LLVMValueRef, nvars);
9943 gpointer *callee_addrs = g_new0 (gpointer, nvars);
9944 GHashTableIter iter;
9945 LLVMValueRef var;
9946 MonoMethod *callee;
9947 gpointer eh_frame;
9948 int i;
9949
9950 /*
9951 * Compute the addresses of the LLVM globals pointing to the
9952 * methods called by the current method. Pass it to the trampoline
9953 * code so it can update them after their corresponding method was
9954 * compiled.
9955 */
9956 g_hash_table_iter_init (&iter, ctx->jit_callees);
9957 i = 0;
9958 while (g_hash_table_iter_next (&iter, NULL, (void**)&var))
9959 callee_vars [i ++] = var;
9960
9961 cfg->native_code = (guint8*)mono_llvm_compile_method (ctx->module->mono_ee, ctx->lmethod, nvars, callee_vars, callee_addrs, &eh_frame);
9962
9963 decode_llvm_eh_info (ctx, eh_frame);
9964
9965 mono_domain_lock (domain);
9966 domain_info = domain_jit_info (domain);
9967 if (!domain_info->llvm_jit_callees)
9968 domain_info->llvm_jit_callees = g_hash_table_new (NULL, NULL);
9969 g_hash_table_iter_init (&iter, ctx->jit_callees);
9970 i = 0;
9971 while (g_hash_table_iter_next (&iter, (void**)&callee, (void**)&var)) {
9972 GSList *addrs = (GSList*)g_hash_table_lookup (domain_info->llvm_jit_callees, callee);
9973 addrs = g_slist_prepend (addrs, callee_addrs [i]);
9974 g_hash_table_insert (domain_info->llvm_jit_callees, callee, addrs);
9975 i ++;
9976 }
9977 mono_domain_unlock (domain);
9978 }
9979
9980 #else
9981
9982 static void
9983 init_jit_module (MonoDomain *domain)
9984 {
9985 g_assert_not_reached ();
9986 }
9987
9988 static void
9989 llvm_jit_finalize_method (EmitContext *ctx)
9990 {
9991 g_assert_not_reached ();
9992 }
9993
9994 #endif
Mirror of the official Mono project git repository