?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge pull request #16242 from lateralusX/lateralusX/windows-fix-local-valuetype...
[mono-project.git] / mono / mini / mini-llvm.c
blob687f3fcad436a1e6bd014322c91d25c815409b80
1 /**
2 * \file
3 * llvm "Backend" for the mono JIT
4 *
5 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
6 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
7 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
8 */
9
10 #include "config.h"
11
12 #include <mono/metadata/debug-helpers.h>
13 #include <mono/metadata/debug-internals.h>
14 #include <mono/metadata/mempool-internals.h>
15 #include <mono/metadata/environment.h>
16 #include <mono/metadata/object-internals.h>
17 #include <mono/metadata/abi-details.h>
18 #include <mono/utils/mono-tls.h>
19 #include <mono/utils/mono-dl.h>
20 #include <mono/utils/mono-time.h>
21 #include <mono/utils/freebsd-dwarf.h>
22
23 #ifndef __STDC_LIMIT_MACROS
24 #define __STDC_LIMIT_MACROS
25 #endif
26 #ifndef __STDC_CONSTANT_MACROS
27 #define __STDC_CONSTANT_MACROS
28 #endif
29
30 #include "llvm-c/BitWriter.h"
31 #include "llvm-c/Analysis.h"
32
33 #include "mini-llvm-cpp.h"
34 #include "llvm-jit.h"
35 #include "aot-compiler.h"
36 #include "mini-llvm.h"
37 #include "mini-runtime.h"
38 #include <mono/utils/mono-math.h>
39
40 #ifndef DISABLE_JIT
41
42 #if defined(TARGET_AMD64) && defined(TARGET_WIN32) && defined(HOST_WIN32) && defined(_MSC_VER)
43 #define TARGET_X86_64_WIN32_MSVC
44 #endif
45
46 #if defined(TARGET_X86_64_WIN32_MSVC)
47 #define TARGET_WIN32_MSVC
48 #endif
49
50 #if LLVM_API_VERSION < 610
51 #error "The version of the mono llvm repository is too old."
52 #endif
53
54 /*
55 * Information associated by mono with LLVM modules.
56 */
57 typedef struct {
58 LLVMModuleRef lmodule;
59 LLVMValueRef throw_icall, rethrow, throw_corlib_exception;
60 GHashTable *llvm_types;
61 LLVMValueRef got_var;
62 const char *got_symbol;
63 const char *get_method_symbol;
64 const char *get_unbox_tramp_symbol;
65 GHashTable *plt_entries;
66 GHashTable *plt_entries_ji;
67 GHashTable *method_to_lmethod;
68 GHashTable *method_to_call_info;
69 GHashTable *lvalue_to_lcalls;
70 GHashTable *direct_callables;
71 char **bb_names;
72 int bb_names_len;
73 GPtrArray *used;
74 LLVMTypeRef ptr_type;
75 GPtrArray *subprogram_mds;
76 MonoEERef *mono_ee;
77 LLVMExecutionEngineRef ee;
78 gboolean external_symbols;
79 gboolean emit_dwarf;
80 int max_got_offset;
81 LLVMValueRef personality;
82 LLVMValueRef *intrins_by_id;
83 gpointer gc_poll_cold_wrapper_compiled;
84
85 /* For AOT */
86 MonoAssembly *assembly;
87 char *global_prefix;
88 MonoAotFileInfo aot_info;
89 const char *jit_got_symbol;
90 const char *eh_frame_symbol;
91 LLVMValueRef get_method, get_unbox_tramp;
92 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
93 LLVMValueRef code_start, code_end;
94 LLVMValueRef inited_var;
95 LLVMValueRef unbox_tramp_indexes;
96 LLVMValueRef unbox_trampolines;
97 LLVMValueRef gc_poll_cold_wrapper;
98 int max_inited_idx, max_method_idx;
99 gboolean has_jitted_code;
100 gboolean static_link;
101 gboolean llvm_only;
102 gboolean llvm_disable_self_init;
103 gboolean interp;
104 GHashTable *idx_to_lmethod;
105 GHashTable *idx_to_unbox_tramp;
106 GPtrArray *callsite_list;
107 LLVMContextRef context;
108 LLVMValueRef sentinel_exception;
109 void *di_builder, *cu;
110 GHashTable *objc_selector_to_var;
111 GPtrArray *cfgs;
112 int unbox_tramp_num, unbox_tramp_elemsize;
113 GHashTable *got_idx_to_type;
114 } MonoLLVMModule;
115
116 /*
117 * Information associated by the backend with mono basic blocks.
118 */
119 typedef struct {
120 LLVMBasicBlockRef bblock, end_bblock;
121 LLVMValueRef finally_ind;
122 gboolean added, invoke_target;
123 /*
124 * If this bblock is the start of a finally clause, this is a list of bblocks it
125 * needs to branch to in ENDFINALLY.
126 */
127 GSList *call_handler_return_bbs;
128 /*
129 * If this bblock is the start of a finally clause, this is the bblock that
130 * CALL_HANDLER needs to branch to.
131 */
132 LLVMBasicBlockRef call_handler_target_bb;
133 /* The list of switch statements generated by ENDFINALLY instructions */
134 GSList *endfinally_switch_ins_list;
135 GSList *phi_nodes;
136 } BBInfo;
137
138 /*
139 * Structure containing emit state
140 */
141 typedef struct {
142 MonoMemPool *mempool;
143
144 /* Maps method names to the corresponding LLVMValueRef */
145 GHashTable *emitted_method_decls;
146
147 MonoCompile *cfg;
148 LLVMValueRef lmethod;
149 MonoLLVMModule *module;
150 LLVMModuleRef lmodule;
151 BBInfo *bblocks;
152 int sindex, default_index, ex_index;
153 LLVMBuilderRef builder;
154 LLVMValueRef *values, *addresses;
155 MonoType **vreg_cli_types;
156 LLVMCallInfo *linfo;
157 MonoMethodSignature *sig;
158 GSList *builders;
159 GHashTable *region_to_handler;
160 GHashTable *clause_to_handler;
161 LLVMBuilderRef alloca_builder;
162 LLVMValueRef last_alloca;
163 LLVMValueRef rgctx_arg;
164 LLVMValueRef this_arg;
165 LLVMTypeRef *vreg_types;
166 gboolean *is_vphi;
167 LLVMTypeRef method_type;
168 LLVMBasicBlockRef init_bb, inited_bb;
169 gboolean *is_dead;
170 gboolean *unreachable;
171 gboolean llvm_only;
172 gboolean has_got_access;
173 gboolean emit_dummy_arg;
174 int this_arg_pindex, rgctx_arg_pindex;
175 LLVMValueRef imt_rgctx_loc;
176 GHashTable *llvm_types;
177 LLVMValueRef dbg_md;
178 MonoDebugMethodInfo *minfo;
179 /* For every clause, the clauses it is nested in */
180 GSList **nested_in;
181 LLVMValueRef ex_var;
182 GHashTable *exc_meta;
183 GPtrArray *callsite_list;
184 GPtrArray *phi_values;
185 GPtrArray *bblock_list;
186 char *method_name;
187 GHashTable *jit_callees;
188 LLVMValueRef long_bb_break_var;
189 } EmitContext;
190
191 typedef struct {
192 MonoBasicBlock *bb;
193 MonoInst *phi;
194 MonoBasicBlock *in_bb;
195 int sreg;
196 } PhiNode;
197
198 /*
199 * Instruction metadata
200 * This is the same as ins_info, but LREG != IREG.
201 */
202 #ifdef MINI_OP
203 #undef MINI_OP
204 #endif
205 #ifdef MINI_OP3
206 #undef MINI_OP3
207 #endif
208 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
209 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
210 #define NONE ' '
211 #define IREG 'i'
212 #define FREG 'f'
213 #define VREG 'v'
214 #define XREG 'x'
215 #define LREG 'l'
216 /* keep in sync with the enum in mini.h */
217 const char
218 mini_llvm_ins_info[] = {
219 #include "mini-ops.h"
220 };
221 #undef MINI_OP
222 #undef MINI_OP3
223
224 #if TARGET_SIZEOF_VOID_P == 4
225 #define GET_LONG_IMM(ins) ((ins)->inst_l)
226 #else
227 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
228 #endif
229
230 #define LLVM_INS_INFO(opcode) (&mini_llvm_ins_info [((opcode) - OP_START - 1) * 4])
231
232 #if 0
233 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
234 #else
235 #define TRACE_FAILURE(msg)
236 #endif
237
238 #ifdef TARGET_X86
239 #define IS_TARGET_X86 1
240 #else
241 #define IS_TARGET_X86 0
242 #endif
243
244 #ifdef TARGET_AMD64
245 #define IS_TARGET_AMD64 1
246 #else
247 #define IS_TARGET_AMD64 0
248 #endif
249
250 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
251
252 static LLVMIntPredicate cond_to_llvm_cond [] = {
253 LLVMIntEQ,
254 LLVMIntNE,
255 LLVMIntSLE,
256 LLVMIntSGE,
257 LLVMIntSLT,
258 LLVMIntSGT,
259 LLVMIntULE,
260 LLVMIntUGE,
261 LLVMIntULT,
262 LLVMIntUGT,
263 };
264
265 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
266 LLVMRealOEQ,
267 LLVMRealUNE,
268 LLVMRealOLE,
269 LLVMRealOGE,
270 LLVMRealOLT,
271 LLVMRealOGT,
272 LLVMRealULE,
273 LLVMRealUGE,
274 LLVMRealULT,
275 LLVMRealUGT,
276 };
277
278 typedef enum {
279 INTRINS_MEMSET,
280 INTRINS_MEMCPY,
281 INTRINS_SADD_OVF_I32,
282 INTRINS_UADD_OVF_I32,
283 INTRINS_SSUB_OVF_I32,
284 INTRINS_USUB_OVF_I32,
285 INTRINS_SMUL_OVF_I32,
286 INTRINS_UMUL_OVF_I32,
287 INTRINS_SADD_OVF_I64,
288 INTRINS_UADD_OVF_I64,
289 INTRINS_SSUB_OVF_I64,
290 INTRINS_USUB_OVF_I64,
291 INTRINS_SMUL_OVF_I64,
292 INTRINS_UMUL_OVF_I64,
293 INTRINS_SIN,
294 INTRINS_COS,
295 INTRINS_SQRT,
296 INTRINS_FABS,
297 INTRINS_ABSF,
298 INTRINS_SINF,
299 INTRINS_COSF,
300 INTRINS_SQRTF,
301 INTRINS_POWF,
302 INTRINS_EXPECT_I8,
303 INTRINS_EXPECT_I1,
304 INTRINS_CTPOP_I32,
305 INTRINS_CTPOP_I64,
306 INTRINS_CTLZ_I32,
307 INTRINS_CTLZ_I64,
308 INTRINS_CTTZ_I32,
309 INTRINS_CTTZ_I64,
310 INTRINS_PEXT_I32,
311 INTRINS_PEXT_I64,
312 INTRINS_PDEP_I32,
313 INTRINS_PDEP_I64,
314 #if defined(TARGET_AMD64) || defined(TARGET_X86)
315 INTRINS_SSE_PMOVMSKB,
316 INTRINS_SSE_PSRLI_W,
317 INTRINS_SSE_PSRAI_W,
318 INTRINS_SSE_PSLLI_W,
319 INTRINS_SSE_PSRLI_D,
320 INTRINS_SSE_PSRAI_D,
321 INTRINS_SSE_PSLLI_D,
322 INTRINS_SSE_PSRLI_Q,
323 INTRINS_SSE_PSLLI_Q,
324 INTRINS_SSE_SQRT_PD,
325 INTRINS_SSE_SQRT_PS,
326 INTRINS_SSE_RSQRT_PS,
327 INTRINS_SSE_RCP_PS,
328 INTRINS_SSE_CVTTPD2DQ,
329 INTRINS_SSE_CVTTPS2DQ,
330 INTRINS_SSE_CVTDQ2PD,
331 INTRINS_SSE_CVTDQ2PS,
332 INTRINS_SSE_CVTPD2DQ,
333 INTRINS_SSE_CVTPS2DQ,
334 INTRINS_SSE_CVTPD2PS,
335 INTRINS_SSE_CVTPS2PD,
336 INTRINS_SSE_CMPPD,
337 INTRINS_SSE_CMPPS,
338 INTRINS_SSE_PACKSSWB,
339 INTRINS_SSE_PACKUSWB,
340 INTRINS_SSE_PACKSSDW,
341 INTRINS_SSE_PACKUSDW,
342 INTRINS_SSE_MINPS,
343 INTRINS_SSE_MAXPS,
344 INTRINS_SSE_HADDPS,
345 INTRINS_SSE_HSUBPS,
346 INTRINS_SSE_ADDSUBPS,
347 INTRINS_SSE_MINPD,
348 INTRINS_SSE_MAXPD,
349 INTRINS_SSE_HADDPD,
350 INTRINS_SSE_HSUBPD,
351 INTRINS_SSE_ADDSUBPD,
352 INTRINS_SSE_PADDSW,
353 INTRINS_SSE_PSUBSW,
354 INTRINS_SSE_PADDUSW,
355 INTRINS_SSE_PSUBUSW,
356 INTRINS_SSE_PAVGW,
357 INTRINS_SSE_PMULHW,
358 INTRINS_SSE_PMULHU,
359 INTRINS_SE_PADDSB,
360 INTRINS_SSE_PSUBSB,
361 INTRINS_SSE_PADDUSB,
362 INTRINS_SSE_PSUBUSB,
363 INTRINS_SSE_PAVGB,
364 INTRINS_SSE_PAUSE,
365 INTRINS_SSE_DPPS,
366 INTRINS_SSE_ROUNDPD,
367 #endif
368 INTRINS_NUM
369 } IntrinsicId;
370
371 static MonoNativeTlsKey current_cfg_tls_id;
372
373 static MonoLLVMModule aot_module;
374
375 static GHashTable *intrins_id_to_name;
376 static GHashTable *intrins_name_to_id;
377
378 static void init_jit_module (MonoDomain *domain);
379
380 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
381 static void emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder);
382 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
383 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
384 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp);
385 static LLVMValueRef get_intrins_by_name (EmitContext *ctx, const char *name);
386 static LLVMValueRef get_intrins (EmitContext *ctx, int id);
387 static LLVMValueRef get_intrins_from_module (LLVMModuleRef lmodule, int id);
388 static void llvm_jit_finalize_method (EmitContext *ctx);
389 static void mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params);
390 static void mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module);
391
392 static inline void
393 set_failure (EmitContext *ctx, const char *message)
394 {
395 TRACE_FAILURE (reason);
396 ctx->cfg->exception_message = g_strdup (message);
397 ctx->cfg->disable_llvm = TRUE;
398 }
399
400 static LLVMValueRef
401 ConstInt32 (int v)
402 {
403 return LLVMConstInt (LLVMInt32Type (), v, FALSE);
404 }
405
406 /*
407 * IntPtrType:
408 *
409 * The LLVM type with width == TARGET_SIZEOF_VOID_P
410 */
411 static LLVMTypeRef
412 IntPtrType (void)
413 {
414 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
415 }
416
417 static LLVMTypeRef
418 ObjRefType (void)
419 {
420 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
421 }
422
423 static LLVMTypeRef
424 ThisType (void)
425 {
426 return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
427 }
428
429 /*
430 * get_vtype_size:
431 *
432 * Return the size of the LLVM representation of the vtype T.
433 */
434 static guint32
435 get_vtype_size (MonoType *t)
436 {
437 int size;
438
439 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
440
441 /* LLVMArgAsIArgs depends on this since it stores whole words */
442 while (size < 2 * TARGET_SIZEOF_VOID_P && mono_is_power_of_two (size) == -1)
443 size ++;
444
445 return size;
446 }
447
448 /*
449 * simd_class_to_llvm_type:
450 *
451 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
452 */
453 static LLVMTypeRef
454 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
455 {
456 const char *klass_name = m_class_get_name (klass);
457 if (!strcmp (klass_name, "Vector2d")) {
458 return LLVMVectorType (LLVMDoubleType (), 2);
459 } else if (!strcmp (klass_name, "Vector2l")) {
460 return LLVMVectorType (LLVMInt64Type (), 2);
461 } else if (!strcmp (klass_name, "Vector2ul")) {
462 return LLVMVectorType (LLVMInt64Type (), 2);
463 } else if (!strcmp (klass_name, "Vector4i")) {
464 return LLVMVectorType (LLVMInt32Type (), 4);
465 } else if (!strcmp (klass_name, "Vector4ui")) {
466 return LLVMVectorType (LLVMInt32Type (), 4);
467 } else if (!strcmp (klass_name, "Vector4f")) {
468 return LLVMVectorType (LLVMFloatType (), 4);
469 } else if (!strcmp (klass_name, "Vector8s")) {
470 return LLVMVectorType (LLVMInt16Type (), 8);
471 } else if (!strcmp (klass_name, "Vector8us")) {
472 return LLVMVectorType (LLVMInt16Type (), 8);
473 } else if (!strcmp (klass_name, "Vector16sb")) {
474 return LLVMVectorType (LLVMInt8Type (), 16);
475 } else if (!strcmp (klass_name, "Vector16b")) {
476 return LLVMVectorType (LLVMInt8Type (), 16);
477 } else if (!strcmp (klass_name, "Vector2")) {
478 /* System.Numerics */
479 return LLVMVectorType (LLVMFloatType (), 4);
480 } else if (!strcmp (klass_name, "Vector3")) {
481 return LLVMVectorType (LLVMFloatType (), 4);
482 } else if (!strcmp (klass_name, "Vector4")) {
483 return LLVMVectorType (LLVMFloatType (), 4);
484 } else if (!strcmp (klass_name, "Vector`1")) {
485 MonoType *etype = mono_class_get_generic_class (klass)->context.class_inst->type_argv [0];
486 int size = mono_class_value_size (klass, NULL);
487 switch (etype->type) {
488 case MONO_TYPE_I1:
489 case MONO_TYPE_U1:
490 return LLVMVectorType (LLVMInt8Type (), size);
491 case MONO_TYPE_I2:
492 case MONO_TYPE_U2:
493 return LLVMVectorType (LLVMInt16Type (), size / 2);
494 case MONO_TYPE_I4:
495 case MONO_TYPE_U4:
496 return LLVMVectorType (LLVMInt32Type (), size / 4);
497 case MONO_TYPE_I8:
498 case MONO_TYPE_U8:
499 return LLVMVectorType (LLVMInt64Type (), size / 8);
500 case MONO_TYPE_R4:
501 return LLVMVectorType (LLVMFloatType (), size / 4);
502 case MONO_TYPE_R8:
503 return LLVMVectorType (LLVMDoubleType (), size / 8);
504 default:
505 g_assert_not_reached ();
506 return NULL;
507 }
508 } else {
509 printf ("%s\n", klass_name);
510 NOT_IMPLEMENTED;
511 return NULL;
512 }
513 }
514
515 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
516 static inline G_GNUC_UNUSED LLVMTypeRef
517 type_to_simd_type (int type)
518 {
519 switch (type) {
520 case MONO_TYPE_I1:
521 return LLVMVectorType (LLVMInt8Type (), 16);
522 case MONO_TYPE_I2:
523 return LLVMVectorType (LLVMInt16Type (), 8);
524 case MONO_TYPE_I4:
525 return LLVMVectorType (LLVMInt32Type (), 4);
526 case MONO_TYPE_I8:
527 return LLVMVectorType (LLVMInt64Type (), 2);
528 case MONO_TYPE_R8:
529 return LLVMVectorType (LLVMDoubleType (), 2);
530 case MONO_TYPE_R4:
531 return LLVMVectorType (LLVMFloatType (), 4);
532 default:
533 g_assert_not_reached ();
534 return NULL;
535 }
536 }
537
538 static LLVMTypeRef
539 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
540 {
541 int i, size, nfields, esize;
542 LLVMTypeRef *eltypes;
543 char *name;
544 MonoType *t;
545 LLVMTypeRef ltype;
546
547 t = m_class_get_byval_arg (klass);
548
549 if (mini_type_is_hfa (t, &nfields, &esize)) {
550 /*
551 * This is needed on arm64 where HFAs are returned in
552 * registers.
553 */
554 /* SIMD types have size 16 in mono_class_value_size () */
555 if (m_class_is_simd_type (klass))
556 nfields = 16/ esize;
557 size = nfields;
558 eltypes = g_new (LLVMTypeRef, size);
559 for (i = 0; i < size; ++i)
560 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
561 } else {
562 size = get_vtype_size (t);
563
564 eltypes = g_new (LLVMTypeRef, size);
565 for (i = 0; i < size; ++i)
566 eltypes [i] = LLVMInt8Type ();
567 }
568
569 name = mono_type_full_name (m_class_get_byval_arg (klass));
570 ltype = LLVMStructCreateNamed (module->context, name);
571 LLVMStructSetBody (ltype, eltypes, size, FALSE);
572 g_free (eltypes);
573 g_free (name);
574
575 return ltype;
576 }
577
578 /*
579 * type_to_llvm_type:
580 *
581 * Return the LLVM type corresponding to T.
582 */
583 static LLVMTypeRef
584 type_to_llvm_type (EmitContext *ctx, MonoType *t)
585 {
586 if (t->byref)
587 return ThisType ();
588
589 t = mini_get_underlying_type (t);
590
591 switch (t->type) {
592 case MONO_TYPE_VOID:
593 return LLVMVoidType ();
594 case MONO_TYPE_I1:
595 return LLVMInt8Type ();
596 case MONO_TYPE_I2:
597 return LLVMInt16Type ();
598 case MONO_TYPE_I4:
599 return LLVMInt32Type ();
600 case MONO_TYPE_U1:
601 return LLVMInt8Type ();
602 case MONO_TYPE_U2:
603 return LLVMInt16Type ();
604 case MONO_TYPE_U4:
605 return LLVMInt32Type ();
606 case MONO_TYPE_I8:
607 case MONO_TYPE_U8:
608 return LLVMInt64Type ();
609 case MONO_TYPE_R4:
610 return LLVMFloatType ();
611 case MONO_TYPE_R8:
612 return LLVMDoubleType ();
613 case MONO_TYPE_I:
614 case MONO_TYPE_U:
615 return IntPtrType ();
616 case MONO_TYPE_OBJECT:
617 return ObjRefType ();
618 case MONO_TYPE_PTR: {
619 MonoClass *klass = mono_class_from_mono_type_internal (t);
620 MonoClass *ptr_klass = m_class_get_element_class (klass);
621 MonoType *ptr_type = m_class_get_byval_arg (ptr_klass);
622 /* Handle primitive pointers */
623 switch (ptr_type->type) {
624 case MONO_TYPE_I1:
625 case MONO_TYPE_I2:
626 case MONO_TYPE_I4:
627 case MONO_TYPE_U1:
628 case MONO_TYPE_U2:
629 case MONO_TYPE_U4:
630 return LLVMPointerType (type_to_llvm_type (ctx, ptr_type), 0);
631 }
632
633 return ObjRefType ();
634 }
635 case MONO_TYPE_VAR:
636 case MONO_TYPE_MVAR:
637 /* Because of generic sharing */
638 return ObjRefType ();
639 case MONO_TYPE_GENERICINST:
640 if (!mono_type_generic_inst_is_valuetype (t))
641 return ObjRefType ();
642 /* Fall through */
643 case MONO_TYPE_VALUETYPE:
644 case MONO_TYPE_TYPEDBYREF: {
645 MonoClass *klass;
646 LLVMTypeRef ltype;
647
648 klass = mono_class_from_mono_type_internal (t);
649
650 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
651 return simd_class_to_llvm_type (ctx, klass);
652
653 if (m_class_is_enumtype (klass))
654 return type_to_llvm_type (ctx, mono_class_enum_basetype_internal (klass));
655
656 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
657 if (!ltype) {
658 ltype = create_llvm_type_for_type (ctx->module, klass);
659 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
660 }
661 return ltype;
662 }
663
664 default:
665 printf ("X: %d\n", t->type);
666 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
667 ctx->cfg->disable_llvm = TRUE;
668 return NULL;
669 }
670 }
671
672 /*
673 * type_is_unsigned:
674 *
675 * Return whenever T is an unsigned int type.
676 */
677 static gboolean
678 type_is_unsigned (EmitContext *ctx, MonoType *t)
679 {
680 t = mini_get_underlying_type (t);
681 if (t->byref)
682 return FALSE;
683 switch (t->type) {
684 case MONO_TYPE_U1:
685 case MONO_TYPE_U2:
686 case MONO_TYPE_CHAR:
687 case MONO_TYPE_U4:
688 case MONO_TYPE_U8:
689 return TRUE;
690 default:
691 return FALSE;
692 }
693 }
694
695 /*
696 * type_to_llvm_arg_type:
697 *
698 * Same as type_to_llvm_type, but treat i8/i16 as i32.
699 */
700 static LLVMTypeRef
701 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
702 {
703 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
704
705 if (ctx->cfg->llvm_only)
706 return ptype;
707
708 /*
709 * This works on all abis except arm64/ios which passes multiple
710 * arguments in one stack slot.
711 */
712 #ifndef TARGET_ARM64
713 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
714 /*
715 * LLVM generates code which only sets the lower bits, while JITted
716 * code expects all the bits to be set.
717 */
718 ptype = LLVMInt32Type ();
719 }
720 #endif
721
722 return ptype;
723 }
724
725 /*
726 * llvm_type_to_stack_type:
727 *
728 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
729 * on the IL stack.
730 */
731 static G_GNUC_UNUSED LLVMTypeRef
732 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
733 {
734 if (type == NULL)
735 return NULL;
736 if (type == LLVMInt8Type ())
737 return LLVMInt32Type ();
738 else if (type == LLVMInt16Type ())
739 return LLVMInt32Type ();
740 else if (!cfg->r4fp && type == LLVMFloatType ())
741 return LLVMDoubleType ();
742 else
743 return type;
744 }
745
746 /*
747 * regtype_to_llvm_type:
748 *
749 * Return the LLVM type corresponding to the regtype C used in instruction
750 * descriptions.
751 */
752 static LLVMTypeRef
753 regtype_to_llvm_type (char c)
754 {
755 switch (c) {
756 case 'i':
757 return LLVMInt32Type ();
758 case 'l':
759 return LLVMInt64Type ();
760 case 'f':
761 return LLVMDoubleType ();
762 default:
763 return NULL;
764 }
765 }
766
767 /*
768 * op_to_llvm_type:
769 *
770 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
771 */
772 static LLVMTypeRef
773 op_to_llvm_type (int opcode)
774 {
775 switch (opcode) {
776 case OP_ICONV_TO_I1:
777 case OP_LCONV_TO_I1:
778 return LLVMInt8Type ();
779 case OP_ICONV_TO_U1:
780 case OP_LCONV_TO_U1:
781 return LLVMInt8Type ();
782 case OP_ICONV_TO_I2:
783 case OP_LCONV_TO_I2:
784 return LLVMInt16Type ();
785 case OP_ICONV_TO_U2:
786 case OP_LCONV_TO_U2:
787 return LLVMInt16Type ();
788 case OP_ICONV_TO_I4:
789 case OP_LCONV_TO_I4:
790 return LLVMInt32Type ();
791 case OP_ICONV_TO_U4:
792 case OP_LCONV_TO_U4:
793 return LLVMInt32Type ();
794 case OP_ICONV_TO_I8:
795 return LLVMInt64Type ();
796 case OP_ICONV_TO_R4:
797 return LLVMFloatType ();
798 case OP_ICONV_TO_R8:
799 return LLVMDoubleType ();
800 case OP_ICONV_TO_U8:
801 return LLVMInt64Type ();
802 case OP_FCONV_TO_I4:
803 return LLVMInt32Type ();
804 case OP_FCONV_TO_I8:
805 return LLVMInt64Type ();
806 case OP_FCONV_TO_I1:
807 case OP_FCONV_TO_U1:
808 case OP_RCONV_TO_I1:
809 case OP_RCONV_TO_U1:
810 return LLVMInt8Type ();
811 case OP_FCONV_TO_I2:
812 case OP_FCONV_TO_U2:
813 case OP_RCONV_TO_I2:
814 case OP_RCONV_TO_U2:
815 return LLVMInt16Type ();
816 case OP_FCONV_TO_U4:
817 case OP_RCONV_TO_U4:
818 return LLVMInt32Type ();
819 case OP_FCONV_TO_U8:
820 case OP_RCONV_TO_U8:
821 return LLVMInt64Type ();
822 case OP_FCONV_TO_I:
823 case OP_FCONV_TO_U:
824 return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
825 case OP_IADD_OVF:
826 case OP_IADD_OVF_UN:
827 case OP_ISUB_OVF:
828 case OP_ISUB_OVF_UN:
829 case OP_IMUL_OVF:
830 case OP_IMUL_OVF_UN:
831 return LLVMInt32Type ();
832 case OP_LADD_OVF:
833 case OP_LADD_OVF_UN:
834 case OP_LSUB_OVF:
835 case OP_LSUB_OVF_UN:
836 case OP_LMUL_OVF:
837 case OP_LMUL_OVF_UN:
838 return LLVMInt64Type ();
839 default:
840 printf ("%s\n", mono_inst_name (opcode));
841 g_assert_not_reached ();
842 return NULL;
843 }
844 }
845
846 #define CLAUSE_START(clause) ((clause)->try_offset)
847 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
848
849 /*
850 * load_store_to_llvm_type:
851 *
852 * Return the size/sign/zero extension corresponding to the load/store opcode
853 * OPCODE.
854 */
855 static LLVMTypeRef
856 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
857 {
858 *sext = FALSE;
859 *zext = FALSE;
860
861 switch (opcode) {
862 case OP_LOADI1_MEMBASE:
863 case OP_STOREI1_MEMBASE_REG:
864 case OP_STOREI1_MEMBASE_IMM:
865 case OP_ATOMIC_LOAD_I1:
866 case OP_ATOMIC_STORE_I1:
867 *size = 1;
868 *sext = TRUE;
869 return LLVMInt8Type ();
870 case OP_LOADU1_MEMBASE:
871 case OP_LOADU1_MEM:
872 case OP_ATOMIC_LOAD_U1:
873 case OP_ATOMIC_STORE_U1:
874 *size = 1;
875 *zext = TRUE;
876 return LLVMInt8Type ();
877 case OP_LOADI2_MEMBASE:
878 case OP_STOREI2_MEMBASE_REG:
879 case OP_STOREI2_MEMBASE_IMM:
880 case OP_ATOMIC_LOAD_I2:
881 case OP_ATOMIC_STORE_I2:
882 *size = 2;
883 *sext = TRUE;
884 return LLVMInt16Type ();
885 case OP_LOADU2_MEMBASE:
886 case OP_LOADU2_MEM:
887 case OP_ATOMIC_LOAD_U2:
888 case OP_ATOMIC_STORE_U2:
889 *size = 2;
890 *zext = TRUE;
891 return LLVMInt16Type ();
892 case OP_LOADI4_MEMBASE:
893 case OP_LOADU4_MEMBASE:
894 case OP_LOADI4_MEM:
895 case OP_LOADU4_MEM:
896 case OP_STOREI4_MEMBASE_REG:
897 case OP_STOREI4_MEMBASE_IMM:
898 case OP_ATOMIC_LOAD_I4:
899 case OP_ATOMIC_STORE_I4:
900 case OP_ATOMIC_LOAD_U4:
901 case OP_ATOMIC_STORE_U4:
902 *size = 4;
903 return LLVMInt32Type ();
904 case OP_LOADI8_MEMBASE:
905 case OP_LOADI8_MEM:
906 case OP_STOREI8_MEMBASE_REG:
907 case OP_STOREI8_MEMBASE_IMM:
908 case OP_ATOMIC_LOAD_I8:
909 case OP_ATOMIC_STORE_I8:
910 case OP_ATOMIC_LOAD_U8:
911 case OP_ATOMIC_STORE_U8:
912 *size = 8;
913 return LLVMInt64Type ();
914 case OP_LOADR4_MEMBASE:
915 case OP_STORER4_MEMBASE_REG:
916 case OP_ATOMIC_LOAD_R4:
917 case OP_ATOMIC_STORE_R4:
918 *size = 4;
919 return LLVMFloatType ();
920 case OP_LOADR8_MEMBASE:
921 case OP_STORER8_MEMBASE_REG:
922 case OP_ATOMIC_LOAD_R8:
923 case OP_ATOMIC_STORE_R8:
924 *size = 8;
925 return LLVMDoubleType ();
926 case OP_LOAD_MEMBASE:
927 case OP_LOAD_MEM:
928 case OP_STORE_MEMBASE_REG:
929 case OP_STORE_MEMBASE_IMM:
930 *size = TARGET_SIZEOF_VOID_P;
931 return IntPtrType ();
932 default:
933 g_assert_not_reached ();
934 return NULL;
935 }
936 }
937
938 /*
939 * ovf_op_to_intrins:
940 *
941 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
942 */
943 static const char*
944 ovf_op_to_intrins (int opcode)
945 {
946 switch (opcode) {
947 case OP_IADD_OVF:
948 return "llvm.sadd.with.overflow.i32";
949 case OP_IADD_OVF_UN:
950 return "llvm.uadd.with.overflow.i32";
951 case OP_ISUB_OVF:
952 return "llvm.ssub.with.overflow.i32";
953 case OP_ISUB_OVF_UN:
954 return "llvm.usub.with.overflow.i32";
955 case OP_IMUL_OVF:
956 return "llvm.smul.with.overflow.i32";
957 case OP_IMUL_OVF_UN:
958 return "llvm.umul.with.overflow.i32";
959 case OP_LADD_OVF:
960 return "llvm.sadd.with.overflow.i64";
961 case OP_LADD_OVF_UN:
962 return "llvm.uadd.with.overflow.i64";
963 case OP_LSUB_OVF:
964 return "llvm.ssub.with.overflow.i64";
965 case OP_LSUB_OVF_UN:
966 return "llvm.usub.with.overflow.i64";
967 case OP_LMUL_OVF:
968 return "llvm.smul.with.overflow.i64";
969 case OP_LMUL_OVF_UN:
970 return "llvm.umul.with.overflow.i64";
971 default:
972 g_assert_not_reached ();
973 return NULL;
974 }
975 }
976
977 static const char*
978 simd_op_to_intrins (int opcode)
979 {
980 switch (opcode) {
981 #if defined(TARGET_X86) || defined(TARGET_AMD64)
982 case OP_MINPD:
983 return "llvm.x86.sse2.min.pd";
984 case OP_MINPS:
985 return "llvm.x86.sse.min.ps";
986 case OP_MAXPD:
987 return "llvm.x86.sse2.max.pd";
988 case OP_MAXPS:
989 return "llvm.x86.sse.max.ps";
990 case OP_HADDPD:
991 return "llvm.x86.sse3.hadd.pd";
992 case OP_HADDPS:
993 return "llvm.x86.sse3.hadd.ps";
994 case OP_HSUBPD:
995 return "llvm.x86.sse3.hsub.pd";
996 case OP_HSUBPS:
997 return "llvm.x86.sse3.hsub.ps";
998 case OP_ADDSUBPS:
999 return "llvm.x86.sse3.addsub.ps";
1000 case OP_ADDSUBPD:
1001 return "llvm.x86.sse3.addsub.pd";
1002 case OP_EXTRACT_MASK:
1003 return "llvm.x86.sse2.pmovmskb.128";
1004 case OP_PSHRW:
1005 case OP_PSHRW_REG:
1006 return "llvm.x86.sse2.psrli.w";
1007 case OP_PSHRD:
1008 case OP_PSHRD_REG:
1009 return "llvm.x86.sse2.psrli.d";
1010 case OP_PSHRQ:
1011 case OP_PSHRQ_REG:
1012 return "llvm.x86.sse2.psrli.q";
1013 case OP_PSHLW:
1014 case OP_PSHLW_REG:
1015 return "llvm.x86.sse2.pslli.w";
1016 case OP_PSHLD:
1017 case OP_PSHLD_REG:
1018 return "llvm.x86.sse2.pslli.d";
1019 case OP_PSHLQ:
1020 case OP_PSHLQ_REG:
1021 return "llvm.x86.sse2.pslli.q";
1022 case OP_PSARW:
1023 case OP_PSARW_REG:
1024 return "llvm.x86.sse2.psrai.w";
1025 case OP_PSARD:
1026 case OP_PSARD_REG:
1027 return "llvm.x86.sse2.psrai.d";
1028 case OP_PADDB_SAT:
1029 return "llvm.x86.sse2.padds.b";
1030 case OP_PADDW_SAT:
1031 return "llvm.x86.sse2.padds.w";
1032 case OP_PSUBB_SAT:
1033 return "llvm.x86.sse2.psubs.b";
1034 case OP_PSUBW_SAT:
1035 return "llvm.x86.sse2.psubs.w";
1036 case OP_PADDB_SAT_UN:
1037 return "llvm.x86.sse2.paddus.b";
1038 case OP_PADDW_SAT_UN:
1039 return "llvm.x86.sse2.paddus.w";
1040 case OP_PSUBB_SAT_UN:
1041 return "llvm.x86.sse2.psubus.b";
1042 case OP_PSUBW_SAT_UN:
1043 return "llvm.x86.sse2.psubus.w";
1044 case OP_PAVGB_UN:
1045 return "llvm.x86.sse2.pavg.b";
1046 case OP_PAVGW_UN:
1047 return "llvm.x86.sse2.pavg.w";
1048 case OP_SQRTPS:
1049 return "llvm.x86.sse.sqrt.ps";
1050 case OP_SQRTPD:
1051 return "llvm.x86.sse2.sqrt.pd";
1052 case OP_RSQRTPS:
1053 return "llvm.x86.sse.rsqrt.ps";
1054 case OP_RCPPS:
1055 return "llvm.x86.sse.rcp.ps";
1056 case OP_CVTDQ2PD:
1057 return "llvm.x86.sse2.cvtdq2pd";
1058 case OP_CVTDQ2PS:
1059 return "llvm.x86.sse2.cvtdq2ps";
1060 case OP_CVTPD2DQ:
1061 return "llvm.x86.sse2.cvtpd2dq";
1062 case OP_CVTPS2DQ:
1063 return "llvm.x86.sse2.cvtps2dq";
1064 case OP_CVTPD2PS:
1065 return "llvm.x86.sse2.cvtpd2ps";
1066 case OP_CVTPS2PD:
1067 return "llvm.x86.sse2.cvtps2pd";
1068 case OP_CVTTPD2DQ:
1069 return "llvm.x86.sse2.cvttpd2dq";
1070 case OP_CVTTPS2DQ:
1071 return "llvm.x86.sse2.cvttps2dq";
1072 case OP_PACKW:
1073 return "llvm.x86.sse2.packsswb.128";
1074 case OP_PACKD:
1075 return "llvm.x86.sse2.packssdw.128";
1076 case OP_PACKW_UN:
1077 return "llvm.x86.sse2.packuswb.128";
1078 case OP_PACKD_UN:
1079 return "llvm.x86.sse41.packusdw";
1080 case OP_PMULW_HIGH:
1081 return "llvm.x86.sse2.pmulh.w";
1082 case OP_PMULW_HIGH_UN:
1083 return "llvm.x86.sse2.pmulhu.w";
1084 case OP_DPPS:
1085 return "llvm.x86.sse41.dpps";
1086 #endif
1087 default:
1088 g_assert_not_reached ();
1089 return NULL;
1090 }
1091 }
1092
1093 static LLVMTypeRef
1094 simd_op_to_llvm_type (int opcode)
1095 {
1096 #if defined(TARGET_X86) || defined(TARGET_AMD64)
1097 switch (opcode) {
1098 case OP_EXTRACT_R8:
1099 case OP_EXPAND_R8:
1100 return type_to_simd_type (MONO_TYPE_R8);
1101 case OP_EXTRACT_I8:
1102 case OP_EXPAND_I8:
1103 return type_to_simd_type (MONO_TYPE_I8);
1104 case OP_EXTRACT_I4:
1105 case OP_EXPAND_I4:
1106 return type_to_simd_type (MONO_TYPE_I4);
1107 case OP_EXTRACT_I2:
1108 case OP_EXTRACT_U2:
1109 case OP_EXTRACTX_U2:
1110 case OP_EXPAND_I2:
1111 return type_to_simd_type (MONO_TYPE_I2);
1112 case OP_EXTRACT_I1:
1113 case OP_EXTRACT_U1:
1114 case OP_EXPAND_I1:
1115 return type_to_simd_type (MONO_TYPE_I1);
1116 case OP_EXPAND_R4:
1117 return type_to_simd_type (MONO_TYPE_R4);
1118 case OP_CVTDQ2PD:
1119 case OP_CVTDQ2PS:
1120 return type_to_simd_type (MONO_TYPE_I4);
1121 case OP_CVTPD2DQ:
1122 case OP_CVTPD2PS:
1123 case OP_CVTTPD2DQ:
1124 return type_to_simd_type (MONO_TYPE_R8);
1125 case OP_CVTPS2DQ:
1126 case OP_CVTPS2PD:
1127 case OP_CVTTPS2DQ:
1128 return type_to_simd_type (MONO_TYPE_R4);
1129 case OP_EXTRACT_MASK:
1130 return type_to_simd_type (MONO_TYPE_I1);
1131 case OP_SQRTPS:
1132 case OP_RSQRTPS:
1133 case OP_RCPPS:
1134 case OP_DUPPS_LOW:
1135 case OP_DUPPS_HIGH:
1136 return type_to_simd_type (MONO_TYPE_R4);
1137 case OP_SQRTPD:
1138 case OP_DUPPD:
1139 return type_to_simd_type (MONO_TYPE_R8);
1140 default:
1141 g_assert_not_reached ();
1142 return NULL;
1143 }
1144 #else
1145 return NULL;
1146 #endif
1147 }
1148
1149 static void
1150 set_cold_cconv (LLVMValueRef func)
1151 {
1152 /*
1153 * xcode10 (watchOS) and ARM/ARM64 doesn't seem to support preserveall, it fails with:
1154 * fatal error: error in backend: Unsupported calling convention
1155 */
1156 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1157 LLVMSetFunctionCallConv (func, LLVMColdCallConv);
1158 #endif
1159 }
1160
1161 static void
1162 set_call_cold_cconv (LLVMValueRef func)
1163 {
1164 #if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
1165 LLVMSetInstructionCallConv (func, LLVMColdCallConv);
1166 #endif
1167 }
1168
1169 /*
1170 * get_bb:
1171 *
1172 * Return the LLVM basic block corresponding to BB.
1173 */
1174 static LLVMBasicBlockRef
1175 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
1176 {
1177 char bb_name_buf [128];
1178 char *bb_name;
1179
1180 if (ctx->bblocks [bb->block_num].bblock == NULL) {
1181 if (bb->flags & BB_EXCEPTION_HANDLER) {
1182 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
1183 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
1184 bb_name = bb_name_buf;
1185 } else if (bb->block_num < 256) {
1186 if (!ctx->module->bb_names) {
1187 ctx->module->bb_names_len = 256;
1188 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
1189 }
1190 if (!ctx->module->bb_names [bb->block_num]) {
1191 char *n;
1192
1193 n = g_strdup_printf ("BB%d", bb->block_num);
1194 mono_memory_barrier ();
1195 ctx->module->bb_names [bb->block_num] = n;
1196 }
1197 bb_name = ctx->module->bb_names [bb->block_num];
1198 } else {
1199 sprintf (bb_name_buf, "BB%d", bb->block_num);
1200 bb_name = bb_name_buf;
1201 }
1202
1203 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1204 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1205 }
1206
1207 return ctx->bblocks [bb->block_num].bblock;
1208 }
1209
1210 /*
1211 * get_end_bb:
1212 *
1213 * Return the last LLVM bblock corresponding to BB.
1214 * This might not be equal to the bb returned by get_bb () since we need to generate
1215 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1216 */
1217 static LLVMBasicBlockRef
1218 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1219 {
1220 get_bb (ctx, bb);
1221 return ctx->bblocks [bb->block_num].end_bblock;
1222 }
1223
1224 static LLVMBasicBlockRef
1225 gen_bb (EmitContext *ctx, const char *prefix)
1226 {
1227 char bb_name [128];
1228
1229 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1230 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1231 }
1232
1233 /*
1234 * resolve_patch:
1235 *
1236 * Return the target of the patch identified by TYPE and TARGET.
1237 */
1238 static gpointer
1239 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1240 {
1241 MonoJumpInfo ji;
1242 ERROR_DECL (error);
1243 gpointer res;
1244
1245 memset (&ji, 0, sizeof (ji));
1246 ji.type = type;
1247 ji.data.target = target;
1248
1249 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, error);
1250 mono_error_assert_ok (error);
1251
1252 return res;
1253 }
1254
1255 /*
1256 * convert_full:
1257 *
1258 * Emit code to convert the LLVM value V to DTYPE.
1259 */
1260 static LLVMValueRef
1261 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1262 {
1263 LLVMTypeRef stype = LLVMTypeOf (v);
1264
1265 if (stype != dtype) {
1266 gboolean ext = FALSE;
1267
1268 /* Extend */
1269 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1270 ext = TRUE;
1271 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1272 ext = TRUE;
1273 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1274 ext = TRUE;
1275
1276 if (ext)
1277 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1278
1279 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1280 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1281
1282 /* Trunc */
1283 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1284 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1285 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1286 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1287 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1288 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1289 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1290 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1291
1292 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1293 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1294 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1295 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1296 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1297 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1298
1299 if (mono_arch_is_soft_float ()) {
1300 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1301 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1302 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1303 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1304 }
1305
1306 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1307 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1308
1309 LLVMDumpValue (v);
1310 printf ("\n");
1311 LLVMDumpValue (LLVMConstNull (dtype));
1312 printf ("\n");
1313 g_assert_not_reached ();
1314 return NULL;
1315 } else {
1316 return v;
1317 }
1318 }
1319
1320 static LLVMValueRef
1321 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1322 {
1323 return convert_full (ctx, v, dtype, FALSE);
1324 }
1325
1326 static void
1327 emit_memset (EmitContext *ctx, LLVMBuilderRef builder, LLVMValueRef v, LLVMValueRef size, int alignment)
1328 {
1329 LLVMValueRef args [5];
1330 int aindex = 0;
1331
1332 args [aindex ++] = v;
1333 args [aindex ++] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
1334 args [aindex ++] = size;
1335 #if LLVM_API_VERSION < 900
1336 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), alignment, FALSE);
1337 #endif
1338 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
1339 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMSET), args, aindex, "");
1340 }
1341
1342 /*
1343 * emit_volatile_load:
1344 *
1345 * If vreg is volatile, emit a load from its address.
1346 */
1347 static LLVMValueRef
1348 emit_volatile_load (EmitContext *ctx, int vreg)
1349 {
1350 MonoType *t;
1351 LLVMValueRef v;
1352
1353 // On arm64, we pass the rgctx in a callee saved
1354 // register on arm64 (x15), and llvm might keep the value in that register
1355 // even through the register is marked as 'reserved' inside llvm.
1356
1357 v = mono_llvm_build_load (ctx->builder, ctx->addresses [vreg], "", TRUE);
1358 t = ctx->vreg_cli_types [vreg];
1359 if (t && !t->byref) {
1360 /*
1361 * Might have to zero extend since llvm doesn't have
1362 * unsigned types.
1363 */
1364 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1365 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1366 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1367 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1368 else if (t->type == MONO_TYPE_U8)
1369 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1370 }
1371
1372 return v;
1373 }
1374
1375 /*
1376 * emit_volatile_store:
1377 *
1378 * If VREG is volatile, emit a store from its value to its address.
1379 */
1380 static void
1381 emit_volatile_store (EmitContext *ctx, int vreg)
1382 {
1383 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1384
1385 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1386 g_assert (ctx->addresses [vreg]);
1387 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1388 }
1389 }
1390
1391 static LLVMTypeRef
1392 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1393 {
1394 LLVMTypeRef ret_type;
1395 LLVMTypeRef *param_types = NULL;
1396 LLVMTypeRef res;
1397 int i, pindex;
1398 MonoType *rtype;
1399
1400 ret_type = type_to_llvm_type (ctx, sig->ret);
1401 if (!ctx_ok (ctx))
1402 return NULL;
1403 rtype = mini_get_underlying_type (sig->ret);
1404
1405 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1406 pindex = 0;
1407
1408 if (sig->hasthis)
1409 param_types [pindex ++] = ThisType ();
1410 for (i = 0; i < sig->param_count; ++i)
1411 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1412
1413 if (!ctx_ok (ctx)) {
1414 g_free (param_types);
1415 return NULL;
1416 }
1417
1418 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1419 g_free (param_types);
1420
1421 return res;
1422 }
1423
1424 /*
1425 * sig_to_llvm_sig_full:
1426 *
1427 * Return the LLVM signature corresponding to the mono signature SIG using the
1428 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1429 */
1430 static LLVMTypeRef
1431 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1432 {
1433 LLVMTypeRef ret_type;
1434 LLVMTypeRef *param_types = NULL;
1435 LLVMTypeRef res;
1436 int i, j, pindex, vret_arg_pindex = 0;
1437 gboolean vretaddr = FALSE;
1438 MonoType *rtype;
1439
1440 if (!cinfo)
1441 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1442
1443 ret_type = type_to_llvm_type (ctx, sig->ret);
1444 if (!ctx_ok (ctx))
1445 return NULL;
1446 rtype = mini_get_underlying_type (sig->ret);
1447
1448 switch (cinfo->ret.storage) {
1449 case LLVMArgVtypeInReg:
1450 /* LLVM models this by returning an aggregate value */
1451 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1452 LLVMTypeRef members [2];
1453
1454 members [0] = IntPtrType ();
1455 ret_type = LLVMStructType (members, 1, FALSE);
1456 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1457 /* Empty struct */
1458 ret_type = LLVMVoidType ();
1459 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1460 LLVMTypeRef members [2];
1461
1462 members [0] = IntPtrType ();
1463 members [1] = IntPtrType ();
1464 ret_type = LLVMStructType (members, 2, FALSE);
1465 } else {
1466 g_assert_not_reached ();
1467 }
1468 break;
1469 case LLVMArgVtypeByVal:
1470 /* Vtype returned normally by val */
1471 break;
1472 case LLVMArgVtypeAsScalar: {
1473 int size = mono_class_value_size (mono_class_from_mono_type_internal (rtype), NULL);
1474 /* LLVM models this by returning an int */
1475 if (size < TARGET_SIZEOF_VOID_P) {
1476 g_assert (cinfo->ret.nslots == 1);
1477 ret_type = LLVMIntType (size * 8);
1478 } else {
1479 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1480 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (target_mgreg_t) * 8);
1481 }
1482 break;
1483 }
1484 case LLVMArgAsIArgs:
1485 ret_type = LLVMArrayType (IntPtrType (), cinfo->ret.nslots);
1486 break;
1487 case LLVMArgFpStruct: {
1488 /* Vtype returned as a fp struct */
1489 LLVMTypeRef members [16];
1490
1491 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1492 for (i = 0; i < cinfo->ret.nslots; ++i)
1493 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1494 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1495 break;
1496 }
1497 case LLVMArgVtypeByRef:
1498 /* Vtype returned using a hidden argument */
1499 ret_type = LLVMVoidType ();
1500 break;
1501 case LLVMArgVtypeRetAddr:
1502 case LLVMArgGsharedvtFixed:
1503 case LLVMArgGsharedvtFixedVtype:
1504 case LLVMArgGsharedvtVariable:
1505 vretaddr = TRUE;
1506 ret_type = LLVMVoidType ();
1507 break;
1508 default:
1509 break;
1510 }
1511
1512 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1513 pindex = 0;
1514 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1515 /*
1516 * Has to be the first argument because of the sret argument attribute
1517 * FIXME: This might conflict with passing 'this' as the first argument, but
1518 * this is only used on arm64 which has a dedicated struct return register.
1519 */
1520 cinfo->vret_arg_pindex = pindex;
1521 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1522 if (!ctx_ok (ctx)) {
1523 g_free (param_types);
1524 return NULL;
1525 }
1526 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1527 pindex ++;
1528 }
1529 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1530 cinfo->rgctx_arg_pindex = pindex;
1531 param_types [pindex] = ctx->module->ptr_type;
1532 pindex ++;
1533 }
1534 if (cinfo->imt_arg) {
1535 cinfo->imt_arg_pindex = pindex;
1536 param_types [pindex] = ctx->module->ptr_type;
1537 pindex ++;
1538 }
1539 if (vretaddr) {
1540 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1541 vret_arg_pindex = pindex;
1542 if (cinfo->vret_arg_index == 1) {
1543 /* Add the slots consumed by the first argument */
1544 LLVMArgInfo *ainfo = &cinfo->args [0];
1545 switch (ainfo->storage) {
1546 case LLVMArgVtypeInReg:
1547 for (j = 0; j < 2; ++j) {
1548 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1549 vret_arg_pindex ++;
1550 }
1551 break;
1552 default:
1553 vret_arg_pindex ++;
1554 }
1555 }
1556
1557 cinfo->vret_arg_pindex = vret_arg_pindex;
1558 }
1559
1560 if (vretaddr && vret_arg_pindex == pindex)
1561 param_types [pindex ++] = IntPtrType ();
1562 if (sig->hasthis) {
1563 cinfo->this_arg_pindex = pindex;
1564 param_types [pindex ++] = ThisType ();
1565 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1566 }
1567 if (vretaddr && vret_arg_pindex == pindex)
1568 param_types [pindex ++] = IntPtrType ();
1569 for (i = 0; i < sig->param_count; ++i) {
1570 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1571
1572 if (vretaddr && vret_arg_pindex == pindex)
1573 param_types [pindex ++] = IntPtrType ();
1574 ainfo->pindex = pindex;
1575
1576 switch (ainfo->storage) {
1577 case LLVMArgVtypeInReg:
1578 for (j = 0; j < 2; ++j) {
1579 switch (ainfo->pair_storage [j]) {
1580 case LLVMArgInIReg:
1581 param_types [pindex ++] = LLVMIntType (TARGET_SIZEOF_VOID_P * 8);
1582 break;
1583 case LLVMArgNone:
1584 break;
1585 default:
1586 g_assert_not_reached ();
1587 }
1588 }
1589 break;
1590 case LLVMArgVtypeByVal:
1591 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1592 if (!ctx_ok (ctx))
1593 break;
1594 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1595 pindex ++;
1596 break;
1597 case LLVMArgAsIArgs:
1598 if (ainfo->esize == 8)
1599 param_types [pindex] = LLVMArrayType (LLVMInt64Type (), ainfo->nslots);
1600 else
1601 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1602 pindex ++;
1603 break;
1604 case LLVMArgVtypeAddr:
1605 case LLVMArgVtypeByRef:
1606 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1607 if (!ctx_ok (ctx))
1608 break;
1609 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1610 pindex ++;
1611 break;
1612 case LLVMArgAsFpArgs: {
1613 int j;
1614
1615 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1616 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1617 param_types [pindex ++] = LLVMDoubleType ();
1618 for (j = 0; j < ainfo->nslots; ++j)
1619 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1620 break;
1621 }
1622 case LLVMArgVtypeAsScalar:
1623 g_assert_not_reached ();
1624 break;
1625 case LLVMArgGsharedvtFixed:
1626 case LLVMArgGsharedvtFixedVtype:
1627 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1628 break;
1629 case LLVMArgGsharedvtVariable:
1630 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1631 break;
1632 default:
1633 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1634 break;
1635 }
1636 }
1637 if (!ctx_ok (ctx)) {
1638 g_free (param_types);
1639 return NULL;
1640 }
1641 if (vretaddr && vret_arg_pindex == pindex)
1642 param_types [pindex ++] = IntPtrType ();
1643 if (ctx->llvm_only && cinfo->rgctx_arg) {
1644 /* Pass the rgctx as the last argument */
1645 cinfo->rgctx_arg_pindex = pindex;
1646 param_types [pindex] = ctx->module->ptr_type;
1647 pindex ++;
1648 } else if (ctx->llvm_only && cinfo->dummy_arg) {
1649 /* Pass a dummy arg last */
1650 cinfo->dummy_arg_pindex = pindex;
1651 param_types [pindex] = ctx->module->ptr_type;
1652 pindex ++;
1653 }
1654
1655 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1656 g_free (param_types);
1657
1658 return res;
1659 }
1660
1661 static LLVMTypeRef
1662 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1663 {
1664 return sig_to_llvm_sig_full (ctx, sig, NULL);
1665 }
1666
1667 /*
1668 * LLVMFunctionType1:
1669 *
1670 * Create an LLVM function type from the arguments.
1671 */
1672 static G_GNUC_UNUSED LLVMTypeRef
1673 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1674 int IsVarArg)
1675 {
1676 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1677 }
1678
1679 /*
1680 * LLVMFunctionType1:
1681 *
1682 * Create an LLVM function type from the arguments.
1683 */
1684 static G_GNUC_UNUSED LLVMTypeRef
1685 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1686 LLVMTypeRef ParamType1,
1687 int IsVarArg)
1688 {
1689 LLVMTypeRef param_types [1];
1690
1691 param_types [0] = ParamType1;
1692
1693 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1694 }
1695
1696 /*
1697 * LLVMFunctionType2:
1698 *
1699 * Create an LLVM function type from the arguments.
1700 */
1701 static G_GNUC_UNUSED LLVMTypeRef
1702 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1703 LLVMTypeRef ParamType1,
1704 LLVMTypeRef ParamType2,
1705 int IsVarArg)
1706 {
1707 LLVMTypeRef param_types [2];
1708
1709 param_types [0] = ParamType1;
1710 param_types [1] = ParamType2;
1711
1712 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1713 }
1714
1715 /*
1716 * LLVMFunctionType3:
1717 *
1718 * Create an LLVM function type from the arguments.
1719 */
1720 static G_GNUC_UNUSED LLVMTypeRef
1721 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1722 LLVMTypeRef ParamType1,
1723 LLVMTypeRef ParamType2,
1724 LLVMTypeRef ParamType3,
1725 int IsVarArg)
1726 {
1727 LLVMTypeRef param_types [3];
1728
1729 param_types [0] = ParamType1;
1730 param_types [1] = ParamType2;
1731 param_types [2] = ParamType3;
1732
1733 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1734 }
1735
1736 static G_GNUC_UNUSED LLVMTypeRef
1737 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1738 LLVMTypeRef ParamType1,
1739 LLVMTypeRef ParamType2,
1740 LLVMTypeRef ParamType3,
1741 LLVMTypeRef ParamType4,
1742 LLVMTypeRef ParamType5,
1743 int IsVarArg)
1744 {
1745 LLVMTypeRef param_types [5];
1746
1747 param_types [0] = ParamType1;
1748 param_types [1] = ParamType2;
1749 param_types [2] = ParamType3;
1750 param_types [3] = ParamType4;
1751 param_types [4] = ParamType5;
1752
1753 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1754 }
1755
1756 /*
1757 * create_builder:
1758 *
1759 * Create an LLVM builder and remember it so it can be freed later.
1760 */
1761 static LLVMBuilderRef
1762 create_builder (EmitContext *ctx)
1763 {
1764 LLVMBuilderRef builder = LLVMCreateBuilder ();
1765
1766 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1767
1768 emit_default_dbg_loc (ctx, builder);
1769
1770 return builder;
1771 }
1772
1773 static char*
1774 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1775 {
1776 char *name;
1777
1778 switch (type) {
1779 case MONO_PATCH_INFO_JIT_ICALL_ID:
1780 name = g_strdup_printf ("jit_icall_%s", mono_find_jit_icall_info ((MonoJitICallId)(gsize)data)->name);
1781 break;
1782 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1783 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1784 name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
1785 break;
1786 }
1787 default:
1788 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1789 break;
1790 }
1791
1792 return name;
1793 }
1794
1795 static int
1796 compute_aot_got_offset (MonoLLVMModule *module, MonoJumpInfo *ji, LLVMTypeRef llvm_type)
1797 {
1798 guint32 got_offset = mono_aot_get_got_offset (ji);
1799
1800 LLVMTypeRef lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (got_offset));
1801
1802 if (!lookup_type) {
1803 lookup_type = llvm_type;
1804 } else if (llvm_type != lookup_type) {
1805 lookup_type = module->ptr_type;
1806 } else {
1807 return got_offset;
1808 }
1809
1810 g_hash_table_insert (module->got_idx_to_type, GINT_TO_POINTER (got_offset), lookup_type);
1811 return got_offset;
1812 }
1813
1814 /* Allocate a GOT slot for TYPE/DATA, and emit IR to load it */
1815 static LLVMValueRef
1816 get_aotconst_typed_module (MonoLLVMModule *module, LLVMBuilderRef builder, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1817 {
1818 guint32 got_offset;
1819 LLVMValueRef indexes [2];
1820 LLVMValueRef got_entry_addr, load;
1821 char *name = NULL;
1822
1823 MonoJumpInfo tmp_ji;
1824 tmp_ji.type = type;
1825 tmp_ji.data.target = data;
1826
1827 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1828
1829 got_offset = compute_aot_got_offset (module, ji, llvm_type);
1830 module->max_got_offset = MAX (module->max_got_offset, got_offset);
1831
1832 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1833 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1834 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
1835
1836 name = get_aotconst_name (type, data, got_offset);
1837 if (llvm_type) {
1838 load = LLVMBuildLoad (builder, got_entry_addr, "");
1839 load = LLVMBuildBitCast (builder, load, llvm_type, name);
1840 } else {
1841 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1842 }
1843 g_free (name);
1844 //set_invariant_load_flag (load);
1845
1846 return load;
1847 }
1848
1849 static LLVMValueRef
1850 get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1851 {
1852 MonoCompile *cfg;
1853 guint32 got_offset;
1854 LLVMValueRef indexes [2];
1855 LLVMValueRef got_entry_addr, load;
1856 LLVMBuilderRef builder = ctx->builder;
1857 char *name = NULL;
1858
1859 cfg = ctx->cfg;
1860
1861 MonoJumpInfo tmp_ji;
1862 tmp_ji.type = type;
1863 tmp_ji.data.target = data;
1864
1865 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1866
1867 ji->next = cfg->patch_info;
1868 cfg->patch_info = ji;
1869
1870 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, llvm_type);
1871 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1872 /*
1873 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1874 * explicitly initialize it.
1875 */
1876 if (!mono_aot_is_shared_got_offset (got_offset)) {
1877 //mono_print_ji (ji);
1878 //printf ("\n");
1879 ctx->cfg->got_access_count ++;
1880 }
1881
1882 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1883 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1884 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1885
1886 name = get_aotconst_name (type, data, got_offset);
1887 if (llvm_type) {
1888 load = LLVMBuildLoad (builder, got_entry_addr, "");
1889 load = convert (ctx, load, llvm_type);
1890 LLVMSetValueName (load, name ? name : "");
1891 } else {
1892 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1893 }
1894 g_free (name);
1895 //set_invariant_load_flag (load);
1896
1897 return load;
1898 }
1899
1900 static LLVMValueRef
1901 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1902 {
1903 return get_aotconst_typed (ctx, type, data, NULL);
1904 }
1905
1906 typedef struct {
1907 MonoJumpInfo *ji;
1908 MonoMethod *method;
1909 LLVMValueRef load;
1910 LLVMTypeRef type;
1911 } CallSite;
1912
1913 static gboolean
1914 method_is_direct_callable (MonoMethod *method)
1915 {
1916 if (method->wrapper_type == MONO_WRAPPER_ALLOC)
1917 return TRUE;
1918 if (method->string_ctor)
1919 return FALSE;
1920 if (method->wrapper_type)
1921 return FALSE;
1922 if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL))
1923 return FALSE;
1924 /* Can't enable this as the callee might fail llvm compilation */
1925 /*
1926 if (!method->is_inflated && (mono_class_get_flags (method->klass) & TYPE_ATTRIBUTE_PUBLIC) && (method->flags & METHOD_ATTRIBUTE_PUBLIC))
1927 return TRUE;
1928 */
1929 return FALSE;
1930 }
1931
1932 static LLVMValueRef
1933 get_callee_llvmonly (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1934 {
1935 LLVMValueRef callee;
1936 char *callee_name = NULL;
1937
1938 if (ctx->module->static_link && ctx->module->assembly->image != mono_get_corlib ()) {
1939 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1940 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1941 g_assert (info);
1942 if (info->func != info->wrapper) {
1943 type = MONO_PATCH_INFO_METHOD;
1944 data = mono_icall_get_wrapper_method (info);
1945 callee_name = mono_aot_get_mangled_method_name ((MonoMethod*)data);
1946 }
1947 } else if (type == MONO_PATCH_INFO_METHOD) {
1948 MonoMethod *method = (MonoMethod*)data;
1949 if (m_class_get_image (method->klass) != ctx->module->assembly->image && method_is_direct_callable (method))
1950 callee_name = mono_aot_get_mangled_method_name (method);
1951 }
1952 }
1953
1954 if (!callee_name)
1955 callee_name = mono_aot_get_direct_call_symbol (type, data);
1956 if (callee_name) {
1957 /* Directly callable */
1958 // FIXME: Locking
1959 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1960 if (!callee) {
1961 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1962
1963 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1964
1965 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1966 } else {
1967 /* LLVMTypeRef's are uniqued */
1968 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
1969 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
1970
1971 g_free (callee_name);
1972 }
1973 return callee;
1974 }
1975
1976 /*
1977 * Change references to jit icalls to the icall wrappers when in corlib, so
1978 * they can be called directly.
1979 */
1980 if (ctx->module->assembly->image == mono_get_corlib () && type == MONO_PATCH_INFO_JIT_ICALL_ID) {
1981 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
1982
1983 if (info->func != info->wrapper) {
1984 type = MONO_PATCH_INFO_METHOD;
1985 data = mono_icall_get_wrapper_method (info);
1986 }
1987 }
1988
1989 /*
1990 * Instead of emitting an indirect call through a got slot, emit a placeholder, and
1991 * replace it with a direct call or an indirect call in mono_llvm_fixup_aot_module ()
1992 * after all methods have been emitted.
1993 */
1994 if (type == MONO_PATCH_INFO_METHOD) {
1995 MonoMethod *method = (MonoMethod*)data;
1996 if (m_class_get_image (method->klass)->assembly == ctx->module->assembly) {
1997 MonoJumpInfo tmp_ji;
1998 tmp_ji.type = type;
1999 tmp_ji.data.target = data;
2000
2001 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
2002 ji->next = ctx->cfg->patch_info;
2003 ctx->cfg->patch_info = ji;
2004 LLVMTypeRef llvm_type = LLVMPointerType (llvm_sig, 0);
2005
2006 ctx->cfg->got_access_count ++;
2007
2008 CallSite *info = g_new0 (CallSite, 1);
2009 info->method = method;
2010 info->ji = ji;
2011 info->type = llvm_type;
2012
2013 /*
2014 * Emit a dummy load to represent the callee, and either replace it with
2015 * a reference to the llvm method for the callee, or from a load from the
2016 * GOT.
2017 */
2018 LLVMValueRef indexes [2];
2019 LLVMValueRef got_entry_addr, load;
2020
2021 LLVMBuilderRef builder = ctx->builder;
2022 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2023 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2024 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
2025
2026 load = LLVMBuildLoad (builder, got_entry_addr, "");
2027 load = convert (ctx, load, llvm_type);
2028 info->load = load;
2029
2030 g_ptr_array_add (ctx->callsite_list, info);
2031
2032 return load;
2033 }
2034 }
2035
2036 /*
2037 * Calls are made through the GOT.
2038 */
2039 callee = get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
2040
2041 return callee;
2042 }
2043
2044 /*
2045 * get_callee:
2046 *
2047 * Return an llvm value representing the callee given by the arguments.
2048 */
2049 static LLVMValueRef
2050 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
2051 {
2052 LLVMValueRef callee;
2053 char *callee_name;
2054 MonoJumpInfo *ji = NULL;
2055
2056 if (ctx->llvm_only)
2057 return get_callee_llvmonly (ctx, llvm_sig, type, data);
2058
2059 callee_name = mono_aot_get_plt_symbol (type, data);
2060 if (!callee_name)
2061 return NULL;
2062
2063 if (ctx->cfg->compile_aot)
2064 /* Add a patch so referenced wrappers can be compiled in full aot mode */
2065 mono_add_patch_info (ctx->cfg, 0, type, data);
2066
2067 // FIXME: Locking
2068 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
2069 if (!callee) {
2070 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
2071
2072 LLVMSetVisibility (callee, LLVMHiddenVisibility);
2073
2074 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
2075 }
2076
2077 if (ctx->cfg->compile_aot) {
2078 ji = g_new0 (MonoJumpInfo, 1);
2079 ji->type = type;
2080 ji->data.target = data;
2081
2082 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
2083 }
2084
2085 return callee;
2086 }
2087
2088 static LLVMValueRef
2089 get_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
2090 {
2091 gpointer target;
2092
2093 // This won't be patched so compile the wrapper immediately
2094 if (type == MONO_PATCH_INFO_JIT_ICALL_ID) {
2095 MonoJitICallInfo * const info = mono_find_jit_icall_info ((MonoJitICallId)(gsize)data);
2096 target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
2097 } else {
2098 target = resolve_patch (ctx->cfg, type, data);
2099 }
2100
2101 LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
2102 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
2103 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
2104 LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
2105 return callee;
2106 }
2107
2108 static int
2109 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
2110 {
2111 MonoMethodHeader *header = cfg->header;
2112 MonoExceptionClause *clause;
2113 int i;
2114
2115 /* Directly */
2116 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
2117 return (bb->region >> 8) - 1;
2118
2119 /* Indirectly */
2120 for (i = 0; i < header->num_clauses; ++i) {
2121 clause = &header->clauses [i];
2122
2123 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
2124 return i;
2125 }
2126
2127 return -1;
2128 }
2129
2130 static MonoExceptionClause *
2131 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
2132 {
2133 if (bb == cfg->bb_init)
2134 return NULL;
2135 // Since they're sorted by nesting we just need
2136 // the first one that the bb is a member of
2137 for (int i = 0; i < cfg->header->num_clauses; i++) {
2138 MonoExceptionClause *curr = &cfg->header->clauses [i];
2139
2140 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
2141 return curr;
2142 }
2143
2144 return NULL;
2145 }
2146
2147 static void
2148 set_metadata_flag (LLVMValueRef v, const char *flag_name)
2149 {
2150 LLVMValueRef md_arg;
2151 int md_kind;
2152
2153 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2154 md_arg = LLVMMDString ("mono", 4);
2155 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2156 }
2157
2158 static void
2159 set_nonnull_load_flag (LLVMValueRef v)
2160 {
2161 LLVMValueRef md_arg;
2162 int md_kind;
2163 const char *flag_name;
2164
2165 flag_name = "nonnull";
2166 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2167 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2168 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2169 }
2170
2171 static void
2172 set_invariant_load_flag (LLVMValueRef v)
2173 {
2174 LLVMValueRef md_arg;
2175 int md_kind;
2176 const char *flag_name;
2177
2178 // FIXME: Cache this
2179 flag_name = "invariant.load";
2180 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
2181 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
2182 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
2183 }
2184
2185 /*
2186 * emit_call:
2187 *
2188 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
2189 * a try region.
2190 */
2191 static LLVMValueRef
2192 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
2193 {
2194 MonoCompile *cfg = ctx->cfg;
2195 LLVMValueRef lcall = NULL;
2196 LLVMBuilderRef builder = *builder_ref;
2197 MonoExceptionClause *clause;
2198
2199 if (ctx->llvm_only) {
2200 clause = get_most_deep_clause (cfg, ctx, bb);
2201
2202 if (clause) {
2203 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2204
2205 /*
2206 * Have to use an invoke instead of a call, branching to the
2207 * handler bblock of the clause containing this bblock.
2208 */
2209 intptr_t key = CLAUSE_END(clause);
2210
2211 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
2212
2213 // FIXME: Find the one that has the lowest end bound for the right start address
2214 // FIXME: Finally + nesting
2215
2216 if (lpad_bb) {
2217 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
2218
2219 /* Use an invoke */
2220 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
2221
2222 builder = ctx->builder = create_builder (ctx);
2223 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2224
2225 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2226 }
2227 }
2228 } else {
2229 int clause_index = get_handler_clause (cfg, bb);
2230
2231 if (clause_index != -1) {
2232 MonoMethodHeader *header = cfg->header;
2233 MonoExceptionClause *ec = &header->clauses [clause_index];
2234 MonoBasicBlock *tblock;
2235 LLVMBasicBlockRef ex_bb, noex_bb;
2236
2237 /*
2238 * Have to use an invoke instead of a call, branching to the
2239 * handler bblock of the clause containing this bblock.
2240 */
2241
2242 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY || ec->flags == MONO_EXCEPTION_CLAUSE_FAULT);
2243
2244 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
2245 g_assert (tblock);
2246
2247 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
2248
2249 ex_bb = get_bb (ctx, tblock);
2250
2251 noex_bb = gen_bb (ctx, "NOEX_BB");
2252
2253 /* Use an invoke */
2254 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
2255
2256 builder = ctx->builder = create_builder (ctx);
2257 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2258
2259 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2260 }
2261 }
2262
2263 if (!lcall) {
2264 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
2265 ctx->builder = builder;
2266 }
2267
2268 if (builder_ref)
2269 *builder_ref = ctx->builder;
2270
2271 return lcall;
2272 }
2273
2274 static LLVMValueRef
2275 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, LLVMValueRef base, const char *name, gboolean is_faulting, BarrierKind barrier)
2276 {
2277 LLVMValueRef res;
2278
2279 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
2280 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
2281 LLVMValueRef cmp;
2282
2283 cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
2284 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
2285 *builder_ref = ctx->builder;
2286 }
2287
2288 /*
2289 * We emit volatile loads for loads which can fault, because otherwise
2290 * LLVM will generate invalid code when encountering a load from a
2291 * NULL address.
2292 */
2293 if (barrier != LLVM_BARRIER_NONE)
2294 res = mono_llvm_build_atomic_load (*builder_ref, addr, name, is_faulting, size, barrier);
2295 else
2296 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting);
2297
2298 /* Mark it with a custom metadata */
2299 /*
2300 if (is_faulting)
2301 set_metadata_flag (res, "mono.faulting.load");
2302 */
2303
2304 return res;
2305 }
2306
2307 static void
2308 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, BarrierKind barrier)
2309 {
2310 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
2311 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
2312 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
2313 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
2314 *builder_ref = ctx->builder;
2315 }
2316
2317 if (barrier != LLVM_BARRIER_NONE)
2318 mono_llvm_build_aligned_store (*builder_ref, value, addr, barrier, size);
2319 else
2320 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
2321 }
2322
2323 static void
2324 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting)
2325 {
2326 emit_store_general (ctx, bb, builder_ref, size, value, addr, base, is_faulting, LLVM_BARRIER_NONE);
2327 }
2328
2329 /*
2330 * emit_cond_system_exception:
2331 *
2332 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2333 * Might set the ctx exception.
2334 */
2335 static void
2336 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
2337 {
2338 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2339 LLVMBuilderRef builder;
2340 MonoClass *exc_class;
2341 LLVMValueRef args [2];
2342 LLVMValueRef callee;
2343 gboolean no_pc = FALSE;
2344 static MonoClass *exc_classes [MONO_EXC_INTRINS_NUM];
2345
2346 if (IS_TARGET_AMD64)
2347 /* Some platforms don't require the pc argument */
2348 no_pc = TRUE;
2349
2350 int exc_id = mini_exception_id_by_name (exc_type);
2351 if (!exc_classes [exc_id])
2352 exc_classes [exc_id] = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
2353 exc_class = exc_classes [exc_id];
2354
2355 ex_bb = gen_bb (ctx, "EX_BB");
2356 if (ctx->llvm_only)
2357 ex2_bb = gen_bb (ctx, "EX2_BB");
2358 noex_bb = gen_bb (ctx, "NOEX_BB");
2359
2360 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2361
2362 /* Emit exception throwing code */
2363 ctx->builder = builder = create_builder (ctx);
2364 LLVMPositionBuilderAtEnd (builder, ex_bb);
2365
2366 if (ctx->cfg->llvm_only) {
2367 LLVMBuildBr (builder, ex2_bb);
2368
2369 ctx->builder = builder = create_builder (ctx);
2370 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2371
2372 if (exc_id == MONO_EXC_NULL_REF) {
2373 static LLVMTypeRef sig;
2374
2375 if (!sig)
2376 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
2377 /* Can't cache this */
2378 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
2379 emit_call (ctx, bb, &builder, callee, NULL, 0);
2380 } else {
2381 static LLVMTypeRef sig;
2382
2383 if (!sig)
2384 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2385 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_throw_corlib_exception));
2386 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2387 emit_call (ctx, bb, &builder, callee, args, 1);
2388 }
2389
2390 LLVMBuildUnreachable (builder);
2391
2392 ctx->builder = builder = create_builder (ctx);
2393 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2394
2395 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2396
2397 ctx->ex_index ++;
2398 return;
2399 }
2400
2401 callee = ctx->module->throw_corlib_exception;
2402 if (!callee) {
2403 LLVMTypeRef sig;
2404
2405 if (no_pc)
2406 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2407 else
2408 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2409
2410 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_throw_corlib_exception_abs_trampoline;
2411
2412 if (ctx->cfg->compile_aot) {
2413 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2414 } else {
2415 /*
2416 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2417 * - On x86, LLVM generated code doesn't push the arguments
2418 * - The trampoline takes the throw address as an arguments, not a pc offset.
2419 */
2420 callee = get_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
2421
2422 /*
2423 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2424 * added by get_jit_callee ().
2425 */
2426 ex2_bb = gen_bb (ctx, "EX2_BB");
2427 LLVMBuildBr (builder, ex2_bb);
2428 ex_bb = ex2_bb;
2429
2430 ctx->builder = builder = create_builder (ctx);
2431 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2432 }
2433 }
2434
2435 args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
2436
2437 /*
2438 * The LLVM mono branch contains changes so a block address can be passed as an
2439 * argument to a call.
2440 */
2441 if (no_pc) {
2442 emit_call (ctx, bb, &builder, callee, args, 1);
2443 } else {
2444 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2445 emit_call (ctx, bb, &builder, callee, args, 2);
2446 }
2447
2448 LLVMBuildUnreachable (builder);
2449
2450 ctx->builder = builder = create_builder (ctx);
2451 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2452
2453 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2454
2455 ctx->ex_index ++;
2456 return;
2457 }
2458
2459 /*
2460 * emit_args_to_vtype:
2461 *
2462 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2463 */
2464 static void
2465 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2466 {
2467 int j, size, nslots;
2468 MonoClass *klass;
2469
2470 t = mini_get_underlying_type (t);
2471 klass = mono_class_from_mono_type_internal (t);
2472 size = mono_class_value_size (klass, NULL);
2473
2474 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
2475 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2476
2477 if (ainfo->storage == LLVMArgAsFpArgs)
2478 nslots = ainfo->nslots;
2479 else
2480 nslots = 2;
2481
2482 for (j = 0; j < nslots; ++j) {
2483 LLVMValueRef index [2], addr, daddr;
2484 int part_size = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2485 LLVMTypeRef part_type;
2486
2487 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2488 part_size ++;
2489
2490 if (ainfo->pair_storage [j] == LLVMArgNone)
2491 continue;
2492
2493 switch (ainfo->pair_storage [j]) {
2494 case LLVMArgInIReg: {
2495 part_type = LLVMIntType (part_size * 8);
2496 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass)) {
2497 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2498 addr = LLVMBuildGEP (builder, address, index, 1, "");
2499 } else {
2500 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2501 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2502 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2503 }
2504 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2505 break;
2506 }
2507 case LLVMArgInFPReg: {
2508 LLVMTypeRef arg_type;
2509
2510 if (ainfo->esize == 8)
2511 arg_type = LLVMDoubleType ();
2512 else
2513 arg_type = LLVMFloatType ();
2514
2515 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2516 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2517 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2518 LLVMBuildStore (builder, args [j], addr);
2519 break;
2520 }
2521 case LLVMArgNone:
2522 break;
2523 default:
2524 g_assert_not_reached ();
2525 }
2526
2527 size -= TARGET_SIZEOF_VOID_P;
2528 }
2529 }
2530
2531 /*
2532 * emit_vtype_to_args:
2533 *
2534 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2535 * into ARGS, and the number of arguments into NARGS.
2536 */
2537 static void
2538 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2539 {
2540 int pindex = 0;
2541 int j, size, nslots;
2542 LLVMTypeRef arg_type;
2543
2544 t = mini_get_underlying_type (t);
2545 size = get_vtype_size (t);
2546
2547 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t)))
2548 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2549
2550 if (ainfo->storage == LLVMArgAsFpArgs)
2551 nslots = ainfo->nslots;
2552 else
2553 nslots = 2;
2554 for (j = 0; j < nslots; ++j) {
2555 LLVMValueRef index [2], addr, daddr;
2556 int partsize = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
2557
2558 if (ainfo->pair_storage [j] == LLVMArgNone)
2559 continue;
2560
2561 switch (ainfo->pair_storage [j]) {
2562 case LLVMArgInIReg:
2563 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
2564 index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
2565 addr = LLVMBuildGEP (builder, address, index, 1, "");
2566 } else {
2567 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2568 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2569 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2570 }
2571 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2572 break;
2573 case LLVMArgInFPReg:
2574 if (ainfo->esize == 8)
2575 arg_type = LLVMDoubleType ();
2576 else
2577 arg_type = LLVMFloatType ();
2578 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2579 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2580 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2581 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2582 break;
2583 case LLVMArgNone:
2584 break;
2585 default:
2586 g_assert_not_reached ();
2587 }
2588 size -= TARGET_SIZEOF_VOID_P;
2589 }
2590
2591 *nargs = pindex;
2592 }
2593
2594 static LLVMValueRef
2595 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2596 {
2597 /*
2598 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2599 * get executed every time control reaches them.
2600 */
2601 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2602
2603 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2604 return ctx->last_alloca;
2605 }
2606
2607 static LLVMValueRef
2608 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2609 {
2610 return build_alloca_llvm_type_name (ctx, t, align, "");
2611 }
2612
2613 static LLVMValueRef
2614 build_alloca (EmitContext *ctx, MonoType *t)
2615 {
2616 MonoClass *k = mono_class_from_mono_type_internal (t);
2617 int align;
2618
2619 g_assert (!mini_is_gsharedvt_variable_type (t));
2620
2621 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2622 align = mono_class_value_size (k, NULL);
2623 else
2624 align = mono_class_min_align (k);
2625
2626 /* Sometimes align is not a power of 2 */
2627 while (mono_is_power_of_two (align) == -1)
2628 align ++;
2629
2630 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2631 }
2632
2633 static LLVMValueRef
2634 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2635 {
2636 /*
2637 * gsharedvt local.
2638 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2639 */
2640 MonoCompile *cfg = ctx->cfg;
2641 LLVMBuilderRef builder = ctx->builder;
2642 LLVMValueRef offset, offset_var;
2643 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2644 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2645 LLVMValueRef ptr;
2646 char *name;
2647
2648 g_assert (info_var);
2649 g_assert (locals_var);
2650
2651 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2652
2653 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * TARGET_SIZEOF_VOID_P), FALSE);
2654 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2655
2656 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2657 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2658
2659 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2660 }
2661
2662 /*
2663 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2664 */
2665 static void
2666 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2667 {
2668 if (!module->used)
2669 module->used = g_ptr_array_sized_new (16);
2670 g_ptr_array_add (module->used, global);
2671 }
2672
2673 static void
2674 emit_llvm_used (MonoLLVMModule *module)
2675 {
2676 LLVMModuleRef lmodule = module->lmodule;
2677 LLVMTypeRef used_type;
2678 LLVMValueRef used, *used_elem;
2679 int i;
2680
2681 if (!module->used)
2682 return;
2683
2684 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2685 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2686 used_elem = g_new0 (LLVMValueRef, module->used->len);
2687 for (i = 0; i < module->used->len; ++i)
2688 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2689 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2690 LLVMSetLinkage (used, LLVMAppendingLinkage);
2691 LLVMSetSection (used, "llvm.metadata");
2692 }
2693
2694 /*
2695 * emit_get_method:
2696 *
2697 * Emit a function mapping method indexes to their code
2698 */
2699 static void
2700 emit_get_method (MonoLLVMModule *module)
2701 {
2702 LLVMModuleRef lmodule = module->lmodule;
2703 LLVMValueRef func, switch_ins, m;
2704 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2705 LLVMBasicBlockRef *bbs = NULL;
2706 LLVMTypeRef rtype;
2707 LLVMBuilderRef builder = LLVMCreateBuilder ();
2708 LLVMValueRef table = NULL;
2709 char *name;
2710 int i;
2711 gboolean emit_table = FALSE;
2712
2713 #ifdef TARGET_WASM
2714 /*
2715 * Emit a table of functions instead of a switch statement,
2716 * its very efficient on wasm. This might be usable on
2717 * other platforms too.
2718 */
2719 emit_table = TRUE;
2720 #endif
2721
2722 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2723
2724 if (emit_table) {
2725 LLVMTypeRef table_type;
2726 LLVMValueRef *table_elems;
2727 char *table_name;
2728
2729 int table_len = module->max_method_idx + 1;
2730 table_type = LLVMArrayType (rtype, table_len);
2731 table_name = g_strdup_printf ("%s_method_table", module->assembly->aname.name);
2732 table = LLVMAddGlobal (lmodule, table_type, table_name);
2733 table_elems = g_new0 (LLVMValueRef, table_len);
2734 for (i = 0; i < table_len; ++i) {
2735 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2736 if (m)
2737 table_elems [i] = LLVMBuildBitCast (builder, m, rtype, "");
2738 else
2739 table_elems [i] = LLVMConstNull (rtype);
2740 }
2741 LLVMSetInitializer (table, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), table_elems, table_len));
2742 }
2743
2744 /*
2745 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2746 * but generating code seems safer.
2747 */
2748 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2749 LLVMSetLinkage (func, LLVMExternalLinkage);
2750 LLVMSetVisibility (func, LLVMHiddenVisibility);
2751 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2752 module->get_method = func;
2753
2754 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2755
2756 /*
2757 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2758 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2759 * then we will have to find another solution.
2760 */
2761
2762 name = g_strdup_printf ("BB_CODE_START");
2763 code_start_bb = LLVMAppendBasicBlock (func, name);
2764 g_free (name);
2765 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2766 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2767
2768 name = g_strdup_printf ("BB_CODE_END");
2769 code_end_bb = LLVMAppendBasicBlock (func, name);
2770 g_free (name);
2771 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2772 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2773
2774 if (emit_table) {
2775 /*
2776 * switch (index) {
2777 * case -1: return code_start;
2778 * case -2: return code_end;
2779 * default: return method_table [index];
2780 */
2781 LLVMBasicBlockRef default_bb = LLVMAppendBasicBlock (func, "DEFAULT");
2782 LLVMPositionBuilderAtEnd (builder, default_bb);
2783 LLVMValueRef base = table;
2784 LLVMValueRef indexes [2];
2785 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2786 indexes [1] = LLVMGetParam (func, 0);
2787 LLVMValueRef addr = LLVMBuildGEP (builder, base, indexes, 2, "");
2788 LLVMValueRef res = mono_llvm_build_load (builder, addr, "", FALSE);
2789 LLVMBuildRet (builder, res);
2790
2791 LLVMPositionBuilderAtEnd (builder, entry_bb);
2792
2793 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), default_bb, 0);
2794 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2795 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2796 } else {
2797 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2798 for (i = 0; i < module->max_method_idx + 1; ++i) {
2799 name = g_strdup_printf ("BB_%d", i);
2800 bb = LLVMAppendBasicBlock (func, name);
2801 g_free (name);
2802 bbs [i] = bb;
2803
2804 LLVMPositionBuilderAtEnd (builder, bb);
2805
2806 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2807 if (m)
2808 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2809 else
2810 LLVMBuildRet (builder, LLVMConstNull (rtype));
2811 }
2812
2813 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2814 LLVMPositionBuilderAtEnd (builder, fail_bb);
2815 LLVMBuildRet (builder, LLVMConstNull (rtype));
2816
2817 LLVMPositionBuilderAtEnd (builder, entry_bb);
2818
2819 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2820 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2821 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2822 for (i = 0; i < module->max_method_idx + 1; ++i) {
2823 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2824 }
2825 }
2826
2827 mark_as_used (module, func);
2828
2829 LLVMDisposeBuilder (builder);
2830 }
2831
2832 /*
2833 * emit_get_unbox_tramp:
2834 *
2835 * Emit a function mapping method indexes to their unbox trampoline
2836 */
2837 static void
2838 emit_get_unbox_tramp (MonoLLVMModule *module)
2839 {
2840 LLVMModuleRef lmodule = module->lmodule;
2841 LLVMValueRef func, switch_ins, m;
2842 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2843 LLVMBasicBlockRef *bbs;
2844 LLVMTypeRef rtype;
2845 LLVMBuilderRef builder = LLVMCreateBuilder ();
2846 char *name;
2847 int i;
2848 gboolean emit_table = FALSE;
2849
2850 /* Similar to emit_get_method () */
2851
2852 #ifndef TARGET_WATCHOS
2853 emit_table = TRUE;
2854 #endif
2855
2856 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2857
2858 if (emit_table) {
2859 // About 10% of methods have an unbox tramp, so emit a table of indexes for them
2860 // that the runtime can search using a binary search
2861 int len = 0;
2862 for (i = 0; i < module->max_method_idx + 1; ++i) {
2863 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2864 if (m)
2865 len ++;
2866 }
2867
2868 LLVMTypeRef table_type, elemtype;
2869 LLVMValueRef *table_elems;
2870 LLVMValueRef table;
2871 char *table_name;
2872 int table_len;
2873 int elemsize;
2874
2875 table_len = len;
2876 elemsize = module->max_method_idx < 65000 ? 2 : 4;
2877
2878 // The index table
2879 elemtype = elemsize == 2 ? LLVMInt16Type () : LLVMInt32Type ();
2880 table_type = LLVMArrayType (elemtype, table_len);
2881 table_name = g_strdup_printf ("%s_unbox_tramp_indexes", module->assembly->aname.name);
2882 table = LLVMAddGlobal (lmodule, table_type, table_name);
2883 table_elems = g_new0 (LLVMValueRef, table_len);
2884 int idx = 0;
2885 for (i = 0; i < module->max_method_idx + 1; ++i) {
2886 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2887 if (m)
2888 table_elems [idx ++] = LLVMConstInt (elemtype, i, FALSE);
2889 }
2890 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2891 module->unbox_tramp_indexes = table;
2892
2893 // The trampoline table
2894 elemtype = rtype;
2895 table_type = LLVMArrayType (elemtype, table_len);
2896 table_name = g_strdup_printf ("%s_unbox_trampolines", module->assembly->aname.name);
2897 table = LLVMAddGlobal (lmodule, table_type, table_name);
2898 table_elems = g_new0 (LLVMValueRef, table_len);
2899 idx = 0;
2900 for (i = 0; i < module->max_method_idx + 1; ++i) {
2901 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2902 if (m)
2903 table_elems [idx ++] = LLVMBuildBitCast (builder, m, rtype, "");
2904 }
2905 LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
2906 module->unbox_trampolines = table;
2907
2908 module->unbox_tramp_num = table_len;
2909 module->unbox_tramp_elemsize = elemsize;
2910 return;
2911 }
2912
2913 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2914 LLVMSetLinkage (func, LLVMExternalLinkage);
2915 LLVMSetVisibility (func, LLVMHiddenVisibility);
2916 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2917 module->get_unbox_tramp = func;
2918
2919 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2920
2921 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2922 for (i = 0; i < module->max_method_idx + 1; ++i) {
2923 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2924 if (!m)
2925 continue;
2926
2927 name = g_strdup_printf ("BB_%d", i);
2928 bb = LLVMAppendBasicBlock (func, name);
2929 g_free (name);
2930 bbs [i] = bb;
2931
2932 LLVMPositionBuilderAtEnd (builder, bb);
2933
2934 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2935 }
2936
2937 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2938 LLVMPositionBuilderAtEnd (builder, fail_bb);
2939 LLVMBuildRet (builder, LLVMConstNull (rtype));
2940
2941 LLVMPositionBuilderAtEnd (builder, entry_bb);
2942
2943 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2944 for (i = 0; i < module->max_method_idx + 1; ++i) {
2945 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2946 if (!m)
2947 continue;
2948
2949 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2950 }
2951
2952 mark_as_used (module, func);
2953 LLVMDisposeBuilder (builder);
2954 }
2955
2956 /* Add a function to mark the beginning of LLVM code */
2957 static void
2958 emit_llvm_code_start (MonoLLVMModule *module)
2959 {
2960 LLVMModuleRef lmodule = module->lmodule;
2961 LLVMValueRef func;
2962 LLVMBasicBlockRef entry_bb;
2963 LLVMBuilderRef builder;
2964
2965 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2966 LLVMSetLinkage (func, LLVMInternalLinkage);
2967 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
2968 module->code_start = func;
2969 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2970 builder = LLVMCreateBuilder ();
2971 LLVMPositionBuilderAtEnd (builder, entry_bb);
2972 LLVMBuildRetVoid (builder);
2973 LLVMDisposeBuilder (builder);
2974 }
2975
2976 static LLVMValueRef
2977 emit_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
2978 {
2979 LLVMModuleRef lmodule = module->lmodule;
2980 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
2981 LLVMBasicBlockRef entry_bb;
2982 LLVMBuilderRef builder;
2983 LLVMTypeRef sig;
2984 MonoJumpInfo *ji;
2985 int got_offset;
2986 const char *wrapper_name = mono_marshal_get_aot_init_wrapper_name (subtype);
2987 char *name = g_strdup_printf ("%s%s", module->global_prefix, wrapper_name);
2988 MonoJitICallId icall_id = MONO_JIT_ICALL_ZeroIsReserved;
2989
2990 switch (subtype) {
2991 case AOT_INIT_METHOD:
2992 icall_id = MONO_JIT_ICALL_mini_llvm_init_method;
2993 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
2994 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
2995 break;
2996 case AOT_INIT_METHOD_GSHARED_MRGCTX:
2997 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_mrgctx; // Deliberate fall-through
2998 case AOT_INIT_METHOD_GSHARED_VTABLE:
2999 /* mrgctx/vtable */
3000 if (!icall_id)
3001 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_vtable;
3002 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
3003 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
3004 break;
3005 case AOT_INIT_METHOD_GSHARED_THIS:
3006 icall_id = MONO_JIT_ICALL_mini_llvm_init_gshared_method_this;
3007 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
3008 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
3009 break;
3010 default:
3011 g_assert_not_reached ();
3012 }
3013
3014 g_assert (icall_id);
3015 LLVMSetLinkage (func, LLVMInternalLinkage);
3016
3017 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3018
3019 set_cold_cconv (func);
3020
3021 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3022 builder = LLVMCreateBuilder ();
3023 LLVMPositionBuilderAtEnd (builder, entry_bb);
3024
3025 /* get_aotconst */
3026 ji = g_new0 (MonoJumpInfo, 1);
3027 ji->type = MONO_PATCH_INFO_AOT_MODULE;
3028 ji = mono_aot_patch_info_dup (ji);
3029 got_offset = compute_aot_got_offset (module, ji, IntPtrType ());
3030 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3031 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3032 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3033 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3034 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
3035 args [1] = LLVMGetParam (func, 0);
3036 if (subtype)
3037 args [2] = LLVMGetParam (func, 1);
3038
3039 ji = g_new0 (MonoJumpInfo, 1);
3040 ji->type = MONO_PATCH_INFO_JIT_ICALL_ID;
3041 ji->data.jit_icall_id = icall_id;
3042 ji = mono_aot_patch_info_dup (ji);
3043 got_offset = compute_aot_got_offset (module, ji, sig);
3044 module->max_got_offset = MAX (module->max_got_offset, got_offset);
3045 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3046 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
3047 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
3048 callee = LLVMBuildLoad (builder, got_entry_addr, "");
3049 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
3050 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
3051
3052 // Set the inited flag
3053 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3054 indexes [1] = LLVMGetParam (func, 0);
3055 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
3056
3057 LLVMBuildRetVoid (builder);
3058
3059 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3060 LLVMDisposeBuilder (builder);
3061 return func;
3062 }
3063
3064 /* Emit a wrapper around the parameterless JIT icall ICALL_ID with a cold calling convention */
3065 static LLVMValueRef
3066 emit_icall_cold_wrapper (MonoLLVMModule *module, LLVMModuleRef lmodule, MonoJitICallId icall_id, gboolean aot)
3067 {
3068 LLVMValueRef func, callee;
3069 LLVMBasicBlockRef entry_bb;
3070 LLVMBuilderRef builder;
3071 LLVMTypeRef sig;
3072 char *name;
3073
3074 name = g_strdup_printf ("%s_icall_cold_wrapper_%d", module->global_prefix, icall_id);
3075
3076 func = LLVMAddFunction (lmodule, name, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3077 sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3078 LLVMSetLinkage (func, LLVMInternalLinkage);
3079 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
3080 set_cold_cconv (func);
3081
3082 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3083 builder = LLVMCreateBuilder ();
3084 LLVMPositionBuilderAtEnd (builder, entry_bb);
3085
3086 if (aot) {
3087 callee = get_aotconst_typed_module (module, builder, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id), LLVMPointerType (sig, 0));
3088 } else {
3089 MonoJitICallInfo * const info = mono_find_jit_icall_info (icall_id);
3090 gpointer target = (gpointer)mono_icall_get_wrapper_full (info, TRUE);
3091
3092 LLVMValueRef tramp_var = LLVMAddGlobal (lmodule, LLVMPointerType (sig, 0), name);
3093 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (sig, 0)));
3094 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3095 callee = LLVMBuildLoad (builder, tramp_var, "");
3096 }
3097 LLVMBuildCall (builder, callee, NULL, 0, "");
3098
3099 LLVMBuildRetVoid (builder);
3100
3101 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3102 LLVMDisposeBuilder (builder);
3103 return func;
3104 }
3105
3106 /*
3107 * Emit wrappers around the C icalls used to initialize llvm methods, to
3108 * make the calling code smaller and to enable usage of the llvm
3109 * cold calling convention.
3110 */
3111 static void
3112 emit_init_icall_wrappers (MonoLLVMModule *module)
3113 {
3114 module->init_method = emit_init_icall_wrapper (module, AOT_INIT_METHOD);
3115 module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_MRGCTX);
3116 module->init_method_gshared_this = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_THIS);
3117 module->init_method_gshared_vtable = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_VTABLE);
3118 }
3119
3120 static LLVMValueRef
3121 get_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
3122 {
3123 switch (subtype) {
3124 case AOT_INIT_METHOD:
3125 return module->init_method;
3126 case AOT_INIT_METHOD_GSHARED_MRGCTX:
3127 return module->init_method_gshared_mrgctx;
3128 case AOT_INIT_METHOD_GSHARED_THIS:
3129 return module->init_method_gshared_this;
3130 case AOT_INIT_METHOD_GSHARED_VTABLE:
3131 return module->init_method_gshared_vtable;
3132 default:
3133 g_assert_not_reached ();
3134 }
3135 }
3136
3137 static void
3138 emit_gc_safepoint_poll (MonoLLVMModule *module)
3139 {
3140 LLVMModuleRef lmodule = module->lmodule;
3141 LLVMValueRef func, flag_addr, val_ptr, val, cmp, args [2], icall_wrapper;
3142 LLVMBasicBlockRef entry_bb, poll_bb, exit_bb;
3143 LLVMBuilderRef builder;
3144 LLVMTypeRef sig;
3145
3146 icall_wrapper = emit_icall_cold_wrapper (module, module->lmodule, MONO_JIT_ICALL_mono_threads_state_poll, TRUE);
3147 module->gc_poll_cold_wrapper = icall_wrapper;
3148
3149 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3150 func = mono_llvm_get_or_insert_gc_safepoint_poll (lmodule);
3151 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3152 LLVMSetLinkage (func, LLVMWeakODRLinkage);
3153
3154 entry_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.entry");
3155 poll_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.poll");
3156 exit_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.exit");
3157
3158 builder = LLVMCreateBuilder ();
3159
3160 /* entry: */
3161 LLVMPositionBuilderAtEnd (builder, entry_bb);
3162
3163 flag_addr = get_aotconst_typed_module (module, builder, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, NULL, LLVMPointerType (IntPtrType (), 0));
3164 val_ptr = LLVMBuildLoad (builder, flag_addr, "");
3165 val = LLVMBuildPtrToInt (builder, val_ptr, IntPtrType (), "");
3166 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
3167 args [0] = cmp;
3168 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
3169 cmp = LLVMBuildCall (builder, get_intrins_from_module (module->lmodule, INTRINS_EXPECT_I1), args, 2, "");
3170 LLVMBuildCondBr (builder, cmp, exit_bb, poll_bb);
3171
3172 /* poll: */
3173 LLVMPositionBuilderAtEnd(builder, poll_bb);
3174
3175 LLVMValueRef call = LLVMBuildCall (builder, icall_wrapper, NULL, 0, "");
3176 set_call_cold_cconv (call);
3177 LLVMBuildBr(builder, exit_bb);
3178
3179 /* exit: */
3180 LLVMPositionBuilderAtEnd(builder, exit_bb);
3181
3182 LLVMBuildRetVoid (builder);
3183
3184 LLVMVerifyFunction(func, LLVMAbortProcessAction);
3185 LLVMDisposeBuilder (builder);
3186 }
3187
3188 static void
3189 emit_llvm_code_end (MonoLLVMModule *module)
3190 {
3191 LLVMModuleRef lmodule = module->lmodule;
3192 LLVMValueRef func;
3193 LLVMBasicBlockRef entry_bb;
3194 LLVMBuilderRef builder;
3195
3196 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3197 LLVMSetLinkage (func, LLVMInternalLinkage);
3198 mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
3199 module->code_end = func;
3200 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
3201 builder = LLVMCreateBuilder ();
3202 LLVMPositionBuilderAtEnd (builder, entry_bb);
3203 LLVMBuildRetVoid (builder);
3204 LLVMDisposeBuilder (builder);
3205 }
3206
3207 static void
3208 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
3209 {
3210 gboolean need_div_check = ctx->cfg->backend->need_div_check;
3211
3212 if (bb->region)
3213 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
3214 need_div_check = TRUE;
3215
3216 if (!need_div_check)
3217 return;
3218
3219 switch (ins->opcode) {
3220 case OP_IDIV:
3221 case OP_LDIV:
3222 case OP_IREM:
3223 case OP_LREM:
3224 case OP_IDIV_UN:
3225 case OP_LDIV_UN:
3226 case OP_IREM_UN:
3227 case OP_LREM_UN:
3228 case OP_IDIV_IMM:
3229 case OP_LDIV_IMM:
3230 case OP_IREM_IMM:
3231 case OP_LREM_IMM:
3232 case OP_IDIV_UN_IMM:
3233 case OP_LDIV_UN_IMM:
3234 case OP_IREM_UN_IMM:
3235 case OP_LREM_UN_IMM: {
3236 LLVMValueRef cmp;
3237 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
3238 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
3239
3240 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
3241 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
3242 if (!ctx_ok (ctx))
3243 break;
3244 builder = ctx->builder;
3245
3246 /* b == -1 && a == 0x80000000 */
3247 if (is_signed) {
3248 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
3249 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
3250 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
3251
3252 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
3253 emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
3254 if (!ctx_ok (ctx))
3255 break;
3256 builder = ctx->builder;
3257 }
3258 break;
3259 }
3260 default:
3261 break;
3262 }
3263 }
3264
3265 /*
3266 * emit_init_method:
3267 *
3268 * Emit code to initialize the GOT slots used by the method.
3269 */
3270 static void
3271 emit_init_method (EmitContext *ctx)
3272 {
3273 LLVMValueRef indexes [16], args [16], callee;
3274 LLVMValueRef inited_var, cmp, call;
3275 LLVMBasicBlockRef inited_bb, notinited_bb;
3276 LLVMBuilderRef builder = ctx->builder;
3277 MonoCompile *cfg = ctx->cfg;
3278
3279 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
3280
3281 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3282 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
3283 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
3284
3285 //WASM doesn't support the "llvm.expect.i8" intrinsic
3286 #ifndef TARGET_WASM
3287 args [0] = inited_var;
3288 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
3289 inited_var = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I8), args, 2, "");
3290 #endif
3291
3292 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
3293
3294 inited_bb = ctx->inited_bb;
3295 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
3296
3297 ctx->cfg->llvmonly_init_cond = LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
3298
3299 builder = ctx->builder = create_builder (ctx);
3300 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
3301
3302 // FIXME: Cache
3303 if (ctx->rgctx_arg && ((cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) ||
3304 mini_method_is_default_method (cfg->method))) {
3305 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3306 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3307 callee = ctx->module->init_method_gshared_mrgctx;
3308 call = LLVMBuildCall (builder, callee, args, 2, "");
3309 } else if (ctx->rgctx_arg) {
3310 /* A vtable is passed as the rgctx argument */
3311 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3312 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
3313 callee = ctx->module->init_method_gshared_vtable;
3314 call = LLVMBuildCall (builder, callee, args, 2, "");
3315 } else if (cfg->gshared) {
3316 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3317 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
3318 callee = ctx->module->init_method_gshared_this;
3319 call = LLVMBuildCall (builder, callee, args, 2, "");
3320 } else {
3321 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
3322 callee = ctx->module->init_method;
3323 call = LLVMBuildCall (builder, callee, args, 1, "");
3324 }
3325
3326 /*
3327 * This enables llvm to keep arguments in their original registers/
3328 * scratch registers, since the call will not clobber them.
3329 */
3330 set_call_cold_cconv (call);
3331
3332 LLVMBuildBr (builder, inited_bb);
3333 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
3334
3335 builder = ctx->builder = create_builder (ctx);
3336 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
3337 }
3338
3339 static void
3340 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
3341 {
3342 /*
3343 * Emit unbox trampoline using a tailcall
3344 */
3345 LLVMValueRef tramp, call, *args;
3346 LLVMBuilderRef builder;
3347 LLVMBasicBlockRef lbb;
3348 LLVMCallInfo *linfo;
3349 char *tramp_name;
3350 int i, nargs;
3351
3352 tramp_name = g_strdup_printf ("ut_%s", method_name);
3353 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
3354 LLVMSetLinkage (tramp, LLVMInternalLinkage);
3355 mono_llvm_add_func_attr (tramp, LLVM_ATTR_OPTIMIZE_FOR_SIZE);
3356 //mono_llvm_add_func_attr (tramp, LLVM_ATTR_NO_UNWIND);
3357 linfo = ctx->linfo;
3358 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
3359 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3360 mono_llvm_add_param_attr (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVM_ATTR_IN_REG);
3361 if (ctx->cfg->vret_addr) {
3362 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
3363 if (linfo->ret.storage == LLVMArgVtypeByRef) {
3364 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
3365 mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
3366 }
3367 }
3368
3369 lbb = LLVMAppendBasicBlock (tramp, "");
3370 builder = LLVMCreateBuilder ();
3371 LLVMPositionBuilderAtEnd (builder, lbb);
3372
3373 nargs = LLVMCountParamTypes (method_type);
3374 args = g_new0 (LLVMValueRef, nargs);
3375 for (i = 0; i < nargs; ++i) {
3376 args [i] = LLVMGetParam (tramp, i);
3377 if (i == ctx->this_arg_pindex) {
3378 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
3379
3380 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
3381 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), MONO_ABI_SIZEOF (MonoObject), FALSE), "");
3382 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
3383 }
3384 }
3385 call = LLVMBuildCall (builder, method, args, nargs, "");
3386 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
3387 mono_llvm_add_instr_attr (call, 1 + ctx->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
3388 if (linfo->ret.storage == LLVMArgVtypeByRef)
3389 mono_llvm_add_instr_attr (call, 1 + linfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
3390
3391 // FIXME: This causes assertions in clang
3392 //mono_llvm_set_must_tailcall (call);
3393 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
3394 LLVMBuildRetVoid (builder);
3395 else
3396 LLVMBuildRet (builder, call);
3397
3398 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
3399 LLVMDisposeBuilder (builder);
3400 }
3401
3402 /*
3403 * emit_entry_bb:
3404 *
3405 * Emit code to load/convert arguments.
3406 */
3407 static void
3408 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
3409 {
3410 int i, j, pindex;
3411 MonoCompile *cfg = ctx->cfg;
3412 MonoMethodSignature *sig = ctx->sig;
3413 LLVMCallInfo *linfo = ctx->linfo;
3414 MonoBasicBlock *bb;
3415 char **names;
3416
3417 LLVMBuilderRef old_builder = ctx->builder;
3418 ctx->builder = builder;
3419
3420 ctx->alloca_builder = create_builder (ctx);
3421
3422 /*
3423 * Handle indirect/volatile variables by allocating memory for them
3424 * using 'alloca', and storing their address in a temporary.
3425 */
3426 for (i = 0; i < cfg->num_varinfo; ++i) {
3427 MonoInst *var = cfg->varinfo [i];
3428 LLVMTypeRef vtype;
3429
3430 if ((var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET))
3431 continue;
3432
3433 #ifdef TARGET_WASM
3434 // For GC stack scanning to work, have to spill all reference variables to the stack
3435 // Some ref variables have type intptr
3436 if (MONO_TYPE_IS_REFERENCE (var->inst_vtype) || var->inst_vtype->type == MONO_TYPE_I)
3437 var->flags |= MONO_INST_INDIRECT;
3438 #endif
3439
3440 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
3441 vtype = type_to_llvm_type (ctx, var->inst_vtype);
3442 if (!ctx_ok (ctx))
3443 return;
3444 /* Could be already created by an OP_VPHI */
3445 if (!ctx->addresses [var->dreg]) {
3446 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
3447 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
3448 }
3449 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
3450 }
3451 }
3452
3453 names = g_new (char *, sig->param_count);
3454 mono_method_get_param_names (cfg->method, (const char **) names);
3455
3456 for (i = 0; i < sig->param_count; ++i) {
3457 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
3458 int reg = cfg->args [i + sig->hasthis]->dreg;
3459 char *name;
3460
3461 pindex = ainfo->pindex;
3462
3463 switch (ainfo->storage) {
3464 case LLVMArgVtypeInReg:
3465 case LLVMArgAsFpArgs: {
3466 LLVMValueRef args [8];
3467 int j;
3468
3469 pindex += ainfo->ndummy_fpargs;
3470
3471 /* The argument is received as a set of int/fp arguments, store them into the real argument */
3472 memset (args, 0, sizeof (args));
3473 if (ainfo->storage == LLVMArgVtypeInReg) {
3474 args [0] = LLVMGetParam (ctx->lmethod, pindex);
3475 if (ainfo->pair_storage [1] != LLVMArgNone)
3476 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
3477 } else {
3478 g_assert (ainfo->nslots <= 8);
3479 for (j = 0; j < ainfo->nslots; ++j)
3480 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
3481 }
3482 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
3483
3484 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
3485
3486 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
3487 /* Treat these as normal values */
3488 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3489 }
3490 break;
3491 }
3492 case LLVMArgVtypeByVal: {
3493 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3494
3495 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
3496 /* Treat these as normal values */
3497 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
3498 }
3499 break;
3500 }
3501 case LLVMArgVtypeAddr:
3502 case LLVMArgVtypeByRef: {
3503 /* The argument is passed by ref */
3504 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3505 break;
3506 }
3507 case LLVMArgAsIArgs: {
3508 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3509 int size;
3510 MonoType *t = mini_get_underlying_type (ainfo->type);
3511
3512 /* The argument is received as an array of ints, store it into the real argument */
3513 ctx->addresses [reg] = build_alloca (ctx, t);
3514
3515 size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
3516 if (size == 0) {
3517 } else if (size < TARGET_SIZEOF_VOID_P) {
3518 /* The upper bits of the registers might not be valid */
3519 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
3520 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
3521 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
3522 } else {
3523 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
3524 }
3525 break;
3526 }
3527 case LLVMArgVtypeAsScalar:
3528 g_assert_not_reached ();
3529 break;
3530 case LLVMArgGsharedvtFixed: {
3531 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
3532 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3533
3534 if (names [i])
3535 name = g_strdup_printf ("arg_%s", names [i]);
3536 else
3537 name = g_strdup_printf ("arg_%d", i);
3538
3539 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
3540 break;
3541 }
3542 case LLVMArgGsharedvtFixedVtype: {
3543 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3544
3545 if (names [i])
3546 name = g_strdup_printf ("vtype_arg_%s", names [i]);
3547 else
3548 name = g_strdup_printf ("vtype_arg_%d", i);
3549
3550 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3551 g_assert (ctx->addresses [reg]);
3552 LLVMSetValueName (ctx->addresses [reg], name);
3553 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
3554 break;
3555 }
3556 case LLVMArgGsharedvtVariable:
3557 /* The IR treats these as variables with addresses */
3558 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3559 break;
3560 default:
3561 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, ainfo->type)), type_is_unsigned (ctx, ainfo->type));
3562 break;
3563 }
3564 }
3565 g_free (names);
3566
3567 if (cfg->vret_addr)
3568 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3569 if (sig->hasthis)
3570 emit_volatile_store (ctx, cfg->args [0]->dreg);
3571 for (i = 0; i < sig->param_count; ++i)
3572 if (!mini_type_is_vtype (sig->params [i]))
3573 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3574
3575 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
3576 LLVMValueRef this_alloc;
3577
3578 /*
3579 * The exception handling code needs the location where the this argument was
3580 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3581 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3582 * location into the LSDA.
3583 */
3584 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3585 /* This volatile store will keep the alloca alive */
3586 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3587
3588 set_metadata_flag (this_alloc, "mono.this");
3589 }
3590
3591 if (cfg->rgctx_var) {
3592 LLVMValueRef rgctx_alloc, store;
3593
3594 /*
3595 * We handle the rgctx arg similarly to the this pointer.
3596 */
3597 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3598 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3599 /* This volatile store will keep the alloca alive */
3600 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3601
3602 set_metadata_flag (rgctx_alloc, "mono.this");
3603 }
3604
3605 /* Initialize the method if needed */
3606 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
3607 /* Emit a location for the initialization code */
3608 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3609 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3610
3611 LLVMBuildBr (ctx->builder, ctx->init_bb);
3612 builder = ctx->builder = create_builder (ctx);
3613 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3614 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3615 }
3616
3617 /* Compute nesting between clauses */
3618 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3619 for (i = 0; i < cfg->header->num_clauses; ++i) {
3620 for (j = 0; j < cfg->header->num_clauses; ++j) {
3621 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3622 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3623
3624 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3625 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3626 }
3627 }
3628
3629 /*
3630 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3631 * it needs to continue normally, or return back to the exception handling system.
3632 */
3633 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3634 int clause_index;
3635 char name [128];
3636
3637 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3638 continue;
3639
3640 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3641 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
3642 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
3643
3644 if (bb->in_scount == 0) {
3645 LLVMValueRef val;
3646
3647 sprintf (name, "finally_ind_bb%d", bb->block_num);
3648 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3649 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3650
3651 ctx->bblocks [bb->block_num].finally_ind = val;
3652 } else {
3653 /* Create a variable to hold the exception var */
3654 if (!ctx->ex_var)
3655 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3656 }
3657
3658 /*
3659 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
3660 * LLVM bblock containing a landing pad causes problems for the
3661 * LLVM optimizer passes.
3662 */
3663 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
3664 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
3665 }
3666 ctx->builder = old_builder;
3667 }
3668
3669 static gboolean
3670 needs_extra_arg (EmitContext *ctx, MonoMethod *method)
3671 {
3672 WrapperInfo *info = NULL;
3673
3674 /*
3675 * When targeting wasm, the caller and callee signature has to match exactly. This means
3676 * that every method which can be called indirectly need an extra arg since the caller
3677 * will call it through an ftnptr and will pass an extra arg.
3678 */
3679 if (!ctx->cfg->llvm_only || !ctx->emit_dummy_arg)
3680 return FALSE;
3681 if (method->wrapper_type)
3682 info = mono_marshal_get_wrapper_info (method);
3683
3684 switch (method->wrapper_type) {
3685 case MONO_WRAPPER_OTHER:
3686 if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
3687 /* Already have an explicit extra arg */
3688 return FALSE;
3689 break;
3690 case MONO_WRAPPER_MANAGED_TO_NATIVE:
3691 if (strstr (method->name, "icall_wrapper"))
3692 /* These are JIT icall wrappers which are only called from JITted code directly */
3693 return FALSE;
3694 /* Normal icalls can be virtual methods which need an extra arg */
3695 break;
3696 case MONO_WRAPPER_RUNTIME_INVOKE:
3697 case MONO_WRAPPER_ALLOC:
3698 case MONO_WRAPPER_CASTCLASS:
3699 case MONO_WRAPPER_WRITE_BARRIER:
3700 return FALSE;
3701 case MONO_WRAPPER_STELEMREF:
3702 if (info->subtype != WRAPPER_SUBTYPE_VIRTUAL_STELEMREF)
3703 return FALSE;
3704 break;
3705 case MONO_WRAPPER_MANAGED_TO_MANAGED:
3706 if (info->subtype == WRAPPER_SUBTYPE_STRING_CTOR)
3707 return FALSE;
3708 break;
3709 default:
3710 break;
3711 }
3712 if (method->string_ctor)
3713 return FALSE;
3714
3715 /* These are called from gsharedvt code with an indirect call which doesn't pass an extra arg */
3716 if (method->klass == mono_get_string_class () && (strstr (method->name, "memcpy") || strstr (method->name, "bzero")))
3717 return FALSE;
3718 return TRUE;
3719 }
3720
3721 static inline gboolean
3722 is_supported_callconv (EmitContext *ctx, MonoCallInst *call)
3723 {
3724 #if defined(TARGET_WIN32) && defined(TARGET_AMD64)
3725 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) ||
3726 (call->signature->call_convention == MONO_CALL_C) ||
3727 (call->signature->call_convention == MONO_CALL_STDCALL);
3728 #else
3729 gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) || ((call->signature->call_convention == MONO_CALL_C) && ctx->llvm_only);
3730 #endif
3731 return result;
3732 }
3733
3734 static void
3735 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3736 {
3737 MonoCompile *cfg = ctx->cfg;
3738 LLVMValueRef *values = ctx->values;
3739 LLVMValueRef *addresses = ctx->addresses;
3740 MonoCallInst *call = (MonoCallInst*)ins;
3741 MonoMethodSignature *sig = call->signature;
3742 LLVMValueRef callee = NULL, lcall;
3743 LLVMValueRef *args;
3744 LLVMCallInfo *cinfo;
3745 GSList *l;
3746 int i, len, nargs;
3747 gboolean vretaddr;
3748 LLVMTypeRef llvm_sig;
3749 gpointer target;
3750 gboolean is_virtual, calli;
3751 LLVMBuilderRef builder = *builder_ref;
3752
3753 /* If both imt and rgctx arg are required, only pass the imt arg, the rgctx trampoline will pass the rgctx */
3754 if (call->imt_arg_reg)
3755 call->rgctx_arg_reg = 0;
3756
3757 if (!is_supported_callconv (ctx, call)) {
3758 set_failure (ctx, "non-default callconv");
3759 return;
3760 }
3761
3762 cinfo = call->cinfo;
3763 g_assert (cinfo);
3764 if (call->rgctx_arg_reg)
3765 cinfo->rgctx_arg = TRUE;
3766 if (call->imt_arg_reg)
3767 cinfo->imt_arg = TRUE;
3768 if (!call->rgctx_arg_reg && call->method && needs_extra_arg (ctx, call->method))
3769 cinfo->dummy_arg = TRUE;
3770
3771 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3772
3773 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3774 if (!ctx_ok (ctx))
3775 return;
3776
3777 int const opcode = ins->opcode;
3778
3779 is_virtual = opcode == OP_VOIDCALL_MEMBASE || opcode == OP_CALL_MEMBASE
3780 || opcode == OP_VCALL_MEMBASE || opcode == OP_LCALL_MEMBASE
3781 || opcode == OP_FCALL_MEMBASE || opcode == OP_RCALL_MEMBASE
3782 || opcode == OP_TAILCALL_MEMBASE;
3783 calli = !call->fptr_is_patch && (opcode == OP_VOIDCALL_REG || opcode == OP_CALL_REG
3784 || opcode == OP_VCALL_REG || opcode == OP_LCALL_REG || opcode == OP_FCALL_REG
3785 || opcode == OP_RCALL_REG || opcode == OP_TAILCALL_REG);
3786
3787 /* FIXME: Avoid creating duplicate methods */
3788
3789 if (ins->flags & MONO_INST_HAS_METHOD) {
3790 if (is_virtual) {
3791 callee = NULL;
3792 } else {
3793 if (cfg->compile_aot) {
3794 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3795 if (!callee) {
3796 set_failure (ctx, "can't encode patch");
3797 return;
3798 }
3799 } else {
3800 ERROR_DECL (error);
3801 static int tramp_index;
3802 char *name;
3803
3804 name = g_strdup_printf ("tramp_%d", tramp_index);
3805 tramp_index ++;
3806
3807 /*
3808 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3809 * Make all calls through a global. The address of the global will be saved in
3810 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3811 * compiled.
3812 */
3813 LLVMValueRef tramp_var = (LLVMValueRef)g_hash_table_lookup (ctx->jit_callees, call->method);
3814 if (!tramp_var) {
3815 target =
3816 mono_create_jit_trampoline (mono_domain_get (),
3817 call->method, error);
3818 if (!is_ok (error)) {
3819 set_failure (ctx, mono_error_get_message (error));
3820 mono_error_cleanup (error);
3821 return;
3822 }
3823
3824 tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
3825 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
3826 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3827 g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
3828 }
3829 callee = LLVMBuildLoad (builder, tramp_var, "");
3830 }
3831 }
3832
3833 if (!cfg->llvm_only && call->method && strstr (m_class_get_name (call->method->klass), "AsyncVoidMethodBuilder")) {
3834 /* LLVM miscompiles async methods */
3835 set_failure (ctx, "#13734");
3836 return;
3837 }
3838 } else if (calli) {
3839 } else {
3840 const MonoJitICallId jit_icall_id = call->jit_icall_id;
3841
3842 if (jit_icall_id) {
3843 if (cfg->compile_aot) {
3844 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3845 if (!callee) {
3846 set_failure (ctx, "can't encode patch");
3847 return;
3848 }
3849 } else {
3850 callee = get_jit_callee (ctx, "", llvm_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (jit_icall_id));
3851 }
3852 } else {
3853 if (cfg->compile_aot) {
3854 callee = NULL;
3855 if (cfg->abs_patches) {
3856 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3857 if (abs_ji) {
3858 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3859 if (!callee) {
3860 set_failure (ctx, "can't encode patch");
3861 return;
3862 }
3863 }
3864 }
3865 if (!callee) {
3866 set_failure (ctx, "aot");
3867 return;
3868 }
3869 } else {
3870 if (cfg->abs_patches) {
3871 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3872 if (abs_ji) {
3873 ERROR_DECL (error);
3874
3875 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, error);
3876 mono_error_assert_ok (error);
3877 callee = get_jit_callee (ctx, "", llvm_sig, abs_ji->type, abs_ji->data.target);
3878 } else {
3879 g_assert_not_reached ();
3880 }
3881 } else {
3882 g_assert_not_reached ();
3883 }
3884 }
3885 }
3886 }
3887
3888 if (is_virtual) {
3889 int size = TARGET_SIZEOF_VOID_P;
3890 LLVMValueRef index;
3891
3892 g_assert (ins->inst_offset % size == 0);
3893 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3894
3895 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3896 } else if (calli) {
3897 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3898 } else {
3899 if (ins->flags & MONO_INST_HAS_METHOD) {
3900 }
3901 }
3902
3903 /*
3904 * Collect and convert arguments
3905 */
3906 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg + call->cinfo->dummy_arg + 1;
3907 len = sizeof (LLVMValueRef) * nargs;
3908 args = g_newa (LLVMValueRef, nargs);
3909 memset (args, 0, len);
3910 l = call->out_ireg_args;
3911
3912 if (call->rgctx_arg_reg) {
3913 g_assert (values [call->rgctx_arg_reg]);
3914 g_assert (cinfo->rgctx_arg_pindex < nargs);
3915 /*
3916 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3917 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3918 * it using a volatile load.
3919 */
3920 #ifdef TARGET_ARM
3921 if (!ctx->imt_rgctx_loc)
3922 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3923 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3924 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3925 #else
3926 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3927 #endif
3928 }
3929 if (call->imt_arg_reg) {
3930 g_assert (!ctx->llvm_only);
3931 g_assert (values [call->imt_arg_reg]);
3932 g_assert (cinfo->imt_arg_pindex < nargs);
3933 #ifdef TARGET_ARM
3934 if (!ctx->imt_rgctx_loc)
3935 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
3936 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3937 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3938 #else
3939 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3940 #endif
3941 }
3942 switch (cinfo->ret.storage) {
3943 case LLVMArgGsharedvtVariable: {
3944 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
3945
3946 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
3947 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
3948 } else {
3949 g_assert (addresses [call->inst.dreg]);
3950 args [cinfo->vret_arg_pindex] = convert (ctx, addresses [call->inst.dreg], IntPtrType ());
3951 }
3952 break;
3953 }
3954 default:
3955 if (vretaddr) {
3956 if (!addresses [call->inst.dreg])
3957 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3958 g_assert (cinfo->vret_arg_pindex < nargs);
3959 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3960 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3961 else
3962 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
3963 }
3964 break;
3965 }
3966
3967 /*
3968 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
3969 * use the real callee for argument type conversion.
3970 */
3971 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
3972 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
3973 LLVMGetParamTypes (callee_type, param_types);
3974
3975 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
3976 guint32 regpair;
3977 int reg, pindex;
3978 LLVMArgInfo *ainfo = &call->cinfo->args [i];
3979
3980 pindex = ainfo->pindex;
3981
3982 regpair = (guint32)(gssize)(l->data);
3983 reg = regpair & 0xffffff;
3984 args [pindex] = values [reg];
3985 switch (ainfo->storage) {
3986 case LLVMArgVtypeInReg:
3987 case LLVMArgAsFpArgs: {
3988 guint32 nargs;
3989 int j;
3990
3991 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
3992 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
3993 pindex += ainfo->ndummy_fpargs;
3994
3995 g_assert (addresses [reg]);
3996 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
3997 pindex += nargs;
3998
3999 // FIXME: alignment
4000 // FIXME: Get rid of the VMOVE
4001 break;
4002 }
4003 case LLVMArgVtypeByVal:
4004 g_assert (addresses [reg]);
4005 args [pindex] = addresses [reg];
4006 break;
4007 case LLVMArgVtypeAddr :
4008 case LLVMArgVtypeByRef: {
4009 g_assert (addresses [reg]);
4010 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4011 break;
4012 }
4013 case LLVMArgAsIArgs:
4014 g_assert (addresses [reg]);
4015 if (ainfo->esize == 8)
4016 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (LLVMInt64Type (), ainfo->nslots), 0)), "");
4017 else
4018 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
4019 break;
4020 case LLVMArgVtypeAsScalar:
4021 g_assert_not_reached ();
4022 break;
4023 case LLVMArgGsharedvtFixed:
4024 case LLVMArgGsharedvtFixedVtype:
4025 g_assert (addresses [reg]);
4026 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
4027 break;
4028 case LLVMArgGsharedvtVariable:
4029 g_assert (addresses [reg]);
4030 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
4031 break;
4032 default:
4033 g_assert (args [pindex]);
4034 if (i == 0 && sig->hasthis)
4035 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
4036 else
4037 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
4038 break;
4039 }
4040 g_assert (pindex <= nargs);
4041
4042 l = l->next;
4043 }
4044
4045
4046 if (call->cinfo->dummy_arg) {
4047 g_assert (call->cinfo->dummy_arg_pindex < nargs);
4048 args [call->cinfo->dummy_arg_pindex] = LLVMConstNull (ctx->module->ptr_type);
4049 }
4050
4051 // FIXME: Align call sites
4052
4053 /*
4054 * Emit the call
4055 */
4056 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
4057
4058 mono_llvm_nonnull_state_update (ctx, lcall, call->method, args, LLVMCountParamTypes (llvm_sig));
4059
4060 // If we just allocated an object, it's not null.
4061 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC) {
4062 mono_llvm_set_call_nonnull_ret (lcall);
4063 }
4064
4065 if (ins->opcode != OP_TAILCALL && ins->opcode != OP_TAILCALL_MEMBASE && LLVMGetInstructionOpcode (lcall) == LLVMCall)
4066 mono_llvm_set_call_notailcall (lcall);
4067
4068 // As per the LLVM docs, a function has a noalias return value if and only if
4069 // it is an allocation function. This is an allocation function.
4070 if (call->method && call->method->wrapper_type == MONO_WRAPPER_ALLOC)
4071 mono_llvm_set_call_noalias_ret (lcall);
4072
4073 /*
4074 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
4075 */
4076 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
4077 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
4078 #endif
4079 /* The two can't be used together, so use only one LLVM calling conv to pass them */
4080 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
4081 if (!sig->pinvoke && !cfg->llvm_only)
4082 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
4083
4084 if (cinfo->ret.storage == LLVMArgVtypeByRef)
4085 mono_llvm_add_instr_attr (lcall, 1 + cinfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
4086 if (!ctx->llvm_only && call->rgctx_arg_reg)
4087 mono_llvm_add_instr_attr (lcall, 1 + cinfo->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
4088 if (call->imt_arg_reg)
4089 mono_llvm_add_instr_attr (lcall, 1 + cinfo->imt_arg_pindex, LLVM_ATTR_IN_REG);
4090
4091 /* Add byval attributes if needed */
4092 for (i = 0; i < sig->param_count; ++i) {
4093 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
4094
4095 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
4096 mono_llvm_add_instr_attr (lcall, 1 + ainfo->pindex, LLVM_ATTR_BY_VAL);
4097 }
4098
4099 /*
4100 * Convert the result
4101 */
4102 switch (cinfo->ret.storage) {
4103 case LLVMArgVtypeInReg: {
4104 LLVMValueRef regs [2];
4105
4106 if (LLVMTypeOf (lcall) == LLVMVoidType ())
4107 /* Empty struct */
4108 break;
4109
4110 if (!addresses [ins->dreg])
4111 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
4112
4113 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
4114 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
4115 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
4116 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
4117 break;
4118 }
4119 case LLVMArgVtypeByVal:
4120 if (!addresses [call->inst.dreg])
4121 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4122 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
4123 break;
4124 case LLVMArgAsIArgs:
4125 case LLVMArgFpStruct:
4126 if (!addresses [call->inst.dreg])
4127 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4128 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4129 break;
4130 case LLVMArgVtypeAsScalar:
4131 if (!addresses [call->inst.dreg])
4132 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
4133 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
4134 break;
4135 case LLVMArgVtypeRetAddr:
4136 case LLVMArgVtypeByRef:
4137 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret))) {
4138 /* Some opcodes like STOREX_MEMBASE access these by value */
4139 g_assert (addresses [call->inst.dreg]);
4140 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4141 }
4142 break;
4143 case LLVMArgGsharedvtVariable:
4144 break;
4145 case LLVMArgGsharedvtFixed:
4146 case LLVMArgGsharedvtFixedVtype:
4147 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
4148 break;
4149 default:
4150 if (sig->ret->type != MONO_TYPE_VOID)
4151 /* If the method returns an unsigned value, need to zext it */
4152 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
4153 break;
4154 }
4155
4156 *builder_ref = ctx->builder;
4157 }
4158
4159 static void
4160 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4161 {
4162 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception : MONO_JIT_ICALL_mono_llvm_throw_exception;
4163 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
4164
4165 LLVMTypeRef exc_type = type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_exception_class ()));
4166
4167 if (!callee) {
4168 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
4169
4170 g_assert (ctx->cfg->compile_aot);
4171 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (icall_id));
4172 }
4173
4174 LLVMValueRef args [2];
4175
4176 args [0] = convert (ctx, exc, exc_type);
4177 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
4178
4179 LLVMBuildUnreachable (ctx->builder);
4180
4181 ctx->builder = create_builder (ctx);
4182 }
4183
4184 static void
4185 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
4186 {
4187 MonoMethodSignature *throw_sig;
4188
4189 LLVMValueRef * const pcallee = rethrow ? &ctx->module->rethrow : &ctx->module->throw_icall;
4190 LLVMValueRef callee = *pcallee;
4191 char const * const icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4192 #ifndef TARGET_X86
4193 const
4194 #endif
4195 MonoJitICallId icall_id = rethrow ? MONO_JIT_ICALL_mono_arch_rethrow_exception : MONO_JIT_ICALL_mono_arch_throw_exception;
4196
4197 if (!callee) {
4198 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
4199 throw_sig->ret = m_class_get_byval_arg (mono_get_void_class ());
4200 throw_sig->params [0] = m_class_get_byval_arg (mono_get_object_class ());
4201 if (ctx->cfg->compile_aot) {
4202 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4203 } else {
4204 #ifdef TARGET_X86
4205 /*
4206 * LLVM doesn't push the exception argument, so we need a different
4207 * trampoline.
4208 */
4209 icall_id = rethrow ? MONO_JIT_ICALL_mono_llvm_rethrow_exception_trampoline : MONO_JIT_ICALL_mono_llvm_throw_exception_trampoline;
4210 #endif
4211 callee = get_jit_callee (ctx, icall_name, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4212 }
4213
4214 mono_memory_barrier ();
4215 }
4216 LLVMValueRef arg;
4217 arg = convert (ctx, exc, type_to_llvm_type (ctx, m_class_get_byval_arg (mono_get_object_class ())));
4218 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
4219 }
4220
4221 static void
4222 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
4223 {
4224 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_resume_exception;
4225 LLVMValueRef callee;
4226
4227 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
4228
4229 g_assert (ctx->cfg->compile_aot);
4230 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4231
4232 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
4233
4234 LLVMBuildUnreachable (ctx->builder);
4235
4236 ctx->builder = create_builder (ctx);
4237 }
4238
4239 static LLVMValueRef
4240 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4241 {
4242 const char *icall_name = "mono_llvm_clear_exception";
4243 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_clear_exception;
4244
4245 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
4246 LLVMValueRef callee = NULL;
4247
4248 if (!callee) {
4249 if (ctx->cfg->compile_aot) {
4250 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4251 } else {
4252 // FIXME: This is broken.
4253 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4254 }
4255 }
4256
4257 g_assert (builder && callee);
4258
4259 return LLVMBuildCall (builder, callee, NULL, 0, "");
4260 }
4261
4262 static LLVMValueRef
4263 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
4264 {
4265 const char *icall_name = "mono_llvm_load_exception";
4266 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_load_exception;
4267
4268 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
4269 LLVMValueRef callee = NULL;
4270
4271 if (!callee) {
4272 if (ctx->cfg->compile_aot) {
4273 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4274 } else {
4275 // FIXME: This is broken.
4276 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
4277 }
4278 }
4279
4280 g_assert (builder && callee);
4281
4282 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
4283 }
4284
4285
4286 static LLVMValueRef
4287 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
4288 {
4289 const char *icall_name = "mono_llvm_match_exception";
4290 const MonoJitICallId icall_id = MONO_JIT_ICALL_mono_llvm_match_exception;
4291
4292 ctx->builder = builder;
4293
4294 LLVMValueRef args[5];
4295 const int num_args = G_N_ELEMENTS (args);
4296
4297 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
4298 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
4299 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
4300 if (ctx->cfg->rgctx_var) {
4301 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
4302 g_assert (rgctx_alloc);
4303 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
4304 } else {
4305 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
4306 }
4307 if (ctx->this_arg)
4308 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
4309 else
4310 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
4311
4312 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
4313 LLVMValueRef callee;
4314 g_assert (ctx->cfg->compile_aot);
4315 ctx->builder = builder;
4316 // get_callee expects ctx->builder to be the emitting builder
4317 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (icall_id));
4318
4319 g_assert (builder && callee);
4320
4321 g_assert (ctx->ex_var);
4322
4323 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
4324 }
4325
4326 // FIXME: This won't work because the code-finding makes this
4327 // not a constant.
4328 /*#define MONO_PERSONALITY_DEBUG*/
4329
4330 #ifdef MONO_PERSONALITY_DEBUG
4331 static const gboolean use_mono_personality_debug = TRUE;
4332 static const char *default_personality_name = "mono_debug_personality";
4333 #else
4334 static const gboolean use_mono_personality_debug = FALSE;
4335 static const char *default_personality_name = "__gxx_personality_v0";
4336 #endif
4337
4338 static LLVMTypeRef
4339 default_cpp_lpad_exc_signature (void)
4340 {
4341 static gboolean inited = FALSE;
4342 static LLVMTypeRef sig;
4343
4344 if (!sig) {
4345 LLVMTypeRef signature [2];
4346 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
4347 signature [1] = LLVMInt32Type ();
4348 sig = LLVMStructType (signature, 2, FALSE);
4349 inited = TRUE;
4350 }
4351
4352 return sig;
4353 }
4354
4355 static LLVMValueRef
4356 get_mono_personality (EmitContext *ctx)
4357 {
4358 LLVMValueRef personality = NULL;
4359 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4360
4361 g_assert (ctx->cfg->compile_aot);
4362 if (!use_mono_personality_debug) {
4363 personality = get_intrins_by_name (ctx, default_personality_name);
4364 } else {
4365 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_debug_personality));
4366 }
4367
4368 g_assert (personality);
4369 return personality;
4370 }
4371
4372 static LLVMBasicBlockRef
4373 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
4374 {
4375 MonoCompile *cfg = ctx->cfg;
4376 LLVMBuilderRef old_builder = ctx->builder;
4377 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
4378
4379 LLVMBuilderRef lpadBuilder = create_builder (ctx);
4380 ctx->builder = lpadBuilder;
4381
4382 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
4383 g_assert (handler_bb);
4384
4385 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4386 LLVMValueRef personality = get_mono_personality (ctx);
4387 g_assert (personality);
4388
4389 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
4390 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
4391 g_free (bb_name);
4392 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4393 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
4394 g_assert (landing_pad);
4395
4396 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
4397 LLVMAddClause (landing_pad, cast);
4398
4399 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
4400 LLVMBuilderRef resume_builder = create_builder (ctx);
4401 ctx->builder = resume_builder;
4402 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
4403
4404 emit_resume_eh (ctx, handler_bb);
4405
4406 // Build match
4407 ctx->builder = lpadBuilder;
4408 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
4409
4410 gboolean finally_only = TRUE;
4411
4412 MonoExceptionClause *group_cursor = group_start;
4413
4414 for (int i = 0; i < group_size; i ++) {
4415 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY || group_cursor->flags & MONO_EXCEPTION_CLAUSE_FAULT))
4416 finally_only = FALSE;
4417
4418 group_cursor++;
4419 }
4420
4421 // FIXME:
4422 // Handle landing pad inlining
4423
4424 if (!finally_only) {
4425 // So at each level of the exception stack we will match the exception again.
4426 // During that match, we need to compare against the handler types for the current
4427 // protected region. We send the try start and end so that we can only check against
4428 // handlers for this lexical protected region.
4429 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
4430
4431 // if returns -1, resume
4432 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
4433
4434 // else move to that target bb
4435 for (int i = 0; i < group_size; i++) {
4436 MonoExceptionClause *clause = group_start + i;
4437 int clause_index = clause - cfg->header->clauses;
4438 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4439 g_assert (handler_bb);
4440 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4441 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4442 }
4443 } else {
4444 int clause_index = group_start - cfg->header->clauses;
4445 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
4446 g_assert (finally_bb);
4447
4448 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
4449 }
4450
4451 ctx->builder = old_builder;
4452
4453 return lpad_bb;
4454 }
4455
4456
4457 static void
4458 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
4459 {
4460 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
4461 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
4462
4463 // Make exception available to catch blocks
4464 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags & MONO_EXCEPTION_CLAUSE_FAULT)) {
4465 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
4466
4467 g_assert (ctx->ex_var);
4468 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
4469
4470 if (bb->in_scount == 1) {
4471 MonoInst *exvar = bb->in_stack [0];
4472 g_assert (!ctx->values [exvar->dreg]);
4473 g_assert (ctx->ex_var);
4474 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
4475 emit_volatile_store (ctx, exvar->dreg);
4476 }
4477
4478 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
4479 }
4480
4481 LLVMBuilderRef handler_builder = create_builder (ctx);
4482 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
4483 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
4484
4485 // Make the handler code end with a jump to cbb
4486 LLVMBuildBr (handler_builder, cbb);
4487 }
4488
4489 static void
4490 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
4491 {
4492 MonoCompile *cfg = ctx->cfg;
4493 LLVMValueRef *values = ctx->values;
4494 LLVMModuleRef lmodule = ctx->lmodule;
4495 BBInfo *bblocks = ctx->bblocks;
4496 LLVMTypeRef i8ptr;
4497 LLVMValueRef personality;
4498 LLVMValueRef landing_pad;
4499 LLVMBasicBlockRef target_bb;
4500 MonoInst *exvar;
4501 static int ti_generator;
4502 char ti_name [128];
4503 LLVMValueRef type_info;
4504 int clause_index;
4505 GSList *l;
4506
4507 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
4508
4509 if (cfg->compile_aot) {
4510 /* Use a dummy personality function */
4511 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
4512 g_assert (personality);
4513 } else {
4514 /* Can't cache this as each method is in its own llvm module */
4515 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
4516 personality = LLVMAddFunction (ctx->lmodule, "mono_personality", personality_type);
4517 mono_llvm_add_func_attr (personality, LLVM_ATTR_NO_UNWIND);
4518 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (personality, "ENTRY");
4519 LLVMBuilderRef builder2 = LLVMCreateBuilder ();
4520 LLVMPositionBuilderAtEnd (builder2, entry_bb);
4521 LLVMBuildRet (builder2, LLVMConstInt (LLVMInt32Type (), 0, FALSE));
4522 LLVMDisposeBuilder (builder2);
4523 }
4524
4525 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
4526
4527 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
4528
4529 /*
4530 * Create the type info
4531 */
4532 sprintf (ti_name, "type_info_%d", ti_generator);
4533 ti_generator ++;
4534
4535 if (cfg->compile_aot) {
4536 /* decode_eh_frame () in aot-runtime.c will decode this */
4537 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4538 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4539
4540 /*
4541 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4542 */
4543 LLVMSetLinkage (type_info, LLVMInternalLinkage);
4544 } else {
4545 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4546 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4547 }
4548
4549 {
4550 LLVMTypeRef members [2], ret_type;
4551
4552 members [0] = i8ptr;
4553 members [1] = LLVMInt32Type ();
4554 ret_type = LLVMStructType (members, 2, FALSE);
4555
4556 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
4557 LLVMAddClause (landing_pad, type_info);
4558
4559 /* Store the exception into the exvar */
4560 if (ctx->ex_var)
4561 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
4562 }
4563
4564 /*
4565 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4566 * code expects control to be transferred to this landing pad even in the
4567 * presence of nested clauses. The landing pad needs to branch to the landing
4568 * pads belonging to nested clauses based on the selector value returned by
4569 * the landing pad instruction, which is passed to the landing pad in a
4570 * register by the EH code.
4571 */
4572 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4573 g_assert (target_bb);
4574
4575 /*
4576 * Branch to the correct landing pad
4577 */
4578 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
4579 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
4580
4581 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
4582 int nesting_clause_index = GPOINTER_TO_INT (l->data);
4583 MonoBasicBlock *handler_bb;
4584
4585 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
4586 g_assert (handler_bb);
4587
4588 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4589 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4590 }
4591
4592 /* Start a new bblock which CALL_HANDLER can branch to */
4593 ctx->builder = builder = create_builder (ctx);
4594 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
4595
4596 ctx->bblocks [bb->block_num].end_bblock = target_bb;
4597
4598 /* Store the exception into the IL level exvar */
4599 if (bb->in_scount == 1) {
4600 g_assert (bb->in_scount == 1);
4601 exvar = bb->in_stack [0];
4602
4603 // FIXME: This is shared with filter clauses ?
4604 g_assert (!values [exvar->dreg]);
4605
4606 g_assert (ctx->ex_var);
4607 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
4608 emit_volatile_store (ctx, exvar->dreg);
4609 }
4610
4611 /* Make normal branches to the start of the clause branch to the new bblock */
4612 bblocks [bb->block_num].bblock = target_bb;
4613 }
4614
4615 //Wasm requires us to canonicalize NaNs.
4616 static LLVMValueRef
4617 get_double_const (MonoCompile *cfg, double val)
4618 {
4619 #ifdef TARGET_WASM
4620 if (mono_isnan (val))
4621 *(gint64 *)&val = 0x7FF8000000000000ll;
4622 #endif
4623 return LLVMConstReal (LLVMDoubleType (), val);
4624 }
4625
4626 static LLVMValueRef
4627 get_float_const (MonoCompile *cfg, float val)
4628 {
4629 #ifdef TARGET_WASM
4630 if (mono_isnan (val))
4631 *(int *)&val = 0x7FC00000;
4632 #endif
4633 if (cfg->r4fp)
4634 return LLVMConstReal (LLVMFloatType (), val);
4635 else
4636 return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val), LLVMDoubleType ());
4637 }
4638
4639 static void
4640 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4641 {
4642 MonoCompile *cfg = ctx->cfg;
4643 MonoMethodSignature *sig = ctx->sig;
4644 LLVMValueRef method = ctx->lmethod;
4645 LLVMValueRef *values = ctx->values;
4646 LLVMValueRef *addresses = ctx->addresses;
4647 LLVMCallInfo *linfo = ctx->linfo;
4648 BBInfo *bblocks = ctx->bblocks;
4649 MonoInst *ins;
4650 LLVMBasicBlockRef cbb;
4651 LLVMBuilderRef builder, starting_builder;
4652 gboolean has_terminator;
4653 LLVMValueRef v;
4654 LLVMValueRef lhs, rhs;
4655 int nins = 0;
4656
4657 cbb = get_end_bb (ctx, bb);
4658
4659 builder = create_builder (ctx);
4660 ctx->builder = builder;
4661 LLVMPositionBuilderAtEnd (builder, cbb);
4662
4663 if (!ctx_ok (ctx))
4664 return;
4665
4666 if (bb->flags & BB_EXCEPTION_HANDLER) {
4667 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4668 set_failure (ctx, "handler without invokes");
4669 return;
4670 }
4671
4672 if (ctx->llvm_only)
4673 emit_llvmonly_handler_start (ctx, bb, cbb);
4674 else
4675 emit_handler_start (ctx, bb, builder);
4676 if (!ctx_ok (ctx))
4677 return;
4678 builder = ctx->builder;
4679 }
4680
4681 /* Handle PHI nodes first */
4682 /* They should be grouped at the start of the bb */
4683 for (ins = bb->code; ins; ins = ins->next) {
4684 emit_dbg_loc (ctx, builder, ins->cil_code);
4685
4686 if (ins->opcode == OP_NOP)
4687 continue;
4688 if (!MONO_IS_PHI (ins))
4689 break;
4690
4691 int i;
4692 gboolean empty = TRUE;
4693
4694 /* Check that all input bblocks really branch to us */
4695 for (i = 0; i < bb->in_count; ++i) {
4696 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4697 ins->inst_phi_args [i + 1] = -1;
4698 else
4699 empty = FALSE;
4700 }
4701
4702 if (empty) {
4703 /* LLVM doesn't like phi instructions with zero operands */
4704 ctx->is_dead [ins->dreg] = TRUE;
4705 continue;
4706 }
4707
4708 /* Created earlier, insert it now */
4709 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4710
4711 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4712 int sreg1 = ins->inst_phi_args [i + 1];
4713 int count, j;
4714
4715 /*
4716 * Count the number of times the incoming bblock branches to us,
4717 * since llvm requires a separate entry for each.
4718 */
4719 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4720 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4721
4722 count = 0;
4723 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4724 if (switch_ins->inst_many_bb [j] == bb)
4725 count ++;
4726 }
4727 } else {
4728 count = 1;
4729 }
4730
4731 /* Remember for later */
4732 for (j = 0; j < count; ++j) {
4733 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4734 node->bb = bb;
4735 node->phi = ins;
4736 node->in_bb = bb->in_bb [i];
4737 node->sreg = sreg1;
4738 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4739 }
4740 }
4741 }
4742 // Add volatile stores for PHI nodes
4743 // These need to be emitted after the PHI nodes
4744 for (ins = bb->code; ins; ins = ins->next) {
4745 const char *spec = LLVM_INS_INFO (ins->opcode);
4746
4747 if (ins->opcode == OP_NOP)
4748 continue;
4749 if (!MONO_IS_PHI (ins))
4750 break;
4751
4752 if (spec [MONO_INST_DEST] != 'v')
4753 emit_volatile_store (ctx, ins->dreg);
4754 }
4755
4756 has_terminator = FALSE;
4757 starting_builder = builder;
4758 for (ins = bb->code; ins; ins = ins->next) {
4759 const char *spec = LLVM_INS_INFO (ins->opcode);
4760 char *dname = NULL;
4761 char dname_buf [128];
4762
4763 emit_dbg_loc (ctx, builder, ins->cil_code);
4764
4765 nins ++;
4766 if (nins > 1000) {
4767 /*
4768 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4769 * Start a new bblock.
4770 * Prevent the bblocks to be merged by doing a volatile load + cond branch
4771 * from localloc-ed memory.
4772 */
4773 if (!cfg->llvm_only)
4774 ;//set_failure (ctx, "basic block too long");
4775
4776 if (!ctx->long_bb_break_var) {
4777 ctx->long_bb_break_var = build_alloca_llvm_type_name (ctx, LLVMInt32Type (), 0, "long_bb_break");
4778 mono_llvm_build_store (ctx->alloca_builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4779 }
4780
4781 cbb = gen_bb (ctx, "CONT_LONG_BB");
4782 LLVMBasicBlockRef dummy_bb = gen_bb (ctx, "CONT_LONG_BB_DUMMY");
4783
4784 LLVMValueRef load = mono_llvm_build_load (builder, ctx->long_bb_break_var, "", TRUE);
4785 /*
4786 * The long_bb_break_var is initialized to 0 in the prolog, so this branch will always go to 'cbb'
4787 * but llvm doesn't know that, so the branch is not going to be eliminated.
4788 */
4789 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntEQ, load, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4790
4791 LLVMBuildCondBr (builder, cmp, cbb, dummy_bb);
4792
4793 /* Emit a dummy false bblock which does nothing but contains a volatile store so it cannot be eliminated */
4794 ctx->builder = builder = create_builder (ctx);
4795 LLVMPositionBuilderAtEnd (builder, dummy_bb);
4796 mono_llvm_build_store (builder, LLVMConstInt (LLVMInt32Type (), 1, FALSE), ctx->long_bb_break_var, TRUE, LLVM_BARRIER_NONE);
4797 LLVMBuildBr (builder, cbb);
4798
4799 ctx->builder = builder = create_builder (ctx);
4800 LLVMPositionBuilderAtEnd (builder, cbb);
4801 ctx->bblocks [bb->block_num].end_bblock = cbb;
4802 nins = 0;
4803
4804 emit_dbg_loc (ctx, builder, ins->cil_code);
4805 }
4806
4807 if (has_terminator)
4808 /* There could be instructions after a terminator, skip them */
4809 break;
4810
4811 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4812 sprintf (dname_buf, "t%d", ins->dreg);
4813 dname = dname_buf;
4814 }
4815
4816 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4817 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4818
4819 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4820 lhs = emit_volatile_load (ctx, ins->sreg1);
4821 } else {
4822 /* It is ok for SETRET to have an uninitialized argument */
4823 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4824 set_failure (ctx, "sreg1");
4825 return;
4826 }
4827 lhs = values [ins->sreg1];
4828 }
4829 } else {
4830 lhs = NULL;
4831 }
4832
4833 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4834 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4835 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4836 rhs = emit_volatile_load (ctx, ins->sreg2);
4837 } else {
4838 if (!values [ins->sreg2]) {
4839 set_failure (ctx, "sreg2");
4840 return;
4841 }
4842 rhs = values [ins->sreg2];
4843 }
4844 } else {
4845 rhs = NULL;
4846 }
4847
4848 //mono_print_ins (ins);
4849 gboolean skip_volatile_store = FALSE;
4850 switch (ins->opcode) {
4851 case OP_NOP:
4852 case OP_NOT_NULL:
4853 case OP_LIVERANGE_START:
4854 case OP_LIVERANGE_END:
4855 break;
4856 case OP_ICONST:
4857 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
4858 break;
4859 case OP_I8CONST:
4860 #if TARGET_SIZEOF_VOID_P == 4
4861 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4862 #else
4863 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
4864 #endif
4865 break;
4866 case OP_R8CONST:
4867 values [ins->dreg] = get_double_const (cfg, *(double*)ins->inst_p0);
4868 break;
4869 case OP_R4CONST:
4870 values [ins->dreg] = get_float_const (cfg, *(float*)ins->inst_p0);
4871 break;
4872 case OP_DUMMY_ICONST:
4873 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4874 break;
4875 case OP_DUMMY_I8CONST:
4876 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
4877 break;
4878 case OP_DUMMY_R8CONST:
4879 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
4880 break;
4881 case OP_BR: {
4882 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
4883 LLVMBuildBr (builder, target_bb);
4884 has_terminator = TRUE;
4885 break;
4886 }
4887 case OP_SWITCH: {
4888 int i;
4889 LLVMValueRef v;
4890 char bb_name [128];
4891 LLVMBasicBlockRef new_bb;
4892 LLVMBuilderRef new_builder;
4893
4894 // The default branch is already handled
4895 // FIXME: Handle it here
4896
4897 /* Start new bblock */
4898 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
4899 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
4900
4901 lhs = convert (ctx, lhs, LLVMInt32Type ());
4902 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
4903 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
4904 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
4905
4906 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
4907 }
4908
4909 new_builder = create_builder (ctx);
4910 LLVMPositionBuilderAtEnd (new_builder, new_bb);
4911 LLVMBuildUnreachable (new_builder);
4912
4913 has_terminator = TRUE;
4914 g_assert (!ins->next);
4915
4916 break;
4917 }
4918
4919 case OP_SETRET:
4920 switch (linfo->ret.storage) {
4921 case LLVMArgVtypeInReg: {
4922 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4923 LLVMValueRef val, addr, retval;
4924 int i;
4925
4926 retval = LLVMGetUndef (ret_type);
4927
4928 if (!addresses [ins->sreg1]) {
4929 /*
4930 * The return type is an LLVM vector type, have to convert between it and the
4931 * real return type which is a struct type.
4932 */
4933 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (sig->ret)));
4934 /* Convert to 2xi64 first */
4935 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
4936
4937 for (i = 0; i < 2; ++i) {
4938 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4939 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
4940 } else {
4941 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4942 }
4943 }
4944 } else {
4945 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
4946 for (i = 0; i < 2; ++i) {
4947 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4948 LLVMValueRef indexes [2], part_addr;
4949
4950 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4951 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
4952 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
4953
4954 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
4955 } else {
4956 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4957 }
4958 }
4959 }
4960 LLVMBuildRet (builder, retval);
4961 break;
4962 }
4963 case LLVMArgVtypeAsScalar: {
4964 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4965 LLVMValueRef retval;
4966
4967 g_assert (addresses [ins->sreg1]);
4968
4969 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
4970 LLVMBuildRet (builder, retval);
4971 break;
4972 }
4973 case LLVMArgVtypeByVal: {
4974 LLVMValueRef retval;
4975
4976 g_assert (addresses [ins->sreg1]);
4977 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
4978 LLVMBuildRet (builder, retval);
4979 break;
4980 }
4981 case LLVMArgVtypeByRef: {
4982 LLVMBuildRetVoid (builder);
4983 break;
4984 }
4985 case LLVMArgGsharedvtFixed: {
4986 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
4987 /* The return value is in lhs, need to store to the vret argument */
4988 /* sreg1 might not be set */
4989 if (lhs) {
4990 g_assert (cfg->vret_addr);
4991 g_assert (values [cfg->vret_addr->dreg]);
4992 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
4993 }
4994 LLVMBuildRetVoid (builder);
4995 break;
4996 }
4997 case LLVMArgGsharedvtFixedVtype: {
4998 /* Already set */
4999 LLVMBuildRetVoid (builder);
5000 break;
5001 }
5002 case LLVMArgGsharedvtVariable: {
5003 /* Already set */
5004 LLVMBuildRetVoid (builder);
5005 break;
5006 }
5007 case LLVMArgVtypeRetAddr: {
5008 LLVMBuildRetVoid (builder);
5009 break;
5010 }
5011 case LLVMArgAsIArgs:
5012 case LLVMArgFpStruct: {
5013 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
5014 LLVMValueRef retval;
5015
5016 g_assert (addresses [ins->sreg1]);
5017 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
5018 LLVMBuildRet (builder, retval);
5019 break;
5020 }
5021 case LLVMArgNone:
5022 case LLVMArgNormal: {
5023 if (!lhs || ctx->is_dead [ins->sreg1]) {
5024 /*
5025 * The method did not set its return value, probably because it
5026 * ends with a throw.
5027 */
5028 if (cfg->vret_addr)
5029 LLVMBuildRetVoid (builder);
5030 else
5031 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
5032 } else {
5033 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
5034 }
5035 has_terminator = TRUE;
5036 break;
5037 }
5038 default:
5039 g_assert_not_reached ();
5040 break;
5041 }
5042 break;
5043 case OP_ICOMPARE:
5044 case OP_FCOMPARE:
5045 case OP_RCOMPARE:
5046 case OP_LCOMPARE:
5047 case OP_COMPARE:
5048 case OP_ICOMPARE_IMM:
5049 case OP_LCOMPARE_IMM:
5050 case OP_COMPARE_IMM: {
5051 CompRelation rel;
5052 LLVMValueRef cmp, args [16];
5053 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
5054 gboolean unlikely = FALSE;
5055
5056 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5057 if (ins->next->inst_false_bb->out_of_line)
5058 likely = TRUE;
5059 else if (ins->next->inst_true_bb->out_of_line)
5060 unlikely = TRUE;
5061 }
5062
5063 if (ins->next->opcode == OP_NOP)
5064 break;
5065
5066 if (ins->next->opcode == OP_BR)
5067 /* The comparison result is not needed */
5068 continue;
5069
5070 rel = mono_opcode_to_cond (ins->next->opcode);
5071
5072 if (ins->opcode == OP_ICOMPARE_IMM) {
5073 lhs = convert (ctx, lhs, LLVMInt32Type ());
5074 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5075 }
5076 if (ins->opcode == OP_LCOMPARE_IMM) {
5077 lhs = convert (ctx, lhs, LLVMInt64Type ());
5078 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5079 }
5080 if (ins->opcode == OP_LCOMPARE) {
5081 lhs = convert (ctx, lhs, LLVMInt64Type ());
5082 rhs = convert (ctx, rhs, LLVMInt64Type ());
5083 }
5084 if (ins->opcode == OP_ICOMPARE) {
5085 lhs = convert (ctx, lhs, LLVMInt32Type ());
5086 rhs = convert (ctx, rhs, LLVMInt32Type ());
5087 }
5088
5089 if (lhs && rhs) {
5090 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5091 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
5092 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
5093 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
5094 }
5095
5096 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
5097 if (ins->opcode == OP_FCOMPARE) {
5098 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5099 } else if (ins->opcode == OP_RCOMPARE) {
5100 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5101 } else if (ins->opcode == OP_COMPARE_IMM) {
5102 LLVMIntPredicate llvm_pred = cond_to_llvm_cond [rel];
5103 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0) {
5104 // We are emitting a NULL check for a pointer
5105 gboolean nonnull = mono_llvm_is_nonnull (lhs);
5106
5107 if (nonnull && llvm_pred == LLVMIntEQ)
5108 cmp = LLVMConstInt (LLVMInt1Type (), FALSE, FALSE);
5109 else if (nonnull && llvm_pred == LLVMIntNE)
5110 cmp = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
5111 else
5112 cmp = LLVMBuildICmp (builder, llvm_pred, lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
5113
5114 } else {
5115 cmp = LLVMBuildICmp (builder, llvm_pred, convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
5116 }
5117 } else if (ins->opcode == OP_LCOMPARE_IMM) {
5118 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5119 }
5120 else if (ins->opcode == OP_COMPARE) {
5121 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
5122 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5123 else
5124 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
5125 } else
5126 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
5127
5128 if (likely || unlikely) {
5129 args [0] = cmp;
5130 args [1] = LLVMConstInt (LLVMInt1Type (), likely ? 1 : 0, FALSE);
5131 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
5132 }
5133
5134 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
5135 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
5136 /*
5137 * If the target bb contains PHI instructions, LLVM requires
5138 * two PHI entries for this bblock, while we only generate one.
5139 * So convert this to an unconditional bblock. (bxc #171).
5140 */
5141 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
5142 } else {
5143 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
5144 }
5145 has_terminator = TRUE;
5146 } else if (MONO_IS_SETCC (ins->next)) {
5147 sprintf (dname_buf, "t%d", ins->next->dreg);
5148 dname = dname_buf;
5149 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5150
5151 /* Add stores for volatile variables */
5152 emit_volatile_store (ctx, ins->next->dreg);
5153 } else if (MONO_IS_COND_EXC (ins->next)) {
5154 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp);
5155 if (!ctx_ok (ctx))
5156 break;
5157 builder = ctx->builder;
5158 } else {
5159 set_failure (ctx, "next");
5160 break;
5161 }
5162
5163 ins = ins->next;
5164 break;
5165 }
5166 case OP_FCEQ:
5167 case OP_FCNEQ:
5168 case OP_FCLT:
5169 case OP_FCLT_UN:
5170 case OP_FCGT:
5171 case OP_FCGT_UN:
5172 case OP_FCGE:
5173 case OP_FCLE: {
5174 CompRelation rel;
5175 LLVMValueRef cmp;
5176
5177 rel = mono_opcode_to_cond (ins->opcode);
5178
5179 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
5180 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5181 break;
5182 }
5183 case OP_RCEQ:
5184 case OP_RCNEQ:
5185 case OP_RCLT:
5186 case OP_RCLT_UN:
5187 case OP_RCGT:
5188 case OP_RCGT_UN: {
5189 CompRelation rel;
5190 LLVMValueRef cmp;
5191
5192 rel = mono_opcode_to_cond (ins->opcode);
5193
5194 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
5195 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
5196 break;
5197 }
5198 case OP_PHI:
5199 case OP_FPHI:
5200 case OP_VPHI:
5201 case OP_XPHI: {
5202 // Handled above
5203 skip_volatile_store = TRUE;
5204 break;
5205 }
5206 case OP_MOVE:
5207 case OP_LMOVE:
5208 case OP_XMOVE:
5209 case OP_SETFRET:
5210 g_assert (lhs);
5211 values [ins->dreg] = lhs;
5212 break;
5213 case OP_FMOVE:
5214 case OP_RMOVE: {
5215 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
5216
5217 g_assert (lhs);
5218 values [ins->dreg] = lhs;
5219
5220 if (var && m_class_get_byval_arg (var->klass)->type == MONO_TYPE_R4) {
5221 /*
5222 * This is added by the spilling pass in case of the JIT,
5223 * but we have to do it ourselves.
5224 */
5225 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
5226 }
5227 break;
5228 }
5229 case OP_MOVE_F_TO_I4: {
5230 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
5231 break;
5232 }
5233 case OP_MOVE_I4_TO_F: {
5234 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
5235 break;
5236 }
5237 case OP_MOVE_F_TO_I8: {
5238 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
5239 break;
5240 }
5241 case OP_MOVE_I8_TO_F: {
5242 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
5243 break;
5244 }
5245 case OP_IADD:
5246 case OP_ISUB:
5247 case OP_IAND:
5248 case OP_IMUL:
5249 case OP_IDIV:
5250 case OP_IDIV_UN:
5251 case OP_IREM:
5252 case OP_IREM_UN:
5253 case OP_IOR:
5254 case OP_IXOR:
5255 case OP_ISHL:
5256 case OP_ISHR:
5257 case OP_ISHR_UN:
5258 case OP_FADD:
5259 case OP_FSUB:
5260 case OP_FMUL:
5261 case OP_FDIV:
5262 case OP_LADD:
5263 case OP_LSUB:
5264 case OP_LMUL:
5265 case OP_LDIV:
5266 case OP_LDIV_UN:
5267 case OP_LREM:
5268 case OP_LREM_UN:
5269 case OP_LAND:
5270 case OP_LOR:
5271 case OP_LXOR:
5272 case OP_LSHL:
5273 case OP_LSHR:
5274 case OP_LSHR_UN:
5275 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5276 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5277
5278 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
5279 if (!ctx_ok (ctx))
5280 break;
5281 builder = ctx->builder;
5282
5283 switch (ins->opcode) {
5284 case OP_IADD:
5285 case OP_LADD:
5286 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
5287 break;
5288 case OP_ISUB:
5289 case OP_LSUB:
5290 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
5291 break;
5292 case OP_IMUL:
5293 case OP_LMUL:
5294 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
5295 break;
5296 case OP_IREM:
5297 case OP_LREM:
5298 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
5299 break;
5300 case OP_IREM_UN:
5301 case OP_LREM_UN:
5302 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
5303 break;
5304 case OP_IDIV:
5305 case OP_LDIV:
5306 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
5307 break;
5308 case OP_IDIV_UN:
5309 case OP_LDIV_UN:
5310 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
5311 break;
5312 case OP_FDIV:
5313 case OP_RDIV:
5314 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5315 break;
5316 case OP_IAND:
5317 case OP_LAND:
5318 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
5319 break;
5320 case OP_IOR:
5321 case OP_LOR:
5322 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
5323 break;
5324 case OP_IXOR:
5325 case OP_LXOR:
5326 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
5327 break;
5328 case OP_ISHL:
5329 case OP_LSHL:
5330 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
5331 break;
5332 case OP_ISHR:
5333 case OP_LSHR:
5334 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
5335 break;
5336 case OP_ISHR_UN:
5337 case OP_LSHR_UN:
5338 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
5339 break;
5340
5341 case OP_FADD:
5342 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5343 break;
5344 case OP_FSUB:
5345 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5346 break;
5347 case OP_FMUL:
5348 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5349 break;
5350
5351 default:
5352 g_assert_not_reached ();
5353 }
5354 break;
5355 case OP_RADD:
5356 case OP_RSUB:
5357 case OP_RMUL:
5358 case OP_RDIV: {
5359 lhs = convert (ctx, lhs, LLVMFloatType ());
5360 rhs = convert (ctx, rhs, LLVMFloatType ());
5361 switch (ins->opcode) {
5362 case OP_RADD:
5363 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
5364 break;
5365 case OP_RSUB:
5366 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
5367 break;
5368 case OP_RMUL:
5369 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
5370 break;
5371 case OP_RDIV:
5372 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
5373 break;
5374 default:
5375 g_assert_not_reached ();
5376 break;
5377 }
5378 break;
5379 }
5380 case OP_IADD_IMM:
5381 case OP_ISUB_IMM:
5382 case OP_IMUL_IMM:
5383 case OP_IREM_IMM:
5384 case OP_IREM_UN_IMM:
5385 case OP_IDIV_IMM:
5386 case OP_IDIV_UN_IMM:
5387 case OP_IAND_IMM:
5388 case OP_IOR_IMM:
5389 case OP_IXOR_IMM:
5390 case OP_ISHL_IMM:
5391 case OP_ISHR_IMM:
5392 case OP_ISHR_UN_IMM:
5393 case OP_LADD_IMM:
5394 case OP_LSUB_IMM:
5395 case OP_LMUL_IMM:
5396 case OP_LREM_IMM:
5397 case OP_LAND_IMM:
5398 case OP_LOR_IMM:
5399 case OP_LXOR_IMM:
5400 case OP_LSHL_IMM:
5401 case OP_LSHR_IMM:
5402 case OP_LSHR_UN_IMM:
5403 case OP_ADD_IMM:
5404 case OP_AND_IMM:
5405 case OP_MUL_IMM:
5406 case OP_SHL_IMM:
5407 case OP_SHR_IMM:
5408 case OP_SHR_UN_IMM: {
5409 LLVMValueRef imm;
5410
5411 if (spec [MONO_INST_SRC1] == 'l') {
5412 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
5413 } else {
5414 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5415 }
5416
5417 emit_div_check (ctx, builder, bb, ins, lhs, imm);
5418 if (!ctx_ok (ctx))
5419 break;
5420 builder = ctx->builder;
5421
5422 #if TARGET_SIZEOF_VOID_P == 4
5423 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
5424 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5425 #endif
5426
5427 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
5428 lhs = convert (ctx, lhs, IntPtrType ());
5429 imm = convert (ctx, imm, LLVMTypeOf (lhs));
5430 switch (ins->opcode) {
5431 case OP_IADD_IMM:
5432 case OP_LADD_IMM:
5433 case OP_ADD_IMM:
5434 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
5435 break;
5436 case OP_ISUB_IMM:
5437 case OP_LSUB_IMM:
5438 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
5439 break;
5440 case OP_IMUL_IMM:
5441 case OP_MUL_IMM:
5442 case OP_LMUL_IMM:
5443 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
5444 break;
5445 case OP_IDIV_IMM:
5446 case OP_LDIV_IMM:
5447 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
5448 break;
5449 case OP_IDIV_UN_IMM:
5450 case OP_LDIV_UN_IMM:
5451 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
5452 break;
5453 case OP_IREM_IMM:
5454 case OP_LREM_IMM:
5455 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
5456 break;
5457 case OP_IREM_UN_IMM:
5458 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
5459 break;
5460 case OP_IAND_IMM:
5461 case OP_LAND_IMM:
5462 case OP_AND_IMM:
5463 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
5464 break;
5465 case OP_IOR_IMM:
5466 case OP_LOR_IMM:
5467 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
5468 break;
5469 case OP_IXOR_IMM:
5470 case OP_LXOR_IMM:
5471 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
5472 break;
5473 case OP_ISHL_IMM:
5474 case OP_LSHL_IMM:
5475 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5476 break;
5477 case OP_SHL_IMM:
5478 if (TARGET_SIZEOF_VOID_P == 8) {
5479 /* The IL is not regular */
5480 lhs = convert (ctx, lhs, LLVMInt64Type ());
5481 imm = convert (ctx, imm, LLVMInt64Type ());
5482 }
5483 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
5484 break;
5485 case OP_ISHR_IMM:
5486 case OP_LSHR_IMM:
5487 case OP_SHR_IMM:
5488 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
5489 break;
5490 case OP_ISHR_UN_IMM:
5491 /* This is used to implement conv.u4, so the lhs could be an i8 */
5492 lhs = convert (ctx, lhs, LLVMInt32Type ());
5493 imm = convert (ctx, imm, LLVMInt32Type ());
5494 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5495 break;
5496 case OP_LSHR_UN_IMM:
5497 case OP_SHR_UN_IMM:
5498 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
5499 break;
5500 default:
5501 g_assert_not_reached ();
5502 }
5503 break;
5504 }
5505 case OP_INEG:
5506 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5507 break;
5508 case OP_LNEG:
5509 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5510 lhs = convert (ctx, lhs, LLVMInt64Type ());
5511 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
5512 break;
5513 case OP_FNEG:
5514 lhs = convert (ctx, lhs, LLVMDoubleType ());
5515 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
5516 break;
5517 case OP_RNEG:
5518 lhs = convert (ctx, lhs, LLVMFloatType ());
5519 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
5520 break;
5521 case OP_INOT: {
5522 guint32 v = 0xffffffff;
5523 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
5524 break;
5525 }
5526 case OP_LNOT: {
5527 if (LLVMTypeOf (lhs) != LLVMInt64Type ())
5528 lhs = convert (ctx, lhs, LLVMInt64Type ());
5529 guint64 v = 0xffffffffffffffffLL;
5530 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
5531 break;
5532 }
5533 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5534 case OP_X86_LEA: {
5535 LLVMValueRef v1, v2;
5536
5537 rhs = LLVMBuildSExt (builder, convert (ctx, rhs, LLVMInt32Type ()), LLVMInt64Type (), "");
5538
5539 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ((unsigned long long)1 << ins->backend.shift_amount), FALSE), "");
5540 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
5541 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
5542 break;
5543 }
5544 #endif
5545
5546 case OP_ICONV_TO_I1:
5547 case OP_ICONV_TO_I2:
5548 case OP_ICONV_TO_I4:
5549 case OP_ICONV_TO_U1:
5550 case OP_ICONV_TO_U2:
5551 case OP_ICONV_TO_U4:
5552 case OP_LCONV_TO_I1:
5553 case OP_LCONV_TO_I2:
5554 case OP_LCONV_TO_U1:
5555 case OP_LCONV_TO_U2:
5556 case OP_LCONV_TO_U4: {
5557 gboolean sign;
5558
5559 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
5560
5561 /* Have to do two casts since our vregs have type int */
5562 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
5563 if (sign)
5564 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
5565 else
5566 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
5567 break;
5568 }
5569 case OP_ICONV_TO_I8:
5570 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
5571 break;
5572 case OP_ICONV_TO_U8:
5573 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
5574 break;
5575 case OP_FCONV_TO_I4:
5576 case OP_RCONV_TO_I4:
5577 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
5578 break;
5579 case OP_FCONV_TO_I1:
5580 case OP_RCONV_TO_I1:
5581 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
5582 break;
5583 case OP_FCONV_TO_U1:
5584 case OP_RCONV_TO_U1:
5585 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
5586 break;
5587 case OP_FCONV_TO_I2:
5588 case OP_RCONV_TO_I2:
5589 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5590 break;
5591 case OP_FCONV_TO_U2:
5592 case OP_RCONV_TO_U2:
5593 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
5594 break;
5595 case OP_FCONV_TO_U4:
5596 case OP_RCONV_TO_U4:
5597 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
5598 break;
5599 case OP_FCONV_TO_U8:
5600 case OP_RCONV_TO_U8:
5601 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt64Type (), dname);
5602 break;
5603 case OP_FCONV_TO_I8:
5604 case OP_RCONV_TO_I8:
5605 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
5606 break;
5607 case OP_FCONV_TO_I:
5608 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
5609 break;
5610 case OP_ICONV_TO_R8:
5611 case OP_LCONV_TO_R8:
5612 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
5613 break;
5614 case OP_ICONV_TO_R_UN:
5615 case OP_LCONV_TO_R_UN:
5616 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
5617 break;
5618 #if TARGET_SIZEOF_VOID_P == 4
5619 case OP_LCONV_TO_U:
5620 #endif
5621 case OP_LCONV_TO_I4:
5622 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5623 break;
5624 case OP_ICONV_TO_R4:
5625 case OP_LCONV_TO_R4:
5626 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
5627 if (cfg->r4fp)
5628 values [ins->dreg] = v;
5629 else
5630 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5631 break;
5632 case OP_FCONV_TO_R4:
5633 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
5634 if (cfg->r4fp)
5635 values [ins->dreg] = v;
5636 else
5637 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5638 break;
5639 case OP_RCONV_TO_R8:
5640 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
5641 break;
5642 case OP_RCONV_TO_R4:
5643 values [ins->dreg] = lhs;
5644 break;
5645 case OP_SEXT_I4:
5646 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5647 break;
5648 case OP_ZEXT_I4:
5649 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5650 break;
5651 case OP_TRUNC_I4:
5652 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5653 break;
5654 case OP_LOCALLOC_IMM: {
5655 LLVMValueRef v;
5656
5657 guint32 size = ins->inst_imm;
5658 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5659
5660 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
5661
5662 if (ins->flags & MONO_INST_INIT)
5663 emit_memset (ctx, builder, v, ConstInt32 (size), MONO_ARCH_FRAME_ALIGNMENT);
5664
5665 values [ins->dreg] = v;
5666 break;
5667 }
5668 case OP_LOCALLOC: {
5669 LLVMValueRef v, size;
5670
5671 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
5672
5673 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
5674
5675 if (ins->flags & MONO_INST_INIT)
5676 emit_memset (ctx, builder, v, size, MONO_ARCH_FRAME_ALIGNMENT);
5677 values [ins->dreg] = v;
5678 break;
5679 }
5680
5681 case OP_LOADI1_MEMBASE:
5682 case OP_LOADU1_MEMBASE:
5683 case OP_LOADI2_MEMBASE:
5684 case OP_LOADU2_MEMBASE:
5685 case OP_LOADI4_MEMBASE:
5686 case OP_LOADU4_MEMBASE:
5687 case OP_LOADI8_MEMBASE:
5688 case OP_LOADR4_MEMBASE:
5689 case OP_LOADR8_MEMBASE:
5690 case OP_LOAD_MEMBASE:
5691 case OP_LOADI8_MEM:
5692 case OP_LOADU1_MEM:
5693 case OP_LOADU2_MEM:
5694 case OP_LOADI4_MEM:
5695 case OP_LOADU4_MEM:
5696 case OP_LOAD_MEM: {
5697 int size = 8;
5698 LLVMValueRef base, index, addr;
5699 LLVMTypeRef t;
5700 gboolean sext = FALSE, zext = FALSE;
5701 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5702
5703 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5704
5705 if (sext || zext)
5706 dname = (char*)"";
5707
5708 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5709 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5710 base = addr;
5711 } else {
5712 /* _MEMBASE */
5713 base = lhs;
5714
5715 if (ins->inst_offset == 0) {
5716 addr = base;
5717 } else if (ins->inst_offset % size != 0) {
5718 /* Unaligned load */
5719 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5720 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5721 } else {
5722 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5723 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5724 }
5725 }
5726
5727 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5728
5729 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, base, dname, is_volatile, LLVM_BARRIER_NONE);
5730
5731 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5732 /*
5733 * These will signal LLVM that these loads do not alias any stores, and
5734 * they can't fail, allowing them to be hoisted out of loops.
5735 */
5736 set_invariant_load_flag (values [ins->dreg]);
5737 }
5738
5739 if (sext)
5740 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5741 else if (zext)
5742 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5743 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5744 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5745 break;
5746 }
5747
5748 case OP_STOREI1_MEMBASE_REG:
5749 case OP_STOREI2_MEMBASE_REG:
5750 case OP_STOREI4_MEMBASE_REG:
5751 case OP_STOREI8_MEMBASE_REG:
5752 case OP_STORER4_MEMBASE_REG:
5753 case OP_STORER8_MEMBASE_REG:
5754 case OP_STORE_MEMBASE_REG: {
5755 int size = 8;
5756 LLVMValueRef index, addr, base;
5757 LLVMTypeRef t;
5758 gboolean sext = FALSE, zext = FALSE;
5759 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5760
5761 if (!values [ins->inst_destbasereg]) {
5762 set_failure (ctx, "inst_destbasereg");
5763 break;
5764 }
5765
5766 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5767
5768 base = values [ins->inst_destbasereg];
5769 if (ins->inst_offset % size != 0) {
5770 /* Unaligned store */
5771 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5772 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5773 } else {
5774 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5775 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5776 }
5777 if (is_volatile && LLVMGetInstructionOpcode (base) == LLVMAlloca && !(ins->flags & MONO_INST_VOLATILE))
5778 /* Storing to an alloca cannot fail */
5779 is_volatile = FALSE;
5780 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5781 break;
5782 }
5783
5784 case OP_STOREI1_MEMBASE_IMM:
5785 case OP_STOREI2_MEMBASE_IMM:
5786 case OP_STOREI4_MEMBASE_IMM:
5787 case OP_STOREI8_MEMBASE_IMM:
5788 case OP_STORE_MEMBASE_IMM: {
5789 int size = 8;
5790 LLVMValueRef index, addr, base;
5791 LLVMTypeRef t;
5792 gboolean sext = FALSE, zext = FALSE;
5793 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
5794
5795 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5796
5797 base = values [ins->inst_destbasereg];
5798 if (ins->inst_offset % size != 0) {
5799 /* Unaligned store */
5800 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5801 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5802 } else {
5803 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5804 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5805 }
5806 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5807 break;
5808 }
5809
5810 case OP_CHECK_THIS:
5811 emit_load (ctx, bb, &builder, TARGET_SIZEOF_VOID_P, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), lhs, "", TRUE, LLVM_BARRIER_NONE);
5812 break;
5813 case OP_OUTARG_VTRETADDR:
5814 break;
5815 case OP_VOIDCALL:
5816 case OP_CALL:
5817 case OP_LCALL:
5818 case OP_FCALL:
5819 case OP_RCALL:
5820 case OP_VCALL:
5821 case OP_VOIDCALL_MEMBASE:
5822 case OP_CALL_MEMBASE:
5823 case OP_LCALL_MEMBASE:
5824 case OP_FCALL_MEMBASE:
5825 case OP_RCALL_MEMBASE:
5826 case OP_VCALL_MEMBASE:
5827 case OP_VOIDCALL_REG:
5828 case OP_CALL_REG:
5829 case OP_LCALL_REG:
5830 case OP_FCALL_REG:
5831 case OP_RCALL_REG:
5832 case OP_VCALL_REG: {
5833 process_call (ctx, bb, &builder, ins);
5834 break;
5835 }
5836 case OP_AOTCONST: {
5837 guint32 got_offset;
5838 LLVMValueRef indexes [2];
5839 MonoJumpInfo *tmp_ji, *ji;
5840 LLVMValueRef got_entry_addr;
5841 char *name;
5842
5843 /*
5844 * FIXME: Can't allocate from the cfg mempool since that is freed if
5845 * the LLVM compile fails.
5846 */
5847 tmp_ji = g_new0 (MonoJumpInfo, 1);
5848 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
5849 tmp_ji->data.target = ins->inst_p0;
5850
5851 ji = mono_aot_patch_info_dup (tmp_ji);
5852 g_free (tmp_ji);
5853
5854 if (ji->type == MONO_PATCH_INFO_ICALL_ADDR) {
5855 char *symbol = mono_aot_get_direct_call_symbol (MONO_PATCH_INFO_ICALL_ADDR_CALL, ji->data.target);
5856 if (symbol) {
5857 /*
5858 * Avoid emitting a got entry for these since the method is directly called, and it might not be
5859 * resolvable at runtime using dlsym ().
5860 */
5861 g_free (symbol);
5862 values [ins->dreg] = LLVMConstInt (IntPtrType (), 0, FALSE);
5863 break;
5864 }
5865 }
5866
5867 ji->next = cfg->patch_info;
5868 cfg->patch_info = ji;
5869
5870 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
5871 got_offset = compute_aot_got_offset (ctx->module, cfg->patch_info, NULL);
5872
5873 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
5874 if (!mono_aot_is_shared_got_offset (got_offset)) {
5875 //mono_print_ji (ji);
5876 //printf ("\n");
5877 ctx->cfg->got_access_count ++;
5878 }
5879
5880 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5881 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
5882 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
5883
5884 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
5885 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
5886 g_free (name);
5887 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5888 if (!cfg->llvm_only || mono_aot_is_shared_got_offset (got_offset)) {
5889 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5890 set_invariant_load_flag (values [ins->dreg]);
5891 }
5892
5893 if (ji->type == MONO_PATCH_INFO_LDSTR)
5894 set_nonnull_load_flag (values [ins->dreg]);
5895 break;
5896 }
5897 case OP_NOT_REACHED:
5898 LLVMBuildUnreachable (builder);
5899 has_terminator = TRUE;
5900 g_assert (bb->block_num < cfg->max_block_num);
5901 ctx->unreachable [bb->block_num] = TRUE;
5902 /* Might have instructions after this */
5903 while (ins->next) {
5904 MonoInst *next = ins->next;
5905 /*
5906 * FIXME: If later code uses the regs defined by these instructions,
5907 * compilation will fail.
5908 */
5909 const char *spec = INS_INFO (next->opcode);
5910 if (spec [MONO_INST_DEST] == 'i')
5911 ctx->values [next->dreg] = LLVMConstNull (LLVMInt32Type ());
5912 MONO_DELETE_INS (bb, next);
5913 }
5914 break;
5915 case OP_LDADDR: {
5916 MonoInst *var = ins->inst_i0;
5917
5918 if (var->opcode == OP_VTARG_ADDR) {
5919 /* The variable contains the vtype address */
5920 values [ins->dreg] = values [var->dreg];
5921 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
5922 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
5923 } else {
5924 values [ins->dreg] = addresses [var->dreg];
5925 }
5926 break;
5927 }
5928 case OP_SIN: {
5929 LLVMValueRef args [1];
5930
5931 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5932 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SIN), args, 1, dname);
5933 break;
5934 }
5935 case OP_SINF: {
5936 LLVMValueRef args [1];
5937
5938 args [0] = convert (ctx, lhs, LLVMFloatType ());
5939 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SINF), args, 1, dname);
5940 break;
5941 }
5942 case OP_COS: {
5943 LLVMValueRef args [1];
5944
5945 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5946 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COS), args, 1, dname);
5947 break;
5948 }
5949 case OP_COSF: {
5950 LLVMValueRef args [1];
5951
5952 args [0] = convert (ctx, lhs, LLVMFloatType ());
5953 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_COSF), args, 1, dname);
5954 break;
5955 }
5956 case OP_SQRT: {
5957 LLVMValueRef args [1];
5958
5959 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5960 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRT), args, 1, dname);
5961 break;
5962 }
5963 case OP_SQRTF: {
5964 LLVMValueRef args [1];
5965
5966 args [0] = convert (ctx, lhs, LLVMFloatType ());
5967 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_SQRTF), args, 1, dname);
5968 break;
5969 }
5970 case OP_ABS: {
5971 LLVMValueRef args [1];
5972
5973 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5974 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
5975 break;
5976 }
5977 case OP_ABSF: {
5978 LLVMValueRef args [1];
5979
5980 #ifdef TARGET_AMD64
5981 args [0] = convert (ctx, lhs, LLVMFloatType ());
5982 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_ABSF), args, 1, dname);
5983 #else
5984 /* llvm.fabs not supported on all platforms */
5985 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5986 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_FABS), args, 1, dname);
5987 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
5988 #endif
5989 break;
5990 }
5991 case OP_RPOW: {
5992 LLVMValueRef args [2];
5993
5994 args [0] = convert (ctx, lhs, LLVMFloatType ());
5995 args [1] = convert (ctx, rhs, LLVMFloatType ());
5996 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_POWF), args, 2, dname);
5997 break;
5998 }
5999
6000 case OP_IMIN:
6001 case OP_LMIN:
6002 case OP_IMAX:
6003 case OP_LMAX:
6004 case OP_IMIN_UN:
6005 case OP_LMIN_UN:
6006 case OP_IMAX_UN:
6007 case OP_LMAX_UN:
6008 case OP_RMAX: {
6009 LLVMValueRef v;
6010
6011 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6012 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
6013
6014 switch (ins->opcode) {
6015 case OP_IMIN:
6016 case OP_LMIN:
6017 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
6018 break;
6019 case OP_IMAX:
6020 case OP_LMAX:
6021 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
6022 break;
6023 case OP_IMIN_UN:
6024 case OP_LMIN_UN:
6025 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
6026 break;
6027 case OP_IMAX_UN:
6028 case OP_LMAX_UN:
6029 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
6030 break;
6031 case OP_RMAX:
6032 v = LLVMBuildFCmp (builder, LLVMRealUGE, lhs, rhs, "");
6033 break;
6034 default:
6035 g_assert_not_reached ();
6036 break;
6037 }
6038 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
6039 break;
6040 }
6041
6042 /*
6043 * See the ARM64 comment in mono/utils/atomic.h for an explanation of why this
6044 * hack is necessary (for now).
6045 */
6046 #ifdef TARGET_ARM64
6047 #define ARM64_ATOMIC_FENCE_FIX mono_llvm_build_fence (builder, LLVM_BARRIER_SEQ)
6048 #else
6049 #define ARM64_ATOMIC_FENCE_FIX
6050 #endif
6051
6052 case OP_ATOMIC_EXCHANGE_I4:
6053 case OP_ATOMIC_EXCHANGE_I8: {
6054 LLVMValueRef args [2];
6055 LLVMTypeRef t;
6056
6057 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
6058 t = LLVMInt32Type ();
6059 else
6060 t = LLVMInt64Type ();
6061
6062 g_assert (ins->inst_offset == 0);
6063
6064 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6065 args [1] = convert (ctx, rhs, t);
6066
6067 ARM64_ATOMIC_FENCE_FIX;
6068 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
6069 ARM64_ATOMIC_FENCE_FIX;
6070 break;
6071 }
6072 case OP_ATOMIC_ADD_I4:
6073 case OP_ATOMIC_ADD_I8: {
6074 LLVMValueRef args [2];
6075 LLVMTypeRef t;
6076
6077 if (ins->opcode == OP_ATOMIC_ADD_I4)
6078 t = LLVMInt32Type ();
6079 else
6080 t = LLVMInt64Type ();
6081
6082 g_assert (ins->inst_offset == 0);
6083
6084 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6085 args [1] = convert (ctx, rhs, t);
6086 ARM64_ATOMIC_FENCE_FIX;
6087 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
6088 ARM64_ATOMIC_FENCE_FIX;
6089 break;
6090 }
6091 case OP_ATOMIC_CAS_I4:
6092 case OP_ATOMIC_CAS_I8: {
6093 LLVMValueRef args [3], val;
6094 LLVMTypeRef t;
6095
6096 if (ins->opcode == OP_ATOMIC_CAS_I4)
6097 t = LLVMInt32Type ();
6098 else
6099 t = LLVMInt64Type ();
6100
6101 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
6102 /* comparand */
6103 args [1] = convert (ctx, values [ins->sreg3], t);
6104 /* new value */
6105 args [2] = convert (ctx, values [ins->sreg2], t);
6106 ARM64_ATOMIC_FENCE_FIX;
6107 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
6108 ARM64_ATOMIC_FENCE_FIX;
6109 /* cmpxchg returns a pair */
6110 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
6111 break;
6112 }
6113 case OP_MEMORY_BARRIER: {
6114 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
6115 break;
6116 }
6117 case OP_ATOMIC_LOAD_I1:
6118 case OP_ATOMIC_LOAD_I2:
6119 case OP_ATOMIC_LOAD_I4:
6120 case OP_ATOMIC_LOAD_I8:
6121 case OP_ATOMIC_LOAD_U1:
6122 case OP_ATOMIC_LOAD_U2:
6123 case OP_ATOMIC_LOAD_U4:
6124 case OP_ATOMIC_LOAD_U8:
6125 case OP_ATOMIC_LOAD_R4:
6126 case OP_ATOMIC_LOAD_R8: {
6127 int size;
6128 gboolean sext, zext;
6129 LLVMTypeRef t;
6130 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
6131 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6132 LLVMValueRef index, addr;
6133
6134 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6135
6136 if (sext || zext)
6137 dname = (char *)"";
6138
6139 if (ins->inst_offset != 0) {
6140 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6141 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
6142 } else {
6143 addr = lhs;
6144 }
6145
6146 addr = convert (ctx, addr, LLVMPointerType (t, 0));
6147
6148 ARM64_ATOMIC_FENCE_FIX;
6149 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, lhs, dname, is_volatile, barrier);
6150 ARM64_ATOMIC_FENCE_FIX;
6151
6152 if (sext)
6153 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6154 else if (zext)
6155 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
6156 break;
6157 }
6158 case OP_ATOMIC_STORE_I1:
6159 case OP_ATOMIC_STORE_I2:
6160 case OP_ATOMIC_STORE_I4:
6161 case OP_ATOMIC_STORE_I8:
6162 case OP_ATOMIC_STORE_U1:
6163 case OP_ATOMIC_STORE_U2:
6164 case OP_ATOMIC_STORE_U4:
6165 case OP_ATOMIC_STORE_U8:
6166 case OP_ATOMIC_STORE_R4:
6167 case OP_ATOMIC_STORE_R8: {
6168 int size;
6169 gboolean sext, zext;
6170 LLVMTypeRef t;
6171 gboolean is_volatile = (ins->flags & (MONO_INST_FAULT | MONO_INST_VOLATILE)) != 0;
6172 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
6173 LLVMValueRef index, addr, value, base;
6174
6175 if (!values [ins->inst_destbasereg]) {
6176 set_failure (ctx, "inst_destbasereg");
6177 break;
6178 }
6179
6180 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
6181
6182 base = values [ins->inst_destbasereg];
6183 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
6184 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
6185 value = convert (ctx, values [ins->sreg1], t);
6186
6187 ARM64_ATOMIC_FENCE_FIX;
6188 emit_store_general (ctx, bb, &builder, size, value, addr, base, is_volatile, barrier);
6189 ARM64_ATOMIC_FENCE_FIX;
6190 break;
6191 }
6192 case OP_RELAXED_NOP: {
6193 #if defined(TARGET_AMD64) || defined(TARGET_X86)
6194 emit_call (ctx, bb, &builder, get_intrins_by_name (ctx, "llvm.x86.sse2.pause"), NULL, 0);
6195 break;
6196 #else
6197 break;
6198 #endif
6199 }
6200 case OP_TLS_GET: {
6201 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
6202 #ifdef TARGET_AMD64
6203 // 257 == FS segment register
6204 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
6205 #else
6206 // 256 == GS segment register
6207 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6208 #endif
6209 // FIXME: XEN
6210 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
6211 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
6212 /* See mono_amd64_emit_tls_get () */
6213 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
6214
6215 // 256 == GS segment register
6216 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
6217 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
6218 #else
6219 set_failure (ctx, "opcode tls-get");
6220 break;
6221 #endif
6222
6223 break;
6224 }
6225 case OP_GC_SAFE_POINT: {
6226 LLVMValueRef val, cmp, callee, call;
6227 LLVMBasicBlockRef poll_bb, cont_bb;
6228 LLVMValueRef args [2];
6229 static LLVMTypeRef sig;
6230 const char *icall_name = "mono_threads_state_poll";
6231
6232 /*
6233 * Create the cold wrapper around the icall, along with a managed method for it so
6234 * unwinding works.
6235 */
6236 if (!cfg->compile_aot && !ctx->module->gc_poll_cold_wrapper_compiled) {
6237 ERROR_DECL (error);
6238 /* Compiling a method here is a bit ugly, but it works */
6239 MonoMethod *wrapper = mono_marshal_get_llvm_func_wrapper (LLVM_FUNC_WRAPPER_GC_POLL);
6240 ctx->module->gc_poll_cold_wrapper_compiled = mono_jit_compile_method (wrapper, error);
6241 mono_error_assert_ok (error);
6242 }
6243
6244 if (!sig)
6245 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
6246
6247 /*
6248 * if (!*sreg1)
6249 * mono_threads_state_poll ();
6250 */
6251 val = mono_llvm_build_load (builder, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
6252 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
6253 poll_bb = gen_bb (ctx, "POLL_BB");
6254 cont_bb = gen_bb (ctx, "CONT_BB");
6255
6256 args [0] = cmp;
6257 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
6258 cmp = LLVMBuildCall (ctx->builder, get_intrins (ctx, INTRINS_EXPECT_I1), args, 2, "");
6259
6260 LLVMBuildCondBr (builder, cmp, cont_bb, poll_bb);
6261
6262 ctx->builder = builder = create_builder (ctx);
6263 LLVMPositionBuilderAtEnd (builder, poll_bb);
6264
6265 if (ctx->cfg->compile_aot) {
6266 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_threads_state_poll));
6267 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6268 } else {
6269 callee = get_jit_callee (ctx, icall_name, sig, MONO_PATCH_INFO_ABS, ctx->module->gc_poll_cold_wrapper_compiled);
6270 call = LLVMBuildCall (builder, callee, NULL, 0, "");
6271 set_call_cold_cconv (call);
6272 }
6273 LLVMBuildBr (builder, cont_bb);
6274
6275 ctx->builder = builder = create_builder (ctx);
6276 LLVMPositionBuilderAtEnd (builder, cont_bb);
6277 ctx->bblocks [bb->block_num].end_bblock = cont_bb;
6278 break;
6279 }
6280
6281 /*
6282 * Overflow opcodes.
6283 */
6284 case OP_IADD_OVF:
6285 case OP_IADD_OVF_UN:
6286 case OP_ISUB_OVF:
6287 case OP_ISUB_OVF_UN:
6288 case OP_IMUL_OVF:
6289 case OP_IMUL_OVF_UN:
6290 case OP_LADD_OVF:
6291 case OP_LADD_OVF_UN:
6292 case OP_LSUB_OVF:
6293 case OP_LSUB_OVF_UN:
6294 case OP_LMUL_OVF:
6295 case OP_LMUL_OVF_UN:
6296 {
6297 LLVMValueRef args [2], val, ovf, func;
6298
6299 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
6300 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
6301 func = get_intrins_by_name (ctx, ovf_op_to_intrins (ins->opcode));
6302 g_assert (func);
6303 val = LLVMBuildCall (builder, func, args, 2, "");
6304 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
6305 ovf = LLVMBuildExtractValue (builder, val, 1, "");
6306 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
6307 if (!ctx_ok (ctx))
6308 break;
6309 builder = ctx->builder;
6310 break;
6311 }
6312
6313 /*
6314 * Valuetypes.
6315 * We currently model them using arrays. Promotion to local vregs is
6316 * disabled for them in mono_handle_global_vregs () in the LLVM case,
6317 * so we always have an entry in cfg->varinfo for them.
6318 * FIXME: Is this needed ?
6319 */
6320 case OP_VZERO: {
6321 MonoClass *klass = ins->klass;
6322
6323 if (!klass) {
6324 // FIXME:
6325 set_failure (ctx, "!klass");
6326 break;
6327 }
6328
6329 if (!addresses [ins->dreg])
6330 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6331 LLVMValueRef ptr = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6332 emit_memset (ctx, builder, ptr, ConstInt32 (mono_class_value_size (klass, NULL)), 0);
6333 break;
6334 }
6335 case OP_DUMMY_VZERO:
6336 break;
6337
6338 case OP_STOREV_MEMBASE:
6339 case OP_LOADV_MEMBASE:
6340 case OP_VMOVE: {
6341 MonoClass *klass = ins->klass;
6342 LLVMValueRef src = NULL, dst, args [5];
6343 gboolean done = FALSE;
6344
6345 if (!klass) {
6346 // FIXME:
6347 set_failure (ctx, "!klass");
6348 break;
6349 }
6350
6351 if (mini_is_gsharedvt_klass (klass)) {
6352 // FIXME:
6353 set_failure (ctx, "gsharedvt");
6354 break;
6355 }
6356
6357 switch (ins->opcode) {
6358 case OP_STOREV_MEMBASE:
6359 if (cfg->gen_write_barriers && m_class_has_references (klass) && ins->inst_destbasereg != cfg->frame_reg &&
6360 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
6361 /* Decomposed earlier */
6362 g_assert_not_reached ();
6363 break;
6364 }
6365 if (!addresses [ins->sreg1]) {
6366 /* SIMD */
6367 g_assert (values [ins->sreg1]);
6368 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (klass)), 0));
6369 LLVMBuildStore (builder, values [ins->sreg1], dst);
6370 done = TRUE;
6371 } else {
6372 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6373 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6374 }
6375 break;
6376 case OP_LOADV_MEMBASE:
6377 if (!addresses [ins->dreg])
6378 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6379 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
6380 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6381 break;
6382 case OP_VMOVE:
6383 if (!addresses [ins->sreg1])
6384 addresses [ins->sreg1] = build_alloca (ctx, m_class_get_byval_arg (klass));
6385 if (!addresses [ins->dreg])
6386 addresses [ins->dreg] = build_alloca (ctx, m_class_get_byval_arg (klass));
6387 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
6388 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
6389 break;
6390 default:
6391 g_assert_not_reached ();
6392 }
6393 if (!ctx_ok (ctx))
6394 break;
6395
6396 if (done)
6397 break;
6398
6399 int aindex = 0;
6400 args [aindex ++] = dst;
6401 args [aindex ++] = src;
6402 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
6403 #if LLVM_API_VERSION < 900
6404 // FIXME: Alignment
6405 args [aindex ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6406 #endif
6407 args [aindex ++] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6408 LLVMBuildCall (builder, get_intrins (ctx, INTRINS_MEMCPY), args, aindex, "");
6409 break;
6410 }
6411 case OP_LLVM_OUTARG_VT: {
6412 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
6413 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
6414
6415 if (ainfo->storage == LLVMArgGsharedvtVariable) {
6416 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
6417
6418 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
6419 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
6420 } else {
6421 g_assert (addresses [ins->sreg1]);
6422 addresses [ins->dreg] = addresses [ins->sreg1];
6423 }
6424 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
6425 if (!addresses [ins->sreg1]) {
6426 addresses [ins->sreg1] = build_alloca (ctx, t);
6427 g_assert (values [ins->sreg1]);
6428 }
6429 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
6430 addresses [ins->dreg] = addresses [ins->sreg1];
6431 } else {
6432 if (!addresses [ins->sreg1]) {
6433 addresses [ins->sreg1] = build_alloca (ctx, t);
6434 g_assert (values [ins->sreg1]);
6435 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
6436 addresses [ins->dreg] = addresses [ins->sreg1];
6437 } else if (ainfo->storage == LLVMArgVtypeAddr || values [ins->sreg1] == addresses [ins->sreg1]) {
6438 /* LLVMArgVtypeByRef/LLVMArgVtypeAddr, have to make a copy */
6439 addresses [ins->dreg] = build_alloca (ctx, t);
6440 LLVMValueRef v = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
6441 LLVMBuildStore (builder, convert (ctx, v, type_to_llvm_type (ctx, t)), addresses [ins->dreg]);
6442 } else {
6443 addresses [ins->dreg] = addresses [ins->sreg1];
6444 }
6445 }
6446 break;
6447 }
6448 case OP_OBJC_GET_SELECTOR: {
6449 const char *name = (const char*)ins->inst_p0;
6450 LLVMValueRef var;
6451
6452 if (!ctx->module->objc_selector_to_var) {
6453 ctx->module->objc_selector_to_var = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
6454
6455 LLVMValueRef info_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), 8), "@OBJC_IMAGE_INFO");
6456 int32_t objc_imageinfo [] = { 0, 16 };
6457 LLVMSetInitializer (info_var, mono_llvm_create_constant_data_array ((uint8_t *) &objc_imageinfo, 8));
6458 LLVMSetLinkage (info_var, LLVMPrivateLinkage);
6459 LLVMSetExternallyInitialized (info_var, TRUE);
6460 LLVMSetSection (info_var, "__DATA, __objc_imageinfo,regular,no_dead_strip");
6461 LLVMSetAlignment (info_var, sizeof (target_mgreg_t));
6462 mark_as_used (ctx->module, info_var);
6463 }
6464
6465 var = (LLVMValueRef)g_hash_table_lookup (ctx->module->objc_selector_to_var, name);
6466 if (!var) {
6467 LLVMValueRef indexes [16];
6468
6469 LLVMValueRef name_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), strlen (name) + 1), "@OBJC_METH_VAR_NAME_");
6470 LLVMSetInitializer (name_var, mono_llvm_create_constant_data_array ((const uint8_t*)name, strlen (name) + 1));
6471 LLVMSetLinkage (name_var, LLVMPrivateLinkage);
6472 LLVMSetSection (name_var, "__TEXT,__objc_methname,cstring_literals");
6473 mark_as_used (ctx->module, name_var);
6474
6475 LLVMValueRef ref_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (LLVMInt8Type (), 0), "@OBJC_SELECTOR_REFERENCES_");
6476
6477 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6478 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, 0);
6479 LLVMSetInitializer (ref_var, LLVMConstGEP (name_var, indexes, 2));
6480 LLVMSetLinkage (ref_var, LLVMPrivateLinkage);
6481 LLVMSetExternallyInitialized (ref_var, TRUE);
6482 LLVMSetSection (ref_var, "__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
6483 LLVMSetAlignment (ref_var, sizeof (target_mgreg_t));
6484 mark_as_used (ctx->module, ref_var);
6485
6486 g_hash_table_insert (ctx->module->objc_selector_to_var, g_strdup (name), ref_var);
6487 var = ref_var;
6488 }
6489
6490 values [ins->dreg] = LLVMBuildLoad (builder, var, "");
6491 break;
6492 }
6493
6494 /*
6495 * SIMD
6496 */
6497 #if defined(TARGET_X86) || defined(TARGET_AMD64)
6498 case OP_XZERO: {
6499 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
6500 break;
6501 }
6502 case OP_LOADX_MEMBASE: {
6503 LLVMTypeRef t = type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass));
6504 LLVMValueRef src;
6505
6506 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6507 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
6508 break;
6509 }
6510 case OP_STOREX_MEMBASE: {
6511 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
6512 LLVMValueRef dest;
6513
6514 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
6515 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
6516 break;
6517 }
6518 case OP_PADDB:
6519 case OP_PADDW:
6520 case OP_PADDD:
6521 case OP_PADDQ:
6522 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
6523 break;
6524 case OP_ADDPD:
6525 case OP_ADDPS:
6526 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
6527 break;
6528 case OP_PSUBB:
6529 case OP_PSUBW:
6530 case OP_PSUBD:
6531 case OP_PSUBQ:
6532 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
6533 break;
6534 case OP_SUBPD:
6535 case OP_SUBPS:
6536 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
6537 break;
6538 case OP_MULPD:
6539 case OP_MULPS:
6540 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
6541 break;
6542 case OP_DIVPD:
6543 case OP_DIVPS:
6544 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
6545 break;
6546 case OP_PAND:
6547 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
6548 break;
6549 case OP_POR:
6550 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
6551 break;
6552 case OP_PXOR:
6553 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
6554 break;
6555 case OP_PMULW:
6556 case OP_PMULD:
6557 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
6558 break;
6559 case OP_ANDPS:
6560 case OP_ANDNPS:
6561 case OP_ORPS:
6562 case OP_XORPS:
6563 case OP_ANDPD:
6564 case OP_ANDNPD:
6565 case OP_ORPD:
6566 case OP_XORPD: {
6567 LLVMTypeRef t, rt;
6568 LLVMValueRef v = NULL;
6569
6570 switch (ins->opcode) {
6571 case OP_ANDPS:
6572 case OP_ANDNPS:
6573 case OP_ORPS:
6574 case OP_XORPS:
6575 t = LLVMVectorType (LLVMInt32Type (), 4);
6576 rt = LLVMVectorType (LLVMFloatType (), 4);
6577 break;
6578 case OP_ANDPD:
6579 case OP_ANDNPD:
6580 case OP_ORPD:
6581 case OP_XORPD:
6582 t = LLVMVectorType (LLVMInt64Type (), 2);
6583 rt = LLVMVectorType (LLVMDoubleType (), 2);
6584 break;
6585 default:
6586 t = LLVMInt32Type ();
6587 rt = LLVMInt32Type ();
6588 g_assert_not_reached ();
6589 }
6590
6591 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6592 rhs = LLVMBuildBitCast (builder, rhs, t, "");
6593 switch (ins->opcode) {
6594 case OP_ANDPS:
6595 case OP_ANDPD:
6596 v = LLVMBuildAnd (builder, lhs, rhs, "");
6597 break;
6598 case OP_ORPS:
6599 case OP_ORPD:
6600 v = LLVMBuildOr (builder, lhs, rhs, "");
6601 break;
6602 case OP_XORPS:
6603 case OP_XORPD:
6604 v = LLVMBuildXor (builder, lhs, rhs, "");
6605 break;
6606 case OP_ANDNPS:
6607 case OP_ANDNPD:
6608 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
6609 break;
6610 }
6611 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
6612 break;
6613 }
6614 case OP_PMIND_UN:
6615 case OP_PMINW_UN:
6616 case OP_PMINB_UN: {
6617 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntULT, lhs, rhs, "");
6618 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6619 break;
6620 }
6621 case OP_PMAXD_UN:
6622 case OP_PMAXW_UN:
6623 case OP_PMAXB_UN: {
6624 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntUGT, lhs, rhs, "");
6625 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6626 break;
6627 }
6628 case OP_PMINW: {
6629 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntSLT, lhs, rhs, "");
6630 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
6631 break;
6632 }
6633 case OP_MINPD:
6634 case OP_MINPS:
6635 case OP_MAXPD:
6636 case OP_MAXPS:
6637 case OP_ADDSUBPD:
6638 case OP_ADDSUBPS:
6639 case OP_HADDPD:
6640 case OP_HADDPS:
6641 case OP_HSUBPD:
6642 case OP_HSUBPS:
6643 case OP_PADDB_SAT:
6644 case OP_PADDW_SAT:
6645 case OP_PSUBB_SAT:
6646 case OP_PSUBW_SAT:
6647 case OP_PADDB_SAT_UN:
6648 case OP_PADDW_SAT_UN:
6649 case OP_PSUBB_SAT_UN:
6650 case OP_PSUBW_SAT_UN:
6651 case OP_PACKW:
6652 case OP_PACKD:
6653 case OP_PACKW_UN:
6654 case OP_PACKD_UN:
6655 case OP_PMULW_HIGH:
6656 case OP_PMULW_HIGH_UN: {
6657 LLVMValueRef args [2];
6658
6659 args [0] = lhs;
6660 args [1] = rhs;
6661
6662 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6663 break;
6664 }
6665 case OP_PAVGB_UN:
6666 case OP_PAVGW_UN: {
6667 LLVMValueRef ones_vec;
6668 LLVMValueRef ones [32];
6669 int vector_size = LLVMGetVectorSize (LLVMTypeOf (lhs));
6670 LLVMTypeRef ext_elem_type = vector_size == 16 ? LLVMInt16Type () : LLVMInt32Type ();
6671
6672 for (int i = 0; i < 32; ++i)
6673 ones [i] = LLVMConstInt (ext_elem_type, 1, FALSE);
6674 ones_vec = LLVMConstVector (ones, vector_size);
6675
6676 LLVMValueRef val;
6677 LLVMTypeRef ext_type = LLVMVectorType (ext_elem_type, vector_size);
6678
6679 /* Have to increase the vector element size to prevent overflows */
6680 /* res = trunc ((zext (lhs) + zext (rhs) + 1) >> 1) */
6681 val = LLVMBuildAdd (builder, LLVMBuildZExt (builder, lhs, ext_type, ""), LLVMBuildZExt (builder, rhs, ext_type, ""), "");
6682 val = LLVMBuildAdd (builder, val, ones_vec, "");
6683 val = LLVMBuildLShr (builder, val, ones_vec, "");
6684 values [ins->dreg] = LLVMBuildTrunc (builder, val, LLVMTypeOf (lhs), "");
6685 break;
6686 }
6687 case OP_PCMPEQB:
6688 case OP_PCMPEQW:
6689 case OP_PCMPEQD:
6690 case OP_PCMPEQQ:
6691 case OP_PCMPGTB: {
6692 LLVMValueRef pcmp;
6693 LLVMTypeRef retType;
6694 int cmpOp;
6695
6696 if (ins->opcode == OP_PCMPGTB)
6697 cmpOp = LLVMIntSGT;
6698 else
6699 cmpOp = LLVMIntEQ;
6700
6701 if (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)) {
6702 pcmp = LLVMBuildICmp (builder, cmpOp, lhs, rhs, "");
6703 retType = LLVMTypeOf (lhs);
6704 } else {
6705 LLVMTypeRef flatType = LLVMVectorType (LLVMInt8Type (), 16);
6706 LLVMValueRef flatRHS = convert (ctx, rhs, flatType);
6707 LLVMValueRef flatLHS = convert (ctx, lhs, flatType);
6708
6709 pcmp = LLVMBuildICmp (builder, cmpOp, flatLHS, flatRHS, "");
6710 retType = flatType;
6711 }
6712
6713 values [ins->dreg] = LLVMBuildSExt (builder, pcmp, retType, "");
6714 break;
6715 }
6716 case OP_EXTRACT_R8:
6717 case OP_EXTRACT_I8:
6718 case OP_EXTRACT_I4:
6719 case OP_EXTRACT_I2:
6720 case OP_EXTRACT_U2:
6721 case OP_EXTRACTX_U2:
6722 case OP_EXTRACT_I1:
6723 case OP_EXTRACT_U1: {
6724 LLVMTypeRef t;
6725 gboolean zext = FALSE;
6726
6727 t = simd_op_to_llvm_type (ins->opcode);
6728
6729 switch (ins->opcode) {
6730 case OP_EXTRACT_R8:
6731 case OP_EXTRACT_I8:
6732 case OP_EXTRACT_I4:
6733 case OP_EXTRACT_I2:
6734 case OP_EXTRACT_I1:
6735 break;
6736 case OP_EXTRACT_U2:
6737 case OP_EXTRACTX_U2:
6738 case OP_EXTRACT_U1:
6739 zext = TRUE;
6740 break;
6741 default:
6742 t = LLVMInt32Type ();
6743 g_assert_not_reached ();
6744 }
6745
6746 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6747 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
6748 if (zext)
6749 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
6750 break;
6751 }
6752
6753 case OP_EXPAND_I1:
6754 case OP_EXPAND_I2:
6755 case OP_EXPAND_I4:
6756 case OP_EXPAND_I8:
6757 case OP_EXPAND_R4:
6758 case OP_EXPAND_R8: {
6759 LLVMTypeRef t;
6760 LLVMValueRef mask [32], v;
6761 int i;
6762
6763 #ifdef ENABLE_NETCORE
6764 t = simd_class_to_llvm_type (ctx, ins->klass);
6765 #else
6766 t = simd_op_to_llvm_type (ins->opcode);
6767 #endif
6768 for (i = 0; i < 32; ++i)
6769 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6770
6771 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
6772
6773 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6774 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
6775 break;
6776 }
6777
6778 case OP_INSERT_I1:
6779 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6780 break;
6781 case OP_INSERT_I2:
6782 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6783 break;
6784 case OP_INSERT_I4:
6785 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6786 break;
6787 case OP_INSERT_I8:
6788 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6789 break;
6790 case OP_INSERT_R4:
6791 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6792 break;
6793 case OP_INSERT_R8:
6794 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6795 break;
6796
6797 case OP_CVTDQ2PD: {
6798 LLVMValueRef indexes [16];
6799
6800 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6801 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6802 LLVMValueRef mask = LLVMConstVector (indexes, 2);
6803 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
6804 values [ins->dreg] = LLVMBuildSIToFP (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
6805 break;
6806 }
6807 case OP_CVTPS2PD: {
6808 LLVMValueRef indexes [16];
6809
6810 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6811 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6812 LLVMValueRef mask = LLVMConstVector (indexes, 2);
6813 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
6814 values [ins->dreg] = LLVMBuildFPExt (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
6815 break;
6816 }
6817 case OP_CVTTPS2DQ:
6818 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMVectorType (LLVMInt32Type (), 4), dname);
6819 break;
6820
6821 case OP_CVTDQ2PS:
6822 case OP_CVTPD2DQ:
6823 case OP_CVTPS2DQ:
6824 case OP_CVTPD2PS:
6825 case OP_CVTTPD2DQ:
6826 case OP_EXTRACT_MASK:
6827 case OP_SQRTPS:
6828 case OP_SQRTPD:
6829 case OP_RSQRTPS:
6830 case OP_RCPPS: {
6831 LLVMValueRef v;
6832
6833 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
6834
6835 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
6836 break;
6837 }
6838 case OP_COMPPS:
6839 case OP_COMPPD: {
6840 LLVMRealPredicate op;
6841
6842 switch (ins->inst_c0) {
6843 case SIMD_COMP_EQ:
6844 op = LLVMRealOEQ;
6845 break;
6846 case SIMD_COMP_LT:
6847 op = LLVMRealOLT;
6848 break;
6849 case SIMD_COMP_LE:
6850 op = LLVMRealOLE;
6851 break;
6852 case SIMD_COMP_UNORD:
6853 op = LLVMRealUNO;
6854 break;
6855 case SIMD_COMP_NEQ:
6856 op = LLVMRealUNE;
6857 break;
6858 case SIMD_COMP_NLT:
6859 op = LLVMRealUGE;
6860 break;
6861 case SIMD_COMP_NLE:
6862 op = LLVMRealUGT;
6863 break;
6864 case SIMD_COMP_ORD:
6865 op = LLVMRealORD;
6866 break;
6867 default:
6868 g_assert_not_reached ();
6869 }
6870
6871 LLVMValueRef cmp = LLVMBuildFCmp (builder, op, lhs, rhs, "");
6872 if (ins->opcode == OP_COMPPD)
6873 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), 2), ""), LLVMTypeOf (lhs), "");
6874 else
6875 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), 4), ""), LLVMTypeOf (lhs), "");
6876 break;
6877 }
6878 case OP_ICONV_TO_X:
6879 /* This is only used for implementing shifts by non-immediate */
6880 values [ins->dreg] = lhs;
6881 break;
6882
6883 case OP_PSHRW:
6884 case OP_PSHRD:
6885 case OP_PSHRQ:
6886 case OP_PSARW:
6887 case OP_PSARD:
6888 case OP_PSHLW:
6889 case OP_PSHLD:
6890 case OP_PSHLQ: {
6891 LLVMValueRef args [3];
6892
6893 args [0] = lhs;
6894 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
6895
6896 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6897 break;
6898 }
6899
6900 case OP_PSHRW_REG:
6901 case OP_PSHRD_REG:
6902 case OP_PSHRQ_REG:
6903 case OP_PSARW_REG:
6904 case OP_PSARD_REG:
6905 case OP_PSHLW_REG:
6906 case OP_PSHLD_REG:
6907 case OP_PSHLQ_REG: {
6908 LLVMValueRef args [3];
6909
6910 args [0] = lhs;
6911 args [1] = values [ins->sreg2];
6912
6913 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6914 break;
6915 }
6916
6917 case OP_SHUFPS:
6918 case OP_SHUFPD:
6919 case OP_PSHUFLED:
6920 case OP_PSHUFLEW_LOW:
6921 case OP_PSHUFLEW_HIGH: {
6922 int mask [16];
6923 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
6924 int i, mask_size = 0;
6925 int imask = ins->inst_c0;
6926
6927 /* Convert the x86 shuffle mask to LLVM's */
6928 switch (ins->opcode) {
6929 case OP_SHUFPS:
6930 mask_size = 4;
6931 mask [0] = ((imask >> 0) & 3);
6932 mask [1] = ((imask >> 2) & 3);
6933 mask [2] = ((imask >> 4) & 3) + 4;
6934 mask [3] = ((imask >> 6) & 3) + 4;
6935 v1 = values [ins->sreg1];
6936 v2 = values [ins->sreg2];
6937 break;
6938 case OP_SHUFPD:
6939 mask_size = 2;
6940 mask [0] = ((imask >> 0) & 1);
6941 mask [1] = ((imask >> 1) & 1) + 2;
6942 v1 = values [ins->sreg1];
6943 v2 = values [ins->sreg2];
6944 break;
6945 case OP_PSHUFLEW_LOW:
6946 mask_size = 8;
6947 mask [0] = ((imask >> 0) & 3);
6948 mask [1] = ((imask >> 2) & 3);
6949 mask [2] = ((imask >> 4) & 3);
6950 mask [3] = ((imask >> 6) & 3);
6951 mask [4] = 4 + 0;
6952 mask [5] = 4 + 1;
6953 mask [6] = 4 + 2;
6954 mask [7] = 4 + 3;
6955 v1 = values [ins->sreg1];
6956 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6957 break;
6958 case OP_PSHUFLEW_HIGH:
6959 mask_size = 8;
6960 mask [0] = 0;
6961 mask [1] = 1;
6962 mask [2] = 2;
6963 mask [3] = 3;
6964 mask [4] = 4 + ((imask >> 0) & 3);
6965 mask [5] = 4 + ((imask >> 2) & 3);
6966 mask [6] = 4 + ((imask >> 4) & 3);
6967 mask [7] = 4 + ((imask >> 6) & 3);
6968 v1 = values [ins->sreg1];
6969 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6970 break;
6971 case OP_PSHUFLED:
6972 mask_size = 4;
6973 mask [0] = ((imask >> 0) & 3);
6974 mask [1] = ((imask >> 2) & 3);
6975 mask [2] = ((imask >> 4) & 3);
6976 mask [3] = ((imask >> 6) & 3);
6977 v1 = values [ins->sreg1];
6978 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6979 break;
6980 default:
6981 g_assert_not_reached ();
6982 }
6983 for (i = 0; i < mask_size; ++i)
6984 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6985
6986 values [ins->dreg] =
6987 LLVMBuildShuffleVector (builder, v1, v2,
6988 LLVMConstVector (mask_values, mask_size), dname);
6989 break;
6990 }
6991
6992 case OP_UNPACK_LOWB:
6993 case OP_UNPACK_LOWW:
6994 case OP_UNPACK_LOWD:
6995 case OP_UNPACK_LOWQ:
6996 case OP_UNPACK_LOWPS:
6997 case OP_UNPACK_LOWPD:
6998 case OP_UNPACK_HIGHB:
6999 case OP_UNPACK_HIGHW:
7000 case OP_UNPACK_HIGHD:
7001 case OP_UNPACK_HIGHQ:
7002 case OP_UNPACK_HIGHPS:
7003 case OP_UNPACK_HIGHPD: {
7004 int mask [16];
7005 LLVMValueRef mask_values [16];
7006 int i, mask_size = 0;
7007 gboolean low = FALSE;
7008
7009 switch (ins->opcode) {
7010 case OP_UNPACK_LOWB:
7011 mask_size = 16;
7012 low = TRUE;
7013 break;
7014 case OP_UNPACK_LOWW:
7015 mask_size = 8;
7016 low = TRUE;
7017 break;
7018 case OP_UNPACK_LOWD:
7019 case OP_UNPACK_LOWPS:
7020 mask_size = 4;
7021 low = TRUE;
7022 break;
7023 case OP_UNPACK_LOWQ:
7024 case OP_UNPACK_LOWPD:
7025 mask_size = 2;
7026 low = TRUE;
7027 break;
7028 case OP_UNPACK_HIGHB:
7029 mask_size = 16;
7030 break;
7031 case OP_UNPACK_HIGHW:
7032 mask_size = 8;
7033 break;
7034 case OP_UNPACK_HIGHD:
7035 case OP_UNPACK_HIGHPS:
7036 mask_size = 4;
7037 break;
7038 case OP_UNPACK_HIGHQ:
7039 case OP_UNPACK_HIGHPD:
7040 mask_size = 2;
7041 break;
7042 default:
7043 g_assert_not_reached ();
7044 }
7045
7046 if (low) {
7047 for (i = 0; i < (mask_size / 2); ++i) {
7048 mask [(i * 2)] = i;
7049 mask [(i * 2) + 1] = mask_size + i;
7050 }
7051 } else {
7052 for (i = 0; i < (mask_size / 2); ++i) {
7053 mask [(i * 2)] = (mask_size / 2) + i;
7054 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
7055 }
7056 }
7057
7058 for (i = 0; i < mask_size; ++i)
7059 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
7060
7061 values [ins->dreg] =
7062 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
7063 LLVMConstVector (mask_values, mask_size), dname);
7064 break;
7065 }
7066
7067 case OP_DUPPD: {
7068 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7069 LLVMValueRef v, val;
7070
7071 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7072 val = LLVMConstNull (t);
7073 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7074 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
7075
7076 values [ins->dreg] = val;
7077 break;
7078 }
7079 case OP_DUPPS_LOW:
7080 case OP_DUPPS_HIGH: {
7081 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
7082 LLVMValueRef v1, v2, val;
7083
7084
7085 if (ins->opcode == OP_DUPPS_LOW) {
7086 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7087 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7088 } else {
7089 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7090 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7091 }
7092 val = LLVMConstNull (t);
7093 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7094 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
7095 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
7096 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
7097
7098 values [ins->dreg] = val;
7099 break;
7100 }
7101
7102 case OP_DPPS: {
7103 LLVMValueRef args [3];
7104
7105 args [0] = lhs;
7106 args [1] = rhs;
7107 /* 0xf1 == multiply all 4 elements, add them together, and store the result to the lowest element */
7108 args [2] = LLVMConstInt (LLVMInt8Type (), 0xf1, FALSE);
7109
7110 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, simd_op_to_intrins (ins->opcode)), args, 3, dname);
7111 break;
7112 }
7113
7114 case OP_FCONV_TO_R8_X: {
7115 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (type_to_simd_type (MONO_TYPE_R8)), lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7116 break;
7117 }
7118
7119 case OP_SSE41_ROUNDPD: {
7120 LLVMValueRef args [3];
7121
7122 args [0] = lhs;
7123 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
7124
7125 values [ins->dreg] = LLVMBuildCall (builder, get_intrins_by_name (ctx, "llvm.x86.sse41.round.pd"), args, 2, dname);
7126 break;
7127 }
7128
7129 #ifdef ENABLE_NETCORE
7130 case OP_XCAST: {
7131 LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass);
7132
7133 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, "");
7134 break;
7135 }
7136 case OP_XCOMPARE_FP: {
7137 LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0];
7138 LLVMValueRef cmp = LLVMBuildFCmp (builder, pred, lhs, rhs, "");
7139 int nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
7140 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
7141 if (ins->inst_c1 == MONO_TYPE_R8)
7142 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), nelems), ""), LLVMTypeOf (lhs), "");
7143 else
7144 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), nelems), ""), LLVMTypeOf (lhs), "");
7145 break;
7146 }
7147 case OP_XCOMPARE: {
7148 LLVMIntPredicate pred = cond_to_llvm_cond [ins->inst_c0];
7149 LLVMValueRef cmp = LLVMBuildICmp (builder, pred, lhs, rhs, "");
7150 g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
7151 values [ins->dreg] = LLVMBuildSExt (builder, cmp, LLVMTypeOf (lhs), "");
7152 break;
7153 }
7154 case OP_XEQUAL: {
7155 LLVMTypeRef t;
7156 LLVMValueRef cmp, mask [32], shuffle;
7157 int nelems;
7158
7159 //%c = icmp sgt <16 x i8> %a0, %a1
7160 if (LLVMGetElementType (LLVMTypeOf (lhs)) == LLVMDoubleType () || LLVMGetElementType (LLVMTypeOf (lhs)) == LLVMFloatType ())
7161 cmp = LLVMBuildFCmp (builder, LLVMRealOEQ, lhs, rhs, "");
7162 else
7163 cmp = LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, "");
7164 nelems = LLVMGetVectorSize (LLVMTypeOf (cmp));
7165 t = LLVMVectorType (LLVMInt8Type (), nelems);
7166 cmp = LLVMBuildSExt (builder, cmp, t, "");
7167 // cmp is a <16 x i8> vector, each element is either 0xff or 0
7168 int half = nelems / 2;
7169 while (half >= 1) {
7170 // AND the top and bottom halfes into the bottom half
7171 for (int i = 0; i < half; ++i)
7172 mask [i] = LLVMConstInt (LLVMInt32Type (), half + i, FALSE);
7173 for (int i = half; i < nelems; ++i)
7174 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7175 shuffle = LLVMBuildShuffleVector (builder, cmp, LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
7176 cmp = LLVMBuildAnd (builder, cmp, shuffle, "");
7177 half = half / 2;
7178 }
7179 // Extract [0]
7180 values [ins->dreg] = LLVMBuildExtractElement (builder, cmp, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
7181 // Maybe convert to 0/1 ?
7182 break;
7183 }
7184 case OP_XBINOP: {
7185 switch (ins->inst_c0) {
7186 case OP_IADD:
7187 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
7188 break;
7189 case OP_ISUB:
7190 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
7191 break;
7192 case OP_IAND:
7193 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
7194 break;
7195 case OP_IOR:
7196 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
7197 break;
7198 case OP_IXOR:
7199 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
7200 break;
7201 case OP_FADD:
7202 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
7203 break;
7204 case OP_FSUB:
7205 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
7206 break;
7207 case OP_FMUL:
7208 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
7209 break;
7210 case OP_FDIV:
7211 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
7212 break;
7213 default:
7214 g_assert_not_reached ();
7215 }
7216 break;
7217 }
7218 case OP_POPCNT32:
7219 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CTPOP_I32), &lhs, 1, "");
7220 break;
7221 case OP_POPCNT64:
7222 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, INTRINS_CTPOP_I64), &lhs, 1, "");
7223 break;
7224 case OP_LZCNT32:
7225 case OP_LZCNT64: {
7226 LLVMValueRef args [2];
7227 args [0] = lhs;
7228 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
7229 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_LZCNT32 ? INTRINS_CTLZ_I32 : INTRINS_CTLZ_I64), args, 2, "");
7230 break;
7231 }
7232 case OP_CTTZ32:
7233 case OP_CTTZ64: {
7234 LLVMValueRef args [2];
7235 args [0] = lhs;
7236 args [1] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
7237 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_CTTZ32 ? INTRINS_CTTZ_I32 : INTRINS_CTTZ_I64), args, 2, "");
7238 break;
7239 }
7240 case OP_PEXT32:
7241 case OP_PEXT64: {
7242 LLVMValueRef args [2];
7243 args [0] = lhs;
7244 args [1] = rhs;
7245 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_PEXT32 ? INTRINS_PEXT_I32 : INTRINS_PEXT_I64), args, 2, "");
7246 break;
7247 }
7248 case OP_PDEP32:
7249 case OP_PDEP64: {
7250 LLVMValueRef args [2];
7251 args [0] = lhs;
7252 args [1] = rhs;
7253 values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_PDEP32 ? INTRINS_PDEP_I32 : INTRINS_PDEP_I64), args, 2, "");
7254 break;
7255 }
7256 #endif /* ENABLE_NETCORE */
7257 #endif /* SIMD */
7258
7259 case OP_DUMMY_USE:
7260 break;
7261
7262 /*
7263 * EXCEPTION HANDLING
7264 */
7265 case OP_IMPLICIT_EXCEPTION:
7266 /* This marks a place where an implicit exception can happen */
7267 if (bb->region != -1)
7268 set_failure (ctx, "implicit-exception");
7269 break;
7270 case OP_THROW:
7271 case OP_RETHROW: {
7272 gboolean rethrow = (ins->opcode == OP_RETHROW);
7273 if (ctx->llvm_only) {
7274 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
7275 has_terminator = TRUE;
7276 ctx->unreachable [bb->block_num] = TRUE;
7277 } else {
7278 emit_throw (ctx, bb, rethrow, lhs);
7279 builder = ctx->builder;
7280 }
7281 break;
7282 }
7283 case OP_CALL_HANDLER: {
7284 /*
7285 * We don't 'call' handlers, but instead simply branch to them.
7286 * The code generated by ENDFINALLY will branch back to us.
7287 */
7288 LLVMBasicBlockRef noex_bb;
7289 GSList *bb_list;
7290 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
7291
7292 bb_list = info->call_handler_return_bbs;
7293
7294 /*
7295 * Set the indicator variable for the finally clause.
7296 */
7297 lhs = info->finally_ind;
7298 g_assert (lhs);
7299 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
7300
7301 /* Branch to the finally clause */
7302 LLVMBuildBr (builder, info->call_handler_target_bb);
7303
7304 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
7305 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
7306
7307 builder = ctx->builder = create_builder (ctx);
7308 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
7309
7310 bblocks [bb->block_num].end_bblock = noex_bb;
7311 break;
7312 }
7313 case OP_START_HANDLER: {
7314 break;
7315 }
7316 case OP_ENDFINALLY: {
7317 LLVMBasicBlockRef resume_bb;
7318 MonoBasicBlock *handler_bb;
7319 LLVMValueRef val, switch_ins, callee;
7320 GSList *bb_list;
7321 BBInfo *info;
7322 gboolean is_fault = MONO_REGION_FLAGS (bb->region) == MONO_EXCEPTION_CLAUSE_FAULT;
7323
7324 /*
7325 * Fault clauses are like finally clauses, but they are only called if an exception is thrown.
7326 */
7327 if (!is_fault) {
7328 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
7329 g_assert (handler_bb);
7330 info = &bblocks [handler_bb->block_num];
7331 lhs = info->finally_ind;
7332 g_assert (lhs);
7333
7334 bb_list = info->call_handler_return_bbs;
7335
7336 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
7337
7338 /* Load the finally variable */
7339 val = LLVMBuildLoad (builder, lhs, "");
7340
7341 /* Reset the variable */
7342 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
7343
7344 /* Branch to either resume_bb, or to the bblocks in bb_list */
7345 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
7346 /*
7347 * The other targets are added at the end to handle OP_CALL_HANDLER
7348 * opcodes processed later.
7349 */
7350 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
7351
7352 builder = ctx->builder = create_builder (ctx);
7353 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
7354 }
7355
7356 if (ctx->llvm_only) {
7357 emit_resume_eh (ctx, bb);
7358 } else {
7359 LLVMTypeRef icall_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
7360 if (ctx->cfg->compile_aot) {
7361 callee = get_callee (ctx, icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7362 } else {
7363 callee = get_jit_callee (ctx, "llvm_resume_unwind_trampoline", icall_sig, MONO_PATCH_INFO_JIT_ICALL_ID, GUINT_TO_POINTER (MONO_JIT_ICALL_mono_llvm_resume_unwind_trampoline));
7364 }
7365 LLVMBuildCall (builder, callee, NULL, 0, "");
7366 LLVMBuildUnreachable (builder);
7367 }
7368
7369 has_terminator = TRUE;
7370 break;
7371 }
7372 case OP_IL_SEQ_POINT:
7373 break;
7374 default: {
7375 char reason [128];
7376
7377 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
7378 set_failure (ctx, reason);
7379 break;
7380 }
7381 }
7382
7383 if (!ctx_ok (ctx))
7384 break;
7385
7386 /* Convert the value to the type required by phi nodes */
7387 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
7388 if (ctx->is_vphi [ins->dreg])
7389 /* vtypes */
7390 values [ins->dreg] = addresses [ins->dreg];
7391 else
7392 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
7393 }
7394
7395 /* Add stores for volatile variables */
7396 if (!skip_volatile_store && spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
7397 emit_volatile_store (ctx, ins->dreg);
7398 }
7399
7400 if (!ctx_ok (ctx))
7401 return;
7402
7403 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
7404 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
7405 }
7406
7407 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
7408 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
7409 LLVMBuildRetVoid (builder);
7410 }
7411
7412 if (bb == cfg->bb_entry)
7413 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
7414 }
7415
7416 /*
7417 * mono_llvm_check_method_supported:
7418 *
7419 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
7420 * compiling a method twice.
7421 */
7422 void
7423 mono_llvm_check_method_supported (MonoCompile *cfg)
7424 {
7425 int i, j;
7426
7427 #ifdef TARGET_WASM
7428 if (mono_method_signature_internal (cfg->method)->call_convention == MONO_CALL_VARARG) {
7429 cfg->exception_message = g_strdup ("vararg callconv");
7430 cfg->disable_llvm = TRUE;
7431 return;
7432 }
7433 #endif
7434
7435 if (cfg->llvm_only)
7436 return;
7437
7438 if (cfg->method->save_lmf) {
7439 cfg->exception_message = g_strdup ("lmf");
7440 cfg->disable_llvm = TRUE;
7441 }
7442 if (cfg->disable_llvm)
7443 return;
7444
7445 /*
7446 * Nested clauses where one of the clauses is a finally clause is
7447 * not supported, because LLVM can't figure out the control flow,
7448 * probably because we resume exception handling by calling our
7449 * own function instead of using the 'resume' llvm instruction.
7450 */
7451 for (i = 0; i < cfg->header->num_clauses; ++i) {
7452 for (j = 0; j < cfg->header->num_clauses; ++j) {
7453 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
7454 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
7455
7456 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
7457 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7458 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
7459 cfg->exception_message = g_strdup ("nested clauses");
7460 cfg->disable_llvm = TRUE;
7461 break;
7462 }
7463 }
7464 }
7465 if (cfg->disable_llvm)
7466 return;
7467
7468 /* FIXME: */
7469 if (cfg->method->dynamic) {
7470 cfg->exception_message = g_strdup ("dynamic.");
7471 cfg->disable_llvm = TRUE;
7472 }
7473 if (cfg->disable_llvm)
7474 return;
7475 }
7476
7477 static LLVMCallInfo*
7478 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
7479 {
7480 LLVMCallInfo *linfo;
7481 int i;
7482
7483 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
7484 int i, n, pindex;
7485
7486 /*
7487 * Gsharedvt methods have the following calling convention:
7488 * - all arguments are passed by ref, even non generic ones
7489 * - the return value is returned by ref too, using a vret
7490 * argument passed after 'this'.
7491 */
7492 n = sig->param_count + sig->hasthis;
7493 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
7494
7495 pindex = 0;
7496 if (sig->hasthis)
7497 linfo->args [pindex ++].storage = LLVMArgNormal;
7498
7499 if (sig->ret->type != MONO_TYPE_VOID) {
7500 if (mini_is_gsharedvt_variable_type (sig->ret))
7501 linfo->ret.storage = LLVMArgGsharedvtVariable;
7502 else if (mini_type_is_vtype (sig->ret))
7503 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
7504 else
7505 linfo->ret.storage = LLVMArgGsharedvtFixed;
7506 linfo->vret_arg_index = pindex;
7507 } else {
7508 linfo->ret.storage = LLVMArgNone;
7509 }
7510
7511 for (i = 0; i < sig->param_count; ++i) {
7512 if (sig->params [i]->byref)
7513 linfo->args [pindex].storage = LLVMArgNormal;
7514 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
7515 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
7516 else if (mini_type_is_vtype (sig->params [i]))
7517 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
7518 else
7519 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
7520 linfo->args [pindex].type = sig->params [i];
7521 pindex ++;
7522 }
7523 return linfo;
7524 }
7525
7526 linfo = mono_arch_get_llvm_call_info (cfg, sig);
7527 linfo->dummy_arg_pindex = -1;
7528 for (i = 0; i < sig->param_count; ++i)
7529 linfo->args [i + sig->hasthis].type = sig->params [i];
7530
7531 return linfo;
7532 }
7533
7534 static void
7535 emit_method_inner (EmitContext *ctx);
7536
7537 static void
7538 free_ctx (EmitContext *ctx)
7539 {
7540 GSList *l;
7541
7542 g_free (ctx->values);
7543 g_free (ctx->addresses);
7544 g_free (ctx->vreg_types);
7545 g_free (ctx->is_vphi);
7546 g_free (ctx->vreg_cli_types);
7547 g_free (ctx->is_dead);
7548 g_free (ctx->unreachable);
7549 g_ptr_array_free (ctx->phi_values, TRUE);
7550 g_free (ctx->bblocks);
7551 g_hash_table_destroy (ctx->region_to_handler);
7552 g_hash_table_destroy (ctx->clause_to_handler);
7553 g_hash_table_destroy (ctx->jit_callees);
7554
7555 g_ptr_array_free (ctx->callsite_list, TRUE);
7556
7557 g_free (ctx->method_name);
7558 g_ptr_array_free (ctx->bblock_list, TRUE);
7559
7560 for (l = ctx->builders; l; l = l->next) {
7561 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
7562 LLVMDisposeBuilder (builder);
7563 }
7564
7565 g_free (ctx);
7566 }
7567
7568 static gboolean
7569 is_externally_callable (EmitContext *ctx, MonoMethod *method)
7570 {
7571 if (ctx->module->llvm_only && ctx->module->static_link && (method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE || method_is_direct_callable (method)))
7572 return TRUE;
7573 return FALSE;
7574 }
7575
7576 /*
7577 * mono_llvm_emit_method:
7578 *
7579 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
7580 */
7581 void
7582 mono_llvm_emit_method (MonoCompile *cfg)
7583 {
7584 EmitContext *ctx;
7585 char *method_name;
7586 int i;
7587
7588 if (cfg->skip)
7589 return;
7590
7591 /* The code below might acquire the loader lock, so use it for global locking */
7592 mono_loader_lock ();
7593
7594 /* Used to communicate with the callbacks */
7595 mono_native_tls_set_value (current_cfg_tls_id, cfg);
7596
7597 ctx = g_new0 (EmitContext, 1);
7598 ctx->cfg = cfg;
7599 ctx->mempool = cfg->mempool;
7600
7601 /*
7602 * This maps vregs to the LLVM instruction defining them
7603 */
7604 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
7605 /*
7606 * This maps vregs for volatile variables to the LLVM instruction defining their
7607 * address.
7608 */
7609 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
7610 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
7611 ctx->is_vphi = g_new0 (gboolean, cfg->next_vreg);
7612 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
7613 ctx->phi_values = g_ptr_array_sized_new (256);
7614 /*
7615 * This signals whenever the vreg was defined by a phi node with no input vars
7616 * (i.e. all its input bblocks end with NOT_REACHABLE).
7617 */
7618 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
7619 /* Whenever the bblock is unreachable */
7620 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
7621 ctx->bblock_list = g_ptr_array_sized_new (256);
7622
7623 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
7624 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
7625 ctx->callsite_list = g_ptr_array_new ();
7626 ctx->jit_callees = g_hash_table_new (NULL, NULL);
7627 if (cfg->compile_aot) {
7628 ctx->module = &aot_module;
7629
7630 if (is_externally_callable (ctx, cfg->method))
7631 method_name = mono_aot_get_mangled_method_name (cfg->method);
7632 else
7633 method_name = mono_aot_get_method_name (cfg);
7634 cfg->llvm_method_name = g_strdup (method_name);
7635 } else {
7636 init_jit_module (cfg->domain);
7637 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
7638 method_name = mono_method_full_name (cfg->method, TRUE);
7639 }
7640 ctx->method_name = method_name;
7641
7642 if (cfg->compile_aot) {
7643 ctx->lmodule = ctx->module->lmodule;
7644 } else {
7645 ctx->lmodule = LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg->method->name));
7646 /* Reset this as it contains values from lmodule */
7647 memset (ctx->module->intrins_by_id, 0, sizeof (LLVMValueRef) * INTRINS_NUM);
7648 }
7649 ctx->llvm_only = ctx->module->llvm_only;
7650 #ifdef TARGET_WASM
7651 ctx->emit_dummy_arg = TRUE;
7652 #endif
7653
7654 emit_method_inner (ctx);
7655
7656 if (!ctx_ok (ctx)) {
7657 if (ctx->lmethod) {
7658 /* Need to add unused phi nodes as they can be referenced by other values */
7659 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
7660 LLVMBuilderRef builder;
7661
7662 builder = create_builder (ctx);
7663 LLVMPositionBuilderAtEnd (builder, phi_bb);
7664
7665 for (i = 0; i < ctx->phi_values->len; ++i) {
7666 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
7667 if (LLVMGetInstructionParent (v) == NULL)
7668 LLVMInsertIntoBuilder (builder, v);
7669 }
7670
7671 if (ctx->module->llvm_only && ctx->module->static_link) {
7672 // Keep a stub for the function since it might be called directly
7673 int nbbs = LLVMCountBasicBlocks (ctx->lmethod);
7674 LLVMBasicBlockRef *bblocks = g_new0 (LLVMBasicBlockRef, nbbs);
7675 LLVMGetBasicBlocks (ctx->lmethod, bblocks);
7676 for (int i = 0; i < nbbs; ++i)
7677 LLVMDeleteBasicBlock (bblocks [i]);
7678
7679 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (ctx->lmethod, "ENTRY");
7680 builder = create_builder (ctx);
7681 LLVMPositionBuilderAtEnd (builder, entry_bb);
7682 ctx->builder = builder;
7683
7684 LLVMTypeRef sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
7685 LLVMValueRef callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, GUINT_TO_POINTER (MONO_JIT_ICALL_mini_llvmonly_throw_nullref_exception));
7686 LLVMBuildCall (builder, callee, NULL, 0, "");
7687 LLVMBuildUnreachable (builder);
7688 } else {
7689 LLVMDeleteFunction (ctx->lmethod);
7690 }
7691 }
7692 }
7693
7694 free_ctx (ctx);
7695
7696 mono_native_tls_set_value (current_cfg_tls_id, NULL);
7697
7698 mono_loader_unlock ();
7699 }
7700
7701 static void
7702 emit_method_inner (EmitContext *ctx)
7703 {
7704 MonoCompile *cfg = ctx->cfg;
7705 MonoMethodSignature *sig;
7706 MonoBasicBlock *bb;
7707 LLVMTypeRef method_type;
7708 LLVMValueRef method = NULL;
7709 LLVMValueRef *values = ctx->values;
7710 int i, max_block_num, bb_index;
7711 gboolean last = FALSE;
7712 LLVMCallInfo *linfo;
7713 LLVMModuleRef lmodule = ctx->lmodule;
7714 BBInfo *bblocks;
7715 GPtrArray *bblock_list = ctx->bblock_list;
7716 MonoMethodHeader *header;
7717 MonoExceptionClause *clause;
7718 char **names;
7719 LLVMBuilderRef entry_builder = NULL;
7720 LLVMBasicBlockRef entry_bb = NULL;
7721
7722 if (cfg->gsharedvt && !cfg->llvm_only) {
7723 set_failure (ctx, "gsharedvt");
7724 return;
7725 }
7726
7727 #if 1
7728 {
7729 static int count = 0;
7730 count ++;
7731
7732 char *llvm_count_str = g_getenv ("LLVM_COUNT");
7733 if (llvm_count_str) {
7734 int lcount = atoi (llvm_count_str);
7735 g_free (llvm_count_str);
7736 if (count == lcount) {
7737 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
7738 fflush (stdout);
7739 last = TRUE;
7740 }
7741 if (count > lcount) {
7742 set_failure (ctx, "count");
7743 return;
7744 }
7745 }
7746 }
7747 #endif
7748
7749 // If we come upon one of the init_method wrappers, we need to find
7750 // the method that we have already emitted and tell LLVM that this
7751 // managed method info for the wrapper is associated with this method
7752 // we constructed ourselves from LLVM IR.
7753 //
7754 // This is necessary to unwind through the init_method, in the case that
7755 // it has to run a static cctor that throws an exception
7756 if (cfg->method->wrapper_type == MONO_WRAPPER_OTHER) {
7757 WrapperInfo *info = mono_marshal_get_wrapper_info (cfg->method);
7758 if (info->subtype == WRAPPER_SUBTYPE_AOT_INIT) {
7759 method = get_init_icall_wrapper (ctx->module, info->d.aot_init.subtype);
7760 ctx->lmethod = method;
7761 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
7762
7763 const char *init_name = mono_marshal_get_aot_init_wrapper_name (info->d.aot_init.subtype);
7764 ctx->method_name = g_strdup_printf ("%s%s", ctx->module->global_prefix, init_name);
7765 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
7766
7767 if (!cfg->llvm_only && ctx->module->external_symbols) {
7768 LLVMSetLinkage (method, LLVMExternalLinkage);
7769 LLVMSetVisibility (method, LLVMHiddenVisibility);
7770 }
7771
7772 goto after_codegen;
7773 } else if (info->subtype == WRAPPER_SUBTYPE_LLVM_FUNC) {
7774 g_assert (info->d.llvm_func.subtype == LLVM_FUNC_WRAPPER_GC_POLL);
7775
7776 if (cfg->compile_aot) {
7777 method = ctx->module->gc_poll_cold_wrapper;
7778 g_assert (method);
7779 } else {
7780 method = emit_icall_cold_wrapper (ctx->module, lmodule, MONO_JIT_ICALL_mono_threads_state_poll, FALSE);
7781 }
7782 ctx->lmethod = method;
7783 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
7784
7785 ctx->method_name = g_strdup (LLVMGetValueName (method)); //g_strdup_printf ("%s_%s", ctx->module->global_prefix, LLVMGetValueName (method));
7786 ctx->cfg->asm_symbol = g_strdup (ctx->method_name);
7787
7788 if (!cfg->llvm_only && ctx->module->external_symbols) {
7789 LLVMSetLinkage (method, LLVMExternalLinkage);
7790 LLVMSetVisibility (method, LLVMHiddenVisibility);
7791 }
7792
7793 goto after_codegen;
7794 }
7795 }
7796
7797 sig = mono_method_signature_internal (cfg->method);
7798 ctx->sig = sig;
7799
7800 linfo = get_llvm_call_info (cfg, sig);
7801 ctx->linfo = linfo;
7802 if (!ctx_ok (ctx))
7803 return;
7804
7805 if (cfg->rgctx_var)
7806 linfo->rgctx_arg = TRUE;
7807 else if (needs_extra_arg (ctx, cfg->method))
7808 linfo->dummy_arg = TRUE;
7809 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
7810 if (!ctx_ok (ctx))
7811 return;
7812
7813 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
7814 ctx->lmethod = method;
7815
7816 if (!cfg->llvm_only)
7817 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
7818
7819 /* if the method doesn't contain
7820 * (1) a call (so it's a leaf method)
7821 * (2) and no loops
7822 * we can skip the GC safepoint on method entry. */
7823 gboolean requires_safepoint;
7824 requires_safepoint = cfg->has_calls;
7825 if (!requires_safepoint) {
7826 for (bb = cfg->bb_entry->next_bb; bb; bb = bb->next_bb) {
7827 if (bb->loop_body_start || (bb->flags & BB_EXCEPTION_HANDLER)) {
7828 requires_safepoint = TRUE;
7829 }
7830 }
7831 }
7832
7833 if (!cfg->llvm_only && cfg->compile_aot && mono_threads_are_safepoints_enabled () && requires_safepoint)
7834 LLVMSetGC (method, "coreclr");
7835 LLVMSetLinkage (method, LLVMPrivateLinkage);
7836
7837 mono_llvm_add_func_attr (method, LLVM_ATTR_UW_TABLE);
7838
7839 if (cfg->compile_aot) {
7840 if (is_externally_callable (ctx, cfg->method)) {
7841 LLVMSetLinkage (method, LLVMExternalLinkage);
7842 } else {
7843 LLVMSetLinkage (method, LLVMInternalLinkage);
7844 //all methods have internal visibility when doing llvm_only
7845 if (!cfg->llvm_only && ctx->module->external_symbols) {
7846 LLVMSetLinkage (method, LLVMExternalLinkage);
7847 LLVMSetVisibility (method, LLVMHiddenVisibility);
7848 }
7849 }
7850 } else {
7851 LLVMSetLinkage (method, LLVMExternalLinkage);
7852 }
7853
7854 if (cfg->method->save_lmf && !cfg->llvm_only) {
7855 set_failure (ctx, "lmf");
7856 return;
7857 }
7858
7859 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
7860 set_failure (ctx, "pinvoke signature");
7861 return;
7862 }
7863
7864 header = cfg->header;
7865 for (i = 0; i < header->num_clauses; ++i) {
7866 clause = &header->clauses [i];
7867 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_FAULT && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
7868 set_failure (ctx, "non-finally/catch/fault clause.");
7869 return;
7870 }
7871 }
7872 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
7873 /* We can't handle inlined methods with clauses */
7874 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
7875
7876 if (linfo->rgctx_arg) {
7877 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
7878 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
7879 /*
7880 * We mark the rgctx parameter with the inreg attribute, which is mapped to
7881 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
7882 * CC_X86_64_Mono in X86CallingConv.td.
7883 */
7884 if (!ctx->llvm_only)
7885 mono_llvm_add_param_attr (ctx->rgctx_arg, LLVM_ATTR_IN_REG);
7886 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
7887 } else {
7888 ctx->rgctx_arg_pindex = -1;
7889 }
7890 if (cfg->vret_addr) {
7891 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
7892 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
7893 if (linfo->ret.storage == LLVMArgVtypeByRef) {
7894 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
7895 mono_llvm_add_param_attr (LLVMGetParam (method, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
7896 }
7897 }
7898
7899 if (sig->hasthis) {
7900 ctx->this_arg_pindex = linfo->this_arg_pindex;
7901 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
7902 values [cfg->args [0]->dreg] = ctx->this_arg;
7903 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
7904 }
7905 if (linfo->dummy_arg)
7906 LLVMSetValueName (LLVMGetParam (method, linfo->dummy_arg_pindex), "dummy_arg");
7907
7908 names = g_new (char *, sig->param_count);
7909 mono_method_get_param_names (cfg->method, (const char **) names);
7910
7911 /* Set parameter names/attributes */
7912 for (i = 0; i < sig->param_count; ++i) {
7913 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
7914 char *name;
7915 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
7916 int j;
7917
7918 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
7919 name = g_strdup_printf ("dummy_%d_%d", i, j);
7920 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
7921 g_free (name);
7922 }
7923
7924 if (ainfo->storage == LLVMArgVtypeInReg && ainfo->pair_storage [0] == LLVMArgNone && ainfo->pair_storage [1] == LLVMArgNone)
7925 continue;
7926
7927 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
7928 if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
7929 if (names [i] && names [i][0] != '\0')
7930 name = g_strdup_printf ("p_arg_%s", names [i]);
7931 else
7932 name = g_strdup_printf ("p_arg_%d", i);
7933 } else {
7934 if (names [i] && names [i][0] != '\0')
7935 name = g_strdup_printf ("arg_%s", names [i]);
7936 else
7937 name = g_strdup_printf ("arg_%d", i);
7938 }
7939 LLVMSetValueName (LLVMGetParam (method, pindex), name);
7940 g_free (name);
7941 if (ainfo->storage == LLVMArgVtypeByVal)
7942 mono_llvm_add_param_attr (LLVMGetParam (method, pindex), LLVM_ATTR_BY_VAL);
7943
7944 if (ainfo->storage == LLVMArgVtypeByRef || ainfo->storage == LLVMArgVtypeAddr) {
7945 /* For OP_LDADDR */
7946 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
7947 }
7948 }
7949 g_free (names);
7950
7951 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
7952 ctx->minfo = mono_debug_lookup_method (cfg->method);
7953 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
7954 }
7955
7956 max_block_num = 0;
7957 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
7958 max_block_num = MAX (max_block_num, bb->block_num);
7959 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
7960
7961 /* Add branches between non-consecutive bblocks */
7962 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7963 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
7964 bb->next_bb != bb->last_ins->inst_false_bb) {
7965
7966 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
7967 inst->opcode = OP_BR;
7968 inst->inst_target_bb = bb->last_ins->inst_false_bb;
7969 mono_bblock_add_inst (bb, inst);
7970 }
7971 }
7972
7973 /*
7974 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
7975 */
7976 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7977 MonoInst *ins;
7978 LLVMBuilderRef builder;
7979 char *dname;
7980 char dname_buf[128];
7981
7982 builder = create_builder (ctx);
7983
7984 for (ins = bb->code; ins; ins = ins->next) {
7985 switch (ins->opcode) {
7986 case OP_PHI:
7987 case OP_FPHI:
7988 case OP_VPHI:
7989 case OP_XPHI: {
7990 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)));
7991
7992 if (!ctx_ok (ctx))
7993 return;
7994
7995 if (ins->opcode == OP_VPHI) {
7996 /* Treat valuetype PHI nodes as operating on the address itself */
7997 g_assert (ins->klass);
7998 phi_type = LLVMPointerType (type_to_llvm_type (ctx, m_class_get_byval_arg (ins->klass)), 0);
7999 }
8000
8001 /*
8002 * Have to precreate these, as they can be referenced by
8003 * earlier instructions.
8004 */
8005 sprintf (dname_buf, "t%d", ins->dreg);
8006 dname = dname_buf;
8007 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
8008
8009 if (ins->opcode == OP_VPHI)
8010 ctx->addresses [ins->dreg] = values [ins->dreg];
8011
8012 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
8013
8014 /*
8015 * Set the expected type of the incoming arguments since these have
8016 * to have the same type.
8017 */
8018 for (i = 0; i < ins->inst_phi_args [0]; i++) {
8019 int sreg1 = ins->inst_phi_args [i + 1];
8020
8021 if (sreg1 != -1) {
8022 if (ins->opcode == OP_VPHI)
8023 ctx->is_vphi [sreg1] = TRUE;
8024 ctx->vreg_types [sreg1] = phi_type;
8025 }
8026 }
8027 break;
8028 }
8029 case OP_LDADDR:
8030 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
8031 break;
8032 default:
8033 break;
8034 }
8035 }
8036 }
8037
8038 /*
8039 * Create an ordering for bblocks, use the depth first order first, then
8040 * put the exception handling bblocks last.
8041 */
8042 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
8043 bb = cfg->bblocks [bb_index];
8044 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
8045 g_ptr_array_add (bblock_list, bb);
8046 bblocks [bb->block_num].added = TRUE;
8047 }
8048 }
8049
8050 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8051 if (!bblocks [bb->block_num].added)
8052 g_ptr_array_add (bblock_list, bb);
8053 }
8054
8055 /*
8056 * Second pass: generate code.
8057 */
8058 // Emit entry point
8059 entry_builder = create_builder (ctx);
8060 entry_bb = get_bb (ctx, cfg->bb_entry);
8061 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
8062 emit_entry_bb (ctx, entry_builder);
8063
8064 // Make landing pads first
8065 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
8066
8067 if (ctx->llvm_only) {
8068 size_t group_index = 0;
8069 while (group_index < cfg->header->num_clauses) {
8070 int count = 0;
8071 size_t cursor = group_index;
8072 while (cursor < cfg->header->num_clauses &&
8073 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
8074 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
8075 count++;
8076 cursor++;
8077 }
8078
8079 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
8080 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
8081 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
8082
8083 group_index = cursor;
8084 }
8085 }
8086
8087 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
8088 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
8089
8090 // Prune unreachable mono BBs.
8091 if (!(bb == cfg->bb_entry || bb->in_count > 0))
8092 continue;
8093
8094 process_bb (ctx, bb);
8095 if (!ctx_ok (ctx))
8096 return;
8097 }
8098 g_hash_table_destroy (ctx->exc_meta);
8099
8100 mono_memory_barrier ();
8101
8102 /* Add incoming phi values */
8103 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8104 GSList *l, *ins_list;
8105
8106 ins_list = bblocks [bb->block_num].phi_nodes;
8107
8108 for (l = ins_list; l; l = l->next) {
8109 PhiNode *node = (PhiNode*)l->data;
8110 MonoInst *phi = node->phi;
8111 int sreg1 = node->sreg;
8112 LLVMBasicBlockRef in_bb;
8113
8114 if (sreg1 == -1)
8115 continue;
8116
8117 in_bb = get_end_bb (ctx, node->in_bb);
8118
8119 if (ctx->unreachable [node->in_bb->block_num])
8120 continue;
8121
8122 if (phi->opcode == OP_VPHI) {
8123 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
8124 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
8125 } else {
8126 if (!values [sreg1]) {
8127 /* Can happen with values in EH clauses */
8128 set_failure (ctx, "incoming phi sreg1");
8129 return;
8130 }
8131 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
8132 set_failure (ctx, "incoming phi arg type mismatch");
8133 return;
8134 }
8135 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
8136 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
8137 }
8138 }
8139 }
8140
8141 /* Nullify empty phi instructions */
8142 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8143 GSList *l, *ins_list;
8144
8145 ins_list = bblocks [bb->block_num].phi_nodes;
8146
8147 for (l = ins_list; l; l = l->next) {
8148 PhiNode *node = (PhiNode*)l->data;
8149 MonoInst *phi = node->phi;
8150 LLVMValueRef phi_ins = values [phi->dreg];
8151
8152 if (!phi_ins)
8153 /* Already removed */
8154 continue;
8155
8156 if (LLVMCountIncoming (phi_ins) == 0) {
8157 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
8158 LLVMInstructionEraseFromParent (phi_ins);
8159 values [phi->dreg] = NULL;
8160 }
8161 }
8162 }
8163
8164 /* Create the SWITCH statements for ENDFINALLY instructions */
8165 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
8166 BBInfo *info = &bblocks [bb->block_num];
8167 GSList *l;
8168 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
8169 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
8170 GSList *bb_list = info->call_handler_return_bbs;
8171
8172 GSList *bb_list_iter;
8173 i = 0;
8174 for (bb_list_iter = bb_list; bb_list_iter; bb_list_iter = g_slist_next (bb_list_iter)) {
8175 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)bb_list_iter->data);
8176 i ++;
8177 }
8178 }
8179 }
8180
8181 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
8182
8183 /* Initialize the method if needed */
8184 if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
8185 // FIXME: Add more shared got entries
8186 ctx->builder = create_builder (ctx);
8187 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
8188
8189 // FIXME: beforefieldinit
8190 /*
8191 * NATIVE_TO_MANAGED methods might be called on a thread not attached to the runtime, so they are initialized when loaded
8192 * in load_method ().
8193 */
8194 gboolean needs_init = ctx->cfg->got_access_count > 0;
8195 MonoMethod *cctor = NULL;
8196 if (!needs_init && (cctor = mono_class_get_cctor (cfg->method->klass))) {
8197 /* Needs init to run the cctor */
8198 if (cfg->method->flags & METHOD_ATTRIBUTE_STATIC)
8199 needs_init = TRUE;
8200 if (cctor == cfg->method)
8201 needs_init = FALSE;
8202
8203 // If we are a constructor, we need to init so the static
8204 // constructor gets called.
8205 if (!strcmp (cfg->method->name, ".ctor"))
8206 needs_init = TRUE;
8207 }
8208 if (cfg->method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED)
8209 needs_init = FALSE;
8210 if (needs_init)
8211 emit_init_method (ctx);
8212 else
8213 LLVMBuildBr (ctx->builder, ctx->inited_bb);
8214
8215 // Was observing LLVM moving field accesses into the caller's method
8216 // body before the init call (the inlined one), leading to NULL derefs
8217 // after the init_method returns (GOT is filled out though)
8218 if (needs_init)
8219 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8220 }
8221
8222 if (mini_get_debug_options ()->llvm_disable_inlining)
8223 mono_llvm_add_func_attr (method, LLVM_ATTR_NO_INLINE);
8224
8225
8226 after_codegen:
8227 if (cfg->llvm_only) {
8228 g_ptr_array_add (ctx->module->cfgs, cfg);
8229
8230 /*
8231 * Add the contents of ctx->callsite_list to module->callsite_list.
8232 * We can't do this earlier, as it contains llvm instructions which can be
8233 * freed if compilation fails.
8234 * FIXME: Get rid of this when all methods can be llvm compiled.
8235 */
8236 for (int i = 0; i < ctx->callsite_list->len; ++i)
8237 g_ptr_array_add (ctx->module->callsite_list, g_ptr_array_index (ctx->callsite_list, i));
8238 }
8239
8240 if (cfg->verbose_level > 1) {
8241 g_print ("\n*** Unoptimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, FALSE));
8242 mono_llvm_dump_value (method);
8243 g_print ("***\n\n");
8244 }
8245
8246 if (cfg->compile_aot && !cfg->llvm_only)
8247 mark_as_used (ctx->module, method);
8248
8249 if (!cfg->llvm_only) {
8250 LLVMValueRef md_args [16];
8251 LLVMValueRef md_node;
8252 int method_index;
8253
8254 if (cfg->compile_aot)
8255 method_index = mono_aot_get_method_index (cfg->orig_method);
8256 else
8257 method_index = 1;
8258 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
8259 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
8260 md_node = LLVMMDNode (md_args, 2);
8261 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
8262 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
8263 }
8264
8265 if (cfg->compile_aot) {
8266 /* Don't generate native code, keep the LLVM IR */
8267 if (cfg->verbose_level) {
8268 char *name = mono_method_get_full_name (cfg->method);
8269 printf ("%s emitted as %s\n", name, ctx->method_name);
8270 g_free (name);
8271 }
8272
8273 //LLVMDumpValue (ctx->lmethod);
8274 //int err = LLVMVerifyFunction(ctx->lmethod, LLVMPrintMessageAction);
8275 //g_assert (err == 0);
8276 } else {
8277 //LLVMVerifyFunction (method, 0);
8278 llvm_jit_finalize_method (ctx);
8279 }
8280
8281 if (ctx->module->method_to_lmethod)
8282 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
8283
8284 if (ctx->module->idx_to_lmethod)
8285 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
8286
8287 if (ctx->llvm_only && m_class_is_valuetype (cfg->orig_method->klass) && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
8288 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
8289 }
8290
8291 /*
8292 * mono_llvm_create_vars:
8293 *
8294 * Same as mono_arch_create_vars () for LLVM.
8295 */
8296 void
8297 mono_llvm_create_vars (MonoCompile *cfg)
8298 {
8299 MonoMethodSignature *sig;
8300
8301 sig = mono_method_signature_internal (cfg->method);
8302 if (cfg->gsharedvt && cfg->llvm_only) {
8303 gboolean vretaddr = FALSE;
8304
8305 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
8306 vretaddr = TRUE;
8307 } else {
8308 MonoMethodSignature *sig = mono_method_signature_internal (cfg->method);
8309 LLVMCallInfo *linfo;
8310
8311 linfo = get_llvm_call_info (cfg, sig);
8312 vretaddr = (linfo->ret.storage == LLVMArgVtypeRetAddr || linfo->ret.storage == LLVMArgVtypeByRef || linfo->ret.storage == LLVMArgGsharedvtFixed || linfo->ret.storage == LLVMArgGsharedvtVariable || linfo->ret.storage == LLVMArgGsharedvtFixedVtype);
8313 }
8314 if (vretaddr) {
8315 /*
8316 * Creating vret_addr forces CEE_SETRET to store the result into it,
8317 * so we don't have to generate any code in our OP_SETRET case.
8318 */
8319 cfg->vret_addr = mono_compile_create_var (cfg, m_class_get_byval_arg (mono_get_intptr_class ()), OP_ARG);
8320 if (G_UNLIKELY (cfg->verbose_level > 1)) {
8321 printf ("vret_addr = ");
8322 mono_print_ins (cfg->vret_addr);
8323 }
8324 }
8325 } else {
8326 mono_arch_create_vars (cfg);
8327 }
8328 }
8329
8330 /*
8331 * mono_llvm_emit_call:
8332 *
8333 * Same as mono_arch_emit_call () for LLVM.
8334 */
8335 void
8336 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
8337 {
8338 MonoInst *in;
8339 MonoMethodSignature *sig;
8340 int i, n, stack_size;
8341 LLVMArgInfo *ainfo;
8342
8343 stack_size = 0;
8344
8345 sig = call->signature;
8346 n = sig->param_count + sig->hasthis;
8347
8348 call->cinfo = get_llvm_call_info (cfg, sig);
8349
8350 if (cfg->disable_llvm)
8351 return;
8352
8353 if (sig->call_convention == MONO_CALL_VARARG) {
8354 cfg->exception_message = g_strdup ("varargs");
8355 cfg->disable_llvm = TRUE;
8356 }
8357
8358 for (i = 0; i < n; ++i) {
8359 MonoInst *ins;
8360
8361 ainfo = call->cinfo->args + i;
8362
8363 in = call->args [i];
8364
8365 /* Simply remember the arguments */
8366 switch (ainfo->storage) {
8367 case LLVMArgNormal: {
8368 MonoType *t = (sig->hasthis && i == 0) ? m_class_get_byval_arg (mono_get_intptr_class ()) : ainfo->type;
8369 int opcode;
8370
8371 opcode = mono_type_to_regmove (cfg, t);
8372 if (opcode == OP_FMOVE) {
8373 MONO_INST_NEW (cfg, ins, OP_FMOVE);
8374 ins->dreg = mono_alloc_freg (cfg);
8375 } else if (opcode == OP_LMOVE) {
8376 MONO_INST_NEW (cfg, ins, OP_LMOVE);
8377 ins->dreg = mono_alloc_lreg (cfg);
8378 } else if (opcode == OP_RMOVE) {
8379 MONO_INST_NEW (cfg, ins, OP_RMOVE);
8380 ins->dreg = mono_alloc_freg (cfg);
8381 } else {
8382 MONO_INST_NEW (cfg, ins, OP_MOVE);
8383 ins->dreg = mono_alloc_ireg (cfg);
8384 }
8385 ins->sreg1 = in->dreg;
8386 break;
8387 }
8388 case LLVMArgVtypeByVal:
8389 case LLVMArgVtypeByRef:
8390 case LLVMArgVtypeInReg:
8391 case LLVMArgVtypeAddr:
8392 case LLVMArgVtypeAsScalar:
8393 case LLVMArgAsIArgs:
8394 case LLVMArgAsFpArgs:
8395 case LLVMArgGsharedvtVariable:
8396 case LLVMArgGsharedvtFixed:
8397 case LLVMArgGsharedvtFixedVtype:
8398 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
8399 ins->dreg = mono_alloc_ireg (cfg);
8400 ins->sreg1 = in->dreg;
8401 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
8402 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
8403 ins->inst_vtype = ainfo->type;
8404 ins->klass = mono_class_from_mono_type_internal (ainfo->type);
8405 break;
8406 default:
8407 cfg->exception_message = g_strdup ("ainfo->storage");
8408 cfg->disable_llvm = TRUE;
8409 return;
8410 }
8411
8412 if (!cfg->disable_llvm) {
8413 MONO_ADD_INS (cfg->cbb, ins);
8414 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
8415 }
8416 }
8417 }
8418
8419 static inline void
8420 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
8421 {
8422 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
8423 }
8424
8425 static inline void
8426 AddFunc1 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1)
8427 {
8428 LLVMTypeRef param_types [4];
8429
8430 param_types [0] = param_type1;
8431
8432 AddFunc (module, name, ret_type, param_types, 1);
8433 }
8434
8435 static inline void
8436 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
8437 {
8438 LLVMTypeRef param_types [4];
8439
8440 param_types [0] = param_type1;
8441 param_types [1] = param_type2;
8442
8443 AddFunc (module, name, ret_type, param_types, 2);
8444 }
8445
8446 typedef struct {
8447 IntrinsicId id;
8448 const char *name;
8449 } IntrinsicDesc;
8450
8451 static IntrinsicDesc intrinsics[] = {
8452 {INTRINS_MEMSET, "llvm.memset.p0i8.i32"},
8453 {INTRINS_MEMCPY, "llvm.memcpy.p0i8.p0i8.i32"},
8454 {INTRINS_SADD_OVF_I32, "llvm.sadd.with.overflow.i32"},
8455 {INTRINS_UADD_OVF_I32, "llvm.uadd.with.overflow.i32"},
8456 {INTRINS_SSUB_OVF_I32, "llvm.ssub.with.overflow.i32"},
8457 {INTRINS_USUB_OVF_I32, "llvm.usub.with.overflow.i32"},
8458 {INTRINS_SMUL_OVF_I32, "llvm.smul.with.overflow.i32"},
8459 {INTRINS_UMUL_OVF_I32, "llvm.umul.with.overflow.i32"},
8460 {INTRINS_SADD_OVF_I64, "llvm.sadd.with.overflow.i64"},
8461 {INTRINS_UADD_OVF_I64, "llvm.uadd.with.overflow.i64"},
8462 {INTRINS_SSUB_OVF_I64, "llvm.ssub.with.overflow.i64"},
8463 {INTRINS_USUB_OVF_I64, "llvm.usub.with.overflow.i64"},
8464 {INTRINS_SMUL_OVF_I64, "llvm.smul.with.overflow.i64"},
8465 {INTRINS_UMUL_OVF_I64, "llvm.umul.with.overflow.i64"},
8466 {INTRINS_SIN, "llvm.sin.f64"},
8467 {INTRINS_COS, "llvm.cos.f64"},
8468 {INTRINS_SQRT, "llvm.sqrt.f64"},
8469 /* This isn't an intrinsic, instead llvm seems to special case it by name */
8470 {INTRINS_FABS, "fabs"},
8471 {INTRINS_ABSF, "llvm.fabs.f32"},
8472 {INTRINS_SINF, "llvm.sin.f32"},
8473 {INTRINS_COSF, "llvm.cos.f32"},
8474 {INTRINS_SQRTF, "llvm.sqrt.f32"},
8475 {INTRINS_POWF, "llvm.pow.f32"},
8476 {INTRINS_EXPECT_I8, "llvm.expect.i8"},
8477 {INTRINS_EXPECT_I1, "llvm.expect.i1"},
8478 {INTRINS_CTPOP_I32, "llvm.ctpop.i32"},
8479 {INTRINS_CTPOP_I64, "llvm.ctpop.i64"},
8480 {INTRINS_CTLZ_I32, "llvm.ctlz.i32"},
8481 {INTRINS_CTLZ_I64, "llvm.ctlz.i64"},
8482 {INTRINS_CTTZ_I32, "llvm.cttz.i32"},
8483 {INTRINS_CTTZ_I64, "llvm.cttz.i64"},
8484 {INTRINS_PEXT_I32, "llvm.x86.bmi.pext.32"},
8485 {INTRINS_PEXT_I64, "llvm.x86.bmi.pext.64"},
8486 {INTRINS_PDEP_I32, "llvm.x86.bmi.pdep.32"},
8487 {INTRINS_PDEP_I64, "llvm.x86.bmi.pdep.64"},
8488 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8489 {INTRINS_SSE_PMOVMSKB, "llvm.x86.sse2.pmovmskb.128"},
8490 {INTRINS_SSE_PSRLI_W, "llvm.x86.sse2.psrli.w"},
8491 {INTRINS_SSE_PSRAI_W, "llvm.x86.sse2.psrai.w"},
8492 {INTRINS_SSE_PSLLI_W, "llvm.x86.sse2.pslli.w"},
8493 {INTRINS_SSE_PSRLI_D, "llvm.x86.sse2.psrli.d"},
8494 {INTRINS_SSE_PSRAI_D, "llvm.x86.sse2.psrai.d"},
8495 {INTRINS_SSE_PSLLI_D, "llvm.x86.sse2.pslli.d"},
8496 {INTRINS_SSE_PSRLI_Q, "llvm.x86.sse2.psrli.q"},
8497 {INTRINS_SSE_PSLLI_Q, "llvm.x86.sse2.pslli.q"},
8498 {INTRINS_SSE_SQRT_PD, "llvm.x86.sse2.sqrt.pd"},
8499 {INTRINS_SSE_SQRT_PS, "llvm.x86.sse.sqrt.ps"},
8500 {INTRINS_SSE_RSQRT_PS, "llvm.x86.sse.rsqrt.ps"},
8501 {INTRINS_SSE_RCP_PS, "llvm.x86.sse.rcp.ps"},
8502 {INTRINS_SSE_CVTTPD2DQ, "llvm.x86.sse2.cvttpd2dq"},
8503 {INTRINS_SSE_CVTTPS2DQ, "llvm.x86.sse2.cvttps2dq"},
8504 {INTRINS_SSE_CVTDQ2PD, "llvm.x86.sse2.cvtdq2pd"},
8505 {INTRINS_SSE_CVTDQ2PS, "llvm.x86.sse2.cvtdq2ps"},
8506 {INTRINS_SSE_CVTPD2DQ, "llvm.x86.sse2.cvtpd2dq"},
8507 {INTRINS_SSE_CVTPS2DQ, "llvm.x86.sse2.cvtps2dq"},
8508 {INTRINS_SSE_CVTPD2PS, "llvm.x86.sse2.cvtpd2ps"},
8509 {INTRINS_SSE_CVTPS2PD, "llvm.x86.sse2.cvtps2pd"},
8510 {INTRINS_SSE_CMPPD, "llvm.x86.sse2.cmp.pd"},
8511 {INTRINS_SSE_CMPPS, "llvm.x86.sse.cmp.ps"},
8512 {INTRINS_SSE_PACKSSWB, "llvm.x86.sse2.packsswb.128"},
8513 {INTRINS_SSE_PACKUSWB, "llvm.x86.sse2.packuswb.128"},
8514 {INTRINS_SSE_PACKSSDW, "llvm.x86.sse2.packssdw.128"},
8515 {INTRINS_SSE_PACKUSDW, "llvm.x86.sse41.packusdw"},
8516 {INTRINS_SSE_MINPS, "llvm.x86.sse.min.ps"},
8517 {INTRINS_SSE_MAXPS, "llvm.x86.sse.max.ps"},
8518 {INTRINS_SSE_HADDPS, "llvm.x86.sse3.hadd.ps"},
8519 {INTRINS_SSE_HSUBPS, "llvm.x86.sse3.hsub.ps"},
8520 {INTRINS_SSE_ADDSUBPS, "llvm.x86.sse3.addsub.ps"},
8521 {INTRINS_SSE_MINPD, "llvm.x86.sse2.min.pd"},
8522 {INTRINS_SSE_MAXPD, "llvm.x86.sse2.max.pd"},
8523 {INTRINS_SSE_HADDPD, "llvm.x86.sse3.hadd.pd"},
8524 {INTRINS_SSE_HSUBPD, "llvm.x86.sse3.hsub.pd"},
8525 {INTRINS_SSE_ADDSUBPD, "llvm.x86.sse3.addsub.pd"},
8526 {INTRINS_SSE_PADDSW, "llvm.x86.sse2.padds.w"},
8527 {INTRINS_SSE_PSUBSW, "llvm.x86.sse2.psubs.w"},
8528 {INTRINS_SSE_PADDUSW, "llvm.x86.sse2.paddus.w"},
8529 {INTRINS_SSE_PSUBUSW, "llvm.x86.sse2.psubus.w"},
8530 {INTRINS_SSE_PAVGW, "llvm.x86.sse2.pavg.w"},
8531 {INTRINS_SSE_PMULHW, "llvm.x86.sse2.pmulh.w"},
8532 {INTRINS_SSE_PMULHU, "llvm.x86.sse2.pmulhu.w"},
8533 {INTRINS_SE_PADDSB, "llvm.x86.sse2.padds.b"},
8534 {INTRINS_SSE_PSUBSB, "llvm.x86.sse2.psubs.b"},
8535 {INTRINS_SSE_PADDUSB, "llvm.x86.sse2.paddus.b"},
8536 {INTRINS_SSE_PSUBUSB, "llvm.x86.sse2.psubus.b"},
8537 {INTRINS_SSE_PAVGB, "llvm.x86.sse2.pavg.b"},
8538 {INTRINS_SSE_PAUSE, "llvm.x86.sse2.pause"},
8539 {INTRINS_SSE_DPPS, "llvm.x86.sse41.dpps"},
8540 {INTRINS_SSE_ROUNDPD, "llvm.x86.sse41.round.pd"}
8541 #endif
8542 };
8543
8544 static void
8545 add_sse_binary (LLVMModuleRef module, const char *name, int type)
8546 {
8547 LLVMTypeRef ret_type = type_to_simd_type (type);
8548 AddFunc2 (module, name, ret_type, ret_type, ret_type);
8549 }
8550
8551 static void
8552 add_intrinsic (LLVMModuleRef module, int id)
8553 {
8554 const char *name;
8555 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8556 LLVMTypeRef ret_type, arg_types [16];
8557 #endif
8558
8559 name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
8560 g_assert (name);
8561
8562 switch (id) {
8563 case INTRINS_MEMSET: {
8564 #if LLVM_API_VERSION >= 900
8565 /* No alignment argument */
8566 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt1Type () };
8567
8568 AddFunc (module, name, LLVMVoidType (), params, 4);
8569 #else
8570 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
8571
8572 AddFunc (module, name, LLVMVoidType (), params, 5);
8573 #endif
8574 break;
8575 }
8576 case INTRINS_MEMCPY: {
8577 #if LLVM_API_VERSION >= 900
8578 /* No alignment argument */
8579 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt1Type () };
8580
8581 AddFunc (module, name, LLVMVoidType (), params, 4);
8582 #else
8583 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
8584
8585 AddFunc (module, name, LLVMVoidType (), params, 5);
8586 #endif
8587 break;
8588 }
8589 case INTRINS_SADD_OVF_I32:
8590 case INTRINS_UADD_OVF_I32:
8591 case INTRINS_SSUB_OVF_I32:
8592 case INTRINS_USUB_OVF_I32:
8593 case INTRINS_SMUL_OVF_I32:
8594 case INTRINS_UMUL_OVF_I32: {
8595 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
8596 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
8597 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
8598
8599 AddFunc (module, name, ret_type, params, 2);
8600 break;
8601 }
8602 case INTRINS_SADD_OVF_I64:
8603 case INTRINS_UADD_OVF_I64:
8604 case INTRINS_SSUB_OVF_I64:
8605 case INTRINS_USUB_OVF_I64:
8606 case INTRINS_SMUL_OVF_I64:
8607 case INTRINS_UMUL_OVF_I64: {
8608 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
8609 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
8610 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
8611
8612 AddFunc (module, name, ret_type, params, 2);
8613 break;
8614 }
8615 case INTRINS_SIN:
8616 case INTRINS_COS:
8617 case INTRINS_SQRT:
8618 case INTRINS_FABS: {
8619 LLVMTypeRef params [] = { LLVMDoubleType () };
8620
8621 AddFunc (module, name, LLVMDoubleType (), params, 1);
8622 break;
8623 }
8624 case INTRINS_SINF:
8625 case INTRINS_COSF:
8626 case INTRINS_SQRTF:
8627 case INTRINS_ABSF: {
8628 LLVMTypeRef params [] = { LLVMFloatType () };
8629
8630 AddFunc (module, name, LLVMFloatType (), params, 1);
8631 break;
8632 }
8633 case INTRINS_POWF: {
8634 LLVMTypeRef params [] = { LLVMFloatType (), LLVMFloatType () };
8635
8636 AddFunc (module, name, LLVMFloatType (), params, 2);
8637 break;
8638 }
8639 case INTRINS_EXPECT_I8:
8640 AddFunc2 (module, name, LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
8641 break;
8642 case INTRINS_EXPECT_I1:
8643 AddFunc2 (module, name, LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type ());
8644 break;
8645 case INTRINS_CTPOP_I32:
8646 AddFunc1 (module, name, LLVMInt32Type (), LLVMInt32Type ());
8647 break;
8648 case INTRINS_CTPOP_I64:
8649 AddFunc1 (module, name, LLVMInt64Type (), LLVMInt64Type ());
8650 break;
8651 case INTRINS_CTLZ_I32:
8652 case INTRINS_CTTZ_I32:
8653 AddFunc2 (module, name, LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type ());
8654 break;
8655 case INTRINS_CTLZ_I64:
8656 case INTRINS_CTTZ_I64:
8657 AddFunc2 (module, name, LLVMInt64Type (), LLVMInt64Type (), LLVMInt1Type ());
8658 break;
8659 case INTRINS_PEXT_I32:
8660 AddFunc2 (module, name, LLVMInt32Type (), LLVMInt32Type (), LLVMInt32Type ());
8661 break;
8662 case INTRINS_PEXT_I64:
8663 AddFunc2 (module, name, LLVMInt64Type (), LLVMInt64Type (), LLVMInt64Type ());
8664 break;
8665 case INTRINS_PDEP_I32:
8666 AddFunc2 (module, name, LLVMInt32Type (), LLVMInt32Type (), LLVMInt32Type ());
8667 break;
8668 case INTRINS_PDEP_I64:
8669 AddFunc2 (module, name, LLVMInt64Type (), LLVMInt64Type (), LLVMInt64Type ());
8670 break;
8671 #if defined(TARGET_AMD64) || defined(TARGET_X86)
8672 case INTRINS_SSE_PMOVMSKB:
8673 /* pmovmskb */
8674 ret_type = LLVMInt32Type ();
8675 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
8676 AddFunc (module, name, ret_type, arg_types, 1);
8677 break;
8678 case INTRINS_SSE_PSRLI_W:
8679 case INTRINS_SSE_PSRAI_W:
8680 case INTRINS_SSE_PSLLI_W:
8681 /* shifts */
8682 ret_type = type_to_simd_type (MONO_TYPE_I2);
8683 arg_types [0] = ret_type;
8684 arg_types [1] = LLVMInt32Type ();
8685 AddFunc (module, name, ret_type, arg_types, 2);
8686 break;
8687 case INTRINS_SSE_PSRLI_D:
8688 case INTRINS_SSE_PSRAI_D:
8689 case INTRINS_SSE_PSLLI_D:
8690 ret_type = type_to_simd_type (MONO_TYPE_I4);
8691 arg_types [0] = ret_type;
8692 arg_types [1] = LLVMInt32Type ();
8693 AddFunc (module, name, ret_type, arg_types, 2);
8694 break;
8695 case INTRINS_SSE_PSRLI_Q:
8696 case INTRINS_SSE_PSLLI_Q:
8697 ret_type = type_to_simd_type (MONO_TYPE_I8);
8698 arg_types [0] = ret_type;
8699 arg_types [1] = LLVMInt32Type ();
8700 AddFunc (module, name, ret_type, arg_types, 2);
8701 break;
8702 case INTRINS_SSE_SQRT_PD:
8703 /* Unary ops */
8704 ret_type = type_to_simd_type (MONO_TYPE_R8);
8705 arg_types [0] = ret_type;
8706 AddFunc (module, name, ret_type, arg_types, 1);
8707 break;
8708 case INTRINS_SSE_SQRT_PS:
8709 ret_type = type_to_simd_type (MONO_TYPE_R4);
8710 arg_types [0] = ret_type;
8711 AddFunc (module, name, ret_type, arg_types, 1);
8712 break;
8713 case INTRINS_SSE_RSQRT_PS:
8714 ret_type = type_to_simd_type (MONO_TYPE_R4);
8715 arg_types [0] = ret_type;
8716 AddFunc (module, name, ret_type, arg_types, 1);
8717 break;
8718 case INTRINS_SSE_RCP_PS:
8719 ret_type = type_to_simd_type (MONO_TYPE_R4);
8720 arg_types [0] = ret_type;
8721 AddFunc (module, name, ret_type, arg_types, 1);
8722 break;
8723 case INTRINS_SSE_CVTTPD2DQ:
8724 ret_type = type_to_simd_type (MONO_TYPE_I4);
8725 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8726 AddFunc (module, name, ret_type, arg_types, 1);
8727 break;
8728 case INTRINS_SSE_CVTTPS2DQ:
8729 ret_type = type_to_simd_type (MONO_TYPE_I4);
8730 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8731 AddFunc (module, name, ret_type, arg_types, 1);
8732 break;
8733 case INTRINS_SSE_CVTDQ2PD:
8734 /* Conversion ops */
8735 ret_type = type_to_simd_type (MONO_TYPE_R8);
8736 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8737 AddFunc (module, name, ret_type, arg_types, 1);
8738 break;
8739 case INTRINS_SSE_CVTDQ2PS:
8740 ret_type = type_to_simd_type (MONO_TYPE_R4);
8741 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8742 AddFunc (module, name, ret_type, arg_types, 1);
8743 break;
8744 case INTRINS_SSE_CVTPD2DQ:
8745 ret_type = type_to_simd_type (MONO_TYPE_I4);
8746 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8747 AddFunc (module, name, ret_type, arg_types, 1);
8748 break;
8749 case INTRINS_SSE_CVTPS2DQ:
8750 ret_type = type_to_simd_type (MONO_TYPE_I4);
8751 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8752 AddFunc (module, name, ret_type, arg_types, 1);
8753 break;
8754 case INTRINS_SSE_CVTPD2PS:
8755 ret_type = type_to_simd_type (MONO_TYPE_R4);
8756 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8757 AddFunc (module, name, ret_type, arg_types, 1);
8758 break;
8759 case INTRINS_SSE_CVTPS2PD:
8760 ret_type = type_to_simd_type (MONO_TYPE_R8);
8761 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8762 AddFunc (module, name, ret_type, arg_types, 1);
8763 break;
8764 case INTRINS_SSE_CMPPD:
8765 /* cmp pd/ps */
8766 ret_type = type_to_simd_type (MONO_TYPE_R8);
8767 arg_types [0] = ret_type;
8768 arg_types [1] = ret_type;
8769 arg_types [2] = LLVMInt8Type ();
8770 AddFunc (module, name, ret_type, arg_types, 3);
8771 break;
8772 case INTRINS_SSE_CMPPS:
8773 ret_type = type_to_simd_type (MONO_TYPE_R4);
8774 arg_types [0] = ret_type;
8775 arg_types [1] = ret_type;
8776 arg_types [2] = LLVMInt8Type ();
8777 AddFunc (module, name, ret_type, arg_types, 3);
8778 break;
8779 case INTRINS_SSE_PACKSSWB:
8780 case INTRINS_SSE_PACKUSWB:
8781 case INTRINS_SSE_PACKSSDW:
8782 /* pack */
8783 ret_type = type_to_simd_type (MONO_TYPE_I1);
8784 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
8785 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
8786 AddFunc (module, name, ret_type, arg_types, 2);
8787 break;
8788 case INTRINS_SSE_PACKUSDW:
8789 ret_type = type_to_simd_type (MONO_TYPE_I2);
8790 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8791 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
8792 AddFunc (module, name, ret_type, arg_types, 2);
8793 break;
8794 /* SSE Binary ops */
8795 case INTRINS_SSE_PADDSW:
8796 case INTRINS_SSE_PSUBSW:
8797 case INTRINS_SSE_PADDUSW:
8798 case INTRINS_SSE_PSUBUSW:
8799 case INTRINS_SSE_PAVGW:
8800 case INTRINS_SSE_PMULHW:
8801 case INTRINS_SSE_PMULHU:
8802 add_sse_binary (module, name, MONO_TYPE_I2);
8803 break;
8804 case INTRINS_SSE_MINPS:
8805 case INTRINS_SSE_MAXPS:
8806 case INTRINS_SSE_HADDPS:
8807 case INTRINS_SSE_HSUBPS:
8808 case INTRINS_SSE_ADDSUBPS:
8809 add_sse_binary (module, name, MONO_TYPE_R4);
8810 break;
8811 case INTRINS_SSE_MINPD:
8812 case INTRINS_SSE_MAXPD:
8813 case INTRINS_SSE_HADDPD:
8814 case INTRINS_SSE_HSUBPD:
8815 case INTRINS_SSE_ADDSUBPD:
8816 add_sse_binary (module, name, MONO_TYPE_R8);
8817 break;
8818 case INTRINS_SE_PADDSB:
8819 case INTRINS_SSE_PSUBSB:
8820 case INTRINS_SSE_PADDUSB:
8821 case INTRINS_SSE_PSUBUSB:
8822 case INTRINS_SSE_PAVGB:
8823 add_sse_binary (module, name, MONO_TYPE_I1);
8824 break;
8825 case INTRINS_SSE_PAUSE:
8826 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
8827 break;
8828 case INTRINS_SSE_DPPS:
8829 ret_type = type_to_simd_type (MONO_TYPE_R4);
8830 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8831 arg_types [1] = type_to_simd_type (MONO_TYPE_R4);
8832 arg_types [2] = LLVMInt8Type ();
8833 AddFunc (module, name, ret_type, arg_types, 3);
8834 break;
8835 case INTRINS_SSE_ROUNDPD:
8836 ret_type = type_to_simd_type (MONO_TYPE_R8);
8837 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8838 arg_types [1] = LLVMInt32Type ();
8839 AddFunc (module, name, ret_type, arg_types, 2);
8840 break;
8841 #endif /* AMD64 || X86 */
8842 default:
8843 g_assert_not_reached ();
8844 break;
8845 }
8846 }
8847
8848 static LLVMValueRef
8849 get_intrins_from_module (LLVMModuleRef lmodule, int id)
8850 {
8851 LLVMValueRef res;
8852
8853 const char *name = (const char*)g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
8854 g_assert (name);
8855
8856 res = LLVMGetNamedFunction (lmodule, name);
8857 if (!res) {
8858 add_intrinsic (lmodule, id);
8859 res = LLVMGetNamedFunction (lmodule, name);
8860 g_assert (res);
8861 }
8862 return res;
8863 }
8864
8865 static LLVMValueRef
8866 get_intrins (EmitContext *ctx, int id)
8867 {
8868 MonoLLVMModule *module = ctx->module;
8869 LLVMValueRef res;
8870
8871 /*
8872 * Every method is emitted into its own module so
8873 * we can add intrinsics on demand.
8874 */
8875 res = module->intrins_by_id [id];
8876 if (!res) {
8877 res = get_intrins_from_module (ctx->lmodule, id);
8878 module->intrins_by_id [id] = res;
8879 }
8880 return res;
8881 }
8882
8883 static LLVMValueRef
8884 get_intrins_by_name (EmitContext *ctx, const char *name)
8885 {
8886 LLVMValueRef res;
8887
8888 /*
8889 * Every method is emitted into its own module so
8890 * we can add intrinsics on demand.
8891 */
8892 res = LLVMGetNamedFunction (ctx->lmodule, name);
8893 if (!res) {
8894 int id = -1;
8895
8896 /* No locking needed */
8897 id = GPOINTER_TO_INT (g_hash_table_lookup (intrins_name_to_id, name));
8898 id --;
8899 if (id == -1)
8900 printf ("%s\n", name);
8901 g_assert (id != -1);
8902 add_intrinsic (ctx->lmodule, id);
8903 res = LLVMGetNamedFunction (ctx->lmodule, name);
8904 g_assert (res);
8905 }
8906
8907 return res;
8908 }
8909
8910 static void
8911 add_intrinsics (LLVMModuleRef module)
8912 {
8913 int i;
8914
8915 /* Emit declarations of instrinsics */
8916 /*
8917 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
8918 * type doesn't seem to do any locking.
8919 */
8920 for (i = 0; i < INTRINS_NUM; ++i)
8921 add_intrinsic (module, i);
8922
8923 /* EH intrinsics */
8924 {
8925 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
8926
8927 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
8928 }
8929
8930 /* Load/Store intrinsics */
8931 {
8932 LLVMTypeRef arg_types [5];
8933 int i;
8934 char name [128];
8935
8936 for (i = 1; i <= 8; i *= 2) {
8937 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
8938 arg_types [1] = LLVMInt32Type ();
8939 arg_types [2] = LLVMInt1Type ();
8940 arg_types [3] = LLVMInt32Type ();
8941 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
8942 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
8943
8944 arg_types [0] = LLVMIntType (i * 8);
8945 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
8946 arg_types [2] = LLVMInt32Type ();
8947 arg_types [3] = LLVMInt1Type ();
8948 arg_types [4] = LLVMInt32Type ();
8949 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
8950 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
8951 }
8952 }
8953 }
8954
8955 static void
8956 add_types (MonoLLVMModule *module)
8957 {
8958 module->ptr_type = LLVMPointerType (TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
8959 }
8960
8961 void
8962 mono_llvm_init (void)
8963 {
8964 GHashTable *h;
8965 int i;
8966
8967 mono_native_tls_alloc (&current_cfg_tls_id, NULL);
8968
8969 h = g_hash_table_new (NULL, NULL);
8970 for (i = 0; i < INTRINS_NUM; ++i)
8971 g_hash_table_insert (h, GINT_TO_POINTER (intrinsics [i].id), (gpointer)intrinsics [i].name);
8972 intrins_id_to_name = h;
8973
8974 h = g_hash_table_new (g_str_hash, g_str_equal);
8975 for (i = 0; i < INTRINS_NUM; ++i)
8976 g_hash_table_insert (h, (gpointer)intrinsics [i].name, GINT_TO_POINTER (intrinsics [i].id + 1));
8977 intrins_name_to_id = h;
8978 }
8979
8980 void
8981 mono_llvm_cleanup (void)
8982 {
8983 MonoLLVMModule *module = &aot_module;
8984
8985 if (module->lmodule)
8986 LLVMDisposeModule (module->lmodule);
8987
8988 if (module->context)
8989 LLVMContextDispose (module->context);
8990 }
8991
8992 void
8993 mono_llvm_free_domain_info (MonoDomain *domain)
8994 {
8995 MonoJitDomainInfo *info = domain_jit_info (domain);
8996 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
8997 int i;
8998
8999 if (!module)
9000 return;
9001
9002 g_hash_table_destroy (module->llvm_types);
9003
9004 mono_llvm_dispose_ee (module->mono_ee);
9005
9006 if (module->bb_names) {
9007 for (i = 0; i < module->bb_names_len; ++i)
9008 g_free (module->bb_names [i]);
9009 g_free (module->bb_names);
9010 }
9011 //LLVMDisposeModule (module->module);
9012
9013 g_free (module);
9014
9015 info->llvm_module = NULL;
9016 }
9017
9018 void
9019 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, int initial_got_size, LLVMModuleFlags flags)
9020 {
9021 MonoLLVMModule *module = &aot_module;
9022 gboolean emit_dwarf = (flags & LLVM_MODULE_FLAG_DWARF) ? 1 : 0;
9023 #ifdef TARGET_WIN32_MSVC
9024 gboolean emit_codeview = (flags & LLVM_MODULE_FLAG_CODEVIEW) ? 1 : 0;
9025 #endif
9026 gboolean static_link = (flags & LLVM_MODULE_FLAG_STATIC) ? 1 : 0;
9027 gboolean llvm_only = (flags & LLVM_MODULE_FLAG_LLVM_ONLY) ? 1 : 0;
9028 gboolean interp = (flags & LLVM_MODULE_FLAG_INTERP) ? 1 : 0;
9029 gboolean llvm_disable_self_init = mini_get_debug_options ()->llvm_disable_self_init;
9030
9031 /* Delete previous module */
9032 g_hash_table_destroy (module->plt_entries);
9033 if (module->lmodule)
9034 LLVMDisposeModule (module->lmodule);
9035
9036 memset (module, 0, sizeof (aot_module));
9037
9038 module->lmodule = LLVMModuleCreateWithName ("aot");
9039 module->assembly = assembly;
9040 module->global_prefix = g_strdup (global_prefix);
9041 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
9042 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
9043 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
9044 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
9045 module->external_symbols = TRUE;
9046 module->emit_dwarf = emit_dwarf;
9047 module->static_link = static_link;
9048 module->llvm_only = llvm_only;
9049 module->llvm_disable_self_init = llvm_disable_self_init && !llvm_only; // llvm_only implies !llvm_disable_self_init
9050 module->interp = interp;
9051 /* The first few entries are reserved */
9052 module->max_got_offset = initial_got_size;
9053 module->context = LLVMGetGlobalContext ();
9054 module->cfgs = g_ptr_array_new ();
9055 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
9056
9057 if (llvm_only)
9058 /* clang ignores our debug info because it has an invalid version */
9059 module->emit_dwarf = FALSE;
9060
9061 add_intrinsics (module->lmodule);
9062 add_types (module);
9063
9064 #ifdef MONO_ARCH_LLVM_TARGET_LAYOUT
9065 LLVMSetDataLayout (module->lmodule, MONO_ARCH_LLVM_TARGET_LAYOUT);
9066 #else
9067 g_assert_not_reached ();
9068 #endif
9069
9070 #ifdef MONO_ARCH_LLVM_TARGET_TRIPLE
9071 LLVMSetTarget (module->lmodule, MONO_ARCH_LLVM_TARGET_TRIPLE);
9072 #endif
9073
9074 if (module->emit_dwarf) {
9075 char *dir, *build_info, *s, *cu_name;
9076
9077 module->di_builder = mono_llvm_create_di_builder (module->lmodule);
9078
9079 // FIXME:
9080 dir = g_strdup (".");
9081 build_info = mono_get_runtime_build_info ();
9082 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
9083 cu_name = g_path_get_basename (assembly->image->name);
9084 module->cu = mono_llvm_di_create_compile_unit (module->di_builder, cu_name, dir, s);
9085 g_free (dir);
9086 g_free (build_info);
9087 g_free (s);
9088 }
9089
9090 #ifdef TARGET_WIN32_MSVC
9091 if (emit_codeview) {
9092 LLVMValueRef codeview_option_args[3];
9093
9094 codeview_option_args[0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9095 codeview_option_args[1] = LLVMMDString ("CodeView", 8);
9096 codeview_option_args[2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
9097
9098 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.module.flags", LLVMMDNode (codeview_option_args, G_N_ELEMENTS (codeview_option_args)));
9099 }
9100
9101 if (!static_link) {
9102 const char linker_options[] = "Linker Options";
9103 const char *default_dynamic_lib_names[] = { "/DEFAULTLIB:msvcrt",
9104 "/DEFAULTLIB:ucrt.lib",
9105 "/DEFAULTLIB:vcruntime.lib" };
9106
9107 LLVMValueRef default_lib_args[G_N_ELEMENTS (default_dynamic_lib_names)];
9108 LLVMValueRef default_lib_nodes[G_N_ELEMENTS(default_dynamic_lib_names)];
9109
9110 const char *default_lib_name = NULL;
9111 for (int i = 0; i < G_N_ELEMENTS (default_dynamic_lib_names); ++i) {
9112 const char *default_lib_name = default_dynamic_lib_names[i];
9113 default_lib_args[i] = LLVMMDString (default_lib_name, strlen (default_lib_name));
9114 default_lib_nodes[i] = LLVMMDNode (default_lib_args + i, 1);
9115 }
9116
9117 LLVMAddNamedMetadataOperand (module->lmodule, "llvm.linker.options", LLVMMDNode (default_lib_args, G_N_ELEMENTS (default_lib_args)));
9118 }
9119 #endif
9120
9121 /* Add GOT */
9122 /*
9123 * We couldn't compute the type of the LLVM global representing the got because
9124 * its size is only known after all the methods have been emitted. So create
9125 * a dummy variable, and replace all uses it with the real got variable when
9126 * its size is known in mono_llvm_emit_aot_module ().
9127 */
9128 {
9129 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
9130
9131 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
9132 module->got_idx_to_type = g_hash_table_new (NULL, NULL);
9133 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
9134 }
9135
9136 /* Add initialization array */
9137 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
9138
9139 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
9140 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
9141
9142
9143 emit_gc_safepoint_poll (module);
9144
9145 emit_llvm_code_start (module);
9146
9147 // Needs idx_to_lmethod
9148 emit_init_icall_wrappers (module);
9149
9150 /* Add a dummy personality function */
9151 if (!use_mono_personality_debug) {
9152 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
9153 LLVMSetLinkage (personality, LLVMExternalLinkage);
9154
9155 //EMCC chockes if the personality function is referenced in the 'used' array
9156 #ifndef TARGET_WASM
9157 mark_as_used (module, personality);
9158 #endif
9159 }
9160
9161 /* Add a reference to the c++ exception we throw/catch */
9162 {
9163 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
9164 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
9165 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
9166 mono_llvm_set_is_constant (module->sentinel_exception);
9167 }
9168
9169 module->llvm_types = g_hash_table_new (NULL, NULL);
9170 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
9171 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
9172 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
9173 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
9174 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
9175 module->method_to_call_info = g_hash_table_new (NULL, NULL);
9176 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
9177 module->callsite_list = g_ptr_array_new ();
9178 }
9179
9180 void
9181 mono_llvm_fixup_aot_module (void)
9182 {
9183 MonoLLVMModule *module = &aot_module;
9184 MonoMethod *method;
9185
9186 if (module->llvm_disable_self_init)
9187 return;
9188
9189 /*
9190 * Replace GOT entries for directly callable methods with the methods themselves.
9191 * It would be easier to implement this by predefining all methods before compiling
9192 * their bodies, but that couldn't handle the case when a method fails to compile
9193 * with llvm.
9194 */
9195
9196 GHashTable *specializable = g_hash_table_new (NULL, NULL);
9197
9198 GHashTable *patches_to_null = g_hash_table_new (mono_patch_info_hash, mono_patch_info_equal);
9199 for (int sindex = 0; sindex < module->callsite_list->len; ++sindex) {
9200 CallSite *site = (CallSite*)g_ptr_array_index (module->callsite_list, sindex);
9201 method = site->method;
9202 LLVMValueRef lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
9203
9204 LLVMValueRef placeholder = (LLVMValueRef)site->load;
9205 LLVMValueRef indexes [2], got_entry_addr, load;
9206 char *name;
9207
9208 if (lmethod && !(method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)) {
9209 mono_llvm_replace_uses_of (placeholder, lmethod);
9210
9211 if (mono_aot_can_specialize (method))
9212 g_hash_table_insert (specializable, lmethod, method);
9213
9214 g_hash_table_insert (patches_to_null, site->ji, site->ji);
9215 } else {
9216 int got_offset = compute_aot_got_offset (module, site->ji, site->type);
9217
9218 module->max_got_offset = MAX (module->max_got_offset, got_offset);
9219
9220 LLVMBuilderRef builder = LLVMCreateBuilder ();
9221 LLVMPositionBuilderBefore (builder, placeholder);
9222 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
9223 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
9224 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
9225
9226 name = get_aotconst_name (site->ji->type, site->ji->data.target, got_offset);
9227 load = LLVMBuildLoad (builder, got_entry_addr, "");
9228 load = LLVMBuildBitCast (builder, load, site->type, name ? name : "");
9229 LLVMReplaceAllUsesWith (placeholder, load);
9230 }
9231 g_free (site);
9232 }
9233
9234 mono_llvm_propagate_nonnull_final (specializable, module);
9235 g_hash_table_destroy (specializable);
9236
9237 for (int i = 0; i < module->cfgs->len; ++i) {
9238 /*
9239 * Nullify the patches pointing to direct calls. This is needed to
9240 * avoid allocating extra got slots, which is a perf problem and it
9241 * makes module->max_got_offset invalid.
9242 * It would be better to just store the patch_info in CallSite, but
9243 * cfg->patch_info is copied in aot-compiler.c.
9244 */
9245 MonoCompile *cfg = (MonoCompile *)g_ptr_array_index (module->cfgs, i);
9246 for (MonoJumpInfo *patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
9247 if (patch_info->type == MONO_PATCH_INFO_METHOD) {
9248 if (g_hash_table_lookup (patches_to_null, patch_info)) {
9249 patch_info->type = MONO_PATCH_INFO_NONE;
9250 /* Nullify the call to init_method () if possible */
9251 g_assert (cfg->got_access_count);
9252 cfg->got_access_count --;
9253 if (cfg->got_access_count == 0) {
9254 LLVMValueRef br = (LLVMValueRef)cfg->llvmonly_init_cond;
9255 if (br)
9256 LLVMSetSuccessor (br, 0, LLVMGetSuccessor (br, 1));
9257 }
9258 }
9259 }
9260 }
9261 }
9262
9263 g_hash_table_destroy (patches_to_null);
9264 }
9265
9266 static LLVMValueRef
9267 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
9268 {
9269 int i;
9270 LLVMValueRef res, *vals;
9271
9272 vals = g_new0 (LLVMValueRef, nvalues);
9273 for (i = 0; i < nvalues; ++i)
9274 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
9275 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
9276 g_free (vals);
9277 return res;
9278 }
9279
9280 static LLVMValueRef
9281 llvm_array_from_bytes (guint8 *values, int nvalues)
9282 {
9283 int i;
9284 LLVMValueRef res, *vals;
9285
9286 vals = g_new0 (LLVMValueRef, nvalues);
9287 for (i = 0; i < nvalues; ++i)
9288 vals [i] = LLVMConstInt (LLVMInt8Type (), values [i], FALSE);
9289 res = LLVMConstArray (LLVMInt8Type (), vals, nvalues);
9290 g_free (vals);
9291 return res;
9292 }
9293 /*
9294 * mono_llvm_emit_aot_file_info:
9295 *
9296 * Emit the MonoAotFileInfo structure.
9297 * Same as emit_aot_file_info () in aot-compiler.c.
9298 */
9299 void
9300 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
9301 {
9302 MonoLLVMModule *module = &aot_module;
9303
9304 /* Save these for later */
9305 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
9306 module->has_jitted_code = has_jitted_code;
9307 }
9308
9309 /*
9310 * mono_llvm_emit_aot_data:
9311 *
9312 * Emit the binary data DATA pointed to by symbol SYMBOL.
9313 */
9314 void
9315 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
9316 {
9317 MonoLLVMModule *module = &aot_module;
9318 LLVMTypeRef type;
9319 LLVMValueRef d;
9320
9321 type = LLVMArrayType (LLVMInt8Type (), data_len);
9322 d = LLVMAddGlobal (module->lmodule, type, symbol);
9323 LLVMSetVisibility (d, LLVMHiddenVisibility);
9324 LLVMSetLinkage (d, LLVMInternalLinkage);
9325 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
9326 LLVMSetAlignment (d, 8);
9327 mono_llvm_set_is_constant (d);
9328 }
9329
9330 /* Add a reference to a global defined in JITted code */
9331 static LLVMValueRef
9332 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
9333 {
9334 char *s;
9335 LLVMValueRef v;
9336
9337 s = g_strdup_printf ("%s%s", module->global_prefix, name);
9338 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
9339 LLVMSetVisibility (v, LLVMHiddenVisibility);
9340 g_free (s);
9341 return v;
9342 }
9343
9344 static void
9345 emit_aot_file_info (MonoLLVMModule *module)
9346 {
9347 LLVMTypeRef file_info_type;
9348 LLVMTypeRef *eltypes, eltype;
9349 LLVMValueRef info_var;
9350 LLVMValueRef *fields;
9351 int i, nfields, tindex;
9352 MonoAotFileInfo *info;
9353 LLVMModuleRef lmodule = module->lmodule;
9354
9355 info = &module->aot_info;
9356
9357 /* Create an LLVM type to represent MonoAotFileInfo */
9358 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 21 + 5;
9359 eltypes = g_new (LLVMTypeRef, nfields);
9360 tindex = 0;
9361 eltypes [tindex ++] = LLVMInt32Type ();
9362 eltypes [tindex ++] = LLVMInt32Type ();
9363 /* Symbols */
9364 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
9365 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
9366 /* Scalars */
9367 for (i = 0; i < 20; ++i)
9368 eltypes [tindex ++] = LLVMInt32Type ();
9369 /* Arrays */
9370 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
9371 for (i = 0; i < 4; ++i)
9372 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
9373 eltypes [tindex ++] = LLVMArrayType (LLVMInt8Type (), 16);
9374 g_assert (tindex == nfields);
9375 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
9376 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
9377
9378 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
9379 if (module->static_link) {
9380 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
9381 LLVMSetLinkage (info_var, LLVMInternalLinkage);
9382 }
9383
9384 #ifdef TARGET_WIN32
9385 if (!module->static_link) {
9386 LLVMSetDLLStorageClass (info_var, LLVMDLLExportStorageClass);
9387 }
9388 #endif
9389
9390 fields = g_new (LLVMValueRef, nfields);
9391 tindex = 0;
9392 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
9393 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
9394
9395 /* Symbols */
9396 /*
9397 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
9398 * for symbols defined in the .s file emitted by the aot compiler.
9399 */
9400 eltype = eltypes [tindex];
9401 if (module->llvm_only)
9402 fields [tindex ++] = LLVMConstNull (eltype);
9403 else
9404 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
9405 fields [tindex ++] = module->got_var;
9406 /* llc defines this directly */
9407 if (!module->llvm_only) {
9408 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
9409 fields [tindex ++] = LLVMConstNull (eltype);
9410 fields [tindex ++] = LLVMConstNull (eltype);
9411 } else {
9412 fields [tindex ++] = LLVMConstNull (eltype);
9413 fields [tindex ++] = module->get_method;
9414 fields [tindex ++] = module->get_unbox_tramp ? module->get_unbox_tramp : LLVMConstNull (eltype);
9415 }
9416 if (module->has_jitted_code) {
9417 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
9418 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
9419 } else {
9420 fields [tindex ++] = LLVMConstNull (eltype);
9421 fields [tindex ++] = LLVMConstNull (eltype);
9422 }
9423 if (!module->llvm_only)
9424 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
9425 else
9426 fields [tindex ++] = LLVMConstNull (eltype);
9427 if (module->llvm_only && module->unbox_tramp_indexes) {
9428 fields [tindex ++] = module->unbox_tramp_indexes;
9429 fields [tindex ++] = module->unbox_trampolines;
9430 } else {
9431 fields [tindex ++] = LLVMConstNull (eltype);
9432 fields [tindex ++] = LLVMConstNull (eltype);
9433 }
9434 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
9435 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
9436 fields [tindex ++] = LLVMConstNull (eltype);
9437 } else {
9438 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
9439 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
9440 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
9441 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
9442 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
9443 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
9444 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
9445 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
9446 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
9447 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
9448 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "weak_field_indexes");
9449 }
9450 /* Not needed (mem_end) */
9451 fields [tindex ++] = LLVMConstNull (eltype);
9452 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
9453 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
9454 if (info->trampoline_size [0]) {
9455 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
9456 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
9457 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_trampolines");
9458 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
9459 fields [tindex ++] = AddJitGlobal (module, eltype, "ftnptr_arg_trampolines");
9460 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_arbitrary_trampolines");
9461 } else {
9462 fields [tindex ++] = LLVMConstNull (eltype);
9463 fields [tindex ++] = LLVMConstNull (eltype);
9464 fields [tindex ++] = LLVMConstNull (eltype);
9465 fields [tindex ++] = LLVMConstNull (eltype);
9466 fields [tindex ++] = LLVMConstNull (eltype);
9467 fields [tindex ++] = LLVMConstNull (eltype);
9468 }
9469 if (module->static_link && !module->llvm_only)
9470 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
9471 else
9472 fields [tindex ++] = LLVMConstNull (eltype);
9473 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
9474 if (!module->llvm_only) {
9475 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
9476 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
9477 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
9478 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
9479 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
9480 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
9481 } else {
9482 fields [tindex ++] = LLVMConstNull (eltype);
9483 fields [tindex ++] = LLVMConstNull (eltype);
9484 fields [tindex ++] = LLVMConstNull (eltype);
9485 fields [tindex ++] = LLVMConstNull (eltype);
9486 fields [tindex ++] = LLVMConstNull (eltype);
9487 fields [tindex ++] = LLVMConstNull (eltype);
9488 }
9489
9490 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i) {
9491 g_assert (fields [2 + i]);
9492 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
9493 }
9494
9495 /* Scalars */
9496 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
9497 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
9498 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
9499 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
9500 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nextra_methods, FALSE);
9501 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
9502 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
9503 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
9504 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
9505 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
9506 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
9507 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
9508 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
9509 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_shift_bits, FALSE);
9510 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->card_table_mask, FALSE);
9511 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
9512 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
9513 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
9514 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_num, FALSE);
9515 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), module->unbox_tramp_elemsize, FALSE);
9516 /* Arrays */
9517 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
9518 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
9519 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
9520 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
9521 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
9522
9523 fields [tindex ++] = llvm_array_from_bytes (info->aotid, 16);
9524 g_assert (tindex == nfields);
9525
9526 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
9527
9528 if (module->static_link) {
9529 char *s, *p;
9530 LLVMValueRef var;
9531
9532 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
9533 /* Get rid of characters which cannot occur in symbols */
9534 p = s;
9535 for (p = s; *p; ++p) {
9536 if (!(isalnum (*p) || *p == '_'))
9537 *p = '_';
9538 }
9539 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
9540 g_free (s);
9541 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
9542 LLVMSetLinkage (var, LLVMExternalLinkage);
9543 }
9544 }
9545
9546 typedef struct {
9547 LLVMValueRef lmethod;
9548 int argument;
9549 } NonnullPropWorkItem;
9550
9551 static void
9552 mono_llvm_nonnull_state_update (EmitContext *ctx, LLVMValueRef lcall, MonoMethod *call_method, LLVMValueRef *args, int num_params)
9553 {
9554 if (!ctx->module->llvm_disable_self_init && mono_aot_can_specialize (call_method)) {
9555 int num_passed = LLVMGetNumArgOperands (lcall);
9556 g_assert (num_params <= num_passed);
9557
9558 g_assert (ctx->module->method_to_call_info);
9559 GArray *call_site_union = (GArray *) g_hash_table_lookup (ctx->module->method_to_call_info, call_method);
9560
9561 if (!call_site_union) {
9562 call_site_union = g_array_sized_new (FALSE, TRUE, sizeof (gint32), num_params);
9563 int zero = 0;
9564 for (int i = 0; i < num_params; i++)
9565 g_array_insert_val (call_site_union, i, zero);
9566 }
9567
9568 for (int i = 0; i < num_params; i++) {
9569 if (mono_llvm_is_nonnull (args [i])) {
9570 g_assert (i < LLVMGetNumArgOperands (lcall));
9571 mono_llvm_set_call_nonnull_arg (lcall, i);
9572 } else {
9573 gint32 *nullable_count = &g_array_index (call_site_union, gint32, i);
9574 *nullable_count = *nullable_count + 1;
9575 }
9576 }
9577
9578 g_hash_table_insert (ctx->module->method_to_call_info, call_method, call_site_union);
9579 }
9580 }
9581
9582 static void
9583 mono_llvm_propagate_nonnull_final (GHashTable *all_specializable, MonoLLVMModule *module)
9584 {
9585 // When we first traverse the mini IL, we mark the things that are
9586 // nonnull (the roots). Then, for all of the methods that can be specialized, we
9587 // see if their call sites have nonnull attributes.
9588
9589 // If so, we mark the function's param. This param has uses to propagate
9590 // the attribute to. This propagation can trigger a need to mark more attributes
9591 // non-null, and so on and so forth.
9592 GSList *queue = NULL;
9593
9594 GHashTableIter iter;
9595 LLVMValueRef lmethod;
9596 MonoMethod *method;
9597 g_hash_table_iter_init (&iter, all_specializable);
9598 while (g_hash_table_iter_next (&iter, (void**)&lmethod, (void**)&method)) {
9599 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, method);
9600
9601 // Basic sanity checking
9602 if (call_site_union)
9603 g_assert (call_site_union->len == LLVMCountParams (lmethod));
9604
9605 // Add root to work queue
9606 for (int i = 0; call_site_union && i < call_site_union->len; i++) {
9607 if (g_array_index (call_site_union, gint32, i) == 0) {
9608 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
9609 item->lmethod = lmethod;
9610 item->argument = i;
9611 queue = g_slist_prepend (queue, item);
9612 }
9613 }
9614 }
9615
9616 // This is essentially reference counting, and we are propagating
9617 // the refcount decrement here. We have less work to do than we may otherwise
9618 // because we are only working with a set of subgraphs of specializable functions.
9619 //
9620 // We rely on being able to see all of the references in the graph.
9621 // This is ensured by the function mono_aot_can_specialize. Everything in
9622 // all_specializable is a function that can be specialized, and is the resulting
9623 // node in the graph after all of the subsitutions are done.
9624 //
9625 // Anything disrupting the direct calls made with self-init will break this optimization.
9626
9627 while (queue) {
9628 // Update the queue state.
9629 // Our only other per-iteration responsibility is now to free current
9630 NonnullPropWorkItem *current = (NonnullPropWorkItem *) queue->data;
9631 queue = queue->next;
9632 g_assert (current->argument < LLVMCountParams (current->lmethod));
9633
9634 // Does the actual leaf-node work here
9635 // Mark the function argument as nonnull for LLVM
9636 mono_llvm_set_func_nonnull_arg (current->lmethod, current->argument);
9637
9638 // The rest of this is for propagating forward nullability changes
9639 // to calls that use the argument that is now nullable.
9640
9641 // Get the actual LLVM value of the argument, so we can see which call instructions
9642 // used that argument
9643 LLVMValueRef caller_argument = LLVMGetParam (current->lmethod, current->argument);
9644
9645 // Iterate over the calls using the newly-non-nullable argument
9646 GSList *calls = mono_llvm_calls_using (caller_argument);
9647 for (GSList *cursor = calls; cursor != NULL; cursor = cursor->next) {
9648
9649 LLVMValueRef lcall = (LLVMValueRef) cursor->data;
9650 LLVMValueRef callee_lmethod = LLVMGetCalledValue (lcall);
9651
9652 // If this wasn't a direct call for which mono_aot_can_specialize is true,
9653 // this lookup won't find a MonoMethod.
9654 MonoMethod *callee_method = (MonoMethod *) g_hash_table_lookup (all_specializable, callee_lmethod);
9655 if (!callee_method)
9656 continue;
9657
9658 // Decrement number of nullable refs at that func's arg offset
9659 GArray *call_site_union = (GArray *) g_hash_table_lookup (module->method_to_call_info, callee_method);
9660
9661 // It has module-local callers and is specializable, should have seen this call site
9662 // and inited this
9663 g_assert (call_site_union);
9664
9665 // The function *definition* parameter arity should always be consistent
9666 int max_params = LLVMCountParams (callee_lmethod);
9667 if (call_site_union->len != max_params) {
9668 mono_llvm_dump_value (callee_lmethod);
9669 g_assert_not_reached ();
9670 }
9671
9672 // Get the values that correspond to the parameters passed to the call
9673 // that used our argument
9674 LLVMValueRef *operands = mono_llvm_call_args (lcall);
9675 for (int call_argument = 0; call_argument < max_params; call_argument++) {
9676 // Every time we used the newly-non-nullable argument, decrement the nullable
9677 // refcount for that function.
9678 if (caller_argument == operands [call_argument]) {
9679 gint32 *nullable_count = &g_array_index (call_site_union, gint32, call_argument);
9680 g_assert (*nullable_count > 0);
9681 *nullable_count = *nullable_count - 1;
9682
9683 // If we caused that callee's parameter to become newly nullable, add to work queue
9684 if (*nullable_count == 0) {
9685 NonnullPropWorkItem *item = g_malloc (sizeof (NonnullPropWorkItem));
9686 item->lmethod = callee_lmethod;
9687 item->argument = call_argument;
9688 queue = g_slist_prepend (queue, item);
9689 }
9690 }
9691 }
9692 g_free (operands);
9693
9694 // Update nullability refcount information for the callee now
9695 g_hash_table_insert (module->method_to_call_info, callee_method, call_site_union);
9696 }
9697 g_slist_free (calls);
9698
9699 g_free (current);
9700 }
9701 }
9702
9703 /*
9704 * Emit the aot module into the LLVM bitcode file FILENAME.
9705 */
9706 void
9707 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
9708 {
9709 LLVMTypeRef got_type, inited_type;
9710 LLVMValueRef real_got, real_inited;
9711 MonoLLVMModule *module = &aot_module;
9712
9713 emit_llvm_code_end (module);
9714
9715 /*
9716 * Create the real got variable and replace all uses of the dummy variable with
9717 * the real one.
9718 */
9719 int size = module->max_got_offset + 1;
9720 LLVMTypeRef *members = g_malloc0 (sizeof (LLVMValueRef) * size);
9721 for (int i = 0; i < size; i++) {
9722 LLVMTypeRef lookup_type = NULL;
9723
9724 lookup_type = (LLVMTypeRef) g_hash_table_lookup (module->got_idx_to_type, GINT_TO_POINTER (i));
9725
9726 if (!lookup_type)
9727 lookup_type = module->ptr_type;
9728
9729 members [i] = LLVMPointerType (lookup_type, 0);
9730 }
9731
9732 got_type = LLVMStructCreateNamed (module->context, g_strdup_printf ("MONO_GOT_%s", cu_name));
9733 LLVMStructSetBody (got_type, members, size, FALSE);
9734 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
9735
9736 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
9737 if (module->external_symbols) {
9738 LLVMSetLinkage (real_got, LLVMExternalLinkage);
9739 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
9740 } else {
9741 LLVMSetLinkage (real_got, LLVMInternalLinkage);
9742 }
9743 mono_llvm_replace_uses_of (module->got_var, real_got);
9744
9745 mark_as_used (&aot_module, real_got);
9746
9747 /* Delete the dummy got so it doesn't become a global */
9748 LLVMDeleteGlobal (module->got_var);
9749 module->got_var = real_got;
9750
9751 /*
9752 * Same for the init_var
9753 */
9754 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
9755 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
9756 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
9757 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
9758 mono_llvm_replace_uses_of (module->inited_var, real_inited);
9759 LLVMDeleteGlobal (module->inited_var);
9760
9761 if (module->llvm_only) {
9762 emit_get_method (&aot_module);
9763 emit_get_unbox_tramp (&aot_module);
9764 }
9765
9766 emit_llvm_used (&aot_module);
9767 emit_dbg_info (&aot_module, filename, cu_name);
9768 emit_aot_file_info (&aot_module);
9769
9770 /* Replace PLT entries for directly callable methods with the methods themselves */
9771 {
9772 GHashTableIter iter;
9773 MonoJumpInfo *ji;
9774 LLVMValueRef callee;
9775
9776 GHashTable *specializable = g_hash_table_new (NULL, NULL);
9777
9778 g_hash_table_iter_init (&iter, module->plt_entries_ji);
9779 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
9780 if (mono_aot_is_direct_callable (ji)) {
9781 LLVMValueRef lmethod;
9782
9783 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
9784 /* The types might not match because the caller might pass an rgctx */
9785 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
9786 mono_llvm_replace_uses_of (callee, lmethod);
9787
9788 if (!module->llvm_disable_self_init && mono_aot_can_specialize (ji->data.method))
9789 g_hash_table_insert (specializable, lmethod, ji->data.method);
9790 mono_aot_mark_unused_llvm_plt_entry (ji);
9791 }
9792 }
9793 }
9794
9795 mono_llvm_propagate_nonnull_final (specializable, module);
9796
9797 g_hash_table_destroy (specializable);
9798 }
9799
9800 /* Note: You can still dump an invalid bitcode file by running `llvm-dis`
9801 * in a debugger, set a breakpoint on `LLVMVerifyModule` and fake its
9802 * result to 0 (indicating success). */
9803 LLVMWriteBitcodeToFile (module->lmodule, filename);
9804
9805 #if 1
9806 {
9807 char *verifier_err;
9808
9809 if (LLVMVerifyModule (module->lmodule, LLVMReturnStatusAction, &verifier_err)) {
9810 printf ("%s\n", verifier_err);
9811 g_assert_not_reached ();
9812 }
9813 }
9814 #endif
9815 }
9816
9817
9818 static LLVMValueRef
9819 md_string (const char *s)
9820 {
9821 return LLVMMDString (s, strlen (s));
9822 }
9823
9824 /* Debugging support */
9825
9826 static void
9827 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
9828 {
9829 LLVMModuleRef lmodule = module->lmodule;
9830 LLVMValueRef args [16], ver;
9831
9832 /*
9833 * This can only be enabled when LLVM code is emitted into a separate object
9834 * file, since the AOT compiler also emits dwarf info,
9835 * and the abbrev indexes will not be correct since llvm has added its own
9836 * abbrevs.
9837 */
9838 if (!module->emit_dwarf)
9839 return;
9840
9841 mono_llvm_di_builder_finalize (module->di_builder);
9842
9843 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9844 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
9845 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9846 ver = LLVMMDNode (args, 3);
9847 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
9848
9849 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
9850 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
9851 args [2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE);
9852 ver = LLVMMDNode (args, 3);
9853 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
9854 }
9855
9856 static LLVMValueRef
9857 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
9858 {
9859 MonoLLVMModule *module = ctx->module;
9860 MonoDebugMethodInfo *minfo = ctx->minfo;
9861 char *source_file, *dir, *filename;
9862 MonoSymSeqPoint *sym_seq_points;
9863 int n_seq_points;
9864
9865 if (!minfo)
9866 return NULL;
9867
9868 mono_debug_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
9869 if (!source_file)
9870 source_file = g_strdup ("<unknown>");
9871 dir = g_path_get_dirname (source_file);
9872 filename = g_path_get_basename (source_file);
9873 g_free (source_file);
9874
9875 return (LLVMValueRef)mono_llvm_di_create_function (module->di_builder, module->cu, method, cfg->method->name, name, dir, filename, n_seq_points ? sym_seq_points [0].line : 1);
9876 }
9877
9878 static void
9879 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
9880 {
9881 MonoCompile *cfg = ctx->cfg;
9882
9883 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
9884 MonoDebugSourceLocation *loc;
9885 LLVMValueRef loc_md;
9886
9887 loc = mono_debug_method_lookup_location (ctx->minfo, cil_code - cfg->header->code);
9888
9889 if (loc) {
9890 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, loc->row, loc->column);
9891 mono_llvm_di_set_location (builder, loc_md);
9892 mono_debug_free_source_location (loc);
9893 }
9894 }
9895 }
9896
9897 static void
9898 emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder)
9899 {
9900 if (ctx->minfo) {
9901 LLVMValueRef loc_md;
9902 loc_md = (LLVMValueRef)mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, 0, 0);
9903 mono_llvm_di_set_location (builder, loc_md);
9904 }
9905 }
9906
9907 /*
9908 DESIGN:
9909 - Emit LLVM IR from the mono IR using the LLVM C API.
9910 - The original arch specific code remains, so we can fall back to it if we run
9911 into something we can't handle.
9912 */
9913
9914 /*
9915 A partial list of issues:
9916 - Handling of opcodes which can throw exceptions.
9917
9918 In the mono JIT, these are implemented using code like this:
9919 method:
9920 <compare>
9921 throw_pos:
9922 b<cond> ex_label
9923 <rest of code>
9924 ex_label:
9925 push throw_pos - method
9926 call <exception trampoline>
9927
9928 The problematic part is push throw_pos - method, which cannot be represented
9929 in the LLVM IR, since it does not support label values.
9930 -> this can be implemented in AOT mode using inline asm + labels, but cannot
9931 be implemented in JIT mode ?
9932 -> a possible but slower implementation would use the normal exception
9933 throwing code but it would need to control the placement of the throw code
9934 (it needs to be exactly after the compare+branch).
9935 -> perhaps add a PC offset intrinsics ?
9936
9937 - efficient implementation of .ovf opcodes.
9938
9939 These are currently implemented as:
9940 <ins which sets the condition codes>
9941 b<cond> ex_label
9942
9943 Some overflow opcodes are now supported by LLVM SVN.
9944
9945 - exception handling, unwinding.
9946 - SSA is disabled for methods with exception handlers
9947 - How to obtain unwind info for LLVM compiled methods ?
9948 -> this is now solved by converting the unwind info generated by LLVM
9949 into our format.
9950 - LLVM uses the c++ exception handling framework, while we use our home grown
9951 code, and couldn't use the c++ one:
9952 - its not supported under VC++, other exotic platforms.
9953 - it might be impossible to support filter clauses with it.
9954
9955 - trampolines.
9956
9957 The trampolines need a predictable call sequence, since they need to disasm
9958 the calling code to obtain register numbers / offsets.
9959
9960 LLVM currently generates this code in non-JIT mode:
9961 mov -0x98(%rax),%eax
9962 callq *%rax
9963 Here, the vtable pointer is lost.
9964 -> solution: use one vtable trampoline per class.
9965
9966 - passing/receiving the IMT pointer/RGCTX.
9967 -> solution: pass them as normal arguments ?
9968
9969 - argument passing.
9970
9971 LLVM does not allow the specification of argument registers etc. This means
9972 that all calls are made according to the platform ABI.
9973
9974 - passing/receiving vtypes.
9975
9976 Vtypes passed/received in registers are handled by the front end by using
9977 a signature with scalar arguments, and loading the parts of the vtype into those
9978 arguments.
9979
9980 Vtypes passed on the stack are handled using the 'byval' attribute.
9981
9982 - ldaddr.
9983
9984 Supported though alloca, we need to emit the load/store code.
9985
9986 - types.
9987
9988 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
9989 typed registers, so we have to keep track of the precise LLVM type of each vreg.
9990 This is made easier because the IR is already in SSA form.
9991 An additional problem is that our IR is not consistent with types, i.e. i32/i64
9992 types are frequently used incorrectly.
9993 */
9994
9995 /*
9996 AOT SUPPORT:
9997 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
9998 it with the file containing the methods emitted by the JIT and the AOT data
9999 structures.
10000 */
10001
10002 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
10003 * - each bblock should end with a branch
10004 * - setting the return value, making cfg->ret non-volatile
10005 * - avoid some transformations in the JIT which make it harder for us to generate
10006 * code.
10007 * - use pointer types to help optimizations.
10008 */
10009
10010 #else /* DISABLE_JIT */
10011
10012 void
10013 mono_llvm_cleanup (void)
10014 {
10015 }
10016
10017 void
10018 mono_llvm_free_domain_info (MonoDomain *domain)
10019 {
10020 }
10021
10022 void
10023 mono_llvm_init (void)
10024 {
10025 }
10026
10027 #endif /* DISABLE_JIT */
10028
10029 #if !defined(DISABLE_JIT) && !defined(MONO_CROSS_COMPILE)
10030
10031 /* LLVM JIT support */
10032
10033 static unsigned char*
10034 alloc_cb (LLVMValueRef function, int size)
10035 {
10036 MonoCompile *cfg;
10037
10038 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10039
10040 if (cfg) {
10041 // FIXME: dynamic
10042 return (unsigned char*)mono_domain_code_reserve (cfg->domain, size);
10043 } else {
10044 return (unsigned char*)mono_domain_code_reserve (mono_domain_get (), size);
10045 }
10046 }
10047
10048 static void
10049 emitted_cb (LLVMValueRef function, void *start, void *end)
10050 {
10051 MonoCompile *cfg;
10052
10053 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10054 g_assert (cfg);
10055 cfg->code_len = (guint8*)end - (guint8*)start;
10056 }
10057
10058 static void
10059 exception_cb (void *data)
10060 {
10061 MonoCompile *cfg;
10062 MonoJitExceptionInfo *ei;
10063 guint32 ei_len, i, j, nested_len, nindex;
10064 gpointer *type_info;
10065 int this_reg, this_offset;
10066
10067 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
10068 g_assert (cfg);
10069
10070 /*
10071 * data points to a DWARF FDE structure, convert it to our unwind format and
10072 * save it.
10073 * An alternative would be to save it directly, and modify our unwinder to work
10074 * with it.
10075 */
10076 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
10077 if (cfg->verbose_level > 1)
10078 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
10079
10080 /* Count nested clauses */
10081 nested_len = 0;
10082 for (i = 0; i < ei_len; ++i) {
10083 gint32 cindex1 = *(gint32*)type_info [i];
10084 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
10085
10086 for (j = 0; j < cfg->header->num_clauses; ++j) {
10087 int cindex2 = j;
10088 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
10089
10090 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
10091 nested_len ++;
10092 }
10093 }
10094 }
10095
10096 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
10097 cfg->llvm_ex_info_len = ei_len + nested_len;
10098 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
10099 /* Fill the rest of the information from the type info */
10100 for (i = 0; i < ei_len; ++i) {
10101 gint32 clause_index = *(gint32*)type_info [i];
10102 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
10103
10104 cfg->llvm_ex_info [i].flags = clause->flags;
10105 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
10106 cfg->llvm_ex_info [i].clause_index = clause_index;
10107 }
10108
10109 /*
10110 * For nested clauses, the LLVM produced exception info associates the try interval with
10111 * the innermost handler, while mono expects it to be associated with all nesting clauses.
10112 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
10113 * and everything else from the nested clause.
10114 */
10115 nindex = ei_len;
10116 for (i = 0; i < ei_len; ++i) {
10117 gint32 cindex1 = *(gint32*)type_info [i];
10118 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
10119
10120 for (j = 0; j < cfg->header->num_clauses; ++j) {
10121 int cindex2 = j;
10122 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
10123 MonoJitExceptionInfo *nesting_ei, *nested_ei;
10124
10125 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
10126 /* clause1 is the nested clause */
10127 nested_ei = &cfg->llvm_ex_info [i];
10128 nesting_ei = &cfg->llvm_ex_info [nindex];
10129 nindex ++;
10130
10131 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
10132
10133 nesting_ei->flags = clause2->flags;
10134 nesting_ei->data.catch_class = clause2->data.catch_class;
10135 nesting_ei->clause_index = cindex2;
10136 }
10137 }
10138 }
10139 g_assert (nindex == ei_len + nested_len);
10140 cfg->llvm_this_reg = this_reg;
10141 cfg->llvm_this_offset = this_offset;
10142
10143 /* type_info [i] is cfg mempool allocated, no need to free it */
10144
10145 g_free (ei);
10146 g_free (type_info);
10147 }
10148
10149 /*
10150 * decode_llvm_eh_info:
10151 *
10152 * Decode the EH table emitted by llvm in jit mode, and store
10153 * the result into cfg.
10154 */
10155 static void
10156 decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame)
10157 {
10158 MonoCompile *cfg = ctx->cfg;
10159 guint8 *cie, *fde;
10160 int fde_len;
10161 MonoLLVMFDEInfo info;
10162 MonoJitExceptionInfo *ei;
10163 guint8 *p = (guint8*)eh_frame;
10164 int version, fde_count, fde_offset;
10165 guint32 ei_len, i, nested_len;
10166 gpointer *type_info;
10167 gint32 *table;
10168 guint8 *unw_info;
10169
10170 /*
10171 * Decode the one element EH table emitted by the MonoException class
10172 * in llvm.
10173 */
10174
10175 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
10176
10177 version = *p;
10178 g_assert (version == 3);
10179 p ++;
10180 p ++;
10181 p = (guint8 *)ALIGN_PTR_TO (p, 4);
10182
10183 fde_count = *(guint32*)p;
10184 p += 4;
10185 table = (gint32*)p;
10186
10187 g_assert (fde_count <= 2);
10188
10189 /* The first entry is the real method */
10190 g_assert (table [0] == 1);
10191 fde_offset = table [1];
10192 table += fde_count * 2;
10193 /* Extra entry */
10194 cfg->code_len = table [0];
10195 fde_len = table [1] - fde_offset;
10196 table += 2;
10197
10198 fde = (guint8*)eh_frame + fde_offset;
10199 cie = (guint8*)table;
10200
10201 /* Compute lengths */
10202 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, NULL, NULL, NULL);
10203
10204 ei = (MonoJitExceptionInfo *)g_malloc0 (info.ex_info_len * sizeof (MonoJitExceptionInfo));
10205 type_info = (gpointer *)g_malloc0 (info.ex_info_len * sizeof (gpointer));
10206 unw_info = (guint8*)g_malloc0 (info.unw_info_len);
10207
10208 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info, ei, type_info, unw_info);
10209
10210 cfg->encoded_unwind_ops = unw_info;
10211 cfg->encoded_unwind_ops_len = info.unw_info_len;
10212 if (cfg->verbose_level > 1)
10213 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
10214 if (info.this_reg != -1) {
10215 cfg->llvm_this_reg = info.this_reg;
10216 cfg->llvm_this_offset = info.this_offset;
10217 }
10218
10219 ei_len = info.ex_info_len;
10220
10221 // Nested clauses are currently disabled
10222 nested_len = 0;
10223
10224 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
10225 cfg->llvm_ex_info_len = ei_len + nested_len;
10226 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
10227 /* Fill the rest of the information from the type info */
10228 for (i = 0; i < ei_len; ++i) {
10229 gint32 clause_index = *(gint32*)type_info [i];
10230 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
10231
10232 cfg->llvm_ex_info [i].flags = clause->flags;
10233 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
10234 cfg->llvm_ex_info [i].clause_index = clause_index;
10235 }
10236 }
10237
10238 static void
10239 init_jit_module (MonoDomain *domain)
10240 {
10241 MonoJitDomainInfo *dinfo;
10242 MonoLLVMModule *module;
10243 char *name;
10244
10245 dinfo = domain_jit_info (domain);
10246 if (dinfo->llvm_module)
10247 return;
10248
10249 mono_loader_lock ();
10250
10251 if (dinfo->llvm_module) {
10252 mono_loader_unlock ();
10253 return;
10254 }
10255
10256 module = g_new0 (MonoLLVMModule, 1);
10257
10258 name = g_strdup_printf ("mono-%s", domain->friendly_name);
10259 module->context = LLVMGetGlobalContext ();
10260 module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM);
10261
10262 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (alloc_cb, emitted_cb, exception_cb, &module->ee);
10263
10264 add_intrinsics (module->lmodule);
10265 add_types (module);
10266
10267 module->llvm_types = g_hash_table_new (NULL, NULL);
10268
10269 mono_memory_barrier ();
10270
10271 dinfo->llvm_module = module;
10272
10273 mono_loader_unlock ();
10274 }
10275
10276 static void
10277 llvm_jit_finalize_method (EmitContext *ctx)
10278 {
10279 MonoCompile *cfg = ctx->cfg;
10280 MonoDomain *domain = mono_domain_get ();
10281 MonoJitDomainInfo *domain_info;
10282 int nvars = g_hash_table_size (ctx->jit_callees);
10283 LLVMValueRef *callee_vars = g_new0 (LLVMValueRef, nvars);
10284 gpointer *callee_addrs = g_new0 (gpointer, nvars);
10285 GHashTableIter iter;
10286 LLVMValueRef var;
10287 MonoMethod *callee;
10288 gpointer eh_frame;
10289 int i;
10290
10291 /*
10292 * Compute the addresses of the LLVM globals pointing to the
10293 * methods called by the current method. Pass it to the trampoline
10294 * code so it can update them after their corresponding method was
10295 * compiled.
10296 */
10297 g_hash_table_iter_init (&iter, ctx->jit_callees);
10298 i = 0;
10299 while (g_hash_table_iter_next (&iter, NULL, (void**)&var))
10300 callee_vars [i ++] = var;
10301
10302 cfg->native_code = (guint8*)mono_llvm_compile_method (ctx->module->mono_ee, ctx->lmethod, nvars, callee_vars, callee_addrs, &eh_frame);
10303 if (cfg->verbose_level > 1) {
10304 g_print ("\n*** Optimized LLVM IR for %s ***\n", mono_method_full_name (cfg->method, FALSE));
10305 mono_llvm_dump_value (ctx->lmethod);
10306 g_print ("***\n\n");
10307 }
10308
10309 decode_llvm_eh_info (ctx, eh_frame);
10310
10311 mono_domain_lock (domain);
10312 domain_info = domain_jit_info (domain);
10313 if (!domain_info->llvm_jit_callees)
10314 domain_info->llvm_jit_callees = g_hash_table_new (NULL, NULL);
10315 g_hash_table_iter_init (&iter, ctx->jit_callees);
10316 i = 0;
10317 while (g_hash_table_iter_next (&iter, (void**)&callee, (void**)&var)) {
10318 GSList *addrs = (GSList*)g_hash_table_lookup (domain_info->llvm_jit_callees, callee);
10319 addrs = g_slist_prepend (addrs, callee_addrs [i]);
10320 g_hash_table_insert (domain_info->llvm_jit_callees, callee, addrs);
10321 i ++;
10322 }
10323 mono_domain_unlock (domain);
10324 }
10325
10326 #else
10327
10328 static void
10329 init_jit_module (MonoDomain *domain)
10330 {
10331 g_assert_not_reached ();
10332 }
10333
10334 static void
10335 llvm_jit_finalize_method (EmitContext *ctx)
10336 {
10337 g_assert_not_reached ();
10338 }
10339
10340 #endif
Mirror of the official Mono project git repository