?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[profiler] Implement call context introspection for enter/leave events.
[mono-project.git] / mono / mini / mini-arm64.c
blob46fb083b2823efaf6034aa894bf141418bb2b31d
1 /**
2 * \file
3 * ARM64 backend for the Mono code generator
4 *
5 * Copyright 2013 Xamarin, Inc (http://www.xamarin.com)
6 *
7 * Based on mini-arm.c:
8 *
9 * Authors:
10 * Paolo Molaro (lupus@ximian.com)
11 * Dietmar Maurer (dietmar@ximian.com)
12 *
13 * (C) 2003 Ximian, Inc.
14 * Copyright 2003-2011 Novell, Inc (http://www.novell.com)
15 * Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
16 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
17 */
18
19 #include "mini.h"
20 #include "cpu-arm64.h"
21 #include "ir-emit.h"
22
23 #include <mono/arch/arm64/arm64-codegen.h>
24 #include <mono/utils/mono-mmap.h>
25 #include <mono/utils/mono-memory-model.h>
26 #include <mono/metadata/abi-details.h>
27
28 /*
29 * Documentation:
30 *
31 * - ARM(R) Architecture Reference Manual, ARMv8, for ARMv8-A architecture profile (DDI0487A_a_armv8_arm.pdf)
32 * - Procedure Call Standard for the ARM 64-bit Architecture (AArch64) (IHI0055B_aapcs64.pdf)
33 * - ELF for the ARM 64-bit Architecture (IHI0056B_aaelf64.pdf)
34 *
35 * Register usage:
36 * - ip0/ip1/lr are used as temporary registers
37 * - r27 is used as the rgctx/imt register
38 * - r28 is used to access arguments passed on the stack
39 * - d15/d16 are used as fp temporary registers
40 */
41
42 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
43
44 #define FP_TEMP_REG ARMREG_D16
45 #define FP_TEMP_REG2 ARMREG_D17
46
47 #define THUNK_SIZE (4 * 4)
48
49 /* The single step trampoline */
50 static gpointer ss_trampoline;
51
52 /* The breakpoint trampoline */
53 static gpointer bp_trampoline;
54
55 static gboolean ios_abi;
56
57 static __attribute__ ((__warn_unused_result__)) guint8* emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset);
58
59 const char*
60 mono_arch_regname (int reg)
61 {
62 static const char * rnames[] = {
63 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9",
64 "r10", "r11", "r12", "r13", "r14", "r15", "r16", "r17", "r18", "r19",
65 "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "fp",
66 "lr", "sp"
67 };
68 if (reg >= 0 && reg < 32)
69 return rnames [reg];
70 return "unknown";
71 }
72
73 const char*
74 mono_arch_fregname (int reg)
75 {
76 static const char * rnames[] = {
77 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9",
78 "d10", "d11", "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19",
79 "d20", "d21", "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29",
80 "d30", "d31"
81 };
82 if (reg >= 0 && reg < 32)
83 return rnames [reg];
84 return "unknown fp";
85 }
86
87 int
88 mono_arch_get_argument_info (MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
89 {
90 NOT_IMPLEMENTED;
91 return 0;
92 }
93
94 #define MAX_ARCH_DELEGATE_PARAMS 7
95
96 static gpointer
97 get_delegate_invoke_impl (gboolean has_target, gboolean param_count, guint32 *code_size)
98 {
99 guint8 *code, *start;
100
101 if (has_target) {
102 start = code = mono_global_codeman_reserve (12);
103
104 /* Replace the this argument with the target */
105 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
106 arm_ldrx (code, ARMREG_R0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, target));
107 arm_brx (code, ARMREG_IP0);
108
109 g_assert ((code - start) <= 12);
110
111 mono_arch_flush_icache (start, 12);
112 } else {
113 int size, i;
114
115 size = 8 + param_count * 4;
116 start = code = mono_global_codeman_reserve (size);
117
118 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
119 /* slide down the arguments */
120 for (i = 0; i < param_count; ++i)
121 arm_movx (code, i, i + 1);
122 arm_brx (code, ARMREG_IP0);
123
124 g_assert ((code - start) <= size);
125
126 mono_arch_flush_icache (start, size);
127 }
128
129 if (code_size)
130 *code_size = code - start;
131
132 return start;
133 }
134
135 /*
136 * mono_arch_get_delegate_invoke_impls:
137 *
138 * Return a list of MonoAotTrampInfo structures for the delegate invoke impl
139 * trampolines.
140 */
141 GSList*
142 mono_arch_get_delegate_invoke_impls (void)
143 {
144 GSList *res = NULL;
145 guint8 *code;
146 guint32 code_len;
147 int i;
148 char *tramp_name;
149
150 code = get_delegate_invoke_impl (TRUE, 0, &code_len);
151 res = g_slist_prepend (res, mono_tramp_info_create ("delegate_invoke_impl_has_target", code, code_len, NULL, NULL));
152
153 for (i = 0; i <= MAX_ARCH_DELEGATE_PARAMS; ++i) {
154 code = get_delegate_invoke_impl (FALSE, i, &code_len);
155 tramp_name = g_strdup_printf ("delegate_invoke_impl_target_%d", i);
156 res = g_slist_prepend (res, mono_tramp_info_create (tramp_name, code, code_len, NULL, NULL));
157 g_free (tramp_name);
158 }
159
160 return res;
161 }
162
163 gpointer
164 mono_arch_get_delegate_invoke_impl (MonoMethodSignature *sig, gboolean has_target)
165 {
166 guint8 *code, *start;
167
168 /*
169 * vtypes are returned in registers, or using the dedicated r8 register, so
170 * they can be supported by delegate invokes.
171 */
172
173 if (has_target) {
174 static guint8* cached = NULL;
175
176 if (cached)
177 return cached;
178
179 if (mono_aot_only)
180 start = mono_aot_get_trampoline ("delegate_invoke_impl_has_target");
181 else
182 start = get_delegate_invoke_impl (TRUE, 0, NULL);
183 mono_memory_barrier ();
184 cached = start;
185 return cached;
186 } else {
187 static guint8* cache [MAX_ARCH_DELEGATE_PARAMS + 1] = {NULL};
188 int i;
189
190 if (sig->param_count > MAX_ARCH_DELEGATE_PARAMS)
191 return NULL;
192 for (i = 0; i < sig->param_count; ++i)
193 if (!mono_is_regsize_var (sig->params [i]))
194 return NULL;
195
196 code = cache [sig->param_count];
197 if (code)
198 return code;
199
200 if (mono_aot_only) {
201 char *name = g_strdup_printf ("delegate_invoke_impl_target_%d", sig->param_count);
202 start = mono_aot_get_trampoline (name);
203 g_free (name);
204 } else {
205 start = get_delegate_invoke_impl (FALSE, sig->param_count, NULL);
206 }
207 mono_memory_barrier ();
208 cache [sig->param_count] = start;
209 return start;
210 }
211
212 return NULL;
213 }
214
215 gpointer
216 mono_arch_get_delegate_virtual_invoke_impl (MonoMethodSignature *sig, MonoMethod *method, int offset, gboolean load_imt_reg)
217 {
218 return NULL;
219 }
220
221 gpointer
222 mono_arch_get_this_arg_from_call (mgreg_t *regs, guint8 *code)
223 {
224 return (gpointer)regs [ARMREG_R0];
225 }
226
227 void
228 mono_arch_cpu_init (void)
229 {
230 }
231
232 void
233 mono_arch_init (void)
234 {
235 mono_aot_register_jit_icall ("mono_arm_throw_exception", mono_arm_throw_exception);
236 mono_aot_register_jit_icall ("mono_arm_resume_unwind", mono_arm_resume_unwind);
237
238 if (!mono_aot_only)
239 bp_trampoline = mini_get_breakpoint_trampoline ();
240
241 mono_arm_gsharedvt_init ();
242
243 #if defined(TARGET_IOS)
244 ios_abi = TRUE;
245 #endif
246 }
247
248 void
249 mono_arch_cleanup (void)
250 {
251 }
252
253 guint32
254 mono_arch_cpu_optimizations (guint32 *exclude_mask)
255 {
256 *exclude_mask = 0;
257 return 0;
258 }
259
260 guint32
261 mono_arch_cpu_enumerate_simd_versions (void)
262 {
263 return 0;
264 }
265
266 void
267 mono_arch_register_lowlevel_calls (void)
268 {
269 }
270
271 void
272 mono_arch_finish_init (void)
273 {
274 }
275
276 /* The maximum length is 2 instructions */
277 static guint8*
278 emit_imm (guint8 *code, int dreg, int imm)
279 {
280 // FIXME: Optimize this
281 if (imm < 0) {
282 gint64 limm = imm;
283 arm_movnx (code, dreg, (~limm) & 0xffff, 0);
284 arm_movkx (code, dreg, (limm >> 16) & 0xffff, 16);
285 } else {
286 arm_movzx (code, dreg, imm & 0xffff, 0);
287 if (imm >> 16)
288 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
289 }
290
291 return code;
292 }
293
294 /* The maximum length is 4 instructions */
295 static guint8*
296 emit_imm64 (guint8 *code, int dreg, guint64 imm)
297 {
298 // FIXME: Optimize this
299 arm_movzx (code, dreg, imm & 0xffff, 0);
300 if ((imm >> 16) & 0xffff)
301 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
302 if ((imm >> 32) & 0xffff)
303 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
304 if ((imm >> 48) & 0xffff)
305 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
306
307 return code;
308 }
309
310 guint8*
311 mono_arm_emit_imm64 (guint8 *code, int dreg, gint64 imm)
312 {
313 return emit_imm64 (code, dreg, imm);
314 }
315
316 /*
317 * emit_imm_template:
318 *
319 * Emit a patchable code sequence for constructing a 64 bit immediate.
320 */
321 static guint8*
322 emit_imm64_template (guint8 *code, int dreg)
323 {
324 arm_movzx (code, dreg, 0, 0);
325 arm_movkx (code, dreg, 0, 16);
326 arm_movkx (code, dreg, 0, 32);
327 arm_movkx (code, dreg, 0, 48);
328
329 return code;
330 }
331
332 static inline __attribute__ ((__warn_unused_result__)) guint8*
333 emit_addw_imm (guint8 *code, int dreg, int sreg, int imm)
334 {
335 if (!arm_is_arith_imm (imm)) {
336 code = emit_imm (code, ARMREG_LR, imm);
337 arm_addw (code, dreg, sreg, ARMREG_LR);
338 } else {
339 arm_addw_imm (code, dreg, sreg, imm);
340 }
341 return code;
342 }
343
344 static inline __attribute__ ((__warn_unused_result__)) guint8*
345 emit_addx_imm (guint8 *code, int dreg, int sreg, int imm)
346 {
347 if (!arm_is_arith_imm (imm)) {
348 code = emit_imm (code, ARMREG_LR, imm);
349 arm_addx (code, dreg, sreg, ARMREG_LR);
350 } else {
351 arm_addx_imm (code, dreg, sreg, imm);
352 }
353 return code;
354 }
355
356 static inline __attribute__ ((__warn_unused_result__)) guint8*
357 emit_subw_imm (guint8 *code, int dreg, int sreg, int imm)
358 {
359 if (!arm_is_arith_imm (imm)) {
360 code = emit_imm (code, ARMREG_LR, imm);
361 arm_subw (code, dreg, sreg, ARMREG_LR);
362 } else {
363 arm_subw_imm (code, dreg, sreg, imm);
364 }
365 return code;
366 }
367
368 static inline __attribute__ ((__warn_unused_result__)) guint8*
369 emit_subx_imm (guint8 *code, int dreg, int sreg, int imm)
370 {
371 if (!arm_is_arith_imm (imm)) {
372 code = emit_imm (code, ARMREG_LR, imm);
373 arm_subx (code, dreg, sreg, ARMREG_LR);
374 } else {
375 arm_subx_imm (code, dreg, sreg, imm);
376 }
377 return code;
378 }
379
380 /* Emit sp+=imm. Clobbers ip0/ip1 */
381 static inline __attribute__ ((__warn_unused_result__)) guint8*
382 emit_addx_sp_imm (guint8 *code, int imm)
383 {
384 code = emit_imm (code, ARMREG_IP0, imm);
385 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
386 arm_addx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
387 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
388 return code;
389 }
390
391 /* Emit sp-=imm. Clobbers ip0/ip1 */
392 static inline __attribute__ ((__warn_unused_result__)) guint8*
393 emit_subx_sp_imm (guint8 *code, int imm)
394 {
395 code = emit_imm (code, ARMREG_IP0, imm);
396 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
397 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
398 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
399 return code;
400 }
401
402 static inline __attribute__ ((__warn_unused_result__)) guint8*
403 emit_andw_imm (guint8 *code, int dreg, int sreg, int imm)
404 {
405 // FIXME:
406 code = emit_imm (code, ARMREG_LR, imm);
407 arm_andw (code, dreg, sreg, ARMREG_LR);
408
409 return code;
410 }
411
412 static inline __attribute__ ((__warn_unused_result__)) guint8*
413 emit_andx_imm (guint8 *code, int dreg, int sreg, int imm)
414 {
415 // FIXME:
416 code = emit_imm (code, ARMREG_LR, imm);
417 arm_andx (code, dreg, sreg, ARMREG_LR);
418
419 return code;
420 }
421
422 static inline __attribute__ ((__warn_unused_result__)) guint8*
423 emit_orrw_imm (guint8 *code, int dreg, int sreg, int imm)
424 {
425 // FIXME:
426 code = emit_imm (code, ARMREG_LR, imm);
427 arm_orrw (code, dreg, sreg, ARMREG_LR);
428
429 return code;
430 }
431
432 static inline __attribute__ ((__warn_unused_result__)) guint8*
433 emit_orrx_imm (guint8 *code, int dreg, int sreg, int imm)
434 {
435 // FIXME:
436 code = emit_imm (code, ARMREG_LR, imm);
437 arm_orrx (code, dreg, sreg, ARMREG_LR);
438
439 return code;
440 }
441
442 static inline __attribute__ ((__warn_unused_result__)) guint8*
443 emit_eorw_imm (guint8 *code, int dreg, int sreg, int imm)
444 {
445 // FIXME:
446 code = emit_imm (code, ARMREG_LR, imm);
447 arm_eorw (code, dreg, sreg, ARMREG_LR);
448
449 return code;
450 }
451
452 static inline __attribute__ ((__warn_unused_result__)) guint8*
453 emit_eorx_imm (guint8 *code, int dreg, int sreg, int imm)
454 {
455 // FIXME:
456 code = emit_imm (code, ARMREG_LR, imm);
457 arm_eorx (code, dreg, sreg, ARMREG_LR);
458
459 return code;
460 }
461
462 static inline __attribute__ ((__warn_unused_result__)) guint8*
463 emit_cmpw_imm (guint8 *code, int sreg, int imm)
464 {
465 if (imm == 0) {
466 arm_cmpw (code, sreg, ARMREG_RZR);
467 } else {
468 // FIXME:
469 code = emit_imm (code, ARMREG_LR, imm);
470 arm_cmpw (code, sreg, ARMREG_LR);
471 }
472
473 return code;
474 }
475
476 static inline __attribute__ ((__warn_unused_result__)) guint8*
477 emit_cmpx_imm (guint8 *code, int sreg, int imm)
478 {
479 if (imm == 0) {
480 arm_cmpx (code, sreg, ARMREG_RZR);
481 } else {
482 // FIXME:
483 code = emit_imm (code, ARMREG_LR, imm);
484 arm_cmpx (code, sreg, ARMREG_LR);
485 }
486
487 return code;
488 }
489
490 static inline __attribute__ ((__warn_unused_result__)) guint8*
491 emit_strb (guint8 *code, int rt, int rn, int imm)
492 {
493 if (arm_is_strb_imm (imm)) {
494 arm_strb (code, rt, rn, imm);
495 } else {
496 g_assert (rt != ARMREG_IP0);
497 g_assert (rn != ARMREG_IP0);
498 code = emit_imm (code, ARMREG_IP0, imm);
499 arm_strb_reg (code, rt, rn, ARMREG_IP0);
500 }
501 return code;
502 }
503
504 static inline __attribute__ ((__warn_unused_result__)) guint8*
505 emit_strh (guint8 *code, int rt, int rn, int imm)
506 {
507 if (arm_is_strh_imm (imm)) {
508 arm_strh (code, rt, rn, imm);
509 } else {
510 g_assert (rt != ARMREG_IP0);
511 g_assert (rn != ARMREG_IP0);
512 code = emit_imm (code, ARMREG_IP0, imm);
513 arm_strh_reg (code, rt, rn, ARMREG_IP0);
514 }
515 return code;
516 }
517
518 static inline __attribute__ ((__warn_unused_result__)) guint8*
519 emit_strw (guint8 *code, int rt, int rn, int imm)
520 {
521 if (arm_is_strw_imm (imm)) {
522 arm_strw (code, rt, rn, imm);
523 } else {
524 g_assert (rt != ARMREG_IP0);
525 g_assert (rn != ARMREG_IP0);
526 code = emit_imm (code, ARMREG_IP0, imm);
527 arm_strw_reg (code, rt, rn, ARMREG_IP0);
528 }
529 return code;
530 }
531
532 static inline __attribute__ ((__warn_unused_result__)) guint8*
533 emit_strfpw (guint8 *code, int rt, int rn, int imm)
534 {
535 if (arm_is_strw_imm (imm)) {
536 arm_strfpw (code, rt, rn, imm);
537 } else {
538 g_assert (rn != ARMREG_IP0);
539 code = emit_imm (code, ARMREG_IP0, imm);
540 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
541 arm_strfpw (code, rt, ARMREG_IP0, 0);
542 }
543 return code;
544 }
545
546 static inline __attribute__ ((__warn_unused_result__)) guint8*
547 emit_strfpx (guint8 *code, int rt, int rn, int imm)
548 {
549 if (arm_is_strx_imm (imm)) {
550 arm_strfpx (code, rt, rn, imm);
551 } else {
552 g_assert (rn != ARMREG_IP0);
553 code = emit_imm (code, ARMREG_IP0, imm);
554 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
555 arm_strfpx (code, rt, ARMREG_IP0, 0);
556 }
557 return code;
558 }
559
560 static inline __attribute__ ((__warn_unused_result__)) guint8*
561 emit_strx (guint8 *code, int rt, int rn, int imm)
562 {
563 if (arm_is_strx_imm (imm)) {
564 arm_strx (code, rt, rn, imm);
565 } else {
566 g_assert (rt != ARMREG_IP0);
567 g_assert (rn != ARMREG_IP0);
568 code = emit_imm (code, ARMREG_IP0, imm);
569 arm_strx_reg (code, rt, rn, ARMREG_IP0);
570 }
571 return code;
572 }
573
574 static inline __attribute__ ((__warn_unused_result__)) guint8*
575 emit_ldrb (guint8 *code, int rt, int rn, int imm)
576 {
577 if (arm_is_pimm12_scaled (imm, 1)) {
578 arm_ldrb (code, rt, rn, imm);
579 } else {
580 g_assert (rt != ARMREG_IP0);
581 g_assert (rn != ARMREG_IP0);
582 code = emit_imm (code, ARMREG_IP0, imm);
583 arm_ldrb_reg (code, rt, rn, ARMREG_IP0);
584 }
585 return code;
586 }
587
588 static inline __attribute__ ((__warn_unused_result__)) guint8*
589 emit_ldrsbx (guint8 *code, int rt, int rn, int imm)
590 {
591 if (arm_is_pimm12_scaled (imm, 1)) {
592 arm_ldrsbx (code, rt, rn, imm);
593 } else {
594 g_assert (rt != ARMREG_IP0);
595 g_assert (rn != ARMREG_IP0);
596 code = emit_imm (code, ARMREG_IP0, imm);
597 arm_ldrsbx_reg (code, rt, rn, ARMREG_IP0);
598 }
599 return code;
600 }
601
602 static inline __attribute__ ((__warn_unused_result__)) guint8*
603 emit_ldrh (guint8 *code, int rt, int rn, int imm)
604 {
605 if (arm_is_pimm12_scaled (imm, 2)) {
606 arm_ldrh (code, rt, rn, imm);
607 } else {
608 g_assert (rt != ARMREG_IP0);
609 g_assert (rn != ARMREG_IP0);
610 code = emit_imm (code, ARMREG_IP0, imm);
611 arm_ldrh_reg (code, rt, rn, ARMREG_IP0);
612 }
613 return code;
614 }
615
616 static inline __attribute__ ((__warn_unused_result__)) guint8*
617 emit_ldrshx (guint8 *code, int rt, int rn, int imm)
618 {
619 if (arm_is_pimm12_scaled (imm, 2)) {
620 arm_ldrshx (code, rt, rn, imm);
621 } else {
622 g_assert (rt != ARMREG_IP0);
623 g_assert (rn != ARMREG_IP0);
624 code = emit_imm (code, ARMREG_IP0, imm);
625 arm_ldrshx_reg (code, rt, rn, ARMREG_IP0);
626 }
627 return code;
628 }
629
630 static inline __attribute__ ((__warn_unused_result__)) guint8*
631 emit_ldrswx (guint8 *code, int rt, int rn, int imm)
632 {
633 if (arm_is_pimm12_scaled (imm, 4)) {
634 arm_ldrswx (code, rt, rn, imm);
635 } else {
636 g_assert (rt != ARMREG_IP0);
637 g_assert (rn != ARMREG_IP0);
638 code = emit_imm (code, ARMREG_IP0, imm);
639 arm_ldrswx_reg (code, rt, rn, ARMREG_IP0);
640 }
641 return code;
642 }
643
644 static inline __attribute__ ((__warn_unused_result__)) guint8*
645 emit_ldrw (guint8 *code, int rt, int rn, int imm)
646 {
647 if (arm_is_pimm12_scaled (imm, 4)) {
648 arm_ldrw (code, rt, rn, imm);
649 } else {
650 g_assert (rn != ARMREG_IP0);
651 code = emit_imm (code, ARMREG_IP0, imm);
652 arm_ldrw_reg (code, rt, rn, ARMREG_IP0);
653 }
654 return code;
655 }
656
657 static inline __attribute__ ((__warn_unused_result__)) guint8*
658 emit_ldrx (guint8 *code, int rt, int rn, int imm)
659 {
660 if (arm_is_pimm12_scaled (imm, 8)) {
661 arm_ldrx (code, rt, rn, imm);
662 } else {
663 g_assert (rn != ARMREG_IP0);
664 code = emit_imm (code, ARMREG_IP0, imm);
665 arm_ldrx_reg (code, rt, rn, ARMREG_IP0);
666 }
667 return code;
668 }
669
670 static inline __attribute__ ((__warn_unused_result__)) guint8*
671 emit_ldrfpw (guint8 *code, int rt, int rn, int imm)
672 {
673 if (arm_is_pimm12_scaled (imm, 4)) {
674 arm_ldrfpw (code, rt, rn, imm);
675 } else {
676 g_assert (rn != ARMREG_IP0);
677 code = emit_imm (code, ARMREG_IP0, imm);
678 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
679 arm_ldrfpw (code, rt, ARMREG_IP0, 0);
680 }
681 return code;
682 }
683
684 static inline __attribute__ ((__warn_unused_result__)) guint8*
685 emit_ldrfpx (guint8 *code, int rt, int rn, int imm)
686 {
687 if (arm_is_pimm12_scaled (imm, 8)) {
688 arm_ldrfpx (code, rt, rn, imm);
689 } else {
690 g_assert (rn != ARMREG_IP0);
691 code = emit_imm (code, ARMREG_IP0, imm);
692 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
693 arm_ldrfpx (code, rt, ARMREG_IP0, 0);
694 }
695 return code;
696 }
697
698 guint8*
699 mono_arm_emit_ldrx (guint8 *code, int rt, int rn, int imm)
700 {
701 return emit_ldrx (code, rt, rn, imm);
702 }
703
704 static guint8*
705 emit_call (MonoCompile *cfg, guint8* code, guint32 patch_type, gconstpointer data)
706 {
707 /*
708 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_IMM);
709 code = emit_imm64_template (code, ARMREG_LR);
710 arm_blrx (code, ARMREG_LR);
711 */
712 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_BL);
713 arm_bl (code, code);
714 cfg->thunk_area += THUNK_SIZE;
715 return code;
716 }
717
718 static guint8*
719 emit_aotconst_full (MonoCompile *cfg, MonoJumpInfo **ji, guint8 *code, guint8 *start, int dreg, guint32 patch_type, gconstpointer data)
720 {
721 if (cfg)
722 mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
723 else
724 *ji = mono_patch_info_list_prepend (*ji, code - start, patch_type, data);
725 /* See arch_emit_got_access () in aot-compiler.c */
726 arm_ldrx_lit (code, dreg, 0);
727 arm_nop (code);
728 arm_nop (code);
729 return code;
730 }
731
732 static guint8*
733 emit_aotconst (MonoCompile *cfg, guint8 *code, int dreg, guint32 patch_type, gconstpointer data)
734 {
735 return emit_aotconst_full (cfg, NULL, code, NULL, dreg, patch_type, data);
736 }
737
738 /*
739 * mono_arm_emit_aotconst:
740 *
741 * Emit code to load an AOT constant into DREG. Usable from trampolines.
742 */
743 guint8*
744 mono_arm_emit_aotconst (gpointer ji, guint8 *code, guint8 *code_start, int dreg, guint32 patch_type, gconstpointer data)
745 {
746 return emit_aotconst_full (NULL, (MonoJumpInfo**)ji, code, code_start, dreg, patch_type, data);
747 }
748
749 gboolean
750 mono_arch_have_fast_tls (void)
751 {
752 #ifdef TARGET_IOS
753 return FALSE;
754 #else
755 return TRUE;
756 #endif
757 }
758
759 static guint8*
760 emit_tls_get (guint8 *code, int dreg, int tls_offset)
761 {
762 arm_mrs (code, dreg, ARM_MRS_REG_TPIDR_EL0);
763 if (tls_offset < 256) {
764 arm_ldrx (code, dreg, dreg, tls_offset);
765 } else {
766 code = emit_addx_imm (code, dreg, dreg, tls_offset);
767 arm_ldrx (code, dreg, dreg, 0);
768 }
769 return code;
770 }
771
772 static guint8*
773 emit_tls_set (guint8 *code, int sreg, int tls_offset)
774 {
775 int tmpreg = ARMREG_IP0;
776
777 g_assert (sreg != tmpreg);
778 arm_mrs (code, tmpreg, ARM_MRS_REG_TPIDR_EL0);
779 if (tls_offset < 256) {
780 arm_strx (code, sreg, tmpreg, tls_offset);
781 } else {
782 code = emit_addx_imm (code, tmpreg, tmpreg, tls_offset);
783 arm_strx (code, sreg, tmpreg, 0);
784 }
785 return code;
786 }
787
788 /*
789 * Emits
790 * - mov sp, fp
791 * - ldrp [fp, lr], [sp], !stack_offfset
792 * Clobbers TEMP_REGS.
793 */
794 __attribute__ ((__warn_unused_result__)) guint8*
795 mono_arm_emit_destroy_frame (guint8 *code, int stack_offset, guint64 temp_regs)
796 {
797 arm_movspx (code, ARMREG_SP, ARMREG_FP);
798
799 if (arm_is_ldpx_imm (stack_offset)) {
800 arm_ldpx_post (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, stack_offset);
801 } else {
802 arm_ldpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
803 /* sp += stack_offset */
804 g_assert (temp_regs & (1 << ARMREG_IP0));
805 if (temp_regs & (1 << ARMREG_IP1)) {
806 code = emit_addx_sp_imm (code, stack_offset);
807 } else {
808 int imm = stack_offset;
809
810 /* Can't use addx_sp_imm () since we can't clobber ip0/ip1 */
811 arm_addx_imm (code, ARMREG_IP0, ARMREG_SP, 0);
812 while (imm > 256) {
813 arm_addx_imm (code, ARMREG_IP0, ARMREG_IP0, 256);
814 imm -= 256;
815 }
816 arm_addx_imm (code, ARMREG_SP, ARMREG_IP0, imm);
817 }
818 }
819 return code;
820 }
821
822 #define is_call_imm(diff) ((gint)(diff) >= -33554432 && (gint)(diff) <= 33554431)
823
824 static guint8*
825 emit_thunk (guint8 *code, gconstpointer target)
826 {
827 guint8 *p = code;
828
829 arm_ldrx_lit (code, ARMREG_IP0, code + 8);
830 arm_brx (code, ARMREG_IP0);
831 *(guint64*)code = (guint64)target;
832 code += sizeof (guint64);
833
834 mono_arch_flush_icache (p, code - p);
835 return code;
836 }
837
838 static gpointer
839 create_thunk (MonoCompile *cfg, MonoDomain *domain, guchar *code, const guchar *target)
840 {
841 MonoJitInfo *ji;
842 MonoThunkJitInfo *info;
843 guint8 *thunks, *p;
844 int thunks_size;
845 guint8 *orig_target;
846 guint8 *target_thunk;
847
848 if (!domain)
849 domain = mono_domain_get ();
850
851 if (cfg) {
852 /*
853 * This can be called multiple times during JITting,
854 * save the current position in cfg->arch to avoid
855 * doing a O(n^2) search.
856 */
857 if (!cfg->arch.thunks) {
858 cfg->arch.thunks = cfg->thunks;
859 cfg->arch.thunks_size = cfg->thunk_area;
860 }
861 thunks = cfg->arch.thunks;
862 thunks_size = cfg->arch.thunks_size;
863 if (!thunks_size) {
864 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, mono_method_full_name (cfg->method, TRUE));
865 g_assert_not_reached ();
866 }
867
868 g_assert (*(guint32*)thunks == 0);
869 emit_thunk (thunks, target);
870
871 cfg->arch.thunks += THUNK_SIZE;
872 cfg->arch.thunks_size -= THUNK_SIZE;
873
874 return thunks;
875 } else {
876 ji = mini_jit_info_table_find (domain, (char*)code, NULL);
877 g_assert (ji);
878 info = mono_jit_info_get_thunk_info (ji);
879 g_assert (info);
880
881 thunks = (guint8*)ji->code_start + info->thunks_offset;
882 thunks_size = info->thunks_size;
883
884 orig_target = mono_arch_get_call_target (code + 4);
885
886 mono_domain_lock (domain);
887
888 target_thunk = NULL;
889 if (orig_target >= thunks && orig_target < thunks + thunks_size) {
890 /* The call already points to a thunk, because of trampolines etc. */
891 target_thunk = orig_target;
892 } else {
893 for (p = thunks; p < thunks + thunks_size; p += THUNK_SIZE) {
894 if (((guint32*)p) [0] == 0) {
895 /* Free entry */
896 target_thunk = p;
897 break;
898 } else if (((guint64*)p) [1] == (guint64)target) {
899 /* Thunk already points to target */
900 target_thunk = p;
901 break;
902 }
903 }
904 }
905
906 //printf ("THUNK: %p %p %p\n", code, target, target_thunk);
907
908 if (!target_thunk) {
909 mono_domain_unlock (domain);
910 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, cfg ? mono_method_full_name (cfg->method, TRUE) : mono_method_full_name (jinfo_get_method (ji), TRUE));
911 g_assert_not_reached ();
912 }
913
914 emit_thunk (target_thunk, target);
915
916 mono_domain_unlock (domain);
917
918 return target_thunk;
919 }
920 }
921
922 static void
923 arm_patch_full (MonoCompile *cfg, MonoDomain *domain, guint8 *code, guint8 *target, int relocation)
924 {
925 switch (relocation) {
926 case MONO_R_ARM64_B:
927 if (arm_is_bl_disp (code, target)) {
928 arm_b (code, target);
929 } else {
930 gpointer thunk;
931
932 thunk = create_thunk (cfg, domain, code, target);
933 g_assert (arm_is_bl_disp (code, thunk));
934 arm_b (code, thunk);
935 }
936 break;
937 case MONO_R_ARM64_BCC: {
938 int cond;
939
940 cond = arm_get_bcc_cond (code);
941 arm_bcc (code, cond, target);
942 break;
943 }
944 case MONO_R_ARM64_CBZ:
945 arm_set_cbz_target (code, target);
946 break;
947 case MONO_R_ARM64_IMM: {
948 guint64 imm = (guint64)target;
949 int dreg;
950
951 /* emit_imm64_template () */
952 dreg = arm_get_movzx_rd (code);
953 arm_movzx (code, dreg, imm & 0xffff, 0);
954 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
955 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
956 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
957 break;
958 }
959 case MONO_R_ARM64_BL:
960 if (arm_is_bl_disp (code, target)) {
961 arm_bl (code, target);
962 } else {
963 gpointer thunk;
964
965 thunk = create_thunk (cfg, domain, code, target);
966 g_assert (arm_is_bl_disp (code, thunk));
967 arm_bl (code, thunk);
968 }
969 break;
970 default:
971 g_assert_not_reached ();
972 }
973 }
974
975 static void
976 arm_patch_rel (guint8 *code, guint8 *target, int relocation)
977 {
978 arm_patch_full (NULL, NULL, code, target, relocation);
979 }
980
981 void
982 mono_arm_patch (guint8 *code, guint8 *target, int relocation)
983 {
984 arm_patch_rel (code, target, relocation);
985 }
986
987 void
988 mono_arch_patch_code_new (MonoCompile *cfg, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, gpointer target)
989 {
990 guint8 *ip;
991
992 ip = ji->ip.i + code;
993
994 switch (ji->type) {
995 case MONO_PATCH_INFO_METHOD_JUMP:
996 /* ji->relocation is not set by the caller */
997 arm_patch_full (cfg, domain, ip, (guint8*)target, MONO_R_ARM64_B);
998 break;
999 default:
1000 arm_patch_full (cfg, domain, ip, (guint8*)target, ji->relocation);
1001 break;
1002 }
1003 }
1004
1005 void
1006 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
1007 {
1008 }
1009
1010 void
1011 mono_arch_flush_register_windows (void)
1012 {
1013 }
1014
1015 MonoMethod*
1016 mono_arch_find_imt_method (mgreg_t *regs, guint8 *code)
1017 {
1018 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1019 }
1020
1021 MonoVTable*
1022 mono_arch_find_static_call_vtable (mgreg_t *regs, guint8 *code)
1023 {
1024 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1025 }
1026
1027 mgreg_t
1028 mono_arch_context_get_int_reg (MonoContext *ctx, int reg)
1029 {
1030 return ctx->regs [reg];
1031 }
1032
1033 void
1034 mono_arch_context_set_int_reg (MonoContext *ctx, int reg, mgreg_t val)
1035 {
1036 ctx->regs [reg] = val;
1037 }
1038
1039 /*
1040 * mono_arch_set_target:
1041 *
1042 * Set the target architecture the JIT backend should generate code for, in the form
1043 * of a GNU target triplet. Only used in AOT mode.
1044 */
1045 void
1046 mono_arch_set_target (char *mtriple)
1047 {
1048 if (strstr (mtriple, "darwin") || strstr (mtriple, "ios")) {
1049 ios_abi = TRUE;
1050 }
1051 }
1052
1053 static void
1054 add_general (CallInfo *cinfo, ArgInfo *ainfo, int size, gboolean sign)
1055 {
1056 if (cinfo->gr >= PARAM_REGS) {
1057 ainfo->storage = ArgOnStack;
1058 if (ios_abi) {
1059 /* Assume size == align */
1060 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1061 ainfo->offset = cinfo->stack_usage;
1062 ainfo->slot_size = size;
1063 ainfo->sign = sign;
1064 cinfo->stack_usage += size;
1065 } else {
1066 ainfo->offset = cinfo->stack_usage;
1067 ainfo->slot_size = 8;
1068 ainfo->sign = FALSE;
1069 /* Put arguments into 8 byte aligned stack slots */
1070 cinfo->stack_usage += 8;
1071 }
1072 } else {
1073 ainfo->storage = ArgInIReg;
1074 ainfo->reg = cinfo->gr;
1075 cinfo->gr ++;
1076 }
1077 }
1078
1079 static void
1080 add_fp (CallInfo *cinfo, ArgInfo *ainfo, gboolean single)
1081 {
1082 int size = single ? 4 : 8;
1083
1084 if (cinfo->fr >= FP_PARAM_REGS) {
1085 ainfo->storage = single ? ArgOnStackR4 : ArgOnStackR8;
1086 if (ios_abi) {
1087 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1088 ainfo->offset = cinfo->stack_usage;
1089 ainfo->slot_size = size;
1090 cinfo->stack_usage += size;
1091 } else {
1092 ainfo->offset = cinfo->stack_usage;
1093 ainfo->slot_size = 8;
1094 /* Put arguments into 8 byte aligned stack slots */
1095 cinfo->stack_usage += 8;
1096 }
1097 } else {
1098 if (single)
1099 ainfo->storage = ArgInFRegR4;
1100 else
1101 ainfo->storage = ArgInFReg;
1102 ainfo->reg = cinfo->fr;
1103 cinfo->fr ++;
1104 }
1105 }
1106
1107 static gboolean
1108 is_hfa (MonoType *t, int *out_nfields, int *out_esize, int *field_offsets)
1109 {
1110 MonoClass *klass;
1111 gpointer iter;
1112 MonoClassField *field;
1113 MonoType *ftype, *prev_ftype = NULL;
1114 int i, nfields = 0;
1115
1116 klass = mono_class_from_mono_type (t);
1117 iter = NULL;
1118 while ((field = mono_class_get_fields (klass, &iter))) {
1119 if (field->type->attrs & FIELD_ATTRIBUTE_STATIC)
1120 continue;
1121 ftype = mono_field_get_type (field);
1122 ftype = mini_get_underlying_type (ftype);
1123
1124 if (MONO_TYPE_ISSTRUCT (ftype)) {
1125 int nested_nfields, nested_esize;
1126 int nested_field_offsets [16];
1127
1128 if (!is_hfa (ftype, &nested_nfields, &nested_esize, nested_field_offsets))
1129 return FALSE;
1130 if (nested_esize == 4)
1131 ftype = &mono_defaults.single_class->byval_arg;
1132 else
1133 ftype = &mono_defaults.double_class->byval_arg;
1134 if (prev_ftype && prev_ftype->type != ftype->type)
1135 return FALSE;
1136 prev_ftype = ftype;
1137 for (i = 0; i < nested_nfields; ++i) {
1138 if (nfields + i < 4)
1139 field_offsets [nfields + i] = field->offset - sizeof (MonoObject) + nested_field_offsets [i];
1140 }
1141 nfields += nested_nfields;
1142 } else {
1143 if (!(!ftype->byref && (ftype->type == MONO_TYPE_R4 || ftype->type == MONO_TYPE_R8)))
1144 return FALSE;
1145 if (prev_ftype && prev_ftype->type != ftype->type)
1146 return FALSE;
1147 prev_ftype = ftype;
1148 if (nfields < 4)
1149 field_offsets [nfields] = field->offset - sizeof (MonoObject);
1150 nfields ++;
1151 }
1152 }
1153 if (nfields == 0 || nfields > 4)
1154 return FALSE;
1155 *out_nfields = nfields;
1156 *out_esize = prev_ftype->type == MONO_TYPE_R4 ? 4 : 8;
1157 return TRUE;
1158 }
1159
1160 static void
1161 add_valuetype (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1162 {
1163 int i, size, align_size, nregs, nfields, esize;
1164 int field_offsets [16];
1165 guint32 align;
1166
1167 size = mini_type_stack_size_full (t, &align, cinfo->pinvoke);
1168 align_size = ALIGN_TO (size, 8);
1169
1170 nregs = align_size / 8;
1171 if (is_hfa (t, &nfields, &esize, field_offsets)) {
1172 /*
1173 * The struct might include nested float structs aligned at 8,
1174 * so need to keep track of the offsets of the individual fields.
1175 */
1176 if (cinfo->fr + nfields <= FP_PARAM_REGS) {
1177 ainfo->storage = ArgHFA;
1178 ainfo->reg = cinfo->fr;
1179 ainfo->nregs = nfields;
1180 ainfo->size = size;
1181 ainfo->esize = esize;
1182 for (i = 0; i < nfields; ++i)
1183 ainfo->foffsets [i] = field_offsets [i];
1184 cinfo->fr += ainfo->nregs;
1185 } else {
1186 ainfo->nfregs_to_skip = FP_PARAM_REGS > cinfo->fr ? FP_PARAM_REGS - cinfo->fr : 0;
1187 cinfo->fr = FP_PARAM_REGS;
1188 size = ALIGN_TO (size, 8);
1189 ainfo->storage = ArgVtypeOnStack;
1190 ainfo->offset = cinfo->stack_usage;
1191 ainfo->size = size;
1192 ainfo->hfa = TRUE;
1193 ainfo->nregs = nfields;
1194 ainfo->esize = esize;
1195 cinfo->stack_usage += size;
1196 }
1197 return;
1198 }
1199
1200 if (align_size > 16) {
1201 ainfo->storage = ArgVtypeByRef;
1202 ainfo->size = size;
1203 return;
1204 }
1205
1206 if (cinfo->gr + nregs > PARAM_REGS) {
1207 size = ALIGN_TO (size, 8);
1208 ainfo->storage = ArgVtypeOnStack;
1209 ainfo->offset = cinfo->stack_usage;
1210 ainfo->size = size;
1211 cinfo->stack_usage += size;
1212 cinfo->gr = PARAM_REGS;
1213 } else {
1214 ainfo->storage = ArgVtypeInIRegs;
1215 ainfo->reg = cinfo->gr;
1216 ainfo->nregs = nregs;
1217 ainfo->size = size;
1218 cinfo->gr += nregs;
1219 }
1220 }
1221
1222 static void
1223 add_param (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1224 {
1225 MonoType *ptype;
1226
1227 ptype = mini_get_underlying_type (t);
1228 switch (ptype->type) {
1229 case MONO_TYPE_I1:
1230 add_general (cinfo, ainfo, 1, TRUE);
1231 break;
1232 case MONO_TYPE_U1:
1233 add_general (cinfo, ainfo, 1, FALSE);
1234 break;
1235 case MONO_TYPE_I2:
1236 add_general (cinfo, ainfo, 2, TRUE);
1237 break;
1238 case MONO_TYPE_U2:
1239 add_general (cinfo, ainfo, 2, FALSE);
1240 break;
1241 case MONO_TYPE_I4:
1242 add_general (cinfo, ainfo, 4, TRUE);
1243 break;
1244 case MONO_TYPE_U4:
1245 add_general (cinfo, ainfo, 4, FALSE);
1246 break;
1247 case MONO_TYPE_I:
1248 case MONO_TYPE_U:
1249 case MONO_TYPE_PTR:
1250 case MONO_TYPE_FNPTR:
1251 case MONO_TYPE_OBJECT:
1252 case MONO_TYPE_U8:
1253 case MONO_TYPE_I8:
1254 add_general (cinfo, ainfo, 8, FALSE);
1255 break;
1256 case MONO_TYPE_R8:
1257 add_fp (cinfo, ainfo, FALSE);
1258 break;
1259 case MONO_TYPE_R4:
1260 add_fp (cinfo, ainfo, TRUE);
1261 break;
1262 case MONO_TYPE_VALUETYPE:
1263 case MONO_TYPE_TYPEDBYREF:
1264 add_valuetype (cinfo, ainfo, ptype);
1265 break;
1266 case MONO_TYPE_VOID:
1267 ainfo->storage = ArgNone;
1268 break;
1269 case MONO_TYPE_GENERICINST:
1270 if (!mono_type_generic_inst_is_valuetype (ptype)) {
1271 add_general (cinfo, ainfo, 8, FALSE);
1272 } else if (mini_is_gsharedvt_variable_type (ptype)) {
1273 /*
1274 * Treat gsharedvt arguments as large vtypes
1275 */
1276 ainfo->storage = ArgVtypeByRef;
1277 ainfo->gsharedvt = TRUE;
1278 } else {
1279 add_valuetype (cinfo, ainfo, ptype);
1280 }
1281 break;
1282 case MONO_TYPE_VAR:
1283 case MONO_TYPE_MVAR:
1284 g_assert (mini_is_gsharedvt_type (ptype));
1285 ainfo->storage = ArgVtypeByRef;
1286 ainfo->gsharedvt = TRUE;
1287 break;
1288 default:
1289 g_assert_not_reached ();
1290 break;
1291 }
1292 }
1293
1294 /*
1295 * get_call_info:
1296 *
1297 * Obtain information about a call according to the calling convention.
1298 */
1299 static CallInfo*
1300 get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
1301 {
1302 CallInfo *cinfo;
1303 ArgInfo *ainfo;
1304 int n, pstart, pindex;
1305
1306 n = sig->hasthis + sig->param_count;
1307
1308 if (mp)
1309 cinfo = mono_mempool_alloc0 (mp, sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1310 else
1311 cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1312
1313 cinfo->nargs = n;
1314 cinfo->pinvoke = sig->pinvoke;
1315
1316 /* Return value */
1317 add_param (cinfo, &cinfo->ret, sig->ret);
1318 if (cinfo->ret.storage == ArgVtypeByRef)
1319 cinfo->ret.reg = ARMREG_R8;
1320 /* Reset state */
1321 cinfo->gr = 0;
1322 cinfo->fr = 0;
1323 cinfo->stack_usage = 0;
1324
1325 /* Parameters */
1326 if (sig->hasthis)
1327 add_general (cinfo, cinfo->args + 0, 8, FALSE);
1328 pstart = 0;
1329 for (pindex = pstart; pindex < sig->param_count; ++pindex) {
1330 ainfo = cinfo->args + sig->hasthis + pindex;
1331
1332 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1333 /* Prevent implicit arguments and sig_cookie from
1334 being passed in registers */
1335 cinfo->gr = PARAM_REGS;
1336 cinfo->fr = FP_PARAM_REGS;
1337 /* Emit the signature cookie just before the implicit arguments */
1338 add_param (cinfo, &cinfo->sig_cookie, &mono_defaults.int_class->byval_arg);
1339 }
1340
1341 add_param (cinfo, ainfo, sig->params [pindex]);
1342 if (ainfo->storage == ArgVtypeByRef) {
1343 /* Pass the argument address in the next register */
1344 if (cinfo->gr >= PARAM_REGS) {
1345 ainfo->storage = ArgVtypeByRefOnStack;
1346 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, 8);
1347 ainfo->offset = cinfo->stack_usage;
1348 cinfo->stack_usage += 8;
1349 } else {
1350 ainfo->reg = cinfo->gr;
1351 cinfo->gr ++;
1352 }
1353 }
1354 }
1355
1356 /* Handle the case where there are no implicit arguments */
1357 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1358 /* Prevent implicit arguments and sig_cookie from
1359 being passed in registers */
1360 cinfo->gr = PARAM_REGS;
1361 cinfo->fr = FP_PARAM_REGS;
1362 /* Emit the signature cookie just before the implicit arguments */
1363 add_param (cinfo, &cinfo->sig_cookie, &mono_defaults.int_class->byval_arg);
1364 }
1365
1366 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, MONO_ARCH_FRAME_ALIGNMENT);
1367
1368 return cinfo;
1369 }
1370
1371 typedef struct {
1372 MonoMethodSignature *sig;
1373 CallInfo *cinfo;
1374 MonoType *rtype;
1375 MonoType **param_types;
1376 int n_fpargs, n_fpret;
1377 } ArchDynCallInfo;
1378
1379 static gboolean
1380 dyn_call_supported (CallInfo *cinfo, MonoMethodSignature *sig)
1381 {
1382 int i;
1383
1384 if (sig->hasthis + sig->param_count > PARAM_REGS + DYN_CALL_STACK_ARGS)
1385 return FALSE;
1386
1387 // FIXME: Add more cases
1388 switch (cinfo->ret.storage) {
1389 case ArgNone:
1390 case ArgInIReg:
1391 case ArgInFReg:
1392 case ArgInFRegR4:
1393 case ArgVtypeByRef:
1394 break;
1395 case ArgVtypeInIRegs:
1396 if (cinfo->ret.nregs > 2)
1397 return FALSE;
1398 break;
1399 case ArgHFA:
1400 break;
1401 default:
1402 return FALSE;
1403 }
1404
1405 for (i = 0; i < cinfo->nargs; ++i) {
1406 ArgInfo *ainfo = &cinfo->args [i];
1407
1408 switch (ainfo->storage) {
1409 case ArgInIReg:
1410 case ArgVtypeInIRegs:
1411 case ArgInFReg:
1412 case ArgInFRegR4:
1413 case ArgHFA:
1414 case ArgVtypeByRef:
1415 break;
1416 case ArgOnStack:
1417 if (ainfo->offset >= DYN_CALL_STACK_ARGS * sizeof (mgreg_t))
1418 return FALSE;
1419 break;
1420 default:
1421 return FALSE;
1422 }
1423 }
1424
1425 return TRUE;
1426 }
1427
1428 MonoDynCallInfo*
1429 mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
1430 {
1431 ArchDynCallInfo *info;
1432 CallInfo *cinfo;
1433 int i;
1434
1435 cinfo = get_call_info (NULL, sig);
1436
1437 if (!dyn_call_supported (cinfo, sig)) {
1438 g_free (cinfo);
1439 return NULL;
1440 }
1441
1442 info = g_new0 (ArchDynCallInfo, 1);
1443 // FIXME: Preprocess the info to speed up start_dyn_call ()
1444 info->sig = sig;
1445 info->cinfo = cinfo;
1446 info->rtype = mini_get_underlying_type (sig->ret);
1447 info->param_types = g_new0 (MonoType*, sig->param_count);
1448 for (i = 0; i < sig->param_count; ++i)
1449 info->param_types [i] = mini_get_underlying_type (sig->params [i]);
1450
1451 switch (cinfo->ret.storage) {
1452 case ArgInFReg:
1453 case ArgInFRegR4:
1454 info->n_fpret = 1;
1455 break;
1456 case ArgHFA:
1457 info->n_fpret = cinfo->ret.nregs;
1458 break;
1459 default:
1460 break;
1461 }
1462
1463 return (MonoDynCallInfo*)info;
1464 }
1465
1466 void
1467 mono_arch_dyn_call_free (MonoDynCallInfo *info)
1468 {
1469 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1470
1471 g_free (ainfo->cinfo);
1472 g_free (ainfo->param_types);
1473 g_free (ainfo);
1474 }
1475
1476 static double
1477 bitcast_r4_to_r8 (float f)
1478 {
1479 float *p = &f;
1480
1481 return *(double*)p;
1482 }
1483
1484 static float
1485 bitcast_r8_to_r4 (double f)
1486 {
1487 double *p = &f;
1488
1489 return *(float*)p;
1490 }
1491
1492 void
1493 mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, guint8 *buf, int buf_len)
1494 {
1495 ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
1496 DynCallArgs *p = (DynCallArgs*)buf;
1497 int aindex, arg_index, greg, i, pindex;
1498 MonoMethodSignature *sig = dinfo->sig;
1499 CallInfo *cinfo = dinfo->cinfo;
1500 int buffer_offset = 0;
1501
1502 g_assert (buf_len >= sizeof (DynCallArgs));
1503
1504 p->res = 0;
1505 p->ret = ret;
1506 p->n_fpargs = dinfo->n_fpargs;
1507 p->n_fpret = dinfo->n_fpret;
1508
1509 arg_index = 0;
1510 greg = 0;
1511 pindex = 0;
1512
1513 if (sig->hasthis)
1514 p->regs [greg ++] = (mgreg_t)*(args [arg_index ++]);
1515
1516 if (cinfo->ret.storage == ArgVtypeByRef)
1517 p->regs [ARMREG_R8] = (mgreg_t)ret;
1518
1519 for (aindex = pindex; aindex < sig->param_count; aindex++) {
1520 MonoType *t = dinfo->param_types [aindex];
1521 gpointer *arg = args [arg_index ++];
1522 ArgInfo *ainfo = &cinfo->args [aindex + sig->hasthis];
1523 int slot = -1;
1524
1525 if (ainfo->storage == ArgOnStack) {
1526 slot = PARAM_REGS + 1 + (ainfo->offset / sizeof (mgreg_t));
1527 } else {
1528 slot = ainfo->reg;
1529 }
1530
1531 if (t->byref) {
1532 p->regs [slot] = (mgreg_t)*arg;
1533 continue;
1534 }
1535
1536 if (ios_abi && ainfo->storage == ArgOnStack) {
1537 guint8 *stack_arg = (guint8*)&(p->regs [PARAM_REGS + 1]) + ainfo->offset;
1538 gboolean handled = TRUE;
1539
1540 /* Special case arguments smaller than 1 machine word */
1541 switch (t->type) {
1542 case MONO_TYPE_U1:
1543 *(guint8*)stack_arg = *(guint8*)arg;
1544 break;
1545 case MONO_TYPE_I1:
1546 *(gint8*)stack_arg = *(gint8*)arg;
1547 break;
1548 case MONO_TYPE_U2:
1549 *(guint16*)stack_arg = *(guint16*)arg;
1550 break;
1551 case MONO_TYPE_I2:
1552 *(gint16*)stack_arg = *(gint16*)arg;
1553 break;
1554 case MONO_TYPE_I4:
1555 *(gint32*)stack_arg = *(gint32*)arg;
1556 break;
1557 case MONO_TYPE_U4:
1558 *(guint32*)stack_arg = *(guint32*)arg;
1559 break;
1560 default:
1561 handled = FALSE;
1562 break;
1563 }
1564 if (handled)
1565 continue;
1566 }
1567
1568 switch (t->type) {
1569 case MONO_TYPE_OBJECT:
1570 case MONO_TYPE_PTR:
1571 case MONO_TYPE_I:
1572 case MONO_TYPE_U:
1573 case MONO_TYPE_I8:
1574 case MONO_TYPE_U8:
1575 p->regs [slot] = (mgreg_t)*arg;
1576 break;
1577 case MONO_TYPE_U1:
1578 p->regs [slot] = *(guint8*)arg;
1579 break;
1580 case MONO_TYPE_I1:
1581 p->regs [slot] = *(gint8*)arg;
1582 break;
1583 case MONO_TYPE_I2:
1584 p->regs [slot] = *(gint16*)arg;
1585 break;
1586 case MONO_TYPE_U2:
1587 p->regs [slot] = *(guint16*)arg;
1588 break;
1589 case MONO_TYPE_I4:
1590 p->regs [slot] = *(gint32*)arg;
1591 break;
1592 case MONO_TYPE_U4:
1593 p->regs [slot] = *(guint32*)arg;
1594 break;
1595 case MONO_TYPE_R4:
1596 p->fpregs [ainfo->reg] = bitcast_r4_to_r8 (*(float*)arg);
1597 p->n_fpargs ++;
1598 break;
1599 case MONO_TYPE_R8:
1600 p->fpregs [ainfo->reg] = *(double*)arg;
1601 p->n_fpargs ++;
1602 break;
1603 case MONO_TYPE_GENERICINST:
1604 if (MONO_TYPE_IS_REFERENCE (t)) {
1605 p->regs [slot] = (mgreg_t)*arg;
1606 break;
1607 } else {
1608 if (t->type == MONO_TYPE_GENERICINST && mono_class_is_nullable (mono_class_from_mono_type (t))) {
1609 MonoClass *klass = mono_class_from_mono_type (t);
1610 guint8 *nullable_buf;
1611 int size;
1612
1613 /*
1614 * Use p->buffer as a temporary buffer since the data needs to be available after this call
1615 * if the nullable param is passed by ref.
1616 */
1617 size = mono_class_value_size (klass, NULL);
1618 nullable_buf = p->buffer + buffer_offset;
1619 buffer_offset += size;
1620 g_assert (buffer_offset <= 256);
1621
1622 /* The argument pointed to by arg is either a boxed vtype or null */
1623 mono_nullable_init (nullable_buf, (MonoObject*)arg, klass);
1624
1625 arg = (gpointer*)nullable_buf;
1626 /* Fall though */
1627 } else {
1628 /* Fall though */
1629 }
1630 }
1631 case MONO_TYPE_VALUETYPE:
1632 switch (ainfo->storage) {
1633 case ArgVtypeInIRegs:
1634 for (i = 0; i < ainfo->nregs; ++i)
1635 p->regs [slot ++] = ((mgreg_t*)arg) [i];
1636 break;
1637 case ArgHFA:
1638 if (ainfo->esize == 4) {
1639 for (i = 0; i < ainfo->nregs; ++i)
1640 p->fpregs [ainfo->reg + i] = bitcast_r4_to_r8 (((float*)arg) [ainfo->foffsets [i] / 4]);
1641 } else {
1642 for (i = 0; i < ainfo->nregs; ++i)
1643 p->fpregs [ainfo->reg + i] = ((double*)arg) [ainfo->foffsets [i] / 8];
1644 }
1645 p->n_fpargs += ainfo->nregs;
1646 break;
1647 case ArgVtypeByRef:
1648 p->regs [slot] = (mgreg_t)arg;
1649 break;
1650 default:
1651 g_assert_not_reached ();
1652 break;
1653 }
1654 break;
1655 default:
1656 g_assert_not_reached ();
1657 }
1658 }
1659 }
1660
1661 void
1662 mono_arch_finish_dyn_call (MonoDynCallInfo *info, guint8 *buf)
1663 {
1664 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1665 CallInfo *cinfo = ainfo->cinfo;
1666 DynCallArgs *args = (DynCallArgs*)buf;
1667 MonoType *ptype = ainfo->rtype;
1668 guint8 *ret = args->ret;
1669 mgreg_t res = args->res;
1670 mgreg_t res2 = args->res2;
1671 int i;
1672
1673 if (cinfo->ret.storage == ArgVtypeByRef)
1674 return;
1675
1676 switch (ptype->type) {
1677 case MONO_TYPE_VOID:
1678 *(gpointer*)ret = NULL;
1679 break;
1680 case MONO_TYPE_OBJECT:
1681 case MONO_TYPE_I:
1682 case MONO_TYPE_U:
1683 case MONO_TYPE_PTR:
1684 *(gpointer*)ret = (gpointer)res;
1685 break;
1686 case MONO_TYPE_I1:
1687 *(gint8*)ret = res;
1688 break;
1689 case MONO_TYPE_U1:
1690 *(guint8*)ret = res;
1691 break;
1692 case MONO_TYPE_I2:
1693 *(gint16*)ret = res;
1694 break;
1695 case MONO_TYPE_U2:
1696 *(guint16*)ret = res;
1697 break;
1698 case MONO_TYPE_I4:
1699 *(gint32*)ret = res;
1700 break;
1701 case MONO_TYPE_U4:
1702 *(guint32*)ret = res;
1703 break;
1704 case MONO_TYPE_I8:
1705 case MONO_TYPE_U8:
1706 *(guint64*)ret = res;
1707 break;
1708 case MONO_TYPE_R4:
1709 *(float*)ret = bitcast_r8_to_r4 (args->fpregs [0]);
1710 break;
1711 case MONO_TYPE_R8:
1712 *(double*)ret = args->fpregs [0];
1713 break;
1714 case MONO_TYPE_GENERICINST:
1715 if (MONO_TYPE_IS_REFERENCE (ptype)) {
1716 *(gpointer*)ret = (gpointer)res;
1717 break;
1718 } else {
1719 /* Fall though */
1720 }
1721 case MONO_TYPE_VALUETYPE:
1722 switch (ainfo->cinfo->ret.storage) {
1723 case ArgVtypeInIRegs:
1724 *(mgreg_t*)ret = res;
1725 if (ainfo->cinfo->ret.nregs > 1)
1726 ((mgreg_t*)ret) [1] = res2;
1727 break;
1728 case ArgHFA:
1729 /* Use the same area for returning fp values */
1730 if (cinfo->ret.esize == 4) {
1731 for (i = 0; i < cinfo->ret.nregs; ++i)
1732 ((float*)ret) [cinfo->ret.foffsets [i] / 4] = bitcast_r8_to_r4 (args->fpregs [i]);
1733 } else {
1734 for (i = 0; i < cinfo->ret.nregs; ++i)
1735 ((double*)ret) [cinfo->ret.foffsets [i] / 8] = args->fpregs [i];
1736 }
1737 break;
1738 default:
1739 g_assert_not_reached ();
1740 break;
1741 }
1742 break;
1743 default:
1744 g_assert_not_reached ();
1745 }
1746 }
1747
1748 #if __APPLE__
1749 void sys_icache_invalidate (void *start, size_t len);
1750 #endif
1751
1752 void
1753 mono_arch_flush_icache (guint8 *code, gint size)
1754 {
1755 #ifndef MONO_CROSS_COMPILE
1756 #if __APPLE__
1757 sys_icache_invalidate (code, size);
1758 #else
1759 /* Don't rely on GCC's __clear_cache implementation, as it caches
1760 * icache/dcache cache line sizes, that can vary between cores on
1761 * big.LITTLE architectures. */
1762 guint64 end = (guint64) (code + size);
1763 guint64 addr;
1764 /* always go with cacheline size of 4 bytes as this code isn't perf critical
1765 * anyway. Reading the cache line size from a machine register can be racy
1766 * on a big.LITTLE architecture if the cores don't have the same cache line
1767 * sizes. */
1768 const size_t icache_line_size = 4;
1769 const size_t dcache_line_size = 4;
1770
1771 addr = (guint64) code & ~(guint64) (dcache_line_size - 1);
1772 for (; addr < end; addr += dcache_line_size)
1773 asm volatile("dc civac, %0" : : "r" (addr) : "memory");
1774 asm volatile("dsb ish" : : : "memory");
1775
1776 addr = (guint64) code & ~(guint64) (icache_line_size - 1);
1777 for (; addr < end; addr += icache_line_size)
1778 asm volatile("ic ivau, %0" : : "r" (addr) : "memory");
1779
1780 asm volatile ("dsb ish" : : : "memory");
1781 asm volatile ("isb" : : : "memory");
1782 #endif
1783 #endif
1784 }
1785
1786 #ifndef DISABLE_JIT
1787
1788 gboolean
1789 mono_arch_opcode_needs_emulation (MonoCompile *cfg, int opcode)
1790 {
1791 NOT_IMPLEMENTED;
1792 return FALSE;
1793 }
1794
1795 GList *
1796 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
1797 {
1798 GList *vars = NULL;
1799 int i;
1800
1801 for (i = 0; i < cfg->num_varinfo; i++) {
1802 MonoInst *ins = cfg->varinfo [i];
1803 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
1804
1805 /* unused vars */
1806 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
1807 continue;
1808
1809 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
1810 (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
1811 continue;
1812
1813 if (mono_is_regsize_var (ins->inst_vtype)) {
1814 g_assert (MONO_VARINFO (cfg, i)->reg == -1);
1815 g_assert (i == vmv->idx);
1816 vars = g_list_prepend (vars, vmv);
1817 }
1818 }
1819
1820 vars = mono_varlist_sort (cfg, vars, 0);
1821
1822 return vars;
1823 }
1824
1825 GList *
1826 mono_arch_get_global_int_regs (MonoCompile *cfg)
1827 {
1828 GList *regs = NULL;
1829 int i;
1830
1831 /* r28 is reserved for cfg->arch.args_reg */
1832 /* r27 is reserved for the imt argument */
1833 for (i = ARMREG_R19; i <= ARMREG_R26; ++i)
1834 regs = g_list_prepend (regs, GUINT_TO_POINTER (i));
1835
1836 return regs;
1837 }
1838
1839 guint32
1840 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
1841 {
1842 MonoInst *ins = cfg->varinfo [vmv->idx];
1843
1844 if (ins->opcode == OP_ARG)
1845 return 1;
1846 else
1847 return 2;
1848 }
1849
1850 void
1851 mono_arch_create_vars (MonoCompile *cfg)
1852 {
1853 MonoMethodSignature *sig;
1854 CallInfo *cinfo;
1855
1856 sig = mono_method_signature (cfg->method);
1857 if (!cfg->arch.cinfo)
1858 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
1859 cinfo = cfg->arch.cinfo;
1860
1861 if (cinfo->ret.storage == ArgVtypeByRef) {
1862 cfg->vret_addr = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1863 cfg->vret_addr->flags |= MONO_INST_VOLATILE;
1864 }
1865
1866 if (cfg->gen_sdb_seq_points) {
1867 MonoInst *ins;
1868
1869 if (cfg->compile_aot) {
1870 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1871 ins->flags |= MONO_INST_VOLATILE;
1872 cfg->arch.seq_point_info_var = ins;
1873 }
1874
1875 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1876 ins->flags |= MONO_INST_VOLATILE;
1877 cfg->arch.ss_tramp_var = ins;
1878
1879 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1880 ins->flags |= MONO_INST_VOLATILE;
1881 cfg->arch.bp_tramp_var = ins;
1882 }
1883
1884 if (cfg->method->save_lmf) {
1885 cfg->create_lmf_var = TRUE;
1886 cfg->lmf_ir = TRUE;
1887 }
1888 }
1889
1890 void
1891 mono_arch_allocate_vars (MonoCompile *cfg)
1892 {
1893 MonoMethodSignature *sig;
1894 MonoInst *ins;
1895 CallInfo *cinfo;
1896 ArgInfo *ainfo;
1897 int i, offset, size, align;
1898 guint32 locals_stack_size, locals_stack_align;
1899 gint32 *offsets;
1900
1901 /*
1902 * Allocate arguments and locals to either register (OP_REGVAR) or to a stack slot (OP_REGOFFSET).
1903 * Compute cfg->stack_offset and update cfg->used_int_regs.
1904 */
1905
1906 sig = mono_method_signature (cfg->method);
1907
1908 if (!cfg->arch.cinfo)
1909 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
1910 cinfo = cfg->arch.cinfo;
1911
1912 /*
1913 * The ARM64 ABI always uses a frame pointer.
1914 * The instruction set prefers positive offsets, so fp points to the bottom of the
1915 * frame, and stack slots are at positive offsets.
1916 * If some arguments are received on the stack, their offsets relative to fp can
1917 * not be computed right now because the stack frame might grow due to spilling
1918 * done by the local register allocator. To solve this, we reserve a register
1919 * which points to them.
1920 * The stack frame looks like this:
1921 * args_reg -> <bottom of parent frame>
1922 * <locals etc>
1923 * fp -> <saved fp+lr>
1924 * sp -> <localloc/params area>
1925 */
1926 cfg->frame_reg = ARMREG_FP;
1927 cfg->flags |= MONO_CFG_HAS_SPILLUP;
1928 offset = 0;
1929
1930 /* Saved fp+lr */
1931 offset += 16;
1932
1933 if (cinfo->stack_usage) {
1934 g_assert (!(cfg->used_int_regs & (1 << ARMREG_R28)));
1935 cfg->arch.args_reg = ARMREG_R28;
1936 cfg->used_int_regs |= 1 << ARMREG_R28;
1937 }
1938
1939 if (cfg->method->save_lmf) {
1940 /* The LMF var is allocated normally */
1941 } else {
1942 /* Callee saved regs */
1943 cfg->arch.saved_gregs_offset = offset;
1944 for (i = 0; i < 32; ++i)
1945 if ((MONO_ARCH_CALLEE_SAVED_REGS & (1 << i)) && (cfg->used_int_regs & (1 << i)))
1946 offset += 8;
1947 }
1948
1949 /* Return value */
1950 switch (cinfo->ret.storage) {
1951 case ArgNone:
1952 break;
1953 case ArgInIReg:
1954 case ArgInFReg:
1955 case ArgInFRegR4:
1956 cfg->ret->opcode = OP_REGVAR;
1957 cfg->ret->dreg = cinfo->ret.reg;
1958 break;
1959 case ArgVtypeInIRegs:
1960 case ArgHFA:
1961 /* Allocate a local to hold the result, the epilog will copy it to the correct place */
1962 cfg->ret->opcode = OP_REGOFFSET;
1963 cfg->ret->inst_basereg = cfg->frame_reg;
1964 cfg->ret->inst_offset = offset;
1965 if (cinfo->ret.storage == ArgHFA)
1966 // FIXME:
1967 offset += 64;
1968 else
1969 offset += 16;
1970 break;
1971 case ArgVtypeByRef:
1972 /* This variable will be initalized in the prolog from R8 */
1973 cfg->vret_addr->opcode = OP_REGOFFSET;
1974 cfg->vret_addr->inst_basereg = cfg->frame_reg;
1975 cfg->vret_addr->inst_offset = offset;
1976 offset += 8;
1977 if (G_UNLIKELY (cfg->verbose_level > 1)) {
1978 printf ("vret_addr =");
1979 mono_print_ins (cfg->vret_addr);
1980 }
1981 break;
1982 default:
1983 g_assert_not_reached ();
1984 break;
1985 }
1986
1987 /* Arguments */
1988 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1989 ainfo = cinfo->args + i;
1990
1991 ins = cfg->args [i];
1992 if (ins->opcode == OP_REGVAR)
1993 continue;
1994
1995 ins->opcode = OP_REGOFFSET;
1996 ins->inst_basereg = cfg->frame_reg;
1997
1998 switch (ainfo->storage) {
1999 case ArgInIReg:
2000 case ArgInFReg:
2001 case ArgInFRegR4:
2002 // FIXME: Use nregs/size
2003 /* These will be copied to the stack in the prolog */
2004 ins->inst_offset = offset;
2005 offset += 8;
2006 break;
2007 case ArgOnStack:
2008 case ArgOnStackR4:
2009 case ArgOnStackR8:
2010 case ArgVtypeOnStack:
2011 /* These are in the parent frame */
2012 g_assert (cfg->arch.args_reg);
2013 ins->inst_basereg = cfg->arch.args_reg;
2014 ins->inst_offset = ainfo->offset;
2015 break;
2016 case ArgVtypeInIRegs:
2017 case ArgHFA:
2018 ins->opcode = OP_REGOFFSET;
2019 ins->inst_basereg = cfg->frame_reg;
2020 /* These arguments are saved to the stack in the prolog */
2021 ins->inst_offset = offset;
2022 if (cfg->verbose_level >= 2)
2023 printf ("arg %d allocated to %s+0x%0x.\n", i, mono_arch_regname (ins->inst_basereg), (int)ins->inst_offset);
2024 if (ainfo->storage == ArgHFA)
2025 // FIXME:
2026 offset += 64;
2027 else
2028 offset += 16;
2029 break;
2030 case ArgVtypeByRefOnStack: {
2031 MonoInst *vtaddr;
2032
2033 if (ainfo->gsharedvt) {
2034 ins->opcode = OP_REGOFFSET;
2035 ins->inst_basereg = cfg->arch.args_reg;
2036 ins->inst_offset = ainfo->offset;
2037 break;
2038 }
2039
2040 /* The vtype address is in the parent frame */
2041 g_assert (cfg->arch.args_reg);
2042 MONO_INST_NEW (cfg, vtaddr, 0);
2043 vtaddr->opcode = OP_REGOFFSET;
2044 vtaddr->inst_basereg = cfg->arch.args_reg;
2045 vtaddr->inst_offset = ainfo->offset;
2046
2047 /* Need an indirection */
2048 ins->opcode = OP_VTARG_ADDR;
2049 ins->inst_left = vtaddr;
2050 break;
2051 }
2052 case ArgVtypeByRef: {
2053 MonoInst *vtaddr;
2054
2055 if (ainfo->gsharedvt) {
2056 ins->opcode = OP_REGOFFSET;
2057 ins->inst_basereg = cfg->frame_reg;
2058 ins->inst_offset = offset;
2059 offset += 8;
2060 break;
2061 }
2062
2063 /* The vtype address is in a register, will be copied to the stack in the prolog */
2064 MONO_INST_NEW (cfg, vtaddr, 0);
2065 vtaddr->opcode = OP_REGOFFSET;
2066 vtaddr->inst_basereg = cfg->frame_reg;
2067 vtaddr->inst_offset = offset;
2068 offset += 8;
2069
2070 /* Need an indirection */
2071 ins->opcode = OP_VTARG_ADDR;
2072 ins->inst_left = vtaddr;
2073 break;
2074 }
2075 default:
2076 g_assert_not_reached ();
2077 break;
2078 }
2079 }
2080
2081 /* Allocate these first so they have a small offset, OP_SEQ_POINT depends on this */
2082 // FIXME: Allocate these to registers
2083 ins = cfg->arch.seq_point_info_var;
2084 if (ins) {
2085 size = 8;
2086 align = 8;
2087 offset += align - 1;
2088 offset &= ~(align - 1);
2089 ins->opcode = OP_REGOFFSET;
2090 ins->inst_basereg = cfg->frame_reg;
2091 ins->inst_offset = offset;
2092 offset += size;
2093 }
2094 ins = cfg->arch.ss_tramp_var;
2095 if (ins) {
2096 size = 8;
2097 align = 8;
2098 offset += align - 1;
2099 offset &= ~(align - 1);
2100 ins->opcode = OP_REGOFFSET;
2101 ins->inst_basereg = cfg->frame_reg;
2102 ins->inst_offset = offset;
2103 offset += size;
2104 }
2105 ins = cfg->arch.bp_tramp_var;
2106 if (ins) {
2107 size = 8;
2108 align = 8;
2109 offset += align - 1;
2110 offset &= ~(align - 1);
2111 ins->opcode = OP_REGOFFSET;
2112 ins->inst_basereg = cfg->frame_reg;
2113 ins->inst_offset = offset;
2114 offset += size;
2115 }
2116
2117 /* Locals */
2118 offsets = mono_allocate_stack_slots (cfg, FALSE, &locals_stack_size, &locals_stack_align);
2119 if (locals_stack_align)
2120 offset = ALIGN_TO (offset, locals_stack_align);
2121
2122 for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
2123 if (offsets [i] != -1) {
2124 ins = cfg->varinfo [i];
2125 ins->opcode = OP_REGOFFSET;
2126 ins->inst_basereg = cfg->frame_reg;
2127 ins->inst_offset = offset + offsets [i];
2128 //printf ("allocated local %d to ", i); mono_print_tree_nl (ins);
2129 }
2130 }
2131 offset += locals_stack_size;
2132
2133 offset = ALIGN_TO (offset, MONO_ARCH_FRAME_ALIGNMENT);
2134
2135 cfg->stack_offset = offset;
2136 }
2137
2138 #ifdef ENABLE_LLVM
2139 LLVMCallInfo*
2140 mono_arch_get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
2141 {
2142 int i, n;
2143 CallInfo *cinfo;
2144 ArgInfo *ainfo;
2145 LLVMCallInfo *linfo;
2146
2147 n = sig->param_count + sig->hasthis;
2148
2149 cinfo = get_call_info (cfg->mempool, sig);
2150
2151 linfo = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
2152
2153 switch (cinfo->ret.storage) {
2154 case ArgInIReg:
2155 case ArgInFReg:
2156 case ArgInFRegR4:
2157 case ArgNone:
2158 break;
2159 case ArgVtypeByRef:
2160 linfo->ret.storage = LLVMArgVtypeByRef;
2161 break;
2162 //
2163 // FIXME: This doesn't work yet since the llvm backend represents these types as an i8
2164 // array which is returned in int regs
2165 //
2166 case ArgHFA:
2167 linfo->ret.storage = LLVMArgFpStruct;
2168 linfo->ret.nslots = cinfo->ret.nregs;
2169 linfo->ret.esize = cinfo->ret.esize;
2170 break;
2171 case ArgVtypeInIRegs:
2172 /* LLVM models this by returning an int */
2173 linfo->ret.storage = LLVMArgVtypeAsScalar;
2174 linfo->ret.nslots = cinfo->ret.nregs;
2175 linfo->ret.esize = cinfo->ret.esize;
2176 break;
2177 default:
2178 g_assert_not_reached ();
2179 break;
2180 }
2181
2182 for (i = 0; i < n; ++i) {
2183 LLVMArgInfo *lainfo = &linfo->args [i];
2184
2185 ainfo = cinfo->args + i;
2186
2187 lainfo->storage = LLVMArgNone;
2188
2189 switch (ainfo->storage) {
2190 case ArgInIReg:
2191 case ArgInFReg:
2192 case ArgInFRegR4:
2193 case ArgOnStack:
2194 case ArgOnStackR4:
2195 case ArgOnStackR8:
2196 lainfo->storage = LLVMArgNormal;
2197 break;
2198 case ArgVtypeByRef:
2199 case ArgVtypeByRefOnStack:
2200 lainfo->storage = LLVMArgVtypeByRef;
2201 break;
2202 case ArgHFA: {
2203 int j;
2204
2205 lainfo->storage = LLVMArgAsFpArgs;
2206 lainfo->nslots = ainfo->nregs;
2207 lainfo->esize = ainfo->esize;
2208 for (j = 0; j < ainfo->nregs; ++j)
2209 lainfo->pair_storage [j] = LLVMArgInFPReg;
2210 break;
2211 }
2212 case ArgVtypeInIRegs:
2213 lainfo->storage = LLVMArgAsIArgs;
2214 lainfo->nslots = ainfo->nregs;
2215 break;
2216 case ArgVtypeOnStack:
2217 if (ainfo->hfa) {
2218 int j;
2219 /* Same as above */
2220 lainfo->storage = LLVMArgAsFpArgs;
2221 lainfo->nslots = ainfo->nregs;
2222 lainfo->esize = ainfo->esize;
2223 lainfo->ndummy_fpargs = ainfo->nfregs_to_skip;
2224 for (j = 0; j < ainfo->nregs; ++j)
2225 lainfo->pair_storage [j] = LLVMArgInFPReg;
2226 } else {
2227 lainfo->storage = LLVMArgAsIArgs;
2228 lainfo->nslots = ainfo->size / 8;
2229 }
2230 break;
2231 default:
2232 g_assert_not_reached ();
2233 break;
2234 }
2235 }
2236
2237 return linfo;
2238 }
2239 #endif
2240
2241 static void
2242 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, ArgStorage storage, int reg, MonoInst *arg)
2243 {
2244 MonoInst *ins;
2245
2246 switch (storage) {
2247 case ArgInIReg:
2248 MONO_INST_NEW (cfg, ins, OP_MOVE);
2249 ins->dreg = mono_alloc_ireg_copy (cfg, arg->dreg);
2250 ins->sreg1 = arg->dreg;
2251 MONO_ADD_INS (cfg->cbb, ins);
2252 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, FALSE);
2253 break;
2254 case ArgInFReg:
2255 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2256 ins->dreg = mono_alloc_freg (cfg);
2257 ins->sreg1 = arg->dreg;
2258 MONO_ADD_INS (cfg->cbb, ins);
2259 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2260 break;
2261 case ArgInFRegR4:
2262 if (COMPILE_LLVM (cfg))
2263 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2264 else if (cfg->r4fp)
2265 MONO_INST_NEW (cfg, ins, OP_RMOVE);
2266 else
2267 MONO_INST_NEW (cfg, ins, OP_ARM_SETFREG_R4);
2268 ins->dreg = mono_alloc_freg (cfg);
2269 ins->sreg1 = arg->dreg;
2270 MONO_ADD_INS (cfg->cbb, ins);
2271 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2272 break;
2273 default:
2274 g_assert_not_reached ();
2275 break;
2276 }
2277 }
2278
2279 static void
2280 emit_sig_cookie (MonoCompile *cfg, MonoCallInst *call, CallInfo *cinfo)
2281 {
2282 MonoMethodSignature *tmp_sig;
2283 int sig_reg;
2284
2285 if (call->tail_call)
2286 NOT_IMPLEMENTED;
2287
2288 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
2289
2290 /*
2291 * mono_ArgIterator_Setup assumes the signature cookie is
2292 * passed first and all the arguments which were before it are
2293 * passed on the stack after the signature. So compensate by
2294 * passing a different signature.
2295 */
2296 tmp_sig = mono_metadata_signature_dup (call->signature);
2297 tmp_sig->param_count -= call->signature->sentinelpos;
2298 tmp_sig->sentinelpos = 0;
2299 memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
2300
2301 sig_reg = mono_alloc_ireg (cfg);
2302 MONO_EMIT_NEW_SIGNATURECONST (cfg, sig_reg, tmp_sig);
2303
2304 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, cinfo->sig_cookie.offset, sig_reg);
2305 }
2306
2307 void
2308 mono_arch_emit_call (MonoCompile *cfg, MonoCallInst *call)
2309 {
2310 MonoMethodSignature *sig;
2311 MonoInst *arg, *vtarg;
2312 CallInfo *cinfo;
2313 ArgInfo *ainfo;
2314 int i;
2315
2316 sig = call->signature;
2317
2318 cinfo = get_call_info (cfg->mempool, sig);
2319
2320 switch (cinfo->ret.storage) {
2321 case ArgVtypeInIRegs:
2322 case ArgHFA:
2323 /*
2324 * The vtype is returned in registers, save the return area address in a local, and save the vtype into
2325 * the location pointed to by it after call in emit_move_return_value ().
2326 */
2327 if (!cfg->arch.vret_addr_loc) {
2328 cfg->arch.vret_addr_loc = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2329 /* Prevent it from being register allocated or optimized away */
2330 ((MonoInst*)cfg->arch.vret_addr_loc)->flags |= MONO_INST_VOLATILE;
2331 }
2332
2333 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, ((MonoInst*)cfg->arch.vret_addr_loc)->dreg, call->vret_var->dreg);
2334 break;
2335 case ArgVtypeByRef:
2336 /* Pass the vtype return address in R8 */
2337 MONO_INST_NEW (cfg, vtarg, OP_MOVE);
2338 vtarg->sreg1 = call->vret_var->dreg;
2339 vtarg->dreg = mono_alloc_preg (cfg);
2340 MONO_ADD_INS (cfg->cbb, vtarg);
2341
2342 mono_call_inst_add_outarg_reg (cfg, call, vtarg->dreg, cinfo->ret.reg, FALSE);
2343 break;
2344 default:
2345 break;
2346 }
2347
2348 for (i = 0; i < cinfo->nargs; ++i) {
2349 ainfo = cinfo->args + i;
2350 arg = call->args [i];
2351
2352 if ((sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
2353 /* Emit the signature cookie just before the implicit arguments */
2354 emit_sig_cookie (cfg, call, cinfo);
2355 }
2356
2357 switch (ainfo->storage) {
2358 case ArgInIReg:
2359 case ArgInFReg:
2360 case ArgInFRegR4:
2361 add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, arg);
2362 break;
2363 case ArgOnStack:
2364 switch (ainfo->slot_size) {
2365 case 8:
2366 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2367 break;
2368 case 4:
2369 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2370 break;
2371 case 2:
2372 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI2_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2373 break;
2374 case 1:
2375 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2376 break;
2377 default:
2378 g_assert_not_reached ();
2379 break;
2380 }
2381 break;
2382 case ArgOnStackR8:
2383 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER8_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2384 break;
2385 case ArgOnStackR4:
2386 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2387 break;
2388 case ArgVtypeInIRegs:
2389 case ArgVtypeByRef:
2390 case ArgVtypeByRefOnStack:
2391 case ArgVtypeOnStack:
2392 case ArgHFA: {
2393 MonoInst *ins;
2394 guint32 align;
2395 guint32 size;
2396
2397 size = mono_class_value_size (arg->klass, &align);
2398
2399 MONO_INST_NEW (cfg, ins, OP_OUTARG_VT);
2400 ins->sreg1 = arg->dreg;
2401 ins->klass = arg->klass;
2402 ins->backend.size = size;
2403 ins->inst_p0 = call;
2404 ins->inst_p1 = mono_mempool_alloc (cfg->mempool, sizeof (ArgInfo));
2405 memcpy (ins->inst_p1, ainfo, sizeof (ArgInfo));
2406 MONO_ADD_INS (cfg->cbb, ins);
2407 break;
2408 }
2409 default:
2410 g_assert_not_reached ();
2411 break;
2412 }
2413 }
2414
2415 /* Handle the case where there are no implicit arguments */
2416 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (cinfo->nargs == sig->sentinelpos))
2417 emit_sig_cookie (cfg, call, cinfo);
2418
2419 call->call_info = cinfo;
2420 call->stack_usage = cinfo->stack_usage;
2421 }
2422
2423 void
2424 mono_arch_emit_outarg_vt (MonoCompile *cfg, MonoInst *ins, MonoInst *src)
2425 {
2426 MonoCallInst *call = (MonoCallInst*)ins->inst_p0;
2427 ArgInfo *ainfo = ins->inst_p1;
2428 MonoInst *load;
2429 int i;
2430
2431 if (ins->backend.size == 0 && !ainfo->gsharedvt)
2432 return;
2433
2434 switch (ainfo->storage) {
2435 case ArgVtypeInIRegs:
2436 for (i = 0; i < ainfo->nregs; ++i) {
2437 // FIXME: Smaller sizes
2438 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2439 load->dreg = mono_alloc_ireg (cfg);
2440 load->inst_basereg = src->dreg;
2441 load->inst_offset = i * sizeof(mgreg_t);
2442 MONO_ADD_INS (cfg->cbb, load);
2443 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg + i, load);
2444 }
2445 break;
2446 case ArgHFA:
2447 for (i = 0; i < ainfo->nregs; ++i) {
2448 if (ainfo->esize == 4)
2449 MONO_INST_NEW (cfg, load, OP_LOADR4_MEMBASE);
2450 else
2451 MONO_INST_NEW (cfg, load, OP_LOADR8_MEMBASE);
2452 load->dreg = mono_alloc_freg (cfg);
2453 load->inst_basereg = src->dreg;
2454 load->inst_offset = ainfo->foffsets [i];
2455 MONO_ADD_INS (cfg->cbb, load);
2456 add_outarg_reg (cfg, call, ainfo->esize == 4 ? ArgInFRegR4 : ArgInFReg, ainfo->reg + i, load);
2457 }
2458 break;
2459 case ArgVtypeByRef:
2460 case ArgVtypeByRefOnStack: {
2461 MonoInst *vtaddr, *load, *arg;
2462
2463 /* Pass the vtype address in a reg/on the stack */
2464 if (ainfo->gsharedvt) {
2465 load = src;
2466 } else {
2467 /* Make a copy of the argument */
2468 vtaddr = mono_compile_create_var (cfg, &ins->klass->byval_arg, OP_LOCAL);
2469
2470 MONO_INST_NEW (cfg, load, OP_LDADDR);
2471 load->inst_p0 = vtaddr;
2472 vtaddr->flags |= MONO_INST_INDIRECT;
2473 load->type = STACK_MP;
2474 load->klass = vtaddr->klass;
2475 load->dreg = mono_alloc_ireg (cfg);
2476 MONO_ADD_INS (cfg->cbb, load);
2477 mini_emit_memcpy (cfg, load->dreg, 0, src->dreg, 0, ainfo->size, 8);
2478 }
2479
2480 if (ainfo->storage == ArgVtypeByRef) {
2481 MONO_INST_NEW (cfg, arg, OP_MOVE);
2482 arg->dreg = mono_alloc_preg (cfg);
2483 arg->sreg1 = load->dreg;
2484 MONO_ADD_INS (cfg->cbb, arg);
2485 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg, arg);
2486 } else {
2487 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, load->dreg);
2488 }
2489 break;
2490 }
2491 case ArgVtypeOnStack:
2492 for (i = 0; i < ainfo->size / 8; ++i) {
2493 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2494 load->dreg = mono_alloc_ireg (cfg);
2495 load->inst_basereg = src->dreg;
2496 load->inst_offset = i * 8;
2497 MONO_ADD_INS (cfg->cbb, load);
2498 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI8_MEMBASE_REG, ARMREG_SP, ainfo->offset + (i * 8), load->dreg);
2499 }
2500 break;
2501 default:
2502 g_assert_not_reached ();
2503 break;
2504 }
2505 }
2506
2507 void
2508 mono_arch_emit_setret (MonoCompile *cfg, MonoMethod *method, MonoInst *val)
2509 {
2510 MonoMethodSignature *sig;
2511 CallInfo *cinfo;
2512
2513 sig = mono_method_signature (cfg->method);
2514 if (!cfg->arch.cinfo)
2515 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
2516 cinfo = cfg->arch.cinfo;
2517
2518 switch (cinfo->ret.storage) {
2519 case ArgNone:
2520 break;
2521 case ArgInIReg:
2522 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, cfg->ret->dreg, val->dreg);
2523 break;
2524 case ArgInFReg:
2525 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2526 break;
2527 case ArgInFRegR4:
2528 if (COMPILE_LLVM (cfg))
2529 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2530 else if (cfg->r4fp)
2531 MONO_EMIT_NEW_UNALU (cfg, OP_RMOVE, cfg->ret->dreg, val->dreg);
2532 else
2533 MONO_EMIT_NEW_UNALU (cfg, OP_ARM_SETFREG_R4, cfg->ret->dreg, val->dreg);
2534 break;
2535 default:
2536 g_assert_not_reached ();
2537 break;
2538 }
2539 }
2540
2541 gboolean
2542 mono_arch_tail_call_supported (MonoCompile *cfg, MonoMethodSignature *caller_sig, MonoMethodSignature *callee_sig)
2543 {
2544 CallInfo *c1, *c2;
2545 gboolean res;
2546
2547 if (cfg->compile_aot && !cfg->full_aot)
2548 /* OP_TAILCALL doesn't work with AOT */
2549 return FALSE;
2550
2551 c1 = get_call_info (NULL, caller_sig);
2552 c2 = get_call_info (NULL, callee_sig);
2553 res = TRUE;
2554 // FIXME: Relax these restrictions
2555 if (c1->stack_usage != 0)
2556 res = FALSE;
2557 if (c1->stack_usage != c2->stack_usage)
2558 res = FALSE;
2559 if ((c1->ret.storage != ArgNone && c1->ret.storage != ArgInIReg) || c1->ret.storage != c2->ret.storage)
2560 res = FALSE;
2561
2562 g_free (c1);
2563 g_free (c2);
2564
2565 return res;
2566 }
2567
2568 gboolean
2569 mono_arch_is_inst_imm (gint64 imm)
2570 {
2571 return (imm >= -((gint64)1<<31) && imm <= (((gint64)1<<31)-1));
2572 }
2573
2574 void*
2575 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
2576 {
2577 NOT_IMPLEMENTED;
2578 return NULL;
2579 }
2580
2581 void*
2582 mono_arch_instrument_epilog_full (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments, gboolean preserve_argument_registers)
2583 {
2584 NOT_IMPLEMENTED;
2585 return NULL;
2586 }
2587
2588 void
2589 mono_arch_peephole_pass_1 (MonoCompile *cfg, MonoBasicBlock *bb)
2590 {
2591 //NOT_IMPLEMENTED;
2592 }
2593
2594 void
2595 mono_arch_peephole_pass_2 (MonoCompile *cfg, MonoBasicBlock *bb)
2596 {
2597 //NOT_IMPLEMENTED;
2598 }
2599
2600 #define ADD_NEW_INS(cfg,dest,op) do { \
2601 MONO_INST_NEW ((cfg), (dest), (op)); \
2602 mono_bblock_insert_before_ins (bb, ins, (dest)); \
2603 } while (0)
2604
2605 void
2606 mono_arch_lowering_pass (MonoCompile *cfg, MonoBasicBlock *bb)
2607 {
2608 MonoInst *ins, *temp, *last_ins = NULL;
2609
2610 MONO_BB_FOR_EACH_INS (bb, ins) {
2611 switch (ins->opcode) {
2612 case OP_SBB:
2613 case OP_ISBB:
2614 case OP_SUBCC:
2615 case OP_ISUBCC:
2616 if (ins->next && (ins->next->opcode == OP_COND_EXC_C || ins->next->opcode == OP_COND_EXC_IC))
2617 /* ARM sets the C flag to 1 if there was _no_ overflow */
2618 ins->next->opcode = OP_COND_EXC_NC;
2619 break;
2620 case OP_IDIV_IMM:
2621 case OP_IREM_IMM:
2622 case OP_IDIV_UN_IMM:
2623 case OP_IREM_UN_IMM:
2624 case OP_LREM_IMM:
2625 mono_decompose_op_imm (cfg, bb, ins);
2626 break;
2627 case OP_LOCALLOC_IMM:
2628 if (ins->inst_imm > 32) {
2629 ADD_NEW_INS (cfg, temp, OP_ICONST);
2630 temp->inst_c0 = ins->inst_imm;
2631 temp->dreg = mono_alloc_ireg (cfg);
2632 ins->sreg1 = temp->dreg;
2633 ins->opcode = mono_op_imm_to_op (ins->opcode);
2634 }
2635 break;
2636 case OP_ICOMPARE_IMM:
2637 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBEQ) {
2638 ins->next->opcode = OP_ARM64_CBZW;
2639 ins->next->sreg1 = ins->sreg1;
2640 NULLIFY_INS (ins);
2641 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBNE_UN) {
2642 ins->next->opcode = OP_ARM64_CBNZW;
2643 ins->next->sreg1 = ins->sreg1;
2644 NULLIFY_INS (ins);
2645 }
2646 break;
2647 case OP_LCOMPARE_IMM:
2648 case OP_COMPARE_IMM:
2649 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBEQ) {
2650 ins->next->opcode = OP_ARM64_CBZX;
2651 ins->next->sreg1 = ins->sreg1;
2652 NULLIFY_INS (ins);
2653 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBNE_UN) {
2654 ins->next->opcode = OP_ARM64_CBNZX;
2655 ins->next->sreg1 = ins->sreg1;
2656 NULLIFY_INS (ins);
2657 }
2658 break;
2659 case OP_FCOMPARE: {
2660 gboolean swap = FALSE;
2661 int reg;
2662
2663 if (!ins->next) {
2664 /* Optimized away */
2665 NULLIFY_INS (ins);
2666 break;
2667 }
2668
2669 /*
2670 * FP compares with unordered operands set the flags
2671 * to NZCV=0011, which matches some non-unordered compares
2672 * as well, like LE, so have to swap the operands.
2673 */
2674 switch (ins->next->opcode) {
2675 case OP_FBLT:
2676 ins->next->opcode = OP_FBGT;
2677 swap = TRUE;
2678 break;
2679 case OP_FBLE:
2680 ins->next->opcode = OP_FBGE;
2681 swap = TRUE;
2682 break;
2683 default:
2684 break;
2685 }
2686 if (swap) {
2687 reg = ins->sreg1;
2688 ins->sreg1 = ins->sreg2;
2689 ins->sreg2 = reg;
2690 }
2691 break;
2692 }
2693 default:
2694 break;
2695 }
2696
2697 last_ins = ins;
2698 }
2699 bb->last_ins = last_ins;
2700 bb->max_vreg = cfg->next_vreg;
2701 }
2702
2703 void
2704 mono_arch_decompose_long_opts (MonoCompile *cfg, MonoInst *long_ins)
2705 {
2706 }
2707
2708 static int
2709 opcode_to_armcond (int opcode)
2710 {
2711 switch (opcode) {
2712 case OP_IBEQ:
2713 case OP_LBEQ:
2714 case OP_FBEQ:
2715 case OP_CEQ:
2716 case OP_ICEQ:
2717 case OP_LCEQ:
2718 case OP_FCEQ:
2719 case OP_RCEQ:
2720 case OP_COND_EXC_IEQ:
2721 case OP_COND_EXC_EQ:
2722 return ARMCOND_EQ;
2723 case OP_IBGE:
2724 case OP_LBGE:
2725 case OP_FBGE:
2726 case OP_ICGE:
2727 case OP_FCGE:
2728 case OP_RCGE:
2729 return ARMCOND_GE;
2730 case OP_IBGT:
2731 case OP_LBGT:
2732 case OP_FBGT:
2733 case OP_CGT:
2734 case OP_ICGT:
2735 case OP_LCGT:
2736 case OP_FCGT:
2737 case OP_RCGT:
2738 case OP_COND_EXC_IGT:
2739 case OP_COND_EXC_GT:
2740 return ARMCOND_GT;
2741 case OP_IBLE:
2742 case OP_LBLE:
2743 case OP_FBLE:
2744 case OP_ICLE:
2745 case OP_FCLE:
2746 case OP_RCLE:
2747 return ARMCOND_LE;
2748 case OP_IBLT:
2749 case OP_LBLT:
2750 case OP_FBLT:
2751 case OP_CLT:
2752 case OP_ICLT:
2753 case OP_LCLT:
2754 case OP_COND_EXC_ILT:
2755 case OP_COND_EXC_LT:
2756 return ARMCOND_LT;
2757 case OP_IBNE_UN:
2758 case OP_LBNE_UN:
2759 case OP_FBNE_UN:
2760 case OP_ICNEQ:
2761 case OP_FCNEQ:
2762 case OP_RCNEQ:
2763 case OP_COND_EXC_INE_UN:
2764 case OP_COND_EXC_NE_UN:
2765 return ARMCOND_NE;
2766 case OP_IBGE_UN:
2767 case OP_LBGE_UN:
2768 case OP_FBGE_UN:
2769 case OP_ICGE_UN:
2770 case OP_COND_EXC_IGE_UN:
2771 case OP_COND_EXC_GE_UN:
2772 return ARMCOND_HS;
2773 case OP_IBGT_UN:
2774 case OP_LBGT_UN:
2775 case OP_FBGT_UN:
2776 case OP_CGT_UN:
2777 case OP_ICGT_UN:
2778 case OP_LCGT_UN:
2779 case OP_FCGT_UN:
2780 case OP_RCGT_UN:
2781 case OP_COND_EXC_IGT_UN:
2782 case OP_COND_EXC_GT_UN:
2783 return ARMCOND_HI;
2784 case OP_IBLE_UN:
2785 case OP_LBLE_UN:
2786 case OP_FBLE_UN:
2787 case OP_ICLE_UN:
2788 case OP_COND_EXC_ILE_UN:
2789 case OP_COND_EXC_LE_UN:
2790 return ARMCOND_LS;
2791 case OP_IBLT_UN:
2792 case OP_LBLT_UN:
2793 case OP_FBLT_UN:
2794 case OP_CLT_UN:
2795 case OP_ICLT_UN:
2796 case OP_LCLT_UN:
2797 case OP_COND_EXC_ILT_UN:
2798 case OP_COND_EXC_LT_UN:
2799 return ARMCOND_LO;
2800 /*
2801 * FCMP sets the NZCV condition bits as follows:
2802 * eq = 0110
2803 * < = 1000
2804 * > = 0010
2805 * unordered = 0011
2806 * ARMCOND_LT is N!=V, so it matches unordered too, so
2807 * fclt and fclt_un need to be special cased.
2808 */
2809 case OP_FCLT:
2810 case OP_RCLT:
2811 /* N==1 */
2812 return ARMCOND_MI;
2813 case OP_FCLT_UN:
2814 case OP_RCLT_UN:
2815 return ARMCOND_LT;
2816 case OP_COND_EXC_C:
2817 case OP_COND_EXC_IC:
2818 return ARMCOND_CS;
2819 case OP_COND_EXC_OV:
2820 case OP_COND_EXC_IOV:
2821 return ARMCOND_VS;
2822 case OP_COND_EXC_NC:
2823 case OP_COND_EXC_INC:
2824 return ARMCOND_CC;
2825 case OP_COND_EXC_NO:
2826 case OP_COND_EXC_INO:
2827 return ARMCOND_VC;
2828 default:
2829 printf ("%s\n", mono_inst_name (opcode));
2830 g_assert_not_reached ();
2831 return -1;
2832 }
2833 }
2834
2835 /* This clobbers LR */
2836 static inline __attribute__ ((__warn_unused_result__)) guint8*
2837 emit_cond_exc (MonoCompile *cfg, guint8 *code, int opcode, const char *exc_name)
2838 {
2839 int cond;
2840
2841 cond = opcode_to_armcond (opcode);
2842 /* Capture PC */
2843 arm_adrx (code, ARMREG_IP1, code);
2844 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_EXC, exc_name, MONO_R_ARM64_BCC);
2845 arm_bcc (code, cond, 0);
2846 return code;
2847 }
2848
2849 static guint8*
2850 emit_move_return_value (MonoCompile *cfg, guint8 * code, MonoInst *ins)
2851 {
2852 CallInfo *cinfo;
2853 MonoCallInst *call;
2854
2855 call = (MonoCallInst*)ins;
2856 cinfo = call->call_info;
2857 g_assert (cinfo);
2858 switch (cinfo->ret.storage) {
2859 case ArgNone:
2860 break;
2861 case ArgInIReg:
2862 /* LLVM compiled code might only set the bottom bits */
2863 if (call->signature && mini_get_underlying_type (call->signature->ret)->type == MONO_TYPE_I4)
2864 arm_sxtwx (code, call->inst.dreg, cinfo->ret.reg);
2865 else if (call->inst.dreg != cinfo->ret.reg)
2866 arm_movx (code, call->inst.dreg, cinfo->ret.reg);
2867 break;
2868 case ArgInFReg:
2869 if (call->inst.dreg != cinfo->ret.reg)
2870 arm_fmovd (code, call->inst.dreg, cinfo->ret.reg);
2871 break;
2872 case ArgInFRegR4:
2873 if (cfg->r4fp)
2874 arm_fmovs (code, call->inst.dreg, cinfo->ret.reg);
2875 else
2876 arm_fcvt_sd (code, call->inst.dreg, cinfo->ret.reg);
2877 break;
2878 case ArgVtypeInIRegs: {
2879 MonoInst *loc = cfg->arch.vret_addr_loc;
2880 int i;
2881
2882 /* Load the destination address */
2883 g_assert (loc && loc->opcode == OP_REGOFFSET);
2884 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
2885 for (i = 0; i < cinfo->ret.nregs; ++i)
2886 arm_strx (code, cinfo->ret.reg + i, ARMREG_LR, i * 8);
2887 break;
2888 }
2889 case ArgHFA: {
2890 MonoInst *loc = cfg->arch.vret_addr_loc;
2891 int i;
2892
2893 /* Load the destination address */
2894 g_assert (loc && loc->opcode == OP_REGOFFSET);
2895 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
2896 for (i = 0; i < cinfo->ret.nregs; ++i) {
2897 if (cinfo->ret.esize == 4)
2898 arm_strfpw (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
2899 else
2900 arm_strfpx (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
2901 }
2902 break;
2903 }
2904 case ArgVtypeByRef:
2905 break;
2906 default:
2907 g_assert_not_reached ();
2908 break;
2909 }
2910 return code;
2911 }
2912
2913 /*
2914 * emit_branch_island:
2915 *
2916 * Emit a branch island for the conditional branches from cfg->native_code + start_offset to code.
2917 */
2918 static guint8*
2919 emit_branch_island (MonoCompile *cfg, guint8 *code, int start_offset)
2920 {
2921 MonoJumpInfo *ji;
2922 int offset, island_size;
2923
2924 /* Iterate over the patch infos added so far by this bb */
2925 island_size = 0;
2926 for (ji = cfg->patch_info; ji; ji = ji->next) {
2927 if (ji->ip.i < start_offset)
2928 /* The patch infos are in reverse order, so this means the end */
2929 break;
2930 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ)
2931 island_size += 4;
2932 }
2933
2934 if (island_size) {
2935 offset = code - cfg->native_code;
2936 if (offset > (cfg->code_size - island_size - 16)) {
2937 cfg->code_size *= 2;
2938 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
2939 code = cfg->native_code + offset;
2940 }
2941
2942 /* Branch over the island */
2943 arm_b (code, code + 4 + island_size);
2944
2945 for (ji = cfg->patch_info; ji; ji = ji->next) {
2946 if (ji->ip.i < start_offset)
2947 break;
2948 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ) {
2949 /* Rewrite the cond branch so it branches to an uncoditional branch in the branch island */
2950 arm_patch_rel (cfg->native_code + ji->ip.i, code, ji->relocation);
2951 /* Rewrite the patch so it points to the unconditional branch */
2952 ji->ip.i = code - cfg->native_code;
2953 ji->relocation = MONO_R_ARM64_B;
2954 arm_b (code, code);
2955 }
2956 }
2957 }
2958 return code;
2959 }
2960
2961 void
2962 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
2963 {
2964 MonoInst *ins;
2965 MonoCallInst *call;
2966 guint offset;
2967 guint8 *code = cfg->native_code + cfg->code_len;
2968 int start_offset, max_len, dreg, sreg1, sreg2;
2969 mgreg_t imm;
2970
2971 if (cfg->verbose_level > 2)
2972 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
2973
2974 start_offset = code - cfg->native_code;
2975
2976 MONO_BB_FOR_EACH_INS (bb, ins) {
2977 offset = code - cfg->native_code;
2978
2979 max_len = ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
2980
2981 if (offset > (cfg->code_size - max_len - 16)) {
2982 cfg->code_size *= 2;
2983 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
2984 code = cfg->native_code + offset;
2985 }
2986
2987 if (G_UNLIKELY (cfg->arch.cond_branch_islands && offset - start_offset > 4 * 0x1ffff)) {
2988 /* Emit a branch island for large basic blocks */
2989 code = emit_branch_island (cfg, code, start_offset);
2990 offset = code - cfg->native_code;
2991 start_offset = offset;
2992 }
2993
2994 mono_debug_record_line_number (cfg, ins, offset);
2995
2996 dreg = ins->dreg;
2997 sreg1 = ins->sreg1;
2998 sreg2 = ins->sreg2;
2999 imm = ins->inst_imm;
3000
3001 switch (ins->opcode) {
3002 case OP_ICONST:
3003 code = emit_imm (code, dreg, ins->inst_c0);
3004 break;
3005 case OP_I8CONST:
3006 code = emit_imm64 (code, dreg, ins->inst_c0);
3007 break;
3008 case OP_MOVE:
3009 if (dreg != sreg1)
3010 arm_movx (code, dreg, sreg1);
3011 break;
3012 case OP_NOP:
3013 case OP_RELAXED_NOP:
3014 break;
3015 case OP_JUMP_TABLE:
3016 mono_add_patch_info_rel (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0, MONO_R_ARM64_IMM);
3017 code = emit_imm64_template (code, dreg);
3018 break;
3019 case OP_BREAK:
3020 /*
3021 * gdb does not like encountering the hw breakpoint ins in the debugged code.
3022 * So instead of emitting a trap, we emit a call a C function and place a
3023 * breakpoint there.
3024 */
3025 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, (gpointer)"mono_break");
3026 break;
3027 case OP_LOCALLOC: {
3028 guint8 *buf [16];
3029
3030 arm_addx_imm (code, ARMREG_IP0, sreg1, (MONO_ARCH_FRAME_ALIGNMENT - 1));
3031 // FIXME: andx_imm doesn't work yet
3032 code = emit_imm (code, ARMREG_IP1, -MONO_ARCH_FRAME_ALIGNMENT);
3033 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3034 //arm_andx_imm (code, ARMREG_IP0, sreg1, - MONO_ARCH_FRAME_ALIGNMENT);
3035 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
3036 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
3037 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
3038
3039 /* Init */
3040 /* ip1 = pointer, ip0 = end */
3041 arm_addx (code, ARMREG_IP0, ARMREG_IP1, ARMREG_IP0);
3042 buf [0] = code;
3043 arm_cmpx (code, ARMREG_IP1, ARMREG_IP0);
3044 buf [1] = code;
3045 arm_bcc (code, ARMCOND_EQ, 0);
3046 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_IP1, 0);
3047 arm_addx_imm (code, ARMREG_IP1, ARMREG_IP1, 16);
3048 arm_b (code, buf [0]);
3049 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3050
3051 arm_movspx (code, dreg, ARMREG_SP);
3052 if (cfg->param_area)
3053 code = emit_subx_sp_imm (code, cfg->param_area);
3054 break;
3055 }
3056 case OP_LOCALLOC_IMM: {
3057 int imm, offset;
3058
3059 imm = ALIGN_TO (ins->inst_imm, MONO_ARCH_FRAME_ALIGNMENT);
3060 g_assert (arm_is_arith_imm (imm));
3061 arm_subx_imm (code, ARMREG_SP, ARMREG_SP, imm);
3062
3063 /* Init */
3064 g_assert (MONO_ARCH_FRAME_ALIGNMENT == 16);
3065 offset = 0;
3066 while (offset < imm) {
3067 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_SP, offset);
3068 offset += 16;
3069 }
3070 arm_movspx (code, dreg, ARMREG_SP);
3071 if (cfg->param_area)
3072 code = emit_subx_sp_imm (code, cfg->param_area);
3073 break;
3074 }
3075 case OP_AOTCONST:
3076 code = emit_aotconst (cfg, code, dreg, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3077 break;
3078 case OP_OBJC_GET_SELECTOR:
3079 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_OBJC_SELECTOR_REF, ins->inst_p0);
3080 /* See arch_emit_objc_selector_ref () in aot-compiler.c */
3081 arm_ldrx_lit (code, ins->dreg, 0);
3082 arm_nop (code);
3083 arm_nop (code);
3084 break;
3085 case OP_SEQ_POINT: {
3086 MonoInst *info_var = cfg->arch.seq_point_info_var;
3087
3088 /*
3089 * For AOT, we use one got slot per method, which will point to a
3090 * SeqPointInfo structure, containing all the information required
3091 * by the code below.
3092 */
3093 if (cfg->compile_aot) {
3094 g_assert (info_var);
3095 g_assert (info_var->opcode == OP_REGOFFSET);
3096 }
3097
3098 if (ins->flags & MONO_INST_SINGLE_STEP_LOC) {
3099 MonoInst *var = cfg->arch.ss_tramp_var;
3100
3101 g_assert (var);
3102 g_assert (var->opcode == OP_REGOFFSET);
3103 /* Load ss_tramp_var */
3104 /* This is equal to &ss_trampoline */
3105 arm_ldrx (code, ARMREG_IP1, var->inst_basereg, var->inst_offset);
3106 /* Load the trampoline address */
3107 arm_ldrx (code, ARMREG_IP1, ARMREG_IP1, 0);
3108 /* Call it if it is non-null */
3109 arm_cbzx (code, ARMREG_IP1, code + 8);
3110 arm_blrx (code, ARMREG_IP1);
3111 }
3112
3113 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
3114
3115 if (cfg->compile_aot) {
3116 guint32 offset = code - cfg->native_code;
3117 guint32 val;
3118
3119 arm_ldrx (code, ARMREG_IP1, info_var->inst_basereg, info_var->inst_offset);
3120 /* Add the offset */
3121 val = ((offset / 4) * sizeof (guint8*)) + MONO_STRUCT_OFFSET (SeqPointInfo, bp_addrs);
3122 /* Load the info->bp_addrs [offset], which is either 0 or the address of the bp trampoline */
3123 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP1, val);
3124 /* Skip the load if its 0 */
3125 arm_cbzx (code, ARMREG_IP1, code + 8);
3126 /* Call the breakpoint trampoline */
3127 arm_blrx (code, ARMREG_IP1);
3128 } else {
3129 MonoInst *var = cfg->arch.bp_tramp_var;
3130
3131 g_assert (var);
3132 g_assert (var->opcode == OP_REGOFFSET);
3133 /* Load the address of the bp trampoline into IP0 */
3134 arm_ldrx (code, ARMREG_IP0, var->inst_basereg, var->inst_offset);
3135 /*
3136 * A placeholder for a possible breakpoint inserted by
3137 * mono_arch_set_breakpoint ().
3138 */
3139 arm_nop (code);
3140 }
3141 break;
3142 }
3143
3144 /* BRANCH */
3145 case OP_BR:
3146 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_B);
3147 arm_b (code, code);
3148 break;
3149 case OP_BR_REG:
3150 arm_brx (code, sreg1);
3151 break;
3152 case OP_IBEQ:
3153 case OP_IBGE:
3154 case OP_IBGT:
3155 case OP_IBLE:
3156 case OP_IBLT:
3157 case OP_IBNE_UN:
3158 case OP_IBGE_UN:
3159 case OP_IBGT_UN:
3160 case OP_IBLE_UN:
3161 case OP_IBLT_UN:
3162 case OP_LBEQ:
3163 case OP_LBGE:
3164 case OP_LBGT:
3165 case OP_LBLE:
3166 case OP_LBLT:
3167 case OP_LBNE_UN:
3168 case OP_LBGE_UN:
3169 case OP_LBGT_UN:
3170 case OP_LBLE_UN:
3171 case OP_LBLT_UN:
3172 case OP_FBEQ:
3173 case OP_FBNE_UN:
3174 case OP_FBLT:
3175 case OP_FBGT:
3176 case OP_FBGT_UN:
3177 case OP_FBLE:
3178 case OP_FBGE:
3179 case OP_FBGE_UN: {
3180 int cond;
3181
3182 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3183 cond = opcode_to_armcond (ins->opcode);
3184 arm_bcc (code, cond, 0);
3185 break;
3186 }
3187 case OP_FBLT_UN:
3188 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3189 /* For fp compares, ARMCOND_LT is lt or unordered */
3190 arm_bcc (code, ARMCOND_LT, 0);
3191 break;
3192 case OP_FBLE_UN:
3193 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3194 arm_bcc (code, ARMCOND_EQ, 0);
3195 offset = code - cfg->native_code;
3196 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3197 /* For fp compares, ARMCOND_LT is lt or unordered */
3198 arm_bcc (code, ARMCOND_LT, 0);
3199 break;
3200 case OP_ARM64_CBZW:
3201 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3202 arm_cbzw (code, sreg1, 0);
3203 break;
3204 case OP_ARM64_CBZX:
3205 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3206 arm_cbzx (code, sreg1, 0);
3207 break;
3208 case OP_ARM64_CBNZW:
3209 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3210 arm_cbnzw (code, sreg1, 0);
3211 break;
3212 case OP_ARM64_CBNZX:
3213 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3214 arm_cbnzx (code, sreg1, 0);
3215 break;
3216 /* ALU */
3217 case OP_IADD:
3218 arm_addw (code, dreg, sreg1, sreg2);
3219 break;
3220 case OP_LADD:
3221 arm_addx (code, dreg, sreg1, sreg2);
3222 break;
3223 case OP_ISUB:
3224 arm_subw (code, dreg, sreg1, sreg2);
3225 break;
3226 case OP_LSUB:
3227 arm_subx (code, dreg, sreg1, sreg2);
3228 break;
3229 case OP_IAND:
3230 arm_andw (code, dreg, sreg1, sreg2);
3231 break;
3232 case OP_LAND:
3233 arm_andx (code, dreg, sreg1, sreg2);
3234 break;
3235 case OP_IOR:
3236 arm_orrw (code, dreg, sreg1, sreg2);
3237 break;
3238 case OP_LOR:
3239 arm_orrx (code, dreg, sreg1, sreg2);
3240 break;
3241 case OP_IXOR:
3242 arm_eorw (code, dreg, sreg1, sreg2);
3243 break;
3244 case OP_LXOR:
3245 arm_eorx (code, dreg, sreg1, sreg2);
3246 break;
3247 case OP_INEG:
3248 arm_negw (code, dreg, sreg1);
3249 break;
3250 case OP_LNEG:
3251 arm_negx (code, dreg, sreg1);
3252 break;
3253 case OP_INOT:
3254 arm_mvnw (code, dreg, sreg1);
3255 break;
3256 case OP_LNOT:
3257 arm_mvnx (code, dreg, sreg1);
3258 break;
3259 case OP_IADDCC:
3260 arm_addsw (code, dreg, sreg1, sreg2);
3261 break;
3262 case OP_ADDCC:
3263 case OP_LADDCC:
3264 arm_addsx (code, dreg, sreg1, sreg2);
3265 break;
3266 case OP_ISUBCC:
3267 arm_subsw (code, dreg, sreg1, sreg2);
3268 break;
3269 case OP_LSUBCC:
3270 case OP_SUBCC:
3271 arm_subsx (code, dreg, sreg1, sreg2);
3272 break;
3273 case OP_ICOMPARE:
3274 arm_cmpw (code, sreg1, sreg2);
3275 break;
3276 case OP_COMPARE:
3277 case OP_LCOMPARE:
3278 arm_cmpx (code, sreg1, sreg2);
3279 break;
3280 case OP_IADD_IMM:
3281 code = emit_addw_imm (code, dreg, sreg1, imm);
3282 break;
3283 case OP_LADD_IMM:
3284 case OP_ADD_IMM:
3285 code = emit_addx_imm (code, dreg, sreg1, imm);
3286 break;
3287 case OP_ISUB_IMM:
3288 code = emit_subw_imm (code, dreg, sreg1, imm);
3289 break;
3290 case OP_LSUB_IMM:
3291 code = emit_subx_imm (code, dreg, sreg1, imm);
3292 break;
3293 case OP_IAND_IMM:
3294 code = emit_andw_imm (code, dreg, sreg1, imm);
3295 break;
3296 case OP_LAND_IMM:
3297 case OP_AND_IMM:
3298 code = emit_andx_imm (code, dreg, sreg1, imm);
3299 break;
3300 case OP_IOR_IMM:
3301 code = emit_orrw_imm (code, dreg, sreg1, imm);
3302 break;
3303 case OP_LOR_IMM:
3304 code = emit_orrx_imm (code, dreg, sreg1, imm);
3305 break;
3306 case OP_IXOR_IMM:
3307 code = emit_eorw_imm (code, dreg, sreg1, imm);
3308 break;
3309 case OP_LXOR_IMM:
3310 code = emit_eorx_imm (code, dreg, sreg1, imm);
3311 break;
3312 case OP_ICOMPARE_IMM:
3313 code = emit_cmpw_imm (code, sreg1, imm);
3314 break;
3315 case OP_LCOMPARE_IMM:
3316 case OP_COMPARE_IMM:
3317 if (imm == 0) {
3318 arm_cmpx (code, sreg1, ARMREG_RZR);
3319 } else {
3320 // FIXME: 32 vs 64 bit issues for 0xffffffff
3321 code = emit_imm64 (code, ARMREG_LR, imm);
3322 arm_cmpx (code, sreg1, ARMREG_LR);
3323 }
3324 break;
3325 case OP_ISHL:
3326 arm_lslvw (code, dreg, sreg1, sreg2);
3327 break;
3328 case OP_LSHL:
3329 arm_lslvx (code, dreg, sreg1, sreg2);
3330 break;
3331 case OP_ISHR:
3332 arm_asrvw (code, dreg, sreg1, sreg2);
3333 break;
3334 case OP_LSHR:
3335 arm_asrvx (code, dreg, sreg1, sreg2);
3336 break;
3337 case OP_ISHR_UN:
3338 arm_lsrvw (code, dreg, sreg1, sreg2);
3339 break;
3340 case OP_LSHR_UN:
3341 arm_lsrvx (code, dreg, sreg1, sreg2);
3342 break;
3343 case OP_ISHL_IMM:
3344 if (imm == 0)
3345 arm_movx (code, dreg, sreg1);
3346 else
3347 arm_lslw (code, dreg, sreg1, imm);
3348 break;
3349 case OP_LSHL_IMM:
3350 if (imm == 0)
3351 arm_movx (code, dreg, sreg1);
3352 else
3353 arm_lslx (code, dreg, sreg1, imm);
3354 break;
3355 case OP_ISHR_IMM:
3356 if (imm == 0)
3357 arm_movx (code, dreg, sreg1);
3358 else
3359 arm_asrw (code, dreg, sreg1, imm);
3360 break;
3361 case OP_LSHR_IMM:
3362 case OP_SHR_IMM:
3363 if (imm == 0)
3364 arm_movx (code, dreg, sreg1);
3365 else
3366 arm_asrx (code, dreg, sreg1, imm);
3367 break;
3368 case OP_ISHR_UN_IMM:
3369 if (imm == 0)
3370 arm_movx (code, dreg, sreg1);
3371 else
3372 arm_lsrw (code, dreg, sreg1, imm);
3373 break;
3374 case OP_SHR_UN_IMM:
3375 case OP_LSHR_UN_IMM:
3376 if (imm == 0)
3377 arm_movx (code, dreg, sreg1);
3378 else
3379 arm_lsrx (code, dreg, sreg1, imm);
3380 break;
3381
3382 /* 64BIT ALU */
3383 case OP_SEXT_I4:
3384 arm_sxtwx (code, dreg, sreg1);
3385 break;
3386 case OP_ZEXT_I4:
3387 /* Clean out the upper word */
3388 arm_movw (code, dreg, sreg1);
3389 break;
3390 case OP_SHL_IMM:
3391 arm_lslx (code, dreg, sreg1, imm);
3392 break;
3393
3394 /* MULTIPLY/DIVISION */
3395 case OP_IDIV:
3396 case OP_IREM:
3397 // FIXME: Optimize this
3398 /* Check for zero */
3399 arm_cmpx_imm (code, sreg2, 0);
3400 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3401 /* Check for INT_MIN/-1 */
3402 code = emit_imm (code, ARMREG_IP0, 0x80000000);
3403 arm_cmpx (code, sreg1, ARMREG_IP0);
3404 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3405 code = emit_imm (code, ARMREG_IP0, 0xffffffff);
3406 arm_cmpx (code, sreg2, ARMREG_IP0);
3407 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3408 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3409 arm_cmpx_imm (code, ARMREG_IP0, 1);
3410 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "OverflowException");
3411 if (ins->opcode == OP_IREM) {
3412 arm_sdivw (code, ARMREG_LR, sreg1, sreg2);
3413 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3414 } else {
3415 arm_sdivw (code, dreg, sreg1, sreg2);
3416 }
3417 break;
3418 case OP_IDIV_UN:
3419 arm_cmpx_imm (code, sreg2, 0);
3420 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3421 arm_udivw (code, dreg, sreg1, sreg2);
3422 break;
3423 case OP_IREM_UN:
3424 arm_cmpx_imm (code, sreg2, 0);
3425 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3426 arm_udivw (code, ARMREG_LR, sreg1, sreg2);
3427 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3428 break;
3429 case OP_LDIV:
3430 case OP_LREM:
3431 // FIXME: Optimize this
3432 /* Check for zero */
3433 arm_cmpx_imm (code, sreg2, 0);
3434 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3435 /* Check for INT64_MIN/-1 */
3436 code = emit_imm64 (code, ARMREG_IP0, 0x8000000000000000);
3437 arm_cmpx (code, sreg1, ARMREG_IP0);
3438 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3439 code = emit_imm64 (code, ARMREG_IP0, 0xffffffffffffffff);
3440 arm_cmpx (code, sreg2, ARMREG_IP0);
3441 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3442 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3443 arm_cmpx_imm (code, ARMREG_IP0, 1);
3444 /* 64 bit uses ArithmeticException */
3445 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "ArithmeticException");
3446 if (ins->opcode == OP_LREM) {
3447 arm_sdivx (code, ARMREG_LR, sreg1, sreg2);
3448 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3449 } else {
3450 arm_sdivx (code, dreg, sreg1, sreg2);
3451 }
3452 break;
3453 case OP_LDIV_UN:
3454 arm_cmpx_imm (code, sreg2, 0);
3455 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3456 arm_udivx (code, dreg, sreg1, sreg2);
3457 break;
3458 case OP_LREM_UN:
3459 arm_cmpx_imm (code, sreg2, 0);
3460 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3461 arm_udivx (code, ARMREG_LR, sreg1, sreg2);
3462 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3463 break;
3464 case OP_IMUL:
3465 arm_mulw (code, dreg, sreg1, sreg2);
3466 break;
3467 case OP_LMUL:
3468 arm_mulx (code, dreg, sreg1, sreg2);
3469 break;
3470 case OP_IMUL_IMM:
3471 code = emit_imm (code, ARMREG_LR, imm);
3472 arm_mulw (code, dreg, sreg1, ARMREG_LR);
3473 break;
3474 case OP_MUL_IMM:
3475 case OP_LMUL_IMM:
3476 code = emit_imm (code, ARMREG_LR, imm);
3477 arm_mulx (code, dreg, sreg1, ARMREG_LR);
3478 break;
3479
3480 /* CONVERSIONS */
3481 case OP_ICONV_TO_I1:
3482 case OP_LCONV_TO_I1:
3483 arm_sxtbx (code, dreg, sreg1);
3484 break;
3485 case OP_ICONV_TO_I2:
3486 case OP_LCONV_TO_I2:
3487 arm_sxthx (code, dreg, sreg1);
3488 break;
3489 case OP_ICONV_TO_U1:
3490 case OP_LCONV_TO_U1:
3491 arm_uxtbw (code, dreg, sreg1);
3492 break;
3493 case OP_ICONV_TO_U2:
3494 case OP_LCONV_TO_U2:
3495 arm_uxthw (code, dreg, sreg1);
3496 break;
3497
3498 /* CSET */
3499 case OP_CEQ:
3500 case OP_ICEQ:
3501 case OP_LCEQ:
3502 case OP_CLT:
3503 case OP_ICLT:
3504 case OP_LCLT:
3505 case OP_CGT:
3506 case OP_ICGT:
3507 case OP_LCGT:
3508 case OP_CLT_UN:
3509 case OP_ICLT_UN:
3510 case OP_LCLT_UN:
3511 case OP_CGT_UN:
3512 case OP_ICGT_UN:
3513 case OP_LCGT_UN:
3514 case OP_ICNEQ:
3515 case OP_ICGE:
3516 case OP_ICLE:
3517 case OP_ICGE_UN:
3518 case OP_ICLE_UN: {
3519 int cond;
3520
3521 cond = opcode_to_armcond (ins->opcode);
3522 arm_cset (code, cond, dreg);
3523 break;
3524 }
3525 case OP_FCEQ:
3526 case OP_FCLT:
3527 case OP_FCLT_UN:
3528 case OP_FCGT:
3529 case OP_FCGT_UN:
3530 case OP_FCNEQ:
3531 case OP_FCLE:
3532 case OP_FCGE: {
3533 int cond;
3534
3535 cond = opcode_to_armcond (ins->opcode);
3536 arm_fcmpd (code, sreg1, sreg2);
3537 arm_cset (code, cond, dreg);
3538 break;
3539 }
3540
3541 /* MEMORY */
3542 case OP_LOADI1_MEMBASE:
3543 code = emit_ldrsbx (code, dreg, ins->inst_basereg, ins->inst_offset);
3544 break;
3545 case OP_LOADU1_MEMBASE:
3546 code = emit_ldrb (code, dreg, ins->inst_basereg, ins->inst_offset);
3547 break;
3548 case OP_LOADI2_MEMBASE:
3549 code = emit_ldrshx (code, dreg, ins->inst_basereg, ins->inst_offset);
3550 break;
3551 case OP_LOADU2_MEMBASE:
3552 code = emit_ldrh (code, dreg, ins->inst_basereg, ins->inst_offset);
3553 break;
3554 case OP_LOADI4_MEMBASE:
3555 code = emit_ldrswx (code, dreg, ins->inst_basereg, ins->inst_offset);
3556 break;
3557 case OP_LOADU4_MEMBASE:
3558 code = emit_ldrw (code, dreg, ins->inst_basereg, ins->inst_offset);
3559 break;
3560 case OP_LOAD_MEMBASE:
3561 case OP_LOADI8_MEMBASE:
3562 code = emit_ldrx (code, dreg, ins->inst_basereg, ins->inst_offset);
3563 break;
3564 case OP_STOREI1_MEMBASE_IMM:
3565 case OP_STOREI2_MEMBASE_IMM:
3566 case OP_STOREI4_MEMBASE_IMM:
3567 case OP_STORE_MEMBASE_IMM:
3568 case OP_STOREI8_MEMBASE_IMM: {
3569 int immreg;
3570
3571 if (imm != 0) {
3572 code = emit_imm (code, ARMREG_LR, imm);
3573 immreg = ARMREG_LR;
3574 } else {
3575 immreg = ARMREG_RZR;
3576 }
3577
3578 switch (ins->opcode) {
3579 case OP_STOREI1_MEMBASE_IMM:
3580 code = emit_strb (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3581 break;
3582 case OP_STOREI2_MEMBASE_IMM:
3583 code = emit_strh (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3584 break;
3585 case OP_STOREI4_MEMBASE_IMM:
3586 code = emit_strw (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3587 break;
3588 case OP_STORE_MEMBASE_IMM:
3589 case OP_STOREI8_MEMBASE_IMM:
3590 code = emit_strx (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3591 break;
3592 default:
3593 g_assert_not_reached ();
3594 break;
3595 }
3596 break;
3597 }
3598 case OP_STOREI1_MEMBASE_REG:
3599 code = emit_strb (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3600 break;
3601 case OP_STOREI2_MEMBASE_REG:
3602 code = emit_strh (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3603 break;
3604 case OP_STOREI4_MEMBASE_REG:
3605 code = emit_strw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3606 break;
3607 case OP_STORE_MEMBASE_REG:
3608 case OP_STOREI8_MEMBASE_REG:
3609 code = emit_strx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3610 break;
3611 case OP_TLS_GET:
3612 code = emit_tls_get (code, dreg, ins->inst_offset);
3613 break;
3614 case OP_TLS_SET:
3615 code = emit_tls_set (code, sreg1, ins->inst_offset);
3616 break;
3617 /* Atomic */
3618 case OP_MEMORY_BARRIER:
3619 arm_dmb (code, 0);
3620 break;
3621 case OP_ATOMIC_ADD_I4: {
3622 guint8 *buf [16];
3623
3624 buf [0] = code;
3625 arm_ldxrw (code, ARMREG_IP0, sreg1);
3626 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3627 arm_stlxrw (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3628 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3629
3630 arm_dmb (code, 0);
3631 arm_movx (code, dreg, ARMREG_IP0);
3632 break;
3633 }
3634 case OP_ATOMIC_ADD_I8: {
3635 guint8 *buf [16];
3636
3637 buf [0] = code;
3638 arm_ldxrx (code, ARMREG_IP0, sreg1);
3639 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3640 arm_stlxrx (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3641 arm_cbnzx (code, ARMREG_IP1, buf [0]);
3642
3643 arm_dmb (code, 0);
3644 arm_movx (code, dreg, ARMREG_IP0);
3645 break;
3646 }
3647 case OP_ATOMIC_EXCHANGE_I4: {
3648 guint8 *buf [16];
3649
3650 buf [0] = code;
3651 arm_ldxrw (code, ARMREG_IP0, sreg1);
3652 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3653 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3654
3655 arm_dmb (code, 0);
3656 arm_movx (code, dreg, ARMREG_IP0);
3657 break;
3658 }
3659 case OP_ATOMIC_EXCHANGE_I8: {
3660 guint8 *buf [16];
3661
3662 buf [0] = code;
3663 arm_ldxrx (code, ARMREG_IP0, sreg1);
3664 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3665 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3666
3667 arm_dmb (code, 0);
3668 arm_movx (code, dreg, ARMREG_IP0);
3669 break;
3670 }
3671 case OP_ATOMIC_CAS_I4: {
3672 guint8 *buf [16];
3673
3674 /* sreg2 is the value, sreg3 is the comparand */
3675 buf [0] = code;
3676 arm_ldxrw (code, ARMREG_IP0, sreg1);
3677 arm_cmpw (code, ARMREG_IP0, ins->sreg3);
3678 buf [1] = code;
3679 arm_bcc (code, ARMCOND_NE, 0);
3680 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3681 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3682 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3683
3684 arm_dmb (code, 0);
3685 arm_movx (code, dreg, ARMREG_IP0);
3686 break;
3687 }
3688 case OP_ATOMIC_CAS_I8: {
3689 guint8 *buf [16];
3690
3691 buf [0] = code;
3692 arm_ldxrx (code, ARMREG_IP0, sreg1);
3693 arm_cmpx (code, ARMREG_IP0, ins->sreg3);
3694 buf [1] = code;
3695 arm_bcc (code, ARMCOND_NE, 0);
3696 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3697 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3698 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3699
3700 arm_dmb (code, 0);
3701 arm_movx (code, dreg, ARMREG_IP0);
3702 break;
3703 }
3704 case OP_ATOMIC_LOAD_I1: {
3705 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3706 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3707 arm_dmb (code, 0);
3708 arm_ldarb (code, ins->dreg, ARMREG_LR);
3709 arm_sxtbx (code, ins->dreg, ins->dreg);
3710 break;
3711 }
3712 case OP_ATOMIC_LOAD_U1: {
3713 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3714 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3715 arm_dmb (code, 0);
3716 arm_ldarb (code, ins->dreg, ARMREG_LR);
3717 arm_uxtbx (code, ins->dreg, ins->dreg);
3718 break;
3719 }
3720 case OP_ATOMIC_LOAD_I2: {
3721 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3722 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3723 arm_dmb (code, 0);
3724 arm_ldarh (code, ins->dreg, ARMREG_LR);
3725 arm_sxthx (code, ins->dreg, ins->dreg);
3726 break;
3727 }
3728 case OP_ATOMIC_LOAD_U2: {
3729 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3730 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3731 arm_dmb (code, 0);
3732 arm_ldarh (code, ins->dreg, ARMREG_LR);
3733 arm_uxthx (code, ins->dreg, ins->dreg);
3734 break;
3735 }
3736 case OP_ATOMIC_LOAD_I4: {
3737 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3738 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3739 arm_dmb (code, 0);
3740 arm_ldarw (code, ins->dreg, ARMREG_LR);
3741 arm_sxtwx (code, ins->dreg, ins->dreg);
3742 break;
3743 }
3744 case OP_ATOMIC_LOAD_U4: {
3745 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3746 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3747 arm_dmb (code, 0);
3748 arm_ldarw (code, ins->dreg, ARMREG_LR);
3749 arm_movw (code, ins->dreg, ins->dreg); /* Clear upper half of the register. */
3750 break;
3751 }
3752 case OP_ATOMIC_LOAD_I8:
3753 case OP_ATOMIC_LOAD_U8: {
3754 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3755 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3756 arm_dmb (code, 0);
3757 arm_ldarx (code, ins->dreg, ARMREG_LR);
3758 break;
3759 }
3760 case OP_ATOMIC_LOAD_R4: {
3761 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3762 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3763 arm_dmb (code, 0);
3764 if (cfg->r4fp) {
3765 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3766 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3767 } else {
3768 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3769 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
3770 arm_fcvt_sd (code, ins->dreg, FP_TEMP_REG);
3771 }
3772 break;
3773 }
3774 case OP_ATOMIC_LOAD_R8: {
3775 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3776 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3777 arm_dmb (code, 0);
3778 arm_ldarx (code, ARMREG_LR, ARMREG_LR);
3779 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3780 break;
3781 }
3782 case OP_ATOMIC_STORE_I1:
3783 case OP_ATOMIC_STORE_U1: {
3784 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3785 arm_stlrb (code, ARMREG_LR, ins->sreg1);
3786 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3787 arm_dmb (code, 0);
3788 break;
3789 }
3790 case OP_ATOMIC_STORE_I2:
3791 case OP_ATOMIC_STORE_U2: {
3792 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3793 arm_stlrh (code, ARMREG_LR, ins->sreg1);
3794 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3795 arm_dmb (code, 0);
3796 break;
3797 }
3798 case OP_ATOMIC_STORE_I4:
3799 case OP_ATOMIC_STORE_U4: {
3800 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3801 arm_stlrw (code, ARMREG_LR, ins->sreg1);
3802 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3803 arm_dmb (code, 0);
3804 break;
3805 }
3806 case OP_ATOMIC_STORE_I8:
3807 case OP_ATOMIC_STORE_U8: {
3808 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3809 arm_stlrx (code, ARMREG_LR, ins->sreg1);
3810 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3811 arm_dmb (code, 0);
3812 break;
3813 }
3814 case OP_ATOMIC_STORE_R4: {
3815 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3816 if (cfg->r4fp) {
3817 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
3818 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
3819 } else {
3820 arm_fcvt_ds (code, FP_TEMP_REG, ins->sreg1);
3821 arm_fmov_double_to_rx (code, ARMREG_IP0, FP_TEMP_REG);
3822 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
3823 }
3824 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3825 arm_dmb (code, 0);
3826 break;
3827 }
3828 case OP_ATOMIC_STORE_R8: {
3829 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3830 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
3831 arm_stlrx (code, ARMREG_LR, ARMREG_IP0);
3832 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3833 arm_dmb (code, 0);
3834 break;
3835 }
3836
3837 /* FP */
3838 case OP_R8CONST: {
3839 guint64 imm = *(guint64*)ins->inst_p0;
3840
3841 if (imm == 0) {
3842 arm_fmov_rx_to_double (code, dreg, ARMREG_RZR);
3843 } else {
3844 code = emit_imm64 (code, ARMREG_LR, imm);
3845 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3846 }
3847 break;
3848 }
3849 case OP_R4CONST: {
3850 guint64 imm = *(guint32*)ins->inst_p0;
3851
3852 code = emit_imm64 (code, ARMREG_LR, imm);
3853 if (cfg->r4fp) {
3854 arm_fmov_rx_to_double (code, dreg, ARMREG_LR);
3855 } else {
3856 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
3857 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3858 }
3859 break;
3860 }
3861 case OP_LOADR8_MEMBASE:
3862 code = emit_ldrfpx (code, dreg, ins->inst_basereg, ins->inst_offset);
3863 break;
3864 case OP_LOADR4_MEMBASE:
3865 if (cfg->r4fp) {
3866 code = emit_ldrfpw (code, dreg, ins->inst_basereg, ins->inst_offset);
3867 } else {
3868 code = emit_ldrfpw (code, FP_TEMP_REG, ins->inst_basereg, ins->inst_offset);
3869 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3870 }
3871 break;
3872 case OP_STORER8_MEMBASE_REG:
3873 code = emit_strfpx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3874 break;
3875 case OP_STORER4_MEMBASE_REG:
3876 if (cfg->r4fp) {
3877 code = emit_strfpw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3878 } else {
3879 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
3880 code = emit_strfpw (code, FP_TEMP_REG, ins->inst_destbasereg, ins->inst_offset);
3881 }
3882 break;
3883 case OP_FMOVE:
3884 if (dreg != sreg1)
3885 arm_fmovd (code, dreg, sreg1);
3886 break;
3887 case OP_RMOVE:
3888 if (dreg != sreg1)
3889 arm_fmovs (code, dreg, sreg1);
3890 break;
3891 case OP_MOVE_F_TO_I4:
3892 if (cfg->r4fp) {
3893 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
3894 } else {
3895 arm_fcvt_ds (code, ins->dreg, ins->sreg1);
3896 arm_fmov_double_to_rx (code, ins->dreg, ins->dreg);
3897 }
3898 break;
3899 case OP_MOVE_I4_TO_F:
3900 if (cfg->r4fp) {
3901 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3902 } else {
3903 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3904 arm_fcvt_sd (code, ins->dreg, ins->dreg);
3905 }
3906 break;
3907 case OP_MOVE_F_TO_I8:
3908 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
3909 break;
3910 case OP_MOVE_I8_TO_F:
3911 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3912 break;
3913 case OP_FCOMPARE:
3914 arm_fcmpd (code, sreg1, sreg2);
3915 break;
3916 case OP_RCOMPARE:
3917 arm_fcmps (code, sreg1, sreg2);
3918 break;
3919 case OP_FCONV_TO_I1:
3920 arm_fcvtzs_dx (code, dreg, sreg1);
3921 arm_sxtbx (code, dreg, dreg);
3922 break;
3923 case OP_FCONV_TO_U1:
3924 arm_fcvtzu_dx (code, dreg, sreg1);
3925 arm_uxtbw (code, dreg, dreg);
3926 break;
3927 case OP_FCONV_TO_I2:
3928 arm_fcvtzs_dx (code, dreg, sreg1);
3929 arm_sxthx (code, dreg, dreg);
3930 break;
3931 case OP_FCONV_TO_U2:
3932 arm_fcvtzu_dx (code, dreg, sreg1);
3933 arm_uxthw (code, dreg, dreg);
3934 break;
3935 case OP_FCONV_TO_I4:
3936 arm_fcvtzs_dx (code, dreg, sreg1);
3937 arm_sxtwx (code, dreg, dreg);
3938 break;
3939 case OP_FCONV_TO_U4:
3940 arm_fcvtzu_dx (code, dreg, sreg1);
3941 break;
3942 case OP_FCONV_TO_I8:
3943 arm_fcvtzs_dx (code, dreg, sreg1);
3944 break;
3945 case OP_FCONV_TO_U8:
3946 arm_fcvtzu_dx (code, dreg, sreg1);
3947 break;
3948 case OP_FCONV_TO_R4:
3949 if (cfg->r4fp) {
3950 arm_fcvt_ds (code, dreg, sreg1);
3951 } else {
3952 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
3953 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3954 }
3955 break;
3956 case OP_ICONV_TO_R4:
3957 if (cfg->r4fp) {
3958 arm_scvtf_rw_to_s (code, dreg, sreg1);
3959 } else {
3960 arm_scvtf_rw_to_s (code, FP_TEMP_REG, sreg1);
3961 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3962 }
3963 break;
3964 case OP_LCONV_TO_R4:
3965 if (cfg->r4fp) {
3966 arm_scvtf_rx_to_s (code, dreg, sreg1);
3967 } else {
3968 arm_scvtf_rx_to_s (code, FP_TEMP_REG, sreg1);
3969 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3970 }
3971 break;
3972 case OP_ICONV_TO_R8:
3973 arm_scvtf_rw_to_d (code, dreg, sreg1);
3974 break;
3975 case OP_LCONV_TO_R8:
3976 arm_scvtf_rx_to_d (code, dreg, sreg1);
3977 break;
3978 case OP_ICONV_TO_R_UN:
3979 arm_ucvtf_rw_to_d (code, dreg, sreg1);
3980 break;
3981 case OP_LCONV_TO_R_UN:
3982 arm_ucvtf_rx_to_d (code, dreg, sreg1);
3983 break;
3984 case OP_FADD:
3985 arm_fadd_d (code, dreg, sreg1, sreg2);
3986 break;
3987 case OP_FSUB:
3988 arm_fsub_d (code, dreg, sreg1, sreg2);
3989 break;
3990 case OP_FMUL:
3991 arm_fmul_d (code, dreg, sreg1, sreg2);
3992 break;
3993 case OP_FDIV:
3994 arm_fdiv_d (code, dreg, sreg1, sreg2);
3995 break;
3996 case OP_FREM:
3997 /* Emulated */
3998 g_assert_not_reached ();
3999 break;
4000 case OP_FNEG:
4001 arm_fneg_d (code, dreg, sreg1);
4002 break;
4003 case OP_ARM_SETFREG_R4:
4004 arm_fcvt_ds (code, dreg, sreg1);
4005 break;
4006 case OP_CKFINITE:
4007 /* Check for infinity */
4008 code = emit_imm64 (code, ARMREG_LR, 0x7fefffffffffffffLL);
4009 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
4010 arm_fabs_d (code, FP_TEMP_REG2, sreg1);
4011 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG);
4012 code = emit_cond_exc (cfg, code, OP_COND_EXC_GT, "ArithmeticException");
4013 /* Check for nans */
4014 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG2);
4015 code = emit_cond_exc (cfg, code, OP_COND_EXC_OV, "ArithmeticException");
4016 arm_fmovd (code, dreg, sreg1);
4017 break;
4018
4019 /* R4 */
4020 case OP_RADD:
4021 arm_fadd_s (code, dreg, sreg1, sreg2);
4022 break;
4023 case OP_RSUB:
4024 arm_fsub_s (code, dreg, sreg1, sreg2);
4025 break;
4026 case OP_RMUL:
4027 arm_fmul_s (code, dreg, sreg1, sreg2);
4028 break;
4029 case OP_RDIV:
4030 arm_fdiv_s (code, dreg, sreg1, sreg2);
4031 break;
4032 case OP_RNEG:
4033 arm_fneg_s (code, dreg, sreg1);
4034 break;
4035 case OP_RCONV_TO_I1:
4036 arm_fcvtzs_sx (code, dreg, sreg1);
4037 arm_sxtbx (code, dreg, dreg);
4038 break;
4039 case OP_RCONV_TO_U1:
4040 arm_fcvtzu_sx (code, dreg, sreg1);
4041 arm_uxtbw (code, dreg, dreg);
4042 break;
4043 case OP_RCONV_TO_I2:
4044 arm_fcvtzs_sx (code, dreg, sreg1);
4045 arm_sxthx (code, dreg, dreg);
4046 break;
4047 case OP_RCONV_TO_U2:
4048 arm_fcvtzu_sx (code, dreg, sreg1);
4049 arm_uxthw (code, dreg, dreg);
4050 break;
4051 case OP_RCONV_TO_I4:
4052 arm_fcvtzs_sx (code, dreg, sreg1);
4053 arm_sxtwx (code, dreg, dreg);
4054 break;
4055 case OP_RCONV_TO_U4:
4056 arm_fcvtzu_sx (code, dreg, sreg1);
4057 break;
4058 case OP_RCONV_TO_I8:
4059 arm_fcvtzs_sx (code, dreg, sreg1);
4060 break;
4061 case OP_RCONV_TO_U8:
4062 arm_fcvtzu_sx (code, dreg, sreg1);
4063 break;
4064 case OP_RCONV_TO_R8:
4065 arm_fcvt_sd (code, dreg, sreg1);
4066 break;
4067 case OP_RCONV_TO_R4:
4068 if (dreg != sreg1)
4069 arm_fmovs (code, dreg, sreg1);
4070 break;
4071 case OP_RCEQ:
4072 case OP_RCLT:
4073 case OP_RCLT_UN:
4074 case OP_RCGT:
4075 case OP_RCGT_UN:
4076 case OP_RCNEQ:
4077 case OP_RCLE:
4078 case OP_RCGE: {
4079 int cond;
4080
4081 cond = opcode_to_armcond (ins->opcode);
4082 arm_fcmps (code, sreg1, sreg2);
4083 arm_cset (code, cond, dreg);
4084 break;
4085 }
4086
4087 /* CALLS */
4088 case OP_VOIDCALL:
4089 case OP_CALL:
4090 case OP_LCALL:
4091 case OP_FCALL:
4092 case OP_RCALL:
4093 case OP_VCALL2:
4094 call = (MonoCallInst*)ins;
4095 if (ins->flags & MONO_INST_HAS_METHOD)
4096 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method);
4097 else
4098 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr);
4099 code = emit_move_return_value (cfg, code, ins);
4100 break;
4101 case OP_VOIDCALL_REG:
4102 case OP_CALL_REG:
4103 case OP_LCALL_REG:
4104 case OP_FCALL_REG:
4105 case OP_RCALL_REG:
4106 case OP_VCALL2_REG:
4107 arm_blrx (code, sreg1);
4108 code = emit_move_return_value (cfg, code, ins);
4109 break;
4110 case OP_VOIDCALL_MEMBASE:
4111 case OP_CALL_MEMBASE:
4112 case OP_LCALL_MEMBASE:
4113 case OP_FCALL_MEMBASE:
4114 case OP_RCALL_MEMBASE:
4115 case OP_VCALL2_MEMBASE:
4116 code = emit_ldrx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4117 arm_blrx (code, ARMREG_IP0);
4118 code = emit_move_return_value (cfg, code, ins);
4119 break;
4120 case OP_TAILCALL: {
4121 MonoCallInst *call = (MonoCallInst*)ins;
4122
4123 g_assert (!cfg->method->save_lmf);
4124
4125 // FIXME: Copy stack arguments
4126
4127 /* Restore registers */
4128 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
4129
4130 /* Destroy frame */
4131 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, ((1 << ARMREG_IP0) | (1 << ARMREG_IP1)));
4132
4133 if (cfg->compile_aot) {
4134 /* This is not a PLT patch */
4135 code = emit_aotconst (cfg, code, ARMREG_IP0, MONO_PATCH_INFO_METHOD_JUMP, call->method);
4136 arm_brx (code, ARMREG_IP0);
4137 } else {
4138 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, call->method, MONO_R_ARM64_B);
4139 arm_b (code, code);
4140 cfg->thunk_area += THUNK_SIZE;
4141 }
4142 ins->flags |= MONO_INST_GC_CALLSITE;
4143 ins->backend.pc_offset = code - cfg->native_code;
4144 break;
4145 }
4146 case OP_ARGLIST:
4147 g_assert (cfg->arch.cinfo);
4148 code = emit_addx_imm (code, ARMREG_IP0, cfg->arch.args_reg, ((CallInfo*)cfg->arch.cinfo)->sig_cookie.offset);
4149 arm_strx (code, ARMREG_IP0, sreg1, 0);
4150 break;
4151 case OP_DYN_CALL: {
4152 MonoInst *var = cfg->dyn_call_var;
4153 guint8 *labels [16];
4154 int i;
4155
4156 /*
4157 * sreg1 points to a DynCallArgs structure initialized by mono_arch_start_dyn_call ().
4158 * sreg2 is the function to call.
4159 */
4160
4161 g_assert (var->opcode == OP_REGOFFSET);
4162
4163 arm_movx (code, ARMREG_LR, sreg1);
4164 arm_movx (code, ARMREG_IP1, sreg2);
4165
4166 /* Save args buffer */
4167 code = emit_strx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4168
4169 /* Set fp argument regs */
4170 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpargs));
4171 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4172 labels [0] = code;
4173 arm_bcc (code, ARMCOND_EQ, 0);
4174 for (i = 0; i < 8; ++i)
4175 code = emit_ldrfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4176 arm_patch_rel (labels [0], code, MONO_R_ARM64_BCC);
4177
4178 /* Set stack args */
4179 for (i = 0; i < DYN_CALL_STACK_ARGS; ++i) {
4180 code = emit_ldrx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, regs) + ((PARAM_REGS + 1 + i) * sizeof (mgreg_t)));
4181 code = emit_strx (code, ARMREG_R0, ARMREG_SP, i * sizeof (mgreg_t));
4182 }
4183
4184 /* Set argument registers + r8 */
4185 code = mono_arm_emit_load_regarray (code, 0x1ff, ARMREG_LR, 0);
4186
4187 /* Make the call */
4188 arm_blrx (code, ARMREG_IP1);
4189
4190 /* Save result */
4191 code = emit_ldrx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4192 arm_strx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res));
4193 arm_strx (code, ARMREG_R1, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res2));
4194 /* Save fp result */
4195 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpret));
4196 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4197 labels [1] = code;
4198 arm_bcc (code, ARMCOND_EQ, 0);
4199 for (i = 0; i < 8; ++i)
4200 code = emit_strfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4201 arm_patch_rel (labels [1], code, MONO_R_ARM64_BCC);
4202 break;
4203 }
4204
4205 case OP_GENERIC_CLASS_INIT: {
4206 int byte_offset;
4207 guint8 *jump;
4208
4209 byte_offset = MONO_STRUCT_OFFSET (MonoVTable, initialized);
4210
4211 /* Load vtable->initialized */
4212 arm_ldrsbx (code, ARMREG_IP0, sreg1, byte_offset);
4213 jump = code;
4214 arm_cbnzx (code, ARMREG_IP0, 0);
4215
4216 /* Slowpath */
4217 g_assert (sreg1 == ARMREG_R0);
4218 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4219 (gpointer)"mono_generic_class_init");
4220
4221 mono_arm_patch (jump, code, MONO_R_ARM64_CBZ);
4222 break;
4223 }
4224
4225 case OP_CHECK_THIS:
4226 arm_ldrx (code, ARMREG_LR, sreg1, 0);
4227 break;
4228 case OP_NOT_NULL:
4229 case OP_NOT_REACHED:
4230 case OP_DUMMY_USE:
4231 break;
4232 case OP_IL_SEQ_POINT:
4233 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
4234 break;
4235
4236 /* EH */
4237 case OP_COND_EXC_C:
4238 case OP_COND_EXC_IC:
4239 case OP_COND_EXC_OV:
4240 case OP_COND_EXC_IOV:
4241 case OP_COND_EXC_NC:
4242 case OP_COND_EXC_INC:
4243 case OP_COND_EXC_NO:
4244 case OP_COND_EXC_INO:
4245 case OP_COND_EXC_EQ:
4246 case OP_COND_EXC_IEQ:
4247 case OP_COND_EXC_NE_UN:
4248 case OP_COND_EXC_INE_UN:
4249 case OP_COND_EXC_ILT:
4250 case OP_COND_EXC_LT:
4251 case OP_COND_EXC_ILT_UN:
4252 case OP_COND_EXC_LT_UN:
4253 case OP_COND_EXC_IGT:
4254 case OP_COND_EXC_GT:
4255 case OP_COND_EXC_IGT_UN:
4256 case OP_COND_EXC_GT_UN:
4257 case OP_COND_EXC_IGE:
4258 case OP_COND_EXC_GE:
4259 case OP_COND_EXC_IGE_UN:
4260 case OP_COND_EXC_GE_UN:
4261 case OP_COND_EXC_ILE:
4262 case OP_COND_EXC_LE:
4263 case OP_COND_EXC_ILE_UN:
4264 case OP_COND_EXC_LE_UN:
4265 code = emit_cond_exc (cfg, code, ins->opcode, ins->inst_p1);
4266 break;
4267 case OP_THROW:
4268 if (sreg1 != ARMREG_R0)
4269 arm_movx (code, ARMREG_R0, sreg1);
4270 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4271 (gpointer)"mono_arch_throw_exception");
4272 break;
4273 case OP_RETHROW:
4274 if (sreg1 != ARMREG_R0)
4275 arm_movx (code, ARMREG_R0, sreg1);
4276 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4277 (gpointer)"mono_arch_rethrow_exception");
4278 break;
4279 case OP_CALL_HANDLER:
4280 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_BL);
4281 arm_bl (code, 0);
4282 cfg->thunk_area += THUNK_SIZE;
4283 mono_cfg_add_try_hole (cfg, ins->inst_eh_block, code, bb);
4284 break;
4285 case OP_START_HANDLER: {
4286 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4287
4288 /* Save caller address */
4289 code = emit_strx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4290
4291 /*
4292 * Reserve a param area, see test_0_finally_param_area ().
4293 * This is needed because the param area is not set up when
4294 * we are called from EH code.
4295 */
4296 if (cfg->param_area)
4297 code = emit_subx_sp_imm (code, cfg->param_area);
4298 break;
4299 }
4300 case OP_ENDFINALLY:
4301 case OP_ENDFILTER: {
4302 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4303
4304 if (cfg->param_area)
4305 code = emit_addx_sp_imm (code, cfg->param_area);
4306
4307 if (ins->opcode == OP_ENDFILTER && sreg1 != ARMREG_R0)
4308 arm_movx (code, ARMREG_R0, sreg1);
4309
4310 /* Return to either after the branch in OP_CALL_HANDLER, or to the EH code */
4311 code = emit_ldrx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4312 arm_brx (code, ARMREG_LR);
4313 break;
4314 }
4315 case OP_GET_EX_OBJ:
4316 if (ins->dreg != ARMREG_R0)
4317 arm_movx (code, ins->dreg, ARMREG_R0);
4318 break;
4319 case OP_GC_SAFE_POINT: {
4320 #if defined (USE_COOP_GC)
4321 guint8 *buf [1];
4322
4323 arm_ldrx (code, ARMREG_IP1, ins->sreg1, 0);
4324 /* Call it if it is non-null */
4325 buf [0] = code;
4326 arm_cbzx (code, ARMREG_IP1, 0);
4327 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_threads_state_poll");
4328 mono_arm_patch (buf [0], code, MONO_R_ARM64_CBZ);
4329 #endif
4330 break;
4331 }
4332 case OP_FILL_PROF_CALL_CTX:
4333 for (int i = 0; i < MONO_MAX_IREGS; i++)
4334 if ((MONO_ARCH_CALLEE_SAVED_REGS & (1 << i)) || i == ARMREG_SP || i == ARMREG_FP)
4335 arm_strx (code, i, ins->sreg1, MONO_STRUCT_OFFSET (MonoContext, regs) + i * sizeof (mgreg_t));
4336 break;
4337 default:
4338 g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
4339 g_assert_not_reached ();
4340 }
4341
4342 if ((cfg->opt & MONO_OPT_BRANCH) && ((code - cfg->native_code - offset) > max_len)) {
4343 g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %d)",
4344 mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
4345 g_assert_not_reached ();
4346 }
4347 }
4348
4349 /*
4350 * If the compiled code size is larger than the bcc displacement (19 bits signed),
4351 * insert branch islands between/inside basic blocks.
4352 */
4353 if (cfg->arch.cond_branch_islands)
4354 code = emit_branch_island (cfg, code, start_offset);
4355
4356 cfg->code_len = code - cfg->native_code;
4357 }
4358
4359 static guint8*
4360 emit_move_args (MonoCompile *cfg, guint8 *code)
4361 {
4362 MonoInst *ins;
4363 CallInfo *cinfo;
4364 ArgInfo *ainfo;
4365 int i, part;
4366
4367 cinfo = cfg->arch.cinfo;
4368 g_assert (cinfo);
4369 for (i = 0; i < cinfo->nargs; ++i) {
4370 ainfo = cinfo->args + i;
4371 ins = cfg->args [i];
4372
4373 if (ins->opcode == OP_REGVAR) {
4374 switch (ainfo->storage) {
4375 case ArgInIReg:
4376 arm_movx (code, ins->dreg, ainfo->reg);
4377 break;
4378 case ArgOnStack:
4379 switch (ainfo->slot_size) {
4380 case 1:
4381 if (ainfo->sign)
4382 code = emit_ldrsbx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4383 else
4384 code = emit_ldrb (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4385 break;
4386 case 2:
4387 if (ainfo->sign)
4388 code = emit_ldrshx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4389 else
4390 code = emit_ldrh (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4391 break;
4392 case 4:
4393 if (ainfo->sign)
4394 code = emit_ldrswx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4395 else
4396 code = emit_ldrw (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4397 break;
4398 default:
4399 code = emit_ldrx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4400 break;
4401 }
4402 break;
4403 default:
4404 g_assert_not_reached ();
4405 break;
4406 }
4407 } else {
4408 if (ainfo->storage != ArgVtypeByRef && ainfo->storage != ArgVtypeByRefOnStack)
4409 g_assert (ins->opcode == OP_REGOFFSET);
4410
4411 switch (ainfo->storage) {
4412 case ArgInIReg:
4413 /* Stack slots for arguments have size 8 */
4414 code = emit_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4415 break;
4416 case ArgInFReg:
4417 code = emit_strfpx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4418 break;
4419 case ArgInFRegR4:
4420 code = emit_strfpw (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4421 break;
4422 case ArgOnStack:
4423 case ArgOnStackR4:
4424 case ArgOnStackR8:
4425 case ArgVtypeByRefOnStack:
4426 case ArgVtypeOnStack:
4427 break;
4428 case ArgVtypeByRef: {
4429 MonoInst *addr_arg = ins->inst_left;
4430
4431 if (ainfo->gsharedvt) {
4432 g_assert (ins->opcode == OP_GSHAREDVT_ARG_REGOFFSET);
4433 arm_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4434 } else {
4435 g_assert (ins->opcode == OP_VTARG_ADDR);
4436 g_assert (addr_arg->opcode == OP_REGOFFSET);
4437 arm_strx (code, ainfo->reg, addr_arg->inst_basereg, addr_arg->inst_offset);
4438 }
4439 break;
4440 }
4441 case ArgVtypeInIRegs:
4442 for (part = 0; part < ainfo->nregs; part ++) {
4443 code = emit_strx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + (part * 8));
4444 }
4445 break;
4446 case ArgHFA:
4447 for (part = 0; part < ainfo->nregs; part ++) {
4448 if (ainfo->esize == 4)
4449 code = emit_strfpw (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4450 else
4451 code = emit_strfpx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4452 }
4453 break;
4454 default:
4455 g_assert_not_reached ();
4456 break;
4457 }
4458 }
4459 }
4460
4461 return code;
4462 }
4463
4464 /*
4465 * emit_store_regarray:
4466 *
4467 * Emit code to store the registers in REGS into the appropriate elements of
4468 * the register array at BASEREG+OFFSET.
4469 */
4470 static __attribute__ ((__warn_unused_result__)) guint8*
4471 emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4472 {
4473 int i;
4474
4475 for (i = 0; i < 32; ++i) {
4476 if (regs & (1 << i)) {
4477 if (i + 1 < 32 && (regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4478 arm_stpx (code, i, i + 1, basereg, offset + (i * 8));
4479 i++;
4480 } else if (i == ARMREG_SP) {
4481 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4482 arm_strx (code, ARMREG_IP1, basereg, offset + (i * 8));
4483 } else {
4484 arm_strx (code, i, basereg, offset + (i * 8));
4485 }
4486 }
4487 }
4488 return code;
4489 }
4490
4491 /*
4492 * emit_load_regarray:
4493 *
4494 * Emit code to load the registers in REGS from the appropriate elements of
4495 * the register array at BASEREG+OFFSET.
4496 */
4497 static __attribute__ ((__warn_unused_result__)) guint8*
4498 emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4499 {
4500 int i;
4501
4502 for (i = 0; i < 32; ++i) {
4503 if (regs & (1 << i)) {
4504 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4505 if (offset + (i * 8) < 500)
4506 arm_ldpx (code, i, i + 1, basereg, offset + (i * 8));
4507 else {
4508 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4509 code = emit_ldrx (code, i + 1, basereg, offset + ((i + 1) * 8));
4510 }
4511 i++;
4512 } else if (i == ARMREG_SP) {
4513 g_assert_not_reached ();
4514 } else {
4515 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4516 }
4517 }
4518 }
4519 return code;
4520 }
4521
4522 /*
4523 * emit_store_regset:
4524 *
4525 * Emit code to store the registers in REGS into consecutive memory locations starting
4526 * at BASEREG+OFFSET.
4527 */
4528 static __attribute__ ((__warn_unused_result__)) guint8*
4529 emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4530 {
4531 int i, pos;
4532
4533 pos = 0;
4534 for (i = 0; i < 32; ++i) {
4535 if (regs & (1 << i)) {
4536 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4537 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4538 i++;
4539 pos++;
4540 } else if (i == ARMREG_SP) {
4541 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4542 arm_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4543 } else {
4544 arm_strx (code, i, basereg, offset + (pos * 8));
4545 }
4546 pos++;
4547 }
4548 }
4549 return code;
4550 }
4551
4552 /*
4553 * emit_load_regset:
4554 *
4555 * Emit code to load the registers in REGS from consecutive memory locations starting
4556 * at BASEREG+OFFSET.
4557 */
4558 static __attribute__ ((__warn_unused_result__)) guint8*
4559 emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset)
4560 {
4561 int i, pos;
4562
4563 pos = 0;
4564 for (i = 0; i < 32; ++i) {
4565 if (regs & (1 << i)) {
4566 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4567 arm_ldpx (code, i, i + 1, basereg, offset + (pos * 8));
4568 i++;
4569 pos++;
4570 } else if (i == ARMREG_SP) {
4571 g_assert_not_reached ();
4572 } else {
4573 arm_ldrx (code, i, basereg, offset + (pos * 8));
4574 }
4575 pos++;
4576 }
4577 }
4578 return code;
4579 }
4580
4581 __attribute__ ((__warn_unused_result__)) guint8*
4582 mono_arm_emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4583 {
4584 return emit_load_regarray (code, regs, basereg, offset);
4585 }
4586
4587 __attribute__ ((__warn_unused_result__)) guint8*
4588 mono_arm_emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4589 {
4590 return emit_store_regarray (code, regs, basereg, offset);
4591 }
4592
4593 __attribute__ ((__warn_unused_result__)) guint8*
4594 mono_arm_emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4595 {
4596 return emit_store_regset (code, regs, basereg, offset);
4597 }
4598
4599 /* Same as emit_store_regset, but emit unwind info too */
4600 /* CFA_OFFSET is the offset between the CFA and basereg */
4601 static __attribute__ ((__warn_unused_result__)) guint8*
4602 emit_store_regset_cfa (MonoCompile *cfg, guint8 *code, guint64 regs, int basereg, int offset, int cfa_offset, guint64 no_cfa_regset)
4603 {
4604 int i, j, pos, nregs;
4605 guint32 cfa_regset = regs & ~no_cfa_regset;
4606
4607 pos = 0;
4608 for (i = 0; i < 32; ++i) {
4609 nregs = 1;
4610 if (regs & (1 << i)) {
4611 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4612 if (offset < 256) {
4613 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4614 } else {
4615 code = emit_strx (code, i, basereg, offset + (pos * 8));
4616 code = emit_strx (code, i + 1, basereg, offset + (pos * 8) + 8);
4617 }
4618 nregs = 2;
4619 } else if (i == ARMREG_SP) {
4620 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4621 code = emit_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4622 } else {
4623 code = emit_strx (code, i, basereg, offset + (pos * 8));
4624 }
4625
4626 for (j = 0; j < nregs; ++j) {
4627 if (cfa_regset & (1 << (i + j)))
4628 mono_emit_unwind_op_offset (cfg, code, i + j, (- cfa_offset) + offset + ((pos + j) * 8));
4629 }
4630
4631 i += nregs - 1;
4632 pos += nregs;
4633 }
4634 }
4635 return code;
4636 }
4637
4638 /*
4639 * emit_setup_lmf:
4640 *
4641 * Emit code to initialize an LMF structure at LMF_OFFSET.
4642 * Clobbers ip0/ip1.
4643 */
4644 static guint8*
4645 emit_setup_lmf (MonoCompile *cfg, guint8 *code, gint32 lmf_offset, int cfa_offset)
4646 {
4647 /*
4648 * The LMF should contain all the state required to be able to reconstruct the machine state
4649 * at the current point of execution. Since the LMF is only read during EH, only callee
4650 * saved etc. registers need to be saved.
4651 * FIXME: Save callee saved fp regs, JITted code doesn't use them, but native code does, and they
4652 * need to be restored during EH.
4653 */
4654
4655 /* pc */
4656 arm_adrx (code, ARMREG_LR, code);
4657 code = emit_strx (code, ARMREG_LR, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, pc));
4658 /* gregs + fp + sp */
4659 /* Don't emit unwind info for sp/fp, they are already handled in the prolog */
4660 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_LMF_REGS, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs), cfa_offset, (1 << ARMREG_FP) | (1 << ARMREG_SP));
4661
4662 return code;
4663 }
4664
4665 guint8 *
4666 mono_arch_emit_prolog (MonoCompile *cfg)
4667 {
4668 MonoMethod *method = cfg->method;
4669 MonoMethodSignature *sig;
4670 MonoBasicBlock *bb;
4671 guint8 *code;
4672 int cfa_offset, max_offset;
4673
4674 sig = mono_method_signature (method);
4675 cfg->code_size = 256 + sig->param_count * 64;
4676 code = cfg->native_code = g_malloc (cfg->code_size);
4677
4678 /* This can be unaligned */
4679 cfg->stack_offset = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
4680
4681 /*
4682 * - Setup frame
4683 */
4684 cfa_offset = 0;
4685 mono_emit_unwind_op_def_cfa (cfg, code, ARMREG_SP, 0);
4686
4687 /* Setup frame */
4688 if (arm_is_ldpx_imm (-cfg->stack_offset)) {
4689 arm_stpx_pre (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, -cfg->stack_offset);
4690 } else {
4691 /* sp -= cfg->stack_offset */
4692 /* This clobbers ip0/ip1 */
4693 code = emit_subx_sp_imm (code, cfg->stack_offset);
4694 arm_stpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
4695 }
4696 cfa_offset += cfg->stack_offset;
4697 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
4698 mono_emit_unwind_op_offset (cfg, code, ARMREG_FP, (- cfa_offset) + 0);
4699 mono_emit_unwind_op_offset (cfg, code, ARMREG_LR, (- cfa_offset) + 8);
4700 arm_movspx (code, ARMREG_FP, ARMREG_SP);
4701 mono_emit_unwind_op_def_cfa_reg (cfg, code, ARMREG_FP);
4702 if (cfg->param_area) {
4703 /* The param area is below the frame pointer */
4704 code = emit_subx_sp_imm (code, cfg->param_area);
4705 }
4706
4707 if (cfg->method->save_lmf) {
4708 code = emit_setup_lmf (cfg, code, cfg->lmf_var->inst_offset, cfa_offset);
4709 } else {
4710 /* Save gregs */
4711 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset, cfa_offset, 0);
4712 }
4713
4714 /* Setup args reg */
4715 if (cfg->arch.args_reg) {
4716 /* The register was already saved above */
4717 code = emit_addx_imm (code, cfg->arch.args_reg, ARMREG_FP, cfg->stack_offset);
4718 }
4719
4720 /* Save return area addr received in R8 */
4721 if (cfg->vret_addr) {
4722 MonoInst *ins = cfg->vret_addr;
4723
4724 g_assert (ins->opcode == OP_REGOFFSET);
4725 code = emit_strx (code, ARMREG_R8, ins->inst_basereg, ins->inst_offset);
4726 }
4727
4728 /* Save mrgctx received in MONO_ARCH_RGCTX_REG */
4729 if (cfg->rgctx_var) {
4730 MonoInst *ins = cfg->rgctx_var;
4731
4732 g_assert (ins->opcode == OP_REGOFFSET);
4733
4734 code = emit_strx (code, MONO_ARCH_RGCTX_REG, ins->inst_basereg, ins->inst_offset);
4735 }
4736
4737 /*
4738 * Move arguments to their registers/stack locations.
4739 */
4740 code = emit_move_args (cfg, code);
4741
4742 /* Initialize seq_point_info_var */
4743 if (cfg->arch.seq_point_info_var) {
4744 MonoInst *ins = cfg->arch.seq_point_info_var;
4745
4746 /* Initialize the variable from a GOT slot */
4747 code = emit_aotconst (cfg, code, ARMREG_IP0, MONO_PATCH_INFO_SEQ_POINT_INFO, cfg->method);
4748 g_assert (ins->opcode == OP_REGOFFSET);
4749 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4750
4751 /* Initialize ss_tramp_var */
4752 ins = cfg->arch.ss_tramp_var;
4753 g_assert (ins->opcode == OP_REGOFFSET);
4754
4755 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP0, MONO_STRUCT_OFFSET (SeqPointInfo, ss_tramp_addr));
4756 code = emit_strx (code, ARMREG_IP1, ins->inst_basereg, ins->inst_offset);
4757 } else {
4758 MonoInst *ins;
4759
4760 if (cfg->arch.ss_tramp_var) {
4761 /* Initialize ss_tramp_var */
4762 ins = cfg->arch.ss_tramp_var;
4763 g_assert (ins->opcode == OP_REGOFFSET);
4764
4765 code = emit_imm64 (code, ARMREG_IP0, (guint64)&ss_trampoline);
4766 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4767 }
4768
4769 if (cfg->arch.bp_tramp_var) {
4770 /* Initialize bp_tramp_var */
4771 ins = cfg->arch.bp_tramp_var;
4772 g_assert (ins->opcode == OP_REGOFFSET);
4773
4774 code = emit_imm64 (code, ARMREG_IP0, (guint64)bp_trampoline);
4775 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4776 }
4777 }
4778
4779 max_offset = 0;
4780 if (cfg->opt & MONO_OPT_BRANCH) {
4781 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4782 MonoInst *ins;
4783 bb->max_offset = max_offset;
4784
4785 MONO_BB_FOR_EACH_INS (bb, ins) {
4786 max_offset += ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
4787 }
4788 }
4789 }
4790 if (max_offset > 0x3ffff * 4)
4791 cfg->arch.cond_branch_islands = TRUE;
4792
4793 return code;
4794 }
4795
4796 static guint8*
4797 realloc_code (MonoCompile *cfg, int size)
4798 {
4799 while (cfg->code_len + size > (cfg->code_size - 16)) {
4800 cfg->code_size *= 2;
4801 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
4802 cfg->stat_code_reallocs++;
4803 }
4804 return cfg->native_code + cfg->code_len;
4805 }
4806
4807 void
4808 mono_arch_emit_epilog (MonoCompile *cfg)
4809 {
4810 CallInfo *cinfo;
4811 int max_epilog_size;
4812 guint8 *code;
4813 int i;
4814
4815 max_epilog_size = 16 + 20*4;
4816 code = realloc_code (cfg, max_epilog_size);
4817
4818 if (cfg->method->save_lmf) {
4819 code = mono_arm_emit_load_regarray (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->lmf_var->inst_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs) - (MONO_ARCH_FIRST_LMF_REG * 8));
4820 } else {
4821 /* Restore gregs */
4822 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
4823 }
4824
4825 /* Load returned vtypes into registers if needed */
4826 cinfo = cfg->arch.cinfo;
4827 switch (cinfo->ret.storage) {
4828 case ArgVtypeInIRegs: {
4829 MonoInst *ins = cfg->ret;
4830
4831 for (i = 0; i < cinfo->ret.nregs; ++i)
4832 code = emit_ldrx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + (i * 8));
4833 break;
4834 }
4835 case ArgHFA: {
4836 MonoInst *ins = cfg->ret;
4837
4838 for (i = 0; i < cinfo->ret.nregs; ++i) {
4839 if (cinfo->ret.esize == 4)
4840 code = emit_ldrfpw (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
4841 else
4842 code = emit_ldrfpx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
4843 }
4844 break;
4845 }
4846 default:
4847 break;
4848 }
4849
4850 /* Destroy frame */
4851 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, ((1 << ARMREG_IP0) | (1 << ARMREG_IP1)));
4852
4853 arm_retx (code, ARMREG_LR);
4854
4855 g_assert (code - (cfg->native_code + cfg->code_len) < max_epilog_size);
4856
4857 cfg->code_len = code - cfg->native_code;
4858 }
4859
4860 void
4861 mono_arch_emit_exceptions (MonoCompile *cfg)
4862 {
4863 MonoJumpInfo *ji;
4864 MonoClass *exc_class;
4865 guint8 *code, *ip;
4866 guint8* exc_throw_pos [MONO_EXC_INTRINS_NUM];
4867 guint8 exc_throw_found [MONO_EXC_INTRINS_NUM];
4868 int i, id, size = 0;
4869
4870 for (i = 0; i < MONO_EXC_INTRINS_NUM; i++) {
4871 exc_throw_pos [i] = NULL;
4872 exc_throw_found [i] = 0;
4873 }
4874
4875 for (ji = cfg->patch_info; ji; ji = ji->next) {
4876 if (ji->type == MONO_PATCH_INFO_EXC) {
4877 i = mini_exception_id_by_name (ji->data.target);
4878 if (!exc_throw_found [i]) {
4879 size += 32;
4880 exc_throw_found [i] = TRUE;
4881 }
4882 }
4883 }
4884
4885 code = realloc_code (cfg, size);
4886
4887 /* Emit code to raise corlib exceptions */
4888 for (ji = cfg->patch_info; ji; ji = ji->next) {
4889 if (ji->type != MONO_PATCH_INFO_EXC)
4890 continue;
4891
4892 ip = cfg->native_code + ji->ip.i;
4893
4894 id = mini_exception_id_by_name (ji->data.target);
4895
4896 if (exc_throw_pos [id]) {
4897 /* ip points to the bcc () in OP_COND_EXC_... */
4898 arm_patch_rel (ip, exc_throw_pos [id], ji->relocation);
4899 ji->type = MONO_PATCH_INFO_NONE;
4900 continue;
4901 }
4902
4903 exc_throw_pos [id] = code;
4904 arm_patch_rel (ip, code, ji->relocation);
4905
4906 /* We are being branched to from the code generated by emit_cond_exc (), the pc is in ip1 */
4907
4908 /* r0 = type token */
4909 exc_class = mono_class_load_from_name (mono_defaults.corlib, "System", ji->data.name);
4910 code = emit_imm (code, ARMREG_R0, exc_class->type_token - MONO_TOKEN_TYPE_DEF);
4911 /* r1 = throw ip */
4912 arm_movx (code, ARMREG_R1, ARMREG_IP1);
4913 /* Branch to the corlib exception throwing trampoline */
4914 ji->ip.i = code - cfg->native_code;
4915 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
4916 ji->data.name = "mono_arch_throw_corlib_exception";
4917 ji->relocation = MONO_R_ARM64_BL;
4918 arm_bl (code, 0);
4919 cfg->thunk_area += THUNK_SIZE;
4920 }
4921
4922 cfg->code_len = code - cfg->native_code;
4923
4924 g_assert (cfg->code_len < cfg->code_size);
4925 }
4926
4927 MonoInst*
4928 mono_arch_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
4929 {
4930 return NULL;
4931 }
4932
4933 gboolean
4934 mono_arch_print_tree (MonoInst *tree, int arity)
4935 {
4936 return FALSE;
4937 }
4938
4939 guint32
4940 mono_arch_get_patch_offset (guint8 *code)
4941 {
4942 return 0;
4943 }
4944
4945 gpointer
4946 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
4947 gpointer fail_tramp)
4948 {
4949 int i, buf_len, imt_reg;
4950 guint8 *buf, *code;
4951
4952 #if DEBUG_IMT
4953 printf ("building IMT trampoline for class %s %s entries %d code size %d code at %p end %p vtable %p\n", vtable->klass->name_space, vtable->klass->name, count, size, start, ((guint8*)start) + size, vtable);
4954 for (i = 0; i < count; ++i) {
4955 MonoIMTCheckItem *item = imt_entries [i];
4956 printf ("method %d (%p) %s vtable slot %p is_equals %d chunk size %d\n", i, item->key, item->key->name, &vtable->vtable [item->value.vtable_slot], item->is_equals, item->chunk_size);
4957 }
4958 #endif
4959
4960 buf_len = 0;
4961 for (i = 0; i < count; ++i) {
4962 MonoIMTCheckItem *item = imt_entries [i];
4963 if (item->is_equals) {
4964 gboolean fail_case = !item->check_target_idx && fail_tramp;
4965
4966 if (item->check_target_idx || fail_case) {
4967 if (!item->compare_done || fail_case) {
4968 buf_len += 4 * 4 + 4;
4969 }
4970 buf_len += 4;
4971 if (item->has_target_code) {
4972 buf_len += 5 * 4;
4973 } else {
4974 buf_len += 6 * 4;
4975 }
4976 if (fail_case) {
4977 buf_len += 5 * 4;
4978 }
4979 } else {
4980 buf_len += 6 * 4;
4981 }
4982 } else {
4983 buf_len += 6 * 4;
4984 }
4985 }
4986
4987 if (fail_tramp)
4988 buf = mono_method_alloc_generic_virtual_trampoline (domain, buf_len);
4989 else
4990 buf = mono_domain_code_reserve (domain, buf_len);
4991 code = buf;
4992
4993 /*
4994 * We are called by JITted code, which passes in the IMT argument in
4995 * MONO_ARCH_RGCTX_REG (r27). We need to preserve all caller saved regs
4996 * except ip0/ip1.
4997 */
4998 imt_reg = MONO_ARCH_RGCTX_REG;
4999 for (i = 0; i < count; ++i) {
5000 MonoIMTCheckItem *item = imt_entries [i];
5001
5002 item->code_target = code;
5003
5004 if (item->is_equals) {
5005 /*
5006 * Check the imt argument against item->key, if equals, jump to either
5007 * item->value.target_code or to vtable [item->value.vtable_slot].
5008 * If fail_tramp is set, jump to it if not-equals.
5009 */
5010 gboolean fail_case = !item->check_target_idx && fail_tramp;
5011
5012 if (item->check_target_idx || fail_case) {
5013 /* Compare imt_reg with item->key */
5014 if (!item->compare_done || fail_case) {
5015 // FIXME: Optimize this
5016 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5017 arm_cmpx (code, imt_reg, ARMREG_IP0);
5018 }
5019 item->jmp_code = code;
5020 arm_bcc (code, ARMCOND_NE, 0);
5021 /* Jump to target if equals */
5022 if (item->has_target_code) {
5023 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->value.target_code);
5024 arm_brx (code, ARMREG_IP0);
5025 } else {
5026 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5027
5028 code = emit_imm64 (code, ARMREG_IP0, imm);
5029 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5030 arm_brx (code, ARMREG_IP0);
5031 }
5032
5033 if (fail_case) {
5034 arm_patch_rel (item->jmp_code, code, MONO_R_ARM64_BCC);
5035 item->jmp_code = NULL;
5036 code = emit_imm64 (code, ARMREG_IP0, (guint64)fail_tramp);
5037 arm_brx (code, ARMREG_IP0);
5038 }
5039 } else {
5040 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5041
5042 code = emit_imm64 (code, ARMREG_IP0, imm);
5043 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5044 arm_brx (code, ARMREG_IP0);
5045 }
5046 } else {
5047 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5048 arm_cmpx (code, imt_reg, ARMREG_IP0);
5049 item->jmp_code = code;
5050 arm_bcc (code, ARMCOND_HS, 0);
5051 }
5052 }
5053 /* Patch the branches */
5054 for (i = 0; i < count; ++i) {
5055 MonoIMTCheckItem *item = imt_entries [i];
5056 if (item->jmp_code && item->check_target_idx)
5057 arm_patch_rel (item->jmp_code, imt_entries [item->check_target_idx]->code_target, MONO_R_ARM64_BCC);
5058 }
5059
5060 g_assert ((code - buf) < buf_len);
5061
5062 mono_arch_flush_icache (buf, code - buf);
5063
5064 return buf;
5065 }
5066
5067 GSList *
5068 mono_arch_get_trampolines (gboolean aot)
5069 {
5070 return mono_arm_get_exception_trampolines (aot);
5071 }
5072
5073 #else /* DISABLE_JIT */
5074
5075 gpointer
5076 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
5077 gpointer fail_tramp)
5078 {
5079 g_assert_not_reached ();
5080 return NULL;
5081 }
5082
5083 #endif /* !DISABLE_JIT */
5084
5085 #ifdef MONO_ARCH_SOFT_DEBUG_SUPPORTED
5086
5087 void
5088 mono_arch_set_breakpoint (MonoJitInfo *ji, guint8 *ip)
5089 {
5090 guint8 *code = ip;
5091 guint32 native_offset = ip - (guint8*)ji->code_start;
5092
5093 if (ji->from_aot) {
5094 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5095
5096 g_assert (native_offset % 4 == 0);
5097 g_assert (info->bp_addrs [native_offset / 4] == 0);
5098 info->bp_addrs [native_offset / 4] = mini_get_breakpoint_trampoline ();
5099 } else {
5100 /* ip points to an ldrx */
5101 code += 4;
5102 arm_blrx (code, ARMREG_IP0);
5103 mono_arch_flush_icache (ip, code - ip);
5104 }
5105 }
5106
5107 void
5108 mono_arch_clear_breakpoint (MonoJitInfo *ji, guint8 *ip)
5109 {
5110 guint8 *code = ip;
5111
5112 if (ji->from_aot) {
5113 guint32 native_offset = ip - (guint8*)ji->code_start;
5114 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5115
5116 g_assert (native_offset % 4 == 0);
5117 info->bp_addrs [native_offset / 4] = NULL;
5118 } else {
5119 /* ip points to an ldrx */
5120 code += 4;
5121 arm_nop (code);
5122 mono_arch_flush_icache (ip, code - ip);
5123 }
5124 }
5125
5126 void
5127 mono_arch_start_single_stepping (void)
5128 {
5129 ss_trampoline = mini_get_single_step_trampoline ();
5130 }
5131
5132 void
5133 mono_arch_stop_single_stepping (void)
5134 {
5135 ss_trampoline = NULL;
5136 }
5137
5138 gboolean
5139 mono_arch_is_single_step_event (void *info, void *sigctx)
5140 {
5141 /* We use soft breakpoints on arm64 */
5142 return FALSE;
5143 }
5144
5145 gboolean
5146 mono_arch_is_breakpoint_event (void *info, void *sigctx)
5147 {
5148 /* We use soft breakpoints on arm64 */
5149 return FALSE;
5150 }
5151
5152 void
5153 mono_arch_skip_breakpoint (MonoContext *ctx, MonoJitInfo *ji)
5154 {
5155 g_assert_not_reached ();
5156 }
5157
5158 void
5159 mono_arch_skip_single_step (MonoContext *ctx)
5160 {
5161 g_assert_not_reached ();
5162 }
5163
5164 gpointer
5165 mono_arch_get_seq_point_info (MonoDomain *domain, guint8 *code)
5166 {
5167 SeqPointInfo *info;
5168 MonoJitInfo *ji;
5169
5170 // FIXME: Add a free function
5171
5172 mono_domain_lock (domain);
5173 info = g_hash_table_lookup (domain_jit_info (domain)->arch_seq_points,
5174 code);
5175 mono_domain_unlock (domain);
5176
5177 if (!info) {
5178 ji = mono_jit_info_table_find (domain, (char*)code);
5179 g_assert (ji);
5180
5181 info = g_malloc0 (sizeof (SeqPointInfo) + (ji->code_size / 4) * sizeof(guint8*));
5182
5183 info->ss_tramp_addr = &ss_trampoline;
5184
5185 mono_domain_lock (domain);
5186 g_hash_table_insert (domain_jit_info (domain)->arch_seq_points,
5187 code, info);
5188 mono_domain_unlock (domain);
5189 }
5190
5191 return info;
5192 }
5193
5194 void
5195 mono_arch_init_lmf_ext (MonoLMFExt *ext, gpointer prev_lmf)
5196 {
5197 ext->lmf.previous_lmf = prev_lmf;
5198 /* Mark that this is a MonoLMFExt */
5199 ext->lmf.previous_lmf = (gpointer)(((gssize)ext->lmf.previous_lmf) | 2);
5200 ext->lmf.gregs [MONO_ARCH_LMF_REG_SP] = (gssize)ext;
5201 }
5202
5203 #endif /* MONO_ARCH_SOFT_DEBUG_SUPPORTED */
5204
5205 gboolean
5206 mono_arch_opcode_supported (int opcode)
5207 {
5208 switch (opcode) {
5209 case OP_ATOMIC_ADD_I4:
5210 case OP_ATOMIC_ADD_I8:
5211 case OP_ATOMIC_EXCHANGE_I4:
5212 case OP_ATOMIC_EXCHANGE_I8:
5213 case OP_ATOMIC_CAS_I4:
5214 case OP_ATOMIC_CAS_I8:
5215 case OP_ATOMIC_LOAD_I1:
5216 case OP_ATOMIC_LOAD_I2:
5217 case OP_ATOMIC_LOAD_I4:
5218 case OP_ATOMIC_LOAD_I8:
5219 case OP_ATOMIC_LOAD_U1:
5220 case OP_ATOMIC_LOAD_U2:
5221 case OP_ATOMIC_LOAD_U4:
5222 case OP_ATOMIC_LOAD_U8:
5223 case OP_ATOMIC_LOAD_R4:
5224 case OP_ATOMIC_LOAD_R8:
5225 case OP_ATOMIC_STORE_I1:
5226 case OP_ATOMIC_STORE_I2:
5227 case OP_ATOMIC_STORE_I4:
5228 case OP_ATOMIC_STORE_I8:
5229 case OP_ATOMIC_STORE_U1:
5230 case OP_ATOMIC_STORE_U2:
5231 case OP_ATOMIC_STORE_U4:
5232 case OP_ATOMIC_STORE_U8:
5233 case OP_ATOMIC_STORE_R4:
5234 case OP_ATOMIC_STORE_R8:
5235 return TRUE;
5236 default:
5237 return FALSE;
5238 }
5239 }
5240
5241 CallInfo*
5242 mono_arch_get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
5243 {
5244 return get_call_info (mp, sig);
5245 }
5246
Mirror of the official Mono project git repository