?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[build] Skips RemoteExecuted bases tests on monodroid
[mono-project.git] / mono / mini / mini-arm64.c
blob6f6ba0b38a8a1cfbd11daaea98ae98bc5951cdac
1 /**
2 * \file
3 * ARM64 backend for the Mono code generator
4 *
5 * Copyright 2013 Xamarin, Inc (http://www.xamarin.com)
6 *
7 * Based on mini-arm.c:
8 *
9 * Authors:
10 * Paolo Molaro (lupus@ximian.com)
11 * Dietmar Maurer (dietmar@ximian.com)
12 *
13 * (C) 2003 Ximian, Inc.
14 * Copyright 2003-2011 Novell, Inc (http://www.novell.com)
15 * Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
16 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
17 */
18
19 #include "mini.h"
20 #include "cpu-arm64.h"
21 #include "ir-emit.h"
22 #include "aot-runtime.h"
23 #include "mini-runtime.h"
24
25 #include <mono/arch/arm64/arm64-codegen.h>
26 #include <mono/utils/mono-mmap.h>
27 #include <mono/utils/mono-memory-model.h>
28 #include <mono/metadata/abi-details.h>
29
30 #include "interp/interp.h"
31
32 /*
33 * Documentation:
34 *
35 * - ARM(R) Architecture Reference Manual, ARMv8, for ARMv8-A architecture profile (DDI0487A_a_armv8_arm.pdf)
36 * - Procedure Call Standard for the ARM 64-bit Architecture (AArch64) (IHI0055B_aapcs64.pdf)
37 * - ELF for the ARM 64-bit Architecture (IHI0056B_aaelf64.pdf)
38 *
39 * Register usage:
40 * - ip0/ip1/lr are used as temporary registers
41 * - r27 is used as the rgctx/imt register
42 * - r28 is used to access arguments passed on the stack
43 * - d15/d16 are used as fp temporary registers
44 */
45
46 #define FP_TEMP_REG ARMREG_D16
47 #define FP_TEMP_REG2 ARMREG_D17
48
49 #define THUNK_SIZE (4 * 4)
50
51 /* The single step trampoline */
52 static gpointer ss_trampoline;
53
54 /* The breakpoint trampoline */
55 static gpointer bp_trampoline;
56
57 static gboolean ios_abi;
58
59 static __attribute__ ((__warn_unused_result__)) guint8* emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset);
60
61 const char*
62 mono_arch_regname (int reg)
63 {
64 static const char * rnames[] = {
65 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9",
66 "r10", "r11", "r12", "r13", "r14", "r15", "r16", "r17", "r18", "r19",
67 "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "fp",
68 "lr", "sp"
69 };
70 if (reg >= 0 && reg < 32)
71 return rnames [reg];
72 return "unknown";
73 }
74
75 const char*
76 mono_arch_fregname (int reg)
77 {
78 static const char * rnames[] = {
79 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9",
80 "d10", "d11", "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19",
81 "d20", "d21", "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29",
82 "d30", "d31"
83 };
84 if (reg >= 0 && reg < 32)
85 return rnames [reg];
86 return "unknown fp";
87 }
88
89 int
90 mono_arch_get_argument_info (MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
91 {
92 NOT_IMPLEMENTED;
93 return 0;
94 }
95
96 #define MAX_ARCH_DELEGATE_PARAMS 7
97
98 static gpointer
99 get_delegate_invoke_impl (gboolean has_target, gboolean param_count, guint32 *code_size)
100 {
101 guint8 *code, *start;
102
103 if (has_target) {
104 start = code = mono_global_codeman_reserve (12);
105
106 /* Replace the this argument with the target */
107 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
108 arm_ldrx (code, ARMREG_R0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, target));
109 arm_brx (code, ARMREG_IP0);
110
111 g_assert ((code - start) <= 12);
112
113 mono_arch_flush_icache (start, 12);
114 MONO_PROFILER_RAISE (jit_code_buffer, (start, code - start, MONO_PROFILER_CODE_BUFFER_DELEGATE_INVOKE, NULL));
115 } else {
116 int size, i;
117
118 size = 8 + param_count * 4;
119 start = code = mono_global_codeman_reserve (size);
120
121 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
122 /* slide down the arguments */
123 for (i = 0; i < param_count; ++i)
124 arm_movx (code, i, i + 1);
125 arm_brx (code, ARMREG_IP0);
126
127 g_assert ((code - start) <= size);
128
129 mono_arch_flush_icache (start, size);
130 MONO_PROFILER_RAISE (jit_code_buffer, (start, code - start, MONO_PROFILER_CODE_BUFFER_DELEGATE_INVOKE, NULL));
131 }
132
133 if (code_size)
134 *code_size = code - start;
135
136 return start;
137 }
138
139 /*
140 * mono_arch_get_delegate_invoke_impls:
141 *
142 * Return a list of MonoAotTrampInfo structures for the delegate invoke impl
143 * trampolines.
144 */
145 GSList*
146 mono_arch_get_delegate_invoke_impls (void)
147 {
148 GSList *res = NULL;
149 guint8 *code;
150 guint32 code_len;
151 int i;
152 char *tramp_name;
153
154 code = get_delegate_invoke_impl (TRUE, 0, &code_len);
155 res = g_slist_prepend (res, mono_tramp_info_create ("delegate_invoke_impl_has_target", code, code_len, NULL, NULL));
156
157 for (i = 0; i <= MAX_ARCH_DELEGATE_PARAMS; ++i) {
158 code = get_delegate_invoke_impl (FALSE, i, &code_len);
159 tramp_name = g_strdup_printf ("delegate_invoke_impl_target_%d", i);
160 res = g_slist_prepend (res, mono_tramp_info_create (tramp_name, code, code_len, NULL, NULL));
161 g_free (tramp_name);
162 }
163
164 return res;
165 }
166
167 gpointer
168 mono_arch_get_delegate_invoke_impl (MonoMethodSignature *sig, gboolean has_target)
169 {
170 guint8 *code, *start;
171
172 /*
173 * vtypes are returned in registers, or using the dedicated r8 register, so
174 * they can be supported by delegate invokes.
175 */
176
177 if (has_target) {
178 static guint8* cached = NULL;
179
180 if (cached)
181 return cached;
182
183 if (mono_ee_features.use_aot_trampolines)
184 start = mono_aot_get_trampoline ("delegate_invoke_impl_has_target");
185 else
186 start = get_delegate_invoke_impl (TRUE, 0, NULL);
187 mono_memory_barrier ();
188 cached = start;
189 return cached;
190 } else {
191 static guint8* cache [MAX_ARCH_DELEGATE_PARAMS + 1] = {NULL};
192 int i;
193
194 if (sig->param_count > MAX_ARCH_DELEGATE_PARAMS)
195 return NULL;
196 for (i = 0; i < sig->param_count; ++i)
197 if (!mono_is_regsize_var (sig->params [i]))
198 return NULL;
199
200 code = cache [sig->param_count];
201 if (code)
202 return code;
203
204 if (mono_ee_features.use_aot_trampolines) {
205 char *name = g_strdup_printf ("delegate_invoke_impl_target_%d", sig->param_count);
206 start = mono_aot_get_trampoline (name);
207 g_free (name);
208 } else {
209 start = get_delegate_invoke_impl (FALSE, sig->param_count, NULL);
210 }
211 mono_memory_barrier ();
212 cache [sig->param_count] = start;
213 return start;
214 }
215
216 return NULL;
217 }
218
219 gpointer
220 mono_arch_get_delegate_virtual_invoke_impl (MonoMethodSignature *sig, MonoMethod *method, int offset, gboolean load_imt_reg)
221 {
222 return NULL;
223 }
224
225 gpointer
226 mono_arch_get_this_arg_from_call (mgreg_t *regs, guint8 *code)
227 {
228 return (gpointer)regs [ARMREG_R0];
229 }
230
231 void
232 mono_arch_cpu_init (void)
233 {
234 }
235
236 void
237 mono_arch_init (void)
238 {
239 mono_aot_register_jit_icall ("mono_arm_throw_exception", mono_arm_throw_exception);
240 mono_aot_register_jit_icall ("mono_arm_resume_unwind", mono_arm_resume_unwind);
241
242 if (!mono_aot_only)
243 bp_trampoline = mini_get_breakpoint_trampoline ();
244
245 mono_arm_gsharedvt_init ();
246
247 #if defined(TARGET_IOS)
248 ios_abi = TRUE;
249 #endif
250 }
251
252 void
253 mono_arch_cleanup (void)
254 {
255 }
256
257 guint32
258 mono_arch_cpu_optimizations (guint32 *exclude_mask)
259 {
260 *exclude_mask = 0;
261 return 0;
262 }
263
264 guint32
265 mono_arch_cpu_enumerate_simd_versions (void)
266 {
267 return 0;
268 }
269
270 void
271 mono_arch_register_lowlevel_calls (void)
272 {
273 }
274
275 void
276 mono_arch_finish_init (void)
277 {
278 }
279
280 /* The maximum length is 2 instructions */
281 static guint8*
282 emit_imm (guint8 *code, int dreg, int imm)
283 {
284 // FIXME: Optimize this
285 if (imm < 0) {
286 gint64 limm = imm;
287 arm_movnx (code, dreg, (~limm) & 0xffff, 0);
288 arm_movkx (code, dreg, (limm >> 16) & 0xffff, 16);
289 } else {
290 arm_movzx (code, dreg, imm & 0xffff, 0);
291 if (imm >> 16)
292 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
293 }
294
295 return code;
296 }
297
298 /* The maximum length is 4 instructions */
299 static guint8*
300 emit_imm64 (guint8 *code, int dreg, guint64 imm)
301 {
302 // FIXME: Optimize this
303 arm_movzx (code, dreg, imm & 0xffff, 0);
304 if ((imm >> 16) & 0xffff)
305 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
306 if ((imm >> 32) & 0xffff)
307 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
308 if ((imm >> 48) & 0xffff)
309 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
310
311 return code;
312 }
313
314 guint8*
315 mono_arm_emit_imm64 (guint8 *code, int dreg, gint64 imm)
316 {
317 return emit_imm64 (code, dreg, imm);
318 }
319
320 /*
321 * emit_imm_template:
322 *
323 * Emit a patchable code sequence for constructing a 64 bit immediate.
324 */
325 static guint8*
326 emit_imm64_template (guint8 *code, int dreg)
327 {
328 arm_movzx (code, dreg, 0, 0);
329 arm_movkx (code, dreg, 0, 16);
330 arm_movkx (code, dreg, 0, 32);
331 arm_movkx (code, dreg, 0, 48);
332
333 return code;
334 }
335
336 static inline __attribute__ ((__warn_unused_result__)) guint8*
337 emit_addw_imm (guint8 *code, int dreg, int sreg, int imm)
338 {
339 if (!arm_is_arith_imm (imm)) {
340 code = emit_imm (code, ARMREG_LR, imm);
341 arm_addw (code, dreg, sreg, ARMREG_LR);
342 } else {
343 arm_addw_imm (code, dreg, sreg, imm);
344 }
345 return code;
346 }
347
348 static inline __attribute__ ((__warn_unused_result__)) guint8*
349 emit_addx_imm (guint8 *code, int dreg, int sreg, int imm)
350 {
351 if (!arm_is_arith_imm (imm)) {
352 code = emit_imm (code, ARMREG_LR, imm);
353 arm_addx (code, dreg, sreg, ARMREG_LR);
354 } else {
355 arm_addx_imm (code, dreg, sreg, imm);
356 }
357 return code;
358 }
359
360 static inline __attribute__ ((__warn_unused_result__)) guint8*
361 emit_subw_imm (guint8 *code, int dreg, int sreg, int imm)
362 {
363 if (!arm_is_arith_imm (imm)) {
364 code = emit_imm (code, ARMREG_LR, imm);
365 arm_subw (code, dreg, sreg, ARMREG_LR);
366 } else {
367 arm_subw_imm (code, dreg, sreg, imm);
368 }
369 return code;
370 }
371
372 static inline __attribute__ ((__warn_unused_result__)) guint8*
373 emit_subx_imm (guint8 *code, int dreg, int sreg, int imm)
374 {
375 if (!arm_is_arith_imm (imm)) {
376 code = emit_imm (code, ARMREG_LR, imm);
377 arm_subx (code, dreg, sreg, ARMREG_LR);
378 } else {
379 arm_subx_imm (code, dreg, sreg, imm);
380 }
381 return code;
382 }
383
384 /* Emit sp+=imm. Clobbers ip0/ip1 */
385 static inline __attribute__ ((__warn_unused_result__)) guint8*
386 emit_addx_sp_imm (guint8 *code, int imm)
387 {
388 code = emit_imm (code, ARMREG_IP0, imm);
389 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
390 arm_addx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
391 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
392 return code;
393 }
394
395 /* Emit sp-=imm. Clobbers ip0/ip1 */
396 static inline __attribute__ ((__warn_unused_result__)) guint8*
397 emit_subx_sp_imm (guint8 *code, int imm)
398 {
399 code = emit_imm (code, ARMREG_IP0, imm);
400 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
401 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
402 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
403 return code;
404 }
405
406 static inline __attribute__ ((__warn_unused_result__)) guint8*
407 emit_andw_imm (guint8 *code, int dreg, int sreg, int imm)
408 {
409 // FIXME:
410 code = emit_imm (code, ARMREG_LR, imm);
411 arm_andw (code, dreg, sreg, ARMREG_LR);
412
413 return code;
414 }
415
416 static inline __attribute__ ((__warn_unused_result__)) guint8*
417 emit_andx_imm (guint8 *code, int dreg, int sreg, int imm)
418 {
419 // FIXME:
420 code = emit_imm (code, ARMREG_LR, imm);
421 arm_andx (code, dreg, sreg, ARMREG_LR);
422
423 return code;
424 }
425
426 static inline __attribute__ ((__warn_unused_result__)) guint8*
427 emit_orrw_imm (guint8 *code, int dreg, int sreg, int imm)
428 {
429 // FIXME:
430 code = emit_imm (code, ARMREG_LR, imm);
431 arm_orrw (code, dreg, sreg, ARMREG_LR);
432
433 return code;
434 }
435
436 static inline __attribute__ ((__warn_unused_result__)) guint8*
437 emit_orrx_imm (guint8 *code, int dreg, int sreg, int imm)
438 {
439 // FIXME:
440 code = emit_imm (code, ARMREG_LR, imm);
441 arm_orrx (code, dreg, sreg, ARMREG_LR);
442
443 return code;
444 }
445
446 static inline __attribute__ ((__warn_unused_result__)) guint8*
447 emit_eorw_imm (guint8 *code, int dreg, int sreg, int imm)
448 {
449 // FIXME:
450 code = emit_imm (code, ARMREG_LR, imm);
451 arm_eorw (code, dreg, sreg, ARMREG_LR);
452
453 return code;
454 }
455
456 static inline __attribute__ ((__warn_unused_result__)) guint8*
457 emit_eorx_imm (guint8 *code, int dreg, int sreg, int imm)
458 {
459 // FIXME:
460 code = emit_imm (code, ARMREG_LR, imm);
461 arm_eorx (code, dreg, sreg, ARMREG_LR);
462
463 return code;
464 }
465
466 static inline __attribute__ ((__warn_unused_result__)) guint8*
467 emit_cmpw_imm (guint8 *code, int sreg, int imm)
468 {
469 if (imm == 0) {
470 arm_cmpw (code, sreg, ARMREG_RZR);
471 } else {
472 // FIXME:
473 code = emit_imm (code, ARMREG_LR, imm);
474 arm_cmpw (code, sreg, ARMREG_LR);
475 }
476
477 return code;
478 }
479
480 static inline __attribute__ ((__warn_unused_result__)) guint8*
481 emit_cmpx_imm (guint8 *code, int sreg, int imm)
482 {
483 if (imm == 0) {
484 arm_cmpx (code, sreg, ARMREG_RZR);
485 } else {
486 // FIXME:
487 code = emit_imm (code, ARMREG_LR, imm);
488 arm_cmpx (code, sreg, ARMREG_LR);
489 }
490
491 return code;
492 }
493
494 static inline __attribute__ ((__warn_unused_result__)) guint8*
495 emit_strb (guint8 *code, int rt, int rn, int imm)
496 {
497 if (arm_is_strb_imm (imm)) {
498 arm_strb (code, rt, rn, imm);
499 } else {
500 g_assert (rt != ARMREG_IP0);
501 g_assert (rn != ARMREG_IP0);
502 code = emit_imm (code, ARMREG_IP0, imm);
503 arm_strb_reg (code, rt, rn, ARMREG_IP0);
504 }
505 return code;
506 }
507
508 static inline __attribute__ ((__warn_unused_result__)) guint8*
509 emit_strh (guint8 *code, int rt, int rn, int imm)
510 {
511 if (arm_is_strh_imm (imm)) {
512 arm_strh (code, rt, rn, imm);
513 } else {
514 g_assert (rt != ARMREG_IP0);
515 g_assert (rn != ARMREG_IP0);
516 code = emit_imm (code, ARMREG_IP0, imm);
517 arm_strh_reg (code, rt, rn, ARMREG_IP0);
518 }
519 return code;
520 }
521
522 static inline __attribute__ ((__warn_unused_result__)) guint8*
523 emit_strw (guint8 *code, int rt, int rn, int imm)
524 {
525 if (arm_is_strw_imm (imm)) {
526 arm_strw (code, rt, rn, imm);
527 } else {
528 g_assert (rt != ARMREG_IP0);
529 g_assert (rn != ARMREG_IP0);
530 code = emit_imm (code, ARMREG_IP0, imm);
531 arm_strw_reg (code, rt, rn, ARMREG_IP0);
532 }
533 return code;
534 }
535
536 static inline __attribute__ ((__warn_unused_result__)) guint8*
537 emit_strfpw (guint8 *code, int rt, int rn, int imm)
538 {
539 if (arm_is_strw_imm (imm)) {
540 arm_strfpw (code, rt, rn, imm);
541 } else {
542 g_assert (rn != ARMREG_IP0);
543 code = emit_imm (code, ARMREG_IP0, imm);
544 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
545 arm_strfpw (code, rt, ARMREG_IP0, 0);
546 }
547 return code;
548 }
549
550 static inline __attribute__ ((__warn_unused_result__)) guint8*
551 emit_strfpx (guint8 *code, int rt, int rn, int imm)
552 {
553 if (arm_is_strx_imm (imm)) {
554 arm_strfpx (code, rt, rn, imm);
555 } else {
556 g_assert (rn != ARMREG_IP0);
557 code = emit_imm (code, ARMREG_IP0, imm);
558 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
559 arm_strfpx (code, rt, ARMREG_IP0, 0);
560 }
561 return code;
562 }
563
564 static inline __attribute__ ((__warn_unused_result__)) guint8*
565 emit_strx (guint8 *code, int rt, int rn, int imm)
566 {
567 if (arm_is_strx_imm (imm)) {
568 arm_strx (code, rt, rn, imm);
569 } else {
570 g_assert (rt != ARMREG_IP0);
571 g_assert (rn != ARMREG_IP0);
572 code = emit_imm (code, ARMREG_IP0, imm);
573 arm_strx_reg (code, rt, rn, ARMREG_IP0);
574 }
575 return code;
576 }
577
578 static inline __attribute__ ((__warn_unused_result__)) guint8*
579 emit_ldrb (guint8 *code, int rt, int rn, int imm)
580 {
581 if (arm_is_pimm12_scaled (imm, 1)) {
582 arm_ldrb (code, rt, rn, imm);
583 } else {
584 g_assert (rt != ARMREG_IP0);
585 g_assert (rn != ARMREG_IP0);
586 code = emit_imm (code, ARMREG_IP0, imm);
587 arm_ldrb_reg (code, rt, rn, ARMREG_IP0);
588 }
589 return code;
590 }
591
592 static inline __attribute__ ((__warn_unused_result__)) guint8*
593 emit_ldrsbx (guint8 *code, int rt, int rn, int imm)
594 {
595 if (arm_is_pimm12_scaled (imm, 1)) {
596 arm_ldrsbx (code, rt, rn, imm);
597 } else {
598 g_assert (rt != ARMREG_IP0);
599 g_assert (rn != ARMREG_IP0);
600 code = emit_imm (code, ARMREG_IP0, imm);
601 arm_ldrsbx_reg (code, rt, rn, ARMREG_IP0);
602 }
603 return code;
604 }
605
606 static inline __attribute__ ((__warn_unused_result__)) guint8*
607 emit_ldrh (guint8 *code, int rt, int rn, int imm)
608 {
609 if (arm_is_pimm12_scaled (imm, 2)) {
610 arm_ldrh (code, rt, rn, imm);
611 } else {
612 g_assert (rt != ARMREG_IP0);
613 g_assert (rn != ARMREG_IP0);
614 code = emit_imm (code, ARMREG_IP0, imm);
615 arm_ldrh_reg (code, rt, rn, ARMREG_IP0);
616 }
617 return code;
618 }
619
620 static inline __attribute__ ((__warn_unused_result__)) guint8*
621 emit_ldrshx (guint8 *code, int rt, int rn, int imm)
622 {
623 if (arm_is_pimm12_scaled (imm, 2)) {
624 arm_ldrshx (code, rt, rn, imm);
625 } else {
626 g_assert (rt != ARMREG_IP0);
627 g_assert (rn != ARMREG_IP0);
628 code = emit_imm (code, ARMREG_IP0, imm);
629 arm_ldrshx_reg (code, rt, rn, ARMREG_IP0);
630 }
631 return code;
632 }
633
634 static inline __attribute__ ((__warn_unused_result__)) guint8*
635 emit_ldrswx (guint8 *code, int rt, int rn, int imm)
636 {
637 if (arm_is_pimm12_scaled (imm, 4)) {
638 arm_ldrswx (code, rt, rn, imm);
639 } else {
640 g_assert (rt != ARMREG_IP0);
641 g_assert (rn != ARMREG_IP0);
642 code = emit_imm (code, ARMREG_IP0, imm);
643 arm_ldrswx_reg (code, rt, rn, ARMREG_IP0);
644 }
645 return code;
646 }
647
648 static inline __attribute__ ((__warn_unused_result__)) guint8*
649 emit_ldrw (guint8 *code, int rt, int rn, int imm)
650 {
651 if (arm_is_pimm12_scaled (imm, 4)) {
652 arm_ldrw (code, rt, rn, imm);
653 } else {
654 g_assert (rn != ARMREG_IP0);
655 code = emit_imm (code, ARMREG_IP0, imm);
656 arm_ldrw_reg (code, rt, rn, ARMREG_IP0);
657 }
658 return code;
659 }
660
661 static inline __attribute__ ((__warn_unused_result__)) guint8*
662 emit_ldrx (guint8 *code, int rt, int rn, int imm)
663 {
664 if (arm_is_pimm12_scaled (imm, 8)) {
665 arm_ldrx (code, rt, rn, imm);
666 } else {
667 g_assert (rn != ARMREG_IP0);
668 code = emit_imm (code, ARMREG_IP0, imm);
669 arm_ldrx_reg (code, rt, rn, ARMREG_IP0);
670 }
671 return code;
672 }
673
674 static inline __attribute__ ((__warn_unused_result__)) guint8*
675 emit_ldrfpw (guint8 *code, int rt, int rn, int imm)
676 {
677 if (arm_is_pimm12_scaled (imm, 4)) {
678 arm_ldrfpw (code, rt, rn, imm);
679 } else {
680 g_assert (rn != ARMREG_IP0);
681 code = emit_imm (code, ARMREG_IP0, imm);
682 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
683 arm_ldrfpw (code, rt, ARMREG_IP0, 0);
684 }
685 return code;
686 }
687
688 static inline __attribute__ ((__warn_unused_result__)) guint8*
689 emit_ldrfpx (guint8 *code, int rt, int rn, int imm)
690 {
691 if (arm_is_pimm12_scaled (imm, 8)) {
692 arm_ldrfpx (code, rt, rn, imm);
693 } else {
694 g_assert (rn != ARMREG_IP0);
695 code = emit_imm (code, ARMREG_IP0, imm);
696 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
697 arm_ldrfpx (code, rt, ARMREG_IP0, 0);
698 }
699 return code;
700 }
701
702 guint8*
703 mono_arm_emit_ldrx (guint8 *code, int rt, int rn, int imm)
704 {
705 return emit_ldrx (code, rt, rn, imm);
706 }
707
708 static guint8*
709 emit_call (MonoCompile *cfg, guint8* code, guint32 patch_type, gconstpointer data)
710 {
711 /*
712 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_IMM);
713 code = emit_imm64_template (code, ARMREG_LR);
714 arm_blrx (code, ARMREG_LR);
715 */
716 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_BL);
717 arm_bl (code, code);
718 cfg->thunk_area += THUNK_SIZE;
719 return code;
720 }
721
722 static guint8*
723 emit_aotconst_full (MonoCompile *cfg, MonoJumpInfo **ji, guint8 *code, guint8 *start, int dreg, guint32 patch_type, gconstpointer data)
724 {
725 if (cfg)
726 mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
727 else
728 *ji = mono_patch_info_list_prepend (*ji, code - start, patch_type, data);
729 /* See arch_emit_got_access () in aot-compiler.c */
730 arm_ldrx_lit (code, dreg, 0);
731 arm_nop (code);
732 arm_nop (code);
733 return code;
734 }
735
736 static guint8*
737 emit_aotconst (MonoCompile *cfg, guint8 *code, int dreg, guint32 patch_type, gconstpointer data)
738 {
739 return emit_aotconst_full (cfg, NULL, code, NULL, dreg, patch_type, data);
740 }
741
742 /*
743 * mono_arm_emit_aotconst:
744 *
745 * Emit code to load an AOT constant into DREG. Usable from trampolines.
746 */
747 guint8*
748 mono_arm_emit_aotconst (gpointer ji, guint8 *code, guint8 *code_start, int dreg, guint32 patch_type, gconstpointer data)
749 {
750 return emit_aotconst_full (NULL, (MonoJumpInfo**)ji, code, code_start, dreg, patch_type, data);
751 }
752
753 gboolean
754 mono_arch_have_fast_tls (void)
755 {
756 #ifdef TARGET_IOS
757 return FALSE;
758 #else
759 return TRUE;
760 #endif
761 }
762
763 static guint8*
764 emit_tls_get (guint8 *code, int dreg, int tls_offset)
765 {
766 arm_mrs (code, dreg, ARM_MRS_REG_TPIDR_EL0);
767 if (tls_offset < 256) {
768 arm_ldrx (code, dreg, dreg, tls_offset);
769 } else {
770 code = emit_addx_imm (code, dreg, dreg, tls_offset);
771 arm_ldrx (code, dreg, dreg, 0);
772 }
773 return code;
774 }
775
776 static guint8*
777 emit_tls_set (guint8 *code, int sreg, int tls_offset)
778 {
779 int tmpreg = ARMREG_IP0;
780
781 g_assert (sreg != tmpreg);
782 arm_mrs (code, tmpreg, ARM_MRS_REG_TPIDR_EL0);
783 if (tls_offset < 256) {
784 arm_strx (code, sreg, tmpreg, tls_offset);
785 } else {
786 code = emit_addx_imm (code, tmpreg, tmpreg, tls_offset);
787 arm_strx (code, sreg, tmpreg, 0);
788 }
789 return code;
790 }
791
792 /*
793 * Emits
794 * - mov sp, fp
795 * - ldrp [fp, lr], [sp], !stack_offfset
796 * Clobbers TEMP_REGS.
797 */
798 __attribute__ ((__warn_unused_result__)) guint8*
799 mono_arm_emit_destroy_frame (guint8 *code, int stack_offset, guint64 temp_regs)
800 {
801 // At least one of these registers must be available, or both.
802 gboolean const temp0 = (temp_regs & (1 << ARMREG_IP0)) != 0;
803 gboolean const temp1 = (temp_regs & (1 << ARMREG_IP1)) != 0;
804 g_assert (temp0 || temp1);
805 int const temp = temp0 ? ARMREG_IP0 : ARMREG_IP1;
806
807 arm_movspx (code, ARMREG_SP, ARMREG_FP);
808
809 if (arm_is_ldpx_imm (stack_offset)) {
810 arm_ldpx_post (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, stack_offset);
811 } else {
812 arm_ldpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
813 /* sp += stack_offset */
814 if (temp0 && temp1) {
815 code = emit_addx_sp_imm (code, stack_offset);
816 } else {
817 int imm = stack_offset;
818
819 /* Can't use addx_sp_imm () since we can't clobber both ip0/ip1 */
820 arm_addx_imm (code, temp, ARMREG_SP, 0);
821 while (imm > 256) {
822 arm_addx_imm (code, temp, temp, 256);
823 imm -= 256;
824 }
825 arm_addx_imm (code, ARMREG_SP, temp, imm);
826 }
827 }
828 return code;
829 }
830
831 #define is_call_imm(diff) ((gint)(diff) >= -33554432 && (gint)(diff) <= 33554431)
832
833 static guint8*
834 emit_thunk (guint8 *code, gconstpointer target)
835 {
836 guint8 *p = code;
837
838 arm_ldrx_lit (code, ARMREG_IP0, code + 8);
839 arm_brx (code, ARMREG_IP0);
840 *(guint64*)code = (guint64)target;
841 code += sizeof (guint64);
842
843 mono_arch_flush_icache (p, code - p);
844 return code;
845 }
846
847 static gpointer
848 create_thunk (MonoCompile *cfg, MonoDomain *domain, guchar *code, const guchar *target)
849 {
850 MonoJitInfo *ji;
851 MonoThunkJitInfo *info;
852 guint8 *thunks, *p;
853 int thunks_size;
854 guint8 *orig_target;
855 guint8 *target_thunk;
856
857 if (!domain)
858 domain = mono_domain_get ();
859
860 if (cfg) {
861 /*
862 * This can be called multiple times during JITting,
863 * save the current position in cfg->arch to avoid
864 * doing a O(n^2) search.
865 */
866 if (!cfg->arch.thunks) {
867 cfg->arch.thunks = cfg->thunks;
868 cfg->arch.thunks_size = cfg->thunk_area;
869 }
870 thunks = cfg->arch.thunks;
871 thunks_size = cfg->arch.thunks_size;
872 if (!thunks_size) {
873 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, mono_method_full_name (cfg->method, TRUE));
874 g_assert_not_reached ();
875 }
876
877 g_assert (*(guint32*)thunks == 0);
878 emit_thunk (thunks, target);
879
880 cfg->arch.thunks += THUNK_SIZE;
881 cfg->arch.thunks_size -= THUNK_SIZE;
882
883 return thunks;
884 } else {
885 ji = mini_jit_info_table_find (domain, (char*)code, NULL);
886 g_assert (ji);
887 info = mono_jit_info_get_thunk_info (ji);
888 g_assert (info);
889
890 thunks = (guint8*)ji->code_start + info->thunks_offset;
891 thunks_size = info->thunks_size;
892
893 orig_target = mono_arch_get_call_target (code + 4);
894
895 mono_domain_lock (domain);
896
897 target_thunk = NULL;
898 if (orig_target >= thunks && orig_target < thunks + thunks_size) {
899 /* The call already points to a thunk, because of trampolines etc. */
900 target_thunk = orig_target;
901 } else {
902 for (p = thunks; p < thunks + thunks_size; p += THUNK_SIZE) {
903 if (((guint32*)p) [0] == 0) {
904 /* Free entry */
905 target_thunk = p;
906 break;
907 } else if (((guint64*)p) [1] == (guint64)target) {
908 /* Thunk already points to target */
909 target_thunk = p;
910 break;
911 }
912 }
913 }
914
915 //printf ("THUNK: %p %p %p\n", code, target, target_thunk);
916
917 if (!target_thunk) {
918 mono_domain_unlock (domain);
919 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, cfg ? mono_method_full_name (cfg->method, TRUE) : mono_method_full_name (jinfo_get_method (ji), TRUE));
920 g_assert_not_reached ();
921 }
922
923 emit_thunk (target_thunk, target);
924
925 mono_domain_unlock (domain);
926
927 return target_thunk;
928 }
929 }
930
931 static void
932 arm_patch_full (MonoCompile *cfg, MonoDomain *domain, guint8 *code, guint8 *target, int relocation)
933 {
934 switch (relocation) {
935 case MONO_R_ARM64_B:
936 if (arm_is_bl_disp (code, target)) {
937 arm_b (code, target);
938 } else {
939 gpointer thunk;
940
941 thunk = create_thunk (cfg, domain, code, target);
942 g_assert (arm_is_bl_disp (code, thunk));
943 arm_b (code, thunk);
944 }
945 break;
946 case MONO_R_ARM64_BCC: {
947 int cond;
948
949 cond = arm_get_bcc_cond (code);
950 arm_bcc (code, cond, target);
951 break;
952 }
953 case MONO_R_ARM64_CBZ:
954 arm_set_cbz_target (code, target);
955 break;
956 case MONO_R_ARM64_IMM: {
957 guint64 imm = (guint64)target;
958 int dreg;
959
960 /* emit_imm64_template () */
961 dreg = arm_get_movzx_rd (code);
962 arm_movzx (code, dreg, imm & 0xffff, 0);
963 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
964 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
965 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
966 break;
967 }
968 case MONO_R_ARM64_BL:
969 if (arm_is_bl_disp (code, target)) {
970 arm_bl (code, target);
971 } else {
972 gpointer thunk;
973
974 thunk = create_thunk (cfg, domain, code, target);
975 g_assert (arm_is_bl_disp (code, thunk));
976 arm_bl (code, thunk);
977 }
978 break;
979 default:
980 g_assert_not_reached ();
981 }
982 }
983
984 static void
985 arm_patch_rel (guint8 *code, guint8 *target, int relocation)
986 {
987 arm_patch_full (NULL, NULL, code, target, relocation);
988 }
989
990 void
991 mono_arm_patch (guint8 *code, guint8 *target, int relocation)
992 {
993 arm_patch_rel (code, target, relocation);
994 }
995
996 void
997 mono_arch_patch_code_new (MonoCompile *cfg, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, gpointer target)
998 {
999 guint8 *ip;
1000
1001 ip = ji->ip.i + code;
1002
1003 switch (ji->type) {
1004 case MONO_PATCH_INFO_METHOD_JUMP:
1005 /* ji->relocation is not set by the caller */
1006 arm_patch_full (cfg, domain, ip, (guint8*)target, MONO_R_ARM64_B);
1007 break;
1008 default:
1009 arm_patch_full (cfg, domain, ip, (guint8*)target, ji->relocation);
1010 break;
1011 }
1012 }
1013
1014 void
1015 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
1016 {
1017 }
1018
1019 void
1020 mono_arch_flush_register_windows (void)
1021 {
1022 }
1023
1024 MonoMethod*
1025 mono_arch_find_imt_method (mgreg_t *regs, guint8 *code)
1026 {
1027 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1028 }
1029
1030 MonoVTable*
1031 mono_arch_find_static_call_vtable (mgreg_t *regs, guint8 *code)
1032 {
1033 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1034 }
1035
1036 mgreg_t
1037 mono_arch_context_get_int_reg (MonoContext *ctx, int reg)
1038 {
1039 return ctx->regs [reg];
1040 }
1041
1042 void
1043 mono_arch_context_set_int_reg (MonoContext *ctx, int reg, mgreg_t val)
1044 {
1045 ctx->regs [reg] = val;
1046 }
1047
1048 /*
1049 * mono_arch_set_target:
1050 *
1051 * Set the target architecture the JIT backend should generate code for, in the form
1052 * of a GNU target triplet. Only used in AOT mode.
1053 */
1054 void
1055 mono_arch_set_target (char *mtriple)
1056 {
1057 if (strstr (mtriple, "darwin") || strstr (mtriple, "ios")) {
1058 ios_abi = TRUE;
1059 }
1060 }
1061
1062 static void
1063 add_general (CallInfo *cinfo, ArgInfo *ainfo, int size, gboolean sign)
1064 {
1065 if (cinfo->gr >= PARAM_REGS) {
1066 ainfo->storage = ArgOnStack;
1067 if (ios_abi) {
1068 /* Assume size == align */
1069 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1070 ainfo->offset = cinfo->stack_usage;
1071 ainfo->slot_size = size;
1072 ainfo->sign = sign;
1073 cinfo->stack_usage += size;
1074 } else {
1075 ainfo->offset = cinfo->stack_usage;
1076 ainfo->slot_size = 8;
1077 ainfo->sign = FALSE;
1078 /* Put arguments into 8 byte aligned stack slots */
1079 cinfo->stack_usage += 8;
1080 }
1081 } else {
1082 ainfo->storage = ArgInIReg;
1083 ainfo->reg = cinfo->gr;
1084 cinfo->gr ++;
1085 }
1086 }
1087
1088 static void
1089 add_fp (CallInfo *cinfo, ArgInfo *ainfo, gboolean single)
1090 {
1091 int size = single ? 4 : 8;
1092
1093 if (cinfo->fr >= FP_PARAM_REGS) {
1094 ainfo->storage = single ? ArgOnStackR4 : ArgOnStackR8;
1095 if (ios_abi) {
1096 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1097 ainfo->offset = cinfo->stack_usage;
1098 ainfo->slot_size = size;
1099 cinfo->stack_usage += size;
1100 } else {
1101 ainfo->offset = cinfo->stack_usage;
1102 ainfo->slot_size = 8;
1103 /* Put arguments into 8 byte aligned stack slots */
1104 cinfo->stack_usage += 8;
1105 }
1106 } else {
1107 if (single)
1108 ainfo->storage = ArgInFRegR4;
1109 else
1110 ainfo->storage = ArgInFReg;
1111 ainfo->reg = cinfo->fr;
1112 cinfo->fr ++;
1113 }
1114 }
1115
1116 static gboolean
1117 is_hfa (MonoType *t, int *out_nfields, int *out_esize, int *field_offsets)
1118 {
1119 MonoClass *klass;
1120 gpointer iter;
1121 MonoClassField *field;
1122 MonoType *ftype, *prev_ftype = NULL;
1123 int i, nfields = 0;
1124
1125 klass = mono_class_from_mono_type (t);
1126 iter = NULL;
1127 while ((field = mono_class_get_fields (klass, &iter))) {
1128 if (field->type->attrs & FIELD_ATTRIBUTE_STATIC)
1129 continue;
1130 ftype = mono_field_get_type (field);
1131 ftype = mini_get_underlying_type (ftype);
1132
1133 if (MONO_TYPE_ISSTRUCT (ftype)) {
1134 int nested_nfields, nested_esize;
1135 int nested_field_offsets [16];
1136
1137 if (!is_hfa (ftype, &nested_nfields, &nested_esize, nested_field_offsets))
1138 return FALSE;
1139 if (nested_esize == 4)
1140 ftype = m_class_get_byval_arg (mono_defaults.single_class);
1141 else
1142 ftype = m_class_get_byval_arg (mono_defaults.double_class);
1143 if (prev_ftype && prev_ftype->type != ftype->type)
1144 return FALSE;
1145 prev_ftype = ftype;
1146 for (i = 0; i < nested_nfields; ++i) {
1147 if (nfields + i < 4)
1148 field_offsets [nfields + i] = field->offset - sizeof (MonoObject) + nested_field_offsets [i];
1149 }
1150 nfields += nested_nfields;
1151 } else {
1152 if (!(!ftype->byref && (ftype->type == MONO_TYPE_R4 || ftype->type == MONO_TYPE_R8)))
1153 return FALSE;
1154 if (prev_ftype && prev_ftype->type != ftype->type)
1155 return FALSE;
1156 prev_ftype = ftype;
1157 if (nfields < 4)
1158 field_offsets [nfields] = field->offset - sizeof (MonoObject);
1159 nfields ++;
1160 }
1161 }
1162 if (nfields == 0 || nfields > 4)
1163 return FALSE;
1164 *out_nfields = nfields;
1165 *out_esize = prev_ftype->type == MONO_TYPE_R4 ? 4 : 8;
1166 return TRUE;
1167 }
1168
1169 static void
1170 add_valuetype (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1171 {
1172 int i, size, align_size, nregs, nfields, esize;
1173 int field_offsets [16];
1174 guint32 align;
1175
1176 size = mini_type_stack_size_full (t, &align, cinfo->pinvoke);
1177 align_size = ALIGN_TO (size, 8);
1178
1179 nregs = align_size / 8;
1180 if (is_hfa (t, &nfields, &esize, field_offsets)) {
1181 /*
1182 * The struct might include nested float structs aligned at 8,
1183 * so need to keep track of the offsets of the individual fields.
1184 */
1185 if (cinfo->fr + nfields <= FP_PARAM_REGS) {
1186 ainfo->storage = ArgHFA;
1187 ainfo->reg = cinfo->fr;
1188 ainfo->nregs = nfields;
1189 ainfo->size = size;
1190 ainfo->esize = esize;
1191 for (i = 0; i < nfields; ++i)
1192 ainfo->foffsets [i] = field_offsets [i];
1193 cinfo->fr += ainfo->nregs;
1194 } else {
1195 ainfo->nfregs_to_skip = FP_PARAM_REGS > cinfo->fr ? FP_PARAM_REGS - cinfo->fr : 0;
1196 cinfo->fr = FP_PARAM_REGS;
1197 size = ALIGN_TO (size, 8);
1198 ainfo->storage = ArgVtypeOnStack;
1199 ainfo->offset = cinfo->stack_usage;
1200 ainfo->size = size;
1201 ainfo->hfa = TRUE;
1202 ainfo->nregs = nfields;
1203 ainfo->esize = esize;
1204 cinfo->stack_usage += size;
1205 }
1206 return;
1207 }
1208
1209 if (align_size > 16) {
1210 ainfo->storage = ArgVtypeByRef;
1211 ainfo->size = size;
1212 return;
1213 }
1214
1215 if (cinfo->gr + nregs > PARAM_REGS) {
1216 size = ALIGN_TO (size, 8);
1217 ainfo->storage = ArgVtypeOnStack;
1218 ainfo->offset = cinfo->stack_usage;
1219 ainfo->size = size;
1220 cinfo->stack_usage += size;
1221 cinfo->gr = PARAM_REGS;
1222 } else {
1223 ainfo->storage = ArgVtypeInIRegs;
1224 ainfo->reg = cinfo->gr;
1225 ainfo->nregs = nregs;
1226 ainfo->size = size;
1227 cinfo->gr += nregs;
1228 }
1229 }
1230
1231 static void
1232 add_param (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1233 {
1234 MonoType *ptype;
1235
1236 ptype = mini_get_underlying_type (t);
1237 switch (ptype->type) {
1238 case MONO_TYPE_I1:
1239 add_general (cinfo, ainfo, 1, TRUE);
1240 break;
1241 case MONO_TYPE_U1:
1242 add_general (cinfo, ainfo, 1, FALSE);
1243 break;
1244 case MONO_TYPE_I2:
1245 add_general (cinfo, ainfo, 2, TRUE);
1246 break;
1247 case MONO_TYPE_U2:
1248 add_general (cinfo, ainfo, 2, FALSE);
1249 break;
1250 case MONO_TYPE_I4:
1251 add_general (cinfo, ainfo, 4, TRUE);
1252 break;
1253 case MONO_TYPE_U4:
1254 add_general (cinfo, ainfo, 4, FALSE);
1255 break;
1256 case MONO_TYPE_I:
1257 case MONO_TYPE_U:
1258 case MONO_TYPE_PTR:
1259 case MONO_TYPE_FNPTR:
1260 case MONO_TYPE_OBJECT:
1261 case MONO_TYPE_U8:
1262 case MONO_TYPE_I8:
1263 add_general (cinfo, ainfo, 8, FALSE);
1264 break;
1265 case MONO_TYPE_R8:
1266 add_fp (cinfo, ainfo, FALSE);
1267 break;
1268 case MONO_TYPE_R4:
1269 add_fp (cinfo, ainfo, TRUE);
1270 break;
1271 case MONO_TYPE_VALUETYPE:
1272 case MONO_TYPE_TYPEDBYREF:
1273 add_valuetype (cinfo, ainfo, ptype);
1274 break;
1275 case MONO_TYPE_VOID:
1276 ainfo->storage = ArgNone;
1277 break;
1278 case MONO_TYPE_GENERICINST:
1279 if (!mono_type_generic_inst_is_valuetype (ptype)) {
1280 add_general (cinfo, ainfo, 8, FALSE);
1281 } else if (mini_is_gsharedvt_variable_type (ptype)) {
1282 /*
1283 * Treat gsharedvt arguments as large vtypes
1284 */
1285 ainfo->storage = ArgVtypeByRef;
1286 ainfo->gsharedvt = TRUE;
1287 } else {
1288 add_valuetype (cinfo, ainfo, ptype);
1289 }
1290 break;
1291 case MONO_TYPE_VAR:
1292 case MONO_TYPE_MVAR:
1293 g_assert (mini_is_gsharedvt_type (ptype));
1294 ainfo->storage = ArgVtypeByRef;
1295 ainfo->gsharedvt = TRUE;
1296 break;
1297 default:
1298 g_assert_not_reached ();
1299 break;
1300 }
1301 }
1302
1303 /*
1304 * get_call_info:
1305 *
1306 * Obtain information about a call according to the calling convention.
1307 */
1308 static CallInfo*
1309 get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
1310 {
1311 CallInfo *cinfo;
1312 ArgInfo *ainfo;
1313 int n, pstart, pindex;
1314
1315 n = sig->hasthis + sig->param_count;
1316
1317 if (mp)
1318 cinfo = mono_mempool_alloc0 (mp, sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1319 else
1320 cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1321
1322 cinfo->nargs = n;
1323 cinfo->pinvoke = sig->pinvoke;
1324
1325 /* Return value */
1326 add_param (cinfo, &cinfo->ret, sig->ret);
1327 if (cinfo->ret.storage == ArgVtypeByRef)
1328 cinfo->ret.reg = ARMREG_R8;
1329 /* Reset state */
1330 cinfo->gr = 0;
1331 cinfo->fr = 0;
1332 cinfo->stack_usage = 0;
1333
1334 /* Parameters */
1335 if (sig->hasthis)
1336 add_general (cinfo, cinfo->args + 0, 8, FALSE);
1337 pstart = 0;
1338 for (pindex = pstart; pindex < sig->param_count; ++pindex) {
1339 ainfo = cinfo->args + sig->hasthis + pindex;
1340
1341 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1342 /* Prevent implicit arguments and sig_cookie from
1343 being passed in registers */
1344 cinfo->gr = PARAM_REGS;
1345 cinfo->fr = FP_PARAM_REGS;
1346 /* Emit the signature cookie just before the implicit arguments */
1347 add_param (cinfo, &cinfo->sig_cookie, mono_get_int_type ());
1348 }
1349
1350 add_param (cinfo, ainfo, sig->params [pindex]);
1351 if (ainfo->storage == ArgVtypeByRef) {
1352 /* Pass the argument address in the next register */
1353 if (cinfo->gr >= PARAM_REGS) {
1354 ainfo->storage = ArgVtypeByRefOnStack;
1355 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, 8);
1356 ainfo->offset = cinfo->stack_usage;
1357 cinfo->stack_usage += 8;
1358 } else {
1359 ainfo->reg = cinfo->gr;
1360 cinfo->gr ++;
1361 }
1362 }
1363 }
1364
1365 /* Handle the case where there are no implicit arguments */
1366 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1367 /* Prevent implicit arguments and sig_cookie from
1368 being passed in registers */
1369 cinfo->gr = PARAM_REGS;
1370 cinfo->fr = FP_PARAM_REGS;
1371 /* Emit the signature cookie just before the implicit arguments */
1372 add_param (cinfo, &cinfo->sig_cookie, mono_get_int_type ());
1373 }
1374
1375 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, MONO_ARCH_FRAME_ALIGNMENT);
1376
1377 return cinfo;
1378 }
1379
1380 static int
1381 arg_need_temp (ArgInfo *ainfo)
1382 {
1383 if (ainfo->storage == ArgHFA && ainfo->esize == 4)
1384 return ainfo->size;
1385 return 0;
1386 }
1387
1388 static gpointer
1389 arg_get_storage (CallContext *ccontext, ArgInfo *ainfo)
1390 {
1391 switch (ainfo->storage) {
1392 case ArgVtypeInIRegs:
1393 case ArgInIReg:
1394 return &ccontext->gregs [ainfo->reg];
1395 case ArgInFReg:
1396 case ArgInFRegR4:
1397 case ArgHFA:
1398 return &ccontext->fregs [ainfo->reg];
1399 case ArgOnStack:
1400 case ArgOnStackR4:
1401 case ArgOnStackR8:
1402 case ArgVtypeOnStack:
1403 return ccontext->stack + ainfo->offset;
1404 case ArgVtypeByRef:
1405 return (gpointer) ccontext->gregs [ainfo->reg];
1406 default:
1407 g_error ("Arg storage type not yet supported");
1408 }
1409 }
1410
1411 static void
1412 arg_get_val (CallContext *ccontext, ArgInfo *ainfo, gpointer dest)
1413 {
1414 g_assert (arg_need_temp (ainfo));
1415
1416 float *dest_float = (float*)dest;
1417 for (int k = 0; k < ainfo->nregs; k++) {
1418 *dest_float = *(float*)&ccontext->fregs [ainfo->reg + k];
1419 dest_float++;
1420 }
1421 }
1422
1423 static void
1424 arg_set_val (CallContext *ccontext, ArgInfo *ainfo, gpointer src)
1425 {
1426 g_assert (arg_need_temp (ainfo));
1427
1428 float *src_float = (float*)src;
1429 for (int k = 0; k < ainfo->nregs; k++) {
1430 *(float*)&ccontext->fregs [ainfo->reg + k] = *src_float;
1431 src_float++;
1432 }
1433 }
1434
1435 /* Set arguments in the ccontext (for i2n entry) */
1436 void
1437 mono_arch_set_native_call_context_args (CallContext *ccontext, gpointer frame, MonoMethodSignature *sig)
1438 {
1439 MonoEECallbacks *interp_cb = mini_get_interp_callbacks ();
1440 CallInfo *cinfo = get_call_info (NULL, sig);
1441 gpointer storage;
1442 ArgInfo *ainfo;
1443
1444 memset (ccontext, 0, sizeof (CallContext));
1445
1446 ccontext->stack_size = ALIGN_TO (cinfo->stack_usage, MONO_ARCH_FRAME_ALIGNMENT);
1447 if (ccontext->stack_size)
1448 ccontext->stack = calloc (1, ccontext->stack_size);
1449
1450 if (sig->ret->type != MONO_TYPE_VOID) {
1451 ainfo = &cinfo->ret;
1452 if (ainfo->storage == ArgVtypeByRef) {
1453 storage = interp_cb->frame_arg_to_storage ((MonoInterpFrameHandle)frame, sig, -1);
1454 ccontext->gregs [cinfo->ret.reg] = (mgreg_t)storage;
1455 }
1456 }
1457
1458 g_assert (!sig->hasthis);
1459
1460 for (int i = 0; i < sig->param_count; i++) {
1461 ainfo = &cinfo->args [i];
1462
1463 if (ainfo->storage == ArgVtypeByRef) {
1464 ccontext->gregs [ainfo->reg] = (mgreg_t)interp_cb->frame_arg_to_storage ((MonoInterpFrameHandle)frame, sig, i);
1465 continue;
1466 }
1467
1468 int temp_size = arg_need_temp (ainfo);
1469
1470 if (temp_size)
1471 storage = alloca (temp_size);
1472 else
1473 storage = arg_get_storage (ccontext, ainfo);
1474
1475 interp_cb->frame_arg_to_data ((MonoInterpFrameHandle)frame, sig, i, storage);
1476 if (temp_size)
1477 arg_set_val (ccontext, ainfo, storage);
1478 }
1479
1480 g_free (cinfo);
1481 }
1482
1483 /* Set return value in the ccontext (for n2i return) */
1484 void
1485 mono_arch_set_native_call_context_ret (CallContext *ccontext, gpointer frame, MonoMethodSignature *sig)
1486 {
1487 MonoEECallbacks *interp_cb = mini_get_interp_callbacks ();
1488 CallInfo *cinfo = get_call_info (NULL, sig);
1489 gpointer storage;
1490 ArgInfo *ainfo;
1491
1492 if (sig->ret->type != MONO_TYPE_VOID) {
1493 ainfo = &cinfo->ret;
1494 if (ainfo->storage != ArgVtypeByRef) {
1495 int temp_size = arg_need_temp (ainfo);
1496
1497 if (temp_size)
1498 storage = alloca (temp_size);
1499 else
1500 storage = arg_get_storage (ccontext, ainfo);
1501 memset (ccontext, 0, sizeof (CallContext)); // FIXME
1502 interp_cb->frame_arg_to_data ((MonoInterpFrameHandle)frame, sig, -1, storage);
1503 if (temp_size)
1504 arg_set_val (ccontext, ainfo, storage);
1505 }
1506 }
1507
1508 g_free (cinfo);
1509 }
1510
1511 /* Gets the arguments from ccontext (for n2i entry) */
1512 void
1513 mono_arch_get_native_call_context_args (CallContext *ccontext, gpointer frame, MonoMethodSignature *sig)
1514 {
1515 MonoEECallbacks *interp_cb = mini_get_interp_callbacks ();
1516 CallInfo *cinfo = get_call_info (NULL, sig);
1517 gpointer storage;
1518 ArgInfo *ainfo;
1519
1520 if (sig->ret->type != MONO_TYPE_VOID) {
1521 ainfo = &cinfo->ret;
1522 if (ainfo->storage == ArgVtypeByRef) {
1523 storage = (gpointer) ccontext->gregs [cinfo->ret.reg];
1524 interp_cb->frame_arg_set_storage ((MonoInterpFrameHandle)frame, sig, -1, storage);
1525 }
1526 }
1527
1528 for (int i = 0; i < sig->param_count + sig->hasthis; i++) {
1529 ainfo = &cinfo->args [i];
1530 int temp_size = arg_need_temp (ainfo);
1531
1532 if (temp_size) {
1533 storage = alloca (temp_size);
1534 arg_get_val (ccontext, ainfo, storage);
1535 } else {
1536 storage = arg_get_storage (ccontext, ainfo);
1537 }
1538 interp_cb->data_to_frame_arg ((MonoInterpFrameHandle)frame, sig, i, storage);
1539 }
1540
1541 g_free (cinfo);
1542 }
1543
1544 /* Gets the return value from ccontext (for i2n exit) */
1545 void
1546 mono_arch_get_native_call_context_ret (CallContext *ccontext, gpointer frame, MonoMethodSignature *sig)
1547 {
1548 MonoEECallbacks *interp_cb = mini_get_interp_callbacks ();
1549 CallInfo *cinfo = get_call_info (NULL, sig);
1550 ArgInfo *ainfo;
1551 gpointer storage;
1552
1553 if (sig->ret->type != MONO_TYPE_VOID) {
1554 ainfo = &cinfo->ret;
1555 if (ainfo->storage != ArgVtypeByRef) {
1556 int temp_size = arg_need_temp (ainfo);
1557
1558 if (temp_size) {
1559 storage = alloca (temp_size);
1560 arg_get_val (ccontext, ainfo, storage);
1561 } else {
1562 storage = arg_get_storage (ccontext, ainfo);
1563 }
1564 interp_cb->data_to_frame_arg ((MonoInterpFrameHandle)frame, sig, -1, storage);
1565 }
1566 }
1567
1568 g_free (cinfo);
1569 }
1570
1571 typedef struct {
1572 MonoMethodSignature *sig;
1573 CallInfo *cinfo;
1574 MonoType *rtype;
1575 MonoType **param_types;
1576 int n_fpargs, n_fpret;
1577 } ArchDynCallInfo;
1578
1579 static gboolean
1580 dyn_call_supported (CallInfo *cinfo, MonoMethodSignature *sig)
1581 {
1582 int i;
1583
1584 // FIXME: Add more cases
1585 switch (cinfo->ret.storage) {
1586 case ArgNone:
1587 case ArgInIReg:
1588 case ArgInFReg:
1589 case ArgInFRegR4:
1590 case ArgVtypeByRef:
1591 break;
1592 case ArgVtypeInIRegs:
1593 if (cinfo->ret.nregs > 2)
1594 return FALSE;
1595 break;
1596 case ArgHFA:
1597 break;
1598 default:
1599 return FALSE;
1600 }
1601
1602 for (i = 0; i < cinfo->nargs; ++i) {
1603 ArgInfo *ainfo = &cinfo->args [i];
1604
1605 switch (ainfo->storage) {
1606 case ArgInIReg:
1607 case ArgVtypeInIRegs:
1608 case ArgInFReg:
1609 case ArgInFRegR4:
1610 case ArgHFA:
1611 case ArgVtypeByRef:
1612 case ArgOnStack:
1613 case ArgVtypeOnStack:
1614 break;
1615 default:
1616 return FALSE;
1617 }
1618 }
1619
1620 return TRUE;
1621 }
1622
1623 MonoDynCallInfo*
1624 mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
1625 {
1626 ArchDynCallInfo *info;
1627 CallInfo *cinfo;
1628 int i;
1629
1630 cinfo = get_call_info (NULL, sig);
1631
1632 if (!dyn_call_supported (cinfo, sig)) {
1633 g_free (cinfo);
1634 return NULL;
1635 }
1636
1637 info = g_new0 (ArchDynCallInfo, 1);
1638 // FIXME: Preprocess the info to speed up start_dyn_call ()
1639 info->sig = sig;
1640 info->cinfo = cinfo;
1641 info->rtype = mini_get_underlying_type (sig->ret);
1642 info->param_types = g_new0 (MonoType*, sig->param_count);
1643 for (i = 0; i < sig->param_count; ++i)
1644 info->param_types [i] = mini_get_underlying_type (sig->params [i]);
1645
1646 switch (cinfo->ret.storage) {
1647 case ArgInFReg:
1648 case ArgInFRegR4:
1649 info->n_fpret = 1;
1650 break;
1651 case ArgHFA:
1652 info->n_fpret = cinfo->ret.nregs;
1653 break;
1654 default:
1655 break;
1656 }
1657
1658 return (MonoDynCallInfo*)info;
1659 }
1660
1661 void
1662 mono_arch_dyn_call_free (MonoDynCallInfo *info)
1663 {
1664 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1665
1666 g_free (ainfo->cinfo);
1667 g_free (ainfo->param_types);
1668 g_free (ainfo);
1669 }
1670
1671 int
1672 mono_arch_dyn_call_get_buf_size (MonoDynCallInfo *info)
1673 {
1674 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1675
1676 g_assert (ainfo->cinfo->stack_usage % MONO_ARCH_FRAME_ALIGNMENT == 0);
1677 return sizeof (DynCallArgs) + ainfo->cinfo->stack_usage;
1678 }
1679
1680 static double
1681 bitcast_r4_to_r8 (float f)
1682 {
1683 float *p = &f;
1684
1685 return *(double*)p;
1686 }
1687
1688 static float
1689 bitcast_r8_to_r4 (double f)
1690 {
1691 double *p = &f;
1692
1693 return *(float*)p;
1694 }
1695
1696 void
1697 mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, guint8 *buf)
1698 {
1699 ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
1700 DynCallArgs *p = (DynCallArgs*)buf;
1701 int aindex, arg_index, greg, i, pindex;
1702 MonoMethodSignature *sig = dinfo->sig;
1703 CallInfo *cinfo = dinfo->cinfo;
1704 int buffer_offset = 0;
1705
1706 p->res = 0;
1707 p->ret = ret;
1708 p->n_fpargs = dinfo->n_fpargs;
1709 p->n_fpret = dinfo->n_fpret;
1710 p->n_stackargs = cinfo->stack_usage / sizeof (mgreg_t);
1711
1712 arg_index = 0;
1713 greg = 0;
1714 pindex = 0;
1715
1716 if (sig->hasthis)
1717 p->regs [greg ++] = (mgreg_t)*(args [arg_index ++]);
1718
1719 if (cinfo->ret.storage == ArgVtypeByRef)
1720 p->regs [ARMREG_R8] = (mgreg_t)ret;
1721
1722 for (aindex = pindex; aindex < sig->param_count; aindex++) {
1723 MonoType *t = dinfo->param_types [aindex];
1724 gpointer *arg = args [arg_index ++];
1725 ArgInfo *ainfo = &cinfo->args [aindex + sig->hasthis];
1726 int slot = -1;
1727
1728 if (ainfo->storage == ArgOnStack || ainfo->storage == ArgVtypeOnStack) {
1729 slot = PARAM_REGS + 1 + (ainfo->offset / sizeof (mgreg_t));
1730 } else {
1731 slot = ainfo->reg;
1732 }
1733
1734 if (t->byref) {
1735 p->regs [slot] = (mgreg_t)*arg;
1736 continue;
1737 }
1738
1739 if (ios_abi && ainfo->storage == ArgOnStack) {
1740 guint8 *stack_arg = (guint8*)&(p->regs [PARAM_REGS + 1]) + ainfo->offset;
1741 gboolean handled = TRUE;
1742
1743 /* Special case arguments smaller than 1 machine word */
1744 switch (t->type) {
1745 case MONO_TYPE_U1:
1746 *(guint8*)stack_arg = *(guint8*)arg;
1747 break;
1748 case MONO_TYPE_I1:
1749 *(gint8*)stack_arg = *(gint8*)arg;
1750 break;
1751 case MONO_TYPE_U2:
1752 *(guint16*)stack_arg = *(guint16*)arg;
1753 break;
1754 case MONO_TYPE_I2:
1755 *(gint16*)stack_arg = *(gint16*)arg;
1756 break;
1757 case MONO_TYPE_I4:
1758 *(gint32*)stack_arg = *(gint32*)arg;
1759 break;
1760 case MONO_TYPE_U4:
1761 *(guint32*)stack_arg = *(guint32*)arg;
1762 break;
1763 default:
1764 handled = FALSE;
1765 break;
1766 }
1767 if (handled)
1768 continue;
1769 }
1770
1771 switch (t->type) {
1772 case MONO_TYPE_OBJECT:
1773 case MONO_TYPE_PTR:
1774 case MONO_TYPE_I:
1775 case MONO_TYPE_U:
1776 case MONO_TYPE_I8:
1777 case MONO_TYPE_U8:
1778 p->regs [slot] = (mgreg_t)*arg;
1779 break;
1780 case MONO_TYPE_U1:
1781 p->regs [slot] = *(guint8*)arg;
1782 break;
1783 case MONO_TYPE_I1:
1784 p->regs [slot] = *(gint8*)arg;
1785 break;
1786 case MONO_TYPE_I2:
1787 p->regs [slot] = *(gint16*)arg;
1788 break;
1789 case MONO_TYPE_U2:
1790 p->regs [slot] = *(guint16*)arg;
1791 break;
1792 case MONO_TYPE_I4:
1793 p->regs [slot] = *(gint32*)arg;
1794 break;
1795 case MONO_TYPE_U4:
1796 p->regs [slot] = *(guint32*)arg;
1797 break;
1798 case MONO_TYPE_R4:
1799 p->fpregs [ainfo->reg] = bitcast_r4_to_r8 (*(float*)arg);
1800 p->n_fpargs ++;
1801 break;
1802 case MONO_TYPE_R8:
1803 p->fpregs [ainfo->reg] = *(double*)arg;
1804 p->n_fpargs ++;
1805 break;
1806 case MONO_TYPE_GENERICINST:
1807 if (MONO_TYPE_IS_REFERENCE (t)) {
1808 p->regs [slot] = (mgreg_t)*arg;
1809 break;
1810 } else {
1811 if (t->type == MONO_TYPE_GENERICINST && mono_class_is_nullable (mono_class_from_mono_type (t))) {
1812 MonoClass *klass = mono_class_from_mono_type (t);
1813 guint8 *nullable_buf;
1814 int size;
1815
1816 /*
1817 * Use p->buffer as a temporary buffer since the data needs to be available after this call
1818 * if the nullable param is passed by ref.
1819 */
1820 size = mono_class_value_size (klass, NULL);
1821 nullable_buf = p->buffer + buffer_offset;
1822 buffer_offset += size;
1823 g_assert (buffer_offset <= 256);
1824
1825 /* The argument pointed to by arg is either a boxed vtype or null */
1826 mono_nullable_init (nullable_buf, (MonoObject*)arg, klass);
1827
1828 arg = (gpointer*)nullable_buf;
1829 /* Fall though */
1830 } else {
1831 /* Fall though */
1832 }
1833 }
1834 case MONO_TYPE_VALUETYPE:
1835 switch (ainfo->storage) {
1836 case ArgVtypeInIRegs:
1837 for (i = 0; i < ainfo->nregs; ++i)
1838 p->regs [slot ++] = ((mgreg_t*)arg) [i];
1839 break;
1840 case ArgHFA:
1841 if (ainfo->esize == 4) {
1842 for (i = 0; i < ainfo->nregs; ++i)
1843 p->fpregs [ainfo->reg + i] = bitcast_r4_to_r8 (((float*)arg) [ainfo->foffsets [i] / 4]);
1844 } else {
1845 for (i = 0; i < ainfo->nregs; ++i)
1846 p->fpregs [ainfo->reg + i] = ((double*)arg) [ainfo->foffsets [i] / 8];
1847 }
1848 p->n_fpargs += ainfo->nregs;
1849 break;
1850 case ArgVtypeByRef:
1851 p->regs [slot] = (mgreg_t)arg;
1852 break;
1853 case ArgVtypeOnStack:
1854 for (i = 0; i < ainfo->size / 8; ++i)
1855 p->regs [slot ++] = ((mgreg_t*)arg) [i];
1856 break;
1857 default:
1858 g_assert_not_reached ();
1859 break;
1860 }
1861 break;
1862 default:
1863 g_assert_not_reached ();
1864 }
1865 }
1866 }
1867
1868 void
1869 mono_arch_finish_dyn_call (MonoDynCallInfo *info, guint8 *buf)
1870 {
1871 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1872 CallInfo *cinfo = ainfo->cinfo;
1873 DynCallArgs *args = (DynCallArgs*)buf;
1874 MonoType *ptype = ainfo->rtype;
1875 guint8 *ret = args->ret;
1876 mgreg_t res = args->res;
1877 mgreg_t res2 = args->res2;
1878 int i;
1879
1880 if (cinfo->ret.storage == ArgVtypeByRef)
1881 return;
1882
1883 switch (ptype->type) {
1884 case MONO_TYPE_VOID:
1885 *(gpointer*)ret = NULL;
1886 break;
1887 case MONO_TYPE_OBJECT:
1888 case MONO_TYPE_I:
1889 case MONO_TYPE_U:
1890 case MONO_TYPE_PTR:
1891 *(gpointer*)ret = (gpointer)res;
1892 break;
1893 case MONO_TYPE_I1:
1894 *(gint8*)ret = res;
1895 break;
1896 case MONO_TYPE_U1:
1897 *(guint8*)ret = res;
1898 break;
1899 case MONO_TYPE_I2:
1900 *(gint16*)ret = res;
1901 break;
1902 case MONO_TYPE_U2:
1903 *(guint16*)ret = res;
1904 break;
1905 case MONO_TYPE_I4:
1906 *(gint32*)ret = res;
1907 break;
1908 case MONO_TYPE_U4:
1909 *(guint32*)ret = res;
1910 break;
1911 case MONO_TYPE_I8:
1912 case MONO_TYPE_U8:
1913 *(guint64*)ret = res;
1914 break;
1915 case MONO_TYPE_R4:
1916 *(float*)ret = bitcast_r8_to_r4 (args->fpregs [0]);
1917 break;
1918 case MONO_TYPE_R8:
1919 *(double*)ret = args->fpregs [0];
1920 break;
1921 case MONO_TYPE_GENERICINST:
1922 if (MONO_TYPE_IS_REFERENCE (ptype)) {
1923 *(gpointer*)ret = (gpointer)res;
1924 break;
1925 } else {
1926 /* Fall though */
1927 }
1928 case MONO_TYPE_VALUETYPE:
1929 switch (ainfo->cinfo->ret.storage) {
1930 case ArgVtypeInIRegs:
1931 *(mgreg_t*)ret = res;
1932 if (ainfo->cinfo->ret.nregs > 1)
1933 ((mgreg_t*)ret) [1] = res2;
1934 break;
1935 case ArgHFA:
1936 /* Use the same area for returning fp values */
1937 if (cinfo->ret.esize == 4) {
1938 for (i = 0; i < cinfo->ret.nregs; ++i)
1939 ((float*)ret) [cinfo->ret.foffsets [i] / 4] = bitcast_r8_to_r4 (args->fpregs [i]);
1940 } else {
1941 for (i = 0; i < cinfo->ret.nregs; ++i)
1942 ((double*)ret) [cinfo->ret.foffsets [i] / 8] = args->fpregs [i];
1943 }
1944 break;
1945 default:
1946 g_assert_not_reached ();
1947 break;
1948 }
1949 break;
1950 default:
1951 g_assert_not_reached ();
1952 }
1953 }
1954
1955 #if __APPLE__
1956 void sys_icache_invalidate (void *start, size_t len);
1957 #endif
1958
1959 void
1960 mono_arch_flush_icache (guint8 *code, gint size)
1961 {
1962 #ifndef MONO_CROSS_COMPILE
1963 #if __APPLE__
1964 sys_icache_invalidate (code, size);
1965 #else
1966 /* Don't rely on GCC's __clear_cache implementation, as it caches
1967 * icache/dcache cache line sizes, that can vary between cores on
1968 * big.LITTLE architectures. */
1969 guint64 end = (guint64) (code + size);
1970 guint64 addr;
1971 /* always go with cacheline size of 4 bytes as this code isn't perf critical
1972 * anyway. Reading the cache line size from a machine register can be racy
1973 * on a big.LITTLE architecture if the cores don't have the same cache line
1974 * sizes. */
1975 const size_t icache_line_size = 4;
1976 const size_t dcache_line_size = 4;
1977
1978 addr = (guint64) code & ~(guint64) (dcache_line_size - 1);
1979 for (; addr < end; addr += dcache_line_size)
1980 asm volatile("dc civac, %0" : : "r" (addr) : "memory");
1981 asm volatile("dsb ish" : : : "memory");
1982
1983 addr = (guint64) code & ~(guint64) (icache_line_size - 1);
1984 for (; addr < end; addr += icache_line_size)
1985 asm volatile("ic ivau, %0" : : "r" (addr) : "memory");
1986
1987 asm volatile ("dsb ish" : : : "memory");
1988 asm volatile ("isb" : : : "memory");
1989 #endif
1990 #endif
1991 }
1992
1993 #ifndef DISABLE_JIT
1994
1995 gboolean
1996 mono_arch_opcode_needs_emulation (MonoCompile *cfg, int opcode)
1997 {
1998 NOT_IMPLEMENTED;
1999 return FALSE;
2000 }
2001
2002 GList *
2003 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
2004 {
2005 GList *vars = NULL;
2006 int i;
2007
2008 for (i = 0; i < cfg->num_varinfo; i++) {
2009 MonoInst *ins = cfg->varinfo [i];
2010 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
2011
2012 /* unused vars */
2013 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
2014 continue;
2015
2016 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
2017 (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
2018 continue;
2019
2020 if (mono_is_regsize_var (ins->inst_vtype)) {
2021 g_assert (MONO_VARINFO (cfg, i)->reg == -1);
2022 g_assert (i == vmv->idx);
2023 vars = g_list_prepend (vars, vmv);
2024 }
2025 }
2026
2027 vars = mono_varlist_sort (cfg, vars, 0);
2028
2029 return vars;
2030 }
2031
2032 GList *
2033 mono_arch_get_global_int_regs (MonoCompile *cfg)
2034 {
2035 GList *regs = NULL;
2036 int i;
2037
2038 /* r28 is reserved for cfg->arch.args_reg */
2039 /* r27 is reserved for the imt argument */
2040 for (i = ARMREG_R19; i <= ARMREG_R26; ++i)
2041 regs = g_list_prepend (regs, GUINT_TO_POINTER (i));
2042
2043 return regs;
2044 }
2045
2046 guint32
2047 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
2048 {
2049 MonoInst *ins = cfg->varinfo [vmv->idx];
2050
2051 if (ins->opcode == OP_ARG)
2052 return 1;
2053 else
2054 return 2;
2055 }
2056
2057 void
2058 mono_arch_create_vars (MonoCompile *cfg)
2059 {
2060 MonoMethodSignature *sig;
2061 CallInfo *cinfo;
2062
2063 sig = mono_method_signature (cfg->method);
2064 if (!cfg->arch.cinfo)
2065 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
2066 cinfo = cfg->arch.cinfo;
2067
2068 if (cinfo->ret.storage == ArgVtypeByRef) {
2069 cfg->vret_addr = mono_compile_create_var (cfg, mono_get_int_type (), OP_LOCAL);
2070 cfg->vret_addr->flags |= MONO_INST_VOLATILE;
2071 }
2072
2073 if (cfg->gen_sdb_seq_points) {
2074 MonoInst *ins;
2075
2076 if (cfg->compile_aot) {
2077 ins = mono_compile_create_var (cfg, mono_get_int_type (), OP_LOCAL);
2078 ins->flags |= MONO_INST_VOLATILE;
2079 cfg->arch.seq_point_info_var = ins;
2080 }
2081
2082 ins = mono_compile_create_var (cfg, mono_get_int_type (), OP_LOCAL);
2083 ins->flags |= MONO_INST_VOLATILE;
2084 cfg->arch.ss_tramp_var = ins;
2085
2086 ins = mono_compile_create_var (cfg, mono_get_int_type (), OP_LOCAL);
2087 ins->flags |= MONO_INST_VOLATILE;
2088 cfg->arch.bp_tramp_var = ins;
2089 }
2090
2091 if (cfg->method->save_lmf) {
2092 cfg->create_lmf_var = TRUE;
2093 cfg->lmf_ir = TRUE;
2094 }
2095 }
2096
2097 void
2098 mono_arch_allocate_vars (MonoCompile *cfg)
2099 {
2100 MonoMethodSignature *sig;
2101 MonoInst *ins;
2102 CallInfo *cinfo;
2103 ArgInfo *ainfo;
2104 int i, offset, size, align;
2105 guint32 locals_stack_size, locals_stack_align;
2106 gint32 *offsets;
2107
2108 /*
2109 * Allocate arguments and locals to either register (OP_REGVAR) or to a stack slot (OP_REGOFFSET).
2110 * Compute cfg->stack_offset and update cfg->used_int_regs.
2111 */
2112
2113 sig = mono_method_signature (cfg->method);
2114
2115 if (!cfg->arch.cinfo)
2116 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
2117 cinfo = cfg->arch.cinfo;
2118
2119 /*
2120 * The ARM64 ABI always uses a frame pointer.
2121 * The instruction set prefers positive offsets, so fp points to the bottom of the
2122 * frame, and stack slots are at positive offsets.
2123 * If some arguments are received on the stack, their offsets relative to fp can
2124 * not be computed right now because the stack frame might grow due to spilling
2125 * done by the local register allocator. To solve this, we reserve a register
2126 * which points to them.
2127 * The stack frame looks like this:
2128 * args_reg -> <bottom of parent frame>
2129 * <locals etc>
2130 * fp -> <saved fp+lr>
2131 * sp -> <localloc/params area>
2132 */
2133 cfg->frame_reg = ARMREG_FP;
2134 cfg->flags |= MONO_CFG_HAS_SPILLUP;
2135 offset = 0;
2136
2137 /* Saved fp+lr */
2138 offset += 16;
2139
2140 if (cinfo->stack_usage) {
2141 g_assert (!(cfg->used_int_regs & (1 << ARMREG_R28)));
2142 cfg->arch.args_reg = ARMREG_R28;
2143 cfg->used_int_regs |= 1 << ARMREG_R28;
2144 }
2145
2146 if (cfg->method->save_lmf) {
2147 /* The LMF var is allocated normally */
2148 } else {
2149 /* Callee saved regs */
2150 cfg->arch.saved_gregs_offset = offset;
2151 for (i = 0; i < 32; ++i)
2152 if ((MONO_ARCH_CALLEE_SAVED_REGS & (1 << i)) && (cfg->used_int_regs & (1 << i)))
2153 offset += 8;
2154 }
2155
2156 /* Return value */
2157 switch (cinfo->ret.storage) {
2158 case ArgNone:
2159 break;
2160 case ArgInIReg:
2161 case ArgInFReg:
2162 case ArgInFRegR4:
2163 cfg->ret->opcode = OP_REGVAR;
2164 cfg->ret->dreg = cinfo->ret.reg;
2165 break;
2166 case ArgVtypeInIRegs:
2167 case ArgHFA:
2168 /* Allocate a local to hold the result, the epilog will copy it to the correct place */
2169 cfg->ret->opcode = OP_REGOFFSET;
2170 cfg->ret->inst_basereg = cfg->frame_reg;
2171 cfg->ret->inst_offset = offset;
2172 if (cinfo->ret.storage == ArgHFA)
2173 // FIXME:
2174 offset += 64;
2175 else
2176 offset += 16;
2177 break;
2178 case ArgVtypeByRef:
2179 /* This variable will be initalized in the prolog from R8 */
2180 cfg->vret_addr->opcode = OP_REGOFFSET;
2181 cfg->vret_addr->inst_basereg = cfg->frame_reg;
2182 cfg->vret_addr->inst_offset = offset;
2183 offset += 8;
2184 if (G_UNLIKELY (cfg->verbose_level > 1)) {
2185 printf ("vret_addr =");
2186 mono_print_ins (cfg->vret_addr);
2187 }
2188 break;
2189 default:
2190 g_assert_not_reached ();
2191 break;
2192 }
2193
2194 /* Arguments */
2195 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
2196 ainfo = cinfo->args + i;
2197
2198 ins = cfg->args [i];
2199 if (ins->opcode == OP_REGVAR)
2200 continue;
2201
2202 ins->opcode = OP_REGOFFSET;
2203 ins->inst_basereg = cfg->frame_reg;
2204
2205 switch (ainfo->storage) {
2206 case ArgInIReg:
2207 case ArgInFReg:
2208 case ArgInFRegR4:
2209 // FIXME: Use nregs/size
2210 /* These will be copied to the stack in the prolog */
2211 ins->inst_offset = offset;
2212 offset += 8;
2213 break;
2214 case ArgOnStack:
2215 case ArgOnStackR4:
2216 case ArgOnStackR8:
2217 case ArgVtypeOnStack:
2218 /* These are in the parent frame */
2219 g_assert (cfg->arch.args_reg);
2220 ins->inst_basereg = cfg->arch.args_reg;
2221 ins->inst_offset = ainfo->offset;
2222 break;
2223 case ArgVtypeInIRegs:
2224 case ArgHFA:
2225 ins->opcode = OP_REGOFFSET;
2226 ins->inst_basereg = cfg->frame_reg;
2227 /* These arguments are saved to the stack in the prolog */
2228 ins->inst_offset = offset;
2229 if (cfg->verbose_level >= 2)
2230 printf ("arg %d allocated to %s+0x%0x.\n", i, mono_arch_regname (ins->inst_basereg), (int)ins->inst_offset);
2231 if (ainfo->storage == ArgHFA)
2232 // FIXME:
2233 offset += 64;
2234 else
2235 offset += 16;
2236 break;
2237 case ArgVtypeByRefOnStack: {
2238 MonoInst *vtaddr;
2239
2240 if (ainfo->gsharedvt) {
2241 ins->opcode = OP_REGOFFSET;
2242 ins->inst_basereg = cfg->arch.args_reg;
2243 ins->inst_offset = ainfo->offset;
2244 break;
2245 }
2246
2247 /* The vtype address is in the parent frame */
2248 g_assert (cfg->arch.args_reg);
2249 MONO_INST_NEW (cfg, vtaddr, 0);
2250 vtaddr->opcode = OP_REGOFFSET;
2251 vtaddr->inst_basereg = cfg->arch.args_reg;
2252 vtaddr->inst_offset = ainfo->offset;
2253
2254 /* Need an indirection */
2255 ins->opcode = OP_VTARG_ADDR;
2256 ins->inst_left = vtaddr;
2257 break;
2258 }
2259 case ArgVtypeByRef: {
2260 MonoInst *vtaddr;
2261
2262 if (ainfo->gsharedvt) {
2263 ins->opcode = OP_REGOFFSET;
2264 ins->inst_basereg = cfg->frame_reg;
2265 ins->inst_offset = offset;
2266 offset += 8;
2267 break;
2268 }
2269
2270 /* The vtype address is in a register, will be copied to the stack in the prolog */
2271 MONO_INST_NEW (cfg, vtaddr, 0);
2272 vtaddr->opcode = OP_REGOFFSET;
2273 vtaddr->inst_basereg = cfg->frame_reg;
2274 vtaddr->inst_offset = offset;
2275 offset += 8;
2276
2277 /* Need an indirection */
2278 ins->opcode = OP_VTARG_ADDR;
2279 ins->inst_left = vtaddr;
2280 break;
2281 }
2282 default:
2283 g_assert_not_reached ();
2284 break;
2285 }
2286 }
2287
2288 /* Allocate these first so they have a small offset, OP_SEQ_POINT depends on this */
2289 // FIXME: Allocate these to registers
2290 ins = cfg->arch.seq_point_info_var;
2291 if (ins) {
2292 size = 8;
2293 align = 8;
2294 offset += align - 1;
2295 offset &= ~(align - 1);
2296 ins->opcode = OP_REGOFFSET;
2297 ins->inst_basereg = cfg->frame_reg;
2298 ins->inst_offset = offset;
2299 offset += size;
2300 }
2301 ins = cfg->arch.ss_tramp_var;
2302 if (ins) {
2303 size = 8;
2304 align = 8;
2305 offset += align - 1;
2306 offset &= ~(align - 1);
2307 ins->opcode = OP_REGOFFSET;
2308 ins->inst_basereg = cfg->frame_reg;
2309 ins->inst_offset = offset;
2310 offset += size;
2311 }
2312 ins = cfg->arch.bp_tramp_var;
2313 if (ins) {
2314 size = 8;
2315 align = 8;
2316 offset += align - 1;
2317 offset &= ~(align - 1);
2318 ins->opcode = OP_REGOFFSET;
2319 ins->inst_basereg = cfg->frame_reg;
2320 ins->inst_offset = offset;
2321 offset += size;
2322 }
2323
2324 /* Locals */
2325 offsets = mono_allocate_stack_slots (cfg, FALSE, &locals_stack_size, &locals_stack_align);
2326 if (locals_stack_align)
2327 offset = ALIGN_TO (offset, locals_stack_align);
2328
2329 for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
2330 if (offsets [i] != -1) {
2331 ins = cfg->varinfo [i];
2332 ins->opcode = OP_REGOFFSET;
2333 ins->inst_basereg = cfg->frame_reg;
2334 ins->inst_offset = offset + offsets [i];
2335 //printf ("allocated local %d to ", i); mono_print_tree_nl (ins);
2336 }
2337 }
2338 offset += locals_stack_size;
2339
2340 offset = ALIGN_TO (offset, MONO_ARCH_FRAME_ALIGNMENT);
2341
2342 cfg->stack_offset = offset;
2343 }
2344
2345 #ifdef ENABLE_LLVM
2346 LLVMCallInfo*
2347 mono_arch_get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
2348 {
2349 int i, n;
2350 CallInfo *cinfo;
2351 ArgInfo *ainfo;
2352 LLVMCallInfo *linfo;
2353
2354 n = sig->param_count + sig->hasthis;
2355
2356 cinfo = get_call_info (cfg->mempool, sig);
2357
2358 linfo = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
2359
2360 switch (cinfo->ret.storage) {
2361 case ArgInIReg:
2362 case ArgInFReg:
2363 case ArgInFRegR4:
2364 case ArgNone:
2365 break;
2366 case ArgVtypeByRef:
2367 linfo->ret.storage = LLVMArgVtypeByRef;
2368 break;
2369 //
2370 // FIXME: This doesn't work yet since the llvm backend represents these types as an i8
2371 // array which is returned in int regs
2372 //
2373 case ArgHFA:
2374 linfo->ret.storage = LLVMArgFpStruct;
2375 linfo->ret.nslots = cinfo->ret.nregs;
2376 linfo->ret.esize = cinfo->ret.esize;
2377 break;
2378 case ArgVtypeInIRegs:
2379 /* LLVM models this by returning an int */
2380 linfo->ret.storage = LLVMArgVtypeAsScalar;
2381 linfo->ret.nslots = cinfo->ret.nregs;
2382 linfo->ret.esize = cinfo->ret.esize;
2383 break;
2384 default:
2385 g_assert_not_reached ();
2386 break;
2387 }
2388
2389 for (i = 0; i < n; ++i) {
2390 LLVMArgInfo *lainfo = &linfo->args [i];
2391
2392 ainfo = cinfo->args + i;
2393
2394 lainfo->storage = LLVMArgNone;
2395
2396 switch (ainfo->storage) {
2397 case ArgInIReg:
2398 case ArgInFReg:
2399 case ArgInFRegR4:
2400 case ArgOnStack:
2401 case ArgOnStackR4:
2402 case ArgOnStackR8:
2403 lainfo->storage = LLVMArgNormal;
2404 break;
2405 case ArgVtypeByRef:
2406 case ArgVtypeByRefOnStack:
2407 lainfo->storage = LLVMArgVtypeByRef;
2408 break;
2409 case ArgHFA: {
2410 int j;
2411
2412 lainfo->storage = LLVMArgAsFpArgs;
2413 lainfo->nslots = ainfo->nregs;
2414 lainfo->esize = ainfo->esize;
2415 for (j = 0; j < ainfo->nregs; ++j)
2416 lainfo->pair_storage [j] = LLVMArgInFPReg;
2417 break;
2418 }
2419 case ArgVtypeInIRegs:
2420 lainfo->storage = LLVMArgAsIArgs;
2421 lainfo->nslots = ainfo->nregs;
2422 break;
2423 case ArgVtypeOnStack:
2424 if (ainfo->hfa) {
2425 int j;
2426 /* Same as above */
2427 lainfo->storage = LLVMArgAsFpArgs;
2428 lainfo->nslots = ainfo->nregs;
2429 lainfo->esize = ainfo->esize;
2430 lainfo->ndummy_fpargs = ainfo->nfregs_to_skip;
2431 for (j = 0; j < ainfo->nregs; ++j)
2432 lainfo->pair_storage [j] = LLVMArgInFPReg;
2433 } else {
2434 lainfo->storage = LLVMArgAsIArgs;
2435 lainfo->nslots = ainfo->size / 8;
2436 }
2437 break;
2438 default:
2439 g_assert_not_reached ();
2440 break;
2441 }
2442 }
2443
2444 return linfo;
2445 }
2446 #endif
2447
2448 static void
2449 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, ArgStorage storage, int reg, MonoInst *arg)
2450 {
2451 MonoInst *ins;
2452
2453 switch (storage) {
2454 case ArgInIReg:
2455 MONO_INST_NEW (cfg, ins, OP_MOVE);
2456 ins->dreg = mono_alloc_ireg_copy (cfg, arg->dreg);
2457 ins->sreg1 = arg->dreg;
2458 MONO_ADD_INS (cfg->cbb, ins);
2459 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, FALSE);
2460 break;
2461 case ArgInFReg:
2462 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2463 ins->dreg = mono_alloc_freg (cfg);
2464 ins->sreg1 = arg->dreg;
2465 MONO_ADD_INS (cfg->cbb, ins);
2466 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2467 break;
2468 case ArgInFRegR4:
2469 if (COMPILE_LLVM (cfg))
2470 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2471 else if (cfg->r4fp)
2472 MONO_INST_NEW (cfg, ins, OP_RMOVE);
2473 else
2474 MONO_INST_NEW (cfg, ins, OP_ARM_SETFREG_R4);
2475 ins->dreg = mono_alloc_freg (cfg);
2476 ins->sreg1 = arg->dreg;
2477 MONO_ADD_INS (cfg->cbb, ins);
2478 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2479 break;
2480 default:
2481 g_assert_not_reached ();
2482 break;
2483 }
2484 }
2485
2486 static void
2487 emit_sig_cookie (MonoCompile *cfg, MonoCallInst *call, CallInfo *cinfo)
2488 {
2489 MonoMethodSignature *tmp_sig;
2490 int sig_reg;
2491
2492 if (MONO_IS_TAILCALL_OPCODE (call))
2493 NOT_IMPLEMENTED;
2494
2495 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
2496
2497 /*
2498 * mono_ArgIterator_Setup assumes the signature cookie is
2499 * passed first and all the arguments which were before it are
2500 * passed on the stack after the signature. So compensate by
2501 * passing a different signature.
2502 */
2503 tmp_sig = mono_metadata_signature_dup (call->signature);
2504 tmp_sig->param_count -= call->signature->sentinelpos;
2505 tmp_sig->sentinelpos = 0;
2506 memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
2507
2508 sig_reg = mono_alloc_ireg (cfg);
2509 MONO_EMIT_NEW_SIGNATURECONST (cfg, sig_reg, tmp_sig);
2510
2511 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, cinfo->sig_cookie.offset, sig_reg);
2512 }
2513
2514 void
2515 mono_arch_emit_call (MonoCompile *cfg, MonoCallInst *call)
2516 {
2517 MonoMethodSignature *sig;
2518 MonoInst *arg, *vtarg;
2519 CallInfo *cinfo;
2520 ArgInfo *ainfo;
2521 int i;
2522
2523 sig = call->signature;
2524
2525 cinfo = get_call_info (cfg->mempool, sig);
2526
2527 switch (cinfo->ret.storage) {
2528 case ArgVtypeInIRegs:
2529 case ArgHFA:
2530 if (MONO_IS_TAILCALL_OPCODE (call))
2531 break;
2532 /*
2533 * The vtype is returned in registers, save the return area address in a local, and save the vtype into
2534 * the location pointed to by it after call in emit_move_return_value ().
2535 */
2536 if (!cfg->arch.vret_addr_loc) {
2537 cfg->arch.vret_addr_loc = mono_compile_create_var (cfg, mono_get_int_type (), OP_LOCAL);
2538 /* Prevent it from being register allocated or optimized away */
2539 ((MonoInst*)cfg->arch.vret_addr_loc)->flags |= MONO_INST_VOLATILE;
2540 }
2541
2542 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, ((MonoInst*)cfg->arch.vret_addr_loc)->dreg, call->vret_var->dreg);
2543 break;
2544 case ArgVtypeByRef:
2545 /* Pass the vtype return address in R8 */
2546 g_assert (!MONO_IS_TAILCALL_OPCODE (call) || call->vret_var == cfg->vret_addr);
2547 MONO_INST_NEW (cfg, vtarg, OP_MOVE);
2548 vtarg->sreg1 = call->vret_var->dreg;
2549 vtarg->dreg = mono_alloc_preg (cfg);
2550 MONO_ADD_INS (cfg->cbb, vtarg);
2551
2552 mono_call_inst_add_outarg_reg (cfg, call, vtarg->dreg, cinfo->ret.reg, FALSE);
2553 break;
2554 default:
2555 break;
2556 }
2557
2558 for (i = 0; i < cinfo->nargs; ++i) {
2559 ainfo = cinfo->args + i;
2560 arg = call->args [i];
2561
2562 if ((sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
2563 /* Emit the signature cookie just before the implicit arguments */
2564 emit_sig_cookie (cfg, call, cinfo);
2565 }
2566
2567 switch (ainfo->storage) {
2568 case ArgInIReg:
2569 case ArgInFReg:
2570 case ArgInFRegR4:
2571 add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, arg);
2572 break;
2573 case ArgOnStack:
2574 switch (ainfo->slot_size) {
2575 case 8:
2576 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2577 break;
2578 case 4:
2579 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2580 break;
2581 case 2:
2582 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI2_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2583 break;
2584 case 1:
2585 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2586 break;
2587 default:
2588 g_assert_not_reached ();
2589 break;
2590 }
2591 break;
2592 case ArgOnStackR8:
2593 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER8_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2594 break;
2595 case ArgOnStackR4:
2596 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2597 break;
2598 case ArgVtypeInIRegs:
2599 case ArgVtypeByRef:
2600 case ArgVtypeByRefOnStack:
2601 case ArgVtypeOnStack:
2602 case ArgHFA: {
2603 MonoInst *ins;
2604 guint32 align;
2605 guint32 size;
2606
2607 size = mono_class_value_size (arg->klass, &align);
2608
2609 MONO_INST_NEW (cfg, ins, OP_OUTARG_VT);
2610 ins->sreg1 = arg->dreg;
2611 ins->klass = arg->klass;
2612 ins->backend.size = size;
2613 ins->inst_p0 = call;
2614 ins->inst_p1 = mono_mempool_alloc (cfg->mempool, sizeof (ArgInfo));
2615 memcpy (ins->inst_p1, ainfo, sizeof (ArgInfo));
2616 MONO_ADD_INS (cfg->cbb, ins);
2617 break;
2618 }
2619 default:
2620 g_assert_not_reached ();
2621 break;
2622 }
2623 }
2624
2625 /* Handle the case where there are no implicit arguments */
2626 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (cinfo->nargs == sig->sentinelpos))
2627 emit_sig_cookie (cfg, call, cinfo);
2628
2629 call->call_info = cinfo;
2630 call->stack_usage = cinfo->stack_usage;
2631 }
2632
2633 void
2634 mono_arch_emit_outarg_vt (MonoCompile *cfg, MonoInst *ins, MonoInst *src)
2635 {
2636 MonoCallInst *call = (MonoCallInst*)ins->inst_p0;
2637 ArgInfo *ainfo = ins->inst_p1;
2638 MonoInst *load;
2639 int i;
2640
2641 if (ins->backend.size == 0 && !ainfo->gsharedvt)
2642 return;
2643
2644 switch (ainfo->storage) {
2645 case ArgVtypeInIRegs:
2646 for (i = 0; i < ainfo->nregs; ++i) {
2647 // FIXME: Smaller sizes
2648 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2649 load->dreg = mono_alloc_ireg (cfg);
2650 load->inst_basereg = src->dreg;
2651 load->inst_offset = i * sizeof(mgreg_t);
2652 MONO_ADD_INS (cfg->cbb, load);
2653 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg + i, load);
2654 }
2655 break;
2656 case ArgHFA:
2657 for (i = 0; i < ainfo->nregs; ++i) {
2658 if (ainfo->esize == 4)
2659 MONO_INST_NEW (cfg, load, OP_LOADR4_MEMBASE);
2660 else
2661 MONO_INST_NEW (cfg, load, OP_LOADR8_MEMBASE);
2662 load->dreg = mono_alloc_freg (cfg);
2663 load->inst_basereg = src->dreg;
2664 load->inst_offset = ainfo->foffsets [i];
2665 MONO_ADD_INS (cfg->cbb, load);
2666 add_outarg_reg (cfg, call, ainfo->esize == 4 ? ArgInFRegR4 : ArgInFReg, ainfo->reg + i, load);
2667 }
2668 break;
2669 case ArgVtypeByRef:
2670 case ArgVtypeByRefOnStack: {
2671 MonoInst *vtaddr, *load, *arg;
2672
2673 /* Pass the vtype address in a reg/on the stack */
2674 if (ainfo->gsharedvt) {
2675 load = src;
2676 } else {
2677 /* Make a copy of the argument */
2678 vtaddr = mono_compile_create_var (cfg, m_class_get_byval_arg (ins->klass), OP_LOCAL);
2679
2680 MONO_INST_NEW (cfg, load, OP_LDADDR);
2681 load->inst_p0 = vtaddr;
2682 vtaddr->flags |= MONO_INST_INDIRECT;
2683 load->type = STACK_MP;
2684 load->klass = vtaddr->klass;
2685 load->dreg = mono_alloc_ireg (cfg);
2686 MONO_ADD_INS (cfg->cbb, load);
2687 mini_emit_memcpy (cfg, load->dreg, 0, src->dreg, 0, ainfo->size, 8);
2688 }
2689
2690 if (ainfo->storage == ArgVtypeByRef) {
2691 MONO_INST_NEW (cfg, arg, OP_MOVE);
2692 arg->dreg = mono_alloc_preg (cfg);
2693 arg->sreg1 = load->dreg;
2694 MONO_ADD_INS (cfg->cbb, arg);
2695 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg, arg);
2696 } else {
2697 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, load->dreg);
2698 }
2699 break;
2700 }
2701 case ArgVtypeOnStack:
2702 for (i = 0; i < ainfo->size / 8; ++i) {
2703 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2704 load->dreg = mono_alloc_ireg (cfg);
2705 load->inst_basereg = src->dreg;
2706 load->inst_offset = i * 8;
2707 MONO_ADD_INS (cfg->cbb, load);
2708 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI8_MEMBASE_REG, ARMREG_SP, ainfo->offset + (i * 8), load->dreg);
2709 }
2710 break;
2711 default:
2712 g_assert_not_reached ();
2713 break;
2714 }
2715 }
2716
2717 void
2718 mono_arch_emit_setret (MonoCompile *cfg, MonoMethod *method, MonoInst *val)
2719 {
2720 MonoMethodSignature *sig;
2721 CallInfo *cinfo;
2722
2723 sig = mono_method_signature (cfg->method);
2724 if (!cfg->arch.cinfo)
2725 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
2726 cinfo = cfg->arch.cinfo;
2727
2728 switch (cinfo->ret.storage) {
2729 case ArgNone:
2730 break;
2731 case ArgInIReg:
2732 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, cfg->ret->dreg, val->dreg);
2733 break;
2734 case ArgInFReg:
2735 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2736 break;
2737 case ArgInFRegR4:
2738 if (COMPILE_LLVM (cfg))
2739 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2740 else if (cfg->r4fp)
2741 MONO_EMIT_NEW_UNALU (cfg, OP_RMOVE, cfg->ret->dreg, val->dreg);
2742 else
2743 MONO_EMIT_NEW_UNALU (cfg, OP_ARM_SETFREG_R4, cfg->ret->dreg, val->dreg);
2744 break;
2745 default:
2746 g_assert_not_reached ();
2747 break;
2748 }
2749 }
2750
2751 static const gboolean debug_tailcall = FALSE;
2752
2753 static gboolean
2754 is_supported_tailcall_helper (gboolean value, const char *svalue)
2755 {
2756 if (!value && debug_tailcall)
2757 g_print ("%s %s\n", __func__, svalue);
2758 return value;
2759 }
2760
2761 #define IS_SUPPORTED_TAILCALL(x) (is_supported_tailcall_helper((x), #x))
2762
2763 gboolean
2764 mono_arch_tailcall_supported (MonoCompile *cfg, MonoMethodSignature *caller_sig, MonoMethodSignature *callee_sig)
2765 {
2766 g_assert (caller_sig);
2767 g_assert (callee_sig);
2768
2769 CallInfo *caller_info = get_call_info (NULL, caller_sig);
2770 CallInfo *callee_info = get_call_info (NULL, callee_sig);
2771
2772 gboolean res = IS_SUPPORTED_TAILCALL (callee_info->stack_usage <= caller_info->stack_usage)
2773 && IS_SUPPORTED_TAILCALL (caller_info->ret.storage == callee_info->ret.storage);
2774
2775 // FIXME Limit stack_usage to 1G. emit_ldrx / strx has 32bit limits.
2776 res &= IS_SUPPORTED_TAILCALL (callee_info->stack_usage < (1 << 30));
2777 res &= IS_SUPPORTED_TAILCALL (caller_info->stack_usage < (1 << 30));
2778
2779 // valuetype parameters are the address of a local
2780 const ArgInfo *ainfo;
2781 ainfo = callee_info->args + callee_sig->hasthis;
2782 for (int i = 0; res && i < callee_sig->param_count; ++i) {
2783 res = IS_SUPPORTED_TAILCALL (ainfo [i].storage != ArgVtypeByRef)
2784 && IS_SUPPORTED_TAILCALL (ainfo [i].storage != ArgVtypeByRefOnStack);
2785 }
2786
2787 g_free (caller_info);
2788 g_free (callee_info);
2789
2790 return res;
2791 }
2792
2793 gboolean
2794 mono_arch_is_inst_imm (int opcode, int imm_opcode, gint64 imm)
2795 {
2796 return (imm >= -((gint64)1<<31) && imm <= (((gint64)1<<31)-1));
2797 }
2798
2799 void*
2800 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
2801 {
2802 NOT_IMPLEMENTED;
2803 return NULL;
2804 }
2805
2806 void*
2807 mono_arch_instrument_epilog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
2808 {
2809 NOT_IMPLEMENTED;
2810 return NULL;
2811 }
2812
2813 void
2814 mono_arch_peephole_pass_1 (MonoCompile *cfg, MonoBasicBlock *bb)
2815 {
2816 //NOT_IMPLEMENTED;
2817 }
2818
2819 void
2820 mono_arch_peephole_pass_2 (MonoCompile *cfg, MonoBasicBlock *bb)
2821 {
2822 //NOT_IMPLEMENTED;
2823 }
2824
2825 #define ADD_NEW_INS(cfg,dest,op) do { \
2826 MONO_INST_NEW ((cfg), (dest), (op)); \
2827 mono_bblock_insert_before_ins (bb, ins, (dest)); \
2828 } while (0)
2829
2830 void
2831 mono_arch_lowering_pass (MonoCompile *cfg, MonoBasicBlock *bb)
2832 {
2833 MonoInst *ins, *temp, *last_ins = NULL;
2834
2835 MONO_BB_FOR_EACH_INS (bb, ins) {
2836 switch (ins->opcode) {
2837 case OP_SBB:
2838 case OP_ISBB:
2839 case OP_SUBCC:
2840 case OP_ISUBCC:
2841 if (ins->next && (ins->next->opcode == OP_COND_EXC_C || ins->next->opcode == OP_COND_EXC_IC))
2842 /* ARM sets the C flag to 1 if there was _no_ overflow */
2843 ins->next->opcode = OP_COND_EXC_NC;
2844 break;
2845 case OP_IDIV_IMM:
2846 case OP_IREM_IMM:
2847 case OP_IDIV_UN_IMM:
2848 case OP_IREM_UN_IMM:
2849 case OP_LREM_IMM:
2850 mono_decompose_op_imm (cfg, bb, ins);
2851 break;
2852 case OP_LOCALLOC_IMM:
2853 if (ins->inst_imm > 32) {
2854 ADD_NEW_INS (cfg, temp, OP_ICONST);
2855 temp->inst_c0 = ins->inst_imm;
2856 temp->dreg = mono_alloc_ireg (cfg);
2857 ins->sreg1 = temp->dreg;
2858 ins->opcode = mono_op_imm_to_op (ins->opcode);
2859 }
2860 break;
2861 case OP_ICOMPARE_IMM:
2862 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBEQ) {
2863 ins->next->opcode = OP_ARM64_CBZW;
2864 ins->next->sreg1 = ins->sreg1;
2865 NULLIFY_INS (ins);
2866 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBNE_UN) {
2867 ins->next->opcode = OP_ARM64_CBNZW;
2868 ins->next->sreg1 = ins->sreg1;
2869 NULLIFY_INS (ins);
2870 }
2871 break;
2872 case OP_LCOMPARE_IMM:
2873 case OP_COMPARE_IMM:
2874 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBEQ) {
2875 ins->next->opcode = OP_ARM64_CBZX;
2876 ins->next->sreg1 = ins->sreg1;
2877 NULLIFY_INS (ins);
2878 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBNE_UN) {
2879 ins->next->opcode = OP_ARM64_CBNZX;
2880 ins->next->sreg1 = ins->sreg1;
2881 NULLIFY_INS (ins);
2882 }
2883 break;
2884 case OP_FCOMPARE:
2885 case OP_RCOMPARE: {
2886 gboolean swap = FALSE;
2887 int reg;
2888
2889 if (!ins->next) {
2890 /* Optimized away */
2891 NULLIFY_INS (ins);
2892 break;
2893 }
2894
2895 /*
2896 * FP compares with unordered operands set the flags
2897 * to NZCV=0011, which matches some non-unordered compares
2898 * as well, like LE, so have to swap the operands.
2899 */
2900 switch (ins->next->opcode) {
2901 case OP_FBLT:
2902 ins->next->opcode = OP_FBGT;
2903 swap = TRUE;
2904 break;
2905 case OP_FBLE:
2906 ins->next->opcode = OP_FBGE;
2907 swap = TRUE;
2908 break;
2909 case OP_RBLT:
2910 ins->next->opcode = OP_RBGT;
2911 swap = TRUE;
2912 break;
2913 case OP_RBLE:
2914 ins->next->opcode = OP_RBGE;
2915 swap = TRUE;
2916 break;
2917 default:
2918 break;
2919 }
2920 if (swap) {
2921 reg = ins->sreg1;
2922 ins->sreg1 = ins->sreg2;
2923 ins->sreg2 = reg;
2924 }
2925 break;
2926 }
2927 default:
2928 break;
2929 }
2930
2931 last_ins = ins;
2932 }
2933 bb->last_ins = last_ins;
2934 bb->max_vreg = cfg->next_vreg;
2935 }
2936
2937 void
2938 mono_arch_decompose_long_opts (MonoCompile *cfg, MonoInst *long_ins)
2939 {
2940 }
2941
2942 static int
2943 opcode_to_armcond (int opcode)
2944 {
2945 switch (opcode) {
2946 case OP_IBEQ:
2947 case OP_LBEQ:
2948 case OP_FBEQ:
2949 case OP_CEQ:
2950 case OP_ICEQ:
2951 case OP_LCEQ:
2952 case OP_FCEQ:
2953 case OP_RCEQ:
2954 case OP_COND_EXC_IEQ:
2955 case OP_COND_EXC_EQ:
2956 return ARMCOND_EQ;
2957 case OP_IBGE:
2958 case OP_LBGE:
2959 case OP_FBGE:
2960 case OP_ICGE:
2961 case OP_FCGE:
2962 case OP_RCGE:
2963 return ARMCOND_GE;
2964 case OP_IBGT:
2965 case OP_LBGT:
2966 case OP_FBGT:
2967 case OP_CGT:
2968 case OP_ICGT:
2969 case OP_LCGT:
2970 case OP_FCGT:
2971 case OP_RCGT:
2972 case OP_COND_EXC_IGT:
2973 case OP_COND_EXC_GT:
2974 return ARMCOND_GT;
2975 case OP_IBLE:
2976 case OP_LBLE:
2977 case OP_FBLE:
2978 case OP_ICLE:
2979 case OP_FCLE:
2980 case OP_RCLE:
2981 return ARMCOND_LE;
2982 case OP_IBLT:
2983 case OP_LBLT:
2984 case OP_FBLT:
2985 case OP_CLT:
2986 case OP_ICLT:
2987 case OP_LCLT:
2988 case OP_COND_EXC_ILT:
2989 case OP_COND_EXC_LT:
2990 return ARMCOND_LT;
2991 case OP_IBNE_UN:
2992 case OP_LBNE_UN:
2993 case OP_FBNE_UN:
2994 case OP_ICNEQ:
2995 case OP_FCNEQ:
2996 case OP_RCNEQ:
2997 case OP_COND_EXC_INE_UN:
2998 case OP_COND_EXC_NE_UN:
2999 return ARMCOND_NE;
3000 case OP_IBGE_UN:
3001 case OP_LBGE_UN:
3002 case OP_FBGE_UN:
3003 case OP_ICGE_UN:
3004 case OP_COND_EXC_IGE_UN:
3005 case OP_COND_EXC_GE_UN:
3006 return ARMCOND_HS;
3007 case OP_IBGT_UN:
3008 case OP_LBGT_UN:
3009 case OP_FBGT_UN:
3010 case OP_CGT_UN:
3011 case OP_ICGT_UN:
3012 case OP_LCGT_UN:
3013 case OP_FCGT_UN:
3014 case OP_RCGT_UN:
3015 case OP_COND_EXC_IGT_UN:
3016 case OP_COND_EXC_GT_UN:
3017 return ARMCOND_HI;
3018 case OP_IBLE_UN:
3019 case OP_LBLE_UN:
3020 case OP_FBLE_UN:
3021 case OP_ICLE_UN:
3022 case OP_COND_EXC_ILE_UN:
3023 case OP_COND_EXC_LE_UN:
3024 return ARMCOND_LS;
3025 case OP_IBLT_UN:
3026 case OP_LBLT_UN:
3027 case OP_FBLT_UN:
3028 case OP_CLT_UN:
3029 case OP_ICLT_UN:
3030 case OP_LCLT_UN:
3031 case OP_COND_EXC_ILT_UN:
3032 case OP_COND_EXC_LT_UN:
3033 return ARMCOND_LO;
3034 /*
3035 * FCMP sets the NZCV condition bits as follows:
3036 * eq = 0110
3037 * < = 1000
3038 * > = 0010
3039 * unordered = 0011
3040 * ARMCOND_LT is N!=V, so it matches unordered too, so
3041 * fclt and fclt_un need to be special cased.
3042 */
3043 case OP_FCLT:
3044 case OP_RCLT:
3045 /* N==1 */
3046 return ARMCOND_MI;
3047 case OP_FCLT_UN:
3048 case OP_RCLT_UN:
3049 return ARMCOND_LT;
3050 case OP_COND_EXC_C:
3051 case OP_COND_EXC_IC:
3052 return ARMCOND_CS;
3053 case OP_COND_EXC_OV:
3054 case OP_COND_EXC_IOV:
3055 return ARMCOND_VS;
3056 case OP_COND_EXC_NC:
3057 case OP_COND_EXC_INC:
3058 return ARMCOND_CC;
3059 case OP_COND_EXC_NO:
3060 case OP_COND_EXC_INO:
3061 return ARMCOND_VC;
3062 default:
3063 printf ("%s\n", mono_inst_name (opcode));
3064 g_assert_not_reached ();
3065 return -1;
3066 }
3067 }
3068
3069 /* This clobbers LR */
3070 static inline __attribute__ ((__warn_unused_result__)) guint8*
3071 emit_cond_exc (MonoCompile *cfg, guint8 *code, int opcode, const char *exc_name)
3072 {
3073 int cond;
3074
3075 cond = opcode_to_armcond (opcode);
3076 /* Capture PC */
3077 arm_adrx (code, ARMREG_IP1, code);
3078 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_EXC, exc_name, MONO_R_ARM64_BCC);
3079 arm_bcc (code, cond, 0);
3080 return code;
3081 }
3082
3083 static guint8*
3084 emit_move_return_value (MonoCompile *cfg, guint8 * code, MonoInst *ins)
3085 {
3086 CallInfo *cinfo;
3087 MonoCallInst *call;
3088
3089 call = (MonoCallInst*)ins;
3090 cinfo = call->call_info;
3091 g_assert (cinfo);
3092 switch (cinfo->ret.storage) {
3093 case ArgNone:
3094 break;
3095 case ArgInIReg:
3096 /* LLVM compiled code might only set the bottom bits */
3097 if (call->signature && mini_get_underlying_type (call->signature->ret)->type == MONO_TYPE_I4)
3098 arm_sxtwx (code, call->inst.dreg, cinfo->ret.reg);
3099 else if (call->inst.dreg != cinfo->ret.reg)
3100 arm_movx (code, call->inst.dreg, cinfo->ret.reg);
3101 break;
3102 case ArgInFReg:
3103 if (call->inst.dreg != cinfo->ret.reg)
3104 arm_fmovd (code, call->inst.dreg, cinfo->ret.reg);
3105 break;
3106 case ArgInFRegR4:
3107 if (cfg->r4fp)
3108 arm_fmovs (code, call->inst.dreg, cinfo->ret.reg);
3109 else
3110 arm_fcvt_sd (code, call->inst.dreg, cinfo->ret.reg);
3111 break;
3112 case ArgVtypeInIRegs: {
3113 MonoInst *loc = cfg->arch.vret_addr_loc;
3114 int i;
3115
3116 /* Load the destination address */
3117 g_assert (loc && loc->opcode == OP_REGOFFSET);
3118 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
3119 for (i = 0; i < cinfo->ret.nregs; ++i)
3120 arm_strx (code, cinfo->ret.reg + i, ARMREG_LR, i * 8);
3121 break;
3122 }
3123 case ArgHFA: {
3124 MonoInst *loc = cfg->arch.vret_addr_loc;
3125 int i;
3126
3127 /* Load the destination address */
3128 g_assert (loc && loc->opcode == OP_REGOFFSET);
3129 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
3130 for (i = 0; i < cinfo->ret.nregs; ++i) {
3131 if (cinfo->ret.esize == 4)
3132 arm_strfpw (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
3133 else
3134 arm_strfpx (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
3135 }
3136 break;
3137 }
3138 case ArgVtypeByRef:
3139 break;
3140 default:
3141 g_assert_not_reached ();
3142 break;
3143 }
3144 return code;
3145 }
3146
3147 /*
3148 * emit_branch_island:
3149 *
3150 * Emit a branch island for the conditional branches from cfg->native_code + start_offset to code.
3151 */
3152 static guint8*
3153 emit_branch_island (MonoCompile *cfg, guint8 *code, int start_offset)
3154 {
3155 MonoJumpInfo *ji;
3156
3157 /* Iterate over the patch infos added so far by this bb */
3158 int island_size = 0;
3159 for (ji = cfg->patch_info; ji; ji = ji->next) {
3160 if (ji->ip.i < start_offset)
3161 /* The patch infos are in reverse order, so this means the end */
3162 break;
3163 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ)
3164 island_size += 4;
3165 }
3166
3167 if (island_size) {
3168 code = realloc_code (cfg, island_size);
3169
3170 /* Branch over the island */
3171 arm_b (code, code + 4 + island_size);
3172
3173 for (ji = cfg->patch_info; ji; ji = ji->next) {
3174 if (ji->ip.i < start_offset)
3175 break;
3176 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ) {
3177 /* Rewrite the cond branch so it branches to an unconditional branch in the branch island */
3178 arm_patch_rel (cfg->native_code + ji->ip.i, code, ji->relocation);
3179 /* Rewrite the patch so it points to the unconditional branch */
3180 ji->ip.i = code - cfg->native_code;
3181 ji->relocation = MONO_R_ARM64_B;
3182 arm_b (code, code);
3183 }
3184 }
3185 set_code_cursor (cfg, code);
3186 }
3187 return code;
3188 }
3189
3190 void
3191 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
3192 {
3193 MonoInst *ins;
3194 MonoCallInst *call;
3195 guint8 *code = cfg->native_code + cfg->code_len;
3196 int start_offset, max_len, dreg, sreg1, sreg2;
3197 mgreg_t imm;
3198
3199 if (cfg->verbose_level > 2)
3200 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
3201
3202 start_offset = code - cfg->native_code;
3203 g_assert (start_offset <= cfg->code_size);
3204
3205 MONO_BB_FOR_EACH_INS (bb, ins) {
3206 guint offset = code - cfg->native_code;
3207 set_code_cursor (cfg, code);
3208 max_len = ins_get_size (ins->opcode);
3209 code = realloc_code (cfg, max_len);
3210
3211 if (G_UNLIKELY (cfg->arch.cond_branch_islands && offset - start_offset > 4 * 0x1ffff)) {
3212 /* Emit a branch island for large basic blocks */
3213 code = emit_branch_island (cfg, code, start_offset);
3214 offset = code - cfg->native_code;
3215 start_offset = offset;
3216 }
3217
3218 mono_debug_record_line_number (cfg, ins, offset);
3219
3220 dreg = ins->dreg;
3221 sreg1 = ins->sreg1;
3222 sreg2 = ins->sreg2;
3223 imm = ins->inst_imm;
3224
3225 switch (ins->opcode) {
3226 case OP_ICONST:
3227 code = emit_imm (code, dreg, ins->inst_c0);
3228 break;
3229 case OP_I8CONST:
3230 code = emit_imm64 (code, dreg, ins->inst_c0);
3231 break;
3232 case OP_MOVE:
3233 if (dreg != sreg1)
3234 arm_movx (code, dreg, sreg1);
3235 break;
3236 case OP_NOP:
3237 case OP_RELAXED_NOP:
3238 break;
3239 case OP_JUMP_TABLE:
3240 mono_add_patch_info_rel (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0, MONO_R_ARM64_IMM);
3241 code = emit_imm64_template (code, dreg);
3242 break;
3243 case OP_BREAK:
3244 /*
3245 * gdb does not like encountering the hw breakpoint ins in the debugged code.
3246 * So instead of emitting a trap, we emit a call a C function and place a
3247 * breakpoint there.
3248 */
3249 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, (gpointer)"mono_break");
3250 break;
3251 case OP_LOCALLOC: {
3252 guint8 *buf [16];
3253
3254 arm_addx_imm (code, ARMREG_IP0, sreg1, (MONO_ARCH_FRAME_ALIGNMENT - 1));
3255 // FIXME: andx_imm doesn't work yet
3256 code = emit_imm (code, ARMREG_IP1, -MONO_ARCH_FRAME_ALIGNMENT);
3257 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3258 //arm_andx_imm (code, ARMREG_IP0, sreg1, - MONO_ARCH_FRAME_ALIGNMENT);
3259 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
3260 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
3261 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
3262
3263 /* Init */
3264 /* ip1 = pointer, ip0 = end */
3265 arm_addx (code, ARMREG_IP0, ARMREG_IP1, ARMREG_IP0);
3266 buf [0] = code;
3267 arm_cmpx (code, ARMREG_IP1, ARMREG_IP0);
3268 buf [1] = code;
3269 arm_bcc (code, ARMCOND_EQ, 0);
3270 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_IP1, 0);
3271 arm_addx_imm (code, ARMREG_IP1, ARMREG_IP1, 16);
3272 arm_b (code, buf [0]);
3273 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3274
3275 arm_movspx (code, dreg, ARMREG_SP);
3276 if (cfg->param_area)
3277 code = emit_subx_sp_imm (code, cfg->param_area);
3278 break;
3279 }
3280 case OP_LOCALLOC_IMM: {
3281 int imm, offset;
3282
3283 imm = ALIGN_TO (ins->inst_imm, MONO_ARCH_FRAME_ALIGNMENT);
3284 g_assert (arm_is_arith_imm (imm));
3285 arm_subx_imm (code, ARMREG_SP, ARMREG_SP, imm);
3286
3287 /* Init */
3288 g_assert (MONO_ARCH_FRAME_ALIGNMENT == 16);
3289 offset = 0;
3290 while (offset < imm) {
3291 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_SP, offset);
3292 offset += 16;
3293 }
3294 arm_movspx (code, dreg, ARMREG_SP);
3295 if (cfg->param_area)
3296 code = emit_subx_sp_imm (code, cfg->param_area);
3297 break;
3298 }
3299 case OP_AOTCONST:
3300 code = emit_aotconst (cfg, code, dreg, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3301 break;
3302 case OP_OBJC_GET_SELECTOR:
3303 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_OBJC_SELECTOR_REF, ins->inst_p0);
3304 /* See arch_emit_objc_selector_ref () in aot-compiler.c */
3305 arm_ldrx_lit (code, ins->dreg, 0);
3306 arm_nop (code);
3307 arm_nop (code);
3308 break;
3309 case OP_SEQ_POINT: {
3310 MonoInst *info_var = cfg->arch.seq_point_info_var;
3311
3312 /*
3313 * For AOT, we use one got slot per method, which will point to a
3314 * SeqPointInfo structure, containing all the information required
3315 * by the code below.
3316 */
3317 if (cfg->compile_aot) {
3318 g_assert (info_var);
3319 g_assert (info_var->opcode == OP_REGOFFSET);
3320 }
3321
3322 if (ins->flags & MONO_INST_SINGLE_STEP_LOC) {
3323 MonoInst *var = cfg->arch.ss_tramp_var;
3324
3325 g_assert (var);
3326 g_assert (var->opcode == OP_REGOFFSET);
3327 /* Load ss_tramp_var */
3328 /* This is equal to &ss_trampoline */
3329 arm_ldrx (code, ARMREG_IP1, var->inst_basereg, var->inst_offset);
3330 /* Load the trampoline address */
3331 arm_ldrx (code, ARMREG_IP1, ARMREG_IP1, 0);
3332 /* Call it if it is non-null */
3333 arm_cbzx (code, ARMREG_IP1, code + 8);
3334 arm_blrx (code, ARMREG_IP1);
3335 }
3336
3337 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
3338
3339 if (cfg->compile_aot) {
3340 const guint32 offset = code - cfg->native_code;
3341 guint32 val;
3342
3343 arm_ldrx (code, ARMREG_IP1, info_var->inst_basereg, info_var->inst_offset);
3344 /* Add the offset */
3345 val = ((offset / 4) * sizeof (guint8*)) + MONO_STRUCT_OFFSET (SeqPointInfo, bp_addrs);
3346 /* Load the info->bp_addrs [offset], which is either 0 or the address of the bp trampoline */
3347 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP1, val);
3348 /* Skip the load if its 0 */
3349 arm_cbzx (code, ARMREG_IP1, code + 8);
3350 /* Call the breakpoint trampoline */
3351 arm_blrx (code, ARMREG_IP1);
3352 } else {
3353 MonoInst *var = cfg->arch.bp_tramp_var;
3354
3355 g_assert (var);
3356 g_assert (var->opcode == OP_REGOFFSET);
3357 /* Load the address of the bp trampoline into IP0 */
3358 arm_ldrx (code, ARMREG_IP0, var->inst_basereg, var->inst_offset);
3359 /*
3360 * A placeholder for a possible breakpoint inserted by
3361 * mono_arch_set_breakpoint ().
3362 */
3363 arm_nop (code);
3364 }
3365 break;
3366 }
3367
3368 /* BRANCH */
3369 case OP_BR:
3370 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_B);
3371 arm_b (code, code);
3372 break;
3373 case OP_BR_REG:
3374 arm_brx (code, sreg1);
3375 break;
3376 case OP_IBEQ:
3377 case OP_IBGE:
3378 case OP_IBGT:
3379 case OP_IBLE:
3380 case OP_IBLT:
3381 case OP_IBNE_UN:
3382 case OP_IBGE_UN:
3383 case OP_IBGT_UN:
3384 case OP_IBLE_UN:
3385 case OP_IBLT_UN:
3386 case OP_LBEQ:
3387 case OP_LBGE:
3388 case OP_LBGT:
3389 case OP_LBLE:
3390 case OP_LBLT:
3391 case OP_LBNE_UN:
3392 case OP_LBGE_UN:
3393 case OP_LBGT_UN:
3394 case OP_LBLE_UN:
3395 case OP_LBLT_UN:
3396 case OP_FBEQ:
3397 case OP_FBNE_UN:
3398 case OP_FBLT:
3399 case OP_FBGT:
3400 case OP_FBGT_UN:
3401 case OP_FBLE:
3402 case OP_FBGE:
3403 case OP_FBGE_UN: {
3404 int cond;
3405
3406 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3407 cond = opcode_to_armcond (ins->opcode);
3408 arm_bcc (code, cond, 0);
3409 break;
3410 }
3411 case OP_FBLT_UN:
3412 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3413 /* For fp compares, ARMCOND_LT is lt or unordered */
3414 arm_bcc (code, ARMCOND_LT, 0);
3415 break;
3416 case OP_FBLE_UN:
3417 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3418 arm_bcc (code, ARMCOND_EQ, 0);
3419 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3420 /* For fp compares, ARMCOND_LT is lt or unordered */
3421 arm_bcc (code, ARMCOND_LT, 0);
3422 break;
3423 case OP_ARM64_CBZW:
3424 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3425 arm_cbzw (code, sreg1, 0);
3426 break;
3427 case OP_ARM64_CBZX:
3428 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3429 arm_cbzx (code, sreg1, 0);
3430 break;
3431 case OP_ARM64_CBNZW:
3432 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3433 arm_cbnzw (code, sreg1, 0);
3434 break;
3435 case OP_ARM64_CBNZX:
3436 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3437 arm_cbnzx (code, sreg1, 0);
3438 break;
3439 /* ALU */
3440 case OP_IADD:
3441 arm_addw (code, dreg, sreg1, sreg2);
3442 break;
3443 case OP_LADD:
3444 arm_addx (code, dreg, sreg1, sreg2);
3445 break;
3446 case OP_ISUB:
3447 arm_subw (code, dreg, sreg1, sreg2);
3448 break;
3449 case OP_LSUB:
3450 arm_subx (code, dreg, sreg1, sreg2);
3451 break;
3452 case OP_IAND:
3453 arm_andw (code, dreg, sreg1, sreg2);
3454 break;
3455 case OP_LAND:
3456 arm_andx (code, dreg, sreg1, sreg2);
3457 break;
3458 case OP_IOR:
3459 arm_orrw (code, dreg, sreg1, sreg2);
3460 break;
3461 case OP_LOR:
3462 arm_orrx (code, dreg, sreg1, sreg2);
3463 break;
3464 case OP_IXOR:
3465 arm_eorw (code, dreg, sreg1, sreg2);
3466 break;
3467 case OP_LXOR:
3468 arm_eorx (code, dreg, sreg1, sreg2);
3469 break;
3470 case OP_INEG:
3471 arm_negw (code, dreg, sreg1);
3472 break;
3473 case OP_LNEG:
3474 arm_negx (code, dreg, sreg1);
3475 break;
3476 case OP_INOT:
3477 arm_mvnw (code, dreg, sreg1);
3478 break;
3479 case OP_LNOT:
3480 arm_mvnx (code, dreg, sreg1);
3481 break;
3482 case OP_IADDCC:
3483 arm_addsw (code, dreg, sreg1, sreg2);
3484 break;
3485 case OP_ADDCC:
3486 case OP_LADDCC:
3487 arm_addsx (code, dreg, sreg1, sreg2);
3488 break;
3489 case OP_ISUBCC:
3490 arm_subsw (code, dreg, sreg1, sreg2);
3491 break;
3492 case OP_LSUBCC:
3493 case OP_SUBCC:
3494 arm_subsx (code, dreg, sreg1, sreg2);
3495 break;
3496 case OP_ICOMPARE:
3497 arm_cmpw (code, sreg1, sreg2);
3498 break;
3499 case OP_COMPARE:
3500 case OP_LCOMPARE:
3501 arm_cmpx (code, sreg1, sreg2);
3502 break;
3503 case OP_IADD_IMM:
3504 code = emit_addw_imm (code, dreg, sreg1, imm);
3505 break;
3506 case OP_LADD_IMM:
3507 case OP_ADD_IMM:
3508 code = emit_addx_imm (code, dreg, sreg1, imm);
3509 break;
3510 case OP_ISUB_IMM:
3511 code = emit_subw_imm (code, dreg, sreg1, imm);
3512 break;
3513 case OP_LSUB_IMM:
3514 code = emit_subx_imm (code, dreg, sreg1, imm);
3515 break;
3516 case OP_IAND_IMM:
3517 code = emit_andw_imm (code, dreg, sreg1, imm);
3518 break;
3519 case OP_LAND_IMM:
3520 case OP_AND_IMM:
3521 code = emit_andx_imm (code, dreg, sreg1, imm);
3522 break;
3523 case OP_IOR_IMM:
3524 code = emit_orrw_imm (code, dreg, sreg1, imm);
3525 break;
3526 case OP_LOR_IMM:
3527 code = emit_orrx_imm (code, dreg, sreg1, imm);
3528 break;
3529 case OP_IXOR_IMM:
3530 code = emit_eorw_imm (code, dreg, sreg1, imm);
3531 break;
3532 case OP_LXOR_IMM:
3533 code = emit_eorx_imm (code, dreg, sreg1, imm);
3534 break;
3535 case OP_ICOMPARE_IMM:
3536 code = emit_cmpw_imm (code, sreg1, imm);
3537 break;
3538 case OP_LCOMPARE_IMM:
3539 case OP_COMPARE_IMM:
3540 if (imm == 0) {
3541 arm_cmpx (code, sreg1, ARMREG_RZR);
3542 } else {
3543 // FIXME: 32 vs 64 bit issues for 0xffffffff
3544 code = emit_imm64 (code, ARMREG_LR, imm);
3545 arm_cmpx (code, sreg1, ARMREG_LR);
3546 }
3547 break;
3548 case OP_ISHL:
3549 arm_lslvw (code, dreg, sreg1, sreg2);
3550 break;
3551 case OP_LSHL:
3552 arm_lslvx (code, dreg, sreg1, sreg2);
3553 break;
3554 case OP_ISHR:
3555 arm_asrvw (code, dreg, sreg1, sreg2);
3556 break;
3557 case OP_LSHR:
3558 arm_asrvx (code, dreg, sreg1, sreg2);
3559 break;
3560 case OP_ISHR_UN:
3561 arm_lsrvw (code, dreg, sreg1, sreg2);
3562 break;
3563 case OP_LSHR_UN:
3564 arm_lsrvx (code, dreg, sreg1, sreg2);
3565 break;
3566 case OP_ISHL_IMM:
3567 if (imm == 0)
3568 arm_movx (code, dreg, sreg1);
3569 else
3570 arm_lslw (code, dreg, sreg1, imm);
3571 break;
3572 case OP_SHL_IMM:
3573 case OP_LSHL_IMM:
3574 if (imm == 0)
3575 arm_movx (code, dreg, sreg1);
3576 else
3577 arm_lslx (code, dreg, sreg1, imm);
3578 break;
3579 case OP_ISHR_IMM:
3580 if (imm == 0)
3581 arm_movx (code, dreg, sreg1);
3582 else
3583 arm_asrw (code, dreg, sreg1, imm);
3584 break;
3585 case OP_LSHR_IMM:
3586 case OP_SHR_IMM:
3587 if (imm == 0)
3588 arm_movx (code, dreg, sreg1);
3589 else
3590 arm_asrx (code, dreg, sreg1, imm);
3591 break;
3592 case OP_ISHR_UN_IMM:
3593 if (imm == 0)
3594 arm_movx (code, dreg, sreg1);
3595 else
3596 arm_lsrw (code, dreg, sreg1, imm);
3597 break;
3598 case OP_SHR_UN_IMM:
3599 case OP_LSHR_UN_IMM:
3600 if (imm == 0)
3601 arm_movx (code, dreg, sreg1);
3602 else
3603 arm_lsrx (code, dreg, sreg1, imm);
3604 break;
3605
3606 /* 64BIT ALU */
3607 case OP_SEXT_I4:
3608 arm_sxtwx (code, dreg, sreg1);
3609 break;
3610 case OP_ZEXT_I4:
3611 /* Clean out the upper word */
3612 arm_movw (code, dreg, sreg1);
3613 break;
3614
3615 /* MULTIPLY/DIVISION */
3616 case OP_IDIV:
3617 case OP_IREM:
3618 // FIXME: Optimize this
3619 /* Check for zero */
3620 arm_cmpx_imm (code, sreg2, 0);
3621 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3622 /* Check for INT_MIN/-1 */
3623 code = emit_imm (code, ARMREG_IP0, 0x80000000);
3624 arm_cmpx (code, sreg1, ARMREG_IP0);
3625 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3626 code = emit_imm (code, ARMREG_IP0, 0xffffffff);
3627 arm_cmpx (code, sreg2, ARMREG_IP0);
3628 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3629 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3630 arm_cmpx_imm (code, ARMREG_IP0, 1);
3631 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "OverflowException");
3632 if (ins->opcode == OP_IREM) {
3633 arm_sdivw (code, ARMREG_LR, sreg1, sreg2);
3634 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3635 } else {
3636 arm_sdivw (code, dreg, sreg1, sreg2);
3637 }
3638 break;
3639 case OP_IDIV_UN:
3640 arm_cmpx_imm (code, sreg2, 0);
3641 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3642 arm_udivw (code, dreg, sreg1, sreg2);
3643 break;
3644 case OP_IREM_UN:
3645 arm_cmpx_imm (code, sreg2, 0);
3646 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3647 arm_udivw (code, ARMREG_LR, sreg1, sreg2);
3648 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3649 break;
3650 case OP_LDIV:
3651 case OP_LREM:
3652 // FIXME: Optimize this
3653 /* Check for zero */
3654 arm_cmpx_imm (code, sreg2, 0);
3655 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3656 /* Check for INT64_MIN/-1 */
3657 code = emit_imm64 (code, ARMREG_IP0, 0x8000000000000000);
3658 arm_cmpx (code, sreg1, ARMREG_IP0);
3659 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3660 code = emit_imm64 (code, ARMREG_IP0, 0xffffffffffffffff);
3661 arm_cmpx (code, sreg2, ARMREG_IP0);
3662 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3663 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3664 arm_cmpx_imm (code, ARMREG_IP0, 1);
3665 /* 64 bit uses OverflowException */
3666 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "OverflowException");
3667 if (ins->opcode == OP_LREM) {
3668 arm_sdivx (code, ARMREG_LR, sreg1, sreg2);
3669 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3670 } else {
3671 arm_sdivx (code, dreg, sreg1, sreg2);
3672 }
3673 break;
3674 case OP_LDIV_UN:
3675 arm_cmpx_imm (code, sreg2, 0);
3676 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3677 arm_udivx (code, dreg, sreg1, sreg2);
3678 break;
3679 case OP_LREM_UN:
3680 arm_cmpx_imm (code, sreg2, 0);
3681 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3682 arm_udivx (code, ARMREG_LR, sreg1, sreg2);
3683 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3684 break;
3685 case OP_IMUL:
3686 arm_mulw (code, dreg, sreg1, sreg2);
3687 break;
3688 case OP_LMUL:
3689 arm_mulx (code, dreg, sreg1, sreg2);
3690 break;
3691 case OP_IMUL_IMM:
3692 code = emit_imm (code, ARMREG_LR, imm);
3693 arm_mulw (code, dreg, sreg1, ARMREG_LR);
3694 break;
3695 case OP_MUL_IMM:
3696 case OP_LMUL_IMM:
3697 code = emit_imm (code, ARMREG_LR, imm);
3698 arm_mulx (code, dreg, sreg1, ARMREG_LR);
3699 break;
3700
3701 /* CONVERSIONS */
3702 case OP_ICONV_TO_I1:
3703 case OP_LCONV_TO_I1:
3704 arm_sxtbx (code, dreg, sreg1);
3705 break;
3706 case OP_ICONV_TO_I2:
3707 case OP_LCONV_TO_I2:
3708 arm_sxthx (code, dreg, sreg1);
3709 break;
3710 case OP_ICONV_TO_U1:
3711 case OP_LCONV_TO_U1:
3712 arm_uxtbw (code, dreg, sreg1);
3713 break;
3714 case OP_ICONV_TO_U2:
3715 case OP_LCONV_TO_U2:
3716 arm_uxthw (code, dreg, sreg1);
3717 break;
3718
3719 /* CSET */
3720 case OP_CEQ:
3721 case OP_ICEQ:
3722 case OP_LCEQ:
3723 case OP_CLT:
3724 case OP_ICLT:
3725 case OP_LCLT:
3726 case OP_CGT:
3727 case OP_ICGT:
3728 case OP_LCGT:
3729 case OP_CLT_UN:
3730 case OP_ICLT_UN:
3731 case OP_LCLT_UN:
3732 case OP_CGT_UN:
3733 case OP_ICGT_UN:
3734 case OP_LCGT_UN:
3735 case OP_ICNEQ:
3736 case OP_ICGE:
3737 case OP_ICLE:
3738 case OP_ICGE_UN:
3739 case OP_ICLE_UN: {
3740 int cond;
3741
3742 cond = opcode_to_armcond (ins->opcode);
3743 arm_cset (code, cond, dreg);
3744 break;
3745 }
3746 case OP_FCEQ:
3747 case OP_FCLT:
3748 case OP_FCLT_UN:
3749 case OP_FCGT:
3750 case OP_FCGT_UN:
3751 case OP_FCNEQ:
3752 case OP_FCLE:
3753 case OP_FCGE: {
3754 int cond;
3755
3756 cond = opcode_to_armcond (ins->opcode);
3757 arm_fcmpd (code, sreg1, sreg2);
3758 arm_cset (code, cond, dreg);
3759 break;
3760 }
3761
3762 /* MEMORY */
3763 case OP_LOADI1_MEMBASE:
3764 code = emit_ldrsbx (code, dreg, ins->inst_basereg, ins->inst_offset);
3765 break;
3766 case OP_LOADU1_MEMBASE:
3767 code = emit_ldrb (code, dreg, ins->inst_basereg, ins->inst_offset);
3768 break;
3769 case OP_LOADI2_MEMBASE:
3770 code = emit_ldrshx (code, dreg, ins->inst_basereg, ins->inst_offset);
3771 break;
3772 case OP_LOADU2_MEMBASE:
3773 code = emit_ldrh (code, dreg, ins->inst_basereg, ins->inst_offset);
3774 break;
3775 case OP_LOADI4_MEMBASE:
3776 code = emit_ldrswx (code, dreg, ins->inst_basereg, ins->inst_offset);
3777 break;
3778 case OP_LOADU4_MEMBASE:
3779 code = emit_ldrw (code, dreg, ins->inst_basereg, ins->inst_offset);
3780 break;
3781 case OP_LOAD_MEMBASE:
3782 case OP_LOADI8_MEMBASE:
3783 code = emit_ldrx (code, dreg, ins->inst_basereg, ins->inst_offset);
3784 break;
3785 case OP_STOREI1_MEMBASE_IMM:
3786 case OP_STOREI2_MEMBASE_IMM:
3787 case OP_STOREI4_MEMBASE_IMM:
3788 case OP_STORE_MEMBASE_IMM:
3789 case OP_STOREI8_MEMBASE_IMM: {
3790 int immreg;
3791
3792 if (imm != 0) {
3793 code = emit_imm (code, ARMREG_LR, imm);
3794 immreg = ARMREG_LR;
3795 } else {
3796 immreg = ARMREG_RZR;
3797 }
3798
3799 switch (ins->opcode) {
3800 case OP_STOREI1_MEMBASE_IMM:
3801 code = emit_strb (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3802 break;
3803 case OP_STOREI2_MEMBASE_IMM:
3804 code = emit_strh (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3805 break;
3806 case OP_STOREI4_MEMBASE_IMM:
3807 code = emit_strw (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3808 break;
3809 case OP_STORE_MEMBASE_IMM:
3810 case OP_STOREI8_MEMBASE_IMM:
3811 code = emit_strx (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3812 break;
3813 default:
3814 g_assert_not_reached ();
3815 break;
3816 }
3817 break;
3818 }
3819 case OP_STOREI1_MEMBASE_REG:
3820 code = emit_strb (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3821 break;
3822 case OP_STOREI2_MEMBASE_REG:
3823 code = emit_strh (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3824 break;
3825 case OP_STOREI4_MEMBASE_REG:
3826 code = emit_strw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3827 break;
3828 case OP_STORE_MEMBASE_REG:
3829 case OP_STOREI8_MEMBASE_REG:
3830 code = emit_strx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3831 break;
3832 case OP_TLS_GET:
3833 code = emit_tls_get (code, dreg, ins->inst_offset);
3834 break;
3835 case OP_TLS_SET:
3836 code = emit_tls_set (code, sreg1, ins->inst_offset);
3837 break;
3838 /* Atomic */
3839 case OP_MEMORY_BARRIER:
3840 arm_dmb (code, ARM_DMB_ISH);
3841 break;
3842 case OP_ATOMIC_ADD_I4: {
3843 guint8 *buf [16];
3844
3845 buf [0] = code;
3846 arm_ldxrw (code, ARMREG_IP0, sreg1);
3847 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3848 arm_stlxrw (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3849 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3850
3851 arm_dmb (code, ARM_DMB_ISH);
3852 arm_movx (code, dreg, ARMREG_IP0);
3853 break;
3854 }
3855 case OP_ATOMIC_ADD_I8: {
3856 guint8 *buf [16];
3857
3858 buf [0] = code;
3859 arm_ldxrx (code, ARMREG_IP0, sreg1);
3860 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3861 arm_stlxrx (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3862 arm_cbnzx (code, ARMREG_IP1, buf [0]);
3863
3864 arm_dmb (code, ARM_DMB_ISH);
3865 arm_movx (code, dreg, ARMREG_IP0);
3866 break;
3867 }
3868 case OP_ATOMIC_EXCHANGE_I4: {
3869 guint8 *buf [16];
3870
3871 buf [0] = code;
3872 arm_ldxrw (code, ARMREG_IP0, sreg1);
3873 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3874 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3875
3876 arm_dmb (code, ARM_DMB_ISH);
3877 arm_movx (code, dreg, ARMREG_IP0);
3878 break;
3879 }
3880 case OP_ATOMIC_EXCHANGE_I8: {
3881 guint8 *buf [16];
3882
3883 buf [0] = code;
3884 arm_ldxrx (code, ARMREG_IP0, sreg1);
3885 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3886 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3887
3888 arm_dmb (code, ARM_DMB_ISH);
3889 arm_movx (code, dreg, ARMREG_IP0);
3890 break;
3891 }
3892 case OP_ATOMIC_CAS_I4: {
3893 guint8 *buf [16];
3894
3895 /* sreg2 is the value, sreg3 is the comparand */
3896 buf [0] = code;
3897 arm_ldxrw (code, ARMREG_IP0, sreg1);
3898 arm_cmpw (code, ARMREG_IP0, ins->sreg3);
3899 buf [1] = code;
3900 arm_bcc (code, ARMCOND_NE, 0);
3901 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3902 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3903 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3904
3905 arm_dmb (code, ARM_DMB_ISH);
3906 arm_movx (code, dreg, ARMREG_IP0);
3907 break;
3908 }
3909 case OP_ATOMIC_CAS_I8: {
3910 guint8 *buf [16];
3911
3912 buf [0] = code;
3913 arm_ldxrx (code, ARMREG_IP0, sreg1);
3914 arm_cmpx (code, ARMREG_IP0, ins->sreg3);
3915 buf [1] = code;
3916 arm_bcc (code, ARMCOND_NE, 0);
3917 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3918 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3919 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3920
3921 arm_dmb (code, ARM_DMB_ISH);
3922 arm_movx (code, dreg, ARMREG_IP0);
3923 break;
3924 }
3925 case OP_ATOMIC_LOAD_I1: {
3926 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3927 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3928 arm_dmb (code, ARM_DMB_ISH);
3929 arm_ldarb (code, ins->dreg, ARMREG_LR);
3930 arm_sxtbx (code, ins->dreg, ins->dreg);
3931 break;
3932 }
3933 case OP_ATOMIC_LOAD_U1: {
3934 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3935 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3936 arm_dmb (code, ARM_DMB_ISH);
3937 arm_ldarb (code, ins->dreg, ARMREG_LR);
3938 arm_uxtbx (code, ins->dreg, ins->dreg);
3939 break;
3940 }
3941 case OP_ATOMIC_LOAD_I2: {
3942 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3943 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3944 arm_dmb (code, ARM_DMB_ISH);
3945 arm_ldarh (code, ins->dreg, ARMREG_LR);
3946 arm_sxthx (code, ins->dreg, ins->dreg);
3947 break;
3948 }
3949 case OP_ATOMIC_LOAD_U2: {
3950 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3951 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3952 arm_dmb (code, ARM_DMB_ISH);
3953 arm_ldarh (code, ins->dreg, ARMREG_LR);
3954 arm_uxthx (code, ins->dreg, ins->dreg);
3955 break;
3956 }
3957 case OP_ATOMIC_LOAD_I4: {
3958 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3959 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3960 arm_dmb (code, ARM_DMB_ISH);
3961 arm_ldarw (code, ins->dreg, ARMREG_LR);
3962 arm_sxtwx (code, ins->dreg, ins->dreg);
3963 break;
3964 }
3965 case OP_ATOMIC_LOAD_U4: {
3966 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3967 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3968 arm_dmb (code, ARM_DMB_ISH);
3969 arm_ldarw (code, ins->dreg, ARMREG_LR);
3970 arm_movw (code, ins->dreg, ins->dreg); /* Clear upper half of the register. */
3971 break;
3972 }
3973 case OP_ATOMIC_LOAD_I8:
3974 case OP_ATOMIC_LOAD_U8: {
3975 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3976 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3977 arm_dmb (code, ARM_DMB_ISH);
3978 arm_ldarx (code, ins->dreg, ARMREG_LR);
3979 break;
3980 }
3981 case OP_ATOMIC_LOAD_R4: {
3982 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3983 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3984 arm_dmb (code, ARM_DMB_ISH);
3985 if (cfg->r4fp) {
3986 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3987 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3988 } else {
3989 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3990 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
3991 arm_fcvt_sd (code, ins->dreg, FP_TEMP_REG);
3992 }
3993 break;
3994 }
3995 case OP_ATOMIC_LOAD_R8: {
3996 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3997 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3998 arm_dmb (code, ARM_DMB_ISH);
3999 arm_ldarx (code, ARMREG_LR, ARMREG_LR);
4000 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
4001 break;
4002 }
4003 case OP_ATOMIC_STORE_I1:
4004 case OP_ATOMIC_STORE_U1: {
4005 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
4006 arm_stlrb (code, ARMREG_LR, ins->sreg1);
4007 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
4008 arm_dmb (code, ARM_DMB_ISH);
4009 break;
4010 }
4011 case OP_ATOMIC_STORE_I2:
4012 case OP_ATOMIC_STORE_U2: {
4013 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
4014 arm_stlrh (code, ARMREG_LR, ins->sreg1);
4015 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
4016 arm_dmb (code, ARM_DMB_ISH);
4017 break;
4018 }
4019 case OP_ATOMIC_STORE_I4:
4020 case OP_ATOMIC_STORE_U4: {
4021 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
4022 arm_stlrw (code, ARMREG_LR, ins->sreg1);
4023 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
4024 arm_dmb (code, ARM_DMB_ISH);
4025 break;
4026 }
4027 case OP_ATOMIC_STORE_I8:
4028 case OP_ATOMIC_STORE_U8: {
4029 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
4030 arm_stlrx (code, ARMREG_LR, ins->sreg1);
4031 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
4032 arm_dmb (code, ARM_DMB_ISH);
4033 break;
4034 }
4035 case OP_ATOMIC_STORE_R4: {
4036 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
4037 if (cfg->r4fp) {
4038 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
4039 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
4040 } else {
4041 arm_fcvt_ds (code, FP_TEMP_REG, ins->sreg1);
4042 arm_fmov_double_to_rx (code, ARMREG_IP0, FP_TEMP_REG);
4043 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
4044 }
4045 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
4046 arm_dmb (code, ARM_DMB_ISH);
4047 break;
4048 }
4049 case OP_ATOMIC_STORE_R8: {
4050 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
4051 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
4052 arm_stlrx (code, ARMREG_LR, ARMREG_IP0);
4053 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
4054 arm_dmb (code, ARM_DMB_ISH);
4055 break;
4056 }
4057
4058 /* FP */
4059 case OP_R8CONST: {
4060 guint64 imm = *(guint64*)ins->inst_p0;
4061
4062 if (imm == 0) {
4063 arm_fmov_rx_to_double (code, dreg, ARMREG_RZR);
4064 } else {
4065 code = emit_imm64 (code, ARMREG_LR, imm);
4066 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
4067 }
4068 break;
4069 }
4070 case OP_R4CONST: {
4071 guint64 imm = *(guint32*)ins->inst_p0;
4072
4073 code = emit_imm64 (code, ARMREG_LR, imm);
4074 if (cfg->r4fp) {
4075 arm_fmov_rx_to_double (code, dreg, ARMREG_LR);
4076 } else {
4077 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
4078 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4079 }
4080 break;
4081 }
4082 case OP_LOADR8_MEMBASE:
4083 code = emit_ldrfpx (code, dreg, ins->inst_basereg, ins->inst_offset);
4084 break;
4085 case OP_LOADR4_MEMBASE:
4086 if (cfg->r4fp) {
4087 code = emit_ldrfpw (code, dreg, ins->inst_basereg, ins->inst_offset);
4088 } else {
4089 code = emit_ldrfpw (code, FP_TEMP_REG, ins->inst_basereg, ins->inst_offset);
4090 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4091 }
4092 break;
4093 case OP_STORER8_MEMBASE_REG:
4094 code = emit_strfpx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
4095 break;
4096 case OP_STORER4_MEMBASE_REG:
4097 if (cfg->r4fp) {
4098 code = emit_strfpw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
4099 } else {
4100 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
4101 code = emit_strfpw (code, FP_TEMP_REG, ins->inst_destbasereg, ins->inst_offset);
4102 }
4103 break;
4104 case OP_FMOVE:
4105 if (dreg != sreg1)
4106 arm_fmovd (code, dreg, sreg1);
4107 break;
4108 case OP_RMOVE:
4109 if (dreg != sreg1)
4110 arm_fmovs (code, dreg, sreg1);
4111 break;
4112 case OP_MOVE_F_TO_I4:
4113 if (cfg->r4fp) {
4114 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
4115 } else {
4116 arm_fcvt_ds (code, ins->dreg, ins->sreg1);
4117 arm_fmov_double_to_rx (code, ins->dreg, ins->dreg);
4118 }
4119 break;
4120 case OP_MOVE_I4_TO_F:
4121 if (cfg->r4fp) {
4122 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
4123 } else {
4124 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
4125 arm_fcvt_sd (code, ins->dreg, ins->dreg);
4126 }
4127 break;
4128 case OP_MOVE_F_TO_I8:
4129 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
4130 break;
4131 case OP_MOVE_I8_TO_F:
4132 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
4133 break;
4134 case OP_FCOMPARE:
4135 arm_fcmpd (code, sreg1, sreg2);
4136 break;
4137 case OP_RCOMPARE:
4138 arm_fcmps (code, sreg1, sreg2);
4139 break;
4140 case OP_FCONV_TO_I1:
4141 arm_fcvtzs_dx (code, dreg, sreg1);
4142 arm_sxtbx (code, dreg, dreg);
4143 break;
4144 case OP_FCONV_TO_U1:
4145 arm_fcvtzu_dx (code, dreg, sreg1);
4146 arm_uxtbw (code, dreg, dreg);
4147 break;
4148 case OP_FCONV_TO_I2:
4149 arm_fcvtzs_dx (code, dreg, sreg1);
4150 arm_sxthx (code, dreg, dreg);
4151 break;
4152 case OP_FCONV_TO_U2:
4153 arm_fcvtzu_dx (code, dreg, sreg1);
4154 arm_uxthw (code, dreg, dreg);
4155 break;
4156 case OP_FCONV_TO_I4:
4157 arm_fcvtzs_dx (code, dreg, sreg1);
4158 arm_sxtwx (code, dreg, dreg);
4159 break;
4160 case OP_FCONV_TO_U4:
4161 arm_fcvtzu_dx (code, dreg, sreg1);
4162 break;
4163 case OP_FCONV_TO_I8:
4164 arm_fcvtzs_dx (code, dreg, sreg1);
4165 break;
4166 case OP_FCONV_TO_U8:
4167 arm_fcvtzu_dx (code, dreg, sreg1);
4168 break;
4169 case OP_FCONV_TO_R4:
4170 if (cfg->r4fp) {
4171 arm_fcvt_ds (code, dreg, sreg1);
4172 } else {
4173 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
4174 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4175 }
4176 break;
4177 case OP_ICONV_TO_R4:
4178 if (cfg->r4fp) {
4179 arm_scvtf_rw_to_s (code, dreg, sreg1);
4180 } else {
4181 arm_scvtf_rw_to_s (code, FP_TEMP_REG, sreg1);
4182 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4183 }
4184 break;
4185 case OP_LCONV_TO_R4:
4186 if (cfg->r4fp) {
4187 arm_scvtf_rx_to_s (code, dreg, sreg1);
4188 } else {
4189 arm_scvtf_rx_to_s (code, FP_TEMP_REG, sreg1);
4190 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4191 }
4192 break;
4193 case OP_ICONV_TO_R8:
4194 arm_scvtf_rw_to_d (code, dreg, sreg1);
4195 break;
4196 case OP_LCONV_TO_R8:
4197 arm_scvtf_rx_to_d (code, dreg, sreg1);
4198 break;
4199 case OP_ICONV_TO_R_UN:
4200 arm_ucvtf_rw_to_d (code, dreg, sreg1);
4201 break;
4202 case OP_LCONV_TO_R_UN:
4203 arm_ucvtf_rx_to_d (code, dreg, sreg1);
4204 break;
4205 case OP_FADD:
4206 arm_fadd_d (code, dreg, sreg1, sreg2);
4207 break;
4208 case OP_FSUB:
4209 arm_fsub_d (code, dreg, sreg1, sreg2);
4210 break;
4211 case OP_FMUL:
4212 arm_fmul_d (code, dreg, sreg1, sreg2);
4213 break;
4214 case OP_FDIV:
4215 arm_fdiv_d (code, dreg, sreg1, sreg2);
4216 break;
4217 case OP_FREM:
4218 /* Emulated */
4219 g_assert_not_reached ();
4220 break;
4221 case OP_FNEG:
4222 arm_fneg_d (code, dreg, sreg1);
4223 break;
4224 case OP_ARM_SETFREG_R4:
4225 arm_fcvt_ds (code, dreg, sreg1);
4226 break;
4227 case OP_CKFINITE:
4228 /* Check for infinity */
4229 code = emit_imm64 (code, ARMREG_LR, 0x7fefffffffffffffLL);
4230 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
4231 arm_fabs_d (code, FP_TEMP_REG2, sreg1);
4232 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG);
4233 code = emit_cond_exc (cfg, code, OP_COND_EXC_GT, "ArithmeticException");
4234 /* Check for nans */
4235 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG2);
4236 code = emit_cond_exc (cfg, code, OP_COND_EXC_OV, "ArithmeticException");
4237 arm_fmovd (code, dreg, sreg1);
4238 break;
4239
4240 /* R4 */
4241 case OP_RADD:
4242 arm_fadd_s (code, dreg, sreg1, sreg2);
4243 break;
4244 case OP_RSUB:
4245 arm_fsub_s (code, dreg, sreg1, sreg2);
4246 break;
4247 case OP_RMUL:
4248 arm_fmul_s (code, dreg, sreg1, sreg2);
4249 break;
4250 case OP_RDIV:
4251 arm_fdiv_s (code, dreg, sreg1, sreg2);
4252 break;
4253 case OP_RNEG:
4254 arm_fneg_s (code, dreg, sreg1);
4255 break;
4256 case OP_RCONV_TO_I1:
4257 arm_fcvtzs_sx (code, dreg, sreg1);
4258 arm_sxtbx (code, dreg, dreg);
4259 break;
4260 case OP_RCONV_TO_U1:
4261 arm_fcvtzu_sx (code, dreg, sreg1);
4262 arm_uxtbw (code, dreg, dreg);
4263 break;
4264 case OP_RCONV_TO_I2:
4265 arm_fcvtzs_sx (code, dreg, sreg1);
4266 arm_sxthx (code, dreg, dreg);
4267 break;
4268 case OP_RCONV_TO_U2:
4269 arm_fcvtzu_sx (code, dreg, sreg1);
4270 arm_uxthw (code, dreg, dreg);
4271 break;
4272 case OP_RCONV_TO_I4:
4273 arm_fcvtzs_sx (code, dreg, sreg1);
4274 arm_sxtwx (code, dreg, dreg);
4275 break;
4276 case OP_RCONV_TO_U4:
4277 arm_fcvtzu_sx (code, dreg, sreg1);
4278 break;
4279 case OP_RCONV_TO_I8:
4280 arm_fcvtzs_sx (code, dreg, sreg1);
4281 break;
4282 case OP_RCONV_TO_U8:
4283 arm_fcvtzu_sx (code, dreg, sreg1);
4284 break;
4285 case OP_RCONV_TO_R8:
4286 arm_fcvt_sd (code, dreg, sreg1);
4287 break;
4288 case OP_RCONV_TO_R4:
4289 if (dreg != sreg1)
4290 arm_fmovs (code, dreg, sreg1);
4291 break;
4292 case OP_RCEQ:
4293 case OP_RCLT:
4294 case OP_RCLT_UN:
4295 case OP_RCGT:
4296 case OP_RCGT_UN:
4297 case OP_RCNEQ:
4298 case OP_RCLE:
4299 case OP_RCGE: {
4300 int cond;
4301
4302 cond = opcode_to_armcond (ins->opcode);
4303 arm_fcmps (code, sreg1, sreg2);
4304 arm_cset (code, cond, dreg);
4305 break;
4306 }
4307
4308 /* CALLS */
4309 case OP_VOIDCALL:
4310 case OP_CALL:
4311 case OP_LCALL:
4312 case OP_FCALL:
4313 case OP_RCALL:
4314 case OP_VCALL2:
4315 call = (MonoCallInst*)ins;
4316 if (ins->flags & MONO_INST_HAS_METHOD)
4317 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method);
4318 else
4319 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr);
4320 code = emit_move_return_value (cfg, code, ins);
4321 break;
4322 case OP_VOIDCALL_REG:
4323 case OP_CALL_REG:
4324 case OP_LCALL_REG:
4325 case OP_FCALL_REG:
4326 case OP_RCALL_REG:
4327 case OP_VCALL2_REG:
4328 arm_blrx (code, sreg1);
4329 code = emit_move_return_value (cfg, code, ins);
4330 break;
4331 case OP_VOIDCALL_MEMBASE:
4332 case OP_CALL_MEMBASE:
4333 case OP_LCALL_MEMBASE:
4334 case OP_FCALL_MEMBASE:
4335 case OP_RCALL_MEMBASE:
4336 case OP_VCALL2_MEMBASE:
4337 code = emit_ldrx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4338 arm_blrx (code, ARMREG_IP0);
4339 code = emit_move_return_value (cfg, code, ins);
4340 break;
4341
4342 case OP_TAILCALL_PARAMETER:
4343 // This opcode helps compute sizes, i.e.
4344 // of the subsequent OP_TAILCALL, but contributes no code.
4345 g_assert (ins->next);
4346 break;
4347
4348 case OP_TAILCALL:
4349 case OP_TAILCALL_MEMBASE:
4350 case OP_TAILCALL_REG: {
4351 int branch_reg = ARMREG_IP0;
4352 guint64 free_reg = 1 << ARMREG_IP1;
4353 call = (MonoCallInst*)ins;
4354
4355 g_assert (!cfg->method->save_lmf);
4356
4357 max_len += call->stack_usage / sizeof (mgreg_t) * ins_get_size (OP_TAILCALL_PARAMETER);
4358 while (G_UNLIKELY (offset + max_len > cfg->code_size)) {
4359 cfg->code_size *= 2;
4360 cfg->native_code = (unsigned char *)mono_realloc_native_code (cfg);
4361 code = cfg->native_code + offset;
4362 cfg->stat_code_reallocs++;
4363 }
4364
4365 switch (ins->opcode) {
4366 case OP_TAILCALL:
4367 free_reg = (1 << ARMREG_IP0) | (1 << ARMREG_IP1);
4368 break;
4369
4370 case OP_TAILCALL_REG:
4371 g_assert (sreg1 != -1);
4372 g_assert (sreg1 != ARMREG_IP0);
4373 g_assert (sreg1 != ARMREG_IP1);
4374 g_assert (sreg1 != ARMREG_LR);
4375 g_assert (sreg1 != ARMREG_SP);
4376 g_assert (sreg1 != ARMREG_R28);
4377 if ((sreg1 << 1) & MONO_ARCH_CALLEE_SAVED_REGS) {
4378 arm_movx (code, branch_reg, sreg1);
4379 } else {
4380 free_reg = (1 << ARMREG_IP0) | (1 << ARMREG_IP1);
4381 branch_reg = sreg1;
4382 }
4383 break;
4384
4385 case OP_TAILCALL_MEMBASE:
4386 g_assert (ins->inst_basereg != -1);
4387 g_assert (ins->inst_basereg != ARMREG_IP0);
4388 g_assert (ins->inst_basereg != ARMREG_IP1);
4389 g_assert (ins->inst_basereg != ARMREG_LR);
4390 g_assert (ins->inst_basereg != ARMREG_SP);
4391 g_assert (ins->inst_basereg != ARMREG_R28);
4392 code = emit_ldrx (code, branch_reg, ins->inst_basereg, ins->inst_offset);
4393 break;
4394
4395 default:
4396 g_assert_not_reached ();
4397 }
4398
4399 // Copy stack arguments.
4400 // FIXME a fixed size memcpy is desirable here,
4401 // at least for larger values of stack_usage.
4402 for (int i = 0; i < call->stack_usage; i += sizeof (mgreg_t)) {
4403 code = emit_ldrx (code, ARMREG_LR, ARMREG_SP, i);
4404 code = emit_strx (code, ARMREG_LR, ARMREG_R28, i);
4405 }
4406
4407 /* Restore registers */
4408 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
4409
4410 /* Destroy frame */
4411 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, free_reg);
4412
4413 switch (ins->opcode) {
4414 case OP_TAILCALL:
4415 if (cfg->compile_aot) {
4416 /* This is not a PLT patch */
4417 code = emit_aotconst (cfg, code, branch_reg, MONO_PATCH_INFO_METHOD_JUMP, call->method);
4418 } else {
4419 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, call->method, MONO_R_ARM64_B);
4420 arm_b (code, code);
4421 cfg->thunk_area += THUNK_SIZE;
4422 break;
4423 }
4424 // fallthrough
4425 case OP_TAILCALL_MEMBASE:
4426 case OP_TAILCALL_REG:
4427 arm_brx (code, branch_reg);
4428 break;
4429
4430 default:
4431 g_assert_not_reached ();
4432 }
4433
4434 ins->flags |= MONO_INST_GC_CALLSITE;
4435 ins->backend.pc_offset = code - cfg->native_code;
4436 break;
4437 }
4438 case OP_ARGLIST:
4439 g_assert (cfg->arch.cinfo);
4440 code = emit_addx_imm (code, ARMREG_IP0, cfg->arch.args_reg, ((CallInfo*)cfg->arch.cinfo)->sig_cookie.offset);
4441 arm_strx (code, ARMREG_IP0, sreg1, 0);
4442 break;
4443 case OP_DYN_CALL: {
4444 MonoInst *var = cfg->dyn_call_var;
4445 guint8 *labels [16];
4446 int i;
4447
4448 /*
4449 * sreg1 points to a DynCallArgs structure initialized by mono_arch_start_dyn_call ().
4450 * sreg2 is the function to call.
4451 */
4452
4453 g_assert (var->opcode == OP_REGOFFSET);
4454
4455 arm_movx (code, ARMREG_LR, sreg1);
4456 arm_movx (code, ARMREG_IP1, sreg2);
4457
4458 /* Save args buffer */
4459 code = emit_strx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4460
4461 /* Set fp argument regs */
4462 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpargs));
4463 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4464 labels [0] = code;
4465 arm_bcc (code, ARMCOND_EQ, 0);
4466 for (i = 0; i < 8; ++i)
4467 code = emit_ldrfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4468 arm_patch_rel (labels [0], code, MONO_R_ARM64_BCC);
4469
4470 /* Allocate callee area */
4471 code = emit_ldrx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_stackargs));
4472 arm_lslw (code, ARMREG_R0, ARMREG_R0, 3);
4473 arm_movspx (code, ARMREG_R1, ARMREG_SP);
4474 arm_subx (code, ARMREG_R1, ARMREG_R1, ARMREG_R0);
4475 arm_movspx (code, ARMREG_SP, ARMREG_R1);
4476
4477 /* Set stack args */
4478 /* R1 = limit */
4479 code = emit_ldrx (code, ARMREG_R1, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_stackargs));
4480 /* R2 = pointer into 'regs' */
4481 code = emit_imm (code, ARMREG_R2, MONO_STRUCT_OFFSET (DynCallArgs, regs) + ((PARAM_REGS + 1) * sizeof (mgreg_t)));
4482 arm_addx (code, ARMREG_R2, ARMREG_LR, ARMREG_R2);
4483 /* R3 = pointer to stack */
4484 arm_movspx (code, ARMREG_R3, ARMREG_SP);
4485 labels [0] = code;
4486 arm_b (code, code);
4487 labels [1] = code;
4488 code = emit_ldrx (code, ARMREG_R5, ARMREG_R2, 0);
4489 code = emit_strx (code, ARMREG_R5, ARMREG_R3, 0);
4490 code = emit_addx_imm (code, ARMREG_R2, ARMREG_R2, sizeof (mgreg_t));
4491 code = emit_addx_imm (code, ARMREG_R3, ARMREG_R3, sizeof (mgreg_t));
4492 code = emit_subx_imm (code, ARMREG_R1, ARMREG_R1, 1);
4493 arm_patch_rel (labels [0], code, MONO_R_ARM64_B);
4494 arm_cmpw (code, ARMREG_R1, ARMREG_RZR);
4495 arm_bcc (code, ARMCOND_GT, labels [1]);
4496
4497 /* Set argument registers + r8 */
4498 code = mono_arm_emit_load_regarray (code, 0x1ff, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, regs));
4499
4500 /* Make the call */
4501 arm_blrx (code, ARMREG_IP1);
4502
4503 /* Save result */
4504 code = emit_ldrx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4505 arm_strx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res));
4506 arm_strx (code, ARMREG_R1, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res2));
4507 /* Save fp result */
4508 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpret));
4509 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4510 labels [1] = code;
4511 arm_bcc (code, ARMCOND_EQ, 0);
4512 for (i = 0; i < 8; ++i)
4513 code = emit_strfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4514 arm_patch_rel (labels [1], code, MONO_R_ARM64_BCC);
4515 break;
4516 }
4517
4518 case OP_GENERIC_CLASS_INIT: {
4519 int byte_offset;
4520 guint8 *jump;
4521
4522 byte_offset = MONO_STRUCT_OFFSET (MonoVTable, initialized);
4523
4524 /* Load vtable->initialized */
4525 arm_ldrsbx (code, ARMREG_IP0, sreg1, byte_offset);
4526 jump = code;
4527 arm_cbnzx (code, ARMREG_IP0, 0);
4528
4529 /* Slowpath */
4530 g_assert (sreg1 == ARMREG_R0);
4531 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4532 (gpointer)"mono_generic_class_init");
4533
4534 mono_arm_patch (jump, code, MONO_R_ARM64_CBZ);
4535 break;
4536 }
4537
4538 case OP_CHECK_THIS:
4539 arm_ldrx (code, ARMREG_LR, sreg1, 0);
4540 break;
4541 case OP_NOT_NULL:
4542 case OP_NOT_REACHED:
4543 case OP_DUMMY_USE:
4544 case OP_DUMMY_ICONST:
4545 case OP_DUMMY_I8CONST:
4546 case OP_DUMMY_R8CONST:
4547 case OP_DUMMY_R4CONST:
4548 break;
4549 case OP_IL_SEQ_POINT:
4550 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
4551 break;
4552
4553 /* EH */
4554 case OP_COND_EXC_C:
4555 case OP_COND_EXC_IC:
4556 case OP_COND_EXC_OV:
4557 case OP_COND_EXC_IOV:
4558 case OP_COND_EXC_NC:
4559 case OP_COND_EXC_INC:
4560 case OP_COND_EXC_NO:
4561 case OP_COND_EXC_INO:
4562 case OP_COND_EXC_EQ:
4563 case OP_COND_EXC_IEQ:
4564 case OP_COND_EXC_NE_UN:
4565 case OP_COND_EXC_INE_UN:
4566 case OP_COND_EXC_ILT:
4567 case OP_COND_EXC_LT:
4568 case OP_COND_EXC_ILT_UN:
4569 case OP_COND_EXC_LT_UN:
4570 case OP_COND_EXC_IGT:
4571 case OP_COND_EXC_GT:
4572 case OP_COND_EXC_IGT_UN:
4573 case OP_COND_EXC_GT_UN:
4574 case OP_COND_EXC_IGE:
4575 case OP_COND_EXC_GE:
4576 case OP_COND_EXC_IGE_UN:
4577 case OP_COND_EXC_GE_UN:
4578 case OP_COND_EXC_ILE:
4579 case OP_COND_EXC_LE:
4580 case OP_COND_EXC_ILE_UN:
4581 case OP_COND_EXC_LE_UN:
4582 code = emit_cond_exc (cfg, code, ins->opcode, ins->inst_p1);
4583 break;
4584 case OP_THROW:
4585 if (sreg1 != ARMREG_R0)
4586 arm_movx (code, ARMREG_R0, sreg1);
4587 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4588 (gpointer)"mono_arch_throw_exception");
4589 break;
4590 case OP_RETHROW:
4591 if (sreg1 != ARMREG_R0)
4592 arm_movx (code, ARMREG_R0, sreg1);
4593 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4594 (gpointer)"mono_arch_rethrow_exception");
4595 break;
4596 case OP_CALL_HANDLER:
4597 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_BL);
4598 arm_bl (code, 0);
4599 cfg->thunk_area += THUNK_SIZE;
4600 for (GList *tmp = ins->inst_eh_blocks; tmp != bb->clause_holes; tmp = tmp->prev)
4601 mono_cfg_add_try_hole (cfg, ((MonoLeaveClause *) tmp->data)->clause, code, bb);
4602 break;
4603 case OP_START_HANDLER: {
4604 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4605
4606 /* Save caller address */
4607 code = emit_strx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4608
4609 /*
4610 * Reserve a param area, see test_0_finally_param_area ().
4611 * This is needed because the param area is not set up when
4612 * we are called from EH code.
4613 */
4614 if (cfg->param_area)
4615 code = emit_subx_sp_imm (code, cfg->param_area);
4616 break;
4617 }
4618 case OP_ENDFINALLY:
4619 case OP_ENDFILTER: {
4620 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4621
4622 if (cfg->param_area)
4623 code = emit_addx_sp_imm (code, cfg->param_area);
4624
4625 if (ins->opcode == OP_ENDFILTER && sreg1 != ARMREG_R0)
4626 arm_movx (code, ARMREG_R0, sreg1);
4627
4628 /* Return to either after the branch in OP_CALL_HANDLER, or to the EH code */
4629 code = emit_ldrx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4630 arm_brx (code, ARMREG_LR);
4631 break;
4632 }
4633 case OP_GET_EX_OBJ:
4634 if (ins->dreg != ARMREG_R0)
4635 arm_movx (code, ins->dreg, ARMREG_R0);
4636 break;
4637 case OP_LIVERANGE_START: {
4638 if (cfg->verbose_level > 1)
4639 printf ("R%d START=0x%x\n", MONO_VARINFO (cfg, ins->inst_c0)->vreg, (int)(code - cfg->native_code));
4640 MONO_VARINFO (cfg, ins->inst_c0)->live_range_start = code - cfg->native_code;
4641 break;
4642 }
4643 case OP_LIVERANGE_END: {
4644 if (cfg->verbose_level > 1)
4645 printf ("R%d END=0x%x\n", MONO_VARINFO (cfg, ins->inst_c0)->vreg, (int)(code - cfg->native_code));
4646 MONO_VARINFO (cfg, ins->inst_c0)->live_range_end = code - cfg->native_code;
4647 break;
4648 }
4649 case OP_GC_SAFE_POINT: {
4650 guint8 *buf [1];
4651
4652 g_assert (mono_threads_are_safepoints_enabled ());
4653
4654 arm_ldrx (code, ARMREG_IP1, ins->sreg1, 0);
4655 /* Call it if it is non-null */
4656 buf [0] = code;
4657 arm_cbzx (code, ARMREG_IP1, 0);
4658 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_threads_state_poll");
4659 mono_arm_patch (buf [0], code, MONO_R_ARM64_CBZ);
4660 break;
4661 }
4662 case OP_FILL_PROF_CALL_CTX:
4663 for (int i = 0; i < MONO_MAX_IREGS; i++)
4664 if ((MONO_ARCH_CALLEE_SAVED_REGS & (1 << i)) || i == ARMREG_SP || i == ARMREG_FP)
4665 arm_strx (code, i, ins->sreg1, MONO_STRUCT_OFFSET (MonoContext, regs) + i * sizeof (mgreg_t));
4666 break;
4667 default:
4668 g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
4669 g_assert_not_reached ();
4670 }
4671
4672 if ((cfg->opt & MONO_OPT_BRANCH) && ((code - cfg->native_code - offset) > max_len)) {
4673 g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %d)",
4674 mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
4675 g_assert_not_reached ();
4676 }
4677 }
4678 set_code_cursor (cfg, code);
4679
4680 /*
4681 * If the compiled code size is larger than the bcc displacement (19 bits signed),
4682 * insert branch islands between/inside basic blocks.
4683 */
4684 if (cfg->arch.cond_branch_islands)
4685 code = emit_branch_island (cfg, code, start_offset);
4686 }
4687
4688 static guint8*
4689 emit_move_args (MonoCompile *cfg, guint8 *code)
4690 {
4691 MonoInst *ins;
4692 CallInfo *cinfo;
4693 ArgInfo *ainfo;
4694 int i, part;
4695
4696 cinfo = cfg->arch.cinfo;
4697 g_assert (cinfo);
4698 for (i = 0; i < cinfo->nargs; ++i) {
4699 ainfo = cinfo->args + i;
4700 ins = cfg->args [i];
4701
4702 if (ins->opcode == OP_REGVAR) {
4703 switch (ainfo->storage) {
4704 case ArgInIReg:
4705 arm_movx (code, ins->dreg, ainfo->reg);
4706 break;
4707 case ArgOnStack:
4708 switch (ainfo->slot_size) {
4709 case 1:
4710 if (ainfo->sign)
4711 code = emit_ldrsbx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4712 else
4713 code = emit_ldrb (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4714 break;
4715 case 2:
4716 if (ainfo->sign)
4717 code = emit_ldrshx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4718 else
4719 code = emit_ldrh (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4720 break;
4721 case 4:
4722 if (ainfo->sign)
4723 code = emit_ldrswx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4724 else
4725 code = emit_ldrw (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4726 break;
4727 default:
4728 code = emit_ldrx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4729 break;
4730 }
4731 break;
4732 default:
4733 g_assert_not_reached ();
4734 break;
4735 }
4736 } else {
4737 if (ainfo->storage != ArgVtypeByRef && ainfo->storage != ArgVtypeByRefOnStack)
4738 g_assert (ins->opcode == OP_REGOFFSET);
4739
4740 switch (ainfo->storage) {
4741 case ArgInIReg:
4742 /* Stack slots for arguments have size 8 */
4743 code = emit_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4744 break;
4745 case ArgInFReg:
4746 code = emit_strfpx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4747 break;
4748 case ArgInFRegR4:
4749 code = emit_strfpw (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4750 break;
4751 case ArgOnStack:
4752 case ArgOnStackR4:
4753 case ArgOnStackR8:
4754 case ArgVtypeByRefOnStack:
4755 case ArgVtypeOnStack:
4756 break;
4757 case ArgVtypeByRef: {
4758 MonoInst *addr_arg = ins->inst_left;
4759
4760 if (ainfo->gsharedvt) {
4761 g_assert (ins->opcode == OP_GSHAREDVT_ARG_REGOFFSET);
4762 arm_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4763 } else {
4764 g_assert (ins->opcode == OP_VTARG_ADDR);
4765 g_assert (addr_arg->opcode == OP_REGOFFSET);
4766 arm_strx (code, ainfo->reg, addr_arg->inst_basereg, addr_arg->inst_offset);
4767 }
4768 break;
4769 }
4770 case ArgVtypeInIRegs:
4771 for (part = 0; part < ainfo->nregs; part ++) {
4772 code = emit_strx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + (part * 8));
4773 }
4774 break;
4775 case ArgHFA:
4776 for (part = 0; part < ainfo->nregs; part ++) {
4777 if (ainfo->esize == 4)
4778 code = emit_strfpw (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4779 else
4780 code = emit_strfpx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4781 }
4782 break;
4783 default:
4784 g_assert_not_reached ();
4785 break;
4786 }
4787 }
4788 }
4789
4790 return code;
4791 }
4792
4793 /*
4794 * emit_store_regarray:
4795 *
4796 * Emit code to store the registers in REGS into the appropriate elements of
4797 * the register array at BASEREG+OFFSET.
4798 */
4799 static __attribute__ ((__warn_unused_result__)) guint8*
4800 emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4801 {
4802 int i;
4803
4804 for (i = 0; i < 32; ++i) {
4805 if (regs & (1 << i)) {
4806 if (i + 1 < 32 && (regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4807 arm_stpx (code, i, i + 1, basereg, offset + (i * 8));
4808 i++;
4809 } else if (i == ARMREG_SP) {
4810 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4811 arm_strx (code, ARMREG_IP1, basereg, offset + (i * 8));
4812 } else {
4813 arm_strx (code, i, basereg, offset + (i * 8));
4814 }
4815 }
4816 }
4817 return code;
4818 }
4819
4820 /*
4821 * emit_load_regarray:
4822 *
4823 * Emit code to load the registers in REGS from the appropriate elements of
4824 * the register array at BASEREG+OFFSET.
4825 */
4826 static __attribute__ ((__warn_unused_result__)) guint8*
4827 emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4828 {
4829 int i;
4830
4831 for (i = 0; i < 32; ++i) {
4832 if (regs & (1 << i)) {
4833 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4834 if (offset + (i * 8) < 500)
4835 arm_ldpx (code, i, i + 1, basereg, offset + (i * 8));
4836 else {
4837 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4838 code = emit_ldrx (code, i + 1, basereg, offset + ((i + 1) * 8));
4839 }
4840 i++;
4841 } else if (i == ARMREG_SP) {
4842 g_assert_not_reached ();
4843 } else {
4844 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4845 }
4846 }
4847 }
4848 return code;
4849 }
4850
4851 /*
4852 * emit_store_regset:
4853 *
4854 * Emit code to store the registers in REGS into consecutive memory locations starting
4855 * at BASEREG+OFFSET.
4856 */
4857 static __attribute__ ((__warn_unused_result__)) guint8*
4858 emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4859 {
4860 int i, pos;
4861
4862 pos = 0;
4863 for (i = 0; i < 32; ++i) {
4864 if (regs & (1 << i)) {
4865 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4866 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4867 i++;
4868 pos++;
4869 } else if (i == ARMREG_SP) {
4870 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4871 arm_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4872 } else {
4873 arm_strx (code, i, basereg, offset + (pos * 8));
4874 }
4875 pos++;
4876 }
4877 }
4878 return code;
4879 }
4880
4881 /*
4882 * emit_load_regset:
4883 *
4884 * Emit code to load the registers in REGS from consecutive memory locations starting
4885 * at BASEREG+OFFSET.
4886 */
4887 static __attribute__ ((__warn_unused_result__)) guint8*
4888 emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset)
4889 {
4890 int i, pos;
4891
4892 pos = 0;
4893 for (i = 0; i < 32; ++i) {
4894 if (regs & (1 << i)) {
4895 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4896 arm_ldpx (code, i, i + 1, basereg, offset + (pos * 8));
4897 i++;
4898 pos++;
4899 } else if (i == ARMREG_SP) {
4900 g_assert_not_reached ();
4901 } else {
4902 arm_ldrx (code, i, basereg, offset + (pos * 8));
4903 }
4904 pos++;
4905 }
4906 }
4907 return code;
4908 }
4909
4910 __attribute__ ((__warn_unused_result__)) guint8*
4911 mono_arm_emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4912 {
4913 return emit_load_regarray (code, regs, basereg, offset);
4914 }
4915
4916 __attribute__ ((__warn_unused_result__)) guint8*
4917 mono_arm_emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4918 {
4919 return emit_store_regarray (code, regs, basereg, offset);
4920 }
4921
4922 __attribute__ ((__warn_unused_result__)) guint8*
4923 mono_arm_emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4924 {
4925 return emit_store_regset (code, regs, basereg, offset);
4926 }
4927
4928 /* Same as emit_store_regset, but emit unwind info too */
4929 /* CFA_OFFSET is the offset between the CFA and basereg */
4930 static __attribute__ ((__warn_unused_result__)) guint8*
4931 emit_store_regset_cfa (MonoCompile *cfg, guint8 *code, guint64 regs, int basereg, int offset, int cfa_offset, guint64 no_cfa_regset)
4932 {
4933 int i, j, pos, nregs;
4934 guint32 cfa_regset = regs & ~no_cfa_regset;
4935
4936 pos = 0;
4937 for (i = 0; i < 32; ++i) {
4938 nregs = 1;
4939 if (regs & (1 << i)) {
4940 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4941 if (offset < 256) {
4942 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4943 } else {
4944 code = emit_strx (code, i, basereg, offset + (pos * 8));
4945 code = emit_strx (code, i + 1, basereg, offset + (pos * 8) + 8);
4946 }
4947 nregs = 2;
4948 } else if (i == ARMREG_SP) {
4949 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4950 code = emit_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4951 } else {
4952 code = emit_strx (code, i, basereg, offset + (pos * 8));
4953 }
4954
4955 for (j = 0; j < nregs; ++j) {
4956 if (cfa_regset & (1 << (i + j)))
4957 mono_emit_unwind_op_offset (cfg, code, i + j, (- cfa_offset) + offset + ((pos + j) * 8));
4958 }
4959
4960 i += nregs - 1;
4961 pos += nregs;
4962 }
4963 }
4964 return code;
4965 }
4966
4967 /*
4968 * emit_setup_lmf:
4969 *
4970 * Emit code to initialize an LMF structure at LMF_OFFSET.
4971 * Clobbers ip0/ip1.
4972 */
4973 static guint8*
4974 emit_setup_lmf (MonoCompile *cfg, guint8 *code, gint32 lmf_offset, int cfa_offset)
4975 {
4976 /*
4977 * The LMF should contain all the state required to be able to reconstruct the machine state
4978 * at the current point of execution. Since the LMF is only read during EH, only callee
4979 * saved etc. registers need to be saved.
4980 * FIXME: Save callee saved fp regs, JITted code doesn't use them, but native code does, and they
4981 * need to be restored during EH.
4982 */
4983
4984 /* pc */
4985 arm_adrx (code, ARMREG_LR, code);
4986 code = emit_strx (code, ARMREG_LR, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, pc));
4987 /* gregs + fp + sp */
4988 /* Don't emit unwind info for sp/fp, they are already handled in the prolog */
4989 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_LMF_REGS, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs), cfa_offset, (1 << ARMREG_FP) | (1 << ARMREG_SP));
4990
4991 return code;
4992 }
4993
4994 guint8 *
4995 mono_arch_emit_prolog (MonoCompile *cfg)
4996 {
4997 MonoMethod *method = cfg->method;
4998 MonoMethodSignature *sig;
4999 MonoBasicBlock *bb;
5000 guint8 *code;
5001 int cfa_offset, max_offset;
5002
5003 sig = mono_method_signature (method);
5004 cfg->code_size = 256 + sig->param_count * 64;
5005 code = cfg->native_code = g_malloc (cfg->code_size);
5006
5007 /* This can be unaligned */
5008 cfg->stack_offset = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
5009
5010 /*
5011 * - Setup frame
5012 */
5013 cfa_offset = 0;
5014 mono_emit_unwind_op_def_cfa (cfg, code, ARMREG_SP, 0);
5015
5016 /* Setup frame */
5017 if (arm_is_ldpx_imm (-cfg->stack_offset)) {
5018 arm_stpx_pre (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, -cfg->stack_offset);
5019 } else {
5020 /* sp -= cfg->stack_offset */
5021 /* This clobbers ip0/ip1 */
5022 code = emit_subx_sp_imm (code, cfg->stack_offset);
5023 arm_stpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
5024 }
5025 cfa_offset += cfg->stack_offset;
5026 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
5027 mono_emit_unwind_op_offset (cfg, code, ARMREG_FP, (- cfa_offset) + 0);
5028 mono_emit_unwind_op_offset (cfg, code, ARMREG_LR, (- cfa_offset) + 8);
5029 arm_movspx (code, ARMREG_FP, ARMREG_SP);
5030 mono_emit_unwind_op_def_cfa_reg (cfg, code, ARMREG_FP);
5031 if (cfg->param_area) {
5032 /* The param area is below the frame pointer */
5033 code = emit_subx_sp_imm (code, cfg->param_area);
5034 }
5035
5036 if (cfg->method->save_lmf) {
5037 code = emit_setup_lmf (cfg, code, cfg->lmf_var->inst_offset, cfa_offset);
5038 } else {
5039 /* Save gregs */
5040 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset, cfa_offset, 0);
5041 }
5042
5043 /* Setup args reg */
5044 if (cfg->arch.args_reg) {
5045 /* The register was already saved above */
5046 code = emit_addx_imm (code, cfg->arch.args_reg, ARMREG_FP, cfg->stack_offset);
5047 }
5048
5049 /* Save return area addr received in R8 */
5050 if (cfg->vret_addr) {
5051 MonoInst *ins = cfg->vret_addr;
5052
5053 g_assert (ins->opcode == OP_REGOFFSET);
5054 code = emit_strx (code, ARMREG_R8, ins->inst_basereg, ins->inst_offset);
5055 }
5056
5057 /* Save mrgctx received in MONO_ARCH_RGCTX_REG */
5058 if (cfg->rgctx_var) {
5059 MonoInst *ins = cfg->rgctx_var;
5060
5061 g_assert (ins->opcode == OP_REGOFFSET);
5062
5063 code = emit_strx (code, MONO_ARCH_RGCTX_REG, ins->inst_basereg, ins->inst_offset);
5064 }
5065
5066 /*
5067 * Move arguments to their registers/stack locations.
5068 */
5069 code = emit_move_args (cfg, code);
5070
5071 /* Initialize seq_point_info_var */
5072 if (cfg->arch.seq_point_info_var) {
5073 MonoInst *ins = cfg->arch.seq_point_info_var;
5074
5075 /* Initialize the variable from a GOT slot */
5076 code = emit_aotconst (cfg, code, ARMREG_IP0, MONO_PATCH_INFO_SEQ_POINT_INFO, cfg->method);
5077 g_assert (ins->opcode == OP_REGOFFSET);
5078 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
5079
5080 /* Initialize ss_tramp_var */
5081 ins = cfg->arch.ss_tramp_var;
5082 g_assert (ins->opcode == OP_REGOFFSET);
5083
5084 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP0, MONO_STRUCT_OFFSET (SeqPointInfo, ss_tramp_addr));
5085 code = emit_strx (code, ARMREG_IP1, ins->inst_basereg, ins->inst_offset);
5086 } else {
5087 MonoInst *ins;
5088
5089 if (cfg->arch.ss_tramp_var) {
5090 /* Initialize ss_tramp_var */
5091 ins = cfg->arch.ss_tramp_var;
5092 g_assert (ins->opcode == OP_REGOFFSET);
5093
5094 code = emit_imm64 (code, ARMREG_IP0, (guint64)&ss_trampoline);
5095 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
5096 }
5097
5098 if (cfg->arch.bp_tramp_var) {
5099 /* Initialize bp_tramp_var */
5100 ins = cfg->arch.bp_tramp_var;
5101 g_assert (ins->opcode == OP_REGOFFSET);
5102
5103 code = emit_imm64 (code, ARMREG_IP0, (guint64)bp_trampoline);
5104 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
5105 }
5106 }
5107
5108 max_offset = 0;
5109 if (cfg->opt & MONO_OPT_BRANCH) {
5110 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
5111 MonoInst *ins;
5112 bb->max_offset = max_offset;
5113
5114 MONO_BB_FOR_EACH_INS (bb, ins) {
5115 max_offset += ins_get_size (ins->opcode);
5116 }
5117 }
5118 }
5119 if (max_offset > 0x3ffff * 4)
5120 cfg->arch.cond_branch_islands = TRUE;
5121
5122 return code;
5123 }
5124
5125 void
5126 mono_arch_emit_epilog (MonoCompile *cfg)
5127 {
5128 CallInfo *cinfo;
5129 int max_epilog_size;
5130 guint8 *code;
5131 int i;
5132
5133 max_epilog_size = 16 + 20*4;
5134 code = realloc_code (cfg, max_epilog_size);
5135
5136 if (cfg->method->save_lmf) {
5137 code = mono_arm_emit_load_regarray (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->lmf_var->inst_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs) - (MONO_ARCH_FIRST_LMF_REG * 8));
5138 } else {
5139 /* Restore gregs */
5140 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
5141 }
5142
5143 /* Load returned vtypes into registers if needed */
5144 cinfo = cfg->arch.cinfo;
5145 switch (cinfo->ret.storage) {
5146 case ArgVtypeInIRegs: {
5147 MonoInst *ins = cfg->ret;
5148
5149 for (i = 0; i < cinfo->ret.nregs; ++i)
5150 code = emit_ldrx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + (i * 8));
5151 break;
5152 }
5153 case ArgHFA: {
5154 MonoInst *ins = cfg->ret;
5155
5156 for (i = 0; i < cinfo->ret.nregs; ++i) {
5157 if (cinfo->ret.esize == 4)
5158 code = emit_ldrfpw (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
5159 else
5160 code = emit_ldrfpx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
5161 }
5162 break;
5163 }
5164 default:
5165 break;
5166 }
5167
5168 /* Destroy frame */
5169 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, (1 << ARMREG_IP0) | (1 << ARMREG_IP1));
5170
5171 arm_retx (code, ARMREG_LR);
5172
5173 g_assert (code - (cfg->native_code + cfg->code_len) < max_epilog_size);
5174
5175 set_code_cursor (cfg, code);
5176 }
5177
5178 void
5179 mono_arch_emit_exceptions (MonoCompile *cfg)
5180 {
5181 MonoJumpInfo *ji;
5182 MonoClass *exc_class;
5183 guint8 *code, *ip;
5184 guint8* exc_throw_pos [MONO_EXC_INTRINS_NUM];
5185 guint8 exc_throw_found [MONO_EXC_INTRINS_NUM];
5186 int i, id, size = 0;
5187
5188 for (i = 0; i < MONO_EXC_INTRINS_NUM; i++) {
5189 exc_throw_pos [i] = NULL;
5190 exc_throw_found [i] = 0;
5191 }
5192
5193 for (ji = cfg->patch_info; ji; ji = ji->next) {
5194 if (ji->type == MONO_PATCH_INFO_EXC) {
5195 i = mini_exception_id_by_name (ji->data.target);
5196 if (!exc_throw_found [i]) {
5197 size += 32;
5198 exc_throw_found [i] = TRUE;
5199 }
5200 }
5201 }
5202
5203 code = realloc_code (cfg, size);
5204
5205 /* Emit code to raise corlib exceptions */
5206 for (ji = cfg->patch_info; ji; ji = ji->next) {
5207 if (ji->type != MONO_PATCH_INFO_EXC)
5208 continue;
5209
5210 ip = cfg->native_code + ji->ip.i;
5211
5212 id = mini_exception_id_by_name (ji->data.target);
5213
5214 if (exc_throw_pos [id]) {
5215 /* ip points to the bcc () in OP_COND_EXC_... */
5216 arm_patch_rel (ip, exc_throw_pos [id], ji->relocation);
5217 ji->type = MONO_PATCH_INFO_NONE;
5218 continue;
5219 }
5220
5221 exc_throw_pos [id] = code;
5222 arm_patch_rel (ip, code, ji->relocation);
5223
5224 /* We are being branched to from the code generated by emit_cond_exc (), the pc is in ip1 */
5225
5226 /* r0 = type token */
5227 exc_class = mono_class_load_from_name (mono_defaults.corlib, "System", ji->data.name);
5228 code = emit_imm (code, ARMREG_R0, m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF);
5229 /* r1 = throw ip */
5230 arm_movx (code, ARMREG_R1, ARMREG_IP1);
5231 /* Branch to the corlib exception throwing trampoline */
5232 ji->ip.i = code - cfg->native_code;
5233 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
5234 ji->data.name = "mono_arch_throw_corlib_exception";
5235 ji->relocation = MONO_R_ARM64_BL;
5236 arm_bl (code, 0);
5237 cfg->thunk_area += THUNK_SIZE;
5238 set_code_cursor (cfg, code);
5239 }
5240
5241 set_code_cursor (cfg, code);
5242 }
5243
5244 MonoInst*
5245 mono_arch_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
5246 {
5247 return NULL;
5248 }
5249
5250 guint32
5251 mono_arch_get_patch_offset (guint8 *code)
5252 {
5253 return 0;
5254 }
5255
5256 gpointer
5257 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
5258 gpointer fail_tramp)
5259 {
5260 int i, buf_len, imt_reg;
5261 guint8 *buf, *code;
5262
5263 #if DEBUG_IMT
5264 printf ("building IMT trampoline for class %s %s entries %d code size %d code at %p end %p vtable %p\n", m_class_get_name_space (vtable->klass), m_class_get_name (vtable->klass), count, size, start, ((guint8*)start) + size, vtable);
5265 for (i = 0; i < count; ++i) {
5266 MonoIMTCheckItem *item = imt_entries [i];
5267 printf ("method %d (%p) %s vtable slot %p is_equals %d chunk size %d\n", i, item->key, item->key->name, &vtable->vtable [item->value.vtable_slot], item->is_equals, item->chunk_size);
5268 }
5269 #endif
5270
5271 buf_len = 0;
5272 for (i = 0; i < count; ++i) {
5273 MonoIMTCheckItem *item = imt_entries [i];
5274 if (item->is_equals) {
5275 gboolean fail_case = !item->check_target_idx && fail_tramp;
5276
5277 if (item->check_target_idx || fail_case) {
5278 if (!item->compare_done || fail_case) {
5279 buf_len += 4 * 4 + 4;
5280 }
5281 buf_len += 4;
5282 if (item->has_target_code) {
5283 buf_len += 5 * 4;
5284 } else {
5285 buf_len += 6 * 4;
5286 }
5287 if (fail_case) {
5288 buf_len += 5 * 4;
5289 }
5290 } else {
5291 buf_len += 6 * 4;
5292 }
5293 } else {
5294 buf_len += 6 * 4;
5295 }
5296 }
5297
5298 if (fail_tramp)
5299 buf = mono_method_alloc_generic_virtual_trampoline (domain, buf_len);
5300 else
5301 buf = mono_domain_code_reserve (domain, buf_len);
5302 code = buf;
5303
5304 /*
5305 * We are called by JITted code, which passes in the IMT argument in
5306 * MONO_ARCH_RGCTX_REG (r27). We need to preserve all caller saved regs
5307 * except ip0/ip1.
5308 */
5309 imt_reg = MONO_ARCH_RGCTX_REG;
5310 for (i = 0; i < count; ++i) {
5311 MonoIMTCheckItem *item = imt_entries [i];
5312
5313 item->code_target = code;
5314
5315 if (item->is_equals) {
5316 /*
5317 * Check the imt argument against item->key, if equals, jump to either
5318 * item->value.target_code or to vtable [item->value.vtable_slot].
5319 * If fail_tramp is set, jump to it if not-equals.
5320 */
5321 gboolean fail_case = !item->check_target_idx && fail_tramp;
5322
5323 if (item->check_target_idx || fail_case) {
5324 /* Compare imt_reg with item->key */
5325 if (!item->compare_done || fail_case) {
5326 // FIXME: Optimize this
5327 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5328 arm_cmpx (code, imt_reg, ARMREG_IP0);
5329 }
5330 item->jmp_code = code;
5331 arm_bcc (code, ARMCOND_NE, 0);
5332 /* Jump to target if equals */
5333 if (item->has_target_code) {
5334 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->value.target_code);
5335 arm_brx (code, ARMREG_IP0);
5336 } else {
5337 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5338
5339 code = emit_imm64 (code, ARMREG_IP0, imm);
5340 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5341 arm_brx (code, ARMREG_IP0);
5342 }
5343
5344 if (fail_case) {
5345 arm_patch_rel (item->jmp_code, code, MONO_R_ARM64_BCC);
5346 item->jmp_code = NULL;
5347 code = emit_imm64 (code, ARMREG_IP0, (guint64)fail_tramp);
5348 arm_brx (code, ARMREG_IP0);
5349 }
5350 } else {
5351 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5352
5353 code = emit_imm64 (code, ARMREG_IP0, imm);
5354 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5355 arm_brx (code, ARMREG_IP0);
5356 }
5357 } else {
5358 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5359 arm_cmpx (code, imt_reg, ARMREG_IP0);
5360 item->jmp_code = code;
5361 arm_bcc (code, ARMCOND_HS, 0);
5362 }
5363 }
5364 /* Patch the branches */
5365 for (i = 0; i < count; ++i) {
5366 MonoIMTCheckItem *item = imt_entries [i];
5367 if (item->jmp_code && item->check_target_idx)
5368 arm_patch_rel (item->jmp_code, imt_entries [item->check_target_idx]->code_target, MONO_R_ARM64_BCC);
5369 }
5370
5371 g_assert ((code - buf) < buf_len);
5372
5373 mono_arch_flush_icache (buf, code - buf);
5374 MONO_PROFILER_RAISE (jit_code_buffer, (buf, code - buf, MONO_PROFILER_CODE_BUFFER_IMT_TRAMPOLINE, NULL));
5375
5376 return buf;
5377 }
5378
5379 GSList *
5380 mono_arch_get_trampolines (gboolean aot)
5381 {
5382 return mono_arm_get_exception_trampolines (aot);
5383 }
5384
5385 #else /* DISABLE_JIT */
5386
5387 gpointer
5388 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
5389 gpointer fail_tramp)
5390 {
5391 g_assert_not_reached ();
5392 return NULL;
5393 }
5394
5395 #endif /* !DISABLE_JIT */
5396
5397 #ifdef MONO_ARCH_SOFT_DEBUG_SUPPORTED
5398
5399 void
5400 mono_arch_set_breakpoint (MonoJitInfo *ji, guint8 *ip)
5401 {
5402 guint8 *code = ip;
5403 guint32 native_offset = ip - (guint8*)ji->code_start;
5404
5405 if (ji->from_aot) {
5406 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5407
5408 g_assert (native_offset % 4 == 0);
5409 g_assert (info->bp_addrs [native_offset / 4] == 0);
5410 info->bp_addrs [native_offset / 4] = mini_get_breakpoint_trampoline ();
5411 } else {
5412 /* ip points to an ldrx */
5413 code += 4;
5414 arm_blrx (code, ARMREG_IP0);
5415 mono_arch_flush_icache (ip, code - ip);
5416 }
5417 }
5418
5419 void
5420 mono_arch_clear_breakpoint (MonoJitInfo *ji, guint8 *ip)
5421 {
5422 guint8 *code = ip;
5423
5424 if (ji->from_aot) {
5425 guint32 native_offset = ip - (guint8*)ji->code_start;
5426 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5427
5428 g_assert (native_offset % 4 == 0);
5429 info->bp_addrs [native_offset / 4] = NULL;
5430 } else {
5431 /* ip points to an ldrx */
5432 code += 4;
5433 arm_nop (code);
5434 mono_arch_flush_icache (ip, code - ip);
5435 }
5436 }
5437
5438 void
5439 mono_arch_start_single_stepping (void)
5440 {
5441 ss_trampoline = mini_get_single_step_trampoline ();
5442 }
5443
5444 void
5445 mono_arch_stop_single_stepping (void)
5446 {
5447 ss_trampoline = NULL;
5448 }
5449
5450 gboolean
5451 mono_arch_is_single_step_event (void *info, void *sigctx)
5452 {
5453 /* We use soft breakpoints on arm64 */
5454 return FALSE;
5455 }
5456
5457 gboolean
5458 mono_arch_is_breakpoint_event (void *info, void *sigctx)
5459 {
5460 /* We use soft breakpoints on arm64 */
5461 return FALSE;
5462 }
5463
5464 void
5465 mono_arch_skip_breakpoint (MonoContext *ctx, MonoJitInfo *ji)
5466 {
5467 g_assert_not_reached ();
5468 }
5469
5470 void
5471 mono_arch_skip_single_step (MonoContext *ctx)
5472 {
5473 g_assert_not_reached ();
5474 }
5475
5476 gpointer
5477 mono_arch_get_seq_point_info (MonoDomain *domain, guint8 *code)
5478 {
5479 SeqPointInfo *info;
5480 MonoJitInfo *ji;
5481
5482 // FIXME: Add a free function
5483
5484 mono_domain_lock (domain);
5485 info = g_hash_table_lookup (domain_jit_info (domain)->arch_seq_points,
5486 code);
5487 mono_domain_unlock (domain);
5488
5489 if (!info) {
5490 ji = mono_jit_info_table_find (domain, code);
5491 g_assert (ji);
5492
5493 info = g_malloc0 (sizeof (SeqPointInfo) + (ji->code_size / 4) * sizeof(guint8*));
5494
5495 info->ss_tramp_addr = &ss_trampoline;
5496
5497 mono_domain_lock (domain);
5498 g_hash_table_insert (domain_jit_info (domain)->arch_seq_points,
5499 code, info);
5500 mono_domain_unlock (domain);
5501 }
5502
5503 return info;
5504 }
5505
5506 #endif /* MONO_ARCH_SOFT_DEBUG_SUPPORTED */
5507
5508 gboolean
5509 mono_arch_opcode_supported (int opcode)
5510 {
5511 switch (opcode) {
5512 case OP_ATOMIC_ADD_I4:
5513 case OP_ATOMIC_ADD_I8:
5514 case OP_ATOMIC_EXCHANGE_I4:
5515 case OP_ATOMIC_EXCHANGE_I8:
5516 case OP_ATOMIC_CAS_I4:
5517 case OP_ATOMIC_CAS_I8:
5518 case OP_ATOMIC_LOAD_I1:
5519 case OP_ATOMIC_LOAD_I2:
5520 case OP_ATOMIC_LOAD_I4:
5521 case OP_ATOMIC_LOAD_I8:
5522 case OP_ATOMIC_LOAD_U1:
5523 case OP_ATOMIC_LOAD_U2:
5524 case OP_ATOMIC_LOAD_U4:
5525 case OP_ATOMIC_LOAD_U8:
5526 case OP_ATOMIC_LOAD_R4:
5527 case OP_ATOMIC_LOAD_R8:
5528 case OP_ATOMIC_STORE_I1:
5529 case OP_ATOMIC_STORE_I2:
5530 case OP_ATOMIC_STORE_I4:
5531 case OP_ATOMIC_STORE_I8:
5532 case OP_ATOMIC_STORE_U1:
5533 case OP_ATOMIC_STORE_U2:
5534 case OP_ATOMIC_STORE_U4:
5535 case OP_ATOMIC_STORE_U8:
5536 case OP_ATOMIC_STORE_R4:
5537 case OP_ATOMIC_STORE_R8:
5538 return TRUE;
5539 default:
5540 return FALSE;
5541 }
5542 }
5543
5544 CallInfo*
5545 mono_arch_get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
5546 {
5547 return get_call_info (mp, sig);
5548 }
5549
Mirror of the official Mono project git repository