?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Move a line_ref variable from tcc_preprocess() function into struct BufferedFile.
[tinycc.git] / arm64-gen.c
blob5444f297d67f229c7e1ed8f79fb34b61355b0b48
1 /*
2 * A64 code generator for TCC
3 *
4 * Copyright (c) 2014-2015 Edmund Grimley Evans
5 *
6 * Copying and distribution of this file, with or without modification,
7 * are permitted in any medium without royalty provided the copyright
8 * notice and this notice are preserved. This file is offered as-is,
9 * without any warranty.
10 */
11
12 #ifdef TARGET_DEFS_ONLY
13
14 // Number of registers available to allocator:
15 #define NB_REGS 28 // x0-x18, x30, v0-v7
16
17 #define TREG_R(x) (x) // x = 0..18
18 #define TREG_R30 19
19 #define TREG_F(x) (x + 20) // x = 0..7
20
21 // Register classes sorted from more general to more precise:
22 #define RC_INT (1 << 0)
23 #define RC_FLOAT (1 << 1)
24 #define RC_R(x) (1 << (2 + (x))) // x = 0..18
25 #define RC_R30 (1 << 21)
26 #define RC_F(x) (1 << (22 + (x))) // x = 0..7
27
28 #define RC_IRET (RC_R(0)) // int return register class
29 #define RC_FRET (RC_F(0)) // float return register class
30
31 #define REG_IRET (TREG_R(0)) // int return register number
32 #define REG_FRET (TREG_F(0)) // float return register number
33
34 #define PTR_SIZE 8
35
36 #define LDOUBLE_SIZE 16
37 #define LDOUBLE_ALIGN 16
38
39 #define MAX_ALIGN 16
40
41 #define CHAR_IS_UNSIGNED
42
43 /******************************************************/
44 /* ELF defines */
45
46 #define EM_TCC_TARGET EM_AARCH64
47
48 #define R_DATA_32 R_AARCH64_ABS32
49 #define R_DATA_PTR R_AARCH64_ABS64
50 #define R_JMP_SLOT R_AARCH64_JUMP_SLOT
51 #define R_COPY R_AARCH64_COPY
52
53 #define ELF_START_ADDR 0x00400000
54 #define ELF_PAGE_SIZE 0x1000
55
56 /******************************************************/
57 #else /* ! TARGET_DEFS_ONLY */
58 /******************************************************/
59 #include "tcc.h"
60 #include <assert.h>
61
62 ST_DATA const int reg_classes[NB_REGS] = {
63 RC_INT | RC_R(0),
64 RC_INT | RC_R(1),
65 RC_INT | RC_R(2),
66 RC_INT | RC_R(3),
67 RC_INT | RC_R(4),
68 RC_INT | RC_R(5),
69 RC_INT | RC_R(6),
70 RC_INT | RC_R(7),
71 RC_INT | RC_R(8),
72 RC_INT | RC_R(9),
73 RC_INT | RC_R(10),
74 RC_INT | RC_R(11),
75 RC_INT | RC_R(12),
76 RC_INT | RC_R(13),
77 RC_INT | RC_R(14),
78 RC_INT | RC_R(15),
79 RC_INT | RC_R(16),
80 RC_INT | RC_R(17),
81 RC_INT | RC_R(18),
82 RC_R30, // not in RC_INT as we make special use of x30
83 RC_FLOAT | RC_F(0),
84 RC_FLOAT | RC_F(1),
85 RC_FLOAT | RC_F(2),
86 RC_FLOAT | RC_F(3),
87 RC_FLOAT | RC_F(4),
88 RC_FLOAT | RC_F(5),
89 RC_FLOAT | RC_F(6),
90 RC_FLOAT | RC_F(7)
91 };
92
93 #define IS_FREG(x) ((x) >= TREG_F(0))
94
95 static uint32_t intr(int r)
96 {
97 assert(TREG_R(0) <= r && r <= TREG_R30);
98 return r < TREG_R30 ? r : 30;
99 }
100
101 static uint32_t fltr(int r)
102 {
103 assert(TREG_F(0) <= r && r <= TREG_F(7));
104 return r - TREG_F(0);
105 }
106
107 // Add an instruction to text section:
108 ST_FUNC void o(unsigned int c)
109 {
110 int ind1 = ind + 4;
111 if (ind1 > cur_text_section->data_allocated)
112 section_realloc(cur_text_section, ind1);
113 *(uint32_t *)(cur_text_section->data + ind) = c;
114 ind = ind1;
115 }
116
117 static int arm64_encode_bimm64(uint64_t x)
118 {
119 int neg = x & 1;
120 int rep, pos, len;
121
122 if (neg)
123 x = ~x;
124 if (!x)
125 return -1;
126
127 if (x >> 2 == (x & (((uint64_t)1 << (64 - 2)) - 1)))
128 rep = 2, x &= ((uint64_t)1 << 2) - 1;
129 else if (x >> 4 == (x & (((uint64_t)1 << (64 - 4)) - 1)))
130 rep = 4, x &= ((uint64_t)1 << 4) - 1;
131 else if (x >> 8 == (x & (((uint64_t)1 << (64 - 8)) - 1)))
132 rep = 8, x &= ((uint64_t)1 << 8) - 1;
133 else if (x >> 16 == (x & (((uint64_t)1 << (64 - 16)) - 1)))
134 rep = 16, x &= ((uint64_t)1 << 16) - 1;
135 else if (x >> 32 == (x & (((uint64_t)1 << (64 - 32)) - 1)))
136 rep = 32, x &= ((uint64_t)1 << 32) - 1;
137 else
138 rep = 64;
139
140 pos = 0;
141 if (!(x & (((uint64_t)1 << 32) - 1))) x >>= 32, pos += 32;
142 if (!(x & (((uint64_t)1 << 16) - 1))) x >>= 16, pos += 16;
143 if (!(x & (((uint64_t)1 << 8) - 1))) x >>= 8, pos += 8;
144 if (!(x & (((uint64_t)1 << 4) - 1))) x >>= 4, pos += 4;
145 if (!(x & (((uint64_t)1 << 2) - 1))) x >>= 2, pos += 2;
146 if (!(x & (((uint64_t)1 << 1) - 1))) x >>= 1, pos += 1;
147
148 len = 0;
149 if (!(~x & (((uint64_t)1 << 32) - 1))) x >>= 32, len += 32;
150 if (!(~x & (((uint64_t)1 << 16) - 1))) x >>= 16, len += 16;
151 if (!(~x & (((uint64_t)1 << 8) - 1))) x >>= 8, len += 8;
152 if (!(~x & (((uint64_t)1 << 4) - 1))) x >>= 4, len += 4;
153 if (!(~x & (((uint64_t)1 << 2) - 1))) x >>= 2, len += 2;
154 if (!(~x & (((uint64_t)1 << 1) - 1))) x >>= 1, len += 1;
155
156 if (x)
157 return -1;
158 if (neg) {
159 pos = (pos + len) & (rep - 1);
160 len = rep - len;
161 }
162 return ((0x1000 & rep << 6) | (((rep - 1) ^ 31) << 1 & 63) |
163 ((rep - pos) & (rep - 1)) << 6 | (len - 1));
164 }
165
166 static uint32_t arm64_movi(int r, uint64_t x)
167 {
168 uint64_t m = 0xffff;
169 int e;
170 if (!(x & ~m))
171 return 0x52800000 | r | x << 5; // movz w(r),#(x)
172 if (!(x & ~(m << 16)))
173 return 0x52a00000 | r | x >> 11; // movz w(r),#(x >> 16),lsl #16
174 if (!(x & ~(m << 32)))
175 return 0xd2c00000 | r | x >> 27; // movz x(r),#(x >> 32),lsl #32
176 if (!(x & ~(m << 48)))
177 return 0xd2e00000 | r | x >> 43; // movz x(r),#(x >> 48),lsl #48
178 if ((x & ~m) == m << 16)
179 return (0x12800000 | r |
180 (~x << 5 & 0x1fffe0)); // movn w(r),#(~x)
181 if ((x & ~(m << 16)) == m)
182 return (0x12a00000 | r |
183 (~x >> 11 & 0x1fffe0)); // movn w(r),#(~x >> 16),lsl #16
184 if (!~(x | m))
185 return (0x92800000 | r |
186 (~x << 5 & 0x1fffe0)); // movn x(r),#(~x)
187 if (!~(x | m << 16))
188 return (0x92a00000 | r |
189 (~x >> 11 & 0x1fffe0)); // movn x(r),#(~x >> 16),lsl #16
190 if (!~(x | m << 32))
191 return (0x92c00000 | r |
192 (~x >> 27 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32
193 if (!~(x | m << 48))
194 return (0x92e00000 | r |
195 (~x >> 43 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32
196 if (!(x >> 32) && (e = arm64_encode_bimm64(x | x << 32)) >= 0)
197 return 0x320003e0 | r | (uint32_t)e << 10; // movi w(r),#(x)
198 if ((e = arm64_encode_bimm64(x)) >= 0)
199 return 0xb20003e0 | r | (uint32_t)e << 10; // movi x(r),#(x)
200 return 0;
201 }
202
203 static void arm64_movimm(int r, uint64_t x)
204 {
205 uint32_t i;
206 if ((i = arm64_movi(r, x)))
207 o(i); // a single MOV
208 else {
209 // MOVZ/MOVN and 1-3 MOVKs
210 int z = 0, m = 0;
211 uint32_t mov1 = 0xd2800000; // movz
212 uint64_t x1 = x;
213 for (i = 0; i < 64; i += 16) {
214 z += !(x >> i & 0xffff);
215 m += !(~x >> i & 0xffff);
216 }
217 if (m > z) {
218 x1 = ~x;
219 mov1 = 0x92800000; // movn
220 }
221 for (i = 0; i < 64; i += 16)
222 if (x1 >> i & 0xffff) {
223 o(mov1 | r | (x1 >> i & 0xffff) << 5 | i << 17);
224 // movz/movn x(r),#(*),lsl #(i)
225 break;
226 }
227 for (i += 16; i < 64; i += 16)
228 if (x1 >> i & 0xffff)
229 o(0xf2800000 | r | (x >> i & 0xffff) << 5 | i << 17);
230 // movk x(r),#(*),lsl #(i)
231 }
232 }
233
234 // Patch all branches in list pointed to by t to branch to a:
235 ST_FUNC void gsym_addr(int t_, int a_)
236 {
237 uint32_t t = t_;
238 uint32_t a = a_;
239 while (t) {
240 uint32_t *ptr = (uint32_t *)(cur_text_section->data + t);
241 uint32_t next = *ptr;
242 if (a - t + 0x8000000 >= 0x10000000)
243 tcc_error("branch out of range");
244 *ptr = (a - t == 4 ? 0xd503201f : // nop
245 0x14000000 | ((a - t) >> 2 & 0x3ffffff)); // b
246 t = next;
247 }
248 }
249
250 // Patch all branches in list pointed to by t to branch to current location:
251 ST_FUNC void gsym(int t)
252 {
253 gsym_addr(t, ind);
254 }
255
256 static int arm64_type_size(int t)
257 {
258 switch (t & VT_BTYPE) {
259 case VT_INT: return 2;
260 case VT_BYTE: return 0;
261 case VT_SHORT: return 1;
262 case VT_PTR: return 3;
263 case VT_ENUM: return 2;
264 case VT_FUNC: return 3;
265 case VT_FLOAT: return 2;
266 case VT_DOUBLE: return 3;
267 case VT_LDOUBLE: return 4;
268 case VT_BOOL: return 0;
269 case VT_LLONG: return 3;
270 }
271 assert(0);
272 return 0;
273 }
274
275 static void arm64_spoff(int reg, uint64_t off)
276 {
277 uint32_t sub = off >> 63;
278 if (sub)
279 off = -off;
280 if (off < 4096)
281 o(0x910003e0 | sub << 30 | reg | off << 10);
282 // (add|sub) x(reg),sp,#(off)
283 else {
284 arm64_movimm(30, off); // use x30 for offset
285 o(0x8b3e63e0 | sub << 30 | reg); // (add|sub) x(reg),sp,x30
286 }
287 }
288
289 static void arm64_ldrx(int sg, int sz, int dst, int bas, uint64_t off)
290 {
291 if (sz >= 2)
292 sg = 0;
293 if (!(off & ~(0xfff << sz)))
294 o(0x39400000 | dst | bas << 5 | off << (10 - sz) |
295 !!sg << 23 | sz << 30); // ldr(*) x(dst),[x(bas),#(off)]
296 else if (off < 256 || -off <= 256)
297 o(0x38400000 | dst | bas << 5 | (off & 511) << 12 |
298 !!sg << 23 | sz << 30); // ldur(*) x(dst),[x(bas),#(off)]
299 else {
300 arm64_movimm(30, off); // use x30 for offset
301 o(0x38206800 | dst | bas << 5 | 30 << 16 |
302 (!!sg + 1) << 22 | sz << 30); // ldr(*) x(dst),[x(bas),x30]
303 }
304 }
305
306 static void arm64_ldrv(int sz, int dst, int bas, uint64_t off)
307 {
308 if (!(off & ~(0xfff << sz)))
309 o(0x3d400000 | dst | bas << 5 | off << (10 - sz) |
310 (sz & 4) << 21 | (sz & 3) << 30); // ldr (s|d|q)(dst),[x(bas),#(off)]
311 else if (off < 256 || -off <= 256)
312 o(0x3c400000 | dst | bas << 5 | (off & 511) << 12 |
313 (sz & 4) << 21 | (sz & 3) << 30); // ldur (s|d|q)(dst),[x(bas),#(off)]
314 else {
315 arm64_movimm(30, off); // use x30 for offset
316 o(0x3c606800 | dst | bas << 5 | 30 << 16 | sz << 30 | (sz & 4) << 21);
317 // ldr (s|d|q)(dst),[x(bas),x30]
318 }
319 }
320
321 static void arm64_ldrs(int reg, int size)
322 {
323 // Use x30 for intermediate value in some cases.
324 switch (size) {
325 default: assert(0); break;
326 case 1:
327 arm64_ldrx(0, 0, reg, reg, 0);
328 break;
329 case 2:
330 arm64_ldrx(0, 1, reg, reg, 0);
331 break;
332 case 3:
333 arm64_ldrx(0, 1, 30, reg, 0);
334 arm64_ldrx(0, 0, reg, reg, 2);
335 o(0x2a0043c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #16
336 break;
337 case 4:
338 arm64_ldrx(0, 2, reg, reg, 0);
339 break;
340 case 5:
341 arm64_ldrx(0, 2, 30, reg, 0);
342 arm64_ldrx(0, 0, reg, reg, 4);
343 o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32
344 break;
345 case 6:
346 arm64_ldrx(0, 2, 30, reg, 0);
347 arm64_ldrx(0, 1, reg, reg, 4);
348 o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32
349 break;
350 case 7:
351 arm64_ldrx(0, 2, 30, reg, 0);
352 arm64_ldrx(0, 2, reg, reg, 3);
353 o(0x53087c00 | reg | reg << 5); // lsr w(reg), w(reg), #8
354 o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32
355 break;
356 case 8:
357 arm64_ldrx(0, 3, reg, reg, 0);
358 break;
359 case 9:
360 arm64_ldrx(0, 0, reg + 1, reg, 8);
361 arm64_ldrx(0, 3, reg, reg, 0);
362 break;
363 case 10:
364 arm64_ldrx(0, 1, reg + 1, reg, 8);
365 arm64_ldrx(0, 3, reg, reg, 0);
366 break;
367 case 11:
368 arm64_ldrx(0, 2, reg + 1, reg, 7);
369 o(0x53087c00 | (reg+1) | (reg+1) << 5); // lsr w(reg+1), w(reg+1), #8
370 arm64_ldrx(0, 3, reg, reg, 0);
371 break;
372 case 12:
373 arm64_ldrx(0, 2, reg + 1, reg, 8);
374 arm64_ldrx(0, 3, reg, reg, 0);
375 break;
376 case 13:
377 arm64_ldrx(0, 3, reg + 1, reg, 5);
378 o(0xd358fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #24
379 arm64_ldrx(0, 3, reg, reg, 0);
380 break;
381 case 14:
382 arm64_ldrx(0, 3, reg + 1, reg, 6);
383 o(0xd350fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #16
384 arm64_ldrx(0, 3, reg, reg, 0);
385 break;
386 case 15:
387 arm64_ldrx(0, 3, reg + 1, reg, 7);
388 o(0xd348fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #8
389 arm64_ldrx(0, 3, reg, reg, 0);
390 break;
391 case 16:
392 o(0xa9400000 | reg | (reg+1) << 10 | reg << 5);
393 // ldp x(reg),x(reg+1),[x(reg)]
394 break;
395 }
396 }
397
398 static void arm64_strx(int sz, int dst, int bas, uint64_t off)
399 {
400 if (!(off & ~(0xfff << sz)))
401 o(0x39000000 | dst | bas << 5 | off << (10 - sz) | sz << 30);
402 // str(*) x(dst),[x(bas],#(off)]
403 else if (off < 256 || -off <= 256)
404 o(0x38000000 | dst | bas << 5 | (off & 511) << 12 | sz << 30);
405 // stur(*) x(dst),[x(bas],#(off)]
406 else {
407 arm64_movimm(30, off); // use x30 for offset
408 o(0x38206800 | dst | bas << 5 | 30 << 16 | sz << 30);
409 // str(*) x(dst),[x(bas),x30]
410 }
411 }
412
413 static void arm64_strv(int sz, int dst, int bas, uint64_t off)
414 {
415 if (!(off & ~(0xfff << sz)))
416 o(0x3d000000 | dst | bas << 5 | off << (10 - sz) |
417 (sz & 4) << 21 | (sz & 3) << 30); // str (s|d|q)(dst),[x(bas),#(off)]
418 else if (off < 256 || -off <= 256)
419 o(0x3c000000 | dst | bas << 5 | (off & 511) << 12 |
420 (sz & 4) << 21 | (sz & 3) << 30); // stur (s|d|q)(dst),[x(bas),#(off)]
421 else {
422 arm64_movimm(30, off); // use x30 for offset
423 o(0x3c206800 | dst | bas << 5 | 30 << 16 | sz << 30 | (sz & 4) << 21);
424 // str (s|d|q)(dst),[x(bas),x30]
425 }
426 }
427
428 static void arm64_sym(int r, Sym *sym, unsigned long addend)
429 {
430 // Currently TCC's linker does not generate COPY relocations for
431 // STT_OBJECTs when tcc is invoked with "-run". This typically
432 // results in "R_AARCH64_ADR_PREL_PG_HI21 relocation failed" when
433 // a program refers to stdin. A workaround is to avoid that
434 // relocation and use only relocations with unlimited range.
435 int avoid_adrp = 1;
436
437 if (avoid_adrp || (sym->type.t & VT_WEAK)) {
438 // (GCC uses a R_AARCH64_ABS64 in this case.)
439 greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G0_NC, addend);
440 o(0xd2800000 | r); // mov x(rt),#0,lsl #0
441 greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G1_NC, addend);
442 o(0xf2a00000 | r); // movk x(rt),#0,lsl #16
443 greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G2_NC, addend);
444 o(0xf2c00000 | r); // movk x(rt),#0,lsl #32
445 greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G3, addend);
446 o(0xf2e00000 | r); // movk x(rt),#0,lsl #48
447 }
448 else {
449 greloca(cur_text_section, sym, ind, R_AARCH64_ADR_PREL_PG_HI21, addend);
450 o(0x90000000 | r);
451 greloca(cur_text_section, sym, ind, R_AARCH64_ADD_ABS_LO12_NC, addend);
452 o(0x91000000 | r | r << 5);
453 }
454 }
455
456 ST_FUNC void load(int r, SValue *sv)
457 {
458 int svtt = sv->type.t;
459 int svr = sv->r & ~VT_LVAL_TYPE;
460 int svrv = svr & VT_VALMASK;
461 uint64_t svcul = (int32_t)sv->c.ul;
462
463 if (svr == (VT_LOCAL | VT_LVAL)) {
464 if (IS_FREG(r))
465 arm64_ldrv(arm64_type_size(svtt), fltr(r), 29, svcul);
466 else
467 arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
468 intr(r), 29, svcul);
469 return;
470 }
471
472 if ((svr & ~VT_VALMASK) == VT_LVAL && svrv < VT_CONST) {
473 if (IS_FREG(r))
474 arm64_ldrv(arm64_type_size(svtt), fltr(r), intr(svrv), 0);
475 else
476 arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
477 intr(r), intr(svrv), 0);
478 return;
479 }
480
481 if (svr == (VT_CONST | VT_LVAL | VT_SYM)) {
482 arm64_sym(30, sv->sym, svcul); // use x30 for address
483 if (IS_FREG(r))
484 arm64_ldrv(arm64_type_size(svtt), fltr(r), 30, 0);
485 else
486 arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
487 intr(r), 30, 0);
488 return;
489 }
490
491 if (svr == (VT_CONST | VT_SYM)) {
492 arm64_sym(intr(r), sv->sym, svcul);
493 return;
494 }
495
496 if (svr == VT_CONST) {
497 if ((svtt & VT_BTYPE) != VT_VOID)
498 arm64_movimm(intr(r),
499 arm64_type_size(svtt) == 3 ? sv->c.ull : svcul);
500 return;
501 }
502
503 if (svr < VT_CONST) {
504 if (IS_FREG(r) && IS_FREG(svr))
505 if (svtt == VT_LDOUBLE)
506 o(0x4ea01c00 | fltr(r) | fltr(svr) << 5);
507 // mov v(r).16b,v(svr).16b
508 else
509 o(0x1e604000 | fltr(r) | fltr(svr) << 5); // fmov d(r),d(svr)
510 else if (!IS_FREG(r) && !IS_FREG(svr))
511 o(0xaa0003e0 | intr(r) | intr(svr) << 16); // mov x(r),x(svr)
512 else
513 assert(0);
514 return;
515 }
516
517 if (svr == VT_LOCAL) {
518 if (-svcul < 0x1000)
519 o(0xd10003a0 | intr(r) | -svcul << 10); // sub x(r),x29,#...
520 else {
521 arm64_movimm(30, -svcul); // use x30 for offset
522 o(0xcb0003a0 | intr(r) | 30 << 16); // sub x(r),x29,x30
523 }
524 return;
525 }
526
527 if (svr == VT_JMP || svr == VT_JMPI) {
528 int t = (svr == VT_JMPI);
529 arm64_movimm(intr(r), t);
530 o(0x14000002); // b .+8
531 gsym(svcul);
532 arm64_movimm(intr(r), t ^ 1);
533 return;
534 }
535
536 if (svr == (VT_LLOCAL | VT_LVAL)) {
537 arm64_ldrx(0, 3, 30, 29, svcul); // use x30 for offset
538 if (IS_FREG(r))
539 arm64_ldrv(arm64_type_size(svtt), fltr(r), 30, 0);
540 else
541 arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
542 intr(r), 30, 0);
543 return;
544 }
545
546 printf("load(%x, (%x, %x, %llx))\n", r, svtt, sv->r, (long long)svcul);
547 assert(0);
548 }
549
550 ST_FUNC void store(int r, SValue *sv)
551 {
552 int svtt = sv->type.t;
553 int svr = sv->r & ~VT_LVAL_TYPE;
554 int svrv = svr & VT_VALMASK;
555 uint64_t svcul = (int32_t)sv->c.ul;
556
557 if (svr == (VT_LOCAL | VT_LVAL)) {
558 if (IS_FREG(r))
559 arm64_strv(arm64_type_size(svtt), fltr(r), 29, svcul);
560 else
561 arm64_strx(arm64_type_size(svtt), intr(r), 29, svcul);
562 return;
563 }
564
565 if ((svr & ~VT_VALMASK) == VT_LVAL && svrv < VT_CONST) {
566 if (IS_FREG(r))
567 arm64_strv(arm64_type_size(svtt), fltr(r), intr(svrv), 0);
568 else
569 arm64_strx(arm64_type_size(svtt), intr(r), intr(svrv), 0);
570 return;
571 }
572
573 if (svr == (VT_CONST | VT_LVAL | VT_SYM)) {
574 arm64_sym(30, sv->sym, svcul); // use x30 for address
575 if (IS_FREG(r))
576 arm64_strv(arm64_type_size(svtt), fltr(r), 30, 0);
577 else
578 arm64_strx(arm64_type_size(svtt), intr(r), 30, 0);
579 return;
580 }
581
582 printf("store(%x, (%x, %x, %llx))\n", r, svtt, sv->r, (long long)svcul);
583 assert(0);
584 }
585
586 static void arm64_gen_bl_or_b(int b)
587 {
588 if ((vtop->r & (VT_VALMASK | VT_LVAL)) == VT_CONST) {
589 assert(!b);
590 if (vtop->r & VT_SYM)
591 greloc(cur_text_section, vtop->sym, ind, R_AARCH64_CALL26);
592 else
593 assert(0);
594 o(0x94000000); // bl .
595 }
596 else
597 o(0xd61f0000 | !b << 21 | intr(gv(RC_R30)) << 5); // br/blr
598 }
599
600 static int arm64_hfa_aux(CType *type, int *fsize, int num)
601 {
602 if (is_float(type->t)) {
603 int a, n = type_size(type, &a);
604 if (num >= 4 || (*fsize && *fsize != n))
605 return -1;
606 *fsize = n;
607 return num + 1;
608 }
609 else if ((type->t & VT_BTYPE) == VT_STRUCT) {
610 int is_struct = 0; // rather than union
611 Sym *field;
612 for (field = type->ref->next; field; field = field->next)
613 if (field->c) {
614 is_struct = 1;
615 break;
616 }
617 if (is_struct) {
618 int num0 = num;
619 for (field = type->ref->next; field; field = field->next) {
620 if (field->c != (num - num0) * *fsize)
621 return -1;
622 num = arm64_hfa_aux(&field->type, fsize, num);
623 if (num == -1)
624 return -1;
625 }
626 if (type->ref->c != (num - num0) * *fsize)
627 return -1;
628 return num;
629 }
630 else { // union
631 int num0 = num;
632 for (field = type->ref->next; field; field = field->next) {
633 int num1 = arm64_hfa_aux(&field->type, fsize, num0);
634 if (num1 == -1)
635 return -1;
636 num = num1 < num ? num : num1;
637 }
638 if (type->ref->c != (num - num0) * *fsize)
639 return -1;
640 return num;
641 }
642 }
643 else if (type->t & VT_ARRAY) {
644 int num1;
645 if (!type->ref->c)
646 return num;
647 num1 = arm64_hfa_aux(&type->ref->type, fsize, num);
648 if (num1 == -1 || (num1 != num && type->ref->c > 4))
649 return -1;
650 num1 = num + type->ref->c * (num1 - num);
651 if (num1 > 4)
652 return -1;
653 return num1;
654 }
655 return -1;
656 }
657
658 static int arm64_hfa(CType *type, int *fsize)
659 {
660 if ((type->t & VT_BTYPE) == VT_STRUCT || (type->t & VT_ARRAY)) {
661 int sz = 0;
662 int n = arm64_hfa_aux(type, &sz, 0);
663 if (0 < n && n <= 4) {
664 if (fsize)
665 *fsize = sz;
666 return n;
667 }
668 }
669 return 0;
670 }
671
672 static unsigned long arm64_pcs_aux(int n, CType **type, unsigned long *a)
673 {
674 int nx = 0; // next integer register
675 int nv = 0; // next vector register
676 unsigned long ns = 32; // next stack offset
677 int i;
678
679 for (i = 0; i < n; i++) {
680 int hfa = arm64_hfa(type[i], 0);
681 int size, align;
682
683 if ((type[i]->t & VT_ARRAY) ||
684 (type[i]->t & VT_BTYPE) == VT_FUNC)
685 size = align = 8;
686 else
687 size = type_size(type[i], &align);
688
689 if (hfa)
690 // B.2
691 ;
692 else if (size > 16) {
693 // B.3: replace with pointer
694 if (nx < 8)
695 a[i] = nx++ << 1 | 1;
696 else {
697 ns = (ns + 7) & ~7;
698 a[i] = ns | 1;
699 ns += 8;
700 }
701 continue;
702 }
703 else if ((type[i]->t & VT_BTYPE) == VT_STRUCT)
704 // B.4
705 size = (size + 7) & ~7;
706
707 // C.1
708 if (is_float(type[i]->t) && nv < 8) {
709 a[i] = 16 + (nv++ << 1);
710 continue;
711 }
712
713 // C.2
714 if (hfa && nv + hfa <= 8) {
715 a[i] = 16 + (nv << 1);
716 nv += hfa;
717 continue;
718 }
719
720 // C.3
721 if (hfa) {
722 nv = 8;
723 size = (size + 7) & ~7;
724 }
725
726 // C.4
727 if (hfa || (type[i]->t & VT_BTYPE) == VT_LDOUBLE) {
728 ns = (ns + 7) & ~7;
729 ns = (ns + align - 1) & -align;
730 }
731
732 // C.5
733 if ((type[i]->t & VT_BTYPE) == VT_FLOAT)
734 size = 8;
735
736 // C.6
737 if (hfa || is_float(type[i]->t)) {
738 a[i] = ns;
739 ns += size;
740 continue;
741 }
742
743 // C.7
744 if ((type[i]->t & VT_BTYPE) != VT_STRUCT && size <= 8 && nx < 8) {
745 a[i] = nx++ << 1;
746 continue;
747 }
748
749 // C.8
750 if (align == 16)
751 nx = (nx + 1) & ~1;
752
753 // C.9
754 if ((type[i]->t & VT_BTYPE) != VT_STRUCT && size == 16 && nx < 7) {
755 a[i] = nx << 1;
756 nx += 2;
757 continue;
758 }
759
760 // C.10
761 if ((type[i]->t & VT_BTYPE) == VT_STRUCT && size <= (8 - nx) * 8) {
762 a[i] = nx << 1;
763 nx += (size + 7) >> 3;
764 continue;
765 }
766
767 // C.11
768 nx = 8;
769
770 // C.12
771 ns = (ns + 7) & ~7;
772 ns = (ns + align - 1) & -align;
773
774 // C.13
775 if ((type[i]->t & VT_BTYPE) == VT_STRUCT) {
776 a[i] = ns;
777 ns += size;
778 continue;
779 }
780
781 // C.14
782 if (size < 8)
783 size = 8;
784
785 // C.15
786 a[i] = ns;
787 ns += size;
788 }
789
790 return ns - 32;
791 }
792
793 static unsigned long arm64_pcs(int n, CType **type, unsigned long *a)
794 {
795 unsigned long stack;
796
797 // Return type:
798 if ((type[0]->t & VT_BTYPE) == VT_VOID)
799 a[0] = -1;
800 else {
801 arm64_pcs_aux(1, type, a);
802 assert(a[0] == 0 || a[0] == 1 || a[0] == 16);
803 }
804
805 // Argument types:
806 stack = arm64_pcs_aux(n, type + 1, a + 1);
807
808 if (0) {
809 int i;
810 for (i = 0; i <= n; i++) {
811 if (!i)
812 printf("arm64_pcs return: ");
813 else
814 printf("arm64_pcs arg %d: ", i);
815 if (a[i] == (unsigned long)-1)
816 printf("void\n");
817 else if (a[i] == 1 && !i)
818 printf("X8 pointer\n");
819 else if (a[i] < 16)
820 printf("X%lu%s\n", a[i] / 2, a[i] & 1 ? " pointer" : "");
821 else if (a[i] < 32)
822 printf("V%lu\n", a[i] / 2 - 8);
823 else
824 printf("stack %lu%s\n",
825 (a[i] - 32) & ~1, a[i] & 1 ? " pointer" : "");
826 }
827 }
828
829 return stack;
830 }
831
832 ST_FUNC void gfunc_call(int nb_args)
833 {
834 CType *return_type;
835 CType **t;
836 unsigned long *a, *a1;
837 unsigned long stack;
838 int i;
839
840 return_type = &vtop[-nb_args].type.ref->type;
841 if ((return_type->t & VT_BTYPE) == VT_STRUCT)
842 --nb_args;
843
844 t = tcc_malloc((nb_args + 1) * sizeof(*t));
845 a = tcc_malloc((nb_args + 1) * sizeof(*a));
846 a1 = tcc_malloc((nb_args + 1) * sizeof(*a1));
847
848 t[0] = return_type;
849 for (i = 0; i < nb_args; i++)
850 t[nb_args - i] = &vtop[-i].type;
851
852 stack = arm64_pcs(nb_args, t, a);
853
854 // Allocate space for structs replaced by pointer:
855 for (i = nb_args; i; i--)
856 if (a[i] & 1) {
857 SValue *arg = &vtop[i - nb_args];
858 int align, size = type_size(&arg->type, &align);
859 assert((arg->type.t & VT_BTYPE) == VT_STRUCT);
860 stack = (stack + align - 1) & -align;
861 a1[i] = stack;
862 stack += size;
863 }
864
865 stack = (stack + 15) >> 4 << 4;
866
867 assert(stack < 0x1000);
868 if (stack)
869 o(0xd10003ff | stack << 10); // sub sp,sp,#(n)
870
871 // First pass: set all values on stack
872 for (i = nb_args; i; i--) {
873 vpushv(vtop - nb_args + i);
874
875 if (a[i] & 1) {
876 // struct replaced by pointer
877 int r = get_reg(RC_INT);
878 arm64_spoff(intr(r), a1[i]);
879 vset(&vtop->type, r | VT_LVAL, 0);
880 vswap();
881 vstore();
882 if (a[i] >= 32) {
883 // pointer on stack
884 r = get_reg(RC_INT);
885 arm64_spoff(intr(r), a1[i]);
886 arm64_strx(3, intr(r), 31, (a[i] - 32) >> 1 << 1);
887 }
888 }
889 else if (a[i] >= 32) {
890 // value on stack
891 if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
892 int r = get_reg(RC_INT);
893 arm64_spoff(intr(r), a[i] - 32);
894 vset(&vtop->type, r | VT_LVAL, 0);
895 vswap();
896 vstore();
897 }
898 else if (is_float(vtop->type.t)) {
899 gv(RC_FLOAT);
900 arm64_strv(arm64_type_size(vtop[0].type.t),
901 fltr(vtop[0].r), 31, a[i] - 32);
902 }
903 else {
904 gv(RC_INT);
905 arm64_strx(arm64_type_size(vtop[0].type.t),
906 intr(vtop[0].r), 31, a[i] - 32);
907 }
908 }
909
910 --vtop;
911 }
912
913 // Second pass: assign values to registers
914 for (i = nb_args; i; i--, vtop--) {
915 if (a[i] < 16 && !(a[i] & 1)) {
916 // value in general-purpose registers
917 if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
918 int align, size = type_size(&vtop->type, &align);
919 vtop->type.t = VT_PTR;
920 gaddrof();
921 gv(RC_R(a[i] / 2));
922 arm64_ldrs(a[i] / 2, size);
923 }
924 else
925 gv(RC_R(a[i] / 2));
926 }
927 else if (a[i] < 16)
928 // struct replaced by pointer in register
929 arm64_spoff(a[i] / 2, a1[i]);
930 else if (a[i] < 32) {
931 // value in floating-point registers
932 if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
933 int j, sz, n = arm64_hfa(&vtop->type, &sz);
934 vtop->type.t = VT_PTR;
935 gaddrof();
936 gv(RC_R30);
937 for (j = 0; j < n; j++)
938 o(0x3d4003c0 |
939 (sz & 16) << 19 | -(sz & 8) << 27 | (sz & 4) << 29 |
940 (a[i] / 2 - 8 + j) |
941 j << 10); // ldr ([sdq])(*),[x30,#(j * sz)]
942 }
943 else
944 gv(RC_F(a[i] / 2 - 8));
945 }
946 }
947
948 if ((return_type->t & VT_BTYPE) == VT_STRUCT) {
949 if (a[0] == 1) {
950 // indirect return: set x8 and discard the stack value
951 gv(RC_R(8));
952 --vtop;
953 }
954 else
955 // return in registers: keep the address for after the call
956 vswap();
957 }
958
959 save_regs(0);
960 arm64_gen_bl_or_b(0);
961 --vtop;
962 if (stack)
963 o(0x910003ff | stack << 10); // add sp,sp,#(n)
964
965 {
966 int rt = return_type->t;
967 int bt = rt & VT_BTYPE;
968 if (bt == VT_BYTE || bt == VT_SHORT)
969 // Promote small integers:
970 o(0x13001c00 | (bt == VT_SHORT) << 13 |
971 !!(rt & VT_UNSIGNED) << 30); // [su]xt[bh] w0,w0
972 else if (bt == VT_STRUCT && !(a[0] & 1)) {
973 // A struct was returned in registers, so write it out:
974 gv(RC_R(8));
975 --vtop;
976 if (a[0] == 0) {
977 int align, size = type_size(return_type, &align);
978 assert(size <= 16);
979 if (size > 8)
980 o(0xa9000500); // stp x0,x1,[x8]
981 else if (size)
982 arm64_strx(size > 4 ? 3 : size > 2 ? 2 : size > 1, 0, 8, 0);
983
984 }
985 else if (a[0] == 16) {
986 int j, sz, n = arm64_hfa(return_type, &sz);
987 for (j = 0; j < n; j++)
988 o(0x3d000100 |
989 (sz & 16) << 19 | -(sz & 8) << 27 | (sz & 4) << 29 |
990 (a[i] / 2 - 8 + j) |
991 j << 10); // str ([sdq])(*),[x8,#(j * sz)]
992 }
993 }
994 }
995
996 tcc_free(a1);
997 tcc_free(a);
998 tcc_free(t);
999 }
1000
1001 static unsigned long arm64_func_va_list_stack;
1002 static int arm64_func_va_list_gr_offs;
1003 static int arm64_func_va_list_vr_offs;
1004 static int arm64_func_sub_sp_offset;
1005
1006 ST_FUNC void gfunc_prolog(CType *func_type)
1007 {
1008 int n = 0;
1009 int i = 0;
1010 Sym *sym;
1011 CType **t;
1012 unsigned long *a;
1013
1014 // Why doesn't the caller (gen_function) set func_vt?
1015 func_vt = func_type->ref->type;
1016 func_vc = 144; // offset of where x8 is stored
1017
1018 for (sym = func_type->ref; sym; sym = sym->next)
1019 ++n;
1020 t = tcc_malloc(n * sizeof(*t));
1021 a = tcc_malloc(n * sizeof(*a));
1022
1023 for (sym = func_type->ref; sym; sym = sym->next)
1024 t[i++] = &sym->type;
1025
1026 arm64_func_va_list_stack = arm64_pcs(n - 1, t, a);
1027
1028 o(0xa9b27bfd); // stp x29,x30,[sp,#-224]!
1029 o(0xad0087e0); // stp q0,q1,[sp,#16]
1030 o(0xad018fe2); // stp q2,q3,[sp,#48]
1031 o(0xad0297e4); // stp q4,q5,[sp,#80]
1032 o(0xad039fe6); // stp q6,q7,[sp,#112]
1033 o(0xa90923e8); // stp x8,x8,[sp,#144]
1034 o(0xa90a07e0); // stp x0,x1,[sp,#160]
1035 o(0xa90b0fe2); // stp x2,x3,[sp,#176]
1036 o(0xa90c17e4); // stp x4,x5,[sp,#192]
1037 o(0xa90d1fe6); // stp x6,x7,[sp,#208]
1038
1039 arm64_func_va_list_gr_offs = -64;
1040 arm64_func_va_list_vr_offs = -128;
1041
1042 for (i = 1, sym = func_type->ref->next; sym; i++, sym = sym->next) {
1043 int off = (a[i] < 16 ? 160 + a[i] / 2 * 8 :
1044 a[i] < 32 ? 16 + (a[i] - 16) / 2 * 16 :
1045 224 + ((a[i] - 32) >> 1 << 1));
1046 sym_push(sym->v & ~SYM_FIELD, &sym->type,
1047 (a[i] & 1 ? VT_LLOCAL : VT_LOCAL) | lvalue_type(sym->type.t),
1048 off);
1049
1050 if (a[i] < 16) {
1051 int align, size = type_size(&sym->type, &align);
1052 arm64_func_va_list_gr_offs = (a[i] / 2 - 7 +
1053 (!(a[i] & 1) && size > 8)) * 8;
1054 }
1055 else if (a[i] < 32) {
1056 int hfa = arm64_hfa(&sym->type, 0);
1057 arm64_func_va_list_vr_offs = (a[i] / 2 - 16 +
1058 (hfa ? hfa : 1)) * 16;
1059 }
1060
1061 // HFAs of float and double need to be written differently:
1062 if (16 <= a[i] && a[i] < 32 && (sym->type.t & VT_BTYPE) == VT_STRUCT) {
1063 int j, sz, k = arm64_hfa(&sym->type, &sz);
1064 if (sz < 16)
1065 for (j = 0; j < k; j++) {
1066 o(0x3d0003e0 | -(sz & 8) << 27 | (sz & 4) << 29 |
1067 ((a[i] - 16) / 2 + j) | (off / sz + j) << 10);
1068 // str ([sdq])(*),[sp,#(j * sz)]
1069 }
1070 }
1071 }
1072
1073 tcc_free(a);
1074 tcc_free(t);
1075
1076 o(0x910003fd); // mov x29,sp
1077 arm64_func_sub_sp_offset = ind;
1078 // In gfunc_epilog these will be replaced with code to decrement SP:
1079 o(0xd503201f); // nop
1080 o(0xd503201f); // nop
1081 loc = 0;
1082 }
1083
1084 ST_FUNC void gen_va_start(void)
1085 {
1086 int r;
1087 --vtop; // we don't need the "arg"
1088 gaddrof();
1089 r = intr(gv(RC_INT));
1090
1091 if (arm64_func_va_list_stack) {
1092 //xx could use add (immediate) here
1093 arm64_movimm(30, arm64_func_va_list_stack + 224);
1094 o(0x8b1e03be); // add x30,x29,x30
1095 }
1096 else
1097 o(0x910383be); // add x30,x29,#224
1098 o(0xf900001e | r << 5); // str x30,[x(r)]
1099
1100 if (arm64_func_va_list_gr_offs) {
1101 if (arm64_func_va_list_stack)
1102 o(0x910383be); // add x30,x29,#224
1103 o(0xf900041e | r << 5); // str x30,[x(r),#8]
1104 }
1105
1106 if (arm64_func_va_list_vr_offs) {
1107 o(0x910243be); // add x30,x29,#144
1108 o(0xf900081e | r << 5); // str x30,[x(r),#16]
1109 }
1110
1111 arm64_movimm(30, arm64_func_va_list_gr_offs);
1112 o(0xb900181e | r << 5); // str w30,[x(r),#24]
1113
1114 arm64_movimm(30, arm64_func_va_list_vr_offs);
1115 o(0xb9001c1e | r << 5); // str w30,[x(r),#28]
1116
1117 --vtop;
1118 }
1119
1120 ST_FUNC void gen_va_arg(CType *t)
1121 {
1122 int align, size = type_size(t, &align);
1123 int fsize, hfa = arm64_hfa(t, &fsize);
1124 uint32_t r0, r1;
1125
1126 if (is_float(t->t)) {
1127 hfa = 1;
1128 fsize = size;
1129 }
1130
1131 gaddrof();
1132 r0 = intr(gv(RC_INT));
1133 r1 = get_reg(RC_INT);
1134 vtop[0].r = r1 | lvalue_type(t->t);
1135 r1 = intr(r1);
1136
1137 if (!hfa) {
1138 uint32_t n = size > 16 ? 8 : (size + 7) & -8;
1139 o(0xb940181e | r0 << 5); // ldr w30,[x(r0),#24] // __gr_offs
1140 if (align == 16) {
1141 assert(0); // this path untested but needed for __uint128_t
1142 o(0x11003fde); // add w30,w30,#15
1143 o(0x121c6fde); // and w30,w30,#-16
1144 }
1145 o(0x310003c0 | r1 | n << 10); // adds w(r1),w30,#(n)
1146 o(0x540000ad); // b.le .+20
1147 o(0xf9400000 | r1 | r0 << 5); // ldr x(r1),[x(r0)] // __stack
1148 o(0x9100001e | r1 << 5 | n << 10); // add x30,x(r1),#(n)
1149 o(0xf900001e | r0 << 5); // str x30,[x(r0)] // __stack
1150 o(0x14000004); // b .+16
1151 o(0xb9001800 | r1 | r0 << 5); // str w(r1),[x(r0),#24] // __gr_offs
1152 o(0xf9400400 | r1 | r0 << 5); // ldr x(r1),[x(r0),#8] // __gr_top
1153 o(0x8b3ec000 | r1 | r1 << 5); // add x(r1),x(r1),w30,sxtw
1154 if (size > 16)
1155 o(0xf9400000 | r1 | r1 << 5); // ldr x(r1),[x(r1)]
1156 }
1157 else {
1158 uint32_t rsz = hfa << 4;
1159 uint32_t ssz = (size + 7) & -(uint32_t)8;
1160 uint32_t b1, b2;
1161 o(0xb9401c1e | r0 << 5); // ldr w30,[x(r0),#28] // __vr_offs
1162 o(0x310003c0 | r1 | rsz << 10); // adds w(r1),w30,#(rsz)
1163 b1 = ind; o(0x5400000d); // b.le lab1
1164 o(0xf9400000 | r1 | r0 << 5); // ldr x(r1),[x(r0)] // __stack
1165 if (fsize == 16) {
1166 o(0x91003c00 | r1 | r1 << 5); // add x(r1),x(r1),#15
1167 o(0x927cec00 | r1 | r1 << 5); // and x(r1),x(r1),#-16
1168 }
1169 o(0x9100001e | r1 << 5 | ssz << 10); // add x30,x(r1),#(ssz)
1170 o(0xf900001e | r0 << 5); // str x30,[x(r0)] // __stack
1171 b2 = ind; o(0x14000000); // b lab2
1172 // lab1:
1173 *(uint32_t *)(cur_text_section->data + b1) =
1174 (0x5400000d | (ind - b1) << 3);
1175 o(0xb9001c00 | r1 | r0 << 5); // str w(r1),[x(r0),#28] // __vr_offs
1176 o(0xf9400800 | r1 | r0 << 5); // ldr x(r1),[x(r0),#16] // __vr_top
1177 if (hfa == 1 || fsize == 16)
1178 o(0x8b3ec000 | r1 | r1 << 5); // add x(r1),x(r1),w30,sxtw
1179 else {
1180 // We need to change the layout of this HFA.
1181 // Get some space on the stack using global variable "loc":
1182 loc = (loc - size) & -(uint32_t)align;
1183 o(0x8b3ec000 | 30 | r1 << 5); // add x30,x(r1),w30,sxtw
1184 arm64_movimm(r1, loc);
1185 o(0x8b0003a0 | r1 | r1 << 16); // add x(r1),x29,x(r1)
1186 o(0x4c402bdc | (uint32_t)fsize << 7 |
1187 (uint32_t)(hfa == 2) << 15 |
1188 (uint32_t)(hfa == 3) << 14); // ld1 {v28.(4s|2d),...},[x30]
1189 o(0x0d00801c | r1 << 5 | (fsize == 8) << 10 |
1190 (uint32_t)(hfa != 2) << 13 |
1191 (uint32_t)(hfa != 3) << 21); // st(hfa) {v28.(s|d),...}[0],[x(r1)]
1192 }
1193 // lab2:
1194 *(uint32_t *)(cur_text_section->data + b2) =
1195 (0x14000000 | (ind - b2) >> 2);
1196 }
1197 }
1198
1199 ST_FUNC int gfunc_sret(CType *vt, int variadic, CType *ret, int *align)
1200 {
1201 return 0;
1202 }
1203
1204 ST_FUNC void greturn(void)
1205 {
1206 CType *t = &func_vt;
1207 unsigned long a;
1208
1209 arm64_pcs(0, &t, &a);
1210 switch (a) {
1211 case -1:
1212 break;
1213 case 0:
1214 if ((func_vt.t & VT_BTYPE) == VT_STRUCT) {
1215 int align, size = type_size(&func_vt, &align);
1216 gaddrof();
1217 gv(RC_R(0));
1218 arm64_ldrs(0, size);
1219 }
1220 else
1221 gv(RC_IRET);
1222 break;
1223 case 1: {
1224 CType type = func_vt;
1225 mk_pointer(&type);
1226 vset(&type, VT_LOCAL | VT_LVAL, func_vc);
1227 indir();
1228 vswap();
1229 vstore();
1230 break;
1231 }
1232 case 16:
1233 if ((func_vt.t & VT_BTYPE) == VT_STRUCT) {
1234 int j, sz, n = arm64_hfa(&vtop->type, &sz);
1235 gaddrof();
1236 gv(RC_R(0));
1237 for (j = 0; j < n; j++)
1238 o(0x3d400000 |
1239 (sz & 16) << 19 | -(sz & 8) << 27 | (sz & 4) << 29 |
1240 j | j << 10); // ldr ([sdq])(*),[x0,#(j * sz)]
1241 }
1242 else
1243 gv(RC_FRET);
1244 break;
1245 default:
1246 assert(0);
1247 }
1248 }
1249
1250 ST_FUNC void gfunc_epilog(void)
1251 {
1252 if (loc) {
1253 // Insert instructions to subtract size of stack frame from SP.
1254 uint32_t *ptr =
1255 (uint32_t *)(cur_text_section->data + arm64_func_sub_sp_offset);
1256 uint64_t diff = (-loc + 15) & ~15;
1257 if (!(diff >> 24)) {
1258 if (diff & 0xfff) // sub sp,sp,#(diff & 0xfff)
1259 ptr[0] = 0xd10003ff | (diff & 0xfff) << 10;
1260 if (diff >> 12) // sub sp,sp,#(diff >> 12),lsl #12
1261 ptr[1] = 0xd14003ff | (diff >> 12) << 10;
1262 }
1263 else {
1264 // In this case we may subtract more than necessary,
1265 // but always less than 17/16 of what we were aiming for.
1266 int i = 0;
1267 int j = 0;
1268 while (diff >> 20) {
1269 diff = (diff + 0xffff) >> 16;
1270 ++i;
1271 }
1272 while (diff >> 16) {
1273 diff = (diff + 1) >> 1;
1274 ++j;
1275 }
1276 ptr[0] = 0xd2800010 | diff << 5 | i << 21;
1277 // mov x16,#(diff),lsl #(16 * i)
1278 ptr[1] = 0xcb3063ff | j << 10;
1279 // sub sp,sp,x16,lsl #(j)
1280 }
1281 }
1282 o(0x910003bf); // mov sp,x29
1283 o(0xa8ce7bfd); // ldp x29,x30,[sp],#224
1284
1285 o(0xd65f03c0); // ret
1286 }
1287
1288 // Generate forward branch to label:
1289 ST_FUNC int gjmp(int t)
1290 {
1291 int r = ind;
1292 o(t);
1293 return r;
1294 }
1295
1296 // Generate branch to known address:
1297 ST_FUNC void gjmp_addr(int a)
1298 {
1299 assert(a - ind + 0x8000000 < 0x10000000);
1300 o(0x14000000 | ((a - ind) >> 2 & 0x3ffffff));
1301 }
1302
1303 ST_FUNC int gtst(int inv, int t)
1304 {
1305 int bt = vtop->type.t & VT_BTYPE;
1306 if (bt == VT_LDOUBLE) {
1307 int a, b, f = fltr(gv(RC_FLOAT));
1308 a = get_reg(RC_INT);
1309 vpushi(0);
1310 vtop[0].r = a;
1311 b = get_reg(RC_INT);
1312 a = intr(a);
1313 b = intr(b);
1314 o(0x4e083c00 | a | f << 5); // mov x(a),v(f).d[0]
1315 o(0x4e183c00 | b | f << 5); // mov x(b),v(f).d[1]
1316 o(0xaa000400 | a | a << 5 | b << 16); // orr x(a),x(a),x(b),lsl #1
1317 o(0xb4000040 | a | !!inv << 24); // cbz/cbnz x(a),.+8
1318 --vtop;
1319 }
1320 else if (bt == VT_FLOAT || bt == VT_DOUBLE) {
1321 int a = fltr(gv(RC_FLOAT));
1322 o(0x1e202008 | a << 5 | (bt != VT_FLOAT) << 22); // fcmp
1323 o(0x54000040 | !!inv); // b.eq/b.ne .+8
1324 }
1325 else {
1326 int ll = (bt == VT_PTR || bt == VT_LLONG);
1327 int a = intr(gv(RC_INT));
1328 o(0x34000040 | a | !!inv << 24 | ll << 31); // cbz/cbnz wA,.+8
1329 }
1330 --vtop;
1331 return gjmp(t);
1332 }
1333
1334 static void arm64_gen_opil(int op, int l)
1335 {
1336 int x, a, b;
1337 gv2(RC_INT, RC_INT);
1338 assert(vtop[-1].r < VT_CONST && vtop[0].r < VT_CONST);
1339 a = intr(vtop[-1].r);
1340 b = intr(vtop[0].r);
1341 vtop -= 2;
1342 x = get_reg(RC_INT);
1343 ++vtop;
1344 vtop[0].r = x;
1345 x = intr(x);
1346
1347 switch (op) {
1348 case '%':
1349 // Use x30 for quotient:
1350 o(0x1ac00c00 | l << 31 | 30 | a << 5 | b << 16); // sdiv
1351 o(0x1b008000 | l << 31 | x | 30 << 5 | b << 16 | a << 10); // msub
1352 break;
1353 case '&':
1354 o(0x0a000000 | l << 31 | x | a << 5 | b << 16); // and
1355 break;
1356 case '*':
1357 o(0x1b007c00 | l << 31 | x | a << 5 | b << 16); // mul
1358 break;
1359 case '+':
1360 o(0x0b000000 | l << 31 | x | a << 5 | b << 16); // add
1361 break;
1362 case '-':
1363 o(0x4b000000 | l << 31 | x | a << 5 | b << 16); // sub
1364 break;
1365 case '/':
1366 o(0x1ac00c00 | l << 31 | x | a << 5 | b << 16); // sdiv
1367 break;
1368 case '^':
1369 o(0x4a000000 | l << 31 | x | a << 5 | b << 16); // eor
1370 break;
1371 case '|':
1372 o(0x2a000000 | l << 31 | x | a << 5 | b << 16); // orr
1373 break;
1374 case TOK_EQ:
1375 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1376 o(0x1a9f17e0 | x); // cset wA,eq
1377 break;
1378 case TOK_GE:
1379 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1380 o(0x1a9fb7e0 | x); // cset wA,ge
1381 break;
1382 case TOK_GT:
1383 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1384 o(0x1a9fd7e0 | x); // cset wA,gt
1385 break;
1386 case TOK_LE:
1387 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1388 o(0x1a9fc7e0 | x); // cset wA,le
1389 break;
1390 case TOK_LT:
1391 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1392 o(0x1a9fa7e0 | x); // cset wA,lt
1393 break;
1394 case TOK_NE:
1395 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1396 o(0x1a9f07e0 | x); // cset wA,ne
1397 break;
1398 case TOK_SAR:
1399 o(0x1ac02800 | l << 31 | x | a << 5 | b << 16); // asr
1400 break;
1401 case TOK_SHL:
1402 o(0x1ac02000 | l << 31 | x | a << 5 | b << 16); // lsl
1403 break;
1404 case TOK_SHR:
1405 o(0x1ac02400 | l << 31 | x | a << 5 | b << 16); // lsr
1406 break;
1407 case TOK_UDIV:
1408 case TOK_PDIV:
1409 o(0x1ac00800 | l << 31 | x | a << 5 | b << 16); // udiv
1410 break;
1411 case TOK_UGE:
1412 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1413 o(0x1a9f37e0 | x); // cset wA,cs
1414 break;
1415 case TOK_UGT:
1416 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1417 o(0x1a9f97e0 | x); // cset wA,hi
1418 break;
1419 case TOK_ULT:
1420 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1421 o(0x1a9f27e0 | x); // cset wA,cc
1422 break;
1423 case TOK_ULE:
1424 o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
1425 o(0x1a9f87e0 | x); // cset wA,ls
1426 break;
1427 case TOK_UMOD:
1428 // Use x30 for quotient:
1429 o(0x1ac00800 | l << 31 | 30 | a << 5 | b << 16); // udiv
1430 o(0x1b008000 | l << 31 | x | 30 << 5 | b << 16 | a << 10); // msub
1431 break;
1432 default:
1433 assert(0);
1434 }
1435 }
1436
1437 ST_FUNC void gen_opi(int op)
1438 {
1439 arm64_gen_opil(op, 0);
1440 }
1441
1442 ST_FUNC void gen_opl(int op)
1443 {
1444 arm64_gen_opil(op, 1);
1445 }
1446
1447 ST_FUNC void gen_opf(int op)
1448 {
1449 int x, a, b, dbl;
1450
1451 if (vtop[0].type.t == VT_LDOUBLE) {
1452 CType type = vtop[0].type;
1453 int func = 0;
1454 int cond = -1;
1455 switch (op) {
1456 case '*': func = TOK___multf3; break;
1457 case '+': func = TOK___addtf3; break;
1458 case '-': func = TOK___subtf3; break;
1459 case '/': func = TOK___divtf3; break;
1460 case TOK_EQ: func = TOK___eqtf2; cond = 1; break;
1461 case TOK_NE: func = TOK___netf2; cond = 0; break;
1462 case TOK_LT: func = TOK___lttf2; cond = 10; break;
1463 case TOK_GE: func = TOK___getf2; cond = 11; break;
1464 case TOK_LE: func = TOK___letf2; cond = 12; break;
1465 case TOK_GT: func = TOK___gttf2; cond = 13; break;
1466 default: assert(0); break;
1467 }
1468 vpush_global_sym(&func_old_type, func);
1469 vrott(3);
1470 gfunc_call(2);
1471 vpushi(0);
1472 vtop->r = cond < 0 ? REG_FRET : REG_IRET;
1473 if (cond < 0)
1474 vtop->type = type;
1475 else {
1476 o(0x7100001f); // cmp w0,#0
1477 o(0x1a9f07e0 | cond << 12); // cset w0,(cond)
1478 }
1479 return;
1480 }
1481
1482 dbl = vtop[0].type.t != VT_FLOAT;
1483 gv2(RC_FLOAT, RC_FLOAT);
1484 assert(vtop[-1].r < VT_CONST && vtop[0].r < VT_CONST);
1485 a = fltr(vtop[-1].r);
1486 b = fltr(vtop[0].r);
1487 vtop -= 2;
1488 switch (op) {
1489 case TOK_EQ: case TOK_NE:
1490 case TOK_LT: case TOK_GE: case TOK_LE: case TOK_GT:
1491 x = get_reg(RC_INT);
1492 ++vtop;
1493 vtop[0].r = x;
1494 x = intr(x);
1495 break;
1496 default:
1497 x = get_reg(RC_FLOAT);
1498 ++vtop;
1499 vtop[0].r = x;
1500 x = fltr(x);
1501 break;
1502 }
1503
1504 switch (op) {
1505 case '*':
1506 o(0x1e200800 | dbl << 22 | x | a << 5 | b << 16); // fmul
1507 break;
1508 case '+':
1509 o(0x1e202800 | dbl << 22 | x | a << 5 | b << 16); // fadd
1510 break;
1511 case '-':
1512 o(0x1e203800 | dbl << 22 | x | a << 5 | b << 16); // fsub
1513 break;
1514 case '/':
1515 o(0x1e201800 | dbl << 22 | x | a << 5 | b << 16); // fdiv
1516 break;
1517 case TOK_EQ:
1518 o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
1519 o(0x1a9f17e0 | x); // cset w(x),eq
1520 break;
1521 case TOK_GE:
1522 o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
1523 o(0x1a9fb7e0 | x); // cset w(x),ge
1524 break;
1525 case TOK_GT:
1526 o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
1527 o(0x1a9fd7e0 | x); // cset w(x),gt
1528 break;
1529 case TOK_LE:
1530 o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
1531 o(0x1a9f87e0 | x); // cset w(x),ls
1532 break;
1533 case TOK_LT:
1534 o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
1535 o(0x1a9f57e0 | x); // cset w(x),mi
1536 break;
1537 case TOK_NE:
1538 o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
1539 o(0x1a9f07e0 | x); // cset w(x),ne
1540 break;
1541 default:
1542 assert(0);
1543 }
1544 }
1545
1546 // Generate sign extension from 32 to 64 bits:
1547 ST_FUNC void gen_cvt_sxtw(void)
1548 {
1549 int r = intr(gv(RC_INT));
1550 o(0x93407c00 | r | r << 5); // sxtw x(r),w(r)
1551 }
1552
1553 ST_FUNC void gen_cvt_itof(int t)
1554 {
1555 if (t == VT_LDOUBLE) {
1556 int f = vtop->type.t;
1557 int func = (f & VT_BTYPE) == VT_LLONG ?
1558 (f & VT_UNSIGNED ? TOK___floatunditf : TOK___floatditf) :
1559 (f & VT_UNSIGNED ? TOK___floatunsitf : TOK___floatsitf);
1560 vpush_global_sym(&func_old_type, func);
1561 vrott(2);
1562 gfunc_call(1);
1563 vpushi(0);
1564 vtop->type.t = t;
1565 vtop->r = REG_FRET;
1566 return;
1567 }
1568 else {
1569 int d, n = intr(gv(RC_INT));
1570 int s = !(vtop->type.t & VT_UNSIGNED);
1571 int l = ((vtop->type.t & VT_BTYPE) == VT_LLONG);
1572 --vtop;
1573 d = get_reg(RC_FLOAT);
1574 ++vtop;
1575 vtop[0].r = d;
1576 o(0x1e220000 | !s << 16 | (t != VT_FLOAT) << 22 | fltr(d) |
1577 l << 31 | n << 5); // [us]cvtf [sd](d),[wx](n)
1578 }
1579 }
1580
1581 ST_FUNC void gen_cvt_ftoi(int t)
1582 {
1583 if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
1584 int func = (t & VT_BTYPE) == VT_LLONG ?
1585 (t & VT_UNSIGNED ? TOK___fixunstfdi : TOK___fixtfdi) :
1586 (t & VT_UNSIGNED ? TOK___fixunstfsi : TOK___fixtfsi);
1587 vpush_global_sym(&func_old_type, func);
1588 vrott(2);
1589 gfunc_call(1);
1590 vpushi(0);
1591 vtop->type.t = t;
1592 vtop->r = REG_IRET;
1593 return;
1594 }
1595 else {
1596 int d, n = fltr(gv(RC_FLOAT));
1597 int l = ((vtop->type.t & VT_BTYPE) != VT_FLOAT);
1598 --vtop;
1599 d = get_reg(RC_INT);
1600 ++vtop;
1601 vtop[0].r = d;
1602 o(0x1e380000 |
1603 !!(t & VT_UNSIGNED) << 16 |
1604 ((t & VT_BTYPE) == VT_LLONG) << 31 | intr(d) |
1605 l << 22 | n << 5); // fcvtz[su] [wx](d),[sd](n)
1606 }
1607 }
1608
1609 ST_FUNC void gen_cvt_ftof(int t)
1610 {
1611 int f = vtop[0].type.t;
1612 assert(t == VT_FLOAT || t == VT_DOUBLE || t == VT_LDOUBLE);
1613 assert(f == VT_FLOAT || f == VT_DOUBLE || f == VT_LDOUBLE);
1614 if (t == f)
1615 return;
1616
1617 if (t == VT_LDOUBLE || f == VT_LDOUBLE) {
1618 int func = (t == VT_LDOUBLE) ?
1619 (f == VT_FLOAT ? TOK___extendsftf2 : TOK___extenddftf2) :
1620 (t == VT_FLOAT ? TOK___trunctfsf2 : TOK___trunctfdf2);
1621 vpush_global_sym(&func_old_type, func);
1622 vrott(2);
1623 gfunc_call(1);
1624 vpushi(0);
1625 vtop->type.t = t;
1626 vtop->r = REG_FRET;
1627 }
1628 else {
1629 int x, a;
1630 gv(RC_FLOAT);
1631 assert(vtop[0].r < VT_CONST);
1632 a = fltr(vtop[0].r);
1633 --vtop;
1634 x = get_reg(RC_FLOAT);
1635 ++vtop;
1636 vtop[0].r = x;
1637 x = fltr(x);
1638
1639 if (f == VT_FLOAT)
1640 o(0x1e22c000 | x | a << 5); // fcvt d(x),s(a)
1641 else
1642 o(0x1e624000 | x | a << 5); // fcvt s(x),d(a)
1643 }
1644 }
1645
1646 ST_FUNC void ggoto(void)
1647 {
1648 arm64_gen_bl_or_b(1);
1649 --vtop;
1650 }
1651
1652 ST_FUNC void gen_vla_sp_save(int addr) {
1653 tcc_error("variable length arrays unsupported for this target");
1654 }
1655
1656 ST_FUNC void gen_vla_sp_restore(int addr) {
1657 tcc_error("variable length arrays unsupported for this target");
1658 }
1659
1660 ST_FUNC void gen_vla_alloc(CType *type, int align) {
1661 tcc_error("variable length arrays unsupported for this target");
1662 }
1663
1664 /* end of A64 code generator */
1665 /*************************************************************/
1666 #endif
1667 /*************************************************************/
Tiny C Compiler main repository