?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
acpi: allow using object as offset for OperationRegion
[qemu.git] / tcg / ppc / tcg-target.inc.c
blob8c1c2dfa9b227b1249b81ada09685a689a8f9413
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "tcg-be-ldst.h"
26
27 #if defined _CALL_DARWIN || defined __APPLE__
28 #define TCG_TARGET_CALL_DARWIN
29 #endif
30 #ifdef _CALL_SYSV
31 # define TCG_TARGET_CALL_ALIGN_ARGS 1
32 #endif
33
34 /* For some memory operations, we need a scratch that isn't R0. For the AIX
35 calling convention, we can re-use the TOC register since we'll be reloading
36 it at every call. Otherwise R12 will do nicely as neither a call-saved
37 register nor a parameter register. */
38 #ifdef _CALL_AIX
39 # define TCG_REG_TMP1 TCG_REG_R2
40 #else
41 # define TCG_REG_TMP1 TCG_REG_R12
42 #endif
43
44 /* For the 64-bit target, we don't like the 5 insn sequence needed to build
45 full 64-bit addresses. Better to have a base register to which we can
46 apply a 32-bit displacement.
47
48 There are generally three items of interest:
49 (1) helper functions in the main executable,
50 (2) TranslationBlock data structures,
51 (3) the return address in the epilogue.
52
53 For user-only, we USE_STATIC_CODE_GEN_BUFFER, so the code_gen_buffer
54 will be inside the main executable, and thus near enough to make a
55 pointer to the epilogue be within 2GB of all helper functions.
56
57 For softmmu, we'll let the kernel choose the address of code_gen_buffer,
58 and odds are it'll be somewhere close to the main malloc arena, and so
59 a pointer to the epilogue will be within 2GB of the TranslationBlocks.
60
61 For --enable-pie, everything will be kinda near everything else,
62 somewhere in high memory.
63
64 Thus we choose to keep the return address in a call-saved register. */
65 #define TCG_REG_RA TCG_REG_R31
66 #define USE_REG_RA (TCG_TARGET_REG_BITS == 64)
67
68 /* Shorthand for size of a pointer. Avoid promotion to unsigned. */
69 #define SZP ((int)sizeof(void *))
70
71 /* Shorthand for size of a register. */
72 #define SZR (TCG_TARGET_REG_BITS / 8)
73
74 #define TCG_CT_CONST_S16 0x100
75 #define TCG_CT_CONST_U16 0x200
76 #define TCG_CT_CONST_S32 0x400
77 #define TCG_CT_CONST_U32 0x800
78 #define TCG_CT_CONST_ZERO 0x1000
79 #define TCG_CT_CONST_MONE 0x2000
80
81 static tcg_insn_unit *tb_ret_addr;
82
83 #include "elf.h"
84 static bool have_isa_2_06;
85 #define HAVE_ISA_2_06 have_isa_2_06
86 #define HAVE_ISEL have_isa_2_06
87
88 #ifndef CONFIG_SOFTMMU
89 #define TCG_GUEST_BASE_REG 30
90 #endif
91
92 #ifndef NDEBUG
93 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
94 "r0",
95 "r1",
96 "r2",
97 "r3",
98 "r4",
99 "r5",
100 "r6",
101 "r7",
102 "r8",
103 "r9",
104 "r10",
105 "r11",
106 "r12",
107 "r13",
108 "r14",
109 "r15",
110 "r16",
111 "r17",
112 "r18",
113 "r19",
114 "r20",
115 "r21",
116 "r22",
117 "r23",
118 "r24",
119 "r25",
120 "r26",
121 "r27",
122 "r28",
123 "r29",
124 "r30",
125 "r31"
126 };
127 #endif
128
129 static const int tcg_target_reg_alloc_order[] = {
130 TCG_REG_R14, /* call saved registers */
131 TCG_REG_R15,
132 TCG_REG_R16,
133 TCG_REG_R17,
134 TCG_REG_R18,
135 TCG_REG_R19,
136 TCG_REG_R20,
137 TCG_REG_R21,
138 TCG_REG_R22,
139 TCG_REG_R23,
140 TCG_REG_R24,
141 TCG_REG_R25,
142 TCG_REG_R26,
143 TCG_REG_R27,
144 TCG_REG_R28,
145 TCG_REG_R29,
146 TCG_REG_R30,
147 TCG_REG_R31,
148 TCG_REG_R12, /* call clobbered, non-arguments */
149 TCG_REG_R11,
150 TCG_REG_R2,
151 TCG_REG_R13,
152 TCG_REG_R10, /* call clobbered, arguments */
153 TCG_REG_R9,
154 TCG_REG_R8,
155 TCG_REG_R7,
156 TCG_REG_R6,
157 TCG_REG_R5,
158 TCG_REG_R4,
159 TCG_REG_R3,
160 };
161
162 static const int tcg_target_call_iarg_regs[] = {
163 TCG_REG_R3,
164 TCG_REG_R4,
165 TCG_REG_R5,
166 TCG_REG_R6,
167 TCG_REG_R7,
168 TCG_REG_R8,
169 TCG_REG_R9,
170 TCG_REG_R10
171 };
172
173 static const int tcg_target_call_oarg_regs[] = {
174 TCG_REG_R3,
175 TCG_REG_R4
176 };
177
178 static const int tcg_target_callee_save_regs[] = {
179 #ifdef TCG_TARGET_CALL_DARWIN
180 TCG_REG_R11,
181 #endif
182 TCG_REG_R14,
183 TCG_REG_R15,
184 TCG_REG_R16,
185 TCG_REG_R17,
186 TCG_REG_R18,
187 TCG_REG_R19,
188 TCG_REG_R20,
189 TCG_REG_R21,
190 TCG_REG_R22,
191 TCG_REG_R23,
192 TCG_REG_R24,
193 TCG_REG_R25,
194 TCG_REG_R26,
195 TCG_REG_R27, /* currently used for the global env */
196 TCG_REG_R28,
197 TCG_REG_R29,
198 TCG_REG_R30,
199 TCG_REG_R31
200 };
201
202 static inline bool in_range_b(tcg_target_long target)
203 {
204 return target == sextract64(target, 0, 26);
205 }
206
207 static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target)
208 {
209 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
210 assert(in_range_b(disp));
211 return disp & 0x3fffffc;
212 }
213
214 static void reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
215 {
216 *pc = (*pc & ~0x3fffffc) | reloc_pc24_val(pc, target);
217 }
218
219 static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target)
220 {
221 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
222 assert(disp == (int16_t) disp);
223 return disp & 0xfffc;
224 }
225
226 static void reloc_pc14(tcg_insn_unit *pc, tcg_insn_unit *target)
227 {
228 *pc = (*pc & ~0xfffc) | reloc_pc14_val(pc, target);
229 }
230
231 static inline void tcg_out_b_noaddr(TCGContext *s, int insn)
232 {
233 unsigned retrans = *s->code_ptr & 0x3fffffc;
234 tcg_out32(s, insn | retrans);
235 }
236
237 static inline void tcg_out_bc_noaddr(TCGContext *s, int insn)
238 {
239 unsigned retrans = *s->code_ptr & 0xfffc;
240 tcg_out32(s, insn | retrans);
241 }
242
243 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
244 intptr_t value, intptr_t addend)
245 {
246 tcg_insn_unit *target = (tcg_insn_unit *)value;
247
248 assert(addend == 0);
249 switch (type) {
250 case R_PPC_REL14:
251 reloc_pc14(code_ptr, target);
252 break;
253 case R_PPC_REL24:
254 reloc_pc24(code_ptr, target);
255 break;
256 default:
257 tcg_abort();
258 }
259 }
260
261 /* parse target specific constraints */
262 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
263 {
264 const char *ct_str;
265
266 ct_str = *pct_str;
267 switch (ct_str[0]) {
268 case 'A': case 'B': case 'C': case 'D':
269 ct->ct |= TCG_CT_REG;
270 tcg_regset_set_reg(ct->u.regs, 3 + ct_str[0] - 'A');
271 break;
272 case 'r':
273 ct->ct |= TCG_CT_REG;
274 tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
275 break;
276 case 'L': /* qemu_ld constraint */
277 ct->ct |= TCG_CT_REG;
278 tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
279 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
280 #ifdef CONFIG_SOFTMMU
281 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
282 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
283 #endif
284 break;
285 case 'S': /* qemu_st constraint */
286 ct->ct |= TCG_CT_REG;
287 tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
288 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
289 #ifdef CONFIG_SOFTMMU
290 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
291 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
292 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
293 #endif
294 break;
295 case 'I':
296 ct->ct |= TCG_CT_CONST_S16;
297 break;
298 case 'J':
299 ct->ct |= TCG_CT_CONST_U16;
300 break;
301 case 'M':
302 ct->ct |= TCG_CT_CONST_MONE;
303 break;
304 case 'T':
305 ct->ct |= TCG_CT_CONST_S32;
306 break;
307 case 'U':
308 ct->ct |= TCG_CT_CONST_U32;
309 break;
310 case 'Z':
311 ct->ct |= TCG_CT_CONST_ZERO;
312 break;
313 default:
314 return -1;
315 }
316 ct_str++;
317 *pct_str = ct_str;
318 return 0;
319 }
320
321 /* test if a constant matches the constraint */
322 static int tcg_target_const_match(tcg_target_long val, TCGType type,
323 const TCGArgConstraint *arg_ct)
324 {
325 int ct = arg_ct->ct;
326 if (ct & TCG_CT_CONST) {
327 return 1;
328 }
329
330 /* The only 32-bit constraint we use aside from
331 TCG_CT_CONST is TCG_CT_CONST_S16. */
332 if (type == TCG_TYPE_I32) {
333 val = (int32_t)val;
334 }
335
336 if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
337 return 1;
338 } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
339 return 1;
340 } else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
341 return 1;
342 } else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
343 return 1;
344 } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
345 return 1;
346 } else if ((ct & TCG_CT_CONST_MONE) && val == -1) {
347 return 1;
348 }
349 return 0;
350 }
351
352 #define OPCD(opc) ((opc)<<26)
353 #define XO19(opc) (OPCD(19)|((opc)<<1))
354 #define MD30(opc) (OPCD(30)|((opc)<<2))
355 #define MDS30(opc) (OPCD(30)|((opc)<<1))
356 #define XO31(opc) (OPCD(31)|((opc)<<1))
357 #define XO58(opc) (OPCD(58)|(opc))
358 #define XO62(opc) (OPCD(62)|(opc))
359
360 #define B OPCD( 18)
361 #define BC OPCD( 16)
362 #define LBZ OPCD( 34)
363 #define LHZ OPCD( 40)
364 #define LHA OPCD( 42)
365 #define LWZ OPCD( 32)
366 #define STB OPCD( 38)
367 #define STH OPCD( 44)
368 #define STW OPCD( 36)
369
370 #define STD XO62( 0)
371 #define STDU XO62( 1)
372 #define STDX XO31(149)
373
374 #define LD XO58( 0)
375 #define LDX XO31( 21)
376 #define LDU XO58( 1)
377 #define LWA XO58( 2)
378 #define LWAX XO31(341)
379
380 #define ADDIC OPCD( 12)
381 #define ADDI OPCD( 14)
382 #define ADDIS OPCD( 15)
383 #define ORI OPCD( 24)
384 #define ORIS OPCD( 25)
385 #define XORI OPCD( 26)
386 #define XORIS OPCD( 27)
387 #define ANDI OPCD( 28)
388 #define ANDIS OPCD( 29)
389 #define MULLI OPCD( 7)
390 #define CMPLI OPCD( 10)
391 #define CMPI OPCD( 11)
392 #define SUBFIC OPCD( 8)
393
394 #define LWZU OPCD( 33)
395 #define STWU OPCD( 37)
396
397 #define RLWIMI OPCD( 20)
398 #define RLWINM OPCD( 21)
399 #define RLWNM OPCD( 23)
400
401 #define RLDICL MD30( 0)
402 #define RLDICR MD30( 1)
403 #define RLDIMI MD30( 3)
404 #define RLDCL MDS30( 8)
405
406 #define BCLR XO19( 16)
407 #define BCCTR XO19(528)
408 #define CRAND XO19(257)
409 #define CRANDC XO19(129)
410 #define CRNAND XO19(225)
411 #define CROR XO19(449)
412 #define CRNOR XO19( 33)
413
414 #define EXTSB XO31(954)
415 #define EXTSH XO31(922)
416 #define EXTSW XO31(986)
417 #define ADD XO31(266)
418 #define ADDE XO31(138)
419 #define ADDME XO31(234)
420 #define ADDZE XO31(202)
421 #define ADDC XO31( 10)
422 #define AND XO31( 28)
423 #define SUBF XO31( 40)
424 #define SUBFC XO31( 8)
425 #define SUBFE XO31(136)
426 #define SUBFME XO31(232)
427 #define SUBFZE XO31(200)
428 #define OR XO31(444)
429 #define XOR XO31(316)
430 #define MULLW XO31(235)
431 #define MULHW XO31( 75)
432 #define MULHWU XO31( 11)
433 #define DIVW XO31(491)
434 #define DIVWU XO31(459)
435 #define CMP XO31( 0)
436 #define CMPL XO31( 32)
437 #define LHBRX XO31(790)
438 #define LWBRX XO31(534)
439 #define LDBRX XO31(532)
440 #define STHBRX XO31(918)
441 #define STWBRX XO31(662)
442 #define STDBRX XO31(660)
443 #define MFSPR XO31(339)
444 #define MTSPR XO31(467)
445 #define SRAWI XO31(824)
446 #define NEG XO31(104)
447 #define MFCR XO31( 19)
448 #define MFOCRF (MFCR | (1u << 20))
449 #define NOR XO31(124)
450 #define CNTLZW XO31( 26)
451 #define CNTLZD XO31( 58)
452 #define ANDC XO31( 60)
453 #define ORC XO31(412)
454 #define EQV XO31(284)
455 #define NAND XO31(476)
456 #define ISEL XO31( 15)
457
458 #define MULLD XO31(233)
459 #define MULHD XO31( 73)
460 #define MULHDU XO31( 9)
461 #define DIVD XO31(489)
462 #define DIVDU XO31(457)
463
464 #define LBZX XO31( 87)
465 #define LHZX XO31(279)
466 #define LHAX XO31(343)
467 #define LWZX XO31( 23)
468 #define STBX XO31(215)
469 #define STHX XO31(407)
470 #define STWX XO31(151)
471
472 #define SPR(a, b) ((((a)<<5)|(b))<<11)
473 #define LR SPR(8, 0)
474 #define CTR SPR(9, 0)
475
476 #define SLW XO31( 24)
477 #define SRW XO31(536)
478 #define SRAW XO31(792)
479
480 #define SLD XO31( 27)
481 #define SRD XO31(539)
482 #define SRAD XO31(794)
483 #define SRADI XO31(413<<1)
484
485 #define TW XO31( 4)
486 #define TRAP (TW | TO(31))
487
488 #define NOP ORI /* ori 0,0,0 */
489
490 #define RT(r) ((r)<<21)
491 #define RS(r) ((r)<<21)
492 #define RA(r) ((r)<<16)
493 #define RB(r) ((r)<<11)
494 #define TO(t) ((t)<<21)
495 #define SH(s) ((s)<<11)
496 #define MB(b) ((b)<<6)
497 #define ME(e) ((e)<<1)
498 #define BO(o) ((o)<<21)
499 #define MB64(b) ((b)<<5)
500 #define FXM(b) (1 << (19 - (b)))
501
502 #define LK 1
503
504 #define TAB(t, a, b) (RT(t) | RA(a) | RB(b))
505 #define SAB(s, a, b) (RS(s) | RA(a) | RB(b))
506 #define TAI(s, a, i) (RT(s) | RA(a) | ((i) & 0xffff))
507 #define SAI(s, a, i) (RS(s) | RA(a) | ((i) & 0xffff))
508
509 #define BF(n) ((n)<<23)
510 #define BI(n, c) (((c)+((n)*4))<<16)
511 #define BT(n, c) (((c)+((n)*4))<<21)
512 #define BA(n, c) (((c)+((n)*4))<<16)
513 #define BB(n, c) (((c)+((n)*4))<<11)
514 #define BC_(n, c) (((c)+((n)*4))<<6)
515
516 #define BO_COND_TRUE BO(12)
517 #define BO_COND_FALSE BO( 4)
518 #define BO_ALWAYS BO(20)
519
520 enum {
521 CR_LT,
522 CR_GT,
523 CR_EQ,
524 CR_SO
525 };
526
527 static const uint32_t tcg_to_bc[] = {
528 [TCG_COND_EQ] = BC | BI(7, CR_EQ) | BO_COND_TRUE,
529 [TCG_COND_NE] = BC | BI(7, CR_EQ) | BO_COND_FALSE,
530 [TCG_COND_LT] = BC | BI(7, CR_LT) | BO_COND_TRUE,
531 [TCG_COND_GE] = BC | BI(7, CR_LT) | BO_COND_FALSE,
532 [TCG_COND_LE] = BC | BI(7, CR_GT) | BO_COND_FALSE,
533 [TCG_COND_GT] = BC | BI(7, CR_GT) | BO_COND_TRUE,
534 [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE,
535 [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE,
536 [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE,
537 [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE,
538 };
539
540 /* The low bit here is set if the RA and RB fields must be inverted. */
541 static const uint32_t tcg_to_isel[] = {
542 [TCG_COND_EQ] = ISEL | BC_(7, CR_EQ),
543 [TCG_COND_NE] = ISEL | BC_(7, CR_EQ) | 1,
544 [TCG_COND_LT] = ISEL | BC_(7, CR_LT),
545 [TCG_COND_GE] = ISEL | BC_(7, CR_LT) | 1,
546 [TCG_COND_LE] = ISEL | BC_(7, CR_GT) | 1,
547 [TCG_COND_GT] = ISEL | BC_(7, CR_GT),
548 [TCG_COND_LTU] = ISEL | BC_(7, CR_LT),
549 [TCG_COND_GEU] = ISEL | BC_(7, CR_LT) | 1,
550 [TCG_COND_LEU] = ISEL | BC_(7, CR_GT) | 1,
551 [TCG_COND_GTU] = ISEL | BC_(7, CR_GT),
552 };
553
554 static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
555 TCGReg base, tcg_target_long offset);
556
557 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
558 {
559 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
560 if (ret != arg) {
561 tcg_out32(s, OR | SAB(arg, ret, arg));
562 }
563 }
564
565 static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
566 int sh, int mb)
567 {
568 assert(TCG_TARGET_REG_BITS == 64);
569 sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1);
570 mb = MB64((mb >> 5) | ((mb << 1) & 0x3f));
571 tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb);
572 }
573
574 static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
575 int sh, int mb, int me)
576 {
577 tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me));
578 }
579
580 static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
581 {
582 tcg_out_rld(s, RLDICL, dst, src, 0, 32);
583 }
584
585 static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
586 {
587 tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
588 }
589
590 static inline void tcg_out_shli64(TCGContext *s, TCGReg dst, TCGReg src, int c)
591 {
592 tcg_out_rld(s, RLDICR, dst, src, c, 63 - c);
593 }
594
595 static inline void tcg_out_shri32(TCGContext *s, TCGReg dst, TCGReg src, int c)
596 {
597 tcg_out_rlw(s, RLWINM, dst, src, 32 - c, c, 31);
598 }
599
600 static inline void tcg_out_shri64(TCGContext *s, TCGReg dst, TCGReg src, int c)
601 {
602 tcg_out_rld(s, RLDICL, dst, src, 64 - c, c);
603 }
604
605 static void tcg_out_movi32(TCGContext *s, TCGReg ret, int32_t arg)
606 {
607 if (arg == (int16_t) arg) {
608 tcg_out32(s, ADDI | TAI(ret, 0, arg));
609 } else {
610 tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16));
611 if (arg & 0xffff) {
612 tcg_out32(s, ORI | SAI(ret, ret, arg));
613 }
614 }
615 }
616
617 static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
618 tcg_target_long arg)
619 {
620 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
621 if (type == TCG_TYPE_I32 || arg == (int32_t)arg) {
622 tcg_out_movi32(s, ret, arg);
623 } else if (arg == (uint32_t)arg && !(arg & 0x8000)) {
624 tcg_out32(s, ADDI | TAI(ret, 0, arg));
625 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
626 } else {
627 int32_t high;
628
629 if (USE_REG_RA) {
630 intptr_t diff = arg - (intptr_t)tb_ret_addr;
631 if (diff == (int32_t)diff) {
632 tcg_out_mem_long(s, ADDI, ADD, ret, TCG_REG_RA, diff);
633 return;
634 }
635 }
636
637 high = arg >> 31 >> 1;
638 tcg_out_movi32(s, ret, high);
639 if (high) {
640 tcg_out_shli64(s, ret, ret, 32);
641 }
642 if (arg & 0xffff0000) {
643 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
644 }
645 if (arg & 0xffff) {
646 tcg_out32(s, ORI | SAI(ret, ret, arg));
647 }
648 }
649 }
650
651 static bool mask_operand(uint32_t c, int *mb, int *me)
652 {
653 uint32_t lsb, test;
654
655 /* Accept a bit pattern like:
656 0....01....1
657 1....10....0
658 0..01..10..0
659 Keep track of the transitions. */
660 if (c == 0 || c == -1) {
661 return false;
662 }
663 test = c;
664 lsb = test & -test;
665 test += lsb;
666 if (test & (test - 1)) {
667 return false;
668 }
669
670 *me = clz32(lsb);
671 *mb = test ? clz32(test & -test) + 1 : 0;
672 return true;
673 }
674
675 static bool mask64_operand(uint64_t c, int *mb, int *me)
676 {
677 uint64_t lsb;
678
679 if (c == 0) {
680 return false;
681 }
682
683 lsb = c & -c;
684 /* Accept 1..10..0. */
685 if (c == -lsb) {
686 *mb = 0;
687 *me = clz64(lsb);
688 return true;
689 }
690 /* Accept 0..01..1. */
691 if (lsb == 1 && (c & (c + 1)) == 0) {
692 *mb = clz64(c + 1) + 1;
693 *me = 63;
694 return true;
695 }
696 return false;
697 }
698
699 static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
700 {
701 int mb, me;
702
703 if (mask_operand(c, &mb, &me)) {
704 tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
705 } else if ((c & 0xffff) == c) {
706 tcg_out32(s, ANDI | SAI(src, dst, c));
707 return;
708 } else if ((c & 0xffff0000) == c) {
709 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
710 return;
711 } else {
712 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
713 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
714 }
715 }
716
717 static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
718 {
719 int mb, me;
720
721 assert(TCG_TARGET_REG_BITS == 64);
722 if (mask64_operand(c, &mb, &me)) {
723 if (mb == 0) {
724 tcg_out_rld(s, RLDICR, dst, src, 0, me);
725 } else {
726 tcg_out_rld(s, RLDICL, dst, src, 0, mb);
727 }
728 } else if ((c & 0xffff) == c) {
729 tcg_out32(s, ANDI | SAI(src, dst, c));
730 return;
731 } else if ((c & 0xffff0000) == c) {
732 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
733 return;
734 } else {
735 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
736 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
737 }
738 }
739
740 static void tcg_out_zori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c,
741 int op_lo, int op_hi)
742 {
743 if (c >> 16) {
744 tcg_out32(s, op_hi | SAI(src, dst, c >> 16));
745 src = dst;
746 }
747 if (c & 0xffff) {
748 tcg_out32(s, op_lo | SAI(src, dst, c));
749 src = dst;
750 }
751 }
752
753 static void tcg_out_ori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
754 {
755 tcg_out_zori32(s, dst, src, c, ORI, ORIS);
756 }
757
758 static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
759 {
760 tcg_out_zori32(s, dst, src, c, XORI, XORIS);
761 }
762
763 static void tcg_out_b(TCGContext *s, int mask, tcg_insn_unit *target)
764 {
765 ptrdiff_t disp = tcg_pcrel_diff(s, target);
766 if (in_range_b(disp)) {
767 tcg_out32(s, B | (disp & 0x3fffffc) | mask);
768 } else {
769 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, (uintptr_t)target);
770 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | CTR);
771 tcg_out32(s, BCCTR | BO_ALWAYS | mask);
772 }
773 }
774
775 static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
776 TCGReg base, tcg_target_long offset)
777 {
778 tcg_target_long orig = offset, l0, l1, extra = 0, align = 0;
779 bool is_store = false;
780 TCGReg rs = TCG_REG_TMP1;
781
782 switch (opi) {
783 case LD: case LWA:
784 align = 3;
785 /* FALLTHRU */
786 default:
787 if (rt != TCG_REG_R0) {
788 rs = rt;
789 break;
790 }
791 break;
792 case STD:
793 align = 3;
794 /* FALLTHRU */
795 case STB: case STH: case STW:
796 is_store = true;
797 break;
798 }
799
800 /* For unaligned, or very large offsets, use the indexed form. */
801 if (offset & align || offset != (int32_t)offset) {
802 if (rs == base) {
803 rs = TCG_REG_R0;
804 }
805 tcg_debug_assert(!is_store || rs != rt);
806 tcg_out_movi(s, TCG_TYPE_PTR, rs, orig);
807 tcg_out32(s, opx | TAB(rt, base, rs));
808 return;
809 }
810
811 l0 = (int16_t)offset;
812 offset = (offset - l0) >> 16;
813 l1 = (int16_t)offset;
814
815 if (l1 < 0 && orig >= 0) {
816 extra = 0x4000;
817 l1 = (int16_t)(offset - 0x4000);
818 }
819 if (l1) {
820 tcg_out32(s, ADDIS | TAI(rs, base, l1));
821 base = rs;
822 }
823 if (extra) {
824 tcg_out32(s, ADDIS | TAI(rs, base, extra));
825 base = rs;
826 }
827 if (opi != ADDI || base != rt || l0 != 0) {
828 tcg_out32(s, opi | TAI(rt, base, l0));
829 }
830 }
831
832 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
833 TCGReg arg1, intptr_t arg2)
834 {
835 int opi, opx;
836
837 assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
838 if (type == TCG_TYPE_I32) {
839 opi = LWZ, opx = LWZX;
840 } else {
841 opi = LD, opx = LDX;
842 }
843 tcg_out_mem_long(s, opi, opx, ret, arg1, arg2);
844 }
845
846 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
847 TCGReg arg1, intptr_t arg2)
848 {
849 int opi, opx;
850
851 assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
852 if (type == TCG_TYPE_I32) {
853 opi = STW, opx = STWX;
854 } else {
855 opi = STD, opx = STDX;
856 }
857 tcg_out_mem_long(s, opi, opx, arg, arg1, arg2);
858 }
859
860 static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
861 int const_arg2, int cr, TCGType type)
862 {
863 int imm;
864 uint32_t op;
865
866 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
867
868 /* Simplify the comparisons below wrt CMPI. */
869 if (type == TCG_TYPE_I32) {
870 arg2 = (int32_t)arg2;
871 }
872
873 switch (cond) {
874 case TCG_COND_EQ:
875 case TCG_COND_NE:
876 if (const_arg2) {
877 if ((int16_t) arg2 == arg2) {
878 op = CMPI;
879 imm = 1;
880 break;
881 } else if ((uint16_t) arg2 == arg2) {
882 op = CMPLI;
883 imm = 1;
884 break;
885 }
886 }
887 op = CMPL;
888 imm = 0;
889 break;
890
891 case TCG_COND_LT:
892 case TCG_COND_GE:
893 case TCG_COND_LE:
894 case TCG_COND_GT:
895 if (const_arg2) {
896 if ((int16_t) arg2 == arg2) {
897 op = CMPI;
898 imm = 1;
899 break;
900 }
901 }
902 op = CMP;
903 imm = 0;
904 break;
905
906 case TCG_COND_LTU:
907 case TCG_COND_GEU:
908 case TCG_COND_LEU:
909 case TCG_COND_GTU:
910 if (const_arg2) {
911 if ((uint16_t) arg2 == arg2) {
912 op = CMPLI;
913 imm = 1;
914 break;
915 }
916 }
917 op = CMPL;
918 imm = 0;
919 break;
920
921 default:
922 tcg_abort();
923 }
924 op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
925
926 if (imm) {
927 tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff));
928 } else {
929 if (const_arg2) {
930 tcg_out_movi(s, type, TCG_REG_R0, arg2);
931 arg2 = TCG_REG_R0;
932 }
933 tcg_out32(s, op | RA(arg1) | RB(arg2));
934 }
935 }
936
937 static void tcg_out_setcond_eq0(TCGContext *s, TCGType type,
938 TCGReg dst, TCGReg src)
939 {
940 if (type == TCG_TYPE_I32) {
941 tcg_out32(s, CNTLZW | RS(src) | RA(dst));
942 tcg_out_shri32(s, dst, dst, 5);
943 } else {
944 tcg_out32(s, CNTLZD | RS(src) | RA(dst));
945 tcg_out_shri64(s, dst, dst, 6);
946 }
947 }
948
949 static void tcg_out_setcond_ne0(TCGContext *s, TCGReg dst, TCGReg src)
950 {
951 /* X != 0 implies X + -1 generates a carry. Extra addition
952 trickery means: R = X-1 + ~X + C = X-1 + (-X+1) + C = C. */
953 if (dst != src) {
954 tcg_out32(s, ADDIC | TAI(dst, src, -1));
955 tcg_out32(s, SUBFE | TAB(dst, dst, src));
956 } else {
957 tcg_out32(s, ADDIC | TAI(TCG_REG_R0, src, -1));
958 tcg_out32(s, SUBFE | TAB(dst, TCG_REG_R0, src));
959 }
960 }
961
962 static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2,
963 bool const_arg2)
964 {
965 if (const_arg2) {
966 if ((uint32_t)arg2 == arg2) {
967 tcg_out_xori32(s, TCG_REG_R0, arg1, arg2);
968 } else {
969 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, arg2);
970 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, TCG_REG_R0));
971 }
972 } else {
973 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, arg2));
974 }
975 return TCG_REG_R0;
976 }
977
978 static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
979 TCGArg arg0, TCGArg arg1, TCGArg arg2,
980 int const_arg2)
981 {
982 int crop, sh;
983
984 assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
985
986 /* Ignore high bits of a potential constant arg2. */
987 if (type == TCG_TYPE_I32) {
988 arg2 = (uint32_t)arg2;
989 }
990
991 /* Handle common and trivial cases before handling anything else. */
992 if (arg2 == 0) {
993 switch (cond) {
994 case TCG_COND_EQ:
995 tcg_out_setcond_eq0(s, type, arg0, arg1);
996 return;
997 case TCG_COND_NE:
998 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
999 tcg_out_ext32u(s, TCG_REG_R0, arg1);
1000 arg1 = TCG_REG_R0;
1001 }
1002 tcg_out_setcond_ne0(s, arg0, arg1);
1003 return;
1004 case TCG_COND_GE:
1005 tcg_out32(s, NOR | SAB(arg1, arg0, arg1));
1006 arg1 = arg0;
1007 /* FALLTHRU */
1008 case TCG_COND_LT:
1009 /* Extract the sign bit. */
1010 if (type == TCG_TYPE_I32) {
1011 tcg_out_shri32(s, arg0, arg1, 31);
1012 } else {
1013 tcg_out_shri64(s, arg0, arg1, 63);
1014 }
1015 return;
1016 default:
1017 break;
1018 }
1019 }
1020
1021 /* If we have ISEL, we can implement everything with 3 or 4 insns.
1022 All other cases below are also at least 3 insns, so speed up the
1023 code generator by not considering them and always using ISEL. */
1024 if (HAVE_ISEL) {
1025 int isel, tab;
1026
1027 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
1028
1029 isel = tcg_to_isel[cond];
1030
1031 tcg_out_movi(s, type, arg0, 1);
1032 if (isel & 1) {
1033 /* arg0 = (bc ? 0 : 1) */
1034 tab = TAB(arg0, 0, arg0);
1035 isel &= ~1;
1036 } else {
1037 /* arg0 = (bc ? 1 : 0) */
1038 tcg_out_movi(s, type, TCG_REG_R0, 0);
1039 tab = TAB(arg0, arg0, TCG_REG_R0);
1040 }
1041 tcg_out32(s, isel | tab);
1042 return;
1043 }
1044
1045 switch (cond) {
1046 case TCG_COND_EQ:
1047 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1048 tcg_out_setcond_eq0(s, type, arg0, arg1);
1049 return;
1050
1051 case TCG_COND_NE:
1052 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1053 /* Discard the high bits only once, rather than both inputs. */
1054 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
1055 tcg_out_ext32u(s, TCG_REG_R0, arg1);
1056 arg1 = TCG_REG_R0;
1057 }
1058 tcg_out_setcond_ne0(s, arg0, arg1);
1059 return;
1060
1061 case TCG_COND_GT:
1062 case TCG_COND_GTU:
1063 sh = 30;
1064 crop = 0;
1065 goto crtest;
1066
1067 case TCG_COND_LT:
1068 case TCG_COND_LTU:
1069 sh = 29;
1070 crop = 0;
1071 goto crtest;
1072
1073 case TCG_COND_GE:
1074 case TCG_COND_GEU:
1075 sh = 31;
1076 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_LT) | BB(7, CR_LT);
1077 goto crtest;
1078
1079 case TCG_COND_LE:
1080 case TCG_COND_LEU:
1081 sh = 31;
1082 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_GT) | BB(7, CR_GT);
1083 crtest:
1084 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
1085 if (crop) {
1086 tcg_out32(s, crop);
1087 }
1088 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
1089 tcg_out_rlw(s, RLWINM, arg0, TCG_REG_R0, sh, 31, 31);
1090 break;
1091
1092 default:
1093 tcg_abort();
1094 }
1095 }
1096
1097 static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l)
1098 {
1099 if (l->has_value) {
1100 tcg_out32(s, bc | reloc_pc14_val(s->code_ptr, l->u.value_ptr));
1101 } else {
1102 tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0);
1103 tcg_out_bc_noaddr(s, bc);
1104 }
1105 }
1106
1107 static void tcg_out_brcond(TCGContext *s, TCGCond cond,
1108 TCGArg arg1, TCGArg arg2, int const_arg2,
1109 TCGLabel *l, TCGType type)
1110 {
1111 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
1112 tcg_out_bc(s, tcg_to_bc[cond], l);
1113 }
1114
1115 static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
1116 TCGArg dest, TCGArg c1, TCGArg c2, TCGArg v1,
1117 TCGArg v2, bool const_c2)
1118 {
1119 /* If for some reason both inputs are zero, don't produce bad code. */
1120 if (v1 == 0 && v2 == 0) {
1121 tcg_out_movi(s, type, dest, 0);
1122 return;
1123 }
1124
1125 tcg_out_cmp(s, cond, c1, c2, const_c2, 7, type);
1126
1127 if (HAVE_ISEL) {
1128 int isel = tcg_to_isel[cond];
1129
1130 /* Swap the V operands if the operation indicates inversion. */
1131 if (isel & 1) {
1132 int t = v1;
1133 v1 = v2;
1134 v2 = t;
1135 isel &= ~1;
1136 }
1137 /* V1 == 0 is handled by isel; V2 == 0 must be handled by hand. */
1138 if (v2 == 0) {
1139 tcg_out_movi(s, type, TCG_REG_R0, 0);
1140 }
1141 tcg_out32(s, isel | TAB(dest, v1, v2));
1142 } else {
1143 if (dest == v2) {
1144 cond = tcg_invert_cond(cond);
1145 v2 = v1;
1146 } else if (dest != v1) {
1147 if (v1 == 0) {
1148 tcg_out_movi(s, type, dest, 0);
1149 } else {
1150 tcg_out_mov(s, type, dest, v1);
1151 }
1152 }
1153 /* Branch forward over one insn */
1154 tcg_out32(s, tcg_to_bc[cond] | 8);
1155 if (v2 == 0) {
1156 tcg_out_movi(s, type, dest, 0);
1157 } else {
1158 tcg_out_mov(s, type, dest, v2);
1159 }
1160 }
1161 }
1162
1163 static void tcg_out_cmp2(TCGContext *s, const TCGArg *args,
1164 const int *const_args)
1165 {
1166 static const struct { uint8_t bit1, bit2; } bits[] = {
1167 [TCG_COND_LT ] = { CR_LT, CR_LT },
1168 [TCG_COND_LE ] = { CR_LT, CR_GT },
1169 [TCG_COND_GT ] = { CR_GT, CR_GT },
1170 [TCG_COND_GE ] = { CR_GT, CR_LT },
1171 [TCG_COND_LTU] = { CR_LT, CR_LT },
1172 [TCG_COND_LEU] = { CR_LT, CR_GT },
1173 [TCG_COND_GTU] = { CR_GT, CR_GT },
1174 [TCG_COND_GEU] = { CR_GT, CR_LT },
1175 };
1176
1177 TCGCond cond = args[4], cond2;
1178 TCGArg al, ah, bl, bh;
1179 int blconst, bhconst;
1180 int op, bit1, bit2;
1181
1182 al = args[0];
1183 ah = args[1];
1184 bl = args[2];
1185 bh = args[3];
1186 blconst = const_args[2];
1187 bhconst = const_args[3];
1188
1189 switch (cond) {
1190 case TCG_COND_EQ:
1191 op = CRAND;
1192 goto do_equality;
1193 case TCG_COND_NE:
1194 op = CRNAND;
1195 do_equality:
1196 tcg_out_cmp(s, cond, al, bl, blconst, 6, TCG_TYPE_I32);
1197 tcg_out_cmp(s, cond, ah, bh, bhconst, 7, TCG_TYPE_I32);
1198 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
1199 break;
1200
1201 case TCG_COND_LT:
1202 case TCG_COND_LE:
1203 case TCG_COND_GT:
1204 case TCG_COND_GE:
1205 case TCG_COND_LTU:
1206 case TCG_COND_LEU:
1207 case TCG_COND_GTU:
1208 case TCG_COND_GEU:
1209 bit1 = bits[cond].bit1;
1210 bit2 = bits[cond].bit2;
1211 op = (bit1 != bit2 ? CRANDC : CRAND);
1212 cond2 = tcg_unsigned_cond(cond);
1213
1214 tcg_out_cmp(s, cond, ah, bh, bhconst, 6, TCG_TYPE_I32);
1215 tcg_out_cmp(s, cond2, al, bl, blconst, 7, TCG_TYPE_I32);
1216 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2));
1217 tcg_out32(s, CROR | BT(7, CR_EQ) | BA(6, bit1) | BB(7, CR_EQ));
1218 break;
1219
1220 default:
1221 tcg_abort();
1222 }
1223 }
1224
1225 static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
1226 const int *const_args)
1227 {
1228 tcg_out_cmp2(s, args + 1, const_args + 1);
1229 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
1230 tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, 31, 31, 31);
1231 }
1232
1233 static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
1234 const int *const_args)
1235 {
1236 tcg_out_cmp2(s, args, const_args);
1237 tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5]));
1238 }
1239
1240 void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
1241 {
1242 tcg_insn_unit i1, i2;
1243 uint64_t pair;
1244 intptr_t diff = addr - jmp_addr;
1245
1246 if (in_range_b(diff)) {
1247 i1 = B | (diff & 0x3fffffc);
1248 i2 = NOP;
1249 } else if (USE_REG_RA) {
1250 intptr_t lo, hi;
1251 diff = addr - (uintptr_t)tb_ret_addr;
1252 lo = (int16_t)diff;
1253 hi = (int32_t)(diff - lo);
1254 assert(diff == hi + lo);
1255 i1 = ADDIS | TAI(TCG_REG_TMP1, TCG_REG_RA, hi >> 16);
1256 i2 = ADDI | TAI(TCG_REG_TMP1, TCG_REG_TMP1, lo);
1257 } else {
1258 assert(TCG_TARGET_REG_BITS == 32 || addr == (int32_t)addr);
1259 i1 = ADDIS | TAI(TCG_REG_TMP1, 0, addr >> 16);
1260 i2 = ORI | SAI(TCG_REG_TMP1, TCG_REG_TMP1, addr);
1261 }
1262 #ifdef HOST_WORDS_BIGENDIAN
1263 pair = (uint64_t)i1 << 32 | i2;
1264 #else
1265 pair = (uint64_t)i2 << 32 | i1;
1266 #endif
1267
1268 /* ??? __atomic_store_8, presuming there's some way to do that
1269 for 32-bit, otherwise this is good enough for 64-bit. */
1270 *(uint64_t *)jmp_addr = pair;
1271 flush_icache_range(jmp_addr, jmp_addr + 8);
1272 }
1273
1274 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
1275 {
1276 #ifdef _CALL_AIX
1277 /* Look through the descriptor. If the branch is in range, and we
1278 don't have to spend too much effort on building the toc. */
1279 void *tgt = ((void **)target)[0];
1280 uintptr_t toc = ((uintptr_t *)target)[1];
1281 intptr_t diff = tcg_pcrel_diff(s, tgt);
1282
1283 if (in_range_b(diff) && toc == (uint32_t)toc) {
1284 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc);
1285 tcg_out_b(s, LK, tgt);
1286 } else {
1287 /* Fold the low bits of the constant into the addresses below. */
1288 intptr_t arg = (intptr_t)target;
1289 int ofs = (int16_t)arg;
1290
1291 if (ofs + 8 < 0x8000) {
1292 arg -= ofs;
1293 } else {
1294 ofs = 0;
1295 }
1296 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, arg);
1297 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs);
1298 tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR);
1299 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP);
1300 tcg_out32(s, BCCTR | BO_ALWAYS | LK);
1301 }
1302 #elif defined(_CALL_ELF) && _CALL_ELF == 2
1303 intptr_t diff;
1304
1305 /* In the ELFv2 ABI, we have to set up r12 to contain the destination
1306 address, which the callee uses to compute its TOC address. */
1307 /* FIXME: when the branch is in range, we could avoid r12 load if we
1308 knew that the destination uses the same TOC, and what its local
1309 entry point offset is. */
1310 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R12, (intptr_t)target);
1311
1312 diff = tcg_pcrel_diff(s, target);
1313 if (in_range_b(diff)) {
1314 tcg_out_b(s, LK, target);
1315 } else {
1316 tcg_out32(s, MTSPR | RS(TCG_REG_R12) | CTR);
1317 tcg_out32(s, BCCTR | BO_ALWAYS | LK);
1318 }
1319 #else
1320 tcg_out_b(s, LK, target);
1321 #endif
1322 }
1323
1324 static const uint32_t qemu_ldx_opc[16] = {
1325 [MO_UB] = LBZX,
1326 [MO_UW] = LHZX,
1327 [MO_UL] = LWZX,
1328 [MO_Q] = LDX,
1329 [MO_SW] = LHAX,
1330 [MO_SL] = LWAX,
1331 [MO_BSWAP | MO_UB] = LBZX,
1332 [MO_BSWAP | MO_UW] = LHBRX,
1333 [MO_BSWAP | MO_UL] = LWBRX,
1334 [MO_BSWAP | MO_Q] = LDBRX,
1335 };
1336
1337 static const uint32_t qemu_stx_opc[16] = {
1338 [MO_UB] = STBX,
1339 [MO_UW] = STHX,
1340 [MO_UL] = STWX,
1341 [MO_Q] = STDX,
1342 [MO_BSWAP | MO_UB] = STBX,
1343 [MO_BSWAP | MO_UW] = STHBRX,
1344 [MO_BSWAP | MO_UL] = STWBRX,
1345 [MO_BSWAP | MO_Q] = STDBRX,
1346 };
1347
1348 static const uint32_t qemu_exts_opc[4] = {
1349 EXTSB, EXTSH, EXTSW, 0
1350 };
1351
1352 #if defined (CONFIG_SOFTMMU)
1353 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
1354 * int mmu_idx, uintptr_t ra)
1355 */
1356 static void * const qemu_ld_helpers[16] = {
1357 [MO_UB] = helper_ret_ldub_mmu,
1358 [MO_LEUW] = helper_le_lduw_mmu,
1359 [MO_LEUL] = helper_le_ldul_mmu,
1360 [MO_LEQ] = helper_le_ldq_mmu,
1361 [MO_BEUW] = helper_be_lduw_mmu,
1362 [MO_BEUL] = helper_be_ldul_mmu,
1363 [MO_BEQ] = helper_be_ldq_mmu,
1364 };
1365
1366 /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
1367 * uintxx_t val, int mmu_idx, uintptr_t ra)
1368 */
1369 static void * const qemu_st_helpers[16] = {
1370 [MO_UB] = helper_ret_stb_mmu,
1371 [MO_LEUW] = helper_le_stw_mmu,
1372 [MO_LEUL] = helper_le_stl_mmu,
1373 [MO_LEQ] = helper_le_stq_mmu,
1374 [MO_BEUW] = helper_be_stw_mmu,
1375 [MO_BEUL] = helper_be_stl_mmu,
1376 [MO_BEQ] = helper_be_stq_mmu,
1377 };
1378
1379 /* Perform the TLB load and compare. Places the result of the comparison
1380 in CR7, loads the addend of the TLB into R3, and returns the register
1381 containing the guest address (zero-extended into R4). Clobbers R0 and R2. */
1382
1383 static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc,
1384 TCGReg addrlo, TCGReg addrhi,
1385 int mem_index, bool is_read)
1386 {
1387 int cmp_off
1388 = (is_read
1389 ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
1390 : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
1391 int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
1392 TCGReg base = TCG_AREG0;
1393 TCGMemOp s_bits = opc & MO_SIZE;
1394
1395 /* Extract the page index, shifted into place for tlb index. */
1396 if (TCG_TARGET_REG_BITS == 64) {
1397 if (TARGET_LONG_BITS == 32) {
1398 /* Zero-extend the address into a place helpful for further use. */
1399 tcg_out_ext32u(s, TCG_REG_R4, addrlo);
1400 addrlo = TCG_REG_R4;
1401 } else {
1402 tcg_out_rld(s, RLDICL, TCG_REG_R3, addrlo,
1403 64 - TARGET_PAGE_BITS, 64 - CPU_TLB_BITS);
1404 }
1405 }
1406
1407 /* Compensate for very large offsets. */
1408 if (add_off >= 0x8000) {
1409 /* Most target env are smaller than 32k; none are larger than 64k.
1410 Simplify the logic here merely to offset by 0x7ff0, giving us a
1411 range just shy of 64k. Check this assumption. */
1412 QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
1413 tlb_table[NB_MMU_MODES - 1][1])
1414 > 0x7ff0 + 0x7fff);
1415 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, base, 0x7ff0));
1416 base = TCG_REG_TMP1;
1417 cmp_off -= 0x7ff0;
1418 add_off -= 0x7ff0;
1419 }
1420
1421 /* Extraction and shifting, part 2. */
1422 if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
1423 tcg_out_rlw(s, RLWINM, TCG_REG_R3, addrlo,
1424 32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
1425 32 - (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS),
1426 31 - CPU_TLB_ENTRY_BITS);
1427 } else {
1428 tcg_out_shli64(s, TCG_REG_R3, TCG_REG_R3, CPU_TLB_ENTRY_BITS);
1429 }
1430
1431 tcg_out32(s, ADD | TAB(TCG_REG_R3, TCG_REG_R3, base));
1432
1433 /* Load the tlb comparator. */
1434 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
1435 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R4, TCG_REG_R3, cmp_off);
1436 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP1, TCG_REG_R3, cmp_off + 4);
1437 } else {
1438 tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP1, TCG_REG_R3, cmp_off);
1439 }
1440
1441 /* Load the TLB addend for use on the fast path. Do this asap
1442 to minimize any load use delay. */
1443 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3, add_off);
1444
1445 /* Clear the non-page, non-alignment bits from the address */
1446 if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
1447 /* We don't support unaligned accesses on 32-bits, preserve
1448 * the bottom bits and thus trigger a comparison failure on
1449 * unaligned accesses
1450 */
1451 tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,
1452 (32 - s_bits) & 31, 31 - TARGET_PAGE_BITS);
1453 } else if (s_bits) {
1454 /* > byte access, we need to handle alignment */
1455 if ((opc & MO_AMASK) == MO_ALIGN) {
1456 /* Alignment required by the front-end, same as 32-bits */
1457 tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,
1458 64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - s_bits);
1459 tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
1460 } else {
1461 /* We support unaligned accesses, we need to make sure we fail
1462 * if we cross a page boundary. The trick is to add the
1463 * access_size-1 to the address before masking the low bits.
1464 * That will make the address overflow to the next page if we
1465 * cross a page boundary which will then force a mismatch of
1466 * the TLB compare since the next page cannot possibly be in
1467 * the same TLB index.
1468 */
1469 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, (1 << s_bits) - 1));
1470 tcg_out_rld(s, RLDICR, TCG_REG_R0, TCG_REG_R0,
1471 0, 63 - TARGET_PAGE_BITS);
1472 }
1473 } else {
1474 /* Byte access, just chop off the bits below the page index */
1475 tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo, 0, 63 - TARGET_PAGE_BITS);
1476 }
1477
1478 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
1479 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
1480 0, 7, TCG_TYPE_I32);
1481 tcg_out_cmp(s, TCG_COND_EQ, addrhi, TCG_REG_R4, 0, 6, TCG_TYPE_I32);
1482 tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
1483 } else {
1484 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
1485 0, 7, TCG_TYPE_TL);
1486 }
1487
1488 return addrlo;
1489 }
1490
1491 /* Record the context of a call to the out of line helper code for the slow
1492 path for a load or store, so that we can later generate the correct
1493 helper code. */
1494 static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
1495 TCGReg datalo_reg, TCGReg datahi_reg,
1496 TCGReg addrlo_reg, TCGReg addrhi_reg,
1497 tcg_insn_unit *raddr, tcg_insn_unit *lptr)
1498 {
1499 TCGLabelQemuLdst *label = new_ldst_label(s);
1500
1501 label->is_ld = is_ld;
1502 label->oi = oi;
1503 label->datalo_reg = datalo_reg;
1504 label->datahi_reg = datahi_reg;
1505 label->addrlo_reg = addrlo_reg;
1506 label->addrhi_reg = addrhi_reg;
1507 label->raddr = raddr;
1508 label->label_ptr[0] = lptr;
1509 }
1510
1511 static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1512 {
1513 TCGMemOpIdx oi = lb->oi;
1514 TCGMemOp opc = get_memop(oi);
1515 TCGReg hi, lo, arg = TCG_REG_R3;
1516
1517 reloc_pc14(lb->label_ptr[0], s->code_ptr);
1518
1519 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
1520
1521 lo = lb->addrlo_reg;
1522 hi = lb->addrhi_reg;
1523 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
1524 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1525 arg |= 1;
1526 #endif
1527 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
1528 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
1529 } else {
1530 /* If the address needed to be zero-extended, we'll have already
1531 placed it in R4. The only remaining case is 64-bit guest. */
1532 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
1533 }
1534
1535 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
1536 tcg_out32(s, MFSPR | RT(arg) | LR);
1537
1538 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
1539
1540 lo = lb->datalo_reg;
1541 hi = lb->datahi_reg;
1542 if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
1543 tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4);
1544 tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3);
1545 } else if (opc & MO_SIGN) {
1546 uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
1547 tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3));
1548 } else {
1549 tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3);
1550 }
1551
1552 tcg_out_b(s, 0, lb->raddr);
1553 }
1554
1555 static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1556 {
1557 TCGMemOpIdx oi = lb->oi;
1558 TCGMemOp opc = get_memop(oi);
1559 TCGMemOp s_bits = opc & MO_SIZE;
1560 TCGReg hi, lo, arg = TCG_REG_R3;
1561
1562 reloc_pc14(lb->label_ptr[0], s->code_ptr);
1563
1564 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
1565
1566 lo = lb->addrlo_reg;
1567 hi = lb->addrhi_reg;
1568 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
1569 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1570 arg |= 1;
1571 #endif
1572 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
1573 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
1574 } else {
1575 /* If the address needed to be zero-extended, we'll have already
1576 placed it in R4. The only remaining case is 64-bit guest. */
1577 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
1578 }
1579
1580 lo = lb->datalo_reg;
1581 hi = lb->datahi_reg;
1582 if (TCG_TARGET_REG_BITS == 32) {
1583 switch (s_bits) {
1584 case MO_64:
1585 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1586 arg |= 1;
1587 #endif
1588 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
1589 /* FALLTHRU */
1590 case MO_32:
1591 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
1592 break;
1593 default:
1594 tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31);
1595 break;
1596 }
1597 } else {
1598 if (s_bits == MO_64) {
1599 tcg_out_mov(s, TCG_TYPE_I64, arg++, lo);
1600 } else {
1601 tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits));
1602 }
1603 }
1604
1605 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
1606 tcg_out32(s, MFSPR | RT(arg) | LR);
1607
1608 tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
1609
1610 tcg_out_b(s, 0, lb->raddr);
1611 }
1612 #endif /* SOFTMMU */
1613
1614 static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
1615 {
1616 TCGReg datalo, datahi, addrlo, rbase;
1617 TCGReg addrhi __attribute__((unused));
1618 TCGMemOpIdx oi;
1619 TCGMemOp opc, s_bits;
1620 #ifdef CONFIG_SOFTMMU
1621 int mem_index;
1622 tcg_insn_unit *label_ptr;
1623 #endif
1624
1625 datalo = *args++;
1626 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
1627 addrlo = *args++;
1628 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
1629 oi = *args++;
1630 opc = get_memop(oi);
1631 s_bits = opc & MO_SIZE;
1632
1633 #ifdef CONFIG_SOFTMMU
1634 mem_index = get_mmuidx(oi);
1635 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true);
1636
1637 /* Load a pointer into the current opcode w/conditional branch-link. */
1638 label_ptr = s->code_ptr;
1639 tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
1640
1641 rbase = TCG_REG_R3;
1642 #else /* !CONFIG_SOFTMMU */
1643 rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
1644 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
1645 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
1646 addrlo = TCG_REG_TMP1;
1647 }
1648 #endif
1649
1650 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
1651 if (opc & MO_BSWAP) {
1652 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
1653 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
1654 tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0));
1655 } else if (rbase != 0) {
1656 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
1657 tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo));
1658 tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0));
1659 } else if (addrlo == datahi) {
1660 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
1661 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
1662 } else {
1663 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
1664 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
1665 }
1666 } else {
1667 uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)];
1668 if (!HAVE_ISA_2_06 && insn == LDBRX) {
1669 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
1670 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
1671 tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0));
1672 tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0);
1673 } else if (insn) {
1674 tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
1675 } else {
1676 insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
1677 tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
1678 insn = qemu_exts_opc[s_bits];
1679 tcg_out32(s, insn | RA(datalo) | RS(datalo));
1680 }
1681 }
1682
1683 #ifdef CONFIG_SOFTMMU
1684 add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
1685 s->code_ptr, label_ptr);
1686 #endif
1687 }
1688
1689 static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
1690 {
1691 TCGReg datalo, datahi, addrlo, rbase;
1692 TCGReg addrhi __attribute__((unused));
1693 TCGMemOpIdx oi;
1694 TCGMemOp opc, s_bits;
1695 #ifdef CONFIG_SOFTMMU
1696 int mem_index;
1697 tcg_insn_unit *label_ptr;
1698 #endif
1699
1700 datalo = *args++;
1701 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
1702 addrlo = *args++;
1703 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
1704 oi = *args++;
1705 opc = get_memop(oi);
1706 s_bits = opc & MO_SIZE;
1707
1708 #ifdef CONFIG_SOFTMMU
1709 mem_index = get_mmuidx(oi);
1710 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false);
1711
1712 /* Load a pointer into the current opcode w/conditional branch-link. */
1713 label_ptr = s->code_ptr;
1714 tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
1715
1716 rbase = TCG_REG_R3;
1717 #else /* !CONFIG_SOFTMMU */
1718 rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
1719 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
1720 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
1721 addrlo = TCG_REG_TMP1;
1722 }
1723 #endif
1724
1725 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
1726 if (opc & MO_BSWAP) {
1727 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
1728 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
1729 tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0));
1730 } else if (rbase != 0) {
1731 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
1732 tcg_out32(s, STWX | SAB(datahi, rbase, addrlo));
1733 tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0));
1734 } else {
1735 tcg_out32(s, STW | TAI(datahi, addrlo, 0));
1736 tcg_out32(s, STW | TAI(datalo, addrlo, 4));
1737 }
1738 } else {
1739 uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)];
1740 if (!HAVE_ISA_2_06 && insn == STDBRX) {
1741 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
1742 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4));
1743 tcg_out_shri64(s, TCG_REG_R0, datalo, 32);
1744 tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1));
1745 } else {
1746 tcg_out32(s, insn | SAB(datalo, rbase, addrlo));
1747 }
1748 }
1749
1750 #ifdef CONFIG_SOFTMMU
1751 add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
1752 s->code_ptr, label_ptr);
1753 #endif
1754 }
1755
1756 /* Parameters for function call generation, used in tcg.c. */
1757 #define TCG_TARGET_STACK_ALIGN 16
1758 #define TCG_TARGET_EXTEND_ARGS 1
1759
1760 #ifdef _CALL_AIX
1761 # define LINK_AREA_SIZE (6 * SZR)
1762 # define LR_OFFSET (1 * SZR)
1763 # define TCG_TARGET_CALL_STACK_OFFSET (LINK_AREA_SIZE + 8 * SZR)
1764 #elif defined(TCG_TARGET_CALL_DARWIN)
1765 # define LINK_AREA_SIZE (6 * SZR)
1766 # define LR_OFFSET (2 * SZR)
1767 #elif TCG_TARGET_REG_BITS == 64
1768 # if defined(_CALL_ELF) && _CALL_ELF == 2
1769 # define LINK_AREA_SIZE (4 * SZR)
1770 # define LR_OFFSET (1 * SZR)
1771 # endif
1772 #else /* TCG_TARGET_REG_BITS == 32 */
1773 # if defined(_CALL_SYSV)
1774 # define LINK_AREA_SIZE (2 * SZR)
1775 # define LR_OFFSET (1 * SZR)
1776 # endif
1777 #endif
1778 #ifndef LR_OFFSET
1779 # error "Unhandled abi"
1780 #endif
1781 #ifndef TCG_TARGET_CALL_STACK_OFFSET
1782 # define TCG_TARGET_CALL_STACK_OFFSET LINK_AREA_SIZE
1783 #endif
1784
1785 #define CPU_TEMP_BUF_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
1786 #define REG_SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * SZR)
1787
1788 #define FRAME_SIZE ((TCG_TARGET_CALL_STACK_OFFSET \
1789 + TCG_STATIC_CALL_ARGS_SIZE \
1790 + CPU_TEMP_BUF_SIZE \
1791 + REG_SAVE_SIZE \
1792 + TCG_TARGET_STACK_ALIGN - 1) \
1793 & -TCG_TARGET_STACK_ALIGN)
1794
1795 #define REG_SAVE_BOT (FRAME_SIZE - REG_SAVE_SIZE)
1796
1797 static void tcg_target_qemu_prologue(TCGContext *s)
1798 {
1799 int i;
1800
1801 #ifdef _CALL_AIX
1802 void **desc = (void **)s->code_ptr;
1803 desc[0] = desc + 2; /* entry point */
1804 desc[1] = 0; /* environment pointer */
1805 s->code_ptr = (void *)(desc + 2); /* skip over descriptor */
1806 #endif
1807
1808 tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE,
1809 CPU_TEMP_BUF_SIZE);
1810
1811 /* Prologue */
1812 tcg_out32(s, MFSPR | RT(TCG_REG_R0) | LR);
1813 tcg_out32(s, (SZR == 8 ? STDU : STWU)
1814 | SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE));
1815
1816 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
1817 tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
1818 TCG_REG_R1, REG_SAVE_BOT + i * SZR);
1819 }
1820 tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
1821
1822 #ifndef CONFIG_SOFTMMU
1823 if (guest_base) {
1824 tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
1825 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
1826 }
1827 #endif
1828
1829 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
1830 tcg_out32(s, MTSPR | RS(tcg_target_call_iarg_regs[1]) | CTR);
1831
1832 if (USE_REG_RA) {
1833 #ifdef _CALL_AIX
1834 /* Make the caller load the value as the TOC into R2. */
1835 tb_ret_addr = s->code_ptr + 2;
1836 desc[1] = tb_ret_addr;
1837 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_RA, TCG_REG_R2);
1838 tcg_out32(s, BCCTR | BO_ALWAYS);
1839 #elif defined(_CALL_ELF) && _CALL_ELF == 2
1840 /* Compute from the incoming R12 value. */
1841 tb_ret_addr = s->code_ptr + 2;
1842 tcg_out32(s, ADDI | TAI(TCG_REG_RA, TCG_REG_R12,
1843 tcg_ptr_byte_diff(tb_ret_addr, s->code_buf)));
1844 tcg_out32(s, BCCTR | BO_ALWAYS);
1845 #else
1846 /* Reserve max 5 insns for the constant load. */
1847 tb_ret_addr = s->code_ptr + 6;
1848 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (intptr_t)tb_ret_addr);
1849 tcg_out32(s, BCCTR | BO_ALWAYS);
1850 while (s->code_ptr < tb_ret_addr) {
1851 tcg_out32(s, NOP);
1852 }
1853 #endif
1854 } else {
1855 tcg_out32(s, BCCTR | BO_ALWAYS);
1856 tb_ret_addr = s->code_ptr;
1857 }
1858
1859 /* Epilogue */
1860 assert(tb_ret_addr == s->code_ptr);
1861
1862 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
1863 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
1864 tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
1865 TCG_REG_R1, REG_SAVE_BOT + i * SZR);
1866 }
1867 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | LR);
1868 tcg_out32(s, ADDI | TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE));
1869 tcg_out32(s, BCLR | BO_ALWAYS);
1870 }
1871
1872 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
1873 const int *const_args)
1874 {
1875 TCGArg a0, a1, a2;
1876 int c;
1877
1878 switch (opc) {
1879 case INDEX_op_exit_tb:
1880 if (USE_REG_RA) {
1881 ptrdiff_t disp = tcg_pcrel_diff(s, tb_ret_addr);
1882
1883 /* Use a direct branch if we can, otherwise use the value in RA.
1884 Note that the direct branch is always backward, thus we need
1885 to account for the possibility of 5 insns from the movi. */
1886 if (!in_range_b(disp - 20)) {
1887 tcg_out32(s, MTSPR | RS(TCG_REG_RA) | CTR);
1888 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
1889 tcg_out32(s, BCCTR | BO_ALWAYS);
1890 break;
1891 }
1892 }
1893 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
1894 tcg_out_b(s, 0, tb_ret_addr);
1895 break;
1896 case INDEX_op_goto_tb:
1897 tcg_debug_assert(s->tb_jmp_offset);
1898 /* Direct jump. Ensure the next insns are 8-byte aligned. */
1899 if ((uintptr_t)s->code_ptr & 7) {
1900 tcg_out32(s, NOP);
1901 }
1902 s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
1903 /* To be replaced by either a branch+nop or a load into TMP1. */
1904 s->code_ptr += 2;
1905 tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
1906 tcg_out32(s, BCCTR | BO_ALWAYS);
1907 s->tb_next_offset[args[0]] = tcg_current_code_size(s);
1908 break;
1909 case INDEX_op_br:
1910 {
1911 TCGLabel *l = arg_label(args[0]);
1912
1913 if (l->has_value) {
1914 tcg_out_b(s, 0, l->u.value_ptr);
1915 } else {
1916 tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0);
1917 tcg_out_b_noaddr(s, B);
1918 }
1919 }
1920 break;
1921 case INDEX_op_ld8u_i32:
1922 case INDEX_op_ld8u_i64:
1923 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
1924 break;
1925 case INDEX_op_ld8s_i32:
1926 case INDEX_op_ld8s_i64:
1927 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
1928 tcg_out32(s, EXTSB | RS(args[0]) | RA(args[0]));
1929 break;
1930 case INDEX_op_ld16u_i32:
1931 case INDEX_op_ld16u_i64:
1932 tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]);
1933 break;
1934 case INDEX_op_ld16s_i32:
1935 case INDEX_op_ld16s_i64:
1936 tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]);
1937 break;
1938 case INDEX_op_ld_i32:
1939 case INDEX_op_ld32u_i64:
1940 tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]);
1941 break;
1942 case INDEX_op_ld32s_i64:
1943 tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]);
1944 break;
1945 case INDEX_op_ld_i64:
1946 tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]);
1947 break;
1948 case INDEX_op_st8_i32:
1949 case INDEX_op_st8_i64:
1950 tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]);
1951 break;
1952 case INDEX_op_st16_i32:
1953 case INDEX_op_st16_i64:
1954 tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]);
1955 break;
1956 case INDEX_op_st_i32:
1957 case INDEX_op_st32_i64:
1958 tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]);
1959 break;
1960 case INDEX_op_st_i64:
1961 tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]);
1962 break;
1963
1964 case INDEX_op_add_i32:
1965 a0 = args[0], a1 = args[1], a2 = args[2];
1966 if (const_args[2]) {
1967 do_addi_32:
1968 tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2);
1969 } else {
1970 tcg_out32(s, ADD | TAB(a0, a1, a2));
1971 }
1972 break;
1973 case INDEX_op_sub_i32:
1974 a0 = args[0], a1 = args[1], a2 = args[2];
1975 if (const_args[1]) {
1976 if (const_args[2]) {
1977 tcg_out_movi(s, TCG_TYPE_I32, a0, a1 - a2);
1978 } else {
1979 tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
1980 }
1981 } else if (const_args[2]) {
1982 a2 = -a2;
1983 goto do_addi_32;
1984 } else {
1985 tcg_out32(s, SUBF | TAB(a0, a2, a1));
1986 }
1987 break;
1988
1989 case INDEX_op_and_i32:
1990 a0 = args[0], a1 = args[1], a2 = args[2];
1991 if (const_args[2]) {
1992 tcg_out_andi32(s, a0, a1, a2);
1993 } else {
1994 tcg_out32(s, AND | SAB(a1, a0, a2));
1995 }
1996 break;
1997 case INDEX_op_and_i64:
1998 a0 = args[0], a1 = args[1], a2 = args[2];
1999 if (const_args[2]) {
2000 tcg_out_andi64(s, a0, a1, a2);
2001 } else {
2002 tcg_out32(s, AND | SAB(a1, a0, a2));
2003 }
2004 break;
2005 case INDEX_op_or_i64:
2006 case INDEX_op_or_i32:
2007 a0 = args[0], a1 = args[1], a2 = args[2];
2008 if (const_args[2]) {
2009 tcg_out_ori32(s, a0, a1, a2);
2010 } else {
2011 tcg_out32(s, OR | SAB(a1, a0, a2));
2012 }
2013 break;
2014 case INDEX_op_xor_i64:
2015 case INDEX_op_xor_i32:
2016 a0 = args[0], a1 = args[1], a2 = args[2];
2017 if (const_args[2]) {
2018 tcg_out_xori32(s, a0, a1, a2);
2019 } else {
2020 tcg_out32(s, XOR | SAB(a1, a0, a2));
2021 }
2022 break;
2023 case INDEX_op_andc_i32:
2024 a0 = args[0], a1 = args[1], a2 = args[2];
2025 if (const_args[2]) {
2026 tcg_out_andi32(s, a0, a1, ~a2);
2027 } else {
2028 tcg_out32(s, ANDC | SAB(a1, a0, a2));
2029 }
2030 break;
2031 case INDEX_op_andc_i64:
2032 a0 = args[0], a1 = args[1], a2 = args[2];
2033 if (const_args[2]) {
2034 tcg_out_andi64(s, a0, a1, ~a2);
2035 } else {
2036 tcg_out32(s, ANDC | SAB(a1, a0, a2));
2037 }
2038 break;
2039 case INDEX_op_orc_i32:
2040 if (const_args[2]) {
2041 tcg_out_ori32(s, args[0], args[1], ~args[2]);
2042 break;
2043 }
2044 /* FALLTHRU */
2045 case INDEX_op_orc_i64:
2046 tcg_out32(s, ORC | SAB(args[1], args[0], args[2]));
2047 break;
2048 case INDEX_op_eqv_i32:
2049 if (const_args[2]) {
2050 tcg_out_xori32(s, args[0], args[1], ~args[2]);
2051 break;
2052 }
2053 /* FALLTHRU */
2054 case INDEX_op_eqv_i64:
2055 tcg_out32(s, EQV | SAB(args[1], args[0], args[2]));
2056 break;
2057 case INDEX_op_nand_i32:
2058 case INDEX_op_nand_i64:
2059 tcg_out32(s, NAND | SAB(args[1], args[0], args[2]));
2060 break;
2061 case INDEX_op_nor_i32:
2062 case INDEX_op_nor_i64:
2063 tcg_out32(s, NOR | SAB(args[1], args[0], args[2]));
2064 break;
2065
2066 case INDEX_op_mul_i32:
2067 a0 = args[0], a1 = args[1], a2 = args[2];
2068 if (const_args[2]) {
2069 tcg_out32(s, MULLI | TAI(a0, a1, a2));
2070 } else {
2071 tcg_out32(s, MULLW | TAB(a0, a1, a2));
2072 }
2073 break;
2074
2075 case INDEX_op_div_i32:
2076 tcg_out32(s, DIVW | TAB(args[0], args[1], args[2]));
2077 break;
2078
2079 case INDEX_op_divu_i32:
2080 tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2]));
2081 break;
2082
2083 case INDEX_op_shl_i32:
2084 if (const_args[2]) {
2085 tcg_out_shli32(s, args[0], args[1], args[2]);
2086 } else {
2087 tcg_out32(s, SLW | SAB(args[1], args[0], args[2]));
2088 }
2089 break;
2090 case INDEX_op_shr_i32:
2091 if (const_args[2]) {
2092 tcg_out_shri32(s, args[0], args[1], args[2]);
2093 } else {
2094 tcg_out32(s, SRW | SAB(args[1], args[0], args[2]));
2095 }
2096 break;
2097 case INDEX_op_sar_i32:
2098 if (const_args[2]) {
2099 tcg_out32(s, SRAWI | RS(args[1]) | RA(args[0]) | SH(args[2]));
2100 } else {
2101 tcg_out32(s, SRAW | SAB(args[1], args[0], args[2]));
2102 }
2103 break;
2104 case INDEX_op_rotl_i32:
2105 if (const_args[2]) {
2106 tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31);
2107 } else {
2108 tcg_out32(s, RLWNM | SAB(args[1], args[0], args[2])
2109 | MB(0) | ME(31));
2110 }
2111 break;
2112 case INDEX_op_rotr_i32:
2113 if (const_args[2]) {
2114 tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31);
2115 } else {
2116 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32));
2117 tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0)
2118 | MB(0) | ME(31));
2119 }
2120 break;
2121
2122 case INDEX_op_brcond_i32:
2123 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
2124 arg_label(args[3]), TCG_TYPE_I32);
2125 break;
2126 case INDEX_op_brcond_i64:
2127 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
2128 arg_label(args[3]), TCG_TYPE_I64);
2129 break;
2130 case INDEX_op_brcond2_i32:
2131 tcg_out_brcond2(s, args, const_args);
2132 break;
2133
2134 case INDEX_op_neg_i32:
2135 case INDEX_op_neg_i64:
2136 tcg_out32(s, NEG | RT(args[0]) | RA(args[1]));
2137 break;
2138
2139 case INDEX_op_not_i32:
2140 case INDEX_op_not_i64:
2141 tcg_out32(s, NOR | SAB(args[1], args[0], args[1]));
2142 break;
2143
2144 case INDEX_op_add_i64:
2145 a0 = args[0], a1 = args[1], a2 = args[2];
2146 if (const_args[2]) {
2147 do_addi_64:
2148 tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2);
2149 } else {
2150 tcg_out32(s, ADD | TAB(a0, a1, a2));
2151 }
2152 break;
2153 case INDEX_op_sub_i64:
2154 a0 = args[0], a1 = args[1], a2 = args[2];
2155 if (const_args[1]) {
2156 if (const_args[2]) {
2157 tcg_out_movi(s, TCG_TYPE_I64, a0, a1 - a2);
2158 } else {
2159 tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
2160 }
2161 } else if (const_args[2]) {
2162 a2 = -a2;
2163 goto do_addi_64;
2164 } else {
2165 tcg_out32(s, SUBF | TAB(a0, a2, a1));
2166 }
2167 break;
2168
2169 case INDEX_op_shl_i64:
2170 if (const_args[2]) {
2171 tcg_out_shli64(s, args[0], args[1], args[2]);
2172 } else {
2173 tcg_out32(s, SLD | SAB(args[1], args[0], args[2]));
2174 }
2175 break;
2176 case INDEX_op_shr_i64:
2177 if (const_args[2]) {
2178 tcg_out_shri64(s, args[0], args[1], args[2]);
2179 } else {
2180 tcg_out32(s, SRD | SAB(args[1], args[0], args[2]));
2181 }
2182 break;
2183 case INDEX_op_sar_i64:
2184 if (const_args[2]) {
2185 int sh = SH(args[2] & 0x1f) | (((args[2] >> 5) & 1) << 1);
2186 tcg_out32(s, SRADI | RA(args[0]) | RS(args[1]) | sh);
2187 } else {
2188 tcg_out32(s, SRAD | SAB(args[1], args[0], args[2]));
2189 }
2190 break;
2191 case INDEX_op_rotl_i64:
2192 if (const_args[2]) {
2193 tcg_out_rld(s, RLDICL, args[0], args[1], args[2], 0);
2194 } else {
2195 tcg_out32(s, RLDCL | SAB(args[1], args[0], args[2]) | MB64(0));
2196 }
2197 break;
2198 case INDEX_op_rotr_i64:
2199 if (const_args[2]) {
2200 tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0);
2201 } else {
2202 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64));
2203 tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0));
2204 }
2205 break;
2206
2207 case INDEX_op_mul_i64:
2208 a0 = args[0], a1 = args[1], a2 = args[2];
2209 if (const_args[2]) {
2210 tcg_out32(s, MULLI | TAI(a0, a1, a2));
2211 } else {
2212 tcg_out32(s, MULLD | TAB(a0, a1, a2));
2213 }
2214 break;
2215 case INDEX_op_div_i64:
2216 tcg_out32(s, DIVD | TAB(args[0], args[1], args[2]));
2217 break;
2218 case INDEX_op_divu_i64:
2219 tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2]));
2220 break;
2221
2222 case INDEX_op_qemu_ld_i32:
2223 tcg_out_qemu_ld(s, args, false);
2224 break;
2225 case INDEX_op_qemu_ld_i64:
2226 tcg_out_qemu_ld(s, args, true);
2227 break;
2228 case INDEX_op_qemu_st_i32:
2229 tcg_out_qemu_st(s, args, false);
2230 break;
2231 case INDEX_op_qemu_st_i64:
2232 tcg_out_qemu_st(s, args, true);
2233 break;
2234
2235 case INDEX_op_ext8s_i32:
2236 case INDEX_op_ext8s_i64:
2237 c = EXTSB;
2238 goto gen_ext;
2239 case INDEX_op_ext16s_i32:
2240 case INDEX_op_ext16s_i64:
2241 c = EXTSH;
2242 goto gen_ext;
2243 case INDEX_op_ext_i32_i64:
2244 case INDEX_op_ext32s_i64:
2245 c = EXTSW;
2246 goto gen_ext;
2247 gen_ext:
2248 tcg_out32(s, c | RS(args[1]) | RA(args[0]));
2249 break;
2250 case INDEX_op_extu_i32_i64:
2251 tcg_out_ext32u(s, args[0], args[1]);
2252 break;
2253
2254 case INDEX_op_setcond_i32:
2255 tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
2256 const_args[2]);
2257 break;
2258 case INDEX_op_setcond_i64:
2259 tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2],
2260 const_args[2]);
2261 break;
2262 case INDEX_op_setcond2_i32:
2263 tcg_out_setcond2(s, args, const_args);
2264 break;
2265
2266 case INDEX_op_bswap16_i32:
2267 case INDEX_op_bswap16_i64:
2268 a0 = args[0], a1 = args[1];
2269 /* a1 = abcd */
2270 if (a0 != a1) {
2271 /* a0 = (a1 r<< 24) & 0xff # 000c */
2272 tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
2273 /* a0 = (a0 & ~0xff00) | (a1 r<< 8) & 0xff00 # 00dc */
2274 tcg_out_rlw(s, RLWIMI, a0, a1, 8, 16, 23);
2275 } else {
2276 /* r0 = (a1 r<< 8) & 0xff00 # 00d0 */
2277 tcg_out_rlw(s, RLWINM, TCG_REG_R0, a1, 8, 16, 23);
2278 /* a0 = (a1 r<< 24) & 0xff # 000c */
2279 tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
2280 /* a0 = a0 | r0 # 00dc */
2281 tcg_out32(s, OR | SAB(TCG_REG_R0, a0, a0));
2282 }
2283 break;
2284
2285 case INDEX_op_bswap32_i32:
2286 case INDEX_op_bswap32_i64:
2287 /* Stolen from gcc's builtin_bswap32 */
2288 a1 = args[1];
2289 a0 = args[0] == a1 ? TCG_REG_R0 : args[0];
2290
2291 /* a1 = args[1] # abcd */
2292 /* a0 = rotate_left (a1, 8) # bcda */
2293 tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
2294 /* a0 = (a0 & ~0xff000000) | ((a1 r<< 24) & 0xff000000) # dcda */
2295 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
2296 /* a0 = (a0 & ~0x0000ff00) | ((a1 r<< 24) & 0x0000ff00) # dcba */
2297 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
2298
2299 if (a0 == TCG_REG_R0) {
2300 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
2301 }
2302 break;
2303
2304 case INDEX_op_bswap64_i64:
2305 a0 = args[0], a1 = args[1], a2 = TCG_REG_R0;
2306 if (a0 == a1) {
2307 a0 = TCG_REG_R0;
2308 a2 = a1;
2309 }
2310
2311 /* a1 = # abcd efgh */
2312 /* a0 = rl32(a1, 8) # 0000 fghe */
2313 tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
2314 /* a0 = dep(a0, rl32(a1, 24), 0xff000000) # 0000 hghe */
2315 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
2316 /* a0 = dep(a0, rl32(a1, 24), 0x0000ff00) # 0000 hgfe */
2317 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
2318
2319 /* a0 = rl64(a0, 32) # hgfe 0000 */
2320 /* a2 = rl64(a1, 32) # efgh abcd */
2321 tcg_out_rld(s, RLDICL, a0, a0, 32, 0);
2322 tcg_out_rld(s, RLDICL, a2, a1, 32, 0);
2323
2324 /* a0 = dep(a0, rl32(a2, 8), 0xffffffff) # hgfe bcda */
2325 tcg_out_rlw(s, RLWIMI, a0, a2, 8, 0, 31);
2326 /* a0 = dep(a0, rl32(a2, 24), 0xff000000) # hgfe dcda */
2327 tcg_out_rlw(s, RLWIMI, a0, a2, 24, 0, 7);
2328 /* a0 = dep(a0, rl32(a2, 24), 0x0000ff00) # hgfe dcba */
2329 tcg_out_rlw(s, RLWIMI, a0, a2, 24, 16, 23);
2330
2331 if (a0 == 0) {
2332 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
2333 }
2334 break;
2335
2336 case INDEX_op_deposit_i32:
2337 if (const_args[2]) {
2338 uint32_t mask = ((2u << (args[4] - 1)) - 1) << args[3];
2339 tcg_out_andi32(s, args[0], args[0], ~mask);
2340 } else {
2341 tcg_out_rlw(s, RLWIMI, args[0], args[2], args[3],
2342 32 - args[3] - args[4], 31 - args[3]);
2343 }
2344 break;
2345 case INDEX_op_deposit_i64:
2346 if (const_args[2]) {
2347 uint64_t mask = ((2ull << (args[4] - 1)) - 1) << args[3];
2348 tcg_out_andi64(s, args[0], args[0], ~mask);
2349 } else {
2350 tcg_out_rld(s, RLDIMI, args[0], args[2], args[3],
2351 64 - args[3] - args[4]);
2352 }
2353 break;
2354
2355 case INDEX_op_movcond_i32:
2356 tcg_out_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], args[2],
2357 args[3], args[4], const_args[2]);
2358 break;
2359 case INDEX_op_movcond_i64:
2360 tcg_out_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], args[2],
2361 args[3], args[4], const_args[2]);
2362 break;
2363
2364 #if TCG_TARGET_REG_BITS == 64
2365 case INDEX_op_add2_i64:
2366 #else
2367 case INDEX_op_add2_i32:
2368 #endif
2369 /* Note that the CA bit is defined based on the word size of the
2370 environment. So in 64-bit mode it's always carry-out of bit 63.
2371 The fallback code using deposit works just as well for 32-bit. */
2372 a0 = args[0], a1 = args[1];
2373 if (a0 == args[3] || (!const_args[5] && a0 == args[5])) {
2374 a0 = TCG_REG_R0;
2375 }
2376 if (const_args[4]) {
2377 tcg_out32(s, ADDIC | TAI(a0, args[2], args[4]));
2378 } else {
2379 tcg_out32(s, ADDC | TAB(a0, args[2], args[4]));
2380 }
2381 if (const_args[5]) {
2382 tcg_out32(s, (args[5] ? ADDME : ADDZE) | RT(a1) | RA(args[3]));
2383 } else {
2384 tcg_out32(s, ADDE | TAB(a1, args[3], args[5]));
2385 }
2386 if (a0 != args[0]) {
2387 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
2388 }
2389 break;
2390
2391 #if TCG_TARGET_REG_BITS == 64
2392 case INDEX_op_sub2_i64:
2393 #else
2394 case INDEX_op_sub2_i32:
2395 #endif
2396 a0 = args[0], a1 = args[1];
2397 if (a0 == args[5] || (!const_args[3] && a0 == args[3])) {
2398 a0 = TCG_REG_R0;
2399 }
2400 if (const_args[2]) {
2401 tcg_out32(s, SUBFIC | TAI(a0, args[4], args[2]));
2402 } else {
2403 tcg_out32(s, SUBFC | TAB(a0, args[4], args[2]));
2404 }
2405 if (const_args[3]) {
2406 tcg_out32(s, (args[3] ? SUBFME : SUBFZE) | RT(a1) | RA(args[5]));
2407 } else {
2408 tcg_out32(s, SUBFE | TAB(a1, args[5], args[3]));
2409 }
2410 if (a0 != args[0]) {
2411 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
2412 }
2413 break;
2414
2415 case INDEX_op_muluh_i32:
2416 tcg_out32(s, MULHWU | TAB(args[0], args[1], args[2]));
2417 break;
2418 case INDEX_op_mulsh_i32:
2419 tcg_out32(s, MULHW | TAB(args[0], args[1], args[2]));
2420 break;
2421 case INDEX_op_muluh_i64:
2422 tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2]));
2423 break;
2424 case INDEX_op_mulsh_i64:
2425 tcg_out32(s, MULHD | TAB(args[0], args[1], args[2]));
2426 break;
2427
2428 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
2429 case INDEX_op_mov_i64:
2430 case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */
2431 case INDEX_op_movi_i64:
2432 case INDEX_op_call: /* Always emitted via tcg_out_call. */
2433 default:
2434 tcg_abort();
2435 }
2436 }
2437
2438 static const TCGTargetOpDef ppc_op_defs[] = {
2439 { INDEX_op_exit_tb, { } },
2440 { INDEX_op_goto_tb, { } },
2441 { INDEX_op_br, { } },
2442
2443 { INDEX_op_ld8u_i32, { "r", "r" } },
2444 { INDEX_op_ld8s_i32, { "r", "r" } },
2445 { INDEX_op_ld16u_i32, { "r", "r" } },
2446 { INDEX_op_ld16s_i32, { "r", "r" } },
2447 { INDEX_op_ld_i32, { "r", "r" } },
2448
2449 { INDEX_op_st8_i32, { "r", "r" } },
2450 { INDEX_op_st16_i32, { "r", "r" } },
2451 { INDEX_op_st_i32, { "r", "r" } },
2452
2453 { INDEX_op_add_i32, { "r", "r", "ri" } },
2454 { INDEX_op_mul_i32, { "r", "r", "rI" } },
2455 { INDEX_op_div_i32, { "r", "r", "r" } },
2456 { INDEX_op_divu_i32, { "r", "r", "r" } },
2457 { INDEX_op_sub_i32, { "r", "rI", "ri" } },
2458 { INDEX_op_and_i32, { "r", "r", "ri" } },
2459 { INDEX_op_or_i32, { "r", "r", "ri" } },
2460 { INDEX_op_xor_i32, { "r", "r", "ri" } },
2461 { INDEX_op_andc_i32, { "r", "r", "ri" } },
2462 { INDEX_op_orc_i32, { "r", "r", "ri" } },
2463 { INDEX_op_eqv_i32, { "r", "r", "ri" } },
2464 { INDEX_op_nand_i32, { "r", "r", "r" } },
2465 { INDEX_op_nor_i32, { "r", "r", "r" } },
2466
2467 { INDEX_op_shl_i32, { "r", "r", "ri" } },
2468 { INDEX_op_shr_i32, { "r", "r", "ri" } },
2469 { INDEX_op_sar_i32, { "r", "r", "ri" } },
2470 { INDEX_op_rotl_i32, { "r", "r", "ri" } },
2471 { INDEX_op_rotr_i32, { "r", "r", "ri" } },
2472
2473 { INDEX_op_neg_i32, { "r", "r" } },
2474 { INDEX_op_not_i32, { "r", "r" } },
2475 { INDEX_op_ext8s_i32, { "r", "r" } },
2476 { INDEX_op_ext16s_i32, { "r", "r" } },
2477 { INDEX_op_bswap16_i32, { "r", "r" } },
2478 { INDEX_op_bswap32_i32, { "r", "r" } },
2479
2480 { INDEX_op_brcond_i32, { "r", "ri" } },
2481 { INDEX_op_setcond_i32, { "r", "r", "ri" } },
2482 { INDEX_op_movcond_i32, { "r", "r", "ri", "rZ", "rZ" } },
2483
2484 { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
2485
2486 { INDEX_op_muluh_i32, { "r", "r", "r" } },
2487 { INDEX_op_mulsh_i32, { "r", "r", "r" } },
2488
2489 #if TCG_TARGET_REG_BITS == 64
2490 { INDEX_op_ld8u_i64, { "r", "r" } },
2491 { INDEX_op_ld8s_i64, { "r", "r" } },
2492 { INDEX_op_ld16u_i64, { "r", "r" } },
2493 { INDEX_op_ld16s_i64, { "r", "r" } },
2494 { INDEX_op_ld32u_i64, { "r", "r" } },
2495 { INDEX_op_ld32s_i64, { "r", "r" } },
2496 { INDEX_op_ld_i64, { "r", "r" } },
2497
2498 { INDEX_op_st8_i64, { "r", "r" } },
2499 { INDEX_op_st16_i64, { "r", "r" } },
2500 { INDEX_op_st32_i64, { "r", "r" } },
2501 { INDEX_op_st_i64, { "r", "r" } },
2502
2503 { INDEX_op_add_i64, { "r", "r", "rT" } },
2504 { INDEX_op_sub_i64, { "r", "rI", "rT" } },
2505 { INDEX_op_and_i64, { "r", "r", "ri" } },
2506 { INDEX_op_or_i64, { "r", "r", "rU" } },
2507 { INDEX_op_xor_i64, { "r", "r", "rU" } },
2508 { INDEX_op_andc_i64, { "r", "r", "ri" } },
2509 { INDEX_op_orc_i64, { "r", "r", "r" } },
2510 { INDEX_op_eqv_i64, { "r", "r", "r" } },
2511 { INDEX_op_nand_i64, { "r", "r", "r" } },
2512 { INDEX_op_nor_i64, { "r", "r", "r" } },
2513
2514 { INDEX_op_shl_i64, { "r", "r", "ri" } },
2515 { INDEX_op_shr_i64, { "r", "r", "ri" } },
2516 { INDEX_op_sar_i64, { "r", "r", "ri" } },
2517 { INDEX_op_rotl_i64, { "r", "r", "ri" } },
2518 { INDEX_op_rotr_i64, { "r", "r", "ri" } },
2519
2520 { INDEX_op_mul_i64, { "r", "r", "rI" } },
2521 { INDEX_op_div_i64, { "r", "r", "r" } },
2522 { INDEX_op_divu_i64, { "r", "r", "r" } },
2523
2524 { INDEX_op_neg_i64, { "r", "r" } },
2525 { INDEX_op_not_i64, { "r", "r" } },
2526 { INDEX_op_ext8s_i64, { "r", "r" } },
2527 { INDEX_op_ext16s_i64, { "r", "r" } },
2528 { INDEX_op_ext32s_i64, { "r", "r" } },
2529 { INDEX_op_ext_i32_i64, { "r", "r" } },
2530 { INDEX_op_extu_i32_i64, { "r", "r" } },
2531 { INDEX_op_bswap16_i64, { "r", "r" } },
2532 { INDEX_op_bswap32_i64, { "r", "r" } },
2533 { INDEX_op_bswap64_i64, { "r", "r" } },
2534
2535 { INDEX_op_brcond_i64, { "r", "ri" } },
2536 { INDEX_op_setcond_i64, { "r", "r", "ri" } },
2537 { INDEX_op_movcond_i64, { "r", "r", "ri", "rZ", "rZ" } },
2538
2539 { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
2540
2541 { INDEX_op_mulsh_i64, { "r", "r", "r" } },
2542 { INDEX_op_muluh_i64, { "r", "r", "r" } },
2543 #endif
2544
2545 #if TCG_TARGET_REG_BITS == 32
2546 { INDEX_op_brcond2_i32, { "r", "r", "ri", "ri" } },
2547 { INDEX_op_setcond2_i32, { "r", "r", "r", "ri", "ri" } },
2548 #endif
2549
2550 #if TCG_TARGET_REG_BITS == 64
2551 { INDEX_op_add2_i64, { "r", "r", "r", "r", "rI", "rZM" } },
2552 { INDEX_op_sub2_i64, { "r", "r", "rI", "rZM", "r", "r" } },
2553 #else
2554 { INDEX_op_add2_i32, { "r", "r", "r", "r", "rI", "rZM" } },
2555 { INDEX_op_sub2_i32, { "r", "r", "rI", "rZM", "r", "r" } },
2556 #endif
2557
2558 #if TCG_TARGET_REG_BITS == 64
2559 { INDEX_op_qemu_ld_i32, { "r", "L" } },
2560 { INDEX_op_qemu_st_i32, { "S", "S" } },
2561 { INDEX_op_qemu_ld_i64, { "r", "L" } },
2562 { INDEX_op_qemu_st_i64, { "S", "S" } },
2563 #elif TARGET_LONG_BITS == 32
2564 { INDEX_op_qemu_ld_i32, { "r", "L" } },
2565 { INDEX_op_qemu_st_i32, { "S", "S" } },
2566 { INDEX_op_qemu_ld_i64, { "L", "L", "L" } },
2567 { INDEX_op_qemu_st_i64, { "S", "S", "S" } },
2568 #else
2569 { INDEX_op_qemu_ld_i32, { "r", "L", "L" } },
2570 { INDEX_op_qemu_st_i32, { "S", "S", "S" } },
2571 { INDEX_op_qemu_ld_i64, { "L", "L", "L", "L" } },
2572 { INDEX_op_qemu_st_i64, { "S", "S", "S", "S" } },
2573 #endif
2574
2575 { -1 },
2576 };
2577
2578 static void tcg_target_init(TCGContext *s)
2579 {
2580 unsigned long hwcap = qemu_getauxval(AT_HWCAP);
2581 if (hwcap & PPC_FEATURE_ARCH_2_06) {
2582 have_isa_2_06 = true;
2583 }
2584
2585 tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
2586 tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
2587 tcg_regset_set32(tcg_target_call_clobber_regs, 0,
2588 (1 << TCG_REG_R0) |
2589 (1 << TCG_REG_R2) |
2590 (1 << TCG_REG_R3) |
2591 (1 << TCG_REG_R4) |
2592 (1 << TCG_REG_R5) |
2593 (1 << TCG_REG_R6) |
2594 (1 << TCG_REG_R7) |
2595 (1 << TCG_REG_R8) |
2596 (1 << TCG_REG_R9) |
2597 (1 << TCG_REG_R10) |
2598 (1 << TCG_REG_R11) |
2599 (1 << TCG_REG_R12));
2600
2601 tcg_regset_clear(s->reserved_regs);
2602 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */
2603 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */
2604 #if defined(_CALL_SYSV)
2605 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* toc pointer */
2606 #endif
2607 #if defined(_CALL_SYSV) || TCG_TARGET_REG_BITS == 64
2608 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */
2609 #endif
2610 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); /* mem temp */
2611 if (USE_REG_RA) {
2612 tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA); /* return addr */
2613 }
2614
2615 tcg_add_target_add_op_defs(ppc_op_defs);
2616 }
2617
2618 #ifdef __ELF__
2619 typedef struct {
2620 DebugFrameCIE cie;
2621 DebugFrameFDEHeader fde;
2622 uint8_t fde_def_cfa[4];
2623 uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3];
2624 } DebugFrame;
2625
2626 /* We're expecting a 2 byte uleb128 encoded value. */
2627 QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
2628
2629 #if TCG_TARGET_REG_BITS == 64
2630 # define ELF_HOST_MACHINE EM_PPC64
2631 #else
2632 # define ELF_HOST_MACHINE EM_PPC
2633 #endif
2634
2635 static DebugFrame debug_frame = {
2636 .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
2637 .cie.id = -1,
2638 .cie.version = 1,
2639 .cie.code_align = 1,
2640 .cie.data_align = (-SZR & 0x7f), /* sleb128 -SZR */
2641 .cie.return_column = 65,
2642
2643 /* Total FDE size does not include the "len" member. */
2644 .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
2645
2646 .fde_def_cfa = {
2647 12, TCG_REG_R1, /* DW_CFA_def_cfa r1, ... */
2648 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
2649 (FRAME_SIZE >> 7)
2650 },
2651 .fde_reg_ofs = {
2652 /* DW_CFA_offset_extended_sf, lr, LR_OFFSET */
2653 0x11, 65, (LR_OFFSET / -SZR) & 0x7f,
2654 }
2655 };
2656
2657 void tcg_register_jit(void *buf, size_t buf_size)
2658 {
2659 uint8_t *p = &debug_frame.fde_reg_ofs[3];
2660 int i;
2661
2662 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) {
2663 p[0] = 0x80 + tcg_target_callee_save_regs[i];
2664 p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * SZR)) / SZR;
2665 }
2666
2667 debug_frame.fde.func_start = (uintptr_t)buf;
2668 debug_frame.fde.func_len = buf_size;
2669
2670 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
2671 }
2672 #endif /* __ELF__ */
2673
2674 static size_t dcache_bsize = 16;
2675 static size_t icache_bsize = 16;
2676
2677 void flush_icache_range(uintptr_t start, uintptr_t stop)
2678 {
2679 uintptr_t p, start1, stop1;
2680 size_t dsize = dcache_bsize;
2681 size_t isize = icache_bsize;
2682
2683 start1 = start & ~(dsize - 1);
2684 stop1 = (stop + dsize - 1) & ~(dsize - 1);
2685 for (p = start1; p < stop1; p += dsize) {
2686 asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
2687 }
2688 asm volatile ("sync" : : : "memory");
2689
2690 start &= start & ~(isize - 1);
2691 stop1 = (stop + isize - 1) & ~(isize - 1);
2692 for (p = start1; p < stop1; p += isize) {
2693 asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
2694 }
2695 asm volatile ("sync" : : : "memory");
2696 asm volatile ("isync" : : : "memory");
2697 }
2698
2699 #if defined _AIX
2700 #include <sys/systemcfg.h>
2701
2702 static void __attribute__((constructor)) tcg_cache_init(void)
2703 {
2704 icache_bsize = _system_configuration.icache_line;
2705 dcache_bsize = _system_configuration.dcache_line;
2706 }
2707
2708 #elif defined __linux__
2709 static void __attribute__((constructor)) tcg_cache_init(void)
2710 {
2711 unsigned long dsize = qemu_getauxval(AT_DCACHEBSIZE);
2712 unsigned long isize = qemu_getauxval(AT_ICACHEBSIZE);
2713
2714 if (dsize == 0 || isize == 0) {
2715 if (dsize == 0) {
2716 fprintf(stderr, "getauxval AT_DCACHEBSIZE failed\n");
2717 }
2718 if (isize == 0) {
2719 fprintf(stderr, "getauxval AT_ICACHEBSIZE failed\n");
2720 }
2721 exit(1);
2722 }
2723 dcache_bsize = dsize;
2724 icache_bsize = isize;
2725 }
2726
2727 #elif defined __APPLE__
2728 #include <sys/sysctl.h>
2729
2730 static void __attribute__((constructor)) tcg_cache_init(void)
2731 {
2732 size_t len;
2733 unsigned cacheline;
2734 int name[2] = { CTL_HW, HW_CACHELINE };
2735
2736 len = sizeof(cacheline);
2737 if (sysctl(name, 2, &cacheline, &len, NULL, 0)) {
2738 perror("sysctl CTL_HW HW_CACHELINE failed");
2739 exit(1);
2740 }
2741 dcache_bsize = cacheline;
2742 icache_bsize = cacheline;
2743 }
2744
2745 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
2746 #include <sys/sysctl.h>
2747
2748 static void __attribute__((constructor)) tcg_cache_init(void)
2749 {
2750 size_t len = 4;
2751 unsigned cacheline;
2752
2753 if (sysctlbyname ("machdep.cacheline_size", &cacheline, &len, NULL, 0)) {
2754 fprintf(stderr, "sysctlbyname machdep.cacheline_size failed: %s\n",
2755 strerror(errno));
2756 exit(1);
2757 }
2758 dcache_bsize = cacheline;
2759 icache_bsize = cacheline;
2760 }
2761 #endif
QEmu