?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
target/ppc: Don't compile ppc_tlb_invalid_all without TCG
[qemu/armbru.git] / tcg / tcg.c
blob5150ed700eeeb5b0691af5190f354274c9f56f33
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
27
28 #include "qemu/osdep.h"
29
30 /* Define to jump the ELF file used to communicate with GDB. */
31 #undef DEBUG_JIT
32
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
38 #include "qemu/cacheflush.h"
39
40 /* Note: the long term plan is to reduce the dependencies on the QEMU
41 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43 #define NO_CPU_IO_DEFS
44
45 #include "exec/exec-all.h"
46 #include "tcg/tcg-op.h"
47
48 #if UINTPTR_MAX == UINT32_MAX
49 # define ELF_CLASS ELFCLASS32
50 #else
51 # define ELF_CLASS ELFCLASS64
52 #endif
53 #ifdef HOST_WORDS_BIGENDIAN
54 # define ELF_DATA ELFDATA2MSB
55 #else
56 # define ELF_DATA ELFDATA2LSB
57 #endif
58
59 #include "elf.h"
60 #include "exec/log.h"
61 #include "tcg-internal.h"
62
63 #ifdef CONFIG_TCG_INTERPRETER
64 #include <ffi.h>
65 #endif
66
67 /* Forward declarations for functions declared in tcg-target.c.inc and
68 used here. */
69 static void tcg_target_init(TCGContext *s);
70 static void tcg_target_qemu_prologue(TCGContext *s);
71 static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
72 intptr_t value, intptr_t addend);
73
74 /* The CIE and FDE header definitions will be common to all hosts. */
75 typedef struct {
76 uint32_t len __attribute__((aligned((sizeof(void *)))));
77 uint32_t id;
78 uint8_t version;
79 char augmentation[1];
80 uint8_t code_align;
81 uint8_t data_align;
82 uint8_t return_column;
83 } DebugFrameCIE;
84
85 typedef struct QEMU_PACKED {
86 uint32_t len __attribute__((aligned((sizeof(void *)))));
87 uint32_t cie_offset;
88 uintptr_t func_start;
89 uintptr_t func_len;
90 } DebugFrameFDEHeader;
91
92 typedef struct QEMU_PACKED {
93 DebugFrameCIE cie;
94 DebugFrameFDEHeader fde;
95 } DebugFrameHeader;
96
97 static void tcg_register_jit_int(const void *buf, size_t size,
98 const void *debug_frame,
99 size_t debug_frame_size)
100 __attribute__((unused));
101
102 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
103 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
104 intptr_t arg2);
105 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
106 static void tcg_out_movi(TCGContext *s, TCGType type,
107 TCGReg ret, tcg_target_long arg);
108 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
109 const TCGArg args[TCG_MAX_OP_ARGS],
110 const int const_args[TCG_MAX_OP_ARGS]);
111 #if TCG_TARGET_MAYBE_vec
112 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, TCGReg src);
114 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg base, intptr_t offset);
116 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, int64_t arg);
118 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
119 unsigned vecl, unsigned vece,
120 const TCGArg args[TCG_MAX_OP_ARGS],
121 const int const_args[TCG_MAX_OP_ARGS]);
122 #else
123 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
124 TCGReg dst, TCGReg src)
125 {
126 g_assert_not_reached();
127 }
128 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
129 TCGReg dst, TCGReg base, intptr_t offset)
130 {
131 g_assert_not_reached();
132 }
133 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
134 TCGReg dst, int64_t arg)
135 {
136 g_assert_not_reached();
137 }
138 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
139 unsigned vecl, unsigned vece,
140 const TCGArg args[TCG_MAX_OP_ARGS],
141 const int const_args[TCG_MAX_OP_ARGS])
142 {
143 g_assert_not_reached();
144 }
145 #endif
146 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
147 intptr_t arg2);
148 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
149 TCGReg base, intptr_t ofs);
150 #ifdef CONFIG_TCG_INTERPRETER
151 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
152 ffi_cif *cif);
153 #else
154 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
155 #endif
156 static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
157 #ifdef TCG_TARGET_NEED_LDST_LABELS
158 static int tcg_out_ldst_finalize(TCGContext *s);
159 #endif
160
161 TCGContext tcg_init_ctx;
162 __thread TCGContext *tcg_ctx;
163
164 TCGContext **tcg_ctxs;
165 unsigned int tcg_cur_ctxs;
166 unsigned int tcg_max_ctxs;
167 TCGv_env cpu_env = 0;
168 const void *tcg_code_gen_epilogue;
169 uintptr_t tcg_splitwx_diff;
170
171 #ifndef CONFIG_TCG_INTERPRETER
172 tcg_prologue_fn *tcg_qemu_tb_exec;
173 #endif
174
175 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
176 static TCGRegSet tcg_target_call_clobber_regs;
177
178 #if TCG_TARGET_INSN_UNIT_SIZE == 1
179 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
180 {
181 *s->code_ptr++ = v;
182 }
183
184 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
185 uint8_t v)
186 {
187 *p = v;
188 }
189 #endif
190
191 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
192 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
193 {
194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
195 *s->code_ptr++ = v;
196 } else {
197 tcg_insn_unit *p = s->code_ptr;
198 memcpy(p, &v, sizeof(v));
199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
200 }
201 }
202
203 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
204 uint16_t v)
205 {
206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
207 *p = v;
208 } else {
209 memcpy(p, &v, sizeof(v));
210 }
211 }
212 #endif
213
214 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
215 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
216 {
217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
218 *s->code_ptr++ = v;
219 } else {
220 tcg_insn_unit *p = s->code_ptr;
221 memcpy(p, &v, sizeof(v));
222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
223 }
224 }
225
226 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
227 uint32_t v)
228 {
229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
230 *p = v;
231 } else {
232 memcpy(p, &v, sizeof(v));
233 }
234 }
235 #endif
236
237 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
238 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
239 {
240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
241 *s->code_ptr++ = v;
242 } else {
243 tcg_insn_unit *p = s->code_ptr;
244 memcpy(p, &v, sizeof(v));
245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
246 }
247 }
248
249 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
250 uint64_t v)
251 {
252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
253 *p = v;
254 } else {
255 memcpy(p, &v, sizeof(v));
256 }
257 }
258 #endif
259
260 /* label relocation processing */
261
262 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
263 TCGLabel *l, intptr_t addend)
264 {
265 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
266
267 r->type = type;
268 r->ptr = code_ptr;
269 r->addend = addend;
270 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
271 }
272
273 static void tcg_out_label(TCGContext *s, TCGLabel *l)
274 {
275 tcg_debug_assert(!l->has_value);
276 l->has_value = 1;
277 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
278 }
279
280 TCGLabel *gen_new_label(void)
281 {
282 TCGContext *s = tcg_ctx;
283 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
284
285 memset(l, 0, sizeof(TCGLabel));
286 l->id = s->nb_labels++;
287 QSIMPLEQ_INIT(&l->relocs);
288
289 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
290
291 return l;
292 }
293
294 static bool tcg_resolve_relocs(TCGContext *s)
295 {
296 TCGLabel *l;
297
298 QSIMPLEQ_FOREACH(l, &s->labels, next) {
299 TCGRelocation *r;
300 uintptr_t value = l->u.value;
301
302 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
303 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
304 return false;
305 }
306 }
307 }
308 return true;
309 }
310
311 static void set_jmp_reset_offset(TCGContext *s, int which)
312 {
313 /*
314 * We will check for overflow at the end of the opcode loop in
315 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
316 */
317 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
318 }
319
320 /* Signal overflow, starting over with fewer guest insns. */
321 static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
322 {
323 siglongjmp(s->jmp_trans, -2);
324 }
325
326 #define C_PFX1(P, A) P##A
327 #define C_PFX2(P, A, B) P##A##_##B
328 #define C_PFX3(P, A, B, C) P##A##_##B##_##C
329 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
330 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
331 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
332
333 /* Define an enumeration for the various combinations. */
334
335 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
336 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
337 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
338 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
339
340 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
341 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
342 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
343 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
344
345 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
346
347 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
348 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
349 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
350 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
351
352 typedef enum {
353 #include "tcg-target-con-set.h"
354 } TCGConstraintSetIndex;
355
356 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
357
358 #undef C_O0_I1
359 #undef C_O0_I2
360 #undef C_O0_I3
361 #undef C_O0_I4
362 #undef C_O1_I1
363 #undef C_O1_I2
364 #undef C_O1_I3
365 #undef C_O1_I4
366 #undef C_N1_I2
367 #undef C_O2_I1
368 #undef C_O2_I2
369 #undef C_O2_I3
370 #undef C_O2_I4
371
372 /* Put all of the constraint sets into an array, indexed by the enum. */
373
374 #define C_O0_I1(I1) { .args_ct_str = { #I1 } },
375 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
376 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
377 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
378
379 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
380 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
381 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
382 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
383
384 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
385
386 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
387 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
388 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
389 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
390
391 static const TCGTargetOpDef constraint_sets[] = {
392 #include "tcg-target-con-set.h"
393 };
394
395
396 #undef C_O0_I1
397 #undef C_O0_I2
398 #undef C_O0_I3
399 #undef C_O0_I4
400 #undef C_O1_I1
401 #undef C_O1_I2
402 #undef C_O1_I3
403 #undef C_O1_I4
404 #undef C_N1_I2
405 #undef C_O2_I1
406 #undef C_O2_I2
407 #undef C_O2_I3
408 #undef C_O2_I4
409
410 /* Expand the enumerator to be returned from tcg_target_op_def(). */
411
412 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
413 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
414 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
415 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
416
417 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
418 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
419 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
420 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
421
422 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
423
424 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
425 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
426 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
427 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
428
429 #include "tcg-target.c.inc"
430
431 static void alloc_tcg_plugin_context(TCGContext *s)
432 {
433 #ifdef CONFIG_PLUGIN
434 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
435 s->plugin_tb->insns =
436 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
437 #endif
438 }
439
440 /*
441 * All TCG threads except the parent (i.e. the one that called tcg_context_init
442 * and registered the target's TCG globals) must register with this function
443 * before initiating translation.
444 *
445 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
446 * of tcg_region_init() for the reasoning behind this.
447 *
448 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
449 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
450 * is not used anymore for translation once this function is called.
451 *
452 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
453 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
454 */
455 #ifdef CONFIG_USER_ONLY
456 void tcg_register_thread(void)
457 {
458 tcg_ctx = &tcg_init_ctx;
459 }
460 #else
461 void tcg_register_thread(void)
462 {
463 TCGContext *s = g_malloc(sizeof(*s));
464 unsigned int i, n;
465
466 *s = tcg_init_ctx;
467
468 /* Relink mem_base. */
469 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
470 if (tcg_init_ctx.temps[i].mem_base) {
471 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
472 tcg_debug_assert(b >= 0 && b < n);
473 s->temps[i].mem_base = &s->temps[b];
474 }
475 }
476
477 /* Claim an entry in tcg_ctxs */
478 n = qatomic_fetch_inc(&tcg_cur_ctxs);
479 g_assert(n < tcg_max_ctxs);
480 qatomic_set(&tcg_ctxs[n], s);
481
482 if (n > 0) {
483 alloc_tcg_plugin_context(s);
484 tcg_region_initial_alloc(s);
485 }
486
487 tcg_ctx = s;
488 }
489 #endif /* !CONFIG_USER_ONLY */
490
491 /* pool based memory allocation */
492 void *tcg_malloc_internal(TCGContext *s, int size)
493 {
494 TCGPool *p;
495 int pool_size;
496
497 if (size > TCG_POOL_CHUNK_SIZE) {
498 /* big malloc: insert a new pool (XXX: could optimize) */
499 p = g_malloc(sizeof(TCGPool) + size);
500 p->size = size;
501 p->next = s->pool_first_large;
502 s->pool_first_large = p;
503 return p->data;
504 } else {
505 p = s->pool_current;
506 if (!p) {
507 p = s->pool_first;
508 if (!p)
509 goto new_pool;
510 } else {
511 if (!p->next) {
512 new_pool:
513 pool_size = TCG_POOL_CHUNK_SIZE;
514 p = g_malloc(sizeof(TCGPool) + pool_size);
515 p->size = pool_size;
516 p->next = NULL;
517 if (s->pool_current)
518 s->pool_current->next = p;
519 else
520 s->pool_first = p;
521 } else {
522 p = p->next;
523 }
524 }
525 }
526 s->pool_current = p;
527 s->pool_cur = p->data + size;
528 s->pool_end = p->data + p->size;
529 return p->data;
530 }
531
532 void tcg_pool_reset(TCGContext *s)
533 {
534 TCGPool *p, *t;
535 for (p = s->pool_first_large; p; p = t) {
536 t = p->next;
537 g_free(p);
538 }
539 s->pool_first_large = NULL;
540 s->pool_cur = s->pool_end = NULL;
541 s->pool_current = NULL;
542 }
543
544 #include "exec/helper-proto.h"
545
546 static const TCGHelperInfo all_helpers[] = {
547 #include "exec/helper-tcg.h"
548 };
549 static GHashTable *helper_table;
550
551 #ifdef CONFIG_TCG_INTERPRETER
552 static GHashTable *ffi_table;
553
554 static ffi_type * const typecode_to_ffi[8] = {
555 [dh_typecode_void] = &ffi_type_void,
556 [dh_typecode_i32] = &ffi_type_uint32,
557 [dh_typecode_s32] = &ffi_type_sint32,
558 [dh_typecode_i64] = &ffi_type_uint64,
559 [dh_typecode_s64] = &ffi_type_sint64,
560 [dh_typecode_ptr] = &ffi_type_pointer,
561 };
562 #endif
563
564 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
565 static void process_op_defs(TCGContext *s);
566 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
567 TCGReg reg, const char *name);
568
569 static void tcg_context_init(unsigned max_cpus)
570 {
571 TCGContext *s = &tcg_init_ctx;
572 int op, total_args, n, i;
573 TCGOpDef *def;
574 TCGArgConstraint *args_ct;
575 TCGTemp *ts;
576
577 memset(s, 0, sizeof(*s));
578 s->nb_globals = 0;
579
580 /* Count total number of arguments and allocate the corresponding
581 space */
582 total_args = 0;
583 for(op = 0; op < NB_OPS; op++) {
584 def = &tcg_op_defs[op];
585 n = def->nb_iargs + def->nb_oargs;
586 total_args += n;
587 }
588
589 args_ct = g_new0(TCGArgConstraint, total_args);
590
591 for(op = 0; op < NB_OPS; op++) {
592 def = &tcg_op_defs[op];
593 def->args_ct = args_ct;
594 n = def->nb_iargs + def->nb_oargs;
595 args_ct += n;
596 }
597
598 /* Register helpers. */
599 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
600 helper_table = g_hash_table_new(NULL, NULL);
601
602 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
603 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
604 (gpointer)&all_helpers[i]);
605 }
606
607 #ifdef CONFIG_TCG_INTERPRETER
608 /* g_direct_hash/equal for direct comparisons on uint32_t. */
609 ffi_table = g_hash_table_new(NULL, NULL);
610 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
611 struct {
612 ffi_cif cif;
613 ffi_type *args[];
614 } *ca;
615 uint32_t typemask = all_helpers[i].typemask;
616 gpointer hash = (gpointer)(uintptr_t)typemask;
617 ffi_status status;
618 int nargs;
619
620 if (g_hash_table_lookup(ffi_table, hash)) {
621 continue;
622 }
623
624 /* Ignoring the return type, find the last non-zero field. */
625 nargs = 32 - clz32(typemask >> 3);
626 nargs = DIV_ROUND_UP(nargs, 3);
627
628 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
629 ca->cif.rtype = typecode_to_ffi[typemask & 7];
630 ca->cif.nargs = nargs;
631
632 if (nargs != 0) {
633 ca->cif.arg_types = ca->args;
634 for (i = 0; i < nargs; ++i) {
635 int typecode = extract32(typemask, (i + 1) * 3, 3);
636 ca->args[i] = typecode_to_ffi[typecode];
637 }
638 }
639
640 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
641 ca->cif.rtype, ca->cif.arg_types);
642 assert(status == FFI_OK);
643
644 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
645 }
646 #endif
647
648 tcg_target_init(s);
649 process_op_defs(s);
650
651 /* Reverse the order of the saved registers, assuming they're all at
652 the start of tcg_target_reg_alloc_order. */
653 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
654 int r = tcg_target_reg_alloc_order[n];
655 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
656 break;
657 }
658 }
659 for (i = 0; i < n; ++i) {
660 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
661 }
662 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
663 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
664 }
665
666 alloc_tcg_plugin_context(s);
667
668 tcg_ctx = s;
669 /*
670 * In user-mode we simply share the init context among threads, since we
671 * use a single region. See the documentation tcg_region_init() for the
672 * reasoning behind this.
673 * In softmmu we will have at most max_cpus TCG threads.
674 */
675 #ifdef CONFIG_USER_ONLY
676 tcg_ctxs = &tcg_ctx;
677 tcg_cur_ctxs = 1;
678 tcg_max_ctxs = 1;
679 #else
680 tcg_max_ctxs = max_cpus;
681 tcg_ctxs = g_new0(TCGContext *, max_cpus);
682 #endif
683
684 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
685 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
686 cpu_env = temp_tcgv_ptr(ts);
687 }
688
689 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
690 {
691 tcg_context_init(max_cpus);
692 tcg_region_init(tb_size, splitwx, max_cpus);
693 }
694
695 /*
696 * Allocate TBs right before their corresponding translated code, making
697 * sure that TBs and code are on different cache lines.
698 */
699 TranslationBlock *tcg_tb_alloc(TCGContext *s)
700 {
701 uintptr_t align = qemu_icache_linesize;
702 TranslationBlock *tb;
703 void *next;
704
705 retry:
706 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
707 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
708
709 if (unlikely(next > s->code_gen_highwater)) {
710 if (tcg_region_alloc(s)) {
711 return NULL;
712 }
713 goto retry;
714 }
715 qatomic_set(&s->code_gen_ptr, next);
716 s->data_gen_ptr = NULL;
717 return tb;
718 }
719
720 void tcg_prologue_init(TCGContext *s)
721 {
722 size_t prologue_size;
723
724 s->code_ptr = s->code_gen_ptr;
725 s->code_buf = s->code_gen_ptr;
726 s->data_gen_ptr = NULL;
727
728 #ifndef CONFIG_TCG_INTERPRETER
729 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
730 #endif
731
732 #ifdef TCG_TARGET_NEED_POOL_LABELS
733 s->pool_labels = NULL;
734 #endif
735
736 qemu_thread_jit_write();
737 /* Generate the prologue. */
738 tcg_target_qemu_prologue(s);
739
740 #ifdef TCG_TARGET_NEED_POOL_LABELS
741 /* Allow the prologue to put e.g. guest_base into a pool entry. */
742 {
743 int result = tcg_out_pool_finalize(s);
744 tcg_debug_assert(result == 0);
745 }
746 #endif
747
748 prologue_size = tcg_current_code_size(s);
749
750 #ifndef CONFIG_TCG_INTERPRETER
751 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
752 (uintptr_t)s->code_buf, prologue_size);
753 #endif
754
755 tcg_region_prologue_set(s);
756
757 #ifdef DEBUG_DISAS
758 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
759 FILE *logfile = qemu_log_lock();
760 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
761 if (s->data_gen_ptr) {
762 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
763 size_t data_size = prologue_size - code_size;
764 size_t i;
765
766 log_disas(s->code_gen_ptr, code_size);
767
768 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
769 if (sizeof(tcg_target_ulong) == 8) {
770 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
771 (uintptr_t)s->data_gen_ptr + i,
772 *(uint64_t *)(s->data_gen_ptr + i));
773 } else {
774 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
775 (uintptr_t)s->data_gen_ptr + i,
776 *(uint32_t *)(s->data_gen_ptr + i));
777 }
778 }
779 } else {
780 log_disas(s->code_gen_ptr, prologue_size);
781 }
782 qemu_log("\n");
783 qemu_log_flush();
784 qemu_log_unlock(logfile);
785 }
786 #endif
787
788 #ifndef CONFIG_TCG_INTERPRETER
789 /*
790 * Assert that goto_ptr is implemented completely, setting an epilogue.
791 * For tci, we use NULL as the signal to return from the interpreter,
792 * so skip this check.
793 */
794 if (TCG_TARGET_HAS_goto_ptr) {
795 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
796 }
797 #endif
798 }
799
800 void tcg_func_start(TCGContext *s)
801 {
802 tcg_pool_reset(s);
803 s->nb_temps = s->nb_globals;
804
805 /* No temps have been previously allocated for size or locality. */
806 memset(s->free_temps, 0, sizeof(s->free_temps));
807
808 /* No constant temps have been previously allocated. */
809 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
810 if (s->const_table[i]) {
811 g_hash_table_remove_all(s->const_table[i]);
812 }
813 }
814
815 s->nb_ops = 0;
816 s->nb_labels = 0;
817 s->current_frame_offset = s->frame_start;
818
819 #ifdef CONFIG_DEBUG_TCG
820 s->goto_tb_issue_mask = 0;
821 #endif
822
823 QTAILQ_INIT(&s->ops);
824 QTAILQ_INIT(&s->free_ops);
825 QSIMPLEQ_INIT(&s->labels);
826 }
827
828 static TCGTemp *tcg_temp_alloc(TCGContext *s)
829 {
830 int n = s->nb_temps++;
831
832 if (n >= TCG_MAX_TEMPS) {
833 tcg_raise_tb_overflow(s);
834 }
835 return memset(&s->temps[n], 0, sizeof(TCGTemp));
836 }
837
838 static TCGTemp *tcg_global_alloc(TCGContext *s)
839 {
840 TCGTemp *ts;
841
842 tcg_debug_assert(s->nb_globals == s->nb_temps);
843 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
844 s->nb_globals++;
845 ts = tcg_temp_alloc(s);
846 ts->kind = TEMP_GLOBAL;
847
848 return ts;
849 }
850
851 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
852 TCGReg reg, const char *name)
853 {
854 TCGTemp *ts;
855
856 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
857 tcg_abort();
858 }
859
860 ts = tcg_global_alloc(s);
861 ts->base_type = type;
862 ts->type = type;
863 ts->kind = TEMP_FIXED;
864 ts->reg = reg;
865 ts->name = name;
866 tcg_regset_set_reg(s->reserved_regs, reg);
867
868 return ts;
869 }
870
871 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
872 {
873 s->frame_start = start;
874 s->frame_end = start + size;
875 s->frame_temp
876 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
877 }
878
879 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
880 intptr_t offset, const char *name)
881 {
882 TCGContext *s = tcg_ctx;
883 TCGTemp *base_ts = tcgv_ptr_temp(base);
884 TCGTemp *ts = tcg_global_alloc(s);
885 int indirect_reg = 0, bigendian = 0;
886 #ifdef HOST_WORDS_BIGENDIAN
887 bigendian = 1;
888 #endif
889
890 switch (base_ts->kind) {
891 case TEMP_FIXED:
892 break;
893 case TEMP_GLOBAL:
894 /* We do not support double-indirect registers. */
895 tcg_debug_assert(!base_ts->indirect_reg);
896 base_ts->indirect_base = 1;
897 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
898 ? 2 : 1);
899 indirect_reg = 1;
900 break;
901 default:
902 g_assert_not_reached();
903 }
904
905 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
906 TCGTemp *ts2 = tcg_global_alloc(s);
907 char buf[64];
908
909 ts->base_type = TCG_TYPE_I64;
910 ts->type = TCG_TYPE_I32;
911 ts->indirect_reg = indirect_reg;
912 ts->mem_allocated = 1;
913 ts->mem_base = base_ts;
914 ts->mem_offset = offset + bigendian * 4;
915 pstrcpy(buf, sizeof(buf), name);
916 pstrcat(buf, sizeof(buf), "_0");
917 ts->name = strdup(buf);
918
919 tcg_debug_assert(ts2 == ts + 1);
920 ts2->base_type = TCG_TYPE_I64;
921 ts2->type = TCG_TYPE_I32;
922 ts2->indirect_reg = indirect_reg;
923 ts2->mem_allocated = 1;
924 ts2->mem_base = base_ts;
925 ts2->mem_offset = offset + (1 - bigendian) * 4;
926 pstrcpy(buf, sizeof(buf), name);
927 pstrcat(buf, sizeof(buf), "_1");
928 ts2->name = strdup(buf);
929 } else {
930 ts->base_type = type;
931 ts->type = type;
932 ts->indirect_reg = indirect_reg;
933 ts->mem_allocated = 1;
934 ts->mem_base = base_ts;
935 ts->mem_offset = offset;
936 ts->name = name;
937 }
938 return ts;
939 }
940
941 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
942 {
943 TCGContext *s = tcg_ctx;
944 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
945 TCGTemp *ts;
946 int idx, k;
947
948 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
949 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
950 if (idx < TCG_MAX_TEMPS) {
951 /* There is already an available temp with the right type. */
952 clear_bit(idx, s->free_temps[k].l);
953
954 ts = &s->temps[idx];
955 ts->temp_allocated = 1;
956 tcg_debug_assert(ts->base_type == type);
957 tcg_debug_assert(ts->kind == kind);
958 } else {
959 ts = tcg_temp_alloc(s);
960 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
961 TCGTemp *ts2 = tcg_temp_alloc(s);
962
963 ts->base_type = type;
964 ts->type = TCG_TYPE_I32;
965 ts->temp_allocated = 1;
966 ts->kind = kind;
967
968 tcg_debug_assert(ts2 == ts + 1);
969 ts2->base_type = TCG_TYPE_I64;
970 ts2->type = TCG_TYPE_I32;
971 ts2->temp_allocated = 1;
972 ts2->kind = kind;
973 } else {
974 ts->base_type = type;
975 ts->type = type;
976 ts->temp_allocated = 1;
977 ts->kind = kind;
978 }
979 }
980
981 #if defined(CONFIG_DEBUG_TCG)
982 s->temps_in_use++;
983 #endif
984 return ts;
985 }
986
987 TCGv_vec tcg_temp_new_vec(TCGType type)
988 {
989 TCGTemp *t;
990
991 #ifdef CONFIG_DEBUG_TCG
992 switch (type) {
993 case TCG_TYPE_V64:
994 assert(TCG_TARGET_HAS_v64);
995 break;
996 case TCG_TYPE_V128:
997 assert(TCG_TARGET_HAS_v128);
998 break;
999 case TCG_TYPE_V256:
1000 assert(TCG_TARGET_HAS_v256);
1001 break;
1002 default:
1003 g_assert_not_reached();
1004 }
1005 #endif
1006
1007 t = tcg_temp_new_internal(type, 0);
1008 return temp_tcgv_vec(t);
1009 }
1010
1011 /* Create a new temp of the same type as an existing temp. */
1012 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1013 {
1014 TCGTemp *t = tcgv_vec_temp(match);
1015
1016 tcg_debug_assert(t->temp_allocated != 0);
1017
1018 t = tcg_temp_new_internal(t->base_type, 0);
1019 return temp_tcgv_vec(t);
1020 }
1021
1022 void tcg_temp_free_internal(TCGTemp *ts)
1023 {
1024 TCGContext *s = tcg_ctx;
1025 int k, idx;
1026
1027 /* In order to simplify users of tcg_constant_*, silently ignore free. */
1028 if (ts->kind == TEMP_CONST) {
1029 return;
1030 }
1031
1032 #if defined(CONFIG_DEBUG_TCG)
1033 s->temps_in_use--;
1034 if (s->temps_in_use < 0) {
1035 fprintf(stderr, "More temporaries freed than allocated!\n");
1036 }
1037 #endif
1038
1039 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
1040 tcg_debug_assert(ts->temp_allocated != 0);
1041 ts->temp_allocated = 0;
1042
1043 idx = temp_idx(ts);
1044 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
1045 set_bit(idx, s->free_temps[k].l);
1046 }
1047
1048 TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1049 {
1050 TCGContext *s = tcg_ctx;
1051 GHashTable *h = s->const_table[type];
1052 TCGTemp *ts;
1053
1054 if (h == NULL) {
1055 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1056 s->const_table[type] = h;
1057 }
1058
1059 ts = g_hash_table_lookup(h, &val);
1060 if (ts == NULL) {
1061 ts = tcg_temp_alloc(s);
1062
1063 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1064 TCGTemp *ts2 = tcg_temp_alloc(s);
1065
1066 ts->base_type = TCG_TYPE_I64;
1067 ts->type = TCG_TYPE_I32;
1068 ts->kind = TEMP_CONST;
1069 ts->temp_allocated = 1;
1070 /*
1071 * Retain the full value of the 64-bit constant in the low
1072 * part, so that the hash table works. Actual uses will
1073 * truncate the value to the low part.
1074 */
1075 ts->val = val;
1076
1077 tcg_debug_assert(ts2 == ts + 1);
1078 ts2->base_type = TCG_TYPE_I64;
1079 ts2->type = TCG_TYPE_I32;
1080 ts2->kind = TEMP_CONST;
1081 ts2->temp_allocated = 1;
1082 ts2->val = val >> 32;
1083 } else {
1084 ts->base_type = type;
1085 ts->type = type;
1086 ts->kind = TEMP_CONST;
1087 ts->temp_allocated = 1;
1088 ts->val = val;
1089 }
1090 g_hash_table_insert(h, &ts->val, ts);
1091 }
1092
1093 return ts;
1094 }
1095
1096 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1097 {
1098 val = dup_const(vece, val);
1099 return temp_tcgv_vec(tcg_constant_internal(type, val));
1100 }
1101
1102 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1103 {
1104 TCGTemp *t = tcgv_vec_temp(match);
1105
1106 tcg_debug_assert(t->temp_allocated != 0);
1107 return tcg_constant_vec(t->base_type, vece, val);
1108 }
1109
1110 TCGv_i32 tcg_const_i32(int32_t val)
1111 {
1112 TCGv_i32 t0;
1113 t0 = tcg_temp_new_i32();
1114 tcg_gen_movi_i32(t0, val);
1115 return t0;
1116 }
1117
1118 TCGv_i64 tcg_const_i64(int64_t val)
1119 {
1120 TCGv_i64 t0;
1121 t0 = tcg_temp_new_i64();
1122 tcg_gen_movi_i64(t0, val);
1123 return t0;
1124 }
1125
1126 TCGv_i32 tcg_const_local_i32(int32_t val)
1127 {
1128 TCGv_i32 t0;
1129 t0 = tcg_temp_local_new_i32();
1130 tcg_gen_movi_i32(t0, val);
1131 return t0;
1132 }
1133
1134 TCGv_i64 tcg_const_local_i64(int64_t val)
1135 {
1136 TCGv_i64 t0;
1137 t0 = tcg_temp_local_new_i64();
1138 tcg_gen_movi_i64(t0, val);
1139 return t0;
1140 }
1141
1142 #if defined(CONFIG_DEBUG_TCG)
1143 void tcg_clear_temp_count(void)
1144 {
1145 TCGContext *s = tcg_ctx;
1146 s->temps_in_use = 0;
1147 }
1148
1149 int tcg_check_temp_count(void)
1150 {
1151 TCGContext *s = tcg_ctx;
1152 if (s->temps_in_use) {
1153 /* Clear the count so that we don't give another
1154 * warning immediately next time around.
1155 */
1156 s->temps_in_use = 0;
1157 return 1;
1158 }
1159 return 0;
1160 }
1161 #endif
1162
1163 /* Return true if OP may appear in the opcode stream.
1164 Test the runtime variable that controls each opcode. */
1165 bool tcg_op_supported(TCGOpcode op)
1166 {
1167 const bool have_vec
1168 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1169
1170 switch (op) {
1171 case INDEX_op_discard:
1172 case INDEX_op_set_label:
1173 case INDEX_op_call:
1174 case INDEX_op_br:
1175 case INDEX_op_mb:
1176 case INDEX_op_insn_start:
1177 case INDEX_op_exit_tb:
1178 case INDEX_op_goto_tb:
1179 case INDEX_op_qemu_ld_i32:
1180 case INDEX_op_qemu_st_i32:
1181 case INDEX_op_qemu_ld_i64:
1182 case INDEX_op_qemu_st_i64:
1183 return true;
1184
1185 case INDEX_op_qemu_st8_i32:
1186 return TCG_TARGET_HAS_qemu_st8_i32;
1187
1188 case INDEX_op_goto_ptr:
1189 return TCG_TARGET_HAS_goto_ptr;
1190
1191 case INDEX_op_mov_i32:
1192 case INDEX_op_setcond_i32:
1193 case INDEX_op_brcond_i32:
1194 case INDEX_op_ld8u_i32:
1195 case INDEX_op_ld8s_i32:
1196 case INDEX_op_ld16u_i32:
1197 case INDEX_op_ld16s_i32:
1198 case INDEX_op_ld_i32:
1199 case INDEX_op_st8_i32:
1200 case INDEX_op_st16_i32:
1201 case INDEX_op_st_i32:
1202 case INDEX_op_add_i32:
1203 case INDEX_op_sub_i32:
1204 case INDEX_op_mul_i32:
1205 case INDEX_op_and_i32:
1206 case INDEX_op_or_i32:
1207 case INDEX_op_xor_i32:
1208 case INDEX_op_shl_i32:
1209 case INDEX_op_shr_i32:
1210 case INDEX_op_sar_i32:
1211 return true;
1212
1213 case INDEX_op_movcond_i32:
1214 return TCG_TARGET_HAS_movcond_i32;
1215 case INDEX_op_div_i32:
1216 case INDEX_op_divu_i32:
1217 return TCG_TARGET_HAS_div_i32;
1218 case INDEX_op_rem_i32:
1219 case INDEX_op_remu_i32:
1220 return TCG_TARGET_HAS_rem_i32;
1221 case INDEX_op_div2_i32:
1222 case INDEX_op_divu2_i32:
1223 return TCG_TARGET_HAS_div2_i32;
1224 case INDEX_op_rotl_i32:
1225 case INDEX_op_rotr_i32:
1226 return TCG_TARGET_HAS_rot_i32;
1227 case INDEX_op_deposit_i32:
1228 return TCG_TARGET_HAS_deposit_i32;
1229 case INDEX_op_extract_i32:
1230 return TCG_TARGET_HAS_extract_i32;
1231 case INDEX_op_sextract_i32:
1232 return TCG_TARGET_HAS_sextract_i32;
1233 case INDEX_op_extract2_i32:
1234 return TCG_TARGET_HAS_extract2_i32;
1235 case INDEX_op_add2_i32:
1236 return TCG_TARGET_HAS_add2_i32;
1237 case INDEX_op_sub2_i32:
1238 return TCG_TARGET_HAS_sub2_i32;
1239 case INDEX_op_mulu2_i32:
1240 return TCG_TARGET_HAS_mulu2_i32;
1241 case INDEX_op_muls2_i32:
1242 return TCG_TARGET_HAS_muls2_i32;
1243 case INDEX_op_muluh_i32:
1244 return TCG_TARGET_HAS_muluh_i32;
1245 case INDEX_op_mulsh_i32:
1246 return TCG_TARGET_HAS_mulsh_i32;
1247 case INDEX_op_ext8s_i32:
1248 return TCG_TARGET_HAS_ext8s_i32;
1249 case INDEX_op_ext16s_i32:
1250 return TCG_TARGET_HAS_ext16s_i32;
1251 case INDEX_op_ext8u_i32:
1252 return TCG_TARGET_HAS_ext8u_i32;
1253 case INDEX_op_ext16u_i32:
1254 return TCG_TARGET_HAS_ext16u_i32;
1255 case INDEX_op_bswap16_i32:
1256 return TCG_TARGET_HAS_bswap16_i32;
1257 case INDEX_op_bswap32_i32:
1258 return TCG_TARGET_HAS_bswap32_i32;
1259 case INDEX_op_not_i32:
1260 return TCG_TARGET_HAS_not_i32;
1261 case INDEX_op_neg_i32:
1262 return TCG_TARGET_HAS_neg_i32;
1263 case INDEX_op_andc_i32:
1264 return TCG_TARGET_HAS_andc_i32;
1265 case INDEX_op_orc_i32:
1266 return TCG_TARGET_HAS_orc_i32;
1267 case INDEX_op_eqv_i32:
1268 return TCG_TARGET_HAS_eqv_i32;
1269 case INDEX_op_nand_i32:
1270 return TCG_TARGET_HAS_nand_i32;
1271 case INDEX_op_nor_i32:
1272 return TCG_TARGET_HAS_nor_i32;
1273 case INDEX_op_clz_i32:
1274 return TCG_TARGET_HAS_clz_i32;
1275 case INDEX_op_ctz_i32:
1276 return TCG_TARGET_HAS_ctz_i32;
1277 case INDEX_op_ctpop_i32:
1278 return TCG_TARGET_HAS_ctpop_i32;
1279
1280 case INDEX_op_brcond2_i32:
1281 case INDEX_op_setcond2_i32:
1282 return TCG_TARGET_REG_BITS == 32;
1283
1284 case INDEX_op_mov_i64:
1285 case INDEX_op_setcond_i64:
1286 case INDEX_op_brcond_i64:
1287 case INDEX_op_ld8u_i64:
1288 case INDEX_op_ld8s_i64:
1289 case INDEX_op_ld16u_i64:
1290 case INDEX_op_ld16s_i64:
1291 case INDEX_op_ld32u_i64:
1292 case INDEX_op_ld32s_i64:
1293 case INDEX_op_ld_i64:
1294 case INDEX_op_st8_i64:
1295 case INDEX_op_st16_i64:
1296 case INDEX_op_st32_i64:
1297 case INDEX_op_st_i64:
1298 case INDEX_op_add_i64:
1299 case INDEX_op_sub_i64:
1300 case INDEX_op_mul_i64:
1301 case INDEX_op_and_i64:
1302 case INDEX_op_or_i64:
1303 case INDEX_op_xor_i64:
1304 case INDEX_op_shl_i64:
1305 case INDEX_op_shr_i64:
1306 case INDEX_op_sar_i64:
1307 case INDEX_op_ext_i32_i64:
1308 case INDEX_op_extu_i32_i64:
1309 return TCG_TARGET_REG_BITS == 64;
1310
1311 case INDEX_op_movcond_i64:
1312 return TCG_TARGET_HAS_movcond_i64;
1313 case INDEX_op_div_i64:
1314 case INDEX_op_divu_i64:
1315 return TCG_TARGET_HAS_div_i64;
1316 case INDEX_op_rem_i64:
1317 case INDEX_op_remu_i64:
1318 return TCG_TARGET_HAS_rem_i64;
1319 case INDEX_op_div2_i64:
1320 case INDEX_op_divu2_i64:
1321 return TCG_TARGET_HAS_div2_i64;
1322 case INDEX_op_rotl_i64:
1323 case INDEX_op_rotr_i64:
1324 return TCG_TARGET_HAS_rot_i64;
1325 case INDEX_op_deposit_i64:
1326 return TCG_TARGET_HAS_deposit_i64;
1327 case INDEX_op_extract_i64:
1328 return TCG_TARGET_HAS_extract_i64;
1329 case INDEX_op_sextract_i64:
1330 return TCG_TARGET_HAS_sextract_i64;
1331 case INDEX_op_extract2_i64:
1332 return TCG_TARGET_HAS_extract2_i64;
1333 case INDEX_op_extrl_i64_i32:
1334 return TCG_TARGET_HAS_extrl_i64_i32;
1335 case INDEX_op_extrh_i64_i32:
1336 return TCG_TARGET_HAS_extrh_i64_i32;
1337 case INDEX_op_ext8s_i64:
1338 return TCG_TARGET_HAS_ext8s_i64;
1339 case INDEX_op_ext16s_i64:
1340 return TCG_TARGET_HAS_ext16s_i64;
1341 case INDEX_op_ext32s_i64:
1342 return TCG_TARGET_HAS_ext32s_i64;
1343 case INDEX_op_ext8u_i64:
1344 return TCG_TARGET_HAS_ext8u_i64;
1345 case INDEX_op_ext16u_i64:
1346 return TCG_TARGET_HAS_ext16u_i64;
1347 case INDEX_op_ext32u_i64:
1348 return TCG_TARGET_HAS_ext32u_i64;
1349 case INDEX_op_bswap16_i64:
1350 return TCG_TARGET_HAS_bswap16_i64;
1351 case INDEX_op_bswap32_i64:
1352 return TCG_TARGET_HAS_bswap32_i64;
1353 case INDEX_op_bswap64_i64:
1354 return TCG_TARGET_HAS_bswap64_i64;
1355 case INDEX_op_not_i64:
1356 return TCG_TARGET_HAS_not_i64;
1357 case INDEX_op_neg_i64:
1358 return TCG_TARGET_HAS_neg_i64;
1359 case INDEX_op_andc_i64:
1360 return TCG_TARGET_HAS_andc_i64;
1361 case INDEX_op_orc_i64:
1362 return TCG_TARGET_HAS_orc_i64;
1363 case INDEX_op_eqv_i64:
1364 return TCG_TARGET_HAS_eqv_i64;
1365 case INDEX_op_nand_i64:
1366 return TCG_TARGET_HAS_nand_i64;
1367 case INDEX_op_nor_i64:
1368 return TCG_TARGET_HAS_nor_i64;
1369 case INDEX_op_clz_i64:
1370 return TCG_TARGET_HAS_clz_i64;
1371 case INDEX_op_ctz_i64:
1372 return TCG_TARGET_HAS_ctz_i64;
1373 case INDEX_op_ctpop_i64:
1374 return TCG_TARGET_HAS_ctpop_i64;
1375 case INDEX_op_add2_i64:
1376 return TCG_TARGET_HAS_add2_i64;
1377 case INDEX_op_sub2_i64:
1378 return TCG_TARGET_HAS_sub2_i64;
1379 case INDEX_op_mulu2_i64:
1380 return TCG_TARGET_HAS_mulu2_i64;
1381 case INDEX_op_muls2_i64:
1382 return TCG_TARGET_HAS_muls2_i64;
1383 case INDEX_op_muluh_i64:
1384 return TCG_TARGET_HAS_muluh_i64;
1385 case INDEX_op_mulsh_i64:
1386 return TCG_TARGET_HAS_mulsh_i64;
1387
1388 case INDEX_op_mov_vec:
1389 case INDEX_op_dup_vec:
1390 case INDEX_op_dupm_vec:
1391 case INDEX_op_ld_vec:
1392 case INDEX_op_st_vec:
1393 case INDEX_op_add_vec:
1394 case INDEX_op_sub_vec:
1395 case INDEX_op_and_vec:
1396 case INDEX_op_or_vec:
1397 case INDEX_op_xor_vec:
1398 case INDEX_op_cmp_vec:
1399 return have_vec;
1400 case INDEX_op_dup2_vec:
1401 return have_vec && TCG_TARGET_REG_BITS == 32;
1402 case INDEX_op_not_vec:
1403 return have_vec && TCG_TARGET_HAS_not_vec;
1404 case INDEX_op_neg_vec:
1405 return have_vec && TCG_TARGET_HAS_neg_vec;
1406 case INDEX_op_abs_vec:
1407 return have_vec && TCG_TARGET_HAS_abs_vec;
1408 case INDEX_op_andc_vec:
1409 return have_vec && TCG_TARGET_HAS_andc_vec;
1410 case INDEX_op_orc_vec:
1411 return have_vec && TCG_TARGET_HAS_orc_vec;
1412 case INDEX_op_mul_vec:
1413 return have_vec && TCG_TARGET_HAS_mul_vec;
1414 case INDEX_op_shli_vec:
1415 case INDEX_op_shri_vec:
1416 case INDEX_op_sari_vec:
1417 return have_vec && TCG_TARGET_HAS_shi_vec;
1418 case INDEX_op_shls_vec:
1419 case INDEX_op_shrs_vec:
1420 case INDEX_op_sars_vec:
1421 return have_vec && TCG_TARGET_HAS_shs_vec;
1422 case INDEX_op_shlv_vec:
1423 case INDEX_op_shrv_vec:
1424 case INDEX_op_sarv_vec:
1425 return have_vec && TCG_TARGET_HAS_shv_vec;
1426 case INDEX_op_rotli_vec:
1427 return have_vec && TCG_TARGET_HAS_roti_vec;
1428 case INDEX_op_rotls_vec:
1429 return have_vec && TCG_TARGET_HAS_rots_vec;
1430 case INDEX_op_rotlv_vec:
1431 case INDEX_op_rotrv_vec:
1432 return have_vec && TCG_TARGET_HAS_rotv_vec;
1433 case INDEX_op_ssadd_vec:
1434 case INDEX_op_usadd_vec:
1435 case INDEX_op_sssub_vec:
1436 case INDEX_op_ussub_vec:
1437 return have_vec && TCG_TARGET_HAS_sat_vec;
1438 case INDEX_op_smin_vec:
1439 case INDEX_op_umin_vec:
1440 case INDEX_op_smax_vec:
1441 case INDEX_op_umax_vec:
1442 return have_vec && TCG_TARGET_HAS_minmax_vec;
1443 case INDEX_op_bitsel_vec:
1444 return have_vec && TCG_TARGET_HAS_bitsel_vec;
1445 case INDEX_op_cmpsel_vec:
1446 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
1447
1448 default:
1449 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1450 return true;
1451 }
1452 }
1453
1454 /* Note: we convert the 64 bit args to 32 bit and do some alignment
1455 and endian swap. Maybe it would be better to do the alignment
1456 and endian swap in tcg_reg_alloc_call(). */
1457 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
1458 {
1459 int i, real_args, nb_rets, pi;
1460 unsigned typemask;
1461 const TCGHelperInfo *info;
1462 TCGOp *op;
1463
1464 info = g_hash_table_lookup(helper_table, (gpointer)func);
1465 typemask = info->typemask;
1466
1467 #ifdef CONFIG_PLUGIN
1468 /* detect non-plugin helpers */
1469 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1470 tcg_ctx->plugin_insn->calls_helpers = true;
1471 }
1472 #endif
1473
1474 #if defined(__sparc__) && !defined(__arch64__) \
1475 && !defined(CONFIG_TCG_INTERPRETER)
1476 /* We have 64-bit values in one register, but need to pass as two
1477 separate parameters. Split them. */
1478 int orig_typemask = typemask;
1479 int orig_nargs = nargs;
1480 TCGv_i64 retl, reth;
1481 TCGTemp *split_args[MAX_OPC_PARAM];
1482
1483 retl = NULL;
1484 reth = NULL;
1485 typemask = 0;
1486 for (i = real_args = 0; i < nargs; ++i) {
1487 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1488 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1489
1490 if (is_64bit) {
1491 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1492 TCGv_i32 h = tcg_temp_new_i32();
1493 TCGv_i32 l = tcg_temp_new_i32();
1494 tcg_gen_extr_i64_i32(l, h, orig);
1495 split_args[real_args++] = tcgv_i32_temp(h);
1496 typemask |= dh_typecode_i32 << (real_args * 3);
1497 split_args[real_args++] = tcgv_i32_temp(l);
1498 typemask |= dh_typecode_i32 << (real_args * 3);
1499 } else {
1500 split_args[real_args++] = args[i];
1501 typemask |= argtype << (real_args * 3);
1502 }
1503 }
1504 nargs = real_args;
1505 args = split_args;
1506 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1507 for (i = 0; i < nargs; ++i) {
1508 int argtype = extract32(typemask, (i + 1) * 3, 3);
1509 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1510 bool is_signed = argtype & 1;
1511
1512 if (is_32bit) {
1513 TCGv_i64 temp = tcg_temp_new_i64();
1514 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1515 if (is_signed) {
1516 tcg_gen_ext32s_i64(temp, orig);
1517 } else {
1518 tcg_gen_ext32u_i64(temp, orig);
1519 }
1520 args[i] = tcgv_i64_temp(temp);
1521 }
1522 }
1523 #endif /* TCG_TARGET_EXTEND_ARGS */
1524
1525 op = tcg_emit_op(INDEX_op_call);
1526
1527 pi = 0;
1528 if (ret != NULL) {
1529 #if defined(__sparc__) && !defined(__arch64__) \
1530 && !defined(CONFIG_TCG_INTERPRETER)
1531 if ((typemask & 6) == dh_typecode_i64) {
1532 /* The 32-bit ABI is going to return the 64-bit value in
1533 the %o0/%o1 register pair. Prepare for this by using
1534 two return temporaries, and reassemble below. */
1535 retl = tcg_temp_new_i64();
1536 reth = tcg_temp_new_i64();
1537 op->args[pi++] = tcgv_i64_arg(reth);
1538 op->args[pi++] = tcgv_i64_arg(retl);
1539 nb_rets = 2;
1540 } else {
1541 op->args[pi++] = temp_arg(ret);
1542 nb_rets = 1;
1543 }
1544 #else
1545 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
1546 #ifdef HOST_WORDS_BIGENDIAN
1547 op->args[pi++] = temp_arg(ret + 1);
1548 op->args[pi++] = temp_arg(ret);
1549 #else
1550 op->args[pi++] = temp_arg(ret);
1551 op->args[pi++] = temp_arg(ret + 1);
1552 #endif
1553 nb_rets = 2;
1554 } else {
1555 op->args[pi++] = temp_arg(ret);
1556 nb_rets = 1;
1557 }
1558 #endif
1559 } else {
1560 nb_rets = 0;
1561 }
1562 TCGOP_CALLO(op) = nb_rets;
1563
1564 real_args = 0;
1565 for (i = 0; i < nargs; i++) {
1566 int argtype = extract32(typemask, (i + 1) * 3, 3);
1567 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1568 bool want_align = false;
1569
1570 #if defined(CONFIG_TCG_INTERPRETER)
1571 /*
1572 * Align all arguments, so that they land in predictable places
1573 * for passing off to ffi_call.
1574 */
1575 want_align = true;
1576 #elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1577 /* Some targets want aligned 64 bit args */
1578 want_align = is_64bit;
1579 #endif
1580
1581 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1582 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1583 real_args++;
1584 }
1585
1586 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
1587 /*
1588 * If stack grows up, then we will be placing successive
1589 * arguments at lower addresses, which means we need to
1590 * reverse the order compared to how we would normally
1591 * treat either big or little-endian. For those arguments
1592 * that will wind up in registers, this still works for
1593 * HPPA (the only current STACK_GROWSUP target) since the
1594 * argument registers are *also* allocated in decreasing
1595 * order. If another such target is added, this logic may
1596 * have to get more complicated to differentiate between
1597 * stack arguments and register arguments.
1598 */
1599 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
1600 op->args[pi++] = temp_arg(args[i] + 1);
1601 op->args[pi++] = temp_arg(args[i]);
1602 #else
1603 op->args[pi++] = temp_arg(args[i]);
1604 op->args[pi++] = temp_arg(args[i] + 1);
1605 #endif
1606 real_args += 2;
1607 continue;
1608 }
1609
1610 op->args[pi++] = temp_arg(args[i]);
1611 real_args++;
1612 }
1613 op->args[pi++] = (uintptr_t)func;
1614 op->args[pi++] = (uintptr_t)info;
1615 TCGOP_CALLI(op) = real_args;
1616
1617 /* Make sure the fields didn't overflow. */
1618 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
1619 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
1620
1621 #if defined(__sparc__) && !defined(__arch64__) \
1622 && !defined(CONFIG_TCG_INTERPRETER)
1623 /* Free all of the parts we allocated above. */
1624 for (i = real_args = 0; i < orig_nargs; ++i) {
1625 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1626 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1627
1628 if (is_64bit) {
1629 tcg_temp_free_internal(args[real_args++]);
1630 tcg_temp_free_internal(args[real_args++]);
1631 } else {
1632 real_args++;
1633 }
1634 }
1635 if ((orig_typemask & 6) == dh_typecode_i64) {
1636 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1637 Note that describing these as TCGv_i64 eliminates an unnecessary
1638 zero-extension that tcg_gen_concat_i32_i64 would create. */
1639 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
1640 tcg_temp_free_i64(retl);
1641 tcg_temp_free_i64(reth);
1642 }
1643 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1644 for (i = 0; i < nargs; ++i) {
1645 int argtype = extract32(typemask, (i + 1) * 3, 3);
1646 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1647
1648 if (is_32bit) {
1649 tcg_temp_free_internal(args[i]);
1650 }
1651 }
1652 #endif /* TCG_TARGET_EXTEND_ARGS */
1653 }
1654
1655 static void tcg_reg_alloc_start(TCGContext *s)
1656 {
1657 int i, n;
1658
1659 for (i = 0, n = s->nb_temps; i < n; i++) {
1660 TCGTemp *ts = &s->temps[i];
1661 TCGTempVal val = TEMP_VAL_MEM;
1662
1663 switch (ts->kind) {
1664 case TEMP_CONST:
1665 val = TEMP_VAL_CONST;
1666 break;
1667 case TEMP_FIXED:
1668 val = TEMP_VAL_REG;
1669 break;
1670 case TEMP_GLOBAL:
1671 break;
1672 case TEMP_NORMAL:
1673 val = TEMP_VAL_DEAD;
1674 /* fall through */
1675 case TEMP_LOCAL:
1676 ts->mem_allocated = 0;
1677 break;
1678 default:
1679 g_assert_not_reached();
1680 }
1681 ts->val_type = val;
1682 }
1683
1684 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
1685 }
1686
1687 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1688 TCGTemp *ts)
1689 {
1690 int idx = temp_idx(ts);
1691
1692 switch (ts->kind) {
1693 case TEMP_FIXED:
1694 case TEMP_GLOBAL:
1695 pstrcpy(buf, buf_size, ts->name);
1696 break;
1697 case TEMP_LOCAL:
1698 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
1699 break;
1700 case TEMP_NORMAL:
1701 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
1702 break;
1703 case TEMP_CONST:
1704 switch (ts->type) {
1705 case TCG_TYPE_I32:
1706 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1707 break;
1708 #if TCG_TARGET_REG_BITS > 32
1709 case TCG_TYPE_I64:
1710 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1711 break;
1712 #endif
1713 case TCG_TYPE_V64:
1714 case TCG_TYPE_V128:
1715 case TCG_TYPE_V256:
1716 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1717 64 << (ts->type - TCG_TYPE_V64), ts->val);
1718 break;
1719 default:
1720 g_assert_not_reached();
1721 }
1722 break;
1723 }
1724 return buf;
1725 }
1726
1727 static char *tcg_get_arg_str(TCGContext *s, char *buf,
1728 int buf_size, TCGArg arg)
1729 {
1730 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
1731 }
1732
1733 static const char * const cond_name[] =
1734 {
1735 [TCG_COND_NEVER] = "never",
1736 [TCG_COND_ALWAYS] = "always",
1737 [TCG_COND_EQ] = "eq",
1738 [TCG_COND_NE] = "ne",
1739 [TCG_COND_LT] = "lt",
1740 [TCG_COND_GE] = "ge",
1741 [TCG_COND_LE] = "le",
1742 [TCG_COND_GT] = "gt",
1743 [TCG_COND_LTU] = "ltu",
1744 [TCG_COND_GEU] = "geu",
1745 [TCG_COND_LEU] = "leu",
1746 [TCG_COND_GTU] = "gtu"
1747 };
1748
1749 static const char * const ldst_name[] =
1750 {
1751 [MO_UB] = "ub",
1752 [MO_SB] = "sb",
1753 [MO_LEUW] = "leuw",
1754 [MO_LESW] = "lesw",
1755 [MO_LEUL] = "leul",
1756 [MO_LESL] = "lesl",
1757 [MO_LEQ] = "leq",
1758 [MO_BEUW] = "beuw",
1759 [MO_BESW] = "besw",
1760 [MO_BEUL] = "beul",
1761 [MO_BESL] = "besl",
1762 [MO_BEQ] = "beq",
1763 };
1764
1765 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
1766 #ifdef TARGET_ALIGNED_ONLY
1767 [MO_UNALN >> MO_ASHIFT] = "un+",
1768 [MO_ALIGN >> MO_ASHIFT] = "",
1769 #else
1770 [MO_UNALN >> MO_ASHIFT] = "",
1771 [MO_ALIGN >> MO_ASHIFT] = "al+",
1772 #endif
1773 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1774 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1775 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1776 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1777 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1778 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1779 };
1780
1781 static const char bswap_flag_name[][6] = {
1782 [TCG_BSWAP_IZ] = "iz",
1783 [TCG_BSWAP_OZ] = "oz",
1784 [TCG_BSWAP_OS] = "os",
1785 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1786 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1787 };
1788
1789 static inline bool tcg_regset_single(TCGRegSet d)
1790 {
1791 return (d & (d - 1)) == 0;
1792 }
1793
1794 static inline TCGReg tcg_regset_first(TCGRegSet d)
1795 {
1796 if (TCG_TARGET_NB_REGS <= 32) {
1797 return ctz32(d);
1798 } else {
1799 return ctz64(d);
1800 }
1801 }
1802
1803 static void tcg_dump_ops(TCGContext *s, bool have_prefs)
1804 {
1805 char buf[128];
1806 TCGOp *op;
1807
1808 QTAILQ_FOREACH(op, &s->ops, link) {
1809 int i, k, nb_oargs, nb_iargs, nb_cargs;
1810 const TCGOpDef *def;
1811 TCGOpcode c;
1812 int col = 0;
1813
1814 c = op->opc;
1815 def = &tcg_op_defs[c];
1816
1817 if (c == INDEX_op_insn_start) {
1818 nb_oargs = 0;
1819 col += qemu_log("\n ----");
1820
1821 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1822 target_ulong a;
1823 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1824 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
1825 #else
1826 a = op->args[i];
1827 #endif
1828 col += qemu_log(" " TARGET_FMT_lx, a);
1829 }
1830 } else if (c == INDEX_op_call) {
1831 const TCGHelperInfo *info = tcg_call_info(op);
1832 void *func = tcg_call_func(op);
1833
1834 /* variable number of arguments */
1835 nb_oargs = TCGOP_CALLO(op);
1836 nb_iargs = TCGOP_CALLI(op);
1837 nb_cargs = def->nb_cargs;
1838
1839 col += qemu_log(" %s ", def->name);
1840
1841 /*
1842 * Print the function name from TCGHelperInfo, if available.
1843 * Note that plugins have a template function for the info,
1844 * but the actual function pointer comes from the plugin.
1845 */
1846 if (func == info->func) {
1847 col += qemu_log("%s", info->name);
1848 } else {
1849 col += qemu_log("plugin(%p)", func);
1850 }
1851
1852 col += qemu_log("$0x%x,$%d", info->flags, nb_oargs);
1853 for (i = 0; i < nb_oargs; i++) {
1854 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1855 op->args[i]));
1856 }
1857 for (i = 0; i < nb_iargs; i++) {
1858 TCGArg arg = op->args[nb_oargs + i];
1859 const char *t = "<dummy>";
1860 if (arg != TCG_CALL_DUMMY_ARG) {
1861 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
1862 }
1863 col += qemu_log(",%s", t);
1864 }
1865 } else {
1866 col += qemu_log(" %s ", def->name);
1867
1868 nb_oargs = def->nb_oargs;
1869 nb_iargs = def->nb_iargs;
1870 nb_cargs = def->nb_cargs;
1871
1872 if (def->flags & TCG_OPF_VECTOR) {
1873 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
1874 8 << TCGOP_VECE(op));
1875 }
1876
1877 k = 0;
1878 for (i = 0; i < nb_oargs; i++) {
1879 if (k != 0) {
1880 col += qemu_log(",");
1881 }
1882 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1883 op->args[k++]));
1884 }
1885 for (i = 0; i < nb_iargs; i++) {
1886 if (k != 0) {
1887 col += qemu_log(",");
1888 }
1889 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1890 op->args[k++]));
1891 }
1892 switch (c) {
1893 case INDEX_op_brcond_i32:
1894 case INDEX_op_setcond_i32:
1895 case INDEX_op_movcond_i32:
1896 case INDEX_op_brcond2_i32:
1897 case INDEX_op_setcond2_i32:
1898 case INDEX_op_brcond_i64:
1899 case INDEX_op_setcond_i64:
1900 case INDEX_op_movcond_i64:
1901 case INDEX_op_cmp_vec:
1902 case INDEX_op_cmpsel_vec:
1903 if (op->args[k] < ARRAY_SIZE(cond_name)
1904 && cond_name[op->args[k]]) {
1905 col += qemu_log(",%s", cond_name[op->args[k++]]);
1906 } else {
1907 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
1908 }
1909 i = 1;
1910 break;
1911 case INDEX_op_qemu_ld_i32:
1912 case INDEX_op_qemu_st_i32:
1913 case INDEX_op_qemu_st8_i32:
1914 case INDEX_op_qemu_ld_i64:
1915 case INDEX_op_qemu_st_i64:
1916 {
1917 TCGMemOpIdx oi = op->args[k++];
1918 MemOp op = get_memop(oi);
1919 unsigned ix = get_mmuidx(oi);
1920
1921 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
1922 col += qemu_log(",$0x%x,%u", op, ix);
1923 } else {
1924 const char *s_al, *s_op;
1925 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
1926 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
1927 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
1928 }
1929 i = 1;
1930 }
1931 break;
1932 case INDEX_op_bswap16_i32:
1933 case INDEX_op_bswap16_i64:
1934 case INDEX_op_bswap32_i32:
1935 case INDEX_op_bswap32_i64:
1936 case INDEX_op_bswap64_i64:
1937 {
1938 TCGArg flags = op->args[k];
1939 const char *name = NULL;
1940
1941 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1942 name = bswap_flag_name[flags];
1943 }
1944 if (name) {
1945 col += qemu_log(",%s", name);
1946 } else {
1947 col += qemu_log(",$0x%" TCG_PRIlx, flags);
1948 }
1949 i = k = 1;
1950 }
1951 break;
1952 default:
1953 i = 0;
1954 break;
1955 }
1956 switch (c) {
1957 case INDEX_op_set_label:
1958 case INDEX_op_br:
1959 case INDEX_op_brcond_i32:
1960 case INDEX_op_brcond_i64:
1961 case INDEX_op_brcond2_i32:
1962 col += qemu_log("%s$L%d", k ? "," : "",
1963 arg_label(op->args[k])->id);
1964 i++, k++;
1965 break;
1966 default:
1967 break;
1968 }
1969 for (; i < nb_cargs; i++, k++) {
1970 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
1971 }
1972 }
1973
1974 if (have_prefs || op->life) {
1975
1976 QemuLogFile *logfile;
1977
1978 rcu_read_lock();
1979 logfile = qatomic_rcu_read(&qemu_logfile);
1980 if (logfile) {
1981 for (; col < 40; ++col) {
1982 putc(' ', logfile->fd);
1983 }
1984 }
1985 rcu_read_unlock();
1986 }
1987
1988 if (op->life) {
1989 unsigned life = op->life;
1990
1991 if (life & (SYNC_ARG * 3)) {
1992 qemu_log(" sync:");
1993 for (i = 0; i < 2; ++i) {
1994 if (life & (SYNC_ARG << i)) {
1995 qemu_log(" %d", i);
1996 }
1997 }
1998 }
1999 life /= DEAD_ARG;
2000 if (life) {
2001 qemu_log(" dead:");
2002 for (i = 0; life; ++i, life >>= 1) {
2003 if (life & 1) {
2004 qemu_log(" %d", i);
2005 }
2006 }
2007 }
2008 }
2009
2010 if (have_prefs) {
2011 for (i = 0; i < nb_oargs; ++i) {
2012 TCGRegSet set = op->output_pref[i];
2013
2014 if (i == 0) {
2015 qemu_log(" pref=");
2016 } else {
2017 qemu_log(",");
2018 }
2019 if (set == 0) {
2020 qemu_log("none");
2021 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2022 qemu_log("all");
2023 #ifdef CONFIG_DEBUG_TCG
2024 } else if (tcg_regset_single(set)) {
2025 TCGReg reg = tcg_regset_first(set);
2026 qemu_log("%s", tcg_target_reg_names[reg]);
2027 #endif
2028 } else if (TCG_TARGET_NB_REGS <= 32) {
2029 qemu_log("%#x", (uint32_t)set);
2030 } else {
2031 qemu_log("%#" PRIx64, (uint64_t)set);
2032 }
2033 }
2034 }
2035
2036 qemu_log("\n");
2037 }
2038 }
2039
2040 /* we give more priority to constraints with less registers */
2041 static int get_constraint_priority(const TCGOpDef *def, int k)
2042 {
2043 const TCGArgConstraint *arg_ct = &def->args_ct[k];
2044 int n;
2045
2046 if (arg_ct->oalias) {
2047 /* an alias is equivalent to a single register */
2048 n = 1;
2049 } else {
2050 n = ctpop64(arg_ct->regs);
2051 }
2052 return TCG_TARGET_NB_REGS - n + 1;
2053 }
2054
2055 /* sort from highest priority to lowest */
2056 static void sort_constraints(TCGOpDef *def, int start, int n)
2057 {
2058 int i, j;
2059 TCGArgConstraint *a = def->args_ct;
2060
2061 for (i = 0; i < n; i++) {
2062 a[start + i].sort_index = start + i;
2063 }
2064 if (n <= 1) {
2065 return;
2066 }
2067 for (i = 0; i < n - 1; i++) {
2068 for (j = i + 1; j < n; j++) {
2069 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2070 int p2 = get_constraint_priority(def, a[start + j].sort_index);
2071 if (p1 < p2) {
2072 int tmp = a[start + i].sort_index;
2073 a[start + i].sort_index = a[start + j].sort_index;
2074 a[start + j].sort_index = tmp;
2075 }
2076 }
2077 }
2078 }
2079
2080 static void process_op_defs(TCGContext *s)
2081 {
2082 TCGOpcode op;
2083
2084 for (op = 0; op < NB_OPS; op++) {
2085 TCGOpDef *def = &tcg_op_defs[op];
2086 const TCGTargetOpDef *tdefs;
2087 int i, nb_args;
2088
2089 if (def->flags & TCG_OPF_NOT_PRESENT) {
2090 continue;
2091 }
2092
2093 nb_args = def->nb_iargs + def->nb_oargs;
2094 if (nb_args == 0) {
2095 continue;
2096 }
2097
2098 /*
2099 * Macro magic should make it impossible, but double-check that
2100 * the array index is in range. Since the signness of an enum
2101 * is implementation defined, force the result to unsigned.
2102 */
2103 unsigned con_set = tcg_target_op_def(op);
2104 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2105 tdefs = &constraint_sets[con_set];
2106
2107 for (i = 0; i < nb_args; i++) {
2108 const char *ct_str = tdefs->args_ct_str[i];
2109 /* Incomplete TCGTargetOpDef entry. */
2110 tcg_debug_assert(ct_str != NULL);
2111
2112 while (*ct_str != '\0') {
2113 switch(*ct_str) {
2114 case '0' ... '9':
2115 {
2116 int oarg = *ct_str - '0';
2117 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2118 tcg_debug_assert(oarg < def->nb_oargs);
2119 tcg_debug_assert(def->args_ct[oarg].regs != 0);
2120 def->args_ct[i] = def->args_ct[oarg];
2121 /* The output sets oalias. */
2122 def->args_ct[oarg].oalias = true;
2123 def->args_ct[oarg].alias_index = i;
2124 /* The input sets ialias. */
2125 def->args_ct[i].ialias = true;
2126 def->args_ct[i].alias_index = oarg;
2127 }
2128 ct_str++;
2129 break;
2130 case '&':
2131 def->args_ct[i].newreg = true;
2132 ct_str++;
2133 break;
2134 case 'i':
2135 def->args_ct[i].ct |= TCG_CT_CONST;
2136 ct_str++;
2137 break;
2138
2139 /* Include all of the target-specific constraints. */
2140
2141 #undef CONST
2142 #define CONST(CASE, MASK) \
2143 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2144 #define REGS(CASE, MASK) \
2145 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2146
2147 #include "tcg-target-con-str.h"
2148
2149 #undef REGS
2150 #undef CONST
2151 default:
2152 /* Typo in TCGTargetOpDef constraint. */
2153 g_assert_not_reached();
2154 }
2155 }
2156 }
2157
2158 /* TCGTargetOpDef entry with too much information? */
2159 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
2160
2161 /* sort the constraints (XXX: this is just an heuristic) */
2162 sort_constraints(def, 0, def->nb_oargs);
2163 sort_constraints(def, def->nb_oargs, def->nb_iargs);
2164 }
2165 }
2166
2167 void tcg_op_remove(TCGContext *s, TCGOp *op)
2168 {
2169 TCGLabel *label;
2170
2171 switch (op->opc) {
2172 case INDEX_op_br:
2173 label = arg_label(op->args[0]);
2174 label->refs--;
2175 break;
2176 case INDEX_op_brcond_i32:
2177 case INDEX_op_brcond_i64:
2178 label = arg_label(op->args[3]);
2179 label->refs--;
2180 break;
2181 case INDEX_op_brcond2_i32:
2182 label = arg_label(op->args[5]);
2183 label->refs--;
2184 break;
2185 default:
2186 break;
2187 }
2188
2189 QTAILQ_REMOVE(&s->ops, op, link);
2190 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
2191 s->nb_ops--;
2192
2193 #ifdef CONFIG_PROFILER
2194 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
2195 #endif
2196 }
2197
2198 void tcg_remove_ops_after(TCGOp *op)
2199 {
2200 TCGContext *s = tcg_ctx;
2201
2202 while (true) {
2203 TCGOp *last = tcg_last_op();
2204 if (last == op) {
2205 return;
2206 }
2207 tcg_op_remove(s, last);
2208 }
2209 }
2210
2211 static TCGOp *tcg_op_alloc(TCGOpcode opc)
2212 {
2213 TCGContext *s = tcg_ctx;
2214 TCGOp *op;
2215
2216 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2217 op = tcg_malloc(sizeof(TCGOp));
2218 } else {
2219 op = QTAILQ_FIRST(&s->free_ops);
2220 QTAILQ_REMOVE(&s->free_ops, op, link);
2221 }
2222 memset(op, 0, offsetof(TCGOp, link));
2223 op->opc = opc;
2224 s->nb_ops++;
2225
2226 return op;
2227 }
2228
2229 TCGOp *tcg_emit_op(TCGOpcode opc)
2230 {
2231 TCGOp *op = tcg_op_alloc(opc);
2232 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2233 return op;
2234 }
2235
2236 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2237 {
2238 TCGOp *new_op = tcg_op_alloc(opc);
2239 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
2240 return new_op;
2241 }
2242
2243 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2244 {
2245 TCGOp *new_op = tcg_op_alloc(opc);
2246 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
2247 return new_op;
2248 }
2249
2250 /* Reachable analysis : remove unreachable code. */
2251 static void reachable_code_pass(TCGContext *s)
2252 {
2253 TCGOp *op, *op_next;
2254 bool dead = false;
2255
2256 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2257 bool remove = dead;
2258 TCGLabel *label;
2259
2260 switch (op->opc) {
2261 case INDEX_op_set_label:
2262 label = arg_label(op->args[0]);
2263 if (label->refs == 0) {
2264 /*
2265 * While there is an occasional backward branch, virtually
2266 * all branches generated by the translators are forward.
2267 * Which means that generally we will have already removed
2268 * all references to the label that will be, and there is
2269 * little to be gained by iterating.
2270 */
2271 remove = true;
2272 } else {
2273 /* Once we see a label, insns become live again. */
2274 dead = false;
2275 remove = false;
2276
2277 /*
2278 * Optimization can fold conditional branches to unconditional.
2279 * If we find a label with one reference which is preceded by
2280 * an unconditional branch to it, remove both. This needed to
2281 * wait until the dead code in between them was removed.
2282 */
2283 if (label->refs == 1) {
2284 TCGOp *op_prev = QTAILQ_PREV(op, link);
2285 if (op_prev->opc == INDEX_op_br &&
2286 label == arg_label(op_prev->args[0])) {
2287 tcg_op_remove(s, op_prev);
2288 remove = true;
2289 }
2290 }
2291 }
2292 break;
2293
2294 case INDEX_op_br:
2295 case INDEX_op_exit_tb:
2296 case INDEX_op_goto_ptr:
2297 /* Unconditional branches; everything following is dead. */
2298 dead = true;
2299 break;
2300
2301 case INDEX_op_call:
2302 /* Notice noreturn helper calls, raising exceptions. */
2303 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
2304 dead = true;
2305 }
2306 break;
2307
2308 case INDEX_op_insn_start:
2309 /* Never remove -- we need to keep these for unwind. */
2310 remove = false;
2311 break;
2312
2313 default:
2314 break;
2315 }
2316
2317 if (remove) {
2318 tcg_op_remove(s, op);
2319 }
2320 }
2321 }
2322
2323 #define TS_DEAD 1
2324 #define TS_MEM 2
2325
2326 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2327 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2328
2329 /* For liveness_pass_1, the register preferences for a given temp. */
2330 static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2331 {
2332 return ts->state_ptr;
2333 }
2334
2335 /* For liveness_pass_1, reset the preferences for a given temp to the
2336 * maximal regset for its type.
2337 */
2338 static inline void la_reset_pref(TCGTemp *ts)
2339 {
2340 *la_temp_pref(ts)
2341 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2342 }
2343
2344 /* liveness analysis: end of function: all temps are dead, and globals
2345 should be in memory. */
2346 static void la_func_end(TCGContext *s, int ng, int nt)
2347 {
2348 int i;
2349
2350 for (i = 0; i < ng; ++i) {
2351 s->temps[i].state = TS_DEAD | TS_MEM;
2352 la_reset_pref(&s->temps[i]);
2353 }
2354 for (i = ng; i < nt; ++i) {
2355 s->temps[i].state = TS_DEAD;
2356 la_reset_pref(&s->temps[i]);
2357 }
2358 }
2359
2360 /* liveness analysis: end of basic block: all temps are dead, globals
2361 and local temps should be in memory. */
2362 static void la_bb_end(TCGContext *s, int ng, int nt)
2363 {
2364 int i;
2365
2366 for (i = 0; i < nt; ++i) {
2367 TCGTemp *ts = &s->temps[i];
2368 int state;
2369
2370 switch (ts->kind) {
2371 case TEMP_FIXED:
2372 case TEMP_GLOBAL:
2373 case TEMP_LOCAL:
2374 state = TS_DEAD | TS_MEM;
2375 break;
2376 case TEMP_NORMAL:
2377 case TEMP_CONST:
2378 state = TS_DEAD;
2379 break;
2380 default:
2381 g_assert_not_reached();
2382 }
2383 ts->state = state;
2384 la_reset_pref(ts);
2385 }
2386 }
2387
2388 /* liveness analysis: sync globals back to memory. */
2389 static void la_global_sync(TCGContext *s, int ng)
2390 {
2391 int i;
2392
2393 for (i = 0; i < ng; ++i) {
2394 int state = s->temps[i].state;
2395 s->temps[i].state = state | TS_MEM;
2396 if (state == TS_DEAD) {
2397 /* If the global was previously dead, reset prefs. */
2398 la_reset_pref(&s->temps[i]);
2399 }
2400 }
2401 }
2402
2403 /*
2404 * liveness analysis: conditional branch: all temps are dead,
2405 * globals and local temps should be synced.
2406 */
2407 static void la_bb_sync(TCGContext *s, int ng, int nt)
2408 {
2409 la_global_sync(s, ng);
2410
2411 for (int i = ng; i < nt; ++i) {
2412 TCGTemp *ts = &s->temps[i];
2413 int state;
2414
2415 switch (ts->kind) {
2416 case TEMP_LOCAL:
2417 state = ts->state;
2418 ts->state = state | TS_MEM;
2419 if (state != TS_DEAD) {
2420 continue;
2421 }
2422 break;
2423 case TEMP_NORMAL:
2424 s->temps[i].state = TS_DEAD;
2425 break;
2426 case TEMP_CONST:
2427 continue;
2428 default:
2429 g_assert_not_reached();
2430 }
2431 la_reset_pref(&s->temps[i]);
2432 }
2433 }
2434
2435 /* liveness analysis: sync globals back to memory and kill. */
2436 static void la_global_kill(TCGContext *s, int ng)
2437 {
2438 int i;
2439
2440 for (i = 0; i < ng; i++) {
2441 s->temps[i].state = TS_DEAD | TS_MEM;
2442 la_reset_pref(&s->temps[i]);
2443 }
2444 }
2445
2446 /* liveness analysis: note live globals crossing calls. */
2447 static void la_cross_call(TCGContext *s, int nt)
2448 {
2449 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2450 int i;
2451
2452 for (i = 0; i < nt; i++) {
2453 TCGTemp *ts = &s->temps[i];
2454 if (!(ts->state & TS_DEAD)) {
2455 TCGRegSet *pset = la_temp_pref(ts);
2456 TCGRegSet set = *pset;
2457
2458 set &= mask;
2459 /* If the combination is not possible, restart. */
2460 if (set == 0) {
2461 set = tcg_target_available_regs[ts->type] & mask;
2462 }
2463 *pset = set;
2464 }
2465 }
2466 }
2467
2468 /* Liveness analysis : update the opc_arg_life array to tell if a
2469 given input arguments is dead. Instructions updating dead
2470 temporaries are removed. */
2471 static void liveness_pass_1(TCGContext *s)
2472 {
2473 int nb_globals = s->nb_globals;
2474 int nb_temps = s->nb_temps;
2475 TCGOp *op, *op_prev;
2476 TCGRegSet *prefs;
2477 int i;
2478
2479 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2480 for (i = 0; i < nb_temps; ++i) {
2481 s->temps[i].state_ptr = prefs + i;
2482 }
2483
2484 /* ??? Should be redundant with the exit_tb that ends the TB. */
2485 la_func_end(s, nb_globals, nb_temps);
2486
2487 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
2488 int nb_iargs, nb_oargs;
2489 TCGOpcode opc_new, opc_new2;
2490 bool have_opc_new2;
2491 TCGLifeData arg_life = 0;
2492 TCGTemp *ts;
2493 TCGOpcode opc = op->opc;
2494 const TCGOpDef *def = &tcg_op_defs[opc];
2495
2496 switch (opc) {
2497 case INDEX_op_call:
2498 {
2499 int call_flags;
2500 int nb_call_regs;
2501
2502 nb_oargs = TCGOP_CALLO(op);
2503 nb_iargs = TCGOP_CALLI(op);
2504 call_flags = tcg_call_flags(op);
2505
2506 /* pure functions can be removed if their result is unused */
2507 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
2508 for (i = 0; i < nb_oargs; i++) {
2509 ts = arg_temp(op->args[i]);
2510 if (ts->state != TS_DEAD) {
2511 goto do_not_remove_call;
2512 }
2513 }
2514 goto do_remove;
2515 }
2516 do_not_remove_call:
2517
2518 /* Output args are dead. */
2519 for (i = 0; i < nb_oargs; i++) {
2520 ts = arg_temp(op->args[i]);
2521 if (ts->state & TS_DEAD) {
2522 arg_life |= DEAD_ARG << i;
2523 }
2524 if (ts->state & TS_MEM) {
2525 arg_life |= SYNC_ARG << i;
2526 }
2527 ts->state = TS_DEAD;
2528 la_reset_pref(ts);
2529
2530 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2531 op->output_pref[i] = 0;
2532 }
2533
2534 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2535 TCG_CALL_NO_READ_GLOBALS))) {
2536 la_global_kill(s, nb_globals);
2537 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
2538 la_global_sync(s, nb_globals);
2539 }
2540
2541 /* Record arguments that die in this helper. */
2542 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2543 ts = arg_temp(op->args[i]);
2544 if (ts && ts->state & TS_DEAD) {
2545 arg_life |= DEAD_ARG << i;
2546 }
2547 }
2548
2549 /* For all live registers, remove call-clobbered prefs. */
2550 la_cross_call(s, nb_temps);
2551
2552 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2553
2554 /* Input arguments are live for preceding opcodes. */
2555 for (i = 0; i < nb_iargs; i++) {
2556 ts = arg_temp(op->args[i + nb_oargs]);
2557 if (ts && ts->state & TS_DEAD) {
2558 /* For those arguments that die, and will be allocated
2559 * in registers, clear the register set for that arg,
2560 * to be filled in below. For args that will be on
2561 * the stack, reset to any available reg.
2562 */
2563 *la_temp_pref(ts)
2564 = (i < nb_call_regs ? 0 :
2565 tcg_target_available_regs[ts->type]);
2566 ts->state &= ~TS_DEAD;
2567 }
2568 }
2569
2570 /* For each input argument, add its input register to prefs.
2571 If a temp is used once, this produces a single set bit. */
2572 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2573 ts = arg_temp(op->args[i + nb_oargs]);
2574 if (ts) {
2575 tcg_regset_set_reg(*la_temp_pref(ts),
2576 tcg_target_call_iarg_regs[i]);
2577 }
2578 }
2579 }
2580 break;
2581 case INDEX_op_insn_start:
2582 break;
2583 case INDEX_op_discard:
2584 /* mark the temporary as dead */
2585 ts = arg_temp(op->args[0]);
2586 ts->state = TS_DEAD;
2587 la_reset_pref(ts);
2588 break;
2589
2590 case INDEX_op_add2_i32:
2591 opc_new = INDEX_op_add_i32;
2592 goto do_addsub2;
2593 case INDEX_op_sub2_i32:
2594 opc_new = INDEX_op_sub_i32;
2595 goto do_addsub2;
2596 case INDEX_op_add2_i64:
2597 opc_new = INDEX_op_add_i64;
2598 goto do_addsub2;
2599 case INDEX_op_sub2_i64:
2600 opc_new = INDEX_op_sub_i64;
2601 do_addsub2:
2602 nb_iargs = 4;
2603 nb_oargs = 2;
2604 /* Test if the high part of the operation is dead, but not
2605 the low part. The result can be optimized to a simple
2606 add or sub. This happens often for x86_64 guest when the
2607 cpu mode is set to 32 bit. */
2608 if (arg_temp(op->args[1])->state == TS_DEAD) {
2609 if (arg_temp(op->args[0])->state == TS_DEAD) {
2610 goto do_remove;
2611 }
2612 /* Replace the opcode and adjust the args in place,
2613 leaving 3 unused args at the end. */
2614 op->opc = opc = opc_new;
2615 op->args[1] = op->args[2];
2616 op->args[2] = op->args[4];
2617 /* Fall through and mark the single-word operation live. */
2618 nb_iargs = 2;
2619 nb_oargs = 1;
2620 }
2621 goto do_not_remove;
2622
2623 case INDEX_op_mulu2_i32:
2624 opc_new = INDEX_op_mul_i32;
2625 opc_new2 = INDEX_op_muluh_i32;
2626 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
2627 goto do_mul2;
2628 case INDEX_op_muls2_i32:
2629 opc_new = INDEX_op_mul_i32;
2630 opc_new2 = INDEX_op_mulsh_i32;
2631 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
2632 goto do_mul2;
2633 case INDEX_op_mulu2_i64:
2634 opc_new = INDEX_op_mul_i64;
2635 opc_new2 = INDEX_op_muluh_i64;
2636 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
2637 goto do_mul2;
2638 case INDEX_op_muls2_i64:
2639 opc_new = INDEX_op_mul_i64;
2640 opc_new2 = INDEX_op_mulsh_i64;
2641 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
2642 goto do_mul2;
2643 do_mul2:
2644 nb_iargs = 2;
2645 nb_oargs = 2;
2646 if (arg_temp(op->args[1])->state == TS_DEAD) {
2647 if (arg_temp(op->args[0])->state == TS_DEAD) {
2648 /* Both parts of the operation are dead. */
2649 goto do_remove;
2650 }
2651 /* The high part of the operation is dead; generate the low. */
2652 op->opc = opc = opc_new;
2653 op->args[1] = op->args[2];
2654 op->args[2] = op->args[3];
2655 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
2656 /* The low part of the operation is dead; generate the high. */
2657 op->opc = opc = opc_new2;
2658 op->args[0] = op->args[1];
2659 op->args[1] = op->args[2];
2660 op->args[2] = op->args[3];
2661 } else {
2662 goto do_not_remove;
2663 }
2664 /* Mark the single-word operation live. */
2665 nb_oargs = 1;
2666 goto do_not_remove;
2667
2668 default:
2669 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2670 nb_iargs = def->nb_iargs;
2671 nb_oargs = def->nb_oargs;
2672
2673 /* Test if the operation can be removed because all
2674 its outputs are dead. We assume that nb_oargs == 0
2675 implies side effects */
2676 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
2677 for (i = 0; i < nb_oargs; i++) {
2678 if (arg_temp(op->args[i])->state != TS_DEAD) {
2679 goto do_not_remove;
2680 }
2681 }
2682 goto do_remove;
2683 }
2684 goto do_not_remove;
2685
2686 do_remove:
2687 tcg_op_remove(s, op);
2688 break;
2689
2690 do_not_remove:
2691 for (i = 0; i < nb_oargs; i++) {
2692 ts = arg_temp(op->args[i]);
2693
2694 /* Remember the preference of the uses that followed. */
2695 op->output_pref[i] = *la_temp_pref(ts);
2696
2697 /* Output args are dead. */
2698 if (ts->state & TS_DEAD) {
2699 arg_life |= DEAD_ARG << i;
2700 }
2701 if (ts->state & TS_MEM) {
2702 arg_life |= SYNC_ARG << i;
2703 }
2704 ts->state = TS_DEAD;
2705 la_reset_pref(ts);
2706 }
2707
2708 /* If end of basic block, update. */
2709 if (def->flags & TCG_OPF_BB_EXIT) {
2710 la_func_end(s, nb_globals, nb_temps);
2711 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2712 la_bb_sync(s, nb_globals, nb_temps);
2713 } else if (def->flags & TCG_OPF_BB_END) {
2714 la_bb_end(s, nb_globals, nb_temps);
2715 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2716 la_global_sync(s, nb_globals);
2717 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2718 la_cross_call(s, nb_temps);
2719 }
2720 }
2721
2722 /* Record arguments that die in this opcode. */
2723 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2724 ts = arg_temp(op->args[i]);
2725 if (ts->state & TS_DEAD) {
2726 arg_life |= DEAD_ARG << i;
2727 }
2728 }
2729
2730 /* Input arguments are live for preceding opcodes. */
2731 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2732 ts = arg_temp(op->args[i]);
2733 if (ts->state & TS_DEAD) {
2734 /* For operands that were dead, initially allow
2735 all regs for the type. */
2736 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2737 ts->state &= ~TS_DEAD;
2738 }
2739 }
2740
2741 /* Incorporate constraints for this operand. */
2742 switch (opc) {
2743 case INDEX_op_mov_i32:
2744 case INDEX_op_mov_i64:
2745 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2746 have proper constraints. That said, special case
2747 moves to propagate preferences backward. */
2748 if (IS_DEAD_ARG(1)) {
2749 *la_temp_pref(arg_temp(op->args[0]))
2750 = *la_temp_pref(arg_temp(op->args[1]));
2751 }
2752 break;
2753
2754 default:
2755 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2756 const TCGArgConstraint *ct = &def->args_ct[i];
2757 TCGRegSet set, *pset;
2758
2759 ts = arg_temp(op->args[i]);
2760 pset = la_temp_pref(ts);
2761 set = *pset;
2762
2763 set &= ct->regs;
2764 if (ct->ialias) {
2765 set &= op->output_pref[ct->alias_index];
2766 }
2767 /* If the combination is not possible, restart. */
2768 if (set == 0) {
2769 set = ct->regs;
2770 }
2771 *pset = set;
2772 }
2773 break;
2774 }
2775 break;
2776 }
2777 op->life = arg_life;
2778 }
2779 }
2780
2781 /* Liveness analysis: Convert indirect regs to direct temporaries. */
2782 static bool liveness_pass_2(TCGContext *s)
2783 {
2784 int nb_globals = s->nb_globals;
2785 int nb_temps, i;
2786 bool changes = false;
2787 TCGOp *op, *op_next;
2788
2789 /* Create a temporary for each indirect global. */
2790 for (i = 0; i < nb_globals; ++i) {
2791 TCGTemp *its = &s->temps[i];
2792 if (its->indirect_reg) {
2793 TCGTemp *dts = tcg_temp_alloc(s);
2794 dts->type = its->type;
2795 dts->base_type = its->base_type;
2796 its->state_ptr = dts;
2797 } else {
2798 its->state_ptr = NULL;
2799 }
2800 /* All globals begin dead. */
2801 its->state = TS_DEAD;
2802 }
2803 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2804 TCGTemp *its = &s->temps[i];
2805 its->state_ptr = NULL;
2806 its->state = TS_DEAD;
2807 }
2808
2809 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2810 TCGOpcode opc = op->opc;
2811 const TCGOpDef *def = &tcg_op_defs[opc];
2812 TCGLifeData arg_life = op->life;
2813 int nb_iargs, nb_oargs, call_flags;
2814 TCGTemp *arg_ts, *dir_ts;
2815
2816 if (opc == INDEX_op_call) {
2817 nb_oargs = TCGOP_CALLO(op);
2818 nb_iargs = TCGOP_CALLI(op);
2819 call_flags = tcg_call_flags(op);
2820 } else {
2821 nb_iargs = def->nb_iargs;
2822 nb_oargs = def->nb_oargs;
2823
2824 /* Set flags similar to how calls require. */
2825 if (def->flags & TCG_OPF_COND_BRANCH) {
2826 /* Like reading globals: sync_globals */
2827 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2828 } else if (def->flags & TCG_OPF_BB_END) {
2829 /* Like writing globals: save_globals */
2830 call_flags = 0;
2831 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2832 /* Like reading globals: sync_globals */
2833 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2834 } else {
2835 /* No effect on globals. */
2836 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2837 TCG_CALL_NO_WRITE_GLOBALS);
2838 }
2839 }
2840
2841 /* Make sure that input arguments are available. */
2842 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2843 arg_ts = arg_temp(op->args[i]);
2844 if (arg_ts) {
2845 dir_ts = arg_ts->state_ptr;
2846 if (dir_ts && arg_ts->state == TS_DEAD) {
2847 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
2848 ? INDEX_op_ld_i32
2849 : INDEX_op_ld_i64);
2850 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
2851
2852 lop->args[0] = temp_arg(dir_ts);
2853 lop->args[1] = temp_arg(arg_ts->mem_base);
2854 lop->args[2] = arg_ts->mem_offset;
2855
2856 /* Loaded, but synced with memory. */
2857 arg_ts->state = TS_MEM;
2858 }
2859 }
2860 }
2861
2862 /* Perform input replacement, and mark inputs that became dead.
2863 No action is required except keeping temp_state up to date
2864 so that we reload when needed. */
2865 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2866 arg_ts = arg_temp(op->args[i]);
2867 if (arg_ts) {
2868 dir_ts = arg_ts->state_ptr;
2869 if (dir_ts) {
2870 op->args[i] = temp_arg(dir_ts);
2871 changes = true;
2872 if (IS_DEAD_ARG(i)) {
2873 arg_ts->state = TS_DEAD;
2874 }
2875 }
2876 }
2877 }
2878
2879 /* Liveness analysis should ensure that the following are
2880 all correct, for call sites and basic block end points. */
2881 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2882 /* Nothing to do */
2883 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2884 for (i = 0; i < nb_globals; ++i) {
2885 /* Liveness should see that globals are synced back,
2886 that is, either TS_DEAD or TS_MEM. */
2887 arg_ts = &s->temps[i];
2888 tcg_debug_assert(arg_ts->state_ptr == 0
2889 || arg_ts->state != 0);
2890 }
2891 } else {
2892 for (i = 0; i < nb_globals; ++i) {
2893 /* Liveness should see that globals are saved back,
2894 that is, TS_DEAD, waiting to be reloaded. */
2895 arg_ts = &s->temps[i];
2896 tcg_debug_assert(arg_ts->state_ptr == 0
2897 || arg_ts->state == TS_DEAD);
2898 }
2899 }
2900
2901 /* Outputs become available. */
2902 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2903 arg_ts = arg_temp(op->args[0]);
2904 dir_ts = arg_ts->state_ptr;
2905 if (dir_ts) {
2906 op->args[0] = temp_arg(dir_ts);
2907 changes = true;
2908
2909 /* The output is now live and modified. */
2910 arg_ts->state = 0;
2911
2912 if (NEED_SYNC_ARG(0)) {
2913 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2914 ? INDEX_op_st_i32
2915 : INDEX_op_st_i64);
2916 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2917 TCGTemp *out_ts = dir_ts;
2918
2919 if (IS_DEAD_ARG(0)) {
2920 out_ts = arg_temp(op->args[1]);
2921 arg_ts->state = TS_DEAD;
2922 tcg_op_remove(s, op);
2923 } else {
2924 arg_ts->state = TS_MEM;
2925 }
2926
2927 sop->args[0] = temp_arg(out_ts);
2928 sop->args[1] = temp_arg(arg_ts->mem_base);
2929 sop->args[2] = arg_ts->mem_offset;
2930 } else {
2931 tcg_debug_assert(!IS_DEAD_ARG(0));
2932 }
2933 }
2934 } else {
2935 for (i = 0; i < nb_oargs; i++) {
2936 arg_ts = arg_temp(op->args[i]);
2937 dir_ts = arg_ts->state_ptr;
2938 if (!dir_ts) {
2939 continue;
2940 }
2941 op->args[i] = temp_arg(dir_ts);
2942 changes = true;
2943
2944 /* The output is now live and modified. */
2945 arg_ts->state = 0;
2946
2947 /* Sync outputs upon their last write. */
2948 if (NEED_SYNC_ARG(i)) {
2949 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2950 ? INDEX_op_st_i32
2951 : INDEX_op_st_i64);
2952 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2953
2954 sop->args[0] = temp_arg(dir_ts);
2955 sop->args[1] = temp_arg(arg_ts->mem_base);
2956 sop->args[2] = arg_ts->mem_offset;
2957
2958 arg_ts->state = TS_MEM;
2959 }
2960 /* Drop outputs that are dead. */
2961 if (IS_DEAD_ARG(i)) {
2962 arg_ts->state = TS_DEAD;
2963 }
2964 }
2965 }
2966 }
2967
2968 return changes;
2969 }
2970
2971 #ifdef CONFIG_DEBUG_TCG
2972 static void dump_regs(TCGContext *s)
2973 {
2974 TCGTemp *ts;
2975 int i;
2976 char buf[64];
2977
2978 for(i = 0; i < s->nb_temps; i++) {
2979 ts = &s->temps[i];
2980 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
2981 switch(ts->val_type) {
2982 case TEMP_VAL_REG:
2983 printf("%s", tcg_target_reg_names[ts->reg]);
2984 break;
2985 case TEMP_VAL_MEM:
2986 printf("%d(%s)", (int)ts->mem_offset,
2987 tcg_target_reg_names[ts->mem_base->reg]);
2988 break;
2989 case TEMP_VAL_CONST:
2990 printf("$0x%" PRIx64, ts->val);
2991 break;
2992 case TEMP_VAL_DEAD:
2993 printf("D");
2994 break;
2995 default:
2996 printf("???");
2997 break;
2998 }
2999 printf("\n");
3000 }
3001
3002 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
3003 if (s->reg_to_temp[i] != NULL) {
3004 printf("%s: %s\n",
3005 tcg_target_reg_names[i],
3006 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
3007 }
3008 }
3009 }
3010
3011 static void check_regs(TCGContext *s)
3012 {
3013 int reg;
3014 int k;
3015 TCGTemp *ts;
3016 char buf[64];
3017
3018 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
3019 ts = s->reg_to_temp[reg];
3020 if (ts != NULL) {
3021 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
3022 printf("Inconsistency for register %s:\n",
3023 tcg_target_reg_names[reg]);
3024 goto fail;
3025 }
3026 }
3027 }
3028 for (k = 0; k < s->nb_temps; k++) {
3029 ts = &s->temps[k];
3030 if (ts->val_type == TEMP_VAL_REG
3031 && ts->kind != TEMP_FIXED
3032 && s->reg_to_temp[ts->reg] != ts) {
3033 printf("Inconsistency for temp %s:\n",
3034 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
3035 fail:
3036 printf("reg state:\n");
3037 dump_regs(s);
3038 tcg_abort();
3039 }
3040 }
3041 }
3042 #endif
3043
3044 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
3045 {
3046 intptr_t off, size, align;
3047
3048 switch (ts->type) {
3049 case TCG_TYPE_I32:
3050 size = align = 4;
3051 break;
3052 case TCG_TYPE_I64:
3053 case TCG_TYPE_V64:
3054 size = align = 8;
3055 break;
3056 case TCG_TYPE_V128:
3057 size = align = 16;
3058 break;
3059 case TCG_TYPE_V256:
3060 /* Note that we do not require aligned storage for V256. */
3061 size = 32, align = 16;
3062 break;
3063 default:
3064 g_assert_not_reached();
3065 }
3066
3067 assert(align <= TCG_TARGET_STACK_ALIGN);
3068 off = ROUND_UP(s->current_frame_offset, align);
3069
3070 /* If we've exhausted the stack frame, restart with a smaller TB. */
3071 if (off + size > s->frame_end) {
3072 tcg_raise_tb_overflow(s);
3073 }
3074 s->current_frame_offset = off + size;
3075
3076 ts->mem_offset = off;
3077 #if defined(__sparc__)
3078 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3079 #endif
3080 ts->mem_base = s->frame_temp;
3081 ts->mem_allocated = 1;
3082 }
3083
3084 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
3085
3086 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3087 mark it free; otherwise mark it dead. */
3088 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
3089 {
3090 TCGTempVal new_type;
3091
3092 switch (ts->kind) {
3093 case TEMP_FIXED:
3094 return;
3095 case TEMP_GLOBAL:
3096 case TEMP_LOCAL:
3097 new_type = TEMP_VAL_MEM;
3098 break;
3099 case TEMP_NORMAL:
3100 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3101 break;
3102 case TEMP_CONST:
3103 new_type = TEMP_VAL_CONST;
3104 break;
3105 default:
3106 g_assert_not_reached();
3107 }
3108 if (ts->val_type == TEMP_VAL_REG) {
3109 s->reg_to_temp[ts->reg] = NULL;
3110 }
3111 ts->val_type = new_type;
3112 }
3113
3114 /* Mark a temporary as dead. */
3115 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3116 {
3117 temp_free_or_dead(s, ts, 1);
3118 }
3119
3120 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3121 registers needs to be allocated to store a constant. If 'free_or_dead'
3122 is non-zero, subsequently release the temporary; if it is positive, the
3123 temp is dead; if it is negative, the temp is free. */
3124 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3125 TCGRegSet preferred_regs, int free_or_dead)
3126 {
3127 if (!temp_readonly(ts) && !ts->mem_coherent) {
3128 if (!ts->mem_allocated) {
3129 temp_allocate_frame(s, ts);
3130 }
3131 switch (ts->val_type) {
3132 case TEMP_VAL_CONST:
3133 /* If we're going to free the temp immediately, then we won't
3134 require it later in a register, so attempt to store the
3135 constant to memory directly. */
3136 if (free_or_dead
3137 && tcg_out_sti(s, ts->type, ts->val,
3138 ts->mem_base->reg, ts->mem_offset)) {
3139 break;
3140 }
3141 temp_load(s, ts, tcg_target_available_regs[ts->type],
3142 allocated_regs, preferred_regs);
3143 /* fallthrough */
3144
3145 case TEMP_VAL_REG:
3146 tcg_out_st(s, ts->type, ts->reg,
3147 ts->mem_base->reg, ts->mem_offset);
3148 break;
3149
3150 case TEMP_VAL_MEM:
3151 break;
3152
3153 case TEMP_VAL_DEAD:
3154 default:
3155 tcg_abort();
3156 }
3157 ts->mem_coherent = 1;
3158 }
3159 if (free_or_dead) {
3160 temp_free_or_dead(s, ts, free_or_dead);
3161 }
3162 }
3163
3164 /* free register 'reg' by spilling the corresponding temporary if necessary */
3165 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
3166 {
3167 TCGTemp *ts = s->reg_to_temp[reg];
3168 if (ts != NULL) {
3169 temp_sync(s, ts, allocated_regs, 0, -1);
3170 }
3171 }
3172
3173 /**
3174 * tcg_reg_alloc:
3175 * @required_regs: Set of registers in which we must allocate.
3176 * @allocated_regs: Set of registers which must be avoided.
3177 * @preferred_regs: Set of registers we should prefer.
3178 * @rev: True if we search the registers in "indirect" order.
3179 *
3180 * The allocated register must be in @required_regs & ~@allocated_regs,
3181 * but if we can put it in @preferred_regs we may save a move later.
3182 */
3183 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3184 TCGRegSet allocated_regs,
3185 TCGRegSet preferred_regs, bool rev)
3186 {
3187 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3188 TCGRegSet reg_ct[2];
3189 const int *order;
3190
3191 reg_ct[1] = required_regs & ~allocated_regs;
3192 tcg_debug_assert(reg_ct[1] != 0);
3193 reg_ct[0] = reg_ct[1] & preferred_regs;
3194
3195 /* Skip the preferred_regs option if it cannot be satisfied,
3196 or if the preference made no difference. */
3197 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3198
3199 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3200
3201 /* Try free registers, preferences first. */
3202 for (j = f; j < 2; j++) {
3203 TCGRegSet set = reg_ct[j];
3204
3205 if (tcg_regset_single(set)) {
3206 /* One register in the set. */
3207 TCGReg reg = tcg_regset_first(set);
3208 if (s->reg_to_temp[reg] == NULL) {
3209 return reg;
3210 }
3211 } else {
3212 for (i = 0; i < n; i++) {
3213 TCGReg reg = order[i];
3214 if (s->reg_to_temp[reg] == NULL &&
3215 tcg_regset_test_reg(set, reg)) {
3216 return reg;
3217 }
3218 }
3219 }
3220 }
3221
3222 /* We must spill something. */
3223 for (j = f; j < 2; j++) {
3224 TCGRegSet set = reg_ct[j];
3225
3226 if (tcg_regset_single(set)) {
3227 /* One register in the set. */
3228 TCGReg reg = tcg_regset_first(set);
3229 tcg_reg_free(s, reg, allocated_regs);
3230 return reg;
3231 } else {
3232 for (i = 0; i < n; i++) {
3233 TCGReg reg = order[i];
3234 if (tcg_regset_test_reg(set, reg)) {
3235 tcg_reg_free(s, reg, allocated_regs);
3236 return reg;
3237 }
3238 }
3239 }
3240 }
3241
3242 tcg_abort();
3243 }
3244
3245 /* Make sure the temporary is in a register. If needed, allocate the register
3246 from DESIRED while avoiding ALLOCATED. */
3247 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
3248 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
3249 {
3250 TCGReg reg;
3251
3252 switch (ts->val_type) {
3253 case TEMP_VAL_REG:
3254 return;
3255 case TEMP_VAL_CONST:
3256 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3257 preferred_regs, ts->indirect_base);
3258 if (ts->type <= TCG_TYPE_I64) {
3259 tcg_out_movi(s, ts->type, reg, ts->val);
3260 } else {
3261 uint64_t val = ts->val;
3262 MemOp vece = MO_64;
3263
3264 /*
3265 * Find the minimal vector element that matches the constant.
3266 * The targets will, in general, have to do this search anyway,
3267 * do this generically.
3268 */
3269 if (val == dup_const(MO_8, val)) {
3270 vece = MO_8;
3271 } else if (val == dup_const(MO_16, val)) {
3272 vece = MO_16;
3273 } else if (val == dup_const(MO_32, val)) {
3274 vece = MO_32;
3275 }
3276
3277 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
3278 }
3279 ts->mem_coherent = 0;
3280 break;
3281 case TEMP_VAL_MEM:
3282 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3283 preferred_regs, ts->indirect_base);
3284 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3285 ts->mem_coherent = 1;
3286 break;
3287 case TEMP_VAL_DEAD:
3288 default:
3289 tcg_abort();
3290 }
3291 ts->reg = reg;
3292 ts->val_type = TEMP_VAL_REG;
3293 s->reg_to_temp[reg] = ts;
3294 }
3295
3296 /* Save a temporary to memory. 'allocated_regs' is used in case a
3297 temporary registers needs to be allocated to store a constant. */
3298 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
3299 {
3300 /* The liveness analysis already ensures that globals are back
3301 in memory. Keep an tcg_debug_assert for safety. */
3302 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
3303 }
3304
3305 /* save globals to their canonical location and assume they can be
3306 modified be the following code. 'allocated_regs' is used in case a
3307 temporary registers needs to be allocated to store a constant. */
3308 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
3309 {
3310 int i, n;
3311
3312 for (i = 0, n = s->nb_globals; i < n; i++) {
3313 temp_save(s, &s->temps[i], allocated_regs);
3314 }
3315 }
3316
3317 /* sync globals to their canonical location and assume they can be
3318 read by the following code. 'allocated_regs' is used in case a
3319 temporary registers needs to be allocated to store a constant. */
3320 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3321 {
3322 int i, n;
3323
3324 for (i = 0, n = s->nb_globals; i < n; i++) {
3325 TCGTemp *ts = &s->temps[i];
3326 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
3327 || ts->kind == TEMP_FIXED
3328 || ts->mem_coherent);
3329 }
3330 }
3331
3332 /* at the end of a basic block, we assume all temporaries are dead and
3333 all globals are stored at their canonical location. */
3334 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
3335 {
3336 int i;
3337
3338 for (i = s->nb_globals; i < s->nb_temps; i++) {
3339 TCGTemp *ts = &s->temps[i];
3340
3341 switch (ts->kind) {
3342 case TEMP_LOCAL:
3343 temp_save(s, ts, allocated_regs);
3344 break;
3345 case TEMP_NORMAL:
3346 /* The liveness analysis already ensures that temps are dead.
3347 Keep an tcg_debug_assert for safety. */
3348 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3349 break;
3350 case TEMP_CONST:
3351 /* Similarly, we should have freed any allocated register. */
3352 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3353 break;
3354 default:
3355 g_assert_not_reached();
3356 }
3357 }
3358
3359 save_globals(s, allocated_regs);
3360 }
3361
3362 /*
3363 * At a conditional branch, we assume all temporaries are dead and
3364 * all globals and local temps are synced to their location.
3365 */
3366 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3367 {
3368 sync_globals(s, allocated_regs);
3369
3370 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3371 TCGTemp *ts = &s->temps[i];
3372 /*
3373 * The liveness analysis already ensures that temps are dead.
3374 * Keep tcg_debug_asserts for safety.
3375 */
3376 switch (ts->kind) {
3377 case TEMP_LOCAL:
3378 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
3379 break;
3380 case TEMP_NORMAL:
3381 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3382 break;
3383 case TEMP_CONST:
3384 break;
3385 default:
3386 g_assert_not_reached();
3387 }
3388 }
3389 }
3390
3391 /*
3392 * Specialized code generation for INDEX_op_mov_* with a constant.
3393 */
3394 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
3395 tcg_target_ulong val, TCGLifeData arg_life,
3396 TCGRegSet preferred_regs)
3397 {
3398 /* ENV should not be modified. */
3399 tcg_debug_assert(!temp_readonly(ots));
3400
3401 /* The movi is not explicitly generated here. */
3402 if (ots->val_type == TEMP_VAL_REG) {
3403 s->reg_to_temp[ots->reg] = NULL;
3404 }
3405 ots->val_type = TEMP_VAL_CONST;
3406 ots->val = val;
3407 ots->mem_coherent = 0;
3408 if (NEED_SYNC_ARG(0)) {
3409 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
3410 } else if (IS_DEAD_ARG(0)) {
3411 temp_dead(s, ots);
3412 }
3413 }
3414
3415 /*
3416 * Specialized code generation for INDEX_op_mov_*.
3417 */
3418 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
3419 {
3420 const TCGLifeData arg_life = op->life;
3421 TCGRegSet allocated_regs, preferred_regs;
3422 TCGTemp *ts, *ots;
3423 TCGType otype, itype;
3424
3425 allocated_regs = s->reserved_regs;
3426 preferred_regs = op->output_pref[0];
3427 ots = arg_temp(op->args[0]);
3428 ts = arg_temp(op->args[1]);
3429
3430 /* ENV should not be modified. */
3431 tcg_debug_assert(!temp_readonly(ots));
3432
3433 /* Note that otype != itype for no-op truncation. */
3434 otype = ots->type;
3435 itype = ts->type;
3436
3437 if (ts->val_type == TEMP_VAL_CONST) {
3438 /* propagate constant or generate sti */
3439 tcg_target_ulong val = ts->val;
3440 if (IS_DEAD_ARG(1)) {
3441 temp_dead(s, ts);
3442 }
3443 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
3444 return;
3445 }
3446
3447 /* If the source value is in memory we're going to be forced
3448 to have it in a register in order to perform the copy. Copy
3449 the SOURCE value into its own register first, that way we
3450 don't have to reload SOURCE the next time it is used. */
3451 if (ts->val_type == TEMP_VAL_MEM) {
3452 temp_load(s, ts, tcg_target_available_regs[itype],
3453 allocated_regs, preferred_regs);
3454 }
3455
3456 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
3457 if (IS_DEAD_ARG(0)) {
3458 /* mov to a non-saved dead register makes no sense (even with
3459 liveness analysis disabled). */
3460 tcg_debug_assert(NEED_SYNC_ARG(0));
3461 if (!ots->mem_allocated) {
3462 temp_allocate_frame(s, ots);
3463 }
3464 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
3465 if (IS_DEAD_ARG(1)) {
3466 temp_dead(s, ts);
3467 }
3468 temp_dead(s, ots);
3469 } else {
3470 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3471 /* the mov can be suppressed */
3472 if (ots->val_type == TEMP_VAL_REG) {
3473 s->reg_to_temp[ots->reg] = NULL;
3474 }
3475 ots->reg = ts->reg;
3476 temp_dead(s, ts);
3477 } else {
3478 if (ots->val_type != TEMP_VAL_REG) {
3479 /* When allocating a new register, make sure to not spill the
3480 input one. */
3481 tcg_regset_set_reg(allocated_regs, ts->reg);
3482 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3483 allocated_regs, preferred_regs,
3484 ots->indirect_base);
3485 }
3486 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
3487 /*
3488 * Cross register class move not supported.
3489 * Store the source register into the destination slot
3490 * and leave the destination temp as TEMP_VAL_MEM.
3491 */
3492 assert(!temp_readonly(ots));
3493 if (!ts->mem_allocated) {
3494 temp_allocate_frame(s, ots);
3495 }
3496 tcg_out_st(s, ts->type, ts->reg,
3497 ots->mem_base->reg, ots->mem_offset);
3498 ots->mem_coherent = 1;
3499 temp_free_or_dead(s, ots, -1);
3500 return;
3501 }
3502 }
3503 ots->val_type = TEMP_VAL_REG;
3504 ots->mem_coherent = 0;
3505 s->reg_to_temp[ots->reg] = ots;
3506 if (NEED_SYNC_ARG(0)) {
3507 temp_sync(s, ots, allocated_regs, 0, 0);
3508 }
3509 }
3510 }
3511
3512 /*
3513 * Specialized code generation for INDEX_op_dup_vec.
3514 */
3515 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3516 {
3517 const TCGLifeData arg_life = op->life;
3518 TCGRegSet dup_out_regs, dup_in_regs;
3519 TCGTemp *its, *ots;
3520 TCGType itype, vtype;
3521 intptr_t endian_fixup;
3522 unsigned vece;
3523 bool ok;
3524
3525 ots = arg_temp(op->args[0]);
3526 its = arg_temp(op->args[1]);
3527
3528 /* ENV should not be modified. */
3529 tcg_debug_assert(!temp_readonly(ots));
3530
3531 itype = its->type;
3532 vece = TCGOP_VECE(op);
3533 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3534
3535 if (its->val_type == TEMP_VAL_CONST) {
3536 /* Propagate constant via movi -> dupi. */
3537 tcg_target_ulong val = its->val;
3538 if (IS_DEAD_ARG(1)) {
3539 temp_dead(s, its);
3540 }
3541 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3542 return;
3543 }
3544
3545 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3546 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
3547
3548 /* Allocate the output register now. */
3549 if (ots->val_type != TEMP_VAL_REG) {
3550 TCGRegSet allocated_regs = s->reserved_regs;
3551
3552 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3553 /* Make sure to not spill the input register. */
3554 tcg_regset_set_reg(allocated_regs, its->reg);
3555 }
3556 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3557 op->output_pref[0], ots->indirect_base);
3558 ots->val_type = TEMP_VAL_REG;
3559 ots->mem_coherent = 0;
3560 s->reg_to_temp[ots->reg] = ots;
3561 }
3562
3563 switch (its->val_type) {
3564 case TEMP_VAL_REG:
3565 /*
3566 * The dup constriaints must be broad, covering all possible VECE.
3567 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3568 * to fail, indicating that extra moves are required for that case.
3569 */
3570 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3571 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3572 goto done;
3573 }
3574 /* Try again from memory or a vector input register. */
3575 }
3576 if (!its->mem_coherent) {
3577 /*
3578 * The input register is not synced, and so an extra store
3579 * would be required to use memory. Attempt an integer-vector
3580 * register move first. We do not have a TCGRegSet for this.
3581 */
3582 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3583 break;
3584 }
3585 /* Sync the temp back to its slot and load from there. */
3586 temp_sync(s, its, s->reserved_regs, 0, 0);
3587 }
3588 /* fall through */
3589
3590 case TEMP_VAL_MEM:
3591 #ifdef HOST_WORDS_BIGENDIAN
3592 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3593 endian_fixup -= 1 << vece;
3594 #else
3595 endian_fixup = 0;
3596 #endif
3597 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3598 its->mem_offset + endian_fixup)) {
3599 goto done;
3600 }
3601 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3602 break;
3603
3604 default:
3605 g_assert_not_reached();
3606 }
3607
3608 /* We now have a vector input register, so dup must succeed. */
3609 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3610 tcg_debug_assert(ok);
3611
3612 done:
3613 if (IS_DEAD_ARG(1)) {
3614 temp_dead(s, its);
3615 }
3616 if (NEED_SYNC_ARG(0)) {
3617 temp_sync(s, ots, s->reserved_regs, 0, 0);
3618 }
3619 if (IS_DEAD_ARG(0)) {
3620 temp_dead(s, ots);
3621 }
3622 }
3623
3624 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
3625 {
3626 const TCGLifeData arg_life = op->life;
3627 const TCGOpDef * const def = &tcg_op_defs[op->opc];
3628 TCGRegSet i_allocated_regs;
3629 TCGRegSet o_allocated_regs;
3630 int i, k, nb_iargs, nb_oargs;
3631 TCGReg reg;
3632 TCGArg arg;
3633 const TCGArgConstraint *arg_ct;
3634 TCGTemp *ts;
3635 TCGArg new_args[TCG_MAX_OP_ARGS];
3636 int const_args[TCG_MAX_OP_ARGS];
3637
3638 nb_oargs = def->nb_oargs;
3639 nb_iargs = def->nb_iargs;
3640
3641 /* copy constants */
3642 memcpy(new_args + nb_oargs + nb_iargs,
3643 op->args + nb_oargs + nb_iargs,
3644 sizeof(TCGArg) * def->nb_cargs);
3645
3646 i_allocated_regs = s->reserved_regs;
3647 o_allocated_regs = s->reserved_regs;
3648
3649 /* satisfy input constraints */
3650 for (k = 0; k < nb_iargs; k++) {
3651 TCGRegSet i_preferred_regs, o_preferred_regs;
3652
3653 i = def->args_ct[nb_oargs + k].sort_index;
3654 arg = op->args[i];
3655 arg_ct = &def->args_ct[i];
3656 ts = arg_temp(arg);
3657
3658 if (ts->val_type == TEMP_VAL_CONST
3659 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
3660 /* constant is OK for instruction */
3661 const_args[i] = 1;
3662 new_args[i] = ts->val;
3663 continue;
3664 }
3665
3666 i_preferred_regs = o_preferred_regs = 0;
3667 if (arg_ct->ialias) {
3668 o_preferred_regs = op->output_pref[arg_ct->alias_index];
3669
3670 /*
3671 * If the input is readonly, then it cannot also be an
3672 * output and aliased to itself. If the input is not
3673 * dead after the instruction, we must allocate a new
3674 * register and move it.
3675 */
3676 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3677 goto allocate_in_reg;
3678 }
3679
3680 /*
3681 * Check if the current register has already been allocated
3682 * for another input aliased to an output.
3683 */
3684 if (ts->val_type == TEMP_VAL_REG) {
3685 reg = ts->reg;
3686 for (int k2 = 0; k2 < k; k2++) {
3687 int i2 = def->args_ct[nb_oargs + k2].sort_index;
3688 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
3689 goto allocate_in_reg;
3690 }
3691 }
3692 }
3693 i_preferred_regs = o_preferred_regs;
3694 }
3695
3696 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
3697 reg = ts->reg;
3698
3699 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
3700 allocate_in_reg:
3701 /*
3702 * Allocate a new register matching the constraint
3703 * and move the temporary register into it.
3704 */
3705 temp_load(s, ts, tcg_target_available_regs[ts->type],
3706 i_allocated_regs, 0);
3707 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
3708 o_preferred_regs, ts->indirect_base);
3709 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
3710 /*
3711 * Cross register class move not supported. Sync the
3712 * temp back to its slot and load from there.
3713 */
3714 temp_sync(s, ts, i_allocated_regs, 0, 0);
3715 tcg_out_ld(s, ts->type, reg,
3716 ts->mem_base->reg, ts->mem_offset);
3717 }
3718 }
3719 new_args[i] = reg;
3720 const_args[i] = 0;
3721 tcg_regset_set_reg(i_allocated_regs, reg);
3722 }
3723
3724 /* mark dead temporaries and free the associated registers */
3725 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3726 if (IS_DEAD_ARG(i)) {
3727 temp_dead(s, arg_temp(op->args[i]));
3728 }
3729 }
3730
3731 if (def->flags & TCG_OPF_COND_BRANCH) {
3732 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3733 } else if (def->flags & TCG_OPF_BB_END) {
3734 tcg_reg_alloc_bb_end(s, i_allocated_regs);
3735 } else {
3736 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3737 /* XXX: permit generic clobber register list ? */
3738 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3739 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
3740 tcg_reg_free(s, i, i_allocated_regs);
3741 }
3742 }
3743 }
3744 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3745 /* sync globals if the op has side effects and might trigger
3746 an exception. */
3747 sync_globals(s, i_allocated_regs);
3748 }
3749
3750 /* satisfy the output constraints */
3751 for(k = 0; k < nb_oargs; k++) {
3752 i = def->args_ct[k].sort_index;
3753 arg = op->args[i];
3754 arg_ct = &def->args_ct[i];
3755 ts = arg_temp(arg);
3756
3757 /* ENV should not be modified. */
3758 tcg_debug_assert(!temp_readonly(ts));
3759
3760 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
3761 reg = new_args[arg_ct->alias_index];
3762 } else if (arg_ct->newreg) {
3763 reg = tcg_reg_alloc(s, arg_ct->regs,
3764 i_allocated_regs | o_allocated_regs,
3765 op->output_pref[k], ts->indirect_base);
3766 } else {
3767 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
3768 op->output_pref[k], ts->indirect_base);
3769 }
3770 tcg_regset_set_reg(o_allocated_regs, reg);
3771 if (ts->val_type == TEMP_VAL_REG) {
3772 s->reg_to_temp[ts->reg] = NULL;
3773 }
3774 ts->val_type = TEMP_VAL_REG;
3775 ts->reg = reg;
3776 /*
3777 * Temp value is modified, so the value kept in memory is
3778 * potentially not the same.
3779 */
3780 ts->mem_coherent = 0;
3781 s->reg_to_temp[reg] = ts;
3782 new_args[i] = reg;
3783 }
3784 }
3785
3786 /* emit instruction */
3787 if (def->flags & TCG_OPF_VECTOR) {
3788 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3789 new_args, const_args);
3790 } else {
3791 tcg_out_op(s, op->opc, new_args, const_args);
3792 }
3793
3794 /* move the outputs in the correct register if needed */
3795 for(i = 0; i < nb_oargs; i++) {
3796 ts = arg_temp(op->args[i]);
3797
3798 /* ENV should not be modified. */
3799 tcg_debug_assert(!temp_readonly(ts));
3800
3801 if (NEED_SYNC_ARG(i)) {
3802 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
3803 } else if (IS_DEAD_ARG(i)) {
3804 temp_dead(s, ts);
3805 }
3806 }
3807 }
3808
3809 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3810 {
3811 const TCGLifeData arg_life = op->life;
3812 TCGTemp *ots, *itsl, *itsh;
3813 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3814
3815 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3816 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3817 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3818
3819 ots = arg_temp(op->args[0]);
3820 itsl = arg_temp(op->args[1]);
3821 itsh = arg_temp(op->args[2]);
3822
3823 /* ENV should not be modified. */
3824 tcg_debug_assert(!temp_readonly(ots));
3825
3826 /* Allocate the output register now. */
3827 if (ots->val_type != TEMP_VAL_REG) {
3828 TCGRegSet allocated_regs = s->reserved_regs;
3829 TCGRegSet dup_out_regs =
3830 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3831
3832 /* Make sure to not spill the input registers. */
3833 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3834 tcg_regset_set_reg(allocated_regs, itsl->reg);
3835 }
3836 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3837 tcg_regset_set_reg(allocated_regs, itsh->reg);
3838 }
3839
3840 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3841 op->output_pref[0], ots->indirect_base);
3842 ots->val_type = TEMP_VAL_REG;
3843 ots->mem_coherent = 0;
3844 s->reg_to_temp[ots->reg] = ots;
3845 }
3846
3847 /* Promote dup2 of immediates to dupi_vec. */
3848 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3849 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3850 MemOp vece = MO_64;
3851
3852 if (val == dup_const(MO_8, val)) {
3853 vece = MO_8;
3854 } else if (val == dup_const(MO_16, val)) {
3855 vece = MO_16;
3856 } else if (val == dup_const(MO_32, val)) {
3857 vece = MO_32;
3858 }
3859
3860 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3861 goto done;
3862 }
3863
3864 /* If the two inputs form one 64-bit value, try dupm_vec. */
3865 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3866 if (!itsl->mem_coherent) {
3867 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3868 }
3869 if (!itsh->mem_coherent) {
3870 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3871 }
3872 #ifdef HOST_WORDS_BIGENDIAN
3873 TCGTemp *its = itsh;
3874 #else
3875 TCGTemp *its = itsl;
3876 #endif
3877 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3878 its->mem_base->reg, its->mem_offset)) {
3879 goto done;
3880 }
3881 }
3882
3883 /* Fall back to generic expansion. */
3884 return false;
3885
3886 done:
3887 if (IS_DEAD_ARG(1)) {
3888 temp_dead(s, itsl);
3889 }
3890 if (IS_DEAD_ARG(2)) {
3891 temp_dead(s, itsh);
3892 }
3893 if (NEED_SYNC_ARG(0)) {
3894 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3895 } else if (IS_DEAD_ARG(0)) {
3896 temp_dead(s, ots);
3897 }
3898 return true;
3899 }
3900
3901 #ifdef TCG_TARGET_STACK_GROWSUP
3902 #define STACK_DIR(x) (-(x))
3903 #else
3904 #define STACK_DIR(x) (x)
3905 #endif
3906
3907 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
3908 {
3909 const int nb_oargs = TCGOP_CALLO(op);
3910 const int nb_iargs = TCGOP_CALLI(op);
3911 const TCGLifeData arg_life = op->life;
3912 const TCGHelperInfo *info;
3913 int flags, nb_regs, i;
3914 TCGReg reg;
3915 TCGArg arg;
3916 TCGTemp *ts;
3917 intptr_t stack_offset;
3918 size_t call_stack_size;
3919 tcg_insn_unit *func_addr;
3920 int allocate_args;
3921 TCGRegSet allocated_regs;
3922
3923 func_addr = tcg_call_func(op);
3924 info = tcg_call_info(op);
3925 flags = info->flags;
3926
3927 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
3928 if (nb_regs > nb_iargs) {
3929 nb_regs = nb_iargs;
3930 }
3931
3932 /* assign stack slots first */
3933 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
3934 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
3935 ~(TCG_TARGET_STACK_ALIGN - 1);
3936 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3937 if (allocate_args) {
3938 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3939 preallocate call stack */
3940 tcg_abort();
3941 }
3942
3943 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
3944 for (i = nb_regs; i < nb_iargs; i++) {
3945 arg = op->args[nb_oargs + i];
3946 #ifdef TCG_TARGET_STACK_GROWSUP
3947 stack_offset -= sizeof(tcg_target_long);
3948 #endif
3949 if (arg != TCG_CALL_DUMMY_ARG) {
3950 ts = arg_temp(arg);
3951 temp_load(s, ts, tcg_target_available_regs[ts->type],
3952 s->reserved_regs, 0);
3953 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
3954 }
3955 #ifndef TCG_TARGET_STACK_GROWSUP
3956 stack_offset += sizeof(tcg_target_long);
3957 #endif
3958 }
3959
3960 /* assign input registers */
3961 allocated_regs = s->reserved_regs;
3962 for (i = 0; i < nb_regs; i++) {
3963 arg = op->args[nb_oargs + i];
3964 if (arg != TCG_CALL_DUMMY_ARG) {
3965 ts = arg_temp(arg);
3966 reg = tcg_target_call_iarg_regs[i];
3967
3968 if (ts->val_type == TEMP_VAL_REG) {
3969 if (ts->reg != reg) {
3970 tcg_reg_free(s, reg, allocated_regs);
3971 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
3972 /*
3973 * Cross register class move not supported. Sync the
3974 * temp back to its slot and load from there.
3975 */
3976 temp_sync(s, ts, allocated_regs, 0, 0);
3977 tcg_out_ld(s, ts->type, reg,
3978 ts->mem_base->reg, ts->mem_offset);
3979 }
3980 }
3981 } else {
3982 TCGRegSet arg_set = 0;
3983
3984 tcg_reg_free(s, reg, allocated_regs);
3985 tcg_regset_set_reg(arg_set, reg);
3986 temp_load(s, ts, arg_set, allocated_regs, 0);
3987 }
3988
3989 tcg_regset_set_reg(allocated_regs, reg);
3990 }
3991 }
3992
3993 /* mark dead temporaries and free the associated registers */
3994 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3995 if (IS_DEAD_ARG(i)) {
3996 temp_dead(s, arg_temp(op->args[i]));
3997 }
3998 }
3999
4000 /* clobber call registers */
4001 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4002 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
4003 tcg_reg_free(s, i, allocated_regs);
4004 }
4005 }
4006
4007 /* Save globals if they might be written by the helper, sync them if
4008 they might be read. */
4009 if (flags & TCG_CALL_NO_READ_GLOBALS) {
4010 /* Nothing to do */
4011 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
4012 sync_globals(s, allocated_regs);
4013 } else {
4014 save_globals(s, allocated_regs);
4015 }
4016
4017 #ifdef CONFIG_TCG_INTERPRETER
4018 {
4019 gpointer hash = (gpointer)(uintptr_t)info->typemask;
4020 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
4021 assert(cif != NULL);
4022 tcg_out_call(s, func_addr, cif);
4023 }
4024 #else
4025 tcg_out_call(s, func_addr);
4026 #endif
4027
4028 /* assign output registers and emit moves if needed */
4029 for(i = 0; i < nb_oargs; i++) {
4030 arg = op->args[i];
4031 ts = arg_temp(arg);
4032
4033 /* ENV should not be modified. */
4034 tcg_debug_assert(!temp_readonly(ts));
4035
4036 reg = tcg_target_call_oarg_regs[i];
4037 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4038 if (ts->val_type == TEMP_VAL_REG) {
4039 s->reg_to_temp[ts->reg] = NULL;
4040 }
4041 ts->val_type = TEMP_VAL_REG;
4042 ts->reg = reg;
4043 ts->mem_coherent = 0;
4044 s->reg_to_temp[reg] = ts;
4045 if (NEED_SYNC_ARG(i)) {
4046 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
4047 } else if (IS_DEAD_ARG(i)) {
4048 temp_dead(s, ts);
4049 }
4050 }
4051 }
4052
4053 #ifdef CONFIG_PROFILER
4054
4055 /* avoid copy/paste errors */
4056 #define PROF_ADD(to, from, field) \
4057 do { \
4058 (to)->field += qatomic_read(&((from)->field)); \
4059 } while (0)
4060
4061 #define PROF_MAX(to, from, field) \
4062 do { \
4063 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
4064 if (val__ > (to)->field) { \
4065 (to)->field = val__; \
4066 } \
4067 } while (0)
4068
4069 /* Pass in a zero'ed @prof */
4070 static inline
4071 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4072 {
4073 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
4074 unsigned int i;
4075
4076 for (i = 0; i < n_ctxs; i++) {
4077 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4078 const TCGProfile *orig = &s->prof;
4079
4080 if (counters) {
4081 PROF_ADD(prof, orig, cpu_exec_time);
4082 PROF_ADD(prof, orig, tb_count1);
4083 PROF_ADD(prof, orig, tb_count);
4084 PROF_ADD(prof, orig, op_count);
4085 PROF_MAX(prof, orig, op_count_max);
4086 PROF_ADD(prof, orig, temp_count);
4087 PROF_MAX(prof, orig, temp_count_max);
4088 PROF_ADD(prof, orig, del_op_count);
4089 PROF_ADD(prof, orig, code_in_len);
4090 PROF_ADD(prof, orig, code_out_len);
4091 PROF_ADD(prof, orig, search_out_len);
4092 PROF_ADD(prof, orig, interm_time);
4093 PROF_ADD(prof, orig, code_time);
4094 PROF_ADD(prof, orig, la_time);
4095 PROF_ADD(prof, orig, opt_time);
4096 PROF_ADD(prof, orig, restore_count);
4097 PROF_ADD(prof, orig, restore_time);
4098 }
4099 if (table) {
4100 int i;
4101
4102 for (i = 0; i < NB_OPS; i++) {
4103 PROF_ADD(prof, orig, table_op_count[i]);
4104 }
4105 }
4106 }
4107 }
4108
4109 #undef PROF_ADD
4110 #undef PROF_MAX
4111
4112 static void tcg_profile_snapshot_counters(TCGProfile *prof)
4113 {
4114 tcg_profile_snapshot(prof, true, false);
4115 }
4116
4117 static void tcg_profile_snapshot_table(TCGProfile *prof)
4118 {
4119 tcg_profile_snapshot(prof, false, true);
4120 }
4121
4122 void tcg_dump_op_count(void)
4123 {
4124 TCGProfile prof = {};
4125 int i;
4126
4127 tcg_profile_snapshot_table(&prof);
4128 for (i = 0; i < NB_OPS; i++) {
4129 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name,
4130 prof.table_op_count[i]);
4131 }
4132 }
4133
4134 int64_t tcg_cpu_exec_time(void)
4135 {
4136 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
4137 unsigned int i;
4138 int64_t ret = 0;
4139
4140 for (i = 0; i < n_ctxs; i++) {
4141 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4142 const TCGProfile *prof = &s->prof;
4143
4144 ret += qatomic_read(&prof->cpu_exec_time);
4145 }
4146 return ret;
4147 }
4148 #else
4149 void tcg_dump_op_count(void)
4150 {
4151 qemu_printf("[TCG profiler not compiled]\n");
4152 }
4153
4154 int64_t tcg_cpu_exec_time(void)
4155 {
4156 error_report("%s: TCG profiler not compiled", __func__);
4157 exit(EXIT_FAILURE);
4158 }
4159 #endif
4160
4161
4162 int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
4163 {
4164 #ifdef CONFIG_PROFILER
4165 TCGProfile *prof = &s->prof;
4166 #endif
4167 int i, num_insns;
4168 TCGOp *op;
4169
4170 #ifdef CONFIG_PROFILER
4171 {
4172 int n = 0;
4173
4174 QTAILQ_FOREACH(op, &s->ops, link) {
4175 n++;
4176 }
4177 qatomic_set(&prof->op_count, prof->op_count + n);
4178 if (n > prof->op_count_max) {
4179 qatomic_set(&prof->op_count_max, n);
4180 }
4181
4182 n = s->nb_temps;
4183 qatomic_set(&prof->temp_count, prof->temp_count + n);
4184 if (n > prof->temp_count_max) {
4185 qatomic_set(&prof->temp_count_max, n);
4186 }
4187 }
4188 #endif
4189
4190 #ifdef DEBUG_DISAS
4191 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4192 && qemu_log_in_addr_range(tb->pc))) {
4193 FILE *logfile = qemu_log_lock();
4194 qemu_log("OP:\n");
4195 tcg_dump_ops(s, false);
4196 qemu_log("\n");
4197 qemu_log_unlock(logfile);
4198 }
4199 #endif
4200
4201 #ifdef CONFIG_DEBUG_TCG
4202 /* Ensure all labels referenced have been emitted. */
4203 {
4204 TCGLabel *l;
4205 bool error = false;
4206
4207 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4208 if (unlikely(!l->present) && l->refs) {
4209 qemu_log_mask(CPU_LOG_TB_OP,
4210 "$L%d referenced but not present.\n", l->id);
4211 error = true;
4212 }
4213 }
4214 assert(!error);
4215 }
4216 #endif
4217
4218 #ifdef CONFIG_PROFILER
4219 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
4220 #endif
4221
4222 #ifdef USE_TCG_OPTIMIZATIONS
4223 tcg_optimize(s);
4224 #endif
4225
4226 #ifdef CONFIG_PROFILER
4227 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4228 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
4229 #endif
4230
4231 reachable_code_pass(s);
4232 liveness_pass_1(s);
4233
4234 if (s->nb_indirects > 0) {
4235 #ifdef DEBUG_DISAS
4236 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4237 && qemu_log_in_addr_range(tb->pc))) {
4238 FILE *logfile = qemu_log_lock();
4239 qemu_log("OP before indirect lowering:\n");
4240 tcg_dump_ops(s, false);
4241 qemu_log("\n");
4242 qemu_log_unlock(logfile);
4243 }
4244 #endif
4245 /* Replace indirect temps with direct temps. */
4246 if (liveness_pass_2(s)) {
4247 /* If changes were made, re-run liveness. */
4248 liveness_pass_1(s);
4249 }
4250 }
4251
4252 #ifdef CONFIG_PROFILER
4253 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
4254 #endif
4255
4256 #ifdef DEBUG_DISAS
4257 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4258 && qemu_log_in_addr_range(tb->pc))) {
4259 FILE *logfile = qemu_log_lock();
4260 qemu_log("OP after optimization and liveness analysis:\n");
4261 tcg_dump_ops(s, true);
4262 qemu_log("\n");
4263 qemu_log_unlock(logfile);
4264 }
4265 #endif
4266
4267 tcg_reg_alloc_start(s);
4268
4269 /*
4270 * Reset the buffer pointers when restarting after overflow.
4271 * TODO: Move this into translate-all.c with the rest of the
4272 * buffer management. Having only this done here is confusing.
4273 */
4274 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4275 s->code_ptr = s->code_buf;
4276
4277 #ifdef TCG_TARGET_NEED_LDST_LABELS
4278 QSIMPLEQ_INIT(&s->ldst_labels);
4279 #endif
4280 #ifdef TCG_TARGET_NEED_POOL_LABELS
4281 s->pool_labels = NULL;
4282 #endif
4283
4284 num_insns = -1;
4285 QTAILQ_FOREACH(op, &s->ops, link) {
4286 TCGOpcode opc = op->opc;
4287
4288 #ifdef CONFIG_PROFILER
4289 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
4290 #endif
4291
4292 switch (opc) {
4293 case INDEX_op_mov_i32:
4294 case INDEX_op_mov_i64:
4295 case INDEX_op_mov_vec:
4296 tcg_reg_alloc_mov(s, op);
4297 break;
4298 case INDEX_op_dup_vec:
4299 tcg_reg_alloc_dup(s, op);
4300 break;
4301 case INDEX_op_insn_start:
4302 if (num_insns >= 0) {
4303 size_t off = tcg_current_code_size(s);
4304 s->gen_insn_end_off[num_insns] = off;
4305 /* Assert that we do not overflow our stored offset. */
4306 assert(s->gen_insn_end_off[num_insns] == off);
4307 }
4308 num_insns++;
4309 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4310 target_ulong a;
4311 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4312 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
4313 #else
4314 a = op->args[i];
4315 #endif
4316 s->gen_insn_data[num_insns][i] = a;
4317 }
4318 break;
4319 case INDEX_op_discard:
4320 temp_dead(s, arg_temp(op->args[0]));
4321 break;
4322 case INDEX_op_set_label:
4323 tcg_reg_alloc_bb_end(s, s->reserved_regs);
4324 tcg_out_label(s, arg_label(op->args[0]));
4325 break;
4326 case INDEX_op_call:
4327 tcg_reg_alloc_call(s, op);
4328 break;
4329 case INDEX_op_dup2_vec:
4330 if (tcg_reg_alloc_dup2(s, op)) {
4331 break;
4332 }
4333 /* fall through */
4334 default:
4335 /* Sanity check that we've not introduced any unhandled opcodes. */
4336 tcg_debug_assert(tcg_op_supported(opc));
4337 /* Note: in order to speed up the code, it would be much
4338 faster to have specialized register allocator functions for
4339 some common argument patterns */
4340 tcg_reg_alloc_op(s, op);
4341 break;
4342 }
4343 #ifdef CONFIG_DEBUG_TCG
4344 check_regs(s);
4345 #endif
4346 /* Test for (pending) buffer overflow. The assumption is that any
4347 one operation beginning below the high water mark cannot overrun
4348 the buffer completely. Thus we can test for overflow after
4349 generating code without having to check during generation. */
4350 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
4351 return -1;
4352 }
4353 /* Test for TB overflow, as seen by gen_insn_end_off. */
4354 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4355 return -2;
4356 }
4357 }
4358 tcg_debug_assert(num_insns >= 0);
4359 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
4360
4361 /* Generate TB finalization at the end of block */
4362 #ifdef TCG_TARGET_NEED_LDST_LABELS
4363 i = tcg_out_ldst_finalize(s);
4364 if (i < 0) {
4365 return i;
4366 }
4367 #endif
4368 #ifdef TCG_TARGET_NEED_POOL_LABELS
4369 i = tcg_out_pool_finalize(s);
4370 if (i < 0) {
4371 return i;
4372 }
4373 #endif
4374 if (!tcg_resolve_relocs(s)) {
4375 return -2;
4376 }
4377
4378 #ifndef CONFIG_TCG_INTERPRETER
4379 /* flush instruction cache */
4380 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4381 (uintptr_t)s->code_buf,
4382 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
4383 #endif
4384
4385 return tcg_current_code_size(s);
4386 }
4387
4388 #ifdef CONFIG_PROFILER
4389 void tcg_dump_info(void)
4390 {
4391 TCGProfile prof = {};
4392 const TCGProfile *s;
4393 int64_t tb_count;
4394 int64_t tb_div_count;
4395 int64_t tot;
4396
4397 tcg_profile_snapshot_counters(&prof);
4398 s = &prof;
4399 tb_count = s->tb_count;
4400 tb_div_count = tb_count ? tb_count : 1;
4401 tot = s->interm_time + s->code_time;
4402
4403 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
4404 tot, tot / 2.4e9);
4405 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64
4406 " %0.1f%%)\n",
4407 tb_count, s->tb_count1 - tb_count,
4408 (double)(s->tb_count1 - s->tb_count)
4409 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4410 qemu_printf("avg ops/TB %0.1f max=%d\n",
4411 (double)s->op_count / tb_div_count, s->op_count_max);
4412 qemu_printf("deleted ops/TB %0.2f\n",
4413 (double)s->del_op_count / tb_div_count);
4414 qemu_printf("avg temps/TB %0.2f max=%d\n",
4415 (double)s->temp_count / tb_div_count, s->temp_count_max);
4416 qemu_printf("avg host code/TB %0.1f\n",
4417 (double)s->code_out_len / tb_div_count);
4418 qemu_printf("avg search data/TB %0.1f\n",
4419 (double)s->search_out_len / tb_div_count);
4420
4421 qemu_printf("cycles/op %0.1f\n",
4422 s->op_count ? (double)tot / s->op_count : 0);
4423 qemu_printf("cycles/in byte %0.1f\n",
4424 s->code_in_len ? (double)tot / s->code_in_len : 0);
4425 qemu_printf("cycles/out byte %0.1f\n",
4426 s->code_out_len ? (double)tot / s->code_out_len : 0);
4427 qemu_printf("cycles/search byte %0.1f\n",
4428 s->search_out_len ? (double)tot / s->search_out_len : 0);
4429 if (tot == 0) {
4430 tot = 1;
4431 }
4432 qemu_printf(" gen_interm time %0.1f%%\n",
4433 (double)s->interm_time / tot * 100.0);
4434 qemu_printf(" gen_code time %0.1f%%\n",
4435 (double)s->code_time / tot * 100.0);
4436 qemu_printf("optim./code time %0.1f%%\n",
4437 (double)s->opt_time / (s->code_time ? s->code_time : 1)
4438 * 100.0);
4439 qemu_printf("liveness/code time %0.1f%%\n",
4440 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
4441 qemu_printf("cpu_restore count %" PRId64 "\n",
4442 s->restore_count);
4443 qemu_printf(" avg cycles %0.1f\n",
4444 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
4445 }
4446 #else
4447 void tcg_dump_info(void)
4448 {
4449 qemu_printf("[TCG profiler not compiled]\n");
4450 }
4451 #endif
4452
4453 #ifdef ELF_HOST_MACHINE
4454 /* In order to use this feature, the backend needs to do three things:
4455
4456 (1) Define ELF_HOST_MACHINE to indicate both what value to
4457 put into the ELF image and to indicate support for the feature.
4458
4459 (2) Define tcg_register_jit. This should create a buffer containing
4460 the contents of a .debug_frame section that describes the post-
4461 prologue unwind info for the tcg machine.
4462
4463 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4464 */
4465
4466 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4467 typedef enum {
4468 JIT_NOACTION = 0,
4469 JIT_REGISTER_FN,
4470 JIT_UNREGISTER_FN
4471 } jit_actions_t;
4472
4473 struct jit_code_entry {
4474 struct jit_code_entry *next_entry;
4475 struct jit_code_entry *prev_entry;
4476 const void *symfile_addr;
4477 uint64_t symfile_size;
4478 };
4479
4480 struct jit_descriptor {
4481 uint32_t version;
4482 uint32_t action_flag;
4483 struct jit_code_entry *relevant_entry;
4484 struct jit_code_entry *first_entry;
4485 };
4486
4487 void __jit_debug_register_code(void) __attribute__((noinline));
4488 void __jit_debug_register_code(void)
4489 {
4490 asm("");
4491 }
4492
4493 /* Must statically initialize the version, because GDB may check
4494 the version before we can set it. */
4495 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4496
4497 /* End GDB interface. */
4498
4499 static int find_string(const char *strtab, const char *str)
4500 {
4501 const char *p = strtab + 1;
4502
4503 while (1) {
4504 if (strcmp(p, str) == 0) {
4505 return p - strtab;
4506 }
4507 p += strlen(p) + 1;
4508 }
4509 }
4510
4511 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
4512 const void *debug_frame,
4513 size_t debug_frame_size)
4514 {
4515 struct __attribute__((packed)) DebugInfo {
4516 uint32_t len;
4517 uint16_t version;
4518 uint32_t abbrev;
4519 uint8_t ptr_size;
4520 uint8_t cu_die;
4521 uint16_t cu_lang;
4522 uintptr_t cu_low_pc;
4523 uintptr_t cu_high_pc;
4524 uint8_t fn_die;
4525 char fn_name[16];
4526 uintptr_t fn_low_pc;
4527 uintptr_t fn_high_pc;
4528 uint8_t cu_eoc;
4529 };
4530
4531 struct ElfImage {
4532 ElfW(Ehdr) ehdr;
4533 ElfW(Phdr) phdr;
4534 ElfW(Shdr) shdr[7];
4535 ElfW(Sym) sym[2];
4536 struct DebugInfo di;
4537 uint8_t da[24];
4538 char str[80];
4539 };
4540
4541 struct ElfImage *img;
4542
4543 static const struct ElfImage img_template = {
4544 .ehdr = {
4545 .e_ident[EI_MAG0] = ELFMAG0,
4546 .e_ident[EI_MAG1] = ELFMAG1,
4547 .e_ident[EI_MAG2] = ELFMAG2,
4548 .e_ident[EI_MAG3] = ELFMAG3,
4549 .e_ident[EI_CLASS] = ELF_CLASS,
4550 .e_ident[EI_DATA] = ELF_DATA,
4551 .e_ident[EI_VERSION] = EV_CURRENT,
4552 .e_type = ET_EXEC,
4553 .e_machine = ELF_HOST_MACHINE,
4554 .e_version = EV_CURRENT,
4555 .e_phoff = offsetof(struct ElfImage, phdr),
4556 .e_shoff = offsetof(struct ElfImage, shdr),
4557 .e_ehsize = sizeof(ElfW(Shdr)),
4558 .e_phentsize = sizeof(ElfW(Phdr)),
4559 .e_phnum = 1,
4560 .e_shentsize = sizeof(ElfW(Shdr)),
4561 .e_shnum = ARRAY_SIZE(img->shdr),
4562 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
4563 #ifdef ELF_HOST_FLAGS
4564 .e_flags = ELF_HOST_FLAGS,
4565 #endif
4566 #ifdef ELF_OSABI
4567 .e_ident[EI_OSABI] = ELF_OSABI,
4568 #endif
4569 },
4570 .phdr = {
4571 .p_type = PT_LOAD,
4572 .p_flags = PF_X,
4573 },
4574 .shdr = {
4575 [0] = { .sh_type = SHT_NULL },
4576 /* Trick: The contents of code_gen_buffer are not present in
4577 this fake ELF file; that got allocated elsewhere. Therefore
4578 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4579 will not look for contents. We can record any address. */
4580 [1] = { /* .text */
4581 .sh_type = SHT_NOBITS,
4582 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4583 },
4584 [2] = { /* .debug_info */
4585 .sh_type = SHT_PROGBITS,
4586 .sh_offset = offsetof(struct ElfImage, di),
4587 .sh_size = sizeof(struct DebugInfo),
4588 },
4589 [3] = { /* .debug_abbrev */
4590 .sh_type = SHT_PROGBITS,
4591 .sh_offset = offsetof(struct ElfImage, da),
4592 .sh_size = sizeof(img->da),
4593 },
4594 [4] = { /* .debug_frame */
4595 .sh_type = SHT_PROGBITS,
4596 .sh_offset = sizeof(struct ElfImage),
4597 },
4598 [5] = { /* .symtab */
4599 .sh_type = SHT_SYMTAB,
4600 .sh_offset = offsetof(struct ElfImage, sym),
4601 .sh_size = sizeof(img->sym),
4602 .sh_info = 1,
4603 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4604 .sh_entsize = sizeof(ElfW(Sym)),
4605 },
4606 [6] = { /* .strtab */
4607 .sh_type = SHT_STRTAB,
4608 .sh_offset = offsetof(struct ElfImage, str),
4609 .sh_size = sizeof(img->str),
4610 }
4611 },
4612 .sym = {
4613 [1] = { /* code_gen_buffer */
4614 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4615 .st_shndx = 1,
4616 }
4617 },
4618 .di = {
4619 .len = sizeof(struct DebugInfo) - 4,
4620 .version = 2,
4621 .ptr_size = sizeof(void *),
4622 .cu_die = 1,
4623 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4624 .fn_die = 2,
4625 .fn_name = "code_gen_buffer"
4626 },
4627 .da = {
4628 1, /* abbrev number (the cu) */
4629 0x11, 1, /* DW_TAG_compile_unit, has children */
4630 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4631 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4632 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4633 0, 0, /* end of abbrev */
4634 2, /* abbrev number (the fn) */
4635 0x2e, 0, /* DW_TAG_subprogram, no children */
4636 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4637 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4638 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4639 0, 0, /* end of abbrev */
4640 0 /* no more abbrev */
4641 },
4642 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4643 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
4644 };
4645
4646 /* We only need a single jit entry; statically allocate it. */
4647 static struct jit_code_entry one_entry;
4648
4649 uintptr_t buf = (uintptr_t)buf_ptr;
4650 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
4651 DebugFrameHeader *dfh;
4652
4653 img = g_malloc(img_size);
4654 *img = img_template;
4655
4656 img->phdr.p_vaddr = buf;
4657 img->phdr.p_paddr = buf;
4658 img->phdr.p_memsz = buf_size;
4659
4660 img->shdr[1].sh_name = find_string(img->str, ".text");
4661 img->shdr[1].sh_addr = buf;
4662 img->shdr[1].sh_size = buf_size;
4663
4664 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4665 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4666
4667 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4668 img->shdr[4].sh_size = debug_frame_size;
4669
4670 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4671 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4672
4673 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4674 img->sym[1].st_value = buf;
4675 img->sym[1].st_size = buf_size;
4676
4677 img->di.cu_low_pc = buf;
4678 img->di.cu_high_pc = buf + buf_size;
4679 img->di.fn_low_pc = buf;
4680 img->di.fn_high_pc = buf + buf_size;
4681
4682 dfh = (DebugFrameHeader *)(img + 1);
4683 memcpy(dfh, debug_frame, debug_frame_size);
4684 dfh->fde.func_start = buf;
4685 dfh->fde.func_len = buf_size;
4686
4687 #ifdef DEBUG_JIT
4688 /* Enable this block to be able to debug the ELF image file creation.
4689 One can use readelf, objdump, or other inspection utilities. */
4690 {
4691 FILE *f = fopen("/tmp/qemu.jit", "w+b");
4692 if (f) {
4693 if (fwrite(img, img_size, 1, f) != img_size) {
4694 /* Avoid stupid unused return value warning for fwrite. */
4695 }
4696 fclose(f);
4697 }
4698 }
4699 #endif
4700
4701 one_entry.symfile_addr = img;
4702 one_entry.symfile_size = img_size;
4703
4704 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4705 __jit_debug_descriptor.relevant_entry = &one_entry;
4706 __jit_debug_descriptor.first_entry = &one_entry;
4707 __jit_debug_register_code();
4708 }
4709 #else
4710 /* No support for the feature. Provide the entry point expected by exec.c,
4711 and implement the internal function we declared earlier. */
4712
4713 static void tcg_register_jit_int(const void *buf, size_t size,
4714 const void *debug_frame,
4715 size_t debug_frame_size)
4716 {
4717 }
4718
4719 void tcg_register_jit(const void *buf, size_t buf_size)
4720 {
4721 }
4722 #endif /* ELF_HOST_MACHINE */
4723
4724 #if !TCG_TARGET_MAYBE_vec
4725 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4726 {
4727 g_assert_not_reached();
4728 }
4729 #endif
armbru's QEMU hackery