?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
libvduse: Fix assignment in vring_set_avail_event
[qemu.git] / tcg / tcg.c
blobda91779890cb585560f1c8966d2500f92ae2d949
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
27
28 #include "qemu/osdep.h"
29
30 /* Define to jump the ELF file used to communicate with GDB. */
31 #undef DEBUG_JIT
32
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
38 #include "qemu/cacheflush.h"
39 #include "qemu/cacheinfo.h"
40
41 /* Note: the long term plan is to reduce the dependencies on the QEMU
42 CPU definitions. Currently they are used for qemu_ld/st
43 instructions */
44 #define NO_CPU_IO_DEFS
45
46 #include "exec/exec-all.h"
47 #include "tcg/tcg-op.h"
48
49 #if UINTPTR_MAX == UINT32_MAX
50 # define ELF_CLASS ELFCLASS32
51 #else
52 # define ELF_CLASS ELFCLASS64
53 #endif
54 #if HOST_BIG_ENDIAN
55 # define ELF_DATA ELFDATA2MSB
56 #else
57 # define ELF_DATA ELFDATA2LSB
58 #endif
59
60 #include "elf.h"
61 #include "exec/log.h"
62 #include "tcg/tcg-ldst.h"
63 #include "tcg-internal.h"
64
65 /* Forward declarations for functions declared in tcg-target.c.inc and
66 used here. */
67 static void tcg_target_init(TCGContext *s);
68 static void tcg_target_qemu_prologue(TCGContext *s);
69 static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
70 intptr_t value, intptr_t addend);
71
72 /* The CIE and FDE header definitions will be common to all hosts. */
73 typedef struct {
74 uint32_t len __attribute__((aligned((sizeof(void *)))));
75 uint32_t id;
76 uint8_t version;
77 char augmentation[1];
78 uint8_t code_align;
79 uint8_t data_align;
80 uint8_t return_column;
81 } DebugFrameCIE;
82
83 typedef struct QEMU_PACKED {
84 uint32_t len __attribute__((aligned((sizeof(void *)))));
85 uint32_t cie_offset;
86 uintptr_t func_start;
87 uintptr_t func_len;
88 } DebugFrameFDEHeader;
89
90 typedef struct QEMU_PACKED {
91 DebugFrameCIE cie;
92 DebugFrameFDEHeader fde;
93 } DebugFrameHeader;
94
95 static void tcg_register_jit_int(const void *buf, size_t size,
96 const void *debug_frame,
97 size_t debug_frame_size)
98 __attribute__((unused));
99
100 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
101 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
102 intptr_t arg2);
103 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
104 static void tcg_out_movi(TCGContext *s, TCGType type,
105 TCGReg ret, tcg_target_long arg);
106 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
107 const TCGArg args[TCG_MAX_OP_ARGS],
108 const int const_args[TCG_MAX_OP_ARGS]);
109 #if TCG_TARGET_MAYBE_vec
110 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
111 TCGReg dst, TCGReg src);
112 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, TCGReg base, intptr_t offset);
114 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, int64_t arg);
116 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
117 unsigned vecl, unsigned vece,
118 const TCGArg args[TCG_MAX_OP_ARGS],
119 const int const_args[TCG_MAX_OP_ARGS]);
120 #else
121 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
122 TCGReg dst, TCGReg src)
123 {
124 g_assert_not_reached();
125 }
126 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
127 TCGReg dst, TCGReg base, intptr_t offset)
128 {
129 g_assert_not_reached();
130 }
131 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
132 TCGReg dst, int64_t arg)
133 {
134 g_assert_not_reached();
135 }
136 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
137 unsigned vecl, unsigned vece,
138 const TCGArg args[TCG_MAX_OP_ARGS],
139 const int const_args[TCG_MAX_OP_ARGS])
140 {
141 g_assert_not_reached();
142 }
143 #endif
144 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
145 intptr_t arg2);
146 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
147 TCGReg base, intptr_t ofs);
148 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
149 const TCGHelperInfo *info);
150 static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
151 #ifdef TCG_TARGET_NEED_LDST_LABELS
152 static int tcg_out_ldst_finalize(TCGContext *s);
153 #endif
154
155 TCGContext tcg_init_ctx;
156 __thread TCGContext *tcg_ctx;
157
158 TCGContext **tcg_ctxs;
159 unsigned int tcg_cur_ctxs;
160 unsigned int tcg_max_ctxs;
161 TCGv_env cpu_env = 0;
162 const void *tcg_code_gen_epilogue;
163 uintptr_t tcg_splitwx_diff;
164
165 #ifndef CONFIG_TCG_INTERPRETER
166 tcg_prologue_fn *tcg_qemu_tb_exec;
167 #endif
168
169 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
170 static TCGRegSet tcg_target_call_clobber_regs;
171
172 #if TCG_TARGET_INSN_UNIT_SIZE == 1
173 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
174 {
175 *s->code_ptr++ = v;
176 }
177
178 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
179 uint8_t v)
180 {
181 *p = v;
182 }
183 #endif
184
185 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
186 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
187 {
188 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
189 *s->code_ptr++ = v;
190 } else {
191 tcg_insn_unit *p = s->code_ptr;
192 memcpy(p, &v, sizeof(v));
193 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
194 }
195 }
196
197 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
198 uint16_t v)
199 {
200 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
201 *p = v;
202 } else {
203 memcpy(p, &v, sizeof(v));
204 }
205 }
206 #endif
207
208 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
209 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
210 {
211 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
212 *s->code_ptr++ = v;
213 } else {
214 tcg_insn_unit *p = s->code_ptr;
215 memcpy(p, &v, sizeof(v));
216 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
217 }
218 }
219
220 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
221 uint32_t v)
222 {
223 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
224 *p = v;
225 } else {
226 memcpy(p, &v, sizeof(v));
227 }
228 }
229 #endif
230
231 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
232 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
233 {
234 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
235 *s->code_ptr++ = v;
236 } else {
237 tcg_insn_unit *p = s->code_ptr;
238 memcpy(p, &v, sizeof(v));
239 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
240 }
241 }
242
243 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
244 uint64_t v)
245 {
246 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
247 *p = v;
248 } else {
249 memcpy(p, &v, sizeof(v));
250 }
251 }
252 #endif
253
254 /* label relocation processing */
255
256 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
257 TCGLabel *l, intptr_t addend)
258 {
259 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
260
261 r->type = type;
262 r->ptr = code_ptr;
263 r->addend = addend;
264 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
265 }
266
267 static void tcg_out_label(TCGContext *s, TCGLabel *l)
268 {
269 tcg_debug_assert(!l->has_value);
270 l->has_value = 1;
271 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
272 }
273
274 TCGLabel *gen_new_label(void)
275 {
276 TCGContext *s = tcg_ctx;
277 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
278
279 memset(l, 0, sizeof(TCGLabel));
280 l->id = s->nb_labels++;
281 QSIMPLEQ_INIT(&l->relocs);
282
283 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
284
285 return l;
286 }
287
288 static bool tcg_resolve_relocs(TCGContext *s)
289 {
290 TCGLabel *l;
291
292 QSIMPLEQ_FOREACH(l, &s->labels, next) {
293 TCGRelocation *r;
294 uintptr_t value = l->u.value;
295
296 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
297 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
298 return false;
299 }
300 }
301 }
302 return true;
303 }
304
305 static void set_jmp_reset_offset(TCGContext *s, int which)
306 {
307 /*
308 * We will check for overflow at the end of the opcode loop in
309 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
310 */
311 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
312 }
313
314 /* Signal overflow, starting over with fewer guest insns. */
315 static G_NORETURN
316 void tcg_raise_tb_overflow(TCGContext *s)
317 {
318 siglongjmp(s->jmp_trans, -2);
319 }
320
321 #define C_PFX1(P, A) P##A
322 #define C_PFX2(P, A, B) P##A##_##B
323 #define C_PFX3(P, A, B, C) P##A##_##B##_##C
324 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
325 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
326 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
327
328 /* Define an enumeration for the various combinations. */
329
330 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
331 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
332 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
333 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
334
335 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
336 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
337 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
338 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
339
340 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
341
342 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
343 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
344 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
345 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
346
347 typedef enum {
348 #include "tcg-target-con-set.h"
349 } TCGConstraintSetIndex;
350
351 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
352
353 #undef C_O0_I1
354 #undef C_O0_I2
355 #undef C_O0_I3
356 #undef C_O0_I4
357 #undef C_O1_I1
358 #undef C_O1_I2
359 #undef C_O1_I3
360 #undef C_O1_I4
361 #undef C_N1_I2
362 #undef C_O2_I1
363 #undef C_O2_I2
364 #undef C_O2_I3
365 #undef C_O2_I4
366
367 /* Put all of the constraint sets into an array, indexed by the enum. */
368
369 #define C_O0_I1(I1) { .args_ct_str = { #I1 } },
370 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
371 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
372 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
373
374 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
375 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
376 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
377 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
378
379 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
380
381 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
382 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
383 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
384 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
385
386 static const TCGTargetOpDef constraint_sets[] = {
387 #include "tcg-target-con-set.h"
388 };
389
390
391 #undef C_O0_I1
392 #undef C_O0_I2
393 #undef C_O0_I3
394 #undef C_O0_I4
395 #undef C_O1_I1
396 #undef C_O1_I2
397 #undef C_O1_I3
398 #undef C_O1_I4
399 #undef C_N1_I2
400 #undef C_O2_I1
401 #undef C_O2_I2
402 #undef C_O2_I3
403 #undef C_O2_I4
404
405 /* Expand the enumerator to be returned from tcg_target_op_def(). */
406
407 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
408 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
409 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
410 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
411
412 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
413 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
414 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
415 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
416
417 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
418
419 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
420 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
421 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
422 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
423
424 #include "tcg-target.c.inc"
425
426 static void alloc_tcg_plugin_context(TCGContext *s)
427 {
428 #ifdef CONFIG_PLUGIN
429 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
430 s->plugin_tb->insns =
431 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
432 #endif
433 }
434
435 /*
436 * All TCG threads except the parent (i.e. the one that called tcg_context_init
437 * and registered the target's TCG globals) must register with this function
438 * before initiating translation.
439 *
440 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
441 * of tcg_region_init() for the reasoning behind this.
442 *
443 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
444 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
445 * is not used anymore for translation once this function is called.
446 *
447 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
448 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
449 */
450 #ifdef CONFIG_USER_ONLY
451 void tcg_register_thread(void)
452 {
453 tcg_ctx = &tcg_init_ctx;
454 }
455 #else
456 void tcg_register_thread(void)
457 {
458 TCGContext *s = g_malloc(sizeof(*s));
459 unsigned int i, n;
460
461 *s = tcg_init_ctx;
462
463 /* Relink mem_base. */
464 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
465 if (tcg_init_ctx.temps[i].mem_base) {
466 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
467 tcg_debug_assert(b >= 0 && b < n);
468 s->temps[i].mem_base = &s->temps[b];
469 }
470 }
471
472 /* Claim an entry in tcg_ctxs */
473 n = qatomic_fetch_inc(&tcg_cur_ctxs);
474 g_assert(n < tcg_max_ctxs);
475 qatomic_set(&tcg_ctxs[n], s);
476
477 if (n > 0) {
478 alloc_tcg_plugin_context(s);
479 tcg_region_initial_alloc(s);
480 }
481
482 tcg_ctx = s;
483 }
484 #endif /* !CONFIG_USER_ONLY */
485
486 /* pool based memory allocation */
487 void *tcg_malloc_internal(TCGContext *s, int size)
488 {
489 TCGPool *p;
490 int pool_size;
491
492 if (size > TCG_POOL_CHUNK_SIZE) {
493 /* big malloc: insert a new pool (XXX: could optimize) */
494 p = g_malloc(sizeof(TCGPool) + size);
495 p->size = size;
496 p->next = s->pool_first_large;
497 s->pool_first_large = p;
498 return p->data;
499 } else {
500 p = s->pool_current;
501 if (!p) {
502 p = s->pool_first;
503 if (!p)
504 goto new_pool;
505 } else {
506 if (!p->next) {
507 new_pool:
508 pool_size = TCG_POOL_CHUNK_SIZE;
509 p = g_malloc(sizeof(TCGPool) + pool_size);
510 p->size = pool_size;
511 p->next = NULL;
512 if (s->pool_current) {
513 s->pool_current->next = p;
514 } else {
515 s->pool_first = p;
516 }
517 } else {
518 p = p->next;
519 }
520 }
521 }
522 s->pool_current = p;
523 s->pool_cur = p->data + size;
524 s->pool_end = p->data + p->size;
525 return p->data;
526 }
527
528 void tcg_pool_reset(TCGContext *s)
529 {
530 TCGPool *p, *t;
531 for (p = s->pool_first_large; p; p = t) {
532 t = p->next;
533 g_free(p);
534 }
535 s->pool_first_large = NULL;
536 s->pool_cur = s->pool_end = NULL;
537 s->pool_current = NULL;
538 }
539
540 #include "exec/helper-proto.h"
541
542 static TCGHelperInfo all_helpers[] = {
543 #include "exec/helper-tcg.h"
544 };
545 static GHashTable *helper_table;
546
547 #ifdef CONFIG_TCG_INTERPRETER
548 static ffi_type *typecode_to_ffi(int argmask)
549 {
550 switch (argmask) {
551 case dh_typecode_void:
552 return &ffi_type_void;
553 case dh_typecode_i32:
554 return &ffi_type_uint32;
555 case dh_typecode_s32:
556 return &ffi_type_sint32;
557 case dh_typecode_i64:
558 return &ffi_type_uint64;
559 case dh_typecode_s64:
560 return &ffi_type_sint64;
561 case dh_typecode_ptr:
562 return &ffi_type_pointer;
563 }
564 g_assert_not_reached();
565 }
566
567 static void init_ffi_layouts(void)
568 {
569 /* g_direct_hash/equal for direct comparisons on uint32_t. */
570 GHashTable *ffi_table = g_hash_table_new(NULL, NULL);
571
572 for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
573 TCGHelperInfo *info = &all_helpers[i];
574 unsigned typemask = info->typemask;
575 gpointer hash = (gpointer)(uintptr_t)typemask;
576 struct {
577 ffi_cif cif;
578 ffi_type *args[];
579 } *ca;
580 ffi_status status;
581 int nargs;
582 ffi_cif *cif;
583
584 cif = g_hash_table_lookup(ffi_table, hash);
585 if (cif) {
586 info->cif = cif;
587 continue;
588 }
589
590 /* Ignoring the return type, find the last non-zero field. */
591 nargs = 32 - clz32(typemask >> 3);
592 nargs = DIV_ROUND_UP(nargs, 3);
593
594 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
595 ca->cif.rtype = typecode_to_ffi(typemask & 7);
596 ca->cif.nargs = nargs;
597
598 if (nargs != 0) {
599 ca->cif.arg_types = ca->args;
600 for (int j = 0; j < nargs; ++j) {
601 int typecode = extract32(typemask, (j + 1) * 3, 3);
602 ca->args[j] = typecode_to_ffi(typecode);
603 }
604 }
605
606 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
607 ca->cif.rtype, ca->cif.arg_types);
608 assert(status == FFI_OK);
609
610 cif = &ca->cif;
611 info->cif = cif;
612 g_hash_table_insert(ffi_table, hash, (gpointer)cif);
613 }
614
615 g_hash_table_destroy(ffi_table);
616 }
617 #endif /* CONFIG_TCG_INTERPRETER */
618
619 typedef struct TCGCumulativeArgs {
620 int arg_idx; /* tcg_gen_callN args[] */
621 int info_in_idx; /* TCGHelperInfo in[] */
622 int arg_slot; /* regs+stack slot */
623 int ref_slot; /* stack slots for references */
624 } TCGCumulativeArgs;
625
626 static void layout_arg_even(TCGCumulativeArgs *cum)
627 {
628 cum->arg_slot += cum->arg_slot & 1;
629 }
630
631 static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
632 TCGCallArgumentKind kind)
633 {
634 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
635
636 *loc = (TCGCallArgumentLoc){
637 .kind = kind,
638 .arg_idx = cum->arg_idx,
639 .arg_slot = cum->arg_slot,
640 };
641 cum->info_in_idx++;
642 cum->arg_slot++;
643 }
644
645 static void layout_arg_normal_n(TCGCumulativeArgs *cum,
646 TCGHelperInfo *info, int n)
647 {
648 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
649
650 for (int i = 0; i < n; ++i) {
651 /* Layout all using the same arg_idx, adjusting the subindex. */
652 loc[i] = (TCGCallArgumentLoc){
653 .kind = TCG_CALL_ARG_NORMAL,
654 .arg_idx = cum->arg_idx,
655 .tmp_subindex = i,
656 .arg_slot = cum->arg_slot + i,
657 };
658 }
659 cum->info_in_idx += n;
660 cum->arg_slot += n;
661 }
662
663 static void init_call_layout(TCGHelperInfo *info)
664 {
665 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
666 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
667 unsigned typemask = info->typemask;
668 unsigned typecode;
669 TCGCumulativeArgs cum = { };
670
671 /*
672 * Parse and place any function return value.
673 */
674 typecode = typemask & 7;
675 switch (typecode) {
676 case dh_typecode_void:
677 info->nr_out = 0;
678 break;
679 case dh_typecode_i32:
680 case dh_typecode_s32:
681 case dh_typecode_ptr:
682 info->nr_out = 1;
683 info->out_kind = TCG_CALL_RET_NORMAL;
684 break;
685 case dh_typecode_i64:
686 case dh_typecode_s64:
687 info->nr_out = 64 / TCG_TARGET_REG_BITS;
688 info->out_kind = TCG_CALL_RET_NORMAL;
689 break;
690 default:
691 g_assert_not_reached();
692 }
693 assert(info->nr_out <= ARRAY_SIZE(tcg_target_call_oarg_regs));
694
695 /*
696 * Parse and place function arguments.
697 */
698 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
699 TCGCallArgumentKind kind;
700 TCGType type;
701
702 typecode = typemask & 7;
703 switch (typecode) {
704 case dh_typecode_i32:
705 case dh_typecode_s32:
706 type = TCG_TYPE_I32;
707 break;
708 case dh_typecode_i64:
709 case dh_typecode_s64:
710 type = TCG_TYPE_I64;
711 break;
712 case dh_typecode_ptr:
713 type = TCG_TYPE_PTR;
714 break;
715 default:
716 g_assert_not_reached();
717 }
718
719 switch (type) {
720 case TCG_TYPE_I32:
721 switch (TCG_TARGET_CALL_ARG_I32) {
722 case TCG_CALL_ARG_EVEN:
723 layout_arg_even(&cum);
724 /* fall through */
725 case TCG_CALL_ARG_NORMAL:
726 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
727 break;
728 case TCG_CALL_ARG_EXTEND:
729 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
730 layout_arg_1(&cum, info, kind);
731 break;
732 default:
733 qemu_build_not_reached();
734 }
735 break;
736
737 case TCG_TYPE_I64:
738 switch (TCG_TARGET_CALL_ARG_I64) {
739 case TCG_CALL_ARG_EVEN:
740 layout_arg_even(&cum);
741 /* fall through */
742 case TCG_CALL_ARG_NORMAL:
743 if (TCG_TARGET_REG_BITS == 32) {
744 layout_arg_normal_n(&cum, info, 2);
745 } else {
746 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
747 }
748 break;
749 default:
750 qemu_build_not_reached();
751 }
752 break;
753
754 default:
755 g_assert_not_reached();
756 }
757 }
758 info->nr_in = cum.info_in_idx;
759
760 /* Validate that we didn't overrun the input array. */
761 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
762 /* Validate the backend has enough argument space. */
763 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
764 assert(cum.ref_slot <= max_stk_slots);
765 }
766
767 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
768 static void process_op_defs(TCGContext *s);
769 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
770 TCGReg reg, const char *name);
771
772 static void tcg_context_init(unsigned max_cpus)
773 {
774 TCGContext *s = &tcg_init_ctx;
775 int op, total_args, n, i;
776 TCGOpDef *def;
777 TCGArgConstraint *args_ct;
778 TCGTemp *ts;
779
780 memset(s, 0, sizeof(*s));
781 s->nb_globals = 0;
782
783 /* Count total number of arguments and allocate the corresponding
784 space */
785 total_args = 0;
786 for(op = 0; op < NB_OPS; op++) {
787 def = &tcg_op_defs[op];
788 n = def->nb_iargs + def->nb_oargs;
789 total_args += n;
790 }
791
792 args_ct = g_new0(TCGArgConstraint, total_args);
793
794 for(op = 0; op < NB_OPS; op++) {
795 def = &tcg_op_defs[op];
796 def->args_ct = args_ct;
797 n = def->nb_iargs + def->nb_oargs;
798 args_ct += n;
799 }
800
801 /* Register helpers. */
802 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
803 helper_table = g_hash_table_new(NULL, NULL);
804
805 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
806 init_call_layout(&all_helpers[i]);
807 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
808 (gpointer)&all_helpers[i]);
809 }
810
811 #ifdef CONFIG_TCG_INTERPRETER
812 init_ffi_layouts();
813 #endif
814
815 tcg_target_init(s);
816 process_op_defs(s);
817
818 /* Reverse the order of the saved registers, assuming they're all at
819 the start of tcg_target_reg_alloc_order. */
820 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
821 int r = tcg_target_reg_alloc_order[n];
822 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
823 break;
824 }
825 }
826 for (i = 0; i < n; ++i) {
827 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
828 }
829 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
830 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
831 }
832
833 alloc_tcg_plugin_context(s);
834
835 tcg_ctx = s;
836 /*
837 * In user-mode we simply share the init context among threads, since we
838 * use a single region. See the documentation tcg_region_init() for the
839 * reasoning behind this.
840 * In softmmu we will have at most max_cpus TCG threads.
841 */
842 #ifdef CONFIG_USER_ONLY
843 tcg_ctxs = &tcg_ctx;
844 tcg_cur_ctxs = 1;
845 tcg_max_ctxs = 1;
846 #else
847 tcg_max_ctxs = max_cpus;
848 tcg_ctxs = g_new0(TCGContext *, max_cpus);
849 #endif
850
851 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
852 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
853 cpu_env = temp_tcgv_ptr(ts);
854 }
855
856 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
857 {
858 tcg_context_init(max_cpus);
859 tcg_region_init(tb_size, splitwx, max_cpus);
860 }
861
862 /*
863 * Allocate TBs right before their corresponding translated code, making
864 * sure that TBs and code are on different cache lines.
865 */
866 TranslationBlock *tcg_tb_alloc(TCGContext *s)
867 {
868 uintptr_t align = qemu_icache_linesize;
869 TranslationBlock *tb;
870 void *next;
871
872 retry:
873 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
874 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
875
876 if (unlikely(next > s->code_gen_highwater)) {
877 if (tcg_region_alloc(s)) {
878 return NULL;
879 }
880 goto retry;
881 }
882 qatomic_set(&s->code_gen_ptr, next);
883 s->data_gen_ptr = NULL;
884 return tb;
885 }
886
887 void tcg_prologue_init(TCGContext *s)
888 {
889 size_t prologue_size;
890
891 s->code_ptr = s->code_gen_ptr;
892 s->code_buf = s->code_gen_ptr;
893 s->data_gen_ptr = NULL;
894
895 #ifndef CONFIG_TCG_INTERPRETER
896 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
897 #endif
898
899 #ifdef TCG_TARGET_NEED_POOL_LABELS
900 s->pool_labels = NULL;
901 #endif
902
903 qemu_thread_jit_write();
904 /* Generate the prologue. */
905 tcg_target_qemu_prologue(s);
906
907 #ifdef TCG_TARGET_NEED_POOL_LABELS
908 /* Allow the prologue to put e.g. guest_base into a pool entry. */
909 {
910 int result = tcg_out_pool_finalize(s);
911 tcg_debug_assert(result == 0);
912 }
913 #endif
914
915 prologue_size = tcg_current_code_size(s);
916
917 #ifndef CONFIG_TCG_INTERPRETER
918 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
919 (uintptr_t)s->code_buf, prologue_size);
920 #endif
921
922 #ifdef DEBUG_DISAS
923 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
924 FILE *logfile = qemu_log_trylock();
925 if (logfile) {
926 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
927 if (s->data_gen_ptr) {
928 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
929 size_t data_size = prologue_size - code_size;
930 size_t i;
931
932 disas(logfile, s->code_gen_ptr, code_size);
933
934 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
935 if (sizeof(tcg_target_ulong) == 8) {
936 fprintf(logfile,
937 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
938 (uintptr_t)s->data_gen_ptr + i,
939 *(uint64_t *)(s->data_gen_ptr + i));
940 } else {
941 fprintf(logfile,
942 "0x%08" PRIxPTR ": .long 0x%08x\n",
943 (uintptr_t)s->data_gen_ptr + i,
944 *(uint32_t *)(s->data_gen_ptr + i));
945 }
946 }
947 } else {
948 disas(logfile, s->code_gen_ptr, prologue_size);
949 }
950 fprintf(logfile, "\n");
951 qemu_log_unlock(logfile);
952 }
953 }
954 #endif
955
956 #ifndef CONFIG_TCG_INTERPRETER
957 /*
958 * Assert that goto_ptr is implemented completely, setting an epilogue.
959 * For tci, we use NULL as the signal to return from the interpreter,
960 * so skip this check.
961 */
962 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
963 #endif
964
965 tcg_region_prologue_set(s);
966 }
967
968 void tcg_func_start(TCGContext *s)
969 {
970 tcg_pool_reset(s);
971 s->nb_temps = s->nb_globals;
972
973 /* No temps have been previously allocated for size or locality. */
974 memset(s->free_temps, 0, sizeof(s->free_temps));
975
976 /* No constant temps have been previously allocated. */
977 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
978 if (s->const_table[i]) {
979 g_hash_table_remove_all(s->const_table[i]);
980 }
981 }
982
983 s->nb_ops = 0;
984 s->nb_labels = 0;
985 s->current_frame_offset = s->frame_start;
986
987 #ifdef CONFIG_DEBUG_TCG
988 s->goto_tb_issue_mask = 0;
989 #endif
990
991 QTAILQ_INIT(&s->ops);
992 QTAILQ_INIT(&s->free_ops);
993 QSIMPLEQ_INIT(&s->labels);
994 }
995
996 static TCGTemp *tcg_temp_alloc(TCGContext *s)
997 {
998 int n = s->nb_temps++;
999
1000 if (n >= TCG_MAX_TEMPS) {
1001 tcg_raise_tb_overflow(s);
1002 }
1003 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1004 }
1005
1006 static TCGTemp *tcg_global_alloc(TCGContext *s)
1007 {
1008 TCGTemp *ts;
1009
1010 tcg_debug_assert(s->nb_globals == s->nb_temps);
1011 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
1012 s->nb_globals++;
1013 ts = tcg_temp_alloc(s);
1014 ts->kind = TEMP_GLOBAL;
1015
1016 return ts;
1017 }
1018
1019 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1020 TCGReg reg, const char *name)
1021 {
1022 TCGTemp *ts;
1023
1024 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
1025 tcg_abort();
1026 }
1027
1028 ts = tcg_global_alloc(s);
1029 ts->base_type = type;
1030 ts->type = type;
1031 ts->kind = TEMP_FIXED;
1032 ts->reg = reg;
1033 ts->name = name;
1034 tcg_regset_set_reg(s->reserved_regs, reg);
1035
1036 return ts;
1037 }
1038
1039 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
1040 {
1041 s->frame_start = start;
1042 s->frame_end = start + size;
1043 s->frame_temp
1044 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
1045 }
1046
1047 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1048 intptr_t offset, const char *name)
1049 {
1050 TCGContext *s = tcg_ctx;
1051 TCGTemp *base_ts = tcgv_ptr_temp(base);
1052 TCGTemp *ts = tcg_global_alloc(s);
1053 int indirect_reg = 0;
1054
1055 switch (base_ts->kind) {
1056 case TEMP_FIXED:
1057 break;
1058 case TEMP_GLOBAL:
1059 /* We do not support double-indirect registers. */
1060 tcg_debug_assert(!base_ts->indirect_reg);
1061 base_ts->indirect_base = 1;
1062 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1063 ? 2 : 1);
1064 indirect_reg = 1;
1065 break;
1066 default:
1067 g_assert_not_reached();
1068 }
1069
1070 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1071 TCGTemp *ts2 = tcg_global_alloc(s);
1072 char buf[64];
1073
1074 ts->base_type = TCG_TYPE_I64;
1075 ts->type = TCG_TYPE_I32;
1076 ts->indirect_reg = indirect_reg;
1077 ts->mem_allocated = 1;
1078 ts->mem_base = base_ts;
1079 ts->mem_offset = offset;
1080 pstrcpy(buf, sizeof(buf), name);
1081 pstrcat(buf, sizeof(buf), "_0");
1082 ts->name = strdup(buf);
1083
1084 tcg_debug_assert(ts2 == ts + 1);
1085 ts2->base_type = TCG_TYPE_I64;
1086 ts2->type = TCG_TYPE_I32;
1087 ts2->indirect_reg = indirect_reg;
1088 ts2->mem_allocated = 1;
1089 ts2->mem_base = base_ts;
1090 ts2->mem_offset = offset + 4;
1091 ts2->temp_subindex = 1;
1092 pstrcpy(buf, sizeof(buf), name);
1093 pstrcat(buf, sizeof(buf), "_1");
1094 ts2->name = strdup(buf);
1095 } else {
1096 ts->base_type = type;
1097 ts->type = type;
1098 ts->indirect_reg = indirect_reg;
1099 ts->mem_allocated = 1;
1100 ts->mem_base = base_ts;
1101 ts->mem_offset = offset;
1102 ts->name = name;
1103 }
1104 return ts;
1105 }
1106
1107 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
1108 {
1109 TCGContext *s = tcg_ctx;
1110 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
1111 TCGTemp *ts;
1112 int idx, k;
1113
1114 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
1115 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
1116 if (idx < TCG_MAX_TEMPS) {
1117 /* There is already an available temp with the right type. */
1118 clear_bit(idx, s->free_temps[k].l);
1119
1120 ts = &s->temps[idx];
1121 ts->temp_allocated = 1;
1122 tcg_debug_assert(ts->base_type == type);
1123 tcg_debug_assert(ts->kind == kind);
1124 } else {
1125 ts = tcg_temp_alloc(s);
1126 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1127 TCGTemp *ts2 = tcg_temp_alloc(s);
1128
1129 ts->base_type = type;
1130 ts->type = TCG_TYPE_I32;
1131 ts->temp_allocated = 1;
1132 ts->kind = kind;
1133
1134 tcg_debug_assert(ts2 == ts + 1);
1135 ts2->base_type = TCG_TYPE_I64;
1136 ts2->type = TCG_TYPE_I32;
1137 ts2->temp_allocated = 1;
1138 ts2->temp_subindex = 1;
1139 ts2->kind = kind;
1140 } else {
1141 ts->base_type = type;
1142 ts->type = type;
1143 ts->temp_allocated = 1;
1144 ts->kind = kind;
1145 }
1146 }
1147
1148 #if defined(CONFIG_DEBUG_TCG)
1149 s->temps_in_use++;
1150 #endif
1151 return ts;
1152 }
1153
1154 TCGv_vec tcg_temp_new_vec(TCGType type)
1155 {
1156 TCGTemp *t;
1157
1158 #ifdef CONFIG_DEBUG_TCG
1159 switch (type) {
1160 case TCG_TYPE_V64:
1161 assert(TCG_TARGET_HAS_v64);
1162 break;
1163 case TCG_TYPE_V128:
1164 assert(TCG_TARGET_HAS_v128);
1165 break;
1166 case TCG_TYPE_V256:
1167 assert(TCG_TARGET_HAS_v256);
1168 break;
1169 default:
1170 g_assert_not_reached();
1171 }
1172 #endif
1173
1174 t = tcg_temp_new_internal(type, 0);
1175 return temp_tcgv_vec(t);
1176 }
1177
1178 /* Create a new temp of the same type as an existing temp. */
1179 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1180 {
1181 TCGTemp *t = tcgv_vec_temp(match);
1182
1183 tcg_debug_assert(t->temp_allocated != 0);
1184
1185 t = tcg_temp_new_internal(t->base_type, 0);
1186 return temp_tcgv_vec(t);
1187 }
1188
1189 void tcg_temp_free_internal(TCGTemp *ts)
1190 {
1191 TCGContext *s = tcg_ctx;
1192 int k, idx;
1193
1194 switch (ts->kind) {
1195 case TEMP_CONST:
1196 /*
1197 * In order to simplify users of tcg_constant_*,
1198 * silently ignore free.
1199 */
1200 return;
1201 case TEMP_NORMAL:
1202 case TEMP_LOCAL:
1203 break;
1204 default:
1205 g_assert_not_reached();
1206 }
1207
1208 #if defined(CONFIG_DEBUG_TCG)
1209 s->temps_in_use--;
1210 if (s->temps_in_use < 0) {
1211 fprintf(stderr, "More temporaries freed than allocated!\n");
1212 }
1213 #endif
1214
1215 tcg_debug_assert(ts->temp_allocated != 0);
1216 ts->temp_allocated = 0;
1217
1218 idx = temp_idx(ts);
1219 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
1220 set_bit(idx, s->free_temps[k].l);
1221 }
1222
1223 TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1224 {
1225 TCGContext *s = tcg_ctx;
1226 GHashTable *h = s->const_table[type];
1227 TCGTemp *ts;
1228
1229 if (h == NULL) {
1230 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1231 s->const_table[type] = h;
1232 }
1233
1234 ts = g_hash_table_lookup(h, &val);
1235 if (ts == NULL) {
1236 int64_t *val_ptr;
1237
1238 ts = tcg_temp_alloc(s);
1239
1240 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1241 TCGTemp *ts2 = tcg_temp_alloc(s);
1242
1243 tcg_debug_assert(ts2 == ts + 1);
1244
1245 ts->base_type = TCG_TYPE_I64;
1246 ts->type = TCG_TYPE_I32;
1247 ts->kind = TEMP_CONST;
1248 ts->temp_allocated = 1;
1249
1250 ts2->base_type = TCG_TYPE_I64;
1251 ts2->type = TCG_TYPE_I32;
1252 ts2->kind = TEMP_CONST;
1253 ts2->temp_allocated = 1;
1254 ts2->temp_subindex = 1;
1255
1256 /*
1257 * Retain the full value of the 64-bit constant in the low
1258 * part, so that the hash table works. Actual uses will
1259 * truncate the value to the low part.
1260 */
1261 ts[HOST_BIG_ENDIAN].val = val;
1262 ts[!HOST_BIG_ENDIAN].val = val >> 32;
1263 val_ptr = &ts[HOST_BIG_ENDIAN].val;
1264 } else {
1265 ts->base_type = type;
1266 ts->type = type;
1267 ts->kind = TEMP_CONST;
1268 ts->temp_allocated = 1;
1269 ts->val = val;
1270 val_ptr = &ts->val;
1271 }
1272 g_hash_table_insert(h, val_ptr, ts);
1273 }
1274
1275 return ts;
1276 }
1277
1278 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1279 {
1280 val = dup_const(vece, val);
1281 return temp_tcgv_vec(tcg_constant_internal(type, val));
1282 }
1283
1284 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1285 {
1286 TCGTemp *t = tcgv_vec_temp(match);
1287
1288 tcg_debug_assert(t->temp_allocated != 0);
1289 return tcg_constant_vec(t->base_type, vece, val);
1290 }
1291
1292 TCGv_i32 tcg_const_i32(int32_t val)
1293 {
1294 TCGv_i32 t0;
1295 t0 = tcg_temp_new_i32();
1296 tcg_gen_movi_i32(t0, val);
1297 return t0;
1298 }
1299
1300 TCGv_i64 tcg_const_i64(int64_t val)
1301 {
1302 TCGv_i64 t0;
1303 t0 = tcg_temp_new_i64();
1304 tcg_gen_movi_i64(t0, val);
1305 return t0;
1306 }
1307
1308 TCGv_i32 tcg_const_local_i32(int32_t val)
1309 {
1310 TCGv_i32 t0;
1311 t0 = tcg_temp_local_new_i32();
1312 tcg_gen_movi_i32(t0, val);
1313 return t0;
1314 }
1315
1316 TCGv_i64 tcg_const_local_i64(int64_t val)
1317 {
1318 TCGv_i64 t0;
1319 t0 = tcg_temp_local_new_i64();
1320 tcg_gen_movi_i64(t0, val);
1321 return t0;
1322 }
1323
1324 #if defined(CONFIG_DEBUG_TCG)
1325 void tcg_clear_temp_count(void)
1326 {
1327 TCGContext *s = tcg_ctx;
1328 s->temps_in_use = 0;
1329 }
1330
1331 int tcg_check_temp_count(void)
1332 {
1333 TCGContext *s = tcg_ctx;
1334 if (s->temps_in_use) {
1335 /* Clear the count so that we don't give another
1336 * warning immediately next time around.
1337 */
1338 s->temps_in_use = 0;
1339 return 1;
1340 }
1341 return 0;
1342 }
1343 #endif
1344
1345 /* Return true if OP may appear in the opcode stream.
1346 Test the runtime variable that controls each opcode. */
1347 bool tcg_op_supported(TCGOpcode op)
1348 {
1349 const bool have_vec
1350 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1351
1352 switch (op) {
1353 case INDEX_op_discard:
1354 case INDEX_op_set_label:
1355 case INDEX_op_call:
1356 case INDEX_op_br:
1357 case INDEX_op_mb:
1358 case INDEX_op_insn_start:
1359 case INDEX_op_exit_tb:
1360 case INDEX_op_goto_tb:
1361 case INDEX_op_goto_ptr:
1362 case INDEX_op_qemu_ld_i32:
1363 case INDEX_op_qemu_st_i32:
1364 case INDEX_op_qemu_ld_i64:
1365 case INDEX_op_qemu_st_i64:
1366 return true;
1367
1368 case INDEX_op_qemu_st8_i32:
1369 return TCG_TARGET_HAS_qemu_st8_i32;
1370
1371 case INDEX_op_mov_i32:
1372 case INDEX_op_setcond_i32:
1373 case INDEX_op_brcond_i32:
1374 case INDEX_op_ld8u_i32:
1375 case INDEX_op_ld8s_i32:
1376 case INDEX_op_ld16u_i32:
1377 case INDEX_op_ld16s_i32:
1378 case INDEX_op_ld_i32:
1379 case INDEX_op_st8_i32:
1380 case INDEX_op_st16_i32:
1381 case INDEX_op_st_i32:
1382 case INDEX_op_add_i32:
1383 case INDEX_op_sub_i32:
1384 case INDEX_op_mul_i32:
1385 case INDEX_op_and_i32:
1386 case INDEX_op_or_i32:
1387 case INDEX_op_xor_i32:
1388 case INDEX_op_shl_i32:
1389 case INDEX_op_shr_i32:
1390 case INDEX_op_sar_i32:
1391 return true;
1392
1393 case INDEX_op_movcond_i32:
1394 return TCG_TARGET_HAS_movcond_i32;
1395 case INDEX_op_div_i32:
1396 case INDEX_op_divu_i32:
1397 return TCG_TARGET_HAS_div_i32;
1398 case INDEX_op_rem_i32:
1399 case INDEX_op_remu_i32:
1400 return TCG_TARGET_HAS_rem_i32;
1401 case INDEX_op_div2_i32:
1402 case INDEX_op_divu2_i32:
1403 return TCG_TARGET_HAS_div2_i32;
1404 case INDEX_op_rotl_i32:
1405 case INDEX_op_rotr_i32:
1406 return TCG_TARGET_HAS_rot_i32;
1407 case INDEX_op_deposit_i32:
1408 return TCG_TARGET_HAS_deposit_i32;
1409 case INDEX_op_extract_i32:
1410 return TCG_TARGET_HAS_extract_i32;
1411 case INDEX_op_sextract_i32:
1412 return TCG_TARGET_HAS_sextract_i32;
1413 case INDEX_op_extract2_i32:
1414 return TCG_TARGET_HAS_extract2_i32;
1415 case INDEX_op_add2_i32:
1416 return TCG_TARGET_HAS_add2_i32;
1417 case INDEX_op_sub2_i32:
1418 return TCG_TARGET_HAS_sub2_i32;
1419 case INDEX_op_mulu2_i32:
1420 return TCG_TARGET_HAS_mulu2_i32;
1421 case INDEX_op_muls2_i32:
1422 return TCG_TARGET_HAS_muls2_i32;
1423 case INDEX_op_muluh_i32:
1424 return TCG_TARGET_HAS_muluh_i32;
1425 case INDEX_op_mulsh_i32:
1426 return TCG_TARGET_HAS_mulsh_i32;
1427 case INDEX_op_ext8s_i32:
1428 return TCG_TARGET_HAS_ext8s_i32;
1429 case INDEX_op_ext16s_i32:
1430 return TCG_TARGET_HAS_ext16s_i32;
1431 case INDEX_op_ext8u_i32:
1432 return TCG_TARGET_HAS_ext8u_i32;
1433 case INDEX_op_ext16u_i32:
1434 return TCG_TARGET_HAS_ext16u_i32;
1435 case INDEX_op_bswap16_i32:
1436 return TCG_TARGET_HAS_bswap16_i32;
1437 case INDEX_op_bswap32_i32:
1438 return TCG_TARGET_HAS_bswap32_i32;
1439 case INDEX_op_not_i32:
1440 return TCG_TARGET_HAS_not_i32;
1441 case INDEX_op_neg_i32:
1442 return TCG_TARGET_HAS_neg_i32;
1443 case INDEX_op_andc_i32:
1444 return TCG_TARGET_HAS_andc_i32;
1445 case INDEX_op_orc_i32:
1446 return TCG_TARGET_HAS_orc_i32;
1447 case INDEX_op_eqv_i32:
1448 return TCG_TARGET_HAS_eqv_i32;
1449 case INDEX_op_nand_i32:
1450 return TCG_TARGET_HAS_nand_i32;
1451 case INDEX_op_nor_i32:
1452 return TCG_TARGET_HAS_nor_i32;
1453 case INDEX_op_clz_i32:
1454 return TCG_TARGET_HAS_clz_i32;
1455 case INDEX_op_ctz_i32:
1456 return TCG_TARGET_HAS_ctz_i32;
1457 case INDEX_op_ctpop_i32:
1458 return TCG_TARGET_HAS_ctpop_i32;
1459
1460 case INDEX_op_brcond2_i32:
1461 case INDEX_op_setcond2_i32:
1462 return TCG_TARGET_REG_BITS == 32;
1463
1464 case INDEX_op_mov_i64:
1465 case INDEX_op_setcond_i64:
1466 case INDEX_op_brcond_i64:
1467 case INDEX_op_ld8u_i64:
1468 case INDEX_op_ld8s_i64:
1469 case INDEX_op_ld16u_i64:
1470 case INDEX_op_ld16s_i64:
1471 case INDEX_op_ld32u_i64:
1472 case INDEX_op_ld32s_i64:
1473 case INDEX_op_ld_i64:
1474 case INDEX_op_st8_i64:
1475 case INDEX_op_st16_i64:
1476 case INDEX_op_st32_i64:
1477 case INDEX_op_st_i64:
1478 case INDEX_op_add_i64:
1479 case INDEX_op_sub_i64:
1480 case INDEX_op_mul_i64:
1481 case INDEX_op_and_i64:
1482 case INDEX_op_or_i64:
1483 case INDEX_op_xor_i64:
1484 case INDEX_op_shl_i64:
1485 case INDEX_op_shr_i64:
1486 case INDEX_op_sar_i64:
1487 case INDEX_op_ext_i32_i64:
1488 case INDEX_op_extu_i32_i64:
1489 return TCG_TARGET_REG_BITS == 64;
1490
1491 case INDEX_op_movcond_i64:
1492 return TCG_TARGET_HAS_movcond_i64;
1493 case INDEX_op_div_i64:
1494 case INDEX_op_divu_i64:
1495 return TCG_TARGET_HAS_div_i64;
1496 case INDEX_op_rem_i64:
1497 case INDEX_op_remu_i64:
1498 return TCG_TARGET_HAS_rem_i64;
1499 case INDEX_op_div2_i64:
1500 case INDEX_op_divu2_i64:
1501 return TCG_TARGET_HAS_div2_i64;
1502 case INDEX_op_rotl_i64:
1503 case INDEX_op_rotr_i64:
1504 return TCG_TARGET_HAS_rot_i64;
1505 case INDEX_op_deposit_i64:
1506 return TCG_TARGET_HAS_deposit_i64;
1507 case INDEX_op_extract_i64:
1508 return TCG_TARGET_HAS_extract_i64;
1509 case INDEX_op_sextract_i64:
1510 return TCG_TARGET_HAS_sextract_i64;
1511 case INDEX_op_extract2_i64:
1512 return TCG_TARGET_HAS_extract2_i64;
1513 case INDEX_op_extrl_i64_i32:
1514 return TCG_TARGET_HAS_extrl_i64_i32;
1515 case INDEX_op_extrh_i64_i32:
1516 return TCG_TARGET_HAS_extrh_i64_i32;
1517 case INDEX_op_ext8s_i64:
1518 return TCG_TARGET_HAS_ext8s_i64;
1519 case INDEX_op_ext16s_i64:
1520 return TCG_TARGET_HAS_ext16s_i64;
1521 case INDEX_op_ext32s_i64:
1522 return TCG_TARGET_HAS_ext32s_i64;
1523 case INDEX_op_ext8u_i64:
1524 return TCG_TARGET_HAS_ext8u_i64;
1525 case INDEX_op_ext16u_i64:
1526 return TCG_TARGET_HAS_ext16u_i64;
1527 case INDEX_op_ext32u_i64:
1528 return TCG_TARGET_HAS_ext32u_i64;
1529 case INDEX_op_bswap16_i64:
1530 return TCG_TARGET_HAS_bswap16_i64;
1531 case INDEX_op_bswap32_i64:
1532 return TCG_TARGET_HAS_bswap32_i64;
1533 case INDEX_op_bswap64_i64:
1534 return TCG_TARGET_HAS_bswap64_i64;
1535 case INDEX_op_not_i64:
1536 return TCG_TARGET_HAS_not_i64;
1537 case INDEX_op_neg_i64:
1538 return TCG_TARGET_HAS_neg_i64;
1539 case INDEX_op_andc_i64:
1540 return TCG_TARGET_HAS_andc_i64;
1541 case INDEX_op_orc_i64:
1542 return TCG_TARGET_HAS_orc_i64;
1543 case INDEX_op_eqv_i64:
1544 return TCG_TARGET_HAS_eqv_i64;
1545 case INDEX_op_nand_i64:
1546 return TCG_TARGET_HAS_nand_i64;
1547 case INDEX_op_nor_i64:
1548 return TCG_TARGET_HAS_nor_i64;
1549 case INDEX_op_clz_i64:
1550 return TCG_TARGET_HAS_clz_i64;
1551 case INDEX_op_ctz_i64:
1552 return TCG_TARGET_HAS_ctz_i64;
1553 case INDEX_op_ctpop_i64:
1554 return TCG_TARGET_HAS_ctpop_i64;
1555 case INDEX_op_add2_i64:
1556 return TCG_TARGET_HAS_add2_i64;
1557 case INDEX_op_sub2_i64:
1558 return TCG_TARGET_HAS_sub2_i64;
1559 case INDEX_op_mulu2_i64:
1560 return TCG_TARGET_HAS_mulu2_i64;
1561 case INDEX_op_muls2_i64:
1562 return TCG_TARGET_HAS_muls2_i64;
1563 case INDEX_op_muluh_i64:
1564 return TCG_TARGET_HAS_muluh_i64;
1565 case INDEX_op_mulsh_i64:
1566 return TCG_TARGET_HAS_mulsh_i64;
1567
1568 case INDEX_op_mov_vec:
1569 case INDEX_op_dup_vec:
1570 case INDEX_op_dupm_vec:
1571 case INDEX_op_ld_vec:
1572 case INDEX_op_st_vec:
1573 case INDEX_op_add_vec:
1574 case INDEX_op_sub_vec:
1575 case INDEX_op_and_vec:
1576 case INDEX_op_or_vec:
1577 case INDEX_op_xor_vec:
1578 case INDEX_op_cmp_vec:
1579 return have_vec;
1580 case INDEX_op_dup2_vec:
1581 return have_vec && TCG_TARGET_REG_BITS == 32;
1582 case INDEX_op_not_vec:
1583 return have_vec && TCG_TARGET_HAS_not_vec;
1584 case INDEX_op_neg_vec:
1585 return have_vec && TCG_TARGET_HAS_neg_vec;
1586 case INDEX_op_abs_vec:
1587 return have_vec && TCG_TARGET_HAS_abs_vec;
1588 case INDEX_op_andc_vec:
1589 return have_vec && TCG_TARGET_HAS_andc_vec;
1590 case INDEX_op_orc_vec:
1591 return have_vec && TCG_TARGET_HAS_orc_vec;
1592 case INDEX_op_nand_vec:
1593 return have_vec && TCG_TARGET_HAS_nand_vec;
1594 case INDEX_op_nor_vec:
1595 return have_vec && TCG_TARGET_HAS_nor_vec;
1596 case INDEX_op_eqv_vec:
1597 return have_vec && TCG_TARGET_HAS_eqv_vec;
1598 case INDEX_op_mul_vec:
1599 return have_vec && TCG_TARGET_HAS_mul_vec;
1600 case INDEX_op_shli_vec:
1601 case INDEX_op_shri_vec:
1602 case INDEX_op_sari_vec:
1603 return have_vec && TCG_TARGET_HAS_shi_vec;
1604 case INDEX_op_shls_vec:
1605 case INDEX_op_shrs_vec:
1606 case INDEX_op_sars_vec:
1607 return have_vec && TCG_TARGET_HAS_shs_vec;
1608 case INDEX_op_shlv_vec:
1609 case INDEX_op_shrv_vec:
1610 case INDEX_op_sarv_vec:
1611 return have_vec && TCG_TARGET_HAS_shv_vec;
1612 case INDEX_op_rotli_vec:
1613 return have_vec && TCG_TARGET_HAS_roti_vec;
1614 case INDEX_op_rotls_vec:
1615 return have_vec && TCG_TARGET_HAS_rots_vec;
1616 case INDEX_op_rotlv_vec:
1617 case INDEX_op_rotrv_vec:
1618 return have_vec && TCG_TARGET_HAS_rotv_vec;
1619 case INDEX_op_ssadd_vec:
1620 case INDEX_op_usadd_vec:
1621 case INDEX_op_sssub_vec:
1622 case INDEX_op_ussub_vec:
1623 return have_vec && TCG_TARGET_HAS_sat_vec;
1624 case INDEX_op_smin_vec:
1625 case INDEX_op_umin_vec:
1626 case INDEX_op_smax_vec:
1627 case INDEX_op_umax_vec:
1628 return have_vec && TCG_TARGET_HAS_minmax_vec;
1629 case INDEX_op_bitsel_vec:
1630 return have_vec && TCG_TARGET_HAS_bitsel_vec;
1631 case INDEX_op_cmpsel_vec:
1632 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
1633
1634 default:
1635 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1636 return true;
1637 }
1638 }
1639
1640 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
1641
1642 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
1643 {
1644 const TCGHelperInfo *info;
1645 TCGv_i64 extend_free[MAX_CALL_IARGS];
1646 int n_extend = 0;
1647 TCGOp *op;
1648 int i, n, pi = 0, total_args;
1649
1650 info = g_hash_table_lookup(helper_table, (gpointer)func);
1651 total_args = info->nr_out + info->nr_in + 2;
1652 op = tcg_op_alloc(INDEX_op_call, total_args);
1653
1654 #ifdef CONFIG_PLUGIN
1655 /* detect non-plugin helpers */
1656 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1657 tcg_ctx->plugin_insn->calls_helpers = true;
1658 }
1659 #endif
1660
1661 TCGOP_CALLO(op) = n = info->nr_out;
1662 switch (n) {
1663 case 0:
1664 tcg_debug_assert(ret == NULL);
1665 break;
1666 case 1:
1667 tcg_debug_assert(ret != NULL);
1668 op->args[pi++] = temp_arg(ret);
1669 break;
1670 case 2:
1671 tcg_debug_assert(ret != NULL);
1672 tcg_debug_assert(ret->base_type == ret->type + 1);
1673 tcg_debug_assert(ret->temp_subindex == 0);
1674 op->args[pi++] = temp_arg(ret);
1675 op->args[pi++] = temp_arg(ret + 1);
1676 break;
1677 default:
1678 g_assert_not_reached();
1679 }
1680
1681 TCGOP_CALLI(op) = n = info->nr_in;
1682 for (i = 0; i < n; i++) {
1683 const TCGCallArgumentLoc *loc = &info->in[i];
1684 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
1685
1686 switch (loc->kind) {
1687 case TCG_CALL_ARG_NORMAL:
1688 op->args[pi++] = temp_arg(ts);
1689 break;
1690
1691 case TCG_CALL_ARG_EXTEND_U:
1692 case TCG_CALL_ARG_EXTEND_S:
1693 {
1694 TCGv_i64 temp = tcg_temp_new_i64();
1695 TCGv_i32 orig = temp_tcgv_i32(ts);
1696
1697 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
1698 tcg_gen_ext_i32_i64(temp, orig);
1699 } else {
1700 tcg_gen_extu_i32_i64(temp, orig);
1701 }
1702 op->args[pi++] = tcgv_i64_arg(temp);
1703 extend_free[n_extend++] = temp;
1704 }
1705 break;
1706
1707 default:
1708 g_assert_not_reached();
1709 }
1710 }
1711 op->args[pi++] = (uintptr_t)func;
1712 op->args[pi++] = (uintptr_t)info;
1713 tcg_debug_assert(pi == total_args);
1714
1715 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
1716
1717 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
1718 for (i = 0; i < n_extend; ++i) {
1719 tcg_temp_free_i64(extend_free[i]);
1720 }
1721 }
1722
1723 static void tcg_reg_alloc_start(TCGContext *s)
1724 {
1725 int i, n;
1726
1727 for (i = 0, n = s->nb_temps; i < n; i++) {
1728 TCGTemp *ts = &s->temps[i];
1729 TCGTempVal val = TEMP_VAL_MEM;
1730
1731 switch (ts->kind) {
1732 case TEMP_CONST:
1733 val = TEMP_VAL_CONST;
1734 break;
1735 case TEMP_FIXED:
1736 val = TEMP_VAL_REG;
1737 break;
1738 case TEMP_GLOBAL:
1739 break;
1740 case TEMP_NORMAL:
1741 case TEMP_EBB:
1742 val = TEMP_VAL_DEAD;
1743 /* fall through */
1744 case TEMP_LOCAL:
1745 ts->mem_allocated = 0;
1746 break;
1747 default:
1748 g_assert_not_reached();
1749 }
1750 ts->val_type = val;
1751 }
1752
1753 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
1754 }
1755
1756 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1757 TCGTemp *ts)
1758 {
1759 int idx = temp_idx(ts);
1760
1761 switch (ts->kind) {
1762 case TEMP_FIXED:
1763 case TEMP_GLOBAL:
1764 pstrcpy(buf, buf_size, ts->name);
1765 break;
1766 case TEMP_LOCAL:
1767 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
1768 break;
1769 case TEMP_EBB:
1770 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals);
1771 break;
1772 case TEMP_NORMAL:
1773 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
1774 break;
1775 case TEMP_CONST:
1776 switch (ts->type) {
1777 case TCG_TYPE_I32:
1778 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1779 break;
1780 #if TCG_TARGET_REG_BITS > 32
1781 case TCG_TYPE_I64:
1782 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1783 break;
1784 #endif
1785 case TCG_TYPE_V64:
1786 case TCG_TYPE_V128:
1787 case TCG_TYPE_V256:
1788 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1789 64 << (ts->type - TCG_TYPE_V64), ts->val);
1790 break;
1791 default:
1792 g_assert_not_reached();
1793 }
1794 break;
1795 }
1796 return buf;
1797 }
1798
1799 static char *tcg_get_arg_str(TCGContext *s, char *buf,
1800 int buf_size, TCGArg arg)
1801 {
1802 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
1803 }
1804
1805 static const char * const cond_name[] =
1806 {
1807 [TCG_COND_NEVER] = "never",
1808 [TCG_COND_ALWAYS] = "always",
1809 [TCG_COND_EQ] = "eq",
1810 [TCG_COND_NE] = "ne",
1811 [TCG_COND_LT] = "lt",
1812 [TCG_COND_GE] = "ge",
1813 [TCG_COND_LE] = "le",
1814 [TCG_COND_GT] = "gt",
1815 [TCG_COND_LTU] = "ltu",
1816 [TCG_COND_GEU] = "geu",
1817 [TCG_COND_LEU] = "leu",
1818 [TCG_COND_GTU] = "gtu"
1819 };
1820
1821 static const char * const ldst_name[] =
1822 {
1823 [MO_UB] = "ub",
1824 [MO_SB] = "sb",
1825 [MO_LEUW] = "leuw",
1826 [MO_LESW] = "lesw",
1827 [MO_LEUL] = "leul",
1828 [MO_LESL] = "lesl",
1829 [MO_LEUQ] = "leq",
1830 [MO_BEUW] = "beuw",
1831 [MO_BESW] = "besw",
1832 [MO_BEUL] = "beul",
1833 [MO_BESL] = "besl",
1834 [MO_BEUQ] = "beq",
1835 };
1836
1837 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
1838 #ifdef TARGET_ALIGNED_ONLY
1839 [MO_UNALN >> MO_ASHIFT] = "un+",
1840 [MO_ALIGN >> MO_ASHIFT] = "",
1841 #else
1842 [MO_UNALN >> MO_ASHIFT] = "",
1843 [MO_ALIGN >> MO_ASHIFT] = "al+",
1844 #endif
1845 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1846 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1847 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1848 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1849 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1850 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1851 };
1852
1853 static const char bswap_flag_name[][6] = {
1854 [TCG_BSWAP_IZ] = "iz",
1855 [TCG_BSWAP_OZ] = "oz",
1856 [TCG_BSWAP_OS] = "os",
1857 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1858 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1859 };
1860
1861 static inline bool tcg_regset_single(TCGRegSet d)
1862 {
1863 return (d & (d - 1)) == 0;
1864 }
1865
1866 static inline TCGReg tcg_regset_first(TCGRegSet d)
1867 {
1868 if (TCG_TARGET_NB_REGS <= 32) {
1869 return ctz32(d);
1870 } else {
1871 return ctz64(d);
1872 }
1873 }
1874
1875 /* Return only the number of characters output -- no error return. */
1876 #define ne_fprintf(...) \
1877 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1878
1879 static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
1880 {
1881 char buf[128];
1882 TCGOp *op;
1883
1884 QTAILQ_FOREACH(op, &s->ops, link) {
1885 int i, k, nb_oargs, nb_iargs, nb_cargs;
1886 const TCGOpDef *def;
1887 TCGOpcode c;
1888 int col = 0;
1889
1890 c = op->opc;
1891 def = &tcg_op_defs[c];
1892
1893 if (c == INDEX_op_insn_start) {
1894 nb_oargs = 0;
1895 col += ne_fprintf(f, "\n ----");
1896
1897 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1898 target_ulong a;
1899 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1900 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
1901 #else
1902 a = op->args[i];
1903 #endif
1904 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
1905 }
1906 } else if (c == INDEX_op_call) {
1907 const TCGHelperInfo *info = tcg_call_info(op);
1908 void *func = tcg_call_func(op);
1909
1910 /* variable number of arguments */
1911 nb_oargs = TCGOP_CALLO(op);
1912 nb_iargs = TCGOP_CALLI(op);
1913 nb_cargs = def->nb_cargs;
1914
1915 col += ne_fprintf(f, " %s ", def->name);
1916
1917 /*
1918 * Print the function name from TCGHelperInfo, if available.
1919 * Note that plugins have a template function for the info,
1920 * but the actual function pointer comes from the plugin.
1921 */
1922 if (func == info->func) {
1923 col += ne_fprintf(f, "%s", info->name);
1924 } else {
1925 col += ne_fprintf(f, "plugin(%p)", func);
1926 }
1927
1928 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
1929 for (i = 0; i < nb_oargs; i++) {
1930 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1931 op->args[i]));
1932 }
1933 for (i = 0; i < nb_iargs; i++) {
1934 TCGArg arg = op->args[nb_oargs + i];
1935 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
1936 col += ne_fprintf(f, ",%s", t);
1937 }
1938 } else {
1939 col += ne_fprintf(f, " %s ", def->name);
1940
1941 nb_oargs = def->nb_oargs;
1942 nb_iargs = def->nb_iargs;
1943 nb_cargs = def->nb_cargs;
1944
1945 if (def->flags & TCG_OPF_VECTOR) {
1946 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
1947 8 << TCGOP_VECE(op));
1948 }
1949
1950 k = 0;
1951 for (i = 0; i < nb_oargs; i++) {
1952 const char *sep = k ? "," : "";
1953 col += ne_fprintf(f, "%s%s", sep,
1954 tcg_get_arg_str(s, buf, sizeof(buf),
1955 op->args[k++]));
1956 }
1957 for (i = 0; i < nb_iargs; i++) {
1958 const char *sep = k ? "," : "";
1959 col += ne_fprintf(f, "%s%s", sep,
1960 tcg_get_arg_str(s, buf, sizeof(buf),
1961 op->args[k++]));
1962 }
1963 switch (c) {
1964 case INDEX_op_brcond_i32:
1965 case INDEX_op_setcond_i32:
1966 case INDEX_op_movcond_i32:
1967 case INDEX_op_brcond2_i32:
1968 case INDEX_op_setcond2_i32:
1969 case INDEX_op_brcond_i64:
1970 case INDEX_op_setcond_i64:
1971 case INDEX_op_movcond_i64:
1972 case INDEX_op_cmp_vec:
1973 case INDEX_op_cmpsel_vec:
1974 if (op->args[k] < ARRAY_SIZE(cond_name)
1975 && cond_name[op->args[k]]) {
1976 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
1977 } else {
1978 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
1979 }
1980 i = 1;
1981 break;
1982 case INDEX_op_qemu_ld_i32:
1983 case INDEX_op_qemu_st_i32:
1984 case INDEX_op_qemu_st8_i32:
1985 case INDEX_op_qemu_ld_i64:
1986 case INDEX_op_qemu_st_i64:
1987 {
1988 MemOpIdx oi = op->args[k++];
1989 MemOp op = get_memop(oi);
1990 unsigned ix = get_mmuidx(oi);
1991
1992 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
1993 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
1994 } else {
1995 const char *s_al, *s_op;
1996 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
1997 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
1998 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
1999 }
2000 i = 1;
2001 }
2002 break;
2003 case INDEX_op_bswap16_i32:
2004 case INDEX_op_bswap16_i64:
2005 case INDEX_op_bswap32_i32:
2006 case INDEX_op_bswap32_i64:
2007 case INDEX_op_bswap64_i64:
2008 {
2009 TCGArg flags = op->args[k];
2010 const char *name = NULL;
2011
2012 if (flags < ARRAY_SIZE(bswap_flag_name)) {
2013 name = bswap_flag_name[flags];
2014 }
2015 if (name) {
2016 col += ne_fprintf(f, ",%s", name);
2017 } else {
2018 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
2019 }
2020 i = k = 1;
2021 }
2022 break;
2023 default:
2024 i = 0;
2025 break;
2026 }
2027 switch (c) {
2028 case INDEX_op_set_label:
2029 case INDEX_op_br:
2030 case INDEX_op_brcond_i32:
2031 case INDEX_op_brcond_i64:
2032 case INDEX_op_brcond2_i32:
2033 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2034 arg_label(op->args[k])->id);
2035 i++, k++;
2036 break;
2037 default:
2038 break;
2039 }
2040 for (; i < nb_cargs; i++, k++) {
2041 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2042 op->args[k]);
2043 }
2044 }
2045
2046 if (have_prefs || op->life) {
2047 for (; col < 40; ++col) {
2048 putc(' ', f);
2049 }
2050 }
2051
2052 if (op->life) {
2053 unsigned life = op->life;
2054
2055 if (life & (SYNC_ARG * 3)) {
2056 ne_fprintf(f, " sync:");
2057 for (i = 0; i < 2; ++i) {
2058 if (life & (SYNC_ARG << i)) {
2059 ne_fprintf(f, " %d", i);
2060 }
2061 }
2062 }
2063 life /= DEAD_ARG;
2064 if (life) {
2065 ne_fprintf(f, " dead:");
2066 for (i = 0; life; ++i, life >>= 1) {
2067 if (life & 1) {
2068 ne_fprintf(f, " %d", i);
2069 }
2070 }
2071 }
2072 }
2073
2074 if (have_prefs) {
2075 for (i = 0; i < nb_oargs; ++i) {
2076 TCGRegSet set = output_pref(op, i);
2077
2078 if (i == 0) {
2079 ne_fprintf(f, " pref=");
2080 } else {
2081 ne_fprintf(f, ",");
2082 }
2083 if (set == 0) {
2084 ne_fprintf(f, "none");
2085 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2086 ne_fprintf(f, "all");
2087 #ifdef CONFIG_DEBUG_TCG
2088 } else if (tcg_regset_single(set)) {
2089 TCGReg reg = tcg_regset_first(set);
2090 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
2091 #endif
2092 } else if (TCG_TARGET_NB_REGS <= 32) {
2093 ne_fprintf(f, "0x%x", (uint32_t)set);
2094 } else {
2095 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
2096 }
2097 }
2098 }
2099
2100 putc('\n', f);
2101 }
2102 }
2103
2104 /* we give more priority to constraints with less registers */
2105 static int get_constraint_priority(const TCGOpDef *def, int k)
2106 {
2107 const TCGArgConstraint *arg_ct = &def->args_ct[k];
2108 int n = ctpop64(arg_ct->regs);
2109
2110 /*
2111 * Sort constraints of a single register first, which includes output
2112 * aliases (which must exactly match the input already allocated).
2113 */
2114 if (n == 1 || arg_ct->oalias) {
2115 return INT_MAX;
2116 }
2117
2118 /*
2119 * Sort register pairs next, first then second immediately after.
2120 * Arbitrarily sort multiple pairs by the index of the first reg;
2121 * there shouldn't be many pairs.
2122 */
2123 switch (arg_ct->pair) {
2124 case 1:
2125 case 3:
2126 return (k + 1) * 2;
2127 case 2:
2128 return (arg_ct->pair_index + 1) * 2 - 1;
2129 }
2130
2131 /* Finally, sort by decreasing register count. */
2132 assert(n > 1);
2133 return -n;
2134 }
2135
2136 /* sort from highest priority to lowest */
2137 static void sort_constraints(TCGOpDef *def, int start, int n)
2138 {
2139 int i, j;
2140 TCGArgConstraint *a = def->args_ct;
2141
2142 for (i = 0; i < n; i++) {
2143 a[start + i].sort_index = start + i;
2144 }
2145 if (n <= 1) {
2146 return;
2147 }
2148 for (i = 0; i < n - 1; i++) {
2149 for (j = i + 1; j < n; j++) {
2150 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2151 int p2 = get_constraint_priority(def, a[start + j].sort_index);
2152 if (p1 < p2) {
2153 int tmp = a[start + i].sort_index;
2154 a[start + i].sort_index = a[start + j].sort_index;
2155 a[start + j].sort_index = tmp;
2156 }
2157 }
2158 }
2159 }
2160
2161 static void process_op_defs(TCGContext *s)
2162 {
2163 TCGOpcode op;
2164
2165 for (op = 0; op < NB_OPS; op++) {
2166 TCGOpDef *def = &tcg_op_defs[op];
2167 const TCGTargetOpDef *tdefs;
2168 bool saw_alias_pair = false;
2169 int i, o, i2, o2, nb_args;
2170
2171 if (def->flags & TCG_OPF_NOT_PRESENT) {
2172 continue;
2173 }
2174
2175 nb_args = def->nb_iargs + def->nb_oargs;
2176 if (nb_args == 0) {
2177 continue;
2178 }
2179
2180 /*
2181 * Macro magic should make it impossible, but double-check that
2182 * the array index is in range. Since the signness of an enum
2183 * is implementation defined, force the result to unsigned.
2184 */
2185 unsigned con_set = tcg_target_op_def(op);
2186 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2187 tdefs = &constraint_sets[con_set];
2188
2189 for (i = 0; i < nb_args; i++) {
2190 const char *ct_str = tdefs->args_ct_str[i];
2191 bool input_p = i >= def->nb_oargs;
2192
2193 /* Incomplete TCGTargetOpDef entry. */
2194 tcg_debug_assert(ct_str != NULL);
2195
2196 switch (*ct_str) {
2197 case '0' ... '9':
2198 o = *ct_str - '0';
2199 tcg_debug_assert(input_p);
2200 tcg_debug_assert(o < def->nb_oargs);
2201 tcg_debug_assert(def->args_ct[o].regs != 0);
2202 tcg_debug_assert(!def->args_ct[o].oalias);
2203 def->args_ct[i] = def->args_ct[o];
2204 /* The output sets oalias. */
2205 def->args_ct[o].oalias = 1;
2206 def->args_ct[o].alias_index = i;
2207 /* The input sets ialias. */
2208 def->args_ct[i].ialias = 1;
2209 def->args_ct[i].alias_index = o;
2210 if (def->args_ct[i].pair) {
2211 saw_alias_pair = true;
2212 }
2213 tcg_debug_assert(ct_str[1] == '\0');
2214 continue;
2215
2216 case '&':
2217 tcg_debug_assert(!input_p);
2218 def->args_ct[i].newreg = true;
2219 ct_str++;
2220 break;
2221
2222 case 'p': /* plus */
2223 /* Allocate to the register after the previous. */
2224 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2225 o = i - 1;
2226 tcg_debug_assert(!def->args_ct[o].pair);
2227 tcg_debug_assert(!def->args_ct[o].ct);
2228 def->args_ct[i] = (TCGArgConstraint){
2229 .pair = 2,
2230 .pair_index = o,
2231 .regs = def->args_ct[o].regs << 1,
2232 };
2233 def->args_ct[o].pair = 1;
2234 def->args_ct[o].pair_index = i;
2235 tcg_debug_assert(ct_str[1] == '\0');
2236 continue;
2237
2238 case 'm': /* minus */
2239 /* Allocate to the register before the previous. */
2240 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2241 o = i - 1;
2242 tcg_debug_assert(!def->args_ct[o].pair);
2243 tcg_debug_assert(!def->args_ct[o].ct);
2244 def->args_ct[i] = (TCGArgConstraint){
2245 .pair = 1,
2246 .pair_index = o,
2247 .regs = def->args_ct[o].regs >> 1,
2248 };
2249 def->args_ct[o].pair = 2;
2250 def->args_ct[o].pair_index = i;
2251 tcg_debug_assert(ct_str[1] == '\0');
2252 continue;
2253 }
2254
2255 do {
2256 switch (*ct_str) {
2257 case 'i':
2258 def->args_ct[i].ct |= TCG_CT_CONST;
2259 break;
2260
2261 /* Include all of the target-specific constraints. */
2262
2263 #undef CONST
2264 #define CONST(CASE, MASK) \
2265 case CASE: def->args_ct[i].ct |= MASK; break;
2266 #define REGS(CASE, MASK) \
2267 case CASE: def->args_ct[i].regs |= MASK; break;
2268
2269 #include "tcg-target-con-str.h"
2270
2271 #undef REGS
2272 #undef CONST
2273 default:
2274 case '0' ... '9':
2275 case '&':
2276 case 'p':
2277 case 'm':
2278 /* Typo in TCGTargetOpDef constraint. */
2279 g_assert_not_reached();
2280 }
2281 } while (*++ct_str != '\0');
2282 }
2283
2284 /* TCGTargetOpDef entry with too much information? */
2285 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
2286
2287 /*
2288 * Fix up output pairs that are aliased with inputs.
2289 * When we created the alias, we copied pair from the output.
2290 * There are three cases:
2291 * (1a) Pairs of inputs alias pairs of outputs.
2292 * (1b) One input aliases the first of a pair of outputs.
2293 * (2) One input aliases the second of a pair of outputs.
2294 *
2295 * Case 1a is handled by making sure that the pair_index'es are
2296 * properly updated so that they appear the same as a pair of inputs.
2297 *
2298 * Case 1b is handled by setting the pair_index of the input to
2299 * itself, simply so it doesn't point to an unrelated argument.
2300 * Since we don't encounter the "second" during the input allocation
2301 * phase, nothing happens with the second half of the input pair.
2302 *
2303 * Case 2 is handled by setting the second input to pair=3, the
2304 * first output to pair=3, and the pair_index'es to match.
2305 */
2306 if (saw_alias_pair) {
2307 for (i = def->nb_oargs; i < nb_args; i++) {
2308 /*
2309 * Since [0-9pm] must be alone in the constraint string,
2310 * the only way they can both be set is if the pair comes
2311 * from the output alias.
2312 */
2313 if (!def->args_ct[i].ialias) {
2314 continue;
2315 }
2316 switch (def->args_ct[i].pair) {
2317 case 0:
2318 break;
2319 case 1:
2320 o = def->args_ct[i].alias_index;
2321 o2 = def->args_ct[o].pair_index;
2322 tcg_debug_assert(def->args_ct[o].pair == 1);
2323 tcg_debug_assert(def->args_ct[o2].pair == 2);
2324 if (def->args_ct[o2].oalias) {
2325 /* Case 1a */
2326 i2 = def->args_ct[o2].alias_index;
2327 tcg_debug_assert(def->args_ct[i2].pair == 2);
2328 def->args_ct[i2].pair_index = i;
2329 def->args_ct[i].pair_index = i2;
2330 } else {
2331 /* Case 1b */
2332 def->args_ct[i].pair_index = i;
2333 }
2334 break;
2335 case 2:
2336 o = def->args_ct[i].alias_index;
2337 o2 = def->args_ct[o].pair_index;
2338 tcg_debug_assert(def->args_ct[o].pair == 2);
2339 tcg_debug_assert(def->args_ct[o2].pair == 1);
2340 if (def->args_ct[o2].oalias) {
2341 /* Case 1a */
2342 i2 = def->args_ct[o2].alias_index;
2343 tcg_debug_assert(def->args_ct[i2].pair == 1);
2344 def->args_ct[i2].pair_index = i;
2345 def->args_ct[i].pair_index = i2;
2346 } else {
2347 /* Case 2 */
2348 def->args_ct[i].pair = 3;
2349 def->args_ct[o2].pair = 3;
2350 def->args_ct[i].pair_index = o2;
2351 def->args_ct[o2].pair_index = i;
2352 }
2353 break;
2354 default:
2355 g_assert_not_reached();
2356 }
2357 }
2358 }
2359
2360 /* sort the constraints (XXX: this is just an heuristic) */
2361 sort_constraints(def, 0, def->nb_oargs);
2362 sort_constraints(def, def->nb_oargs, def->nb_iargs);
2363 }
2364 }
2365
2366 void tcg_op_remove(TCGContext *s, TCGOp *op)
2367 {
2368 TCGLabel *label;
2369
2370 switch (op->opc) {
2371 case INDEX_op_br:
2372 label = arg_label(op->args[0]);
2373 label->refs--;
2374 break;
2375 case INDEX_op_brcond_i32:
2376 case INDEX_op_brcond_i64:
2377 label = arg_label(op->args[3]);
2378 label->refs--;
2379 break;
2380 case INDEX_op_brcond2_i32:
2381 label = arg_label(op->args[5]);
2382 label->refs--;
2383 break;
2384 default:
2385 break;
2386 }
2387
2388 QTAILQ_REMOVE(&s->ops, op, link);
2389 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
2390 s->nb_ops--;
2391
2392 #ifdef CONFIG_PROFILER
2393 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
2394 #endif
2395 }
2396
2397 void tcg_remove_ops_after(TCGOp *op)
2398 {
2399 TCGContext *s = tcg_ctx;
2400
2401 while (true) {
2402 TCGOp *last = tcg_last_op();
2403 if (last == op) {
2404 return;
2405 }
2406 tcg_op_remove(s, last);
2407 }
2408 }
2409
2410 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
2411 {
2412 TCGContext *s = tcg_ctx;
2413 TCGOp *op = NULL;
2414
2415 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
2416 QTAILQ_FOREACH(op, &s->free_ops, link) {
2417 if (nargs <= op->nargs) {
2418 QTAILQ_REMOVE(&s->free_ops, op, link);
2419 nargs = op->nargs;
2420 goto found;
2421 }
2422 }
2423 }
2424
2425 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2426 nargs = MAX(4, nargs);
2427 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
2428
2429 found:
2430 memset(op, 0, offsetof(TCGOp, link));
2431 op->opc = opc;
2432 op->nargs = nargs;
2433
2434 /* Check for bitfield overflow. */
2435 tcg_debug_assert(op->nargs == nargs);
2436
2437 s->nb_ops++;
2438 return op;
2439 }
2440
2441 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
2442 {
2443 TCGOp *op = tcg_op_alloc(opc, nargs);
2444 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2445 return op;
2446 }
2447
2448 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
2449 TCGOpcode opc, unsigned nargs)
2450 {
2451 TCGOp *new_op = tcg_op_alloc(opc, nargs);
2452 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
2453 return new_op;
2454 }
2455
2456 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
2457 TCGOpcode opc, unsigned nargs)
2458 {
2459 TCGOp *new_op = tcg_op_alloc(opc, nargs);
2460 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
2461 return new_op;
2462 }
2463
2464 /* Reachable analysis : remove unreachable code. */
2465 static void reachable_code_pass(TCGContext *s)
2466 {
2467 TCGOp *op, *op_next;
2468 bool dead = false;
2469
2470 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2471 bool remove = dead;
2472 TCGLabel *label;
2473
2474 switch (op->opc) {
2475 case INDEX_op_set_label:
2476 label = arg_label(op->args[0]);
2477 if (label->refs == 0) {
2478 /*
2479 * While there is an occasional backward branch, virtually
2480 * all branches generated by the translators are forward.
2481 * Which means that generally we will have already removed
2482 * all references to the label that will be, and there is
2483 * little to be gained by iterating.
2484 */
2485 remove = true;
2486 } else {
2487 /* Once we see a label, insns become live again. */
2488 dead = false;
2489 remove = false;
2490
2491 /*
2492 * Optimization can fold conditional branches to unconditional.
2493 * If we find a label with one reference which is preceded by
2494 * an unconditional branch to it, remove both. This needed to
2495 * wait until the dead code in between them was removed.
2496 */
2497 if (label->refs == 1) {
2498 TCGOp *op_prev = QTAILQ_PREV(op, link);
2499 if (op_prev->opc == INDEX_op_br &&
2500 label == arg_label(op_prev->args[0])) {
2501 tcg_op_remove(s, op_prev);
2502 remove = true;
2503 }
2504 }
2505 }
2506 break;
2507
2508 case INDEX_op_br:
2509 case INDEX_op_exit_tb:
2510 case INDEX_op_goto_ptr:
2511 /* Unconditional branches; everything following is dead. */
2512 dead = true;
2513 break;
2514
2515 case INDEX_op_call:
2516 /* Notice noreturn helper calls, raising exceptions. */
2517 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
2518 dead = true;
2519 }
2520 break;
2521
2522 case INDEX_op_insn_start:
2523 /* Never remove -- we need to keep these for unwind. */
2524 remove = false;
2525 break;
2526
2527 default:
2528 break;
2529 }
2530
2531 if (remove) {
2532 tcg_op_remove(s, op);
2533 }
2534 }
2535 }
2536
2537 #define TS_DEAD 1
2538 #define TS_MEM 2
2539
2540 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2541 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2542
2543 /* For liveness_pass_1, the register preferences for a given temp. */
2544 static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2545 {
2546 return ts->state_ptr;
2547 }
2548
2549 /* For liveness_pass_1, reset the preferences for a given temp to the
2550 * maximal regset for its type.
2551 */
2552 static inline void la_reset_pref(TCGTemp *ts)
2553 {
2554 *la_temp_pref(ts)
2555 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2556 }
2557
2558 /* liveness analysis: end of function: all temps are dead, and globals
2559 should be in memory. */
2560 static void la_func_end(TCGContext *s, int ng, int nt)
2561 {
2562 int i;
2563
2564 for (i = 0; i < ng; ++i) {
2565 s->temps[i].state = TS_DEAD | TS_MEM;
2566 la_reset_pref(&s->temps[i]);
2567 }
2568 for (i = ng; i < nt; ++i) {
2569 s->temps[i].state = TS_DEAD;
2570 la_reset_pref(&s->temps[i]);
2571 }
2572 }
2573
2574 /* liveness analysis: end of basic block: all temps are dead, globals
2575 and local temps should be in memory. */
2576 static void la_bb_end(TCGContext *s, int ng, int nt)
2577 {
2578 int i;
2579
2580 for (i = 0; i < nt; ++i) {
2581 TCGTemp *ts = &s->temps[i];
2582 int state;
2583
2584 switch (ts->kind) {
2585 case TEMP_FIXED:
2586 case TEMP_GLOBAL:
2587 case TEMP_LOCAL:
2588 state = TS_DEAD | TS_MEM;
2589 break;
2590 case TEMP_NORMAL:
2591 case TEMP_EBB:
2592 case TEMP_CONST:
2593 state = TS_DEAD;
2594 break;
2595 default:
2596 g_assert_not_reached();
2597 }
2598 ts->state = state;
2599 la_reset_pref(ts);
2600 }
2601 }
2602
2603 /* liveness analysis: sync globals back to memory. */
2604 static void la_global_sync(TCGContext *s, int ng)
2605 {
2606 int i;
2607
2608 for (i = 0; i < ng; ++i) {
2609 int state = s->temps[i].state;
2610 s->temps[i].state = state | TS_MEM;
2611 if (state == TS_DEAD) {
2612 /* If the global was previously dead, reset prefs. */
2613 la_reset_pref(&s->temps[i]);
2614 }
2615 }
2616 }
2617
2618 /*
2619 * liveness analysis: conditional branch: all temps are dead unless
2620 * explicitly live-across-conditional-branch, globals and local temps
2621 * should be synced.
2622 */
2623 static void la_bb_sync(TCGContext *s, int ng, int nt)
2624 {
2625 la_global_sync(s, ng);
2626
2627 for (int i = ng; i < nt; ++i) {
2628 TCGTemp *ts = &s->temps[i];
2629 int state;
2630
2631 switch (ts->kind) {
2632 case TEMP_LOCAL:
2633 state = ts->state;
2634 ts->state = state | TS_MEM;
2635 if (state != TS_DEAD) {
2636 continue;
2637 }
2638 break;
2639 case TEMP_NORMAL:
2640 s->temps[i].state = TS_DEAD;
2641 break;
2642 case TEMP_EBB:
2643 case TEMP_CONST:
2644 continue;
2645 default:
2646 g_assert_not_reached();
2647 }
2648 la_reset_pref(&s->temps[i]);
2649 }
2650 }
2651
2652 /* liveness analysis: sync globals back to memory and kill. */
2653 static void la_global_kill(TCGContext *s, int ng)
2654 {
2655 int i;
2656
2657 for (i = 0; i < ng; i++) {
2658 s->temps[i].state = TS_DEAD | TS_MEM;
2659 la_reset_pref(&s->temps[i]);
2660 }
2661 }
2662
2663 /* liveness analysis: note live globals crossing calls. */
2664 static void la_cross_call(TCGContext *s, int nt)
2665 {
2666 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2667 int i;
2668
2669 for (i = 0; i < nt; i++) {
2670 TCGTemp *ts = &s->temps[i];
2671 if (!(ts->state & TS_DEAD)) {
2672 TCGRegSet *pset = la_temp_pref(ts);
2673 TCGRegSet set = *pset;
2674
2675 set &= mask;
2676 /* If the combination is not possible, restart. */
2677 if (set == 0) {
2678 set = tcg_target_available_regs[ts->type] & mask;
2679 }
2680 *pset = set;
2681 }
2682 }
2683 }
2684
2685 /* Liveness analysis : update the opc_arg_life array to tell if a
2686 given input arguments is dead. Instructions updating dead
2687 temporaries are removed. */
2688 static void liveness_pass_1(TCGContext *s)
2689 {
2690 int nb_globals = s->nb_globals;
2691 int nb_temps = s->nb_temps;
2692 TCGOp *op, *op_prev;
2693 TCGRegSet *prefs;
2694 int i;
2695
2696 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2697 for (i = 0; i < nb_temps; ++i) {
2698 s->temps[i].state_ptr = prefs + i;
2699 }
2700
2701 /* ??? Should be redundant with the exit_tb that ends the TB. */
2702 la_func_end(s, nb_globals, nb_temps);
2703
2704 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
2705 int nb_iargs, nb_oargs;
2706 TCGOpcode opc_new, opc_new2;
2707 bool have_opc_new2;
2708 TCGLifeData arg_life = 0;
2709 TCGTemp *ts;
2710 TCGOpcode opc = op->opc;
2711 const TCGOpDef *def = &tcg_op_defs[opc];
2712
2713 switch (opc) {
2714 case INDEX_op_call:
2715 {
2716 const TCGHelperInfo *info = tcg_call_info(op);
2717 int call_flags = tcg_call_flags(op);
2718
2719 nb_oargs = TCGOP_CALLO(op);
2720 nb_iargs = TCGOP_CALLI(op);
2721
2722 /* pure functions can be removed if their result is unused */
2723 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
2724 for (i = 0; i < nb_oargs; i++) {
2725 ts = arg_temp(op->args[i]);
2726 if (ts->state != TS_DEAD) {
2727 goto do_not_remove_call;
2728 }
2729 }
2730 goto do_remove;
2731 }
2732 do_not_remove_call:
2733
2734 /* Output args are dead. */
2735 for (i = 0; i < nb_oargs; i++) {
2736 ts = arg_temp(op->args[i]);
2737 if (ts->state & TS_DEAD) {
2738 arg_life |= DEAD_ARG << i;
2739 }
2740 if (ts->state & TS_MEM) {
2741 arg_life |= SYNC_ARG << i;
2742 }
2743 ts->state = TS_DEAD;
2744 la_reset_pref(ts);
2745 }
2746
2747 /* Not used -- it will be tcg_target_call_oarg_reg(). */
2748 memset(op->output_pref, 0, sizeof(op->output_pref));
2749
2750 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2751 TCG_CALL_NO_READ_GLOBALS))) {
2752 la_global_kill(s, nb_globals);
2753 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
2754 la_global_sync(s, nb_globals);
2755 }
2756
2757 /* Record arguments that die in this helper. */
2758 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2759 ts = arg_temp(op->args[i]);
2760 if (ts->state & TS_DEAD) {
2761 arg_life |= DEAD_ARG << i;
2762 }
2763 }
2764
2765 /* For all live registers, remove call-clobbered prefs. */
2766 la_cross_call(s, nb_temps);
2767
2768 /*
2769 * Input arguments are live for preceding opcodes.
2770 *
2771 * For those arguments that die, and will be allocated in
2772 * registers, clear the register set for that arg, to be
2773 * filled in below. For args that will be on the stack,
2774 * reset to any available reg. Process arguments in reverse
2775 * order so that if a temp is used more than once, the stack
2776 * reset to max happens before the register reset to 0.
2777 */
2778 for (i = nb_iargs - 1; i >= 0; i--) {
2779 const TCGCallArgumentLoc *loc = &info->in[i];
2780 ts = arg_temp(op->args[nb_oargs + i]);
2781
2782 if (ts->state & TS_DEAD) {
2783 switch (loc->kind) {
2784 case TCG_CALL_ARG_NORMAL:
2785 case TCG_CALL_ARG_EXTEND_U:
2786 case TCG_CALL_ARG_EXTEND_S:
2787 if (REG_P(loc)) {
2788 *la_temp_pref(ts) = 0;
2789 break;
2790 }
2791 /* fall through */
2792 default:
2793 *la_temp_pref(ts) =
2794 tcg_target_available_regs[ts->type];
2795 break;
2796 }
2797 ts->state &= ~TS_DEAD;
2798 }
2799 }
2800
2801 /*
2802 * For each input argument, add its input register to prefs.
2803 * If a temp is used once, this produces a single set bit;
2804 * if a temp is used multiple times, this produces a set.
2805 */
2806 for (i = 0; i < nb_iargs; i++) {
2807 const TCGCallArgumentLoc *loc = &info->in[i];
2808 ts = arg_temp(op->args[nb_oargs + i]);
2809
2810 switch (loc->kind) {
2811 case TCG_CALL_ARG_NORMAL:
2812 case TCG_CALL_ARG_EXTEND_U:
2813 case TCG_CALL_ARG_EXTEND_S:
2814 if (REG_P(loc)) {
2815 tcg_regset_set_reg(*la_temp_pref(ts),
2816 tcg_target_call_iarg_regs[loc->arg_slot]);
2817 }
2818 break;
2819 default:
2820 break;
2821 }
2822 }
2823 }
2824 break;
2825 case INDEX_op_insn_start:
2826 break;
2827 case INDEX_op_discard:
2828 /* mark the temporary as dead */
2829 ts = arg_temp(op->args[0]);
2830 ts->state = TS_DEAD;
2831 la_reset_pref(ts);
2832 break;
2833
2834 case INDEX_op_add2_i32:
2835 opc_new = INDEX_op_add_i32;
2836 goto do_addsub2;
2837 case INDEX_op_sub2_i32:
2838 opc_new = INDEX_op_sub_i32;
2839 goto do_addsub2;
2840 case INDEX_op_add2_i64:
2841 opc_new = INDEX_op_add_i64;
2842 goto do_addsub2;
2843 case INDEX_op_sub2_i64:
2844 opc_new = INDEX_op_sub_i64;
2845 do_addsub2:
2846 nb_iargs = 4;
2847 nb_oargs = 2;
2848 /* Test if the high part of the operation is dead, but not
2849 the low part. The result can be optimized to a simple
2850 add or sub. This happens often for x86_64 guest when the
2851 cpu mode is set to 32 bit. */
2852 if (arg_temp(op->args[1])->state == TS_DEAD) {
2853 if (arg_temp(op->args[0])->state == TS_DEAD) {
2854 goto do_remove;
2855 }
2856 /* Replace the opcode and adjust the args in place,
2857 leaving 3 unused args at the end. */
2858 op->opc = opc = opc_new;
2859 op->args[1] = op->args[2];
2860 op->args[2] = op->args[4];
2861 /* Fall through and mark the single-word operation live. */
2862 nb_iargs = 2;
2863 nb_oargs = 1;
2864 }
2865 goto do_not_remove;
2866
2867 case INDEX_op_mulu2_i32:
2868 opc_new = INDEX_op_mul_i32;
2869 opc_new2 = INDEX_op_muluh_i32;
2870 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
2871 goto do_mul2;
2872 case INDEX_op_muls2_i32:
2873 opc_new = INDEX_op_mul_i32;
2874 opc_new2 = INDEX_op_mulsh_i32;
2875 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
2876 goto do_mul2;
2877 case INDEX_op_mulu2_i64:
2878 opc_new = INDEX_op_mul_i64;
2879 opc_new2 = INDEX_op_muluh_i64;
2880 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
2881 goto do_mul2;
2882 case INDEX_op_muls2_i64:
2883 opc_new = INDEX_op_mul_i64;
2884 opc_new2 = INDEX_op_mulsh_i64;
2885 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
2886 goto do_mul2;
2887 do_mul2:
2888 nb_iargs = 2;
2889 nb_oargs = 2;
2890 if (arg_temp(op->args[1])->state == TS_DEAD) {
2891 if (arg_temp(op->args[0])->state == TS_DEAD) {
2892 /* Both parts of the operation are dead. */
2893 goto do_remove;
2894 }
2895 /* The high part of the operation is dead; generate the low. */
2896 op->opc = opc = opc_new;
2897 op->args[1] = op->args[2];
2898 op->args[2] = op->args[3];
2899 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
2900 /* The low part of the operation is dead; generate the high. */
2901 op->opc = opc = opc_new2;
2902 op->args[0] = op->args[1];
2903 op->args[1] = op->args[2];
2904 op->args[2] = op->args[3];
2905 } else {
2906 goto do_not_remove;
2907 }
2908 /* Mark the single-word operation live. */
2909 nb_oargs = 1;
2910 goto do_not_remove;
2911
2912 default:
2913 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2914 nb_iargs = def->nb_iargs;
2915 nb_oargs = def->nb_oargs;
2916
2917 /* Test if the operation can be removed because all
2918 its outputs are dead. We assume that nb_oargs == 0
2919 implies side effects */
2920 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
2921 for (i = 0; i < nb_oargs; i++) {
2922 if (arg_temp(op->args[i])->state != TS_DEAD) {
2923 goto do_not_remove;
2924 }
2925 }
2926 goto do_remove;
2927 }
2928 goto do_not_remove;
2929
2930 do_remove:
2931 tcg_op_remove(s, op);
2932 break;
2933
2934 do_not_remove:
2935 for (i = 0; i < nb_oargs; i++) {
2936 ts = arg_temp(op->args[i]);
2937
2938 /* Remember the preference of the uses that followed. */
2939 if (i < ARRAY_SIZE(op->output_pref)) {
2940 op->output_pref[i] = *la_temp_pref(ts);
2941 }
2942
2943 /* Output args are dead. */
2944 if (ts->state & TS_DEAD) {
2945 arg_life |= DEAD_ARG << i;
2946 }
2947 if (ts->state & TS_MEM) {
2948 arg_life |= SYNC_ARG << i;
2949 }
2950 ts->state = TS_DEAD;
2951 la_reset_pref(ts);
2952 }
2953
2954 /* If end of basic block, update. */
2955 if (def->flags & TCG_OPF_BB_EXIT) {
2956 la_func_end(s, nb_globals, nb_temps);
2957 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2958 la_bb_sync(s, nb_globals, nb_temps);
2959 } else if (def->flags & TCG_OPF_BB_END) {
2960 la_bb_end(s, nb_globals, nb_temps);
2961 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2962 la_global_sync(s, nb_globals);
2963 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2964 la_cross_call(s, nb_temps);
2965 }
2966 }
2967
2968 /* Record arguments that die in this opcode. */
2969 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2970 ts = arg_temp(op->args[i]);
2971 if (ts->state & TS_DEAD) {
2972 arg_life |= DEAD_ARG << i;
2973 }
2974 }
2975
2976 /* Input arguments are live for preceding opcodes. */
2977 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2978 ts = arg_temp(op->args[i]);
2979 if (ts->state & TS_DEAD) {
2980 /* For operands that were dead, initially allow
2981 all regs for the type. */
2982 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2983 ts->state &= ~TS_DEAD;
2984 }
2985 }
2986
2987 /* Incorporate constraints for this operand. */
2988 switch (opc) {
2989 case INDEX_op_mov_i32:
2990 case INDEX_op_mov_i64:
2991 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2992 have proper constraints. That said, special case
2993 moves to propagate preferences backward. */
2994 if (IS_DEAD_ARG(1)) {
2995 *la_temp_pref(arg_temp(op->args[0]))
2996 = *la_temp_pref(arg_temp(op->args[1]));
2997 }
2998 break;
2999
3000 default:
3001 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3002 const TCGArgConstraint *ct = &def->args_ct[i];
3003 TCGRegSet set, *pset;
3004
3005 ts = arg_temp(op->args[i]);
3006 pset = la_temp_pref(ts);
3007 set = *pset;
3008
3009 set &= ct->regs;
3010 if (ct->ialias) {
3011 set &= output_pref(op, ct->alias_index);
3012 }
3013 /* If the combination is not possible, restart. */
3014 if (set == 0) {
3015 set = ct->regs;
3016 }
3017 *pset = set;
3018 }
3019 break;
3020 }
3021 break;
3022 }
3023 op->life = arg_life;
3024 }
3025 }
3026
3027 /* Liveness analysis: Convert indirect regs to direct temporaries. */
3028 static bool liveness_pass_2(TCGContext *s)
3029 {
3030 int nb_globals = s->nb_globals;
3031 int nb_temps, i;
3032 bool changes = false;
3033 TCGOp *op, *op_next;
3034
3035 /* Create a temporary for each indirect global. */
3036 for (i = 0; i < nb_globals; ++i) {
3037 TCGTemp *its = &s->temps[i];
3038 if (its->indirect_reg) {
3039 TCGTemp *dts = tcg_temp_alloc(s);
3040 dts->type = its->type;
3041 dts->base_type = its->base_type;
3042 dts->kind = TEMP_EBB;
3043 its->state_ptr = dts;
3044 } else {
3045 its->state_ptr = NULL;
3046 }
3047 /* All globals begin dead. */
3048 its->state = TS_DEAD;
3049 }
3050 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3051 TCGTemp *its = &s->temps[i];
3052 its->state_ptr = NULL;
3053 its->state = TS_DEAD;
3054 }
3055
3056 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3057 TCGOpcode opc = op->opc;
3058 const TCGOpDef *def = &tcg_op_defs[opc];
3059 TCGLifeData arg_life = op->life;
3060 int nb_iargs, nb_oargs, call_flags;
3061 TCGTemp *arg_ts, *dir_ts;
3062
3063 if (opc == INDEX_op_call) {
3064 nb_oargs = TCGOP_CALLO(op);
3065 nb_iargs = TCGOP_CALLI(op);
3066 call_flags = tcg_call_flags(op);
3067 } else {
3068 nb_iargs = def->nb_iargs;
3069 nb_oargs = def->nb_oargs;
3070
3071 /* Set flags similar to how calls require. */
3072 if (def->flags & TCG_OPF_COND_BRANCH) {
3073 /* Like reading globals: sync_globals */
3074 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3075 } else if (def->flags & TCG_OPF_BB_END) {
3076 /* Like writing globals: save_globals */
3077 call_flags = 0;
3078 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3079 /* Like reading globals: sync_globals */
3080 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3081 } else {
3082 /* No effect on globals. */
3083 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3084 TCG_CALL_NO_WRITE_GLOBALS);
3085 }
3086 }
3087
3088 /* Make sure that input arguments are available. */
3089 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3090 arg_ts = arg_temp(op->args[i]);
3091 dir_ts = arg_ts->state_ptr;
3092 if (dir_ts && arg_ts->state == TS_DEAD) {
3093 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3094 ? INDEX_op_ld_i32
3095 : INDEX_op_ld_i64);
3096 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
3097
3098 lop->args[0] = temp_arg(dir_ts);
3099 lop->args[1] = temp_arg(arg_ts->mem_base);
3100 lop->args[2] = arg_ts->mem_offset;
3101
3102 /* Loaded, but synced with memory. */
3103 arg_ts->state = TS_MEM;
3104 }
3105 }
3106
3107 /* Perform input replacement, and mark inputs that became dead.
3108 No action is required except keeping temp_state up to date
3109 so that we reload when needed. */
3110 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3111 arg_ts = arg_temp(op->args[i]);
3112 dir_ts = arg_ts->state_ptr;
3113 if (dir_ts) {
3114 op->args[i] = temp_arg(dir_ts);
3115 changes = true;
3116 if (IS_DEAD_ARG(i)) {
3117 arg_ts->state = TS_DEAD;
3118 }
3119 }
3120 }
3121
3122 /* Liveness analysis should ensure that the following are
3123 all correct, for call sites and basic block end points. */
3124 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
3125 /* Nothing to do */
3126 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
3127 for (i = 0; i < nb_globals; ++i) {
3128 /* Liveness should see that globals are synced back,
3129 that is, either TS_DEAD or TS_MEM. */
3130 arg_ts = &s->temps[i];
3131 tcg_debug_assert(arg_ts->state_ptr == 0
3132 || arg_ts->state != 0);
3133 }
3134 } else {
3135 for (i = 0; i < nb_globals; ++i) {
3136 /* Liveness should see that globals are saved back,
3137 that is, TS_DEAD, waiting to be reloaded. */
3138 arg_ts = &s->temps[i];
3139 tcg_debug_assert(arg_ts->state_ptr == 0
3140 || arg_ts->state == TS_DEAD);
3141 }
3142 }
3143
3144 /* Outputs become available. */
3145 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
3146 arg_ts = arg_temp(op->args[0]);
3147 dir_ts = arg_ts->state_ptr;
3148 if (dir_ts) {
3149 op->args[0] = temp_arg(dir_ts);
3150 changes = true;
3151
3152 /* The output is now live and modified. */
3153 arg_ts->state = 0;
3154
3155 if (NEED_SYNC_ARG(0)) {
3156 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3157 ? INDEX_op_st_i32
3158 : INDEX_op_st_i64);
3159 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
3160 TCGTemp *out_ts = dir_ts;
3161
3162 if (IS_DEAD_ARG(0)) {
3163 out_ts = arg_temp(op->args[1]);
3164 arg_ts->state = TS_DEAD;
3165 tcg_op_remove(s, op);
3166 } else {
3167 arg_ts->state = TS_MEM;
3168 }
3169
3170 sop->args[0] = temp_arg(out_ts);
3171 sop->args[1] = temp_arg(arg_ts->mem_base);
3172 sop->args[2] = arg_ts->mem_offset;
3173 } else {
3174 tcg_debug_assert(!IS_DEAD_ARG(0));
3175 }
3176 }
3177 } else {
3178 for (i = 0; i < nb_oargs; i++) {
3179 arg_ts = arg_temp(op->args[i]);
3180 dir_ts = arg_ts->state_ptr;
3181 if (!dir_ts) {
3182 continue;
3183 }
3184 op->args[i] = temp_arg(dir_ts);
3185 changes = true;
3186
3187 /* The output is now live and modified. */
3188 arg_ts->state = 0;
3189
3190 /* Sync outputs upon their last write. */
3191 if (NEED_SYNC_ARG(i)) {
3192 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3193 ? INDEX_op_st_i32
3194 : INDEX_op_st_i64);
3195 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
3196
3197 sop->args[0] = temp_arg(dir_ts);
3198 sop->args[1] = temp_arg(arg_ts->mem_base);
3199 sop->args[2] = arg_ts->mem_offset;
3200
3201 arg_ts->state = TS_MEM;
3202 }
3203 /* Drop outputs that are dead. */
3204 if (IS_DEAD_ARG(i)) {
3205 arg_ts->state = TS_DEAD;
3206 }
3207 }
3208 }
3209 }
3210
3211 return changes;
3212 }
3213
3214 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
3215 {
3216 int size = tcg_type_size(ts->type);
3217 int align;
3218 intptr_t off;
3219
3220 switch (ts->type) {
3221 case TCG_TYPE_I32:
3222 align = 4;
3223 break;
3224 case TCG_TYPE_I64:
3225 case TCG_TYPE_V64:
3226 align = 8;
3227 break;
3228 case TCG_TYPE_V128:
3229 case TCG_TYPE_V256:
3230 /* Note that we do not require aligned storage for V256. */
3231 align = 16;
3232 break;
3233 default:
3234 g_assert_not_reached();
3235 }
3236
3237 /*
3238 * Assume the stack is sufficiently aligned.
3239 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3240 * and do not require 16 byte vector alignment. This seems slightly
3241 * easier than fully parameterizing the above switch statement.
3242 */
3243 align = MIN(TCG_TARGET_STACK_ALIGN, align);
3244 off = ROUND_UP(s->current_frame_offset, align);
3245
3246 /* If we've exhausted the stack frame, restart with a smaller TB. */
3247 if (off + size > s->frame_end) {
3248 tcg_raise_tb_overflow(s);
3249 }
3250 s->current_frame_offset = off + size;
3251
3252 ts->mem_offset = off;
3253 #if defined(__sparc__)
3254 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3255 #endif
3256 ts->mem_base = s->frame_temp;
3257 ts->mem_allocated = 1;
3258 }
3259
3260 /* Assign @reg to @ts, and update reg_to_temp[]. */
3261 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
3262 {
3263 if (ts->val_type == TEMP_VAL_REG) {
3264 TCGReg old = ts->reg;
3265 tcg_debug_assert(s->reg_to_temp[old] == ts);
3266 if (old == reg) {
3267 return;
3268 }
3269 s->reg_to_temp[old] = NULL;
3270 }
3271 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3272 s->reg_to_temp[reg] = ts;
3273 ts->val_type = TEMP_VAL_REG;
3274 ts->reg = reg;
3275 }
3276
3277 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3278 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
3279 {
3280 tcg_debug_assert(type != TEMP_VAL_REG);
3281 if (ts->val_type == TEMP_VAL_REG) {
3282 TCGReg reg = ts->reg;
3283 tcg_debug_assert(s->reg_to_temp[reg] == ts);
3284 s->reg_to_temp[reg] = NULL;
3285 }
3286 ts->val_type = type;
3287 }
3288
3289 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
3290
3291 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3292 mark it free; otherwise mark it dead. */
3293 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
3294 {
3295 TCGTempVal new_type;
3296
3297 switch (ts->kind) {
3298 case TEMP_FIXED:
3299 return;
3300 case TEMP_GLOBAL:
3301 case TEMP_LOCAL:
3302 new_type = TEMP_VAL_MEM;
3303 break;
3304 case TEMP_NORMAL:
3305 case TEMP_EBB:
3306 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3307 break;
3308 case TEMP_CONST:
3309 new_type = TEMP_VAL_CONST;
3310 break;
3311 default:
3312 g_assert_not_reached();
3313 }
3314 set_temp_val_nonreg(s, ts, new_type);
3315 }
3316
3317 /* Mark a temporary as dead. */
3318 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3319 {
3320 temp_free_or_dead(s, ts, 1);
3321 }
3322
3323 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3324 registers needs to be allocated to store a constant. If 'free_or_dead'
3325 is non-zero, subsequently release the temporary; if it is positive, the
3326 temp is dead; if it is negative, the temp is free. */
3327 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3328 TCGRegSet preferred_regs, int free_or_dead)
3329 {
3330 if (!temp_readonly(ts) && !ts->mem_coherent) {
3331 if (!ts->mem_allocated) {
3332 temp_allocate_frame(s, ts);
3333 }
3334 switch (ts->val_type) {
3335 case TEMP_VAL_CONST:
3336 /* If we're going to free the temp immediately, then we won't
3337 require it later in a register, so attempt to store the
3338 constant to memory directly. */
3339 if (free_or_dead
3340 && tcg_out_sti(s, ts->type, ts->val,
3341 ts->mem_base->reg, ts->mem_offset)) {
3342 break;
3343 }
3344 temp_load(s, ts, tcg_target_available_regs[ts->type],
3345 allocated_regs, preferred_regs);
3346 /* fallthrough */
3347
3348 case TEMP_VAL_REG:
3349 tcg_out_st(s, ts->type, ts->reg,
3350 ts->mem_base->reg, ts->mem_offset);
3351 break;
3352
3353 case TEMP_VAL_MEM:
3354 break;
3355
3356 case TEMP_VAL_DEAD:
3357 default:
3358 tcg_abort();
3359 }
3360 ts->mem_coherent = 1;
3361 }
3362 if (free_or_dead) {
3363 temp_free_or_dead(s, ts, free_or_dead);
3364 }
3365 }
3366
3367 /* free register 'reg' by spilling the corresponding temporary if necessary */
3368 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
3369 {
3370 TCGTemp *ts = s->reg_to_temp[reg];
3371 if (ts != NULL) {
3372 temp_sync(s, ts, allocated_regs, 0, -1);
3373 }
3374 }
3375
3376 /**
3377 * tcg_reg_alloc:
3378 * @required_regs: Set of registers in which we must allocate.
3379 * @allocated_regs: Set of registers which must be avoided.
3380 * @preferred_regs: Set of registers we should prefer.
3381 * @rev: True if we search the registers in "indirect" order.
3382 *
3383 * The allocated register must be in @required_regs & ~@allocated_regs,
3384 * but if we can put it in @preferred_regs we may save a move later.
3385 */
3386 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3387 TCGRegSet allocated_regs,
3388 TCGRegSet preferred_regs, bool rev)
3389 {
3390 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3391 TCGRegSet reg_ct[2];
3392 const int *order;
3393
3394 reg_ct[1] = required_regs & ~allocated_regs;
3395 tcg_debug_assert(reg_ct[1] != 0);
3396 reg_ct[0] = reg_ct[1] & preferred_regs;
3397
3398 /* Skip the preferred_regs option if it cannot be satisfied,
3399 or if the preference made no difference. */
3400 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3401
3402 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3403
3404 /* Try free registers, preferences first. */
3405 for (j = f; j < 2; j++) {
3406 TCGRegSet set = reg_ct[j];
3407
3408 if (tcg_regset_single(set)) {
3409 /* One register in the set. */
3410 TCGReg reg = tcg_regset_first(set);
3411 if (s->reg_to_temp[reg] == NULL) {
3412 return reg;
3413 }
3414 } else {
3415 for (i = 0; i < n; i++) {
3416 TCGReg reg = order[i];
3417 if (s->reg_to_temp[reg] == NULL &&
3418 tcg_regset_test_reg(set, reg)) {
3419 return reg;
3420 }
3421 }
3422 }
3423 }
3424
3425 /* We must spill something. */
3426 for (j = f; j < 2; j++) {
3427 TCGRegSet set = reg_ct[j];
3428
3429 if (tcg_regset_single(set)) {
3430 /* One register in the set. */
3431 TCGReg reg = tcg_regset_first(set);
3432 tcg_reg_free(s, reg, allocated_regs);
3433 return reg;
3434 } else {
3435 for (i = 0; i < n; i++) {
3436 TCGReg reg = order[i];
3437 if (tcg_regset_test_reg(set, reg)) {
3438 tcg_reg_free(s, reg, allocated_regs);
3439 return reg;
3440 }
3441 }
3442 }
3443 }
3444
3445 tcg_abort();
3446 }
3447
3448 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
3449 TCGRegSet allocated_regs,
3450 TCGRegSet preferred_regs, bool rev)
3451 {
3452 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3453 TCGRegSet reg_ct[2];
3454 const int *order;
3455
3456 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3457 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
3458 tcg_debug_assert(reg_ct[1] != 0);
3459 reg_ct[0] = reg_ct[1] & preferred_regs;
3460
3461 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3462
3463 /*
3464 * Skip the preferred_regs option if it cannot be satisfied,
3465 * or if the preference made no difference.
3466 */
3467 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3468
3469 /*
3470 * Minimize the number of flushes by looking for 2 free registers first,
3471 * then a single flush, then two flushes.
3472 */
3473 for (fmin = 2; fmin >= 0; fmin--) {
3474 for (j = k; j < 2; j++) {
3475 TCGRegSet set = reg_ct[j];
3476
3477 for (i = 0; i < n; i++) {
3478 TCGReg reg = order[i];
3479
3480 if (tcg_regset_test_reg(set, reg)) {
3481 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
3482 if (f >= fmin) {
3483 tcg_reg_free(s, reg, allocated_regs);
3484 tcg_reg_free(s, reg + 1, allocated_regs);
3485 return reg;
3486 }
3487 }
3488 }
3489 }
3490 }
3491 tcg_abort();
3492 }
3493
3494 /* Make sure the temporary is in a register. If needed, allocate the register
3495 from DESIRED while avoiding ALLOCATED. */
3496 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
3497 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
3498 {
3499 TCGReg reg;
3500
3501 switch (ts->val_type) {
3502 case TEMP_VAL_REG:
3503 return;
3504 case TEMP_VAL_CONST:
3505 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3506 preferred_regs, ts->indirect_base);
3507 if (ts->type <= TCG_TYPE_I64) {
3508 tcg_out_movi(s, ts->type, reg, ts->val);
3509 } else {
3510 uint64_t val = ts->val;
3511 MemOp vece = MO_64;
3512
3513 /*
3514 * Find the minimal vector element that matches the constant.
3515 * The targets will, in general, have to do this search anyway,
3516 * do this generically.
3517 */
3518 if (val == dup_const(MO_8, val)) {
3519 vece = MO_8;
3520 } else if (val == dup_const(MO_16, val)) {
3521 vece = MO_16;
3522 } else if (val == dup_const(MO_32, val)) {
3523 vece = MO_32;
3524 }
3525
3526 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
3527 }
3528 ts->mem_coherent = 0;
3529 break;
3530 case TEMP_VAL_MEM:
3531 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3532 preferred_regs, ts->indirect_base);
3533 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3534 ts->mem_coherent = 1;
3535 break;
3536 case TEMP_VAL_DEAD:
3537 default:
3538 tcg_abort();
3539 }
3540 set_temp_val_reg(s, ts, reg);
3541 }
3542
3543 /* Save a temporary to memory. 'allocated_regs' is used in case a
3544 temporary registers needs to be allocated to store a constant. */
3545 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
3546 {
3547 /* The liveness analysis already ensures that globals are back
3548 in memory. Keep an tcg_debug_assert for safety. */
3549 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
3550 }
3551
3552 /* save globals to their canonical location and assume they can be
3553 modified be the following code. 'allocated_regs' is used in case a
3554 temporary registers needs to be allocated to store a constant. */
3555 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
3556 {
3557 int i, n;
3558
3559 for (i = 0, n = s->nb_globals; i < n; i++) {
3560 temp_save(s, &s->temps[i], allocated_regs);
3561 }
3562 }
3563
3564 /* sync globals to their canonical location and assume they can be
3565 read by the following code. 'allocated_regs' is used in case a
3566 temporary registers needs to be allocated to store a constant. */
3567 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3568 {
3569 int i, n;
3570
3571 for (i = 0, n = s->nb_globals; i < n; i++) {
3572 TCGTemp *ts = &s->temps[i];
3573 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
3574 || ts->kind == TEMP_FIXED
3575 || ts->mem_coherent);
3576 }
3577 }
3578
3579 /* at the end of a basic block, we assume all temporaries are dead and
3580 all globals are stored at their canonical location. */
3581 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
3582 {
3583 int i;
3584
3585 for (i = s->nb_globals; i < s->nb_temps; i++) {
3586 TCGTemp *ts = &s->temps[i];
3587
3588 switch (ts->kind) {
3589 case TEMP_LOCAL:
3590 temp_save(s, ts, allocated_regs);
3591 break;
3592 case TEMP_NORMAL:
3593 case TEMP_EBB:
3594 /* The liveness analysis already ensures that temps are dead.
3595 Keep an tcg_debug_assert for safety. */
3596 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3597 break;
3598 case TEMP_CONST:
3599 /* Similarly, we should have freed any allocated register. */
3600 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3601 break;
3602 default:
3603 g_assert_not_reached();
3604 }
3605 }
3606
3607 save_globals(s, allocated_regs);
3608 }
3609
3610 /*
3611 * At a conditional branch, we assume all temporaries are dead unless
3612 * explicitly live-across-conditional-branch; all globals and local
3613 * temps are synced to their location.
3614 */
3615 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3616 {
3617 sync_globals(s, allocated_regs);
3618
3619 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3620 TCGTemp *ts = &s->temps[i];
3621 /*
3622 * The liveness analysis already ensures that temps are dead.
3623 * Keep tcg_debug_asserts for safety.
3624 */
3625 switch (ts->kind) {
3626 case TEMP_LOCAL:
3627 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
3628 break;
3629 case TEMP_NORMAL:
3630 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3631 break;
3632 case TEMP_EBB:
3633 case TEMP_CONST:
3634 break;
3635 default:
3636 g_assert_not_reached();
3637 }
3638 }
3639 }
3640
3641 /*
3642 * Specialized code generation for INDEX_op_mov_* with a constant.
3643 */
3644 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
3645 tcg_target_ulong val, TCGLifeData arg_life,
3646 TCGRegSet preferred_regs)
3647 {
3648 /* ENV should not be modified. */
3649 tcg_debug_assert(!temp_readonly(ots));
3650
3651 /* The movi is not explicitly generated here. */
3652 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
3653 ots->val = val;
3654 ots->mem_coherent = 0;
3655 if (NEED_SYNC_ARG(0)) {
3656 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
3657 } else if (IS_DEAD_ARG(0)) {
3658 temp_dead(s, ots);
3659 }
3660 }
3661
3662 /*
3663 * Specialized code generation for INDEX_op_mov_*.
3664 */
3665 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
3666 {
3667 const TCGLifeData arg_life = op->life;
3668 TCGRegSet allocated_regs, preferred_regs;
3669 TCGTemp *ts, *ots;
3670 TCGType otype, itype;
3671 TCGReg oreg, ireg;
3672
3673 allocated_regs = s->reserved_regs;
3674 preferred_regs = output_pref(op, 0);
3675 ots = arg_temp(op->args[0]);
3676 ts = arg_temp(op->args[1]);
3677
3678 /* ENV should not be modified. */
3679 tcg_debug_assert(!temp_readonly(ots));
3680
3681 /* Note that otype != itype for no-op truncation. */
3682 otype = ots->type;
3683 itype = ts->type;
3684
3685 if (ts->val_type == TEMP_VAL_CONST) {
3686 /* propagate constant or generate sti */
3687 tcg_target_ulong val = ts->val;
3688 if (IS_DEAD_ARG(1)) {
3689 temp_dead(s, ts);
3690 }
3691 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
3692 return;
3693 }
3694
3695 /* If the source value is in memory we're going to be forced
3696 to have it in a register in order to perform the copy. Copy
3697 the SOURCE value into its own register first, that way we
3698 don't have to reload SOURCE the next time it is used. */
3699 if (ts->val_type == TEMP_VAL_MEM) {
3700 temp_load(s, ts, tcg_target_available_regs[itype],
3701 allocated_regs, preferred_regs);
3702 }
3703 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
3704 ireg = ts->reg;
3705
3706 if (IS_DEAD_ARG(0)) {
3707 /* mov to a non-saved dead register makes no sense (even with
3708 liveness analysis disabled). */
3709 tcg_debug_assert(NEED_SYNC_ARG(0));
3710 if (!ots->mem_allocated) {
3711 temp_allocate_frame(s, ots);
3712 }
3713 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
3714 if (IS_DEAD_ARG(1)) {
3715 temp_dead(s, ts);
3716 }
3717 temp_dead(s, ots);
3718 return;
3719 }
3720
3721 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3722 /*
3723 * The mov can be suppressed. Kill input first, so that it
3724 * is unlinked from reg_to_temp, then set the output to the
3725 * reg that we saved from the input.
3726 */
3727 temp_dead(s, ts);
3728 oreg = ireg;
3729 } else {
3730 if (ots->val_type == TEMP_VAL_REG) {
3731 oreg = ots->reg;
3732 } else {
3733 /* Make sure to not spill the input register during allocation. */
3734 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3735 allocated_regs | ((TCGRegSet)1 << ireg),
3736 preferred_regs, ots->indirect_base);
3737 }
3738 if (!tcg_out_mov(s, otype, oreg, ireg)) {
3739 /*
3740 * Cross register class move not supported.
3741 * Store the source register into the destination slot
3742 * and leave the destination temp as TEMP_VAL_MEM.
3743 */
3744 assert(!temp_readonly(ots));
3745 if (!ts->mem_allocated) {
3746 temp_allocate_frame(s, ots);
3747 }
3748 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
3749 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
3750 ots->mem_coherent = 1;
3751 return;
3752 }
3753 }
3754 set_temp_val_reg(s, ots, oreg);
3755 ots->mem_coherent = 0;
3756
3757 if (NEED_SYNC_ARG(0)) {
3758 temp_sync(s, ots, allocated_regs, 0, 0);
3759 }
3760 }
3761
3762 /*
3763 * Specialized code generation for INDEX_op_dup_vec.
3764 */
3765 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3766 {
3767 const TCGLifeData arg_life = op->life;
3768 TCGRegSet dup_out_regs, dup_in_regs;
3769 TCGTemp *its, *ots;
3770 TCGType itype, vtype;
3771 unsigned vece;
3772 int lowpart_ofs;
3773 bool ok;
3774
3775 ots = arg_temp(op->args[0]);
3776 its = arg_temp(op->args[1]);
3777
3778 /* ENV should not be modified. */
3779 tcg_debug_assert(!temp_readonly(ots));
3780
3781 itype = its->type;
3782 vece = TCGOP_VECE(op);
3783 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3784
3785 if (its->val_type == TEMP_VAL_CONST) {
3786 /* Propagate constant via movi -> dupi. */
3787 tcg_target_ulong val = its->val;
3788 if (IS_DEAD_ARG(1)) {
3789 temp_dead(s, its);
3790 }
3791 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
3792 return;
3793 }
3794
3795 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3796 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
3797
3798 /* Allocate the output register now. */
3799 if (ots->val_type != TEMP_VAL_REG) {
3800 TCGRegSet allocated_regs = s->reserved_regs;
3801 TCGReg oreg;
3802
3803 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3804 /* Make sure to not spill the input register. */
3805 tcg_regset_set_reg(allocated_regs, its->reg);
3806 }
3807 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3808 output_pref(op, 0), ots->indirect_base);
3809 set_temp_val_reg(s, ots, oreg);
3810 }
3811
3812 switch (its->val_type) {
3813 case TEMP_VAL_REG:
3814 /*
3815 * The dup constriaints must be broad, covering all possible VECE.
3816 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3817 * to fail, indicating that extra moves are required for that case.
3818 */
3819 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3820 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3821 goto done;
3822 }
3823 /* Try again from memory or a vector input register. */
3824 }
3825 if (!its->mem_coherent) {
3826 /*
3827 * The input register is not synced, and so an extra store
3828 * would be required to use memory. Attempt an integer-vector
3829 * register move first. We do not have a TCGRegSet for this.
3830 */
3831 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3832 break;
3833 }
3834 /* Sync the temp back to its slot and load from there. */
3835 temp_sync(s, its, s->reserved_regs, 0, 0);
3836 }
3837 /* fall through */
3838
3839 case TEMP_VAL_MEM:
3840 lowpart_ofs = 0;
3841 if (HOST_BIG_ENDIAN) {
3842 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
3843 }
3844 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3845 its->mem_offset + lowpart_ofs)) {
3846 goto done;
3847 }
3848 /* Load the input into the destination vector register. */
3849 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3850 break;
3851
3852 default:
3853 g_assert_not_reached();
3854 }
3855
3856 /* We now have a vector input register, so dup must succeed. */
3857 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3858 tcg_debug_assert(ok);
3859
3860 done:
3861 ots->mem_coherent = 0;
3862 if (IS_DEAD_ARG(1)) {
3863 temp_dead(s, its);
3864 }
3865 if (NEED_SYNC_ARG(0)) {
3866 temp_sync(s, ots, s->reserved_regs, 0, 0);
3867 }
3868 if (IS_DEAD_ARG(0)) {
3869 temp_dead(s, ots);
3870 }
3871 }
3872
3873 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
3874 {
3875 const TCGLifeData arg_life = op->life;
3876 const TCGOpDef * const def = &tcg_op_defs[op->opc];
3877 TCGRegSet i_allocated_regs;
3878 TCGRegSet o_allocated_regs;
3879 int i, k, nb_iargs, nb_oargs;
3880 TCGReg reg;
3881 TCGArg arg;
3882 const TCGArgConstraint *arg_ct;
3883 TCGTemp *ts;
3884 TCGArg new_args[TCG_MAX_OP_ARGS];
3885 int const_args[TCG_MAX_OP_ARGS];
3886
3887 nb_oargs = def->nb_oargs;
3888 nb_iargs = def->nb_iargs;
3889
3890 /* copy constants */
3891 memcpy(new_args + nb_oargs + nb_iargs,
3892 op->args + nb_oargs + nb_iargs,
3893 sizeof(TCGArg) * def->nb_cargs);
3894
3895 i_allocated_regs = s->reserved_regs;
3896 o_allocated_regs = s->reserved_regs;
3897
3898 /* satisfy input constraints */
3899 for (k = 0; k < nb_iargs; k++) {
3900 TCGRegSet i_preferred_regs, i_required_regs;
3901 bool allocate_new_reg, copyto_new_reg;
3902 TCGTemp *ts2;
3903 int i1, i2;
3904
3905 i = def->args_ct[nb_oargs + k].sort_index;
3906 arg = op->args[i];
3907 arg_ct = &def->args_ct[i];
3908 ts = arg_temp(arg);
3909
3910 if (ts->val_type == TEMP_VAL_CONST
3911 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
3912 /* constant is OK for instruction */
3913 const_args[i] = 1;
3914 new_args[i] = ts->val;
3915 continue;
3916 }
3917
3918 reg = ts->reg;
3919 i_preferred_regs = 0;
3920 i_required_regs = arg_ct->regs;
3921 allocate_new_reg = false;
3922 copyto_new_reg = false;
3923
3924 switch (arg_ct->pair) {
3925 case 0: /* not paired */
3926 if (arg_ct->ialias) {
3927 i_preferred_regs = output_pref(op, arg_ct->alias_index);
3928
3929 /*
3930 * If the input is readonly, then it cannot also be an
3931 * output and aliased to itself. If the input is not
3932 * dead after the instruction, we must allocate a new
3933 * register and move it.
3934 */
3935 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3936 allocate_new_reg = true;
3937 } else if (ts->val_type == TEMP_VAL_REG) {
3938 /*
3939 * Check if the current register has already been
3940 * allocated for another input.
3941 */
3942 allocate_new_reg =
3943 tcg_regset_test_reg(i_allocated_regs, reg);
3944 }
3945 }
3946 if (!allocate_new_reg) {
3947 temp_load(s, ts, i_required_regs, i_allocated_regs,
3948 i_preferred_regs);
3949 reg = ts->reg;
3950 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
3951 }
3952 if (allocate_new_reg) {
3953 /*
3954 * Allocate a new register matching the constraint
3955 * and move the temporary register into it.
3956 */
3957 temp_load(s, ts, tcg_target_available_regs[ts->type],
3958 i_allocated_regs, 0);
3959 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
3960 i_preferred_regs, ts->indirect_base);
3961 copyto_new_reg = true;
3962 }
3963 break;
3964
3965 case 1:
3966 /* First of an input pair; if i1 == i2, the second is an output. */
3967 i1 = i;
3968 i2 = arg_ct->pair_index;
3969 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
3970
3971 /*
3972 * It is easier to default to allocating a new pair
3973 * and to identify a few cases where it's not required.
3974 */
3975 if (arg_ct->ialias) {
3976 i_preferred_regs = output_pref(op, arg_ct->alias_index);
3977 if (IS_DEAD_ARG(i1) &&
3978 IS_DEAD_ARG(i2) &&
3979 !temp_readonly(ts) &&
3980 ts->val_type == TEMP_VAL_REG &&
3981 ts->reg < TCG_TARGET_NB_REGS - 1 &&
3982 tcg_regset_test_reg(i_required_regs, reg) &&
3983 !tcg_regset_test_reg(i_allocated_regs, reg) &&
3984 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
3985 (ts2
3986 ? ts2->val_type == TEMP_VAL_REG &&
3987 ts2->reg == reg + 1 &&
3988 !temp_readonly(ts2)
3989 : s->reg_to_temp[reg + 1] == NULL)) {
3990 break;
3991 }
3992 } else {
3993 /* Without aliasing, the pair must also be an input. */
3994 tcg_debug_assert(ts2);
3995 if (ts->val_type == TEMP_VAL_REG &&
3996 ts2->val_type == TEMP_VAL_REG &&
3997 ts2->reg == reg + 1 &&
3998 tcg_regset_test_reg(i_required_regs, reg)) {
3999 break;
4000 }
4001 }
4002 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4003 0, ts->indirect_base);
4004 goto do_pair;
4005
4006 case 2: /* pair second */
4007 reg = new_args[arg_ct->pair_index] + 1;
4008 goto do_pair;
4009
4010 case 3: /* ialias with second output, no first input */
4011 tcg_debug_assert(arg_ct->ialias);
4012 i_preferred_regs = output_pref(op, arg_ct->alias_index);
4013
4014 if (IS_DEAD_ARG(i) &&
4015 !temp_readonly(ts) &&
4016 ts->val_type == TEMP_VAL_REG &&
4017 reg > 0 &&
4018 s->reg_to_temp[reg - 1] == NULL &&
4019 tcg_regset_test_reg(i_required_regs, reg) &&
4020 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4021 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4022 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4023 break;
4024 }
4025 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4026 i_allocated_regs, 0,
4027 ts->indirect_base);
4028 tcg_regset_set_reg(i_allocated_regs, reg);
4029 reg += 1;
4030 goto do_pair;
4031
4032 do_pair:
4033 /*
4034 * If an aliased input is not dead after the instruction,
4035 * we must allocate a new register and move it.
4036 */
4037 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4038 TCGRegSet t_allocated_regs = i_allocated_regs;
4039
4040 /*
4041 * Because of the alias, and the continued life, make sure
4042 * that the temp is somewhere *other* than the reg pair,
4043 * and we get a copy in reg.
4044 */
4045 tcg_regset_set_reg(t_allocated_regs, reg);
4046 tcg_regset_set_reg(t_allocated_regs, reg + 1);
4047 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
4048 /* If ts was already in reg, copy it somewhere else. */
4049 TCGReg nr;
4050 bool ok;
4051
4052 tcg_debug_assert(ts->kind != TEMP_FIXED);
4053 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
4054 t_allocated_regs, 0, ts->indirect_base);
4055 ok = tcg_out_mov(s, ts->type, nr, reg);
4056 tcg_debug_assert(ok);
4057
4058 set_temp_val_reg(s, ts, nr);
4059 } else {
4060 temp_load(s, ts, tcg_target_available_regs[ts->type],
4061 t_allocated_regs, 0);
4062 copyto_new_reg = true;
4063 }
4064 } else {
4065 /* Preferably allocate to reg, otherwise copy. */
4066 i_required_regs = (TCGRegSet)1 << reg;
4067 temp_load(s, ts, i_required_regs, i_allocated_regs,
4068 i_preferred_regs);
4069 copyto_new_reg = ts->reg != reg;
4070 }
4071 break;
4072
4073 default:
4074 g_assert_not_reached();
4075 }
4076
4077 if (copyto_new_reg) {
4078 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4079 /*
4080 * Cross register class move not supported. Sync the
4081 * temp back to its slot and load from there.
4082 */
4083 temp_sync(s, ts, i_allocated_regs, 0, 0);
4084 tcg_out_ld(s, ts->type, reg,
4085 ts->mem_base->reg, ts->mem_offset);
4086 }
4087 }
4088 new_args[i] = reg;
4089 const_args[i] = 0;
4090 tcg_regset_set_reg(i_allocated_regs, reg);
4091 }
4092
4093 /* mark dead temporaries and free the associated registers */
4094 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4095 if (IS_DEAD_ARG(i)) {
4096 temp_dead(s, arg_temp(op->args[i]));
4097 }
4098 }
4099
4100 if (def->flags & TCG_OPF_COND_BRANCH) {
4101 tcg_reg_alloc_cbranch(s, i_allocated_regs);
4102 } else if (def->flags & TCG_OPF_BB_END) {
4103 tcg_reg_alloc_bb_end(s, i_allocated_regs);
4104 } else {
4105 if (def->flags & TCG_OPF_CALL_CLOBBER) {
4106 /* XXX: permit generic clobber register list ? */
4107 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4108 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
4109 tcg_reg_free(s, i, i_allocated_regs);
4110 }
4111 }
4112 }
4113 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4114 /* sync globals if the op has side effects and might trigger
4115 an exception. */
4116 sync_globals(s, i_allocated_regs);
4117 }
4118
4119 /* satisfy the output constraints */
4120 for(k = 0; k < nb_oargs; k++) {
4121 i = def->args_ct[k].sort_index;
4122 arg = op->args[i];
4123 arg_ct = &def->args_ct[i];
4124 ts = arg_temp(arg);
4125
4126 /* ENV should not be modified. */
4127 tcg_debug_assert(!temp_readonly(ts));
4128
4129 switch (arg_ct->pair) {
4130 case 0: /* not paired */
4131 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
4132 reg = new_args[arg_ct->alias_index];
4133 } else if (arg_ct->newreg) {
4134 reg = tcg_reg_alloc(s, arg_ct->regs,
4135 i_allocated_regs | o_allocated_regs,
4136 output_pref(op, k), ts->indirect_base);
4137 } else {
4138 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
4139 output_pref(op, k), ts->indirect_base);
4140 }
4141 break;
4142
4143 case 1: /* first of pair */
4144 tcg_debug_assert(!arg_ct->newreg);
4145 if (arg_ct->oalias) {
4146 reg = new_args[arg_ct->alias_index];
4147 break;
4148 }
4149 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
4150 output_pref(op, k), ts->indirect_base);
4151 break;
4152
4153 case 2: /* second of pair */
4154 tcg_debug_assert(!arg_ct->newreg);
4155 if (arg_ct->oalias) {
4156 reg = new_args[arg_ct->alias_index];
4157 } else {
4158 reg = new_args[arg_ct->pair_index] + 1;
4159 }
4160 break;
4161
4162 case 3: /* first of pair, aliasing with a second input */
4163 tcg_debug_assert(!arg_ct->newreg);
4164 reg = new_args[arg_ct->pair_index] - 1;
4165 break;
4166
4167 default:
4168 g_assert_not_reached();
4169 }
4170 tcg_regset_set_reg(o_allocated_regs, reg);
4171 set_temp_val_reg(s, ts, reg);
4172 ts->mem_coherent = 0;
4173 new_args[i] = reg;
4174 }
4175 }
4176
4177 /* emit instruction */
4178 if (def->flags & TCG_OPF_VECTOR) {
4179 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4180 new_args, const_args);
4181 } else {
4182 tcg_out_op(s, op->opc, new_args, const_args);
4183 }
4184
4185 /* move the outputs in the correct register if needed */
4186 for(i = 0; i < nb_oargs; i++) {
4187 ts = arg_temp(op->args[i]);
4188
4189 /* ENV should not be modified. */
4190 tcg_debug_assert(!temp_readonly(ts));
4191
4192 if (NEED_SYNC_ARG(i)) {
4193 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
4194 } else if (IS_DEAD_ARG(i)) {
4195 temp_dead(s, ts);
4196 }
4197 }
4198 }
4199
4200 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4201 {
4202 const TCGLifeData arg_life = op->life;
4203 TCGTemp *ots, *itsl, *itsh;
4204 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4205
4206 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4207 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4208 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4209
4210 ots = arg_temp(op->args[0]);
4211 itsl = arg_temp(op->args[1]);
4212 itsh = arg_temp(op->args[2]);
4213
4214 /* ENV should not be modified. */
4215 tcg_debug_assert(!temp_readonly(ots));
4216
4217 /* Allocate the output register now. */
4218 if (ots->val_type != TEMP_VAL_REG) {
4219 TCGRegSet allocated_regs = s->reserved_regs;
4220 TCGRegSet dup_out_regs =
4221 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4222 TCGReg oreg;
4223
4224 /* Make sure to not spill the input registers. */
4225 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4226 tcg_regset_set_reg(allocated_regs, itsl->reg);
4227 }
4228 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4229 tcg_regset_set_reg(allocated_regs, itsh->reg);
4230 }
4231
4232 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
4233 output_pref(op, 0), ots->indirect_base);
4234 set_temp_val_reg(s, ots, oreg);
4235 }
4236
4237 /* Promote dup2 of immediates to dupi_vec. */
4238 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4239 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4240 MemOp vece = MO_64;
4241
4242 if (val == dup_const(MO_8, val)) {
4243 vece = MO_8;
4244 } else if (val == dup_const(MO_16, val)) {
4245 vece = MO_16;
4246 } else if (val == dup_const(MO_32, val)) {
4247 vece = MO_32;
4248 }
4249
4250 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4251 goto done;
4252 }
4253
4254 /* If the two inputs form one 64-bit value, try dupm_vec. */
4255 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
4256 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
4257 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
4258 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
4259
4260 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
4261 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
4262
4263 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4264 its->mem_base->reg, its->mem_offset)) {
4265 goto done;
4266 }
4267 }
4268
4269 /* Fall back to generic expansion. */
4270 return false;
4271
4272 done:
4273 ots->mem_coherent = 0;
4274 if (IS_DEAD_ARG(1)) {
4275 temp_dead(s, itsl);
4276 }
4277 if (IS_DEAD_ARG(2)) {
4278 temp_dead(s, itsh);
4279 }
4280 if (NEED_SYNC_ARG(0)) {
4281 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4282 } else if (IS_DEAD_ARG(0)) {
4283 temp_dead(s, ots);
4284 }
4285 return true;
4286 }
4287
4288 static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
4289 TCGRegSet allocated_regs)
4290 {
4291 if (ts->val_type == TEMP_VAL_REG) {
4292 if (ts->reg != reg) {
4293 tcg_reg_free(s, reg, allocated_regs);
4294 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4295 /*
4296 * Cross register class move not supported. Sync the
4297 * temp back to its slot and load from there.
4298 */
4299 temp_sync(s, ts, allocated_regs, 0, 0);
4300 tcg_out_ld(s, ts->type, reg,
4301 ts->mem_base->reg, ts->mem_offset);
4302 }
4303 }
4304 } else {
4305 TCGRegSet arg_set = 0;
4306
4307 tcg_reg_free(s, reg, allocated_regs);
4308 tcg_regset_set_reg(arg_set, reg);
4309 temp_load(s, ts, arg_set, allocated_regs, 0);
4310 }
4311 }
4312
4313 static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts,
4314 TCGRegSet allocated_regs)
4315 {
4316 /*
4317 * When the destination is on the stack, load up the temp and store.
4318 * If there are many call-saved registers, the temp might live to
4319 * see another use; otherwise it'll be discarded.
4320 */
4321 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
4322 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
4323 TCG_TARGET_CALL_STACK_OFFSET +
4324 stk_slot * sizeof(tcg_target_long));
4325 }
4326
4327 static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
4328 TCGTemp *ts, TCGRegSet *allocated_regs)
4329 {
4330 if (REG_P(l)) {
4331 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
4332 load_arg_reg(s, reg, ts, *allocated_regs);
4333 tcg_regset_set_reg(*allocated_regs, reg);
4334 } else {
4335 load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs),
4336 ts, *allocated_regs);
4337 }
4338 }
4339
4340 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
4341 {
4342 const int nb_oargs = TCGOP_CALLO(op);
4343 const int nb_iargs = TCGOP_CALLI(op);
4344 const TCGLifeData arg_life = op->life;
4345 const TCGHelperInfo *info = tcg_call_info(op);
4346 TCGRegSet allocated_regs = s->reserved_regs;
4347 int i;
4348
4349 /*
4350 * Move inputs into place in reverse order,
4351 * so that we place stacked arguments first.
4352 */
4353 for (i = nb_iargs - 1; i >= 0; --i) {
4354 const TCGCallArgumentLoc *loc = &info->in[i];
4355 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
4356
4357 switch (loc->kind) {
4358 case TCG_CALL_ARG_NORMAL:
4359 case TCG_CALL_ARG_EXTEND_U:
4360 case TCG_CALL_ARG_EXTEND_S:
4361 load_arg_normal(s, loc, ts, &allocated_regs);
4362 break;
4363 default:
4364 g_assert_not_reached();
4365 }
4366 }
4367
4368 /* Mark dead temporaries and free the associated registers. */
4369 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4370 if (IS_DEAD_ARG(i)) {
4371 temp_dead(s, arg_temp(op->args[i]));
4372 }
4373 }
4374
4375 /* Clobber call registers. */
4376 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4377 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
4378 tcg_reg_free(s, i, allocated_regs);
4379 }
4380 }
4381
4382 /*
4383 * Save globals if they might be written by the helper,
4384 * sync them if they might be read.
4385 */
4386 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
4387 /* Nothing to do */
4388 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
4389 sync_globals(s, allocated_regs);
4390 } else {
4391 save_globals(s, allocated_regs);
4392 }
4393
4394 tcg_out_call(s, tcg_call_func(op), info);
4395
4396 /* Assign output registers and emit moves if needed. */
4397 switch (info->out_kind) {
4398 case TCG_CALL_RET_NORMAL:
4399 for (i = 0; i < nb_oargs; i++) {
4400 TCGTemp *ts = arg_temp(op->args[i]);
4401 TCGReg reg = tcg_target_call_oarg_regs[i];
4402
4403 /* ENV should not be modified. */
4404 tcg_debug_assert(!temp_readonly(ts));
4405
4406 set_temp_val_reg(s, ts, reg);
4407 ts->mem_coherent = 0;
4408 }
4409 break;
4410 default:
4411 g_assert_not_reached();
4412 }
4413
4414 /* Flush or discard output registers as needed. */
4415 for (i = 0; i < nb_oargs; i++) {
4416 TCGTemp *ts = arg_temp(op->args[i]);
4417 if (NEED_SYNC_ARG(i)) {
4418 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
4419 } else if (IS_DEAD_ARG(i)) {
4420 temp_dead(s, ts);
4421 }
4422 }
4423 }
4424
4425 #ifdef CONFIG_PROFILER
4426
4427 /* avoid copy/paste errors */
4428 #define PROF_ADD(to, from, field) \
4429 do { \
4430 (to)->field += qatomic_read(&((from)->field)); \
4431 } while (0)
4432
4433 #define PROF_MAX(to, from, field) \
4434 do { \
4435 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
4436 if (val__ > (to)->field) { \
4437 (to)->field = val__; \
4438 } \
4439 } while (0)
4440
4441 /* Pass in a zero'ed @prof */
4442 static inline
4443 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4444 {
4445 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
4446 unsigned int i;
4447
4448 for (i = 0; i < n_ctxs; i++) {
4449 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4450 const TCGProfile *orig = &s->prof;
4451
4452 if (counters) {
4453 PROF_ADD(prof, orig, cpu_exec_time);
4454 PROF_ADD(prof, orig, tb_count1);
4455 PROF_ADD(prof, orig, tb_count);
4456 PROF_ADD(prof, orig, op_count);
4457 PROF_MAX(prof, orig, op_count_max);
4458 PROF_ADD(prof, orig, temp_count);
4459 PROF_MAX(prof, orig, temp_count_max);
4460 PROF_ADD(prof, orig, del_op_count);
4461 PROF_ADD(prof, orig, code_in_len);
4462 PROF_ADD(prof, orig, code_out_len);
4463 PROF_ADD(prof, orig, search_out_len);
4464 PROF_ADD(prof, orig, interm_time);
4465 PROF_ADD(prof, orig, code_time);
4466 PROF_ADD(prof, orig, la_time);
4467 PROF_ADD(prof, orig, opt_time);
4468 PROF_ADD(prof, orig, restore_count);
4469 PROF_ADD(prof, orig, restore_time);
4470 }
4471 if (table) {
4472 int i;
4473
4474 for (i = 0; i < NB_OPS; i++) {
4475 PROF_ADD(prof, orig, table_op_count[i]);
4476 }
4477 }
4478 }
4479 }
4480
4481 #undef PROF_ADD
4482 #undef PROF_MAX
4483
4484 static void tcg_profile_snapshot_counters(TCGProfile *prof)
4485 {
4486 tcg_profile_snapshot(prof, true, false);
4487 }
4488
4489 static void tcg_profile_snapshot_table(TCGProfile *prof)
4490 {
4491 tcg_profile_snapshot(prof, false, true);
4492 }
4493
4494 void tcg_dump_op_count(GString *buf)
4495 {
4496 TCGProfile prof = {};
4497 int i;
4498
4499 tcg_profile_snapshot_table(&prof);
4500 for (i = 0; i < NB_OPS; i++) {
4501 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4502 prof.table_op_count[i]);
4503 }
4504 }
4505
4506 int64_t tcg_cpu_exec_time(void)
4507 {
4508 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
4509 unsigned int i;
4510 int64_t ret = 0;
4511
4512 for (i = 0; i < n_ctxs; i++) {
4513 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4514 const TCGProfile *prof = &s->prof;
4515
4516 ret += qatomic_read(&prof->cpu_exec_time);
4517 }
4518 return ret;
4519 }
4520 #else
4521 void tcg_dump_op_count(GString *buf)
4522 {
4523 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
4524 }
4525
4526 int64_t tcg_cpu_exec_time(void)
4527 {
4528 error_report("%s: TCG profiler not compiled", __func__);
4529 exit(EXIT_FAILURE);
4530 }
4531 #endif
4532
4533
4534 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
4535 {
4536 #ifdef CONFIG_PROFILER
4537 TCGProfile *prof = &s->prof;
4538 #endif
4539 int i, num_insns;
4540 TCGOp *op;
4541
4542 #ifdef CONFIG_PROFILER
4543 {
4544 int n = 0;
4545
4546 QTAILQ_FOREACH(op, &s->ops, link) {
4547 n++;
4548 }
4549 qatomic_set(&prof->op_count, prof->op_count + n);
4550 if (n > prof->op_count_max) {
4551 qatomic_set(&prof->op_count_max, n);
4552 }
4553
4554 n = s->nb_temps;
4555 qatomic_set(&prof->temp_count, prof->temp_count + n);
4556 if (n > prof->temp_count_max) {
4557 qatomic_set(&prof->temp_count_max, n);
4558 }
4559 }
4560 #endif
4561
4562 #ifdef DEBUG_DISAS
4563 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4564 && qemu_log_in_addr_range(pc_start))) {
4565 FILE *logfile = qemu_log_trylock();
4566 if (logfile) {
4567 fprintf(logfile, "OP:\n");
4568 tcg_dump_ops(s, logfile, false);
4569 fprintf(logfile, "\n");
4570 qemu_log_unlock(logfile);
4571 }
4572 }
4573 #endif
4574
4575 #ifdef CONFIG_DEBUG_TCG
4576 /* Ensure all labels referenced have been emitted. */
4577 {
4578 TCGLabel *l;
4579 bool error = false;
4580
4581 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4582 if (unlikely(!l->present) && l->refs) {
4583 qemu_log_mask(CPU_LOG_TB_OP,
4584 "$L%d referenced but not present.\n", l->id);
4585 error = true;
4586 }
4587 }
4588 assert(!error);
4589 }
4590 #endif
4591
4592 #ifdef CONFIG_PROFILER
4593 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
4594 #endif
4595
4596 #ifdef USE_TCG_OPTIMIZATIONS
4597 tcg_optimize(s);
4598 #endif
4599
4600 #ifdef CONFIG_PROFILER
4601 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4602 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
4603 #endif
4604
4605 reachable_code_pass(s);
4606 liveness_pass_1(s);
4607
4608 if (s->nb_indirects > 0) {
4609 #ifdef DEBUG_DISAS
4610 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4611 && qemu_log_in_addr_range(pc_start))) {
4612 FILE *logfile = qemu_log_trylock();
4613 if (logfile) {
4614 fprintf(logfile, "OP before indirect lowering:\n");
4615 tcg_dump_ops(s, logfile, false);
4616 fprintf(logfile, "\n");
4617 qemu_log_unlock(logfile);
4618 }
4619 }
4620 #endif
4621 /* Replace indirect temps with direct temps. */
4622 if (liveness_pass_2(s)) {
4623 /* If changes were made, re-run liveness. */
4624 liveness_pass_1(s);
4625 }
4626 }
4627
4628 #ifdef CONFIG_PROFILER
4629 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
4630 #endif
4631
4632 #ifdef DEBUG_DISAS
4633 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4634 && qemu_log_in_addr_range(pc_start))) {
4635 FILE *logfile = qemu_log_trylock();
4636 if (logfile) {
4637 fprintf(logfile, "OP after optimization and liveness analysis:\n");
4638 tcg_dump_ops(s, logfile, true);
4639 fprintf(logfile, "\n");
4640 qemu_log_unlock(logfile);
4641 }
4642 }
4643 #endif
4644
4645 /* Initialize goto_tb jump offsets. */
4646 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
4647 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
4648 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
4649 if (TCG_TARGET_HAS_direct_jump) {
4650 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
4651 tcg_ctx->tb_jmp_target_addr = NULL;
4652 } else {
4653 tcg_ctx->tb_jmp_insn_offset = NULL;
4654 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
4655 }
4656
4657 tcg_reg_alloc_start(s);
4658
4659 /*
4660 * Reset the buffer pointers when restarting after overflow.
4661 * TODO: Move this into translate-all.c with the rest of the
4662 * buffer management. Having only this done here is confusing.
4663 */
4664 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4665 s->code_ptr = s->code_buf;
4666
4667 #ifdef TCG_TARGET_NEED_LDST_LABELS
4668 QSIMPLEQ_INIT(&s->ldst_labels);
4669 #endif
4670 #ifdef TCG_TARGET_NEED_POOL_LABELS
4671 s->pool_labels = NULL;
4672 #endif
4673
4674 num_insns = -1;
4675 QTAILQ_FOREACH(op, &s->ops, link) {
4676 TCGOpcode opc = op->opc;
4677
4678 #ifdef CONFIG_PROFILER
4679 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
4680 #endif
4681
4682 switch (opc) {
4683 case INDEX_op_mov_i32:
4684 case INDEX_op_mov_i64:
4685 case INDEX_op_mov_vec:
4686 tcg_reg_alloc_mov(s, op);
4687 break;
4688 case INDEX_op_dup_vec:
4689 tcg_reg_alloc_dup(s, op);
4690 break;
4691 case INDEX_op_insn_start:
4692 if (num_insns >= 0) {
4693 size_t off = tcg_current_code_size(s);
4694 s->gen_insn_end_off[num_insns] = off;
4695 /* Assert that we do not overflow our stored offset. */
4696 assert(s->gen_insn_end_off[num_insns] == off);
4697 }
4698 num_insns++;
4699 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4700 target_ulong a;
4701 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4702 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
4703 #else
4704 a = op->args[i];
4705 #endif
4706 s->gen_insn_data[num_insns][i] = a;
4707 }
4708 break;
4709 case INDEX_op_discard:
4710 temp_dead(s, arg_temp(op->args[0]));
4711 break;
4712 case INDEX_op_set_label:
4713 tcg_reg_alloc_bb_end(s, s->reserved_regs);
4714 tcg_out_label(s, arg_label(op->args[0]));
4715 break;
4716 case INDEX_op_call:
4717 tcg_reg_alloc_call(s, op);
4718 break;
4719 case INDEX_op_dup2_vec:
4720 if (tcg_reg_alloc_dup2(s, op)) {
4721 break;
4722 }
4723 /* fall through */
4724 default:
4725 /* Sanity check that we've not introduced any unhandled opcodes. */
4726 tcg_debug_assert(tcg_op_supported(opc));
4727 /* Note: in order to speed up the code, it would be much
4728 faster to have specialized register allocator functions for
4729 some common argument patterns */
4730 tcg_reg_alloc_op(s, op);
4731 break;
4732 }
4733 /* Test for (pending) buffer overflow. The assumption is that any
4734 one operation beginning below the high water mark cannot overrun
4735 the buffer completely. Thus we can test for overflow after
4736 generating code without having to check during generation. */
4737 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
4738 return -1;
4739 }
4740 /* Test for TB overflow, as seen by gen_insn_end_off. */
4741 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4742 return -2;
4743 }
4744 }
4745 tcg_debug_assert(num_insns >= 0);
4746 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
4747
4748 /* Generate TB finalization at the end of block */
4749 #ifdef TCG_TARGET_NEED_LDST_LABELS
4750 i = tcg_out_ldst_finalize(s);
4751 if (i < 0) {
4752 return i;
4753 }
4754 #endif
4755 #ifdef TCG_TARGET_NEED_POOL_LABELS
4756 i = tcg_out_pool_finalize(s);
4757 if (i < 0) {
4758 return i;
4759 }
4760 #endif
4761 if (!tcg_resolve_relocs(s)) {
4762 return -2;
4763 }
4764
4765 #ifndef CONFIG_TCG_INTERPRETER
4766 /* flush instruction cache */
4767 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4768 (uintptr_t)s->code_buf,
4769 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
4770 #endif
4771
4772 return tcg_current_code_size(s);
4773 }
4774
4775 #ifdef CONFIG_PROFILER
4776 void tcg_dump_info(GString *buf)
4777 {
4778 TCGProfile prof = {};
4779 const TCGProfile *s;
4780 int64_t tb_count;
4781 int64_t tb_div_count;
4782 int64_t tot;
4783
4784 tcg_profile_snapshot_counters(&prof);
4785 s = &prof;
4786 tb_count = s->tb_count;
4787 tb_div_count = tb_count ? tb_count : 1;
4788 tot = s->interm_time + s->code_time;
4789
4790 g_string_append_printf(buf, "JIT cycles %" PRId64
4791 " (%0.3f s at 2.4 GHz)\n",
4792 tot, tot / 2.4e9);
4793 g_string_append_printf(buf, "translated TBs %" PRId64
4794 " (aborted=%" PRId64 " %0.1f%%)\n",
4795 tb_count, s->tb_count1 - tb_count,
4796 (double)(s->tb_count1 - s->tb_count)
4797 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4798 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
4799 (double)s->op_count / tb_div_count, s->op_count_max);
4800 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
4801 (double)s->del_op_count / tb_div_count);
4802 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
4803 (double)s->temp_count / tb_div_count,
4804 s->temp_count_max);
4805 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
4806 (double)s->code_out_len / tb_div_count);
4807 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
4808 (double)s->search_out_len / tb_div_count);
4809
4810 g_string_append_printf(buf, "cycles/op %0.1f\n",
4811 s->op_count ? (double)tot / s->op_count : 0);
4812 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
4813 s->code_in_len ? (double)tot / s->code_in_len : 0);
4814 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
4815 s->code_out_len ? (double)tot / s->code_out_len : 0);
4816 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
4817 s->search_out_len ?
4818 (double)tot / s->search_out_len : 0);
4819 if (tot == 0) {
4820 tot = 1;
4821 }
4822 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
4823 (double)s->interm_time / tot * 100.0);
4824 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
4825 (double)s->code_time / tot * 100.0);
4826 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
4827 (double)s->opt_time / (s->code_time ?
4828 s->code_time : 1)
4829 * 100.0);
4830 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
4831 (double)s->la_time / (s->code_time ?
4832 s->code_time : 1) * 100.0);
4833 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
4834 s->restore_count);
4835 g_string_append_printf(buf, " avg cycles %0.1f\n",
4836 s->restore_count ?
4837 (double)s->restore_time / s->restore_count : 0);
4838 }
4839 #else
4840 void tcg_dump_info(GString *buf)
4841 {
4842 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
4843 }
4844 #endif
4845
4846 #ifdef ELF_HOST_MACHINE
4847 /* In order to use this feature, the backend needs to do three things:
4848
4849 (1) Define ELF_HOST_MACHINE to indicate both what value to
4850 put into the ELF image and to indicate support for the feature.
4851
4852 (2) Define tcg_register_jit. This should create a buffer containing
4853 the contents of a .debug_frame section that describes the post-
4854 prologue unwind info for the tcg machine.
4855
4856 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4857 */
4858
4859 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4860 typedef enum {
4861 JIT_NOACTION = 0,
4862 JIT_REGISTER_FN,
4863 JIT_UNREGISTER_FN
4864 } jit_actions_t;
4865
4866 struct jit_code_entry {
4867 struct jit_code_entry *next_entry;
4868 struct jit_code_entry *prev_entry;
4869 const void *symfile_addr;
4870 uint64_t symfile_size;
4871 };
4872
4873 struct jit_descriptor {
4874 uint32_t version;
4875 uint32_t action_flag;
4876 struct jit_code_entry *relevant_entry;
4877 struct jit_code_entry *first_entry;
4878 };
4879
4880 void __jit_debug_register_code(void) __attribute__((noinline));
4881 void __jit_debug_register_code(void)
4882 {
4883 asm("");
4884 }
4885
4886 /* Must statically initialize the version, because GDB may check
4887 the version before we can set it. */
4888 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4889
4890 /* End GDB interface. */
4891
4892 static int find_string(const char *strtab, const char *str)
4893 {
4894 const char *p = strtab + 1;
4895
4896 while (1) {
4897 if (strcmp(p, str) == 0) {
4898 return p - strtab;
4899 }
4900 p += strlen(p) + 1;
4901 }
4902 }
4903
4904 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
4905 const void *debug_frame,
4906 size_t debug_frame_size)
4907 {
4908 struct __attribute__((packed)) DebugInfo {
4909 uint32_t len;
4910 uint16_t version;
4911 uint32_t abbrev;
4912 uint8_t ptr_size;
4913 uint8_t cu_die;
4914 uint16_t cu_lang;
4915 uintptr_t cu_low_pc;
4916 uintptr_t cu_high_pc;
4917 uint8_t fn_die;
4918 char fn_name[16];
4919 uintptr_t fn_low_pc;
4920 uintptr_t fn_high_pc;
4921 uint8_t cu_eoc;
4922 };
4923
4924 struct ElfImage {
4925 ElfW(Ehdr) ehdr;
4926 ElfW(Phdr) phdr;
4927 ElfW(Shdr) shdr[7];
4928 ElfW(Sym) sym[2];
4929 struct DebugInfo di;
4930 uint8_t da[24];
4931 char str[80];
4932 };
4933
4934 struct ElfImage *img;
4935
4936 static const struct ElfImage img_template = {
4937 .ehdr = {
4938 .e_ident[EI_MAG0] = ELFMAG0,
4939 .e_ident[EI_MAG1] = ELFMAG1,
4940 .e_ident[EI_MAG2] = ELFMAG2,
4941 .e_ident[EI_MAG3] = ELFMAG3,
4942 .e_ident[EI_CLASS] = ELF_CLASS,
4943 .e_ident[EI_DATA] = ELF_DATA,
4944 .e_ident[EI_VERSION] = EV_CURRENT,
4945 .e_type = ET_EXEC,
4946 .e_machine = ELF_HOST_MACHINE,
4947 .e_version = EV_CURRENT,
4948 .e_phoff = offsetof(struct ElfImage, phdr),
4949 .e_shoff = offsetof(struct ElfImage, shdr),
4950 .e_ehsize = sizeof(ElfW(Shdr)),
4951 .e_phentsize = sizeof(ElfW(Phdr)),
4952 .e_phnum = 1,
4953 .e_shentsize = sizeof(ElfW(Shdr)),
4954 .e_shnum = ARRAY_SIZE(img->shdr),
4955 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
4956 #ifdef ELF_HOST_FLAGS
4957 .e_flags = ELF_HOST_FLAGS,
4958 #endif
4959 #ifdef ELF_OSABI
4960 .e_ident[EI_OSABI] = ELF_OSABI,
4961 #endif
4962 },
4963 .phdr = {
4964 .p_type = PT_LOAD,
4965 .p_flags = PF_X,
4966 },
4967 .shdr = {
4968 [0] = { .sh_type = SHT_NULL },
4969 /* Trick: The contents of code_gen_buffer are not present in
4970 this fake ELF file; that got allocated elsewhere. Therefore
4971 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4972 will not look for contents. We can record any address. */
4973 [1] = { /* .text */
4974 .sh_type = SHT_NOBITS,
4975 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4976 },
4977 [2] = { /* .debug_info */
4978 .sh_type = SHT_PROGBITS,
4979 .sh_offset = offsetof(struct ElfImage, di),
4980 .sh_size = sizeof(struct DebugInfo),
4981 },
4982 [3] = { /* .debug_abbrev */
4983 .sh_type = SHT_PROGBITS,
4984 .sh_offset = offsetof(struct ElfImage, da),
4985 .sh_size = sizeof(img->da),
4986 },
4987 [4] = { /* .debug_frame */
4988 .sh_type = SHT_PROGBITS,
4989 .sh_offset = sizeof(struct ElfImage),
4990 },
4991 [5] = { /* .symtab */
4992 .sh_type = SHT_SYMTAB,
4993 .sh_offset = offsetof(struct ElfImage, sym),
4994 .sh_size = sizeof(img->sym),
4995 .sh_info = 1,
4996 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4997 .sh_entsize = sizeof(ElfW(Sym)),
4998 },
4999 [6] = { /* .strtab */
5000 .sh_type = SHT_STRTAB,
5001 .sh_offset = offsetof(struct ElfImage, str),
5002 .sh_size = sizeof(img->str),
5003 }
5004 },
5005 .sym = {
5006 [1] = { /* code_gen_buffer */
5007 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
5008 .st_shndx = 1,
5009 }
5010 },
5011 .di = {
5012 .len = sizeof(struct DebugInfo) - 4,
5013 .version = 2,
5014 .ptr_size = sizeof(void *),
5015 .cu_die = 1,
5016 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
5017 .fn_die = 2,
5018 .fn_name = "code_gen_buffer"
5019 },
5020 .da = {
5021 1, /* abbrev number (the cu) */
5022 0x11, 1, /* DW_TAG_compile_unit, has children */
5023 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
5024 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5025 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5026 0, 0, /* end of abbrev */
5027 2, /* abbrev number (the fn) */
5028 0x2e, 0, /* DW_TAG_subprogram, no children */
5029 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
5030 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5031 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5032 0, 0, /* end of abbrev */
5033 0 /* no more abbrev */
5034 },
5035 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5036 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
5037 };
5038
5039 /* We only need a single jit entry; statically allocate it. */
5040 static struct jit_code_entry one_entry;
5041
5042 uintptr_t buf = (uintptr_t)buf_ptr;
5043 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
5044 DebugFrameHeader *dfh;
5045
5046 img = g_malloc(img_size);
5047 *img = img_template;
5048
5049 img->phdr.p_vaddr = buf;
5050 img->phdr.p_paddr = buf;
5051 img->phdr.p_memsz = buf_size;
5052
5053 img->shdr[1].sh_name = find_string(img->str, ".text");
5054 img->shdr[1].sh_addr = buf;
5055 img->shdr[1].sh_size = buf_size;
5056
5057 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
5058 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
5059
5060 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
5061 img->shdr[4].sh_size = debug_frame_size;
5062
5063 img->shdr[5].sh_name = find_string(img->str, ".symtab");
5064 img->shdr[6].sh_name = find_string(img->str, ".strtab");
5065
5066 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
5067 img->sym[1].st_value = buf;
5068 img->sym[1].st_size = buf_size;
5069
5070 img->di.cu_low_pc = buf;
5071 img->di.cu_high_pc = buf + buf_size;
5072 img->di.fn_low_pc = buf;
5073 img->di.fn_high_pc = buf + buf_size;
5074
5075 dfh = (DebugFrameHeader *)(img + 1);
5076 memcpy(dfh, debug_frame, debug_frame_size);
5077 dfh->fde.func_start = buf;
5078 dfh->fde.func_len = buf_size;
5079
5080 #ifdef DEBUG_JIT
5081 /* Enable this block to be able to debug the ELF image file creation.
5082 One can use readelf, objdump, or other inspection utilities. */
5083 {
5084 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5085 FILE *f = fopen(jit, "w+b");
5086 if (f) {
5087 if (fwrite(img, img_size, 1, f) != img_size) {
5088 /* Avoid stupid unused return value warning for fwrite. */
5089 }
5090 fclose(f);
5091 }
5092 }
5093 #endif
5094
5095 one_entry.symfile_addr = img;
5096 one_entry.symfile_size = img_size;
5097
5098 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
5099 __jit_debug_descriptor.relevant_entry = &one_entry;
5100 __jit_debug_descriptor.first_entry = &one_entry;
5101 __jit_debug_register_code();
5102 }
5103 #else
5104 /* No support for the feature. Provide the entry point expected by exec.c,
5105 and implement the internal function we declared earlier. */
5106
5107 static void tcg_register_jit_int(const void *buf, size_t size,
5108 const void *debug_frame,
5109 size_t debug_frame_size)
5110 {
5111 }
5112
5113 void tcg_register_jit(const void *buf, size_t buf_size)
5114 {
5115 }
5116 #endif /* ELF_HOST_MACHINE */
5117
5118 #if !TCG_TARGET_MAYBE_vec
5119 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
5120 {
5121 g_assert_not_reached();
5122 }
5123 #endif
QEmu