?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
linux-user/trace-events: fix minor typo in format string
[qemu.git] / tcg / tcg.c
blobca482c2301ee2d774b2cc05723704653e9f81a7a
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
27
28 #include "qemu/osdep.h"
29
30 /* Define to jump the ELF file used to communicate with GDB. */
31 #undef DEBUG_JIT
32
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
38 #include "qemu/cacheflush.h"
39
40 /* Note: the long term plan is to reduce the dependencies on the QEMU
41 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43 #define NO_CPU_IO_DEFS
44
45 #include "exec/exec-all.h"
46 #include "tcg/tcg-op.h"
47
48 #if UINTPTR_MAX == UINT32_MAX
49 # define ELF_CLASS ELFCLASS32
50 #else
51 # define ELF_CLASS ELFCLASS64
52 #endif
53 #ifdef HOST_WORDS_BIGENDIAN
54 # define ELF_DATA ELFDATA2MSB
55 #else
56 # define ELF_DATA ELFDATA2LSB
57 #endif
58
59 #include "elf.h"
60 #include "exec/log.h"
61 #include "tcg-internal.h"
62
63 /* Forward declarations for functions declared in tcg-target.c.inc and
64 used here. */
65 static void tcg_target_init(TCGContext *s);
66 static void tcg_target_qemu_prologue(TCGContext *s);
67 static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
68 intptr_t value, intptr_t addend);
69
70 /* The CIE and FDE header definitions will be common to all hosts. */
71 typedef struct {
72 uint32_t len __attribute__((aligned((sizeof(void *)))));
73 uint32_t id;
74 uint8_t version;
75 char augmentation[1];
76 uint8_t code_align;
77 uint8_t data_align;
78 uint8_t return_column;
79 } DebugFrameCIE;
80
81 typedef struct QEMU_PACKED {
82 uint32_t len __attribute__((aligned((sizeof(void *)))));
83 uint32_t cie_offset;
84 uintptr_t func_start;
85 uintptr_t func_len;
86 } DebugFrameFDEHeader;
87
88 typedef struct QEMU_PACKED {
89 DebugFrameCIE cie;
90 DebugFrameFDEHeader fde;
91 } DebugFrameHeader;
92
93 static void tcg_register_jit_int(const void *buf, size_t size,
94 const void *debug_frame,
95 size_t debug_frame_size)
96 __attribute__((unused));
97
98 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
99 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
100 intptr_t arg2);
101 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
102 static void tcg_out_movi(TCGContext *s, TCGType type,
103 TCGReg ret, tcg_target_long arg);
104 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
105 const TCGArg args[TCG_MAX_OP_ARGS],
106 const int const_args[TCG_MAX_OP_ARGS]);
107 #if TCG_TARGET_MAYBE_vec
108 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
109 TCGReg dst, TCGReg src);
110 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
111 TCGReg dst, TCGReg base, intptr_t offset);
112 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, int64_t arg);
114 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
115 unsigned vecl, unsigned vece,
116 const TCGArg args[TCG_MAX_OP_ARGS],
117 const int const_args[TCG_MAX_OP_ARGS]);
118 #else
119 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
120 TCGReg dst, TCGReg src)
121 {
122 g_assert_not_reached();
123 }
124 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
125 TCGReg dst, TCGReg base, intptr_t offset)
126 {
127 g_assert_not_reached();
128 }
129 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
130 TCGReg dst, int64_t arg)
131 {
132 g_assert_not_reached();
133 }
134 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
135 unsigned vecl, unsigned vece,
136 const TCGArg args[TCG_MAX_OP_ARGS],
137 const int const_args[TCG_MAX_OP_ARGS])
138 {
139 g_assert_not_reached();
140 }
141 #endif
142 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
143 intptr_t arg2);
144 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
145 TCGReg base, intptr_t ofs);
146 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
147 static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
148 #ifdef TCG_TARGET_NEED_LDST_LABELS
149 static int tcg_out_ldst_finalize(TCGContext *s);
150 #endif
151
152 TCGContext tcg_init_ctx;
153 __thread TCGContext *tcg_ctx;
154
155 TCGContext **tcg_ctxs;
156 unsigned int tcg_cur_ctxs;
157 unsigned int tcg_max_ctxs;
158 TCGv_env cpu_env = 0;
159 const void *tcg_code_gen_epilogue;
160 uintptr_t tcg_splitwx_diff;
161
162 #ifndef CONFIG_TCG_INTERPRETER
163 tcg_prologue_fn *tcg_qemu_tb_exec;
164 #endif
165
166 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
167 static TCGRegSet tcg_target_call_clobber_regs;
168
169 #if TCG_TARGET_INSN_UNIT_SIZE == 1
170 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
171 {
172 *s->code_ptr++ = v;
173 }
174
175 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
176 uint8_t v)
177 {
178 *p = v;
179 }
180 #endif
181
182 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
183 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
184 {
185 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
186 *s->code_ptr++ = v;
187 } else {
188 tcg_insn_unit *p = s->code_ptr;
189 memcpy(p, &v, sizeof(v));
190 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
191 }
192 }
193
194 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
195 uint16_t v)
196 {
197 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
198 *p = v;
199 } else {
200 memcpy(p, &v, sizeof(v));
201 }
202 }
203 #endif
204
205 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
206 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
207 {
208 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
209 *s->code_ptr++ = v;
210 } else {
211 tcg_insn_unit *p = s->code_ptr;
212 memcpy(p, &v, sizeof(v));
213 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
214 }
215 }
216
217 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
218 uint32_t v)
219 {
220 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
221 *p = v;
222 } else {
223 memcpy(p, &v, sizeof(v));
224 }
225 }
226 #endif
227
228 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
229 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
230 {
231 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
232 *s->code_ptr++ = v;
233 } else {
234 tcg_insn_unit *p = s->code_ptr;
235 memcpy(p, &v, sizeof(v));
236 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
237 }
238 }
239
240 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
241 uint64_t v)
242 {
243 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
244 *p = v;
245 } else {
246 memcpy(p, &v, sizeof(v));
247 }
248 }
249 #endif
250
251 /* label relocation processing */
252
253 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
254 TCGLabel *l, intptr_t addend)
255 {
256 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
257
258 r->type = type;
259 r->ptr = code_ptr;
260 r->addend = addend;
261 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
262 }
263
264 static void tcg_out_label(TCGContext *s, TCGLabel *l)
265 {
266 tcg_debug_assert(!l->has_value);
267 l->has_value = 1;
268 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
269 }
270
271 TCGLabel *gen_new_label(void)
272 {
273 TCGContext *s = tcg_ctx;
274 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
275
276 memset(l, 0, sizeof(TCGLabel));
277 l->id = s->nb_labels++;
278 QSIMPLEQ_INIT(&l->relocs);
279
280 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
281
282 return l;
283 }
284
285 static bool tcg_resolve_relocs(TCGContext *s)
286 {
287 TCGLabel *l;
288
289 QSIMPLEQ_FOREACH(l, &s->labels, next) {
290 TCGRelocation *r;
291 uintptr_t value = l->u.value;
292
293 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
294 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
295 return false;
296 }
297 }
298 }
299 return true;
300 }
301
302 static void set_jmp_reset_offset(TCGContext *s, int which)
303 {
304 /*
305 * We will check for overflow at the end of the opcode loop in
306 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
307 */
308 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
309 }
310
311 /* Signal overflow, starting over with fewer guest insns. */
312 static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
313 {
314 siglongjmp(s->jmp_trans, -2);
315 }
316
317 #define C_PFX1(P, A) P##A
318 #define C_PFX2(P, A, B) P##A##_##B
319 #define C_PFX3(P, A, B, C) P##A##_##B##_##C
320 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
321 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
322 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
323
324 /* Define an enumeration for the various combinations. */
325
326 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
327 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
328 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
329 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
330
331 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
332 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
333 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
334 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
335
336 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
337
338 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
339 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
340 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
341 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
342
343 typedef enum {
344 #include "tcg-target-con-set.h"
345 } TCGConstraintSetIndex;
346
347 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
348
349 #undef C_O0_I1
350 #undef C_O0_I2
351 #undef C_O0_I3
352 #undef C_O0_I4
353 #undef C_O1_I1
354 #undef C_O1_I2
355 #undef C_O1_I3
356 #undef C_O1_I4
357 #undef C_N1_I2
358 #undef C_O2_I1
359 #undef C_O2_I2
360 #undef C_O2_I3
361 #undef C_O2_I4
362
363 /* Put all of the constraint sets into an array, indexed by the enum. */
364
365 #define C_O0_I1(I1) { .args_ct_str = { #I1 } },
366 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
367 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
368 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
369
370 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
371 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
372 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
373 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
374
375 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
376
377 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
378 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
379 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
380 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
381
382 static const TCGTargetOpDef constraint_sets[] = {
383 #include "tcg-target-con-set.h"
384 };
385
386
387 #undef C_O0_I1
388 #undef C_O0_I2
389 #undef C_O0_I3
390 #undef C_O0_I4
391 #undef C_O1_I1
392 #undef C_O1_I2
393 #undef C_O1_I3
394 #undef C_O1_I4
395 #undef C_N1_I2
396 #undef C_O2_I1
397 #undef C_O2_I2
398 #undef C_O2_I3
399 #undef C_O2_I4
400
401 /* Expand the enumerator to be returned from tcg_target_op_def(). */
402
403 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
404 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
405 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
406 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
407
408 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
409 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
410 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
411 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
412
413 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
414
415 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
416 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
417 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
418 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
419
420 #include "tcg-target.c.inc"
421
422 static void alloc_tcg_plugin_context(TCGContext *s)
423 {
424 #ifdef CONFIG_PLUGIN
425 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
426 s->plugin_tb->insns =
427 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
428 #endif
429 }
430
431 /*
432 * All TCG threads except the parent (i.e. the one that called tcg_context_init
433 * and registered the target's TCG globals) must register with this function
434 * before initiating translation.
435 *
436 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
437 * of tcg_region_init() for the reasoning behind this.
438 *
439 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
440 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
441 * is not used anymore for translation once this function is called.
442 *
443 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
444 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
445 */
446 #ifdef CONFIG_USER_ONLY
447 void tcg_register_thread(void)
448 {
449 tcg_ctx = &tcg_init_ctx;
450 }
451 #else
452 void tcg_register_thread(void)
453 {
454 TCGContext *s = g_malloc(sizeof(*s));
455 unsigned int i, n;
456
457 *s = tcg_init_ctx;
458
459 /* Relink mem_base. */
460 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
461 if (tcg_init_ctx.temps[i].mem_base) {
462 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
463 tcg_debug_assert(b >= 0 && b < n);
464 s->temps[i].mem_base = &s->temps[b];
465 }
466 }
467
468 /* Claim an entry in tcg_ctxs */
469 n = qatomic_fetch_inc(&tcg_cur_ctxs);
470 g_assert(n < tcg_max_ctxs);
471 qatomic_set(&tcg_ctxs[n], s);
472
473 if (n > 0) {
474 alloc_tcg_plugin_context(s);
475 tcg_region_initial_alloc(s);
476 }
477
478 tcg_ctx = s;
479 }
480 #endif /* !CONFIG_USER_ONLY */
481
482 /* pool based memory allocation */
483 void *tcg_malloc_internal(TCGContext *s, int size)
484 {
485 TCGPool *p;
486 int pool_size;
487
488 if (size > TCG_POOL_CHUNK_SIZE) {
489 /* big malloc: insert a new pool (XXX: could optimize) */
490 p = g_malloc(sizeof(TCGPool) + size);
491 p->size = size;
492 p->next = s->pool_first_large;
493 s->pool_first_large = p;
494 return p->data;
495 } else {
496 p = s->pool_current;
497 if (!p) {
498 p = s->pool_first;
499 if (!p)
500 goto new_pool;
501 } else {
502 if (!p->next) {
503 new_pool:
504 pool_size = TCG_POOL_CHUNK_SIZE;
505 p = g_malloc(sizeof(TCGPool) + pool_size);
506 p->size = pool_size;
507 p->next = NULL;
508 if (s->pool_current)
509 s->pool_current->next = p;
510 else
511 s->pool_first = p;
512 } else {
513 p = p->next;
514 }
515 }
516 }
517 s->pool_current = p;
518 s->pool_cur = p->data + size;
519 s->pool_end = p->data + p->size;
520 return p->data;
521 }
522
523 void tcg_pool_reset(TCGContext *s)
524 {
525 TCGPool *p, *t;
526 for (p = s->pool_first_large; p; p = t) {
527 t = p->next;
528 g_free(p);
529 }
530 s->pool_first_large = NULL;
531 s->pool_cur = s->pool_end = NULL;
532 s->pool_current = NULL;
533 }
534
535 typedef struct TCGHelperInfo {
536 void *func;
537 const char *name;
538 unsigned flags;
539 unsigned sizemask;
540 } TCGHelperInfo;
541
542 #include "exec/helper-proto.h"
543
544 static const TCGHelperInfo all_helpers[] = {
545 #include "exec/helper-tcg.h"
546 };
547 static GHashTable *helper_table;
548
549 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
550 static void process_op_defs(TCGContext *s);
551 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
552 TCGReg reg, const char *name);
553
554 static void tcg_context_init(unsigned max_cpus)
555 {
556 TCGContext *s = &tcg_init_ctx;
557 int op, total_args, n, i;
558 TCGOpDef *def;
559 TCGArgConstraint *args_ct;
560 TCGTemp *ts;
561
562 memset(s, 0, sizeof(*s));
563 s->nb_globals = 0;
564
565 /* Count total number of arguments and allocate the corresponding
566 space */
567 total_args = 0;
568 for(op = 0; op < NB_OPS; op++) {
569 def = &tcg_op_defs[op];
570 n = def->nb_iargs + def->nb_oargs;
571 total_args += n;
572 }
573
574 args_ct = g_new0(TCGArgConstraint, total_args);
575
576 for(op = 0; op < NB_OPS; op++) {
577 def = &tcg_op_defs[op];
578 def->args_ct = args_ct;
579 n = def->nb_iargs + def->nb_oargs;
580 args_ct += n;
581 }
582
583 /* Register helpers. */
584 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
585 helper_table = g_hash_table_new(NULL, NULL);
586
587 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
588 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
589 (gpointer)&all_helpers[i]);
590 }
591
592 tcg_target_init(s);
593 process_op_defs(s);
594
595 /* Reverse the order of the saved registers, assuming they're all at
596 the start of tcg_target_reg_alloc_order. */
597 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
598 int r = tcg_target_reg_alloc_order[n];
599 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
600 break;
601 }
602 }
603 for (i = 0; i < n; ++i) {
604 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
605 }
606 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
607 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
608 }
609
610 alloc_tcg_plugin_context(s);
611
612 tcg_ctx = s;
613 /*
614 * In user-mode we simply share the init context among threads, since we
615 * use a single region. See the documentation tcg_region_init() for the
616 * reasoning behind this.
617 * In softmmu we will have at most max_cpus TCG threads.
618 */
619 #ifdef CONFIG_USER_ONLY
620 tcg_ctxs = &tcg_ctx;
621 tcg_cur_ctxs = 1;
622 tcg_max_ctxs = 1;
623 #else
624 tcg_max_ctxs = max_cpus;
625 tcg_ctxs = g_new0(TCGContext *, max_cpus);
626 #endif
627
628 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
629 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
630 cpu_env = temp_tcgv_ptr(ts);
631 }
632
633 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
634 {
635 tcg_context_init(max_cpus);
636 tcg_region_init(tb_size, splitwx, max_cpus);
637 }
638
639 /*
640 * Allocate TBs right before their corresponding translated code, making
641 * sure that TBs and code are on different cache lines.
642 */
643 TranslationBlock *tcg_tb_alloc(TCGContext *s)
644 {
645 uintptr_t align = qemu_icache_linesize;
646 TranslationBlock *tb;
647 void *next;
648
649 retry:
650 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
651 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
652
653 if (unlikely(next > s->code_gen_highwater)) {
654 if (tcg_region_alloc(s)) {
655 return NULL;
656 }
657 goto retry;
658 }
659 qatomic_set(&s->code_gen_ptr, next);
660 s->data_gen_ptr = NULL;
661 return tb;
662 }
663
664 void tcg_prologue_init(TCGContext *s)
665 {
666 size_t prologue_size;
667
668 s->code_ptr = s->code_gen_ptr;
669 s->code_buf = s->code_gen_ptr;
670 s->data_gen_ptr = NULL;
671
672 #ifndef CONFIG_TCG_INTERPRETER
673 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
674 #endif
675
676 #ifdef TCG_TARGET_NEED_POOL_LABELS
677 s->pool_labels = NULL;
678 #endif
679
680 qemu_thread_jit_write();
681 /* Generate the prologue. */
682 tcg_target_qemu_prologue(s);
683
684 #ifdef TCG_TARGET_NEED_POOL_LABELS
685 /* Allow the prologue to put e.g. guest_base into a pool entry. */
686 {
687 int result = tcg_out_pool_finalize(s);
688 tcg_debug_assert(result == 0);
689 }
690 #endif
691
692 prologue_size = tcg_current_code_size(s);
693
694 #ifndef CONFIG_TCG_INTERPRETER
695 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
696 (uintptr_t)s->code_buf, prologue_size);
697 #endif
698
699 tcg_region_prologue_set(s);
700
701 #ifdef DEBUG_DISAS
702 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
703 FILE *logfile = qemu_log_lock();
704 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
705 if (s->data_gen_ptr) {
706 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
707 size_t data_size = prologue_size - code_size;
708 size_t i;
709
710 log_disas(s->code_gen_ptr, code_size);
711
712 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
713 if (sizeof(tcg_target_ulong) == 8) {
714 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
715 (uintptr_t)s->data_gen_ptr + i,
716 *(uint64_t *)(s->data_gen_ptr + i));
717 } else {
718 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
719 (uintptr_t)s->data_gen_ptr + i,
720 *(uint32_t *)(s->data_gen_ptr + i));
721 }
722 }
723 } else {
724 log_disas(s->code_gen_ptr, prologue_size);
725 }
726 qemu_log("\n");
727 qemu_log_flush();
728 qemu_log_unlock(logfile);
729 }
730 #endif
731
732 /* Assert that goto_ptr is implemented completely. */
733 if (TCG_TARGET_HAS_goto_ptr) {
734 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
735 }
736 }
737
738 void tcg_func_start(TCGContext *s)
739 {
740 tcg_pool_reset(s);
741 s->nb_temps = s->nb_globals;
742
743 /* No temps have been previously allocated for size or locality. */
744 memset(s->free_temps, 0, sizeof(s->free_temps));
745
746 /* No constant temps have been previously allocated. */
747 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
748 if (s->const_table[i]) {
749 g_hash_table_remove_all(s->const_table[i]);
750 }
751 }
752
753 s->nb_ops = 0;
754 s->nb_labels = 0;
755 s->current_frame_offset = s->frame_start;
756
757 #ifdef CONFIG_DEBUG_TCG
758 s->goto_tb_issue_mask = 0;
759 #endif
760
761 QTAILQ_INIT(&s->ops);
762 QTAILQ_INIT(&s->free_ops);
763 QSIMPLEQ_INIT(&s->labels);
764 }
765
766 static TCGTemp *tcg_temp_alloc(TCGContext *s)
767 {
768 int n = s->nb_temps++;
769
770 if (n >= TCG_MAX_TEMPS) {
771 tcg_raise_tb_overflow(s);
772 }
773 return memset(&s->temps[n], 0, sizeof(TCGTemp));
774 }
775
776 static TCGTemp *tcg_global_alloc(TCGContext *s)
777 {
778 TCGTemp *ts;
779
780 tcg_debug_assert(s->nb_globals == s->nb_temps);
781 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
782 s->nb_globals++;
783 ts = tcg_temp_alloc(s);
784 ts->kind = TEMP_GLOBAL;
785
786 return ts;
787 }
788
789 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
790 TCGReg reg, const char *name)
791 {
792 TCGTemp *ts;
793
794 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
795 tcg_abort();
796 }
797
798 ts = tcg_global_alloc(s);
799 ts->base_type = type;
800 ts->type = type;
801 ts->kind = TEMP_FIXED;
802 ts->reg = reg;
803 ts->name = name;
804 tcg_regset_set_reg(s->reserved_regs, reg);
805
806 return ts;
807 }
808
809 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
810 {
811 s->frame_start = start;
812 s->frame_end = start + size;
813 s->frame_temp
814 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
815 }
816
817 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
818 intptr_t offset, const char *name)
819 {
820 TCGContext *s = tcg_ctx;
821 TCGTemp *base_ts = tcgv_ptr_temp(base);
822 TCGTemp *ts = tcg_global_alloc(s);
823 int indirect_reg = 0, bigendian = 0;
824 #ifdef HOST_WORDS_BIGENDIAN
825 bigendian = 1;
826 #endif
827
828 switch (base_ts->kind) {
829 case TEMP_FIXED:
830 break;
831 case TEMP_GLOBAL:
832 /* We do not support double-indirect registers. */
833 tcg_debug_assert(!base_ts->indirect_reg);
834 base_ts->indirect_base = 1;
835 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
836 ? 2 : 1);
837 indirect_reg = 1;
838 break;
839 default:
840 g_assert_not_reached();
841 }
842
843 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
844 TCGTemp *ts2 = tcg_global_alloc(s);
845 char buf[64];
846
847 ts->base_type = TCG_TYPE_I64;
848 ts->type = TCG_TYPE_I32;
849 ts->indirect_reg = indirect_reg;
850 ts->mem_allocated = 1;
851 ts->mem_base = base_ts;
852 ts->mem_offset = offset + bigendian * 4;
853 pstrcpy(buf, sizeof(buf), name);
854 pstrcat(buf, sizeof(buf), "_0");
855 ts->name = strdup(buf);
856
857 tcg_debug_assert(ts2 == ts + 1);
858 ts2->base_type = TCG_TYPE_I64;
859 ts2->type = TCG_TYPE_I32;
860 ts2->indirect_reg = indirect_reg;
861 ts2->mem_allocated = 1;
862 ts2->mem_base = base_ts;
863 ts2->mem_offset = offset + (1 - bigendian) * 4;
864 pstrcpy(buf, sizeof(buf), name);
865 pstrcat(buf, sizeof(buf), "_1");
866 ts2->name = strdup(buf);
867 } else {
868 ts->base_type = type;
869 ts->type = type;
870 ts->indirect_reg = indirect_reg;
871 ts->mem_allocated = 1;
872 ts->mem_base = base_ts;
873 ts->mem_offset = offset;
874 ts->name = name;
875 }
876 return ts;
877 }
878
879 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
880 {
881 TCGContext *s = tcg_ctx;
882 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
883 TCGTemp *ts;
884 int idx, k;
885
886 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
887 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
888 if (idx < TCG_MAX_TEMPS) {
889 /* There is already an available temp with the right type. */
890 clear_bit(idx, s->free_temps[k].l);
891
892 ts = &s->temps[idx];
893 ts->temp_allocated = 1;
894 tcg_debug_assert(ts->base_type == type);
895 tcg_debug_assert(ts->kind == kind);
896 } else {
897 ts = tcg_temp_alloc(s);
898 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
899 TCGTemp *ts2 = tcg_temp_alloc(s);
900
901 ts->base_type = type;
902 ts->type = TCG_TYPE_I32;
903 ts->temp_allocated = 1;
904 ts->kind = kind;
905
906 tcg_debug_assert(ts2 == ts + 1);
907 ts2->base_type = TCG_TYPE_I64;
908 ts2->type = TCG_TYPE_I32;
909 ts2->temp_allocated = 1;
910 ts2->kind = kind;
911 } else {
912 ts->base_type = type;
913 ts->type = type;
914 ts->temp_allocated = 1;
915 ts->kind = kind;
916 }
917 }
918
919 #if defined(CONFIG_DEBUG_TCG)
920 s->temps_in_use++;
921 #endif
922 return ts;
923 }
924
925 TCGv_vec tcg_temp_new_vec(TCGType type)
926 {
927 TCGTemp *t;
928
929 #ifdef CONFIG_DEBUG_TCG
930 switch (type) {
931 case TCG_TYPE_V64:
932 assert(TCG_TARGET_HAS_v64);
933 break;
934 case TCG_TYPE_V128:
935 assert(TCG_TARGET_HAS_v128);
936 break;
937 case TCG_TYPE_V256:
938 assert(TCG_TARGET_HAS_v256);
939 break;
940 default:
941 g_assert_not_reached();
942 }
943 #endif
944
945 t = tcg_temp_new_internal(type, 0);
946 return temp_tcgv_vec(t);
947 }
948
949 /* Create a new temp of the same type as an existing temp. */
950 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
951 {
952 TCGTemp *t = tcgv_vec_temp(match);
953
954 tcg_debug_assert(t->temp_allocated != 0);
955
956 t = tcg_temp_new_internal(t->base_type, 0);
957 return temp_tcgv_vec(t);
958 }
959
960 void tcg_temp_free_internal(TCGTemp *ts)
961 {
962 TCGContext *s = tcg_ctx;
963 int k, idx;
964
965 /* In order to simplify users of tcg_constant_*, silently ignore free. */
966 if (ts->kind == TEMP_CONST) {
967 return;
968 }
969
970 #if defined(CONFIG_DEBUG_TCG)
971 s->temps_in_use--;
972 if (s->temps_in_use < 0) {
973 fprintf(stderr, "More temporaries freed than allocated!\n");
974 }
975 #endif
976
977 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
978 tcg_debug_assert(ts->temp_allocated != 0);
979 ts->temp_allocated = 0;
980
981 idx = temp_idx(ts);
982 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
983 set_bit(idx, s->free_temps[k].l);
984 }
985
986 TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
987 {
988 TCGContext *s = tcg_ctx;
989 GHashTable *h = s->const_table[type];
990 TCGTemp *ts;
991
992 if (h == NULL) {
993 h = g_hash_table_new(g_int64_hash, g_int64_equal);
994 s->const_table[type] = h;
995 }
996
997 ts = g_hash_table_lookup(h, &val);
998 if (ts == NULL) {
999 ts = tcg_temp_alloc(s);
1000
1001 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1002 TCGTemp *ts2 = tcg_temp_alloc(s);
1003
1004 ts->base_type = TCG_TYPE_I64;
1005 ts->type = TCG_TYPE_I32;
1006 ts->kind = TEMP_CONST;
1007 ts->temp_allocated = 1;
1008 /*
1009 * Retain the full value of the 64-bit constant in the low
1010 * part, so that the hash table works. Actual uses will
1011 * truncate the value to the low part.
1012 */
1013 ts->val = val;
1014
1015 tcg_debug_assert(ts2 == ts + 1);
1016 ts2->base_type = TCG_TYPE_I64;
1017 ts2->type = TCG_TYPE_I32;
1018 ts2->kind = TEMP_CONST;
1019 ts2->temp_allocated = 1;
1020 ts2->val = val >> 32;
1021 } else {
1022 ts->base_type = type;
1023 ts->type = type;
1024 ts->kind = TEMP_CONST;
1025 ts->temp_allocated = 1;
1026 ts->val = val;
1027 }
1028 g_hash_table_insert(h, &ts->val, ts);
1029 }
1030
1031 return ts;
1032 }
1033
1034 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1035 {
1036 val = dup_const(vece, val);
1037 return temp_tcgv_vec(tcg_constant_internal(type, val));
1038 }
1039
1040 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1041 {
1042 TCGTemp *t = tcgv_vec_temp(match);
1043
1044 tcg_debug_assert(t->temp_allocated != 0);
1045 return tcg_constant_vec(t->base_type, vece, val);
1046 }
1047
1048 TCGv_i32 tcg_const_i32(int32_t val)
1049 {
1050 TCGv_i32 t0;
1051 t0 = tcg_temp_new_i32();
1052 tcg_gen_movi_i32(t0, val);
1053 return t0;
1054 }
1055
1056 TCGv_i64 tcg_const_i64(int64_t val)
1057 {
1058 TCGv_i64 t0;
1059 t0 = tcg_temp_new_i64();
1060 tcg_gen_movi_i64(t0, val);
1061 return t0;
1062 }
1063
1064 TCGv_i32 tcg_const_local_i32(int32_t val)
1065 {
1066 TCGv_i32 t0;
1067 t0 = tcg_temp_local_new_i32();
1068 tcg_gen_movi_i32(t0, val);
1069 return t0;
1070 }
1071
1072 TCGv_i64 tcg_const_local_i64(int64_t val)
1073 {
1074 TCGv_i64 t0;
1075 t0 = tcg_temp_local_new_i64();
1076 tcg_gen_movi_i64(t0, val);
1077 return t0;
1078 }
1079
1080 #if defined(CONFIG_DEBUG_TCG)
1081 void tcg_clear_temp_count(void)
1082 {
1083 TCGContext *s = tcg_ctx;
1084 s->temps_in_use = 0;
1085 }
1086
1087 int tcg_check_temp_count(void)
1088 {
1089 TCGContext *s = tcg_ctx;
1090 if (s->temps_in_use) {
1091 /* Clear the count so that we don't give another
1092 * warning immediately next time around.
1093 */
1094 s->temps_in_use = 0;
1095 return 1;
1096 }
1097 return 0;
1098 }
1099 #endif
1100
1101 /* Return true if OP may appear in the opcode stream.
1102 Test the runtime variable that controls each opcode. */
1103 bool tcg_op_supported(TCGOpcode op)
1104 {
1105 const bool have_vec
1106 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1107
1108 switch (op) {
1109 case INDEX_op_discard:
1110 case INDEX_op_set_label:
1111 case INDEX_op_call:
1112 case INDEX_op_br:
1113 case INDEX_op_mb:
1114 case INDEX_op_insn_start:
1115 case INDEX_op_exit_tb:
1116 case INDEX_op_goto_tb:
1117 case INDEX_op_qemu_ld_i32:
1118 case INDEX_op_qemu_st_i32:
1119 case INDEX_op_qemu_ld_i64:
1120 case INDEX_op_qemu_st_i64:
1121 return true;
1122
1123 case INDEX_op_qemu_st8_i32:
1124 return TCG_TARGET_HAS_qemu_st8_i32;
1125
1126 case INDEX_op_goto_ptr:
1127 return TCG_TARGET_HAS_goto_ptr;
1128
1129 case INDEX_op_mov_i32:
1130 case INDEX_op_setcond_i32:
1131 case INDEX_op_brcond_i32:
1132 case INDEX_op_ld8u_i32:
1133 case INDEX_op_ld8s_i32:
1134 case INDEX_op_ld16u_i32:
1135 case INDEX_op_ld16s_i32:
1136 case INDEX_op_ld_i32:
1137 case INDEX_op_st8_i32:
1138 case INDEX_op_st16_i32:
1139 case INDEX_op_st_i32:
1140 case INDEX_op_add_i32:
1141 case INDEX_op_sub_i32:
1142 case INDEX_op_mul_i32:
1143 case INDEX_op_and_i32:
1144 case INDEX_op_or_i32:
1145 case INDEX_op_xor_i32:
1146 case INDEX_op_shl_i32:
1147 case INDEX_op_shr_i32:
1148 case INDEX_op_sar_i32:
1149 return true;
1150
1151 case INDEX_op_movcond_i32:
1152 return TCG_TARGET_HAS_movcond_i32;
1153 case INDEX_op_div_i32:
1154 case INDEX_op_divu_i32:
1155 return TCG_TARGET_HAS_div_i32;
1156 case INDEX_op_rem_i32:
1157 case INDEX_op_remu_i32:
1158 return TCG_TARGET_HAS_rem_i32;
1159 case INDEX_op_div2_i32:
1160 case INDEX_op_divu2_i32:
1161 return TCG_TARGET_HAS_div2_i32;
1162 case INDEX_op_rotl_i32:
1163 case INDEX_op_rotr_i32:
1164 return TCG_TARGET_HAS_rot_i32;
1165 case INDEX_op_deposit_i32:
1166 return TCG_TARGET_HAS_deposit_i32;
1167 case INDEX_op_extract_i32:
1168 return TCG_TARGET_HAS_extract_i32;
1169 case INDEX_op_sextract_i32:
1170 return TCG_TARGET_HAS_sextract_i32;
1171 case INDEX_op_extract2_i32:
1172 return TCG_TARGET_HAS_extract2_i32;
1173 case INDEX_op_add2_i32:
1174 return TCG_TARGET_HAS_add2_i32;
1175 case INDEX_op_sub2_i32:
1176 return TCG_TARGET_HAS_sub2_i32;
1177 case INDEX_op_mulu2_i32:
1178 return TCG_TARGET_HAS_mulu2_i32;
1179 case INDEX_op_muls2_i32:
1180 return TCG_TARGET_HAS_muls2_i32;
1181 case INDEX_op_muluh_i32:
1182 return TCG_TARGET_HAS_muluh_i32;
1183 case INDEX_op_mulsh_i32:
1184 return TCG_TARGET_HAS_mulsh_i32;
1185 case INDEX_op_ext8s_i32:
1186 return TCG_TARGET_HAS_ext8s_i32;
1187 case INDEX_op_ext16s_i32:
1188 return TCG_TARGET_HAS_ext16s_i32;
1189 case INDEX_op_ext8u_i32:
1190 return TCG_TARGET_HAS_ext8u_i32;
1191 case INDEX_op_ext16u_i32:
1192 return TCG_TARGET_HAS_ext16u_i32;
1193 case INDEX_op_bswap16_i32:
1194 return TCG_TARGET_HAS_bswap16_i32;
1195 case INDEX_op_bswap32_i32:
1196 return TCG_TARGET_HAS_bswap32_i32;
1197 case INDEX_op_not_i32:
1198 return TCG_TARGET_HAS_not_i32;
1199 case INDEX_op_neg_i32:
1200 return TCG_TARGET_HAS_neg_i32;
1201 case INDEX_op_andc_i32:
1202 return TCG_TARGET_HAS_andc_i32;
1203 case INDEX_op_orc_i32:
1204 return TCG_TARGET_HAS_orc_i32;
1205 case INDEX_op_eqv_i32:
1206 return TCG_TARGET_HAS_eqv_i32;
1207 case INDEX_op_nand_i32:
1208 return TCG_TARGET_HAS_nand_i32;
1209 case INDEX_op_nor_i32:
1210 return TCG_TARGET_HAS_nor_i32;
1211 case INDEX_op_clz_i32:
1212 return TCG_TARGET_HAS_clz_i32;
1213 case INDEX_op_ctz_i32:
1214 return TCG_TARGET_HAS_ctz_i32;
1215 case INDEX_op_ctpop_i32:
1216 return TCG_TARGET_HAS_ctpop_i32;
1217
1218 case INDEX_op_brcond2_i32:
1219 case INDEX_op_setcond2_i32:
1220 return TCG_TARGET_REG_BITS == 32;
1221
1222 case INDEX_op_mov_i64:
1223 case INDEX_op_setcond_i64:
1224 case INDEX_op_brcond_i64:
1225 case INDEX_op_ld8u_i64:
1226 case INDEX_op_ld8s_i64:
1227 case INDEX_op_ld16u_i64:
1228 case INDEX_op_ld16s_i64:
1229 case INDEX_op_ld32u_i64:
1230 case INDEX_op_ld32s_i64:
1231 case INDEX_op_ld_i64:
1232 case INDEX_op_st8_i64:
1233 case INDEX_op_st16_i64:
1234 case INDEX_op_st32_i64:
1235 case INDEX_op_st_i64:
1236 case INDEX_op_add_i64:
1237 case INDEX_op_sub_i64:
1238 case INDEX_op_mul_i64:
1239 case INDEX_op_and_i64:
1240 case INDEX_op_or_i64:
1241 case INDEX_op_xor_i64:
1242 case INDEX_op_shl_i64:
1243 case INDEX_op_shr_i64:
1244 case INDEX_op_sar_i64:
1245 case INDEX_op_ext_i32_i64:
1246 case INDEX_op_extu_i32_i64:
1247 return TCG_TARGET_REG_BITS == 64;
1248
1249 case INDEX_op_movcond_i64:
1250 return TCG_TARGET_HAS_movcond_i64;
1251 case INDEX_op_div_i64:
1252 case INDEX_op_divu_i64:
1253 return TCG_TARGET_HAS_div_i64;
1254 case INDEX_op_rem_i64:
1255 case INDEX_op_remu_i64:
1256 return TCG_TARGET_HAS_rem_i64;
1257 case INDEX_op_div2_i64:
1258 case INDEX_op_divu2_i64:
1259 return TCG_TARGET_HAS_div2_i64;
1260 case INDEX_op_rotl_i64:
1261 case INDEX_op_rotr_i64:
1262 return TCG_TARGET_HAS_rot_i64;
1263 case INDEX_op_deposit_i64:
1264 return TCG_TARGET_HAS_deposit_i64;
1265 case INDEX_op_extract_i64:
1266 return TCG_TARGET_HAS_extract_i64;
1267 case INDEX_op_sextract_i64:
1268 return TCG_TARGET_HAS_sextract_i64;
1269 case INDEX_op_extract2_i64:
1270 return TCG_TARGET_HAS_extract2_i64;
1271 case INDEX_op_extrl_i64_i32:
1272 return TCG_TARGET_HAS_extrl_i64_i32;
1273 case INDEX_op_extrh_i64_i32:
1274 return TCG_TARGET_HAS_extrh_i64_i32;
1275 case INDEX_op_ext8s_i64:
1276 return TCG_TARGET_HAS_ext8s_i64;
1277 case INDEX_op_ext16s_i64:
1278 return TCG_TARGET_HAS_ext16s_i64;
1279 case INDEX_op_ext32s_i64:
1280 return TCG_TARGET_HAS_ext32s_i64;
1281 case INDEX_op_ext8u_i64:
1282 return TCG_TARGET_HAS_ext8u_i64;
1283 case INDEX_op_ext16u_i64:
1284 return TCG_TARGET_HAS_ext16u_i64;
1285 case INDEX_op_ext32u_i64:
1286 return TCG_TARGET_HAS_ext32u_i64;
1287 case INDEX_op_bswap16_i64:
1288 return TCG_TARGET_HAS_bswap16_i64;
1289 case INDEX_op_bswap32_i64:
1290 return TCG_TARGET_HAS_bswap32_i64;
1291 case INDEX_op_bswap64_i64:
1292 return TCG_TARGET_HAS_bswap64_i64;
1293 case INDEX_op_not_i64:
1294 return TCG_TARGET_HAS_not_i64;
1295 case INDEX_op_neg_i64:
1296 return TCG_TARGET_HAS_neg_i64;
1297 case INDEX_op_andc_i64:
1298 return TCG_TARGET_HAS_andc_i64;
1299 case INDEX_op_orc_i64:
1300 return TCG_TARGET_HAS_orc_i64;
1301 case INDEX_op_eqv_i64:
1302 return TCG_TARGET_HAS_eqv_i64;
1303 case INDEX_op_nand_i64:
1304 return TCG_TARGET_HAS_nand_i64;
1305 case INDEX_op_nor_i64:
1306 return TCG_TARGET_HAS_nor_i64;
1307 case INDEX_op_clz_i64:
1308 return TCG_TARGET_HAS_clz_i64;
1309 case INDEX_op_ctz_i64:
1310 return TCG_TARGET_HAS_ctz_i64;
1311 case INDEX_op_ctpop_i64:
1312 return TCG_TARGET_HAS_ctpop_i64;
1313 case INDEX_op_add2_i64:
1314 return TCG_TARGET_HAS_add2_i64;
1315 case INDEX_op_sub2_i64:
1316 return TCG_TARGET_HAS_sub2_i64;
1317 case INDEX_op_mulu2_i64:
1318 return TCG_TARGET_HAS_mulu2_i64;
1319 case INDEX_op_muls2_i64:
1320 return TCG_TARGET_HAS_muls2_i64;
1321 case INDEX_op_muluh_i64:
1322 return TCG_TARGET_HAS_muluh_i64;
1323 case INDEX_op_mulsh_i64:
1324 return TCG_TARGET_HAS_mulsh_i64;
1325
1326 case INDEX_op_mov_vec:
1327 case INDEX_op_dup_vec:
1328 case INDEX_op_dupm_vec:
1329 case INDEX_op_ld_vec:
1330 case INDEX_op_st_vec:
1331 case INDEX_op_add_vec:
1332 case INDEX_op_sub_vec:
1333 case INDEX_op_and_vec:
1334 case INDEX_op_or_vec:
1335 case INDEX_op_xor_vec:
1336 case INDEX_op_cmp_vec:
1337 return have_vec;
1338 case INDEX_op_dup2_vec:
1339 return have_vec && TCG_TARGET_REG_BITS == 32;
1340 case INDEX_op_not_vec:
1341 return have_vec && TCG_TARGET_HAS_not_vec;
1342 case INDEX_op_neg_vec:
1343 return have_vec && TCG_TARGET_HAS_neg_vec;
1344 case INDEX_op_abs_vec:
1345 return have_vec && TCG_TARGET_HAS_abs_vec;
1346 case INDEX_op_andc_vec:
1347 return have_vec && TCG_TARGET_HAS_andc_vec;
1348 case INDEX_op_orc_vec:
1349 return have_vec && TCG_TARGET_HAS_orc_vec;
1350 case INDEX_op_mul_vec:
1351 return have_vec && TCG_TARGET_HAS_mul_vec;
1352 case INDEX_op_shli_vec:
1353 case INDEX_op_shri_vec:
1354 case INDEX_op_sari_vec:
1355 return have_vec && TCG_TARGET_HAS_shi_vec;
1356 case INDEX_op_shls_vec:
1357 case INDEX_op_shrs_vec:
1358 case INDEX_op_sars_vec:
1359 return have_vec && TCG_TARGET_HAS_shs_vec;
1360 case INDEX_op_shlv_vec:
1361 case INDEX_op_shrv_vec:
1362 case INDEX_op_sarv_vec:
1363 return have_vec && TCG_TARGET_HAS_shv_vec;
1364 case INDEX_op_rotli_vec:
1365 return have_vec && TCG_TARGET_HAS_roti_vec;
1366 case INDEX_op_rotls_vec:
1367 return have_vec && TCG_TARGET_HAS_rots_vec;
1368 case INDEX_op_rotlv_vec:
1369 case INDEX_op_rotrv_vec:
1370 return have_vec && TCG_TARGET_HAS_rotv_vec;
1371 case INDEX_op_ssadd_vec:
1372 case INDEX_op_usadd_vec:
1373 case INDEX_op_sssub_vec:
1374 case INDEX_op_ussub_vec:
1375 return have_vec && TCG_TARGET_HAS_sat_vec;
1376 case INDEX_op_smin_vec:
1377 case INDEX_op_umin_vec:
1378 case INDEX_op_smax_vec:
1379 case INDEX_op_umax_vec:
1380 return have_vec && TCG_TARGET_HAS_minmax_vec;
1381 case INDEX_op_bitsel_vec:
1382 return have_vec && TCG_TARGET_HAS_bitsel_vec;
1383 case INDEX_op_cmpsel_vec:
1384 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
1385
1386 default:
1387 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1388 return true;
1389 }
1390 }
1391
1392 /* Note: we convert the 64 bit args to 32 bit and do some alignment
1393 and endian swap. Maybe it would be better to do the alignment
1394 and endian swap in tcg_reg_alloc_call(). */
1395 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
1396 {
1397 int i, real_args, nb_rets, pi;
1398 unsigned sizemask, flags;
1399 TCGHelperInfo *info;
1400 TCGOp *op;
1401
1402 info = g_hash_table_lookup(helper_table, (gpointer)func);
1403 flags = info->flags;
1404 sizemask = info->sizemask;
1405
1406 #ifdef CONFIG_PLUGIN
1407 /* detect non-plugin helpers */
1408 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1409 tcg_ctx->plugin_insn->calls_helpers = true;
1410 }
1411 #endif
1412
1413 #if defined(__sparc__) && !defined(__arch64__) \
1414 && !defined(CONFIG_TCG_INTERPRETER)
1415 /* We have 64-bit values in one register, but need to pass as two
1416 separate parameters. Split them. */
1417 int orig_sizemask = sizemask;
1418 int orig_nargs = nargs;
1419 TCGv_i64 retl, reth;
1420 TCGTemp *split_args[MAX_OPC_PARAM];
1421
1422 retl = NULL;
1423 reth = NULL;
1424 if (sizemask != 0) {
1425 for (i = real_args = 0; i < nargs; ++i) {
1426 int is_64bit = sizemask & (1 << (i+1)*2);
1427 if (is_64bit) {
1428 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1429 TCGv_i32 h = tcg_temp_new_i32();
1430 TCGv_i32 l = tcg_temp_new_i32();
1431 tcg_gen_extr_i64_i32(l, h, orig);
1432 split_args[real_args++] = tcgv_i32_temp(h);
1433 split_args[real_args++] = tcgv_i32_temp(l);
1434 } else {
1435 split_args[real_args++] = args[i];
1436 }
1437 }
1438 nargs = real_args;
1439 args = split_args;
1440 sizemask = 0;
1441 }
1442 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1443 for (i = 0; i < nargs; ++i) {
1444 int is_64bit = sizemask & (1 << (i+1)*2);
1445 int is_signed = sizemask & (2 << (i+1)*2);
1446 if (!is_64bit) {
1447 TCGv_i64 temp = tcg_temp_new_i64();
1448 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1449 if (is_signed) {
1450 tcg_gen_ext32s_i64(temp, orig);
1451 } else {
1452 tcg_gen_ext32u_i64(temp, orig);
1453 }
1454 args[i] = tcgv_i64_temp(temp);
1455 }
1456 }
1457 #endif /* TCG_TARGET_EXTEND_ARGS */
1458
1459 op = tcg_emit_op(INDEX_op_call);
1460
1461 pi = 0;
1462 if (ret != NULL) {
1463 #if defined(__sparc__) && !defined(__arch64__) \
1464 && !defined(CONFIG_TCG_INTERPRETER)
1465 if (orig_sizemask & 1) {
1466 /* The 32-bit ABI is going to return the 64-bit value in
1467 the %o0/%o1 register pair. Prepare for this by using
1468 two return temporaries, and reassemble below. */
1469 retl = tcg_temp_new_i64();
1470 reth = tcg_temp_new_i64();
1471 op->args[pi++] = tcgv_i64_arg(reth);
1472 op->args[pi++] = tcgv_i64_arg(retl);
1473 nb_rets = 2;
1474 } else {
1475 op->args[pi++] = temp_arg(ret);
1476 nb_rets = 1;
1477 }
1478 #else
1479 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) {
1480 #ifdef HOST_WORDS_BIGENDIAN
1481 op->args[pi++] = temp_arg(ret + 1);
1482 op->args[pi++] = temp_arg(ret);
1483 #else
1484 op->args[pi++] = temp_arg(ret);
1485 op->args[pi++] = temp_arg(ret + 1);
1486 #endif
1487 nb_rets = 2;
1488 } else {
1489 op->args[pi++] = temp_arg(ret);
1490 nb_rets = 1;
1491 }
1492 #endif
1493 } else {
1494 nb_rets = 0;
1495 }
1496 TCGOP_CALLO(op) = nb_rets;
1497
1498 real_args = 0;
1499 for (i = 0; i < nargs; i++) {
1500 int is_64bit = sizemask & (1 << (i+1)*2);
1501 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
1502 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1503 /* some targets want aligned 64 bit args */
1504 if (real_args & 1) {
1505 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1506 real_args++;
1507 }
1508 #endif
1509 /* If stack grows up, then we will be placing successive
1510 arguments at lower addresses, which means we need to
1511 reverse the order compared to how we would normally
1512 treat either big or little-endian. For those arguments
1513 that will wind up in registers, this still works for
1514 HPPA (the only current STACK_GROWSUP target) since the
1515 argument registers are *also* allocated in decreasing
1516 order. If another such target is added, this logic may
1517 have to get more complicated to differentiate between
1518 stack arguments and register arguments. */
1519 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
1520 op->args[pi++] = temp_arg(args[i] + 1);
1521 op->args[pi++] = temp_arg(args[i]);
1522 #else
1523 op->args[pi++] = temp_arg(args[i]);
1524 op->args[pi++] = temp_arg(args[i] + 1);
1525 #endif
1526 real_args += 2;
1527 continue;
1528 }
1529
1530 op->args[pi++] = temp_arg(args[i]);
1531 real_args++;
1532 }
1533 op->args[pi++] = (uintptr_t)func;
1534 op->args[pi++] = flags;
1535 TCGOP_CALLI(op) = real_args;
1536
1537 /* Make sure the fields didn't overflow. */
1538 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
1539 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
1540
1541 #if defined(__sparc__) && !defined(__arch64__) \
1542 && !defined(CONFIG_TCG_INTERPRETER)
1543 /* Free all of the parts we allocated above. */
1544 for (i = real_args = 0; i < orig_nargs; ++i) {
1545 int is_64bit = orig_sizemask & (1 << (i+1)*2);
1546 if (is_64bit) {
1547 tcg_temp_free_internal(args[real_args++]);
1548 tcg_temp_free_internal(args[real_args++]);
1549 } else {
1550 real_args++;
1551 }
1552 }
1553 if (orig_sizemask & 1) {
1554 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1555 Note that describing these as TCGv_i64 eliminates an unnecessary
1556 zero-extension that tcg_gen_concat_i32_i64 would create. */
1557 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
1558 tcg_temp_free_i64(retl);
1559 tcg_temp_free_i64(reth);
1560 }
1561 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1562 for (i = 0; i < nargs; ++i) {
1563 int is_64bit = sizemask & (1 << (i+1)*2);
1564 if (!is_64bit) {
1565 tcg_temp_free_internal(args[i]);
1566 }
1567 }
1568 #endif /* TCG_TARGET_EXTEND_ARGS */
1569 }
1570
1571 static void tcg_reg_alloc_start(TCGContext *s)
1572 {
1573 int i, n;
1574
1575 for (i = 0, n = s->nb_temps; i < n; i++) {
1576 TCGTemp *ts = &s->temps[i];
1577 TCGTempVal val = TEMP_VAL_MEM;
1578
1579 switch (ts->kind) {
1580 case TEMP_CONST:
1581 val = TEMP_VAL_CONST;
1582 break;
1583 case TEMP_FIXED:
1584 val = TEMP_VAL_REG;
1585 break;
1586 case TEMP_GLOBAL:
1587 break;
1588 case TEMP_NORMAL:
1589 val = TEMP_VAL_DEAD;
1590 /* fall through */
1591 case TEMP_LOCAL:
1592 ts->mem_allocated = 0;
1593 break;
1594 default:
1595 g_assert_not_reached();
1596 }
1597 ts->val_type = val;
1598 }
1599
1600 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
1601 }
1602
1603 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1604 TCGTemp *ts)
1605 {
1606 int idx = temp_idx(ts);
1607
1608 switch (ts->kind) {
1609 case TEMP_FIXED:
1610 case TEMP_GLOBAL:
1611 pstrcpy(buf, buf_size, ts->name);
1612 break;
1613 case TEMP_LOCAL:
1614 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
1615 break;
1616 case TEMP_NORMAL:
1617 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
1618 break;
1619 case TEMP_CONST:
1620 switch (ts->type) {
1621 case TCG_TYPE_I32:
1622 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1623 break;
1624 #if TCG_TARGET_REG_BITS > 32
1625 case TCG_TYPE_I64:
1626 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1627 break;
1628 #endif
1629 case TCG_TYPE_V64:
1630 case TCG_TYPE_V128:
1631 case TCG_TYPE_V256:
1632 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1633 64 << (ts->type - TCG_TYPE_V64), ts->val);
1634 break;
1635 default:
1636 g_assert_not_reached();
1637 }
1638 break;
1639 }
1640 return buf;
1641 }
1642
1643 static char *tcg_get_arg_str(TCGContext *s, char *buf,
1644 int buf_size, TCGArg arg)
1645 {
1646 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
1647 }
1648
1649 /* Find helper name. */
1650 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
1651 {
1652 const char *ret = NULL;
1653 if (helper_table) {
1654 TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val);
1655 if (info) {
1656 ret = info->name;
1657 }
1658 }
1659 return ret;
1660 }
1661
1662 static const char * const cond_name[] =
1663 {
1664 [TCG_COND_NEVER] = "never",
1665 [TCG_COND_ALWAYS] = "always",
1666 [TCG_COND_EQ] = "eq",
1667 [TCG_COND_NE] = "ne",
1668 [TCG_COND_LT] = "lt",
1669 [TCG_COND_GE] = "ge",
1670 [TCG_COND_LE] = "le",
1671 [TCG_COND_GT] = "gt",
1672 [TCG_COND_LTU] = "ltu",
1673 [TCG_COND_GEU] = "geu",
1674 [TCG_COND_LEU] = "leu",
1675 [TCG_COND_GTU] = "gtu"
1676 };
1677
1678 static const char * const ldst_name[] =
1679 {
1680 [MO_UB] = "ub",
1681 [MO_SB] = "sb",
1682 [MO_LEUW] = "leuw",
1683 [MO_LESW] = "lesw",
1684 [MO_LEUL] = "leul",
1685 [MO_LESL] = "lesl",
1686 [MO_LEQ] = "leq",
1687 [MO_BEUW] = "beuw",
1688 [MO_BESW] = "besw",
1689 [MO_BEUL] = "beul",
1690 [MO_BESL] = "besl",
1691 [MO_BEQ] = "beq",
1692 };
1693
1694 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
1695 #ifdef TARGET_ALIGNED_ONLY
1696 [MO_UNALN >> MO_ASHIFT] = "un+",
1697 [MO_ALIGN >> MO_ASHIFT] = "",
1698 #else
1699 [MO_UNALN >> MO_ASHIFT] = "",
1700 [MO_ALIGN >> MO_ASHIFT] = "al+",
1701 #endif
1702 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1703 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1704 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1705 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1706 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1707 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1708 };
1709
1710 static inline bool tcg_regset_single(TCGRegSet d)
1711 {
1712 return (d & (d - 1)) == 0;
1713 }
1714
1715 static inline TCGReg tcg_regset_first(TCGRegSet d)
1716 {
1717 if (TCG_TARGET_NB_REGS <= 32) {
1718 return ctz32(d);
1719 } else {
1720 return ctz64(d);
1721 }
1722 }
1723
1724 static void tcg_dump_ops(TCGContext *s, bool have_prefs)
1725 {
1726 char buf[128];
1727 TCGOp *op;
1728
1729 QTAILQ_FOREACH(op, &s->ops, link) {
1730 int i, k, nb_oargs, nb_iargs, nb_cargs;
1731 const TCGOpDef *def;
1732 TCGOpcode c;
1733 int col = 0;
1734
1735 c = op->opc;
1736 def = &tcg_op_defs[c];
1737
1738 if (c == INDEX_op_insn_start) {
1739 nb_oargs = 0;
1740 col += qemu_log("\n ----");
1741
1742 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1743 target_ulong a;
1744 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1745 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
1746 #else
1747 a = op->args[i];
1748 #endif
1749 col += qemu_log(" " TARGET_FMT_lx, a);
1750 }
1751 } else if (c == INDEX_op_call) {
1752 /* variable number of arguments */
1753 nb_oargs = TCGOP_CALLO(op);
1754 nb_iargs = TCGOP_CALLI(op);
1755 nb_cargs = def->nb_cargs;
1756
1757 /* function name, flags, out args */
1758 col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
1759 tcg_find_helper(s, op->args[nb_oargs + nb_iargs]),
1760 op->args[nb_oargs + nb_iargs + 1], nb_oargs);
1761 for (i = 0; i < nb_oargs; i++) {
1762 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1763 op->args[i]));
1764 }
1765 for (i = 0; i < nb_iargs; i++) {
1766 TCGArg arg = op->args[nb_oargs + i];
1767 const char *t = "<dummy>";
1768 if (arg != TCG_CALL_DUMMY_ARG) {
1769 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
1770 }
1771 col += qemu_log(",%s", t);
1772 }
1773 } else {
1774 col += qemu_log(" %s ", def->name);
1775
1776 nb_oargs = def->nb_oargs;
1777 nb_iargs = def->nb_iargs;
1778 nb_cargs = def->nb_cargs;
1779
1780 if (def->flags & TCG_OPF_VECTOR) {
1781 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
1782 8 << TCGOP_VECE(op));
1783 }
1784
1785 k = 0;
1786 for (i = 0; i < nb_oargs; i++) {
1787 if (k != 0) {
1788 col += qemu_log(",");
1789 }
1790 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1791 op->args[k++]));
1792 }
1793 for (i = 0; i < nb_iargs; i++) {
1794 if (k != 0) {
1795 col += qemu_log(",");
1796 }
1797 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1798 op->args[k++]));
1799 }
1800 switch (c) {
1801 case INDEX_op_brcond_i32:
1802 case INDEX_op_setcond_i32:
1803 case INDEX_op_movcond_i32:
1804 case INDEX_op_brcond2_i32:
1805 case INDEX_op_setcond2_i32:
1806 case INDEX_op_brcond_i64:
1807 case INDEX_op_setcond_i64:
1808 case INDEX_op_movcond_i64:
1809 case INDEX_op_cmp_vec:
1810 case INDEX_op_cmpsel_vec:
1811 if (op->args[k] < ARRAY_SIZE(cond_name)
1812 && cond_name[op->args[k]]) {
1813 col += qemu_log(",%s", cond_name[op->args[k++]]);
1814 } else {
1815 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
1816 }
1817 i = 1;
1818 break;
1819 case INDEX_op_qemu_ld_i32:
1820 case INDEX_op_qemu_st_i32:
1821 case INDEX_op_qemu_st8_i32:
1822 case INDEX_op_qemu_ld_i64:
1823 case INDEX_op_qemu_st_i64:
1824 {
1825 TCGMemOpIdx oi = op->args[k++];
1826 MemOp op = get_memop(oi);
1827 unsigned ix = get_mmuidx(oi);
1828
1829 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
1830 col += qemu_log(",$0x%x,%u", op, ix);
1831 } else {
1832 const char *s_al, *s_op;
1833 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
1834 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
1835 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
1836 }
1837 i = 1;
1838 }
1839 break;
1840 default:
1841 i = 0;
1842 break;
1843 }
1844 switch (c) {
1845 case INDEX_op_set_label:
1846 case INDEX_op_br:
1847 case INDEX_op_brcond_i32:
1848 case INDEX_op_brcond_i64:
1849 case INDEX_op_brcond2_i32:
1850 col += qemu_log("%s$L%d", k ? "," : "",
1851 arg_label(op->args[k])->id);
1852 i++, k++;
1853 break;
1854 default:
1855 break;
1856 }
1857 for (; i < nb_cargs; i++, k++) {
1858 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
1859 }
1860 }
1861
1862 if (have_prefs || op->life) {
1863
1864 QemuLogFile *logfile;
1865
1866 rcu_read_lock();
1867 logfile = qatomic_rcu_read(&qemu_logfile);
1868 if (logfile) {
1869 for (; col < 40; ++col) {
1870 putc(' ', logfile->fd);
1871 }
1872 }
1873 rcu_read_unlock();
1874 }
1875
1876 if (op->life) {
1877 unsigned life = op->life;
1878
1879 if (life & (SYNC_ARG * 3)) {
1880 qemu_log(" sync:");
1881 for (i = 0; i < 2; ++i) {
1882 if (life & (SYNC_ARG << i)) {
1883 qemu_log(" %d", i);
1884 }
1885 }
1886 }
1887 life /= DEAD_ARG;
1888 if (life) {
1889 qemu_log(" dead:");
1890 for (i = 0; life; ++i, life >>= 1) {
1891 if (life & 1) {
1892 qemu_log(" %d", i);
1893 }
1894 }
1895 }
1896 }
1897
1898 if (have_prefs) {
1899 for (i = 0; i < nb_oargs; ++i) {
1900 TCGRegSet set = op->output_pref[i];
1901
1902 if (i == 0) {
1903 qemu_log(" pref=");
1904 } else {
1905 qemu_log(",");
1906 }
1907 if (set == 0) {
1908 qemu_log("none");
1909 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
1910 qemu_log("all");
1911 #ifdef CONFIG_DEBUG_TCG
1912 } else if (tcg_regset_single(set)) {
1913 TCGReg reg = tcg_regset_first(set);
1914 qemu_log("%s", tcg_target_reg_names[reg]);
1915 #endif
1916 } else if (TCG_TARGET_NB_REGS <= 32) {
1917 qemu_log("%#x", (uint32_t)set);
1918 } else {
1919 qemu_log("%#" PRIx64, (uint64_t)set);
1920 }
1921 }
1922 }
1923
1924 qemu_log("\n");
1925 }
1926 }
1927
1928 /* we give more priority to constraints with less registers */
1929 static int get_constraint_priority(const TCGOpDef *def, int k)
1930 {
1931 const TCGArgConstraint *arg_ct = &def->args_ct[k];
1932 int n;
1933
1934 if (arg_ct->oalias) {
1935 /* an alias is equivalent to a single register */
1936 n = 1;
1937 } else {
1938 n = ctpop64(arg_ct->regs);
1939 }
1940 return TCG_TARGET_NB_REGS - n + 1;
1941 }
1942
1943 /* sort from highest priority to lowest */
1944 static void sort_constraints(TCGOpDef *def, int start, int n)
1945 {
1946 int i, j;
1947 TCGArgConstraint *a = def->args_ct;
1948
1949 for (i = 0; i < n; i++) {
1950 a[start + i].sort_index = start + i;
1951 }
1952 if (n <= 1) {
1953 return;
1954 }
1955 for (i = 0; i < n - 1; i++) {
1956 for (j = i + 1; j < n; j++) {
1957 int p1 = get_constraint_priority(def, a[start + i].sort_index);
1958 int p2 = get_constraint_priority(def, a[start + j].sort_index);
1959 if (p1 < p2) {
1960 int tmp = a[start + i].sort_index;
1961 a[start + i].sort_index = a[start + j].sort_index;
1962 a[start + j].sort_index = tmp;
1963 }
1964 }
1965 }
1966 }
1967
1968 static void process_op_defs(TCGContext *s)
1969 {
1970 TCGOpcode op;
1971
1972 for (op = 0; op < NB_OPS; op++) {
1973 TCGOpDef *def = &tcg_op_defs[op];
1974 const TCGTargetOpDef *tdefs;
1975 int i, nb_args;
1976
1977 if (def->flags & TCG_OPF_NOT_PRESENT) {
1978 continue;
1979 }
1980
1981 nb_args = def->nb_iargs + def->nb_oargs;
1982 if (nb_args == 0) {
1983 continue;
1984 }
1985
1986 /*
1987 * Macro magic should make it impossible, but double-check that
1988 * the array index is in range. Since the signness of an enum
1989 * is implementation defined, force the result to unsigned.
1990 */
1991 unsigned con_set = tcg_target_op_def(op);
1992 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
1993 tdefs = &constraint_sets[con_set];
1994
1995 for (i = 0; i < nb_args; i++) {
1996 const char *ct_str = tdefs->args_ct_str[i];
1997 /* Incomplete TCGTargetOpDef entry. */
1998 tcg_debug_assert(ct_str != NULL);
1999
2000 while (*ct_str != '\0') {
2001 switch(*ct_str) {
2002 case '0' ... '9':
2003 {
2004 int oarg = *ct_str - '0';
2005 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2006 tcg_debug_assert(oarg < def->nb_oargs);
2007 tcg_debug_assert(def->args_ct[oarg].regs != 0);
2008 def->args_ct[i] = def->args_ct[oarg];
2009 /* The output sets oalias. */
2010 def->args_ct[oarg].oalias = true;
2011 def->args_ct[oarg].alias_index = i;
2012 /* The input sets ialias. */
2013 def->args_ct[i].ialias = true;
2014 def->args_ct[i].alias_index = oarg;
2015 }
2016 ct_str++;
2017 break;
2018 case '&':
2019 def->args_ct[i].newreg = true;
2020 ct_str++;
2021 break;
2022 case 'i':
2023 def->args_ct[i].ct |= TCG_CT_CONST;
2024 ct_str++;
2025 break;
2026
2027 /* Include all of the target-specific constraints. */
2028
2029 #undef CONST
2030 #define CONST(CASE, MASK) \
2031 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2032 #define REGS(CASE, MASK) \
2033 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2034
2035 #include "tcg-target-con-str.h"
2036
2037 #undef REGS
2038 #undef CONST
2039 default:
2040 /* Typo in TCGTargetOpDef constraint. */
2041 g_assert_not_reached();
2042 }
2043 }
2044 }
2045
2046 /* TCGTargetOpDef entry with too much information? */
2047 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
2048
2049 /* sort the constraints (XXX: this is just an heuristic) */
2050 sort_constraints(def, 0, def->nb_oargs);
2051 sort_constraints(def, def->nb_oargs, def->nb_iargs);
2052 }
2053 }
2054
2055 void tcg_op_remove(TCGContext *s, TCGOp *op)
2056 {
2057 TCGLabel *label;
2058
2059 switch (op->opc) {
2060 case INDEX_op_br:
2061 label = arg_label(op->args[0]);
2062 label->refs--;
2063 break;
2064 case INDEX_op_brcond_i32:
2065 case INDEX_op_brcond_i64:
2066 label = arg_label(op->args[3]);
2067 label->refs--;
2068 break;
2069 case INDEX_op_brcond2_i32:
2070 label = arg_label(op->args[5]);
2071 label->refs--;
2072 break;
2073 default:
2074 break;
2075 }
2076
2077 QTAILQ_REMOVE(&s->ops, op, link);
2078 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
2079 s->nb_ops--;
2080
2081 #ifdef CONFIG_PROFILER
2082 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
2083 #endif
2084 }
2085
2086 void tcg_remove_ops_after(TCGOp *op)
2087 {
2088 TCGContext *s = tcg_ctx;
2089
2090 while (true) {
2091 TCGOp *last = tcg_last_op();
2092 if (last == op) {
2093 return;
2094 }
2095 tcg_op_remove(s, last);
2096 }
2097 }
2098
2099 static TCGOp *tcg_op_alloc(TCGOpcode opc)
2100 {
2101 TCGContext *s = tcg_ctx;
2102 TCGOp *op;
2103
2104 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2105 op = tcg_malloc(sizeof(TCGOp));
2106 } else {
2107 op = QTAILQ_FIRST(&s->free_ops);
2108 QTAILQ_REMOVE(&s->free_ops, op, link);
2109 }
2110 memset(op, 0, offsetof(TCGOp, link));
2111 op->opc = opc;
2112 s->nb_ops++;
2113
2114 return op;
2115 }
2116
2117 TCGOp *tcg_emit_op(TCGOpcode opc)
2118 {
2119 TCGOp *op = tcg_op_alloc(opc);
2120 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2121 return op;
2122 }
2123
2124 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2125 {
2126 TCGOp *new_op = tcg_op_alloc(opc);
2127 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
2128 return new_op;
2129 }
2130
2131 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2132 {
2133 TCGOp *new_op = tcg_op_alloc(opc);
2134 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
2135 return new_op;
2136 }
2137
2138 /* Reachable analysis : remove unreachable code. */
2139 static void reachable_code_pass(TCGContext *s)
2140 {
2141 TCGOp *op, *op_next;
2142 bool dead = false;
2143
2144 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2145 bool remove = dead;
2146 TCGLabel *label;
2147 int call_flags;
2148
2149 switch (op->opc) {
2150 case INDEX_op_set_label:
2151 label = arg_label(op->args[0]);
2152 if (label->refs == 0) {
2153 /*
2154 * While there is an occasional backward branch, virtually
2155 * all branches generated by the translators are forward.
2156 * Which means that generally we will have already removed
2157 * all references to the label that will be, and there is
2158 * little to be gained by iterating.
2159 */
2160 remove = true;
2161 } else {
2162 /* Once we see a label, insns become live again. */
2163 dead = false;
2164 remove = false;
2165
2166 /*
2167 * Optimization can fold conditional branches to unconditional.
2168 * If we find a label with one reference which is preceded by
2169 * an unconditional branch to it, remove both. This needed to
2170 * wait until the dead code in between them was removed.
2171 */
2172 if (label->refs == 1) {
2173 TCGOp *op_prev = QTAILQ_PREV(op, link);
2174 if (op_prev->opc == INDEX_op_br &&
2175 label == arg_label(op_prev->args[0])) {
2176 tcg_op_remove(s, op_prev);
2177 remove = true;
2178 }
2179 }
2180 }
2181 break;
2182
2183 case INDEX_op_br:
2184 case INDEX_op_exit_tb:
2185 case INDEX_op_goto_ptr:
2186 /* Unconditional branches; everything following is dead. */
2187 dead = true;
2188 break;
2189
2190 case INDEX_op_call:
2191 /* Notice noreturn helper calls, raising exceptions. */
2192 call_flags = op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1];
2193 if (call_flags & TCG_CALL_NO_RETURN) {
2194 dead = true;
2195 }
2196 break;
2197
2198 case INDEX_op_insn_start:
2199 /* Never remove -- we need to keep these for unwind. */
2200 remove = false;
2201 break;
2202
2203 default:
2204 break;
2205 }
2206
2207 if (remove) {
2208 tcg_op_remove(s, op);
2209 }
2210 }
2211 }
2212
2213 #define TS_DEAD 1
2214 #define TS_MEM 2
2215
2216 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2217 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2218
2219 /* For liveness_pass_1, the register preferences for a given temp. */
2220 static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2221 {
2222 return ts->state_ptr;
2223 }
2224
2225 /* For liveness_pass_1, reset the preferences for a given temp to the
2226 * maximal regset for its type.
2227 */
2228 static inline void la_reset_pref(TCGTemp *ts)
2229 {
2230 *la_temp_pref(ts)
2231 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2232 }
2233
2234 /* liveness analysis: end of function: all temps are dead, and globals
2235 should be in memory. */
2236 static void la_func_end(TCGContext *s, int ng, int nt)
2237 {
2238 int i;
2239
2240 for (i = 0; i < ng; ++i) {
2241 s->temps[i].state = TS_DEAD | TS_MEM;
2242 la_reset_pref(&s->temps[i]);
2243 }
2244 for (i = ng; i < nt; ++i) {
2245 s->temps[i].state = TS_DEAD;
2246 la_reset_pref(&s->temps[i]);
2247 }
2248 }
2249
2250 /* liveness analysis: end of basic block: all temps are dead, globals
2251 and local temps should be in memory. */
2252 static void la_bb_end(TCGContext *s, int ng, int nt)
2253 {
2254 int i;
2255
2256 for (i = 0; i < nt; ++i) {
2257 TCGTemp *ts = &s->temps[i];
2258 int state;
2259
2260 switch (ts->kind) {
2261 case TEMP_FIXED:
2262 case TEMP_GLOBAL:
2263 case TEMP_LOCAL:
2264 state = TS_DEAD | TS_MEM;
2265 break;
2266 case TEMP_NORMAL:
2267 case TEMP_CONST:
2268 state = TS_DEAD;
2269 break;
2270 default:
2271 g_assert_not_reached();
2272 }
2273 ts->state = state;
2274 la_reset_pref(ts);
2275 }
2276 }
2277
2278 /* liveness analysis: sync globals back to memory. */
2279 static void la_global_sync(TCGContext *s, int ng)
2280 {
2281 int i;
2282
2283 for (i = 0; i < ng; ++i) {
2284 int state = s->temps[i].state;
2285 s->temps[i].state = state | TS_MEM;
2286 if (state == TS_DEAD) {
2287 /* If the global was previously dead, reset prefs. */
2288 la_reset_pref(&s->temps[i]);
2289 }
2290 }
2291 }
2292
2293 /*
2294 * liveness analysis: conditional branch: all temps are dead,
2295 * globals and local temps should be synced.
2296 */
2297 static void la_bb_sync(TCGContext *s, int ng, int nt)
2298 {
2299 la_global_sync(s, ng);
2300
2301 for (int i = ng; i < nt; ++i) {
2302 TCGTemp *ts = &s->temps[i];
2303 int state;
2304
2305 switch (ts->kind) {
2306 case TEMP_LOCAL:
2307 state = ts->state;
2308 ts->state = state | TS_MEM;
2309 if (state != TS_DEAD) {
2310 continue;
2311 }
2312 break;
2313 case TEMP_NORMAL:
2314 s->temps[i].state = TS_DEAD;
2315 break;
2316 case TEMP_CONST:
2317 continue;
2318 default:
2319 g_assert_not_reached();
2320 }
2321 la_reset_pref(&s->temps[i]);
2322 }
2323 }
2324
2325 /* liveness analysis: sync globals back to memory and kill. */
2326 static void la_global_kill(TCGContext *s, int ng)
2327 {
2328 int i;
2329
2330 for (i = 0; i < ng; i++) {
2331 s->temps[i].state = TS_DEAD | TS_MEM;
2332 la_reset_pref(&s->temps[i]);
2333 }
2334 }
2335
2336 /* liveness analysis: note live globals crossing calls. */
2337 static void la_cross_call(TCGContext *s, int nt)
2338 {
2339 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2340 int i;
2341
2342 for (i = 0; i < nt; i++) {
2343 TCGTemp *ts = &s->temps[i];
2344 if (!(ts->state & TS_DEAD)) {
2345 TCGRegSet *pset = la_temp_pref(ts);
2346 TCGRegSet set = *pset;
2347
2348 set &= mask;
2349 /* If the combination is not possible, restart. */
2350 if (set == 0) {
2351 set = tcg_target_available_regs[ts->type] & mask;
2352 }
2353 *pset = set;
2354 }
2355 }
2356 }
2357
2358 /* Liveness analysis : update the opc_arg_life array to tell if a
2359 given input arguments is dead. Instructions updating dead
2360 temporaries are removed. */
2361 static void liveness_pass_1(TCGContext *s)
2362 {
2363 int nb_globals = s->nb_globals;
2364 int nb_temps = s->nb_temps;
2365 TCGOp *op, *op_prev;
2366 TCGRegSet *prefs;
2367 int i;
2368
2369 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2370 for (i = 0; i < nb_temps; ++i) {
2371 s->temps[i].state_ptr = prefs + i;
2372 }
2373
2374 /* ??? Should be redundant with the exit_tb that ends the TB. */
2375 la_func_end(s, nb_globals, nb_temps);
2376
2377 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
2378 int nb_iargs, nb_oargs;
2379 TCGOpcode opc_new, opc_new2;
2380 bool have_opc_new2;
2381 TCGLifeData arg_life = 0;
2382 TCGTemp *ts;
2383 TCGOpcode opc = op->opc;
2384 const TCGOpDef *def = &tcg_op_defs[opc];
2385
2386 switch (opc) {
2387 case INDEX_op_call:
2388 {
2389 int call_flags;
2390 int nb_call_regs;
2391
2392 nb_oargs = TCGOP_CALLO(op);
2393 nb_iargs = TCGOP_CALLI(op);
2394 call_flags = op->args[nb_oargs + nb_iargs + 1];
2395
2396 /* pure functions can be removed if their result is unused */
2397 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
2398 for (i = 0; i < nb_oargs; i++) {
2399 ts = arg_temp(op->args[i]);
2400 if (ts->state != TS_DEAD) {
2401 goto do_not_remove_call;
2402 }
2403 }
2404 goto do_remove;
2405 }
2406 do_not_remove_call:
2407
2408 /* Output args are dead. */
2409 for (i = 0; i < nb_oargs; i++) {
2410 ts = arg_temp(op->args[i]);
2411 if (ts->state & TS_DEAD) {
2412 arg_life |= DEAD_ARG << i;
2413 }
2414 if (ts->state & TS_MEM) {
2415 arg_life |= SYNC_ARG << i;
2416 }
2417 ts->state = TS_DEAD;
2418 la_reset_pref(ts);
2419
2420 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2421 op->output_pref[i] = 0;
2422 }
2423
2424 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2425 TCG_CALL_NO_READ_GLOBALS))) {
2426 la_global_kill(s, nb_globals);
2427 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
2428 la_global_sync(s, nb_globals);
2429 }
2430
2431 /* Record arguments that die in this helper. */
2432 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2433 ts = arg_temp(op->args[i]);
2434 if (ts && ts->state & TS_DEAD) {
2435 arg_life |= DEAD_ARG << i;
2436 }
2437 }
2438
2439 /* For all live registers, remove call-clobbered prefs. */
2440 la_cross_call(s, nb_temps);
2441
2442 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2443
2444 /* Input arguments are live for preceding opcodes. */
2445 for (i = 0; i < nb_iargs; i++) {
2446 ts = arg_temp(op->args[i + nb_oargs]);
2447 if (ts && ts->state & TS_DEAD) {
2448 /* For those arguments that die, and will be allocated
2449 * in registers, clear the register set for that arg,
2450 * to be filled in below. For args that will be on
2451 * the stack, reset to any available reg.
2452 */
2453 *la_temp_pref(ts)
2454 = (i < nb_call_regs ? 0 :
2455 tcg_target_available_regs[ts->type]);
2456 ts->state &= ~TS_DEAD;
2457 }
2458 }
2459
2460 /* For each input argument, add its input register to prefs.
2461 If a temp is used once, this produces a single set bit. */
2462 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2463 ts = arg_temp(op->args[i + nb_oargs]);
2464 if (ts) {
2465 tcg_regset_set_reg(*la_temp_pref(ts),
2466 tcg_target_call_iarg_regs[i]);
2467 }
2468 }
2469 }
2470 break;
2471 case INDEX_op_insn_start:
2472 break;
2473 case INDEX_op_discard:
2474 /* mark the temporary as dead */
2475 ts = arg_temp(op->args[0]);
2476 ts->state = TS_DEAD;
2477 la_reset_pref(ts);
2478 break;
2479
2480 case INDEX_op_add2_i32:
2481 opc_new = INDEX_op_add_i32;
2482 goto do_addsub2;
2483 case INDEX_op_sub2_i32:
2484 opc_new = INDEX_op_sub_i32;
2485 goto do_addsub2;
2486 case INDEX_op_add2_i64:
2487 opc_new = INDEX_op_add_i64;
2488 goto do_addsub2;
2489 case INDEX_op_sub2_i64:
2490 opc_new = INDEX_op_sub_i64;
2491 do_addsub2:
2492 nb_iargs = 4;
2493 nb_oargs = 2;
2494 /* Test if the high part of the operation is dead, but not
2495 the low part. The result can be optimized to a simple
2496 add or sub. This happens often for x86_64 guest when the
2497 cpu mode is set to 32 bit. */
2498 if (arg_temp(op->args[1])->state == TS_DEAD) {
2499 if (arg_temp(op->args[0])->state == TS_DEAD) {
2500 goto do_remove;
2501 }
2502 /* Replace the opcode and adjust the args in place,
2503 leaving 3 unused args at the end. */
2504 op->opc = opc = opc_new;
2505 op->args[1] = op->args[2];
2506 op->args[2] = op->args[4];
2507 /* Fall through and mark the single-word operation live. */
2508 nb_iargs = 2;
2509 nb_oargs = 1;
2510 }
2511 goto do_not_remove;
2512
2513 case INDEX_op_mulu2_i32:
2514 opc_new = INDEX_op_mul_i32;
2515 opc_new2 = INDEX_op_muluh_i32;
2516 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
2517 goto do_mul2;
2518 case INDEX_op_muls2_i32:
2519 opc_new = INDEX_op_mul_i32;
2520 opc_new2 = INDEX_op_mulsh_i32;
2521 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
2522 goto do_mul2;
2523 case INDEX_op_mulu2_i64:
2524 opc_new = INDEX_op_mul_i64;
2525 opc_new2 = INDEX_op_muluh_i64;
2526 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
2527 goto do_mul2;
2528 case INDEX_op_muls2_i64:
2529 opc_new = INDEX_op_mul_i64;
2530 opc_new2 = INDEX_op_mulsh_i64;
2531 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
2532 goto do_mul2;
2533 do_mul2:
2534 nb_iargs = 2;
2535 nb_oargs = 2;
2536 if (arg_temp(op->args[1])->state == TS_DEAD) {
2537 if (arg_temp(op->args[0])->state == TS_DEAD) {
2538 /* Both parts of the operation are dead. */
2539 goto do_remove;
2540 }
2541 /* The high part of the operation is dead; generate the low. */
2542 op->opc = opc = opc_new;
2543 op->args[1] = op->args[2];
2544 op->args[2] = op->args[3];
2545 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
2546 /* The low part of the operation is dead; generate the high. */
2547 op->opc = opc = opc_new2;
2548 op->args[0] = op->args[1];
2549 op->args[1] = op->args[2];
2550 op->args[2] = op->args[3];
2551 } else {
2552 goto do_not_remove;
2553 }
2554 /* Mark the single-word operation live. */
2555 nb_oargs = 1;
2556 goto do_not_remove;
2557
2558 default:
2559 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2560 nb_iargs = def->nb_iargs;
2561 nb_oargs = def->nb_oargs;
2562
2563 /* Test if the operation can be removed because all
2564 its outputs are dead. We assume that nb_oargs == 0
2565 implies side effects */
2566 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
2567 for (i = 0; i < nb_oargs; i++) {
2568 if (arg_temp(op->args[i])->state != TS_DEAD) {
2569 goto do_not_remove;
2570 }
2571 }
2572 goto do_remove;
2573 }
2574 goto do_not_remove;
2575
2576 do_remove:
2577 tcg_op_remove(s, op);
2578 break;
2579
2580 do_not_remove:
2581 for (i = 0; i < nb_oargs; i++) {
2582 ts = arg_temp(op->args[i]);
2583
2584 /* Remember the preference of the uses that followed. */
2585 op->output_pref[i] = *la_temp_pref(ts);
2586
2587 /* Output args are dead. */
2588 if (ts->state & TS_DEAD) {
2589 arg_life |= DEAD_ARG << i;
2590 }
2591 if (ts->state & TS_MEM) {
2592 arg_life |= SYNC_ARG << i;
2593 }
2594 ts->state = TS_DEAD;
2595 la_reset_pref(ts);
2596 }
2597
2598 /* If end of basic block, update. */
2599 if (def->flags & TCG_OPF_BB_EXIT) {
2600 la_func_end(s, nb_globals, nb_temps);
2601 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2602 la_bb_sync(s, nb_globals, nb_temps);
2603 } else if (def->flags & TCG_OPF_BB_END) {
2604 la_bb_end(s, nb_globals, nb_temps);
2605 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2606 la_global_sync(s, nb_globals);
2607 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2608 la_cross_call(s, nb_temps);
2609 }
2610 }
2611
2612 /* Record arguments that die in this opcode. */
2613 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2614 ts = arg_temp(op->args[i]);
2615 if (ts->state & TS_DEAD) {
2616 arg_life |= DEAD_ARG << i;
2617 }
2618 }
2619
2620 /* Input arguments are live for preceding opcodes. */
2621 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2622 ts = arg_temp(op->args[i]);
2623 if (ts->state & TS_DEAD) {
2624 /* For operands that were dead, initially allow
2625 all regs for the type. */
2626 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2627 ts->state &= ~TS_DEAD;
2628 }
2629 }
2630
2631 /* Incorporate constraints for this operand. */
2632 switch (opc) {
2633 case INDEX_op_mov_i32:
2634 case INDEX_op_mov_i64:
2635 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2636 have proper constraints. That said, special case
2637 moves to propagate preferences backward. */
2638 if (IS_DEAD_ARG(1)) {
2639 *la_temp_pref(arg_temp(op->args[0]))
2640 = *la_temp_pref(arg_temp(op->args[1]));
2641 }
2642 break;
2643
2644 default:
2645 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2646 const TCGArgConstraint *ct = &def->args_ct[i];
2647 TCGRegSet set, *pset;
2648
2649 ts = arg_temp(op->args[i]);
2650 pset = la_temp_pref(ts);
2651 set = *pset;
2652
2653 set &= ct->regs;
2654 if (ct->ialias) {
2655 set &= op->output_pref[ct->alias_index];
2656 }
2657 /* If the combination is not possible, restart. */
2658 if (set == 0) {
2659 set = ct->regs;
2660 }
2661 *pset = set;
2662 }
2663 break;
2664 }
2665 break;
2666 }
2667 op->life = arg_life;
2668 }
2669 }
2670
2671 /* Liveness analysis: Convert indirect regs to direct temporaries. */
2672 static bool liveness_pass_2(TCGContext *s)
2673 {
2674 int nb_globals = s->nb_globals;
2675 int nb_temps, i;
2676 bool changes = false;
2677 TCGOp *op, *op_next;
2678
2679 /* Create a temporary for each indirect global. */
2680 for (i = 0; i < nb_globals; ++i) {
2681 TCGTemp *its = &s->temps[i];
2682 if (its->indirect_reg) {
2683 TCGTemp *dts = tcg_temp_alloc(s);
2684 dts->type = its->type;
2685 dts->base_type = its->base_type;
2686 its->state_ptr = dts;
2687 } else {
2688 its->state_ptr = NULL;
2689 }
2690 /* All globals begin dead. */
2691 its->state = TS_DEAD;
2692 }
2693 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2694 TCGTemp *its = &s->temps[i];
2695 its->state_ptr = NULL;
2696 its->state = TS_DEAD;
2697 }
2698
2699 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2700 TCGOpcode opc = op->opc;
2701 const TCGOpDef *def = &tcg_op_defs[opc];
2702 TCGLifeData arg_life = op->life;
2703 int nb_iargs, nb_oargs, call_flags;
2704 TCGTemp *arg_ts, *dir_ts;
2705
2706 if (opc == INDEX_op_call) {
2707 nb_oargs = TCGOP_CALLO(op);
2708 nb_iargs = TCGOP_CALLI(op);
2709 call_flags = op->args[nb_oargs + nb_iargs + 1];
2710 } else {
2711 nb_iargs = def->nb_iargs;
2712 nb_oargs = def->nb_oargs;
2713
2714 /* Set flags similar to how calls require. */
2715 if (def->flags & TCG_OPF_COND_BRANCH) {
2716 /* Like reading globals: sync_globals */
2717 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2718 } else if (def->flags & TCG_OPF_BB_END) {
2719 /* Like writing globals: save_globals */
2720 call_flags = 0;
2721 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2722 /* Like reading globals: sync_globals */
2723 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2724 } else {
2725 /* No effect on globals. */
2726 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2727 TCG_CALL_NO_WRITE_GLOBALS);
2728 }
2729 }
2730
2731 /* Make sure that input arguments are available. */
2732 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2733 arg_ts = arg_temp(op->args[i]);
2734 if (arg_ts) {
2735 dir_ts = arg_ts->state_ptr;
2736 if (dir_ts && arg_ts->state == TS_DEAD) {
2737 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
2738 ? INDEX_op_ld_i32
2739 : INDEX_op_ld_i64);
2740 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
2741
2742 lop->args[0] = temp_arg(dir_ts);
2743 lop->args[1] = temp_arg(arg_ts->mem_base);
2744 lop->args[2] = arg_ts->mem_offset;
2745
2746 /* Loaded, but synced with memory. */
2747 arg_ts->state = TS_MEM;
2748 }
2749 }
2750 }
2751
2752 /* Perform input replacement, and mark inputs that became dead.
2753 No action is required except keeping temp_state up to date
2754 so that we reload when needed. */
2755 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2756 arg_ts = arg_temp(op->args[i]);
2757 if (arg_ts) {
2758 dir_ts = arg_ts->state_ptr;
2759 if (dir_ts) {
2760 op->args[i] = temp_arg(dir_ts);
2761 changes = true;
2762 if (IS_DEAD_ARG(i)) {
2763 arg_ts->state = TS_DEAD;
2764 }
2765 }
2766 }
2767 }
2768
2769 /* Liveness analysis should ensure that the following are
2770 all correct, for call sites and basic block end points. */
2771 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2772 /* Nothing to do */
2773 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2774 for (i = 0; i < nb_globals; ++i) {
2775 /* Liveness should see that globals are synced back,
2776 that is, either TS_DEAD or TS_MEM. */
2777 arg_ts = &s->temps[i];
2778 tcg_debug_assert(arg_ts->state_ptr == 0
2779 || arg_ts->state != 0);
2780 }
2781 } else {
2782 for (i = 0; i < nb_globals; ++i) {
2783 /* Liveness should see that globals are saved back,
2784 that is, TS_DEAD, waiting to be reloaded. */
2785 arg_ts = &s->temps[i];
2786 tcg_debug_assert(arg_ts->state_ptr == 0
2787 || arg_ts->state == TS_DEAD);
2788 }
2789 }
2790
2791 /* Outputs become available. */
2792 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2793 arg_ts = arg_temp(op->args[0]);
2794 dir_ts = arg_ts->state_ptr;
2795 if (dir_ts) {
2796 op->args[0] = temp_arg(dir_ts);
2797 changes = true;
2798
2799 /* The output is now live and modified. */
2800 arg_ts->state = 0;
2801
2802 if (NEED_SYNC_ARG(0)) {
2803 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2804 ? INDEX_op_st_i32
2805 : INDEX_op_st_i64);
2806 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2807 TCGTemp *out_ts = dir_ts;
2808
2809 if (IS_DEAD_ARG(0)) {
2810 out_ts = arg_temp(op->args[1]);
2811 arg_ts->state = TS_DEAD;
2812 tcg_op_remove(s, op);
2813 } else {
2814 arg_ts->state = TS_MEM;
2815 }
2816
2817 sop->args[0] = temp_arg(out_ts);
2818 sop->args[1] = temp_arg(arg_ts->mem_base);
2819 sop->args[2] = arg_ts->mem_offset;
2820 } else {
2821 tcg_debug_assert(!IS_DEAD_ARG(0));
2822 }
2823 }
2824 } else {
2825 for (i = 0; i < nb_oargs; i++) {
2826 arg_ts = arg_temp(op->args[i]);
2827 dir_ts = arg_ts->state_ptr;
2828 if (!dir_ts) {
2829 continue;
2830 }
2831 op->args[i] = temp_arg(dir_ts);
2832 changes = true;
2833
2834 /* The output is now live and modified. */
2835 arg_ts->state = 0;
2836
2837 /* Sync outputs upon their last write. */
2838 if (NEED_SYNC_ARG(i)) {
2839 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2840 ? INDEX_op_st_i32
2841 : INDEX_op_st_i64);
2842 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2843
2844 sop->args[0] = temp_arg(dir_ts);
2845 sop->args[1] = temp_arg(arg_ts->mem_base);
2846 sop->args[2] = arg_ts->mem_offset;
2847
2848 arg_ts->state = TS_MEM;
2849 }
2850 /* Drop outputs that are dead. */
2851 if (IS_DEAD_ARG(i)) {
2852 arg_ts->state = TS_DEAD;
2853 }
2854 }
2855 }
2856 }
2857
2858 return changes;
2859 }
2860
2861 #ifdef CONFIG_DEBUG_TCG
2862 static void dump_regs(TCGContext *s)
2863 {
2864 TCGTemp *ts;
2865 int i;
2866 char buf[64];
2867
2868 for(i = 0; i < s->nb_temps; i++) {
2869 ts = &s->temps[i];
2870 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
2871 switch(ts->val_type) {
2872 case TEMP_VAL_REG:
2873 printf("%s", tcg_target_reg_names[ts->reg]);
2874 break;
2875 case TEMP_VAL_MEM:
2876 printf("%d(%s)", (int)ts->mem_offset,
2877 tcg_target_reg_names[ts->mem_base->reg]);
2878 break;
2879 case TEMP_VAL_CONST:
2880 printf("$0x%" PRIx64, ts->val);
2881 break;
2882 case TEMP_VAL_DEAD:
2883 printf("D");
2884 break;
2885 default:
2886 printf("???");
2887 break;
2888 }
2889 printf("\n");
2890 }
2891
2892 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
2893 if (s->reg_to_temp[i] != NULL) {
2894 printf("%s: %s\n",
2895 tcg_target_reg_names[i],
2896 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
2897 }
2898 }
2899 }
2900
2901 static void check_regs(TCGContext *s)
2902 {
2903 int reg;
2904 int k;
2905 TCGTemp *ts;
2906 char buf[64];
2907
2908 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
2909 ts = s->reg_to_temp[reg];
2910 if (ts != NULL) {
2911 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
2912 printf("Inconsistency for register %s:\n",
2913 tcg_target_reg_names[reg]);
2914 goto fail;
2915 }
2916 }
2917 }
2918 for (k = 0; k < s->nb_temps; k++) {
2919 ts = &s->temps[k];
2920 if (ts->val_type == TEMP_VAL_REG
2921 && ts->kind != TEMP_FIXED
2922 && s->reg_to_temp[ts->reg] != ts) {
2923 printf("Inconsistency for temp %s:\n",
2924 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
2925 fail:
2926 printf("reg state:\n");
2927 dump_regs(s);
2928 tcg_abort();
2929 }
2930 }
2931 }
2932 #endif
2933
2934 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
2935 {
2936 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
2937 /* Sparc64 stack is accessed with offset of 2047 */
2938 s->current_frame_offset = (s->current_frame_offset +
2939 (tcg_target_long)sizeof(tcg_target_long) - 1) &
2940 ~(sizeof(tcg_target_long) - 1);
2941 #endif
2942 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
2943 s->frame_end) {
2944 tcg_abort();
2945 }
2946 ts->mem_offset = s->current_frame_offset;
2947 ts->mem_base = s->frame_temp;
2948 ts->mem_allocated = 1;
2949 s->current_frame_offset += sizeof(tcg_target_long);
2950 }
2951
2952 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
2953
2954 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
2955 mark it free; otherwise mark it dead. */
2956 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
2957 {
2958 TCGTempVal new_type;
2959
2960 switch (ts->kind) {
2961 case TEMP_FIXED:
2962 return;
2963 case TEMP_GLOBAL:
2964 case TEMP_LOCAL:
2965 new_type = TEMP_VAL_MEM;
2966 break;
2967 case TEMP_NORMAL:
2968 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
2969 break;
2970 case TEMP_CONST:
2971 new_type = TEMP_VAL_CONST;
2972 break;
2973 default:
2974 g_assert_not_reached();
2975 }
2976 if (ts->val_type == TEMP_VAL_REG) {
2977 s->reg_to_temp[ts->reg] = NULL;
2978 }
2979 ts->val_type = new_type;
2980 }
2981
2982 /* Mark a temporary as dead. */
2983 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
2984 {
2985 temp_free_or_dead(s, ts, 1);
2986 }
2987
2988 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
2989 registers needs to be allocated to store a constant. If 'free_or_dead'
2990 is non-zero, subsequently release the temporary; if it is positive, the
2991 temp is dead; if it is negative, the temp is free. */
2992 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
2993 TCGRegSet preferred_regs, int free_or_dead)
2994 {
2995 if (!temp_readonly(ts) && !ts->mem_coherent) {
2996 if (!ts->mem_allocated) {
2997 temp_allocate_frame(s, ts);
2998 }
2999 switch (ts->val_type) {
3000 case TEMP_VAL_CONST:
3001 /* If we're going to free the temp immediately, then we won't
3002 require it later in a register, so attempt to store the
3003 constant to memory directly. */
3004 if (free_or_dead
3005 && tcg_out_sti(s, ts->type, ts->val,
3006 ts->mem_base->reg, ts->mem_offset)) {
3007 break;
3008 }
3009 temp_load(s, ts, tcg_target_available_regs[ts->type],
3010 allocated_regs, preferred_regs);
3011 /* fallthrough */
3012
3013 case TEMP_VAL_REG:
3014 tcg_out_st(s, ts->type, ts->reg,
3015 ts->mem_base->reg, ts->mem_offset);
3016 break;
3017
3018 case TEMP_VAL_MEM:
3019 break;
3020
3021 case TEMP_VAL_DEAD:
3022 default:
3023 tcg_abort();
3024 }
3025 ts->mem_coherent = 1;
3026 }
3027 if (free_or_dead) {
3028 temp_free_or_dead(s, ts, free_or_dead);
3029 }
3030 }
3031
3032 /* free register 'reg' by spilling the corresponding temporary if necessary */
3033 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
3034 {
3035 TCGTemp *ts = s->reg_to_temp[reg];
3036 if (ts != NULL) {
3037 temp_sync(s, ts, allocated_regs, 0, -1);
3038 }
3039 }
3040
3041 /**
3042 * tcg_reg_alloc:
3043 * @required_regs: Set of registers in which we must allocate.
3044 * @allocated_regs: Set of registers which must be avoided.
3045 * @preferred_regs: Set of registers we should prefer.
3046 * @rev: True if we search the registers in "indirect" order.
3047 *
3048 * The allocated register must be in @required_regs & ~@allocated_regs,
3049 * but if we can put it in @preferred_regs we may save a move later.
3050 */
3051 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3052 TCGRegSet allocated_regs,
3053 TCGRegSet preferred_regs, bool rev)
3054 {
3055 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3056 TCGRegSet reg_ct[2];
3057 const int *order;
3058
3059 reg_ct[1] = required_regs & ~allocated_regs;
3060 tcg_debug_assert(reg_ct[1] != 0);
3061 reg_ct[0] = reg_ct[1] & preferred_regs;
3062
3063 /* Skip the preferred_regs option if it cannot be satisfied,
3064 or if the preference made no difference. */
3065 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3066
3067 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3068
3069 /* Try free registers, preferences first. */
3070 for (j = f; j < 2; j++) {
3071 TCGRegSet set = reg_ct[j];
3072
3073 if (tcg_regset_single(set)) {
3074 /* One register in the set. */
3075 TCGReg reg = tcg_regset_first(set);
3076 if (s->reg_to_temp[reg] == NULL) {
3077 return reg;
3078 }
3079 } else {
3080 for (i = 0; i < n; i++) {
3081 TCGReg reg = order[i];
3082 if (s->reg_to_temp[reg] == NULL &&
3083 tcg_regset_test_reg(set, reg)) {
3084 return reg;
3085 }
3086 }
3087 }
3088 }
3089
3090 /* We must spill something. */
3091 for (j = f; j < 2; j++) {
3092 TCGRegSet set = reg_ct[j];
3093
3094 if (tcg_regset_single(set)) {
3095 /* One register in the set. */
3096 TCGReg reg = tcg_regset_first(set);
3097 tcg_reg_free(s, reg, allocated_regs);
3098 return reg;
3099 } else {
3100 for (i = 0; i < n; i++) {
3101 TCGReg reg = order[i];
3102 if (tcg_regset_test_reg(set, reg)) {
3103 tcg_reg_free(s, reg, allocated_regs);
3104 return reg;
3105 }
3106 }
3107 }
3108 }
3109
3110 tcg_abort();
3111 }
3112
3113 /* Make sure the temporary is in a register. If needed, allocate the register
3114 from DESIRED while avoiding ALLOCATED. */
3115 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
3116 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
3117 {
3118 TCGReg reg;
3119
3120 switch (ts->val_type) {
3121 case TEMP_VAL_REG:
3122 return;
3123 case TEMP_VAL_CONST:
3124 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3125 preferred_regs, ts->indirect_base);
3126 if (ts->type <= TCG_TYPE_I64) {
3127 tcg_out_movi(s, ts->type, reg, ts->val);
3128 } else {
3129 uint64_t val = ts->val;
3130 MemOp vece = MO_64;
3131
3132 /*
3133 * Find the minimal vector element that matches the constant.
3134 * The targets will, in general, have to do this search anyway,
3135 * do this generically.
3136 */
3137 if (val == dup_const(MO_8, val)) {
3138 vece = MO_8;
3139 } else if (val == dup_const(MO_16, val)) {
3140 vece = MO_16;
3141 } else if (val == dup_const(MO_32, val)) {
3142 vece = MO_32;
3143 }
3144
3145 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
3146 }
3147 ts->mem_coherent = 0;
3148 break;
3149 case TEMP_VAL_MEM:
3150 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3151 preferred_regs, ts->indirect_base);
3152 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3153 ts->mem_coherent = 1;
3154 break;
3155 case TEMP_VAL_DEAD:
3156 default:
3157 tcg_abort();
3158 }
3159 ts->reg = reg;
3160 ts->val_type = TEMP_VAL_REG;
3161 s->reg_to_temp[reg] = ts;
3162 }
3163
3164 /* Save a temporary to memory. 'allocated_regs' is used in case a
3165 temporary registers needs to be allocated to store a constant. */
3166 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
3167 {
3168 /* The liveness analysis already ensures that globals are back
3169 in memory. Keep an tcg_debug_assert for safety. */
3170 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
3171 }
3172
3173 /* save globals to their canonical location and assume they can be
3174 modified be the following code. 'allocated_regs' is used in case a
3175 temporary registers needs to be allocated to store a constant. */
3176 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
3177 {
3178 int i, n;
3179
3180 for (i = 0, n = s->nb_globals; i < n; i++) {
3181 temp_save(s, &s->temps[i], allocated_regs);
3182 }
3183 }
3184
3185 /* sync globals to their canonical location and assume they can be
3186 read by the following code. 'allocated_regs' is used in case a
3187 temporary registers needs to be allocated to store a constant. */
3188 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3189 {
3190 int i, n;
3191
3192 for (i = 0, n = s->nb_globals; i < n; i++) {
3193 TCGTemp *ts = &s->temps[i];
3194 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
3195 || ts->kind == TEMP_FIXED
3196 || ts->mem_coherent);
3197 }
3198 }
3199
3200 /* at the end of a basic block, we assume all temporaries are dead and
3201 all globals are stored at their canonical location. */
3202 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
3203 {
3204 int i;
3205
3206 for (i = s->nb_globals; i < s->nb_temps; i++) {
3207 TCGTemp *ts = &s->temps[i];
3208
3209 switch (ts->kind) {
3210 case TEMP_LOCAL:
3211 temp_save(s, ts, allocated_regs);
3212 break;
3213 case TEMP_NORMAL:
3214 /* The liveness analysis already ensures that temps are dead.
3215 Keep an tcg_debug_assert for safety. */
3216 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3217 break;
3218 case TEMP_CONST:
3219 /* Similarly, we should have freed any allocated register. */
3220 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3221 break;
3222 default:
3223 g_assert_not_reached();
3224 }
3225 }
3226
3227 save_globals(s, allocated_regs);
3228 }
3229
3230 /*
3231 * At a conditional branch, we assume all temporaries are dead and
3232 * all globals and local temps are synced to their location.
3233 */
3234 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3235 {
3236 sync_globals(s, allocated_regs);
3237
3238 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3239 TCGTemp *ts = &s->temps[i];
3240 /*
3241 * The liveness analysis already ensures that temps are dead.
3242 * Keep tcg_debug_asserts for safety.
3243 */
3244 switch (ts->kind) {
3245 case TEMP_LOCAL:
3246 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
3247 break;
3248 case TEMP_NORMAL:
3249 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3250 break;
3251 case TEMP_CONST:
3252 break;
3253 default:
3254 g_assert_not_reached();
3255 }
3256 }
3257 }
3258
3259 /*
3260 * Specialized code generation for INDEX_op_mov_* with a constant.
3261 */
3262 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
3263 tcg_target_ulong val, TCGLifeData arg_life,
3264 TCGRegSet preferred_regs)
3265 {
3266 /* ENV should not be modified. */
3267 tcg_debug_assert(!temp_readonly(ots));
3268
3269 /* The movi is not explicitly generated here. */
3270 if (ots->val_type == TEMP_VAL_REG) {
3271 s->reg_to_temp[ots->reg] = NULL;
3272 }
3273 ots->val_type = TEMP_VAL_CONST;
3274 ots->val = val;
3275 ots->mem_coherent = 0;
3276 if (NEED_SYNC_ARG(0)) {
3277 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
3278 } else if (IS_DEAD_ARG(0)) {
3279 temp_dead(s, ots);
3280 }
3281 }
3282
3283 /*
3284 * Specialized code generation for INDEX_op_mov_*.
3285 */
3286 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
3287 {
3288 const TCGLifeData arg_life = op->life;
3289 TCGRegSet allocated_regs, preferred_regs;
3290 TCGTemp *ts, *ots;
3291 TCGType otype, itype;
3292
3293 allocated_regs = s->reserved_regs;
3294 preferred_regs = op->output_pref[0];
3295 ots = arg_temp(op->args[0]);
3296 ts = arg_temp(op->args[1]);
3297
3298 /* ENV should not be modified. */
3299 tcg_debug_assert(!temp_readonly(ots));
3300
3301 /* Note that otype != itype for no-op truncation. */
3302 otype = ots->type;
3303 itype = ts->type;
3304
3305 if (ts->val_type == TEMP_VAL_CONST) {
3306 /* propagate constant or generate sti */
3307 tcg_target_ulong val = ts->val;
3308 if (IS_DEAD_ARG(1)) {
3309 temp_dead(s, ts);
3310 }
3311 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
3312 return;
3313 }
3314
3315 /* If the source value is in memory we're going to be forced
3316 to have it in a register in order to perform the copy. Copy
3317 the SOURCE value into its own register first, that way we
3318 don't have to reload SOURCE the next time it is used. */
3319 if (ts->val_type == TEMP_VAL_MEM) {
3320 temp_load(s, ts, tcg_target_available_regs[itype],
3321 allocated_regs, preferred_regs);
3322 }
3323
3324 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
3325 if (IS_DEAD_ARG(0)) {
3326 /* mov to a non-saved dead register makes no sense (even with
3327 liveness analysis disabled). */
3328 tcg_debug_assert(NEED_SYNC_ARG(0));
3329 if (!ots->mem_allocated) {
3330 temp_allocate_frame(s, ots);
3331 }
3332 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
3333 if (IS_DEAD_ARG(1)) {
3334 temp_dead(s, ts);
3335 }
3336 temp_dead(s, ots);
3337 } else {
3338 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3339 /* the mov can be suppressed */
3340 if (ots->val_type == TEMP_VAL_REG) {
3341 s->reg_to_temp[ots->reg] = NULL;
3342 }
3343 ots->reg = ts->reg;
3344 temp_dead(s, ts);
3345 } else {
3346 if (ots->val_type != TEMP_VAL_REG) {
3347 /* When allocating a new register, make sure to not spill the
3348 input one. */
3349 tcg_regset_set_reg(allocated_regs, ts->reg);
3350 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3351 allocated_regs, preferred_regs,
3352 ots->indirect_base);
3353 }
3354 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
3355 /*
3356 * Cross register class move not supported.
3357 * Store the source register into the destination slot
3358 * and leave the destination temp as TEMP_VAL_MEM.
3359 */
3360 assert(!temp_readonly(ots));
3361 if (!ts->mem_allocated) {
3362 temp_allocate_frame(s, ots);
3363 }
3364 tcg_out_st(s, ts->type, ts->reg,
3365 ots->mem_base->reg, ots->mem_offset);
3366 ots->mem_coherent = 1;
3367 temp_free_or_dead(s, ots, -1);
3368 return;
3369 }
3370 }
3371 ots->val_type = TEMP_VAL_REG;
3372 ots->mem_coherent = 0;
3373 s->reg_to_temp[ots->reg] = ots;
3374 if (NEED_SYNC_ARG(0)) {
3375 temp_sync(s, ots, allocated_regs, 0, 0);
3376 }
3377 }
3378 }
3379
3380 /*
3381 * Specialized code generation for INDEX_op_dup_vec.
3382 */
3383 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3384 {
3385 const TCGLifeData arg_life = op->life;
3386 TCGRegSet dup_out_regs, dup_in_regs;
3387 TCGTemp *its, *ots;
3388 TCGType itype, vtype;
3389 intptr_t endian_fixup;
3390 unsigned vece;
3391 bool ok;
3392
3393 ots = arg_temp(op->args[0]);
3394 its = arg_temp(op->args[1]);
3395
3396 /* ENV should not be modified. */
3397 tcg_debug_assert(!temp_readonly(ots));
3398
3399 itype = its->type;
3400 vece = TCGOP_VECE(op);
3401 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3402
3403 if (its->val_type == TEMP_VAL_CONST) {
3404 /* Propagate constant via movi -> dupi. */
3405 tcg_target_ulong val = its->val;
3406 if (IS_DEAD_ARG(1)) {
3407 temp_dead(s, its);
3408 }
3409 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3410 return;
3411 }
3412
3413 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3414 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
3415
3416 /* Allocate the output register now. */
3417 if (ots->val_type != TEMP_VAL_REG) {
3418 TCGRegSet allocated_regs = s->reserved_regs;
3419
3420 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3421 /* Make sure to not spill the input register. */
3422 tcg_regset_set_reg(allocated_regs, its->reg);
3423 }
3424 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3425 op->output_pref[0], ots->indirect_base);
3426 ots->val_type = TEMP_VAL_REG;
3427 ots->mem_coherent = 0;
3428 s->reg_to_temp[ots->reg] = ots;
3429 }
3430
3431 switch (its->val_type) {
3432 case TEMP_VAL_REG:
3433 /*
3434 * The dup constriaints must be broad, covering all possible VECE.
3435 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3436 * to fail, indicating that extra moves are required for that case.
3437 */
3438 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3439 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3440 goto done;
3441 }
3442 /* Try again from memory or a vector input register. */
3443 }
3444 if (!its->mem_coherent) {
3445 /*
3446 * The input register is not synced, and so an extra store
3447 * would be required to use memory. Attempt an integer-vector
3448 * register move first. We do not have a TCGRegSet for this.
3449 */
3450 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3451 break;
3452 }
3453 /* Sync the temp back to its slot and load from there. */
3454 temp_sync(s, its, s->reserved_regs, 0, 0);
3455 }
3456 /* fall through */
3457
3458 case TEMP_VAL_MEM:
3459 #ifdef HOST_WORDS_BIGENDIAN
3460 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3461 endian_fixup -= 1 << vece;
3462 #else
3463 endian_fixup = 0;
3464 #endif
3465 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3466 its->mem_offset + endian_fixup)) {
3467 goto done;
3468 }
3469 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3470 break;
3471
3472 default:
3473 g_assert_not_reached();
3474 }
3475
3476 /* We now have a vector input register, so dup must succeed. */
3477 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3478 tcg_debug_assert(ok);
3479
3480 done:
3481 if (IS_DEAD_ARG(1)) {
3482 temp_dead(s, its);
3483 }
3484 if (NEED_SYNC_ARG(0)) {
3485 temp_sync(s, ots, s->reserved_regs, 0, 0);
3486 }
3487 if (IS_DEAD_ARG(0)) {
3488 temp_dead(s, ots);
3489 }
3490 }
3491
3492 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
3493 {
3494 const TCGLifeData arg_life = op->life;
3495 const TCGOpDef * const def = &tcg_op_defs[op->opc];
3496 TCGRegSet i_allocated_regs;
3497 TCGRegSet o_allocated_regs;
3498 int i, k, nb_iargs, nb_oargs;
3499 TCGReg reg;
3500 TCGArg arg;
3501 const TCGArgConstraint *arg_ct;
3502 TCGTemp *ts;
3503 TCGArg new_args[TCG_MAX_OP_ARGS];
3504 int const_args[TCG_MAX_OP_ARGS];
3505
3506 nb_oargs = def->nb_oargs;
3507 nb_iargs = def->nb_iargs;
3508
3509 /* copy constants */
3510 memcpy(new_args + nb_oargs + nb_iargs,
3511 op->args + nb_oargs + nb_iargs,
3512 sizeof(TCGArg) * def->nb_cargs);
3513
3514 i_allocated_regs = s->reserved_regs;
3515 o_allocated_regs = s->reserved_regs;
3516
3517 /* satisfy input constraints */
3518 for (k = 0; k < nb_iargs; k++) {
3519 TCGRegSet i_preferred_regs, o_preferred_regs;
3520
3521 i = def->args_ct[nb_oargs + k].sort_index;
3522 arg = op->args[i];
3523 arg_ct = &def->args_ct[i];
3524 ts = arg_temp(arg);
3525
3526 if (ts->val_type == TEMP_VAL_CONST
3527 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
3528 /* constant is OK for instruction */
3529 const_args[i] = 1;
3530 new_args[i] = ts->val;
3531 continue;
3532 }
3533
3534 i_preferred_regs = o_preferred_regs = 0;
3535 if (arg_ct->ialias) {
3536 o_preferred_regs = op->output_pref[arg_ct->alias_index];
3537
3538 /*
3539 * If the input is readonly, then it cannot also be an
3540 * output and aliased to itself. If the input is not
3541 * dead after the instruction, we must allocate a new
3542 * register and move it.
3543 */
3544 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3545 goto allocate_in_reg;
3546 }
3547
3548 /*
3549 * Check if the current register has already been allocated
3550 * for another input aliased to an output.
3551 */
3552 if (ts->val_type == TEMP_VAL_REG) {
3553 reg = ts->reg;
3554 for (int k2 = 0; k2 < k; k2++) {
3555 int i2 = def->args_ct[nb_oargs + k2].sort_index;
3556 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
3557 goto allocate_in_reg;
3558 }
3559 }
3560 }
3561 i_preferred_regs = o_preferred_regs;
3562 }
3563
3564 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
3565 reg = ts->reg;
3566
3567 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
3568 allocate_in_reg:
3569 /*
3570 * Allocate a new register matching the constraint
3571 * and move the temporary register into it.
3572 */
3573 temp_load(s, ts, tcg_target_available_regs[ts->type],
3574 i_allocated_regs, 0);
3575 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
3576 o_preferred_regs, ts->indirect_base);
3577 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
3578 /*
3579 * Cross register class move not supported. Sync the
3580 * temp back to its slot and load from there.
3581 */
3582 temp_sync(s, ts, i_allocated_regs, 0, 0);
3583 tcg_out_ld(s, ts->type, reg,
3584 ts->mem_base->reg, ts->mem_offset);
3585 }
3586 }
3587 new_args[i] = reg;
3588 const_args[i] = 0;
3589 tcg_regset_set_reg(i_allocated_regs, reg);
3590 }
3591
3592 /* mark dead temporaries and free the associated registers */
3593 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3594 if (IS_DEAD_ARG(i)) {
3595 temp_dead(s, arg_temp(op->args[i]));
3596 }
3597 }
3598
3599 if (def->flags & TCG_OPF_COND_BRANCH) {
3600 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3601 } else if (def->flags & TCG_OPF_BB_END) {
3602 tcg_reg_alloc_bb_end(s, i_allocated_regs);
3603 } else {
3604 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3605 /* XXX: permit generic clobber register list ? */
3606 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3607 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
3608 tcg_reg_free(s, i, i_allocated_regs);
3609 }
3610 }
3611 }
3612 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3613 /* sync globals if the op has side effects and might trigger
3614 an exception. */
3615 sync_globals(s, i_allocated_regs);
3616 }
3617
3618 /* satisfy the output constraints */
3619 for(k = 0; k < nb_oargs; k++) {
3620 i = def->args_ct[k].sort_index;
3621 arg = op->args[i];
3622 arg_ct = &def->args_ct[i];
3623 ts = arg_temp(arg);
3624
3625 /* ENV should not be modified. */
3626 tcg_debug_assert(!temp_readonly(ts));
3627
3628 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
3629 reg = new_args[arg_ct->alias_index];
3630 } else if (arg_ct->newreg) {
3631 reg = tcg_reg_alloc(s, arg_ct->regs,
3632 i_allocated_regs | o_allocated_regs,
3633 op->output_pref[k], ts->indirect_base);
3634 } else {
3635 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
3636 op->output_pref[k], ts->indirect_base);
3637 }
3638 tcg_regset_set_reg(o_allocated_regs, reg);
3639 if (ts->val_type == TEMP_VAL_REG) {
3640 s->reg_to_temp[ts->reg] = NULL;
3641 }
3642 ts->val_type = TEMP_VAL_REG;
3643 ts->reg = reg;
3644 /*
3645 * Temp value is modified, so the value kept in memory is
3646 * potentially not the same.
3647 */
3648 ts->mem_coherent = 0;
3649 s->reg_to_temp[reg] = ts;
3650 new_args[i] = reg;
3651 }
3652 }
3653
3654 /* emit instruction */
3655 if (def->flags & TCG_OPF_VECTOR) {
3656 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3657 new_args, const_args);
3658 } else {
3659 tcg_out_op(s, op->opc, new_args, const_args);
3660 }
3661
3662 /* move the outputs in the correct register if needed */
3663 for(i = 0; i < nb_oargs; i++) {
3664 ts = arg_temp(op->args[i]);
3665
3666 /* ENV should not be modified. */
3667 tcg_debug_assert(!temp_readonly(ts));
3668
3669 if (NEED_SYNC_ARG(i)) {
3670 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
3671 } else if (IS_DEAD_ARG(i)) {
3672 temp_dead(s, ts);
3673 }
3674 }
3675 }
3676
3677 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3678 {
3679 const TCGLifeData arg_life = op->life;
3680 TCGTemp *ots, *itsl, *itsh;
3681 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3682
3683 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3684 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3685 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3686
3687 ots = arg_temp(op->args[0]);
3688 itsl = arg_temp(op->args[1]);
3689 itsh = arg_temp(op->args[2]);
3690
3691 /* ENV should not be modified. */
3692 tcg_debug_assert(!temp_readonly(ots));
3693
3694 /* Allocate the output register now. */
3695 if (ots->val_type != TEMP_VAL_REG) {
3696 TCGRegSet allocated_regs = s->reserved_regs;
3697 TCGRegSet dup_out_regs =
3698 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3699
3700 /* Make sure to not spill the input registers. */
3701 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3702 tcg_regset_set_reg(allocated_regs, itsl->reg);
3703 }
3704 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3705 tcg_regset_set_reg(allocated_regs, itsh->reg);
3706 }
3707
3708 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3709 op->output_pref[0], ots->indirect_base);
3710 ots->val_type = TEMP_VAL_REG;
3711 ots->mem_coherent = 0;
3712 s->reg_to_temp[ots->reg] = ots;
3713 }
3714
3715 /* Promote dup2 of immediates to dupi_vec. */
3716 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3717 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3718 MemOp vece = MO_64;
3719
3720 if (val == dup_const(MO_8, val)) {
3721 vece = MO_8;
3722 } else if (val == dup_const(MO_16, val)) {
3723 vece = MO_16;
3724 } else if (val == dup_const(MO_32, val)) {
3725 vece = MO_32;
3726 }
3727
3728 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3729 goto done;
3730 }
3731
3732 /* If the two inputs form one 64-bit value, try dupm_vec. */
3733 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3734 if (!itsl->mem_coherent) {
3735 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3736 }
3737 if (!itsh->mem_coherent) {
3738 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3739 }
3740 #ifdef HOST_WORDS_BIGENDIAN
3741 TCGTemp *its = itsh;
3742 #else
3743 TCGTemp *its = itsl;
3744 #endif
3745 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3746 its->mem_base->reg, its->mem_offset)) {
3747 goto done;
3748 }
3749 }
3750
3751 /* Fall back to generic expansion. */
3752 return false;
3753
3754 done:
3755 if (IS_DEAD_ARG(1)) {
3756 temp_dead(s, itsl);
3757 }
3758 if (IS_DEAD_ARG(2)) {
3759 temp_dead(s, itsh);
3760 }
3761 if (NEED_SYNC_ARG(0)) {
3762 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3763 } else if (IS_DEAD_ARG(0)) {
3764 temp_dead(s, ots);
3765 }
3766 return true;
3767 }
3768
3769 #ifdef TCG_TARGET_STACK_GROWSUP
3770 #define STACK_DIR(x) (-(x))
3771 #else
3772 #define STACK_DIR(x) (x)
3773 #endif
3774
3775 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
3776 {
3777 const int nb_oargs = TCGOP_CALLO(op);
3778 const int nb_iargs = TCGOP_CALLI(op);
3779 const TCGLifeData arg_life = op->life;
3780 int flags, nb_regs, i;
3781 TCGReg reg;
3782 TCGArg arg;
3783 TCGTemp *ts;
3784 intptr_t stack_offset;
3785 size_t call_stack_size;
3786 tcg_insn_unit *func_addr;
3787 int allocate_args;
3788 TCGRegSet allocated_regs;
3789
3790 func_addr = (tcg_insn_unit *)(intptr_t)op->args[nb_oargs + nb_iargs];
3791 flags = op->args[nb_oargs + nb_iargs + 1];
3792
3793 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
3794 if (nb_regs > nb_iargs) {
3795 nb_regs = nb_iargs;
3796 }
3797
3798 /* assign stack slots first */
3799 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
3800 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
3801 ~(TCG_TARGET_STACK_ALIGN - 1);
3802 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3803 if (allocate_args) {
3804 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3805 preallocate call stack */
3806 tcg_abort();
3807 }
3808
3809 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
3810 for (i = nb_regs; i < nb_iargs; i++) {
3811 arg = op->args[nb_oargs + i];
3812 #ifdef TCG_TARGET_STACK_GROWSUP
3813 stack_offset -= sizeof(tcg_target_long);
3814 #endif
3815 if (arg != TCG_CALL_DUMMY_ARG) {
3816 ts = arg_temp(arg);
3817 temp_load(s, ts, tcg_target_available_regs[ts->type],
3818 s->reserved_regs, 0);
3819 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
3820 }
3821 #ifndef TCG_TARGET_STACK_GROWSUP
3822 stack_offset += sizeof(tcg_target_long);
3823 #endif
3824 }
3825
3826 /* assign input registers */
3827 allocated_regs = s->reserved_regs;
3828 for (i = 0; i < nb_regs; i++) {
3829 arg = op->args[nb_oargs + i];
3830 if (arg != TCG_CALL_DUMMY_ARG) {
3831 ts = arg_temp(arg);
3832 reg = tcg_target_call_iarg_regs[i];
3833
3834 if (ts->val_type == TEMP_VAL_REG) {
3835 if (ts->reg != reg) {
3836 tcg_reg_free(s, reg, allocated_regs);
3837 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
3838 /*
3839 * Cross register class move not supported. Sync the
3840 * temp back to its slot and load from there.
3841 */
3842 temp_sync(s, ts, allocated_regs, 0, 0);
3843 tcg_out_ld(s, ts->type, reg,
3844 ts->mem_base->reg, ts->mem_offset);
3845 }
3846 }
3847 } else {
3848 TCGRegSet arg_set = 0;
3849
3850 tcg_reg_free(s, reg, allocated_regs);
3851 tcg_regset_set_reg(arg_set, reg);
3852 temp_load(s, ts, arg_set, allocated_regs, 0);
3853 }
3854
3855 tcg_regset_set_reg(allocated_regs, reg);
3856 }
3857 }
3858
3859 /* mark dead temporaries and free the associated registers */
3860 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3861 if (IS_DEAD_ARG(i)) {
3862 temp_dead(s, arg_temp(op->args[i]));
3863 }
3864 }
3865
3866 /* clobber call registers */
3867 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3868 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
3869 tcg_reg_free(s, i, allocated_regs);
3870 }
3871 }
3872
3873 /* Save globals if they might be written by the helper, sync them if
3874 they might be read. */
3875 if (flags & TCG_CALL_NO_READ_GLOBALS) {
3876 /* Nothing to do */
3877 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
3878 sync_globals(s, allocated_regs);
3879 } else {
3880 save_globals(s, allocated_regs);
3881 }
3882
3883 tcg_out_call(s, func_addr);
3884
3885 /* assign output registers and emit moves if needed */
3886 for(i = 0; i < nb_oargs; i++) {
3887 arg = op->args[i];
3888 ts = arg_temp(arg);
3889
3890 /* ENV should not be modified. */
3891 tcg_debug_assert(!temp_readonly(ts));
3892
3893 reg = tcg_target_call_oarg_regs[i];
3894 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3895 if (ts->val_type == TEMP_VAL_REG) {
3896 s->reg_to_temp[ts->reg] = NULL;
3897 }
3898 ts->val_type = TEMP_VAL_REG;
3899 ts->reg = reg;
3900 ts->mem_coherent = 0;
3901 s->reg_to_temp[reg] = ts;
3902 if (NEED_SYNC_ARG(i)) {
3903 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
3904 } else if (IS_DEAD_ARG(i)) {
3905 temp_dead(s, ts);
3906 }
3907 }
3908 }
3909
3910 #ifdef CONFIG_PROFILER
3911
3912 /* avoid copy/paste errors */
3913 #define PROF_ADD(to, from, field) \
3914 do { \
3915 (to)->field += qatomic_read(&((from)->field)); \
3916 } while (0)
3917
3918 #define PROF_MAX(to, from, field) \
3919 do { \
3920 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
3921 if (val__ > (to)->field) { \
3922 (to)->field = val__; \
3923 } \
3924 } while (0)
3925
3926 /* Pass in a zero'ed @prof */
3927 static inline
3928 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
3929 {
3930 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
3931 unsigned int i;
3932
3933 for (i = 0; i < n_ctxs; i++) {
3934 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3935 const TCGProfile *orig = &s->prof;
3936
3937 if (counters) {
3938 PROF_ADD(prof, orig, cpu_exec_time);
3939 PROF_ADD(prof, orig, tb_count1);
3940 PROF_ADD(prof, orig, tb_count);
3941 PROF_ADD(prof, orig, op_count);
3942 PROF_MAX(prof, orig, op_count_max);
3943 PROF_ADD(prof, orig, temp_count);
3944 PROF_MAX(prof, orig, temp_count_max);
3945 PROF_ADD(prof, orig, del_op_count);
3946 PROF_ADD(prof, orig, code_in_len);
3947 PROF_ADD(prof, orig, code_out_len);
3948 PROF_ADD(prof, orig, search_out_len);
3949 PROF_ADD(prof, orig, interm_time);
3950 PROF_ADD(prof, orig, code_time);
3951 PROF_ADD(prof, orig, la_time);
3952 PROF_ADD(prof, orig, opt_time);
3953 PROF_ADD(prof, orig, restore_count);
3954 PROF_ADD(prof, orig, restore_time);
3955 }
3956 if (table) {
3957 int i;
3958
3959 for (i = 0; i < NB_OPS; i++) {
3960 PROF_ADD(prof, orig, table_op_count[i]);
3961 }
3962 }
3963 }
3964 }
3965
3966 #undef PROF_ADD
3967 #undef PROF_MAX
3968
3969 static void tcg_profile_snapshot_counters(TCGProfile *prof)
3970 {
3971 tcg_profile_snapshot(prof, true, false);
3972 }
3973
3974 static void tcg_profile_snapshot_table(TCGProfile *prof)
3975 {
3976 tcg_profile_snapshot(prof, false, true);
3977 }
3978
3979 void tcg_dump_op_count(void)
3980 {
3981 TCGProfile prof = {};
3982 int i;
3983
3984 tcg_profile_snapshot_table(&prof);
3985 for (i = 0; i < NB_OPS; i++) {
3986 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name,
3987 prof.table_op_count[i]);
3988 }
3989 }
3990
3991 int64_t tcg_cpu_exec_time(void)
3992 {
3993 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
3994 unsigned int i;
3995 int64_t ret = 0;
3996
3997 for (i = 0; i < n_ctxs; i++) {
3998 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3999 const TCGProfile *prof = &s->prof;
4000
4001 ret += qatomic_read(&prof->cpu_exec_time);
4002 }
4003 return ret;
4004 }
4005 #else
4006 void tcg_dump_op_count(void)
4007 {
4008 qemu_printf("[TCG profiler not compiled]\n");
4009 }
4010
4011 int64_t tcg_cpu_exec_time(void)
4012 {
4013 error_report("%s: TCG profiler not compiled", __func__);
4014 exit(EXIT_FAILURE);
4015 }
4016 #endif
4017
4018
4019 int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
4020 {
4021 #ifdef CONFIG_PROFILER
4022 TCGProfile *prof = &s->prof;
4023 #endif
4024 int i, num_insns;
4025 TCGOp *op;
4026
4027 #ifdef CONFIG_PROFILER
4028 {
4029 int n = 0;
4030
4031 QTAILQ_FOREACH(op, &s->ops, link) {
4032 n++;
4033 }
4034 qatomic_set(&prof->op_count, prof->op_count + n);
4035 if (n > prof->op_count_max) {
4036 qatomic_set(&prof->op_count_max, n);
4037 }
4038
4039 n = s->nb_temps;
4040 qatomic_set(&prof->temp_count, prof->temp_count + n);
4041 if (n > prof->temp_count_max) {
4042 qatomic_set(&prof->temp_count_max, n);
4043 }
4044 }
4045 #endif
4046
4047 #ifdef DEBUG_DISAS
4048 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4049 && qemu_log_in_addr_range(tb->pc))) {
4050 FILE *logfile = qemu_log_lock();
4051 qemu_log("OP:\n");
4052 tcg_dump_ops(s, false);
4053 qemu_log("\n");
4054 qemu_log_unlock(logfile);
4055 }
4056 #endif
4057
4058 #ifdef CONFIG_DEBUG_TCG
4059 /* Ensure all labels referenced have been emitted. */
4060 {
4061 TCGLabel *l;
4062 bool error = false;
4063
4064 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4065 if (unlikely(!l->present) && l->refs) {
4066 qemu_log_mask(CPU_LOG_TB_OP,
4067 "$L%d referenced but not present.\n", l->id);
4068 error = true;
4069 }
4070 }
4071 assert(!error);
4072 }
4073 #endif
4074
4075 #ifdef CONFIG_PROFILER
4076 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
4077 #endif
4078
4079 #ifdef USE_TCG_OPTIMIZATIONS
4080 tcg_optimize(s);
4081 #endif
4082
4083 #ifdef CONFIG_PROFILER
4084 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4085 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
4086 #endif
4087
4088 reachable_code_pass(s);
4089 liveness_pass_1(s);
4090
4091 if (s->nb_indirects > 0) {
4092 #ifdef DEBUG_DISAS
4093 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4094 && qemu_log_in_addr_range(tb->pc))) {
4095 FILE *logfile = qemu_log_lock();
4096 qemu_log("OP before indirect lowering:\n");
4097 tcg_dump_ops(s, false);
4098 qemu_log("\n");
4099 qemu_log_unlock(logfile);
4100 }
4101 #endif
4102 /* Replace indirect temps with direct temps. */
4103 if (liveness_pass_2(s)) {
4104 /* If changes were made, re-run liveness. */
4105 liveness_pass_1(s);
4106 }
4107 }
4108
4109 #ifdef CONFIG_PROFILER
4110 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
4111 #endif
4112
4113 #ifdef DEBUG_DISAS
4114 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4115 && qemu_log_in_addr_range(tb->pc))) {
4116 FILE *logfile = qemu_log_lock();
4117 qemu_log("OP after optimization and liveness analysis:\n");
4118 tcg_dump_ops(s, true);
4119 qemu_log("\n");
4120 qemu_log_unlock(logfile);
4121 }
4122 #endif
4123
4124 tcg_reg_alloc_start(s);
4125
4126 /*
4127 * Reset the buffer pointers when restarting after overflow.
4128 * TODO: Move this into translate-all.c with the rest of the
4129 * buffer management. Having only this done here is confusing.
4130 */
4131 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4132 s->code_ptr = s->code_buf;
4133
4134 #ifdef TCG_TARGET_NEED_LDST_LABELS
4135 QSIMPLEQ_INIT(&s->ldst_labels);
4136 #endif
4137 #ifdef TCG_TARGET_NEED_POOL_LABELS
4138 s->pool_labels = NULL;
4139 #endif
4140
4141 num_insns = -1;
4142 QTAILQ_FOREACH(op, &s->ops, link) {
4143 TCGOpcode opc = op->opc;
4144
4145 #ifdef CONFIG_PROFILER
4146 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
4147 #endif
4148
4149 switch (opc) {
4150 case INDEX_op_mov_i32:
4151 case INDEX_op_mov_i64:
4152 case INDEX_op_mov_vec:
4153 tcg_reg_alloc_mov(s, op);
4154 break;
4155 case INDEX_op_dup_vec:
4156 tcg_reg_alloc_dup(s, op);
4157 break;
4158 case INDEX_op_insn_start:
4159 if (num_insns >= 0) {
4160 size_t off = tcg_current_code_size(s);
4161 s->gen_insn_end_off[num_insns] = off;
4162 /* Assert that we do not overflow our stored offset. */
4163 assert(s->gen_insn_end_off[num_insns] == off);
4164 }
4165 num_insns++;
4166 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4167 target_ulong a;
4168 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4169 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
4170 #else
4171 a = op->args[i];
4172 #endif
4173 s->gen_insn_data[num_insns][i] = a;
4174 }
4175 break;
4176 case INDEX_op_discard:
4177 temp_dead(s, arg_temp(op->args[0]));
4178 break;
4179 case INDEX_op_set_label:
4180 tcg_reg_alloc_bb_end(s, s->reserved_regs);
4181 tcg_out_label(s, arg_label(op->args[0]));
4182 break;
4183 case INDEX_op_call:
4184 tcg_reg_alloc_call(s, op);
4185 break;
4186 case INDEX_op_dup2_vec:
4187 if (tcg_reg_alloc_dup2(s, op)) {
4188 break;
4189 }
4190 /* fall through */
4191 default:
4192 /* Sanity check that we've not introduced any unhandled opcodes. */
4193 tcg_debug_assert(tcg_op_supported(opc));
4194 /* Note: in order to speed up the code, it would be much
4195 faster to have specialized register allocator functions for
4196 some common argument patterns */
4197 tcg_reg_alloc_op(s, op);
4198 break;
4199 }
4200 #ifdef CONFIG_DEBUG_TCG
4201 check_regs(s);
4202 #endif
4203 /* Test for (pending) buffer overflow. The assumption is that any
4204 one operation beginning below the high water mark cannot overrun
4205 the buffer completely. Thus we can test for overflow after
4206 generating code without having to check during generation. */
4207 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
4208 return -1;
4209 }
4210 /* Test for TB overflow, as seen by gen_insn_end_off. */
4211 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4212 return -2;
4213 }
4214 }
4215 tcg_debug_assert(num_insns >= 0);
4216 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
4217
4218 /* Generate TB finalization at the end of block */
4219 #ifdef TCG_TARGET_NEED_LDST_LABELS
4220 i = tcg_out_ldst_finalize(s);
4221 if (i < 0) {
4222 return i;
4223 }
4224 #endif
4225 #ifdef TCG_TARGET_NEED_POOL_LABELS
4226 i = tcg_out_pool_finalize(s);
4227 if (i < 0) {
4228 return i;
4229 }
4230 #endif
4231 if (!tcg_resolve_relocs(s)) {
4232 return -2;
4233 }
4234
4235 #ifndef CONFIG_TCG_INTERPRETER
4236 /* flush instruction cache */
4237 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4238 (uintptr_t)s->code_buf,
4239 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
4240 #endif
4241
4242 return tcg_current_code_size(s);
4243 }
4244
4245 #ifdef CONFIG_PROFILER
4246 void tcg_dump_info(void)
4247 {
4248 TCGProfile prof = {};
4249 const TCGProfile *s;
4250 int64_t tb_count;
4251 int64_t tb_div_count;
4252 int64_t tot;
4253
4254 tcg_profile_snapshot_counters(&prof);
4255 s = &prof;
4256 tb_count = s->tb_count;
4257 tb_div_count = tb_count ? tb_count : 1;
4258 tot = s->interm_time + s->code_time;
4259
4260 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
4261 tot, tot / 2.4e9);
4262 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64
4263 " %0.1f%%)\n",
4264 tb_count, s->tb_count1 - tb_count,
4265 (double)(s->tb_count1 - s->tb_count)
4266 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4267 qemu_printf("avg ops/TB %0.1f max=%d\n",
4268 (double)s->op_count / tb_div_count, s->op_count_max);
4269 qemu_printf("deleted ops/TB %0.2f\n",
4270 (double)s->del_op_count / tb_div_count);
4271 qemu_printf("avg temps/TB %0.2f max=%d\n",
4272 (double)s->temp_count / tb_div_count, s->temp_count_max);
4273 qemu_printf("avg host code/TB %0.1f\n",
4274 (double)s->code_out_len / tb_div_count);
4275 qemu_printf("avg search data/TB %0.1f\n",
4276 (double)s->search_out_len / tb_div_count);
4277
4278 qemu_printf("cycles/op %0.1f\n",
4279 s->op_count ? (double)tot / s->op_count : 0);
4280 qemu_printf("cycles/in byte %0.1f\n",
4281 s->code_in_len ? (double)tot / s->code_in_len : 0);
4282 qemu_printf("cycles/out byte %0.1f\n",
4283 s->code_out_len ? (double)tot / s->code_out_len : 0);
4284 qemu_printf("cycles/search byte %0.1f\n",
4285 s->search_out_len ? (double)tot / s->search_out_len : 0);
4286 if (tot == 0) {
4287 tot = 1;
4288 }
4289 qemu_printf(" gen_interm time %0.1f%%\n",
4290 (double)s->interm_time / tot * 100.0);
4291 qemu_printf(" gen_code time %0.1f%%\n",
4292 (double)s->code_time / tot * 100.0);
4293 qemu_printf("optim./code time %0.1f%%\n",
4294 (double)s->opt_time / (s->code_time ? s->code_time : 1)
4295 * 100.0);
4296 qemu_printf("liveness/code time %0.1f%%\n",
4297 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
4298 qemu_printf("cpu_restore count %" PRId64 "\n",
4299 s->restore_count);
4300 qemu_printf(" avg cycles %0.1f\n",
4301 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
4302 }
4303 #else
4304 void tcg_dump_info(void)
4305 {
4306 qemu_printf("[TCG profiler not compiled]\n");
4307 }
4308 #endif
4309
4310 #ifdef ELF_HOST_MACHINE
4311 /* In order to use this feature, the backend needs to do three things:
4312
4313 (1) Define ELF_HOST_MACHINE to indicate both what value to
4314 put into the ELF image and to indicate support for the feature.
4315
4316 (2) Define tcg_register_jit. This should create a buffer containing
4317 the contents of a .debug_frame section that describes the post-
4318 prologue unwind info for the tcg machine.
4319
4320 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4321 */
4322
4323 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4324 typedef enum {
4325 JIT_NOACTION = 0,
4326 JIT_REGISTER_FN,
4327 JIT_UNREGISTER_FN
4328 } jit_actions_t;
4329
4330 struct jit_code_entry {
4331 struct jit_code_entry *next_entry;
4332 struct jit_code_entry *prev_entry;
4333 const void *symfile_addr;
4334 uint64_t symfile_size;
4335 };
4336
4337 struct jit_descriptor {
4338 uint32_t version;
4339 uint32_t action_flag;
4340 struct jit_code_entry *relevant_entry;
4341 struct jit_code_entry *first_entry;
4342 };
4343
4344 void __jit_debug_register_code(void) __attribute__((noinline));
4345 void __jit_debug_register_code(void)
4346 {
4347 asm("");
4348 }
4349
4350 /* Must statically initialize the version, because GDB may check
4351 the version before we can set it. */
4352 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4353
4354 /* End GDB interface. */
4355
4356 static int find_string(const char *strtab, const char *str)
4357 {
4358 const char *p = strtab + 1;
4359
4360 while (1) {
4361 if (strcmp(p, str) == 0) {
4362 return p - strtab;
4363 }
4364 p += strlen(p) + 1;
4365 }
4366 }
4367
4368 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
4369 const void *debug_frame,
4370 size_t debug_frame_size)
4371 {
4372 struct __attribute__((packed)) DebugInfo {
4373 uint32_t len;
4374 uint16_t version;
4375 uint32_t abbrev;
4376 uint8_t ptr_size;
4377 uint8_t cu_die;
4378 uint16_t cu_lang;
4379 uintptr_t cu_low_pc;
4380 uintptr_t cu_high_pc;
4381 uint8_t fn_die;
4382 char fn_name[16];
4383 uintptr_t fn_low_pc;
4384 uintptr_t fn_high_pc;
4385 uint8_t cu_eoc;
4386 };
4387
4388 struct ElfImage {
4389 ElfW(Ehdr) ehdr;
4390 ElfW(Phdr) phdr;
4391 ElfW(Shdr) shdr[7];
4392 ElfW(Sym) sym[2];
4393 struct DebugInfo di;
4394 uint8_t da[24];
4395 char str[80];
4396 };
4397
4398 struct ElfImage *img;
4399
4400 static const struct ElfImage img_template = {
4401 .ehdr = {
4402 .e_ident[EI_MAG0] = ELFMAG0,
4403 .e_ident[EI_MAG1] = ELFMAG1,
4404 .e_ident[EI_MAG2] = ELFMAG2,
4405 .e_ident[EI_MAG3] = ELFMAG3,
4406 .e_ident[EI_CLASS] = ELF_CLASS,
4407 .e_ident[EI_DATA] = ELF_DATA,
4408 .e_ident[EI_VERSION] = EV_CURRENT,
4409 .e_type = ET_EXEC,
4410 .e_machine = ELF_HOST_MACHINE,
4411 .e_version = EV_CURRENT,
4412 .e_phoff = offsetof(struct ElfImage, phdr),
4413 .e_shoff = offsetof(struct ElfImage, shdr),
4414 .e_ehsize = sizeof(ElfW(Shdr)),
4415 .e_phentsize = sizeof(ElfW(Phdr)),
4416 .e_phnum = 1,
4417 .e_shentsize = sizeof(ElfW(Shdr)),
4418 .e_shnum = ARRAY_SIZE(img->shdr),
4419 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
4420 #ifdef ELF_HOST_FLAGS
4421 .e_flags = ELF_HOST_FLAGS,
4422 #endif
4423 #ifdef ELF_OSABI
4424 .e_ident[EI_OSABI] = ELF_OSABI,
4425 #endif
4426 },
4427 .phdr = {
4428 .p_type = PT_LOAD,
4429 .p_flags = PF_X,
4430 },
4431 .shdr = {
4432 [0] = { .sh_type = SHT_NULL },
4433 /* Trick: The contents of code_gen_buffer are not present in
4434 this fake ELF file; that got allocated elsewhere. Therefore
4435 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4436 will not look for contents. We can record any address. */
4437 [1] = { /* .text */
4438 .sh_type = SHT_NOBITS,
4439 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4440 },
4441 [2] = { /* .debug_info */
4442 .sh_type = SHT_PROGBITS,
4443 .sh_offset = offsetof(struct ElfImage, di),
4444 .sh_size = sizeof(struct DebugInfo),
4445 },
4446 [3] = { /* .debug_abbrev */
4447 .sh_type = SHT_PROGBITS,
4448 .sh_offset = offsetof(struct ElfImage, da),
4449 .sh_size = sizeof(img->da),
4450 },
4451 [4] = { /* .debug_frame */
4452 .sh_type = SHT_PROGBITS,
4453 .sh_offset = sizeof(struct ElfImage),
4454 },
4455 [5] = { /* .symtab */
4456 .sh_type = SHT_SYMTAB,
4457 .sh_offset = offsetof(struct ElfImage, sym),
4458 .sh_size = sizeof(img->sym),
4459 .sh_info = 1,
4460 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4461 .sh_entsize = sizeof(ElfW(Sym)),
4462 },
4463 [6] = { /* .strtab */
4464 .sh_type = SHT_STRTAB,
4465 .sh_offset = offsetof(struct ElfImage, str),
4466 .sh_size = sizeof(img->str),
4467 }
4468 },
4469 .sym = {
4470 [1] = { /* code_gen_buffer */
4471 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4472 .st_shndx = 1,
4473 }
4474 },
4475 .di = {
4476 .len = sizeof(struct DebugInfo) - 4,
4477 .version = 2,
4478 .ptr_size = sizeof(void *),
4479 .cu_die = 1,
4480 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4481 .fn_die = 2,
4482 .fn_name = "code_gen_buffer"
4483 },
4484 .da = {
4485 1, /* abbrev number (the cu) */
4486 0x11, 1, /* DW_TAG_compile_unit, has children */
4487 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4488 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4489 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4490 0, 0, /* end of abbrev */
4491 2, /* abbrev number (the fn) */
4492 0x2e, 0, /* DW_TAG_subprogram, no children */
4493 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4494 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4495 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4496 0, 0, /* end of abbrev */
4497 0 /* no more abbrev */
4498 },
4499 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4500 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
4501 };
4502
4503 /* We only need a single jit entry; statically allocate it. */
4504 static struct jit_code_entry one_entry;
4505
4506 uintptr_t buf = (uintptr_t)buf_ptr;
4507 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
4508 DebugFrameHeader *dfh;
4509
4510 img = g_malloc(img_size);
4511 *img = img_template;
4512
4513 img->phdr.p_vaddr = buf;
4514 img->phdr.p_paddr = buf;
4515 img->phdr.p_memsz = buf_size;
4516
4517 img->shdr[1].sh_name = find_string(img->str, ".text");
4518 img->shdr[1].sh_addr = buf;
4519 img->shdr[1].sh_size = buf_size;
4520
4521 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4522 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4523
4524 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4525 img->shdr[4].sh_size = debug_frame_size;
4526
4527 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4528 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4529
4530 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4531 img->sym[1].st_value = buf;
4532 img->sym[1].st_size = buf_size;
4533
4534 img->di.cu_low_pc = buf;
4535 img->di.cu_high_pc = buf + buf_size;
4536 img->di.fn_low_pc = buf;
4537 img->di.fn_high_pc = buf + buf_size;
4538
4539 dfh = (DebugFrameHeader *)(img + 1);
4540 memcpy(dfh, debug_frame, debug_frame_size);
4541 dfh->fde.func_start = buf;
4542 dfh->fde.func_len = buf_size;
4543
4544 #ifdef DEBUG_JIT
4545 /* Enable this block to be able to debug the ELF image file creation.
4546 One can use readelf, objdump, or other inspection utilities. */
4547 {
4548 FILE *f = fopen("/tmp/qemu.jit", "w+b");
4549 if (f) {
4550 if (fwrite(img, img_size, 1, f) != img_size) {
4551 /* Avoid stupid unused return value warning for fwrite. */
4552 }
4553 fclose(f);
4554 }
4555 }
4556 #endif
4557
4558 one_entry.symfile_addr = img;
4559 one_entry.symfile_size = img_size;
4560
4561 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4562 __jit_debug_descriptor.relevant_entry = &one_entry;
4563 __jit_debug_descriptor.first_entry = &one_entry;
4564 __jit_debug_register_code();
4565 }
4566 #else
4567 /* No support for the feature. Provide the entry point expected by exec.c,
4568 and implement the internal function we declared earlier. */
4569
4570 static void tcg_register_jit_int(const void *buf, size_t size,
4571 const void *debug_frame,
4572 size_t debug_frame_size)
4573 {
4574 }
4575
4576 void tcg_register_jit(const void *buf, size_t buf_size)
4577 {
4578 }
4579 #endif /* ELF_HOST_MACHINE */
4580
4581 #if !TCG_TARGET_MAYBE_vec
4582 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4583 {
4584 g_assert_not_reached();
4585 }
4586 #endif
QEmu