2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
28 #include "qemu/osdep.h"
30 /* Define to jump the ELF file used to communicate with GDB. */
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
38 #include "qemu/cacheflush.h"
39 #include "qemu/cacheinfo.h"
41 /* Note: the long term plan is to reduce the dependencies on the QEMU
42 CPU definitions. Currently they are used for qemu_ld/st
44 #define NO_CPU_IO_DEFS
46 #include "exec/exec-all.h"
47 #include "tcg/tcg-op.h"
49 #if UINTPTR_MAX == UINT32_MAX
50 # define ELF_CLASS ELFCLASS32
52 # define ELF_CLASS ELFCLASS64
55 # define ELF_DATA ELFDATA2MSB
57 # define ELF_DATA ELFDATA2LSB
62 #include "tcg/tcg-ldst.h"
63 #include "tcg-internal.h"
64 #include "accel/tcg/perf.h"
66 /* Forward declarations for functions declared in tcg-target.c.inc and
68 static void tcg_target_init(TCGContext
*s
);
69 static void tcg_target_qemu_prologue(TCGContext
*s
);
70 static bool patch_reloc(tcg_insn_unit
*code_ptr
, int type
,
71 intptr_t value
, intptr_t addend
);
73 /* The CIE and FDE header definitions will be common to all hosts. */
75 uint32_t len
__attribute__((aligned((sizeof(void *)))));
81 uint8_t return_column
;
84 typedef struct QEMU_PACKED
{
85 uint32_t len
__attribute__((aligned((sizeof(void *)))));
89 } DebugFrameFDEHeader
;
91 typedef struct QEMU_PACKED
{
93 DebugFrameFDEHeader fde
;
96 static void tcg_register_jit_int(const void *buf
, size_t size
,
97 const void *debug_frame
,
98 size_t debug_frame_size
)
99 __attribute__((unused
));
101 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
102 static void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg1
,
104 static bool tcg_out_mov(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg
);
105 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
106 TCGReg ret
, tcg_target_long arg
);
107 static void tcg_out_exit_tb(TCGContext
*s
, uintptr_t arg
);
108 static void tcg_out_goto_tb(TCGContext
*s
, int which
);
109 static void tcg_out_op(TCGContext
*s
, TCGOpcode opc
,
110 const TCGArg args
[TCG_MAX_OP_ARGS
],
111 const int const_args
[TCG_MAX_OP_ARGS
]);
112 #if TCG_TARGET_MAYBE_vec
113 static bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
114 TCGReg dst
, TCGReg src
);
115 static bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
116 TCGReg dst
, TCGReg base
, intptr_t offset
);
117 static void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
118 TCGReg dst
, int64_t arg
);
119 static void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
,
120 unsigned vecl
, unsigned vece
,
121 const TCGArg args
[TCG_MAX_OP_ARGS
],
122 const int const_args
[TCG_MAX_OP_ARGS
]);
124 static inline bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
125 TCGReg dst
, TCGReg src
)
127 g_assert_not_reached();
129 static inline bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
130 TCGReg dst
, TCGReg base
, intptr_t offset
)
132 g_assert_not_reached();
134 static inline void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
135 TCGReg dst
, int64_t arg
)
137 g_assert_not_reached();
139 static inline void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
,
140 unsigned vecl
, unsigned vece
,
141 const TCGArg args
[TCG_MAX_OP_ARGS
],
142 const int const_args
[TCG_MAX_OP_ARGS
])
144 g_assert_not_reached();
147 static void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
, TCGReg arg1
,
149 static bool tcg_out_sti(TCGContext
*s
, TCGType type
, TCGArg val
,
150 TCGReg base
, intptr_t ofs
);
151 static void tcg_out_call(TCGContext
*s
, const tcg_insn_unit
*target
,
152 const TCGHelperInfo
*info
);
153 static bool tcg_target_const_match(int64_t val
, TCGType type
, int ct
);
154 #ifdef TCG_TARGET_NEED_LDST_LABELS
155 static int tcg_out_ldst_finalize(TCGContext
*s
);
158 TCGContext tcg_init_ctx
;
159 __thread TCGContext
*tcg_ctx
;
161 TCGContext
**tcg_ctxs
;
162 unsigned int tcg_cur_ctxs
;
163 unsigned int tcg_max_ctxs
;
164 TCGv_env cpu_env
= 0;
165 const void *tcg_code_gen_epilogue
;
166 uintptr_t tcg_splitwx_diff
;
168 #ifndef CONFIG_TCG_INTERPRETER
169 tcg_prologue_fn
*tcg_qemu_tb_exec
;
172 static TCGRegSet tcg_target_available_regs
[TCG_TYPE_COUNT
];
173 static TCGRegSet tcg_target_call_clobber_regs
;
175 #if TCG_TARGET_INSN_UNIT_SIZE == 1
176 static __attribute__((unused
)) inline void tcg_out8(TCGContext
*s
, uint8_t v
)
181 static __attribute__((unused
)) inline void tcg_patch8(tcg_insn_unit
*p
,
188 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
189 static __attribute__((unused
)) inline void tcg_out16(TCGContext
*s
, uint16_t v
)
191 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
194 tcg_insn_unit
*p
= s
->code_ptr
;
195 memcpy(p
, &v
, sizeof(v
));
196 s
->code_ptr
= p
+ (2 / TCG_TARGET_INSN_UNIT_SIZE
);
200 static __attribute__((unused
)) inline void tcg_patch16(tcg_insn_unit
*p
,
203 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
206 memcpy(p
, &v
, sizeof(v
));
211 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
212 static __attribute__((unused
)) inline void tcg_out32(TCGContext
*s
, uint32_t v
)
214 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
217 tcg_insn_unit
*p
= s
->code_ptr
;
218 memcpy(p
, &v
, sizeof(v
));
219 s
->code_ptr
= p
+ (4 / TCG_TARGET_INSN_UNIT_SIZE
);
223 static __attribute__((unused
)) inline void tcg_patch32(tcg_insn_unit
*p
,
226 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
229 memcpy(p
, &v
, sizeof(v
));
234 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
235 static __attribute__((unused
)) inline void tcg_out64(TCGContext
*s
, uint64_t v
)
237 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
240 tcg_insn_unit
*p
= s
->code_ptr
;
241 memcpy(p
, &v
, sizeof(v
));
242 s
->code_ptr
= p
+ (8 / TCG_TARGET_INSN_UNIT_SIZE
);
246 static __attribute__((unused
)) inline void tcg_patch64(tcg_insn_unit
*p
,
249 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
252 memcpy(p
, &v
, sizeof(v
));
257 /* label relocation processing */
259 static void tcg_out_reloc(TCGContext
*s
, tcg_insn_unit
*code_ptr
, int type
,
260 TCGLabel
*l
, intptr_t addend
)
262 TCGRelocation
*r
= tcg_malloc(sizeof(TCGRelocation
));
267 QSIMPLEQ_INSERT_TAIL(&l
->relocs
, r
, next
);
270 static void tcg_out_label(TCGContext
*s
, TCGLabel
*l
)
272 tcg_debug_assert(!l
->has_value
);
274 l
->u
.value_ptr
= tcg_splitwx_to_rx(s
->code_ptr
);
277 TCGLabel
*gen_new_label(void)
279 TCGContext
*s
= tcg_ctx
;
280 TCGLabel
*l
= tcg_malloc(sizeof(TCGLabel
));
282 memset(l
, 0, sizeof(TCGLabel
));
283 l
->id
= s
->nb_labels
++;
284 QSIMPLEQ_INIT(&l
->relocs
);
286 QSIMPLEQ_INSERT_TAIL(&s
->labels
, l
, next
);
291 static bool tcg_resolve_relocs(TCGContext
*s
)
295 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
297 uintptr_t value
= l
->u
.value
;
299 QSIMPLEQ_FOREACH(r
, &l
->relocs
, next
) {
300 if (!patch_reloc(r
->ptr
, r
->type
, value
, r
->addend
)) {
308 static void set_jmp_reset_offset(TCGContext
*s
, int which
)
311 * We will check for overflow at the end of the opcode loop in
312 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
314 s
->gen_tb
->jmp_reset_offset
[which
] = tcg_current_code_size(s
);
317 static void G_GNUC_UNUSED
set_jmp_insn_offset(TCGContext
*s
, int which
)
320 * We will check for overflow at the end of the opcode loop in
321 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
323 tcg_debug_assert(TCG_TARGET_HAS_direct_jump
);
324 s
->gen_tb
->jmp_target_arg
[which
] = tcg_current_code_size(s
);
327 static uintptr_t G_GNUC_UNUSED
get_jmp_target_addr(TCGContext
*s
, int which
)
330 * Return the read-execute version of the pointer, for the benefit
331 * of any pc-relative addressing mode.
333 return (uintptr_t)tcg_splitwx_to_rx(s
->gen_tb
->jmp_target_arg
+ which
);
336 /* Signal overflow, starting over with fewer guest insns. */
338 void tcg_raise_tb_overflow(TCGContext
*s
)
340 siglongjmp(s
->jmp_trans
, -2);
343 #define C_PFX1(P, A) P##A
344 #define C_PFX2(P, A, B) P##A##_##B
345 #define C_PFX3(P, A, B, C) P##A##_##B##_##C
346 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
347 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
348 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
350 /* Define an enumeration for the various combinations. */
352 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
353 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
354 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
355 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
357 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
358 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
359 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
360 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
362 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
364 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
365 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
366 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
367 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
370 #include "tcg-target-con-set.h"
371 } TCGConstraintSetIndex
;
373 static TCGConstraintSetIndex
tcg_target_op_def(TCGOpcode
);
389 /* Put all of the constraint sets into an array, indexed by the enum. */
391 #define C_O0_I1(I1) { .args_ct_str = { #I1 } },
392 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
393 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
394 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
396 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
397 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
398 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
399 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
401 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
403 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
404 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
405 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
406 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
408 static const TCGTargetOpDef constraint_sets
[] = {
409 #include "tcg-target-con-set.h"
427 /* Expand the enumerator to be returned from tcg_target_op_def(). */
429 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
430 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
431 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
432 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
434 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
435 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
436 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
437 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
439 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
441 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
442 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
443 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
444 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
446 #include "tcg-target.c.inc"
448 static void alloc_tcg_plugin_context(TCGContext
*s
)
451 s
->plugin_tb
= g_new0(struct qemu_plugin_tb
, 1);
452 s
->plugin_tb
->insns
=
453 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn
);
458 * All TCG threads except the parent (i.e. the one that called tcg_context_init
459 * and registered the target's TCG globals) must register with this function
460 * before initiating translation.
462 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
463 * of tcg_region_init() for the reasoning behind this.
465 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
466 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
467 * is not used anymore for translation once this function is called.
469 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
470 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
472 #ifdef CONFIG_USER_ONLY
473 void tcg_register_thread(void)
475 tcg_ctx
= &tcg_init_ctx
;
478 void tcg_register_thread(void)
480 TCGContext
*s
= g_malloc(sizeof(*s
));
485 /* Relink mem_base. */
486 for (i
= 0, n
= tcg_init_ctx
.nb_globals
; i
< n
; ++i
) {
487 if (tcg_init_ctx
.temps
[i
].mem_base
) {
488 ptrdiff_t b
= tcg_init_ctx
.temps
[i
].mem_base
- tcg_init_ctx
.temps
;
489 tcg_debug_assert(b
>= 0 && b
< n
);
490 s
->temps
[i
].mem_base
= &s
->temps
[b
];
494 /* Claim an entry in tcg_ctxs */
495 n
= qatomic_fetch_inc(&tcg_cur_ctxs
);
496 g_assert(n
< tcg_max_ctxs
);
497 qatomic_set(&tcg_ctxs
[n
], s
);
500 alloc_tcg_plugin_context(s
);
501 tcg_region_initial_alloc(s
);
506 #endif /* !CONFIG_USER_ONLY */
508 /* pool based memory allocation */
509 void *tcg_malloc_internal(TCGContext
*s
, int size
)
514 if (size
> TCG_POOL_CHUNK_SIZE
) {
515 /* big malloc: insert a new pool (XXX: could optimize) */
516 p
= g_malloc(sizeof(TCGPool
) + size
);
518 p
->next
= s
->pool_first_large
;
519 s
->pool_first_large
= p
;
530 pool_size
= TCG_POOL_CHUNK_SIZE
;
531 p
= g_malloc(sizeof(TCGPool
) + pool_size
);
534 if (s
->pool_current
) {
535 s
->pool_current
->next
= p
;
545 s
->pool_cur
= p
->data
+ size
;
546 s
->pool_end
= p
->data
+ p
->size
;
550 void tcg_pool_reset(TCGContext
*s
)
553 for (p
= s
->pool_first_large
; p
; p
= t
) {
557 s
->pool_first_large
= NULL
;
558 s
->pool_cur
= s
->pool_end
= NULL
;
559 s
->pool_current
= NULL
;
562 #include "exec/helper-proto.h"
564 static TCGHelperInfo all_helpers
[] = {
565 #include "exec/helper-tcg.h"
567 static GHashTable
*helper_table
;
569 #ifdef CONFIG_TCG_INTERPRETER
570 static ffi_type
*typecode_to_ffi(int argmask
)
573 case dh_typecode_void
:
574 return &ffi_type_void
;
575 case dh_typecode_i32
:
576 return &ffi_type_uint32
;
577 case dh_typecode_s32
:
578 return &ffi_type_sint32
;
579 case dh_typecode_i64
:
580 return &ffi_type_uint64
;
581 case dh_typecode_s64
:
582 return &ffi_type_sint64
;
583 case dh_typecode_ptr
:
584 return &ffi_type_pointer
;
586 g_assert_not_reached();
589 static void init_ffi_layouts(void)
591 /* g_direct_hash/equal for direct comparisons on uint32_t. */
592 GHashTable
*ffi_table
= g_hash_table_new(NULL
, NULL
);
594 for (int i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
595 TCGHelperInfo
*info
= &all_helpers
[i
];
596 unsigned typemask
= info
->typemask
;
597 gpointer hash
= (gpointer
)(uintptr_t)typemask
;
606 cif
= g_hash_table_lookup(ffi_table
, hash
);
612 /* Ignoring the return type, find the last non-zero field. */
613 nargs
= 32 - clz32(typemask
>> 3);
614 nargs
= DIV_ROUND_UP(nargs
, 3);
616 ca
= g_malloc0(sizeof(*ca
) + nargs
* sizeof(ffi_type
*));
617 ca
->cif
.rtype
= typecode_to_ffi(typemask
& 7);
618 ca
->cif
.nargs
= nargs
;
621 ca
->cif
.arg_types
= ca
->args
;
622 for (int j
= 0; j
< nargs
; ++j
) {
623 int typecode
= extract32(typemask
, (j
+ 1) * 3, 3);
624 ca
->args
[j
] = typecode_to_ffi(typecode
);
628 status
= ffi_prep_cif(&ca
->cif
, FFI_DEFAULT_ABI
, nargs
,
629 ca
->cif
.rtype
, ca
->cif
.arg_types
);
630 assert(status
== FFI_OK
);
634 g_hash_table_insert(ffi_table
, hash
, (gpointer
)cif
);
637 g_hash_table_destroy(ffi_table
);
639 #endif /* CONFIG_TCG_INTERPRETER */
641 typedef struct TCGCumulativeArgs
{
642 int arg_idx
; /* tcg_gen_callN args[] */
643 int info_in_idx
; /* TCGHelperInfo in[] */
644 int arg_slot
; /* regs+stack slot */
645 int ref_slot
; /* stack slots for references */
648 static void layout_arg_even(TCGCumulativeArgs
*cum
)
650 cum
->arg_slot
+= cum
->arg_slot
& 1;
653 static void layout_arg_1(TCGCumulativeArgs
*cum
, TCGHelperInfo
*info
,
654 TCGCallArgumentKind kind
)
656 TCGCallArgumentLoc
*loc
= &info
->in
[cum
->info_in_idx
];
658 *loc
= (TCGCallArgumentLoc
){
660 .arg_idx
= cum
->arg_idx
,
661 .arg_slot
= cum
->arg_slot
,
667 static void layout_arg_normal_n(TCGCumulativeArgs
*cum
,
668 TCGHelperInfo
*info
, int n
)
670 TCGCallArgumentLoc
*loc
= &info
->in
[cum
->info_in_idx
];
672 for (int i
= 0; i
< n
; ++i
) {
673 /* Layout all using the same arg_idx, adjusting the subindex. */
674 loc
[i
] = (TCGCallArgumentLoc
){
675 .kind
= TCG_CALL_ARG_NORMAL
,
676 .arg_idx
= cum
->arg_idx
,
678 .arg_slot
= cum
->arg_slot
+ i
,
681 cum
->info_in_idx
+= n
;
685 static void init_call_layout(TCGHelperInfo
*info
)
687 int max_reg_slots
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
688 int max_stk_slots
= TCG_STATIC_CALL_ARGS_SIZE
/ sizeof(tcg_target_long
);
689 unsigned typemask
= info
->typemask
;
691 TCGCumulativeArgs cum
= { };
694 * Parse and place any function return value.
696 typecode
= typemask
& 7;
698 case dh_typecode_void
:
701 case dh_typecode_i32
:
702 case dh_typecode_s32
:
703 case dh_typecode_ptr
:
705 info
->out_kind
= TCG_CALL_RET_NORMAL
;
707 case dh_typecode_i64
:
708 case dh_typecode_s64
:
709 info
->nr_out
= 64 / TCG_TARGET_REG_BITS
;
710 info
->out_kind
= TCG_CALL_RET_NORMAL
;
713 g_assert_not_reached();
715 assert(info
->nr_out
<= ARRAY_SIZE(tcg_target_call_oarg_regs
));
718 * Parse and place function arguments.
720 for (typemask
>>= 3; typemask
; typemask
>>= 3, cum
.arg_idx
++) {
721 TCGCallArgumentKind kind
;
724 typecode
= typemask
& 7;
726 case dh_typecode_i32
:
727 case dh_typecode_s32
:
730 case dh_typecode_i64
:
731 case dh_typecode_s64
:
734 case dh_typecode_ptr
:
738 g_assert_not_reached();
743 switch (TCG_TARGET_CALL_ARG_I32
) {
744 case TCG_CALL_ARG_EVEN
:
745 layout_arg_even(&cum
);
747 case TCG_CALL_ARG_NORMAL
:
748 layout_arg_1(&cum
, info
, TCG_CALL_ARG_NORMAL
);
750 case TCG_CALL_ARG_EXTEND
:
751 kind
= TCG_CALL_ARG_EXTEND_U
+ (typecode
& 1);
752 layout_arg_1(&cum
, info
, kind
);
755 qemu_build_not_reached();
760 switch (TCG_TARGET_CALL_ARG_I64
) {
761 case TCG_CALL_ARG_EVEN
:
762 layout_arg_even(&cum
);
764 case TCG_CALL_ARG_NORMAL
:
765 if (TCG_TARGET_REG_BITS
== 32) {
766 layout_arg_normal_n(&cum
, info
, 2);
768 layout_arg_1(&cum
, info
, TCG_CALL_ARG_NORMAL
);
772 qemu_build_not_reached();
777 g_assert_not_reached();
780 info
->nr_in
= cum
.info_in_idx
;
782 /* Validate that we didn't overrun the input array. */
783 assert(cum
.info_in_idx
<= ARRAY_SIZE(info
->in
));
784 /* Validate the backend has enough argument space. */
785 assert(cum
.arg_slot
<= max_reg_slots
+ max_stk_slots
);
786 assert(cum
.ref_slot
<= max_stk_slots
);
789 static int indirect_reg_alloc_order
[ARRAY_SIZE(tcg_target_reg_alloc_order
)];
790 static void process_op_defs(TCGContext
*s
);
791 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
792 TCGReg reg
, const char *name
);
794 static void tcg_context_init(unsigned max_cpus
)
796 TCGContext
*s
= &tcg_init_ctx
;
797 int op
, total_args
, n
, i
;
799 TCGArgConstraint
*args_ct
;
802 memset(s
, 0, sizeof(*s
));
805 /* Count total number of arguments and allocate the corresponding
808 for(op
= 0; op
< NB_OPS
; op
++) {
809 def
= &tcg_op_defs
[op
];
810 n
= def
->nb_iargs
+ def
->nb_oargs
;
814 args_ct
= g_new0(TCGArgConstraint
, total_args
);
816 for(op
= 0; op
< NB_OPS
; op
++) {
817 def
= &tcg_op_defs
[op
];
818 def
->args_ct
= args_ct
;
819 n
= def
->nb_iargs
+ def
->nb_oargs
;
823 /* Register helpers. */
824 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
825 helper_table
= g_hash_table_new(NULL
, NULL
);
827 for (i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
828 init_call_layout(&all_helpers
[i
]);
829 g_hash_table_insert(helper_table
, (gpointer
)all_helpers
[i
].func
,
830 (gpointer
)&all_helpers
[i
]);
833 #ifdef CONFIG_TCG_INTERPRETER
840 /* Reverse the order of the saved registers, assuming they're all at
841 the start of tcg_target_reg_alloc_order. */
842 for (n
= 0; n
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++n
) {
843 int r
= tcg_target_reg_alloc_order
[n
];
844 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, r
)) {
848 for (i
= 0; i
< n
; ++i
) {
849 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[n
- 1 - i
];
851 for (; i
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++i
) {
852 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[i
];
855 alloc_tcg_plugin_context(s
);
859 * In user-mode we simply share the init context among threads, since we
860 * use a single region. See the documentation tcg_region_init() for the
861 * reasoning behind this.
862 * In softmmu we will have at most max_cpus TCG threads.
864 #ifdef CONFIG_USER_ONLY
869 tcg_max_ctxs
= max_cpus
;
870 tcg_ctxs
= g_new0(TCGContext
*, max_cpus
);
873 tcg_debug_assert(!tcg_regset_test_reg(s
->reserved_regs
, TCG_AREG0
));
874 ts
= tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, TCG_AREG0
, "env");
875 cpu_env
= temp_tcgv_ptr(ts
);
878 void tcg_init(size_t tb_size
, int splitwx
, unsigned max_cpus
)
880 tcg_context_init(max_cpus
);
881 tcg_region_init(tb_size
, splitwx
, max_cpus
);
885 * Allocate TBs right before their corresponding translated code, making
886 * sure that TBs and code are on different cache lines.
888 TranslationBlock
*tcg_tb_alloc(TCGContext
*s
)
890 uintptr_t align
= qemu_icache_linesize
;
891 TranslationBlock
*tb
;
895 tb
= (void *)ROUND_UP((uintptr_t)s
->code_gen_ptr
, align
);
896 next
= (void *)ROUND_UP((uintptr_t)(tb
+ 1), align
);
898 if (unlikely(next
> s
->code_gen_highwater
)) {
899 if (tcg_region_alloc(s
)) {
904 qatomic_set(&s
->code_gen_ptr
, next
);
905 s
->data_gen_ptr
= NULL
;
909 void tcg_prologue_init(TCGContext
*s
)
911 size_t prologue_size
;
913 s
->code_ptr
= s
->code_gen_ptr
;
914 s
->code_buf
= s
->code_gen_ptr
;
915 s
->data_gen_ptr
= NULL
;
917 #ifndef CONFIG_TCG_INTERPRETER
918 tcg_qemu_tb_exec
= (tcg_prologue_fn
*)tcg_splitwx_to_rx(s
->code_ptr
);
921 #ifdef TCG_TARGET_NEED_POOL_LABELS
922 s
->pool_labels
= NULL
;
925 qemu_thread_jit_write();
926 /* Generate the prologue. */
927 tcg_target_qemu_prologue(s
);
929 #ifdef TCG_TARGET_NEED_POOL_LABELS
930 /* Allow the prologue to put e.g. guest_base into a pool entry. */
932 int result
= tcg_out_pool_finalize(s
);
933 tcg_debug_assert(result
== 0);
937 prologue_size
= tcg_current_code_size(s
);
938 perf_report_prologue(s
->code_gen_ptr
, prologue_size
);
940 #ifndef CONFIG_TCG_INTERPRETER
941 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s
->code_buf
),
942 (uintptr_t)s
->code_buf
, prologue_size
);
946 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
947 FILE *logfile
= qemu_log_trylock();
949 fprintf(logfile
, "PROLOGUE: [size=%zu]\n", prologue_size
);
950 if (s
->data_gen_ptr
) {
951 size_t code_size
= s
->data_gen_ptr
- s
->code_gen_ptr
;
952 size_t data_size
= prologue_size
- code_size
;
955 disas(logfile
, s
->code_gen_ptr
, code_size
);
957 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
958 if (sizeof(tcg_target_ulong
) == 8) {
960 "0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
961 (uintptr_t)s
->data_gen_ptr
+ i
,
962 *(uint64_t *)(s
->data_gen_ptr
+ i
));
965 "0x%08" PRIxPTR
": .long 0x%08x\n",
966 (uintptr_t)s
->data_gen_ptr
+ i
,
967 *(uint32_t *)(s
->data_gen_ptr
+ i
));
971 disas(logfile
, s
->code_gen_ptr
, prologue_size
);
973 fprintf(logfile
, "\n");
974 qemu_log_unlock(logfile
);
979 #ifndef CONFIG_TCG_INTERPRETER
981 * Assert that goto_ptr is implemented completely, setting an epilogue.
982 * For tci, we use NULL as the signal to return from the interpreter,
983 * so skip this check.
985 tcg_debug_assert(tcg_code_gen_epilogue
!= NULL
);
988 tcg_region_prologue_set(s
);
991 void tcg_func_start(TCGContext
*s
)
994 s
->nb_temps
= s
->nb_globals
;
996 /* No temps have been previously allocated for size or locality. */
997 memset(s
->free_temps
, 0, sizeof(s
->free_temps
));
999 /* No constant temps have been previously allocated. */
1000 for (int i
= 0; i
< TCG_TYPE_COUNT
; ++i
) {
1001 if (s
->const_table
[i
]) {
1002 g_hash_table_remove_all(s
->const_table
[i
]);
1008 s
->current_frame_offset
= s
->frame_start
;
1010 #ifdef CONFIG_DEBUG_TCG
1011 s
->goto_tb_issue_mask
= 0;
1014 QTAILQ_INIT(&s
->ops
);
1015 QTAILQ_INIT(&s
->free_ops
);
1016 QSIMPLEQ_INIT(&s
->labels
);
1019 static TCGTemp
*tcg_temp_alloc(TCGContext
*s
)
1021 int n
= s
->nb_temps
++;
1023 if (n
>= TCG_MAX_TEMPS
) {
1024 tcg_raise_tb_overflow(s
);
1026 return memset(&s
->temps
[n
], 0, sizeof(TCGTemp
));
1029 static TCGTemp
*tcg_global_alloc(TCGContext
*s
)
1033 tcg_debug_assert(s
->nb_globals
== s
->nb_temps
);
1034 tcg_debug_assert(s
->nb_globals
< TCG_MAX_TEMPS
);
1036 ts
= tcg_temp_alloc(s
);
1037 ts
->kind
= TEMP_GLOBAL
;
1042 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
1043 TCGReg reg
, const char *name
)
1047 if (TCG_TARGET_REG_BITS
== 32 && type
!= TCG_TYPE_I32
) {
1051 ts
= tcg_global_alloc(s
);
1052 ts
->base_type
= type
;
1054 ts
->kind
= TEMP_FIXED
;
1057 tcg_regset_set_reg(s
->reserved_regs
, reg
);
1062 void tcg_set_frame(TCGContext
*s
, TCGReg reg
, intptr_t start
, intptr_t size
)
1064 s
->frame_start
= start
;
1065 s
->frame_end
= start
+ size
;
1067 = tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, reg
, "_frame");
1070 TCGTemp
*tcg_global_mem_new_internal(TCGType type
, TCGv_ptr base
,
1071 intptr_t offset
, const char *name
)
1073 TCGContext
*s
= tcg_ctx
;
1074 TCGTemp
*base_ts
= tcgv_ptr_temp(base
);
1075 TCGTemp
*ts
= tcg_global_alloc(s
);
1076 int indirect_reg
= 0;
1078 switch (base_ts
->kind
) {
1082 /* We do not support double-indirect registers. */
1083 tcg_debug_assert(!base_ts
->indirect_reg
);
1084 base_ts
->indirect_base
= 1;
1085 s
->nb_indirects
+= (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
1090 g_assert_not_reached();
1093 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1094 TCGTemp
*ts2
= tcg_global_alloc(s
);
1097 ts
->base_type
= TCG_TYPE_I64
;
1098 ts
->type
= TCG_TYPE_I32
;
1099 ts
->indirect_reg
= indirect_reg
;
1100 ts
->mem_allocated
= 1;
1101 ts
->mem_base
= base_ts
;
1102 ts
->mem_offset
= offset
;
1103 pstrcpy(buf
, sizeof(buf
), name
);
1104 pstrcat(buf
, sizeof(buf
), "_0");
1105 ts
->name
= strdup(buf
);
1107 tcg_debug_assert(ts2
== ts
+ 1);
1108 ts2
->base_type
= TCG_TYPE_I64
;
1109 ts2
->type
= TCG_TYPE_I32
;
1110 ts2
->indirect_reg
= indirect_reg
;
1111 ts2
->mem_allocated
= 1;
1112 ts2
->mem_base
= base_ts
;
1113 ts2
->mem_offset
= offset
+ 4;
1114 ts2
->temp_subindex
= 1;
1115 pstrcpy(buf
, sizeof(buf
), name
);
1116 pstrcat(buf
, sizeof(buf
), "_1");
1117 ts2
->name
= strdup(buf
);
1119 ts
->base_type
= type
;
1121 ts
->indirect_reg
= indirect_reg
;
1122 ts
->mem_allocated
= 1;
1123 ts
->mem_base
= base_ts
;
1124 ts
->mem_offset
= offset
;
1130 TCGTemp
*tcg_temp_new_internal(TCGType type
, bool temp_local
)
1132 TCGContext
*s
= tcg_ctx
;
1133 TCGTempKind kind
= temp_local
? TEMP_LOCAL
: TEMP_NORMAL
;
1137 k
= type
+ (temp_local
? TCG_TYPE_COUNT
: 0);
1138 idx
= find_first_bit(s
->free_temps
[k
].l
, TCG_MAX_TEMPS
);
1139 if (idx
< TCG_MAX_TEMPS
) {
1140 /* There is already an available temp with the right type. */
1141 clear_bit(idx
, s
->free_temps
[k
].l
);
1143 ts
= &s
->temps
[idx
];
1144 ts
->temp_allocated
= 1;
1145 tcg_debug_assert(ts
->base_type
== type
);
1146 tcg_debug_assert(ts
->kind
== kind
);
1148 ts
= tcg_temp_alloc(s
);
1149 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1150 TCGTemp
*ts2
= tcg_temp_alloc(s
);
1152 ts
->base_type
= type
;
1153 ts
->type
= TCG_TYPE_I32
;
1154 ts
->temp_allocated
= 1;
1157 tcg_debug_assert(ts2
== ts
+ 1);
1158 ts2
->base_type
= TCG_TYPE_I64
;
1159 ts2
->type
= TCG_TYPE_I32
;
1160 ts2
->temp_allocated
= 1;
1161 ts2
->temp_subindex
= 1;
1164 ts
->base_type
= type
;
1166 ts
->temp_allocated
= 1;
1171 #if defined(CONFIG_DEBUG_TCG)
1177 TCGv_vec
tcg_temp_new_vec(TCGType type
)
1181 #ifdef CONFIG_DEBUG_TCG
1184 assert(TCG_TARGET_HAS_v64
);
1187 assert(TCG_TARGET_HAS_v128
);
1190 assert(TCG_TARGET_HAS_v256
);
1193 g_assert_not_reached();
1197 t
= tcg_temp_new_internal(type
, 0);
1198 return temp_tcgv_vec(t
);
1201 /* Create a new temp of the same type as an existing temp. */
1202 TCGv_vec
tcg_temp_new_vec_matching(TCGv_vec match
)
1204 TCGTemp
*t
= tcgv_vec_temp(match
);
1206 tcg_debug_assert(t
->temp_allocated
!= 0);
1208 t
= tcg_temp_new_internal(t
->base_type
, 0);
1209 return temp_tcgv_vec(t
);
1212 void tcg_temp_free_internal(TCGTemp
*ts
)
1214 TCGContext
*s
= tcg_ctx
;
1220 * In order to simplify users of tcg_constant_*,
1221 * silently ignore free.
1228 g_assert_not_reached();
1231 #if defined(CONFIG_DEBUG_TCG)
1233 if (s
->temps_in_use
< 0) {
1234 fprintf(stderr
, "More temporaries freed than allocated!\n");
1238 tcg_debug_assert(ts
->temp_allocated
!= 0);
1239 ts
->temp_allocated
= 0;
1242 k
= ts
->base_type
+ (ts
->kind
== TEMP_NORMAL
? 0 : TCG_TYPE_COUNT
);
1243 set_bit(idx
, s
->free_temps
[k
].l
);
1246 TCGTemp
*tcg_constant_internal(TCGType type
, int64_t val
)
1248 TCGContext
*s
= tcg_ctx
;
1249 GHashTable
*h
= s
->const_table
[type
];
1253 h
= g_hash_table_new(g_int64_hash
, g_int64_equal
);
1254 s
->const_table
[type
] = h
;
1257 ts
= g_hash_table_lookup(h
, &val
);
1261 ts
= tcg_temp_alloc(s
);
1263 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1264 TCGTemp
*ts2
= tcg_temp_alloc(s
);
1266 tcg_debug_assert(ts2
== ts
+ 1);
1268 ts
->base_type
= TCG_TYPE_I64
;
1269 ts
->type
= TCG_TYPE_I32
;
1270 ts
->kind
= TEMP_CONST
;
1271 ts
->temp_allocated
= 1;
1273 ts2
->base_type
= TCG_TYPE_I64
;
1274 ts2
->type
= TCG_TYPE_I32
;
1275 ts2
->kind
= TEMP_CONST
;
1276 ts2
->temp_allocated
= 1;
1277 ts2
->temp_subindex
= 1;
1280 * Retain the full value of the 64-bit constant in the low
1281 * part, so that the hash table works. Actual uses will
1282 * truncate the value to the low part.
1284 ts
[HOST_BIG_ENDIAN
].val
= val
;
1285 ts
[!HOST_BIG_ENDIAN
].val
= val
>> 32;
1286 val_ptr
= &ts
[HOST_BIG_ENDIAN
].val
;
1288 ts
->base_type
= type
;
1290 ts
->kind
= TEMP_CONST
;
1291 ts
->temp_allocated
= 1;
1295 g_hash_table_insert(h
, val_ptr
, ts
);
1301 TCGv_vec
tcg_constant_vec(TCGType type
, unsigned vece
, int64_t val
)
1303 val
= dup_const(vece
, val
);
1304 return temp_tcgv_vec(tcg_constant_internal(type
, val
));
1307 TCGv_vec
tcg_constant_vec_matching(TCGv_vec match
, unsigned vece
, int64_t val
)
1309 TCGTemp
*t
= tcgv_vec_temp(match
);
1311 tcg_debug_assert(t
->temp_allocated
!= 0);
1312 return tcg_constant_vec(t
->base_type
, vece
, val
);
1315 TCGv_i32
tcg_const_i32(int32_t val
)
1318 t0
= tcg_temp_new_i32();
1319 tcg_gen_movi_i32(t0
, val
);
1323 TCGv_i64
tcg_const_i64(int64_t val
)
1326 t0
= tcg_temp_new_i64();
1327 tcg_gen_movi_i64(t0
, val
);
1331 TCGv_i32
tcg_const_local_i32(int32_t val
)
1334 t0
= tcg_temp_local_new_i32();
1335 tcg_gen_movi_i32(t0
, val
);
1339 TCGv_i64
tcg_const_local_i64(int64_t val
)
1342 t0
= tcg_temp_local_new_i64();
1343 tcg_gen_movi_i64(t0
, val
);
1347 #if defined(CONFIG_DEBUG_TCG)
1348 void tcg_clear_temp_count(void)
1350 TCGContext
*s
= tcg_ctx
;
1351 s
->temps_in_use
= 0;
1354 int tcg_check_temp_count(void)
1356 TCGContext
*s
= tcg_ctx
;
1357 if (s
->temps_in_use
) {
1358 /* Clear the count so that we don't give another
1359 * warning immediately next time around.
1361 s
->temps_in_use
= 0;
1368 /* Return true if OP may appear in the opcode stream.
1369 Test the runtime variable that controls each opcode. */
1370 bool tcg_op_supported(TCGOpcode op
)
1373 = TCG_TARGET_HAS_v64
| TCG_TARGET_HAS_v128
| TCG_TARGET_HAS_v256
;
1376 case INDEX_op_discard
:
1377 case INDEX_op_set_label
:
1381 case INDEX_op_insn_start
:
1382 case INDEX_op_exit_tb
:
1383 case INDEX_op_goto_tb
:
1384 case INDEX_op_goto_ptr
:
1385 case INDEX_op_qemu_ld_i32
:
1386 case INDEX_op_qemu_st_i32
:
1387 case INDEX_op_qemu_ld_i64
:
1388 case INDEX_op_qemu_st_i64
:
1391 case INDEX_op_qemu_st8_i32
:
1392 return TCG_TARGET_HAS_qemu_st8_i32
;
1394 case INDEX_op_mov_i32
:
1395 case INDEX_op_setcond_i32
:
1396 case INDEX_op_brcond_i32
:
1397 case INDEX_op_ld8u_i32
:
1398 case INDEX_op_ld8s_i32
:
1399 case INDEX_op_ld16u_i32
:
1400 case INDEX_op_ld16s_i32
:
1401 case INDEX_op_ld_i32
:
1402 case INDEX_op_st8_i32
:
1403 case INDEX_op_st16_i32
:
1404 case INDEX_op_st_i32
:
1405 case INDEX_op_add_i32
:
1406 case INDEX_op_sub_i32
:
1407 case INDEX_op_mul_i32
:
1408 case INDEX_op_and_i32
:
1409 case INDEX_op_or_i32
:
1410 case INDEX_op_xor_i32
:
1411 case INDEX_op_shl_i32
:
1412 case INDEX_op_shr_i32
:
1413 case INDEX_op_sar_i32
:
1416 case INDEX_op_movcond_i32
:
1417 return TCG_TARGET_HAS_movcond_i32
;
1418 case INDEX_op_div_i32
:
1419 case INDEX_op_divu_i32
:
1420 return TCG_TARGET_HAS_div_i32
;
1421 case INDEX_op_rem_i32
:
1422 case INDEX_op_remu_i32
:
1423 return TCG_TARGET_HAS_rem_i32
;
1424 case INDEX_op_div2_i32
:
1425 case INDEX_op_divu2_i32
:
1426 return TCG_TARGET_HAS_div2_i32
;
1427 case INDEX_op_rotl_i32
:
1428 case INDEX_op_rotr_i32
:
1429 return TCG_TARGET_HAS_rot_i32
;
1430 case INDEX_op_deposit_i32
:
1431 return TCG_TARGET_HAS_deposit_i32
;
1432 case INDEX_op_extract_i32
:
1433 return TCG_TARGET_HAS_extract_i32
;
1434 case INDEX_op_sextract_i32
:
1435 return TCG_TARGET_HAS_sextract_i32
;
1436 case INDEX_op_extract2_i32
:
1437 return TCG_TARGET_HAS_extract2_i32
;
1438 case INDEX_op_add2_i32
:
1439 return TCG_TARGET_HAS_add2_i32
;
1440 case INDEX_op_sub2_i32
:
1441 return TCG_TARGET_HAS_sub2_i32
;
1442 case INDEX_op_mulu2_i32
:
1443 return TCG_TARGET_HAS_mulu2_i32
;
1444 case INDEX_op_muls2_i32
:
1445 return TCG_TARGET_HAS_muls2_i32
;
1446 case INDEX_op_muluh_i32
:
1447 return TCG_TARGET_HAS_muluh_i32
;
1448 case INDEX_op_mulsh_i32
:
1449 return TCG_TARGET_HAS_mulsh_i32
;
1450 case INDEX_op_ext8s_i32
:
1451 return TCG_TARGET_HAS_ext8s_i32
;
1452 case INDEX_op_ext16s_i32
:
1453 return TCG_TARGET_HAS_ext16s_i32
;
1454 case INDEX_op_ext8u_i32
:
1455 return TCG_TARGET_HAS_ext8u_i32
;
1456 case INDEX_op_ext16u_i32
:
1457 return TCG_TARGET_HAS_ext16u_i32
;
1458 case INDEX_op_bswap16_i32
:
1459 return TCG_TARGET_HAS_bswap16_i32
;
1460 case INDEX_op_bswap32_i32
:
1461 return TCG_TARGET_HAS_bswap32_i32
;
1462 case INDEX_op_not_i32
:
1463 return TCG_TARGET_HAS_not_i32
;
1464 case INDEX_op_neg_i32
:
1465 return TCG_TARGET_HAS_neg_i32
;
1466 case INDEX_op_andc_i32
:
1467 return TCG_TARGET_HAS_andc_i32
;
1468 case INDEX_op_orc_i32
:
1469 return TCG_TARGET_HAS_orc_i32
;
1470 case INDEX_op_eqv_i32
:
1471 return TCG_TARGET_HAS_eqv_i32
;
1472 case INDEX_op_nand_i32
:
1473 return TCG_TARGET_HAS_nand_i32
;
1474 case INDEX_op_nor_i32
:
1475 return TCG_TARGET_HAS_nor_i32
;
1476 case INDEX_op_clz_i32
:
1477 return TCG_TARGET_HAS_clz_i32
;
1478 case INDEX_op_ctz_i32
:
1479 return TCG_TARGET_HAS_ctz_i32
;
1480 case INDEX_op_ctpop_i32
:
1481 return TCG_TARGET_HAS_ctpop_i32
;
1483 case INDEX_op_brcond2_i32
:
1484 case INDEX_op_setcond2_i32
:
1485 return TCG_TARGET_REG_BITS
== 32;
1487 case INDEX_op_mov_i64
:
1488 case INDEX_op_setcond_i64
:
1489 case INDEX_op_brcond_i64
:
1490 case INDEX_op_ld8u_i64
:
1491 case INDEX_op_ld8s_i64
:
1492 case INDEX_op_ld16u_i64
:
1493 case INDEX_op_ld16s_i64
:
1494 case INDEX_op_ld32u_i64
:
1495 case INDEX_op_ld32s_i64
:
1496 case INDEX_op_ld_i64
:
1497 case INDEX_op_st8_i64
:
1498 case INDEX_op_st16_i64
:
1499 case INDEX_op_st32_i64
:
1500 case INDEX_op_st_i64
:
1501 case INDEX_op_add_i64
:
1502 case INDEX_op_sub_i64
:
1503 case INDEX_op_mul_i64
:
1504 case INDEX_op_and_i64
:
1505 case INDEX_op_or_i64
:
1506 case INDEX_op_xor_i64
:
1507 case INDEX_op_shl_i64
:
1508 case INDEX_op_shr_i64
:
1509 case INDEX_op_sar_i64
:
1510 case INDEX_op_ext_i32_i64
:
1511 case INDEX_op_extu_i32_i64
:
1512 return TCG_TARGET_REG_BITS
== 64;
1514 case INDEX_op_movcond_i64
:
1515 return TCG_TARGET_HAS_movcond_i64
;
1516 case INDEX_op_div_i64
:
1517 case INDEX_op_divu_i64
:
1518 return TCG_TARGET_HAS_div_i64
;
1519 case INDEX_op_rem_i64
:
1520 case INDEX_op_remu_i64
:
1521 return TCG_TARGET_HAS_rem_i64
;
1522 case INDEX_op_div2_i64
:
1523 case INDEX_op_divu2_i64
:
1524 return TCG_TARGET_HAS_div2_i64
;
1525 case INDEX_op_rotl_i64
:
1526 case INDEX_op_rotr_i64
:
1527 return TCG_TARGET_HAS_rot_i64
;
1528 case INDEX_op_deposit_i64
:
1529 return TCG_TARGET_HAS_deposit_i64
;
1530 case INDEX_op_extract_i64
:
1531 return TCG_TARGET_HAS_extract_i64
;
1532 case INDEX_op_sextract_i64
:
1533 return TCG_TARGET_HAS_sextract_i64
;
1534 case INDEX_op_extract2_i64
:
1535 return TCG_TARGET_HAS_extract2_i64
;
1536 case INDEX_op_extrl_i64_i32
:
1537 return TCG_TARGET_HAS_extrl_i64_i32
;
1538 case INDEX_op_extrh_i64_i32
:
1539 return TCG_TARGET_HAS_extrh_i64_i32
;
1540 case INDEX_op_ext8s_i64
:
1541 return TCG_TARGET_HAS_ext8s_i64
;
1542 case INDEX_op_ext16s_i64
:
1543 return TCG_TARGET_HAS_ext16s_i64
;
1544 case INDEX_op_ext32s_i64
:
1545 return TCG_TARGET_HAS_ext32s_i64
;
1546 case INDEX_op_ext8u_i64
:
1547 return TCG_TARGET_HAS_ext8u_i64
;
1548 case INDEX_op_ext16u_i64
:
1549 return TCG_TARGET_HAS_ext16u_i64
;
1550 case INDEX_op_ext32u_i64
:
1551 return TCG_TARGET_HAS_ext32u_i64
;
1552 case INDEX_op_bswap16_i64
:
1553 return TCG_TARGET_HAS_bswap16_i64
;
1554 case INDEX_op_bswap32_i64
:
1555 return TCG_TARGET_HAS_bswap32_i64
;
1556 case INDEX_op_bswap64_i64
:
1557 return TCG_TARGET_HAS_bswap64_i64
;
1558 case INDEX_op_not_i64
:
1559 return TCG_TARGET_HAS_not_i64
;
1560 case INDEX_op_neg_i64
:
1561 return TCG_TARGET_HAS_neg_i64
;
1562 case INDEX_op_andc_i64
:
1563 return TCG_TARGET_HAS_andc_i64
;
1564 case INDEX_op_orc_i64
:
1565 return TCG_TARGET_HAS_orc_i64
;
1566 case INDEX_op_eqv_i64
:
1567 return TCG_TARGET_HAS_eqv_i64
;
1568 case INDEX_op_nand_i64
:
1569 return TCG_TARGET_HAS_nand_i64
;
1570 case INDEX_op_nor_i64
:
1571 return TCG_TARGET_HAS_nor_i64
;
1572 case INDEX_op_clz_i64
:
1573 return TCG_TARGET_HAS_clz_i64
;
1574 case INDEX_op_ctz_i64
:
1575 return TCG_TARGET_HAS_ctz_i64
;
1576 case INDEX_op_ctpop_i64
:
1577 return TCG_TARGET_HAS_ctpop_i64
;
1578 case INDEX_op_add2_i64
:
1579 return TCG_TARGET_HAS_add2_i64
;
1580 case INDEX_op_sub2_i64
:
1581 return TCG_TARGET_HAS_sub2_i64
;
1582 case INDEX_op_mulu2_i64
:
1583 return TCG_TARGET_HAS_mulu2_i64
;
1584 case INDEX_op_muls2_i64
:
1585 return TCG_TARGET_HAS_muls2_i64
;
1586 case INDEX_op_muluh_i64
:
1587 return TCG_TARGET_HAS_muluh_i64
;
1588 case INDEX_op_mulsh_i64
:
1589 return TCG_TARGET_HAS_mulsh_i64
;
1591 case INDEX_op_mov_vec
:
1592 case INDEX_op_dup_vec
:
1593 case INDEX_op_dupm_vec
:
1594 case INDEX_op_ld_vec
:
1595 case INDEX_op_st_vec
:
1596 case INDEX_op_add_vec
:
1597 case INDEX_op_sub_vec
:
1598 case INDEX_op_and_vec
:
1599 case INDEX_op_or_vec
:
1600 case INDEX_op_xor_vec
:
1601 case INDEX_op_cmp_vec
:
1603 case INDEX_op_dup2_vec
:
1604 return have_vec
&& TCG_TARGET_REG_BITS
== 32;
1605 case INDEX_op_not_vec
:
1606 return have_vec
&& TCG_TARGET_HAS_not_vec
;
1607 case INDEX_op_neg_vec
:
1608 return have_vec
&& TCG_TARGET_HAS_neg_vec
;
1609 case INDEX_op_abs_vec
:
1610 return have_vec
&& TCG_TARGET_HAS_abs_vec
;
1611 case INDEX_op_andc_vec
:
1612 return have_vec
&& TCG_TARGET_HAS_andc_vec
;
1613 case INDEX_op_orc_vec
:
1614 return have_vec
&& TCG_TARGET_HAS_orc_vec
;
1615 case INDEX_op_nand_vec
:
1616 return have_vec
&& TCG_TARGET_HAS_nand_vec
;
1617 case INDEX_op_nor_vec
:
1618 return have_vec
&& TCG_TARGET_HAS_nor_vec
;
1619 case INDEX_op_eqv_vec
:
1620 return have_vec
&& TCG_TARGET_HAS_eqv_vec
;
1621 case INDEX_op_mul_vec
:
1622 return have_vec
&& TCG_TARGET_HAS_mul_vec
;
1623 case INDEX_op_shli_vec
:
1624 case INDEX_op_shri_vec
:
1625 case INDEX_op_sari_vec
:
1626 return have_vec
&& TCG_TARGET_HAS_shi_vec
;
1627 case INDEX_op_shls_vec
:
1628 case INDEX_op_shrs_vec
:
1629 case INDEX_op_sars_vec
:
1630 return have_vec
&& TCG_TARGET_HAS_shs_vec
;
1631 case INDEX_op_shlv_vec
:
1632 case INDEX_op_shrv_vec
:
1633 case INDEX_op_sarv_vec
:
1634 return have_vec
&& TCG_TARGET_HAS_shv_vec
;
1635 case INDEX_op_rotli_vec
:
1636 return have_vec
&& TCG_TARGET_HAS_roti_vec
;
1637 case INDEX_op_rotls_vec
:
1638 return have_vec
&& TCG_TARGET_HAS_rots_vec
;
1639 case INDEX_op_rotlv_vec
:
1640 case INDEX_op_rotrv_vec
:
1641 return have_vec
&& TCG_TARGET_HAS_rotv_vec
;
1642 case INDEX_op_ssadd_vec
:
1643 case INDEX_op_usadd_vec
:
1644 case INDEX_op_sssub_vec
:
1645 case INDEX_op_ussub_vec
:
1646 return have_vec
&& TCG_TARGET_HAS_sat_vec
;
1647 case INDEX_op_smin_vec
:
1648 case INDEX_op_umin_vec
:
1649 case INDEX_op_smax_vec
:
1650 case INDEX_op_umax_vec
:
1651 return have_vec
&& TCG_TARGET_HAS_minmax_vec
;
1652 case INDEX_op_bitsel_vec
:
1653 return have_vec
&& TCG_TARGET_HAS_bitsel_vec
;
1654 case INDEX_op_cmpsel_vec
:
1655 return have_vec
&& TCG_TARGET_HAS_cmpsel_vec
;
1658 tcg_debug_assert(op
> INDEX_op_last_generic
&& op
< NB_OPS
);
1663 static TCGOp
*tcg_op_alloc(TCGOpcode opc
, unsigned nargs
);
1665 void tcg_gen_callN(void *func
, TCGTemp
*ret
, int nargs
, TCGTemp
**args
)
1667 const TCGHelperInfo
*info
;
1668 TCGv_i64 extend_free
[MAX_CALL_IARGS
];
1671 int i
, n
, pi
= 0, total_args
;
1673 info
= g_hash_table_lookup(helper_table
, (gpointer
)func
);
1674 total_args
= info
->nr_out
+ info
->nr_in
+ 2;
1675 op
= tcg_op_alloc(INDEX_op_call
, total_args
);
1677 #ifdef CONFIG_PLUGIN
1678 /* detect non-plugin helpers */
1679 if (tcg_ctx
->plugin_insn
&& unlikely(strncmp(info
->name
, "plugin_", 7))) {
1680 tcg_ctx
->plugin_insn
->calls_helpers
= true;
1684 TCGOP_CALLO(op
) = n
= info
->nr_out
;
1687 tcg_debug_assert(ret
== NULL
);
1690 tcg_debug_assert(ret
!= NULL
);
1691 op
->args
[pi
++] = temp_arg(ret
);
1694 tcg_debug_assert(ret
!= NULL
);
1695 tcg_debug_assert(ret
->base_type
== ret
->type
+ 1);
1696 tcg_debug_assert(ret
->temp_subindex
== 0);
1697 op
->args
[pi
++] = temp_arg(ret
);
1698 op
->args
[pi
++] = temp_arg(ret
+ 1);
1701 g_assert_not_reached();
1704 TCGOP_CALLI(op
) = n
= info
->nr_in
;
1705 for (i
= 0; i
< n
; i
++) {
1706 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
1707 TCGTemp
*ts
= args
[loc
->arg_idx
] + loc
->tmp_subindex
;
1709 switch (loc
->kind
) {
1710 case TCG_CALL_ARG_NORMAL
:
1711 op
->args
[pi
++] = temp_arg(ts
);
1714 case TCG_CALL_ARG_EXTEND_U
:
1715 case TCG_CALL_ARG_EXTEND_S
:
1717 TCGv_i64 temp
= tcg_temp_new_i64();
1718 TCGv_i32 orig
= temp_tcgv_i32(ts
);
1720 if (loc
->kind
== TCG_CALL_ARG_EXTEND_S
) {
1721 tcg_gen_ext_i32_i64(temp
, orig
);
1723 tcg_gen_extu_i32_i64(temp
, orig
);
1725 op
->args
[pi
++] = tcgv_i64_arg(temp
);
1726 extend_free
[n_extend
++] = temp
;
1731 g_assert_not_reached();
1734 op
->args
[pi
++] = (uintptr_t)func
;
1735 op
->args
[pi
++] = (uintptr_t)info
;
1736 tcg_debug_assert(pi
== total_args
);
1738 QTAILQ_INSERT_TAIL(&tcg_ctx
->ops
, op
, link
);
1740 tcg_debug_assert(n_extend
< ARRAY_SIZE(extend_free
));
1741 for (i
= 0; i
< n_extend
; ++i
) {
1742 tcg_temp_free_i64(extend_free
[i
]);
1746 static void tcg_reg_alloc_start(TCGContext
*s
)
1750 for (i
= 0, n
= s
->nb_temps
; i
< n
; i
++) {
1751 TCGTemp
*ts
= &s
->temps
[i
];
1752 TCGTempVal val
= TEMP_VAL_MEM
;
1756 val
= TEMP_VAL_CONST
;
1765 val
= TEMP_VAL_DEAD
;
1768 ts
->mem_allocated
= 0;
1771 g_assert_not_reached();
1776 memset(s
->reg_to_temp
, 0, sizeof(s
->reg_to_temp
));
1779 static char *tcg_get_arg_str_ptr(TCGContext
*s
, char *buf
, int buf_size
,
1782 int idx
= temp_idx(ts
);
1787 pstrcpy(buf
, buf_size
, ts
->name
);
1790 snprintf(buf
, buf_size
, "loc%d", idx
- s
->nb_globals
);
1793 snprintf(buf
, buf_size
, "ebb%d", idx
- s
->nb_globals
);
1796 snprintf(buf
, buf_size
, "tmp%d", idx
- s
->nb_globals
);
1801 snprintf(buf
, buf_size
, "$0x%x", (int32_t)ts
->val
);
1803 #if TCG_TARGET_REG_BITS > 32
1805 snprintf(buf
, buf_size
, "$0x%" PRIx64
, ts
->val
);
1811 snprintf(buf
, buf_size
, "v%d$0x%" PRIx64
,
1812 64 << (ts
->type
- TCG_TYPE_V64
), ts
->val
);
1815 g_assert_not_reached();
1822 static char *tcg_get_arg_str(TCGContext
*s
, char *buf
,
1823 int buf_size
, TCGArg arg
)
1825 return tcg_get_arg_str_ptr(s
, buf
, buf_size
, arg_temp(arg
));
1828 static const char * const cond_name
[] =
1830 [TCG_COND_NEVER
] = "never",
1831 [TCG_COND_ALWAYS
] = "always",
1832 [TCG_COND_EQ
] = "eq",
1833 [TCG_COND_NE
] = "ne",
1834 [TCG_COND_LT
] = "lt",
1835 [TCG_COND_GE
] = "ge",
1836 [TCG_COND_LE
] = "le",
1837 [TCG_COND_GT
] = "gt",
1838 [TCG_COND_LTU
] = "ltu",
1839 [TCG_COND_GEU
] = "geu",
1840 [TCG_COND_LEU
] = "leu",
1841 [TCG_COND_GTU
] = "gtu"
1844 static const char * const ldst_name
[] =
1860 static const char * const alignment_name
[(MO_AMASK
>> MO_ASHIFT
) + 1] = {
1861 #ifdef TARGET_ALIGNED_ONLY
1862 [MO_UNALN
>> MO_ASHIFT
] = "un+",
1863 [MO_ALIGN
>> MO_ASHIFT
] = "",
1865 [MO_UNALN
>> MO_ASHIFT
] = "",
1866 [MO_ALIGN
>> MO_ASHIFT
] = "al+",
1868 [MO_ALIGN_2
>> MO_ASHIFT
] = "al2+",
1869 [MO_ALIGN_4
>> MO_ASHIFT
] = "al4+",
1870 [MO_ALIGN_8
>> MO_ASHIFT
] = "al8+",
1871 [MO_ALIGN_16
>> MO_ASHIFT
] = "al16+",
1872 [MO_ALIGN_32
>> MO_ASHIFT
] = "al32+",
1873 [MO_ALIGN_64
>> MO_ASHIFT
] = "al64+",
1876 static const char bswap_flag_name
[][6] = {
1877 [TCG_BSWAP_IZ
] = "iz",
1878 [TCG_BSWAP_OZ
] = "oz",
1879 [TCG_BSWAP_OS
] = "os",
1880 [TCG_BSWAP_IZ
| TCG_BSWAP_OZ
] = "iz,oz",
1881 [TCG_BSWAP_IZ
| TCG_BSWAP_OS
] = "iz,os",
1884 static inline bool tcg_regset_single(TCGRegSet d
)
1886 return (d
& (d
- 1)) == 0;
1889 static inline TCGReg
tcg_regset_first(TCGRegSet d
)
1891 if (TCG_TARGET_NB_REGS
<= 32) {
1898 /* Return only the number of characters output -- no error return. */
1899 #define ne_fprintf(...) \
1900 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1902 static void tcg_dump_ops(TCGContext
*s
, FILE *f
, bool have_prefs
)
1907 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
1908 int i
, k
, nb_oargs
, nb_iargs
, nb_cargs
;
1909 const TCGOpDef
*def
;
1914 def
= &tcg_op_defs
[c
];
1916 if (c
== INDEX_op_insn_start
) {
1918 col
+= ne_fprintf(f
, "\n ----");
1920 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
1922 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1923 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
1927 col
+= ne_fprintf(f
, " " TARGET_FMT_lx
, a
);
1929 } else if (c
== INDEX_op_call
) {
1930 const TCGHelperInfo
*info
= tcg_call_info(op
);
1931 void *func
= tcg_call_func(op
);
1933 /* variable number of arguments */
1934 nb_oargs
= TCGOP_CALLO(op
);
1935 nb_iargs
= TCGOP_CALLI(op
);
1936 nb_cargs
= def
->nb_cargs
;
1938 col
+= ne_fprintf(f
, " %s ", def
->name
);
1941 * Print the function name from TCGHelperInfo, if available.
1942 * Note that plugins have a template function for the info,
1943 * but the actual function pointer comes from the plugin.
1945 if (func
== info
->func
) {
1946 col
+= ne_fprintf(f
, "%s", info
->name
);
1948 col
+= ne_fprintf(f
, "plugin(%p)", func
);
1951 col
+= ne_fprintf(f
, ",$0x%x,$%d", info
->flags
, nb_oargs
);
1952 for (i
= 0; i
< nb_oargs
; i
++) {
1953 col
+= ne_fprintf(f
, ",%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
1956 for (i
= 0; i
< nb_iargs
; i
++) {
1957 TCGArg arg
= op
->args
[nb_oargs
+ i
];
1958 const char *t
= tcg_get_arg_str(s
, buf
, sizeof(buf
), arg
);
1959 col
+= ne_fprintf(f
, ",%s", t
);
1962 col
+= ne_fprintf(f
, " %s ", def
->name
);
1964 nb_oargs
= def
->nb_oargs
;
1965 nb_iargs
= def
->nb_iargs
;
1966 nb_cargs
= def
->nb_cargs
;
1968 if (def
->flags
& TCG_OPF_VECTOR
) {
1969 col
+= ne_fprintf(f
, "v%d,e%d,", 64 << TCGOP_VECL(op
),
1970 8 << TCGOP_VECE(op
));
1974 for (i
= 0; i
< nb_oargs
; i
++) {
1975 const char *sep
= k
? "," : "";
1976 col
+= ne_fprintf(f
, "%s%s", sep
,
1977 tcg_get_arg_str(s
, buf
, sizeof(buf
),
1980 for (i
= 0; i
< nb_iargs
; i
++) {
1981 const char *sep
= k
? "," : "";
1982 col
+= ne_fprintf(f
, "%s%s", sep
,
1983 tcg_get_arg_str(s
, buf
, sizeof(buf
),
1987 case INDEX_op_brcond_i32
:
1988 case INDEX_op_setcond_i32
:
1989 case INDEX_op_movcond_i32
:
1990 case INDEX_op_brcond2_i32
:
1991 case INDEX_op_setcond2_i32
:
1992 case INDEX_op_brcond_i64
:
1993 case INDEX_op_setcond_i64
:
1994 case INDEX_op_movcond_i64
:
1995 case INDEX_op_cmp_vec
:
1996 case INDEX_op_cmpsel_vec
:
1997 if (op
->args
[k
] < ARRAY_SIZE(cond_name
)
1998 && cond_name
[op
->args
[k
]]) {
1999 col
+= ne_fprintf(f
, ",%s", cond_name
[op
->args
[k
++]]);
2001 col
+= ne_fprintf(f
, ",$0x%" TCG_PRIlx
, op
->args
[k
++]);
2005 case INDEX_op_qemu_ld_i32
:
2006 case INDEX_op_qemu_st_i32
:
2007 case INDEX_op_qemu_st8_i32
:
2008 case INDEX_op_qemu_ld_i64
:
2009 case INDEX_op_qemu_st_i64
:
2011 MemOpIdx oi
= op
->args
[k
++];
2012 MemOp op
= get_memop(oi
);
2013 unsigned ix
= get_mmuidx(oi
);
2015 if (op
& ~(MO_AMASK
| MO_BSWAP
| MO_SSIZE
)) {
2016 col
+= ne_fprintf(f
, ",$0x%x,%u", op
, ix
);
2018 const char *s_al
, *s_op
;
2019 s_al
= alignment_name
[(op
& MO_AMASK
) >> MO_ASHIFT
];
2020 s_op
= ldst_name
[op
& (MO_BSWAP
| MO_SSIZE
)];
2021 col
+= ne_fprintf(f
, ",%s%s,%u", s_al
, s_op
, ix
);
2026 case INDEX_op_bswap16_i32
:
2027 case INDEX_op_bswap16_i64
:
2028 case INDEX_op_bswap32_i32
:
2029 case INDEX_op_bswap32_i64
:
2030 case INDEX_op_bswap64_i64
:
2032 TCGArg flags
= op
->args
[k
];
2033 const char *name
= NULL
;
2035 if (flags
< ARRAY_SIZE(bswap_flag_name
)) {
2036 name
= bswap_flag_name
[flags
];
2039 col
+= ne_fprintf(f
, ",%s", name
);
2041 col
+= ne_fprintf(f
, ",$0x%" TCG_PRIlx
, flags
);
2051 case INDEX_op_set_label
:
2053 case INDEX_op_brcond_i32
:
2054 case INDEX_op_brcond_i64
:
2055 case INDEX_op_brcond2_i32
:
2056 col
+= ne_fprintf(f
, "%s$L%d", k
? "," : "",
2057 arg_label(op
->args
[k
])->id
);
2063 for (; i
< nb_cargs
; i
++, k
++) {
2064 col
+= ne_fprintf(f
, "%s$0x%" TCG_PRIlx
, k
? "," : "",
2069 if (have_prefs
|| op
->life
) {
2070 for (; col
< 40; ++col
) {
2076 unsigned life
= op
->life
;
2078 if (life
& (SYNC_ARG
* 3)) {
2079 ne_fprintf(f
, " sync:");
2080 for (i
= 0; i
< 2; ++i
) {
2081 if (life
& (SYNC_ARG
<< i
)) {
2082 ne_fprintf(f
, " %d", i
);
2088 ne_fprintf(f
, " dead:");
2089 for (i
= 0; life
; ++i
, life
>>= 1) {
2091 ne_fprintf(f
, " %d", i
);
2098 for (i
= 0; i
< nb_oargs
; ++i
) {
2099 TCGRegSet set
= output_pref(op
, i
);
2102 ne_fprintf(f
, " pref=");
2107 ne_fprintf(f
, "none");
2108 } else if (set
== MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS
)) {
2109 ne_fprintf(f
, "all");
2110 #ifdef CONFIG_DEBUG_TCG
2111 } else if (tcg_regset_single(set
)) {
2112 TCGReg reg
= tcg_regset_first(set
);
2113 ne_fprintf(f
, "%s", tcg_target_reg_names
[reg
]);
2115 } else if (TCG_TARGET_NB_REGS
<= 32) {
2116 ne_fprintf(f
, "0x%x", (uint32_t)set
);
2118 ne_fprintf(f
, "0x%" PRIx64
, (uint64_t)set
);
2127 /* we give more priority to constraints with less registers */
2128 static int get_constraint_priority(const TCGOpDef
*def
, int k
)
2130 const TCGArgConstraint
*arg_ct
= &def
->args_ct
[k
];
2131 int n
= ctpop64(arg_ct
->regs
);
2134 * Sort constraints of a single register first, which includes output
2135 * aliases (which must exactly match the input already allocated).
2137 if (n
== 1 || arg_ct
->oalias
) {
2142 * Sort register pairs next, first then second immediately after.
2143 * Arbitrarily sort multiple pairs by the index of the first reg;
2144 * there shouldn't be many pairs.
2146 switch (arg_ct
->pair
) {
2151 return (arg_ct
->pair_index
+ 1) * 2 - 1;
2154 /* Finally, sort by decreasing register count. */
2159 /* sort from highest priority to lowest */
2160 static void sort_constraints(TCGOpDef
*def
, int start
, int n
)
2163 TCGArgConstraint
*a
= def
->args_ct
;
2165 for (i
= 0; i
< n
; i
++) {
2166 a
[start
+ i
].sort_index
= start
+ i
;
2171 for (i
= 0; i
< n
- 1; i
++) {
2172 for (j
= i
+ 1; j
< n
; j
++) {
2173 int p1
= get_constraint_priority(def
, a
[start
+ i
].sort_index
);
2174 int p2
= get_constraint_priority(def
, a
[start
+ j
].sort_index
);
2176 int tmp
= a
[start
+ i
].sort_index
;
2177 a
[start
+ i
].sort_index
= a
[start
+ j
].sort_index
;
2178 a
[start
+ j
].sort_index
= tmp
;
2184 static void process_op_defs(TCGContext
*s
)
2188 for (op
= 0; op
< NB_OPS
; op
++) {
2189 TCGOpDef
*def
= &tcg_op_defs
[op
];
2190 const TCGTargetOpDef
*tdefs
;
2191 bool saw_alias_pair
= false;
2192 int i
, o
, i2
, o2
, nb_args
;
2194 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
2198 nb_args
= def
->nb_iargs
+ def
->nb_oargs
;
2204 * Macro magic should make it impossible, but double-check that
2205 * the array index is in range. Since the signness of an enum
2206 * is implementation defined, force the result to unsigned.
2208 unsigned con_set
= tcg_target_op_def(op
);
2209 tcg_debug_assert(con_set
< ARRAY_SIZE(constraint_sets
));
2210 tdefs
= &constraint_sets
[con_set
];
2212 for (i
= 0; i
< nb_args
; i
++) {
2213 const char *ct_str
= tdefs
->args_ct_str
[i
];
2214 bool input_p
= i
>= def
->nb_oargs
;
2216 /* Incomplete TCGTargetOpDef entry. */
2217 tcg_debug_assert(ct_str
!= NULL
);
2222 tcg_debug_assert(input_p
);
2223 tcg_debug_assert(o
< def
->nb_oargs
);
2224 tcg_debug_assert(def
->args_ct
[o
].regs
!= 0);
2225 tcg_debug_assert(!def
->args_ct
[o
].oalias
);
2226 def
->args_ct
[i
] = def
->args_ct
[o
];
2227 /* The output sets oalias. */
2228 def
->args_ct
[o
].oalias
= 1;
2229 def
->args_ct
[o
].alias_index
= i
;
2230 /* The input sets ialias. */
2231 def
->args_ct
[i
].ialias
= 1;
2232 def
->args_ct
[i
].alias_index
= o
;
2233 if (def
->args_ct
[i
].pair
) {
2234 saw_alias_pair
= true;
2236 tcg_debug_assert(ct_str
[1] == '\0');
2240 tcg_debug_assert(!input_p
);
2241 def
->args_ct
[i
].newreg
= true;
2245 case 'p': /* plus */
2246 /* Allocate to the register after the previous. */
2247 tcg_debug_assert(i
> (input_p
? def
->nb_oargs
: 0));
2249 tcg_debug_assert(!def
->args_ct
[o
].pair
);
2250 tcg_debug_assert(!def
->args_ct
[o
].ct
);
2251 def
->args_ct
[i
] = (TCGArgConstraint
){
2254 .regs
= def
->args_ct
[o
].regs
<< 1,
2256 def
->args_ct
[o
].pair
= 1;
2257 def
->args_ct
[o
].pair_index
= i
;
2258 tcg_debug_assert(ct_str
[1] == '\0');
2261 case 'm': /* minus */
2262 /* Allocate to the register before the previous. */
2263 tcg_debug_assert(i
> (input_p
? def
->nb_oargs
: 0));
2265 tcg_debug_assert(!def
->args_ct
[o
].pair
);
2266 tcg_debug_assert(!def
->args_ct
[o
].ct
);
2267 def
->args_ct
[i
] = (TCGArgConstraint
){
2270 .regs
= def
->args_ct
[o
].regs
>> 1,
2272 def
->args_ct
[o
].pair
= 2;
2273 def
->args_ct
[o
].pair_index
= i
;
2274 tcg_debug_assert(ct_str
[1] == '\0');
2281 def
->args_ct
[i
].ct
|= TCG_CT_CONST
;
2284 /* Include all of the target-specific constraints. */
2287 #define CONST(CASE, MASK) \
2288 case CASE: def->args_ct[i].ct |= MASK; break;
2289 #define REGS(CASE, MASK) \
2290 case CASE: def->args_ct[i].regs |= MASK; break;
2292 #include "tcg-target-con-str.h"
2301 /* Typo in TCGTargetOpDef constraint. */
2302 g_assert_not_reached();
2304 } while (*++ct_str
!= '\0');
2307 /* TCGTargetOpDef entry with too much information? */
2308 tcg_debug_assert(i
== TCG_MAX_OP_ARGS
|| tdefs
->args_ct_str
[i
] == NULL
);
2311 * Fix up output pairs that are aliased with inputs.
2312 * When we created the alias, we copied pair from the output.
2313 * There are three cases:
2314 * (1a) Pairs of inputs alias pairs of outputs.
2315 * (1b) One input aliases the first of a pair of outputs.
2316 * (2) One input aliases the second of a pair of outputs.
2318 * Case 1a is handled by making sure that the pair_index'es are
2319 * properly updated so that they appear the same as a pair of inputs.
2321 * Case 1b is handled by setting the pair_index of the input to
2322 * itself, simply so it doesn't point to an unrelated argument.
2323 * Since we don't encounter the "second" during the input allocation
2324 * phase, nothing happens with the second half of the input pair.
2326 * Case 2 is handled by setting the second input to pair=3, the
2327 * first output to pair=3, and the pair_index'es to match.
2329 if (saw_alias_pair
) {
2330 for (i
= def
->nb_oargs
; i
< nb_args
; i
++) {
2332 * Since [0-9pm] must be alone in the constraint string,
2333 * the only way they can both be set is if the pair comes
2334 * from the output alias.
2336 if (!def
->args_ct
[i
].ialias
) {
2339 switch (def
->args_ct
[i
].pair
) {
2343 o
= def
->args_ct
[i
].alias_index
;
2344 o2
= def
->args_ct
[o
].pair_index
;
2345 tcg_debug_assert(def
->args_ct
[o
].pair
== 1);
2346 tcg_debug_assert(def
->args_ct
[o2
].pair
== 2);
2347 if (def
->args_ct
[o2
].oalias
) {
2349 i2
= def
->args_ct
[o2
].alias_index
;
2350 tcg_debug_assert(def
->args_ct
[i2
].pair
== 2);
2351 def
->args_ct
[i2
].pair_index
= i
;
2352 def
->args_ct
[i
].pair_index
= i2
;
2355 def
->args_ct
[i
].pair_index
= i
;
2359 o
= def
->args_ct
[i
].alias_index
;
2360 o2
= def
->args_ct
[o
].pair_index
;
2361 tcg_debug_assert(def
->args_ct
[o
].pair
== 2);
2362 tcg_debug_assert(def
->args_ct
[o2
].pair
== 1);
2363 if (def
->args_ct
[o2
].oalias
) {
2365 i2
= def
->args_ct
[o2
].alias_index
;
2366 tcg_debug_assert(def
->args_ct
[i2
].pair
== 1);
2367 def
->args_ct
[i2
].pair_index
= i
;
2368 def
->args_ct
[i
].pair_index
= i2
;
2371 def
->args_ct
[i
].pair
= 3;
2372 def
->args_ct
[o2
].pair
= 3;
2373 def
->args_ct
[i
].pair_index
= o2
;
2374 def
->args_ct
[o2
].pair_index
= i
;
2378 g_assert_not_reached();
2383 /* sort the constraints (XXX: this is just an heuristic) */
2384 sort_constraints(def
, 0, def
->nb_oargs
);
2385 sort_constraints(def
, def
->nb_oargs
, def
->nb_iargs
);
2389 void tcg_op_remove(TCGContext
*s
, TCGOp
*op
)
2395 label
= arg_label(op
->args
[0]);
2398 case INDEX_op_brcond_i32
:
2399 case INDEX_op_brcond_i64
:
2400 label
= arg_label(op
->args
[3]);
2403 case INDEX_op_brcond2_i32
:
2404 label
= arg_label(op
->args
[5]);
2411 QTAILQ_REMOVE(&s
->ops
, op
, link
);
2412 QTAILQ_INSERT_TAIL(&s
->free_ops
, op
, link
);
2415 #ifdef CONFIG_PROFILER
2416 qatomic_set(&s
->prof
.del_op_count
, s
->prof
.del_op_count
+ 1);
2420 void tcg_remove_ops_after(TCGOp
*op
)
2422 TCGContext
*s
= tcg_ctx
;
2425 TCGOp
*last
= tcg_last_op();
2429 tcg_op_remove(s
, last
);
2433 static TCGOp
*tcg_op_alloc(TCGOpcode opc
, unsigned nargs
)
2435 TCGContext
*s
= tcg_ctx
;
2438 if (unlikely(!QTAILQ_EMPTY(&s
->free_ops
))) {
2439 QTAILQ_FOREACH(op
, &s
->free_ops
, link
) {
2440 if (nargs
<= op
->nargs
) {
2441 QTAILQ_REMOVE(&s
->free_ops
, op
, link
);
2448 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2449 nargs
= MAX(4, nargs
);
2450 op
= tcg_malloc(sizeof(TCGOp
) + sizeof(TCGArg
) * nargs
);
2453 memset(op
, 0, offsetof(TCGOp
, link
));
2457 /* Check for bitfield overflow. */
2458 tcg_debug_assert(op
->nargs
== nargs
);
2464 TCGOp
*tcg_emit_op(TCGOpcode opc
, unsigned nargs
)
2466 TCGOp
*op
= tcg_op_alloc(opc
, nargs
);
2467 QTAILQ_INSERT_TAIL(&tcg_ctx
->ops
, op
, link
);
2471 TCGOp
*tcg_op_insert_before(TCGContext
*s
, TCGOp
*old_op
,
2472 TCGOpcode opc
, unsigned nargs
)
2474 TCGOp
*new_op
= tcg_op_alloc(opc
, nargs
);
2475 QTAILQ_INSERT_BEFORE(old_op
, new_op
, link
);
2479 TCGOp
*tcg_op_insert_after(TCGContext
*s
, TCGOp
*old_op
,
2480 TCGOpcode opc
, unsigned nargs
)
2482 TCGOp
*new_op
= tcg_op_alloc(opc
, nargs
);
2483 QTAILQ_INSERT_AFTER(&s
->ops
, old_op
, new_op
, link
);
2487 /* Reachable analysis : remove unreachable code. */
2488 static void reachable_code_pass(TCGContext
*s
)
2490 TCGOp
*op
, *op_next
;
2493 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2498 case INDEX_op_set_label
:
2499 label
= arg_label(op
->args
[0]);
2500 if (label
->refs
== 0) {
2502 * While there is an occasional backward branch, virtually
2503 * all branches generated by the translators are forward.
2504 * Which means that generally we will have already removed
2505 * all references to the label that will be, and there is
2506 * little to be gained by iterating.
2510 /* Once we see a label, insns become live again. */
2515 * Optimization can fold conditional branches to unconditional.
2516 * If we find a label with one reference which is preceded by
2517 * an unconditional branch to it, remove both. This needed to
2518 * wait until the dead code in between them was removed.
2520 if (label
->refs
== 1) {
2521 TCGOp
*op_prev
= QTAILQ_PREV(op
, link
);
2522 if (op_prev
->opc
== INDEX_op_br
&&
2523 label
== arg_label(op_prev
->args
[0])) {
2524 tcg_op_remove(s
, op_prev
);
2532 case INDEX_op_exit_tb
:
2533 case INDEX_op_goto_ptr
:
2534 /* Unconditional branches; everything following is dead. */
2539 /* Notice noreturn helper calls, raising exceptions. */
2540 if (tcg_call_flags(op
) & TCG_CALL_NO_RETURN
) {
2545 case INDEX_op_insn_start
:
2546 /* Never remove -- we need to keep these for unwind. */
2555 tcg_op_remove(s
, op
);
2563 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2564 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2566 /* For liveness_pass_1, the register preferences for a given temp. */
2567 static inline TCGRegSet
*la_temp_pref(TCGTemp
*ts
)
2569 return ts
->state_ptr
;
2572 /* For liveness_pass_1, reset the preferences for a given temp to the
2573 * maximal regset for its type.
2575 static inline void la_reset_pref(TCGTemp
*ts
)
2578 = (ts
->state
== TS_DEAD
? 0 : tcg_target_available_regs
[ts
->type
]);
2581 /* liveness analysis: end of function: all temps are dead, and globals
2582 should be in memory. */
2583 static void la_func_end(TCGContext
*s
, int ng
, int nt
)
2587 for (i
= 0; i
< ng
; ++i
) {
2588 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2589 la_reset_pref(&s
->temps
[i
]);
2591 for (i
= ng
; i
< nt
; ++i
) {
2592 s
->temps
[i
].state
= TS_DEAD
;
2593 la_reset_pref(&s
->temps
[i
]);
2597 /* liveness analysis: end of basic block: all temps are dead, globals
2598 and local temps should be in memory. */
2599 static void la_bb_end(TCGContext
*s
, int ng
, int nt
)
2603 for (i
= 0; i
< nt
; ++i
) {
2604 TCGTemp
*ts
= &s
->temps
[i
];
2611 state
= TS_DEAD
| TS_MEM
;
2619 g_assert_not_reached();
2626 /* liveness analysis: sync globals back to memory. */
2627 static void la_global_sync(TCGContext
*s
, int ng
)
2631 for (i
= 0; i
< ng
; ++i
) {
2632 int state
= s
->temps
[i
].state
;
2633 s
->temps
[i
].state
= state
| TS_MEM
;
2634 if (state
== TS_DEAD
) {
2635 /* If the global was previously dead, reset prefs. */
2636 la_reset_pref(&s
->temps
[i
]);
2642 * liveness analysis: conditional branch: all temps are dead unless
2643 * explicitly live-across-conditional-branch, globals and local temps
2646 static void la_bb_sync(TCGContext
*s
, int ng
, int nt
)
2648 la_global_sync(s
, ng
);
2650 for (int i
= ng
; i
< nt
; ++i
) {
2651 TCGTemp
*ts
= &s
->temps
[i
];
2657 ts
->state
= state
| TS_MEM
;
2658 if (state
!= TS_DEAD
) {
2663 s
->temps
[i
].state
= TS_DEAD
;
2669 g_assert_not_reached();
2671 la_reset_pref(&s
->temps
[i
]);
2675 /* liveness analysis: sync globals back to memory and kill. */
2676 static void la_global_kill(TCGContext
*s
, int ng
)
2680 for (i
= 0; i
< ng
; i
++) {
2681 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2682 la_reset_pref(&s
->temps
[i
]);
2686 /* liveness analysis: note live globals crossing calls. */
2687 static void la_cross_call(TCGContext
*s
, int nt
)
2689 TCGRegSet mask
= ~tcg_target_call_clobber_regs
;
2692 for (i
= 0; i
< nt
; i
++) {
2693 TCGTemp
*ts
= &s
->temps
[i
];
2694 if (!(ts
->state
& TS_DEAD
)) {
2695 TCGRegSet
*pset
= la_temp_pref(ts
);
2696 TCGRegSet set
= *pset
;
2699 /* If the combination is not possible, restart. */
2701 set
= tcg_target_available_regs
[ts
->type
] & mask
;
2708 /* Liveness analysis : update the opc_arg_life array to tell if a
2709 given input arguments is dead. Instructions updating dead
2710 temporaries are removed. */
2711 static void liveness_pass_1(TCGContext
*s
)
2713 int nb_globals
= s
->nb_globals
;
2714 int nb_temps
= s
->nb_temps
;
2715 TCGOp
*op
, *op_prev
;
2719 prefs
= tcg_malloc(sizeof(TCGRegSet
) * nb_temps
);
2720 for (i
= 0; i
< nb_temps
; ++i
) {
2721 s
->temps
[i
].state_ptr
= prefs
+ i
;
2724 /* ??? Should be redundant with the exit_tb that ends the TB. */
2725 la_func_end(s
, nb_globals
, nb_temps
);
2727 QTAILQ_FOREACH_REVERSE_SAFE(op
, &s
->ops
, link
, op_prev
) {
2728 int nb_iargs
, nb_oargs
;
2729 TCGOpcode opc_new
, opc_new2
;
2731 TCGLifeData arg_life
= 0;
2733 TCGOpcode opc
= op
->opc
;
2734 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2739 const TCGHelperInfo
*info
= tcg_call_info(op
);
2740 int call_flags
= tcg_call_flags(op
);
2742 nb_oargs
= TCGOP_CALLO(op
);
2743 nb_iargs
= TCGOP_CALLI(op
);
2745 /* pure functions can be removed if their result is unused */
2746 if (call_flags
& TCG_CALL_NO_SIDE_EFFECTS
) {
2747 for (i
= 0; i
< nb_oargs
; i
++) {
2748 ts
= arg_temp(op
->args
[i
]);
2749 if (ts
->state
!= TS_DEAD
) {
2750 goto do_not_remove_call
;
2757 /* Output args are dead. */
2758 for (i
= 0; i
< nb_oargs
; i
++) {
2759 ts
= arg_temp(op
->args
[i
]);
2760 if (ts
->state
& TS_DEAD
) {
2761 arg_life
|= DEAD_ARG
<< i
;
2763 if (ts
->state
& TS_MEM
) {
2764 arg_life
|= SYNC_ARG
<< i
;
2766 ts
->state
= TS_DEAD
;
2770 /* Not used -- it will be tcg_target_call_oarg_reg(). */
2771 memset(op
->output_pref
, 0, sizeof(op
->output_pref
));
2773 if (!(call_flags
& (TCG_CALL_NO_WRITE_GLOBALS
|
2774 TCG_CALL_NO_READ_GLOBALS
))) {
2775 la_global_kill(s
, nb_globals
);
2776 } else if (!(call_flags
& TCG_CALL_NO_READ_GLOBALS
)) {
2777 la_global_sync(s
, nb_globals
);
2780 /* Record arguments that die in this helper. */
2781 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2782 ts
= arg_temp(op
->args
[i
]);
2783 if (ts
->state
& TS_DEAD
) {
2784 arg_life
|= DEAD_ARG
<< i
;
2788 /* For all live registers, remove call-clobbered prefs. */
2789 la_cross_call(s
, nb_temps
);
2792 * Input arguments are live for preceding opcodes.
2794 * For those arguments that die, and will be allocated in
2795 * registers, clear the register set for that arg, to be
2796 * filled in below. For args that will be on the stack,
2797 * reset to any available reg. Process arguments in reverse
2798 * order so that if a temp is used more than once, the stack
2799 * reset to max happens before the register reset to 0.
2801 for (i
= nb_iargs
- 1; i
>= 0; i
--) {
2802 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
2803 ts
= arg_temp(op
->args
[nb_oargs
+ i
]);
2805 if (ts
->state
& TS_DEAD
) {
2806 switch (loc
->kind
) {
2807 case TCG_CALL_ARG_NORMAL
:
2808 case TCG_CALL_ARG_EXTEND_U
:
2809 case TCG_CALL_ARG_EXTEND_S
:
2811 *la_temp_pref(ts
) = 0;
2817 tcg_target_available_regs
[ts
->type
];
2820 ts
->state
&= ~TS_DEAD
;
2825 * For each input argument, add its input register to prefs.
2826 * If a temp is used once, this produces a single set bit;
2827 * if a temp is used multiple times, this produces a set.
2829 for (i
= 0; i
< nb_iargs
; i
++) {
2830 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
2831 ts
= arg_temp(op
->args
[nb_oargs
+ i
]);
2833 switch (loc
->kind
) {
2834 case TCG_CALL_ARG_NORMAL
:
2835 case TCG_CALL_ARG_EXTEND_U
:
2836 case TCG_CALL_ARG_EXTEND_S
:
2838 tcg_regset_set_reg(*la_temp_pref(ts
),
2839 tcg_target_call_iarg_regs
[loc
->arg_slot
]);
2848 case INDEX_op_insn_start
:
2850 case INDEX_op_discard
:
2851 /* mark the temporary as dead */
2852 ts
= arg_temp(op
->args
[0]);
2853 ts
->state
= TS_DEAD
;
2857 case INDEX_op_add2_i32
:
2858 opc_new
= INDEX_op_add_i32
;
2860 case INDEX_op_sub2_i32
:
2861 opc_new
= INDEX_op_sub_i32
;
2863 case INDEX_op_add2_i64
:
2864 opc_new
= INDEX_op_add_i64
;
2866 case INDEX_op_sub2_i64
:
2867 opc_new
= INDEX_op_sub_i64
;
2871 /* Test if the high part of the operation is dead, but not
2872 the low part. The result can be optimized to a simple
2873 add or sub. This happens often for x86_64 guest when the
2874 cpu mode is set to 32 bit. */
2875 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2876 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2879 /* Replace the opcode and adjust the args in place,
2880 leaving 3 unused args at the end. */
2881 op
->opc
= opc
= opc_new
;
2882 op
->args
[1] = op
->args
[2];
2883 op
->args
[2] = op
->args
[4];
2884 /* Fall through and mark the single-word operation live. */
2890 case INDEX_op_mulu2_i32
:
2891 opc_new
= INDEX_op_mul_i32
;
2892 opc_new2
= INDEX_op_muluh_i32
;
2893 have_opc_new2
= TCG_TARGET_HAS_muluh_i32
;
2895 case INDEX_op_muls2_i32
:
2896 opc_new
= INDEX_op_mul_i32
;
2897 opc_new2
= INDEX_op_mulsh_i32
;
2898 have_opc_new2
= TCG_TARGET_HAS_mulsh_i32
;
2900 case INDEX_op_mulu2_i64
:
2901 opc_new
= INDEX_op_mul_i64
;
2902 opc_new2
= INDEX_op_muluh_i64
;
2903 have_opc_new2
= TCG_TARGET_HAS_muluh_i64
;
2905 case INDEX_op_muls2_i64
:
2906 opc_new
= INDEX_op_mul_i64
;
2907 opc_new2
= INDEX_op_mulsh_i64
;
2908 have_opc_new2
= TCG_TARGET_HAS_mulsh_i64
;
2913 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2914 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2915 /* Both parts of the operation are dead. */
2918 /* The high part of the operation is dead; generate the low. */
2919 op
->opc
= opc
= opc_new
;
2920 op
->args
[1] = op
->args
[2];
2921 op
->args
[2] = op
->args
[3];
2922 } else if (arg_temp(op
->args
[0])->state
== TS_DEAD
&& have_opc_new2
) {
2923 /* The low part of the operation is dead; generate the high. */
2924 op
->opc
= opc
= opc_new2
;
2925 op
->args
[0] = op
->args
[1];
2926 op
->args
[1] = op
->args
[2];
2927 op
->args
[2] = op
->args
[3];
2931 /* Mark the single-word operation live. */
2936 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2937 nb_iargs
= def
->nb_iargs
;
2938 nb_oargs
= def
->nb_oargs
;
2940 /* Test if the operation can be removed because all
2941 its outputs are dead. We assume that nb_oargs == 0
2942 implies side effects */
2943 if (!(def
->flags
& TCG_OPF_SIDE_EFFECTS
) && nb_oargs
!= 0) {
2944 for (i
= 0; i
< nb_oargs
; i
++) {
2945 if (arg_temp(op
->args
[i
])->state
!= TS_DEAD
) {
2954 tcg_op_remove(s
, op
);
2958 for (i
= 0; i
< nb_oargs
; i
++) {
2959 ts
= arg_temp(op
->args
[i
]);
2961 /* Remember the preference of the uses that followed. */
2962 if (i
< ARRAY_SIZE(op
->output_pref
)) {
2963 op
->output_pref
[i
] = *la_temp_pref(ts
);
2966 /* Output args are dead. */
2967 if (ts
->state
& TS_DEAD
) {
2968 arg_life
|= DEAD_ARG
<< i
;
2970 if (ts
->state
& TS_MEM
) {
2971 arg_life
|= SYNC_ARG
<< i
;
2973 ts
->state
= TS_DEAD
;
2977 /* If end of basic block, update. */
2978 if (def
->flags
& TCG_OPF_BB_EXIT
) {
2979 la_func_end(s
, nb_globals
, nb_temps
);
2980 } else if (def
->flags
& TCG_OPF_COND_BRANCH
) {
2981 la_bb_sync(s
, nb_globals
, nb_temps
);
2982 } else if (def
->flags
& TCG_OPF_BB_END
) {
2983 la_bb_end(s
, nb_globals
, nb_temps
);
2984 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2985 la_global_sync(s
, nb_globals
);
2986 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
2987 la_cross_call(s
, nb_temps
);
2991 /* Record arguments that die in this opcode. */
2992 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2993 ts
= arg_temp(op
->args
[i
]);
2994 if (ts
->state
& TS_DEAD
) {
2995 arg_life
|= DEAD_ARG
<< i
;
2999 /* Input arguments are live for preceding opcodes. */
3000 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
3001 ts
= arg_temp(op
->args
[i
]);
3002 if (ts
->state
& TS_DEAD
) {
3003 /* For operands that were dead, initially allow
3004 all regs for the type. */
3005 *la_temp_pref(ts
) = tcg_target_available_regs
[ts
->type
];
3006 ts
->state
&= ~TS_DEAD
;
3010 /* Incorporate constraints for this operand. */
3012 case INDEX_op_mov_i32
:
3013 case INDEX_op_mov_i64
:
3014 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3015 have proper constraints. That said, special case
3016 moves to propagate preferences backward. */
3017 if (IS_DEAD_ARG(1)) {
3018 *la_temp_pref(arg_temp(op
->args
[0]))
3019 = *la_temp_pref(arg_temp(op
->args
[1]));
3024 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
3025 const TCGArgConstraint
*ct
= &def
->args_ct
[i
];
3026 TCGRegSet set
, *pset
;
3028 ts
= arg_temp(op
->args
[i
]);
3029 pset
= la_temp_pref(ts
);
3034 set
&= output_pref(op
, ct
->alias_index
);
3036 /* If the combination is not possible, restart. */
3046 op
->life
= arg_life
;
3050 /* Liveness analysis: Convert indirect regs to direct temporaries. */
3051 static bool liveness_pass_2(TCGContext
*s
)
3053 int nb_globals
= s
->nb_globals
;
3055 bool changes
= false;
3056 TCGOp
*op
, *op_next
;
3058 /* Create a temporary for each indirect global. */
3059 for (i
= 0; i
< nb_globals
; ++i
) {
3060 TCGTemp
*its
= &s
->temps
[i
];
3061 if (its
->indirect_reg
) {
3062 TCGTemp
*dts
= tcg_temp_alloc(s
);
3063 dts
->type
= its
->type
;
3064 dts
->base_type
= its
->base_type
;
3065 dts
->kind
= TEMP_EBB
;
3066 its
->state_ptr
= dts
;
3068 its
->state_ptr
= NULL
;
3070 /* All globals begin dead. */
3071 its
->state
= TS_DEAD
;
3073 for (nb_temps
= s
->nb_temps
; i
< nb_temps
; ++i
) {
3074 TCGTemp
*its
= &s
->temps
[i
];
3075 its
->state_ptr
= NULL
;
3076 its
->state
= TS_DEAD
;
3079 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
3080 TCGOpcode opc
= op
->opc
;
3081 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
3082 TCGLifeData arg_life
= op
->life
;
3083 int nb_iargs
, nb_oargs
, call_flags
;
3084 TCGTemp
*arg_ts
, *dir_ts
;
3086 if (opc
== INDEX_op_call
) {
3087 nb_oargs
= TCGOP_CALLO(op
);
3088 nb_iargs
= TCGOP_CALLI(op
);
3089 call_flags
= tcg_call_flags(op
);
3091 nb_iargs
= def
->nb_iargs
;
3092 nb_oargs
= def
->nb_oargs
;
3094 /* Set flags similar to how calls require. */
3095 if (def
->flags
& TCG_OPF_COND_BRANCH
) {
3096 /* Like reading globals: sync_globals */
3097 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
3098 } else if (def
->flags
& TCG_OPF_BB_END
) {
3099 /* Like writing globals: save_globals */
3101 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
3102 /* Like reading globals: sync_globals */
3103 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
3105 /* No effect on globals. */
3106 call_flags
= (TCG_CALL_NO_READ_GLOBALS
|
3107 TCG_CALL_NO_WRITE_GLOBALS
);
3111 /* Make sure that input arguments are available. */
3112 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
3113 arg_ts
= arg_temp(op
->args
[i
]);
3114 dir_ts
= arg_ts
->state_ptr
;
3115 if (dir_ts
&& arg_ts
->state
== TS_DEAD
) {
3116 TCGOpcode lopc
= (arg_ts
->type
== TCG_TYPE_I32
3119 TCGOp
*lop
= tcg_op_insert_before(s
, op
, lopc
, 3);
3121 lop
->args
[0] = temp_arg(dir_ts
);
3122 lop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3123 lop
->args
[2] = arg_ts
->mem_offset
;
3125 /* Loaded, but synced with memory. */
3126 arg_ts
->state
= TS_MEM
;
3130 /* Perform input replacement, and mark inputs that became dead.
3131 No action is required except keeping temp_state up to date
3132 so that we reload when needed. */
3133 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
3134 arg_ts
= arg_temp(op
->args
[i
]);
3135 dir_ts
= arg_ts
->state_ptr
;
3137 op
->args
[i
] = temp_arg(dir_ts
);
3139 if (IS_DEAD_ARG(i
)) {
3140 arg_ts
->state
= TS_DEAD
;
3145 /* Liveness analysis should ensure that the following are
3146 all correct, for call sites and basic block end points. */
3147 if (call_flags
& TCG_CALL_NO_READ_GLOBALS
) {
3149 } else if (call_flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
3150 for (i
= 0; i
< nb_globals
; ++i
) {
3151 /* Liveness should see that globals are synced back,
3152 that is, either TS_DEAD or TS_MEM. */
3153 arg_ts
= &s
->temps
[i
];
3154 tcg_debug_assert(arg_ts
->state_ptr
== 0
3155 || arg_ts
->state
!= 0);
3158 for (i
= 0; i
< nb_globals
; ++i
) {
3159 /* Liveness should see that globals are saved back,
3160 that is, TS_DEAD, waiting to be reloaded. */
3161 arg_ts
= &s
->temps
[i
];
3162 tcg_debug_assert(arg_ts
->state_ptr
== 0
3163 || arg_ts
->state
== TS_DEAD
);
3167 /* Outputs become available. */
3168 if (opc
== INDEX_op_mov_i32
|| opc
== INDEX_op_mov_i64
) {
3169 arg_ts
= arg_temp(op
->args
[0]);
3170 dir_ts
= arg_ts
->state_ptr
;
3172 op
->args
[0] = temp_arg(dir_ts
);
3175 /* The output is now live and modified. */
3178 if (NEED_SYNC_ARG(0)) {
3179 TCGOpcode sopc
= (arg_ts
->type
== TCG_TYPE_I32
3182 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
, 3);
3183 TCGTemp
*out_ts
= dir_ts
;
3185 if (IS_DEAD_ARG(0)) {
3186 out_ts
= arg_temp(op
->args
[1]);
3187 arg_ts
->state
= TS_DEAD
;
3188 tcg_op_remove(s
, op
);
3190 arg_ts
->state
= TS_MEM
;
3193 sop
->args
[0] = temp_arg(out_ts
);
3194 sop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3195 sop
->args
[2] = arg_ts
->mem_offset
;
3197 tcg_debug_assert(!IS_DEAD_ARG(0));
3201 for (i
= 0; i
< nb_oargs
; i
++) {
3202 arg_ts
= arg_temp(op
->args
[i
]);
3203 dir_ts
= arg_ts
->state_ptr
;
3207 op
->args
[i
] = temp_arg(dir_ts
);
3210 /* The output is now live and modified. */
3213 /* Sync outputs upon their last write. */
3214 if (NEED_SYNC_ARG(i
)) {
3215 TCGOpcode sopc
= (arg_ts
->type
== TCG_TYPE_I32
3218 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
, 3);
3220 sop
->args
[0] = temp_arg(dir_ts
);
3221 sop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3222 sop
->args
[2] = arg_ts
->mem_offset
;
3224 arg_ts
->state
= TS_MEM
;
3226 /* Drop outputs that are dead. */
3227 if (IS_DEAD_ARG(i
)) {
3228 arg_ts
->state
= TS_DEAD
;
3237 static void temp_allocate_frame(TCGContext
*s
, TCGTemp
*ts
)
3239 int size
= tcg_type_size(ts
->type
);
3253 /* Note that we do not require aligned storage for V256. */
3257 g_assert_not_reached();
3261 * Assume the stack is sufficiently aligned.
3262 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3263 * and do not require 16 byte vector alignment. This seems slightly
3264 * easier than fully parameterizing the above switch statement.
3266 align
= MIN(TCG_TARGET_STACK_ALIGN
, align
);
3267 off
= ROUND_UP(s
->current_frame_offset
, align
);
3269 /* If we've exhausted the stack frame, restart with a smaller TB. */
3270 if (off
+ size
> s
->frame_end
) {
3271 tcg_raise_tb_overflow(s
);
3273 s
->current_frame_offset
= off
+ size
;
3275 ts
->mem_offset
= off
;
3276 #if defined(__sparc__)
3277 ts
->mem_offset
+= TCG_TARGET_STACK_BIAS
;
3279 ts
->mem_base
= s
->frame_temp
;
3280 ts
->mem_allocated
= 1;
3283 /* Assign @reg to @ts, and update reg_to_temp[]. */
3284 static void set_temp_val_reg(TCGContext
*s
, TCGTemp
*ts
, TCGReg reg
)
3286 if (ts
->val_type
== TEMP_VAL_REG
) {
3287 TCGReg old
= ts
->reg
;
3288 tcg_debug_assert(s
->reg_to_temp
[old
] == ts
);
3292 s
->reg_to_temp
[old
] = NULL
;
3294 tcg_debug_assert(s
->reg_to_temp
[reg
] == NULL
);
3295 s
->reg_to_temp
[reg
] = ts
;
3296 ts
->val_type
= TEMP_VAL_REG
;
3300 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3301 static void set_temp_val_nonreg(TCGContext
*s
, TCGTemp
*ts
, TCGTempVal type
)
3303 tcg_debug_assert(type
!= TEMP_VAL_REG
);
3304 if (ts
->val_type
== TEMP_VAL_REG
) {
3305 TCGReg reg
= ts
->reg
;
3306 tcg_debug_assert(s
->reg_to_temp
[reg
] == ts
);
3307 s
->reg_to_temp
[reg
] = NULL
;
3309 ts
->val_type
= type
;
3312 static void temp_load(TCGContext
*, TCGTemp
*, TCGRegSet
, TCGRegSet
, TCGRegSet
);
3314 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3315 mark it free; otherwise mark it dead. */
3316 static void temp_free_or_dead(TCGContext
*s
, TCGTemp
*ts
, int free_or_dead
)
3318 TCGTempVal new_type
;
3325 new_type
= TEMP_VAL_MEM
;
3329 new_type
= free_or_dead
< 0 ? TEMP_VAL_MEM
: TEMP_VAL_DEAD
;
3332 new_type
= TEMP_VAL_CONST
;
3335 g_assert_not_reached();
3337 set_temp_val_nonreg(s
, ts
, new_type
);
3340 /* Mark a temporary as dead. */
3341 static inline void temp_dead(TCGContext
*s
, TCGTemp
*ts
)
3343 temp_free_or_dead(s
, ts
, 1);
3346 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3347 registers needs to be allocated to store a constant. If 'free_or_dead'
3348 is non-zero, subsequently release the temporary; if it is positive, the
3349 temp is dead; if it is negative, the temp is free. */
3350 static void temp_sync(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
,
3351 TCGRegSet preferred_regs
, int free_or_dead
)
3353 if (!temp_readonly(ts
) && !ts
->mem_coherent
) {
3354 if (!ts
->mem_allocated
) {
3355 temp_allocate_frame(s
, ts
);
3357 switch (ts
->val_type
) {
3358 case TEMP_VAL_CONST
:
3359 /* If we're going to free the temp immediately, then we won't
3360 require it later in a register, so attempt to store the
3361 constant to memory directly. */
3363 && tcg_out_sti(s
, ts
->type
, ts
->val
,
3364 ts
->mem_base
->reg
, ts
->mem_offset
)) {
3367 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3368 allocated_regs
, preferred_regs
);
3372 tcg_out_st(s
, ts
->type
, ts
->reg
,
3373 ts
->mem_base
->reg
, ts
->mem_offset
);
3383 ts
->mem_coherent
= 1;
3386 temp_free_or_dead(s
, ts
, free_or_dead
);
3390 /* free register 'reg' by spilling the corresponding temporary if necessary */
3391 static void tcg_reg_free(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
3393 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
3395 temp_sync(s
, ts
, allocated_regs
, 0, -1);
3401 * @required_regs: Set of registers in which we must allocate.
3402 * @allocated_regs: Set of registers which must be avoided.
3403 * @preferred_regs: Set of registers we should prefer.
3404 * @rev: True if we search the registers in "indirect" order.
3406 * The allocated register must be in @required_regs & ~@allocated_regs,
3407 * but if we can put it in @preferred_regs we may save a move later.
3409 static TCGReg
tcg_reg_alloc(TCGContext
*s
, TCGRegSet required_regs
,
3410 TCGRegSet allocated_regs
,
3411 TCGRegSet preferred_regs
, bool rev
)
3413 int i
, j
, f
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
3414 TCGRegSet reg_ct
[2];
3417 reg_ct
[1] = required_regs
& ~allocated_regs
;
3418 tcg_debug_assert(reg_ct
[1] != 0);
3419 reg_ct
[0] = reg_ct
[1] & preferred_regs
;
3421 /* Skip the preferred_regs option if it cannot be satisfied,
3422 or if the preference made no difference. */
3423 f
= reg_ct
[0] == 0 || reg_ct
[0] == reg_ct
[1];
3425 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
3427 /* Try free registers, preferences first. */
3428 for (j
= f
; j
< 2; j
++) {
3429 TCGRegSet set
= reg_ct
[j
];
3431 if (tcg_regset_single(set
)) {
3432 /* One register in the set. */
3433 TCGReg reg
= tcg_regset_first(set
);
3434 if (s
->reg_to_temp
[reg
] == NULL
) {
3438 for (i
= 0; i
< n
; i
++) {
3439 TCGReg reg
= order
[i
];
3440 if (s
->reg_to_temp
[reg
] == NULL
&&
3441 tcg_regset_test_reg(set
, reg
)) {
3448 /* We must spill something. */
3449 for (j
= f
; j
< 2; j
++) {
3450 TCGRegSet set
= reg_ct
[j
];
3452 if (tcg_regset_single(set
)) {
3453 /* One register in the set. */
3454 TCGReg reg
= tcg_regset_first(set
);
3455 tcg_reg_free(s
, reg
, allocated_regs
);
3458 for (i
= 0; i
< n
; i
++) {
3459 TCGReg reg
= order
[i
];
3460 if (tcg_regset_test_reg(set
, reg
)) {
3461 tcg_reg_free(s
, reg
, allocated_regs
);
3471 static TCGReg
tcg_reg_alloc_pair(TCGContext
*s
, TCGRegSet required_regs
,
3472 TCGRegSet allocated_regs
,
3473 TCGRegSet preferred_regs
, bool rev
)
3475 int i
, j
, k
, fmin
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
3476 TCGRegSet reg_ct
[2];
3479 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3480 reg_ct
[1] = required_regs
& ~(allocated_regs
| (allocated_regs
>> 1));
3481 tcg_debug_assert(reg_ct
[1] != 0);
3482 reg_ct
[0] = reg_ct
[1] & preferred_regs
;
3484 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
3487 * Skip the preferred_regs option if it cannot be satisfied,
3488 * or if the preference made no difference.
3490 k
= reg_ct
[0] == 0 || reg_ct
[0] == reg_ct
[1];
3493 * Minimize the number of flushes by looking for 2 free registers first,
3494 * then a single flush, then two flushes.
3496 for (fmin
= 2; fmin
>= 0; fmin
--) {
3497 for (j
= k
; j
< 2; j
++) {
3498 TCGRegSet set
= reg_ct
[j
];
3500 for (i
= 0; i
< n
; i
++) {
3501 TCGReg reg
= order
[i
];
3503 if (tcg_regset_test_reg(set
, reg
)) {
3504 int f
= !s
->reg_to_temp
[reg
] + !s
->reg_to_temp
[reg
+ 1];
3506 tcg_reg_free(s
, reg
, allocated_regs
);
3507 tcg_reg_free(s
, reg
+ 1, allocated_regs
);
3517 /* Make sure the temporary is in a register. If needed, allocate the register
3518 from DESIRED while avoiding ALLOCATED. */
3519 static void temp_load(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet desired_regs
,
3520 TCGRegSet allocated_regs
, TCGRegSet preferred_regs
)
3524 switch (ts
->val_type
) {
3527 case TEMP_VAL_CONST
:
3528 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3529 preferred_regs
, ts
->indirect_base
);
3530 if (ts
->type
<= TCG_TYPE_I64
) {
3531 tcg_out_movi(s
, ts
->type
, reg
, ts
->val
);
3533 uint64_t val
= ts
->val
;
3537 * Find the minimal vector element that matches the constant.
3538 * The targets will, in general, have to do this search anyway,
3539 * do this generically.
3541 if (val
== dup_const(MO_8
, val
)) {
3543 } else if (val
== dup_const(MO_16
, val
)) {
3545 } else if (val
== dup_const(MO_32
, val
)) {
3549 tcg_out_dupi_vec(s
, ts
->type
, vece
, reg
, ts
->val
);
3551 ts
->mem_coherent
= 0;
3554 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3555 preferred_regs
, ts
->indirect_base
);
3556 tcg_out_ld(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
3557 ts
->mem_coherent
= 1;
3563 set_temp_val_reg(s
, ts
, reg
);
3566 /* Save a temporary to memory. 'allocated_regs' is used in case a
3567 temporary registers needs to be allocated to store a constant. */
3568 static void temp_save(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
)
3570 /* The liveness analysis already ensures that globals are back
3571 in memory. Keep an tcg_debug_assert for safety. */
3572 tcg_debug_assert(ts
->val_type
== TEMP_VAL_MEM
|| temp_readonly(ts
));
3575 /* save globals to their canonical location and assume they can be
3576 modified be the following code. 'allocated_regs' is used in case a
3577 temporary registers needs to be allocated to store a constant. */
3578 static void save_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3582 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3583 temp_save(s
, &s
->temps
[i
], allocated_regs
);
3587 /* sync globals to their canonical location and assume they can be
3588 read by the following code. 'allocated_regs' is used in case a
3589 temporary registers needs to be allocated to store a constant. */
3590 static void sync_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3594 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3595 TCGTemp
*ts
= &s
->temps
[i
];
3596 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
3597 || ts
->kind
== TEMP_FIXED
3598 || ts
->mem_coherent
);
3602 /* at the end of a basic block, we assume all temporaries are dead and
3603 all globals are stored at their canonical location. */
3604 static void tcg_reg_alloc_bb_end(TCGContext
*s
, TCGRegSet allocated_regs
)
3608 for (i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
3609 TCGTemp
*ts
= &s
->temps
[i
];
3613 temp_save(s
, ts
, allocated_regs
);
3617 /* The liveness analysis already ensures that temps are dead.
3618 Keep an tcg_debug_assert for safety. */
3619 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
3622 /* Similarly, we should have freed any allocated register. */
3623 tcg_debug_assert(ts
->val_type
== TEMP_VAL_CONST
);
3626 g_assert_not_reached();
3630 save_globals(s
, allocated_regs
);
3634 * At a conditional branch, we assume all temporaries are dead unless
3635 * explicitly live-across-conditional-branch; all globals and local
3636 * temps are synced to their location.
3638 static void tcg_reg_alloc_cbranch(TCGContext
*s
, TCGRegSet allocated_regs
)
3640 sync_globals(s
, allocated_regs
);
3642 for (int i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
3643 TCGTemp
*ts
= &s
->temps
[i
];
3645 * The liveness analysis already ensures that temps are dead.
3646 * Keep tcg_debug_asserts for safety.
3650 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
|| ts
->mem_coherent
);
3653 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
3659 g_assert_not_reached();
3665 * Specialized code generation for INDEX_op_mov_* with a constant.
3667 static void tcg_reg_alloc_do_movi(TCGContext
*s
, TCGTemp
*ots
,
3668 tcg_target_ulong val
, TCGLifeData arg_life
,
3669 TCGRegSet preferred_regs
)
3671 /* ENV should not be modified. */
3672 tcg_debug_assert(!temp_readonly(ots
));
3674 /* The movi is not explicitly generated here. */
3675 set_temp_val_nonreg(s
, ots
, TEMP_VAL_CONST
);
3677 ots
->mem_coherent
= 0;
3678 if (NEED_SYNC_ARG(0)) {
3679 temp_sync(s
, ots
, s
->reserved_regs
, preferred_regs
, IS_DEAD_ARG(0));
3680 } else if (IS_DEAD_ARG(0)) {
3686 * Specialized code generation for INDEX_op_mov_*.
3688 static void tcg_reg_alloc_mov(TCGContext
*s
, const TCGOp
*op
)
3690 const TCGLifeData arg_life
= op
->life
;
3691 TCGRegSet allocated_regs
, preferred_regs
;
3693 TCGType otype
, itype
;
3696 allocated_regs
= s
->reserved_regs
;
3697 preferred_regs
= output_pref(op
, 0);
3698 ots
= arg_temp(op
->args
[0]);
3699 ts
= arg_temp(op
->args
[1]);
3701 /* ENV should not be modified. */
3702 tcg_debug_assert(!temp_readonly(ots
));
3704 /* Note that otype != itype for no-op truncation. */
3708 if (ts
->val_type
== TEMP_VAL_CONST
) {
3709 /* propagate constant or generate sti */
3710 tcg_target_ulong val
= ts
->val
;
3711 if (IS_DEAD_ARG(1)) {
3714 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, preferred_regs
);
3718 /* If the source value is in memory we're going to be forced
3719 to have it in a register in order to perform the copy. Copy
3720 the SOURCE value into its own register first, that way we
3721 don't have to reload SOURCE the next time it is used. */
3722 if (ts
->val_type
== TEMP_VAL_MEM
) {
3723 temp_load(s
, ts
, tcg_target_available_regs
[itype
],
3724 allocated_regs
, preferred_regs
);
3726 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
3729 if (IS_DEAD_ARG(0)) {
3730 /* mov to a non-saved dead register makes no sense (even with
3731 liveness analysis disabled). */
3732 tcg_debug_assert(NEED_SYNC_ARG(0));
3733 if (!ots
->mem_allocated
) {
3734 temp_allocate_frame(s
, ots
);
3736 tcg_out_st(s
, otype
, ireg
, ots
->mem_base
->reg
, ots
->mem_offset
);
3737 if (IS_DEAD_ARG(1)) {
3744 if (IS_DEAD_ARG(1) && ts
->kind
!= TEMP_FIXED
) {
3746 * The mov can be suppressed. Kill input first, so that it
3747 * is unlinked from reg_to_temp, then set the output to the
3748 * reg that we saved from the input.
3753 if (ots
->val_type
== TEMP_VAL_REG
) {
3756 /* Make sure to not spill the input register during allocation. */
3757 oreg
= tcg_reg_alloc(s
, tcg_target_available_regs
[otype
],
3758 allocated_regs
| ((TCGRegSet
)1 << ireg
),
3759 preferred_regs
, ots
->indirect_base
);
3761 if (!tcg_out_mov(s
, otype
, oreg
, ireg
)) {
3763 * Cross register class move not supported.
3764 * Store the source register into the destination slot
3765 * and leave the destination temp as TEMP_VAL_MEM.
3767 assert(!temp_readonly(ots
));
3768 if (!ts
->mem_allocated
) {
3769 temp_allocate_frame(s
, ots
);
3771 tcg_out_st(s
, ts
->type
, ireg
, ots
->mem_base
->reg
, ots
->mem_offset
);
3772 set_temp_val_nonreg(s
, ts
, TEMP_VAL_MEM
);
3773 ots
->mem_coherent
= 1;
3777 set_temp_val_reg(s
, ots
, oreg
);
3778 ots
->mem_coherent
= 0;
3780 if (NEED_SYNC_ARG(0)) {
3781 temp_sync(s
, ots
, allocated_regs
, 0, 0);
3786 * Specialized code generation for INDEX_op_dup_vec.
3788 static void tcg_reg_alloc_dup(TCGContext
*s
, const TCGOp
*op
)
3790 const TCGLifeData arg_life
= op
->life
;
3791 TCGRegSet dup_out_regs
, dup_in_regs
;
3793 TCGType itype
, vtype
;
3798 ots
= arg_temp(op
->args
[0]);
3799 its
= arg_temp(op
->args
[1]);
3801 /* ENV should not be modified. */
3802 tcg_debug_assert(!temp_readonly(ots
));
3805 vece
= TCGOP_VECE(op
);
3806 vtype
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
3808 if (its
->val_type
== TEMP_VAL_CONST
) {
3809 /* Propagate constant via movi -> dupi. */
3810 tcg_target_ulong val
= its
->val
;
3811 if (IS_DEAD_ARG(1)) {
3814 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, output_pref(op
, 0));
3818 dup_out_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[0].regs
;
3819 dup_in_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[1].regs
;
3821 /* Allocate the output register now. */
3822 if (ots
->val_type
!= TEMP_VAL_REG
) {
3823 TCGRegSet allocated_regs
= s
->reserved_regs
;
3826 if (!IS_DEAD_ARG(1) && its
->val_type
== TEMP_VAL_REG
) {
3827 /* Make sure to not spill the input register. */
3828 tcg_regset_set_reg(allocated_regs
, its
->reg
);
3830 oreg
= tcg_reg_alloc(s
, dup_out_regs
, allocated_regs
,
3831 output_pref(op
, 0), ots
->indirect_base
);
3832 set_temp_val_reg(s
, ots
, oreg
);
3835 switch (its
->val_type
) {
3838 * The dup constriaints must be broad, covering all possible VECE.
3839 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3840 * to fail, indicating that extra moves are required for that case.
3842 if (tcg_regset_test_reg(dup_in_regs
, its
->reg
)) {
3843 if (tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, its
->reg
)) {
3846 /* Try again from memory or a vector input register. */
3848 if (!its
->mem_coherent
) {
3850 * The input register is not synced, and so an extra store
3851 * would be required to use memory. Attempt an integer-vector
3852 * register move first. We do not have a TCGRegSet for this.
3854 if (tcg_out_mov(s
, itype
, ots
->reg
, its
->reg
)) {
3857 /* Sync the temp back to its slot and load from there. */
3858 temp_sync(s
, its
, s
->reserved_regs
, 0, 0);
3864 if (HOST_BIG_ENDIAN
) {
3865 lowpart_ofs
= tcg_type_size(itype
) - (1 << vece
);
3867 if (tcg_out_dupm_vec(s
, vtype
, vece
, ots
->reg
, its
->mem_base
->reg
,
3868 its
->mem_offset
+ lowpart_ofs
)) {
3871 /* Load the input into the destination vector register. */
3872 tcg_out_ld(s
, itype
, ots
->reg
, its
->mem_base
->reg
, its
->mem_offset
);
3876 g_assert_not_reached();
3879 /* We now have a vector input register, so dup must succeed. */
3880 ok
= tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, ots
->reg
);
3881 tcg_debug_assert(ok
);
3884 ots
->mem_coherent
= 0;
3885 if (IS_DEAD_ARG(1)) {
3888 if (NEED_SYNC_ARG(0)) {
3889 temp_sync(s
, ots
, s
->reserved_regs
, 0, 0);
3891 if (IS_DEAD_ARG(0)) {
3896 static void tcg_reg_alloc_op(TCGContext
*s
, const TCGOp
*op
)
3898 const TCGLifeData arg_life
= op
->life
;
3899 const TCGOpDef
* const def
= &tcg_op_defs
[op
->opc
];
3900 TCGRegSet i_allocated_regs
;
3901 TCGRegSet o_allocated_regs
;
3902 int i
, k
, nb_iargs
, nb_oargs
;
3905 const TCGArgConstraint
*arg_ct
;
3907 TCGArg new_args
[TCG_MAX_OP_ARGS
];
3908 int const_args
[TCG_MAX_OP_ARGS
];
3910 nb_oargs
= def
->nb_oargs
;
3911 nb_iargs
= def
->nb_iargs
;
3913 /* copy constants */
3914 memcpy(new_args
+ nb_oargs
+ nb_iargs
,
3915 op
->args
+ nb_oargs
+ nb_iargs
,
3916 sizeof(TCGArg
) * def
->nb_cargs
);
3918 i_allocated_regs
= s
->reserved_regs
;
3919 o_allocated_regs
= s
->reserved_regs
;
3921 /* satisfy input constraints */
3922 for (k
= 0; k
< nb_iargs
; k
++) {
3923 TCGRegSet i_preferred_regs
, i_required_regs
;
3924 bool allocate_new_reg
, copyto_new_reg
;
3928 i
= def
->args_ct
[nb_oargs
+ k
].sort_index
;
3930 arg_ct
= &def
->args_ct
[i
];
3933 if (ts
->val_type
== TEMP_VAL_CONST
3934 && tcg_target_const_match(ts
->val
, ts
->type
, arg_ct
->ct
)) {
3935 /* constant is OK for instruction */
3937 new_args
[i
] = ts
->val
;
3942 i_preferred_regs
= 0;
3943 i_required_regs
= arg_ct
->regs
;
3944 allocate_new_reg
= false;
3945 copyto_new_reg
= false;
3947 switch (arg_ct
->pair
) {
3948 case 0: /* not paired */
3949 if (arg_ct
->ialias
) {
3950 i_preferred_regs
= output_pref(op
, arg_ct
->alias_index
);
3953 * If the input is readonly, then it cannot also be an
3954 * output and aliased to itself. If the input is not
3955 * dead after the instruction, we must allocate a new
3956 * register and move it.
3958 if (temp_readonly(ts
) || !IS_DEAD_ARG(i
)) {
3959 allocate_new_reg
= true;
3960 } else if (ts
->val_type
== TEMP_VAL_REG
) {
3962 * Check if the current register has already been
3963 * allocated for another input.
3966 tcg_regset_test_reg(i_allocated_regs
, reg
);
3969 if (!allocate_new_reg
) {
3970 temp_load(s
, ts
, i_required_regs
, i_allocated_regs
,
3973 allocate_new_reg
= !tcg_regset_test_reg(i_required_regs
, reg
);
3975 if (allocate_new_reg
) {
3977 * Allocate a new register matching the constraint
3978 * and move the temporary register into it.
3980 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3981 i_allocated_regs
, 0);
3982 reg
= tcg_reg_alloc(s
, i_required_regs
, i_allocated_regs
,
3983 i_preferred_regs
, ts
->indirect_base
);
3984 copyto_new_reg
= true;
3989 /* First of an input pair; if i1 == i2, the second is an output. */
3991 i2
= arg_ct
->pair_index
;
3992 ts2
= i1
!= i2
? arg_temp(op
->args
[i2
]) : NULL
;
3995 * It is easier to default to allocating a new pair
3996 * and to identify a few cases where it's not required.
3998 if (arg_ct
->ialias
) {
3999 i_preferred_regs
= output_pref(op
, arg_ct
->alias_index
);
4000 if (IS_DEAD_ARG(i1
) &&
4002 !temp_readonly(ts
) &&
4003 ts
->val_type
== TEMP_VAL_REG
&&
4004 ts
->reg
< TCG_TARGET_NB_REGS
- 1 &&
4005 tcg_regset_test_reg(i_required_regs
, reg
) &&
4006 !tcg_regset_test_reg(i_allocated_regs
, reg
) &&
4007 !tcg_regset_test_reg(i_allocated_regs
, reg
+ 1) &&
4009 ? ts2
->val_type
== TEMP_VAL_REG
&&
4010 ts2
->reg
== reg
+ 1 &&
4012 : s
->reg_to_temp
[reg
+ 1] == NULL
)) {
4016 /* Without aliasing, the pair must also be an input. */
4017 tcg_debug_assert(ts2
);
4018 if (ts
->val_type
== TEMP_VAL_REG
&&
4019 ts2
->val_type
== TEMP_VAL_REG
&&
4020 ts2
->reg
== reg
+ 1 &&
4021 tcg_regset_test_reg(i_required_regs
, reg
)) {
4025 reg
= tcg_reg_alloc_pair(s
, i_required_regs
, i_allocated_regs
,
4026 0, ts
->indirect_base
);
4029 case 2: /* pair second */
4030 reg
= new_args
[arg_ct
->pair_index
] + 1;
4033 case 3: /* ialias with second output, no first input */
4034 tcg_debug_assert(arg_ct
->ialias
);
4035 i_preferred_regs
= output_pref(op
, arg_ct
->alias_index
);
4037 if (IS_DEAD_ARG(i
) &&
4038 !temp_readonly(ts
) &&
4039 ts
->val_type
== TEMP_VAL_REG
&&
4041 s
->reg_to_temp
[reg
- 1] == NULL
&&
4042 tcg_regset_test_reg(i_required_regs
, reg
) &&
4043 !tcg_regset_test_reg(i_allocated_regs
, reg
) &&
4044 !tcg_regset_test_reg(i_allocated_regs
, reg
- 1)) {
4045 tcg_regset_set_reg(i_allocated_regs
, reg
- 1);
4048 reg
= tcg_reg_alloc_pair(s
, i_required_regs
>> 1,
4049 i_allocated_regs
, 0,
4051 tcg_regset_set_reg(i_allocated_regs
, reg
);
4057 * If an aliased input is not dead after the instruction,
4058 * we must allocate a new register and move it.
4060 if (arg_ct
->ialias
&& (!IS_DEAD_ARG(i
) || temp_readonly(ts
))) {
4061 TCGRegSet t_allocated_regs
= i_allocated_regs
;
4064 * Because of the alias, and the continued life, make sure
4065 * that the temp is somewhere *other* than the reg pair,
4066 * and we get a copy in reg.
4068 tcg_regset_set_reg(t_allocated_regs
, reg
);
4069 tcg_regset_set_reg(t_allocated_regs
, reg
+ 1);
4070 if (ts
->val_type
== TEMP_VAL_REG
&& ts
->reg
== reg
) {
4071 /* If ts was already in reg, copy it somewhere else. */
4075 tcg_debug_assert(ts
->kind
!= TEMP_FIXED
);
4076 nr
= tcg_reg_alloc(s
, tcg_target_available_regs
[ts
->type
],
4077 t_allocated_regs
, 0, ts
->indirect_base
);
4078 ok
= tcg_out_mov(s
, ts
->type
, nr
, reg
);
4079 tcg_debug_assert(ok
);
4081 set_temp_val_reg(s
, ts
, nr
);
4083 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
4084 t_allocated_regs
, 0);
4085 copyto_new_reg
= true;
4088 /* Preferably allocate to reg, otherwise copy. */
4089 i_required_regs
= (TCGRegSet
)1 << reg
;
4090 temp_load(s
, ts
, i_required_regs
, i_allocated_regs
,
4092 copyto_new_reg
= ts
->reg
!= reg
;
4097 g_assert_not_reached();
4100 if (copyto_new_reg
) {
4101 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
4103 * Cross register class move not supported. Sync the
4104 * temp back to its slot and load from there.
4106 temp_sync(s
, ts
, i_allocated_regs
, 0, 0);
4107 tcg_out_ld(s
, ts
->type
, reg
,
4108 ts
->mem_base
->reg
, ts
->mem_offset
);
4113 tcg_regset_set_reg(i_allocated_regs
, reg
);
4116 /* mark dead temporaries and free the associated registers */
4117 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
4118 if (IS_DEAD_ARG(i
)) {
4119 temp_dead(s
, arg_temp(op
->args
[i
]));
4123 if (def
->flags
& TCG_OPF_COND_BRANCH
) {
4124 tcg_reg_alloc_cbranch(s
, i_allocated_regs
);
4125 } else if (def
->flags
& TCG_OPF_BB_END
) {
4126 tcg_reg_alloc_bb_end(s
, i_allocated_regs
);
4128 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
4129 /* XXX: permit generic clobber register list ? */
4130 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
4131 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
4132 tcg_reg_free(s
, i
, i_allocated_regs
);
4136 if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
4137 /* sync globals if the op has side effects and might trigger
4139 sync_globals(s
, i_allocated_regs
);
4142 /* satisfy the output constraints */
4143 for(k
= 0; k
< nb_oargs
; k
++) {
4144 i
= def
->args_ct
[k
].sort_index
;
4146 arg_ct
= &def
->args_ct
[i
];
4149 /* ENV should not be modified. */
4150 tcg_debug_assert(!temp_readonly(ts
));
4152 switch (arg_ct
->pair
) {
4153 case 0: /* not paired */
4154 if (arg_ct
->oalias
&& !const_args
[arg_ct
->alias_index
]) {
4155 reg
= new_args
[arg_ct
->alias_index
];
4156 } else if (arg_ct
->newreg
) {
4157 reg
= tcg_reg_alloc(s
, arg_ct
->regs
,
4158 i_allocated_regs
| o_allocated_regs
,
4159 output_pref(op
, k
), ts
->indirect_base
);
4161 reg
= tcg_reg_alloc(s
, arg_ct
->regs
, o_allocated_regs
,
4162 output_pref(op
, k
), ts
->indirect_base
);
4166 case 1: /* first of pair */
4167 tcg_debug_assert(!arg_ct
->newreg
);
4168 if (arg_ct
->oalias
) {
4169 reg
= new_args
[arg_ct
->alias_index
];
4172 reg
= tcg_reg_alloc_pair(s
, arg_ct
->regs
, o_allocated_regs
,
4173 output_pref(op
, k
), ts
->indirect_base
);
4176 case 2: /* second of pair */
4177 tcg_debug_assert(!arg_ct
->newreg
);
4178 if (arg_ct
->oalias
) {
4179 reg
= new_args
[arg_ct
->alias_index
];
4181 reg
= new_args
[arg_ct
->pair_index
] + 1;
4185 case 3: /* first of pair, aliasing with a second input */
4186 tcg_debug_assert(!arg_ct
->newreg
);
4187 reg
= new_args
[arg_ct
->pair_index
] - 1;
4191 g_assert_not_reached();
4193 tcg_regset_set_reg(o_allocated_regs
, reg
);
4194 set_temp_val_reg(s
, ts
, reg
);
4195 ts
->mem_coherent
= 0;
4200 /* emit instruction */
4201 if (def
->flags
& TCG_OPF_VECTOR
) {
4202 tcg_out_vec_op(s
, op
->opc
, TCGOP_VECL(op
), TCGOP_VECE(op
),
4203 new_args
, const_args
);
4205 tcg_out_op(s
, op
->opc
, new_args
, const_args
);
4208 /* move the outputs in the correct register if needed */
4209 for(i
= 0; i
< nb_oargs
; i
++) {
4210 ts
= arg_temp(op
->args
[i
]);
4212 /* ENV should not be modified. */
4213 tcg_debug_assert(!temp_readonly(ts
));
4215 if (NEED_SYNC_ARG(i
)) {
4216 temp_sync(s
, ts
, o_allocated_regs
, 0, IS_DEAD_ARG(i
));
4217 } else if (IS_DEAD_ARG(i
)) {
4223 static bool tcg_reg_alloc_dup2(TCGContext
*s
, const TCGOp
*op
)
4225 const TCGLifeData arg_life
= op
->life
;
4226 TCGTemp
*ots
, *itsl
, *itsh
;
4227 TCGType vtype
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
4229 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4230 tcg_debug_assert(TCG_TARGET_REG_BITS
== 32);
4231 tcg_debug_assert(TCGOP_VECE(op
) == MO_64
);
4233 ots
= arg_temp(op
->args
[0]);
4234 itsl
= arg_temp(op
->args
[1]);
4235 itsh
= arg_temp(op
->args
[2]);
4237 /* ENV should not be modified. */
4238 tcg_debug_assert(!temp_readonly(ots
));
4240 /* Allocate the output register now. */
4241 if (ots
->val_type
!= TEMP_VAL_REG
) {
4242 TCGRegSet allocated_regs
= s
->reserved_regs
;
4243 TCGRegSet dup_out_regs
=
4244 tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[0].regs
;
4247 /* Make sure to not spill the input registers. */
4248 if (!IS_DEAD_ARG(1) && itsl
->val_type
== TEMP_VAL_REG
) {
4249 tcg_regset_set_reg(allocated_regs
, itsl
->reg
);
4251 if (!IS_DEAD_ARG(2) && itsh
->val_type
== TEMP_VAL_REG
) {
4252 tcg_regset_set_reg(allocated_regs
, itsh
->reg
);
4255 oreg
= tcg_reg_alloc(s
, dup_out_regs
, allocated_regs
,
4256 output_pref(op
, 0), ots
->indirect_base
);
4257 set_temp_val_reg(s
, ots
, oreg
);
4260 /* Promote dup2 of immediates to dupi_vec. */
4261 if (itsl
->val_type
== TEMP_VAL_CONST
&& itsh
->val_type
== TEMP_VAL_CONST
) {
4262 uint64_t val
= deposit64(itsl
->val
, 32, 32, itsh
->val
);
4265 if (val
== dup_const(MO_8
, val
)) {
4267 } else if (val
== dup_const(MO_16
, val
)) {
4269 } else if (val
== dup_const(MO_32
, val
)) {
4273 tcg_out_dupi_vec(s
, vtype
, vece
, ots
->reg
, val
);
4277 /* If the two inputs form one 64-bit value, try dupm_vec. */
4278 if (itsl
->temp_subindex
== HOST_BIG_ENDIAN
&&
4279 itsh
->temp_subindex
== !HOST_BIG_ENDIAN
&&
4280 itsl
== itsh
+ (HOST_BIG_ENDIAN
? 1 : -1)) {
4281 TCGTemp
*its
= itsl
- HOST_BIG_ENDIAN
;
4283 temp_sync(s
, its
+ 0, s
->reserved_regs
, 0, 0);
4284 temp_sync(s
, its
+ 1, s
->reserved_regs
, 0, 0);
4286 if (tcg_out_dupm_vec(s
, vtype
, MO_64
, ots
->reg
,
4287 its
->mem_base
->reg
, its
->mem_offset
)) {
4292 /* Fall back to generic expansion. */
4296 ots
->mem_coherent
= 0;
4297 if (IS_DEAD_ARG(1)) {
4300 if (IS_DEAD_ARG(2)) {
4303 if (NEED_SYNC_ARG(0)) {
4304 temp_sync(s
, ots
, s
->reserved_regs
, 0, IS_DEAD_ARG(0));
4305 } else if (IS_DEAD_ARG(0)) {
4311 static void load_arg_reg(TCGContext
*s
, TCGReg reg
, TCGTemp
*ts
,
4312 TCGRegSet allocated_regs
)
4314 if (ts
->val_type
== TEMP_VAL_REG
) {
4315 if (ts
->reg
!= reg
) {
4316 tcg_reg_free(s
, reg
, allocated_regs
);
4317 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
4319 * Cross register class move not supported. Sync the
4320 * temp back to its slot and load from there.
4322 temp_sync(s
, ts
, allocated_regs
, 0, 0);
4323 tcg_out_ld(s
, ts
->type
, reg
,
4324 ts
->mem_base
->reg
, ts
->mem_offset
);
4328 TCGRegSet arg_set
= 0;
4330 tcg_reg_free(s
, reg
, allocated_regs
);
4331 tcg_regset_set_reg(arg_set
, reg
);
4332 temp_load(s
, ts
, arg_set
, allocated_regs
, 0);
4336 static void load_arg_stk(TCGContext
*s
, int stk_slot
, TCGTemp
*ts
,
4337 TCGRegSet allocated_regs
)
4340 * When the destination is on the stack, load up the temp and store.
4341 * If there are many call-saved registers, the temp might live to
4342 * see another use; otherwise it'll be discarded.
4344 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
], allocated_regs
, 0);
4345 tcg_out_st(s
, ts
->type
, ts
->reg
, TCG_REG_CALL_STACK
,
4346 TCG_TARGET_CALL_STACK_OFFSET
+
4347 stk_slot
* sizeof(tcg_target_long
));
4350 static void load_arg_normal(TCGContext
*s
, const TCGCallArgumentLoc
*l
,
4351 TCGTemp
*ts
, TCGRegSet
*allocated_regs
)
4354 TCGReg reg
= tcg_target_call_iarg_regs
[l
->arg_slot
];
4355 load_arg_reg(s
, reg
, ts
, *allocated_regs
);
4356 tcg_regset_set_reg(*allocated_regs
, reg
);
4358 load_arg_stk(s
, l
->arg_slot
- ARRAY_SIZE(tcg_target_call_iarg_regs
),
4359 ts
, *allocated_regs
);
4363 static void tcg_reg_alloc_call(TCGContext
*s
, TCGOp
*op
)
4365 const int nb_oargs
= TCGOP_CALLO(op
);
4366 const int nb_iargs
= TCGOP_CALLI(op
);
4367 const TCGLifeData arg_life
= op
->life
;
4368 const TCGHelperInfo
*info
= tcg_call_info(op
);
4369 TCGRegSet allocated_regs
= s
->reserved_regs
;
4373 * Move inputs into place in reverse order,
4374 * so that we place stacked arguments first.
4376 for (i
= nb_iargs
- 1; i
>= 0; --i
) {
4377 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
4378 TCGTemp
*ts
= arg_temp(op
->args
[nb_oargs
+ i
]);
4380 switch (loc
->kind
) {
4381 case TCG_CALL_ARG_NORMAL
:
4382 case TCG_CALL_ARG_EXTEND_U
:
4383 case TCG_CALL_ARG_EXTEND_S
:
4384 load_arg_normal(s
, loc
, ts
, &allocated_regs
);
4387 g_assert_not_reached();
4391 /* Mark dead temporaries and free the associated registers. */
4392 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
4393 if (IS_DEAD_ARG(i
)) {
4394 temp_dead(s
, arg_temp(op
->args
[i
]));
4398 /* Clobber call registers. */
4399 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
4400 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
4401 tcg_reg_free(s
, i
, allocated_regs
);
4406 * Save globals if they might be written by the helper,
4407 * sync them if they might be read.
4409 if (info
->flags
& TCG_CALL_NO_READ_GLOBALS
) {
4411 } else if (info
->flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
4412 sync_globals(s
, allocated_regs
);
4414 save_globals(s
, allocated_regs
);
4417 tcg_out_call(s
, tcg_call_func(op
), info
);
4419 /* Assign output registers and emit moves if needed. */
4420 switch (info
->out_kind
) {
4421 case TCG_CALL_RET_NORMAL
:
4422 for (i
= 0; i
< nb_oargs
; i
++) {
4423 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
4424 TCGReg reg
= tcg_target_call_oarg_regs
[i
];
4426 /* ENV should not be modified. */
4427 tcg_debug_assert(!temp_readonly(ts
));
4429 set_temp_val_reg(s
, ts
, reg
);
4430 ts
->mem_coherent
= 0;
4434 g_assert_not_reached();
4437 /* Flush or discard output registers as needed. */
4438 for (i
= 0; i
< nb_oargs
; i
++) {
4439 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
4440 if (NEED_SYNC_ARG(i
)) {
4441 temp_sync(s
, ts
, s
->reserved_regs
, 0, IS_DEAD_ARG(i
));
4442 } else if (IS_DEAD_ARG(i
)) {
4448 #ifdef CONFIG_PROFILER
4450 /* avoid copy/paste errors */
4451 #define PROF_ADD(to, from, field) \
4453 (to)->field += qatomic_read(&((from)->field)); \
4456 #define PROF_MAX(to, from, field) \
4458 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
4459 if (val__ > (to)->field) { \
4460 (to)->field = val__; \
4464 /* Pass in a zero'ed @prof */
4466 void tcg_profile_snapshot(TCGProfile
*prof
, bool counters
, bool table
)
4468 unsigned int n_ctxs
= qatomic_read(&tcg_cur_ctxs
);
4471 for (i
= 0; i
< n_ctxs
; i
++) {
4472 TCGContext
*s
= qatomic_read(&tcg_ctxs
[i
]);
4473 const TCGProfile
*orig
= &s
->prof
;
4476 PROF_ADD(prof
, orig
, cpu_exec_time
);
4477 PROF_ADD(prof
, orig
, tb_count1
);
4478 PROF_ADD(prof
, orig
, tb_count
);
4479 PROF_ADD(prof
, orig
, op_count
);
4480 PROF_MAX(prof
, orig
, op_count_max
);
4481 PROF_ADD(prof
, orig
, temp_count
);
4482 PROF_MAX(prof
, orig
, temp_count_max
);
4483 PROF_ADD(prof
, orig
, del_op_count
);
4484 PROF_ADD(prof
, orig
, code_in_len
);
4485 PROF_ADD(prof
, orig
, code_out_len
);
4486 PROF_ADD(prof
, orig
, search_out_len
);
4487 PROF_ADD(prof
, orig
, interm_time
);
4488 PROF_ADD(prof
, orig
, code_time
);
4489 PROF_ADD(prof
, orig
, la_time
);
4490 PROF_ADD(prof
, orig
, opt_time
);
4491 PROF_ADD(prof
, orig
, restore_count
);
4492 PROF_ADD(prof
, orig
, restore_time
);
4497 for (i
= 0; i
< NB_OPS
; i
++) {
4498 PROF_ADD(prof
, orig
, table_op_count
[i
]);
4507 static void tcg_profile_snapshot_counters(TCGProfile
*prof
)
4509 tcg_profile_snapshot(prof
, true, false);
4512 static void tcg_profile_snapshot_table(TCGProfile
*prof
)
4514 tcg_profile_snapshot(prof
, false, true);
4517 void tcg_dump_op_count(GString
*buf
)
4519 TCGProfile prof
= {};
4522 tcg_profile_snapshot_table(&prof
);
4523 for (i
= 0; i
< NB_OPS
; i
++) {
4524 g_string_append_printf(buf
, "%s %" PRId64
"\n", tcg_op_defs
[i
].name
,
4525 prof
.table_op_count
[i
]);
4529 int64_t tcg_cpu_exec_time(void)
4531 unsigned int n_ctxs
= qatomic_read(&tcg_cur_ctxs
);
4535 for (i
= 0; i
< n_ctxs
; i
++) {
4536 const TCGContext
*s
= qatomic_read(&tcg_ctxs
[i
]);
4537 const TCGProfile
*prof
= &s
->prof
;
4539 ret
+= qatomic_read(&prof
->cpu_exec_time
);
4544 void tcg_dump_op_count(GString
*buf
)
4546 g_string_append_printf(buf
, "[TCG profiler not compiled]\n");
4549 int64_t tcg_cpu_exec_time(void)
4551 error_report("%s: TCG profiler not compiled", __func__
);
4557 int tcg_gen_code(TCGContext
*s
, TranslationBlock
*tb
, target_ulong pc_start
)
4559 #ifdef CONFIG_PROFILER
4560 TCGProfile
*prof
= &s
->prof
;
4565 #ifdef CONFIG_PROFILER
4569 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4572 qatomic_set(&prof
->op_count
, prof
->op_count
+ n
);
4573 if (n
> prof
->op_count_max
) {
4574 qatomic_set(&prof
->op_count_max
, n
);
4578 qatomic_set(&prof
->temp_count
, prof
->temp_count
+ n
);
4579 if (n
> prof
->temp_count_max
) {
4580 qatomic_set(&prof
->temp_count_max
, n
);
4586 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP
)
4587 && qemu_log_in_addr_range(pc_start
))) {
4588 FILE *logfile
= qemu_log_trylock();
4590 fprintf(logfile
, "OP:\n");
4591 tcg_dump_ops(s
, logfile
, false);
4592 fprintf(logfile
, "\n");
4593 qemu_log_unlock(logfile
);
4598 #ifdef CONFIG_DEBUG_TCG
4599 /* Ensure all labels referenced have been emitted. */
4604 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
4605 if (unlikely(!l
->present
) && l
->refs
) {
4606 qemu_log_mask(CPU_LOG_TB_OP
,
4607 "$L%d referenced but not present.\n", l
->id
);
4615 #ifdef CONFIG_PROFILER
4616 qatomic_set(&prof
->opt_time
, prof
->opt_time
- profile_getclock());
4619 #ifdef USE_TCG_OPTIMIZATIONS
4623 #ifdef CONFIG_PROFILER
4624 qatomic_set(&prof
->opt_time
, prof
->opt_time
+ profile_getclock());
4625 qatomic_set(&prof
->la_time
, prof
->la_time
- profile_getclock());
4628 reachable_code_pass(s
);
4631 if (s
->nb_indirects
> 0) {
4633 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND
)
4634 && qemu_log_in_addr_range(pc_start
))) {
4635 FILE *logfile
= qemu_log_trylock();
4637 fprintf(logfile
, "OP before indirect lowering:\n");
4638 tcg_dump_ops(s
, logfile
, false);
4639 fprintf(logfile
, "\n");
4640 qemu_log_unlock(logfile
);
4644 /* Replace indirect temps with direct temps. */
4645 if (liveness_pass_2(s
)) {
4646 /* If changes were made, re-run liveness. */
4651 #ifdef CONFIG_PROFILER
4652 qatomic_set(&prof
->la_time
, prof
->la_time
+ profile_getclock());
4656 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT
)
4657 && qemu_log_in_addr_range(pc_start
))) {
4658 FILE *logfile
= qemu_log_trylock();
4660 fprintf(logfile
, "OP after optimization and liveness analysis:\n");
4661 tcg_dump_ops(s
, logfile
, true);
4662 fprintf(logfile
, "\n");
4663 qemu_log_unlock(logfile
);
4668 /* Initialize goto_tb jump offsets. */
4669 tb
->jmp_reset_offset
[0] = TB_JMP_OFFSET_INVALID
;
4670 tb
->jmp_reset_offset
[1] = TB_JMP_OFFSET_INVALID
;
4672 tcg_reg_alloc_start(s
);
4675 * Reset the buffer pointers when restarting after overflow.
4676 * TODO: Move this into translate-all.c with the rest of the
4677 * buffer management. Having only this done here is confusing.
4679 s
->code_buf
= tcg_splitwx_to_rw(tb
->tc
.ptr
);
4680 s
->code_ptr
= s
->code_buf
;
4682 #ifdef TCG_TARGET_NEED_LDST_LABELS
4683 QSIMPLEQ_INIT(&s
->ldst_labels
);
4685 #ifdef TCG_TARGET_NEED_POOL_LABELS
4686 s
->pool_labels
= NULL
;
4690 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4691 TCGOpcode opc
= op
->opc
;
4693 #ifdef CONFIG_PROFILER
4694 qatomic_set(&prof
->table_op_count
[opc
], prof
->table_op_count
[opc
] + 1);
4698 case INDEX_op_mov_i32
:
4699 case INDEX_op_mov_i64
:
4700 case INDEX_op_mov_vec
:
4701 tcg_reg_alloc_mov(s
, op
);
4703 case INDEX_op_dup_vec
:
4704 tcg_reg_alloc_dup(s
, op
);
4706 case INDEX_op_insn_start
:
4707 if (num_insns
>= 0) {
4708 size_t off
= tcg_current_code_size(s
);
4709 s
->gen_insn_end_off
[num_insns
] = off
;
4710 /* Assert that we do not overflow our stored offset. */
4711 assert(s
->gen_insn_end_off
[num_insns
] == off
);
4714 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
4716 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4717 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
4721 s
->gen_insn_data
[num_insns
][i
] = a
;
4724 case INDEX_op_discard
:
4725 temp_dead(s
, arg_temp(op
->args
[0]));
4727 case INDEX_op_set_label
:
4728 tcg_reg_alloc_bb_end(s
, s
->reserved_regs
);
4729 tcg_out_label(s
, arg_label(op
->args
[0]));
4732 tcg_reg_alloc_call(s
, op
);
4734 case INDEX_op_exit_tb
:
4735 tcg_out_exit_tb(s
, op
->args
[0]);
4737 case INDEX_op_goto_tb
:
4738 tcg_out_goto_tb(s
, op
->args
[0]);
4740 case INDEX_op_dup2_vec
:
4741 if (tcg_reg_alloc_dup2(s
, op
)) {
4746 /* Sanity check that we've not introduced any unhandled opcodes. */
4747 tcg_debug_assert(tcg_op_supported(opc
));
4748 /* Note: in order to speed up the code, it would be much
4749 faster to have specialized register allocator functions for
4750 some common argument patterns */
4751 tcg_reg_alloc_op(s
, op
);
4754 /* Test for (pending) buffer overflow. The assumption is that any
4755 one operation beginning below the high water mark cannot overrun
4756 the buffer completely. Thus we can test for overflow after
4757 generating code without having to check during generation. */
4758 if (unlikely((void *)s
->code_ptr
> s
->code_gen_highwater
)) {
4761 /* Test for TB overflow, as seen by gen_insn_end_off. */
4762 if (unlikely(tcg_current_code_size(s
) > UINT16_MAX
)) {
4766 tcg_debug_assert(num_insns
>= 0);
4767 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
4769 /* Generate TB finalization at the end of block */
4770 #ifdef TCG_TARGET_NEED_LDST_LABELS
4771 i
= tcg_out_ldst_finalize(s
);
4776 #ifdef TCG_TARGET_NEED_POOL_LABELS
4777 i
= tcg_out_pool_finalize(s
);
4782 if (!tcg_resolve_relocs(s
)) {
4786 #ifndef CONFIG_TCG_INTERPRETER
4787 /* flush instruction cache */
4788 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s
->code_buf
),
4789 (uintptr_t)s
->code_buf
,
4790 tcg_ptr_byte_diff(s
->code_ptr
, s
->code_buf
));
4793 return tcg_current_code_size(s
);
4796 #ifdef CONFIG_PROFILER
4797 void tcg_dump_info(GString
*buf
)
4799 TCGProfile prof
= {};
4800 const TCGProfile
*s
;
4802 int64_t tb_div_count
;
4805 tcg_profile_snapshot_counters(&prof
);
4807 tb_count
= s
->tb_count
;
4808 tb_div_count
= tb_count
? tb_count
: 1;
4809 tot
= s
->interm_time
+ s
->code_time
;
4811 g_string_append_printf(buf
, "JIT cycles %" PRId64
4812 " (%0.3f s at 2.4 GHz)\n",
4814 g_string_append_printf(buf
, "translated TBs %" PRId64
4815 " (aborted=%" PRId64
" %0.1f%%)\n",
4816 tb_count
, s
->tb_count1
- tb_count
,
4817 (double)(s
->tb_count1
- s
->tb_count
)
4818 / (s
->tb_count1
? s
->tb_count1
: 1) * 100.0);
4819 g_string_append_printf(buf
, "avg ops/TB %0.1f max=%d\n",
4820 (double)s
->op_count
/ tb_div_count
, s
->op_count_max
);
4821 g_string_append_printf(buf
, "deleted ops/TB %0.2f\n",
4822 (double)s
->del_op_count
/ tb_div_count
);
4823 g_string_append_printf(buf
, "avg temps/TB %0.2f max=%d\n",
4824 (double)s
->temp_count
/ tb_div_count
,
4826 g_string_append_printf(buf
, "avg host code/TB %0.1f\n",
4827 (double)s
->code_out_len
/ tb_div_count
);
4828 g_string_append_printf(buf
, "avg search data/TB %0.1f\n",
4829 (double)s
->search_out_len
/ tb_div_count
);
4831 g_string_append_printf(buf
, "cycles/op %0.1f\n",
4832 s
->op_count
? (double)tot
/ s
->op_count
: 0);
4833 g_string_append_printf(buf
, "cycles/in byte %0.1f\n",
4834 s
->code_in_len
? (double)tot
/ s
->code_in_len
: 0);
4835 g_string_append_printf(buf
, "cycles/out byte %0.1f\n",
4836 s
->code_out_len
? (double)tot
/ s
->code_out_len
: 0);
4837 g_string_append_printf(buf
, "cycles/search byte %0.1f\n",
4839 (double)tot
/ s
->search_out_len
: 0);
4843 g_string_append_printf(buf
, " gen_interm time %0.1f%%\n",
4844 (double)s
->interm_time
/ tot
* 100.0);
4845 g_string_append_printf(buf
, " gen_code time %0.1f%%\n",
4846 (double)s
->code_time
/ tot
* 100.0);
4847 g_string_append_printf(buf
, "optim./code time %0.1f%%\n",
4848 (double)s
->opt_time
/ (s
->code_time
?
4851 g_string_append_printf(buf
, "liveness/code time %0.1f%%\n",
4852 (double)s
->la_time
/ (s
->code_time
?
4853 s
->code_time
: 1) * 100.0);
4854 g_string_append_printf(buf
, "cpu_restore count %" PRId64
"\n",
4856 g_string_append_printf(buf
, " avg cycles %0.1f\n",
4858 (double)s
->restore_time
/ s
->restore_count
: 0);
4861 void tcg_dump_info(GString
*buf
)
4863 g_string_append_printf(buf
, "[TCG profiler not compiled]\n");
4867 #ifdef ELF_HOST_MACHINE
4868 /* In order to use this feature, the backend needs to do three things:
4870 (1) Define ELF_HOST_MACHINE to indicate both what value to
4871 put into the ELF image and to indicate support for the feature.
4873 (2) Define tcg_register_jit. This should create a buffer containing
4874 the contents of a .debug_frame section that describes the post-
4875 prologue unwind info for the tcg machine.
4877 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4880 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4887 struct jit_code_entry
{
4888 struct jit_code_entry
*next_entry
;
4889 struct jit_code_entry
*prev_entry
;
4890 const void *symfile_addr
;
4891 uint64_t symfile_size
;
4894 struct jit_descriptor
{
4896 uint32_t action_flag
;
4897 struct jit_code_entry
*relevant_entry
;
4898 struct jit_code_entry
*first_entry
;
4901 void __jit_debug_register_code(void) __attribute__((noinline
));
4902 void __jit_debug_register_code(void)
4907 /* Must statically initialize the version, because GDB may check
4908 the version before we can set it. */
4909 struct jit_descriptor __jit_debug_descriptor
= { 1, 0, 0, 0 };
4911 /* End GDB interface. */
4913 static int find_string(const char *strtab
, const char *str
)
4915 const char *p
= strtab
+ 1;
4918 if (strcmp(p
, str
) == 0) {
4925 static void tcg_register_jit_int(const void *buf_ptr
, size_t buf_size
,
4926 const void *debug_frame
,
4927 size_t debug_frame_size
)
4929 struct __attribute__((packed
)) DebugInfo
{
4936 uintptr_t cu_low_pc
;
4937 uintptr_t cu_high_pc
;
4940 uintptr_t fn_low_pc
;
4941 uintptr_t fn_high_pc
;
4950 struct DebugInfo di
;
4955 struct ElfImage
*img
;
4957 static const struct ElfImage img_template
= {
4959 .e_ident
[EI_MAG0
] = ELFMAG0
,
4960 .e_ident
[EI_MAG1
] = ELFMAG1
,
4961 .e_ident
[EI_MAG2
] = ELFMAG2
,
4962 .e_ident
[EI_MAG3
] = ELFMAG3
,
4963 .e_ident
[EI_CLASS
] = ELF_CLASS
,
4964 .e_ident
[EI_DATA
] = ELF_DATA
,
4965 .e_ident
[EI_VERSION
] = EV_CURRENT
,
4967 .e_machine
= ELF_HOST_MACHINE
,
4968 .e_version
= EV_CURRENT
,
4969 .e_phoff
= offsetof(struct ElfImage
, phdr
),
4970 .e_shoff
= offsetof(struct ElfImage
, shdr
),
4971 .e_ehsize
= sizeof(ElfW(Shdr
)),
4972 .e_phentsize
= sizeof(ElfW(Phdr
)),
4974 .e_shentsize
= sizeof(ElfW(Shdr
)),
4975 .e_shnum
= ARRAY_SIZE(img
->shdr
),
4976 .e_shstrndx
= ARRAY_SIZE(img
->shdr
) - 1,
4977 #ifdef ELF_HOST_FLAGS
4978 .e_flags
= ELF_HOST_FLAGS
,
4981 .e_ident
[EI_OSABI
] = ELF_OSABI
,
4989 [0] = { .sh_type
= SHT_NULL
},
4990 /* Trick: The contents of code_gen_buffer are not present in
4991 this fake ELF file; that got allocated elsewhere. Therefore
4992 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4993 will not look for contents. We can record any address. */
4995 .sh_type
= SHT_NOBITS
,
4996 .sh_flags
= SHF_EXECINSTR
| SHF_ALLOC
,
4998 [2] = { /* .debug_info */
4999 .sh_type
= SHT_PROGBITS
,
5000 .sh_offset
= offsetof(struct ElfImage
, di
),
5001 .sh_size
= sizeof(struct DebugInfo
),
5003 [3] = { /* .debug_abbrev */
5004 .sh_type
= SHT_PROGBITS
,
5005 .sh_offset
= offsetof(struct ElfImage
, da
),
5006 .sh_size
= sizeof(img
->da
),
5008 [4] = { /* .debug_frame */
5009 .sh_type
= SHT_PROGBITS
,
5010 .sh_offset
= sizeof(struct ElfImage
),
5012 [5] = { /* .symtab */
5013 .sh_type
= SHT_SYMTAB
,
5014 .sh_offset
= offsetof(struct ElfImage
, sym
),
5015 .sh_size
= sizeof(img
->sym
),
5017 .sh_link
= ARRAY_SIZE(img
->shdr
) - 1,
5018 .sh_entsize
= sizeof(ElfW(Sym
)),
5020 [6] = { /* .strtab */
5021 .sh_type
= SHT_STRTAB
,
5022 .sh_offset
= offsetof(struct ElfImage
, str
),
5023 .sh_size
= sizeof(img
->str
),
5027 [1] = { /* code_gen_buffer */
5028 .st_info
= ELF_ST_INFO(STB_GLOBAL
, STT_FUNC
),
5033 .len
= sizeof(struct DebugInfo
) - 4,
5035 .ptr_size
= sizeof(void *),
5037 .cu_lang
= 0x8001, /* DW_LANG_Mips_Assembler */
5039 .fn_name
= "code_gen_buffer"
5042 1, /* abbrev number (the cu) */
5043 0x11, 1, /* DW_TAG_compile_unit, has children */
5044 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
5045 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5046 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5047 0, 0, /* end of abbrev */
5048 2, /* abbrev number (the fn) */
5049 0x2e, 0, /* DW_TAG_subprogram, no children */
5050 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
5051 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5052 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5053 0, 0, /* end of abbrev */
5054 0 /* no more abbrev */
5056 .str
= "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5057 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
5060 /* We only need a single jit entry; statically allocate it. */
5061 static struct jit_code_entry one_entry
;
5063 uintptr_t buf
= (uintptr_t)buf_ptr
;
5064 size_t img_size
= sizeof(struct ElfImage
) + debug_frame_size
;
5065 DebugFrameHeader
*dfh
;
5067 img
= g_malloc(img_size
);
5068 *img
= img_template
;
5070 img
->phdr
.p_vaddr
= buf
;
5071 img
->phdr
.p_paddr
= buf
;
5072 img
->phdr
.p_memsz
= buf_size
;
5074 img
->shdr
[1].sh_name
= find_string(img
->str
, ".text");
5075 img
->shdr
[1].sh_addr
= buf
;
5076 img
->shdr
[1].sh_size
= buf_size
;
5078 img
->shdr
[2].sh_name
= find_string(img
->str
, ".debug_info");
5079 img
->shdr
[3].sh_name
= find_string(img
->str
, ".debug_abbrev");
5081 img
->shdr
[4].sh_name
= find_string(img
->str
, ".debug_frame");
5082 img
->shdr
[4].sh_size
= debug_frame_size
;
5084 img
->shdr
[5].sh_name
= find_string(img
->str
, ".symtab");
5085 img
->shdr
[6].sh_name
= find_string(img
->str
, ".strtab");
5087 img
->sym
[1].st_name
= find_string(img
->str
, "code_gen_buffer");
5088 img
->sym
[1].st_value
= buf
;
5089 img
->sym
[1].st_size
= buf_size
;
5091 img
->di
.cu_low_pc
= buf
;
5092 img
->di
.cu_high_pc
= buf
+ buf_size
;
5093 img
->di
.fn_low_pc
= buf
;
5094 img
->di
.fn_high_pc
= buf
+ buf_size
;
5096 dfh
= (DebugFrameHeader
*)(img
+ 1);
5097 memcpy(dfh
, debug_frame
, debug_frame_size
);
5098 dfh
->fde
.func_start
= buf
;
5099 dfh
->fde
.func_len
= buf_size
;
5102 /* Enable this block to be able to debug the ELF image file creation.
5103 One can use readelf, objdump, or other inspection utilities. */
5105 g_autofree
char *jit
= g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5106 FILE *f
= fopen(jit
, "w+b");
5108 if (fwrite(img
, img_size
, 1, f
) != img_size
) {
5109 /* Avoid stupid unused return value warning for fwrite. */
5116 one_entry
.symfile_addr
= img
;
5117 one_entry
.symfile_size
= img_size
;
5119 __jit_debug_descriptor
.action_flag
= JIT_REGISTER_FN
;
5120 __jit_debug_descriptor
.relevant_entry
= &one_entry
;
5121 __jit_debug_descriptor
.first_entry
= &one_entry
;
5122 __jit_debug_register_code();
5125 /* No support for the feature. Provide the entry point expected by exec.c,
5126 and implement the internal function we declared earlier. */
5128 static void tcg_register_jit_int(const void *buf
, size_t size
,
5129 const void *debug_frame
,
5130 size_t debug_frame_size
)
5134 void tcg_register_jit(const void *buf
, size_t buf_size
)
5137 #endif /* ELF_HOST_MACHINE */
5139 #if !TCG_TARGET_MAYBE_vec
5140 void tcg_expand_vec_op(TCGOpcode o
, TCGType t
, unsigned e
, TCGArg a0
, ...)
5142 g_assert_not_reached();