2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
28 #include "qemu/osdep.h"
30 /* Define to jump the ELF file used to communicate with GDB. */
33 #include "qemu/cutils.h"
34 #include "qemu/host-utils.h"
35 #include "qemu/timer.h"
37 /* Note: the long term plan is to reduce the dependencies on the QEMU
38 CPU definitions. Currently they are used for qemu_ld/st
40 #define NO_CPU_IO_DEFS
43 #include "exec/cpu-common.h"
44 #include "exec/exec-all.h"
48 #if UINTPTR_MAX == UINT32_MAX
49 # define ELF_CLASS ELFCLASS32
51 # define ELF_CLASS ELFCLASS64
53 #ifdef HOST_WORDS_BIGENDIAN
54 # define ELF_DATA ELFDATA2MSB
56 # define ELF_DATA ELFDATA2LSB
62 /* Forward declarations for functions declared in tcg-target.inc.c and
64 static void tcg_target_init(TCGContext
*s
);
65 static const TCGTargetOpDef
*tcg_target_op_def(TCGOpcode
);
66 static void tcg_target_qemu_prologue(TCGContext
*s
);
67 static void patch_reloc(tcg_insn_unit
*code_ptr
, int type
,
68 intptr_t value
, intptr_t addend
);
70 /* The CIE and FDE header definitions will be common to all hosts. */
72 uint32_t len
__attribute__((aligned((sizeof(void *)))));
78 uint8_t return_column
;
81 typedef struct QEMU_PACKED
{
82 uint32_t len
__attribute__((aligned((sizeof(void *)))));
86 } DebugFrameFDEHeader
;
88 typedef struct QEMU_PACKED
{
90 DebugFrameFDEHeader fde
;
93 static void tcg_register_jit_int(void *buf
, size_t size
,
94 const void *debug_frame
,
95 size_t debug_frame_size
)
96 __attribute__((unused
));
98 /* Forward declarations for functions declared and used in tcg-target.inc.c. */
99 static const char *target_parse_constraint(TCGArgConstraint
*ct
,
100 const char *ct_str
, TCGType type
);
101 static void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg1
,
103 static void tcg_out_mov(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg
);
104 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
105 TCGReg ret
, tcg_target_long arg
);
106 static void tcg_out_op(TCGContext
*s
, TCGOpcode opc
, const TCGArg
*args
,
107 const int *const_args
);
108 static void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
, TCGReg arg1
,
110 static bool tcg_out_sti(TCGContext
*s
, TCGType type
, TCGArg val
,
111 TCGReg base
, intptr_t ofs
);
112 static void tcg_out_call(TCGContext
*s
, tcg_insn_unit
*target
);
113 static int tcg_target_const_match(tcg_target_long val
, TCGType type
,
114 const TCGArgConstraint
*arg_ct
);
115 #ifdef TCG_TARGET_NEED_LDST_LABELS
116 static bool tcg_out_ldst_finalize(TCGContext
*s
);
119 static TCGRegSet tcg_target_available_regs
[2];
120 static TCGRegSet tcg_target_call_clobber_regs
;
122 #if TCG_TARGET_INSN_UNIT_SIZE == 1
123 static __attribute__((unused
)) inline void tcg_out8(TCGContext
*s
, uint8_t v
)
128 static __attribute__((unused
)) inline void tcg_patch8(tcg_insn_unit
*p
,
135 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
136 static __attribute__((unused
)) inline void tcg_out16(TCGContext
*s
, uint16_t v
)
138 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
141 tcg_insn_unit
*p
= s
->code_ptr
;
142 memcpy(p
, &v
, sizeof(v
));
143 s
->code_ptr
= p
+ (2 / TCG_TARGET_INSN_UNIT_SIZE
);
147 static __attribute__((unused
)) inline void tcg_patch16(tcg_insn_unit
*p
,
150 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
153 memcpy(p
, &v
, sizeof(v
));
158 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
159 static __attribute__((unused
)) inline void tcg_out32(TCGContext
*s
, uint32_t v
)
161 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
164 tcg_insn_unit
*p
= s
->code_ptr
;
165 memcpy(p
, &v
, sizeof(v
));
166 s
->code_ptr
= p
+ (4 / TCG_TARGET_INSN_UNIT_SIZE
);
170 static __attribute__((unused
)) inline void tcg_patch32(tcg_insn_unit
*p
,
173 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
176 memcpy(p
, &v
, sizeof(v
));
181 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
182 static __attribute__((unused
)) inline void tcg_out64(TCGContext
*s
, uint64_t v
)
184 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
187 tcg_insn_unit
*p
= s
->code_ptr
;
188 memcpy(p
, &v
, sizeof(v
));
189 s
->code_ptr
= p
+ (8 / TCG_TARGET_INSN_UNIT_SIZE
);
193 static __attribute__((unused
)) inline void tcg_patch64(tcg_insn_unit
*p
,
196 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
199 memcpy(p
, &v
, sizeof(v
));
204 /* label relocation processing */
206 static void tcg_out_reloc(TCGContext
*s
, tcg_insn_unit
*code_ptr
, int type
,
207 TCGLabel
*l
, intptr_t addend
)
212 /* FIXME: This may break relocations on RISC targets that
213 modify instruction fields in place. The caller may not have
214 written the initial value. */
215 patch_reloc(code_ptr
, type
, l
->u
.value
, addend
);
217 /* add a new relocation entry */
218 r
= tcg_malloc(sizeof(TCGRelocation
));
222 r
->next
= l
->u
.first_reloc
;
223 l
->u
.first_reloc
= r
;
227 static void tcg_out_label(TCGContext
*s
, TCGLabel
*l
, tcg_insn_unit
*ptr
)
229 intptr_t value
= (intptr_t)ptr
;
232 tcg_debug_assert(!l
->has_value
);
234 for (r
= l
->u
.first_reloc
; r
!= NULL
; r
= r
->next
) {
235 patch_reloc(r
->ptr
, r
->type
, value
, r
->addend
);
239 l
->u
.value_ptr
= ptr
;
242 TCGLabel
*gen_new_label(void)
244 TCGContext
*s
= &tcg_ctx
;
245 TCGLabel
*l
= tcg_malloc(sizeof(TCGLabel
));
254 #include "tcg-target.inc.c"
256 /* pool based memory allocation */
257 void *tcg_malloc_internal(TCGContext
*s
, int size
)
262 if (size
> TCG_POOL_CHUNK_SIZE
) {
263 /* big malloc: insert a new pool (XXX: could optimize) */
264 p
= g_malloc(sizeof(TCGPool
) + size
);
266 p
->next
= s
->pool_first_large
;
267 s
->pool_first_large
= p
;
278 pool_size
= TCG_POOL_CHUNK_SIZE
;
279 p
= g_malloc(sizeof(TCGPool
) + pool_size
);
283 s
->pool_current
->next
= p
;
292 s
->pool_cur
= p
->data
+ size
;
293 s
->pool_end
= p
->data
+ p
->size
;
297 void tcg_pool_reset(TCGContext
*s
)
300 for (p
= s
->pool_first_large
; p
; p
= t
) {
304 s
->pool_first_large
= NULL
;
305 s
->pool_cur
= s
->pool_end
= NULL
;
306 s
->pool_current
= NULL
;
309 typedef struct TCGHelperInfo
{
316 #include "exec/helper-proto.h"
318 static const TCGHelperInfo all_helpers
[] = {
319 #include "exec/helper-tcg.h"
322 static int indirect_reg_alloc_order
[ARRAY_SIZE(tcg_target_reg_alloc_order
)];
323 static void process_op_defs(TCGContext
*s
);
325 void tcg_context_init(TCGContext
*s
)
327 int op
, total_args
, n
, i
;
329 TCGArgConstraint
*args_ct
;
331 GHashTable
*helper_table
;
333 memset(s
, 0, sizeof(*s
));
336 /* Count total number of arguments and allocate the corresponding
339 for(op
= 0; op
< NB_OPS
; op
++) {
340 def
= &tcg_op_defs
[op
];
341 n
= def
->nb_iargs
+ def
->nb_oargs
;
345 args_ct
= g_malloc(sizeof(TCGArgConstraint
) * total_args
);
346 sorted_args
= g_malloc(sizeof(int) * total_args
);
348 for(op
= 0; op
< NB_OPS
; op
++) {
349 def
= &tcg_op_defs
[op
];
350 def
->args_ct
= args_ct
;
351 def
->sorted_args
= sorted_args
;
352 n
= def
->nb_iargs
+ def
->nb_oargs
;
357 /* Register helpers. */
358 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
359 s
->helpers
= helper_table
= g_hash_table_new(NULL
, NULL
);
361 for (i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
362 g_hash_table_insert(helper_table
, (gpointer
)all_helpers
[i
].func
,
363 (gpointer
)&all_helpers
[i
]);
369 /* Reverse the order of the saved registers, assuming they're all at
370 the start of tcg_target_reg_alloc_order. */
371 for (n
= 0; n
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++n
) {
372 int r
= tcg_target_reg_alloc_order
[n
];
373 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, r
)) {
377 for (i
= 0; i
< n
; ++i
) {
378 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[n
- 1 - i
];
380 for (; i
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++i
) {
381 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[i
];
386 * Allocate TBs right before their corresponding translated code, making
387 * sure that TBs and code are on different cache lines.
389 TranslationBlock
*tcg_tb_alloc(TCGContext
*s
)
391 uintptr_t align
= qemu_icache_linesize
;
392 TranslationBlock
*tb
;
395 tb
= (void *)ROUND_UP((uintptr_t)s
->code_gen_ptr
, align
);
396 next
= (void *)ROUND_UP((uintptr_t)(tb
+ 1), align
);
398 if (unlikely(next
> s
->code_gen_highwater
)) {
401 s
->code_gen_ptr
= next
;
402 s
->data_gen_ptr
= NULL
;
406 void tcg_prologue_init(TCGContext
*s
)
408 size_t prologue_size
, total_size
;
411 /* Put the prologue at the beginning of code_gen_buffer. */
412 buf0
= s
->code_gen_buffer
;
415 s
->code_gen_prologue
= buf0
;
417 /* Generate the prologue. */
418 tcg_target_qemu_prologue(s
);
420 flush_icache_range((uintptr_t)buf0
, (uintptr_t)buf1
);
422 /* Deduct the prologue from the buffer. */
423 prologue_size
= tcg_current_code_size(s
);
424 s
->code_gen_ptr
= buf1
;
425 s
->code_gen_buffer
= buf1
;
427 total_size
= s
->code_gen_buffer_size
- prologue_size
;
428 s
->code_gen_buffer_size
= total_size
;
430 /* Compute a high-water mark, at which we voluntarily flush the buffer
431 and start over. The size here is arbitrary, significantly larger
432 than we expect the code generation for any one opcode to require. */
433 s
->code_gen_highwater
= s
->code_gen_buffer
+ (total_size
- 1024);
435 tcg_register_jit(s
->code_gen_buffer
, total_size
);
438 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
440 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size
);
441 log_disas(buf0
, prologue_size
);
448 /* Assert that goto_ptr is implemented completely. */
449 if (TCG_TARGET_HAS_goto_ptr
) {
450 tcg_debug_assert(s
->code_gen_epilogue
!= NULL
);
454 void tcg_func_start(TCGContext
*s
)
457 s
->nb_temps
= s
->nb_globals
;
459 /* No temps have been previously allocated for size or locality. */
460 memset(s
->free_temps
, 0, sizeof(s
->free_temps
));
463 s
->current_frame_offset
= s
->frame_start
;
465 #ifdef CONFIG_DEBUG_TCG
466 s
->goto_tb_issue_mask
= 0;
469 s
->gen_op_buf
[0].next
= 1;
470 s
->gen_op_buf
[0].prev
= 0;
471 s
->gen_next_op_idx
= 1;
472 s
->gen_next_parm_idx
= 0;
475 static inline int temp_idx(TCGContext
*s
, TCGTemp
*ts
)
477 ptrdiff_t n
= ts
- s
->temps
;
478 tcg_debug_assert(n
>= 0 && n
< s
->nb_temps
);
482 static inline TCGTemp
*tcg_temp_alloc(TCGContext
*s
)
484 int n
= s
->nb_temps
++;
485 tcg_debug_assert(n
< TCG_MAX_TEMPS
);
486 return memset(&s
->temps
[n
], 0, sizeof(TCGTemp
));
489 static inline TCGTemp
*tcg_global_alloc(TCGContext
*s
)
491 tcg_debug_assert(s
->nb_globals
== s
->nb_temps
);
493 return tcg_temp_alloc(s
);
496 static int tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
497 TCGReg reg
, const char *name
)
501 if (TCG_TARGET_REG_BITS
== 32 && type
!= TCG_TYPE_I32
) {
505 ts
= tcg_global_alloc(s
);
506 ts
->base_type
= type
;
511 tcg_regset_set_reg(s
->reserved_regs
, reg
);
513 return temp_idx(s
, ts
);
516 void tcg_set_frame(TCGContext
*s
, TCGReg reg
, intptr_t start
, intptr_t size
)
519 s
->frame_start
= start
;
520 s
->frame_end
= start
+ size
;
521 idx
= tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, reg
, "_frame");
522 s
->frame_temp
= &s
->temps
[idx
];
525 TCGv_i32
tcg_global_reg_new_i32(TCGReg reg
, const char *name
)
527 TCGContext
*s
= &tcg_ctx
;
530 if (tcg_regset_test_reg(s
->reserved_regs
, reg
)) {
533 idx
= tcg_global_reg_new_internal(s
, TCG_TYPE_I32
, reg
, name
);
534 return MAKE_TCGV_I32(idx
);
537 TCGv_i64
tcg_global_reg_new_i64(TCGReg reg
, const char *name
)
539 TCGContext
*s
= &tcg_ctx
;
542 if (tcg_regset_test_reg(s
->reserved_regs
, reg
)) {
545 idx
= tcg_global_reg_new_internal(s
, TCG_TYPE_I64
, reg
, name
);
546 return MAKE_TCGV_I64(idx
);
549 int tcg_global_mem_new_internal(TCGType type
, TCGv_ptr base
,
550 intptr_t offset
, const char *name
)
552 TCGContext
*s
= &tcg_ctx
;
553 TCGTemp
*base_ts
= &s
->temps
[GET_TCGV_PTR(base
)];
554 TCGTemp
*ts
= tcg_global_alloc(s
);
555 int indirect_reg
= 0, bigendian
= 0;
556 #ifdef HOST_WORDS_BIGENDIAN
560 if (!base_ts
->fixed_reg
) {
561 /* We do not support double-indirect registers. */
562 tcg_debug_assert(!base_ts
->indirect_reg
);
563 base_ts
->indirect_base
= 1;
564 s
->nb_indirects
+= (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
569 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
570 TCGTemp
*ts2
= tcg_global_alloc(s
);
573 ts
->base_type
= TCG_TYPE_I64
;
574 ts
->type
= TCG_TYPE_I32
;
575 ts
->indirect_reg
= indirect_reg
;
576 ts
->mem_allocated
= 1;
577 ts
->mem_base
= base_ts
;
578 ts
->mem_offset
= offset
+ bigendian
* 4;
579 pstrcpy(buf
, sizeof(buf
), name
);
580 pstrcat(buf
, sizeof(buf
), "_0");
581 ts
->name
= strdup(buf
);
583 tcg_debug_assert(ts2
== ts
+ 1);
584 ts2
->base_type
= TCG_TYPE_I64
;
585 ts2
->type
= TCG_TYPE_I32
;
586 ts2
->indirect_reg
= indirect_reg
;
587 ts2
->mem_allocated
= 1;
588 ts2
->mem_base
= base_ts
;
589 ts2
->mem_offset
= offset
+ (1 - bigendian
) * 4;
590 pstrcpy(buf
, sizeof(buf
), name
);
591 pstrcat(buf
, sizeof(buf
), "_1");
592 ts2
->name
= strdup(buf
);
594 ts
->base_type
= type
;
596 ts
->indirect_reg
= indirect_reg
;
597 ts
->mem_allocated
= 1;
598 ts
->mem_base
= base_ts
;
599 ts
->mem_offset
= offset
;
602 return temp_idx(s
, ts
);
605 static int tcg_temp_new_internal(TCGType type
, int temp_local
)
607 TCGContext
*s
= &tcg_ctx
;
611 k
= type
+ (temp_local
? TCG_TYPE_COUNT
: 0);
612 idx
= find_first_bit(s
->free_temps
[k
].l
, TCG_MAX_TEMPS
);
613 if (idx
< TCG_MAX_TEMPS
) {
614 /* There is already an available temp with the right type. */
615 clear_bit(idx
, s
->free_temps
[k
].l
);
618 ts
->temp_allocated
= 1;
619 tcg_debug_assert(ts
->base_type
== type
);
620 tcg_debug_assert(ts
->temp_local
== temp_local
);
622 ts
= tcg_temp_alloc(s
);
623 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
624 TCGTemp
*ts2
= tcg_temp_alloc(s
);
626 ts
->base_type
= type
;
627 ts
->type
= TCG_TYPE_I32
;
628 ts
->temp_allocated
= 1;
629 ts
->temp_local
= temp_local
;
631 tcg_debug_assert(ts2
== ts
+ 1);
632 ts2
->base_type
= TCG_TYPE_I64
;
633 ts2
->type
= TCG_TYPE_I32
;
634 ts2
->temp_allocated
= 1;
635 ts2
->temp_local
= temp_local
;
637 ts
->base_type
= type
;
639 ts
->temp_allocated
= 1;
640 ts
->temp_local
= temp_local
;
642 idx
= temp_idx(s
, ts
);
645 #if defined(CONFIG_DEBUG_TCG)
651 TCGv_i32
tcg_temp_new_internal_i32(int temp_local
)
655 idx
= tcg_temp_new_internal(TCG_TYPE_I32
, temp_local
);
656 return MAKE_TCGV_I32(idx
);
659 TCGv_i64
tcg_temp_new_internal_i64(int temp_local
)
663 idx
= tcg_temp_new_internal(TCG_TYPE_I64
, temp_local
);
664 return MAKE_TCGV_I64(idx
);
667 static void tcg_temp_free_internal(int idx
)
669 TCGContext
*s
= &tcg_ctx
;
673 #if defined(CONFIG_DEBUG_TCG)
675 if (s
->temps_in_use
< 0) {
676 fprintf(stderr
, "More temporaries freed than allocated!\n");
680 tcg_debug_assert(idx
>= s
->nb_globals
&& idx
< s
->nb_temps
);
682 tcg_debug_assert(ts
->temp_allocated
!= 0);
683 ts
->temp_allocated
= 0;
685 k
= ts
->base_type
+ (ts
->temp_local
? TCG_TYPE_COUNT
: 0);
686 set_bit(idx
, s
->free_temps
[k
].l
);
689 void tcg_temp_free_i32(TCGv_i32 arg
)
691 tcg_temp_free_internal(GET_TCGV_I32(arg
));
694 void tcg_temp_free_i64(TCGv_i64 arg
)
696 tcg_temp_free_internal(GET_TCGV_I64(arg
));
699 TCGv_i32
tcg_const_i32(int32_t val
)
702 t0
= tcg_temp_new_i32();
703 tcg_gen_movi_i32(t0
, val
);
707 TCGv_i64
tcg_const_i64(int64_t val
)
710 t0
= tcg_temp_new_i64();
711 tcg_gen_movi_i64(t0
, val
);
715 TCGv_i32
tcg_const_local_i32(int32_t val
)
718 t0
= tcg_temp_local_new_i32();
719 tcg_gen_movi_i32(t0
, val
);
723 TCGv_i64
tcg_const_local_i64(int64_t val
)
726 t0
= tcg_temp_local_new_i64();
727 tcg_gen_movi_i64(t0
, val
);
731 #if defined(CONFIG_DEBUG_TCG)
732 void tcg_clear_temp_count(void)
734 TCGContext
*s
= &tcg_ctx
;
738 int tcg_check_temp_count(void)
740 TCGContext
*s
= &tcg_ctx
;
741 if (s
->temps_in_use
) {
742 /* Clear the count so that we don't give another
743 * warning immediately next time around.
752 /* Return true if OP may appear in the opcode stream.
753 Test the runtime variable that controls each opcode. */
754 bool tcg_op_supported(TCGOpcode op
)
757 case INDEX_op_discard
:
758 case INDEX_op_set_label
:
762 case INDEX_op_insn_start
:
763 case INDEX_op_exit_tb
:
764 case INDEX_op_goto_tb
:
765 case INDEX_op_qemu_ld_i32
:
766 case INDEX_op_qemu_st_i32
:
767 case INDEX_op_qemu_ld_i64
:
768 case INDEX_op_qemu_st_i64
:
771 case INDEX_op_goto_ptr
:
772 return TCG_TARGET_HAS_goto_ptr
;
774 case INDEX_op_mov_i32
:
775 case INDEX_op_movi_i32
:
776 case INDEX_op_setcond_i32
:
777 case INDEX_op_brcond_i32
:
778 case INDEX_op_ld8u_i32
:
779 case INDEX_op_ld8s_i32
:
780 case INDEX_op_ld16u_i32
:
781 case INDEX_op_ld16s_i32
:
782 case INDEX_op_ld_i32
:
783 case INDEX_op_st8_i32
:
784 case INDEX_op_st16_i32
:
785 case INDEX_op_st_i32
:
786 case INDEX_op_add_i32
:
787 case INDEX_op_sub_i32
:
788 case INDEX_op_mul_i32
:
789 case INDEX_op_and_i32
:
790 case INDEX_op_or_i32
:
791 case INDEX_op_xor_i32
:
792 case INDEX_op_shl_i32
:
793 case INDEX_op_shr_i32
:
794 case INDEX_op_sar_i32
:
797 case INDEX_op_movcond_i32
:
798 return TCG_TARGET_HAS_movcond_i32
;
799 case INDEX_op_div_i32
:
800 case INDEX_op_divu_i32
:
801 return TCG_TARGET_HAS_div_i32
;
802 case INDEX_op_rem_i32
:
803 case INDEX_op_remu_i32
:
804 return TCG_TARGET_HAS_rem_i32
;
805 case INDEX_op_div2_i32
:
806 case INDEX_op_divu2_i32
:
807 return TCG_TARGET_HAS_div2_i32
;
808 case INDEX_op_rotl_i32
:
809 case INDEX_op_rotr_i32
:
810 return TCG_TARGET_HAS_rot_i32
;
811 case INDEX_op_deposit_i32
:
812 return TCG_TARGET_HAS_deposit_i32
;
813 case INDEX_op_extract_i32
:
814 return TCG_TARGET_HAS_extract_i32
;
815 case INDEX_op_sextract_i32
:
816 return TCG_TARGET_HAS_sextract_i32
;
817 case INDEX_op_add2_i32
:
818 return TCG_TARGET_HAS_add2_i32
;
819 case INDEX_op_sub2_i32
:
820 return TCG_TARGET_HAS_sub2_i32
;
821 case INDEX_op_mulu2_i32
:
822 return TCG_TARGET_HAS_mulu2_i32
;
823 case INDEX_op_muls2_i32
:
824 return TCG_TARGET_HAS_muls2_i32
;
825 case INDEX_op_muluh_i32
:
826 return TCG_TARGET_HAS_muluh_i32
;
827 case INDEX_op_mulsh_i32
:
828 return TCG_TARGET_HAS_mulsh_i32
;
829 case INDEX_op_ext8s_i32
:
830 return TCG_TARGET_HAS_ext8s_i32
;
831 case INDEX_op_ext16s_i32
:
832 return TCG_TARGET_HAS_ext16s_i32
;
833 case INDEX_op_ext8u_i32
:
834 return TCG_TARGET_HAS_ext8u_i32
;
835 case INDEX_op_ext16u_i32
:
836 return TCG_TARGET_HAS_ext16u_i32
;
837 case INDEX_op_bswap16_i32
:
838 return TCG_TARGET_HAS_bswap16_i32
;
839 case INDEX_op_bswap32_i32
:
840 return TCG_TARGET_HAS_bswap32_i32
;
841 case INDEX_op_not_i32
:
842 return TCG_TARGET_HAS_not_i32
;
843 case INDEX_op_neg_i32
:
844 return TCG_TARGET_HAS_neg_i32
;
845 case INDEX_op_andc_i32
:
846 return TCG_TARGET_HAS_andc_i32
;
847 case INDEX_op_orc_i32
:
848 return TCG_TARGET_HAS_orc_i32
;
849 case INDEX_op_eqv_i32
:
850 return TCG_TARGET_HAS_eqv_i32
;
851 case INDEX_op_nand_i32
:
852 return TCG_TARGET_HAS_nand_i32
;
853 case INDEX_op_nor_i32
:
854 return TCG_TARGET_HAS_nor_i32
;
855 case INDEX_op_clz_i32
:
856 return TCG_TARGET_HAS_clz_i32
;
857 case INDEX_op_ctz_i32
:
858 return TCG_TARGET_HAS_ctz_i32
;
859 case INDEX_op_ctpop_i32
:
860 return TCG_TARGET_HAS_ctpop_i32
;
862 case INDEX_op_brcond2_i32
:
863 case INDEX_op_setcond2_i32
:
864 return TCG_TARGET_REG_BITS
== 32;
866 case INDEX_op_mov_i64
:
867 case INDEX_op_movi_i64
:
868 case INDEX_op_setcond_i64
:
869 case INDEX_op_brcond_i64
:
870 case INDEX_op_ld8u_i64
:
871 case INDEX_op_ld8s_i64
:
872 case INDEX_op_ld16u_i64
:
873 case INDEX_op_ld16s_i64
:
874 case INDEX_op_ld32u_i64
:
875 case INDEX_op_ld32s_i64
:
876 case INDEX_op_ld_i64
:
877 case INDEX_op_st8_i64
:
878 case INDEX_op_st16_i64
:
879 case INDEX_op_st32_i64
:
880 case INDEX_op_st_i64
:
881 case INDEX_op_add_i64
:
882 case INDEX_op_sub_i64
:
883 case INDEX_op_mul_i64
:
884 case INDEX_op_and_i64
:
885 case INDEX_op_or_i64
:
886 case INDEX_op_xor_i64
:
887 case INDEX_op_shl_i64
:
888 case INDEX_op_shr_i64
:
889 case INDEX_op_sar_i64
:
890 case INDEX_op_ext_i32_i64
:
891 case INDEX_op_extu_i32_i64
:
892 return TCG_TARGET_REG_BITS
== 64;
894 case INDEX_op_movcond_i64
:
895 return TCG_TARGET_HAS_movcond_i64
;
896 case INDEX_op_div_i64
:
897 case INDEX_op_divu_i64
:
898 return TCG_TARGET_HAS_div_i64
;
899 case INDEX_op_rem_i64
:
900 case INDEX_op_remu_i64
:
901 return TCG_TARGET_HAS_rem_i64
;
902 case INDEX_op_div2_i64
:
903 case INDEX_op_divu2_i64
:
904 return TCG_TARGET_HAS_div2_i64
;
905 case INDEX_op_rotl_i64
:
906 case INDEX_op_rotr_i64
:
907 return TCG_TARGET_HAS_rot_i64
;
908 case INDEX_op_deposit_i64
:
909 return TCG_TARGET_HAS_deposit_i64
;
910 case INDEX_op_extract_i64
:
911 return TCG_TARGET_HAS_extract_i64
;
912 case INDEX_op_sextract_i64
:
913 return TCG_TARGET_HAS_sextract_i64
;
914 case INDEX_op_extrl_i64_i32
:
915 return TCG_TARGET_HAS_extrl_i64_i32
;
916 case INDEX_op_extrh_i64_i32
:
917 return TCG_TARGET_HAS_extrh_i64_i32
;
918 case INDEX_op_ext8s_i64
:
919 return TCG_TARGET_HAS_ext8s_i64
;
920 case INDEX_op_ext16s_i64
:
921 return TCG_TARGET_HAS_ext16s_i64
;
922 case INDEX_op_ext32s_i64
:
923 return TCG_TARGET_HAS_ext32s_i64
;
924 case INDEX_op_ext8u_i64
:
925 return TCG_TARGET_HAS_ext8u_i64
;
926 case INDEX_op_ext16u_i64
:
927 return TCG_TARGET_HAS_ext16u_i64
;
928 case INDEX_op_ext32u_i64
:
929 return TCG_TARGET_HAS_ext32u_i64
;
930 case INDEX_op_bswap16_i64
:
931 return TCG_TARGET_HAS_bswap16_i64
;
932 case INDEX_op_bswap32_i64
:
933 return TCG_TARGET_HAS_bswap32_i64
;
934 case INDEX_op_bswap64_i64
:
935 return TCG_TARGET_HAS_bswap64_i64
;
936 case INDEX_op_not_i64
:
937 return TCG_TARGET_HAS_not_i64
;
938 case INDEX_op_neg_i64
:
939 return TCG_TARGET_HAS_neg_i64
;
940 case INDEX_op_andc_i64
:
941 return TCG_TARGET_HAS_andc_i64
;
942 case INDEX_op_orc_i64
:
943 return TCG_TARGET_HAS_orc_i64
;
944 case INDEX_op_eqv_i64
:
945 return TCG_TARGET_HAS_eqv_i64
;
946 case INDEX_op_nand_i64
:
947 return TCG_TARGET_HAS_nand_i64
;
948 case INDEX_op_nor_i64
:
949 return TCG_TARGET_HAS_nor_i64
;
950 case INDEX_op_clz_i64
:
951 return TCG_TARGET_HAS_clz_i64
;
952 case INDEX_op_ctz_i64
:
953 return TCG_TARGET_HAS_ctz_i64
;
954 case INDEX_op_ctpop_i64
:
955 return TCG_TARGET_HAS_ctpop_i64
;
956 case INDEX_op_add2_i64
:
957 return TCG_TARGET_HAS_add2_i64
;
958 case INDEX_op_sub2_i64
:
959 return TCG_TARGET_HAS_sub2_i64
;
960 case INDEX_op_mulu2_i64
:
961 return TCG_TARGET_HAS_mulu2_i64
;
962 case INDEX_op_muls2_i64
:
963 return TCG_TARGET_HAS_muls2_i64
;
964 case INDEX_op_muluh_i64
:
965 return TCG_TARGET_HAS_muluh_i64
;
966 case INDEX_op_mulsh_i64
:
967 return TCG_TARGET_HAS_mulsh_i64
;
972 g_assert_not_reached();
975 /* Note: we convert the 64 bit args to 32 bit and do some alignment
976 and endian swap. Maybe it would be better to do the alignment
977 and endian swap in tcg_reg_alloc_call(). */
978 void tcg_gen_callN(TCGContext
*s
, void *func
, TCGArg ret
,
979 int nargs
, TCGArg
*args
)
981 int i
, real_args
, nb_rets
, pi
, pi_first
;
982 unsigned sizemask
, flags
;
985 info
= g_hash_table_lookup(s
->helpers
, (gpointer
)func
);
987 sizemask
= info
->sizemask
;
989 #if defined(__sparc__) && !defined(__arch64__) \
990 && !defined(CONFIG_TCG_INTERPRETER)
991 /* We have 64-bit values in one register, but need to pass as two
992 separate parameters. Split them. */
993 int orig_sizemask
= sizemask
;
994 int orig_nargs
= nargs
;
997 TCGV_UNUSED_I64(retl
);
998 TCGV_UNUSED_I64(reth
);
1000 TCGArg
*split_args
= __builtin_alloca(sizeof(TCGArg
) * nargs
* 2);
1001 for (i
= real_args
= 0; i
< nargs
; ++i
) {
1002 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1004 TCGv_i64 orig
= MAKE_TCGV_I64(args
[i
]);
1005 TCGv_i32 h
= tcg_temp_new_i32();
1006 TCGv_i32 l
= tcg_temp_new_i32();
1007 tcg_gen_extr_i64_i32(l
, h
, orig
);
1008 split_args
[real_args
++] = GET_TCGV_I32(h
);
1009 split_args
[real_args
++] = GET_TCGV_I32(l
);
1011 split_args
[real_args
++] = args
[i
];
1018 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1019 for (i
= 0; i
< nargs
; ++i
) {
1020 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1021 int is_signed
= sizemask
& (2 << (i
+1)*2);
1023 TCGv_i64 temp
= tcg_temp_new_i64();
1024 TCGv_i64 orig
= MAKE_TCGV_I64(args
[i
]);
1026 tcg_gen_ext32s_i64(temp
, orig
);
1028 tcg_gen_ext32u_i64(temp
, orig
);
1030 args
[i
] = GET_TCGV_I64(temp
);
1033 #endif /* TCG_TARGET_EXTEND_ARGS */
1035 pi_first
= pi
= s
->gen_next_parm_idx
;
1036 if (ret
!= TCG_CALL_DUMMY_ARG
) {
1037 #if defined(__sparc__) && !defined(__arch64__) \
1038 && !defined(CONFIG_TCG_INTERPRETER)
1039 if (orig_sizemask
& 1) {
1040 /* The 32-bit ABI is going to return the 64-bit value in
1041 the %o0/%o1 register pair. Prepare for this by using
1042 two return temporaries, and reassemble below. */
1043 retl
= tcg_temp_new_i64();
1044 reth
= tcg_temp_new_i64();
1045 s
->gen_opparam_buf
[pi
++] = GET_TCGV_I64(reth
);
1046 s
->gen_opparam_buf
[pi
++] = GET_TCGV_I64(retl
);
1049 s
->gen_opparam_buf
[pi
++] = ret
;
1053 if (TCG_TARGET_REG_BITS
< 64 && (sizemask
& 1)) {
1054 #ifdef HOST_WORDS_BIGENDIAN
1055 s
->gen_opparam_buf
[pi
++] = ret
+ 1;
1056 s
->gen_opparam_buf
[pi
++] = ret
;
1058 s
->gen_opparam_buf
[pi
++] = ret
;
1059 s
->gen_opparam_buf
[pi
++] = ret
+ 1;
1063 s
->gen_opparam_buf
[pi
++] = ret
;
1071 for (i
= 0; i
< nargs
; i
++) {
1072 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1073 if (TCG_TARGET_REG_BITS
< 64 && is_64bit
) {
1074 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1075 /* some targets want aligned 64 bit args */
1076 if (real_args
& 1) {
1077 s
->gen_opparam_buf
[pi
++] = TCG_CALL_DUMMY_ARG
;
1081 /* If stack grows up, then we will be placing successive
1082 arguments at lower addresses, which means we need to
1083 reverse the order compared to how we would normally
1084 treat either big or little-endian. For those arguments
1085 that will wind up in registers, this still works for
1086 HPPA (the only current STACK_GROWSUP target) since the
1087 argument registers are *also* allocated in decreasing
1088 order. If another such target is added, this logic may
1089 have to get more complicated to differentiate between
1090 stack arguments and register arguments. */
1091 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
1092 s
->gen_opparam_buf
[pi
++] = args
[i
] + 1;
1093 s
->gen_opparam_buf
[pi
++] = args
[i
];
1095 s
->gen_opparam_buf
[pi
++] = args
[i
];
1096 s
->gen_opparam_buf
[pi
++] = args
[i
] + 1;
1102 s
->gen_opparam_buf
[pi
++] = args
[i
];
1105 s
->gen_opparam_buf
[pi
++] = (uintptr_t)func
;
1106 s
->gen_opparam_buf
[pi
++] = flags
;
1108 i
= s
->gen_next_op_idx
;
1109 tcg_debug_assert(i
< OPC_BUF_SIZE
);
1110 tcg_debug_assert(pi
<= OPPARAM_BUF_SIZE
);
1112 /* Set links for sequential allocation during translation. */
1113 s
->gen_op_buf
[i
] = (TCGOp
){
1114 .opc
= INDEX_op_call
,
1122 /* Make sure the calli field didn't overflow. */
1123 tcg_debug_assert(s
->gen_op_buf
[i
].calli
== real_args
);
1125 s
->gen_op_buf
[0].prev
= i
;
1126 s
->gen_next_op_idx
= i
+ 1;
1127 s
->gen_next_parm_idx
= pi
;
1129 #if defined(__sparc__) && !defined(__arch64__) \
1130 && !defined(CONFIG_TCG_INTERPRETER)
1131 /* Free all of the parts we allocated above. */
1132 for (i
= real_args
= 0; i
< orig_nargs
; ++i
) {
1133 int is_64bit
= orig_sizemask
& (1 << (i
+1)*2);
1135 TCGv_i32 h
= MAKE_TCGV_I32(args
[real_args
++]);
1136 TCGv_i32 l
= MAKE_TCGV_I32(args
[real_args
++]);
1137 tcg_temp_free_i32(h
);
1138 tcg_temp_free_i32(l
);
1143 if (orig_sizemask
& 1) {
1144 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1145 Note that describing these as TCGv_i64 eliminates an unnecessary
1146 zero-extension that tcg_gen_concat_i32_i64 would create. */
1147 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret
), retl
, reth
);
1148 tcg_temp_free_i64(retl
);
1149 tcg_temp_free_i64(reth
);
1151 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1152 for (i
= 0; i
< nargs
; ++i
) {
1153 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1155 TCGv_i64 temp
= MAKE_TCGV_I64(args
[i
]);
1156 tcg_temp_free_i64(temp
);
1159 #endif /* TCG_TARGET_EXTEND_ARGS */
1162 static void tcg_reg_alloc_start(TCGContext
*s
)
1166 for(i
= 0; i
< s
->nb_globals
; i
++) {
1168 if (ts
->fixed_reg
) {
1169 ts
->val_type
= TEMP_VAL_REG
;
1171 ts
->val_type
= TEMP_VAL_MEM
;
1174 for(i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
1176 if (ts
->temp_local
) {
1177 ts
->val_type
= TEMP_VAL_MEM
;
1179 ts
->val_type
= TEMP_VAL_DEAD
;
1181 ts
->mem_allocated
= 0;
1185 memset(s
->reg_to_temp
, 0, sizeof(s
->reg_to_temp
));
1188 static char *tcg_get_arg_str_ptr(TCGContext
*s
, char *buf
, int buf_size
,
1191 int idx
= temp_idx(s
, ts
);
1193 if (idx
< s
->nb_globals
) {
1194 pstrcpy(buf
, buf_size
, ts
->name
);
1195 } else if (ts
->temp_local
) {
1196 snprintf(buf
, buf_size
, "loc%d", idx
- s
->nb_globals
);
1198 snprintf(buf
, buf_size
, "tmp%d", idx
- s
->nb_globals
);
1203 static char *tcg_get_arg_str_idx(TCGContext
*s
, char *buf
,
1204 int buf_size
, int idx
)
1206 tcg_debug_assert(idx
>= 0 && idx
< s
->nb_temps
);
1207 return tcg_get_arg_str_ptr(s
, buf
, buf_size
, &s
->temps
[idx
]);
1210 /* Find helper name. */
1211 static inline const char *tcg_find_helper(TCGContext
*s
, uintptr_t val
)
1213 const char *ret
= NULL
;
1215 TCGHelperInfo
*info
= g_hash_table_lookup(s
->helpers
, (gpointer
)val
);
1223 static const char * const cond_name
[] =
1225 [TCG_COND_NEVER
] = "never",
1226 [TCG_COND_ALWAYS
] = "always",
1227 [TCG_COND_EQ
] = "eq",
1228 [TCG_COND_NE
] = "ne",
1229 [TCG_COND_LT
] = "lt",
1230 [TCG_COND_GE
] = "ge",
1231 [TCG_COND_LE
] = "le",
1232 [TCG_COND_GT
] = "gt",
1233 [TCG_COND_LTU
] = "ltu",
1234 [TCG_COND_GEU
] = "geu",
1235 [TCG_COND_LEU
] = "leu",
1236 [TCG_COND_GTU
] = "gtu"
1239 static const char * const ldst_name
[] =
1255 static const char * const alignment_name
[(MO_AMASK
>> MO_ASHIFT
) + 1] = {
1257 [MO_UNALN
>> MO_ASHIFT
] = "un+",
1258 [MO_ALIGN
>> MO_ASHIFT
] = "",
1260 [MO_UNALN
>> MO_ASHIFT
] = "",
1261 [MO_ALIGN
>> MO_ASHIFT
] = "al+",
1263 [MO_ALIGN_2
>> MO_ASHIFT
] = "al2+",
1264 [MO_ALIGN_4
>> MO_ASHIFT
] = "al4+",
1265 [MO_ALIGN_8
>> MO_ASHIFT
] = "al8+",
1266 [MO_ALIGN_16
>> MO_ASHIFT
] = "al16+",
1267 [MO_ALIGN_32
>> MO_ASHIFT
] = "al32+",
1268 [MO_ALIGN_64
>> MO_ASHIFT
] = "al64+",
1271 void tcg_dump_ops(TCGContext
*s
)
1277 for (oi
= s
->gen_op_buf
[0].next
; oi
!= 0; oi
= op
->next
) {
1278 int i
, k
, nb_oargs
, nb_iargs
, nb_cargs
;
1279 const TCGOpDef
*def
;
1284 op
= &s
->gen_op_buf
[oi
];
1286 def
= &tcg_op_defs
[c
];
1287 args
= &s
->gen_opparam_buf
[op
->args
];
1289 if (c
== INDEX_op_insn_start
) {
1290 col
+= qemu_log("%s ----", oi
!= s
->gen_op_buf
[0].next
? "\n" : "");
1292 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
1294 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1295 a
= ((target_ulong
)args
[i
* 2 + 1] << 32) | args
[i
* 2];
1299 col
+= qemu_log(" " TARGET_FMT_lx
, a
);
1301 } else if (c
== INDEX_op_call
) {
1302 /* variable number of arguments */
1303 nb_oargs
= op
->callo
;
1304 nb_iargs
= op
->calli
;
1305 nb_cargs
= def
->nb_cargs
;
1307 /* function name, flags, out args */
1308 col
+= qemu_log(" %s %s,$0x%" TCG_PRIlx
",$%d", def
->name
,
1309 tcg_find_helper(s
, args
[nb_oargs
+ nb_iargs
]),
1310 args
[nb_oargs
+ nb_iargs
+ 1], nb_oargs
);
1311 for (i
= 0; i
< nb_oargs
; i
++) {
1312 col
+= qemu_log(",%s", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
),
1315 for (i
= 0; i
< nb_iargs
; i
++) {
1316 TCGArg arg
= args
[nb_oargs
+ i
];
1317 const char *t
= "<dummy>";
1318 if (arg
!= TCG_CALL_DUMMY_ARG
) {
1319 t
= tcg_get_arg_str_idx(s
, buf
, sizeof(buf
), arg
);
1321 col
+= qemu_log(",%s", t
);
1324 col
+= qemu_log(" %s ", def
->name
);
1326 nb_oargs
= def
->nb_oargs
;
1327 nb_iargs
= def
->nb_iargs
;
1328 nb_cargs
= def
->nb_cargs
;
1331 for (i
= 0; i
< nb_oargs
; i
++) {
1333 col
+= qemu_log(",");
1335 col
+= qemu_log("%s", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
),
1338 for (i
= 0; i
< nb_iargs
; i
++) {
1340 col
+= qemu_log(",");
1342 col
+= qemu_log("%s", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
),
1346 case INDEX_op_brcond_i32
:
1347 case INDEX_op_setcond_i32
:
1348 case INDEX_op_movcond_i32
:
1349 case INDEX_op_brcond2_i32
:
1350 case INDEX_op_setcond2_i32
:
1351 case INDEX_op_brcond_i64
:
1352 case INDEX_op_setcond_i64
:
1353 case INDEX_op_movcond_i64
:
1354 if (args
[k
] < ARRAY_SIZE(cond_name
) && cond_name
[args
[k
]]) {
1355 col
+= qemu_log(",%s", cond_name
[args
[k
++]]);
1357 col
+= qemu_log(",$0x%" TCG_PRIlx
, args
[k
++]);
1361 case INDEX_op_qemu_ld_i32
:
1362 case INDEX_op_qemu_st_i32
:
1363 case INDEX_op_qemu_ld_i64
:
1364 case INDEX_op_qemu_st_i64
:
1366 TCGMemOpIdx oi
= args
[k
++];
1367 TCGMemOp op
= get_memop(oi
);
1368 unsigned ix
= get_mmuidx(oi
);
1370 if (op
& ~(MO_AMASK
| MO_BSWAP
| MO_SSIZE
)) {
1371 col
+= qemu_log(",$0x%x,%u", op
, ix
);
1373 const char *s_al
, *s_op
;
1374 s_al
= alignment_name
[(op
& MO_AMASK
) >> MO_ASHIFT
];
1375 s_op
= ldst_name
[op
& (MO_BSWAP
| MO_SSIZE
)];
1376 col
+= qemu_log(",%s%s,%u", s_al
, s_op
, ix
);
1386 case INDEX_op_set_label
:
1388 case INDEX_op_brcond_i32
:
1389 case INDEX_op_brcond_i64
:
1390 case INDEX_op_brcond2_i32
:
1391 col
+= qemu_log("%s$L%d", k
? "," : "", arg_label(args
[k
])->id
);
1397 for (; i
< nb_cargs
; i
++, k
++) {
1398 col
+= qemu_log("%s$0x%" TCG_PRIlx
, k
? "," : "", args
[k
]);
1402 unsigned life
= op
->life
;
1404 for (; col
< 48; ++col
) {
1405 putc(' ', qemu_logfile
);
1408 if (life
& (SYNC_ARG
* 3)) {
1410 for (i
= 0; i
< 2; ++i
) {
1411 if (life
& (SYNC_ARG
<< i
)) {
1419 for (i
= 0; life
; ++i
, life
>>= 1) {
1430 /* we give more priority to constraints with less registers */
1431 static int get_constraint_priority(const TCGOpDef
*def
, int k
)
1433 const TCGArgConstraint
*arg_ct
;
1436 arg_ct
= &def
->args_ct
[k
];
1437 if (arg_ct
->ct
& TCG_CT_ALIAS
) {
1438 /* an alias is equivalent to a single register */
1441 if (!(arg_ct
->ct
& TCG_CT_REG
))
1444 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
1445 if (tcg_regset_test_reg(arg_ct
->u
.regs
, i
))
1449 return TCG_TARGET_NB_REGS
- n
+ 1;
1452 /* sort from highest priority to lowest */
1453 static void sort_constraints(TCGOpDef
*def
, int start
, int n
)
1455 int i
, j
, p1
, p2
, tmp
;
1457 for(i
= 0; i
< n
; i
++)
1458 def
->sorted_args
[start
+ i
] = start
+ i
;
1461 for(i
= 0; i
< n
- 1; i
++) {
1462 for(j
= i
+ 1; j
< n
; j
++) {
1463 p1
= get_constraint_priority(def
, def
->sorted_args
[start
+ i
]);
1464 p2
= get_constraint_priority(def
, def
->sorted_args
[start
+ j
]);
1466 tmp
= def
->sorted_args
[start
+ i
];
1467 def
->sorted_args
[start
+ i
] = def
->sorted_args
[start
+ j
];
1468 def
->sorted_args
[start
+ j
] = tmp
;
1474 static void process_op_defs(TCGContext
*s
)
1478 for (op
= 0; op
< NB_OPS
; op
++) {
1479 TCGOpDef
*def
= &tcg_op_defs
[op
];
1480 const TCGTargetOpDef
*tdefs
;
1484 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
1488 nb_args
= def
->nb_iargs
+ def
->nb_oargs
;
1493 tdefs
= tcg_target_op_def(op
);
1494 /* Missing TCGTargetOpDef entry. */
1495 tcg_debug_assert(tdefs
!= NULL
);
1497 type
= (def
->flags
& TCG_OPF_64BIT
? TCG_TYPE_I64
: TCG_TYPE_I32
);
1498 for (i
= 0; i
< nb_args
; i
++) {
1499 const char *ct_str
= tdefs
->args_ct_str
[i
];
1500 /* Incomplete TCGTargetOpDef entry. */
1501 tcg_debug_assert(ct_str
!= NULL
);
1503 def
->args_ct
[i
].u
.regs
= 0;
1504 def
->args_ct
[i
].ct
= 0;
1505 while (*ct_str
!= '\0') {
1509 int oarg
= *ct_str
- '0';
1510 tcg_debug_assert(ct_str
== tdefs
->args_ct_str
[i
]);
1511 tcg_debug_assert(oarg
< def
->nb_oargs
);
1512 tcg_debug_assert(def
->args_ct
[oarg
].ct
& TCG_CT_REG
);
1513 /* TCG_CT_ALIAS is for the output arguments.
1514 The input is tagged with TCG_CT_IALIAS. */
1515 def
->args_ct
[i
] = def
->args_ct
[oarg
];
1516 def
->args_ct
[oarg
].ct
|= TCG_CT_ALIAS
;
1517 def
->args_ct
[oarg
].alias_index
= i
;
1518 def
->args_ct
[i
].ct
|= TCG_CT_IALIAS
;
1519 def
->args_ct
[i
].alias_index
= oarg
;
1524 def
->args_ct
[i
].ct
|= TCG_CT_NEWREG
;
1528 def
->args_ct
[i
].ct
|= TCG_CT_CONST
;
1532 ct_str
= target_parse_constraint(&def
->args_ct
[i
],
1534 /* Typo in TCGTargetOpDef constraint. */
1535 tcg_debug_assert(ct_str
!= NULL
);
1540 /* TCGTargetOpDef entry with too much information? */
1541 tcg_debug_assert(i
== TCG_MAX_OP_ARGS
|| tdefs
->args_ct_str
[i
] == NULL
);
1543 /* sort the constraints (XXX: this is just an heuristic) */
1544 sort_constraints(def
, 0, def
->nb_oargs
);
1545 sort_constraints(def
, def
->nb_oargs
, def
->nb_iargs
);
1549 void tcg_op_remove(TCGContext
*s
, TCGOp
*op
)
1551 int next
= op
->next
;
1552 int prev
= op
->prev
;
1554 /* We should never attempt to remove the list terminator. */
1555 tcg_debug_assert(op
!= &s
->gen_op_buf
[0]);
1557 s
->gen_op_buf
[next
].prev
= prev
;
1558 s
->gen_op_buf
[prev
].next
= next
;
1560 memset(op
, 0, sizeof(*op
));
1562 #ifdef CONFIG_PROFILER
1567 TCGOp
*tcg_op_insert_before(TCGContext
*s
, TCGOp
*old_op
,
1568 TCGOpcode opc
, int nargs
)
1570 int oi
= s
->gen_next_op_idx
;
1571 int pi
= s
->gen_next_parm_idx
;
1572 int prev
= old_op
->prev
;
1573 int next
= old_op
- s
->gen_op_buf
;
1576 tcg_debug_assert(oi
< OPC_BUF_SIZE
);
1577 tcg_debug_assert(pi
+ nargs
<= OPPARAM_BUF_SIZE
);
1578 s
->gen_next_op_idx
= oi
+ 1;
1579 s
->gen_next_parm_idx
= pi
+ nargs
;
1581 new_op
= &s
->gen_op_buf
[oi
];
1588 s
->gen_op_buf
[prev
].next
= oi
;
1594 TCGOp
*tcg_op_insert_after(TCGContext
*s
, TCGOp
*old_op
,
1595 TCGOpcode opc
, int nargs
)
1597 int oi
= s
->gen_next_op_idx
;
1598 int pi
= s
->gen_next_parm_idx
;
1599 int prev
= old_op
- s
->gen_op_buf
;
1600 int next
= old_op
->next
;
1603 tcg_debug_assert(oi
< OPC_BUF_SIZE
);
1604 tcg_debug_assert(pi
+ nargs
<= OPPARAM_BUF_SIZE
);
1605 s
->gen_next_op_idx
= oi
+ 1;
1606 s
->gen_next_parm_idx
= pi
+ nargs
;
1608 new_op
= &s
->gen_op_buf
[oi
];
1615 s
->gen_op_buf
[next
].prev
= oi
;
1624 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
1625 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
1627 /* liveness analysis: end of function: all temps are dead, and globals
1628 should be in memory. */
1629 static inline void tcg_la_func_end(TCGContext
*s
, uint8_t *temp_state
)
1631 memset(temp_state
, TS_DEAD
| TS_MEM
, s
->nb_globals
);
1632 memset(temp_state
+ s
->nb_globals
, TS_DEAD
, s
->nb_temps
- s
->nb_globals
);
1635 /* liveness analysis: end of basic block: all temps are dead, globals
1636 and local temps should be in memory. */
1637 static inline void tcg_la_bb_end(TCGContext
*s
, uint8_t *temp_state
)
1641 tcg_la_func_end(s
, temp_state
);
1642 for (i
= s
->nb_globals
, n
= s
->nb_temps
; i
< n
; i
++) {
1643 if (s
->temps
[i
].temp_local
) {
1644 temp_state
[i
] |= TS_MEM
;
1649 /* Liveness analysis : update the opc_arg_life array to tell if a
1650 given input arguments is dead. Instructions updating dead
1651 temporaries are removed. */
1652 static void liveness_pass_1(TCGContext
*s
, uint8_t *temp_state
)
1654 int nb_globals
= s
->nb_globals
;
1657 tcg_la_func_end(s
, temp_state
);
1659 for (oi
= s
->gen_op_buf
[0].prev
; oi
!= 0; oi
= oi_prev
) {
1660 int i
, nb_iargs
, nb_oargs
;
1661 TCGOpcode opc_new
, opc_new2
;
1663 TCGLifeData arg_life
= 0;
1666 TCGOp
* const op
= &s
->gen_op_buf
[oi
];
1667 TCGArg
* const args
= &s
->gen_opparam_buf
[op
->args
];
1668 TCGOpcode opc
= op
->opc
;
1669 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
1678 nb_oargs
= op
->callo
;
1679 nb_iargs
= op
->calli
;
1680 call_flags
= args
[nb_oargs
+ nb_iargs
+ 1];
1682 /* pure functions can be removed if their result is unused */
1683 if (call_flags
& TCG_CALL_NO_SIDE_EFFECTS
) {
1684 for (i
= 0; i
< nb_oargs
; i
++) {
1686 if (temp_state
[arg
] != TS_DEAD
) {
1687 goto do_not_remove_call
;
1694 /* output args are dead */
1695 for (i
= 0; i
< nb_oargs
; i
++) {
1697 if (temp_state
[arg
] & TS_DEAD
) {
1698 arg_life
|= DEAD_ARG
<< i
;
1700 if (temp_state
[arg
] & TS_MEM
) {
1701 arg_life
|= SYNC_ARG
<< i
;
1703 temp_state
[arg
] = TS_DEAD
;
1706 if (!(call_flags
& (TCG_CALL_NO_WRITE_GLOBALS
|
1707 TCG_CALL_NO_READ_GLOBALS
))) {
1708 /* globals should go back to memory */
1709 memset(temp_state
, TS_DEAD
| TS_MEM
, nb_globals
);
1710 } else if (!(call_flags
& TCG_CALL_NO_READ_GLOBALS
)) {
1711 /* globals should be synced to memory */
1712 for (i
= 0; i
< nb_globals
; i
++) {
1713 temp_state
[i
] |= TS_MEM
;
1717 /* record arguments that die in this helper */
1718 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
1720 if (arg
!= TCG_CALL_DUMMY_ARG
) {
1721 if (temp_state
[arg
] & TS_DEAD
) {
1722 arg_life
|= DEAD_ARG
<< i
;
1726 /* input arguments are live for preceding opcodes */
1727 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
1729 if (arg
!= TCG_CALL_DUMMY_ARG
) {
1730 temp_state
[arg
] &= ~TS_DEAD
;
1736 case INDEX_op_insn_start
:
1738 case INDEX_op_discard
:
1739 /* mark the temporary as dead */
1740 temp_state
[args
[0]] = TS_DEAD
;
1743 case INDEX_op_add2_i32
:
1744 opc_new
= INDEX_op_add_i32
;
1746 case INDEX_op_sub2_i32
:
1747 opc_new
= INDEX_op_sub_i32
;
1749 case INDEX_op_add2_i64
:
1750 opc_new
= INDEX_op_add_i64
;
1752 case INDEX_op_sub2_i64
:
1753 opc_new
= INDEX_op_sub_i64
;
1757 /* Test if the high part of the operation is dead, but not
1758 the low part. The result can be optimized to a simple
1759 add or sub. This happens often for x86_64 guest when the
1760 cpu mode is set to 32 bit. */
1761 if (temp_state
[args
[1]] == TS_DEAD
) {
1762 if (temp_state
[args
[0]] == TS_DEAD
) {
1765 /* Replace the opcode and adjust the args in place,
1766 leaving 3 unused args at the end. */
1767 op
->opc
= opc
= opc_new
;
1770 /* Fall through and mark the single-word operation live. */
1776 case INDEX_op_mulu2_i32
:
1777 opc_new
= INDEX_op_mul_i32
;
1778 opc_new2
= INDEX_op_muluh_i32
;
1779 have_opc_new2
= TCG_TARGET_HAS_muluh_i32
;
1781 case INDEX_op_muls2_i32
:
1782 opc_new
= INDEX_op_mul_i32
;
1783 opc_new2
= INDEX_op_mulsh_i32
;
1784 have_opc_new2
= TCG_TARGET_HAS_mulsh_i32
;
1786 case INDEX_op_mulu2_i64
:
1787 opc_new
= INDEX_op_mul_i64
;
1788 opc_new2
= INDEX_op_muluh_i64
;
1789 have_opc_new2
= TCG_TARGET_HAS_muluh_i64
;
1791 case INDEX_op_muls2_i64
:
1792 opc_new
= INDEX_op_mul_i64
;
1793 opc_new2
= INDEX_op_mulsh_i64
;
1794 have_opc_new2
= TCG_TARGET_HAS_mulsh_i64
;
1799 if (temp_state
[args
[1]] == TS_DEAD
) {
1800 if (temp_state
[args
[0]] == TS_DEAD
) {
1801 /* Both parts of the operation are dead. */
1804 /* The high part of the operation is dead; generate the low. */
1805 op
->opc
= opc
= opc_new
;
1808 } else if (temp_state
[args
[0]] == TS_DEAD
&& have_opc_new2
) {
1809 /* The low part of the operation is dead; generate the high. */
1810 op
->opc
= opc
= opc_new2
;
1817 /* Mark the single-word operation live. */
1822 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
1823 nb_iargs
= def
->nb_iargs
;
1824 nb_oargs
= def
->nb_oargs
;
1826 /* Test if the operation can be removed because all
1827 its outputs are dead. We assume that nb_oargs == 0
1828 implies side effects */
1829 if (!(def
->flags
& TCG_OPF_SIDE_EFFECTS
) && nb_oargs
!= 0) {
1830 for (i
= 0; i
< nb_oargs
; i
++) {
1831 if (temp_state
[args
[i
]] != TS_DEAD
) {
1836 tcg_op_remove(s
, op
);
1839 /* output args are dead */
1840 for (i
= 0; i
< nb_oargs
; i
++) {
1842 if (temp_state
[arg
] & TS_DEAD
) {
1843 arg_life
|= DEAD_ARG
<< i
;
1845 if (temp_state
[arg
] & TS_MEM
) {
1846 arg_life
|= SYNC_ARG
<< i
;
1848 temp_state
[arg
] = TS_DEAD
;
1851 /* if end of basic block, update */
1852 if (def
->flags
& TCG_OPF_BB_END
) {
1853 tcg_la_bb_end(s
, temp_state
);
1854 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
1855 /* globals should be synced to memory */
1856 for (i
= 0; i
< nb_globals
; i
++) {
1857 temp_state
[i
] |= TS_MEM
;
1861 /* record arguments that die in this opcode */
1862 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
1864 if (temp_state
[arg
] & TS_DEAD
) {
1865 arg_life
|= DEAD_ARG
<< i
;
1868 /* input arguments are live for preceding opcodes */
1869 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
1870 temp_state
[args
[i
]] &= ~TS_DEAD
;
1875 op
->life
= arg_life
;
1879 /* Liveness analysis: Convert indirect regs to direct temporaries. */
1880 static bool liveness_pass_2(TCGContext
*s
, uint8_t *temp_state
)
1882 int nb_globals
= s
->nb_globals
;
1885 bool changes
= false;
1887 dir_temps
= tcg_malloc(nb_globals
* sizeof(int16_t));
1888 memset(dir_temps
, 0, nb_globals
* sizeof(int16_t));
1890 /* Create a temporary for each indirect global. */
1891 for (i
= 0; i
< nb_globals
; ++i
) {
1892 TCGTemp
*its
= &s
->temps
[i
];
1893 if (its
->indirect_reg
) {
1894 TCGTemp
*dts
= tcg_temp_alloc(s
);
1895 dts
->type
= its
->type
;
1896 dts
->base_type
= its
->base_type
;
1897 dir_temps
[i
] = temp_idx(s
, dts
);
1901 memset(temp_state
, TS_DEAD
, nb_globals
);
1903 for (oi
= s
->gen_op_buf
[0].next
; oi
!= 0; oi
= oi_next
) {
1904 TCGOp
*op
= &s
->gen_op_buf
[oi
];
1905 TCGArg
*args
= &s
->gen_opparam_buf
[op
->args
];
1906 TCGOpcode opc
= op
->opc
;
1907 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
1908 TCGLifeData arg_life
= op
->life
;
1909 int nb_iargs
, nb_oargs
, call_flags
;
1914 if (opc
== INDEX_op_call
) {
1915 nb_oargs
= op
->callo
;
1916 nb_iargs
= op
->calli
;
1917 call_flags
= args
[nb_oargs
+ nb_iargs
+ 1];
1919 nb_iargs
= def
->nb_iargs
;
1920 nb_oargs
= def
->nb_oargs
;
1922 /* Set flags similar to how calls require. */
1923 if (def
->flags
& TCG_OPF_BB_END
) {
1924 /* Like writing globals: save_globals */
1926 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
1927 /* Like reading globals: sync_globals */
1928 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
1930 /* No effect on globals. */
1931 call_flags
= (TCG_CALL_NO_READ_GLOBALS
|
1932 TCG_CALL_NO_WRITE_GLOBALS
);
1936 /* Make sure that input arguments are available. */
1937 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
1939 /* Note this unsigned test catches TCG_CALL_ARG_DUMMY too. */
1940 if (arg
< nb_globals
) {
1941 dir
= dir_temps
[arg
];
1942 if (dir
!= 0 && temp_state
[arg
] == TS_DEAD
) {
1943 TCGTemp
*its
= &s
->temps
[arg
];
1944 TCGOpcode lopc
= (its
->type
== TCG_TYPE_I32
1947 TCGOp
*lop
= tcg_op_insert_before(s
, op
, lopc
, 3);
1948 TCGArg
*largs
= &s
->gen_opparam_buf
[lop
->args
];
1951 largs
[1] = temp_idx(s
, its
->mem_base
);
1952 largs
[2] = its
->mem_offset
;
1954 /* Loaded, but synced with memory. */
1955 temp_state
[arg
] = TS_MEM
;
1960 /* Perform input replacement, and mark inputs that became dead.
1961 No action is required except keeping temp_state up to date
1962 so that we reload when needed. */
1963 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
1965 if (arg
< nb_globals
) {
1966 dir
= dir_temps
[arg
];
1970 if (IS_DEAD_ARG(i
)) {
1971 temp_state
[arg
] = TS_DEAD
;
1977 /* Liveness analysis should ensure that the following are
1978 all correct, for call sites and basic block end points. */
1979 if (call_flags
& TCG_CALL_NO_READ_GLOBALS
) {
1981 } else if (call_flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
1982 for (i
= 0; i
< nb_globals
; ++i
) {
1983 /* Liveness should see that globals are synced back,
1984 that is, either TS_DEAD or TS_MEM. */
1985 tcg_debug_assert(dir_temps
[i
] == 0
1986 || temp_state
[i
] != 0);
1989 for (i
= 0; i
< nb_globals
; ++i
) {
1990 /* Liveness should see that globals are saved back,
1991 that is, TS_DEAD, waiting to be reloaded. */
1992 tcg_debug_assert(dir_temps
[i
] == 0
1993 || temp_state
[i
] == TS_DEAD
);
1997 /* Outputs become available. */
1998 for (i
= 0; i
< nb_oargs
; i
++) {
2000 if (arg
>= nb_globals
) {
2003 dir
= dir_temps
[arg
];
2010 /* The output is now live and modified. */
2011 temp_state
[arg
] = 0;
2013 /* Sync outputs upon their last write. */
2014 if (NEED_SYNC_ARG(i
)) {
2015 TCGTemp
*its
= &s
->temps
[arg
];
2016 TCGOpcode sopc
= (its
->type
== TCG_TYPE_I32
2019 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
, 3);
2020 TCGArg
*sargs
= &s
->gen_opparam_buf
[sop
->args
];
2023 sargs
[1] = temp_idx(s
, its
->mem_base
);
2024 sargs
[2] = its
->mem_offset
;
2026 temp_state
[arg
] = TS_MEM
;
2028 /* Drop outputs that are dead. */
2029 if (IS_DEAD_ARG(i
)) {
2030 temp_state
[arg
] = TS_DEAD
;
2038 #ifdef CONFIG_DEBUG_TCG
2039 static void dump_regs(TCGContext
*s
)
2045 for(i
= 0; i
< s
->nb_temps
; i
++) {
2047 printf(" %10s: ", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
), i
));
2048 switch(ts
->val_type
) {
2050 printf("%s", tcg_target_reg_names
[ts
->reg
]);
2053 printf("%d(%s)", (int)ts
->mem_offset
,
2054 tcg_target_reg_names
[ts
->mem_base
->reg
]);
2056 case TEMP_VAL_CONST
:
2057 printf("$0x%" TCG_PRIlx
, ts
->val
);
2069 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2070 if (s
->reg_to_temp
[i
] != NULL
) {
2072 tcg_target_reg_names
[i
],
2073 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), s
->reg_to_temp
[i
]));
2078 static void check_regs(TCGContext
*s
)
2085 for (reg
= 0; reg
< TCG_TARGET_NB_REGS
; reg
++) {
2086 ts
= s
->reg_to_temp
[reg
];
2088 if (ts
->val_type
!= TEMP_VAL_REG
|| ts
->reg
!= reg
) {
2089 printf("Inconsistency for register %s:\n",
2090 tcg_target_reg_names
[reg
]);
2095 for (k
= 0; k
< s
->nb_temps
; k
++) {
2097 if (ts
->val_type
== TEMP_VAL_REG
&& !ts
->fixed_reg
2098 && s
->reg_to_temp
[ts
->reg
] != ts
) {
2099 printf("Inconsistency for temp %s:\n",
2100 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
2102 printf("reg state:\n");
2110 static void temp_allocate_frame(TCGContext
*s
, int temp
)
2113 ts
= &s
->temps
[temp
];
2114 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
2115 /* Sparc64 stack is accessed with offset of 2047 */
2116 s
->current_frame_offset
= (s
->current_frame_offset
+
2117 (tcg_target_long
)sizeof(tcg_target_long
) - 1) &
2118 ~(sizeof(tcg_target_long
) - 1);
2120 if (s
->current_frame_offset
+ (tcg_target_long
)sizeof(tcg_target_long
) >
2124 ts
->mem_offset
= s
->current_frame_offset
;
2125 ts
->mem_base
= s
->frame_temp
;
2126 ts
->mem_allocated
= 1;
2127 s
->current_frame_offset
+= sizeof(tcg_target_long
);
2130 static void temp_load(TCGContext
*, TCGTemp
*, TCGRegSet
, TCGRegSet
);
2132 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
2133 mark it free; otherwise mark it dead. */
2134 static void temp_free_or_dead(TCGContext
*s
, TCGTemp
*ts
, int free_or_dead
)
2136 if (ts
->fixed_reg
) {
2139 if (ts
->val_type
== TEMP_VAL_REG
) {
2140 s
->reg_to_temp
[ts
->reg
] = NULL
;
2142 ts
->val_type
= (free_or_dead
< 0
2144 || temp_idx(s
, ts
) < s
->nb_globals
2145 ? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
2148 /* Mark a temporary as dead. */
2149 static inline void temp_dead(TCGContext
*s
, TCGTemp
*ts
)
2151 temp_free_or_dead(s
, ts
, 1);
2154 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
2155 registers needs to be allocated to store a constant. If 'free_or_dead'
2156 is non-zero, subsequently release the temporary; if it is positive, the
2157 temp is dead; if it is negative, the temp is free. */
2158 static void temp_sync(TCGContext
*s
, TCGTemp
*ts
,
2159 TCGRegSet allocated_regs
, int free_or_dead
)
2161 if (ts
->fixed_reg
) {
2164 if (!ts
->mem_coherent
) {
2165 if (!ts
->mem_allocated
) {
2166 temp_allocate_frame(s
, temp_idx(s
, ts
));
2168 switch (ts
->val_type
) {
2169 case TEMP_VAL_CONST
:
2170 /* If we're going to free the temp immediately, then we won't
2171 require it later in a register, so attempt to store the
2172 constant to memory directly. */
2174 && tcg_out_sti(s
, ts
->type
, ts
->val
,
2175 ts
->mem_base
->reg
, ts
->mem_offset
)) {
2178 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
2183 tcg_out_st(s
, ts
->type
, ts
->reg
,
2184 ts
->mem_base
->reg
, ts
->mem_offset
);
2194 ts
->mem_coherent
= 1;
2197 temp_free_or_dead(s
, ts
, free_or_dead
);
2201 /* free register 'reg' by spilling the corresponding temporary if necessary */
2202 static void tcg_reg_free(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
2204 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
2206 temp_sync(s
, ts
, allocated_regs
, -1);
2210 /* Allocate a register belonging to reg1 & ~reg2 */
2211 static TCGReg
tcg_reg_alloc(TCGContext
*s
, TCGRegSet desired_regs
,
2212 TCGRegSet allocated_regs
, bool rev
)
2214 int i
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
2219 reg_ct
= desired_regs
& ~allocated_regs
;
2220 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
2222 /* first try free registers */
2223 for(i
= 0; i
< n
; i
++) {
2225 if (tcg_regset_test_reg(reg_ct
, reg
) && s
->reg_to_temp
[reg
] == NULL
)
2229 /* XXX: do better spill choice */
2230 for(i
= 0; i
< n
; i
++) {
2232 if (tcg_regset_test_reg(reg_ct
, reg
)) {
2233 tcg_reg_free(s
, reg
, allocated_regs
);
2241 /* Make sure the temporary is in a register. If needed, allocate the register
2242 from DESIRED while avoiding ALLOCATED. */
2243 static void temp_load(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet desired_regs
,
2244 TCGRegSet allocated_regs
)
2248 switch (ts
->val_type
) {
2251 case TEMP_VAL_CONST
:
2252 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
, ts
->indirect_base
);
2253 tcg_out_movi(s
, ts
->type
, reg
, ts
->val
);
2254 ts
->mem_coherent
= 0;
2257 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
, ts
->indirect_base
);
2258 tcg_out_ld(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
2259 ts
->mem_coherent
= 1;
2266 ts
->val_type
= TEMP_VAL_REG
;
2267 s
->reg_to_temp
[reg
] = ts
;
2270 /* Save a temporary to memory. 'allocated_regs' is used in case a
2271 temporary registers needs to be allocated to store a constant. */
2272 static void temp_save(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
)
2274 /* The liveness analysis already ensures that globals are back
2275 in memory. Keep an tcg_debug_assert for safety. */
2276 tcg_debug_assert(ts
->val_type
== TEMP_VAL_MEM
|| ts
->fixed_reg
);
2279 /* save globals to their canonical location and assume they can be
2280 modified be the following code. 'allocated_regs' is used in case a
2281 temporary registers needs to be allocated to store a constant. */
2282 static void save_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
2286 for (i
= 0; i
< s
->nb_globals
; i
++) {
2287 temp_save(s
, &s
->temps
[i
], allocated_regs
);
2291 /* sync globals to their canonical location and assume they can be
2292 read by the following code. 'allocated_regs' is used in case a
2293 temporary registers needs to be allocated to store a constant. */
2294 static void sync_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
2298 for (i
= 0; i
< s
->nb_globals
; i
++) {
2299 TCGTemp
*ts
= &s
->temps
[i
];
2300 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
2302 || ts
->mem_coherent
);
2306 /* at the end of a basic block, we assume all temporaries are dead and
2307 all globals are stored at their canonical location. */
2308 static void tcg_reg_alloc_bb_end(TCGContext
*s
, TCGRegSet allocated_regs
)
2312 for (i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
2313 TCGTemp
*ts
= &s
->temps
[i
];
2314 if (ts
->temp_local
) {
2315 temp_save(s
, ts
, allocated_regs
);
2317 /* The liveness analysis already ensures that temps are dead.
2318 Keep an tcg_debug_assert for safety. */
2319 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
2323 save_globals(s
, allocated_regs
);
2326 static void tcg_reg_alloc_do_movi(TCGContext
*s
, TCGTemp
*ots
,
2327 tcg_target_ulong val
, TCGLifeData arg_life
)
2329 if (ots
->fixed_reg
) {
2330 /* For fixed registers, we do not do any constant propagation. */
2331 tcg_out_movi(s
, ots
->type
, ots
->reg
, val
);
2335 /* The movi is not explicitly generated here. */
2336 if (ots
->val_type
== TEMP_VAL_REG
) {
2337 s
->reg_to_temp
[ots
->reg
] = NULL
;
2339 ots
->val_type
= TEMP_VAL_CONST
;
2341 ots
->mem_coherent
= 0;
2342 if (NEED_SYNC_ARG(0)) {
2343 temp_sync(s
, ots
, s
->reserved_regs
, IS_DEAD_ARG(0));
2344 } else if (IS_DEAD_ARG(0)) {
2349 static void tcg_reg_alloc_movi(TCGContext
*s
, const TCGArg
*args
,
2350 TCGLifeData arg_life
)
2352 TCGTemp
*ots
= &s
->temps
[args
[0]];
2353 tcg_target_ulong val
= args
[1];
2355 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
);
2358 static void tcg_reg_alloc_mov(TCGContext
*s
, const TCGOpDef
*def
,
2359 const TCGArg
*args
, TCGLifeData arg_life
)
2361 TCGRegSet allocated_regs
;
2363 TCGType otype
, itype
;
2365 allocated_regs
= s
->reserved_regs
;
2366 ots
= &s
->temps
[args
[0]];
2367 ts
= &s
->temps
[args
[1]];
2369 /* Note that otype != itype for no-op truncation. */
2373 if (ts
->val_type
== TEMP_VAL_CONST
) {
2374 /* propagate constant or generate sti */
2375 tcg_target_ulong val
= ts
->val
;
2376 if (IS_DEAD_ARG(1)) {
2379 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
);
2383 /* If the source value is in memory we're going to be forced
2384 to have it in a register in order to perform the copy. Copy
2385 the SOURCE value into its own register first, that way we
2386 don't have to reload SOURCE the next time it is used. */
2387 if (ts
->val_type
== TEMP_VAL_MEM
) {
2388 temp_load(s
, ts
, tcg_target_available_regs
[itype
], allocated_regs
);
2391 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
2392 if (IS_DEAD_ARG(0) && !ots
->fixed_reg
) {
2393 /* mov to a non-saved dead register makes no sense (even with
2394 liveness analysis disabled). */
2395 tcg_debug_assert(NEED_SYNC_ARG(0));
2396 if (!ots
->mem_allocated
) {
2397 temp_allocate_frame(s
, args
[0]);
2399 tcg_out_st(s
, otype
, ts
->reg
, ots
->mem_base
->reg
, ots
->mem_offset
);
2400 if (IS_DEAD_ARG(1)) {
2405 if (IS_DEAD_ARG(1) && !ts
->fixed_reg
&& !ots
->fixed_reg
) {
2406 /* the mov can be suppressed */
2407 if (ots
->val_type
== TEMP_VAL_REG
) {
2408 s
->reg_to_temp
[ots
->reg
] = NULL
;
2413 if (ots
->val_type
!= TEMP_VAL_REG
) {
2414 /* When allocating a new register, make sure to not spill the
2416 tcg_regset_set_reg(allocated_regs
, ts
->reg
);
2417 ots
->reg
= tcg_reg_alloc(s
, tcg_target_available_regs
[otype
],
2418 allocated_regs
, ots
->indirect_base
);
2420 tcg_out_mov(s
, otype
, ots
->reg
, ts
->reg
);
2422 ots
->val_type
= TEMP_VAL_REG
;
2423 ots
->mem_coherent
= 0;
2424 s
->reg_to_temp
[ots
->reg
] = ots
;
2425 if (NEED_SYNC_ARG(0)) {
2426 temp_sync(s
, ots
, allocated_regs
, 0);
2431 static void tcg_reg_alloc_op(TCGContext
*s
,
2432 const TCGOpDef
*def
, TCGOpcode opc
,
2433 const TCGArg
*args
, TCGLifeData arg_life
)
2435 TCGRegSet i_allocated_regs
;
2436 TCGRegSet o_allocated_regs
;
2437 int i
, k
, nb_iargs
, nb_oargs
;
2440 const TCGArgConstraint
*arg_ct
;
2442 TCGArg new_args
[TCG_MAX_OP_ARGS
];
2443 int const_args
[TCG_MAX_OP_ARGS
];
2445 nb_oargs
= def
->nb_oargs
;
2446 nb_iargs
= def
->nb_iargs
;
2448 /* copy constants */
2449 memcpy(new_args
+ nb_oargs
+ nb_iargs
,
2450 args
+ nb_oargs
+ nb_iargs
,
2451 sizeof(TCGArg
) * def
->nb_cargs
);
2453 i_allocated_regs
= s
->reserved_regs
;
2454 o_allocated_regs
= s
->reserved_regs
;
2456 /* satisfy input constraints */
2457 for(k
= 0; k
< nb_iargs
; k
++) {
2458 i
= def
->sorted_args
[nb_oargs
+ k
];
2460 arg_ct
= &def
->args_ct
[i
];
2461 ts
= &s
->temps
[arg
];
2463 if (ts
->val_type
== TEMP_VAL_CONST
2464 && tcg_target_const_match(ts
->val
, ts
->type
, arg_ct
)) {
2465 /* constant is OK for instruction */
2467 new_args
[i
] = ts
->val
;
2471 temp_load(s
, ts
, arg_ct
->u
.regs
, i_allocated_regs
);
2473 if (arg_ct
->ct
& TCG_CT_IALIAS
) {
2474 if (ts
->fixed_reg
) {
2475 /* if fixed register, we must allocate a new register
2476 if the alias is not the same register */
2477 if (arg
!= args
[arg_ct
->alias_index
])
2478 goto allocate_in_reg
;
2480 /* if the input is aliased to an output and if it is
2481 not dead after the instruction, we must allocate
2482 a new register and move it */
2483 if (!IS_DEAD_ARG(i
)) {
2484 goto allocate_in_reg
;
2486 /* check if the current register has already been allocated
2487 for another input aliased to an output */
2489 for (k2
= 0 ; k2
< k
; k2
++) {
2490 i2
= def
->sorted_args
[nb_oargs
+ k2
];
2491 if ((def
->args_ct
[i2
].ct
& TCG_CT_IALIAS
) &&
2492 (new_args
[i2
] == ts
->reg
)) {
2493 goto allocate_in_reg
;
2499 if (tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
2500 /* nothing to do : the constraint is satisfied */
2503 /* allocate a new register matching the constraint
2504 and move the temporary register into it */
2505 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, i_allocated_regs
,
2507 tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
);
2511 tcg_regset_set_reg(i_allocated_regs
, reg
);
2515 /* mark dead temporaries and free the associated registers */
2516 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2517 if (IS_DEAD_ARG(i
)) {
2518 temp_dead(s
, &s
->temps
[args
[i
]]);
2522 if (def
->flags
& TCG_OPF_BB_END
) {
2523 tcg_reg_alloc_bb_end(s
, i_allocated_regs
);
2525 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
2526 /* XXX: permit generic clobber register list ? */
2527 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2528 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
2529 tcg_reg_free(s
, i
, i_allocated_regs
);
2533 if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2534 /* sync globals if the op has side effects and might trigger
2536 sync_globals(s
, i_allocated_regs
);
2539 /* satisfy the output constraints */
2540 for(k
= 0; k
< nb_oargs
; k
++) {
2541 i
= def
->sorted_args
[k
];
2543 arg_ct
= &def
->args_ct
[i
];
2544 ts
= &s
->temps
[arg
];
2545 if ((arg_ct
->ct
& TCG_CT_ALIAS
)
2546 && !const_args
[arg_ct
->alias_index
]) {
2547 reg
= new_args
[arg_ct
->alias_index
];
2548 } else if (arg_ct
->ct
& TCG_CT_NEWREG
) {
2549 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
,
2550 i_allocated_regs
| o_allocated_regs
,
2553 /* if fixed register, we try to use it */
2555 if (ts
->fixed_reg
&&
2556 tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
2559 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, o_allocated_regs
,
2562 tcg_regset_set_reg(o_allocated_regs
, reg
);
2563 /* if a fixed register is used, then a move will be done afterwards */
2564 if (!ts
->fixed_reg
) {
2565 if (ts
->val_type
== TEMP_VAL_REG
) {
2566 s
->reg_to_temp
[ts
->reg
] = NULL
;
2568 ts
->val_type
= TEMP_VAL_REG
;
2570 /* temp value is modified, so the value kept in memory is
2571 potentially not the same */
2572 ts
->mem_coherent
= 0;
2573 s
->reg_to_temp
[reg
] = ts
;
2580 /* emit instruction */
2581 tcg_out_op(s
, opc
, new_args
, const_args
);
2583 /* move the outputs in the correct register if needed */
2584 for(i
= 0; i
< nb_oargs
; i
++) {
2585 ts
= &s
->temps
[args
[i
]];
2587 if (ts
->fixed_reg
&& ts
->reg
!= reg
) {
2588 tcg_out_mov(s
, ts
->type
, ts
->reg
, reg
);
2590 if (NEED_SYNC_ARG(i
)) {
2591 temp_sync(s
, ts
, o_allocated_regs
, IS_DEAD_ARG(i
));
2592 } else if (IS_DEAD_ARG(i
)) {
2598 #ifdef TCG_TARGET_STACK_GROWSUP
2599 #define STACK_DIR(x) (-(x))
2601 #define STACK_DIR(x) (x)
2604 static void tcg_reg_alloc_call(TCGContext
*s
, int nb_oargs
, int nb_iargs
,
2605 const TCGArg
* const args
, TCGLifeData arg_life
)
2607 int flags
, nb_regs
, i
;
2611 intptr_t stack_offset
;
2612 size_t call_stack_size
;
2613 tcg_insn_unit
*func_addr
;
2615 TCGRegSet allocated_regs
;
2617 func_addr
= (tcg_insn_unit
*)(intptr_t)args
[nb_oargs
+ nb_iargs
];
2618 flags
= args
[nb_oargs
+ nb_iargs
+ 1];
2620 nb_regs
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
2621 if (nb_regs
> nb_iargs
) {
2625 /* assign stack slots first */
2626 call_stack_size
= (nb_iargs
- nb_regs
) * sizeof(tcg_target_long
);
2627 call_stack_size
= (call_stack_size
+ TCG_TARGET_STACK_ALIGN
- 1) &
2628 ~(TCG_TARGET_STACK_ALIGN
- 1);
2629 allocate_args
= (call_stack_size
> TCG_STATIC_CALL_ARGS_SIZE
);
2630 if (allocate_args
) {
2631 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
2632 preallocate call stack */
2636 stack_offset
= TCG_TARGET_CALL_STACK_OFFSET
;
2637 for(i
= nb_regs
; i
< nb_iargs
; i
++) {
2638 arg
= args
[nb_oargs
+ i
];
2639 #ifdef TCG_TARGET_STACK_GROWSUP
2640 stack_offset
-= sizeof(tcg_target_long
);
2642 if (arg
!= TCG_CALL_DUMMY_ARG
) {
2643 ts
= &s
->temps
[arg
];
2644 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
2646 tcg_out_st(s
, ts
->type
, ts
->reg
, TCG_REG_CALL_STACK
, stack_offset
);
2648 #ifndef TCG_TARGET_STACK_GROWSUP
2649 stack_offset
+= sizeof(tcg_target_long
);
2653 /* assign input registers */
2654 allocated_regs
= s
->reserved_regs
;
2655 for(i
= 0; i
< nb_regs
; i
++) {
2656 arg
= args
[nb_oargs
+ i
];
2657 if (arg
!= TCG_CALL_DUMMY_ARG
) {
2658 ts
= &s
->temps
[arg
];
2659 reg
= tcg_target_call_iarg_regs
[i
];
2660 tcg_reg_free(s
, reg
, allocated_regs
);
2662 if (ts
->val_type
== TEMP_VAL_REG
) {
2663 if (ts
->reg
!= reg
) {
2664 tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
);
2667 TCGRegSet arg_set
= 0;
2669 tcg_regset_set_reg(arg_set
, reg
);
2670 temp_load(s
, ts
, arg_set
, allocated_regs
);
2673 tcg_regset_set_reg(allocated_regs
, reg
);
2677 /* mark dead temporaries and free the associated registers */
2678 for(i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2679 if (IS_DEAD_ARG(i
)) {
2680 temp_dead(s
, &s
->temps
[args
[i
]]);
2684 /* clobber call registers */
2685 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2686 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
2687 tcg_reg_free(s
, i
, allocated_regs
);
2691 /* Save globals if they might be written by the helper, sync them if
2692 they might be read. */
2693 if (flags
& TCG_CALL_NO_READ_GLOBALS
) {
2695 } else if (flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
2696 sync_globals(s
, allocated_regs
);
2698 save_globals(s
, allocated_regs
);
2701 tcg_out_call(s
, func_addr
);
2703 /* assign output registers and emit moves if needed */
2704 for(i
= 0; i
< nb_oargs
; i
++) {
2706 ts
= &s
->temps
[arg
];
2707 reg
= tcg_target_call_oarg_regs
[i
];
2708 tcg_debug_assert(s
->reg_to_temp
[reg
] == NULL
);
2710 if (ts
->fixed_reg
) {
2711 if (ts
->reg
!= reg
) {
2712 tcg_out_mov(s
, ts
->type
, ts
->reg
, reg
);
2715 if (ts
->val_type
== TEMP_VAL_REG
) {
2716 s
->reg_to_temp
[ts
->reg
] = NULL
;
2718 ts
->val_type
= TEMP_VAL_REG
;
2720 ts
->mem_coherent
= 0;
2721 s
->reg_to_temp
[reg
] = ts
;
2722 if (NEED_SYNC_ARG(i
)) {
2723 temp_sync(s
, ts
, allocated_regs
, IS_DEAD_ARG(i
));
2724 } else if (IS_DEAD_ARG(i
)) {
2731 #ifdef CONFIG_PROFILER
2733 static int64_t tcg_table_op_count
[NB_OPS
];
2735 void tcg_dump_op_count(FILE *f
, fprintf_function cpu_fprintf
)
2739 for (i
= 0; i
< NB_OPS
; i
++) {
2740 cpu_fprintf(f
, "%s %" PRId64
"\n", tcg_op_defs
[i
].name
,
2741 tcg_table_op_count
[i
]);
2745 void tcg_dump_op_count(FILE *f
, fprintf_function cpu_fprintf
)
2747 cpu_fprintf(f
, "[TCG profiler not compiled]\n");
2752 int tcg_gen_code(TCGContext
*s
, TranslationBlock
*tb
)
2754 int i
, oi
, oi_next
, num_insns
;
2756 #ifdef CONFIG_PROFILER
2760 n
= s
->gen_op_buf
[0].prev
+ 1;
2762 if (n
> s
->op_count_max
) {
2763 s
->op_count_max
= n
;
2768 if (n
> s
->temp_count_max
) {
2769 s
->temp_count_max
= n
;
2775 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP
)
2776 && qemu_log_in_addr_range(tb
->pc
))) {
2785 #ifdef CONFIG_PROFILER
2786 s
->opt_time
-= profile_getclock();
2789 #ifdef USE_TCG_OPTIMIZATIONS
2793 #ifdef CONFIG_PROFILER
2794 s
->opt_time
+= profile_getclock();
2795 s
->la_time
-= profile_getclock();
2799 uint8_t *temp_state
= tcg_malloc(s
->nb_temps
+ s
->nb_indirects
);
2801 liveness_pass_1(s
, temp_state
);
2803 if (s
->nb_indirects
> 0) {
2805 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND
)
2806 && qemu_log_in_addr_range(tb
->pc
))) {
2808 qemu_log("OP before indirect lowering:\n");
2814 /* Replace indirect temps with direct temps. */
2815 if (liveness_pass_2(s
, temp_state
)) {
2816 /* If changes were made, re-run liveness. */
2817 liveness_pass_1(s
, temp_state
);
2822 #ifdef CONFIG_PROFILER
2823 s
->la_time
+= profile_getclock();
2827 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT
)
2828 && qemu_log_in_addr_range(tb
->pc
))) {
2830 qemu_log("OP after optimization and liveness analysis:\n");
2837 tcg_reg_alloc_start(s
);
2839 s
->code_buf
= tb
->tc_ptr
;
2840 s
->code_ptr
= tb
->tc_ptr
;
2842 #ifdef TCG_TARGET_NEED_LDST_LABELS
2843 s
->ldst_labels
= NULL
;
2845 #ifdef TCG_TARGET_NEED_POOL_LABELS
2846 s
->pool_labels
= NULL
;
2850 for (oi
= s
->gen_op_buf
[0].next
; oi
!= 0; oi
= oi_next
) {
2851 TCGOp
* const op
= &s
->gen_op_buf
[oi
];
2852 TCGArg
* const args
= &s
->gen_opparam_buf
[op
->args
];
2853 TCGOpcode opc
= op
->opc
;
2854 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2855 TCGLifeData arg_life
= op
->life
;
2858 #ifdef CONFIG_PROFILER
2859 tcg_table_op_count
[opc
]++;
2863 case INDEX_op_mov_i32
:
2864 case INDEX_op_mov_i64
:
2865 tcg_reg_alloc_mov(s
, def
, args
, arg_life
);
2867 case INDEX_op_movi_i32
:
2868 case INDEX_op_movi_i64
:
2869 tcg_reg_alloc_movi(s
, args
, arg_life
);
2871 case INDEX_op_insn_start
:
2872 if (num_insns
>= 0) {
2873 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
2876 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
2878 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2879 a
= ((target_ulong
)args
[i
* 2 + 1] << 32) | args
[i
* 2];
2883 s
->gen_insn_data
[num_insns
][i
] = a
;
2886 case INDEX_op_discard
:
2887 temp_dead(s
, &s
->temps
[args
[0]]);
2889 case INDEX_op_set_label
:
2890 tcg_reg_alloc_bb_end(s
, s
->reserved_regs
);
2891 tcg_out_label(s
, arg_label(args
[0]), s
->code_ptr
);
2894 tcg_reg_alloc_call(s
, op
->callo
, op
->calli
, args
, arg_life
);
2897 /* Sanity check that we've not introduced any unhandled opcodes. */
2898 tcg_debug_assert(tcg_op_supported(opc
));
2899 /* Note: in order to speed up the code, it would be much
2900 faster to have specialized register allocator functions for
2901 some common argument patterns */
2902 tcg_reg_alloc_op(s
, def
, opc
, args
, arg_life
);
2905 #ifdef CONFIG_DEBUG_TCG
2908 /* Test for (pending) buffer overflow. The assumption is that any
2909 one operation beginning below the high water mark cannot overrun
2910 the buffer completely. Thus we can test for overflow after
2911 generating code without having to check during generation. */
2912 if (unlikely((void *)s
->code_ptr
> s
->code_gen_highwater
)) {
2916 tcg_debug_assert(num_insns
>= 0);
2917 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
2919 /* Generate TB finalization at the end of block */
2920 #ifdef TCG_TARGET_NEED_LDST_LABELS
2921 if (!tcg_out_ldst_finalize(s
)) {
2925 #ifdef TCG_TARGET_NEED_POOL_LABELS
2926 if (!tcg_out_pool_finalize(s
)) {
2931 /* flush instruction cache */
2932 flush_icache_range((uintptr_t)s
->code_buf
, (uintptr_t)s
->code_ptr
);
2934 return tcg_current_code_size(s
);
2937 #ifdef CONFIG_PROFILER
2938 void tcg_dump_info(FILE *f
, fprintf_function cpu_fprintf
)
2940 TCGContext
*s
= &tcg_ctx
;
2941 int64_t tb_count
= s
->tb_count
;
2942 int64_t tb_div_count
= tb_count
? tb_count
: 1;
2943 int64_t tot
= s
->interm_time
+ s
->code_time
;
2945 cpu_fprintf(f
, "JIT cycles %" PRId64
" (%0.3f s at 2.4 GHz)\n",
2947 cpu_fprintf(f
, "translated TBs %" PRId64
" (aborted=%" PRId64
" %0.1f%%)\n",
2948 tb_count
, s
->tb_count1
- tb_count
,
2949 (double)(s
->tb_count1
- s
->tb_count
)
2950 / (s
->tb_count1
? s
->tb_count1
: 1) * 100.0);
2951 cpu_fprintf(f
, "avg ops/TB %0.1f max=%d\n",
2952 (double)s
->op_count
/ tb_div_count
, s
->op_count_max
);
2953 cpu_fprintf(f
, "deleted ops/TB %0.2f\n",
2954 (double)s
->del_op_count
/ tb_div_count
);
2955 cpu_fprintf(f
, "avg temps/TB %0.2f max=%d\n",
2956 (double)s
->temp_count
/ tb_div_count
, s
->temp_count_max
);
2957 cpu_fprintf(f
, "avg host code/TB %0.1f\n",
2958 (double)s
->code_out_len
/ tb_div_count
);
2959 cpu_fprintf(f
, "avg search data/TB %0.1f\n",
2960 (double)s
->search_out_len
/ tb_div_count
);
2962 cpu_fprintf(f
, "cycles/op %0.1f\n",
2963 s
->op_count
? (double)tot
/ s
->op_count
: 0);
2964 cpu_fprintf(f
, "cycles/in byte %0.1f\n",
2965 s
->code_in_len
? (double)tot
/ s
->code_in_len
: 0);
2966 cpu_fprintf(f
, "cycles/out byte %0.1f\n",
2967 s
->code_out_len
? (double)tot
/ s
->code_out_len
: 0);
2968 cpu_fprintf(f
, "cycles/search byte %0.1f\n",
2969 s
->search_out_len
? (double)tot
/ s
->search_out_len
: 0);
2973 cpu_fprintf(f
, " gen_interm time %0.1f%%\n",
2974 (double)s
->interm_time
/ tot
* 100.0);
2975 cpu_fprintf(f
, " gen_code time %0.1f%%\n",
2976 (double)s
->code_time
/ tot
* 100.0);
2977 cpu_fprintf(f
, "optim./code time %0.1f%%\n",
2978 (double)s
->opt_time
/ (s
->code_time
? s
->code_time
: 1)
2980 cpu_fprintf(f
, "liveness/code time %0.1f%%\n",
2981 (double)s
->la_time
/ (s
->code_time
? s
->code_time
: 1) * 100.0);
2982 cpu_fprintf(f
, "cpu_restore count %" PRId64
"\n",
2984 cpu_fprintf(f
, " avg cycles %0.1f\n",
2985 s
->restore_count
? (double)s
->restore_time
/ s
->restore_count
: 0);
2988 void tcg_dump_info(FILE *f
, fprintf_function cpu_fprintf
)
2990 cpu_fprintf(f
, "[TCG profiler not compiled]\n");
2994 #ifdef ELF_HOST_MACHINE
2995 /* In order to use this feature, the backend needs to do three things:
2997 (1) Define ELF_HOST_MACHINE to indicate both what value to
2998 put into the ELF image and to indicate support for the feature.
3000 (2) Define tcg_register_jit. This should create a buffer containing
3001 the contents of a .debug_frame section that describes the post-
3002 prologue unwind info for the tcg machine.
3004 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
3007 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
3014 struct jit_code_entry
{
3015 struct jit_code_entry
*next_entry
;
3016 struct jit_code_entry
*prev_entry
;
3017 const void *symfile_addr
;
3018 uint64_t symfile_size
;
3021 struct jit_descriptor
{
3023 uint32_t action_flag
;
3024 struct jit_code_entry
*relevant_entry
;
3025 struct jit_code_entry
*first_entry
;
3028 void __jit_debug_register_code(void) __attribute__((noinline
));
3029 void __jit_debug_register_code(void)
3034 /* Must statically initialize the version, because GDB may check
3035 the version before we can set it. */
3036 struct jit_descriptor __jit_debug_descriptor
= { 1, 0, 0, 0 };
3038 /* End GDB interface. */
3040 static int find_string(const char *strtab
, const char *str
)
3042 const char *p
= strtab
+ 1;
3045 if (strcmp(p
, str
) == 0) {
3052 static void tcg_register_jit_int(void *buf_ptr
, size_t buf_size
,
3053 const void *debug_frame
,
3054 size_t debug_frame_size
)
3056 struct __attribute__((packed
)) DebugInfo
{
3063 uintptr_t cu_low_pc
;
3064 uintptr_t cu_high_pc
;
3067 uintptr_t fn_low_pc
;
3068 uintptr_t fn_high_pc
;
3077 struct DebugInfo di
;
3082 struct ElfImage
*img
;
3084 static const struct ElfImage img_template
= {
3086 .e_ident
[EI_MAG0
] = ELFMAG0
,
3087 .e_ident
[EI_MAG1
] = ELFMAG1
,
3088 .e_ident
[EI_MAG2
] = ELFMAG2
,
3089 .e_ident
[EI_MAG3
] = ELFMAG3
,
3090 .e_ident
[EI_CLASS
] = ELF_CLASS
,
3091 .e_ident
[EI_DATA
] = ELF_DATA
,
3092 .e_ident
[EI_VERSION
] = EV_CURRENT
,
3094 .e_machine
= ELF_HOST_MACHINE
,
3095 .e_version
= EV_CURRENT
,
3096 .e_phoff
= offsetof(struct ElfImage
, phdr
),
3097 .e_shoff
= offsetof(struct ElfImage
, shdr
),
3098 .e_ehsize
= sizeof(ElfW(Shdr
)),
3099 .e_phentsize
= sizeof(ElfW(Phdr
)),
3101 .e_shentsize
= sizeof(ElfW(Shdr
)),
3102 .e_shnum
= ARRAY_SIZE(img
->shdr
),
3103 .e_shstrndx
= ARRAY_SIZE(img
->shdr
) - 1,
3104 #ifdef ELF_HOST_FLAGS
3105 .e_flags
= ELF_HOST_FLAGS
,
3108 .e_ident
[EI_OSABI
] = ELF_OSABI
,
3116 [0] = { .sh_type
= SHT_NULL
},
3117 /* Trick: The contents of code_gen_buffer are not present in
3118 this fake ELF file; that got allocated elsewhere. Therefore
3119 we mark .text as SHT_NOBITS (similar to .bss) so that readers
3120 will not look for contents. We can record any address. */
3122 .sh_type
= SHT_NOBITS
,
3123 .sh_flags
= SHF_EXECINSTR
| SHF_ALLOC
,
3125 [2] = { /* .debug_info */
3126 .sh_type
= SHT_PROGBITS
,
3127 .sh_offset
= offsetof(struct ElfImage
, di
),
3128 .sh_size
= sizeof(struct DebugInfo
),
3130 [3] = { /* .debug_abbrev */
3131 .sh_type
= SHT_PROGBITS
,
3132 .sh_offset
= offsetof(struct ElfImage
, da
),
3133 .sh_size
= sizeof(img
->da
),
3135 [4] = { /* .debug_frame */
3136 .sh_type
= SHT_PROGBITS
,
3137 .sh_offset
= sizeof(struct ElfImage
),
3139 [5] = { /* .symtab */
3140 .sh_type
= SHT_SYMTAB
,
3141 .sh_offset
= offsetof(struct ElfImage
, sym
),
3142 .sh_size
= sizeof(img
->sym
),
3144 .sh_link
= ARRAY_SIZE(img
->shdr
) - 1,
3145 .sh_entsize
= sizeof(ElfW(Sym
)),
3147 [6] = { /* .strtab */
3148 .sh_type
= SHT_STRTAB
,
3149 .sh_offset
= offsetof(struct ElfImage
, str
),
3150 .sh_size
= sizeof(img
->str
),
3154 [1] = { /* code_gen_buffer */
3155 .st_info
= ELF_ST_INFO(STB_GLOBAL
, STT_FUNC
),
3160 .len
= sizeof(struct DebugInfo
) - 4,
3162 .ptr_size
= sizeof(void *),
3164 .cu_lang
= 0x8001, /* DW_LANG_Mips_Assembler */
3166 .fn_name
= "code_gen_buffer"
3169 1, /* abbrev number (the cu) */
3170 0x11, 1, /* DW_TAG_compile_unit, has children */
3171 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
3172 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
3173 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
3174 0, 0, /* end of abbrev */
3175 2, /* abbrev number (the fn) */
3176 0x2e, 0, /* DW_TAG_subprogram, no children */
3177 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
3178 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
3179 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
3180 0, 0, /* end of abbrev */
3181 0 /* no more abbrev */
3183 .str
= "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
3184 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
3187 /* We only need a single jit entry; statically allocate it. */
3188 static struct jit_code_entry one_entry
;
3190 uintptr_t buf
= (uintptr_t)buf_ptr
;
3191 size_t img_size
= sizeof(struct ElfImage
) + debug_frame_size
;
3192 DebugFrameHeader
*dfh
;
3194 img
= g_malloc(img_size
);
3195 *img
= img_template
;
3197 img
->phdr
.p_vaddr
= buf
;
3198 img
->phdr
.p_paddr
= buf
;
3199 img
->phdr
.p_memsz
= buf_size
;
3201 img
->shdr
[1].sh_name
= find_string(img
->str
, ".text");
3202 img
->shdr
[1].sh_addr
= buf
;
3203 img
->shdr
[1].sh_size
= buf_size
;
3205 img
->shdr
[2].sh_name
= find_string(img
->str
, ".debug_info");
3206 img
->shdr
[3].sh_name
= find_string(img
->str
, ".debug_abbrev");
3208 img
->shdr
[4].sh_name
= find_string(img
->str
, ".debug_frame");
3209 img
->shdr
[4].sh_size
= debug_frame_size
;
3211 img
->shdr
[5].sh_name
= find_string(img
->str
, ".symtab");
3212 img
->shdr
[6].sh_name
= find_string(img
->str
, ".strtab");
3214 img
->sym
[1].st_name
= find_string(img
->str
, "code_gen_buffer");
3215 img
->sym
[1].st_value
= buf
;
3216 img
->sym
[1].st_size
= buf_size
;
3218 img
->di
.cu_low_pc
= buf
;
3219 img
->di
.cu_high_pc
= buf
+ buf_size
;
3220 img
->di
.fn_low_pc
= buf
;
3221 img
->di
.fn_high_pc
= buf
+ buf_size
;
3223 dfh
= (DebugFrameHeader
*)(img
+ 1);
3224 memcpy(dfh
, debug_frame
, debug_frame_size
);
3225 dfh
->fde
.func_start
= buf
;
3226 dfh
->fde
.func_len
= buf_size
;
3229 /* Enable this block to be able to debug the ELF image file creation.
3230 One can use readelf, objdump, or other inspection utilities. */
3232 FILE *f
= fopen("/tmp/qemu.jit", "w+b");
3234 if (fwrite(img
, img_size
, 1, f
) != img_size
) {
3235 /* Avoid stupid unused return value warning for fwrite. */
3242 one_entry
.symfile_addr
= img
;
3243 one_entry
.symfile_size
= img_size
;
3245 __jit_debug_descriptor
.action_flag
= JIT_REGISTER_FN
;
3246 __jit_debug_descriptor
.relevant_entry
= &one_entry
;
3247 __jit_debug_descriptor
.first_entry
= &one_entry
;
3248 __jit_debug_register_code();
3251 /* No support for the feature. Provide the entry point expected by exec.c,
3252 and implement the internal function we declared earlier. */
3254 static void tcg_register_jit_int(void *buf
, size_t size
,
3255 const void *debug_frame
,
3256 size_t debug_frame_size
)
3260 void tcg_register_jit(void *buf
, size_t buf_size
)
3263 #endif /* ELF_HOST_MACHINE */