2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 /* define it to use liveness analysis (better code) */
26 #define USE_LIVENESS_ANALYSIS
27 #define USE_TCG_OPTIMIZATIONS
29 #include "qemu/osdep.h"
31 /* Define to jump the ELF file used to communicate with GDB. */
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/timer.h"
38 /* Note: the long term plan is to reduce the dependencies on the QEMU
39 CPU definitions. Currently they are used for qemu_ld/st
41 #define NO_CPU_IO_DEFS
46 #if UINTPTR_MAX == UINT32_MAX
47 # define ELF_CLASS ELFCLASS32
49 # define ELF_CLASS ELFCLASS64
51 #ifdef HOST_WORDS_BIGENDIAN
52 # define ELF_DATA ELFDATA2MSB
54 # define ELF_DATA ELFDATA2LSB
60 /* Forward declarations for functions declared in tcg-target.inc.c and
62 static void tcg_target_init(TCGContext
*s
);
63 static void tcg_target_qemu_prologue(TCGContext
*s
);
64 static void patch_reloc(tcg_insn_unit
*code_ptr
, int type
,
65 intptr_t value
, intptr_t addend
);
67 /* The CIE and FDE header definitions will be common to all hosts. */
69 uint32_t len
__attribute__((aligned((sizeof(void *)))));
75 uint8_t return_column
;
78 typedef struct QEMU_PACKED
{
79 uint32_t len
__attribute__((aligned((sizeof(void *)))));
83 } DebugFrameFDEHeader
;
85 typedef struct QEMU_PACKED
{
87 DebugFrameFDEHeader fde
;
90 static void tcg_register_jit_int(void *buf
, size_t size
,
91 const void *debug_frame
,
92 size_t debug_frame_size
)
93 __attribute__((unused
));
95 /* Forward declarations for functions declared and used in tcg-target.inc.c. */
96 static int target_parse_constraint(TCGArgConstraint
*ct
, const char **pct_str
);
97 static void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg1
,
99 static void tcg_out_mov(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg
);
100 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
101 TCGReg ret
, tcg_target_long arg
);
102 static void tcg_out_op(TCGContext
*s
, TCGOpcode opc
, const TCGArg
*args
,
103 const int *const_args
);
104 static void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
, TCGReg arg1
,
106 static void tcg_out_call(TCGContext
*s
, tcg_insn_unit
*target
);
107 static int tcg_target_const_match(tcg_target_long val
, TCGType type
,
108 const TCGArgConstraint
*arg_ct
);
109 static void tcg_out_tb_init(TCGContext
*s
);
110 static bool tcg_out_tb_finalize(TCGContext
*s
);
113 static TCGRegSet tcg_target_available_regs
[2];
114 static TCGRegSet tcg_target_call_clobber_regs
;
116 #if TCG_TARGET_INSN_UNIT_SIZE == 1
117 static __attribute__((unused
)) inline void tcg_out8(TCGContext
*s
, uint8_t v
)
122 static __attribute__((unused
)) inline void tcg_patch8(tcg_insn_unit
*p
,
129 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
130 static __attribute__((unused
)) inline void tcg_out16(TCGContext
*s
, uint16_t v
)
132 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
135 tcg_insn_unit
*p
= s
->code_ptr
;
136 memcpy(p
, &v
, sizeof(v
));
137 s
->code_ptr
= p
+ (2 / TCG_TARGET_INSN_UNIT_SIZE
);
141 static __attribute__((unused
)) inline void tcg_patch16(tcg_insn_unit
*p
,
144 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
147 memcpy(p
, &v
, sizeof(v
));
152 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
153 static __attribute__((unused
)) inline void tcg_out32(TCGContext
*s
, uint32_t v
)
155 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
158 tcg_insn_unit
*p
= s
->code_ptr
;
159 memcpy(p
, &v
, sizeof(v
));
160 s
->code_ptr
= p
+ (4 / TCG_TARGET_INSN_UNIT_SIZE
);
164 static __attribute__((unused
)) inline void tcg_patch32(tcg_insn_unit
*p
,
167 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
170 memcpy(p
, &v
, sizeof(v
));
175 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
176 static __attribute__((unused
)) inline void tcg_out64(TCGContext
*s
, uint64_t v
)
178 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
181 tcg_insn_unit
*p
= s
->code_ptr
;
182 memcpy(p
, &v
, sizeof(v
));
183 s
->code_ptr
= p
+ (8 / TCG_TARGET_INSN_UNIT_SIZE
);
187 static __attribute__((unused
)) inline void tcg_patch64(tcg_insn_unit
*p
,
190 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
193 memcpy(p
, &v
, sizeof(v
));
198 /* label relocation processing */
200 static void tcg_out_reloc(TCGContext
*s
, tcg_insn_unit
*code_ptr
, int type
,
201 TCGLabel
*l
, intptr_t addend
)
206 /* FIXME: This may break relocations on RISC targets that
207 modify instruction fields in place. The caller may not have
208 written the initial value. */
209 patch_reloc(code_ptr
, type
, l
->u
.value
, addend
);
211 /* add a new relocation entry */
212 r
= tcg_malloc(sizeof(TCGRelocation
));
216 r
->next
= l
->u
.first_reloc
;
217 l
->u
.first_reloc
= r
;
221 static void tcg_out_label(TCGContext
*s
, TCGLabel
*l
, tcg_insn_unit
*ptr
)
223 intptr_t value
= (intptr_t)ptr
;
226 tcg_debug_assert(!l
->has_value
);
228 for (r
= l
->u
.first_reloc
; r
!= NULL
; r
= r
->next
) {
229 patch_reloc(r
->ptr
, r
->type
, value
, r
->addend
);
233 l
->u
.value_ptr
= ptr
;
236 TCGLabel
*gen_new_label(void)
238 TCGContext
*s
= &tcg_ctx
;
239 TCGLabel
*l
= tcg_malloc(sizeof(TCGLabel
));
248 #include "tcg-target.inc.c"
250 /* pool based memory allocation */
251 void *tcg_malloc_internal(TCGContext
*s
, int size
)
256 if (size
> TCG_POOL_CHUNK_SIZE
) {
257 /* big malloc: insert a new pool (XXX: could optimize) */
258 p
= g_malloc(sizeof(TCGPool
) + size
);
260 p
->next
= s
->pool_first_large
;
261 s
->pool_first_large
= p
;
272 pool_size
= TCG_POOL_CHUNK_SIZE
;
273 p
= g_malloc(sizeof(TCGPool
) + pool_size
);
277 s
->pool_current
->next
= p
;
286 s
->pool_cur
= p
->data
+ size
;
287 s
->pool_end
= p
->data
+ p
->size
;
291 void tcg_pool_reset(TCGContext
*s
)
294 for (p
= s
->pool_first_large
; p
; p
= t
) {
298 s
->pool_first_large
= NULL
;
299 s
->pool_cur
= s
->pool_end
= NULL
;
300 s
->pool_current
= NULL
;
303 typedef struct TCGHelperInfo
{
310 #include "exec/helper-proto.h"
312 static const TCGHelperInfo all_helpers
[] = {
313 #include "exec/helper-tcg.h"
316 static int indirect_reg_alloc_order
[ARRAY_SIZE(tcg_target_reg_alloc_order
)];
318 void tcg_context_init(TCGContext
*s
)
320 int op
, total_args
, n
, i
;
322 TCGArgConstraint
*args_ct
;
324 GHashTable
*helper_table
;
326 memset(s
, 0, sizeof(*s
));
329 /* Count total number of arguments and allocate the corresponding
332 for(op
= 0; op
< NB_OPS
; op
++) {
333 def
= &tcg_op_defs
[op
];
334 n
= def
->nb_iargs
+ def
->nb_oargs
;
338 args_ct
= g_malloc(sizeof(TCGArgConstraint
) * total_args
);
339 sorted_args
= g_malloc(sizeof(int) * total_args
);
341 for(op
= 0; op
< NB_OPS
; op
++) {
342 def
= &tcg_op_defs
[op
];
343 def
->args_ct
= args_ct
;
344 def
->sorted_args
= sorted_args
;
345 n
= def
->nb_iargs
+ def
->nb_oargs
;
350 /* Register helpers. */
351 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
352 s
->helpers
= helper_table
= g_hash_table_new(NULL
, NULL
);
354 for (i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
355 g_hash_table_insert(helper_table
, (gpointer
)all_helpers
[i
].func
,
356 (gpointer
)&all_helpers
[i
]);
361 /* Reverse the order of the saved registers, assuming they're all at
362 the start of tcg_target_reg_alloc_order. */
363 for (n
= 0; n
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++n
) {
364 int r
= tcg_target_reg_alloc_order
[n
];
365 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, r
)) {
369 for (i
= 0; i
< n
; ++i
) {
370 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[n
- 1 - i
];
372 for (; i
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++i
) {
373 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[i
];
377 void tcg_prologue_init(TCGContext
*s
)
379 size_t prologue_size
, total_size
;
382 /* Put the prologue at the beginning of code_gen_buffer. */
383 buf0
= s
->code_gen_buffer
;
386 s
->code_gen_prologue
= buf0
;
388 /* Generate the prologue. */
389 tcg_target_qemu_prologue(s
);
391 flush_icache_range((uintptr_t)buf0
, (uintptr_t)buf1
);
393 /* Deduct the prologue from the buffer. */
394 prologue_size
= tcg_current_code_size(s
);
395 s
->code_gen_ptr
= buf1
;
396 s
->code_gen_buffer
= buf1
;
398 total_size
= s
->code_gen_buffer_size
- prologue_size
;
399 s
->code_gen_buffer_size
= total_size
;
401 /* Compute a high-water mark, at which we voluntarily flush the buffer
402 and start over. The size here is arbitrary, significantly larger
403 than we expect the code generation for any one opcode to require. */
404 s
->code_gen_highwater
= s
->code_gen_buffer
+ (total_size
- 1024);
406 tcg_register_jit(s
->code_gen_buffer
, total_size
);
409 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
410 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size
);
411 log_disas(buf0
, prologue_size
);
418 void tcg_func_start(TCGContext
*s
)
421 s
->nb_temps
= s
->nb_globals
;
423 /* No temps have been previously allocated for size or locality. */
424 memset(s
->free_temps
, 0, sizeof(s
->free_temps
));
427 s
->current_frame_offset
= s
->frame_start
;
429 #ifdef CONFIG_DEBUG_TCG
430 s
->goto_tb_issue_mask
= 0;
433 s
->gen_first_op_idx
= 0;
434 s
->gen_last_op_idx
= -1;
435 s
->gen_next_op_idx
= 0;
436 s
->gen_next_parm_idx
= 0;
438 s
->be
= tcg_malloc(sizeof(TCGBackendData
));
441 static inline int temp_idx(TCGContext
*s
, TCGTemp
*ts
)
443 ptrdiff_t n
= ts
- s
->temps
;
444 tcg_debug_assert(n
>= 0 && n
< s
->nb_temps
);
448 static inline TCGTemp
*tcg_temp_alloc(TCGContext
*s
)
450 int n
= s
->nb_temps
++;
451 tcg_debug_assert(n
< TCG_MAX_TEMPS
);
452 return memset(&s
->temps
[n
], 0, sizeof(TCGTemp
));
455 static inline TCGTemp
*tcg_global_alloc(TCGContext
*s
)
457 tcg_debug_assert(s
->nb_globals
== s
->nb_temps
);
459 return tcg_temp_alloc(s
);
462 static int tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
463 TCGReg reg
, const char *name
)
467 if (TCG_TARGET_REG_BITS
== 32 && type
!= TCG_TYPE_I32
) {
471 ts
= tcg_global_alloc(s
);
472 ts
->base_type
= type
;
477 tcg_regset_set_reg(s
->reserved_regs
, reg
);
479 return temp_idx(s
, ts
);
482 void tcg_set_frame(TCGContext
*s
, TCGReg reg
, intptr_t start
, intptr_t size
)
485 s
->frame_start
= start
;
486 s
->frame_end
= start
+ size
;
487 idx
= tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, reg
, "_frame");
488 s
->frame_temp
= &s
->temps
[idx
];
491 TCGv_i32
tcg_global_reg_new_i32(TCGReg reg
, const char *name
)
493 TCGContext
*s
= &tcg_ctx
;
496 if (tcg_regset_test_reg(s
->reserved_regs
, reg
)) {
499 idx
= tcg_global_reg_new_internal(s
, TCG_TYPE_I32
, reg
, name
);
500 return MAKE_TCGV_I32(idx
);
503 TCGv_i64
tcg_global_reg_new_i64(TCGReg reg
, const char *name
)
505 TCGContext
*s
= &tcg_ctx
;
508 if (tcg_regset_test_reg(s
->reserved_regs
, reg
)) {
511 idx
= tcg_global_reg_new_internal(s
, TCG_TYPE_I64
, reg
, name
);
512 return MAKE_TCGV_I64(idx
);
515 int tcg_global_mem_new_internal(TCGType type
, TCGv_ptr base
,
516 intptr_t offset
, const char *name
)
518 TCGContext
*s
= &tcg_ctx
;
519 TCGTemp
*base_ts
= &s
->temps
[GET_TCGV_PTR(base
)];
520 TCGTemp
*ts
= tcg_global_alloc(s
);
521 int indirect_reg
= 0, bigendian
= 0;
522 #ifdef HOST_WORDS_BIGENDIAN
526 if (!base_ts
->fixed_reg
) {
528 base_ts
->indirect_base
= 1;
531 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
532 TCGTemp
*ts2
= tcg_global_alloc(s
);
535 ts
->base_type
= TCG_TYPE_I64
;
536 ts
->type
= TCG_TYPE_I32
;
537 ts
->indirect_reg
= indirect_reg
;
538 ts
->mem_allocated
= 1;
539 ts
->mem_base
= base_ts
;
540 ts
->mem_offset
= offset
+ bigendian
* 4;
541 pstrcpy(buf
, sizeof(buf
), name
);
542 pstrcat(buf
, sizeof(buf
), "_0");
543 ts
->name
= strdup(buf
);
545 tcg_debug_assert(ts2
== ts
+ 1);
546 ts2
->base_type
= TCG_TYPE_I64
;
547 ts2
->type
= TCG_TYPE_I32
;
548 ts2
->indirect_reg
= indirect_reg
;
549 ts2
->mem_allocated
= 1;
550 ts2
->mem_base
= base_ts
;
551 ts2
->mem_offset
= offset
+ (1 - bigendian
) * 4;
552 pstrcpy(buf
, sizeof(buf
), name
);
553 pstrcat(buf
, sizeof(buf
), "_1");
554 ts
->name
= strdup(buf
);
556 ts
->base_type
= type
;
558 ts
->indirect_reg
= indirect_reg
;
559 ts
->mem_allocated
= 1;
560 ts
->mem_base
= base_ts
;
561 ts
->mem_offset
= offset
;
564 return temp_idx(s
, ts
);
567 static int tcg_temp_new_internal(TCGType type
, int temp_local
)
569 TCGContext
*s
= &tcg_ctx
;
573 k
= type
+ (temp_local
? TCG_TYPE_COUNT
: 0);
574 idx
= find_first_bit(s
->free_temps
[k
].l
, TCG_MAX_TEMPS
);
575 if (idx
< TCG_MAX_TEMPS
) {
576 /* There is already an available temp with the right type. */
577 clear_bit(idx
, s
->free_temps
[k
].l
);
580 ts
->temp_allocated
= 1;
581 tcg_debug_assert(ts
->base_type
== type
);
582 tcg_debug_assert(ts
->temp_local
== temp_local
);
584 ts
= tcg_temp_alloc(s
);
585 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
586 TCGTemp
*ts2
= tcg_temp_alloc(s
);
588 ts
->base_type
= type
;
589 ts
->type
= TCG_TYPE_I32
;
590 ts
->temp_allocated
= 1;
591 ts
->temp_local
= temp_local
;
593 tcg_debug_assert(ts2
== ts
+ 1);
594 ts2
->base_type
= TCG_TYPE_I64
;
595 ts2
->type
= TCG_TYPE_I32
;
596 ts2
->temp_allocated
= 1;
597 ts2
->temp_local
= temp_local
;
599 ts
->base_type
= type
;
601 ts
->temp_allocated
= 1;
602 ts
->temp_local
= temp_local
;
604 idx
= temp_idx(s
, ts
);
607 #if defined(CONFIG_DEBUG_TCG)
613 TCGv_i32
tcg_temp_new_internal_i32(int temp_local
)
617 idx
= tcg_temp_new_internal(TCG_TYPE_I32
, temp_local
);
618 return MAKE_TCGV_I32(idx
);
621 TCGv_i64
tcg_temp_new_internal_i64(int temp_local
)
625 idx
= tcg_temp_new_internal(TCG_TYPE_I64
, temp_local
);
626 return MAKE_TCGV_I64(idx
);
629 static void tcg_temp_free_internal(int idx
)
631 TCGContext
*s
= &tcg_ctx
;
635 #if defined(CONFIG_DEBUG_TCG)
637 if (s
->temps_in_use
< 0) {
638 fprintf(stderr
, "More temporaries freed than allocated!\n");
642 tcg_debug_assert(idx
>= s
->nb_globals
&& idx
< s
->nb_temps
);
644 tcg_debug_assert(ts
->temp_allocated
!= 0);
645 ts
->temp_allocated
= 0;
647 k
= ts
->base_type
+ (ts
->temp_local
? TCG_TYPE_COUNT
: 0);
648 set_bit(idx
, s
->free_temps
[k
].l
);
651 void tcg_temp_free_i32(TCGv_i32 arg
)
653 tcg_temp_free_internal(GET_TCGV_I32(arg
));
656 void tcg_temp_free_i64(TCGv_i64 arg
)
658 tcg_temp_free_internal(GET_TCGV_I64(arg
));
661 TCGv_i32
tcg_const_i32(int32_t val
)
664 t0
= tcg_temp_new_i32();
665 tcg_gen_movi_i32(t0
, val
);
669 TCGv_i64
tcg_const_i64(int64_t val
)
672 t0
= tcg_temp_new_i64();
673 tcg_gen_movi_i64(t0
, val
);
677 TCGv_i32
tcg_const_local_i32(int32_t val
)
680 t0
= tcg_temp_local_new_i32();
681 tcg_gen_movi_i32(t0
, val
);
685 TCGv_i64
tcg_const_local_i64(int64_t val
)
688 t0
= tcg_temp_local_new_i64();
689 tcg_gen_movi_i64(t0
, val
);
693 #if defined(CONFIG_DEBUG_TCG)
694 void tcg_clear_temp_count(void)
696 TCGContext
*s
= &tcg_ctx
;
700 int tcg_check_temp_count(void)
702 TCGContext
*s
= &tcg_ctx
;
703 if (s
->temps_in_use
) {
704 /* Clear the count so that we don't give another
705 * warning immediately next time around.
714 /* Note: we convert the 64 bit args to 32 bit and do some alignment
715 and endian swap. Maybe it would be better to do the alignment
716 and endian swap in tcg_reg_alloc_call(). */
717 void tcg_gen_callN(TCGContext
*s
, void *func
, TCGArg ret
,
718 int nargs
, TCGArg
*args
)
720 int i
, real_args
, nb_rets
, pi
, pi_first
;
721 unsigned sizemask
, flags
;
724 info
= g_hash_table_lookup(s
->helpers
, (gpointer
)func
);
726 sizemask
= info
->sizemask
;
728 #if defined(__sparc__) && !defined(__arch64__) \
729 && !defined(CONFIG_TCG_INTERPRETER)
730 /* We have 64-bit values in one register, but need to pass as two
731 separate parameters. Split them. */
732 int orig_sizemask
= sizemask
;
733 int orig_nargs
= nargs
;
736 TCGV_UNUSED_I64(retl
);
737 TCGV_UNUSED_I64(reth
);
739 TCGArg
*split_args
= __builtin_alloca(sizeof(TCGArg
) * nargs
* 2);
740 for (i
= real_args
= 0; i
< nargs
; ++i
) {
741 int is_64bit
= sizemask
& (1 << (i
+1)*2);
743 TCGv_i64 orig
= MAKE_TCGV_I64(args
[i
]);
744 TCGv_i32 h
= tcg_temp_new_i32();
745 TCGv_i32 l
= tcg_temp_new_i32();
746 tcg_gen_extr_i64_i32(l
, h
, orig
);
747 split_args
[real_args
++] = GET_TCGV_I32(h
);
748 split_args
[real_args
++] = GET_TCGV_I32(l
);
750 split_args
[real_args
++] = args
[i
];
757 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
758 for (i
= 0; i
< nargs
; ++i
) {
759 int is_64bit
= sizemask
& (1 << (i
+1)*2);
760 int is_signed
= sizemask
& (2 << (i
+1)*2);
762 TCGv_i64 temp
= tcg_temp_new_i64();
763 TCGv_i64 orig
= MAKE_TCGV_I64(args
[i
]);
765 tcg_gen_ext32s_i64(temp
, orig
);
767 tcg_gen_ext32u_i64(temp
, orig
);
769 args
[i
] = GET_TCGV_I64(temp
);
772 #endif /* TCG_TARGET_EXTEND_ARGS */
774 pi_first
= pi
= s
->gen_next_parm_idx
;
775 if (ret
!= TCG_CALL_DUMMY_ARG
) {
776 #if defined(__sparc__) && !defined(__arch64__) \
777 && !defined(CONFIG_TCG_INTERPRETER)
778 if (orig_sizemask
& 1) {
779 /* The 32-bit ABI is going to return the 64-bit value in
780 the %o0/%o1 register pair. Prepare for this by using
781 two return temporaries, and reassemble below. */
782 retl
= tcg_temp_new_i64();
783 reth
= tcg_temp_new_i64();
784 s
->gen_opparam_buf
[pi
++] = GET_TCGV_I64(reth
);
785 s
->gen_opparam_buf
[pi
++] = GET_TCGV_I64(retl
);
788 s
->gen_opparam_buf
[pi
++] = ret
;
792 if (TCG_TARGET_REG_BITS
< 64 && (sizemask
& 1)) {
793 #ifdef HOST_WORDS_BIGENDIAN
794 s
->gen_opparam_buf
[pi
++] = ret
+ 1;
795 s
->gen_opparam_buf
[pi
++] = ret
;
797 s
->gen_opparam_buf
[pi
++] = ret
;
798 s
->gen_opparam_buf
[pi
++] = ret
+ 1;
802 s
->gen_opparam_buf
[pi
++] = ret
;
810 for (i
= 0; i
< nargs
; i
++) {
811 int is_64bit
= sizemask
& (1 << (i
+1)*2);
812 if (TCG_TARGET_REG_BITS
< 64 && is_64bit
) {
813 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
814 /* some targets want aligned 64 bit args */
816 s
->gen_opparam_buf
[pi
++] = TCG_CALL_DUMMY_ARG
;
820 /* If stack grows up, then we will be placing successive
821 arguments at lower addresses, which means we need to
822 reverse the order compared to how we would normally
823 treat either big or little-endian. For those arguments
824 that will wind up in registers, this still works for
825 HPPA (the only current STACK_GROWSUP target) since the
826 argument registers are *also* allocated in decreasing
827 order. If another such target is added, this logic may
828 have to get more complicated to differentiate between
829 stack arguments and register arguments. */
830 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
831 s
->gen_opparam_buf
[pi
++] = args
[i
] + 1;
832 s
->gen_opparam_buf
[pi
++] = args
[i
];
834 s
->gen_opparam_buf
[pi
++] = args
[i
];
835 s
->gen_opparam_buf
[pi
++] = args
[i
] + 1;
841 s
->gen_opparam_buf
[pi
++] = args
[i
];
844 s
->gen_opparam_buf
[pi
++] = (uintptr_t)func
;
845 s
->gen_opparam_buf
[pi
++] = flags
;
847 i
= s
->gen_next_op_idx
;
848 tcg_debug_assert(i
< OPC_BUF_SIZE
);
849 tcg_debug_assert(pi
<= OPPARAM_BUF_SIZE
);
851 /* Set links for sequential allocation during translation. */
852 s
->gen_op_buf
[i
] = (TCGOp
){
853 .opc
= INDEX_op_call
,
861 /* Make sure the calli field didn't overflow. */
862 tcg_debug_assert(s
->gen_op_buf
[i
].calli
== real_args
);
864 s
->gen_last_op_idx
= i
;
865 s
->gen_next_op_idx
= i
+ 1;
866 s
->gen_next_parm_idx
= pi
;
868 #if defined(__sparc__) && !defined(__arch64__) \
869 && !defined(CONFIG_TCG_INTERPRETER)
870 /* Free all of the parts we allocated above. */
871 for (i
= real_args
= 0; i
< orig_nargs
; ++i
) {
872 int is_64bit
= orig_sizemask
& (1 << (i
+1)*2);
874 TCGv_i32 h
= MAKE_TCGV_I32(args
[real_args
++]);
875 TCGv_i32 l
= MAKE_TCGV_I32(args
[real_args
++]);
876 tcg_temp_free_i32(h
);
877 tcg_temp_free_i32(l
);
882 if (orig_sizemask
& 1) {
883 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
884 Note that describing these as TCGv_i64 eliminates an unnecessary
885 zero-extension that tcg_gen_concat_i32_i64 would create. */
886 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret
), retl
, reth
);
887 tcg_temp_free_i64(retl
);
888 tcg_temp_free_i64(reth
);
890 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
891 for (i
= 0; i
< nargs
; ++i
) {
892 int is_64bit
= sizemask
& (1 << (i
+1)*2);
894 TCGv_i64 temp
= MAKE_TCGV_I64(args
[i
]);
895 tcg_temp_free_i64(temp
);
898 #endif /* TCG_TARGET_EXTEND_ARGS */
901 static void tcg_reg_alloc_start(TCGContext
*s
)
905 for(i
= 0; i
< s
->nb_globals
; i
++) {
908 ts
->val_type
= TEMP_VAL_REG
;
910 ts
->val_type
= TEMP_VAL_MEM
;
913 for(i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
915 if (ts
->temp_local
) {
916 ts
->val_type
= TEMP_VAL_MEM
;
918 ts
->val_type
= TEMP_VAL_DEAD
;
920 ts
->mem_allocated
= 0;
924 memset(s
->reg_to_temp
, 0, sizeof(s
->reg_to_temp
));
927 static char *tcg_get_arg_str_ptr(TCGContext
*s
, char *buf
, int buf_size
,
930 int idx
= temp_idx(s
, ts
);
932 if (idx
< s
->nb_globals
) {
933 pstrcpy(buf
, buf_size
, ts
->name
);
934 } else if (ts
->temp_local
) {
935 snprintf(buf
, buf_size
, "loc%d", idx
- s
->nb_globals
);
937 snprintf(buf
, buf_size
, "tmp%d", idx
- s
->nb_globals
);
942 static char *tcg_get_arg_str_idx(TCGContext
*s
, char *buf
,
943 int buf_size
, int idx
)
945 tcg_debug_assert(idx
>= 0 && idx
< s
->nb_temps
);
946 return tcg_get_arg_str_ptr(s
, buf
, buf_size
, &s
->temps
[idx
]);
949 /* Find helper name. */
950 static inline const char *tcg_find_helper(TCGContext
*s
, uintptr_t val
)
952 const char *ret
= NULL
;
954 TCGHelperInfo
*info
= g_hash_table_lookup(s
->helpers
, (gpointer
)val
);
962 static const char * const cond_name
[] =
964 [TCG_COND_NEVER
] = "never",
965 [TCG_COND_ALWAYS
] = "always",
966 [TCG_COND_EQ
] = "eq",
967 [TCG_COND_NE
] = "ne",
968 [TCG_COND_LT
] = "lt",
969 [TCG_COND_GE
] = "ge",
970 [TCG_COND_LE
] = "le",
971 [TCG_COND_GT
] = "gt",
972 [TCG_COND_LTU
] = "ltu",
973 [TCG_COND_GEU
] = "geu",
974 [TCG_COND_LEU
] = "leu",
975 [TCG_COND_GTU
] = "gtu"
978 static const char * const ldst_name
[] =
994 void tcg_dump_ops(TCGContext
*s
)
1000 for (oi
= s
->gen_first_op_idx
; oi
>= 0; oi
= op
->next
) {
1001 int i
, k
, nb_oargs
, nb_iargs
, nb_cargs
;
1002 const TCGOpDef
*def
;
1006 op
= &s
->gen_op_buf
[oi
];
1008 def
= &tcg_op_defs
[c
];
1009 args
= &s
->gen_opparam_buf
[op
->args
];
1011 if (c
== INDEX_op_insn_start
) {
1012 qemu_log("%s ----", oi
!= s
->gen_first_op_idx
? "\n" : "");
1014 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
1016 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1017 a
= ((target_ulong
)args
[i
* 2 + 1] << 32) | args
[i
* 2];
1021 qemu_log(" " TARGET_FMT_lx
, a
);
1023 } else if (c
== INDEX_op_call
) {
1024 /* variable number of arguments */
1025 nb_oargs
= op
->callo
;
1026 nb_iargs
= op
->calli
;
1027 nb_cargs
= def
->nb_cargs
;
1029 /* function name, flags, out args */
1030 qemu_log(" %s %s,$0x%" TCG_PRIlx
",$%d", def
->name
,
1031 tcg_find_helper(s
, args
[nb_oargs
+ nb_iargs
]),
1032 args
[nb_oargs
+ nb_iargs
+ 1], nb_oargs
);
1033 for (i
= 0; i
< nb_oargs
; i
++) {
1034 qemu_log(",%s", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
),
1037 for (i
= 0; i
< nb_iargs
; i
++) {
1038 TCGArg arg
= args
[nb_oargs
+ i
];
1039 const char *t
= "<dummy>";
1040 if (arg
!= TCG_CALL_DUMMY_ARG
) {
1041 t
= tcg_get_arg_str_idx(s
, buf
, sizeof(buf
), arg
);
1046 qemu_log(" %s ", def
->name
);
1048 nb_oargs
= def
->nb_oargs
;
1049 nb_iargs
= def
->nb_iargs
;
1050 nb_cargs
= def
->nb_cargs
;
1053 for (i
= 0; i
< nb_oargs
; i
++) {
1057 qemu_log("%s", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
),
1060 for (i
= 0; i
< nb_iargs
; i
++) {
1064 qemu_log("%s", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
),
1068 case INDEX_op_brcond_i32
:
1069 case INDEX_op_setcond_i32
:
1070 case INDEX_op_movcond_i32
:
1071 case INDEX_op_brcond2_i32
:
1072 case INDEX_op_setcond2_i32
:
1073 case INDEX_op_brcond_i64
:
1074 case INDEX_op_setcond_i64
:
1075 case INDEX_op_movcond_i64
:
1076 if (args
[k
] < ARRAY_SIZE(cond_name
) && cond_name
[args
[k
]]) {
1077 qemu_log(",%s", cond_name
[args
[k
++]]);
1079 qemu_log(",$0x%" TCG_PRIlx
, args
[k
++]);
1083 case INDEX_op_qemu_ld_i32
:
1084 case INDEX_op_qemu_st_i32
:
1085 case INDEX_op_qemu_ld_i64
:
1086 case INDEX_op_qemu_st_i64
:
1088 TCGMemOpIdx oi
= args
[k
++];
1089 TCGMemOp op
= get_memop(oi
);
1090 unsigned ix
= get_mmuidx(oi
);
1092 if (op
& ~(MO_AMASK
| MO_BSWAP
| MO_SSIZE
)) {
1093 qemu_log(",$0x%x,%u", op
, ix
);
1095 const char *s_al
= "", *s_op
;
1096 if (op
& MO_AMASK
) {
1097 if ((op
& MO_AMASK
) == MO_ALIGN
) {
1103 s_op
= ldst_name
[op
& (MO_BSWAP
| MO_SSIZE
)];
1104 qemu_log(",%s%s,%u", s_al
, s_op
, ix
);
1114 case INDEX_op_set_label
:
1116 case INDEX_op_brcond_i32
:
1117 case INDEX_op_brcond_i64
:
1118 case INDEX_op_brcond2_i32
:
1119 qemu_log("%s$L%d", k
? "," : "", arg_label(args
[k
])->id
);
1125 for (; i
< nb_cargs
; i
++, k
++) {
1126 qemu_log("%s$0x%" TCG_PRIlx
, k
? "," : "", args
[k
]);
1133 /* we give more priority to constraints with less registers */
1134 static int get_constraint_priority(const TCGOpDef
*def
, int k
)
1136 const TCGArgConstraint
*arg_ct
;
1139 arg_ct
= &def
->args_ct
[k
];
1140 if (arg_ct
->ct
& TCG_CT_ALIAS
) {
1141 /* an alias is equivalent to a single register */
1144 if (!(arg_ct
->ct
& TCG_CT_REG
))
1147 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
1148 if (tcg_regset_test_reg(arg_ct
->u
.regs
, i
))
1152 return TCG_TARGET_NB_REGS
- n
+ 1;
1155 /* sort from highest priority to lowest */
1156 static void sort_constraints(TCGOpDef
*def
, int start
, int n
)
1158 int i
, j
, p1
, p2
, tmp
;
1160 for(i
= 0; i
< n
; i
++)
1161 def
->sorted_args
[start
+ i
] = start
+ i
;
1164 for(i
= 0; i
< n
- 1; i
++) {
1165 for(j
= i
+ 1; j
< n
; j
++) {
1166 p1
= get_constraint_priority(def
, def
->sorted_args
[start
+ i
]);
1167 p2
= get_constraint_priority(def
, def
->sorted_args
[start
+ j
]);
1169 tmp
= def
->sorted_args
[start
+ i
];
1170 def
->sorted_args
[start
+ i
] = def
->sorted_args
[start
+ j
];
1171 def
->sorted_args
[start
+ j
] = tmp
;
1177 void tcg_add_target_add_op_defs(const TCGTargetOpDef
*tdefs
)
1185 if (tdefs
->op
== (TCGOpcode
)-1)
1188 tcg_debug_assert((unsigned)op
< NB_OPS
);
1189 def
= &tcg_op_defs
[op
];
1190 #if defined(CONFIG_DEBUG_TCG)
1191 /* Duplicate entry in op definitions? */
1192 tcg_debug_assert(!def
->used
);
1195 nb_args
= def
->nb_iargs
+ def
->nb_oargs
;
1196 for(i
= 0; i
< nb_args
; i
++) {
1197 ct_str
= tdefs
->args_ct_str
[i
];
1198 /* Incomplete TCGTargetOpDef entry? */
1199 tcg_debug_assert(ct_str
!= NULL
);
1200 tcg_regset_clear(def
->args_ct
[i
].u
.regs
);
1201 def
->args_ct
[i
].ct
= 0;
1202 if (ct_str
[0] >= '0' && ct_str
[0] <= '9') {
1204 oarg
= ct_str
[0] - '0';
1205 tcg_debug_assert(oarg
< def
->nb_oargs
);
1206 tcg_debug_assert(def
->args_ct
[oarg
].ct
& TCG_CT_REG
);
1207 /* TCG_CT_ALIAS is for the output arguments. The input
1208 argument is tagged with TCG_CT_IALIAS. */
1209 def
->args_ct
[i
] = def
->args_ct
[oarg
];
1210 def
->args_ct
[oarg
].ct
= TCG_CT_ALIAS
;
1211 def
->args_ct
[oarg
].alias_index
= i
;
1212 def
->args_ct
[i
].ct
|= TCG_CT_IALIAS
;
1213 def
->args_ct
[i
].alias_index
= oarg
;
1216 if (*ct_str
== '\0')
1220 def
->args_ct
[i
].ct
|= TCG_CT_CONST
;
1224 if (target_parse_constraint(&def
->args_ct
[i
], &ct_str
) < 0) {
1225 fprintf(stderr
, "Invalid constraint '%s' for arg %d of operation '%s'\n",
1226 ct_str
, i
, def
->name
);
1234 /* TCGTargetOpDef entry with too much information? */
1235 tcg_debug_assert(i
== TCG_MAX_OP_ARGS
|| tdefs
->args_ct_str
[i
] == NULL
);
1237 /* sort the constraints (XXX: this is just an heuristic) */
1238 sort_constraints(def
, 0, def
->nb_oargs
);
1239 sort_constraints(def
, def
->nb_oargs
, def
->nb_iargs
);
1245 printf("%s: sorted=", def
->name
);
1246 for(i
= 0; i
< def
->nb_oargs
+ def
->nb_iargs
; i
++)
1247 printf(" %d", def
->sorted_args
[i
]);
1254 #if defined(CONFIG_DEBUG_TCG)
1256 for (op
= 0; op
< tcg_op_defs_max
; op
++) {
1257 const TCGOpDef
*def
= &tcg_op_defs
[op
];
1258 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
1259 /* Wrong entry in op definitions? */
1261 fprintf(stderr
, "Invalid op definition for %s\n", def
->name
);
1265 /* Missing entry in op definitions? */
1267 fprintf(stderr
, "Missing op definition for %s\n", def
->name
);
1278 void tcg_op_remove(TCGContext
*s
, TCGOp
*op
)
1280 int next
= op
->next
;
1281 int prev
= op
->prev
;
1284 s
->gen_op_buf
[next
].prev
= prev
;
1286 s
->gen_last_op_idx
= prev
;
1289 s
->gen_op_buf
[prev
].next
= next
;
1291 s
->gen_first_op_idx
= next
;
1294 memset(op
, -1, sizeof(*op
));
1296 #ifdef CONFIG_PROFILER
1301 #ifdef USE_LIVENESS_ANALYSIS
1302 /* liveness analysis: end of function: all temps are dead, and globals
1303 should be in memory. */
1304 static inline void tcg_la_func_end(TCGContext
*s
, uint8_t *dead_temps
,
1307 memset(dead_temps
, 1, s
->nb_temps
);
1308 memset(mem_temps
, 1, s
->nb_globals
);
1309 memset(mem_temps
+ s
->nb_globals
, 0, s
->nb_temps
- s
->nb_globals
);
1312 /* liveness analysis: end of basic block: all temps are dead, globals
1313 and local temps should be in memory. */
1314 static inline void tcg_la_bb_end(TCGContext
*s
, uint8_t *dead_temps
,
1319 memset(dead_temps
, 1, s
->nb_temps
);
1320 memset(mem_temps
, 1, s
->nb_globals
);
1321 for(i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
1322 mem_temps
[i
] = s
->temps
[i
].temp_local
;
1326 /* Liveness analysis : update the opc_dead_args array to tell if a
1327 given input arguments is dead. Instructions updating dead
1328 temporaries are removed. */
1329 static void tcg_liveness_analysis(TCGContext
*s
)
1331 uint8_t *dead_temps
, *mem_temps
;
1332 int oi
, oi_prev
, nb_ops
;
1334 nb_ops
= s
->gen_next_op_idx
;
1335 s
->op_dead_args
= tcg_malloc(nb_ops
* sizeof(uint16_t));
1336 s
->op_sync_args
= tcg_malloc(nb_ops
* sizeof(uint8_t));
1338 dead_temps
= tcg_malloc(s
->nb_temps
);
1339 mem_temps
= tcg_malloc(s
->nb_temps
);
1340 tcg_la_func_end(s
, dead_temps
, mem_temps
);
1342 for (oi
= s
->gen_last_op_idx
; oi
>= 0; oi
= oi_prev
) {
1343 int i
, nb_iargs
, nb_oargs
;
1344 TCGOpcode opc_new
, opc_new2
;
1350 TCGOp
* const op
= &s
->gen_op_buf
[oi
];
1351 TCGArg
* const args
= &s
->gen_opparam_buf
[op
->args
];
1352 TCGOpcode opc
= op
->opc
;
1353 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
1362 nb_oargs
= op
->callo
;
1363 nb_iargs
= op
->calli
;
1364 call_flags
= args
[nb_oargs
+ nb_iargs
+ 1];
1366 /* pure functions can be removed if their result is unused */
1367 if (call_flags
& TCG_CALL_NO_SIDE_EFFECTS
) {
1368 for (i
= 0; i
< nb_oargs
; i
++) {
1370 if (!dead_temps
[arg
] || mem_temps
[arg
]) {
1371 goto do_not_remove_call
;
1378 /* output args are dead */
1381 for (i
= 0; i
< nb_oargs
; i
++) {
1383 if (dead_temps
[arg
]) {
1384 dead_args
|= (1 << i
);
1386 if (mem_temps
[arg
]) {
1387 sync_args
|= (1 << i
);
1389 dead_temps
[arg
] = 1;
1393 if (!(call_flags
& TCG_CALL_NO_READ_GLOBALS
)) {
1394 /* globals should be synced to memory */
1395 memset(mem_temps
, 1, s
->nb_globals
);
1397 if (!(call_flags
& (TCG_CALL_NO_WRITE_GLOBALS
|
1398 TCG_CALL_NO_READ_GLOBALS
))) {
1399 /* globals should go back to memory */
1400 memset(dead_temps
, 1, s
->nb_globals
);
1403 /* record arguments that die in this helper */
1404 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
1406 if (arg
!= TCG_CALL_DUMMY_ARG
) {
1407 if (dead_temps
[arg
]) {
1408 dead_args
|= (1 << i
);
1412 /* input arguments are live for preceding opcodes */
1413 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
1415 dead_temps
[arg
] = 0;
1417 s
->op_dead_args
[oi
] = dead_args
;
1418 s
->op_sync_args
[oi
] = sync_args
;
1422 case INDEX_op_insn_start
:
1424 case INDEX_op_discard
:
1425 /* mark the temporary as dead */
1426 dead_temps
[args
[0]] = 1;
1427 mem_temps
[args
[0]] = 0;
1430 case INDEX_op_add2_i32
:
1431 opc_new
= INDEX_op_add_i32
;
1433 case INDEX_op_sub2_i32
:
1434 opc_new
= INDEX_op_sub_i32
;
1436 case INDEX_op_add2_i64
:
1437 opc_new
= INDEX_op_add_i64
;
1439 case INDEX_op_sub2_i64
:
1440 opc_new
= INDEX_op_sub_i64
;
1444 /* Test if the high part of the operation is dead, but not
1445 the low part. The result can be optimized to a simple
1446 add or sub. This happens often for x86_64 guest when the
1447 cpu mode is set to 32 bit. */
1448 if (dead_temps
[args
[1]] && !mem_temps
[args
[1]]) {
1449 if (dead_temps
[args
[0]] && !mem_temps
[args
[0]]) {
1452 /* Replace the opcode and adjust the args in place,
1453 leaving 3 unused args at the end. */
1454 op
->opc
= opc
= opc_new
;
1457 /* Fall through and mark the single-word operation live. */
1463 case INDEX_op_mulu2_i32
:
1464 opc_new
= INDEX_op_mul_i32
;
1465 opc_new2
= INDEX_op_muluh_i32
;
1466 have_opc_new2
= TCG_TARGET_HAS_muluh_i32
;
1468 case INDEX_op_muls2_i32
:
1469 opc_new
= INDEX_op_mul_i32
;
1470 opc_new2
= INDEX_op_mulsh_i32
;
1471 have_opc_new2
= TCG_TARGET_HAS_mulsh_i32
;
1473 case INDEX_op_mulu2_i64
:
1474 opc_new
= INDEX_op_mul_i64
;
1475 opc_new2
= INDEX_op_muluh_i64
;
1476 have_opc_new2
= TCG_TARGET_HAS_muluh_i64
;
1478 case INDEX_op_muls2_i64
:
1479 opc_new
= INDEX_op_mul_i64
;
1480 opc_new2
= INDEX_op_mulsh_i64
;
1481 have_opc_new2
= TCG_TARGET_HAS_mulsh_i64
;
1486 if (dead_temps
[args
[1]] && !mem_temps
[args
[1]]) {
1487 if (dead_temps
[args
[0]] && !mem_temps
[args
[0]]) {
1488 /* Both parts of the operation are dead. */
1491 /* The high part of the operation is dead; generate the low. */
1492 op
->opc
= opc
= opc_new
;
1495 } else if (have_opc_new2
&& dead_temps
[args
[0]]
1496 && !mem_temps
[args
[0]]) {
1497 /* The low part of the operation is dead; generate the high. */
1498 op
->opc
= opc
= opc_new2
;
1505 /* Mark the single-word operation live. */
1510 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
1511 nb_iargs
= def
->nb_iargs
;
1512 nb_oargs
= def
->nb_oargs
;
1514 /* Test if the operation can be removed because all
1515 its outputs are dead. We assume that nb_oargs == 0
1516 implies side effects */
1517 if (!(def
->flags
& TCG_OPF_SIDE_EFFECTS
) && nb_oargs
!= 0) {
1518 for (i
= 0; i
< nb_oargs
; i
++) {
1520 if (!dead_temps
[arg
] || mem_temps
[arg
]) {
1525 tcg_op_remove(s
, op
);
1528 /* output args are dead */
1531 for (i
= 0; i
< nb_oargs
; i
++) {
1533 if (dead_temps
[arg
]) {
1534 dead_args
|= (1 << i
);
1536 if (mem_temps
[arg
]) {
1537 sync_args
|= (1 << i
);
1539 dead_temps
[arg
] = 1;
1543 /* if end of basic block, update */
1544 if (def
->flags
& TCG_OPF_BB_END
) {
1545 tcg_la_bb_end(s
, dead_temps
, mem_temps
);
1546 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
1547 /* globals should be synced to memory */
1548 memset(mem_temps
, 1, s
->nb_globals
);
1551 /* record arguments that die in this opcode */
1552 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
1554 if (dead_temps
[arg
]) {
1555 dead_args
|= (1 << i
);
1558 /* input arguments are live for preceding opcodes */
1559 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
1561 dead_temps
[arg
] = 0;
1563 s
->op_dead_args
[oi
] = dead_args
;
1564 s
->op_sync_args
[oi
] = sync_args
;
1571 /* dummy liveness analysis */
1572 static void tcg_liveness_analysis(TCGContext
*s
)
1574 int nb_ops
= s
->gen_next_op_idx
;
1576 s
->op_dead_args
= tcg_malloc(nb_ops
* sizeof(uint16_t));
1577 memset(s
->op_dead_args
, 0, nb_ops
* sizeof(uint16_t));
1578 s
->op_sync_args
= tcg_malloc(nb_ops
* sizeof(uint8_t));
1579 memset(s
->op_sync_args
, 0, nb_ops
* sizeof(uint8_t));
1583 #ifdef CONFIG_DEBUG_TCG
1584 static void dump_regs(TCGContext
*s
)
1590 for(i
= 0; i
< s
->nb_temps
; i
++) {
1592 printf(" %10s: ", tcg_get_arg_str_idx(s
, buf
, sizeof(buf
), i
));
1593 switch(ts
->val_type
) {
1595 printf("%s", tcg_target_reg_names
[ts
->reg
]);
1598 printf("%d(%s)", (int)ts
->mem_offset
,
1599 tcg_target_reg_names
[ts
->mem_base
->reg
]);
1601 case TEMP_VAL_CONST
:
1602 printf("$0x%" TCG_PRIlx
, ts
->val
);
1614 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
1615 if (s
->reg_to_temp
[i
] != NULL
) {
1617 tcg_target_reg_names
[i
],
1618 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), s
->reg_to_temp
[i
]));
1623 static void check_regs(TCGContext
*s
)
1630 for (reg
= 0; reg
< TCG_TARGET_NB_REGS
; reg
++) {
1631 ts
= s
->reg_to_temp
[reg
];
1633 if (ts
->val_type
!= TEMP_VAL_REG
|| ts
->reg
!= reg
) {
1634 printf("Inconsistency for register %s:\n",
1635 tcg_target_reg_names
[reg
]);
1640 for (k
= 0; k
< s
->nb_temps
; k
++) {
1642 if (ts
->val_type
== TEMP_VAL_REG
&& !ts
->fixed_reg
1643 && s
->reg_to_temp
[ts
->reg
] != ts
) {
1644 printf("Inconsistency for temp %s:\n",
1645 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
1647 printf("reg state:\n");
1655 static void temp_allocate_frame(TCGContext
*s
, int temp
)
1658 ts
= &s
->temps
[temp
];
1659 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
1660 /* Sparc64 stack is accessed with offset of 2047 */
1661 s
->current_frame_offset
= (s
->current_frame_offset
+
1662 (tcg_target_long
)sizeof(tcg_target_long
) - 1) &
1663 ~(sizeof(tcg_target_long
) - 1);
1665 if (s
->current_frame_offset
+ (tcg_target_long
)sizeof(tcg_target_long
) >
1669 ts
->mem_offset
= s
->current_frame_offset
;
1670 ts
->mem_base
= s
->frame_temp
;
1671 ts
->mem_allocated
= 1;
1672 s
->current_frame_offset
+= sizeof(tcg_target_long
);
1675 static void temp_load(TCGContext
*, TCGTemp
*, TCGRegSet
, TCGRegSet
);
1677 /* sync register 'reg' by saving it to the corresponding temporary */
1678 static void tcg_reg_sync(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
1680 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
1682 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
1683 if (!ts
->mem_coherent
&& !ts
->fixed_reg
) {
1684 if (!ts
->mem_allocated
) {
1685 temp_allocate_frame(s
, temp_idx(s
, ts
));
1686 } else if (ts
->indirect_reg
) {
1687 tcg_regset_set_reg(allocated_regs
, ts
->reg
);
1688 temp_load(s
, ts
->mem_base
,
1689 tcg_target_available_regs
[TCG_TYPE_PTR
],
1692 tcg_out_st(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
1694 ts
->mem_coherent
= 1;
1697 /* free register 'reg' by spilling the corresponding temporary if necessary */
1698 static void tcg_reg_free(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
1700 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
1703 tcg_reg_sync(s
, reg
, allocated_regs
);
1704 ts
->val_type
= TEMP_VAL_MEM
;
1705 s
->reg_to_temp
[reg
] = NULL
;
1709 /* Allocate a register belonging to reg1 & ~reg2 */
1710 static TCGReg
tcg_reg_alloc(TCGContext
*s
, TCGRegSet desired_regs
,
1711 TCGRegSet allocated_regs
, bool rev
)
1713 int i
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
1718 tcg_regset_andnot(reg_ct
, desired_regs
, allocated_regs
);
1719 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
1721 /* first try free registers */
1722 for(i
= 0; i
< n
; i
++) {
1724 if (tcg_regset_test_reg(reg_ct
, reg
) && s
->reg_to_temp
[reg
] == NULL
)
1728 /* XXX: do better spill choice */
1729 for(i
= 0; i
< n
; i
++) {
1731 if (tcg_regset_test_reg(reg_ct
, reg
)) {
1732 tcg_reg_free(s
, reg
, allocated_regs
);
1740 /* Make sure the temporary is in a register. If needed, allocate the register
1741 from DESIRED while avoiding ALLOCATED. */
1742 static void temp_load(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet desired_regs
,
1743 TCGRegSet allocated_regs
)
1747 switch (ts
->val_type
) {
1750 case TEMP_VAL_CONST
:
1751 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
, ts
->indirect_base
);
1752 tcg_out_movi(s
, ts
->type
, reg
, ts
->val
);
1753 ts
->mem_coherent
= 0;
1756 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
, ts
->indirect_base
);
1757 if (ts
->indirect_reg
) {
1758 tcg_regset_set_reg(allocated_regs
, reg
);
1759 temp_load(s
, ts
->mem_base
,
1760 tcg_target_available_regs
[TCG_TYPE_PTR
],
1763 tcg_out_ld(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
1764 ts
->mem_coherent
= 1;
1771 ts
->val_type
= TEMP_VAL_REG
;
1772 s
->reg_to_temp
[reg
] = ts
;
1775 /* mark a temporary as dead. */
1776 static inline void temp_dead(TCGContext
*s
, TCGTemp
*ts
)
1778 if (ts
->fixed_reg
) {
1781 if (ts
->val_type
== TEMP_VAL_REG
) {
1782 s
->reg_to_temp
[ts
->reg
] = NULL
;
1784 ts
->val_type
= (temp_idx(s
, ts
) < s
->nb_globals
|| ts
->temp_local
1785 ? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
1788 /* sync a temporary to memory. 'allocated_regs' is used in case a
1789 temporary registers needs to be allocated to store a constant. */
1790 static void temp_sync(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
)
1792 if (ts
->fixed_reg
) {
1795 switch (ts
->val_type
) {
1796 case TEMP_VAL_CONST
:
1797 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
], allocated_regs
);
1800 tcg_reg_sync(s
, ts
->reg
, allocated_regs
);
1810 /* save a temporary to memory. 'allocated_regs' is used in case a
1811 temporary registers needs to be allocated to store a constant. */
1812 static inline void temp_save(TCGContext
*s
, TCGTemp
*ts
,
1813 TCGRegSet allocated_regs
)
1815 #ifdef USE_LIVENESS_ANALYSIS
1816 /* ??? Liveness does not yet incorporate indirect bases. */
1817 if (!ts
->indirect_base
) {
1818 /* The liveness analysis already ensures that globals are back
1819 in memory. Keep an tcg_debug_assert for safety. */
1820 tcg_debug_assert(ts
->val_type
== TEMP_VAL_MEM
|| ts
->fixed_reg
);
1824 temp_sync(s
, ts
, allocated_regs
);
1828 /* save globals to their canonical location and assume they can be
1829 modified be the following code. 'allocated_regs' is used in case a
1830 temporary registers needs to be allocated to store a constant. */
1831 static void save_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
1835 for (i
= 0; i
< s
->nb_globals
; i
++) {
1836 temp_save(s
, &s
->temps
[i
], allocated_regs
);
1840 /* sync globals to their canonical location and assume they can be
1841 read by the following code. 'allocated_regs' is used in case a
1842 temporary registers needs to be allocated to store a constant. */
1843 static void sync_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
1847 for (i
= 0; i
< s
->nb_globals
; i
++) {
1848 TCGTemp
*ts
= &s
->temps
[i
];
1849 #ifdef USE_LIVENESS_ANALYSIS
1850 /* ??? Liveness does not yet incorporate indirect bases. */
1851 if (!ts
->indirect_base
) {
1852 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
1854 || ts
->mem_coherent
);
1858 temp_sync(s
, ts
, allocated_regs
);
1862 /* at the end of a basic block, we assume all temporaries are dead and
1863 all globals are stored at their canonical location. */
1864 static void tcg_reg_alloc_bb_end(TCGContext
*s
, TCGRegSet allocated_regs
)
1868 for (i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
1869 TCGTemp
*ts
= &s
->temps
[i
];
1870 if (ts
->temp_local
) {
1871 temp_save(s
, ts
, allocated_regs
);
1873 #ifdef USE_LIVENESS_ANALYSIS
1874 /* ??? Liveness does not yet incorporate indirect bases. */
1875 if (!ts
->indirect_base
) {
1876 /* The liveness analysis already ensures that temps are dead.
1877 Keep an tcg_debug_assert for safety. */
1878 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
1886 save_globals(s
, allocated_regs
);
1889 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1)
1890 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1)
1892 static void tcg_reg_alloc_movi(TCGContext
*s
, const TCGArg
*args
,
1893 uint16_t dead_args
, uint8_t sync_args
)
1896 tcg_target_ulong val
;
1898 ots
= &s
->temps
[args
[0]];
1901 if (ots
->fixed_reg
) {
1902 /* for fixed registers, we do not do any constant
1904 tcg_out_movi(s
, ots
->type
, ots
->reg
, val
);
1906 /* The movi is not explicitly generated here */
1907 if (ots
->val_type
== TEMP_VAL_REG
) {
1908 s
->reg_to_temp
[ots
->reg
] = NULL
;
1910 ots
->val_type
= TEMP_VAL_CONST
;
1913 if (NEED_SYNC_ARG(0)) {
1914 temp_sync(s
, ots
, s
->reserved_regs
);
1916 if (IS_DEAD_ARG(0)) {
1921 static void tcg_reg_alloc_mov(TCGContext
*s
, const TCGOpDef
*def
,
1922 const TCGArg
*args
, uint16_t dead_args
,
1925 TCGRegSet allocated_regs
;
1927 TCGType otype
, itype
;
1929 tcg_regset_set(allocated_regs
, s
->reserved_regs
);
1930 ots
= &s
->temps
[args
[0]];
1931 ts
= &s
->temps
[args
[1]];
1933 /* Note that otype != itype for no-op truncation. */
1937 /* If the source value is not in a register, and we're going to be
1938 forced to have it in a register in order to perform the copy,
1939 then copy the SOURCE value into its own register first. That way
1940 we don't have to reload SOURCE the next time it is used. */
1941 if (((NEED_SYNC_ARG(0) || ots
->fixed_reg
) && ts
->val_type
!= TEMP_VAL_REG
)
1942 || ts
->val_type
== TEMP_VAL_MEM
) {
1943 temp_load(s
, ts
, tcg_target_available_regs
[itype
], allocated_regs
);
1946 if (IS_DEAD_ARG(0) && !ots
->fixed_reg
) {
1947 /* mov to a non-saved dead register makes no sense (even with
1948 liveness analysis disabled). */
1949 tcg_debug_assert(NEED_SYNC_ARG(0));
1950 /* The code above should have moved the temp to a register. */
1951 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
1952 if (!ots
->mem_allocated
) {
1953 temp_allocate_frame(s
, args
[0]);
1955 if (ots
->indirect_reg
) {
1956 tcg_regset_set_reg(allocated_regs
, ts
->reg
);
1957 temp_load(s
, ots
->mem_base
,
1958 tcg_target_available_regs
[TCG_TYPE_PTR
],
1961 tcg_out_st(s
, otype
, ts
->reg
, ots
->mem_base
->reg
, ots
->mem_offset
);
1962 if (IS_DEAD_ARG(1)) {
1966 } else if (ts
->val_type
== TEMP_VAL_CONST
) {
1967 /* propagate constant */
1968 if (ots
->val_type
== TEMP_VAL_REG
) {
1969 s
->reg_to_temp
[ots
->reg
] = NULL
;
1971 ots
->val_type
= TEMP_VAL_CONST
;
1973 if (IS_DEAD_ARG(1)) {
1977 /* The code in the first if block should have moved the
1978 temp to a register. */
1979 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
1980 if (IS_DEAD_ARG(1) && !ts
->fixed_reg
&& !ots
->fixed_reg
) {
1981 /* the mov can be suppressed */
1982 if (ots
->val_type
== TEMP_VAL_REG
) {
1983 s
->reg_to_temp
[ots
->reg
] = NULL
;
1988 if (ots
->val_type
!= TEMP_VAL_REG
) {
1989 /* When allocating a new register, make sure to not spill the
1991 tcg_regset_set_reg(allocated_regs
, ts
->reg
);
1992 ots
->reg
= tcg_reg_alloc(s
, tcg_target_available_regs
[otype
],
1993 allocated_regs
, ots
->indirect_base
);
1995 tcg_out_mov(s
, otype
, ots
->reg
, ts
->reg
);
1997 ots
->val_type
= TEMP_VAL_REG
;
1998 ots
->mem_coherent
= 0;
1999 s
->reg_to_temp
[ots
->reg
] = ots
;
2000 if (NEED_SYNC_ARG(0)) {
2001 tcg_reg_sync(s
, ots
->reg
, allocated_regs
);
2006 static void tcg_reg_alloc_op(TCGContext
*s
,
2007 const TCGOpDef
*def
, TCGOpcode opc
,
2008 const TCGArg
*args
, uint16_t dead_args
,
2011 TCGRegSet allocated_regs
;
2012 int i
, k
, nb_iargs
, nb_oargs
;
2015 const TCGArgConstraint
*arg_ct
;
2017 TCGArg new_args
[TCG_MAX_OP_ARGS
];
2018 int const_args
[TCG_MAX_OP_ARGS
];
2020 nb_oargs
= def
->nb_oargs
;
2021 nb_iargs
= def
->nb_iargs
;
2023 /* copy constants */
2024 memcpy(new_args
+ nb_oargs
+ nb_iargs
,
2025 args
+ nb_oargs
+ nb_iargs
,
2026 sizeof(TCGArg
) * def
->nb_cargs
);
2028 /* satisfy input constraints */
2029 tcg_regset_set(allocated_regs
, s
->reserved_regs
);
2030 for(k
= 0; k
< nb_iargs
; k
++) {
2031 i
= def
->sorted_args
[nb_oargs
+ k
];
2033 arg_ct
= &def
->args_ct
[i
];
2034 ts
= &s
->temps
[arg
];
2036 if (ts
->val_type
== TEMP_VAL_CONST
2037 && tcg_target_const_match(ts
->val
, ts
->type
, arg_ct
)) {
2038 /* constant is OK for instruction */
2040 new_args
[i
] = ts
->val
;
2044 temp_load(s
, ts
, arg_ct
->u
.regs
, allocated_regs
);
2046 if (arg_ct
->ct
& TCG_CT_IALIAS
) {
2047 if (ts
->fixed_reg
) {
2048 /* if fixed register, we must allocate a new register
2049 if the alias is not the same register */
2050 if (arg
!= args
[arg_ct
->alias_index
])
2051 goto allocate_in_reg
;
2053 /* if the input is aliased to an output and if it is
2054 not dead after the instruction, we must allocate
2055 a new register and move it */
2056 if (!IS_DEAD_ARG(i
)) {
2057 goto allocate_in_reg
;
2059 /* check if the current register has already been allocated
2060 for another input aliased to an output */
2062 for (k2
= 0 ; k2
< k
; k2
++) {
2063 i2
= def
->sorted_args
[nb_oargs
+ k2
];
2064 if ((def
->args_ct
[i2
].ct
& TCG_CT_IALIAS
) &&
2065 (new_args
[i2
] == ts
->reg
)) {
2066 goto allocate_in_reg
;
2072 if (tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
2073 /* nothing to do : the constraint is satisfied */
2076 /* allocate a new register matching the constraint
2077 and move the temporary register into it */
2078 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, allocated_regs
,
2080 tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
);
2084 tcg_regset_set_reg(allocated_regs
, reg
);
2088 /* mark dead temporaries and free the associated registers */
2089 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2090 if (IS_DEAD_ARG(i
)) {
2091 temp_dead(s
, &s
->temps
[args
[i
]]);
2095 if (def
->flags
& TCG_OPF_BB_END
) {
2096 tcg_reg_alloc_bb_end(s
, allocated_regs
);
2098 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
2099 /* XXX: permit generic clobber register list ? */
2100 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2101 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
2102 tcg_reg_free(s
, i
, allocated_regs
);
2106 if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2107 /* sync globals if the op has side effects and might trigger
2109 sync_globals(s
, allocated_regs
);
2112 /* satisfy the output constraints */
2113 tcg_regset_set(allocated_regs
, s
->reserved_regs
);
2114 for(k
= 0; k
< nb_oargs
; k
++) {
2115 i
= def
->sorted_args
[k
];
2117 arg_ct
= &def
->args_ct
[i
];
2118 ts
= &s
->temps
[arg
];
2119 if (arg_ct
->ct
& TCG_CT_ALIAS
) {
2120 reg
= new_args
[arg_ct
->alias_index
];
2122 /* if fixed register, we try to use it */
2124 if (ts
->fixed_reg
&&
2125 tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
2128 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, allocated_regs
,
2131 tcg_regset_set_reg(allocated_regs
, reg
);
2132 /* if a fixed register is used, then a move will be done afterwards */
2133 if (!ts
->fixed_reg
) {
2134 if (ts
->val_type
== TEMP_VAL_REG
) {
2135 s
->reg_to_temp
[ts
->reg
] = NULL
;
2137 ts
->val_type
= TEMP_VAL_REG
;
2139 /* temp value is modified, so the value kept in memory is
2140 potentially not the same */
2141 ts
->mem_coherent
= 0;
2142 s
->reg_to_temp
[reg
] = ts
;
2149 /* emit instruction */
2150 tcg_out_op(s
, opc
, new_args
, const_args
);
2152 /* move the outputs in the correct register if needed */
2153 for(i
= 0; i
< nb_oargs
; i
++) {
2154 ts
= &s
->temps
[args
[i
]];
2156 if (ts
->fixed_reg
&& ts
->reg
!= reg
) {
2157 tcg_out_mov(s
, ts
->type
, ts
->reg
, reg
);
2159 if (NEED_SYNC_ARG(i
)) {
2160 tcg_reg_sync(s
, reg
, allocated_regs
);
2162 if (IS_DEAD_ARG(i
)) {
2168 #ifdef TCG_TARGET_STACK_GROWSUP
2169 #define STACK_DIR(x) (-(x))
2171 #define STACK_DIR(x) (x)
2174 static void tcg_reg_alloc_call(TCGContext
*s
, int nb_oargs
, int nb_iargs
,
2175 const TCGArg
* const args
, uint16_t dead_args
,
2178 int flags
, nb_regs
, i
;
2182 intptr_t stack_offset
;
2183 size_t call_stack_size
;
2184 tcg_insn_unit
*func_addr
;
2186 TCGRegSet allocated_regs
;
2188 func_addr
= (tcg_insn_unit
*)(intptr_t)args
[nb_oargs
+ nb_iargs
];
2189 flags
= args
[nb_oargs
+ nb_iargs
+ 1];
2191 nb_regs
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
2192 if (nb_regs
> nb_iargs
) {
2196 /* assign stack slots first */
2197 call_stack_size
= (nb_iargs
- nb_regs
) * sizeof(tcg_target_long
);
2198 call_stack_size
= (call_stack_size
+ TCG_TARGET_STACK_ALIGN
- 1) &
2199 ~(TCG_TARGET_STACK_ALIGN
- 1);
2200 allocate_args
= (call_stack_size
> TCG_STATIC_CALL_ARGS_SIZE
);
2201 if (allocate_args
) {
2202 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
2203 preallocate call stack */
2207 stack_offset
= TCG_TARGET_CALL_STACK_OFFSET
;
2208 for(i
= nb_regs
; i
< nb_iargs
; i
++) {
2209 arg
= args
[nb_oargs
+ i
];
2210 #ifdef TCG_TARGET_STACK_GROWSUP
2211 stack_offset
-= sizeof(tcg_target_long
);
2213 if (arg
!= TCG_CALL_DUMMY_ARG
) {
2214 ts
= &s
->temps
[arg
];
2215 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
2217 tcg_out_st(s
, ts
->type
, ts
->reg
, TCG_REG_CALL_STACK
, stack_offset
);
2219 #ifndef TCG_TARGET_STACK_GROWSUP
2220 stack_offset
+= sizeof(tcg_target_long
);
2224 /* assign input registers */
2225 tcg_regset_set(allocated_regs
, s
->reserved_regs
);
2226 for(i
= 0; i
< nb_regs
; i
++) {
2227 arg
= args
[nb_oargs
+ i
];
2228 if (arg
!= TCG_CALL_DUMMY_ARG
) {
2229 ts
= &s
->temps
[arg
];
2230 reg
= tcg_target_call_iarg_regs
[i
];
2231 tcg_reg_free(s
, reg
, allocated_regs
);
2233 if (ts
->val_type
== TEMP_VAL_REG
) {
2234 if (ts
->reg
!= reg
) {
2235 tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
);
2240 tcg_regset_clear(arg_set
);
2241 tcg_regset_set_reg(arg_set
, reg
);
2242 temp_load(s
, ts
, arg_set
, allocated_regs
);
2245 tcg_regset_set_reg(allocated_regs
, reg
);
2249 /* mark dead temporaries and free the associated registers */
2250 for(i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2251 if (IS_DEAD_ARG(i
)) {
2252 temp_dead(s
, &s
->temps
[args
[i
]]);
2256 /* clobber call registers */
2257 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2258 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
2259 tcg_reg_free(s
, i
, allocated_regs
);
2263 /* Save globals if they might be written by the helper, sync them if
2264 they might be read. */
2265 if (flags
& TCG_CALL_NO_READ_GLOBALS
) {
2267 } else if (flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
2268 sync_globals(s
, allocated_regs
);
2270 save_globals(s
, allocated_regs
);
2273 tcg_out_call(s
, func_addr
);
2275 /* assign output registers and emit moves if needed */
2276 for(i
= 0; i
< nb_oargs
; i
++) {
2278 ts
= &s
->temps
[arg
];
2279 reg
= tcg_target_call_oarg_regs
[i
];
2280 tcg_debug_assert(s
->reg_to_temp
[reg
] == NULL
);
2282 if (ts
->fixed_reg
) {
2283 if (ts
->reg
!= reg
) {
2284 tcg_out_mov(s
, ts
->type
, ts
->reg
, reg
);
2287 if (ts
->val_type
== TEMP_VAL_REG
) {
2288 s
->reg_to_temp
[ts
->reg
] = NULL
;
2290 ts
->val_type
= TEMP_VAL_REG
;
2292 ts
->mem_coherent
= 0;
2293 s
->reg_to_temp
[reg
] = ts
;
2294 if (NEED_SYNC_ARG(i
)) {
2295 tcg_reg_sync(s
, reg
, allocated_regs
);
2297 if (IS_DEAD_ARG(i
)) {
2304 #ifdef CONFIG_PROFILER
2306 static int64_t tcg_table_op_count
[NB_OPS
];
2308 void tcg_dump_op_count(FILE *f
, fprintf_function cpu_fprintf
)
2312 for (i
= 0; i
< NB_OPS
; i
++) {
2313 cpu_fprintf(f
, "%s %" PRId64
"\n", tcg_op_defs
[i
].name
,
2314 tcg_table_op_count
[i
]);
2318 void tcg_dump_op_count(FILE *f
, fprintf_function cpu_fprintf
)
2320 cpu_fprintf(f
, "[TCG profiler not compiled]\n");
2325 int tcg_gen_code(TCGContext
*s
, TranslationBlock
*tb
)
2327 int i
, oi
, oi_next
, num_insns
;
2329 #ifdef CONFIG_PROFILER
2333 n
= s
->gen_last_op_idx
+ 1;
2335 if (n
> s
->op_count_max
) {
2336 s
->op_count_max
= n
;
2341 if (n
> s
->temp_count_max
) {
2342 s
->temp_count_max
= n
;
2348 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP
)
2349 && qemu_log_in_addr_range(tb
->pc
))) {
2356 #ifdef CONFIG_PROFILER
2357 s
->opt_time
-= profile_getclock();
2360 #ifdef USE_TCG_OPTIMIZATIONS
2364 #ifdef CONFIG_PROFILER
2365 s
->opt_time
+= profile_getclock();
2366 s
->la_time
-= profile_getclock();
2369 tcg_liveness_analysis(s
);
2371 #ifdef CONFIG_PROFILER
2372 s
->la_time
+= profile_getclock();
2376 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT
)
2377 && qemu_log_in_addr_range(tb
->pc
))) {
2378 qemu_log("OP after optimization and liveness analysis:\n");
2384 tcg_reg_alloc_start(s
);
2386 s
->code_buf
= tb
->tc_ptr
;
2387 s
->code_ptr
= tb
->tc_ptr
;
2392 for (oi
= s
->gen_first_op_idx
; oi
>= 0; oi
= oi_next
) {
2393 TCGOp
* const op
= &s
->gen_op_buf
[oi
];
2394 TCGArg
* const args
= &s
->gen_opparam_buf
[op
->args
];
2395 TCGOpcode opc
= op
->opc
;
2396 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2397 uint16_t dead_args
= s
->op_dead_args
[oi
];
2398 uint8_t sync_args
= s
->op_sync_args
[oi
];
2401 #ifdef CONFIG_PROFILER
2402 tcg_table_op_count
[opc
]++;
2406 case INDEX_op_mov_i32
:
2407 case INDEX_op_mov_i64
:
2408 tcg_reg_alloc_mov(s
, def
, args
, dead_args
, sync_args
);
2410 case INDEX_op_movi_i32
:
2411 case INDEX_op_movi_i64
:
2412 tcg_reg_alloc_movi(s
, args
, dead_args
, sync_args
);
2414 case INDEX_op_insn_start
:
2415 if (num_insns
>= 0) {
2416 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
2419 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
2421 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2422 a
= ((target_ulong
)args
[i
* 2 + 1] << 32) | args
[i
* 2];
2426 s
->gen_insn_data
[num_insns
][i
] = a
;
2429 case INDEX_op_discard
:
2430 temp_dead(s
, &s
->temps
[args
[0]]);
2432 case INDEX_op_set_label
:
2433 tcg_reg_alloc_bb_end(s
, s
->reserved_regs
);
2434 tcg_out_label(s
, arg_label(args
[0]), s
->code_ptr
);
2437 tcg_reg_alloc_call(s
, op
->callo
, op
->calli
, args
,
2438 dead_args
, sync_args
);
2441 /* Sanity check that we've not introduced any unhandled opcodes. */
2442 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
2445 /* Note: in order to speed up the code, it would be much
2446 faster to have specialized register allocator functions for
2447 some common argument patterns */
2448 tcg_reg_alloc_op(s
, def
, opc
, args
, dead_args
, sync_args
);
2451 #ifdef CONFIG_DEBUG_TCG
2454 /* Test for (pending) buffer overflow. The assumption is that any
2455 one operation beginning below the high water mark cannot overrun
2456 the buffer completely. Thus we can test for overflow after
2457 generating code without having to check during generation. */
2458 if (unlikely((void *)s
->code_ptr
> s
->code_gen_highwater
)) {
2462 tcg_debug_assert(num_insns
>= 0);
2463 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
2465 /* Generate TB finalization at the end of block */
2466 if (!tcg_out_tb_finalize(s
)) {
2470 /* flush instruction cache */
2471 flush_icache_range((uintptr_t)s
->code_buf
, (uintptr_t)s
->code_ptr
);
2473 return tcg_current_code_size(s
);
2476 #ifdef CONFIG_PROFILER
2477 void tcg_dump_info(FILE *f
, fprintf_function cpu_fprintf
)
2479 TCGContext
*s
= &tcg_ctx
;
2480 int64_t tb_count
= s
->tb_count
;
2481 int64_t tb_div_count
= tb_count
? tb_count
: 1;
2482 int64_t tot
= s
->interm_time
+ s
->code_time
;
2484 cpu_fprintf(f
, "JIT cycles %" PRId64
" (%0.3f s at 2.4 GHz)\n",
2486 cpu_fprintf(f
, "translated TBs %" PRId64
" (aborted=%" PRId64
" %0.1f%%)\n",
2487 tb_count
, s
->tb_count1
- tb_count
,
2488 (double)(s
->tb_count1
- s
->tb_count
)
2489 / (s
->tb_count1
? s
->tb_count1
: 1) * 100.0);
2490 cpu_fprintf(f
, "avg ops/TB %0.1f max=%d\n",
2491 (double)s
->op_count
/ tb_div_count
, s
->op_count_max
);
2492 cpu_fprintf(f
, "deleted ops/TB %0.2f\n",
2493 (double)s
->del_op_count
/ tb_div_count
);
2494 cpu_fprintf(f
, "avg temps/TB %0.2f max=%d\n",
2495 (double)s
->temp_count
/ tb_div_count
, s
->temp_count_max
);
2496 cpu_fprintf(f
, "avg host code/TB %0.1f\n",
2497 (double)s
->code_out_len
/ tb_div_count
);
2498 cpu_fprintf(f
, "avg search data/TB %0.1f\n",
2499 (double)s
->search_out_len
/ tb_div_count
);
2501 cpu_fprintf(f
, "cycles/op %0.1f\n",
2502 s
->op_count
? (double)tot
/ s
->op_count
: 0);
2503 cpu_fprintf(f
, "cycles/in byte %0.1f\n",
2504 s
->code_in_len
? (double)tot
/ s
->code_in_len
: 0);
2505 cpu_fprintf(f
, "cycles/out byte %0.1f\n",
2506 s
->code_out_len
? (double)tot
/ s
->code_out_len
: 0);
2507 cpu_fprintf(f
, "cycles/search byte %0.1f\n",
2508 s
->search_out_len
? (double)tot
/ s
->search_out_len
: 0);
2512 cpu_fprintf(f
, " gen_interm time %0.1f%%\n",
2513 (double)s
->interm_time
/ tot
* 100.0);
2514 cpu_fprintf(f
, " gen_code time %0.1f%%\n",
2515 (double)s
->code_time
/ tot
* 100.0);
2516 cpu_fprintf(f
, "optim./code time %0.1f%%\n",
2517 (double)s
->opt_time
/ (s
->code_time
? s
->code_time
: 1)
2519 cpu_fprintf(f
, "liveness/code time %0.1f%%\n",
2520 (double)s
->la_time
/ (s
->code_time
? s
->code_time
: 1) * 100.0);
2521 cpu_fprintf(f
, "cpu_restore count %" PRId64
"\n",
2523 cpu_fprintf(f
, " avg cycles %0.1f\n",
2524 s
->restore_count
? (double)s
->restore_time
/ s
->restore_count
: 0);
2527 void tcg_dump_info(FILE *f
, fprintf_function cpu_fprintf
)
2529 cpu_fprintf(f
, "[TCG profiler not compiled]\n");
2533 #ifdef ELF_HOST_MACHINE
2534 /* In order to use this feature, the backend needs to do three things:
2536 (1) Define ELF_HOST_MACHINE to indicate both what value to
2537 put into the ELF image and to indicate support for the feature.
2539 (2) Define tcg_register_jit. This should create a buffer containing
2540 the contents of a .debug_frame section that describes the post-
2541 prologue unwind info for the tcg machine.
2543 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
2546 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
2553 struct jit_code_entry
{
2554 struct jit_code_entry
*next_entry
;
2555 struct jit_code_entry
*prev_entry
;
2556 const void *symfile_addr
;
2557 uint64_t symfile_size
;
2560 struct jit_descriptor
{
2562 uint32_t action_flag
;
2563 struct jit_code_entry
*relevant_entry
;
2564 struct jit_code_entry
*first_entry
;
2567 void __jit_debug_register_code(void) __attribute__((noinline
));
2568 void __jit_debug_register_code(void)
2573 /* Must statically initialize the version, because GDB may check
2574 the version before we can set it. */
2575 struct jit_descriptor __jit_debug_descriptor
= { 1, 0, 0, 0 };
2577 /* End GDB interface. */
2579 static int find_string(const char *strtab
, const char *str
)
2581 const char *p
= strtab
+ 1;
2584 if (strcmp(p
, str
) == 0) {
2591 static void tcg_register_jit_int(void *buf_ptr
, size_t buf_size
,
2592 const void *debug_frame
,
2593 size_t debug_frame_size
)
2595 struct __attribute__((packed
)) DebugInfo
{
2602 uintptr_t cu_low_pc
;
2603 uintptr_t cu_high_pc
;
2606 uintptr_t fn_low_pc
;
2607 uintptr_t fn_high_pc
;
2616 struct DebugInfo di
;
2621 struct ElfImage
*img
;
2623 static const struct ElfImage img_template
= {
2625 .e_ident
[EI_MAG0
] = ELFMAG0
,
2626 .e_ident
[EI_MAG1
] = ELFMAG1
,
2627 .e_ident
[EI_MAG2
] = ELFMAG2
,
2628 .e_ident
[EI_MAG3
] = ELFMAG3
,
2629 .e_ident
[EI_CLASS
] = ELF_CLASS
,
2630 .e_ident
[EI_DATA
] = ELF_DATA
,
2631 .e_ident
[EI_VERSION
] = EV_CURRENT
,
2633 .e_machine
= ELF_HOST_MACHINE
,
2634 .e_version
= EV_CURRENT
,
2635 .e_phoff
= offsetof(struct ElfImage
, phdr
),
2636 .e_shoff
= offsetof(struct ElfImage
, shdr
),
2637 .e_ehsize
= sizeof(ElfW(Shdr
)),
2638 .e_phentsize
= sizeof(ElfW(Phdr
)),
2640 .e_shentsize
= sizeof(ElfW(Shdr
)),
2641 .e_shnum
= ARRAY_SIZE(img
->shdr
),
2642 .e_shstrndx
= ARRAY_SIZE(img
->shdr
) - 1,
2643 #ifdef ELF_HOST_FLAGS
2644 .e_flags
= ELF_HOST_FLAGS
,
2647 .e_ident
[EI_OSABI
] = ELF_OSABI
,
2655 [0] = { .sh_type
= SHT_NULL
},
2656 /* Trick: The contents of code_gen_buffer are not present in
2657 this fake ELF file; that got allocated elsewhere. Therefore
2658 we mark .text as SHT_NOBITS (similar to .bss) so that readers
2659 will not look for contents. We can record any address. */
2661 .sh_type
= SHT_NOBITS
,
2662 .sh_flags
= SHF_EXECINSTR
| SHF_ALLOC
,
2664 [2] = { /* .debug_info */
2665 .sh_type
= SHT_PROGBITS
,
2666 .sh_offset
= offsetof(struct ElfImage
, di
),
2667 .sh_size
= sizeof(struct DebugInfo
),
2669 [3] = { /* .debug_abbrev */
2670 .sh_type
= SHT_PROGBITS
,
2671 .sh_offset
= offsetof(struct ElfImage
, da
),
2672 .sh_size
= sizeof(img
->da
),
2674 [4] = { /* .debug_frame */
2675 .sh_type
= SHT_PROGBITS
,
2676 .sh_offset
= sizeof(struct ElfImage
),
2678 [5] = { /* .symtab */
2679 .sh_type
= SHT_SYMTAB
,
2680 .sh_offset
= offsetof(struct ElfImage
, sym
),
2681 .sh_size
= sizeof(img
->sym
),
2683 .sh_link
= ARRAY_SIZE(img
->shdr
) - 1,
2684 .sh_entsize
= sizeof(ElfW(Sym
)),
2686 [6] = { /* .strtab */
2687 .sh_type
= SHT_STRTAB
,
2688 .sh_offset
= offsetof(struct ElfImage
, str
),
2689 .sh_size
= sizeof(img
->str
),
2693 [1] = { /* code_gen_buffer */
2694 .st_info
= ELF_ST_INFO(STB_GLOBAL
, STT_FUNC
),
2699 .len
= sizeof(struct DebugInfo
) - 4,
2701 .ptr_size
= sizeof(void *),
2703 .cu_lang
= 0x8001, /* DW_LANG_Mips_Assembler */
2705 .fn_name
= "code_gen_buffer"
2708 1, /* abbrev number (the cu) */
2709 0x11, 1, /* DW_TAG_compile_unit, has children */
2710 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
2711 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
2712 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
2713 0, 0, /* end of abbrev */
2714 2, /* abbrev number (the fn) */
2715 0x2e, 0, /* DW_TAG_subprogram, no children */
2716 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
2717 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
2718 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
2719 0, 0, /* end of abbrev */
2720 0 /* no more abbrev */
2722 .str
= "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
2723 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
2726 /* We only need a single jit entry; statically allocate it. */
2727 static struct jit_code_entry one_entry
;
2729 uintptr_t buf
= (uintptr_t)buf_ptr
;
2730 size_t img_size
= sizeof(struct ElfImage
) + debug_frame_size
;
2731 DebugFrameHeader
*dfh
;
2733 img
= g_malloc(img_size
);
2734 *img
= img_template
;
2736 img
->phdr
.p_vaddr
= buf
;
2737 img
->phdr
.p_paddr
= buf
;
2738 img
->phdr
.p_memsz
= buf_size
;
2740 img
->shdr
[1].sh_name
= find_string(img
->str
, ".text");
2741 img
->shdr
[1].sh_addr
= buf
;
2742 img
->shdr
[1].sh_size
= buf_size
;
2744 img
->shdr
[2].sh_name
= find_string(img
->str
, ".debug_info");
2745 img
->shdr
[3].sh_name
= find_string(img
->str
, ".debug_abbrev");
2747 img
->shdr
[4].sh_name
= find_string(img
->str
, ".debug_frame");
2748 img
->shdr
[4].sh_size
= debug_frame_size
;
2750 img
->shdr
[5].sh_name
= find_string(img
->str
, ".symtab");
2751 img
->shdr
[6].sh_name
= find_string(img
->str
, ".strtab");
2753 img
->sym
[1].st_name
= find_string(img
->str
, "code_gen_buffer");
2754 img
->sym
[1].st_value
= buf
;
2755 img
->sym
[1].st_size
= buf_size
;
2757 img
->di
.cu_low_pc
= buf
;
2758 img
->di
.cu_high_pc
= buf
+ buf_size
;
2759 img
->di
.fn_low_pc
= buf
;
2760 img
->di
.fn_high_pc
= buf
+ buf_size
;
2762 dfh
= (DebugFrameHeader
*)(img
+ 1);
2763 memcpy(dfh
, debug_frame
, debug_frame_size
);
2764 dfh
->fde
.func_start
= buf
;
2765 dfh
->fde
.func_len
= buf_size
;
2768 /* Enable this block to be able to debug the ELF image file creation.
2769 One can use readelf, objdump, or other inspection utilities. */
2771 FILE *f
= fopen("/tmp/qemu.jit", "w+b");
2773 if (fwrite(img
, img_size
, 1, f
) != img_size
) {
2774 /* Avoid stupid unused return value warning for fwrite. */
2781 one_entry
.symfile_addr
= img
;
2782 one_entry
.symfile_size
= img_size
;
2784 __jit_debug_descriptor
.action_flag
= JIT_REGISTER_FN
;
2785 __jit_debug_descriptor
.relevant_entry
= &one_entry
;
2786 __jit_debug_descriptor
.first_entry
= &one_entry
;
2787 __jit_debug_register_code();
2790 /* No support for the feature. Provide the entry point expected by exec.c,
2791 and implement the internal function we declared earlier. */
2793 static void tcg_register_jit_int(void *buf
, size_t size
,
2794 const void *debug_frame
,
2795 size_t debug_frame_size
)
2799 void tcg_register_jit(void *buf
, size_t buf_size
)
2802 #endif /* ELF_HOST_MACHINE */