2 * Optimizations for Tiny Code Generator for QEMU
4 * Copyright (c) 2010 Samsung Electronics.
5 * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 #include "qemu/osdep.h"
27 #include "tcg/tcg-op.h"
28 #include "tcg-internal.h"
30 #define CASE_OP_32_64(x) \
31 glue(glue(case INDEX_op_, x), _i32): \
32 glue(glue(case INDEX_op_, x), _i64)
34 #define CASE_OP_32_64_VEC(x) \
35 glue(glue(case INDEX_op_, x), _i32): \
36 glue(glue(case INDEX_op_, x), _i64): \
37 glue(glue(case INDEX_op_, x), _vec)
39 typedef struct TempOptInfo
{
47 static inline TempOptInfo
*ts_info(TCGTemp
*ts
)
52 static inline TempOptInfo
*arg_info(TCGArg arg
)
54 return ts_info(arg_temp(arg
));
57 static inline bool ts_is_const(TCGTemp
*ts
)
59 return ts_info(ts
)->is_const
;
62 static inline bool arg_is_const(TCGArg arg
)
64 return ts_is_const(arg_temp(arg
));
67 static inline bool ts_is_copy(TCGTemp
*ts
)
69 return ts_info(ts
)->next_copy
!= ts
;
72 /* Reset TEMP's state, possibly removing the temp for the list of copies. */
73 static void reset_ts(TCGTemp
*ts
)
75 TempOptInfo
*ti
= ts_info(ts
);
76 TempOptInfo
*pi
= ts_info(ti
->prev_copy
);
77 TempOptInfo
*ni
= ts_info(ti
->next_copy
);
79 ni
->prev_copy
= ti
->prev_copy
;
80 pi
->next_copy
= ti
->next_copy
;
87 static void reset_temp(TCGArg arg
)
89 reset_ts(arg_temp(arg
));
92 /* Initialize and activate a temporary. */
93 static void init_ts_info(TCGTempSet
*temps_used
, TCGTemp
*ts
)
95 size_t idx
= temp_idx(ts
);
98 if (test_bit(idx
, temps_used
->l
)) {
101 set_bit(idx
, temps_used
->l
);
105 ti
= tcg_malloc(sizeof(TempOptInfo
));
111 if (ts
->kind
== TEMP_CONST
) {
115 if (TCG_TARGET_REG_BITS
> 32 && ts
->type
== TCG_TYPE_I32
) {
116 /* High bits of a 32-bit quantity are garbage. */
117 ti
->mask
|= ~0xffffffffull
;
120 ti
->is_const
= false;
125 static void init_arg_info(TCGTempSet
*temps_used
, TCGArg arg
)
127 init_ts_info(temps_used
, arg_temp(arg
));
130 static TCGTemp
*find_better_copy(TCGContext
*s
, TCGTemp
*ts
)
134 /* If this is already readonly, we can't do better. */
135 if (temp_readonly(ts
)) {
140 for (i
= ts_info(ts
)->next_copy
; i
!= ts
; i
= ts_info(i
)->next_copy
) {
141 if (temp_readonly(i
)) {
143 } else if (i
->kind
> ts
->kind
) {
144 if (i
->kind
== TEMP_GLOBAL
) {
146 } else if (i
->kind
== TEMP_LOCAL
) {
152 /* If we didn't find a better representation, return the same temp. */
153 return g
? g
: l
? l
: ts
;
156 static bool ts_are_copies(TCGTemp
*ts1
, TCGTemp
*ts2
)
164 if (!ts_is_copy(ts1
) || !ts_is_copy(ts2
)) {
168 for (i
= ts_info(ts1
)->next_copy
; i
!= ts1
; i
= ts_info(i
)->next_copy
) {
177 static bool args_are_copies(TCGArg arg1
, TCGArg arg2
)
179 return ts_are_copies(arg_temp(arg1
), arg_temp(arg2
));
182 static void tcg_opt_gen_mov(TCGContext
*s
, TCGOp
*op
, TCGArg dst
, TCGArg src
)
184 TCGTemp
*dst_ts
= arg_temp(dst
);
185 TCGTemp
*src_ts
= arg_temp(src
);
192 if (ts_are_copies(dst_ts
, src_ts
)) {
193 tcg_op_remove(s
, op
);
198 di
= ts_info(dst_ts
);
199 si
= ts_info(src_ts
);
200 def
= &tcg_op_defs
[op
->opc
];
201 if (def
->flags
& TCG_OPF_VECTOR
) {
202 new_op
= INDEX_op_mov_vec
;
203 } else if (def
->flags
& TCG_OPF_64BIT
) {
204 new_op
= INDEX_op_mov_i64
;
206 new_op
= INDEX_op_mov_i32
;
209 /* TCGOP_VECL and TCGOP_VECE remain unchanged. */
214 if (TCG_TARGET_REG_BITS
> 32 && new_op
== INDEX_op_mov_i32
) {
215 /* High bits of the destination are now garbage. */
216 mask
|= ~0xffffffffull
;
220 if (src_ts
->type
== dst_ts
->type
) {
221 TempOptInfo
*ni
= ts_info(si
->next_copy
);
223 di
->next_copy
= si
->next_copy
;
224 di
->prev_copy
= src_ts
;
225 ni
->prev_copy
= dst_ts
;
226 si
->next_copy
= dst_ts
;
227 di
->is_const
= si
->is_const
;
232 static void tcg_opt_gen_movi(TCGContext
*s
, TCGTempSet
*temps_used
,
233 TCGOp
*op
, TCGArg dst
, uint64_t val
)
235 const TCGOpDef
*def
= &tcg_op_defs
[op
->opc
];
239 if (def
->flags
& TCG_OPF_VECTOR
) {
240 type
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
241 } else if (def
->flags
& TCG_OPF_64BIT
) {
247 /* Convert movi to mov with constant temp. */
248 tv
= tcg_constant_internal(type
, val
);
249 init_ts_info(temps_used
, tv
);
250 tcg_opt_gen_mov(s
, op
, dst
, temp_arg(tv
));
253 static uint64_t do_constant_folding_2(TCGOpcode op
, uint64_t x
, uint64_t y
)
276 case INDEX_op_shl_i32
:
277 return (uint32_t)x
<< (y
& 31);
279 case INDEX_op_shl_i64
:
280 return (uint64_t)x
<< (y
& 63);
282 case INDEX_op_shr_i32
:
283 return (uint32_t)x
>> (y
& 31);
285 case INDEX_op_shr_i64
:
286 return (uint64_t)x
>> (y
& 63);
288 case INDEX_op_sar_i32
:
289 return (int32_t)x
>> (y
& 31);
291 case INDEX_op_sar_i64
:
292 return (int64_t)x
>> (y
& 63);
294 case INDEX_op_rotr_i32
:
295 return ror32(x
, y
& 31);
297 case INDEX_op_rotr_i64
:
298 return ror64(x
, y
& 63);
300 case INDEX_op_rotl_i32
:
301 return rol32(x
, y
& 31);
303 case INDEX_op_rotl_i64
:
304 return rol64(x
, y
& 63);
327 case INDEX_op_clz_i32
:
328 return (uint32_t)x
? clz32(x
) : y
;
330 case INDEX_op_clz_i64
:
331 return x
? clz64(x
) : y
;
333 case INDEX_op_ctz_i32
:
334 return (uint32_t)x
? ctz32(x
) : y
;
336 case INDEX_op_ctz_i64
:
337 return x
? ctz64(x
) : y
;
339 case INDEX_op_ctpop_i32
:
342 case INDEX_op_ctpop_i64
:
345 CASE_OP_32_64(ext8s
):
348 CASE_OP_32_64(ext16s
):
351 CASE_OP_32_64(ext8u
):
354 CASE_OP_32_64(ext16u
):
357 CASE_OP_32_64(bswap16
):
359 return y
& TCG_BSWAP_OS
? (int16_t)x
: x
;
361 CASE_OP_32_64(bswap32
):
363 return y
& TCG_BSWAP_OS
? (int32_t)x
: x
;
365 case INDEX_op_bswap64_i64
:
368 case INDEX_op_ext_i32_i64
:
369 case INDEX_op_ext32s_i64
:
372 case INDEX_op_extu_i32_i64
:
373 case INDEX_op_extrl_i64_i32
:
374 case INDEX_op_ext32u_i64
:
377 case INDEX_op_extrh_i64_i32
:
378 return (uint64_t)x
>> 32;
380 case INDEX_op_muluh_i32
:
381 return ((uint64_t)(uint32_t)x
* (uint32_t)y
) >> 32;
382 case INDEX_op_mulsh_i32
:
383 return ((int64_t)(int32_t)x
* (int32_t)y
) >> 32;
385 case INDEX_op_muluh_i64
:
386 mulu64(&l64
, &h64
, x
, y
);
388 case INDEX_op_mulsh_i64
:
389 muls64(&l64
, &h64
, x
, y
);
392 case INDEX_op_div_i32
:
393 /* Avoid crashing on divide by zero, otherwise undefined. */
394 return (int32_t)x
/ ((int32_t)y
? : 1);
395 case INDEX_op_divu_i32
:
396 return (uint32_t)x
/ ((uint32_t)y
? : 1);
397 case INDEX_op_div_i64
:
398 return (int64_t)x
/ ((int64_t)y
? : 1);
399 case INDEX_op_divu_i64
:
400 return (uint64_t)x
/ ((uint64_t)y
? : 1);
402 case INDEX_op_rem_i32
:
403 return (int32_t)x
% ((int32_t)y
? : 1);
404 case INDEX_op_remu_i32
:
405 return (uint32_t)x
% ((uint32_t)y
? : 1);
406 case INDEX_op_rem_i64
:
407 return (int64_t)x
% ((int64_t)y
? : 1);
408 case INDEX_op_remu_i64
:
409 return (uint64_t)x
% ((uint64_t)y
? : 1);
413 "Unrecognized operation %d in do_constant_folding.\n", op
);
418 static uint64_t do_constant_folding(TCGOpcode op
, uint64_t x
, uint64_t y
)
420 const TCGOpDef
*def
= &tcg_op_defs
[op
];
421 uint64_t res
= do_constant_folding_2(op
, x
, y
);
422 if (!(def
->flags
& TCG_OPF_64BIT
)) {
428 static bool do_constant_folding_cond_32(uint32_t x
, uint32_t y
, TCGCond c
)
436 return (int32_t)x
< (int32_t)y
;
438 return (int32_t)x
>= (int32_t)y
;
440 return (int32_t)x
<= (int32_t)y
;
442 return (int32_t)x
> (int32_t)y
;
456 static bool do_constant_folding_cond_64(uint64_t x
, uint64_t y
, TCGCond c
)
464 return (int64_t)x
< (int64_t)y
;
466 return (int64_t)x
>= (int64_t)y
;
468 return (int64_t)x
<= (int64_t)y
;
470 return (int64_t)x
> (int64_t)y
;
484 static bool do_constant_folding_cond_eq(TCGCond c
)
504 /* Return 2 if the condition can't be simplified, and the result
505 of the condition (0 or 1) if it can */
506 static TCGArg
do_constant_folding_cond(TCGOpcode op
, TCGArg x
,
509 uint64_t xv
= arg_info(x
)->val
;
510 uint64_t yv
= arg_info(y
)->val
;
512 if (arg_is_const(x
) && arg_is_const(y
)) {
513 const TCGOpDef
*def
= &tcg_op_defs
[op
];
514 tcg_debug_assert(!(def
->flags
& TCG_OPF_VECTOR
));
515 if (def
->flags
& TCG_OPF_64BIT
) {
516 return do_constant_folding_cond_64(xv
, yv
, c
);
518 return do_constant_folding_cond_32(xv
, yv
, c
);
520 } else if (args_are_copies(x
, y
)) {
521 return do_constant_folding_cond_eq(c
);
522 } else if (arg_is_const(y
) && yv
== 0) {
535 /* Return 2 if the condition can't be simplified, and the result
536 of the condition (0 or 1) if it can */
537 static TCGArg
do_constant_folding_cond2(TCGArg
*p1
, TCGArg
*p2
, TCGCond c
)
539 TCGArg al
= p1
[0], ah
= p1
[1];
540 TCGArg bl
= p2
[0], bh
= p2
[1];
542 if (arg_is_const(bl
) && arg_is_const(bh
)) {
543 tcg_target_ulong blv
= arg_info(bl
)->val
;
544 tcg_target_ulong bhv
= arg_info(bh
)->val
;
545 uint64_t b
= deposit64(blv
, 32, 32, bhv
);
547 if (arg_is_const(al
) && arg_is_const(ah
)) {
548 tcg_target_ulong alv
= arg_info(al
)->val
;
549 tcg_target_ulong ahv
= arg_info(ah
)->val
;
550 uint64_t a
= deposit64(alv
, 32, 32, ahv
);
551 return do_constant_folding_cond_64(a
, b
, c
);
564 if (args_are_copies(al
, bl
) && args_are_copies(ah
, bh
)) {
565 return do_constant_folding_cond_eq(c
);
570 static bool swap_commutative(TCGArg dest
, TCGArg
*p1
, TCGArg
*p2
)
572 TCGArg a1
= *p1
, a2
= *p2
;
574 sum
+= arg_is_const(a1
);
575 sum
-= arg_is_const(a2
);
577 /* Prefer the constant in second argument, and then the form
578 op a, a, b, which is better handled on non-RISC hosts. */
579 if (sum
> 0 || (sum
== 0 && dest
== a2
)) {
587 static bool swap_commutative2(TCGArg
*p1
, TCGArg
*p2
)
590 sum
+= arg_is_const(p1
[0]);
591 sum
+= arg_is_const(p1
[1]);
592 sum
-= arg_is_const(p2
[0]);
593 sum
-= arg_is_const(p2
[1]);
596 t
= p1
[0], p1
[0] = p2
[0], p2
[0] = t
;
597 t
= p1
[1], p1
[1] = p2
[1], p2
[1] = t
;
603 /* Propagate constants and copies, fold constant expressions. */
604 void tcg_optimize(TCGContext
*s
)
606 int nb_temps
, nb_globals
, i
;
607 TCGOp
*op
, *op_next
, *prev_mb
= NULL
;
608 TCGTempSet temps_used
;
610 /* Array VALS has an element for each temp.
611 If this temp holds a constant then its value is kept in VALS' element.
612 If this temp is a copy of other ones then the other copies are
613 available through the doubly linked circular list. */
615 nb_temps
= s
->nb_temps
;
616 nb_globals
= s
->nb_globals
;
618 memset(&temps_used
, 0, sizeof(temps_used
));
619 for (i
= 0; i
< nb_temps
; ++i
) {
620 s
->temps
[i
].state_ptr
= NULL
;
623 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
624 uint64_t mask
, partmask
, affected
, tmp
;
625 int nb_oargs
, nb_iargs
;
626 TCGOpcode opc
= op
->opc
;
627 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
629 /* Count the arguments, and initialize the temps that are
631 if (opc
== INDEX_op_call
) {
632 nb_oargs
= TCGOP_CALLO(op
);
633 nb_iargs
= TCGOP_CALLI(op
);
634 for (i
= 0; i
< nb_oargs
+ nb_iargs
; i
++) {
635 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
637 init_ts_info(&temps_used
, ts
);
641 nb_oargs
= def
->nb_oargs
;
642 nb_iargs
= def
->nb_iargs
;
643 for (i
= 0; i
< nb_oargs
+ nb_iargs
; i
++) {
644 init_arg_info(&temps_used
, op
->args
[i
]);
648 /* Do copy propagation */
649 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
650 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
651 if (ts
&& ts_is_copy(ts
)) {
652 op
->args
[i
] = temp_arg(find_better_copy(s
, ts
));
656 /* For commutative operations make constant second argument */
658 CASE_OP_32_64_VEC(add
):
659 CASE_OP_32_64_VEC(mul
):
660 CASE_OP_32_64_VEC(and):
661 CASE_OP_32_64_VEC(or):
662 CASE_OP_32_64_VEC(xor):
666 CASE_OP_32_64(muluh
):
667 CASE_OP_32_64(mulsh
):
668 swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2]);
670 CASE_OP_32_64(brcond
):
671 if (swap_commutative(-1, &op
->args
[0], &op
->args
[1])) {
672 op
->args
[2] = tcg_swap_cond(op
->args
[2]);
675 CASE_OP_32_64(setcond
):
676 if (swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2])) {
677 op
->args
[3] = tcg_swap_cond(op
->args
[3]);
680 CASE_OP_32_64(movcond
):
681 if (swap_commutative(-1, &op
->args
[1], &op
->args
[2])) {
682 op
->args
[5] = tcg_swap_cond(op
->args
[5]);
684 /* For movcond, we canonicalize the "false" input reg to match
685 the destination reg so that the tcg backend can implement
686 a "move if true" operation. */
687 if (swap_commutative(op
->args
[0], &op
->args
[4], &op
->args
[3])) {
688 op
->args
[5] = tcg_invert_cond(op
->args
[5]);
692 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[4]);
693 swap_commutative(op
->args
[1], &op
->args
[3], &op
->args
[5]);
695 CASE_OP_32_64(mulu2
):
696 CASE_OP_32_64(muls2
):
697 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[3]);
699 case INDEX_op_brcond2_i32
:
700 if (swap_commutative2(&op
->args
[0], &op
->args
[2])) {
701 op
->args
[4] = tcg_swap_cond(op
->args
[4]);
704 case INDEX_op_setcond2_i32
:
705 if (swap_commutative2(&op
->args
[1], &op
->args
[3])) {
706 op
->args
[5] = tcg_swap_cond(op
->args
[5]);
713 /* Simplify expressions for "shift/rot r, 0, a => movi r, 0",
714 and "sub r, 0, a => neg r, a" case. */
721 if (arg_is_const(op
->args
[1])
722 && arg_info(op
->args
[1])->val
== 0) {
723 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], 0);
727 CASE_OP_32_64_VEC(sub
):
732 if (arg_is_const(op
->args
[2])) {
733 /* Proceed with possible constant folding. */
736 if (opc
== INDEX_op_sub_i32
) {
737 neg_op
= INDEX_op_neg_i32
;
738 have_neg
= TCG_TARGET_HAS_neg_i32
;
739 } else if (opc
== INDEX_op_sub_i64
) {
740 neg_op
= INDEX_op_neg_i64
;
741 have_neg
= TCG_TARGET_HAS_neg_i64
;
742 } else if (TCG_TARGET_HAS_neg_vec
) {
743 TCGType type
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
744 unsigned vece
= TCGOP_VECE(op
);
745 neg_op
= INDEX_op_neg_vec
;
746 have_neg
= tcg_can_emit_vec_op(neg_op
, type
, vece
) > 0;
753 if (arg_is_const(op
->args
[1])
754 && arg_info(op
->args
[1])->val
== 0) {
756 reset_temp(op
->args
[0]);
757 op
->args
[1] = op
->args
[2];
762 CASE_OP_32_64_VEC(xor):
764 if (!arg_is_const(op
->args
[1])
765 && arg_is_const(op
->args
[2])
766 && arg_info(op
->args
[2])->val
== -1) {
772 if (!arg_is_const(op
->args
[1])
773 && arg_is_const(op
->args
[2])
774 && arg_info(op
->args
[2])->val
== 0) {
779 CASE_OP_32_64_VEC(andc
):
780 if (!arg_is_const(op
->args
[2])
781 && arg_is_const(op
->args
[1])
782 && arg_info(op
->args
[1])->val
== -1) {
787 CASE_OP_32_64_VEC(orc
):
789 if (!arg_is_const(op
->args
[2])
790 && arg_is_const(op
->args
[1])
791 && arg_info(op
->args
[1])->val
== 0) {
801 if (def
->flags
& TCG_OPF_VECTOR
) {
802 not_op
= INDEX_op_not_vec
;
803 have_not
= TCG_TARGET_HAS_not_vec
;
804 } else if (def
->flags
& TCG_OPF_64BIT
) {
805 not_op
= INDEX_op_not_i64
;
806 have_not
= TCG_TARGET_HAS_not_i64
;
808 not_op
= INDEX_op_not_i32
;
809 have_not
= TCG_TARGET_HAS_not_i32
;
815 reset_temp(op
->args
[0]);
816 op
->args
[1] = op
->args
[i
];
823 /* Simplify expression for "op r, a, const => mov r, a" cases */
825 CASE_OP_32_64_VEC(add
):
826 CASE_OP_32_64_VEC(sub
):
827 CASE_OP_32_64_VEC(or):
828 CASE_OP_32_64_VEC(xor):
829 CASE_OP_32_64_VEC(andc
):
835 if (!arg_is_const(op
->args
[1])
836 && arg_is_const(op
->args
[2])
837 && arg_info(op
->args
[2])->val
== 0) {
838 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
842 CASE_OP_32_64_VEC(and):
843 CASE_OP_32_64_VEC(orc
):
845 if (!arg_is_const(op
->args
[1])
846 && arg_is_const(op
->args
[2])
847 && arg_info(op
->args
[2])->val
== -1) {
848 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
856 /* Simplify using known-zero bits. Currently only ops with a single
857 output argument is supported. */
861 CASE_OP_32_64(ext8s
):
862 if ((arg_info(op
->args
[1])->mask
& 0x80) != 0) {
866 CASE_OP_32_64(ext8u
):
869 CASE_OP_32_64(ext16s
):
870 if ((arg_info(op
->args
[1])->mask
& 0x8000) != 0) {
874 CASE_OP_32_64(ext16u
):
877 case INDEX_op_ext32s_i64
:
878 if ((arg_info(op
->args
[1])->mask
& 0x80000000) != 0) {
882 case INDEX_op_ext32u_i64
:
887 mask
= arg_info(op
->args
[2])->mask
;
888 if (arg_is_const(op
->args
[2])) {
890 affected
= arg_info(op
->args
[1])->mask
& ~mask
;
892 mask
= arg_info(op
->args
[1])->mask
& mask
;
895 case INDEX_op_ext_i32_i64
:
896 if ((arg_info(op
->args
[1])->mask
& 0x80000000) != 0) {
900 case INDEX_op_extu_i32_i64
:
901 /* We do not compute affected as it is a size changing op. */
902 mask
= (uint32_t)arg_info(op
->args
[1])->mask
;
906 /* Known-zeros does not imply known-ones. Therefore unless
907 op->args[2] is constant, we can't infer anything from it. */
908 if (arg_is_const(op
->args
[2])) {
909 mask
= ~arg_info(op
->args
[2])->mask
;
912 /* But we certainly know nothing outside args[1] may be set. */
913 mask
= arg_info(op
->args
[1])->mask
;
916 case INDEX_op_sar_i32
:
917 if (arg_is_const(op
->args
[2])) {
918 tmp
= arg_info(op
->args
[2])->val
& 31;
919 mask
= (int32_t)arg_info(op
->args
[1])->mask
>> tmp
;
922 case INDEX_op_sar_i64
:
923 if (arg_is_const(op
->args
[2])) {
924 tmp
= arg_info(op
->args
[2])->val
& 63;
925 mask
= (int64_t)arg_info(op
->args
[1])->mask
>> tmp
;
929 case INDEX_op_shr_i32
:
930 if (arg_is_const(op
->args
[2])) {
931 tmp
= arg_info(op
->args
[2])->val
& 31;
932 mask
= (uint32_t)arg_info(op
->args
[1])->mask
>> tmp
;
935 case INDEX_op_shr_i64
:
936 if (arg_is_const(op
->args
[2])) {
937 tmp
= arg_info(op
->args
[2])->val
& 63;
938 mask
= (uint64_t)arg_info(op
->args
[1])->mask
>> tmp
;
942 case INDEX_op_extrl_i64_i32
:
943 mask
= (uint32_t)arg_info(op
->args
[1])->mask
;
945 case INDEX_op_extrh_i64_i32
:
946 mask
= (uint64_t)arg_info(op
->args
[1])->mask
>> 32;
950 if (arg_is_const(op
->args
[2])) {
951 tmp
= arg_info(op
->args
[2])->val
& (TCG_TARGET_REG_BITS
- 1);
952 mask
= arg_info(op
->args
[1])->mask
<< tmp
;
957 /* Set to 1 all bits to the left of the rightmost. */
958 mask
= -(arg_info(op
->args
[1])->mask
959 & -arg_info(op
->args
[1])->mask
);
962 CASE_OP_32_64(deposit
):
963 mask
= deposit64(arg_info(op
->args
[1])->mask
,
964 op
->args
[3], op
->args
[4],
965 arg_info(op
->args
[2])->mask
);
968 CASE_OP_32_64(extract
):
969 mask
= extract64(arg_info(op
->args
[1])->mask
,
970 op
->args
[2], op
->args
[3]);
971 if (op
->args
[2] == 0) {
972 affected
= arg_info(op
->args
[1])->mask
& ~mask
;
975 CASE_OP_32_64(sextract
):
976 mask
= sextract64(arg_info(op
->args
[1])->mask
,
977 op
->args
[2], op
->args
[3]);
978 if (op
->args
[2] == 0 && (tcg_target_long
)mask
>= 0) {
979 affected
= arg_info(op
->args
[1])->mask
& ~mask
;
985 mask
= arg_info(op
->args
[1])->mask
| arg_info(op
->args
[2])->mask
;
988 case INDEX_op_clz_i32
:
989 case INDEX_op_ctz_i32
:
990 mask
= arg_info(op
->args
[2])->mask
| 31;
993 case INDEX_op_clz_i64
:
994 case INDEX_op_ctz_i64
:
995 mask
= arg_info(op
->args
[2])->mask
| 63;
998 case INDEX_op_ctpop_i32
:
1001 case INDEX_op_ctpop_i64
:
1005 CASE_OP_32_64(setcond
):
1006 case INDEX_op_setcond2_i32
:
1010 CASE_OP_32_64(movcond
):
1011 mask
= arg_info(op
->args
[3])->mask
| arg_info(op
->args
[4])->mask
;
1014 CASE_OP_32_64(ld8u
):
1017 CASE_OP_32_64(ld16u
):
1020 case INDEX_op_ld32u_i64
:
1024 CASE_OP_32_64(qemu_ld
):
1026 MemOpIdx oi
= op
->args
[nb_oargs
+ nb_iargs
];
1027 MemOp mop
= get_memop(oi
);
1028 if (!(mop
& MO_SIGN
)) {
1029 mask
= (2ULL << ((8 << (mop
& MO_SIZE
)) - 1)) - 1;
1034 CASE_OP_32_64(bswap16
):
1035 mask
= arg_info(op
->args
[1])->mask
;
1036 if (mask
<= 0xffff) {
1037 op
->args
[2] |= TCG_BSWAP_IZ
;
1039 mask
= bswap16(mask
);
1040 switch (op
->args
[2] & (TCG_BSWAP_OZ
| TCG_BSWAP_OS
)) {
1044 mask
= (int16_t)mask
;
1046 default: /* undefined high bits */
1047 mask
|= MAKE_64BIT_MASK(16, 48);
1052 case INDEX_op_bswap32_i64
:
1053 mask
= arg_info(op
->args
[1])->mask
;
1054 if (mask
<= 0xffffffffu
) {
1055 op
->args
[2] |= TCG_BSWAP_IZ
;
1057 mask
= bswap32(mask
);
1058 switch (op
->args
[2] & (TCG_BSWAP_OZ
| TCG_BSWAP_OS
)) {
1062 mask
= (int32_t)mask
;
1064 default: /* undefined high bits */
1065 mask
|= MAKE_64BIT_MASK(32, 32);
1074 /* 32-bit ops generate 32-bit results. For the result is zero test
1075 below, we can ignore high bits, but for further optimizations we
1076 need to record that the high bits contain garbage. */
1078 if (!(def
->flags
& TCG_OPF_64BIT
)) {
1079 mask
|= ~(tcg_target_ulong
)0xffffffffu
;
1080 partmask
&= 0xffffffffu
;
1081 affected
&= 0xffffffffu
;
1084 if (partmask
== 0) {
1085 tcg_debug_assert(nb_oargs
== 1);
1086 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], 0);
1089 if (affected
== 0) {
1090 tcg_debug_assert(nb_oargs
== 1);
1091 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1095 /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
1097 CASE_OP_32_64_VEC(and):
1098 CASE_OP_32_64_VEC(mul
):
1099 CASE_OP_32_64(muluh
):
1100 CASE_OP_32_64(mulsh
):
1101 if (arg_is_const(op
->args
[2])
1102 && arg_info(op
->args
[2])->val
== 0) {
1103 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], 0);
1111 /* Simplify expression for "op r, a, a => mov r, a" cases */
1113 CASE_OP_32_64_VEC(or):
1114 CASE_OP_32_64_VEC(and):
1115 if (args_are_copies(op
->args
[1], op
->args
[2])) {
1116 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1124 /* Simplify expression for "op r, a, a => movi r, 0" cases */
1126 CASE_OP_32_64_VEC(andc
):
1127 CASE_OP_32_64_VEC(sub
):
1128 CASE_OP_32_64_VEC(xor):
1129 if (args_are_copies(op
->args
[1], op
->args
[2])) {
1130 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], 0);
1138 /* Propagate constants through copy operations and do constant
1139 folding. Constants will be substituted to arguments by register
1140 allocator where needed and possible. Also detect copies. */
1142 CASE_OP_32_64_VEC(mov
):
1143 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1146 case INDEX_op_dup_vec
:
1147 if (arg_is_const(op
->args
[1])) {
1148 tmp
= arg_info(op
->args
[1])->val
;
1149 tmp
= dup_const(TCGOP_VECE(op
), tmp
);
1150 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1155 case INDEX_op_dup2_vec
:
1156 assert(TCG_TARGET_REG_BITS
== 32);
1157 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1158 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0],
1159 deposit64(arg_info(op
->args
[1])->val
, 32, 32,
1160 arg_info(op
->args
[2])->val
));
1162 } else if (args_are_copies(op
->args
[1], op
->args
[2])) {
1163 op
->opc
= INDEX_op_dup_vec
;
1164 TCGOP_VECE(op
) = MO_32
;
1171 CASE_OP_32_64(ext8s
):
1172 CASE_OP_32_64(ext8u
):
1173 CASE_OP_32_64(ext16s
):
1174 CASE_OP_32_64(ext16u
):
1175 CASE_OP_32_64(ctpop
):
1176 case INDEX_op_ext32s_i64
:
1177 case INDEX_op_ext32u_i64
:
1178 case INDEX_op_ext_i32_i64
:
1179 case INDEX_op_extu_i32_i64
:
1180 case INDEX_op_extrl_i64_i32
:
1181 case INDEX_op_extrh_i64_i32
:
1182 if (arg_is_const(op
->args
[1])) {
1183 tmp
= do_constant_folding(opc
, arg_info(op
->args
[1])->val
, 0);
1184 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1189 CASE_OP_32_64(bswap16
):
1190 CASE_OP_32_64(bswap32
):
1191 case INDEX_op_bswap64_i64
:
1192 if (arg_is_const(op
->args
[1])) {
1193 tmp
= do_constant_folding(opc
, arg_info(op
->args
[1])->val
,
1195 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1209 CASE_OP_32_64(rotl
):
1210 CASE_OP_32_64(rotr
):
1211 CASE_OP_32_64(andc
):
1214 CASE_OP_32_64(nand
):
1216 CASE_OP_32_64(muluh
):
1217 CASE_OP_32_64(mulsh
):
1219 CASE_OP_32_64(divu
):
1221 CASE_OP_32_64(remu
):
1222 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1223 tmp
= do_constant_folding(opc
, arg_info(op
->args
[1])->val
,
1224 arg_info(op
->args
[2])->val
);
1225 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1232 if (arg_is_const(op
->args
[1])) {
1233 TCGArg v
= arg_info(op
->args
[1])->val
;
1235 tmp
= do_constant_folding(opc
, v
, 0);
1236 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1238 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[2]);
1244 CASE_OP_32_64(deposit
):
1245 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1246 tmp
= deposit64(arg_info(op
->args
[1])->val
,
1247 op
->args
[3], op
->args
[4],
1248 arg_info(op
->args
[2])->val
);
1249 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1254 CASE_OP_32_64(extract
):
1255 if (arg_is_const(op
->args
[1])) {
1256 tmp
= extract64(arg_info(op
->args
[1])->val
,
1257 op
->args
[2], op
->args
[3]);
1258 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1263 CASE_OP_32_64(sextract
):
1264 if (arg_is_const(op
->args
[1])) {
1265 tmp
= sextract64(arg_info(op
->args
[1])->val
,
1266 op
->args
[2], op
->args
[3]);
1267 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1272 CASE_OP_32_64(extract2
):
1273 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1274 uint64_t v1
= arg_info(op
->args
[1])->val
;
1275 uint64_t v2
= arg_info(op
->args
[2])->val
;
1276 int shr
= op
->args
[3];
1278 if (opc
== INDEX_op_extract2_i64
) {
1279 tmp
= (v1
>> shr
) | (v2
<< (64 - shr
));
1281 tmp
= (int32_t)(((uint32_t)v1
>> shr
) |
1282 ((uint32_t)v2
<< (32 - shr
)));
1284 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1289 CASE_OP_32_64(setcond
):
1290 tmp
= do_constant_folding_cond(opc
, op
->args
[1],
1291 op
->args
[2], op
->args
[3]);
1293 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1298 CASE_OP_32_64(brcond
):
1299 tmp
= do_constant_folding_cond(opc
, op
->args
[0],
1300 op
->args
[1], op
->args
[2]);
1303 memset(&temps_used
, 0, sizeof(temps_used
));
1304 op
->opc
= INDEX_op_br
;
1305 op
->args
[0] = op
->args
[3];
1307 tcg_op_remove(s
, op
);
1313 CASE_OP_32_64(movcond
):
1314 tmp
= do_constant_folding_cond(opc
, op
->args
[1],
1315 op
->args
[2], op
->args
[5]);
1317 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[4-tmp
]);
1320 if (arg_is_const(op
->args
[3]) && arg_is_const(op
->args
[4])) {
1321 uint64_t tv
= arg_info(op
->args
[3])->val
;
1322 uint64_t fv
= arg_info(op
->args
[4])->val
;
1323 TCGCond cond
= op
->args
[5];
1325 if (fv
== 1 && tv
== 0) {
1326 cond
= tcg_invert_cond(cond
);
1327 } else if (!(tv
== 1 && fv
== 0)) {
1331 op
->opc
= opc
= (opc
== INDEX_op_movcond_i32
1332 ? INDEX_op_setcond_i32
1333 : INDEX_op_setcond_i64
);
1338 case INDEX_op_add2_i32
:
1339 case INDEX_op_sub2_i32
:
1340 if (arg_is_const(op
->args
[2]) && arg_is_const(op
->args
[3])
1341 && arg_is_const(op
->args
[4]) && arg_is_const(op
->args
[5])) {
1342 uint32_t al
= arg_info(op
->args
[2])->val
;
1343 uint32_t ah
= arg_info(op
->args
[3])->val
;
1344 uint32_t bl
= arg_info(op
->args
[4])->val
;
1345 uint32_t bh
= arg_info(op
->args
[5])->val
;
1346 uint64_t a
= ((uint64_t)ah
<< 32) | al
;
1347 uint64_t b
= ((uint64_t)bh
<< 32) | bl
;
1349 TCGOp
*op2
= tcg_op_insert_before(s
, op
, INDEX_op_mov_i32
);
1351 if (opc
== INDEX_op_add2_i32
) {
1359 tcg_opt_gen_movi(s
, &temps_used
, op
, rl
, (int32_t)a
);
1360 tcg_opt_gen_movi(s
, &temps_used
, op2
, rh
, (int32_t)(a
>> 32));
1365 case INDEX_op_mulu2_i32
:
1366 if (arg_is_const(op
->args
[2]) && arg_is_const(op
->args
[3])) {
1367 uint32_t a
= arg_info(op
->args
[2])->val
;
1368 uint32_t b
= arg_info(op
->args
[3])->val
;
1369 uint64_t r
= (uint64_t)a
* b
;
1371 TCGOp
*op2
= tcg_op_insert_before(s
, op
, INDEX_op_mov_i32
);
1375 tcg_opt_gen_movi(s
, &temps_used
, op
, rl
, (int32_t)r
);
1376 tcg_opt_gen_movi(s
, &temps_used
, op2
, rh
, (int32_t)(r
>> 32));
1381 case INDEX_op_brcond2_i32
:
1382 tmp
= do_constant_folding_cond2(&op
->args
[0], &op
->args
[2],
1387 memset(&temps_used
, 0, sizeof(temps_used
));
1388 op
->opc
= INDEX_op_br
;
1389 op
->args
[0] = op
->args
[5];
1392 tcg_op_remove(s
, op
);
1394 } else if ((op
->args
[4] == TCG_COND_LT
1395 || op
->args
[4] == TCG_COND_GE
)
1396 && arg_is_const(op
->args
[2])
1397 && arg_info(op
->args
[2])->val
== 0
1398 && arg_is_const(op
->args
[3])
1399 && arg_info(op
->args
[3])->val
== 0) {
1400 /* Simplify LT/GE comparisons vs zero to a single compare
1401 vs the high word of the input. */
1403 memset(&temps_used
, 0, sizeof(temps_used
));
1404 op
->opc
= INDEX_op_brcond_i32
;
1405 op
->args
[0] = op
->args
[1];
1406 op
->args
[1] = op
->args
[3];
1407 op
->args
[2] = op
->args
[4];
1408 op
->args
[3] = op
->args
[5];
1409 } else if (op
->args
[4] == TCG_COND_EQ
) {
1410 /* Simplify EQ comparisons where one of the pairs
1411 can be simplified. */
1412 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1413 op
->args
[0], op
->args
[2],
1416 goto do_brcond_false
;
1417 } else if (tmp
== 1) {
1418 goto do_brcond_high
;
1420 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1421 op
->args
[1], op
->args
[3],
1424 goto do_brcond_false
;
1425 } else if (tmp
!= 1) {
1429 memset(&temps_used
, 0, sizeof(temps_used
));
1430 op
->opc
= INDEX_op_brcond_i32
;
1431 op
->args
[1] = op
->args
[2];
1432 op
->args
[2] = op
->args
[4];
1433 op
->args
[3] = op
->args
[5];
1434 } else if (op
->args
[4] == TCG_COND_NE
) {
1435 /* Simplify NE comparisons where one of the pairs
1436 can be simplified. */
1437 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1438 op
->args
[0], op
->args
[2],
1441 goto do_brcond_high
;
1442 } else if (tmp
== 1) {
1443 goto do_brcond_true
;
1445 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1446 op
->args
[1], op
->args
[3],
1450 } else if (tmp
== 1) {
1451 goto do_brcond_true
;
1459 case INDEX_op_setcond2_i32
:
1460 tmp
= do_constant_folding_cond2(&op
->args
[1], &op
->args
[3],
1464 tcg_opt_gen_movi(s
, &temps_used
, op
, op
->args
[0], tmp
);
1465 } else if ((op
->args
[5] == TCG_COND_LT
1466 || op
->args
[5] == TCG_COND_GE
)
1467 && arg_is_const(op
->args
[3])
1468 && arg_info(op
->args
[3])->val
== 0
1469 && arg_is_const(op
->args
[4])
1470 && arg_info(op
->args
[4])->val
== 0) {
1471 /* Simplify LT/GE comparisons vs zero to a single compare
1472 vs the high word of the input. */
1474 reset_temp(op
->args
[0]);
1475 arg_info(op
->args
[0])->mask
= 1;
1476 op
->opc
= INDEX_op_setcond_i32
;
1477 op
->args
[1] = op
->args
[2];
1478 op
->args
[2] = op
->args
[4];
1479 op
->args
[3] = op
->args
[5];
1480 } else if (op
->args
[5] == TCG_COND_EQ
) {
1481 /* Simplify EQ comparisons where one of the pairs
1482 can be simplified. */
1483 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1484 op
->args
[1], op
->args
[3],
1487 goto do_setcond_const
;
1488 } else if (tmp
== 1) {
1489 goto do_setcond_high
;
1491 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1492 op
->args
[2], op
->args
[4],
1495 goto do_setcond_high
;
1496 } else if (tmp
!= 1) {
1500 reset_temp(op
->args
[0]);
1501 arg_info(op
->args
[0])->mask
= 1;
1502 op
->opc
= INDEX_op_setcond_i32
;
1503 op
->args
[2] = op
->args
[3];
1504 op
->args
[3] = op
->args
[5];
1505 } else if (op
->args
[5] == TCG_COND_NE
) {
1506 /* Simplify NE comparisons where one of the pairs
1507 can be simplified. */
1508 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1509 op
->args
[1], op
->args
[3],
1512 goto do_setcond_high
;
1513 } else if (tmp
== 1) {
1514 goto do_setcond_const
;
1516 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1517 op
->args
[2], op
->args
[4],
1520 goto do_setcond_low
;
1521 } else if (tmp
== 1) {
1522 goto do_setcond_const
;
1531 if (!(tcg_call_flags(op
)
1532 & (TCG_CALL_NO_READ_GLOBALS
| TCG_CALL_NO_WRITE_GLOBALS
))) {
1533 for (i
= 0; i
< nb_globals
; i
++) {
1534 if (test_bit(i
, temps_used
.l
)) {
1535 reset_ts(&s
->temps
[i
]);
1539 goto do_reset_output
;
1543 /* Default case: we know nothing about operation (or were unable
1544 to compute the operation result) so no propagation is done.
1545 We trash everything if the operation is the end of a basic
1546 block, otherwise we only trash the output args. "mask" is
1547 the non-zero bits mask for the first output arg. */
1548 if (def
->flags
& TCG_OPF_BB_END
) {
1549 memset(&temps_used
, 0, sizeof(temps_used
));
1552 for (i
= 0; i
< nb_oargs
; i
++) {
1553 reset_temp(op
->args
[i
]);
1554 /* Save the corresponding known-zero bits mask for the
1555 first output argument (only one supported so far). */
1557 arg_info(op
->args
[i
])->mask
= mask
;
1564 /* Eliminate duplicate and redundant fence instructions. */
1568 /* Merge two barriers of the same type into one,
1569 * or a weaker barrier into a stronger one,
1570 * or two weaker barriers into a stronger one.
1571 * mb X; mb Y => mb X|Y
1572 * mb; strl => mb; st
1573 * ldaq; mb => ld; mb
1574 * ldaq; strl => ld; mb; st
1575 * Other combinations are also merged into a strong
1576 * barrier. This is stricter than specified but for
1577 * the purposes of TCG is better than not optimizing.
1579 prev_mb
->args
[0] |= op
->args
[0];
1580 tcg_op_remove(s
, op
);
1584 /* Opcodes that end the block stop the optimization. */
1585 if ((def
->flags
& TCG_OPF_BB_END
) == 0) {
1589 case INDEX_op_qemu_ld_i32
:
1590 case INDEX_op_qemu_ld_i64
:
1591 case INDEX_op_qemu_st_i32
:
1592 case INDEX_op_qemu_st8_i32
:
1593 case INDEX_op_qemu_st_i64
:
1595 /* Opcodes that touch guest memory stop the optimization. */
1599 } else if (opc
== INDEX_op_mb
) {