tcg/optimize: fix uninitialized variable
[qemu/ar7.git] / tcg / optimize.c
blob752cc5c56b671eac78fec6a04270518b6cff4581
1 /*
2 * Optimizations for Tiny Code Generator for QEMU
4 * Copyright (c) 2010 Samsung Electronics.
5 * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 * THE SOFTWARE.
26 #include "qemu/osdep.h"
27 #include "qemu/int128.h"
28 #include "qemu/interval-tree.h"
29 #include "tcg/tcg-op-common.h"
30 #include "tcg-internal.h"
32 #define CASE_OP_32_64(x) \
33 glue(glue(case INDEX_op_, x), _i32): \
34 glue(glue(case INDEX_op_, x), _i64)
36 #define CASE_OP_32_64_VEC(x) \
37 glue(glue(case INDEX_op_, x), _i32): \
38 glue(glue(case INDEX_op_, x), _i64): \
39 glue(glue(case INDEX_op_, x), _vec)
41 typedef struct MemCopyInfo {
42 IntervalTreeNode itree;
43 QSIMPLEQ_ENTRY (MemCopyInfo) next;
44 TCGTemp *ts;
45 TCGType type;
46 } MemCopyInfo;
48 typedef struct TempOptInfo {
49 bool is_const;
50 TCGTemp *prev_copy;
51 TCGTemp *next_copy;
52 QSIMPLEQ_HEAD(, MemCopyInfo) mem_copy;
53 uint64_t val;
54 uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */
55 uint64_t s_mask; /* a left-aligned mask of clrsb(value) bits. */
56 } TempOptInfo;
58 typedef struct OptContext {
59 TCGContext *tcg;
60 TCGOp *prev_mb;
61 TCGTempSet temps_used;
63 IntervalTreeRoot mem_copy;
64 QSIMPLEQ_HEAD(, MemCopyInfo) mem_free;
66 /* In flight values from optimization. */
67 uint64_t a_mask; /* mask bit is 0 iff value identical to first input */
68 uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */
69 uint64_t s_mask; /* mask of clrsb(value) bits */
70 TCGType type;
71 } OptContext;
73 /* Calculate the smask for a specific value. */
74 static uint64_t smask_from_value(uint64_t value)
76 int rep = clrsb64(value);
77 return ~(~0ull >> rep);
81 * Calculate the smask for a given set of known-zeros.
82 * If there are lots of zeros on the left, we can consider the remainder
83 * an unsigned field, and thus the corresponding signed field is one bit
84 * larger.
86 static uint64_t smask_from_zmask(uint64_t zmask)
89 * Only the 0 bits are significant for zmask, thus the msb itself
90 * must be zero, else we have no sign information.
92 int rep = clz64(zmask);
93 if (rep == 0) {
94 return 0;
96 rep -= 1;
97 return ~(~0ull >> rep);
101 * Recreate a properly left-aligned smask after manipulation.
102 * Some bit-shuffling, particularly shifts and rotates, may
103 * retain sign bits on the left, but may scatter disconnected
104 * sign bits on the right. Retain only what remains to the left.
106 static uint64_t smask_from_smask(int64_t smask)
108 /* Only the 1 bits are significant for smask */
109 return smask_from_zmask(~smask);
112 static inline TempOptInfo *ts_info(TCGTemp *ts)
114 return ts->state_ptr;
117 static inline TempOptInfo *arg_info(TCGArg arg)
119 return ts_info(arg_temp(arg));
122 static inline bool ts_is_const(TCGTemp *ts)
124 return ts_info(ts)->is_const;
127 static inline bool ts_is_const_val(TCGTemp *ts, uint64_t val)
129 TempOptInfo *ti = ts_info(ts);
130 return ti->is_const && ti->val == val;
133 static inline bool arg_is_const(TCGArg arg)
135 return ts_is_const(arg_temp(arg));
138 static inline bool arg_is_const_val(TCGArg arg, uint64_t val)
140 return ts_is_const_val(arg_temp(arg), val);
143 static inline bool ts_is_copy(TCGTemp *ts)
145 return ts_info(ts)->next_copy != ts;
148 static TCGTemp *cmp_better_copy(TCGTemp *a, TCGTemp *b)
150 return a->kind < b->kind ? b : a;
153 /* Initialize and activate a temporary. */
154 static void init_ts_info(OptContext *ctx, TCGTemp *ts)
156 size_t idx = temp_idx(ts);
157 TempOptInfo *ti;
159 if (test_bit(idx, ctx->temps_used.l)) {
160 return;
162 set_bit(idx, ctx->temps_used.l);
164 ti = ts->state_ptr;
165 if (ti == NULL) {
166 ti = tcg_malloc(sizeof(TempOptInfo));
167 ts->state_ptr = ti;
170 ti->next_copy = ts;
171 ti->prev_copy = ts;
172 QSIMPLEQ_INIT(&ti->mem_copy);
173 if (ts->kind == TEMP_CONST) {
174 ti->is_const = true;
175 ti->val = ts->val;
176 ti->z_mask = ts->val;
177 ti->s_mask = smask_from_value(ts->val);
178 } else {
179 ti->is_const = false;
180 ti->z_mask = -1;
181 ti->s_mask = 0;
185 static MemCopyInfo *mem_copy_first(OptContext *ctx, intptr_t s, intptr_t l)
187 IntervalTreeNode *r = interval_tree_iter_first(&ctx->mem_copy, s, l);
188 return r ? container_of(r, MemCopyInfo, itree) : NULL;
191 static MemCopyInfo *mem_copy_next(MemCopyInfo *mem, intptr_t s, intptr_t l)
193 IntervalTreeNode *r = interval_tree_iter_next(&mem->itree, s, l);
194 return r ? container_of(r, MemCopyInfo, itree) : NULL;
197 static void remove_mem_copy(OptContext *ctx, MemCopyInfo *mc)
199 TCGTemp *ts = mc->ts;
200 TempOptInfo *ti = ts_info(ts);
202 interval_tree_remove(&mc->itree, &ctx->mem_copy);
203 QSIMPLEQ_REMOVE(&ti->mem_copy, mc, MemCopyInfo, next);
204 QSIMPLEQ_INSERT_TAIL(&ctx->mem_free, mc, next);
207 static void remove_mem_copy_in(OptContext *ctx, intptr_t s, intptr_t l)
209 while (true) {
210 MemCopyInfo *mc = mem_copy_first(ctx, s, l);
211 if (!mc) {
212 break;
214 remove_mem_copy(ctx, mc);
218 static void remove_mem_copy_all(OptContext *ctx)
220 remove_mem_copy_in(ctx, 0, -1);
221 tcg_debug_assert(interval_tree_is_empty(&ctx->mem_copy));
224 static TCGTemp *find_better_copy(TCGTemp *ts)
226 TCGTemp *i, *ret;
228 /* If this is already readonly, we can't do better. */
229 if (temp_readonly(ts)) {
230 return ts;
233 ret = ts;
234 for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) {
235 ret = cmp_better_copy(ret, i);
237 return ret;
240 static void move_mem_copies(TCGTemp *dst_ts, TCGTemp *src_ts)
242 TempOptInfo *si = ts_info(src_ts);
243 TempOptInfo *di = ts_info(dst_ts);
244 MemCopyInfo *mc;
246 QSIMPLEQ_FOREACH(mc, &si->mem_copy, next) {
247 tcg_debug_assert(mc->ts == src_ts);
248 mc->ts = dst_ts;
250 QSIMPLEQ_CONCAT(&di->mem_copy, &si->mem_copy);
253 /* Reset TEMP's state, possibly removing the temp for the list of copies. */
254 static void reset_ts(OptContext *ctx, TCGTemp *ts)
256 TempOptInfo *ti = ts_info(ts);
257 TCGTemp *pts = ti->prev_copy;
258 TCGTemp *nts = ti->next_copy;
259 TempOptInfo *pi = ts_info(pts);
260 TempOptInfo *ni = ts_info(nts);
262 ni->prev_copy = ti->prev_copy;
263 pi->next_copy = ti->next_copy;
264 ti->next_copy = ts;
265 ti->prev_copy = ts;
266 ti->is_const = false;
267 ti->z_mask = -1;
268 ti->s_mask = 0;
270 if (!QSIMPLEQ_EMPTY(&ti->mem_copy)) {
271 if (ts == nts) {
272 /* Last temp copy being removed, the mem copies die. */
273 MemCopyInfo *mc;
274 QSIMPLEQ_FOREACH(mc, &ti->mem_copy, next) {
275 interval_tree_remove(&mc->itree, &ctx->mem_copy);
277 QSIMPLEQ_CONCAT(&ctx->mem_free, &ti->mem_copy);
278 } else {
279 move_mem_copies(find_better_copy(nts), ts);
284 static void reset_temp(OptContext *ctx, TCGArg arg)
286 reset_ts(ctx, arg_temp(arg));
289 static void record_mem_copy(OptContext *ctx, TCGType type,
290 TCGTemp *ts, intptr_t start, intptr_t last)
292 MemCopyInfo *mc;
293 TempOptInfo *ti;
295 mc = QSIMPLEQ_FIRST(&ctx->mem_free);
296 if (mc) {
297 QSIMPLEQ_REMOVE_HEAD(&ctx->mem_free, next);
298 } else {
299 mc = tcg_malloc(sizeof(*mc));
302 memset(mc, 0, sizeof(*mc));
303 mc->itree.start = start;
304 mc->itree.last = last;
305 mc->type = type;
306 interval_tree_insert(&mc->itree, &ctx->mem_copy);
308 ts = find_better_copy(ts);
309 ti = ts_info(ts);
310 mc->ts = ts;
311 QSIMPLEQ_INSERT_TAIL(&ti->mem_copy, mc, next);
314 static bool ts_are_copies(TCGTemp *ts1, TCGTemp *ts2)
316 TCGTemp *i;
318 if (ts1 == ts2) {
319 return true;
322 if (!ts_is_copy(ts1) || !ts_is_copy(ts2)) {
323 return false;
326 for (i = ts_info(ts1)->next_copy; i != ts1; i = ts_info(i)->next_copy) {
327 if (i == ts2) {
328 return true;
332 return false;
335 static bool args_are_copies(TCGArg arg1, TCGArg arg2)
337 return ts_are_copies(arg_temp(arg1), arg_temp(arg2));
340 static TCGTemp *find_mem_copy_for(OptContext *ctx, TCGType type, intptr_t s)
342 MemCopyInfo *mc;
344 for (mc = mem_copy_first(ctx, s, s); mc; mc = mem_copy_next(mc, s, s)) {
345 if (mc->itree.start == s && mc->type == type) {
346 return find_better_copy(mc->ts);
349 return NULL;
352 static TCGArg arg_new_constant(OptContext *ctx, uint64_t val)
354 TCGType type = ctx->type;
355 TCGTemp *ts;
357 if (type == TCG_TYPE_I32) {
358 val = (int32_t)val;
361 ts = tcg_constant_internal(type, val);
362 init_ts_info(ctx, ts);
364 return temp_arg(ts);
367 static TCGArg arg_new_temp(OptContext *ctx)
369 TCGTemp *ts = tcg_temp_new_internal(ctx->type, TEMP_EBB);
370 init_ts_info(ctx, ts);
371 return temp_arg(ts);
374 static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src)
376 TCGTemp *dst_ts = arg_temp(dst);
377 TCGTemp *src_ts = arg_temp(src);
378 TempOptInfo *di;
379 TempOptInfo *si;
380 TCGOpcode new_op;
382 if (ts_are_copies(dst_ts, src_ts)) {
383 tcg_op_remove(ctx->tcg, op);
384 return true;
387 reset_ts(ctx, dst_ts);
388 di = ts_info(dst_ts);
389 si = ts_info(src_ts);
391 switch (ctx->type) {
392 case TCG_TYPE_I32:
393 new_op = INDEX_op_mov_i32;
394 break;
395 case TCG_TYPE_I64:
396 new_op = INDEX_op_mov_i64;
397 break;
398 case TCG_TYPE_V64:
399 case TCG_TYPE_V128:
400 case TCG_TYPE_V256:
401 /* TCGOP_VECL and TCGOP_VECE remain unchanged. */
402 new_op = INDEX_op_mov_vec;
403 break;
404 default:
405 g_assert_not_reached();
407 op->opc = new_op;
408 op->args[0] = dst;
409 op->args[1] = src;
411 di->z_mask = si->z_mask;
412 di->s_mask = si->s_mask;
414 if (src_ts->type == dst_ts->type) {
415 TempOptInfo *ni = ts_info(si->next_copy);
417 di->next_copy = si->next_copy;
418 di->prev_copy = src_ts;
419 ni->prev_copy = dst_ts;
420 si->next_copy = dst_ts;
421 di->is_const = si->is_const;
422 di->val = si->val;
424 if (!QSIMPLEQ_EMPTY(&si->mem_copy)
425 && cmp_better_copy(src_ts, dst_ts) == dst_ts) {
426 move_mem_copies(dst_ts, src_ts);
429 return true;
432 static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op,
433 TCGArg dst, uint64_t val)
435 /* Convert movi to mov with constant temp. */
436 return tcg_opt_gen_mov(ctx, op, dst, arg_new_constant(ctx, val));
439 static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y)
441 uint64_t l64, h64;
443 switch (op) {
444 CASE_OP_32_64(add):
445 return x + y;
447 CASE_OP_32_64(sub):
448 return x - y;
450 CASE_OP_32_64(mul):
451 return x * y;
453 CASE_OP_32_64_VEC(and):
454 return x & y;
456 CASE_OP_32_64_VEC(or):
457 return x | y;
459 CASE_OP_32_64_VEC(xor):
460 return x ^ y;
462 case INDEX_op_shl_i32:
463 return (uint32_t)x << (y & 31);
465 case INDEX_op_shl_i64:
466 return (uint64_t)x << (y & 63);
468 case INDEX_op_shr_i32:
469 return (uint32_t)x >> (y & 31);
471 case INDEX_op_shr_i64:
472 return (uint64_t)x >> (y & 63);
474 case INDEX_op_sar_i32:
475 return (int32_t)x >> (y & 31);
477 case INDEX_op_sar_i64:
478 return (int64_t)x >> (y & 63);
480 case INDEX_op_rotr_i32:
481 return ror32(x, y & 31);
483 case INDEX_op_rotr_i64:
484 return ror64(x, y & 63);
486 case INDEX_op_rotl_i32:
487 return rol32(x, y & 31);
489 case INDEX_op_rotl_i64:
490 return rol64(x, y & 63);
492 CASE_OP_32_64_VEC(not):
493 return ~x;
495 CASE_OP_32_64(neg):
496 return -x;
498 CASE_OP_32_64_VEC(andc):
499 return x & ~y;
501 CASE_OP_32_64_VEC(orc):
502 return x | ~y;
504 CASE_OP_32_64_VEC(eqv):
505 return ~(x ^ y);
507 CASE_OP_32_64_VEC(nand):
508 return ~(x & y);
510 CASE_OP_32_64_VEC(nor):
511 return ~(x | y);
513 case INDEX_op_clz_i32:
514 return (uint32_t)x ? clz32(x) : y;
516 case INDEX_op_clz_i64:
517 return x ? clz64(x) : y;
519 case INDEX_op_ctz_i32:
520 return (uint32_t)x ? ctz32(x) : y;
522 case INDEX_op_ctz_i64:
523 return x ? ctz64(x) : y;
525 case INDEX_op_ctpop_i32:
526 return ctpop32(x);
528 case INDEX_op_ctpop_i64:
529 return ctpop64(x);
531 CASE_OP_32_64(ext8s):
532 return (int8_t)x;
534 CASE_OP_32_64(ext16s):
535 return (int16_t)x;
537 CASE_OP_32_64(ext8u):
538 return (uint8_t)x;
540 CASE_OP_32_64(ext16u):
541 return (uint16_t)x;
543 CASE_OP_32_64(bswap16):
544 x = bswap16(x);
545 return y & TCG_BSWAP_OS ? (int16_t)x : x;
547 CASE_OP_32_64(bswap32):
548 x = bswap32(x);
549 return y & TCG_BSWAP_OS ? (int32_t)x : x;
551 case INDEX_op_bswap64_i64:
552 return bswap64(x);
554 case INDEX_op_ext_i32_i64:
555 case INDEX_op_ext32s_i64:
556 return (int32_t)x;
558 case INDEX_op_extu_i32_i64:
559 case INDEX_op_extrl_i64_i32:
560 case INDEX_op_ext32u_i64:
561 return (uint32_t)x;
563 case INDEX_op_extrh_i64_i32:
564 return (uint64_t)x >> 32;
566 case INDEX_op_muluh_i32:
567 return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
568 case INDEX_op_mulsh_i32:
569 return ((int64_t)(int32_t)x * (int32_t)y) >> 32;
571 case INDEX_op_muluh_i64:
572 mulu64(&l64, &h64, x, y);
573 return h64;
574 case INDEX_op_mulsh_i64:
575 muls64(&l64, &h64, x, y);
576 return h64;
578 case INDEX_op_div_i32:
579 /* Avoid crashing on divide by zero, otherwise undefined. */
580 return (int32_t)x / ((int32_t)y ? : 1);
581 case INDEX_op_divu_i32:
582 return (uint32_t)x / ((uint32_t)y ? : 1);
583 case INDEX_op_div_i64:
584 return (int64_t)x / ((int64_t)y ? : 1);
585 case INDEX_op_divu_i64:
586 return (uint64_t)x / ((uint64_t)y ? : 1);
588 case INDEX_op_rem_i32:
589 return (int32_t)x % ((int32_t)y ? : 1);
590 case INDEX_op_remu_i32:
591 return (uint32_t)x % ((uint32_t)y ? : 1);
592 case INDEX_op_rem_i64:
593 return (int64_t)x % ((int64_t)y ? : 1);
594 case INDEX_op_remu_i64:
595 return (uint64_t)x % ((uint64_t)y ? : 1);
597 default:
598 g_assert_not_reached();
602 static uint64_t do_constant_folding(TCGOpcode op, TCGType type,
603 uint64_t x, uint64_t y)
605 uint64_t res = do_constant_folding_2(op, x, y);
606 if (type == TCG_TYPE_I32) {
607 res = (int32_t)res;
609 return res;
612 static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
614 switch (c) {
615 case TCG_COND_EQ:
616 return x == y;
617 case TCG_COND_NE:
618 return x != y;
619 case TCG_COND_LT:
620 return (int32_t)x < (int32_t)y;
621 case TCG_COND_GE:
622 return (int32_t)x >= (int32_t)y;
623 case TCG_COND_LE:
624 return (int32_t)x <= (int32_t)y;
625 case TCG_COND_GT:
626 return (int32_t)x > (int32_t)y;
627 case TCG_COND_LTU:
628 return x < y;
629 case TCG_COND_GEU:
630 return x >= y;
631 case TCG_COND_LEU:
632 return x <= y;
633 case TCG_COND_GTU:
634 return x > y;
635 case TCG_COND_TSTEQ:
636 return (x & y) == 0;
637 case TCG_COND_TSTNE:
638 return (x & y) != 0;
639 case TCG_COND_ALWAYS:
640 case TCG_COND_NEVER:
641 break;
643 g_assert_not_reached();
646 static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
648 switch (c) {
649 case TCG_COND_EQ:
650 return x == y;
651 case TCG_COND_NE:
652 return x != y;
653 case TCG_COND_LT:
654 return (int64_t)x < (int64_t)y;
655 case TCG_COND_GE:
656 return (int64_t)x >= (int64_t)y;
657 case TCG_COND_LE:
658 return (int64_t)x <= (int64_t)y;
659 case TCG_COND_GT:
660 return (int64_t)x > (int64_t)y;
661 case TCG_COND_LTU:
662 return x < y;
663 case TCG_COND_GEU:
664 return x >= y;
665 case TCG_COND_LEU:
666 return x <= y;
667 case TCG_COND_GTU:
668 return x > y;
669 case TCG_COND_TSTEQ:
670 return (x & y) == 0;
671 case TCG_COND_TSTNE:
672 return (x & y) != 0;
673 case TCG_COND_ALWAYS:
674 case TCG_COND_NEVER:
675 break;
677 g_assert_not_reached();
680 static int do_constant_folding_cond_eq(TCGCond c)
682 switch (c) {
683 case TCG_COND_GT:
684 case TCG_COND_LTU:
685 case TCG_COND_LT:
686 case TCG_COND_GTU:
687 case TCG_COND_NE:
688 return 0;
689 case TCG_COND_GE:
690 case TCG_COND_GEU:
691 case TCG_COND_LE:
692 case TCG_COND_LEU:
693 case TCG_COND_EQ:
694 return 1;
695 case TCG_COND_TSTEQ:
696 case TCG_COND_TSTNE:
697 return -1;
698 case TCG_COND_ALWAYS:
699 case TCG_COND_NEVER:
700 break;
702 g_assert_not_reached();
706 * Return -1 if the condition can't be simplified,
707 * and the result of the condition (0 or 1) if it can.
709 static int do_constant_folding_cond(TCGType type, TCGArg x,
710 TCGArg y, TCGCond c)
712 if (arg_is_const(x) && arg_is_const(y)) {
713 uint64_t xv = arg_info(x)->val;
714 uint64_t yv = arg_info(y)->val;
716 switch (type) {
717 case TCG_TYPE_I32:
718 return do_constant_folding_cond_32(xv, yv, c);
719 case TCG_TYPE_I64:
720 return do_constant_folding_cond_64(xv, yv, c);
721 default:
722 /* Only scalar comparisons are optimizable */
723 return -1;
725 } else if (args_are_copies(x, y)) {
726 return do_constant_folding_cond_eq(c);
727 } else if (arg_is_const_val(y, 0)) {
728 switch (c) {
729 case TCG_COND_LTU:
730 case TCG_COND_TSTNE:
731 return 0;
732 case TCG_COND_GEU:
733 case TCG_COND_TSTEQ:
734 return 1;
735 default:
736 return -1;
739 return -1;
743 * swap_commutative:
744 * @dest: TCGArg of the destination argument, or NO_DEST.
745 * @p1: first paired argument
746 * @p2: second paired argument
748 * If *@p1 is a constant and *@p2 is not, swap.
749 * If *@p2 matches @dest, swap.
750 * Return true if a swap was performed.
753 #define NO_DEST temp_arg(NULL)
755 static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
757 TCGArg a1 = *p1, a2 = *p2;
758 int sum = 0;
759 sum += arg_is_const(a1);
760 sum -= arg_is_const(a2);
762 /* Prefer the constant in second argument, and then the form
763 op a, a, b, which is better handled on non-RISC hosts. */
764 if (sum > 0 || (sum == 0 && dest == a2)) {
765 *p1 = a2;
766 *p2 = a1;
767 return true;
769 return false;
772 static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
774 int sum = 0;
775 sum += arg_is_const(p1[0]);
776 sum += arg_is_const(p1[1]);
777 sum -= arg_is_const(p2[0]);
778 sum -= arg_is_const(p2[1]);
779 if (sum > 0) {
780 TCGArg t;
781 t = p1[0], p1[0] = p2[0], p2[0] = t;
782 t = p1[1], p1[1] = p2[1], p2[1] = t;
783 return true;
785 return false;
789 * Return -1 if the condition can't be simplified,
790 * and the result of the condition (0 or 1) if it can.
792 static int do_constant_folding_cond1(OptContext *ctx, TCGOp *op, TCGArg dest,
793 TCGArg *p1, TCGArg *p2, TCGArg *pcond)
795 TCGCond cond;
796 bool swap;
797 int r;
799 swap = swap_commutative(dest, p1, p2);
800 cond = *pcond;
801 if (swap) {
802 *pcond = cond = tcg_swap_cond(cond);
805 r = do_constant_folding_cond(ctx->type, *p1, *p2, cond);
806 if (r >= 0) {
807 return r;
809 if (!is_tst_cond(cond)) {
810 return -1;
814 * TSTNE x,x -> NE x,0
815 * TSTNE x,-1 -> NE x,0
817 if (args_are_copies(*p1, *p2) || arg_is_const_val(*p2, -1)) {
818 *p2 = arg_new_constant(ctx, 0);
819 *pcond = tcg_tst_eqne_cond(cond);
820 return -1;
823 /* TSTNE x,sign -> LT x,0 */
824 if (arg_is_const_val(*p2, (ctx->type == TCG_TYPE_I32
825 ? INT32_MIN : INT64_MIN))) {
826 *p2 = arg_new_constant(ctx, 0);
827 *pcond = tcg_tst_ltge_cond(cond);
828 return -1;
831 /* Expand to AND with a temporary if no backend support. */
832 if (!TCG_TARGET_HAS_tst) {
833 TCGOpcode and_opc = (ctx->type == TCG_TYPE_I32
834 ? INDEX_op_and_i32 : INDEX_op_and_i64);
835 TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, and_opc, 3);
836 TCGArg tmp = arg_new_temp(ctx);
838 op2->args[0] = tmp;
839 op2->args[1] = *p1;
840 op2->args[2] = *p2;
842 *p1 = tmp;
843 *p2 = arg_new_constant(ctx, 0);
844 *pcond = tcg_tst_eqne_cond(cond);
846 return -1;
849 static int do_constant_folding_cond2(OptContext *ctx, TCGOp *op, TCGArg *args)
851 TCGArg al, ah, bl, bh;
852 TCGCond c;
853 bool swap;
854 int r;
856 swap = swap_commutative2(args, args + 2);
857 c = args[4];
858 if (swap) {
859 args[4] = c = tcg_swap_cond(c);
862 al = args[0];
863 ah = args[1];
864 bl = args[2];
865 bh = args[3];
867 if (arg_is_const(bl) && arg_is_const(bh)) {
868 tcg_target_ulong blv = arg_info(bl)->val;
869 tcg_target_ulong bhv = arg_info(bh)->val;
870 uint64_t b = deposit64(blv, 32, 32, bhv);
872 if (arg_is_const(al) && arg_is_const(ah)) {
873 tcg_target_ulong alv = arg_info(al)->val;
874 tcg_target_ulong ahv = arg_info(ah)->val;
875 uint64_t a = deposit64(alv, 32, 32, ahv);
877 r = do_constant_folding_cond_64(a, b, c);
878 if (r >= 0) {
879 return r;
883 if (b == 0) {
884 switch (c) {
885 case TCG_COND_LTU:
886 case TCG_COND_TSTNE:
887 return 0;
888 case TCG_COND_GEU:
889 case TCG_COND_TSTEQ:
890 return 1;
891 default:
892 break;
896 /* TSTNE x,-1 -> NE x,0 */
897 if (b == -1 && is_tst_cond(c)) {
898 args[3] = args[2] = arg_new_constant(ctx, 0);
899 args[4] = tcg_tst_eqne_cond(c);
900 return -1;
903 /* TSTNE x,sign -> LT x,0 */
904 if (b == INT64_MIN && is_tst_cond(c)) {
905 /* bl must be 0, so copy that to bh */
906 args[3] = bl;
907 args[4] = tcg_tst_ltge_cond(c);
908 return -1;
912 if (args_are_copies(al, bl) && args_are_copies(ah, bh)) {
913 r = do_constant_folding_cond_eq(c);
914 if (r >= 0) {
915 return r;
918 /* TSTNE x,x -> NE x,0 */
919 if (is_tst_cond(c)) {
920 args[3] = args[2] = arg_new_constant(ctx, 0);
921 args[4] = tcg_tst_eqne_cond(c);
922 return -1;
926 /* Expand to AND with a temporary if no backend support. */
927 if (!TCG_TARGET_HAS_tst && is_tst_cond(c)) {
928 TCGOp *op1 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3);
929 TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3);
930 TCGArg t1 = arg_new_temp(ctx);
931 TCGArg t2 = arg_new_temp(ctx);
933 op1->args[0] = t1;
934 op1->args[1] = al;
935 op1->args[2] = bl;
936 op2->args[0] = t2;
937 op2->args[1] = ah;
938 op2->args[2] = bh;
940 args[0] = t1;
941 args[1] = t2;
942 args[3] = args[2] = arg_new_constant(ctx, 0);
943 args[4] = tcg_tst_eqne_cond(c);
945 return -1;
948 static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args)
950 for (int i = 0; i < nb_args; i++) {
951 TCGTemp *ts = arg_temp(op->args[i]);
952 init_ts_info(ctx, ts);
956 static void copy_propagate(OptContext *ctx, TCGOp *op,
957 int nb_oargs, int nb_iargs)
959 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
960 TCGTemp *ts = arg_temp(op->args[i]);
961 if (ts_is_copy(ts)) {
962 op->args[i] = temp_arg(find_better_copy(ts));
967 static void finish_folding(OptContext *ctx, TCGOp *op)
969 const TCGOpDef *def = &tcg_op_defs[op->opc];
970 int i, nb_oargs;
973 * We only optimize extended basic blocks. If the opcode ends a BB
974 * and is not a conditional branch, reset all temp data.
976 if (def->flags & TCG_OPF_BB_END) {
977 ctx->prev_mb = NULL;
978 if (!(def->flags & TCG_OPF_COND_BRANCH)) {
979 memset(&ctx->temps_used, 0, sizeof(ctx->temps_used));
980 remove_mem_copy_all(ctx);
982 return;
985 nb_oargs = def->nb_oargs;
986 for (i = 0; i < nb_oargs; i++) {
987 TCGTemp *ts = arg_temp(op->args[i]);
988 reset_ts(ctx, ts);
990 * Save the corresponding known-zero/sign bits mask for the
991 * first output argument (only one supported so far).
993 if (i == 0) {
994 ts_info(ts)->z_mask = ctx->z_mask;
995 ts_info(ts)->s_mask = ctx->s_mask;
1001 * The fold_* functions return true when processing is complete,
1002 * usually by folding the operation to a constant or to a copy,
1003 * and calling tcg_opt_gen_{mov,movi}. They may do other things,
1004 * like collect information about the value produced, for use in
1005 * optimizing a subsequent operation.
1007 * These first fold_* functions are all helpers, used by other
1008 * folders for more specific operations.
1011 static bool fold_const1(OptContext *ctx, TCGOp *op)
1013 if (arg_is_const(op->args[1])) {
1014 uint64_t t;
1016 t = arg_info(op->args[1])->val;
1017 t = do_constant_folding(op->opc, ctx->type, t, 0);
1018 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1020 return false;
1023 static bool fold_const2(OptContext *ctx, TCGOp *op)
1025 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1026 uint64_t t1 = arg_info(op->args[1])->val;
1027 uint64_t t2 = arg_info(op->args[2])->val;
1029 t1 = do_constant_folding(op->opc, ctx->type, t1, t2);
1030 return tcg_opt_gen_movi(ctx, op, op->args[0], t1);
1032 return false;
1035 static bool fold_commutative(OptContext *ctx, TCGOp *op)
1037 swap_commutative(op->args[0], &op->args[1], &op->args[2]);
1038 return false;
1041 static bool fold_const2_commutative(OptContext *ctx, TCGOp *op)
1043 swap_commutative(op->args[0], &op->args[1], &op->args[2]);
1044 return fold_const2(ctx, op);
1047 static bool fold_masks(OptContext *ctx, TCGOp *op)
1049 uint64_t a_mask = ctx->a_mask;
1050 uint64_t z_mask = ctx->z_mask;
1051 uint64_t s_mask = ctx->s_mask;
1054 * 32-bit ops generate 32-bit results, which for the purpose of
1055 * simplifying tcg are sign-extended. Certainly that's how we
1056 * represent our constants elsewhere. Note that the bits will
1057 * be reset properly for a 64-bit value when encountering the
1058 * type changing opcodes.
1060 if (ctx->type == TCG_TYPE_I32) {
1061 a_mask = (int32_t)a_mask;
1062 z_mask = (int32_t)z_mask;
1063 s_mask |= MAKE_64BIT_MASK(32, 32);
1064 ctx->z_mask = z_mask;
1065 ctx->s_mask = s_mask;
1068 if (z_mask == 0) {
1069 return tcg_opt_gen_movi(ctx, op, op->args[0], 0);
1071 if (a_mask == 0) {
1072 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1074 return false;
1078 * Convert @op to NOT, if NOT is supported by the host.
1079 * Return true f the conversion is successful, which will still
1080 * indicate that the processing is complete.
1082 static bool fold_not(OptContext *ctx, TCGOp *op);
1083 static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx)
1085 TCGOpcode not_op;
1086 bool have_not;
1088 switch (ctx->type) {
1089 case TCG_TYPE_I32:
1090 not_op = INDEX_op_not_i32;
1091 have_not = TCG_TARGET_HAS_not_i32;
1092 break;
1093 case TCG_TYPE_I64:
1094 not_op = INDEX_op_not_i64;
1095 have_not = TCG_TARGET_HAS_not_i64;
1096 break;
1097 case TCG_TYPE_V64:
1098 case TCG_TYPE_V128:
1099 case TCG_TYPE_V256:
1100 not_op = INDEX_op_not_vec;
1101 have_not = TCG_TARGET_HAS_not_vec;
1102 break;
1103 default:
1104 g_assert_not_reached();
1106 if (have_not) {
1107 op->opc = not_op;
1108 op->args[1] = op->args[idx];
1109 return fold_not(ctx, op);
1111 return false;
1114 /* If the binary operation has first argument @i, fold to @i. */
1115 static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
1117 if (arg_is_const_val(op->args[1], i)) {
1118 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1120 return false;
1123 /* If the binary operation has first argument @i, fold to NOT. */
1124 static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i)
1126 if (arg_is_const_val(op->args[1], i)) {
1127 return fold_to_not(ctx, op, 2);
1129 return false;
1132 /* If the binary operation has second argument @i, fold to @i. */
1133 static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
1135 if (arg_is_const_val(op->args[2], i)) {
1136 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1138 return false;
1141 /* If the binary operation has second argument @i, fold to identity. */
1142 static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i)
1144 if (arg_is_const_val(op->args[2], i)) {
1145 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1147 return false;
1150 /* If the binary operation has second argument @i, fold to NOT. */
1151 static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i)
1153 if (arg_is_const_val(op->args[2], i)) {
1154 return fold_to_not(ctx, op, 1);
1156 return false;
1159 /* If the binary operation has both arguments equal, fold to @i. */
1160 static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
1162 if (args_are_copies(op->args[1], op->args[2])) {
1163 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1165 return false;
1168 /* If the binary operation has both arguments equal, fold to identity. */
1169 static bool fold_xx_to_x(OptContext *ctx, TCGOp *op)
1171 if (args_are_copies(op->args[1], op->args[2])) {
1172 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1174 return false;
1178 * These outermost fold_<op> functions are sorted alphabetically.
1180 * The ordering of the transformations should be:
1181 * 1) those that produce a constant
1182 * 2) those that produce a copy
1183 * 3) those that produce information about the result value.
1186 static bool fold_add(OptContext *ctx, TCGOp *op)
1188 if (fold_const2_commutative(ctx, op) ||
1189 fold_xi_to_x(ctx, op, 0)) {
1190 return true;
1192 return false;
1195 /* We cannot as yet do_constant_folding with vectors. */
1196 static bool fold_add_vec(OptContext *ctx, TCGOp *op)
1198 if (fold_commutative(ctx, op) ||
1199 fold_xi_to_x(ctx, op, 0)) {
1200 return true;
1202 return false;
1205 static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add)
1207 bool a_const = arg_is_const(op->args[2]) && arg_is_const(op->args[3]);
1208 bool b_const = arg_is_const(op->args[4]) && arg_is_const(op->args[5]);
1210 if (a_const && b_const) {
1211 uint64_t al = arg_info(op->args[2])->val;
1212 uint64_t ah = arg_info(op->args[3])->val;
1213 uint64_t bl = arg_info(op->args[4])->val;
1214 uint64_t bh = arg_info(op->args[5])->val;
1215 TCGArg rl, rh;
1216 TCGOp *op2;
1218 if (ctx->type == TCG_TYPE_I32) {
1219 uint64_t a = deposit64(al, 32, 32, ah);
1220 uint64_t b = deposit64(bl, 32, 32, bh);
1222 if (add) {
1223 a += b;
1224 } else {
1225 a -= b;
1228 al = sextract64(a, 0, 32);
1229 ah = sextract64(a, 32, 32);
1230 } else {
1231 Int128 a = int128_make128(al, ah);
1232 Int128 b = int128_make128(bl, bh);
1234 if (add) {
1235 a = int128_add(a, b);
1236 } else {
1237 a = int128_sub(a, b);
1240 al = int128_getlo(a);
1241 ah = int128_gethi(a);
1244 rl = op->args[0];
1245 rh = op->args[1];
1247 /* The proper opcode is supplied by tcg_opt_gen_mov. */
1248 op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2);
1250 tcg_opt_gen_movi(ctx, op, rl, al);
1251 tcg_opt_gen_movi(ctx, op2, rh, ah);
1252 return true;
1255 /* Fold sub2 r,x,i to add2 r,x,-i */
1256 if (!add && b_const) {
1257 uint64_t bl = arg_info(op->args[4])->val;
1258 uint64_t bh = arg_info(op->args[5])->val;
1260 /* Negate the two parts without assembling and disassembling. */
1261 bl = -bl;
1262 bh = ~bh + !bl;
1264 op->opc = (ctx->type == TCG_TYPE_I32
1265 ? INDEX_op_add2_i32 : INDEX_op_add2_i64);
1266 op->args[4] = arg_new_constant(ctx, bl);
1267 op->args[5] = arg_new_constant(ctx, bh);
1269 return false;
1272 static bool fold_add2(OptContext *ctx, TCGOp *op)
1274 /* Note that the high and low parts may be independently swapped. */
1275 swap_commutative(op->args[0], &op->args[2], &op->args[4]);
1276 swap_commutative(op->args[1], &op->args[3], &op->args[5]);
1278 return fold_addsub2(ctx, op, true);
1281 static bool fold_and(OptContext *ctx, TCGOp *op)
1283 uint64_t z1, z2;
1285 if (fold_const2_commutative(ctx, op) ||
1286 fold_xi_to_i(ctx, op, 0) ||
1287 fold_xi_to_x(ctx, op, -1) ||
1288 fold_xx_to_x(ctx, op)) {
1289 return true;
1292 z1 = arg_info(op->args[1])->z_mask;
1293 z2 = arg_info(op->args[2])->z_mask;
1294 ctx->z_mask = z1 & z2;
1297 * Sign repetitions are perforce all identical, whether they are 1 or 0.
1298 * Bitwise operations preserve the relative quantity of the repetitions.
1300 ctx->s_mask = arg_info(op->args[1])->s_mask
1301 & arg_info(op->args[2])->s_mask;
1304 * Known-zeros does not imply known-ones. Therefore unless
1305 * arg2 is constant, we can't infer affected bits from it.
1307 if (arg_is_const(op->args[2])) {
1308 ctx->a_mask = z1 & ~z2;
1311 return fold_masks(ctx, op);
1314 static bool fold_andc(OptContext *ctx, TCGOp *op)
1316 uint64_t z1;
1318 if (fold_const2(ctx, op) ||
1319 fold_xx_to_i(ctx, op, 0) ||
1320 fold_xi_to_x(ctx, op, 0) ||
1321 fold_ix_to_not(ctx, op, -1)) {
1322 return true;
1325 z1 = arg_info(op->args[1])->z_mask;
1328 * Known-zeros does not imply known-ones. Therefore unless
1329 * arg2 is constant, we can't infer anything from it.
1331 if (arg_is_const(op->args[2])) {
1332 uint64_t z2 = ~arg_info(op->args[2])->z_mask;
1333 ctx->a_mask = z1 & ~z2;
1334 z1 &= z2;
1336 ctx->z_mask = z1;
1338 ctx->s_mask = arg_info(op->args[1])->s_mask
1339 & arg_info(op->args[2])->s_mask;
1340 return fold_masks(ctx, op);
1343 static bool fold_brcond(OptContext *ctx, TCGOp *op)
1345 int i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[0],
1346 &op->args[1], &op->args[2]);
1347 if (i == 0) {
1348 tcg_op_remove(ctx->tcg, op);
1349 return true;
1351 if (i > 0) {
1352 op->opc = INDEX_op_br;
1353 op->args[0] = op->args[3];
1355 return false;
1358 static bool fold_brcond2(OptContext *ctx, TCGOp *op)
1360 TCGCond cond;
1361 TCGArg label;
1362 int i, inv = 0;
1364 i = do_constant_folding_cond2(ctx, op, &op->args[0]);
1365 cond = op->args[4];
1366 label = op->args[5];
1367 if (i >= 0) {
1368 goto do_brcond_const;
1371 switch (cond) {
1372 case TCG_COND_LT:
1373 case TCG_COND_GE:
1375 * Simplify LT/GE comparisons vs zero to a single compare
1376 * vs the high word of the input.
1378 if (arg_is_const_val(op->args[2], 0) &&
1379 arg_is_const_val(op->args[3], 0)) {
1380 goto do_brcond_high;
1382 break;
1384 case TCG_COND_NE:
1385 inv = 1;
1386 QEMU_FALLTHROUGH;
1387 case TCG_COND_EQ:
1389 * Simplify EQ/NE comparisons where one of the pairs
1390 * can be simplified.
1392 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[0],
1393 op->args[2], cond);
1394 switch (i ^ inv) {
1395 case 0:
1396 goto do_brcond_const;
1397 case 1:
1398 goto do_brcond_high;
1401 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1],
1402 op->args[3], cond);
1403 switch (i ^ inv) {
1404 case 0:
1405 goto do_brcond_const;
1406 case 1:
1407 goto do_brcond_low;
1409 break;
1411 case TCG_COND_TSTEQ:
1412 case TCG_COND_TSTNE:
1413 if (arg_is_const_val(op->args[2], 0)) {
1414 goto do_brcond_high;
1416 if (arg_is_const_val(op->args[3], 0)) {
1417 goto do_brcond_low;
1419 break;
1421 default:
1422 break;
1424 do_brcond_low:
1425 op->opc = INDEX_op_brcond_i32;
1426 op->args[1] = op->args[2];
1427 op->args[2] = cond;
1428 op->args[3] = label;
1429 return fold_brcond(ctx, op);
1431 do_brcond_high:
1432 op->opc = INDEX_op_brcond_i32;
1433 op->args[0] = op->args[1];
1434 op->args[1] = op->args[3];
1435 op->args[2] = cond;
1436 op->args[3] = label;
1437 return fold_brcond(ctx, op);
1439 do_brcond_const:
1440 if (i == 0) {
1441 tcg_op_remove(ctx->tcg, op);
1442 return true;
1444 op->opc = INDEX_op_br;
1445 op->args[0] = label;
1446 break;
1448 return false;
1451 static bool fold_bswap(OptContext *ctx, TCGOp *op)
1453 uint64_t z_mask, s_mask, sign;
1455 if (arg_is_const(op->args[1])) {
1456 uint64_t t = arg_info(op->args[1])->val;
1458 t = do_constant_folding(op->opc, ctx->type, t, op->args[2]);
1459 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1462 z_mask = arg_info(op->args[1])->z_mask;
1464 switch (op->opc) {
1465 case INDEX_op_bswap16_i32:
1466 case INDEX_op_bswap16_i64:
1467 z_mask = bswap16(z_mask);
1468 sign = INT16_MIN;
1469 break;
1470 case INDEX_op_bswap32_i32:
1471 case INDEX_op_bswap32_i64:
1472 z_mask = bswap32(z_mask);
1473 sign = INT32_MIN;
1474 break;
1475 case INDEX_op_bswap64_i64:
1476 z_mask = bswap64(z_mask);
1477 sign = INT64_MIN;
1478 break;
1479 default:
1480 g_assert_not_reached();
1482 s_mask = smask_from_zmask(z_mask);
1484 switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) {
1485 case TCG_BSWAP_OZ:
1486 break;
1487 case TCG_BSWAP_OS:
1488 /* If the sign bit may be 1, force all the bits above to 1. */
1489 if (z_mask & sign) {
1490 z_mask |= sign;
1491 s_mask = sign << 1;
1493 break;
1494 default:
1495 /* The high bits are undefined: force all bits above the sign to 1. */
1496 z_mask |= sign << 1;
1497 s_mask = 0;
1498 break;
1500 ctx->z_mask = z_mask;
1501 ctx->s_mask = s_mask;
1503 return fold_masks(ctx, op);
1506 static bool fold_call(OptContext *ctx, TCGOp *op)
1508 TCGContext *s = ctx->tcg;
1509 int nb_oargs = TCGOP_CALLO(op);
1510 int nb_iargs = TCGOP_CALLI(op);
1511 int flags, i;
1513 init_arguments(ctx, op, nb_oargs + nb_iargs);
1514 copy_propagate(ctx, op, nb_oargs, nb_iargs);
1516 /* If the function reads or writes globals, reset temp data. */
1517 flags = tcg_call_flags(op);
1518 if (!(flags & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
1519 int nb_globals = s->nb_globals;
1521 for (i = 0; i < nb_globals; i++) {
1522 if (test_bit(i, ctx->temps_used.l)) {
1523 reset_ts(ctx, &ctx->tcg->temps[i]);
1528 /* If the function has side effects, reset mem data. */
1529 if (!(flags & TCG_CALL_NO_SIDE_EFFECTS)) {
1530 remove_mem_copy_all(ctx);
1533 /* Reset temp data for outputs. */
1534 for (i = 0; i < nb_oargs; i++) {
1535 reset_temp(ctx, op->args[i]);
1538 /* Stop optimizing MB across calls. */
1539 ctx->prev_mb = NULL;
1540 return true;
1543 static bool fold_count_zeros(OptContext *ctx, TCGOp *op)
1545 uint64_t z_mask;
1547 if (arg_is_const(op->args[1])) {
1548 uint64_t t = arg_info(op->args[1])->val;
1550 if (t != 0) {
1551 t = do_constant_folding(op->opc, ctx->type, t, 0);
1552 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1554 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]);
1557 switch (ctx->type) {
1558 case TCG_TYPE_I32:
1559 z_mask = 31;
1560 break;
1561 case TCG_TYPE_I64:
1562 z_mask = 63;
1563 break;
1564 default:
1565 g_assert_not_reached();
1567 ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask;
1568 ctx->s_mask = smask_from_zmask(ctx->z_mask);
1569 return false;
1572 static bool fold_ctpop(OptContext *ctx, TCGOp *op)
1574 if (fold_const1(ctx, op)) {
1575 return true;
1578 switch (ctx->type) {
1579 case TCG_TYPE_I32:
1580 ctx->z_mask = 32 | 31;
1581 break;
1582 case TCG_TYPE_I64:
1583 ctx->z_mask = 64 | 63;
1584 break;
1585 default:
1586 g_assert_not_reached();
1588 ctx->s_mask = smask_from_zmask(ctx->z_mask);
1589 return false;
1592 static bool fold_deposit(OptContext *ctx, TCGOp *op)
1594 TCGOpcode and_opc;
1596 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1597 uint64_t t1 = arg_info(op->args[1])->val;
1598 uint64_t t2 = arg_info(op->args[2])->val;
1600 t1 = deposit64(t1, op->args[3], op->args[4], t2);
1601 return tcg_opt_gen_movi(ctx, op, op->args[0], t1);
1604 switch (ctx->type) {
1605 case TCG_TYPE_I32:
1606 and_opc = INDEX_op_and_i32;
1607 break;
1608 case TCG_TYPE_I64:
1609 and_opc = INDEX_op_and_i64;
1610 break;
1611 default:
1612 g_assert_not_reached();
1615 /* Inserting a value into zero at offset 0. */
1616 if (arg_is_const_val(op->args[1], 0) && op->args[3] == 0) {
1617 uint64_t mask = MAKE_64BIT_MASK(0, op->args[4]);
1619 op->opc = and_opc;
1620 op->args[1] = op->args[2];
1621 op->args[2] = arg_new_constant(ctx, mask);
1622 ctx->z_mask = mask & arg_info(op->args[1])->z_mask;
1623 return false;
1626 /* Inserting zero into a value. */
1627 if (arg_is_const_val(op->args[2], 0)) {
1628 uint64_t mask = deposit64(-1, op->args[3], op->args[4], 0);
1630 op->opc = and_opc;
1631 op->args[2] = arg_new_constant(ctx, mask);
1632 ctx->z_mask = mask & arg_info(op->args[1])->z_mask;
1633 return false;
1636 ctx->z_mask = deposit64(arg_info(op->args[1])->z_mask,
1637 op->args[3], op->args[4],
1638 arg_info(op->args[2])->z_mask);
1639 return false;
1642 static bool fold_divide(OptContext *ctx, TCGOp *op)
1644 if (fold_const2(ctx, op) ||
1645 fold_xi_to_x(ctx, op, 1)) {
1646 return true;
1648 return false;
1651 static bool fold_dup(OptContext *ctx, TCGOp *op)
1653 if (arg_is_const(op->args[1])) {
1654 uint64_t t = arg_info(op->args[1])->val;
1655 t = dup_const(TCGOP_VECE(op), t);
1656 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1658 return false;
1661 static bool fold_dup2(OptContext *ctx, TCGOp *op)
1663 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1664 uint64_t t = deposit64(arg_info(op->args[1])->val, 32, 32,
1665 arg_info(op->args[2])->val);
1666 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1669 if (args_are_copies(op->args[1], op->args[2])) {
1670 op->opc = INDEX_op_dup_vec;
1671 TCGOP_VECE(op) = MO_32;
1673 return false;
1676 static bool fold_eqv(OptContext *ctx, TCGOp *op)
1678 if (fold_const2_commutative(ctx, op) ||
1679 fold_xi_to_x(ctx, op, -1) ||
1680 fold_xi_to_not(ctx, op, 0)) {
1681 return true;
1684 ctx->s_mask = arg_info(op->args[1])->s_mask
1685 & arg_info(op->args[2])->s_mask;
1686 return false;
1689 static bool fold_extract(OptContext *ctx, TCGOp *op)
1691 uint64_t z_mask_old, z_mask;
1692 int pos = op->args[2];
1693 int len = op->args[3];
1695 if (arg_is_const(op->args[1])) {
1696 uint64_t t;
1698 t = arg_info(op->args[1])->val;
1699 t = extract64(t, pos, len);
1700 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1703 z_mask_old = arg_info(op->args[1])->z_mask;
1704 z_mask = extract64(z_mask_old, pos, len);
1705 if (pos == 0) {
1706 ctx->a_mask = z_mask_old ^ z_mask;
1708 ctx->z_mask = z_mask;
1709 ctx->s_mask = smask_from_zmask(z_mask);
1711 return fold_masks(ctx, op);
1714 static bool fold_extract2(OptContext *ctx, TCGOp *op)
1716 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1717 uint64_t v1 = arg_info(op->args[1])->val;
1718 uint64_t v2 = arg_info(op->args[2])->val;
1719 int shr = op->args[3];
1721 if (op->opc == INDEX_op_extract2_i64) {
1722 v1 >>= shr;
1723 v2 <<= 64 - shr;
1724 } else {
1725 v1 = (uint32_t)v1 >> shr;
1726 v2 = (uint64_t)((int32_t)v2 << (32 - shr));
1728 return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2);
1730 return false;
1733 static bool fold_exts(OptContext *ctx, TCGOp *op)
1735 uint64_t s_mask_old, s_mask, z_mask, sign;
1736 bool type_change = false;
1738 if (fold_const1(ctx, op)) {
1739 return true;
1742 z_mask = arg_info(op->args[1])->z_mask;
1743 s_mask = arg_info(op->args[1])->s_mask;
1744 s_mask_old = s_mask;
1746 switch (op->opc) {
1747 CASE_OP_32_64(ext8s):
1748 sign = INT8_MIN;
1749 z_mask = (uint8_t)z_mask;
1750 break;
1751 CASE_OP_32_64(ext16s):
1752 sign = INT16_MIN;
1753 z_mask = (uint16_t)z_mask;
1754 break;
1755 case INDEX_op_ext_i32_i64:
1756 type_change = true;
1757 QEMU_FALLTHROUGH;
1758 case INDEX_op_ext32s_i64:
1759 sign = INT32_MIN;
1760 z_mask = (uint32_t)z_mask;
1761 break;
1762 default:
1763 g_assert_not_reached();
1766 if (z_mask & sign) {
1767 z_mask |= sign;
1769 s_mask |= sign << 1;
1771 ctx->z_mask = z_mask;
1772 ctx->s_mask = s_mask;
1773 if (!type_change) {
1774 ctx->a_mask = s_mask & ~s_mask_old;
1777 return fold_masks(ctx, op);
1780 static bool fold_extu(OptContext *ctx, TCGOp *op)
1782 uint64_t z_mask_old, z_mask;
1783 bool type_change = false;
1785 if (fold_const1(ctx, op)) {
1786 return true;
1789 z_mask_old = z_mask = arg_info(op->args[1])->z_mask;
1791 switch (op->opc) {
1792 CASE_OP_32_64(ext8u):
1793 z_mask = (uint8_t)z_mask;
1794 break;
1795 CASE_OP_32_64(ext16u):
1796 z_mask = (uint16_t)z_mask;
1797 break;
1798 case INDEX_op_extrl_i64_i32:
1799 case INDEX_op_extu_i32_i64:
1800 type_change = true;
1801 QEMU_FALLTHROUGH;
1802 case INDEX_op_ext32u_i64:
1803 z_mask = (uint32_t)z_mask;
1804 break;
1805 case INDEX_op_extrh_i64_i32:
1806 type_change = true;
1807 z_mask >>= 32;
1808 break;
1809 default:
1810 g_assert_not_reached();
1813 ctx->z_mask = z_mask;
1814 ctx->s_mask = smask_from_zmask(z_mask);
1815 if (!type_change) {
1816 ctx->a_mask = z_mask_old ^ z_mask;
1818 return fold_masks(ctx, op);
1821 static bool fold_mb(OptContext *ctx, TCGOp *op)
1823 /* Eliminate duplicate and redundant fence instructions. */
1824 if (ctx->prev_mb) {
1826 * Merge two barriers of the same type into one,
1827 * or a weaker barrier into a stronger one,
1828 * or two weaker barriers into a stronger one.
1829 * mb X; mb Y => mb X|Y
1830 * mb; strl => mb; st
1831 * ldaq; mb => ld; mb
1832 * ldaq; strl => ld; mb; st
1833 * Other combinations are also merged into a strong
1834 * barrier. This is stricter than specified but for
1835 * the purposes of TCG is better than not optimizing.
1837 ctx->prev_mb->args[0] |= op->args[0];
1838 tcg_op_remove(ctx->tcg, op);
1839 } else {
1840 ctx->prev_mb = op;
1842 return true;
1845 static bool fold_mov(OptContext *ctx, TCGOp *op)
1847 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1850 static bool fold_movcond(OptContext *ctx, TCGOp *op)
1852 int i;
1855 * Canonicalize the "false" input reg to match the destination reg so
1856 * that the tcg backend can implement a "move if true" operation.
1858 if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) {
1859 op->args[5] = tcg_invert_cond(op->args[5]);
1862 i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[1],
1863 &op->args[2], &op->args[5]);
1864 if (i >= 0) {
1865 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]);
1868 ctx->z_mask = arg_info(op->args[3])->z_mask
1869 | arg_info(op->args[4])->z_mask;
1870 ctx->s_mask = arg_info(op->args[3])->s_mask
1871 & arg_info(op->args[4])->s_mask;
1873 if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) {
1874 uint64_t tv = arg_info(op->args[3])->val;
1875 uint64_t fv = arg_info(op->args[4])->val;
1876 TCGOpcode opc, negopc = 0;
1877 TCGCond cond = op->args[5];
1879 switch (ctx->type) {
1880 case TCG_TYPE_I32:
1881 opc = INDEX_op_setcond_i32;
1882 if (TCG_TARGET_HAS_negsetcond_i32) {
1883 negopc = INDEX_op_negsetcond_i32;
1885 tv = (int32_t)tv;
1886 fv = (int32_t)fv;
1887 break;
1888 case TCG_TYPE_I64:
1889 opc = INDEX_op_setcond_i64;
1890 if (TCG_TARGET_HAS_negsetcond_i64) {
1891 negopc = INDEX_op_negsetcond_i64;
1893 break;
1894 default:
1895 g_assert_not_reached();
1898 if (tv == 1 && fv == 0) {
1899 op->opc = opc;
1900 op->args[3] = cond;
1901 } else if (fv == 1 && tv == 0) {
1902 op->opc = opc;
1903 op->args[3] = tcg_invert_cond(cond);
1904 } else if (negopc) {
1905 if (tv == -1 && fv == 0) {
1906 op->opc = negopc;
1907 op->args[3] = cond;
1908 } else if (fv == -1 && tv == 0) {
1909 op->opc = negopc;
1910 op->args[3] = tcg_invert_cond(cond);
1914 return false;
1917 static bool fold_mul(OptContext *ctx, TCGOp *op)
1919 if (fold_const2(ctx, op) ||
1920 fold_xi_to_i(ctx, op, 0) ||
1921 fold_xi_to_x(ctx, op, 1)) {
1922 return true;
1924 return false;
1927 static bool fold_mul_highpart(OptContext *ctx, TCGOp *op)
1929 if (fold_const2_commutative(ctx, op) ||
1930 fold_xi_to_i(ctx, op, 0)) {
1931 return true;
1933 return false;
1936 static bool fold_multiply2(OptContext *ctx, TCGOp *op)
1938 swap_commutative(op->args[0], &op->args[2], &op->args[3]);
1940 if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) {
1941 uint64_t a = arg_info(op->args[2])->val;
1942 uint64_t b = arg_info(op->args[3])->val;
1943 uint64_t h, l;
1944 TCGArg rl, rh;
1945 TCGOp *op2;
1947 switch (op->opc) {
1948 case INDEX_op_mulu2_i32:
1949 l = (uint64_t)(uint32_t)a * (uint32_t)b;
1950 h = (int32_t)(l >> 32);
1951 l = (int32_t)l;
1952 break;
1953 case INDEX_op_muls2_i32:
1954 l = (int64_t)(int32_t)a * (int32_t)b;
1955 h = l >> 32;
1956 l = (int32_t)l;
1957 break;
1958 case INDEX_op_mulu2_i64:
1959 mulu64(&l, &h, a, b);
1960 break;
1961 case INDEX_op_muls2_i64:
1962 muls64(&l, &h, a, b);
1963 break;
1964 default:
1965 g_assert_not_reached();
1968 rl = op->args[0];
1969 rh = op->args[1];
1971 /* The proper opcode is supplied by tcg_opt_gen_mov. */
1972 op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2);
1974 tcg_opt_gen_movi(ctx, op, rl, l);
1975 tcg_opt_gen_movi(ctx, op2, rh, h);
1976 return true;
1978 return false;
1981 static bool fold_nand(OptContext *ctx, TCGOp *op)
1983 if (fold_const2_commutative(ctx, op) ||
1984 fold_xi_to_not(ctx, op, -1)) {
1985 return true;
1988 ctx->s_mask = arg_info(op->args[1])->s_mask
1989 & arg_info(op->args[2])->s_mask;
1990 return false;
1993 static bool fold_neg(OptContext *ctx, TCGOp *op)
1995 uint64_t z_mask;
1997 if (fold_const1(ctx, op)) {
1998 return true;
2001 /* Set to 1 all bits to the left of the rightmost. */
2002 z_mask = arg_info(op->args[1])->z_mask;
2003 ctx->z_mask = -(z_mask & -z_mask);
2006 * Because of fold_sub_to_neg, we want to always return true,
2007 * via finish_folding.
2009 finish_folding(ctx, op);
2010 return true;
2013 static bool fold_nor(OptContext *ctx, TCGOp *op)
2015 if (fold_const2_commutative(ctx, op) ||
2016 fold_xi_to_not(ctx, op, 0)) {
2017 return true;
2020 ctx->s_mask = arg_info(op->args[1])->s_mask
2021 & arg_info(op->args[2])->s_mask;
2022 return false;
2025 static bool fold_not(OptContext *ctx, TCGOp *op)
2027 if (fold_const1(ctx, op)) {
2028 return true;
2031 ctx->s_mask = arg_info(op->args[1])->s_mask;
2033 /* Because of fold_to_not, we want to always return true, via finish. */
2034 finish_folding(ctx, op);
2035 return true;
2038 static bool fold_or(OptContext *ctx, TCGOp *op)
2040 if (fold_const2_commutative(ctx, op) ||
2041 fold_xi_to_x(ctx, op, 0) ||
2042 fold_xx_to_x(ctx, op)) {
2043 return true;
2046 ctx->z_mask = arg_info(op->args[1])->z_mask
2047 | arg_info(op->args[2])->z_mask;
2048 ctx->s_mask = arg_info(op->args[1])->s_mask
2049 & arg_info(op->args[2])->s_mask;
2050 return fold_masks(ctx, op);
2053 static bool fold_orc(OptContext *ctx, TCGOp *op)
2055 if (fold_const2(ctx, op) ||
2056 fold_xx_to_i(ctx, op, -1) ||
2057 fold_xi_to_x(ctx, op, -1) ||
2058 fold_ix_to_not(ctx, op, 0)) {
2059 return true;
2062 ctx->s_mask = arg_info(op->args[1])->s_mask
2063 & arg_info(op->args[2])->s_mask;
2064 return false;
2067 static bool fold_qemu_ld(OptContext *ctx, TCGOp *op)
2069 const TCGOpDef *def = &tcg_op_defs[op->opc];
2070 MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs];
2071 MemOp mop = get_memop(oi);
2072 int width = 8 * memop_size(mop);
2074 if (width < 64) {
2075 ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width);
2076 if (!(mop & MO_SIGN)) {
2077 ctx->z_mask = MAKE_64BIT_MASK(0, width);
2078 ctx->s_mask <<= 1;
2082 /* Opcodes that touch guest memory stop the mb optimization. */
2083 ctx->prev_mb = NULL;
2084 return false;
2087 static bool fold_qemu_st(OptContext *ctx, TCGOp *op)
2089 /* Opcodes that touch guest memory stop the mb optimization. */
2090 ctx->prev_mb = NULL;
2091 return false;
2094 static bool fold_remainder(OptContext *ctx, TCGOp *op)
2096 if (fold_const2(ctx, op) ||
2097 fold_xx_to_i(ctx, op, 0)) {
2098 return true;
2100 return false;
2103 static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg)
2105 TCGOpcode and_opc, sub_opc, xor_opc, neg_opc, shr_opc;
2106 TCGOpcode uext_opc = 0, sext_opc = 0;
2107 TCGCond cond = op->args[3];
2108 TCGArg ret, src1, src2;
2109 TCGOp *op2;
2110 uint64_t val;
2111 int sh;
2112 bool inv;
2114 if (!is_tst_cond(cond) || !arg_is_const(op->args[2])) {
2115 return;
2118 src2 = op->args[2];
2119 val = arg_info(src2)->val;
2120 if (!is_power_of_2(val)) {
2121 return;
2123 sh = ctz64(val);
2125 switch (ctx->type) {
2126 case TCG_TYPE_I32:
2127 and_opc = INDEX_op_and_i32;
2128 sub_opc = INDEX_op_sub_i32;
2129 xor_opc = INDEX_op_xor_i32;
2130 shr_opc = INDEX_op_shr_i32;
2131 neg_opc = INDEX_op_neg_i32;
2132 if (TCG_TARGET_extract_i32_valid(sh, 1)) {
2133 uext_opc = TCG_TARGET_HAS_extract_i32 ? INDEX_op_extract_i32 : 0;
2134 sext_opc = TCG_TARGET_HAS_sextract_i32 ? INDEX_op_sextract_i32 : 0;
2136 break;
2137 case TCG_TYPE_I64:
2138 and_opc = INDEX_op_and_i64;
2139 sub_opc = INDEX_op_sub_i64;
2140 xor_opc = INDEX_op_xor_i64;
2141 shr_opc = INDEX_op_shr_i64;
2142 neg_opc = INDEX_op_neg_i64;
2143 if (TCG_TARGET_extract_i64_valid(sh, 1)) {
2144 uext_opc = TCG_TARGET_HAS_extract_i64 ? INDEX_op_extract_i64 : 0;
2145 sext_opc = TCG_TARGET_HAS_sextract_i64 ? INDEX_op_sextract_i64 : 0;
2147 break;
2148 default:
2149 g_assert_not_reached();
2152 ret = op->args[0];
2153 src1 = op->args[1];
2154 inv = cond == TCG_COND_TSTEQ;
2156 if (sh && sext_opc && neg && !inv) {
2157 op->opc = sext_opc;
2158 op->args[1] = src1;
2159 op->args[2] = sh;
2160 op->args[3] = 1;
2161 return;
2162 } else if (sh && uext_opc) {
2163 op->opc = uext_opc;
2164 op->args[1] = src1;
2165 op->args[2] = sh;
2166 op->args[3] = 1;
2167 } else {
2168 if (sh) {
2169 op2 = tcg_op_insert_before(ctx->tcg, op, shr_opc, 3);
2170 op2->args[0] = ret;
2171 op2->args[1] = src1;
2172 op2->args[2] = arg_new_constant(ctx, sh);
2173 src1 = ret;
2175 op->opc = and_opc;
2176 op->args[1] = src1;
2177 op->args[2] = arg_new_constant(ctx, 1);
2180 if (neg && inv) {
2181 op2 = tcg_op_insert_after(ctx->tcg, op, sub_opc, 3);
2182 op2->args[0] = ret;
2183 op2->args[1] = ret;
2184 op2->args[2] = arg_new_constant(ctx, 1);
2185 } else if (inv) {
2186 op2 = tcg_op_insert_after(ctx->tcg, op, xor_opc, 3);
2187 op2->args[0] = ret;
2188 op2->args[1] = ret;
2189 op2->args[2] = arg_new_constant(ctx, 1);
2190 } else if (neg) {
2191 op2 = tcg_op_insert_after(ctx->tcg, op, neg_opc, 2);
2192 op2->args[0] = ret;
2193 op2->args[1] = ret;
2197 static bool fold_setcond(OptContext *ctx, TCGOp *op)
2199 int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1],
2200 &op->args[2], &op->args[3]);
2201 if (i >= 0) {
2202 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
2204 fold_setcond_tst_pow2(ctx, op, false);
2206 ctx->z_mask = 1;
2207 ctx->s_mask = smask_from_zmask(1);
2208 return false;
2211 static bool fold_negsetcond(OptContext *ctx, TCGOp *op)
2213 int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1],
2214 &op->args[2], &op->args[3]);
2215 if (i >= 0) {
2216 return tcg_opt_gen_movi(ctx, op, op->args[0], -i);
2218 fold_setcond_tst_pow2(ctx, op, true);
2220 /* Value is {0,-1} so all bits are repetitions of the sign. */
2221 ctx->s_mask = -1;
2222 return false;
2225 static bool fold_setcond2(OptContext *ctx, TCGOp *op)
2227 TCGCond cond;
2228 int i, inv = 0;
2230 i = do_constant_folding_cond2(ctx, op, &op->args[1]);
2231 cond = op->args[5];
2232 if (i >= 0) {
2233 goto do_setcond_const;
2236 switch (cond) {
2237 case TCG_COND_LT:
2238 case TCG_COND_GE:
2240 * Simplify LT/GE comparisons vs zero to a single compare
2241 * vs the high word of the input.
2243 if (arg_is_const_val(op->args[3], 0) &&
2244 arg_is_const_val(op->args[4], 0)) {
2245 goto do_setcond_high;
2247 break;
2249 case TCG_COND_NE:
2250 inv = 1;
2251 QEMU_FALLTHROUGH;
2252 case TCG_COND_EQ:
2254 * Simplify EQ/NE comparisons where one of the pairs
2255 * can be simplified.
2257 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1],
2258 op->args[3], cond);
2259 switch (i ^ inv) {
2260 case 0:
2261 goto do_setcond_const;
2262 case 1:
2263 goto do_setcond_high;
2266 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[2],
2267 op->args[4], cond);
2268 switch (i ^ inv) {
2269 case 0:
2270 goto do_setcond_const;
2271 case 1:
2272 goto do_setcond_low;
2274 break;
2276 case TCG_COND_TSTEQ:
2277 case TCG_COND_TSTNE:
2278 if (arg_is_const_val(op->args[2], 0)) {
2279 goto do_setcond_high;
2281 if (arg_is_const_val(op->args[4], 0)) {
2282 goto do_setcond_low;
2284 break;
2286 default:
2287 break;
2289 do_setcond_low:
2290 op->args[2] = op->args[3];
2291 op->args[3] = cond;
2292 op->opc = INDEX_op_setcond_i32;
2293 return fold_setcond(ctx, op);
2295 do_setcond_high:
2296 op->args[1] = op->args[2];
2297 op->args[2] = op->args[4];
2298 op->args[3] = cond;
2299 op->opc = INDEX_op_setcond_i32;
2300 return fold_setcond(ctx, op);
2303 ctx->z_mask = 1;
2304 ctx->s_mask = smask_from_zmask(1);
2305 return false;
2307 do_setcond_const:
2308 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
2311 static bool fold_sextract(OptContext *ctx, TCGOp *op)
2313 uint64_t z_mask, s_mask, s_mask_old;
2314 int pos = op->args[2];
2315 int len = op->args[3];
2317 if (arg_is_const(op->args[1])) {
2318 uint64_t t;
2320 t = arg_info(op->args[1])->val;
2321 t = sextract64(t, pos, len);
2322 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
2325 z_mask = arg_info(op->args[1])->z_mask;
2326 z_mask = sextract64(z_mask, pos, len);
2327 ctx->z_mask = z_mask;
2329 s_mask_old = arg_info(op->args[1])->s_mask;
2330 s_mask = sextract64(s_mask_old, pos, len);
2331 s_mask |= MAKE_64BIT_MASK(len, 64 - len);
2332 ctx->s_mask = s_mask;
2334 if (pos == 0) {
2335 ctx->a_mask = s_mask & ~s_mask_old;
2338 return fold_masks(ctx, op);
2341 static bool fold_shift(OptContext *ctx, TCGOp *op)
2343 uint64_t s_mask, z_mask, sign;
2345 if (fold_const2(ctx, op) ||
2346 fold_ix_to_i(ctx, op, 0) ||
2347 fold_xi_to_x(ctx, op, 0)) {
2348 return true;
2351 s_mask = arg_info(op->args[1])->s_mask;
2352 z_mask = arg_info(op->args[1])->z_mask;
2354 if (arg_is_const(op->args[2])) {
2355 int sh = arg_info(op->args[2])->val;
2357 ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh);
2359 s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh);
2360 ctx->s_mask = smask_from_smask(s_mask);
2362 return fold_masks(ctx, op);
2365 switch (op->opc) {
2366 CASE_OP_32_64(sar):
2368 * Arithmetic right shift will not reduce the number of
2369 * input sign repetitions.
2371 ctx->s_mask = s_mask;
2372 break;
2373 CASE_OP_32_64(shr):
2375 * If the sign bit is known zero, then logical right shift
2376 * will not reduced the number of input sign repetitions.
2378 sign = (s_mask & -s_mask) >> 1;
2379 if (!(z_mask & sign)) {
2380 ctx->s_mask = s_mask;
2382 break;
2383 default:
2384 break;
2387 return false;
2390 static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op)
2392 TCGOpcode neg_op;
2393 bool have_neg;
2395 if (!arg_is_const(op->args[1]) || arg_info(op->args[1])->val != 0) {
2396 return false;
2399 switch (ctx->type) {
2400 case TCG_TYPE_I32:
2401 neg_op = INDEX_op_neg_i32;
2402 have_neg = true;
2403 break;
2404 case TCG_TYPE_I64:
2405 neg_op = INDEX_op_neg_i64;
2406 have_neg = true;
2407 break;
2408 case TCG_TYPE_V64:
2409 case TCG_TYPE_V128:
2410 case TCG_TYPE_V256:
2411 neg_op = INDEX_op_neg_vec;
2412 have_neg = (TCG_TARGET_HAS_neg_vec &&
2413 tcg_can_emit_vec_op(neg_op, ctx->type, TCGOP_VECE(op)) > 0);
2414 break;
2415 default:
2416 g_assert_not_reached();
2418 if (have_neg) {
2419 op->opc = neg_op;
2420 op->args[1] = op->args[2];
2421 return fold_neg(ctx, op);
2423 return false;
2426 /* We cannot as yet do_constant_folding with vectors. */
2427 static bool fold_sub_vec(OptContext *ctx, TCGOp *op)
2429 if (fold_xx_to_i(ctx, op, 0) ||
2430 fold_xi_to_x(ctx, op, 0) ||
2431 fold_sub_to_neg(ctx, op)) {
2432 return true;
2434 return false;
2437 static bool fold_sub(OptContext *ctx, TCGOp *op)
2439 if (fold_const2(ctx, op) || fold_sub_vec(ctx, op)) {
2440 return true;
2443 /* Fold sub r,x,i to add r,x,-i */
2444 if (arg_is_const(op->args[2])) {
2445 uint64_t val = arg_info(op->args[2])->val;
2447 op->opc = (ctx->type == TCG_TYPE_I32
2448 ? INDEX_op_add_i32 : INDEX_op_add_i64);
2449 op->args[2] = arg_new_constant(ctx, -val);
2451 return false;
2454 static bool fold_sub2(OptContext *ctx, TCGOp *op)
2456 return fold_addsub2(ctx, op, false);
2459 static bool fold_tcg_ld(OptContext *ctx, TCGOp *op)
2461 /* We can't do any folding with a load, but we can record bits. */
2462 switch (op->opc) {
2463 CASE_OP_32_64(ld8s):
2464 ctx->s_mask = MAKE_64BIT_MASK(8, 56);
2465 break;
2466 CASE_OP_32_64(ld8u):
2467 ctx->z_mask = MAKE_64BIT_MASK(0, 8);
2468 ctx->s_mask = MAKE_64BIT_MASK(9, 55);
2469 break;
2470 CASE_OP_32_64(ld16s):
2471 ctx->s_mask = MAKE_64BIT_MASK(16, 48);
2472 break;
2473 CASE_OP_32_64(ld16u):
2474 ctx->z_mask = MAKE_64BIT_MASK(0, 16);
2475 ctx->s_mask = MAKE_64BIT_MASK(17, 47);
2476 break;
2477 case INDEX_op_ld32s_i64:
2478 ctx->s_mask = MAKE_64BIT_MASK(32, 32);
2479 break;
2480 case INDEX_op_ld32u_i64:
2481 ctx->z_mask = MAKE_64BIT_MASK(0, 32);
2482 ctx->s_mask = MAKE_64BIT_MASK(33, 31);
2483 break;
2484 default:
2485 g_assert_not_reached();
2487 return false;
2490 static bool fold_tcg_ld_memcopy(OptContext *ctx, TCGOp *op)
2492 TCGTemp *dst, *src;
2493 intptr_t ofs;
2494 TCGType type;
2496 if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
2497 return false;
2500 type = ctx->type;
2501 ofs = op->args[2];
2502 dst = arg_temp(op->args[0]);
2503 src = find_mem_copy_for(ctx, type, ofs);
2504 if (src && src->base_type == type) {
2505 return tcg_opt_gen_mov(ctx, op, temp_arg(dst), temp_arg(src));
2508 reset_ts(ctx, dst);
2509 record_mem_copy(ctx, type, dst, ofs, ofs + tcg_type_size(type) - 1);
2510 return true;
2513 static bool fold_tcg_st(OptContext *ctx, TCGOp *op)
2515 intptr_t ofs = op->args[2];
2516 intptr_t lm1;
2518 if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
2519 remove_mem_copy_all(ctx);
2520 return false;
2523 switch (op->opc) {
2524 CASE_OP_32_64(st8):
2525 lm1 = 0;
2526 break;
2527 CASE_OP_32_64(st16):
2528 lm1 = 1;
2529 break;
2530 case INDEX_op_st32_i64:
2531 case INDEX_op_st_i32:
2532 lm1 = 3;
2533 break;
2534 case INDEX_op_st_i64:
2535 lm1 = 7;
2536 break;
2537 case INDEX_op_st_vec:
2538 lm1 = tcg_type_size(ctx->type) - 1;
2539 break;
2540 default:
2541 g_assert_not_reached();
2543 remove_mem_copy_in(ctx, ofs, ofs + lm1);
2544 return false;
2547 static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op)
2549 TCGTemp *src;
2550 intptr_t ofs, last;
2551 TCGType type;
2553 if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
2554 fold_tcg_st(ctx, op);
2555 return false;
2558 src = arg_temp(op->args[0]);
2559 ofs = op->args[2];
2560 type = ctx->type;
2563 * Eliminate duplicate stores of a constant.
2564 * This happens frequently when the target ISA zero-extends.
2566 if (ts_is_const(src)) {
2567 TCGTemp *prev = find_mem_copy_for(ctx, type, ofs);
2568 if (src == prev) {
2569 tcg_op_remove(ctx->tcg, op);
2570 return true;
2574 last = ofs + tcg_type_size(type) - 1;
2575 remove_mem_copy_in(ctx, ofs, last);
2576 record_mem_copy(ctx, type, src, ofs, last);
2577 return false;
2580 static bool fold_xor(OptContext *ctx, TCGOp *op)
2582 if (fold_const2_commutative(ctx, op) ||
2583 fold_xx_to_i(ctx, op, 0) ||
2584 fold_xi_to_x(ctx, op, 0) ||
2585 fold_xi_to_not(ctx, op, -1)) {
2586 return true;
2589 ctx->z_mask = arg_info(op->args[1])->z_mask
2590 | arg_info(op->args[2])->z_mask;
2591 ctx->s_mask = arg_info(op->args[1])->s_mask
2592 & arg_info(op->args[2])->s_mask;
2593 return fold_masks(ctx, op);
2596 /* Propagate constants and copies, fold constant expressions. */
2597 void tcg_optimize(TCGContext *s)
2599 int nb_temps, i;
2600 TCGOp *op, *op_next;
2601 OptContext ctx = { .tcg = s };
2603 QSIMPLEQ_INIT(&ctx.mem_free);
2605 /* Array VALS has an element for each temp.
2606 If this temp holds a constant then its value is kept in VALS' element.
2607 If this temp is a copy of other ones then the other copies are
2608 available through the doubly linked circular list. */
2610 nb_temps = s->nb_temps;
2611 for (i = 0; i < nb_temps; ++i) {
2612 s->temps[i].state_ptr = NULL;
2615 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2616 TCGOpcode opc = op->opc;
2617 const TCGOpDef *def;
2618 bool done = false;
2620 /* Calls are special. */
2621 if (opc == INDEX_op_call) {
2622 fold_call(&ctx, op);
2623 continue;
2626 def = &tcg_op_defs[opc];
2627 init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs);
2628 copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs);
2630 /* Pre-compute the type of the operation. */
2631 if (def->flags & TCG_OPF_VECTOR) {
2632 ctx.type = TCG_TYPE_V64 + TCGOP_VECL(op);
2633 } else if (def->flags & TCG_OPF_64BIT) {
2634 ctx.type = TCG_TYPE_I64;
2635 } else {
2636 ctx.type = TCG_TYPE_I32;
2639 /* Assume all bits affected, no bits known zero, no sign reps. */
2640 ctx.a_mask = -1;
2641 ctx.z_mask = -1;
2642 ctx.s_mask = 0;
2645 * Process each opcode.
2646 * Sorted alphabetically by opcode as much as possible.
2648 switch (opc) {
2649 CASE_OP_32_64(add):
2650 done = fold_add(&ctx, op);
2651 break;
2652 case INDEX_op_add_vec:
2653 done = fold_add_vec(&ctx, op);
2654 break;
2655 CASE_OP_32_64(add2):
2656 done = fold_add2(&ctx, op);
2657 break;
2658 CASE_OP_32_64_VEC(and):
2659 done = fold_and(&ctx, op);
2660 break;
2661 CASE_OP_32_64_VEC(andc):
2662 done = fold_andc(&ctx, op);
2663 break;
2664 CASE_OP_32_64(brcond):
2665 done = fold_brcond(&ctx, op);
2666 break;
2667 case INDEX_op_brcond2_i32:
2668 done = fold_brcond2(&ctx, op);
2669 break;
2670 CASE_OP_32_64(bswap16):
2671 CASE_OP_32_64(bswap32):
2672 case INDEX_op_bswap64_i64:
2673 done = fold_bswap(&ctx, op);
2674 break;
2675 CASE_OP_32_64(clz):
2676 CASE_OP_32_64(ctz):
2677 done = fold_count_zeros(&ctx, op);
2678 break;
2679 CASE_OP_32_64(ctpop):
2680 done = fold_ctpop(&ctx, op);
2681 break;
2682 CASE_OP_32_64(deposit):
2683 done = fold_deposit(&ctx, op);
2684 break;
2685 CASE_OP_32_64(div):
2686 CASE_OP_32_64(divu):
2687 done = fold_divide(&ctx, op);
2688 break;
2689 case INDEX_op_dup_vec:
2690 done = fold_dup(&ctx, op);
2691 break;
2692 case INDEX_op_dup2_vec:
2693 done = fold_dup2(&ctx, op);
2694 break;
2695 CASE_OP_32_64_VEC(eqv):
2696 done = fold_eqv(&ctx, op);
2697 break;
2698 CASE_OP_32_64(extract):
2699 done = fold_extract(&ctx, op);
2700 break;
2701 CASE_OP_32_64(extract2):
2702 done = fold_extract2(&ctx, op);
2703 break;
2704 CASE_OP_32_64(ext8s):
2705 CASE_OP_32_64(ext16s):
2706 case INDEX_op_ext32s_i64:
2707 case INDEX_op_ext_i32_i64:
2708 done = fold_exts(&ctx, op);
2709 break;
2710 CASE_OP_32_64(ext8u):
2711 CASE_OP_32_64(ext16u):
2712 case INDEX_op_ext32u_i64:
2713 case INDEX_op_extu_i32_i64:
2714 case INDEX_op_extrl_i64_i32:
2715 case INDEX_op_extrh_i64_i32:
2716 done = fold_extu(&ctx, op);
2717 break;
2718 CASE_OP_32_64(ld8s):
2719 CASE_OP_32_64(ld8u):
2720 CASE_OP_32_64(ld16s):
2721 CASE_OP_32_64(ld16u):
2722 case INDEX_op_ld32s_i64:
2723 case INDEX_op_ld32u_i64:
2724 done = fold_tcg_ld(&ctx, op);
2725 break;
2726 case INDEX_op_ld_i32:
2727 case INDEX_op_ld_i64:
2728 case INDEX_op_ld_vec:
2729 done = fold_tcg_ld_memcopy(&ctx, op);
2730 break;
2731 CASE_OP_32_64(st8):
2732 CASE_OP_32_64(st16):
2733 case INDEX_op_st32_i64:
2734 done = fold_tcg_st(&ctx, op);
2735 break;
2736 case INDEX_op_st_i32:
2737 case INDEX_op_st_i64:
2738 case INDEX_op_st_vec:
2739 done = fold_tcg_st_memcopy(&ctx, op);
2740 break;
2741 case INDEX_op_mb:
2742 done = fold_mb(&ctx, op);
2743 break;
2744 CASE_OP_32_64_VEC(mov):
2745 done = fold_mov(&ctx, op);
2746 break;
2747 CASE_OP_32_64(movcond):
2748 done = fold_movcond(&ctx, op);
2749 break;
2750 CASE_OP_32_64(mul):
2751 done = fold_mul(&ctx, op);
2752 break;
2753 CASE_OP_32_64(mulsh):
2754 CASE_OP_32_64(muluh):
2755 done = fold_mul_highpart(&ctx, op);
2756 break;
2757 CASE_OP_32_64(muls2):
2758 CASE_OP_32_64(mulu2):
2759 done = fold_multiply2(&ctx, op);
2760 break;
2761 CASE_OP_32_64_VEC(nand):
2762 done = fold_nand(&ctx, op);
2763 break;
2764 CASE_OP_32_64(neg):
2765 done = fold_neg(&ctx, op);
2766 break;
2767 CASE_OP_32_64_VEC(nor):
2768 done = fold_nor(&ctx, op);
2769 break;
2770 CASE_OP_32_64_VEC(not):
2771 done = fold_not(&ctx, op);
2772 break;
2773 CASE_OP_32_64_VEC(or):
2774 done = fold_or(&ctx, op);
2775 break;
2776 CASE_OP_32_64_VEC(orc):
2777 done = fold_orc(&ctx, op);
2778 break;
2779 case INDEX_op_qemu_ld_a32_i32:
2780 case INDEX_op_qemu_ld_a64_i32:
2781 case INDEX_op_qemu_ld_a32_i64:
2782 case INDEX_op_qemu_ld_a64_i64:
2783 case INDEX_op_qemu_ld_a32_i128:
2784 case INDEX_op_qemu_ld_a64_i128:
2785 done = fold_qemu_ld(&ctx, op);
2786 break;
2787 case INDEX_op_qemu_st8_a32_i32:
2788 case INDEX_op_qemu_st8_a64_i32:
2789 case INDEX_op_qemu_st_a32_i32:
2790 case INDEX_op_qemu_st_a64_i32:
2791 case INDEX_op_qemu_st_a32_i64:
2792 case INDEX_op_qemu_st_a64_i64:
2793 case INDEX_op_qemu_st_a32_i128:
2794 case INDEX_op_qemu_st_a64_i128:
2795 done = fold_qemu_st(&ctx, op);
2796 break;
2797 CASE_OP_32_64(rem):
2798 CASE_OP_32_64(remu):
2799 done = fold_remainder(&ctx, op);
2800 break;
2801 CASE_OP_32_64(rotl):
2802 CASE_OP_32_64(rotr):
2803 CASE_OP_32_64(sar):
2804 CASE_OP_32_64(shl):
2805 CASE_OP_32_64(shr):
2806 done = fold_shift(&ctx, op);
2807 break;
2808 CASE_OP_32_64(setcond):
2809 done = fold_setcond(&ctx, op);
2810 break;
2811 CASE_OP_32_64(negsetcond):
2812 done = fold_negsetcond(&ctx, op);
2813 break;
2814 case INDEX_op_setcond2_i32:
2815 done = fold_setcond2(&ctx, op);
2816 break;
2817 CASE_OP_32_64(sextract):
2818 done = fold_sextract(&ctx, op);
2819 break;
2820 CASE_OP_32_64(sub):
2821 done = fold_sub(&ctx, op);
2822 break;
2823 case INDEX_op_sub_vec:
2824 done = fold_sub_vec(&ctx, op);
2825 break;
2826 CASE_OP_32_64(sub2):
2827 done = fold_sub2(&ctx, op);
2828 break;
2829 CASE_OP_32_64_VEC(xor):
2830 done = fold_xor(&ctx, op);
2831 break;
2832 default:
2833 break;
2836 if (!done) {
2837 finish_folding(&ctx, op);