2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2018 Linaro, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "tcg/tcg-op.h"
23 #include "tcg/tcg-mo.h"
24 #include "tcg-internal.h"
27 /* Reduce the number of ifdefs below. This assumes that all uses of
28 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
29 the compiler can eliminate. */
30 #if TCG_TARGET_REG_BITS == 64
31 extern TCGv_i32
TCGV_LOW_link_error(TCGv_i64
);
32 extern TCGv_i32
TCGV_HIGH_link_error(TCGv_i64
);
33 #define TCGV_LOW TCGV_LOW_link_error
34 #define TCGV_HIGH TCGV_HIGH_link_error
38 * Vector optional opcode tracking.
39 * Except for the basic logical operations (and, or, xor), and
40 * data movement (mov, ld, st, dupi), many vector opcodes are
41 * optional and may not be supported on the host. Thank Intel
42 * for the irregularity in their instruction set.
44 * The gvec expanders allow custom vector operations to be composed,
45 * generally via the .fniv callback in the GVecGen* structures. At
46 * the same time, in deciding whether to use this hook we need to
47 * know if the host supports the required operations. This is
48 * presented as an array of opcodes, terminated by 0. Each opcode
49 * is assumed to be expanded with the given VECE.
51 * For debugging, we want to validate this array. Therefore, when
52 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
53 * will validate that their opcode is present in the list.
55 #ifdef CONFIG_DEBUG_TCG
56 void tcg_assert_listed_vecop(TCGOpcode op
)
58 const TCGOpcode
*p
= tcg_ctx
->vecop_list
;
65 g_assert_not_reached();
70 bool tcg_can_emit_vecop_list(const TCGOpcode
*list
,
71 TCGType type
, unsigned vece
)
77 for (; *list
; ++list
) {
78 TCGOpcode opc
= *list
;
80 #ifdef CONFIG_DEBUG_TCG
82 case INDEX_op_and_vec
:
84 case INDEX_op_xor_vec
:
85 case INDEX_op_mov_vec
:
86 case INDEX_op_dup_vec
:
87 case INDEX_op_dup2_vec
:
90 case INDEX_op_bitsel_vec
:
91 /* These opcodes are mandatory and should not be listed. */
92 g_assert_not_reached();
93 case INDEX_op_not_vec
:
94 /* These opcodes have generic expansions using the above. */
95 g_assert_not_reached();
101 if (tcg_can_emit_vec_op(opc
, type
, vece
)) {
106 * The opcode list is created by front ends based on what they
107 * actually invoke. We must mirror the logic in the routines
108 * below for generic expansions using other opcodes.
111 case INDEX_op_neg_vec
:
112 if (tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
)) {
116 case INDEX_op_abs_vec
:
117 if (tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
)
118 && (tcg_can_emit_vec_op(INDEX_op_smax_vec
, type
, vece
) > 0
119 || tcg_can_emit_vec_op(INDEX_op_sari_vec
, type
, vece
) > 0
120 || tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
))) {
124 case INDEX_op_usadd_vec
:
125 if (tcg_can_emit_vec_op(INDEX_op_umin_vec
, type
, vece
) ||
126 tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
)) {
130 case INDEX_op_ussub_vec
:
131 if (tcg_can_emit_vec_op(INDEX_op_umax_vec
, type
, vece
) ||
132 tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
)) {
136 case INDEX_op_cmpsel_vec
:
137 case INDEX_op_smin_vec
:
138 case INDEX_op_smax_vec
:
139 case INDEX_op_umin_vec
:
140 case INDEX_op_umax_vec
:
141 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
)) {
153 void vec_gen_2(TCGOpcode opc
, TCGType type
, unsigned vece
, TCGArg r
, TCGArg a
)
155 TCGOp
*op
= tcg_emit_op(opc
, 2);
156 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
157 TCGOP_VECE(op
) = vece
;
162 void vec_gen_3(TCGOpcode opc
, TCGType type
, unsigned vece
,
163 TCGArg r
, TCGArg a
, TCGArg b
)
165 TCGOp
*op
= tcg_emit_op(opc
, 3);
166 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
167 TCGOP_VECE(op
) = vece
;
173 void vec_gen_4(TCGOpcode opc
, TCGType type
, unsigned vece
,
174 TCGArg r
, TCGArg a
, TCGArg b
, TCGArg c
)
176 TCGOp
*op
= tcg_emit_op(opc
, 4);
177 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
178 TCGOP_VECE(op
) = vece
;
185 static void vec_gen_6(TCGOpcode opc
, TCGType type
, unsigned vece
, TCGArg r
,
186 TCGArg a
, TCGArg b
, TCGArg c
, TCGArg d
, TCGArg e
)
188 TCGOp
*op
= tcg_emit_op(opc
, 6);
189 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
190 TCGOP_VECE(op
) = vece
;
199 static void vec_gen_op2(TCGOpcode opc
, unsigned vece
, TCGv_vec r
, TCGv_vec a
)
201 TCGTemp
*rt
= tcgv_vec_temp(r
);
202 TCGTemp
*at
= tcgv_vec_temp(a
);
203 TCGType type
= rt
->base_type
;
205 /* Must enough inputs for the output. */
206 tcg_debug_assert(at
->base_type
>= type
);
207 vec_gen_2(opc
, type
, vece
, temp_arg(rt
), temp_arg(at
));
210 static void vec_gen_op3(TCGOpcode opc
, unsigned vece
,
211 TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
213 TCGTemp
*rt
= tcgv_vec_temp(r
);
214 TCGTemp
*at
= tcgv_vec_temp(a
);
215 TCGTemp
*bt
= tcgv_vec_temp(b
);
216 TCGType type
= rt
->base_type
;
218 /* Must enough inputs for the output. */
219 tcg_debug_assert(at
->base_type
>= type
);
220 tcg_debug_assert(bt
->base_type
>= type
);
221 vec_gen_3(opc
, type
, vece
, temp_arg(rt
), temp_arg(at
), temp_arg(bt
));
224 void tcg_gen_mov_vec(TCGv_vec r
, TCGv_vec a
)
227 vec_gen_op2(INDEX_op_mov_vec
, 0, r
, a
);
231 TCGv_vec
tcg_const_zeros_vec(TCGType type
)
233 TCGv_vec ret
= tcg_temp_new_vec(type
);
234 tcg_gen_dupi_vec(MO_64
, ret
, 0);
238 TCGv_vec
tcg_const_ones_vec(TCGType type
)
240 TCGv_vec ret
= tcg_temp_new_vec(type
);
241 tcg_gen_dupi_vec(MO_64
, ret
, -1);
245 TCGv_vec
tcg_const_zeros_vec_matching(TCGv_vec m
)
247 TCGTemp
*t
= tcgv_vec_temp(m
);
248 return tcg_const_zeros_vec(t
->base_type
);
251 TCGv_vec
tcg_const_ones_vec_matching(TCGv_vec m
)
253 TCGTemp
*t
= tcgv_vec_temp(m
);
254 return tcg_const_ones_vec(t
->base_type
);
257 void tcg_gen_dupi_vec(unsigned vece
, TCGv_vec r
, uint64_t a
)
259 TCGTemp
*rt
= tcgv_vec_temp(r
);
260 tcg_gen_mov_vec(r
, tcg_constant_vec(rt
->base_type
, vece
, a
));
263 void tcg_gen_dup_i64_vec(unsigned vece
, TCGv_vec r
, TCGv_i64 a
)
265 TCGArg ri
= tcgv_vec_arg(r
);
266 TCGTemp
*rt
= arg_temp(ri
);
267 TCGType type
= rt
->base_type
;
269 if (TCG_TARGET_REG_BITS
== 64) {
270 TCGArg ai
= tcgv_i64_arg(a
);
271 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
272 } else if (vece
== MO_64
) {
273 TCGArg al
= tcgv_i32_arg(TCGV_LOW(a
));
274 TCGArg ah
= tcgv_i32_arg(TCGV_HIGH(a
));
275 vec_gen_3(INDEX_op_dup2_vec
, type
, MO_64
, ri
, al
, ah
);
277 TCGArg ai
= tcgv_i32_arg(TCGV_LOW(a
));
278 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
282 void tcg_gen_dup_i32_vec(unsigned vece
, TCGv_vec r
, TCGv_i32 a
)
284 TCGArg ri
= tcgv_vec_arg(r
);
285 TCGArg ai
= tcgv_i32_arg(a
);
286 TCGTemp
*rt
= arg_temp(ri
);
287 TCGType type
= rt
->base_type
;
289 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
292 void tcg_gen_dup_mem_vec(unsigned vece
, TCGv_vec r
, TCGv_ptr b
,
295 TCGArg ri
= tcgv_vec_arg(r
);
296 TCGArg bi
= tcgv_ptr_arg(b
);
297 TCGTemp
*rt
= arg_temp(ri
);
298 TCGType type
= rt
->base_type
;
300 vec_gen_3(INDEX_op_dupm_vec
, type
, vece
, ri
, bi
, ofs
);
303 static void vec_gen_ldst(TCGOpcode opc
, TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
305 TCGArg ri
= tcgv_vec_arg(r
);
306 TCGArg bi
= tcgv_ptr_arg(b
);
307 TCGTemp
*rt
= arg_temp(ri
);
308 TCGType type
= rt
->base_type
;
310 vec_gen_3(opc
, type
, 0, ri
, bi
, o
);
313 void tcg_gen_ld_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
315 vec_gen_ldst(INDEX_op_ld_vec
, r
, b
, o
);
318 void tcg_gen_st_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
320 vec_gen_ldst(INDEX_op_st_vec
, r
, b
, o
);
323 void tcg_gen_stl_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
, TCGType low_type
)
325 TCGArg ri
= tcgv_vec_arg(r
);
326 TCGArg bi
= tcgv_ptr_arg(b
);
327 TCGTemp
*rt
= arg_temp(ri
);
328 TCGType type
= rt
->base_type
;
330 tcg_debug_assert(low_type
>= TCG_TYPE_V64
);
331 tcg_debug_assert(low_type
<= type
);
332 vec_gen_3(INDEX_op_st_vec
, low_type
, 0, ri
, bi
, o
);
335 void tcg_gen_and_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
337 vec_gen_op3(INDEX_op_and_vec
, 0, r
, a
, b
);
340 void tcg_gen_or_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
342 vec_gen_op3(INDEX_op_or_vec
, 0, r
, a
, b
);
345 void tcg_gen_xor_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
347 vec_gen_op3(INDEX_op_xor_vec
, 0, r
, a
, b
);
350 void tcg_gen_andc_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
352 if (TCG_TARGET_HAS_andc_vec
) {
353 vec_gen_op3(INDEX_op_andc_vec
, 0, r
, a
, b
);
355 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
356 tcg_gen_not_vec(0, t
, b
);
357 tcg_gen_and_vec(0, r
, a
, t
);
358 tcg_temp_free_vec(t
);
362 void tcg_gen_orc_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
364 if (TCG_TARGET_HAS_orc_vec
) {
365 vec_gen_op3(INDEX_op_orc_vec
, 0, r
, a
, b
);
367 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
368 tcg_gen_not_vec(0, t
, b
);
369 tcg_gen_or_vec(0, r
, a
, t
);
370 tcg_temp_free_vec(t
);
374 void tcg_gen_nand_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
376 if (TCG_TARGET_HAS_nand_vec
) {
377 vec_gen_op3(INDEX_op_nand_vec
, 0, r
, a
, b
);
379 tcg_gen_and_vec(0, r
, a
, b
);
380 tcg_gen_not_vec(0, r
, r
);
384 void tcg_gen_nor_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
386 if (TCG_TARGET_HAS_nor_vec
) {
387 vec_gen_op3(INDEX_op_nor_vec
, 0, r
, a
, b
);
389 tcg_gen_or_vec(0, r
, a
, b
);
390 tcg_gen_not_vec(0, r
, r
);
394 void tcg_gen_eqv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
396 if (TCG_TARGET_HAS_eqv_vec
) {
397 vec_gen_op3(INDEX_op_eqv_vec
, 0, r
, a
, b
);
399 tcg_gen_xor_vec(0, r
, a
, b
);
400 tcg_gen_not_vec(0, r
, r
);
404 static bool do_op2(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGOpcode opc
)
406 TCGTemp
*rt
= tcgv_vec_temp(r
);
407 TCGTemp
*at
= tcgv_vec_temp(a
);
408 TCGArg ri
= temp_arg(rt
);
409 TCGArg ai
= temp_arg(at
);
410 TCGType type
= rt
->base_type
;
413 tcg_debug_assert(at
->base_type
>= type
);
414 tcg_assert_listed_vecop(opc
);
415 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
417 vec_gen_2(opc
, type
, vece
, ri
, ai
);
418 } else if (can
< 0) {
419 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
420 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
);
421 tcg_swap_vecop_list(hold_list
);
428 void tcg_gen_not_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
430 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
432 if (!TCG_TARGET_HAS_not_vec
|| !do_op2(vece
, r
, a
, INDEX_op_not_vec
)) {
433 TCGv_vec t
= tcg_const_ones_vec_matching(r
);
434 tcg_gen_xor_vec(0, r
, a
, t
);
435 tcg_temp_free_vec(t
);
437 tcg_swap_vecop_list(hold_list
);
440 void tcg_gen_neg_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
442 const TCGOpcode
*hold_list
;
444 tcg_assert_listed_vecop(INDEX_op_neg_vec
);
445 hold_list
= tcg_swap_vecop_list(NULL
);
447 if (!TCG_TARGET_HAS_neg_vec
|| !do_op2(vece
, r
, a
, INDEX_op_neg_vec
)) {
448 TCGv_vec t
= tcg_const_zeros_vec_matching(r
);
449 tcg_gen_sub_vec(vece
, r
, t
, a
);
450 tcg_temp_free_vec(t
);
452 tcg_swap_vecop_list(hold_list
);
455 void tcg_gen_abs_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
457 const TCGOpcode
*hold_list
;
459 tcg_assert_listed_vecop(INDEX_op_abs_vec
);
460 hold_list
= tcg_swap_vecop_list(NULL
);
462 if (!do_op2(vece
, r
, a
, INDEX_op_abs_vec
)) {
463 TCGType type
= tcgv_vec_temp(r
)->base_type
;
464 TCGv_vec t
= tcg_temp_new_vec(type
);
466 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
));
467 if (tcg_can_emit_vec_op(INDEX_op_smax_vec
, type
, vece
) > 0) {
468 tcg_gen_neg_vec(vece
, t
, a
);
469 tcg_gen_smax_vec(vece
, r
, a
, t
);
471 if (tcg_can_emit_vec_op(INDEX_op_sari_vec
, type
, vece
) > 0) {
472 tcg_gen_sari_vec(vece
, t
, a
, (8 << vece
) - 1);
474 tcg_gen_cmp_vec(TCG_COND_LT
, vece
, t
, a
,
475 tcg_constant_vec(type
, vece
, 0));
477 tcg_gen_xor_vec(vece
, r
, a
, t
);
478 tcg_gen_sub_vec(vece
, r
, r
, t
);
481 tcg_temp_free_vec(t
);
483 tcg_swap_vecop_list(hold_list
);
486 static void do_shifti(TCGOpcode opc
, unsigned vece
,
487 TCGv_vec r
, TCGv_vec a
, int64_t i
)
489 TCGTemp
*rt
= tcgv_vec_temp(r
);
490 TCGTemp
*at
= tcgv_vec_temp(a
);
491 TCGArg ri
= temp_arg(rt
);
492 TCGArg ai
= temp_arg(at
);
493 TCGType type
= rt
->base_type
;
496 tcg_debug_assert(at
->base_type
== type
);
497 tcg_debug_assert(i
>= 0 && i
< (8 << vece
));
498 tcg_assert_listed_vecop(opc
);
501 tcg_gen_mov_vec(r
, a
);
505 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
507 vec_gen_3(opc
, type
, vece
, ri
, ai
, i
);
509 /* We leave the choice of expansion via scalar or vector shift
510 to the target. Often, but not always, dupi can feed a vector
511 shift easier than a scalar. */
512 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
513 tcg_debug_assert(can
< 0);
514 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, i
);
515 tcg_swap_vecop_list(hold_list
);
519 void tcg_gen_shli_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
521 do_shifti(INDEX_op_shli_vec
, vece
, r
, a
, i
);
524 void tcg_gen_shri_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
526 do_shifti(INDEX_op_shri_vec
, vece
, r
, a
, i
);
529 void tcg_gen_sari_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
531 do_shifti(INDEX_op_sari_vec
, vece
, r
, a
, i
);
534 void tcg_gen_rotli_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
536 do_shifti(INDEX_op_rotli_vec
, vece
, r
, a
, i
);
539 void tcg_gen_rotri_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
541 int bits
= 8 << vece
;
542 tcg_debug_assert(i
>= 0 && i
< bits
);
543 do_shifti(INDEX_op_rotli_vec
, vece
, r
, a
, -i
& (bits
- 1));
546 void tcg_gen_cmp_vec(TCGCond cond
, unsigned vece
,
547 TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
549 TCGTemp
*rt
= tcgv_vec_temp(r
);
550 TCGTemp
*at
= tcgv_vec_temp(a
);
551 TCGTemp
*bt
= tcgv_vec_temp(b
);
552 TCGArg ri
= temp_arg(rt
);
553 TCGArg ai
= temp_arg(at
);
554 TCGArg bi
= temp_arg(bt
);
555 TCGType type
= rt
->base_type
;
558 tcg_debug_assert(at
->base_type
>= type
);
559 tcg_debug_assert(bt
->base_type
>= type
);
560 tcg_assert_listed_vecop(INDEX_op_cmp_vec
);
561 can
= tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
);
563 vec_gen_4(INDEX_op_cmp_vec
, type
, vece
, ri
, ai
, bi
, cond
);
565 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
566 tcg_debug_assert(can
< 0);
567 tcg_expand_vec_op(INDEX_op_cmp_vec
, type
, vece
, ri
, ai
, bi
, cond
);
568 tcg_swap_vecop_list(hold_list
);
572 static bool do_op3(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
573 TCGv_vec b
, TCGOpcode opc
)
575 TCGTemp
*rt
= tcgv_vec_temp(r
);
576 TCGTemp
*at
= tcgv_vec_temp(a
);
577 TCGTemp
*bt
= tcgv_vec_temp(b
);
578 TCGArg ri
= temp_arg(rt
);
579 TCGArg ai
= temp_arg(at
);
580 TCGArg bi
= temp_arg(bt
);
581 TCGType type
= rt
->base_type
;
584 tcg_debug_assert(at
->base_type
>= type
);
585 tcg_debug_assert(bt
->base_type
>= type
);
586 tcg_assert_listed_vecop(opc
);
587 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
589 vec_gen_3(opc
, type
, vece
, ri
, ai
, bi
);
590 } else if (can
< 0) {
591 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
592 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, bi
);
593 tcg_swap_vecop_list(hold_list
);
600 static void do_op3_nofail(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
601 TCGv_vec b
, TCGOpcode opc
)
603 bool ok
= do_op3(vece
, r
, a
, b
, opc
);
604 tcg_debug_assert(ok
);
607 void tcg_gen_add_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
609 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_add_vec
);
612 void tcg_gen_sub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
614 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sub_vec
);
617 void tcg_gen_mul_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
619 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_mul_vec
);
622 void tcg_gen_ssadd_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
624 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_ssadd_vec
);
627 void tcg_gen_usadd_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
629 if (!do_op3(vece
, r
, a
, b
, INDEX_op_usadd_vec
)) {
630 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
631 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
633 /* usadd(a, b) = min(a, ~b) + b */
634 tcg_gen_not_vec(vece
, t
, b
);
635 tcg_gen_umin_vec(vece
, t
, t
, a
);
636 tcg_gen_add_vec(vece
, r
, t
, b
);
638 tcg_temp_free_vec(t
);
639 tcg_swap_vecop_list(hold_list
);
643 void tcg_gen_sssub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
645 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sssub_vec
);
648 void tcg_gen_ussub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
650 if (!do_op3(vece
, r
, a
, b
, INDEX_op_ussub_vec
)) {
651 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
652 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
654 /* ussub(a, b) = max(a, b) - b */
655 tcg_gen_umax_vec(vece
, t
, a
, b
);
656 tcg_gen_sub_vec(vece
, r
, t
, b
);
658 tcg_temp_free_vec(t
);
659 tcg_swap_vecop_list(hold_list
);
663 static void do_minmax(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
664 TCGv_vec b
, TCGOpcode opc
, TCGCond cond
)
666 if (!do_op3(vece
, r
, a
, b
, opc
)) {
667 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
668 tcg_gen_cmpsel_vec(cond
, vece
, r
, a
, b
, a
, b
);
669 tcg_swap_vecop_list(hold_list
);
673 void tcg_gen_smin_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
675 do_minmax(vece
, r
, a
, b
, INDEX_op_smin_vec
, TCG_COND_LT
);
678 void tcg_gen_umin_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
680 do_minmax(vece
, r
, a
, b
, INDEX_op_umin_vec
, TCG_COND_LTU
);
683 void tcg_gen_smax_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
685 do_minmax(vece
, r
, a
, b
, INDEX_op_smax_vec
, TCG_COND_GT
);
688 void tcg_gen_umax_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
690 do_minmax(vece
, r
, a
, b
, INDEX_op_umax_vec
, TCG_COND_GTU
);
693 void tcg_gen_shlv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
695 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_shlv_vec
);
698 void tcg_gen_shrv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
700 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_shrv_vec
);
703 void tcg_gen_sarv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
705 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sarv_vec
);
708 void tcg_gen_rotlv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
710 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_rotlv_vec
);
713 void tcg_gen_rotrv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
715 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_rotrv_vec
);
718 static void do_shifts(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
719 TCGv_i32 s
, TCGOpcode opc
)
721 TCGTemp
*rt
= tcgv_vec_temp(r
);
722 TCGTemp
*at
= tcgv_vec_temp(a
);
723 TCGTemp
*st
= tcgv_i32_temp(s
);
724 TCGArg ri
= temp_arg(rt
);
725 TCGArg ai
= temp_arg(at
);
726 TCGArg si
= temp_arg(st
);
727 TCGType type
= rt
->base_type
;
730 tcg_debug_assert(at
->base_type
>= type
);
731 tcg_assert_listed_vecop(opc
);
732 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
734 vec_gen_3(opc
, type
, vece
, ri
, ai
, si
);
735 } else if (can
< 0) {
736 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
737 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, si
);
738 tcg_swap_vecop_list(hold_list
);
740 g_assert_not_reached();
744 void tcg_gen_shls_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
746 do_shifts(vece
, r
, a
, b
, INDEX_op_shls_vec
);
749 void tcg_gen_shrs_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
751 do_shifts(vece
, r
, a
, b
, INDEX_op_shrs_vec
);
754 void tcg_gen_sars_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
756 do_shifts(vece
, r
, a
, b
, INDEX_op_sars_vec
);
759 void tcg_gen_rotls_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 s
)
761 do_shifts(vece
, r
, a
, s
, INDEX_op_rotls_vec
);
764 void tcg_gen_bitsel_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
765 TCGv_vec b
, TCGv_vec c
)
767 TCGTemp
*rt
= tcgv_vec_temp(r
);
768 TCGTemp
*at
= tcgv_vec_temp(a
);
769 TCGTemp
*bt
= tcgv_vec_temp(b
);
770 TCGTemp
*ct
= tcgv_vec_temp(c
);
771 TCGType type
= rt
->base_type
;
773 tcg_debug_assert(at
->base_type
>= type
);
774 tcg_debug_assert(bt
->base_type
>= type
);
775 tcg_debug_assert(ct
->base_type
>= type
);
777 if (TCG_TARGET_HAS_bitsel_vec
) {
778 vec_gen_4(INDEX_op_bitsel_vec
, type
, MO_8
,
779 temp_arg(rt
), temp_arg(at
), temp_arg(bt
), temp_arg(ct
));
781 TCGv_vec t
= tcg_temp_new_vec(type
);
782 tcg_gen_and_vec(MO_8
, t
, a
, b
);
783 tcg_gen_andc_vec(MO_8
, r
, c
, a
);
784 tcg_gen_or_vec(MO_8
, r
, r
, t
);
785 tcg_temp_free_vec(t
);
789 void tcg_gen_cmpsel_vec(TCGCond cond
, unsigned vece
, TCGv_vec r
,
790 TCGv_vec a
, TCGv_vec b
, TCGv_vec c
, TCGv_vec d
)
792 TCGTemp
*rt
= tcgv_vec_temp(r
);
793 TCGTemp
*at
= tcgv_vec_temp(a
);
794 TCGTemp
*bt
= tcgv_vec_temp(b
);
795 TCGTemp
*ct
= tcgv_vec_temp(c
);
796 TCGTemp
*dt
= tcgv_vec_temp(d
);
797 TCGArg ri
= temp_arg(rt
);
798 TCGArg ai
= temp_arg(at
);
799 TCGArg bi
= temp_arg(bt
);
800 TCGArg ci
= temp_arg(ct
);
801 TCGArg di
= temp_arg(dt
);
802 TCGType type
= rt
->base_type
;
803 const TCGOpcode
*hold_list
;
806 tcg_debug_assert(at
->base_type
>= type
);
807 tcg_debug_assert(bt
->base_type
>= type
);
808 tcg_debug_assert(ct
->base_type
>= type
);
809 tcg_debug_assert(dt
->base_type
>= type
);
811 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec
);
812 hold_list
= tcg_swap_vecop_list(NULL
);
813 can
= tcg_can_emit_vec_op(INDEX_op_cmpsel_vec
, type
, vece
);
816 vec_gen_6(INDEX_op_cmpsel_vec
, type
, vece
, ri
, ai
, bi
, ci
, di
, cond
);
817 } else if (can
< 0) {
818 tcg_expand_vec_op(INDEX_op_cmpsel_vec
, type
, vece
,
819 ri
, ai
, bi
, ci
, di
, cond
);
821 TCGv_vec t
= tcg_temp_new_vec(type
);
822 tcg_gen_cmp_vec(cond
, vece
, t
, a
, b
);
823 tcg_gen_bitsel_vec(vece
, r
, t
, c
, d
);
824 tcg_temp_free_vec(t
);
826 tcg_swap_vecop_list(hold_list
);