2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2018 Linaro, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-mo.h"
26 /* Reduce the number of ifdefs below. This assumes that all uses of
27 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
28 the compiler can eliminate. */
29 #if TCG_TARGET_REG_BITS == 64
30 extern TCGv_i32
TCGV_LOW_link_error(TCGv_i64
);
31 extern TCGv_i32
TCGV_HIGH_link_error(TCGv_i64
);
32 #define TCGV_LOW TCGV_LOW_link_error
33 #define TCGV_HIGH TCGV_HIGH_link_error
37 * Vector optional opcode tracking.
38 * Except for the basic logical operations (and, or, xor), and
39 * data movement (mov, ld, st, dupi), many vector opcodes are
40 * optional and may not be supported on the host. Thank Intel
41 * for the irregularity in their instruction set.
43 * The gvec expanders allow custom vector operations to be composed,
44 * generally via the .fniv callback in the GVecGen* structures. At
45 * the same time, in deciding whether to use this hook we need to
46 * know if the host supports the required operations. This is
47 * presented as an array of opcodes, terminated by 0. Each opcode
48 * is assumed to be expanded with the given VECE.
50 * For debugging, we want to validate this array. Therefore, when
51 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
52 * will validate that their opcode is present in the list.
54 #ifdef CONFIG_DEBUG_TCG
55 void tcg_assert_listed_vecop(TCGOpcode op
)
57 const TCGOpcode
*p
= tcg_ctx
->vecop_list
;
64 g_assert_not_reached();
69 bool tcg_can_emit_vecop_list(const TCGOpcode
*list
,
70 TCGType type
, unsigned vece
)
76 for (; *list
; ++list
) {
77 TCGOpcode opc
= *list
;
79 #ifdef CONFIG_DEBUG_TCG
81 case INDEX_op_and_vec
:
83 case INDEX_op_xor_vec
:
84 case INDEX_op_mov_vec
:
85 case INDEX_op_dup_vec
:
86 case INDEX_op_dup2_vec
:
89 case INDEX_op_bitsel_vec
:
90 /* These opcodes are mandatory and should not be listed. */
91 g_assert_not_reached();
92 case INDEX_op_not_vec
:
93 /* These opcodes have generic expansions using the above. */
94 g_assert_not_reached();
100 if (tcg_can_emit_vec_op(opc
, type
, vece
)) {
105 * The opcode list is created by front ends based on what they
106 * actually invoke. We must mirror the logic in the routines
107 * below for generic expansions using other opcodes.
110 case INDEX_op_neg_vec
:
111 if (tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
)) {
115 case INDEX_op_abs_vec
:
116 if (tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
)
117 && (tcg_can_emit_vec_op(INDEX_op_smax_vec
, type
, vece
) > 0
118 || tcg_can_emit_vec_op(INDEX_op_sari_vec
, type
, vece
) > 0
119 || tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
))) {
123 case INDEX_op_cmpsel_vec
:
124 case INDEX_op_smin_vec
:
125 case INDEX_op_smax_vec
:
126 case INDEX_op_umin_vec
:
127 case INDEX_op_umax_vec
:
128 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
)) {
140 void vec_gen_2(TCGOpcode opc
, TCGType type
, unsigned vece
, TCGArg r
, TCGArg a
)
142 TCGOp
*op
= tcg_emit_op(opc
);
143 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
144 TCGOP_VECE(op
) = vece
;
149 void vec_gen_3(TCGOpcode opc
, TCGType type
, unsigned vece
,
150 TCGArg r
, TCGArg a
, TCGArg b
)
152 TCGOp
*op
= tcg_emit_op(opc
);
153 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
154 TCGOP_VECE(op
) = vece
;
160 void vec_gen_4(TCGOpcode opc
, TCGType type
, unsigned vece
,
161 TCGArg r
, TCGArg a
, TCGArg b
, TCGArg c
)
163 TCGOp
*op
= tcg_emit_op(opc
);
164 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
165 TCGOP_VECE(op
) = vece
;
172 static void vec_gen_6(TCGOpcode opc
, TCGType type
, unsigned vece
, TCGArg r
,
173 TCGArg a
, TCGArg b
, TCGArg c
, TCGArg d
, TCGArg e
)
175 TCGOp
*op
= tcg_emit_op(opc
);
176 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
177 TCGOP_VECE(op
) = vece
;
186 static void vec_gen_op2(TCGOpcode opc
, unsigned vece
, TCGv_vec r
, TCGv_vec a
)
188 TCGTemp
*rt
= tcgv_vec_temp(r
);
189 TCGTemp
*at
= tcgv_vec_temp(a
);
190 TCGType type
= rt
->base_type
;
192 /* Must enough inputs for the output. */
193 tcg_debug_assert(at
->base_type
>= type
);
194 vec_gen_2(opc
, type
, vece
, temp_arg(rt
), temp_arg(at
));
197 static void vec_gen_op3(TCGOpcode opc
, unsigned vece
,
198 TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
200 TCGTemp
*rt
= tcgv_vec_temp(r
);
201 TCGTemp
*at
= tcgv_vec_temp(a
);
202 TCGTemp
*bt
= tcgv_vec_temp(b
);
203 TCGType type
= rt
->base_type
;
205 /* Must enough inputs for the output. */
206 tcg_debug_assert(at
->base_type
>= type
);
207 tcg_debug_assert(bt
->base_type
>= type
);
208 vec_gen_3(opc
, type
, vece
, temp_arg(rt
), temp_arg(at
), temp_arg(bt
));
211 void tcg_gen_mov_vec(TCGv_vec r
, TCGv_vec a
)
214 vec_gen_op2(INDEX_op_mov_vec
, 0, r
, a
);
218 TCGv_vec
tcg_const_zeros_vec(TCGType type
)
220 TCGv_vec ret
= tcg_temp_new_vec(type
);
221 tcg_gen_dupi_vec(MO_64
, ret
, 0);
225 TCGv_vec
tcg_const_ones_vec(TCGType type
)
227 TCGv_vec ret
= tcg_temp_new_vec(type
);
228 tcg_gen_dupi_vec(MO_64
, ret
, -1);
232 TCGv_vec
tcg_const_zeros_vec_matching(TCGv_vec m
)
234 TCGTemp
*t
= tcgv_vec_temp(m
);
235 return tcg_const_zeros_vec(t
->base_type
);
238 TCGv_vec
tcg_const_ones_vec_matching(TCGv_vec m
)
240 TCGTemp
*t
= tcgv_vec_temp(m
);
241 return tcg_const_ones_vec(t
->base_type
);
244 void tcg_gen_dupi_vec(unsigned vece
, TCGv_vec r
, uint64_t a
)
246 TCGTemp
*rt
= tcgv_vec_temp(r
);
247 tcg_gen_mov_vec(r
, tcg_constant_vec(rt
->base_type
, vece
, a
));
250 void tcg_gen_dup_i64_vec(unsigned vece
, TCGv_vec r
, TCGv_i64 a
)
252 TCGArg ri
= tcgv_vec_arg(r
);
253 TCGTemp
*rt
= arg_temp(ri
);
254 TCGType type
= rt
->base_type
;
256 if (TCG_TARGET_REG_BITS
== 64) {
257 TCGArg ai
= tcgv_i64_arg(a
);
258 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
259 } else if (vece
== MO_64
) {
260 TCGArg al
= tcgv_i32_arg(TCGV_LOW(a
));
261 TCGArg ah
= tcgv_i32_arg(TCGV_HIGH(a
));
262 vec_gen_3(INDEX_op_dup2_vec
, type
, MO_64
, ri
, al
, ah
);
264 TCGArg ai
= tcgv_i32_arg(TCGV_LOW(a
));
265 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
269 void tcg_gen_dup_i32_vec(unsigned vece
, TCGv_vec r
, TCGv_i32 a
)
271 TCGArg ri
= tcgv_vec_arg(r
);
272 TCGArg ai
= tcgv_i32_arg(a
);
273 TCGTemp
*rt
= arg_temp(ri
);
274 TCGType type
= rt
->base_type
;
276 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
279 void tcg_gen_dup_mem_vec(unsigned vece
, TCGv_vec r
, TCGv_ptr b
,
282 TCGArg ri
= tcgv_vec_arg(r
);
283 TCGArg bi
= tcgv_ptr_arg(b
);
284 TCGTemp
*rt
= arg_temp(ri
);
285 TCGType type
= rt
->base_type
;
287 vec_gen_3(INDEX_op_dupm_vec
, type
, vece
, ri
, bi
, ofs
);
290 static void vec_gen_ldst(TCGOpcode opc
, TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
292 TCGArg ri
= tcgv_vec_arg(r
);
293 TCGArg bi
= tcgv_ptr_arg(b
);
294 TCGTemp
*rt
= arg_temp(ri
);
295 TCGType type
= rt
->base_type
;
297 vec_gen_3(opc
, type
, 0, ri
, bi
, o
);
300 void tcg_gen_ld_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
302 vec_gen_ldst(INDEX_op_ld_vec
, r
, b
, o
);
305 void tcg_gen_st_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
307 vec_gen_ldst(INDEX_op_st_vec
, r
, b
, o
);
310 void tcg_gen_stl_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
, TCGType low_type
)
312 TCGArg ri
= tcgv_vec_arg(r
);
313 TCGArg bi
= tcgv_ptr_arg(b
);
314 TCGTemp
*rt
= arg_temp(ri
);
315 TCGType type
= rt
->base_type
;
317 tcg_debug_assert(low_type
>= TCG_TYPE_V64
);
318 tcg_debug_assert(low_type
<= type
);
319 vec_gen_3(INDEX_op_st_vec
, low_type
, 0, ri
, bi
, o
);
322 void tcg_gen_and_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
324 vec_gen_op3(INDEX_op_and_vec
, 0, r
, a
, b
);
327 void tcg_gen_or_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
329 vec_gen_op3(INDEX_op_or_vec
, 0, r
, a
, b
);
332 void tcg_gen_xor_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
334 vec_gen_op3(INDEX_op_xor_vec
, 0, r
, a
, b
);
337 void tcg_gen_andc_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
339 if (TCG_TARGET_HAS_andc_vec
) {
340 vec_gen_op3(INDEX_op_andc_vec
, 0, r
, a
, b
);
342 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
343 tcg_gen_not_vec(0, t
, b
);
344 tcg_gen_and_vec(0, r
, a
, t
);
345 tcg_temp_free_vec(t
);
349 void tcg_gen_orc_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
351 if (TCG_TARGET_HAS_orc_vec
) {
352 vec_gen_op3(INDEX_op_orc_vec
, 0, r
, a
, b
);
354 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
355 tcg_gen_not_vec(0, t
, b
);
356 tcg_gen_or_vec(0, r
, a
, t
);
357 tcg_temp_free_vec(t
);
361 void tcg_gen_nand_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
363 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
364 tcg_gen_and_vec(0, r
, a
, b
);
365 tcg_gen_not_vec(0, r
, r
);
368 void tcg_gen_nor_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
370 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
371 tcg_gen_or_vec(0, r
, a
, b
);
372 tcg_gen_not_vec(0, r
, r
);
375 void tcg_gen_eqv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
377 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
378 tcg_gen_xor_vec(0, r
, a
, b
);
379 tcg_gen_not_vec(0, r
, r
);
382 static bool do_op2(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGOpcode opc
)
384 TCGTemp
*rt
= tcgv_vec_temp(r
);
385 TCGTemp
*at
= tcgv_vec_temp(a
);
386 TCGArg ri
= temp_arg(rt
);
387 TCGArg ai
= temp_arg(at
);
388 TCGType type
= rt
->base_type
;
391 tcg_debug_assert(at
->base_type
>= type
);
392 tcg_assert_listed_vecop(opc
);
393 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
395 vec_gen_2(opc
, type
, vece
, ri
, ai
);
396 } else if (can
< 0) {
397 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
398 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
);
399 tcg_swap_vecop_list(hold_list
);
406 void tcg_gen_not_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
408 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
410 if (!TCG_TARGET_HAS_not_vec
|| !do_op2(vece
, r
, a
, INDEX_op_not_vec
)) {
411 TCGv_vec t
= tcg_const_ones_vec_matching(r
);
412 tcg_gen_xor_vec(0, r
, a
, t
);
413 tcg_temp_free_vec(t
);
415 tcg_swap_vecop_list(hold_list
);
418 void tcg_gen_neg_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
420 const TCGOpcode
*hold_list
;
422 tcg_assert_listed_vecop(INDEX_op_neg_vec
);
423 hold_list
= tcg_swap_vecop_list(NULL
);
425 if (!TCG_TARGET_HAS_neg_vec
|| !do_op2(vece
, r
, a
, INDEX_op_neg_vec
)) {
426 TCGv_vec t
= tcg_const_zeros_vec_matching(r
);
427 tcg_gen_sub_vec(vece
, r
, t
, a
);
428 tcg_temp_free_vec(t
);
430 tcg_swap_vecop_list(hold_list
);
433 void tcg_gen_abs_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
435 const TCGOpcode
*hold_list
;
437 tcg_assert_listed_vecop(INDEX_op_abs_vec
);
438 hold_list
= tcg_swap_vecop_list(NULL
);
440 if (!do_op2(vece
, r
, a
, INDEX_op_abs_vec
)) {
441 TCGType type
= tcgv_vec_temp(r
)->base_type
;
442 TCGv_vec t
= tcg_temp_new_vec(type
);
444 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
));
445 if (tcg_can_emit_vec_op(INDEX_op_smax_vec
, type
, vece
) > 0) {
446 tcg_gen_neg_vec(vece
, t
, a
);
447 tcg_gen_smax_vec(vece
, r
, a
, t
);
449 if (tcg_can_emit_vec_op(INDEX_op_sari_vec
, type
, vece
) > 0) {
450 tcg_gen_sari_vec(vece
, t
, a
, (8 << vece
) - 1);
452 tcg_gen_cmp_vec(TCG_COND_LT
, vece
, t
, a
,
453 tcg_constant_vec(type
, vece
, 0));
455 tcg_gen_xor_vec(vece
, r
, a
, t
);
456 tcg_gen_sub_vec(vece
, r
, r
, t
);
459 tcg_temp_free_vec(t
);
461 tcg_swap_vecop_list(hold_list
);
464 static void do_shifti(TCGOpcode opc
, unsigned vece
,
465 TCGv_vec r
, TCGv_vec a
, int64_t i
)
467 TCGTemp
*rt
= tcgv_vec_temp(r
);
468 TCGTemp
*at
= tcgv_vec_temp(a
);
469 TCGArg ri
= temp_arg(rt
);
470 TCGArg ai
= temp_arg(at
);
471 TCGType type
= rt
->base_type
;
474 tcg_debug_assert(at
->base_type
== type
);
475 tcg_debug_assert(i
>= 0 && i
< (8 << vece
));
476 tcg_assert_listed_vecop(opc
);
479 tcg_gen_mov_vec(r
, a
);
483 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
485 vec_gen_3(opc
, type
, vece
, ri
, ai
, i
);
487 /* We leave the choice of expansion via scalar or vector shift
488 to the target. Often, but not always, dupi can feed a vector
489 shift easier than a scalar. */
490 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
491 tcg_debug_assert(can
< 0);
492 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, i
);
493 tcg_swap_vecop_list(hold_list
);
497 void tcg_gen_shli_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
499 do_shifti(INDEX_op_shli_vec
, vece
, r
, a
, i
);
502 void tcg_gen_shri_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
504 do_shifti(INDEX_op_shri_vec
, vece
, r
, a
, i
);
507 void tcg_gen_sari_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
509 do_shifti(INDEX_op_sari_vec
, vece
, r
, a
, i
);
512 void tcg_gen_rotli_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
514 do_shifti(INDEX_op_rotli_vec
, vece
, r
, a
, i
);
517 void tcg_gen_rotri_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
519 int bits
= 8 << vece
;
520 tcg_debug_assert(i
>= 0 && i
< bits
);
521 do_shifti(INDEX_op_rotli_vec
, vece
, r
, a
, -i
& (bits
- 1));
524 void tcg_gen_cmp_vec(TCGCond cond
, unsigned vece
,
525 TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
527 TCGTemp
*rt
= tcgv_vec_temp(r
);
528 TCGTemp
*at
= tcgv_vec_temp(a
);
529 TCGTemp
*bt
= tcgv_vec_temp(b
);
530 TCGArg ri
= temp_arg(rt
);
531 TCGArg ai
= temp_arg(at
);
532 TCGArg bi
= temp_arg(bt
);
533 TCGType type
= rt
->base_type
;
536 tcg_debug_assert(at
->base_type
>= type
);
537 tcg_debug_assert(bt
->base_type
>= type
);
538 tcg_assert_listed_vecop(INDEX_op_cmp_vec
);
539 can
= tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
);
541 vec_gen_4(INDEX_op_cmp_vec
, type
, vece
, ri
, ai
, bi
, cond
);
543 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
544 tcg_debug_assert(can
< 0);
545 tcg_expand_vec_op(INDEX_op_cmp_vec
, type
, vece
, ri
, ai
, bi
, cond
);
546 tcg_swap_vecop_list(hold_list
);
550 static bool do_op3(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
551 TCGv_vec b
, TCGOpcode opc
)
553 TCGTemp
*rt
= tcgv_vec_temp(r
);
554 TCGTemp
*at
= tcgv_vec_temp(a
);
555 TCGTemp
*bt
= tcgv_vec_temp(b
);
556 TCGArg ri
= temp_arg(rt
);
557 TCGArg ai
= temp_arg(at
);
558 TCGArg bi
= temp_arg(bt
);
559 TCGType type
= rt
->base_type
;
562 tcg_debug_assert(at
->base_type
>= type
);
563 tcg_debug_assert(bt
->base_type
>= type
);
564 tcg_assert_listed_vecop(opc
);
565 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
567 vec_gen_3(opc
, type
, vece
, ri
, ai
, bi
);
568 } else if (can
< 0) {
569 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
570 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, bi
);
571 tcg_swap_vecop_list(hold_list
);
578 static void do_op3_nofail(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
579 TCGv_vec b
, TCGOpcode opc
)
581 bool ok
= do_op3(vece
, r
, a
, b
, opc
);
582 tcg_debug_assert(ok
);
585 void tcg_gen_add_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
587 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_add_vec
);
590 void tcg_gen_sub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
592 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sub_vec
);
595 void tcg_gen_mul_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
597 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_mul_vec
);
600 void tcg_gen_ssadd_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
602 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_ssadd_vec
);
605 void tcg_gen_usadd_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
607 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_usadd_vec
);
610 void tcg_gen_sssub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
612 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sssub_vec
);
615 void tcg_gen_ussub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
617 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_ussub_vec
);
620 static void do_minmax(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
621 TCGv_vec b
, TCGOpcode opc
, TCGCond cond
)
623 if (!do_op3(vece
, r
, a
, b
, opc
)) {
624 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
625 tcg_gen_cmpsel_vec(cond
, vece
, r
, a
, b
, a
, b
);
626 tcg_swap_vecop_list(hold_list
);
630 void tcg_gen_smin_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
632 do_minmax(vece
, r
, a
, b
, INDEX_op_smin_vec
, TCG_COND_LT
);
635 void tcg_gen_umin_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
637 do_minmax(vece
, r
, a
, b
, INDEX_op_umin_vec
, TCG_COND_LTU
);
640 void tcg_gen_smax_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
642 do_minmax(vece
, r
, a
, b
, INDEX_op_smax_vec
, TCG_COND_GT
);
645 void tcg_gen_umax_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
647 do_minmax(vece
, r
, a
, b
, INDEX_op_umax_vec
, TCG_COND_GTU
);
650 void tcg_gen_shlv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
652 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_shlv_vec
);
655 void tcg_gen_shrv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
657 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_shrv_vec
);
660 void tcg_gen_sarv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
662 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sarv_vec
);
665 void tcg_gen_rotlv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
667 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_rotlv_vec
);
670 void tcg_gen_rotrv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
672 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_rotrv_vec
);
675 static void do_shifts(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
676 TCGv_i32 s
, TCGOpcode opc
)
678 TCGTemp
*rt
= tcgv_vec_temp(r
);
679 TCGTemp
*at
= tcgv_vec_temp(a
);
680 TCGTemp
*st
= tcgv_i32_temp(s
);
681 TCGArg ri
= temp_arg(rt
);
682 TCGArg ai
= temp_arg(at
);
683 TCGArg si
= temp_arg(st
);
684 TCGType type
= rt
->base_type
;
687 tcg_debug_assert(at
->base_type
>= type
);
688 tcg_assert_listed_vecop(opc
);
689 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
691 vec_gen_3(opc
, type
, vece
, ri
, ai
, si
);
692 } else if (can
< 0) {
693 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
694 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, si
);
695 tcg_swap_vecop_list(hold_list
);
697 g_assert_not_reached();
701 void tcg_gen_shls_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
703 do_shifts(vece
, r
, a
, b
, INDEX_op_shls_vec
);
706 void tcg_gen_shrs_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
708 do_shifts(vece
, r
, a
, b
, INDEX_op_shrs_vec
);
711 void tcg_gen_sars_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
713 do_shifts(vece
, r
, a
, b
, INDEX_op_sars_vec
);
716 void tcg_gen_rotls_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 s
)
718 do_shifts(vece
, r
, a
, s
, INDEX_op_rotls_vec
);
721 void tcg_gen_bitsel_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
722 TCGv_vec b
, TCGv_vec c
)
724 TCGTemp
*rt
= tcgv_vec_temp(r
);
725 TCGTemp
*at
= tcgv_vec_temp(a
);
726 TCGTemp
*bt
= tcgv_vec_temp(b
);
727 TCGTemp
*ct
= tcgv_vec_temp(c
);
728 TCGType type
= rt
->base_type
;
730 tcg_debug_assert(at
->base_type
>= type
);
731 tcg_debug_assert(bt
->base_type
>= type
);
732 tcg_debug_assert(ct
->base_type
>= type
);
734 if (TCG_TARGET_HAS_bitsel_vec
) {
735 vec_gen_4(INDEX_op_bitsel_vec
, type
, MO_8
,
736 temp_arg(rt
), temp_arg(at
), temp_arg(bt
), temp_arg(ct
));
738 TCGv_vec t
= tcg_temp_new_vec(type
);
739 tcg_gen_and_vec(MO_8
, t
, a
, b
);
740 tcg_gen_andc_vec(MO_8
, r
, c
, a
);
741 tcg_gen_or_vec(MO_8
, r
, r
, t
);
742 tcg_temp_free_vec(t
);
746 void tcg_gen_cmpsel_vec(TCGCond cond
, unsigned vece
, TCGv_vec r
,
747 TCGv_vec a
, TCGv_vec b
, TCGv_vec c
, TCGv_vec d
)
749 TCGTemp
*rt
= tcgv_vec_temp(r
);
750 TCGTemp
*at
= tcgv_vec_temp(a
);
751 TCGTemp
*bt
= tcgv_vec_temp(b
);
752 TCGTemp
*ct
= tcgv_vec_temp(c
);
753 TCGTemp
*dt
= tcgv_vec_temp(d
);
754 TCGArg ri
= temp_arg(rt
);
755 TCGArg ai
= temp_arg(at
);
756 TCGArg bi
= temp_arg(bt
);
757 TCGArg ci
= temp_arg(ct
);
758 TCGArg di
= temp_arg(dt
);
759 TCGType type
= rt
->base_type
;
760 const TCGOpcode
*hold_list
;
763 tcg_debug_assert(at
->base_type
>= type
);
764 tcg_debug_assert(bt
->base_type
>= type
);
765 tcg_debug_assert(ct
->base_type
>= type
);
766 tcg_debug_assert(dt
->base_type
>= type
);
768 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec
);
769 hold_list
= tcg_swap_vecop_list(NULL
);
770 can
= tcg_can_emit_vec_op(INDEX_op_cmpsel_vec
, type
, vece
);
773 vec_gen_6(INDEX_op_cmpsel_vec
, type
, vece
, ri
, ai
, bi
, ci
, di
, cond
);
774 } else if (can
< 0) {
775 tcg_expand_vec_op(INDEX_op_cmpsel_vec
, type
, vece
,
776 ri
, ai
, bi
, ci
, di
, cond
);
778 TCGv_vec t
= tcg_temp_new_vec(type
);
779 tcg_gen_cmp_vec(cond
, vece
, t
, a
, b
);
780 tcg_gen_bitsel_vec(vece
, r
, t
, c
, d
);
781 tcg_temp_free_vec(t
);
783 tcg_swap_vecop_list(hold_list
);